Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify matmul

eugene10b86ceca51c
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index 92ff957d9..b7f677d03 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -52,7 +52,10 @@ pub fn bindgen(
let entry_name = dynamic_mangle(kernel.name.as_ref(), variants_kernel_format);
let base_index = specialize_indices.get(&kernel.name).copied();
- let (specialize_args, specialize_setup): (Vec<TokenStream>, Vec<TokenStream>) = kernel
+ let (specialize_args, (specialize_setup, specialize_arg_names)): (
+ Vec<TokenStream>,
+ (Vec<TokenStream>, Vec<Ident>),
+ ) = kernel
.arguments
.iter()
.filter(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Specialize(_))))
@@ -68,7 +71,7 @@ pub fn bindgen(
let setup = quote! {
function_constants.set_value(&#arg_name, #idx);
};
- (arg_def, setup)
+ (arg_def, (setup, arg_name))
})
.unzip();
@@ -86,6 +89,12 @@ pub fn bindgen(
} else {
quote! { None }
};
+ let cache_key = if has_specialize {
+ let format_str = format!("{{}}{}", repeat_n("_{}", specialize_arg_names.len()).join(""));
+ quote! { &format!(#format_str, &entry_name #(, #specialize_arg_names)*) }
+ } else {
+ quote! { &entry_name }
+ };
let mut arg_count: usize = 0;
let mut indirect_flag = false;
@@ -305,8 +314,9 @@ pub fn bindgen(
type Backend = crate::backends::metal::Metal;
fn new(context: &MetalContext #(, #variants_extra_arguments)* #(, #specialize_args)*) -> Result<Self, MetalError> {
+ let entry_name = #entry_name;
#function_constants_init
- let pipeline = context.compute_pipeline_state(&#entry_name, #function_constants_arg)?;
+ let pipeline = context.compute_pipeline_state(#cache_key, &entry_name, #function_constants_arg)?;
Ok(Self { pipeline #(, #conditional_buffer_sets)* })
}
diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rs
index dbcc1d012..bd1fbd8e2 100644
--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs
+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs
@@ -42,33 +42,3 @@ pub struct GEMMSpiltKParams {
pub split_k_partition_size: i32,
pub gemm_k_iterations_aligned: i32,
}
-
-/// Split-K MLP Fused GEMM parameters.
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
-pub struct GEMMSpiltKMlpFusedParams {
- pub M: i32,
- pub N: i32,
- pub K: i32,
- pub lda: i32,
- pub ldb: i32,
- pub ldc: i32,
- pub tiles_n: i32,
- pub tiles_m: i32,
- pub split_k_partitions: i32,
- pub split_k_partition_stride: i32,
- pub split_k_partition_size: i32,
- pub gemm_k_iterations_aligned: i32,
- pub hidden_dim: i32,
-}
-
-/// AddMM parameters (alpha * A @ B + beta * C).
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
-pub struct GEMMAddMMParams {
- pub ldc: i32,
- pub fdc: i32,
- pub batch_stride_c: i64,
- pub alpha: f32,
- pub beta: f32,
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs
new file mode 100644
index 000000000..cb79a4815
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs
@@ -0,0 +1,17 @@
+use super::{gemm, gemv, split_k};
+
+#[derive(Debug, Clone)]
+pub enum MatmulDispatchDescriptor {
+ Gemv(gemv::DispatchDescriptor),
+ SplitK(split_k::DispatchDescriptor),
+ Gemm(gemm::DispatchDescriptor),
+}
+
+impl MatmulDispatchDescriptor {
+ pub fn bias_is_fused(&self) -> bool {
+ match self {
+ MatmulDispatchDescriptor::Gemv(d) => d.bias_is_fused(),
+ MatmulDispatchDescriptor::SplitK(_) | MatmulDispatchDescriptor::Gemm(_) => false,
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
new file mode 100644
index 000000000..44e1d380b
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
@@ -0,0 +1,9 @@
+use super::{super::grid_size::GridSize, specialization::Specialization};
+use crate::backends::common::gpu_types::GEMMParams;
+
+#[derive(Debug, Clone)]
+pub struct DispatchDescriptor {
+ pub specialization: Specialization,
+ pub params: GEMMParams,
+ pub threadgroups: GridSize,
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
new file mode 100644
index 000000000..0d4b30556
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
@@ -0,0 +1,96 @@
+use std::collections::HashMap;
+
+use super::{
+ super::matmul_arguments::MatmulArguments, dispatch_descriptor::DispatchDescriptor, specialization::Specialization,
+};
+use crate::{
+ DataType,
+ backends::common::{Backend, Kernels, kernel::MatmulGemmKernel},
+};
+
+pub struct GemmKernel<B: Backend> {
+ data_type: DataType,
+ pipelines: HashMap<Specialization, <B::Kernels as Kernels>::MatmulGemmKernel>,
+}
+
+impl<B: Backend> GemmKernel<B>
+where
+ B::Error: From<String>,
+{
+ pub fn new(data_type: DataType) -> Result<Self, B::Error> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(B::Error::from(format!("Unsupported dtype for GEMM: {data_type:?}")));
+ }
+ Ok(Self {
+ data_type,
+ pipelines: HashMap::new(),
+ })
+ }
+
+ pub fn precompile(
+ &mut self,
+ context: &B::Context,
+ ) -> Result<(), B::Error> {
+ for &config in Specialization::precompile_configs(self.data_type) {
+ self.get_or_create_kernel(context, config)?;
+ }
+ Ok(())
+ }
+
+ fn get_or_create_kernel(
+ &mut self,
+ context: &B::Context,
+ config: Specialization,
+ ) -> Result<&<B::Kernels as Kernels>::MatmulGemmKernel, B::Error> {
+ if !self.pipelines.contains_key(&config) {
+ let kernel = <B::Kernels as Kernels>::MatmulGemmKernel::new(
+ context,
+ self.data_type,
+ config.block_rows as u32,
+ config.block_cols as u32,
+ config.block_depth as u32,
+ config.warps_per_row as u32,
+ config.warps_per_col as u32,
+ config.align_m,
+ config.align_n,
+ config.align_k,
+ )?;
+ self.pipelines.insert(config, kernel);
+ }
+ Ok(self.pipelines.get(&config).unwrap())
+ }
+
+ pub fn encode(
+ &mut self,
+ context: &B::Context,
+ arguments: &MatmulArguments<B>,
+ dispatch_descriptor: &DispatchDescriptor,
+ encoder: &B::ComputeEncoder,
+ ) -> Result<(), B::Error> {
+ let config = dispatch_descriptor.specialization;
+
+ let group_count_x = u32::try_from(dispatch_descriptor.threadgroups.x).map_err(|_| {
+ B::Error::from(format!("GEMM group count x overflows u32: {}", dispatch_descriptor.threadgroups.x))
+ })?;
+ let group_count_y = u32::try_from(dispatch_descriptor.threadgroups.y).map_err(|_| {
+ B::Error::from(format!("GEMM group count y overflows u32: {}", dispatch_descriptor.threadgroups.y))
+ })?;
+ let group_count_z = u32::try_from(dispatch_descriptor.threadgroups.z).map_err(|_| {
+ B::Error::from(format!("GEMM group count z overflows u32: {}", dispatch_descriptor.threadgroups.z))
+ })?;
+
+ let pipeline = self.get_or_create_kernel(context, config)?;
+ pipeline.encode(
+ (arguments.a, arguments.a_offset as usize),
+ arguments.b,
+ arguments.d,
+ std::slice::from_ref(&dispatch_descriptor.params),
+ group_count_x,
+ group_count_y,
+ group_count_z,
+ encoder,
+ );
+
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs
new file mode 100644
index 000000000..a4cbe5d36
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs
@@ -0,0 +1,7 @@
+mod dispatch_descriptor;
+mod kernel;
+pub mod specialization;
+
+pub use dispatch_descriptor::DispatchDescriptor;
+pub use kernel::GemmKernel;
+pub use specialization::Specialization;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
new file mode 100644
index 000000000..4c349a4c2
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
@@ -0,0 +1,138 @@
+use crate::DataType;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Specialization {
+ pub block_rows: i32,
+ pub block_cols: i32,
+ pub block_depth: i32,
+ pub warps_per_row: u64,
+ pub warps_per_col: u64,
+ pub swizzle_log2: i32,
+ pub align_m: bool,
+ pub align_n: bool,
+ pub align_k: bool,
+}
+
+impl Specialization {
+ pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
+ match data_type {
+ DataType::BF16 => &[
+ Self {
+ block_rows: 64,
+ block_cols: 32,
+ block_depth: 32,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: false,
+ align_n: true,
+ align_k: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 32,
+ block_depth: 32,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: true,
+ align_n: true,
+ align_k: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ block_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: false,
+ align_n: true,
+ align_k: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ block_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: true,
+ align_n: false,
+ align_k: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ block_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: true,
+ align_n: true,
+ align_k: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ block_depth: 16,
+ warps_per_row: 1,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: true,
+ align_n: true,
+ align_k: true,
+ },
+ ],
+ DataType::F16 => &[
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ block_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: true,
+ align_n: true,
+ align_k: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ block_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: false,
+ align_n: true,
+ align_k: true,
+ },
+ ],
+ DataType::F32 => &[
+ Self {
+ block_rows: 32,
+ block_cols: 64,
+ block_depth: 16,
+ warps_per_row: 1,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: false,
+ align_n: true,
+ align_k: true,
+ },
+ Self {
+ block_rows: 32,
+ block_cols: 64,
+ block_depth: 16,
+ warps_per_row: 1,
+ warps_per_col: 2,
+ swizzle_log2: 0,
+ align_m: true,
+ align_n: true,
+ align_k: true,
+ },
+ ],
+ _ => &[],
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
new file mode 100644
index 000000000..e458f0f87
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
@@ -0,0 +1,161 @@
+use std::sync::OnceLock;
+
+use super::{super::matmul_arguments::MatmulArguments, specialization::Specialization};
+use crate::{DataType, backends::common::Backend};
+
+const DEFAULT_GEMV_MAX_BATCH: i32 = 8;
+
+static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();
+
+fn max_gemv_batch_threshold() -> i32 {
+ *GEMV_MAX_BATCH.get_or_init(|| {
+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
+ })
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum OutputSource {
+ None,
+ Bias,
+}
+
+#[derive(Debug, Clone)]
+pub struct DispatchDescriptor {
+ pub specialization: Specialization,
+ pub matrix_is_rhs: bool,
+ pub output_source: OutputSource,
+ pub input_dimension: i32,
+ pub output_dimension: i32,
+ pub matrix_leading_dim: i32,
+ pub alpha: f32,
+ pub beta: f32,
+ pub batch_shape: [i32; 1],
+ pub vector_batch_stride: [i64; 1],
+ pub matrix_batch_stride: [i64; 1],
+ pub bias_batch_stride: [i64; 1],
+ pub bias_stride: i32,
+ pub batch_rows: i32,
+}
+
+impl DispatchDescriptor {
+ pub fn try_new<B: Backend>(
+ data_type: DataType,
+ arguments: &MatmulArguments<B>,
+ ) -> Result<Option<Self>, B::Error>
+ where
+ B::Error: From<String>,
+ {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(B::Error::from(format!("Unsupported data type for GEMV: {data_type:?}")));
+ }
+
+ if !arguments.transpose_b {
+ return Ok(None);
+ }
+
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+
+ let max_gemv_batch = max_gemv_batch_threshold();
+
+ if n == 1 {
+ if m != 1 {
+ return Ok(None);
+ }
+ } else if m > max_gemv_batch {
+ return Ok(None);
+ }
+
+ let matrix_is_rhs = n != 1;
+ let transpose_matrix = if matrix_is_rhs {
+ !arguments.transpose_b
+ } else {
+ false
+ };
+
+ let output_source = if arguments.bias.is_some() {
+ OutputSource::Bias
+ } else {
+ OutputSource::None
+ };
+
+ let (apply_output_scale_and_accumulate, alpha, beta, bias_stride) = match output_source {
+ OutputSource::None => (false, 1.0f32, 0.0f32, 0),
+ OutputSource::Bias => (true, 1.0f32, 1.0f32, 1),
+ };
+
+ let output_dimension = if matrix_is_rhs {
+ arguments.output_dim
+ } else {
+ arguments.batch
+ };
+
+ let specialization = Specialization::select(
+ transpose_matrix,
+ arguments.input_dim,
+ output_dimension,
+ apply_output_scale_and_accumulate,
+ );
+
+ let input_dimension = arguments.input_dim;
+ let matrix_leading_dim = if matrix_is_rhs {
+ arguments.ldb
+ } else {
+ arguments.lda
+ };
+
+ let batch_shape = [if arguments.batch_count > 1 {
+ arguments.batch_count
+ } else {
+ 1
+ }];
+
+ let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);
+ let elements_per_matrix_b = if arguments.transpose_b {
+ (arguments.output_dim as i64) * (arguments.ldb as i64)
+ } else {
+ (arguments.input_dim as i64) * (arguments.ldb as i64)
+ };
+
+ let vector_batch_stride = [if matrix_is_rhs {
+ elements_per_matrix_a
+ } else {
+ elements_per_matrix_b
+ }];
+
+ let matrix_batch_stride = [if matrix_is_rhs {
+ elements_per_matrix_b
+ } else {
+ elements_per_matrix_a
+ }];
+
+ let bias_batch_stride = [if arguments.batch_count > 1 {
+ (output_dimension as i64) * (arguments.ldd as i64)
+ } else {
+ 0
+ }];
+
+ let batch_rows = arguments.batch;
+
+ Ok(Some(Self {
+ specialization,
+ matrix_is_rhs,
+ output_source,
+ input_dimension,
+ output_dimension,
+ matrix_leading_dim,
+ alpha,
+ beta,
+ batch_shape,
+ vector_batch_stride,
+ matrix_batch_stride,
+ bias_batch_stride,
+ bias_stride,
+ batch_rows,
+ }))
+ }
+
+ pub fn bias_is_fused(&self) -> bool {
+ matches!(self.output_source, OutputSource::Bias)
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
new file mode 100644
index 000000000..950fc9980
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
@@ -0,0 +1,142 @@
+use std::collections::HashMap;
+
+use super::{
+ super::matmul_arguments::MatmulArguments,
+ dispatch_descriptor::{DispatchDescriptor, OutputSource},
+ specialization::Specialization,
+};
+use crate::{
+ DataType,
+ backends::common::{Backend, Kernels, kernel::MatmulGemvKernel},
+};
+
+pub struct GemvKernel<B: Backend> {
+ data_type: DataType,
+ pipelines: HashMap<Specialization, <B::Kernels as Kernels>::MatmulGemvKernel>,
+}
+
+impl<B: Backend> GemvKernel<B>
+where
+ B::Error: From<String>,
+{
+ pub fn new(data_type: DataType) -> Result<Self, B::Error> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(B::Error::from(format!("Unsupported data type for GEMV: {data_type:?}")));
+ }
+ Ok(Self {
+ data_type,
+ pipelines: HashMap::new(),
+ })
+ }
+
+ pub fn precompile(
+ &mut self,
+ context: &B::Context,
+ ) -> Result<(), B::Error> {
+ for &config in Specialization::precompile_configs(self.data_type) {
+ self.get_or_create_kernel(context, config)?;
+ }
+ Ok(())
+ }
+
+ fn get_or_create_kernel(
+ &mut self,
+ context: &B::Context,
+ config: Specialization,
+ ) -> Result<&<B::Kernels as Kernels>::MatmulGemvKernel, B::Error> {
+ if !self.pipelines.contains_key(&config) {
+ let kernel = <B::Kernels as Kernels>::MatmulGemvKernel::new(
+ context,
+ self.data_type,
+ config.threadgroup_rows,
+ config.threadgroup_cols,
+ config.threads_per_simdgroup_row,
+ config.threads_per_simdgroup_col,
+ config.elements_per_thread_row,
+ config.elements_per_thread_col,
+ config.apply_output_scale_and_accumulate,
+ )?;
+ self.pipelines.insert(config, kernel);
+ }
+ Ok(self.pipelines.get(&config).unwrap())
+ }
+
+ pub fn encode(
+ &mut self,
+ context: &B::Context,
+ arguments: &MatmulArguments<B>,
+ dispatch_descriptor: &DispatchDescriptor,
+ encoder: &B::ComputeEncoder,
+ ) -> Result<(), B::Error> {
+ let config = dispatch_descriptor.specialization;
+ let pipeline = self.get_or_create_kernel(context, config)?;
+
+ let (matrix, matrix_offset) = if dispatch_descriptor.matrix_is_rhs {
+ (arguments.b, 0usize)
+ } else {
+ (arguments.a, arguments.a_offset as usize)
+ };
+ let (input_vector, input_vector_offset) = if dispatch_descriptor.matrix_is_rhs {
+ (arguments.a, arguments.a_offset as usize)
+ } else {
+ (arguments.b, 0usize)
+ };
+
+ let output_source = if config.apply_output_scale_and_accumulate {
+ match dispatch_descriptor.output_source {
+ OutputSource::None => {
+ return Err(B::Error::from(
+ "GEMV descriptor mismatch: apply_output_scale_and_accumulate=true but output_source=None"
+ .to_owned(),
+ ));
+ },
+ OutputSource::Bias => Some(
+ arguments.bias.ok_or_else(|| B::Error::from("GEMV descriptor requires bias buffer".to_owned()))?,
+ ),
+ }
+ } else {
+ None
+ };
+
+ let vector_batch_stride = [i32::try_from(dispatch_descriptor.vector_batch_stride[0]).map_err(|_| {
+ B::Error::from(format!(
+ "GEMV path requires i32 vector_batch_stride but got {}",
+ dispatch_descriptor.vector_batch_stride[0]
+ ))
+ })?];
+ let matrix_batch_stride = [i32::try_from(dispatch_descriptor.matrix_batch_stride[0]).map_err(|_| {
+ B::Error::from(format!(
+ "GEMV path requires i32 matrix_batch_stride but got {}",
+ dispatch_descriptor.matrix_batch_stride[0]
+ ))
+ })?];
+ let output_source_batch_stride = [i32::try_from(dispatch_descriptor.bias_batch_stride[0]).map_err(|_| {
+ B::Error::from(format!(
+ "GEMV path requires i32 output_source_batch_stride but got {}",
+ dispatch_descriptor.bias_batch_stride[0]
+ ))
+ })?];
+
+ pipeline.encode(
+ (matrix, matrix_offset),
+ (input_vector, input_vector_offset),
+ output_source.map(|buffer| (buffer, 0usize)),
+ arguments.d,
+ dispatch_descriptor.input_dimension,
+ dispatch_descriptor.output_dimension,
+ dispatch_descriptor.matrix_leading_dim,
+ dispatch_descriptor.alpha,
+ dispatch_descriptor.beta,
+ &dispatch_descriptor.batch_shape,
+ &vector_batch_stride,
+ &matrix_batch_stride,
+ &output_source_batch_stride,
+ dispatch_descriptor.bias_stride,
+ dispatch_descriptor.batch_rows,
+ config.output_rows_per_threadgroup() as i32,
+ encoder,
+ );
+
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs
new file mode 100644
index 000000000..eba171b69
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs
@@ -0,0 +1,6 @@
+mod dispatch_descriptor;
+mod kernel;
+mod specialization;
+
+pub use dispatch_descriptor::DispatchDescriptor;
+pub use kernel::GemvKernel;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rs
new file mode 100644
index 000000000..87fca97f1
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rs
@@ -0,0 +1,161 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Specialization {
+ pub threadgroup_rows: u32,
+ pub threadgroup_cols: u32,
+ pub threads_per_simdgroup_row: u32,
+ pub threads_per_simdgroup_col: u32,
+ pub elements_per_thread_row: u32,
+ pub elements_per_thread_col: u32,
+ pub apply_output_scale_and_accumulate: bool,
+}
+
+impl Specialization {
+ pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
+ use crate::DataType;
+ match data_type {
+ DataType::BF16 => &[
+ Self {
+ threadgroup_rows: 4,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ apply_output_scale_and_accumulate: false,
+ },
+ Self {
+ threadgroup_rows: 4,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ apply_output_scale_and_accumulate: true,
+ },
+ Self {
+ threadgroup_rows: 8,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ apply_output_scale_and_accumulate: false,
+ },
+ Self {
+ threadgroup_rows: 8,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ apply_output_scale_and_accumulate: true,
+ },
+ ],
+ DataType::F16 => &[Self {
+ threadgroup_rows: 8,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ apply_output_scale_and_accumulate: false,
+ }],
+ DataType::F32 => &[Self {
+ threadgroup_rows: 8,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ apply_output_scale_and_accumulate: false,
+ }],
+ _ => &[],
+ }
+ }
+
+ pub fn output_rows_per_threadgroup(&self) -> u32 {
+ self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row
+ }
+
+ pub fn select(
+ transpose_matrix: bool,
+ input_dimension: i32,
+ output_dimension: i32,
+ apply_output_scale_and_accumulate: bool,
+ ) -> Self {
+ let (threadgroup_rows, threadgroup_cols);
+ let (threads_per_simdgroup_row, threads_per_simdgroup_col);
+ let (elements_per_thread_row, elements_per_thread_col);
+
+ if transpose_matrix {
+ let mut simdgroup_thread_rows = 8;
+ let mut simdgroup_thread_cols = 4;
+ if input_dimension >= 8192 && output_dimension >= 2048 {
+ simdgroup_thread_rows = 4;
+ simdgroup_thread_cols = 8;
+ }
+
+ let threadgroup_simd_cols = if output_dimension >= 2048 {
+ 16
+ } else if output_dimension >= 512 {
+ 4
+ } else {
+ 2
+ };
+
+ let thread_output_cols = if output_dimension < 4 {
+ 1
+ } else {
+ 4
+ };
+
+ threadgroup_rows = 1;
+ threadgroup_cols = threadgroup_simd_cols;
+ threads_per_simdgroup_row = simdgroup_thread_rows;
+ threads_per_simdgroup_col = simdgroup_thread_cols;
+ elements_per_thread_row = 4;
+ elements_per_thread_col = thread_output_cols;
+ } else {
+ let threadgroup_simd_rows;
+ let mut simdgroup_thread_rows = 1;
+ let mut simdgroup_thread_cols = 32;
+ let mut threadgroup_simd_cols = 1;
+
+ if input_dimension <= 64 {
+ threadgroup_simd_rows = 1;
+ simdgroup_thread_rows = 8;
+ simdgroup_thread_cols = 4;
+ } else if input_dimension >= 16 * output_dimension {
+ threadgroup_simd_rows = 1;
+ threadgroup_simd_cols = 8;
+ } else if output_dimension >= 4096 {
+ threadgroup_simd_rows = 8;
+ } else {
+ threadgroup_simd_rows = 4;
+ }
+
+ let thread_output_rows = if output_dimension < 4 {
+ 1
+ } else {
+ 4
+ };
+
+ threadgroup_rows = threadgroup_simd_rows;
+ threadgroup_cols = threadgroup_simd_cols;
+ threads_per_simdgroup_row = simdgroup_thread_rows;
+ threads_per_simdgroup_col = simdgroup_thread_cols;
+ elements_per_thread_row = thread_output_rows;
+ elements_per_thread_col = 4;
+ }
+
+ Self {
+ threadgroup_rows,
+ threadgroup_cols,
+ threads_per_simdgroup_row,
+ threads_per_simdgroup_col,
+ elements_per_thread_row,
+ elements_per_thread_col,
+ apply_output_scale_and_accumulate,
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs b/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs
new file mode 100644
index 000000000..cee1e39fb
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs
@@ -0,0 +1,6 @@
+#[derive(Debug, Clone, Copy)]
+pub struct GridSize {
+ pub x: usize,
+ pub y: usize,
+ pub z: usize,
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
new file mode 100644
index 000000000..2473e601a
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -0,0 +1,146 @@
+use super::{
+ dispatch_descriptor::MatmulDispatchDescriptor, gemm::GemmKernel, gemv::GemvKernel,
+ matmul_arguments::MatmulArguments, split_k::SplitKKernel,
+};
+use crate::{
+ DataType,
+ backends::common::{Backend, Kernels, kernel::TensorAddBiasKernel},
+};
+
+pub struct MatmulKernel<B: Backend> {
+ pub(crate) data_type: DataType,
+ gemm: Option<GemmKernel<B>>,
+ gemv: Option<GemvKernel<B>>,
+ splitk: Option<SplitKKernel<B>>,
+ bias_add: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
+}
+
+impl<B: Backend> MatmulKernel<B>
+where
+ B::Error: From<String>,
+{
+ pub fn new(data_type: DataType) -> Result<Self, B::Error> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(B::Error::from(format!("Unsupported dtype for MatmulKernel: {data_type:?}")));
+ }
+
+ Ok(Self {
+ data_type,
+ gemm: None,
+ gemv: None,
+ splitk: None,
+ bias_add: None,
+ })
+ }
+
+ pub fn precompile(
+ &mut self,
+ context: &B::Context,
+ ) -> Result<(), B::Error> {
+ let gemm = self.get_or_create_gemm()?;
+ gemm.precompile(context)?;
+
+ let gemv = self.get_or_create_gemv()?;
+ gemv.precompile(context)?;
+
+ let splitk = self.get_or_create_splitk()?;
+ splitk.precompile(context)?;
+
+ Ok(())
+ }
+
+ fn get_or_create_gemm(&mut self) -> Result<&mut GemmKernel<B>, B::Error> {
+ if self.gemm.is_none() {
+ self.gemm = Some(GemmKernel::<B>::new(self.data_type)?);
+ }
+ Ok(self.gemm.as_mut().unwrap())
+ }
+
+ fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel<B>, B::Error> {
+ if self.gemv.is_none() {
+ self.gemv = Some(GemvKernel::<B>::new(self.data_type)?);
+ }
+ Ok(self.gemv.as_mut().unwrap())
+ }
+
+ fn get_or_create_splitk(&mut self) -> Result<&mut SplitKKernel<B>, B::Error> {
+ if self.splitk.is_none() {
+ self.splitk = Some(SplitKKernel::<B>::new(self.data_type)?);
+ }
+ Ok(self.splitk.as_mut().unwrap())
+ }
+
+ fn encode_dispatch_descriptor(
+ &mut self,
+ context: &B::Context,
+ arguments: &MatmulArguments<B>,
+ dispatch_descriptor: &MatmulDispatchDescriptor,
+ encoder: &B::ComputeEncoder,
+ ) -> Result<(), B::Error> {
+ match dispatch_descriptor {
+ MatmulDispatchDescriptor::Gemv(d) => {
+ let gemv = self.get_or_create_gemv()?;
+ gemv.encode(context, arguments, d, encoder)
+ },
+ MatmulDispatchDescriptor::SplitK(d) => {
+ let splitk = self.get_or_create_splitk()?;
+ splitk.encode(context, arguments, d, encoder)
+ },
+ MatmulDispatchDescriptor::Gemm(d) => {
+ let gemm = self.get_or_create_gemm()?;
+ gemm.encode(context, arguments, d, encoder)
+ },
+ }
+ }
+
+ pub fn encode_with_descriptor(
+ &mut self,
+ context: &B::Context,
+ arguments: MatmulArguments<B>,
+ dispatch_descriptor: &MatmulDispatchDescriptor,
+ encoder: &B::ComputeEncoder,
+ ) -> Result<(), B::Error> {
+ self.encode_dispatch_descriptor(context, &arguments, dispatch_descriptor, encoder)?;
+
+ if let Some(bias) = arguments.bias {
+ if !dispatch_descriptor.bias_is_fused() {
+ self.apply_bias_add(context, &arguments, bias, encoder)?;
+ }
+ }
+
+ Ok(())
+ }
+
+ fn apply_bias_add(
+ &mut self,
+ context: &B::Context,
+ arguments: &MatmulArguments<B>,
+ bias: &B::NativeBuffer,
+ encoder: &B::ComputeEncoder,
+ ) -> Result<(), B::Error> {
+ let m = arguments.batch as usize;
+ let n = arguments.output_dim as usize;
+ let batch_count = arguments.batch_count as usize;
+ let total_len = m * n * batch_count;
+ if total_len == 0 {
+ return Ok(());
+ }
+
+ if self.bias_add.is_none() {
+ self.bias_add = Some(<B::Kernels as Kernels>::TensorAddBiasKernel::new(context, self.data_type)?);
+ }
+ let bias_add = self.bias_add.as_ref().unwrap();
+ bias_add.encode(arguments.d, bias, arguments.d, n as u32, total_len as u32, encoder);
+ Ok(())
+ }
+
+ pub fn apply_batch_collapse(arguments: &mut MatmulArguments<B>) {
+ if arguments.batch_count <= 1 {
+ return;
+ }
+ if arguments.lda == arguments.input_dim && arguments.transpose_b {
+ arguments.batch *= arguments.batch_count;
+ arguments.batch_count = 1;
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
similarity index 50%
rename from crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs
rename to crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
index b01d3cd5a..22ac9571f 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
@@ -1,15 +1,13 @@
-use metal::MTLBuffer;
-use objc2::{rc::Retained, runtime::ProtocolObject};
+use crate::backends::common::Backend;
#[derive(Debug, Clone)]
-pub struct MatmulArguments<'a> {
- pub a: &'a Retained<ProtocolObject<dyn MTLBuffer>>,
+pub struct MatmulArguments<'a, B: Backend> {
+ pub a: &'a B::NativeBuffer,
/// Byte offset into `a` (used for slicing the batch dimension).
pub a_offset: u64,
- pub b: &'a Retained<ProtocolObject<dyn MTLBuffer>>,
- pub c: Option<&'a Retained<ProtocolObject<dyn MTLBuffer>>>,
- pub d: &'a Retained<ProtocolObject<dyn MTLBuffer>>,
- pub bias: Option<&'a Retained<ProtocolObject<dyn MTLBuffer>>>,
+ pub b: &'a B::NativeBuffer,
+ pub d: &'a B::NativeBuffer,
+ pub bias: Option<&'a B::NativeBuffer>,
/// M dimension - batch/number of tokens (rows of A, rows of D)
pub batch: i32,
/// K dimension - input_dim/reduction dimension (cols of A, rows of B)
@@ -21,9 +19,5 @@ pub struct MatmulArguments<'a> {
pub ldd: i32,
/// Number of batched matrix multiplications (z-dimension)
pub batch_count: i32,
- /// Scaling factors for fused addmm (alpha * A @ B + beta * C)
- pub alpha: f32,
- pub beta: f32,
- pub transpose_a: bool,
pub transpose_b: bool,
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
index bc496f181..41fe391d7 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
@@ -1,6 +1,17 @@
+mod dispatch_descriptor;
mod full_precision;
+pub mod gemm;
+pub mod gemv;
+mod grid_size;
+mod kernel;
+mod matmul_arguments;
+pub mod split_k;
+pub use dispatch_descriptor::MatmulDispatchDescriptor;
pub use full_precision::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel};
+pub use grid_size::GridSize;
+pub use kernel::MatmulKernel;
+pub use matmul_arguments::MatmulArguments;
use super::Kernels;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs
similarity index 59%
rename from crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs
rename to crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs
index 21c0e7523..11cd3d489 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs
@@ -1,39 +1,29 @@
-use metal::MTLSize;
-
-use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};
+use super::{
+ super::{grid_size::GridSize, matmul_arguments::MatmulArguments},
+ specialization::Specialization,
+ tile_configuration::{TileConfiguration, select_tile_configuration},
+};
use crate::{
DataType,
- backends::metal::{
- context::MetalContext,
- error::MetalError,
- kernel::matmul::common::{GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments},
- },
+ backends::common::{Backend, gpu_types::GEMMSpiltKParams as SplitKGEMMParams},
};
#[derive(Debug, Clone)]
-pub(crate) struct DispatchDescriptor {
- pub(crate) pipeline_configuration: PipelineConfiguration,
- pub(crate) params: SplitKGEMMParams,
- pub(crate) partition_count: i32,
- pub(crate) output_elements_per_partition: i32,
- pub(crate) accumulator_bytes: usize,
- pub(crate) partial_threadgroups: MTLSize,
- pub(crate) partial_threads_per_threadgroup: MTLSize,
- pub(crate) accum_total_threads: MTLSize,
- pub(crate) accum_threads_per_threadgroup: MTLSize,
+pub struct DispatchDescriptor {
+ pub params: SplitKGEMMParams,
+ pub partition_count: i32,
+ pub output_elements_per_partition: i32,
+ pub accumulator_bytes: usize,
+ pub partial_threadgroups: GridSize,
+ pub accum_total_threads: GridSize,
}
impl DispatchDescriptor {
- pub(crate) fn try_new(
- _context: &MetalContext,
+ pub fn try_new<B: Backend>(
data_type: DataType,
- arguments: &MatmulArguments,
- ) -> Result<Option<Self>, MetalError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MetalError::Generic(format!("Unsupported dtype for Split-K: {:?}", data_type)));
- }
-
- if arguments.c.is_some() {
+ arguments: &MatmulArguments<B>,
+ ) -> Result<Option<Self>, B::Error> {
+ if !matches!(data_type, DataType::BF16) {
return Ok(None);
}
@@ -51,14 +41,17 @@ impl DispatchDescriptor {
let mn_aligned = (m % tile.tile_rows) == 0 && (n % tile.tile_cols) == 0;
let k_aligned = (k % tile.tile_depth) == 0;
- let pipeline_configuration = PipelineConfiguration {
+ let specialization = Specialization {
tile,
- transpose_a: arguments.transpose_a,
transpose_b: arguments.transpose_b,
mn_aligned,
k_aligned,
};
+ if !is_supported_specialization(&specialization) {
+ return Ok(None);
+ }
+
let tile_count_m = (m + tile.tile_rows - 1) / tile.tile_rows;
let tile_count_n = (n + tile.tile_cols - 1) / tile.tile_cols;
@@ -84,23 +77,25 @@ impl DispatchDescriptor {
gemm_k_iterations_aligned: gemm_k_iterations,
};
- let partial_threads_per_threadgroup =
- MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);
- let partial_threadgroups = MTLSize::new(tile_count_n as usize, tile_count_m as usize, partition_count as usize);
+ let partial_threadgroups = GridSize {
+ x: tile_count_n as usize,
+ y: tile_count_m as usize,
+ z: partition_count as usize,
+ };
- let accum_total_threads = MTLSize::new(n as usize, m as usize, 1);
- let accum_threads_per_threadgroup = MTLSize::new(16.min(n as usize), 16.min(m as usize), 1);
+ let accum_total_threads = GridSize {
+ x: n as usize,
+ y: m as usize,
+ z: 1,
+ };
Ok(Some(Self {
- pipeline_configuration,
params,
partition_count,
output_elements_per_partition,
accumulator_bytes,
partial_threadgroups,
- partial_threads_per_threadgroup,
accum_total_threads,
- accum_threads_per_threadgroup,
}))
}
}
@@ -135,3 +130,15 @@ fn should_use_splitk(
let k_tiles = k / 16;
(m_tiles * n_tiles) <= 32 && k_tiles >= 8
}
+
+fn is_supported_specialization(config: &Specialization) -> bool {
+ let supported_tile = TileConfiguration {
+ tile_rows: 16,
+ tile_cols: 32,
+ tile_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ };
+
+ config.tile == supported_tile && config.transpose_b && !config.mn_aligned && config.k_aligned
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
new file mode 100644
index 000000000..33b31a032
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
@@ -0,0 +1,142 @@
+use super::{super::matmul_arguments::MatmulArguments, dispatch_descriptor::DispatchDescriptor};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend, Context, Kernels,
+ kernel::{MatmulSplitKAccumBfloat16Kernel, MatmulSplitKPartialBfloat16Kernel},
+ },
+};
+
+pub struct SplitKKernel<B: Backend> {
+ data_type: DataType,
+ partial_bfloat16: Option<<B::Kernels as Kernels>::MatmulSplitKPartialBfloat16Kernel>,
+ accum_bfloat16: Option<<B::Kernels as Kernels>::MatmulSplitKAccumBfloat16Kernel>,
+ accumulator_buffer: Option<B::NativeBuffer>,
+ accumulator_buffer_bytes: usize,
+}
+
+impl<B: Backend> SplitKKernel<B>
+where
+ B::Error: From<String>,
+{
+ pub fn new(data_type: DataType) -> Result<Self, B::Error> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(B::Error::from(format!("Unsupported dtype for Split-K: {data_type:?}")));
+ }
+ Ok(Self {
+ data_type,
+ partial_bfloat16: None,
+ accum_bfloat16: None,
+ accumulator_buffer: None,
+ accumulator_buffer_bytes: 0,
+ })
+ }
+
+ pub fn precompile(
+ &mut self,
+ context: &B::Context,
+ ) -> Result<(), B::Error> {
+ if !matches!(self.data_type, DataType::BF16) {
+ return Ok(());
+ }
+ self.ensure_kernels(context)
+ }
+
+ fn ensure_kernels(
+ &mut self,
+ context: &B::Context,
+ ) -> Result<(), B::Error> {
+ if !matches!(self.data_type, DataType::BF16) {
+ return Err(B::Error::from(format!("Split-K currently supports BF16 only, got {:?}", self.data_type)));
+ }
+ if self.partial_bfloat16.is_none() {
+ self.partial_bfloat16 = Some(<B::Kernels as Kernels>::MatmulSplitKPartialBfloat16Kernel::new(context)?);
+ }
+ if self.accum_bfloat16.is_none() {
+ self.accum_bfloat16 = Some(<B::Kernels as Kernels>::MatmulSplitKAccumBfloat16Kernel::new(context)?);
+ }
+ Ok(())
+ }
+
+ pub fn encode(
+ &mut self,
+ context: &B::Context,
+ arguments: &MatmulArguments<B>,
+ dispatch_descriptor: &DispatchDescriptor,
+ encoder: &B::ComputeEncoder,
+ ) -> Result<(), B::Error> {
+ self.ensure_kernels(context)?;
+ self.ensure_accumulator_buffer(context, dispatch_descriptor.accumulator_bytes)?;
+ let accumulator_buffer =
+ self.accumulator_buffer.as_ref().cloned().expect("Accumulator buffer must be initialized");
+
+ let partial_group_count_x = u32::try_from(dispatch_descriptor.partial_threadgroups.x).map_err(|_| {
+ B::Error::from(format!(
+ "Split-K partial group count x overflows u32: {}",
+ dispatch_descriptor.partial_threadgroups.x
+ ))
+ })?;
+ let partial_group_count_y = u32::try_from(dispatch_descriptor.partial_threadgroups.y).map_err(|_| {
+ B::Error::from(format!(
+ "Split-K partial group count y overflows u32: {}",
+ dispatch_descriptor.partial_threadgroups.y
+ ))
+ })?;
+ let partial_group_count_z = u32::try_from(dispatch_descriptor.partial_threadgroups.z).map_err(|_| {
+ B::Error::from(format!(
+ "Split-K partial group count z overflows u32: {}",
+ dispatch_descriptor.partial_threadgroups.z
+ ))
+ })?;
+ let partial = self.partial_bfloat16.as_ref().unwrap();
+ partial.encode(
+ (arguments.a, arguments.a_offset as usize),
+ arguments.b,
+ &accumulator_buffer,
+ std::slice::from_ref(&dispatch_descriptor.params),
+ partial_group_count_x,
+ partial_group_count_y,
+ partial_group_count_z,
+ encoder,
+ );
+
+ let accum_total_threads_x = u32::try_from(dispatch_descriptor.accum_total_threads.x).map_err(|_| {
+ B::Error::from(format!(
+ "Split-K accum total threads x overflows u32: {}",
+ dispatch_descriptor.accum_total_threads.x
+ ))
+ })?;
+ let accum_total_threads_y = u32::try_from(dispatch_descriptor.accum_total_threads.y).map_err(|_| {
+ B::Error::from(format!(
+ "Split-K accum total threads y overflows u32: {}",
+ dispatch_descriptor.accum_total_threads.y
+ ))
+ })?;
+ let accum = self.accum_bfloat16.as_ref().unwrap();
+ accum.encode(
+ &accumulator_buffer,
+ arguments.d,
+ dispatch_descriptor.partition_count,
+ dispatch_descriptor.output_elements_per_partition,
+ arguments.ldd,
+ accum_total_threads_x,
+ accum_total_threads_y,
+ encoder,
+ );
+
+ Ok(())
+ }
+
+ fn ensure_accumulator_buffer(
+ &mut self,
+ context: &B::Context,
+ required_bytes: usize,
+ ) -> Result<(), B::Error> {
+ if required_bytes <= self.accumulator_buffer_bytes && self.accumulator_buffer.is_some() {
+ return Ok(());
+ }
+ self.accumulator_buffer = Some(context.create_buffer(required_bytes)?);
+ self.accumulator_buffer_bytes = required_bytes;
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs
new file mode 100644
index 000000000..0fcefe437
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs
@@ -0,0 +1,7 @@
+mod dispatch_descriptor;
+mod kernel;
+mod specialization;
+mod tile_configuration;
+
+pub use dispatch_descriptor::DispatchDescriptor;
+pub use kernel::SplitKKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/specialization.rs
similarity index 77%
rename from crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs
rename to crates/uzu/src/backends/common/kernel/matmul/split_k/specialization.rs
index 1eab77254..dcd6b7e60 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/specialization.rs
@@ -1,9 +1,8 @@
use super::tile_configuration::TileConfiguration;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct PipelineConfiguration {
+pub struct Specialization {
pub tile: TileConfiguration,
- pub transpose_a: bool,
pub transpose_b: bool,
pub mn_aligned: bool,
pub k_aligned: bool,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/tile_configuration.rs
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rs
rename to crates/uzu/src/backends/common/kernel/matmul/split_k/tile_configuration.rs
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index 87dadbc6c..ca3e45905 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -156,18 +156,6 @@ impl MetalContext {
}
pub fn compute_pipeline_state(
- &self,
- function_name: &str,
- constants: Option<&MTLFunctionConstantValues>,
- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {
- // Only cache pipelines without constants
- if constants.is_some() {
- return self.library.compute_pipeline_state(function_name, constants);
- }
- self.compute_pipeline_state_cached(function_name, function_name, None)
- }
-
- pub fn compute_pipeline_state_cached(
&self,
cache_key: &str,
function_name: &str,
diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.h
index 0fe96df40..db0130600 100644
--- a/crates/uzu/src/backends/metal/generated/matmul.h
+++ b/crates/uzu/src/backends/metal/generated/matmul.h
@@ -48,32 +48,6 @@ typedef struct {
int32_t gemm_k_iterations_aligned;
} GEMMSpiltKParams;
-/**Split-K MLP Fused GEMM parameters. */
-typedef struct {
- int32_t M;
- int32_t N;
- int32_t K;
- int32_t lda;
- int32_t ldb;
- int32_t ldc;
- int32_t tiles_n;
- int32_t tiles_m;
- int32_t split_k_partitions;
- int32_t split_k_partition_stride;
- int32_t split_k_partition_size;
- int32_t gemm_k_iterations_aligned;
- int32_t hidden_dim;
-} GEMMSpiltKMlpFusedParams;
-
-/**AddMM parameters (alpha * A @ B + beta * C). */
-typedef struct {
- int32_t ldc;
- int32_t fdc;
- int64_t batch_stride_c;
- float alpha;
- float beta;
-} GEMMAddMMParams;
-
#ifdef __METAL_VERSION__
} // namespace matmul
} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
index 6751a3863..3ce63cb23 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
@@ -1,7 +1 @@
-mod matmul_arguments;
-mod transpose_configuration;
-
-pub use matmul_arguments::MatmulArguments;
-pub(crate) use transpose_configuration::transpose_configuration;
-
-pub use crate::backends::common::gpu_types::{GEMMAddMMParams, GEMMParams, GEMMSpiltKMlpFusedParams, GEMMSpiltKParams};
+pub use crate::backends::common::gpu_types::{GEMMParams, GEMMSpiltKParams};
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h
index bb8c8b8b2..40056c256 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h
@@ -4,294 +4,74 @@
#include "mma.h"
#include "params.h"
#include "transforms.h"
-#include "../utils.h"
using namespace metal;
///////////////////////////////////////////////////////////////////////////////
-// GEMM kernel class
+// GEMM kernel helpers
///////////////////////////////////////////////////////////////////////////////
namespace steel {
-template <bool M_aligned, bool N_aligned, bool K_aligned>
-struct LoopAlignment {};
-
template <
typename T,
typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- bool MN_aligned,
- bool K_aligned,
- typename AccumType = typename AccumHelper<T>::accum_type,
+ typename AccumType = float,
typename Epilogue = TransformNone<U, AccumType>>
-struct GEMMKernel {
- STEEL_CONST short tgp_padding_a = 16 / sizeof(T);
- STEEL_CONST short tgp_padding_b = 16 / sizeof(T);
- STEEL_CONST short tgp_mem_size_a =
- transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);
- STEEL_CONST short tgp_mem_size_b =
- transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b);
- STEEL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;
-
- STEEL_CONST short tgp_size = WM * WN * 32;
-
- using loader_a_t = BlockLoader<
- T,
- transpose_a ? BK : BM,
- transpose_a ? BM : BK,
- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
- !transpose_a,
- tgp_size>;
- using loader_b_t = BlockLoader<
- T,
- transpose_b ? BN : BK,
- transpose_b ? BK : BN,
- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
- transpose_b,
- tgp_size>;
- using mma_t = BlockMMA<
- T,
- U,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
- AccumType,
- Epilogue>;
-
- /* Main kernel function */
- template <bool M_aligned, bool N_aligned, bool K_aligned_>
- static METAL_FUNC void gemm_loop(
- threadgroup T* As [[threadgroup(0)]],
- threadgroup T* Bs [[threadgroup(1)]],
- const int gemm_k_iterations,
- thread loader_a_t& loader_a,
- thread loader_b_t& loader_b,
- thread mma_t& mma_op,
- thread const short& tgp_bm,
- thread const short& tgp_bn,
- thread const short& lbk,
- LoopAlignment<M_aligned, N_aligned, K_aligned_> l = {}
- ) {
- // Appease the compiler
- (void)l;
-
- short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);
-
- short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);
-
- for (int k = 0; k < gemm_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- // Load elements into threadgroup
- if (M_aligned) {
- loader_a.load_unsafe();
- } else {
- loader_a.load_safe(tile_dims_A);
- }
-
- if (N_aligned) {
- loader_b.load_unsafe();
- } else {
- loader_b.load_safe(tile_dims_B);
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
+METAL_FUNC void gemm_loop(
+ threadgroup T* As,
+ threadgroup T* Bs,
+ const int gemm_k_iterations,
+ thread BlockLoader<T>& loader_a,
+ thread BlockLoader<T>& loader_b,
+ thread BlockMMA<T, U, AccumType, Epilogue>& mma_op,
+ thread const short& tgp_bm,
+ thread const short& tgp_bn,
+ thread const short& lbk,
+ const short BK,
+ const bool transpose_a,
+ const bool transpose_b,
+ const bool align_m,
+ const bool align_n,
+ const bool align_k
+) {
+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);
+ short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);
+
+ for (int k = 0; k < gemm_k_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (align_m) {
+ loader_a.load_unsafe();
+ } else {
+ loader_a.load_safe(tile_dims_A);
}
- if (!K_aligned_) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- short2 tile_dims_A_last =
- transpose_a ? short2(tgp_bm, lbk) : short2(lbk, tgp_bm);
- short2 tile_dims_B_last =
- transpose_b ? short2(lbk, tgp_bn) : short2(tgp_bn, lbk);
-
- loader_a.load_safe(tile_dims_A_last);
- loader_b.load_safe(tile_dims_B_last);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_op.mma(As, Bs);
+ if (align_n) {
+ loader_b.load_unsafe();
+ } else {
+ loader_b.load_safe(tile_dims_B);
}
- }
-
- /* Main kernel function */
- static METAL_FUNC void run(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- device U* D [[buffer(2)]],
- const constant GEMMParams* params [[buffer(3)]],
- threadgroup T* As [[threadgroup(0)]],
- threadgroup T* Bs [[threadgroup(1)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]]
- ) {
- // Pacifying compiler
- (void)lid;
-
- const int tid_y = ((tid.y) << params->swizzle_log) +
- ((tid.x) & ((1 << params->swizzle_log) - 1));
- const int tid_x = (tid.x) >> params->swizzle_log;
-
- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
- return;
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- // Find block in A, B, C
- const int c_row = tid_y * BM;
- const int c_col = tid_x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
- A += transpose_a ? c_row_long : c_row_long * params->lda;
- B += transpose_b ? c_col_long * params->ldb : c_col_long;
- D += c_row_long * params->ldd + c_col_long;
-
- // Prepare threadgroup loading operations
- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);
-
- // Prepare threadgroup mma operation
- thread mma_t mma_op(simd_group_id, simd_lane_id);
-
- int gemm_k_iterations = params->gemm_k_iterations_aligned;
-
- ///////////////////////////////////////////////////////////////////////////////
- // MNK aligned loop
- if (MN_aligned) {
- for (int k = 0; k < gemm_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- // Load elements into threadgroup
- loader_a.load_unsafe();
- loader_b.load_unsafe();
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- // Loop tail
- if (!K_aligned) {
- int lbk = params->K - params->gemm_k_iterations_aligned * BK;
- short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);
- short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);
-
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_op.mma(As, Bs);
- }
-
- // Store results to device memory
- mma_op.store_result(D, params->ldd);
- return;
-
- }
- ///////////////////////////////////////////////////////////////////////////////
- // MN unaligned loop
- else { // Loop over K - unaligned case
- short tgp_bm = min(BM, params->M - c_row);
- short tgp_bn = min(BN, params->N - c_col);
- short leftover_bk = params->K - params->gemm_k_iterations_aligned * BK;
-
- if (tgp_bm == BM && tgp_bn == BN) {
- gemm_loop<true, true, K_aligned>(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk
- );
-
- mma_op.store_result(D, params->ldd);
- return;
-
- } else if (tgp_bn == BN) {
- gemm_loop<false, true, K_aligned>(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk
- );
-
- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
- return;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(As, Bs);
+ loader_a.next();
+ loader_b.next();
+ }
- } else if (tgp_bm == BM) {
- gemm_loop<true, false, K_aligned>(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk
- );
+ if (!align_k) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
- return;
+ short2 tile_dims_A_last =
+ transpose_a ? short2(tgp_bm, lbk) : short2(lbk, tgp_bm);
+ short2 tile_dims_B_last =
+ transpose_b ? short2(lbk, tgp_bn) : short2(tgp_bn, lbk);
- } else {
- gemm_loop<false, false, K_aligned>(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk
- );
+ loader_a.load_safe(tile_dims_A_last);
+ loader_b.load_safe(tile_dims_B_last);
- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
- return;
- }
- }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(As, Bs);
}
-};
+}
} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.h
deleted file mode 100644
index 39f560249..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.h
+++ /dev/null
@@ -1,159 +0,0 @@
-
-
-#pragma once
-
-#include "nax.h"
-#include "params.h"
-#include "transforms.h"
-#include "../utils.h"
-
-using namespace metal;
-
-namespace steel {
-
-template <
- typename T,
- short SM,
- short SN,
- short SK,
- short BK,
- bool transpose_a,
- bool transpose_b,
- bool kAlignedM,
- bool kAlignedN,
- bool kAlignedK,
- short UM,
- short UN,
- short UK,
- typename AccumType = float>
-auto gemm_loop(
- const device T* A,
- const device T* B,
- const constant GEMMParams* params [[buffer(4)]],
- const short sgp_sm,
- const short sgp_sn
-) {
- constexpr short TM = SM / UM;
- constexpr short TN = SN / UN;
- constexpr short TK = SK / UK;
-
- constexpr int RA = transpose_a ? TK : TM;
- constexpr int CA = transpose_a ? TM : TK;
-
- constexpr int RB = transpose_b ? TN : TK;
- constexpr int CB = transpose_b ? TK : TN;
-
- using DSubTile = NAXSubTile<AccumType, UM, UN>;
- using ASubTile =
- NAXSubTile<T, (transpose_a ? UK : UM), (transpose_a ? UM : UK)>;
- using BSubTile =
- NAXSubTile<T, (transpose_b ? UN : UK), (transpose_b ? UK : UN)>;
-
- NAXTile<AccumType, TM, TN, DSubTile> Dtile;
- Dtile.clear();
-
- int gemm_k_iterations_ = params->gemm_k_iterations_aligned;
-
- STEEL_PRAGMA_NO_UNROLL
- for (int kk0 = 0; kk0 < gemm_k_iterations_; kk0++) {
- threadgroup_barrier(mem_flags::mem_none);
-
- STEEL_PRAGMA_NO_UNROLL
- for (int kk1 = 0; kk1 < BK; kk1 += SK) {
- NAXTile<T, RA, CA, ASubTile> Atile;
- NAXTile<T, RB, CB, BSubTile> Btile;
- const int k = kk1;
-
- volatile int compiler_barrier;
-
- const int A_offset = transpose_a ? k * params->lda : k;
- const int B_offset = transpose_b ? k : k * params->ldb;
-
- if constexpr (kAlignedM) {
- Atile.load(A + A_offset, params->lda);
- } else {
- const short rmax = transpose_a ? SK : sgp_sm;
- const short cmax = transpose_a ? sgp_sm : SK;
- Atile.load_safe(A + A_offset, params->lda, short2(cmax, rmax));
- }
-
- if constexpr (kAlignedN) {
- Btile.load(B + B_offset, params->ldb);
- } else {
- const short rmax = transpose_b ? sgp_sn : SK;
- const short cmax = transpose_b ? SK : sgp_sn;
- Btile.load_safe(B + B_offset, params->ldb, short2(cmax, rmax));
- }
-
- tile_matmad_nax(
- Dtile,
- Atile,
- metal::bool_constant<transpose_a>{},
- Btile,
- metal::bool_constant<transpose_b>{}
- );
-
- (void)compiler_barrier;
- }
-
- A += transpose_a ? (BK * params->lda) : BK;
- B += transpose_b ? BK : (BK * params->ldb);
- }
-
- if constexpr (!kAlignedK) {
- simdgroup_barrier(mem_flags::mem_none);
-
- const short rem_bk = params->K - gemm_k_iterations_ * BK;
-
- STEEL_PRAGMA_NO_UNROLL
- for (int kk1 = 0; kk1 < rem_bk; kk1 += SK) {
- NAXTile<T, 1, 1, ASubTile> Atile;
- NAXTile<T, 1, 1, BSubTile> Btile;
-
- STEEL_PRAGMA_UNROLL
- for (int mm = 0; mm < TM; mm++) {
- STEEL_PRAGMA_UNROLL
- for (int nn = 0; nn < TN; nn++) {
- STEEL_PRAGMA_UNROLL
- for (int kk = 0; kk < TK; kk++) {
- const int m = mm * UM;
- const int n = nn * UN;
- const int k = kk1 + kk * UK;
- const short psk = max(0, rem_bk - k);
-
- const int A_offset =
- transpose_a ? (m + k * params->lda) : (m * params->lda + k);
- const int B_offset =
- transpose_b ? (k + n * params->ldb) : (k * params->ldb + n);
-
- {
- const short psm = kAlignedM ? SM : max(0, sgp_sm - m);
- const short rmax = transpose_a ? psk : psm;
- const short cmax = transpose_a ? psm : psk;
- Atile.load_safe(A + A_offset, params->lda, short2(cmax, rmax));
- }
-
- {
- const short psn = kAlignedN ? SN : max(0, sgp_sn - n);
- const short rmax = transpose_b ? psn : psk;
- const short cmax = transpose_b ? psk : psn;
- Btile.load_safe(B + B_offset, params->ldb, short2(cmax, rmax));
- }
-
- subtile_matmad_nax(
- Dtile.subtile_at(mm, nn),
- Atile.subtile_at(0, 0),
- metal::bool_constant<transpose_a>{},
- Btile.subtile_at(0, 0),
- metal::bool_constant<transpose_b>{}
- );
- }
- }
- }
- }
- }
-
- return Dtile;
-}
-
-} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h
index 7aab96e78..4aa7cc56c 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h
@@ -10,27 +10,19 @@
namespace steel {
-template <
- typename T,
- short BROWS,
- short BCOLS,
- short dst_ld,
- short reduction_dim,
- short tgp_size,
- short alignment = 1,
- short n_reads = (BCOLS * BROWS) / (tgp_size),
- short TCOLS = BCOLS / n_reads,
- short TROWS = tgp_size / TCOLS>
+template <typename T>
struct BlockLoader {
- STEEL_CONST short n_rows = (BROWS + TROWS - 1) / TROWS;
- STEEL_CONST short vec_size = n_reads;
+ // Tile params
+ const short BROWS;
+ const short dst_ld;
+ const short vec_size;
+ const short TROWS;
// Leading dimension for src
const int src_ld;
const int tile_stride;
// Thread location indices
- const short thread_idx;
const short bi;
const short bj;
@@ -38,41 +30,35 @@ struct BlockLoader {
threadgroup T* dst;
const device T* src;
- struct alignas(alignment * sizeof(T)) ReadVector {
- uint8_t v[sizeof(T) * vec_size];
- };
-
/* Constructor */
METAL_FUNC BlockLoader(
const device T* src_,
const int src_ld_,
threadgroup T* dst_,
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
+ ushort simd_group_id,
+ ushort simd_lane_id,
+ short BROWS_,
+ short BCOLS_,
+ short dst_ld_,
+ short reduction_dim,
+ short tgp_size
)
- : src_ld(src_ld_), tile_stride(reduction_dim ? BCOLS : BROWS * src_ld),
- thread_idx(simd_group_id * 32 + simd_lane_id), bi(thread_idx / TCOLS),
- bj(vec_size * (thread_idx % TCOLS)), dst(dst_ + bi * dst_ld + bj),
- src(src_ + bi * src_ld + bj) {}
-
- /* Apply operation to threadgroup without bound checking */
- template <typename UnaryOp>
- METAL_FUNC void apply_inplace_op(thread const UnaryOp& op) const {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < BROWS; i += TROWS) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- dst[i * dst_ld + j] = op.apply(dst[i * dst_ld + j]);
- }
- }
- }
+ : BROWS(BROWS_), dst_ld(dst_ld_), vec_size((BCOLS_ * BROWS_) / tgp_size),
+ TROWS(tgp_size / (BCOLS_ / ((BCOLS_ * BROWS_) / tgp_size))),
+ src_ld(src_ld_), tile_stride(reduction_dim ? BCOLS_ : BROWS_ * src_ld_),
+ bi(short(simd_group_id * 32 + simd_lane_id) /
+ short(BCOLS_ / ((BCOLS_ * BROWS_) / tgp_size))),
+ bj(short((BCOLS_ * BROWS_) / tgp_size) *
+ (short(simd_group_id * 32 + simd_lane_id) %
+ short(BCOLS_ / ((BCOLS_ * BROWS_) / tgp_size)))),
+ dst(dst_ + bi * dst_ld_ + bj), src(src_ + bi * src_ld_ + bj) {}
/* Load from device memory into threadgroup memory - without bound checking */
METAL_FUNC void load_unsafe() const {
- STEEL_PRAGMA_UNROLL
for (short i = 0; i < BROWS; i += TROWS) {
- *((threadgroup ReadVector*)(&dst[i * dst_ld])) =
- *((const device ReadVector*)(&src[i * src_ld]));
+ for (short j = 0; j < vec_size; j++) {
+ dst[i * dst_ld + j] = src[i * src_ld + j];
+ }
}
}
@@ -80,11 +66,8 @@ struct BlockLoader {
METAL_FUNC void load_safe(short2 src_tile_dim) const {
src_tile_dim = src_tile_dim - short2(bj, bi);
- // Skip loading if thread has no valid reads
if (src_tile_dim.x <= 0 || src_tile_dim.y <= 0) {
- STEEL_PRAGMA_UNROLL
for (short i = 0; i < BROWS; i += TROWS) {
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < vec_size; j++) {
dst[i * dst_ld + j] = T(0);
}
@@ -92,34 +75,11 @@ struct BlockLoader {
return;
}
- // Use fast thread memory for bound checks
- bool tmp_idx[vec_size];
- T tmp_val[vec_size];
-
- STEEL_PRAGMA_UNROLL
for (short i = 0; i < BROWS; i += TROWS) {
- // Make sure tmp_idx only contains valid indices
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- tmp_idx[j] = (i < src_tile_dim.y) && (j < src_tile_dim.x);
- }
-
- // Read valid indices into tmp_val
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- tmp_val[j] = src[(tmp_idx[j] ? i * src_ld + j : 0)];
- }
-
- // Zero out unneeded values
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- tmp_val[j] = tmp_idx[j] ? tmp_val[j] : T(0);
- }
-
- // Copy values to threadgroup memory
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < vec_size; j++) {
- dst[i * dst_ld + j] = tmp_val[j];
+ bool valid = (i < src_tile_dim.y) && (j < src_tile_dim.x);
+ T val = src[(valid ? i * src_ld + j : 0)];
+ dst[i * dst_ld + j] = valid ? val : T(0);
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h
index 3bf58387a..989b22008 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h
@@ -7,8 +7,8 @@
#include <metal_stdlib>
#include "../defines.h"
+#include "../utils/type_traits.h"
#include "transforms.h"
-#include "../utils/integral_constant.h"
using namespace metal;
@@ -18,20 +18,16 @@ using namespace metal;
namespace steel {
-template <typename T, int kFragRows_, int kFragCols_>
-struct BaseMMAFrag {
- static_assert(
- kFragRows_ == 8,
- "Only 8 x 8 fragment matrices are currently supported"
- );
- static_assert(
- kFragCols_ == 8,
- "Only 8 x 8 fragment matrices are currently supported"
- );
-};
+// Max tile dimensions across all configurations.
+// GEMM: TM up to 8 (64/(8*1)), TN up to 4 (64/(8*2))
+// Split-K: TM up to 1, TN up to 2
+// tile_matmad K dim is always 1
+#define STEEL_MAX_TM 8
+#define STEEL_MAX_TN 4
+#define STEEL_MAX_CTILE_FRAGS (STEEL_MAX_TM * STEEL_MAX_TN) // 32
template <typename T>
-struct BaseMMAFrag<T, 8, 8> {
+struct BaseMMAFrag {
STEEL_CONST int kFragRows = 8;
STEEL_CONST int kFragCols = 8;
@@ -40,11 +36,6 @@ struct BaseMMAFrag<T, 8, 8> {
STEEL_CONST int kElemRows = 1;
STEEL_CONST int kElemCols = 2;
- static_assert(
- kElemRows * kElemCols == kElemsPerFrag,
- "MMAFrag shape is not consistent with MMAFrag size"
- );
-
typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;
typedef metal::vec<T, kElemsPerFrag> frag_type;
@@ -57,12 +48,12 @@ struct BaseMMAFrag<T, 8, 8> {
return short2{fn, fm};
}
- template <typename SrcPtrType, typename StrX, typename StrY>
+ template <typename SrcPtrType>
METAL_FUNC static constexpr void load(
thread frag_type& dst,
SrcPtrType src,
- StrX str_x,
- StrY str_y
+ int str_x,
+ int str_y
) {
STEEL_PRAGMA_UNROLL
for (short i = 0; i < kElemRows; i++) {
@@ -73,23 +64,16 @@ struct BaseMMAFrag<T, 8, 8> {
}
}
- template <
- typename SrcPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX,
- typename OffY>
+ template <typename SrcPtrType>
METAL_FUNC static constexpr void load_safe(
thread frag_type& dst,
SrcPtrType src,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = Int<0>{},
- OffY off_y = Int<0>{}
+ int str_x,
+ int str_y,
+ int lim_x,
+ int lim_y,
+ int off_x = 0,
+ int off_y = 0
) {
STEEL_PRAGMA_UNROLL
for (short i = 0; i < kElemRows; i++) {
@@ -105,12 +89,12 @@ struct BaseMMAFrag<T, 8, 8> {
}
}
- template <typename DstPtrType, typename StrX, typename StrY>
+ template <typename DstPtrType>
METAL_FUNC static constexpr void store(
const thread frag_type& src,
DstPtrType dst,
- StrX str_x,
- StrY str_y
+ int str_x,
+ int str_y
) {
using U = pointer_element_t<DstPtrType>;
@@ -123,23 +107,16 @@ struct BaseMMAFrag<T, 8, 8> {
}
}
- template <
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX,
- typename OffY>
+ template <typename DstPtrType>
METAL_FUNC static constexpr void store_safe(
const thread frag_type& src,
DstPtrType dst,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = Int<0>{},
- OffY off_y = Int<0>{}
+ int str_x,
+ int str_y,
+ int lim_x,
+ int lim_y,
+ int off_x = 0,
+ int off_y = 0
) {
using U = pointer_element_t<DstPtrType>;
@@ -155,27 +132,18 @@ struct BaseMMAFrag<T, 8, 8> {
}
}
- template <
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename StartX,
- typename StopX,
- typename StartY,
- typename StopY,
- typename OffX,
- typename OffY>
+ template <typename DstPtrType>
METAL_FUNC static constexpr void store_slice(
const thread frag_type& src,
DstPtrType dst,
- StrX str_x,
- StrY str_y,
- StartX start_x,
- StopX stop_x,
- StartY start_y,
- StopY stop_y,
- OffX off_x = Int<0>{},
- OffY off_y = Int<0>{}
+ int str_x,
+ int str_y,
+ int start_x,
+ int stop_x,
+ int start_y,
+ int stop_y,
+ int off_x = 0,
+ int off_y = 0
) {
using U = pointer_element_t<DstPtrType>;
@@ -222,37 +190,34 @@ struct BaseMMAFrag<T, 8, 8> {
}
};
-template <
- typename T,
- int kTileRows_,
- int kTileCols_,
- class MMAFrag_ = BaseMMAFrag<T, 8, 8>>
+template <typename T>
struct MMATile {
- using MMAFrag_t = MMAFrag_;
+ using MMAFrag_t = BaseMMAFrag<T>;
using elem_type = T;
STEEL_CONST int kFragRows = MMAFrag_t::kFragRows;
STEEL_CONST int kFragCols = MMAFrag_t::kFragCols;
STEEL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;
- STEEL_CONST int kTileRows = kTileRows_;
- STEEL_CONST int kTileCols = kTileCols_;
-
- STEEL_CONST int kRows = kTileRows * kFragRows;
- STEEL_CONST int kCols = kTileCols * kFragCols;
-
- STEEL_CONST int kNumFrags = kTileRows * kTileCols;
- STEEL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;
-
typedef typename MMAFrag_t::mat_type mat_type;
typedef typename MMAFrag_t::frag_type frag_type;
- frag_type val_frags[kNumFrags] = {frag_type(0)};
+ // Max-sized array — runtime kTileRows/kTileCols control how much is used
+ frag_type val_frags[STEEL_MAX_CTILE_FRAGS] = {};
- METAL_FUNC MMATile() thread {}
+ short kTileRows;
+ short kTileCols;
+ short kNumFrags;
+
+ METAL_FUNC MMATile() thread : kTileRows(0), kTileCols(0), kNumFrags(0) {}
+
+ METAL_FUNC MMATile(short tile_rows, short tile_cols) thread
+ : kTileRows(tile_rows),
+ kTileCols(tile_cols),
+ kNumFrags(tile_rows* tile_cols) {}
METAL_FUNC constexpr void clear() {
STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kNumFrags; ++i) {
+ for (short i = 0; i < STEEL_MAX_CTILE_FRAGS; ++i) {
val_frags[i] = frag_type(0);
}
}
@@ -268,15 +233,6 @@ struct MMATile {
return val_frags[i * kTileCols + j];
}
- METAL_FUNC mat_type mat_at(const short i, const short j) {
- mat_type val_mat;
- STEEL_PRAGMA_UNROLL
- for (short ii = 0; ii < kElemsPerFrag; ++ii) {
- val_mat.thread_elements()[ii] = frag_at(i, j)[ii];
- }
- return val_mat;
- }
-
METAL_FUNC thread elem_type* elems() {
return reinterpret_cast<thread elem_type*>(val_frags);
}
@@ -285,11 +241,17 @@ struct MMATile {
return reinterpret_cast<const thread elem_type*>(val_frags);
}
- template <typename U, int w_x, int w_y, int str_x, int str_y>
- METAL_FUNC void load(const threadgroup U* src) {
- STEEL_PRAGMA_UNROLL
+ METAL_FUNC short elems_per_tile() const { return kNumFrags * kElemsPerFrag; }
+
+ template <typename U>
+ METAL_FUNC void load(
+ const threadgroup U* src,
+ int w_x,
+ int w_y,
+ int str_x,
+ int str_y
+ ) {
for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < kTileCols; ++j) {
MMAFrag_t::load(
frag_at(i, j),
@@ -297,18 +259,22 @@ struct MMATile {
src[(i * kFragRows) * w_x * str_x +
(j * kFragCols) * w_y * str_y]
),
- Int<str_x>{},
- Int<str_y>{}
+ str_x,
+ str_y
);
}
}
}
- template <typename U, int w_x, int w_y, int str_x, int str_y>
- METAL_FUNC void store(threadgroup U* dst) const {
- STEEL_PRAGMA_UNROLL
+ template <typename U>
+ METAL_FUNC void store(
+ threadgroup U* dst,
+ int w_x,
+ int w_y,
+ int str_x,
+ int str_y
+ ) const {
for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < kTileCols; ++j) {
MMAFrag_t::store(
frag_at(i, j),
@@ -316,60 +282,56 @@ struct MMATile {
dst[(i * kFragRows) * w_x * str_x +
(j * kFragCols) * w_y * str_y]
),
- Int<str_x>{},
- Int<str_y>{}
+ str_x,
+ str_y
);
}
}
}
- template <typename U, int w_x, int w_y>
- METAL_FUNC void load(const device U* src, const int ld) {
- STEEL_PRAGMA_UNROLL
+ template <typename U>
+ METAL_FUNC void load(const device U* src, const int ld, int w_x, int w_y) {
for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < kTileCols; ++j) {
MMAFrag_t::load(
frag_at(i, j),
&(src[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
ld,
- Int<1>{}
+ 1
);
}
}
}
- template <typename U, int w_x, int w_y>
- METAL_FUNC void store(device U* dst, const int ld) const {
- STEEL_PRAGMA_UNROLL
+ template <typename U>
+ METAL_FUNC void store(device U* dst, const int ld, int w_x, int w_y) const {
for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < kTileCols; ++j) {
MMAFrag_t::store(
frag_at(i, j),
&(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
ld,
- Int<1>{}
+ 1
);
}
}
}
- template <typename U, int w_x, int w_y>
+ template <typename U>
METAL_FUNC void load_safe(
const device U* src,
const int ld,
- const short2 src_tile_dims
+ const short2 src_tile_dims,
+ int w_x,
+ int w_y
) {
- STEEL_PRAGMA_UNROLL
for (int i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
for (int j = 0; j < kTileCols; ++j) {
MMAFrag_t::load_safe(
frag_at(i, j),
src,
ld,
- Int<1>{},
+ 1,
src_tile_dims.y,
src_tile_dims.x,
(i * kFragRows) * w_x,
@@ -379,21 +341,21 @@ struct MMATile {
}
}
- template <typename U, int w_x, int w_y>
+ template <typename U>
METAL_FUNC void store_safe(
device U* dst,
const int ld,
- const short2 dst_tile_dims
+ const short2 dst_tile_dims,
+ int w_x,
+ int w_y
) const {
- STEEL_PRAGMA_UNROLL
for (int i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
for (int j = 0; j < kTileCols; ++j) {
MMAFrag_t::store_safe(
frag_at(i, j),
dst,
ld,
- Int<1>{},
+ 1,
dst_tile_dims.y,
dst_tile_dims.x,
(i * kFragRows) * w_x,
@@ -403,22 +365,22 @@ struct MMATile {
}
}
- template <typename U, int w_x, int w_y>
+ template <typename U>
METAL_FUNC void store_slice(
device U* dst,
const int ld,
const short2 start,
- const short2 stop
+ const short2 stop,
+ int w_x,
+ int w_y
) const {
- STEEL_PRAGMA_UNROLL
for (int i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
for (int j = 0; j < kTileCols; ++j) {
MMAFrag_t::store_slice(
frag_at(i, j),
dst,
ld,
- Int<1>{},
+ 1,
start.y,
stop.y,
start.x,
@@ -431,21 +393,18 @@ struct MMATile {
}
};
-template <typename T, typename U, int M, int N, int K>
+template <typename T, typename U>
METAL_FUNC void tile_matmad(
- thread MMATile<T, M, N>& D,
- thread MMATile<U, M, K>& A,
- thread MMATile<U, K, N>& B,
- thread MMATile<T, M, N>& C
+ thread MMATile<T>& D,
+ thread MMATile<U>& A,
+ thread MMATile<U>& B,
+ thread MMATile<T>& C
) {
- STEEL_PRAGMA_UNROLL
- for (short m = 0; m < M; ++m) {
- STEEL_PRAGMA_UNROLL
- for (short n = 0; n < N; ++n) {
- short n_serp = (m % 2) ? (N - 1 - n) : n;
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < K; ++k) {
- MMATile<T, M, N>::MMAFrag_t::mma(
+ for (short m = 0; m < A.kTileRows; ++m) {
+ for (short n = 0; n < B.kTileCols; ++n) {
+ short n_serp = (m % 2) ? (B.kTileCols - 1 - n) : n;
+ for (short k = 0; k < A.kTileCols; ++k) {
+ BaseMMAFrag<T>::mma(
D.frag_at(m, n_serp),
A.frag_at(m, k),
B.frag_at(k, n_serp),
@@ -465,62 +424,60 @@ struct TransformNone<complex64_t, InT> {
template <
typename T,
typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- short lda_tgp,
- short ldb_tgp,
typename AccumType = float,
typename Epilogue = TransformNone<U, AccumType>>
struct BlockMMA {
- // MMAFrag size
STEEL_CONST short kFragSize = 8;
- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
+ using MMAFrag_acc_t = BaseMMAFrag<AccumType>;
- // Warp tile simdgroup matrix strides along M
- STEEL_CONST short TM_stride = kFragSize * WM;
- // Warp tile simdgroup matrix strides along M
- STEEL_CONST short TN_stride = kFragSize * WN;
+ // Tile params (set at construction)
+ short BM, BN, BK, WM, WN;
+ bool transpose_a, transpose_b;
+ short lda_tgp, ldb_tgp;
- // Warp tile size along M
- STEEL_CONST short TM = BM / (kFragSize * WM);
- // Warp tile size along N
- STEEL_CONST short TN = BN / (kFragSize * WN);
-
- // Threadgroup A strides
- STEEL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
- STEEL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K
-
- // Threadgroup B strides
- STEEL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K
- STEEL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
-
- // Threadgroup strides along K
- STEEL_CONST short tile_stride_a = kFragSize * A_str_k;
- STEEL_CONST short tile_stride_b = kFragSize * B_str_k;
+ // Derived values
+ short TM_stride, TN_stride;
+ short TM, TN;
+ short A_str_m, A_str_k;
+ short B_str_k, B_str_n;
+ short tile_stride_a, tile_stride_b;
// Simdgroup matrices
- MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;
- MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;
- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;
+ MMATile<AccumType> Atile;
+ MMATile<AccumType> Btile;
+ MMATile<AccumType> Ctile;
// Offsets within threadgroup
short sm;
short sn;
-
short As_offset;
short Bs_offset;
/* Constructor */
METAL_FUNC BlockMMA(
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- ) {
- // Determine thread position in simdgroup matrix
+ ushort simd_group_id,
+ ushort simd_lane_id,
+ short BM_,
+ short BN_,
+ short BK_,
+ short WM_,
+ short WN_,
+ bool transpose_a_,
+ bool transpose_b_,
+ short lda_tgp_,
+ short ldb_tgp_
+ )
+ : BM(BM_), BN(BN_), BK(BK_), WM(WM_), WN(WN_), transpose_a(transpose_a_),
+ transpose_b(transpose_b_), lda_tgp(lda_tgp_), ldb_tgp(ldb_tgp_),
+ TM_stride(kFragSize * WM_), TN_stride(kFragSize * WN_),
+ TM(BM_ / (kFragSize * WM_)), TN(BN_ / (kFragSize * WN_)),
+ A_str_m(transpose_a_ ? 1 : lda_tgp_),
+ A_str_k(transpose_a_ ? lda_tgp_ : 1),
+ B_str_k(transpose_b_ ? 1 : ldb_tgp_),
+ B_str_n(transpose_b_ ? ldb_tgp_ : 1),
+ tile_stride_a(kFragSize * (transpose_a_ ? lda_tgp_ : 1)),
+ tile_stride_b(kFragSize * (transpose_b_ ? 1 : ldb_tgp_)), Atile(TM, 1),
+ Btile(1, TN), Ctile(TM, TN) {
short tm = kFragSize * (simd_group_id / WN);
short tn = kFragSize * (simd_group_id % WN);
@@ -528,9 +485,8 @@ struct BlockMMA {
sm = simd_coord.y;
sn = simd_coord.x;
- // Determine thread and simdgroup offset
- As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K
- Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N
+ As_offset = (tm + sm) * A_str_m + (sn)*A_str_k;
+ Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n;
sm += tm;
sn += tn;
@@ -538,26 +494,16 @@ struct BlockMMA {
/* (BM, BK) X (BK, BN) multiply accumulate function */
METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {
- // Adjust for simdgroup and thread location
As += As_offset;
Bs += Bs_offset;
- // Iterate over BK in blocks of kFragSize
- STEEL_PRAGMA_UNROLL
for (short kk = 0; kk < BK; kk += kFragSize) {
simdgroup_barrier(mem_flags::mem_none);
-
- Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);
-
+ Atile.load(As, WM, 1, A_str_m, A_str_k);
simdgroup_barrier(mem_flags::mem_none);
-
- Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);
-
+ Btile.load(Bs, 1, WN, B_str_k, B_str_n);
simdgroup_barrier(mem_flags::mem_none);
-
tile_matmad(Ctile, Atile, Btile, Ctile);
-
- // Progress to next simdgroup tile
As += tile_stride_a;
Bs += tile_stride_b;
}
@@ -565,40 +511,11 @@ struct BlockMMA {
/* Store results from simdgroup_matrix results into device memory */
METAL_FUNC void store_result(device U* D, const int ldd) {
- // Apply epilogue
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
- }
-
- // Adjust for simdgroup and thread location
- D += sm * ldd + sn;
-
- Ctile.template store<U, WM, WN>(D, ldd);
- }
-
- METAL_FUNC void store_result_slice(
- device U* D,
- const int ldd,
- short2 start,
- short2 stop
- ) {
- // Apply epilogue
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ for (short i = 0; i < Ctile.elems_per_tile(); i++) {
Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
}
-
D += sm * ldd + sn;
- start -= short2(sn, sm);
- stop -= short2(sn, sm);
-
- // TODO: Check the start as well
- if (stop.y <= 0 || stop.x <= 0) {
- return;
- }
-
- Ctile.template store_slice<U, WM, WN>(D, ldd, start, stop);
+ Ctile.store(D, ldd, WM, WN);
}
METAL_FUNC void store_result_safe(
@@ -606,30 +523,14 @@ struct BlockMMA {
const int ldd,
short2 dst_tile_dims
) {
- // Apply epilogue
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ for (short i = 0; i < Ctile.elems_per_tile(); i++) {
Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
}
-
- // Adjust for simdgroup and thread location
D += sm * ldd + sn;
dst_tile_dims -= short2(sn, sm);
-
if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
return;
-
- Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);
- }
-
- /* Apply epilogue */
- template <typename UnaryEpilogue>
- METAL_FUNC void apply_epilogue(thread const UnaryEpilogue& epilogue_op) {
- // Loop over all simdgroup tiles
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
- Ctile.elems()[i] = epilogue_op.apply(Ctile.elems()[i]);
- }
+ Ctile.store_safe(D, ldd, dst_tile_dims, WM, WN);
}
/* Apply epilogue */
@@ -640,28 +541,24 @@ struct BlockMMA {
const int fdc,
thread const BinaryEpilogue& epilogue_op
) {
- // Adjust for simdgroup and thread location
C += (sm)*ldc + (sn)*fdc;
- // Loop over all simdgroup tiles
- STEEL_PRAGMA_UNROLL
for (short i = 0; i < TM; i++) {
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < TN; j++) {
- // Get accumulated result and associated offset in C
thread auto& accum = Ctile.frag_at(i, j);
int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;
- // Apply epilogue
STEEL_PRAGMA_UNROLL
- for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {
+ for (short k = 0; k < kElemsPerFrag; k++) {
accum[k] = epilogue_op.apply(accum[k], C[offset_c + k * fdc]);
}
}
}
}
- /* Apply epilogue */
+ STEEL_CONST short kElemsPerFrag = MMAFrag_acc_t::kElemsPerFrag;
+
+ /* Apply epilogue safe */
template <typename BinaryEpilogue>
METAL_FUNC void apply_epilogue_safe(
const device U* C,
@@ -670,44 +567,35 @@ struct BlockMMA {
short2 dst_tile_dims,
thread const BinaryEpilogue& epilogue_op
) {
- // Adjust for simdgroup and thread location
C += (sm)*ldc + (sn)*fdc;
dst_tile_dims -= short2(sn, sm);
if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
return;
- // Loop over all simdgroup tiles
- STEEL_PRAGMA_UNROLL
for (short i = 0; i < TM; i++) {
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < TN; j++) {
- // Get accumulated result and associated offset in C
thread auto& accum = Ctile.frag_at(i, j);
int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;
- constexpr short kelems = decltype(Ctile)::kElemsPerFrag;
-
- // Read C
- U c_elems[kelems] = {0};
+ U c_elems[2] = {0};
STEEL_PRAGMA_UNROLL
- for (short k = 0; k < kelems; k++) {
+ for (short k = 0; k < kElemsPerFrag; k++) {
if ((j * TN_stride + k) < dst_tile_dims.x) {
c_elems[k] = C[offset_c + k * fdc];
}
}
- // Apply epilogue
STEEL_PRAGMA_UNROLL
- for (short k = 0; k < kelems; k++) {
+ for (short k = 0; k < kElemsPerFrag; k++) {
accum[k] = epilogue_op.apply(accum[k], c_elems[k]);
}
}
}
}
- /* Store results from simdgroup_matrix results into device memory */
+ /* Store results with epilogue from C source */
METAL_FUNC void store_result(
device U* D,
const int ldd,
@@ -716,25 +604,17 @@ struct BlockMMA {
const int fdc,
thread const Epilogue& epilogue_op
) const {
- // Adjust for simdgroup and thread location
C += (sm)*ldc + (sn)*fdc;
D += (sm)*ldd + sn;
- constexpr short kelems = decltype(Ctile)::kElemsPerFrag;
-
- // Loop over all simdgroup tiles
- STEEL_PRAGMA_UNROLL
for (short i = 0; i < TM; i++) {
- STEEL_PRAGMA_UNROLL
for (short j = 0; j < TN; j++) {
- // Get accumulated result and associated offset in C
thread const auto& accum = Ctile.frag_at(i, j);
int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;
int offset_d = (i * TM_stride) * ldd + (j * TN_stride);
- // Apply epilogue
STEEL_PRAGMA_UNROLL
- for (short k = 0; k < kelems; k++) {
+ for (short k = 0; k < kElemsPerFrag; k++) {
D[offset_d + k] = epilogue_op.apply(accum[k], C[offset_c + k * fdc]);
}
}
@@ -750,7 +630,6 @@ struct BlockMMA {
short2 dst_tile_dims,
thread const Epilogue& epilogue_op
) const {
- // Adjust for simdgroup and thread location
C += (sm)*ldc + (sn)*fdc;
D += (sm)*ldd + sn;
dst_tile_dims -= short2(sn, sm);
@@ -758,21 +637,15 @@ struct BlockMMA {
if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
return;
- constexpr short kelems = decltype(Ctile)::kElemsPerFrag;
-
- STEEL_PRAGMA_UNROLL
for (int i = 0; i < TM; i++) {
if (i * TM_stride < dst_tile_dims.y) {
- STEEL_PRAGMA_UNROLL
for (int j = 0; j < TN; j++) {
- // Get accumulated result and associated offset in C
thread const auto& accum = Ctile.frag_at(i, j);
int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;
int offset_d = (i * TM_stride) * ldd + (j * TN_stride);
- // Apply epilogue
STEEL_PRAGMA_UNROLL
- for (short k = 0; k < kelems; k++) {
+ for (short k = 0; k < kElemsPerFrag; k++) {
if ((j * TN_stride + k) < dst_tile_dims.x) {
D[offset_d + k] =
epilogue_op.apply(accum[k], C[offset_c + k * fdc]);
@@ -784,426 +657,4 @@ struct BlockMMA {
}
};
-template <
- typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- short lda_tgp,
- short ldb_tgp,
- typename AccumType,
- typename Epilogue>
-struct BlockMMA<
- complex64_t,
- U,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- lda_tgp,
- ldb_tgp,
- AccumType,
- Epilogue> {
- static_assert(
- metal::is_same_v<AccumType, float>,
- "BlockMMA<complex64_t,...> expects float accumulators"
- );
- static_assert(
- metal::is_same_v<U, complex64_t>,
- "For complex BlockMMA, U must be complex64_t; use a different epilogue "
- "for projections"
- );
- // MMAFrag size
- STEEL_CONST short kFragSize = 8;
- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
-
- // Warp tile simdgroup matrix strides along M
- STEEL_CONST short TM_stride = kFragSize * WM;
- // Warp tile simdgroup matrix strides along M
- STEEL_CONST short TN_stride = kFragSize * WN;
-
- // Warp tile size along M
- STEEL_CONST short TM = BM / (kFragSize * WM);
- // Warp tile size along N
- STEEL_CONST short TN = BN / (kFragSize * WN);
-
- // Threadgroup A strides
- STEEL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
- STEEL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K
-
- // Threadgroup B strides
- STEEL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K
- STEEL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
-
- // Threadgroup strides along K
- STEEL_CONST short tile_stride_a = kFragSize * A_str_k;
- STEEL_CONST short tile_stride_b = kFragSize * B_str_k;
-
- // When indexing complex as float[2]
- STEEL_CONST short A_str_m_f = A_str_m * 2;
- STEEL_CONST short A_str_k_f = A_str_k * 2;
- STEEL_CONST short B_str_k_f = B_str_k * 2;
- STEEL_CONST short B_str_n_f = B_str_n * 2;
- STEEL_CONST short tile_stride_a_f = tile_stride_a * 2;
- STEEL_CONST short tile_stride_b_f = tile_stride_b * 2;
-
- // Accumulators (real/imag)
- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile_r;
- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile_i;
-
- // Offsets within threadgroup
- short sm, sn;
- short As_offset, Bs_offset;
-
- /* Constructor */
- METAL_FUNC BlockMMA(
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- ) {
- // Determine thread position in simdgroup matrix
- short tm = kFragSize * (simd_group_id / WN);
- short tn = kFragSize * (simd_group_id % WN);
-
- short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);
- sm = simd_coord.y;
- sn = simd_coord.x;
-
- // Determine thread and simdgroup offset
- As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // (M,K)
- Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // (K,N)
-
- sm += tm;
- sn += tn;
- }
-
- /* Karatsuba MMA: 3 real MMAs per K-chunk */
- METAL_FUNC void mma(
- const threadgroup complex64_t* As,
- const threadgroup complex64_t* Bs
- ) {
- // Adjust for simdgroup and thread location
- As += As_offset;
- Bs += Bs_offset;
- threadgroup const float* As_f =
- reinterpret_cast<threadgroup const float*>(As);
- threadgroup const float* Bs_f =
- reinterpret_cast<threadgroup const float*>(Bs);
-
- // Iterate over BK in blocks of kFragSize
- STEEL_PRAGMA_UNROLL
- for (short kk = 0; kk < BK; kk += kFragSize) {
- simdgroup_barrier(mem_flags::mem_none);
-
- MMATile<AccumType, TM, 1, MMAFrag_acc_t> Ar, Ai;
- Ar.template load<float, WM, 1, A_str_m_f, A_str_k_f>(As_f + 0);
- Ai.template load<float, WM, 1, A_str_m_f, A_str_k_f>(As_f + 1);
-
- simdgroup_barrier(mem_flags::mem_none);
-
- MMATile<AccumType, 1, TN, MMAFrag_acc_t> Br, Bi;
- Br.template load<float, 1, WN, B_str_k_f, B_str_n_f>(Bs_f + 0);
- Bi.template load<float, 1, WN, B_str_k_f, B_str_n_f>(Bs_f + 1);
-
- simdgroup_barrier(mem_flags::mem_none);
-
- // P = Ar*Br ; Q = Ai*Bi ; R = (Ar+Ai)*(Br+Bi)
- MMATile<AccumType, TM, TN, MMAFrag_acc_t> P, Q, R;
-
- tile_matmad(P, Ar, Br, P);
- tile_matmad(Q, Ai, Bi, Q);
-
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ar)::kElemsPerTile; ++i)
- Ar.elems()[i] += Ai.elems()[i];
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Br)::kElemsPerTile; ++i)
- Br.elems()[i] += Bi.elems()[i];
-
- tile_matmad(R, Ar, Br, R);
-
- // C_r += P - Q ; C_i -= Q
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile_r)::kElemsPerTile; ++i) {
- const auto p = P.elems()[i];
- const auto q = Q.elems()[i];
- const auto r = R.elems()[i];
- Ctile_r.elems()[i] += (p - q);
- Ctile_i.elems()[i] += (r - p - q);
- }
-
- // Progress to next simdgroup tile
- As_f += tile_stride_a_f;
- Bs_f += tile_stride_b_f;
- }
- }
-
- /* Store results from simdgroup_matrix results into device memory */
- METAL_FUNC void store_result(device U* D, const int ldd) {
- // Adjust for simdgroup and thread location
- D += sm * ldd + sn;
-
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < TM; i++) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < TN; j++) {
- thread const auto& r = Ctile_r.frag_at(i, j);
- thread const auto& im = Ctile_i.frag_at(i, j);
- int off = (i * TM_stride) * ldd + (j * TN_stride);
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < decltype(Ctile_r)::kElemsPerFrag; k++) {
- D[off + k] = Epilogue::apply(complex64_t(r[k], im[k]));
- }
- }
- }
- }
-
- METAL_FUNC void store_result_slice(
- device U* D,
- const int ldd,
- short2 start,
- short2 stop
- ) {
- D += sm * ldd + sn;
- start -= short2(sn, sm);
- stop -= short2(sn, sm);
-
- if (stop.y <= 0 || stop.x <= 0)
- return;
-
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < TM; ++i) {
- const int row = i * TM_stride;
- if (row >= start.y && row < stop.y) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < TN; ++j) {
- const int off = row * ldd + (j * TN_stride);
- thread const auto& r = Ctile_r.frag_at(i, j);
- thread const auto& im = Ctile_i.frag_at(i, j);
-
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < decltype(Ctile_r)::kElemsPerFrag; ++k) {
- const int col = j * TN_stride + k;
- if (col >= start.x && col < stop.x) {
- D[off + k] = Epilogue::apply(complex64_t(r[k], im[k]));
- }
- }
- }
- }
- }
- }
-
- METAL_FUNC void store_result_safe(
- device U* D,
- const int ldd,
- short2 dst_tile_dims
- ) {
- D += sm * ldd + sn;
- dst_tile_dims -= short2(sn, sm);
- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
- return;
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < TM; i++) {
- if (i * TM_stride < dst_tile_dims.y) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < TN; j++) {
- int off = (i * TM_stride) * ldd + (j * TN_stride);
- thread const auto& r = Ctile_r.frag_at(i, j);
- thread const auto& im = Ctile_i.frag_at(i, j);
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < decltype(Ctile_r)::kElemsPerFrag; k++) {
- if ((j * TN_stride + k) < dst_tile_dims.x) {
- D[off + k] = Epilogue::apply(complex64_t(r[k], im[k]));
- }
- }
- }
- }
- }
- }
-
- /* Apply epilogue */
- template <typename UnaryEpilogue>
- METAL_FUNC void apply_epilogue(thread const UnaryEpilogue& epilogue_op) {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile_r)::kElemsPerTile; i++) {
- complex64_t out = epilogue_op.apply(
- complex64_t(Ctile_r.elems()[i], Ctile_i.elems()[i])
- );
- Ctile_r.elems()[i] = out.real;
- Ctile_i.elems()[i] = out.imag;
- }
- }
-
- /* Apply epilogue */
- template <typename BinaryEpilogue>
- METAL_FUNC void apply_epilogue(
- const device U* C,
- const int ldc,
- const int fdc,
- thread const BinaryEpilogue& epilogue_op
- ) {
- // Adjust for simdgroup and thread location
- C += (sm)*ldc + (sn)*fdc;
-
- // Loop over all simdgroup tiles
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < TM; i++) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < TN; j++) {
- // Get accumulated result and associated offset in Cr, Ci
- thread auto& r = Ctile_r.frag_at(i, j);
- thread auto& im = Ctile_i.frag_at(i, j);
- int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;
-
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < decltype(Ctile_r)::kElemsPerFrag; k++) {
- complex64_t out = epilogue_op.apply(
- complex64_t(r[k], im[k]),
- C[offset_c + k * fdc]
- );
- r[k] = out.real;
- im[k] = out.imag;
- }
- }
- }
- }
-
- /* Apply epilogue */
- template <typename BinaryEpilogue>
- METAL_FUNC void apply_epilogue_safe(
- const device U* C,
- const int ldc,
- const int fdc,
- short2 dst_tile_dims,
- thread const BinaryEpilogue& epilogue_op
- ) {
- // Adjust for simdgroup and thread location
- C += (sm)*ldc + (sn)*fdc;
- dst_tile_dims -= short2(sn, sm);
-
- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
- return;
-
- // Loop over all simdgroup tiles
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < TM; i++) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < TN; j++) {
- // Get accumulated result and associated offset in Cr, Ci
- thread auto& r = Ctile_r.frag_at(i, j);
- thread auto& im = Ctile_i.frag_at(i, j);
- int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;
-
- constexpr short kelems = decltype(Ctile_r)::kElemsPerFrag;
- complex64_t tmp[kelems];
-
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < kelems; k++) {
- if ((j * TN_stride + k) < dst_tile_dims.x &&
- (i * TM_stride) < dst_tile_dims.y) {
- tmp[k] = C[offset_c + k * fdc];
- } else {
- tmp[k] = complex64_t(0.0f, 0.0f);
- }
- }
-
- // Apply epilogue
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < kelems; k++) {
- complex64_t out = epilogue_op.apply(complex64_t(r[k], im[k]), tmp[k]);
- r[k] = out.real;
- im[k] = out.imag;
- }
- }
- }
- }
-
- /* Store results from simdgroup_matrix results into device memory */
- METAL_FUNC void store_result(
- device U* D,
- const int ldd,
- const device U* C,
- const int ldc,
- const int fdc,
- thread const Epilogue& epilogue_op
- ) const {
- // Adjust for simdgroup and thread location
- C += (sm)*ldc + (sn)*fdc;
- D += (sm)*ldd + sn;
-
- constexpr short kelems = decltype(Ctile_r)::kElemsPerFrag;
-
- // Loop over all simdgroup tiles
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < TM; i++) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < TN; j++) {
- // Get accumulated result and associated offset in Cr, Ci
- thread const auto& r = Ctile_r.frag_at(i, j);
- thread const auto& im = Ctile_i.frag_at(i, j);
- int off_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;
- int off_d = (i * TM_stride) * ldd + (j * TN_stride);
-
- // Apply epilogue
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < kelems; k++) {
- D[off_d + k] =
- epilogue_op.apply(complex64_t(r[k], im[k]), C[off_c + k * fdc]);
- }
- }
- }
- }
-
- METAL_FUNC void store_result_safe(
- device U* D,
- const int ldd,
- const device U* C,
- const int ldc,
- const int fdc,
- short2 dst_tile_dims,
- thread const Epilogue& epilogue_op
- ) const {
- // Adjust for simdgroup and thread location
- C += (sm)*ldc + (sn)*fdc;
- D += (sm)*ldd + sn;
- dst_tile_dims -= short2(sn, sm);
-
- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
- return;
-
- constexpr short kelems = decltype(Ctile_r)::kElemsPerFrag;
-
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < TM; i++) {
- if (i * TM_stride < dst_tile_dims.y) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < TN; j++) {
- // Get accumulated result and associated offset in Cr, Ci
- thread const auto& r = Ctile_r.frag_at(i, j);
- thread const auto& im = Ctile_i.frag_at(i, j);
- int off_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;
- int off_d = (i * TM_stride) * ldd + (j * TN_stride);
-
- // Apply epilogue
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < kelems; k++) {
- if ((j * TN_stride + k) < dst_tile_dims.x) {
- D[off_d + k] = epilogue_op.apply(
- complex64_t(r[k], im[k]),
- C[off_c + k * fdc]
- );
- }
- }
- }
- }
- }
- }
-};
-
} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.h
deleted file mode 100644
index e1d665fc5..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.h
+++ /dev/null
@@ -1,1144 +0,0 @@
-
-
-#pragma once
-
-#include <metal_simdgroup>
-#include <metal_simdgroup_matrix>
-#include <metal_stdlib>
-
-#include "../defines.h"
-#include "transforms.h"
-#include "../utils/integral_constant.h"
-
-#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>
-
-using namespace metal;
-
-///////////////////////////////////////////////////////////////////////////////
-// MMA helper
-///////////////////////////////////////////////////////////////////////////////
-
-namespace steel {
-
-///////////////////////////////////////////////////////////////////////////////
-// NAX Steel with new tiles
-///////////////////////////////////////////////////////////////////////////////
-
-struct BaseNAXFrag {
- STEEL_CONST short kFragRows = 16;
- STEEL_CONST short kFragCols = 16;
-
- STEEL_CONST short kElemsPerFrag = (kFragRows * kFragCols) / 32;
-
- STEEL_CONST short kElemRows = 2;
- STEEL_CONST short kElemCols = 4;
-
- STEEL_CONST short kElemRowsJump = 8;
-
- static_assert(
- kElemRows * kElemCols == kElemsPerFrag,
- "MMAFrag shape is not consistent with MMAFrag size"
- );
-
- template <typename U>
- using dtype_frag_t = typename metal::vec<U, kElemsPerFrag>;
-
- METAL_FUNC static short2 get_coord() {
- const ushort simd_lane_id = __metal_get_thread_index_in_simdgroup(ushort());
- const short qid = simd_lane_id >> 2;
- const short fm = ((qid & 4) | ((simd_lane_id >> 1) & 3));
- const short fn = ((qid & 2) | (simd_lane_id & 1)) * 4;
- return short2{fn, fm};
- }
-
- METAL_FUNC static short2 get_coord(short idx) {
- const ushort simd_lane_id = __metal_get_thread_index_in_simdgroup(ushort());
- const short qid = simd_lane_id >> 2;
- const short fm = ((qid & 4) | ((simd_lane_id >> 1) & 3)) + (idx >> 2) * 8;
- const short fn = ((qid & 2) | (simd_lane_id & 1)) * 4 + idx % 4;
- return short2{fn, fm};
- }
-
- template <
- typename T,
- typename SrcPtrType,
- typename StrX,
- typename StrY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC static constexpr void load(
- thread dtype_frag_t<T>& dst,
- SrcPtrType src,
- StrX str_x,
- StrY str_y,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- const short2 sc = get_coord();
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- const auto r = off_x + i * kElemRowsJump + sc.y;
- const auto c = off_y + sc.x;
-
- if constexpr (metal::is_same_v<StrY, Int<1>>) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[i * kElemCols + j] = static_cast<T>(src[r * str_x + c + j]);
- }
- } else {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[i * kElemCols + j] =
- static_cast<T>(src[r * str_x + (c + j) * str_y]);
- }
- }
- }
- }
-
- template <
- typename T,
- typename SrcPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC static constexpr void load_rows(
- thread dtype_frag_t<T>& dst,
- SrcPtrType src,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- const short2 sc = get_coord();
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- const auto r = off_x + i * kElemRowsJump + sc.y;
- const auto c = off_y + sc.x;
-
- if (r < lim_x) {
- if constexpr (metal::is_same_v<StrY, Int<1>>) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[i * kElemCols + j] = static_cast<T>(src[r * str_x + (c + j)]);
- }
- } else {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[i * kElemCols + j] =
- static_cast<T>(src[r * str_x + (c + j) * str_y]);
- }
- }
-
- } else {
- dst = dtype_frag_t<T>(0);
- }
- }
- }
-
- template <
- typename T,
- typename SrcPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC static constexpr void load_safe(
- thread dtype_frag_t<T>& dst,
- SrcPtrType src,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- const short2 sc = get_coord();
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- const auto r = off_x + i * kElemRowsJump + sc.y;
- const auto c = off_y + sc.x;
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- if (r < lim_x && (c + j) < lim_y) {
- dst[i * kElemCols + j] =
- static_cast<T>(src[r * str_x + (c + j) * str_y]);
- } else {
- dst[i * kElemCols + j] = T(0);
- }
- }
- }
- }
-
- template <
- typename T,
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC static constexpr void store(
- const thread dtype_frag_t<T>& src,
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- using U = pointer_element_t<DstPtrType>;
-
- const short2 sc = get_coord();
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- const auto r = off_x + i * kElemRowsJump + sc.y;
- const auto c = off_y + sc.x;
-
- if constexpr (metal::is_same_v<StrY, Int<1>>) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[r * str_x + c + j] = static_cast<U>(src[i * kElemCols + j]);
- }
- } else {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[r * str_x + (c + j) * str_y] =
- static_cast<U>(src[i * kElemCols + j]);
- }
- }
- }
- }
-
- template <
- typename T,
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC static constexpr void store_rows(
- const thread dtype_frag_t<T>& src,
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- using U = pointer_element_t<DstPtrType>;
-
- const short2 sc = get_coord();
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- const auto r = off_x + i * kElemRowsJump + sc.y;
- const auto c = off_y + sc.x;
-
- if (r < lim_x) {
- if constexpr (metal::is_same_v<StrY, Int<1>>) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[r * str_x + c + j] = static_cast<U>(src[i * kElemCols + j]);
- }
- } else {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[r * str_x + (c + j) * str_y] =
- static_cast<U>(src[i * kElemCols + j]);
- }
- }
- }
- }
- }
-
- template <
- typename T,
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC static constexpr void store_safe(
- const thread dtype_frag_t<T>& src,
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- using U = pointer_element_t<DstPtrType>;
-
- const short2 sc = get_coord();
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- const auto r = off_x + i * kElemRowsJump + sc.y;
- const auto c = off_y + sc.x;
-
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- if (r < lim_x && (c + j) < lim_y) {
- dst[r * str_x + (c + j) * str_y] =
- static_cast<U>(src[i * kElemCols + j]);
- }
- }
- }
- }
-
- template <
- typename T,
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename StartX,
- typename StopX,
- typename StartY,
- typename StopY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC static constexpr void store_slice(
- const thread dtype_frag_t<T>& src,
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- StartX start_x,
- StopX stop_x,
- StartY start_y,
- StopY stop_y,
- OffX off_x = Int<0>{},
- OffY off_y = Int<0>{}
- ) {
- using U = pointer_element_t<DstPtrType>;
-
- const short2 sc = get_coord();
-
- const_for_loop<0, kElemRows, 1>([&](auto idx_row) {
- const auto r = off_x + idx_row * Int<kElemRowsJump>{};
- if (r >= stop_x - sc.y || r < start_x - sc.y) {
- return;
- }
-
- const_for_loop<0, kElemCols, 1>([&](auto idx_col) {
- const auto c = off_y + idx_col;
- if (c >= stop_y - sc.x || c < start_y - sc.x) {
- return;
- }
-
- const auto src_idx = idx_row * Int<kElemCols>{} + idx_col;
- dst[(r + sc.y) * str_x + (c + sc.x) * str_y] =
- static_cast<U>(src[src_idx]);
- });
- });
- }
-
- template <typename Op, typename T>
- METAL_FUNC static constexpr void row_reduce(
- thread const dtype_frag_t<T>& inp_vals,
- thread T* reduced_vals
- ) {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- T thr_reduce = Op::apply(
- Op::apply(inp_vals[i * kElemCols + 0], inp_vals[i * kElemCols + 1]),
- Op::apply(inp_vals[i * kElemCols + 2], inp_vals[i * kElemCols + 3])
- );
-
- T qgr_reduce = simd_shuffle_xor(thr_reduce, ushort(1));
- qgr_reduce = Op::apply(thr_reduce, qgr_reduce);
-
- T sgr_reduce = simd_shuffle_xor(qgr_reduce, ushort(8));
- sgr_reduce = Op::apply(qgr_reduce, sgr_reduce);
-
- reduced_vals[i] = Op::apply(reduced_vals[i], sgr_reduce);
- }
- }
-
- template <typename Op, typename T>
- METAL_FUNC static constexpr void row_bin_op(
- thread dtype_frag_t<T>& inp_vals,
- thread T* row_vals
- ) {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- inp_vals[i * kElemCols + j] =
- Op::apply(inp_vals[i * kElemCols + j], row_vals[i]);
- }
- }
- }
-};
-
-template <
- typename T,
- short kRows_,
- short kCols_,
- typename NAXFrag_t = BaseNAXFrag>
-struct NAXSubTile {
- STEEL_CONST short kRows = kRows_;
- STEEL_CONST short kCols = kCols_;
-
- STEEL_CONST short kFragRows = NAXFrag_t::kFragRows;
- STEEL_CONST short kFragCols = NAXFrag_t::kFragCols;
- STEEL_CONST short kElemsPerFrag = NAXFrag_t::kElemsPerFrag;
-
- STEEL_CONST short kSubTileRows = kRows / kFragRows;
- STEEL_CONST short kSubTileCols = kCols / kFragCols;
-
- STEEL_CONST short kNumFrags = kSubTileRows * kSubTileCols;
- STEEL_CONST short kElemsPerSubTile = kNumFrags * kElemsPerFrag;
-
- STEEL_CONST int kRowsPerThread = kSubTileRows * NAXFrag_t::kElemRows;
- STEEL_CONST int kColsPerThread = kSubTileCols * NAXFrag_t::kElemCols;
-
- STEEL_CONST short kFragThrRows = NAXFrag_t::kElemRows;
- STEEL_CONST short kFragThrCols = NAXFrag_t::kElemCols;
- STEEL_CONST short kFragRowsJump = NAXFrag_t::kElemRowsJump;
-
- using frag_type = typename NAXFrag_t::template dtype_frag_t<T>;
-
- frag_type val_frags[kNumFrags];
-
- METAL_FUNC constexpr void clear() {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kNumFrags; ++i) {
- val_frags[i] = frag_type(0);
- }
- }
-
- METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {
- return val_frags[i * kSubTileCols + j];
- }
-
- METAL_FUNC constexpr const thread frag_type& frag_at(
- const short i,
- const short j
- ) const {
- return val_frags[i * kSubTileCols + j];
- }
-
- template <int i, int j>
- METAL_FUNC constexpr thread frag_type& frag_at() {
- return val_frags[i * kSubTileCols + j];
- }
-
- template <int i, int j>
- METAL_FUNC constexpr const thread frag_type& frag_at() const {
- return val_frags[i * kSubTileCols + j];
- }
-
- METAL_FUNC thread T* elems() {
- return reinterpret_cast<thread T*>(val_frags);
- }
-
- METAL_FUNC const thread T* elems() const {
- return reinterpret_cast<const thread T*>(val_frags);
- }
-
- template <typename Op>
- METAL_FUNC void row_reduce(thread metal::vec<T, kRowsPerThread>& vals) const {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kSubTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kSubTileCols; ++j) {
- NAXFrag_t::template row_reduce<Op>(
- frag_at(i, j),
- &vals[i * kFragThrRows]
- );
- }
- }
- }
-
- template <typename Op>
- METAL_FUNC void row_bin_op(thread metal::vec<T, kRowsPerThread>& vals) {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kSubTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kSubTileCols; ++j) {
- NAXFrag_t::template row_bin_op<Op>(
- frag_at(i, j),
- &vals[i * kFragThrRows]
- );
- }
- }
- }
-
- template <
- typename SrcPtrType,
- typename StrX,
- typename StrY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC constexpr void load(
- SrcPtrType src,
- StrX str_x,
- StrY str_y,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kSubTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kSubTileCols; ++j) {
- NAXFrag_t::load(
- frag_at(i, j),
- src,
- str_x,
- str_y,
- off_x + i * kFragRows,
- off_y + j * kFragCols
- );
- }
- }
- }
-
- template <
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC constexpr void store(
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- OffX off_x = {},
- OffY off_y = {}
- ) const {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kSubTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kSubTileCols; ++j) {
- NAXFrag_t::store(
- frag_at(i, j),
- dst,
- str_x,
- str_y,
- off_x + i * kFragRows,
- off_y + j * kFragCols
- );
- }
- }
- }
-
- template <
- typename SrcPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC constexpr void load_rows(
- SrcPtrType src,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < kSubTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < kSubTileCols; ++j) {
- NAXFrag_t::load_rows(
- frag_at(i, j),
- src,
- str_x,
- str_y,
- lim_x,
- off_x + (i * kFragRows),
- off_y + (j * kFragCols)
- );
- }
- }
- }
-
- template <
- typename SrcPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC constexpr void load_safe(
- SrcPtrType src,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = {},
- OffY off_y = {}
- ) {
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < kSubTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < kSubTileCols; ++j) {
- NAXFrag_t::load_safe(
- frag_at(i, j),
- src,
- str_x,
- str_y,
- lim_x,
- lim_y,
- off_x + (i * kFragRows),
- off_y + (j * kFragCols)
- );
- }
- }
- }
-
- template <
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC constexpr void store_safe(
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = {},
- OffY off_y = {}
- ) const {
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < kSubTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < kSubTileCols; ++j) {
- NAXFrag_t::store_safe(
- frag_at(i, j),
- dst,
- str_x,
- str_y,
- lim_x,
- lim_y,
- off_x + (i * kFragRows),
- off_y + (j * kFragCols)
- );
- }
- }
- }
-
- template <
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC constexpr void store_rows(
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- OffX off_x = {},
- OffY off_y = {}
- ) const {
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < kSubTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < kSubTileCols; ++j) {
- NAXFrag_t::store_safe(
- frag_at(i, j),
- dst,
- str_x,
- str_y,
- lim_x,
- off_x + (i * kFragRows),
- off_y + (j * kFragCols)
- );
- }
- }
- }
-
- template <
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename StartX,
- typename StopX,
- typename StartY,
- typename StopY,
- typename OffX = Int<0>,
- typename OffY = Int<0>>
- METAL_FUNC constexpr void store_slice(
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- StartX start_x,
- StopX stop_x,
- StartY start_y,
- StopY stop_y,
- OffX off_x = Int<0>{},
- OffY off_y = Int<0>{}
- ) const {
- const_for_loop<0, kSubTileRows, 1>([&](auto idx_row) {
- const_for_loop<0, kSubTileCols, 1>([&](auto idx_col) {
- NAXFrag_t::store_slice(
- frag_at<idx_row.value, idx_col.value>(),
- dst,
- str_x,
- str_y,
- start_x,
- stop_x,
- start_y,
- stop_y,
- off_x + idx_row * Int<kFragRows>{},
- off_y + idx_col * Int<kFragCols>{}
- );
- });
- });
- }
-};
-
-template <
- short RC,
- short CC,
- short RA,
- short CA,
- short RB,
- short CB,
- typename CType,
- typename AType,
- typename BType,
- bool transpose_a,
- bool transpose_b,
- typename NAXFrag_t = BaseNAXFrag>
-METAL_FUNC void subtile_matmad_nax(
- thread NAXSubTile<CType, RC, CC, NAXFrag_t>& C,
- thread NAXSubTile<AType, RA, CA, NAXFrag_t>& A,
- metal::bool_constant<transpose_a>,
- thread NAXSubTile<BType, RB, CB, NAXFrag_t>& B,
- metal::bool_constant<transpose_b>
-) {
- // Static checks
- constexpr short FMa = transpose_a ? CA : RA;
- constexpr short FMc = RC;
- static_assert(FMa == FMc, "NAX matmul: M dimensions do not match");
-
- constexpr short FNb = transpose_b ? RB : CB;
- constexpr short FNc = CC;
- static_assert(FNb == FNc, "NAX matmul: N dimensions do not match");
-
- constexpr short FKa = transpose_a ? RA : CA;
- constexpr short FKb = transpose_b ? CB : RB;
- static_assert(FKa == FKb, "NAX matmul: N dimensions do not match");
-
- constexpr short FM = FMc;
- constexpr short FN = FNc;
- constexpr short FK = FKa;
-
- constexpr int TM = FM / 16;
- constexpr int TN = FN / 16;
- constexpr int TK = FK / 16;
-
- // Create Matmul descriptor
- constexpr auto desc = mpp::tensor_ops::matmul2d_descriptor(
- FM,
- FN,
- FK,
- transpose_a,
- transpose_b,
- true,
- mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate
- );
-
- // Create matmul op
- mpp::tensor_ops::matmul2d<desc, metal::execution_simdgroup> gemm_op;
-
- // Create matmul operands in registers
- auto ct_a =
- gemm_op.template get_left_input_cooperative_tensor<AType, BType, CType>();
- auto ct_b =
- gemm_op
- .template get_right_input_cooperative_tensor<AType, BType, CType>();
-
- // Create matmul output in register
- auto ct_c = gemm_op.template get_destination_cooperative_tensor<
- decltype(ct_a),
- decltype(ct_b),
- CType>();
-
- // Load A in to left operand registers
- STEEL_PRAGMA_UNROLL
- for (short mm = 0; mm < TM; mm++) {
- STEEL_PRAGMA_UNROLL
- for (short kk = 0; kk < TK; kk++) {
- const short fi = transpose_a ? kk : mm;
- const short fj = transpose_a ? mm : kk;
-
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < 8; i++) {
- ct_a[(TK * mm + kk) * 8 + i] = A.frag_at(fi, fj)[i];
- }
- }
- }
-
- // Load B into right operand registers
- STEEL_PRAGMA_UNROLL
- for (short nn = 0; nn < TN; nn++) {
- STEEL_PRAGMA_UNROLL
- for (short kk = 0; kk < TK; kk++) {
- const short fi = transpose_b ? nn : kk;
- const short fj = transpose_b ? kk : nn;
-
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < 8; i++) {
- ct_b[(TN * kk + nn) * 8 + i] = B.frag_at(fi, fj)[i];
- }
- }
- }
-
- // Load C into output registers (op handles accumulation)
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < ct_c.get_capacity(); i++) {
- ct_c[i] = C.elems()[i];
- }
-
- // Do matmul
- gemm_op.run(ct_a, ct_b, ct_c);
-
- // Copy out results
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < ct_c.get_capacity(); i++) {
- C.elems()[i] = ct_c[i];
- }
-}
-
-template <typename T, short kTileRows_, short kTileCols_, class NAXSubTile_>
-struct NAXTile {
- using NAXSubTile_t = NAXSubTile_;
- using elem_type = T;
- STEEL_CONST short kSubTileRows = NAXSubTile_t::kRows;
- STEEL_CONST short kSubTileCols = NAXSubTile_t::kCols;
- STEEL_CONST short kElemsPerSubTile = NAXSubTile_t::kElemsPerSubTile;
-
- STEEL_CONST short kTileRows = kTileRows_;
- STEEL_CONST short kTileCols = kTileCols_;
-
- STEEL_CONST short kRows = kTileRows * kSubTileRows;
- STEEL_CONST short kCols = kTileCols * kSubTileCols;
-
- STEEL_CONST short kSubTiles = kTileRows * kTileCols;
- STEEL_CONST short kElemsPerTile = kSubTiles * kElemsPerSubTile;
-
- STEEL_CONST short kRowsPerThread = kTileRows * NAXSubTile_t::kRowsPerThread;
- STEEL_CONST short kColsPerThread = kTileCols * NAXSubTile_t::kColsPerThread;
-
- STEEL_CONST short kSubTileThrRows = NAXSubTile_t::kRowsPerThread;
- STEEL_CONST short kSubTileThrCols = NAXSubTile_t::kColsPerThread;
-
- NAXSubTile_t val_subtiles[kSubTiles];
-
- METAL_FUNC NAXTile() thread {}
-
- METAL_FUNC constexpr void clear() {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kSubTiles; ++i) {
- val_subtiles[i].clear();
- }
- }
-
- METAL_FUNC constexpr thread NAXSubTile_t& subtile_at(
- const short i,
- const short j
- ) {
- return val_subtiles[i * kTileCols + j];
- }
-
- METAL_FUNC constexpr const thread NAXSubTile_t& subtile_at(
- const short i,
- const short j
- ) const {
- return val_subtiles[i * kTileCols + j];
- }
-
- template <int i, int j>
- METAL_FUNC constexpr const thread NAXSubTile_t& subtile_at() const {
- return val_subtiles[i * kTileCols + j];
- }
-
- METAL_FUNC thread elem_type* elems() {
- return reinterpret_cast<thread elem_type*>(val_subtiles[0].elems());
- }
-
- METAL_FUNC const thread elem_type* elems() const {
- return reinterpret_cast<const thread elem_type*>(val_subtiles[0].elems());
- }
-
- template <typename Op>
- METAL_FUNC void row_reduce(thread metal::vec<T, kRowsPerThread>& vals) const {
- auto sub_rows = (thread metal::vec<T, kSubTileThrRows>*)(&vals);
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).template row_reduce<Op>(sub_rows[i]);
- }
- }
- }
-
- template <typename Op>
- METAL_FUNC void row_bin_op(thread metal::vec<T, kRowsPerThread>& vals) {
- auto sub_rows = (thread metal::vec<T, kSubTileThrRows>*)(&vals);
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).template row_bin_op<Op>(sub_rows[i]);
- }
- }
- }
-
- template <typename U, int str_x, int str_y>
- METAL_FUNC void load(const threadgroup U* src) {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).load(
- src,
- Int<str_x>{},
- Int<str_y>{},
- i * kSubTileRows,
- j * kSubTileCols
- );
- }
- }
- }
-
- template <typename U, int str_x, int str_y>
- METAL_FUNC void store(threadgroup U* dst) const {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).store(
- dst,
- Int<str_x>{},
- Int<str_y>{},
- i * kSubTileRows,
- j * kSubTileCols
- );
- }
- }
- }
-
- template <typename U>
- METAL_FUNC void load(const device U* src, const int ld) {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).load(
- &src[(i * kSubTileRows * ld + j * kSubTileCols)],
- ld,
- Int<1>{}
- );
- }
- }
- }
-
- template <typename U>
- METAL_FUNC void store(device U* dst, const int ld) const {
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).store(
- &dst[(i * kSubTileRows * ld + j * kSubTileCols)],
- ld,
- Int<1>{}
- );
- }
- }
- }
-
- template <typename U>
- METAL_FUNC void load_rows(
- const device U* src,
- const int ld,
- const short n_rows
- ) {
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).load_rows(
- &src[(i * kSubTileRows) * ld + (j * kSubTileCols)],
- ld,
- Int<1>{},
- n_rows - i * kSubTileRows
- );
- }
- }
- }
-
- template <typename U>
- METAL_FUNC void load_safe(
- const device U* src,
- const int ld,
- const short2 src_tile_dims
- ) {
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).load_safe(
- src,
- ld,
- Int<1>{},
- src_tile_dims.y,
- src_tile_dims.x,
- i * kSubTileRows,
- j * kSubTileCols
- );
- }
- }
- }
-
- template <typename U>
- METAL_FUNC void store_rows(
- device U* dst,
- const int ld,
- const short n_rows
- ) const {
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).store_rows(
- &dst[(i * kSubTileRows) * ld + (j * kSubTileCols)],
- ld,
- Int<1>{},
- n_rows - i * kSubTileRows
- );
- }
- }
- }
-
- template <typename U>
- METAL_FUNC void store_safe(
- device U* dst,
- const int ld,
- const short2 dst_tile_dims
- ) const {
- STEEL_PRAGMA_UNROLL
- for (int i = 0; i < kTileRows; ++i) {
- STEEL_PRAGMA_UNROLL
- for (int j = 0; j < kTileCols; ++j) {
- subtile_at(i, j).store_safe(
- dst,
- ld,
- Int<1>{},
- dst_tile_dims.y,
- dst_tile_dims.x,
- i * kSubTileRows,
- j * kSubTileCols
- );
- }
- }
- }
-
- template <typename U>
- METAL_FUNC void store_slice(
- device U* dst,
- const int ld,
- const short2 start,
- const short2 stop
- ) const {
- const_for_loop<0, kTileRows, 1>([&](auto idx_row) {
- const_for_loop<0, kTileCols, 1>([&](auto idx_col) {
- subtile_at<idx_row.value, idx_col.value>().store_slice(
- dst,
- ld,
- Int<1>{},
- start.y,
- stop.y,
- start.x,
- stop.x,
- idx_row * Int<kSubTileRows>{},
- idx_col * Int<kSubTileCols>{}
- );
- });
- });
- }
-};
-
-template <
- class CTile,
- class ATile,
- class BTile,
- bool transpose_a,
- bool transpose_b>
-METAL_FUNC void tile_matmad_nax(
- thread CTile& C,
- thread ATile& A,
- metal::bool_constant<transpose_a>,
- thread BTile& B,
- metal::bool_constant<transpose_b>
-) {
- // Static checks
- constexpr short TMa = transpose_a ? ATile::kTileCols : ATile::kTileRows;
- constexpr short TMc = CTile::kTileRows;
- static_assert(TMa == TMc, "NAX tile matmul: M dimensions do not match");
-
- constexpr short FMa = transpose_a ? ATile::kSubTileCols : ATile::kSubTileRows;
- constexpr short FMc = CTile::kSubTileRows;
- static_assert(FMa == FMc, "NAX subtile matmul: M dimensions do not match");
-
- constexpr short TNb = transpose_b ? BTile::kTileRows : BTile::kTileCols;
- constexpr short TNc = CTile::kTileCols;
- static_assert(TNb == TNc, "NAX tile matmul: N dimensions do not match");
-
- constexpr short FNb = transpose_b ? BTile::kSubTileRows : BTile::kSubTileCols;
- constexpr short FNc = CTile::kSubTileCols;
- static_assert(FNb == FNc, "NAX subtile matmul: N dimensions do not match");
-
- constexpr short TKa = transpose_a ? ATile::kTileRows : ATile::kTileCols;
- constexpr short TKb = transpose_b ? BTile::kTileCols : BTile::kTileRows;
- static_assert(TKa == TKb, "NAX tile matmul: K dimensions do not match");
-
- constexpr short FKa = transpose_a ? ATile::kSubTileRows : ATile::kSubTileCols;
- constexpr short FKb = transpose_b ? BTile::kSubTileCols : BTile::kSubTileRows;
- static_assert(FKa == FKb, "NAX subtile matmul: K dimensions do not match");
-
- constexpr short TM = TMc;
- constexpr short TN = TNc;
- constexpr short TK = TKa;
-
- // Do matmul here
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < TM; ++i) {
- STEEL_PRAGMA_UNROLL
- for (short j = 0; j < TN; ++j) {
- STEEL_PRAGMA_UNROLL
- for (short k = 0; k < TK; ++k) {
- const short ra = transpose_a ? k : i;
- const short ca = transpose_a ? i : k;
- const short rb = transpose_b ? j : k;
- const short cb = transpose_b ? k : j;
-
- subtile_matmad_nax(
- C.subtile_at(i, j),
- A.subtile_at(ra, ca),
- metal::bool_constant<transpose_a>{},
- B.subtile_at(rb, cb),
- metal::bool_constant<transpose_b>{}
- );
- }
- }
- }
-}
-
-} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h
index e94df081f..fd8cb42ce 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h
@@ -49,14 +49,4 @@ struct GEMMSpiltKParams {
const int gemm_k_iterations_aligned;
};
-struct GEMMAddMMParams {
- const int ldc;
- const int fdc;
-
- const int64_t batch_stride_c;
-
- const float alpha;
- const float beta;
-};
-
} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h
index 87246822c..f0cf5a745 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h
@@ -2,8 +2,6 @@
#pragma once
-#include "../utils.h"
-
///////////////////////////////////////////////////////////////////////////////
// Transforms and Epilogues
///////////////////////////////////////////////////////////////////////////////
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.h
deleted file mode 100644
index 78952e7b7..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-
-METAL_FUNC ulong2 elem_to_loc_broadcast(
- uint elem,
- constant const int* shape,
- constant const int64_t* a_strides,
- constant const int64_t* b_strides,
- int ndim
-) {
- ulong loc_a{0};
- ulong loc_b{0};
- for (int i = ndim - 1; i >= 0 && elem > 0; --i) {
- int pos_in_dim = (elem % shape[i]);
- elem /= shape[i];
- loc_a += pos_in_dim * a_strides[i];
- loc_b += pos_in_dim * b_strides[i];
- }
- return ulong2(loc_a, loc_b);
-}
-
-METAL_FUNC ulong3 elem_to_loc_broadcast(
- uint elem,
- constant const int* shape,
- constant const int64_t* a_strides,
- constant const int64_t* b_strides,
- constant const int64_t* c_strides,
- int ndim
-) {
- ulong loc_a{0};
- ulong loc_b{0};
- ulong loc_c{0};
- for (int i = ndim - 1; i >= 0 && elem > 0; --i) {
- int pos_in_dim = (elem % shape[i]);
- elem /= shape[i];
- loc_a += pos_in_dim * a_strides[i];
- loc_b += pos_in_dim * b_strides[i];
- loc_c += pos_in_dim * c_strides[i];
- }
- return ulong3(loc_a, loc_b, loc_c);
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h
deleted file mode 100644
index 5cf7e9645..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h
+++ /dev/null
@@ -1,131 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-#include "type_traits.h"
-#include "../../../../definitions.metal"
-
-#pragma METAL internals : enable
-
-namespace steel {
-
-///////////////////////////////////////////////////////////////////////////////
-// Integral constant with casting
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename T, T v>
-struct integral_constant {
- static constexpr constant T value = v;
- using value_type = T;
- using type = integral_constant;
-
- METAL_FUNC constexpr operator value_type() const noexcept { return value; }
-
- // METAL_FUNC constexpr value_type operator()() const noexcept {
- // return value;
- // }
-};
-
-template <bool B>
-using bool_constant = integral_constant<bool, B>;
-using true_type = bool_constant<true>;
-using false_type = bool_constant<false>;
-
-template <class T>
-struct is_integral : bool_constant<metal::is_integral<T>::value> {};
-
-template <class T, T v>
-struct is_integral<integral_constant<T, v>>
- : bool_constant<metal::is_integral<T>::value> {};
-
-template <typename T>
-constexpr constant bool is_integral_v = is_integral<T>::value;
-
-template <int val>
-using Int = integral_constant<int, val>;
-
-///////////////////////////////////////////////////////////////////////////////
-// Binary Operators on Integral constants
-///////////////////////////////////////////////////////////////////////////////
-
-#define integral_const_binop(__op__, __operator__) \
- template <typename T, T tv, typename U, U uv> \
- METAL_FUNC constexpr auto __operator__( \
- integral_constant<T, tv>, \
- integral_constant<U, uv> \
- ) { \
- constexpr auto res = tv __op__ uv; \
- return integral_constant<decltype(res), res>{}; \
- }
-
-integral_const_binop(+, operator+);
-integral_const_binop(-, operator-);
-integral_const_binop(*, operator*);
-integral_const_binop(/, operator/);
-
-integral_const_binop(==, operator==);
-integral_const_binop(!=, operator!=);
-integral_const_binop(<, operator<);
-integral_const_binop(>, operator>);
-integral_const_binop(<=, operator<=);
-integral_const_binop(>=, operator>=);
-
-integral_const_binop(&&, operator&&);
-integral_const_binop(||, operator||);
-
-template <typename T, typename = metal::enable_if_t<!is_integral_v<T>>>
-METAL_FUNC constexpr auto operator||(true_type, T) {
- return true_type{};
-}
-template <typename T, typename = metal::enable_if_t<!is_integral_v<T>>>
-METAL_FUNC constexpr auto operator||(T, true_type) {
- return true_type{};
-}
-
-template <typename T, typename = metal::enable_if_t<!is_integral_v<T>>>
-METAL_FUNC constexpr auto operator&&(false_type, T) {
- return false_type{};
-}
-
-template <typename T, typename = metal::enable_if_t<!is_integral_v<T>>>
-METAL_FUNC constexpr auto operator&&(T, false_type) {
- return false_type{};
-}
-
-// Dispatch utilities
-template <typename F>
-void dispatch_bool(bool v, F f) {
- if (v) {
- f(true_type{});
- } else {
- f(false_type{});
- }
-}
-
-template <int start, int stop, int step, typename F>
-constexpr void const_for_loop(F f) {
- IF_CONSTEXPR(start < stop) {
- constexpr auto idx = Int<start>{};
- f(idx);
- const_for_loop<start + step, stop, step, F>(f);
- }
-}
-
-#undef integral_const_binop
-
-///////////////////////////////////////////////////////////////////////////////
-// Reduction operators
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename T>
-METAL_FUNC constexpr T sum(T x) {
- return x;
-}
-
-template <typename T, typename... Us>
-METAL_FUNC constexpr auto sum(T x, Us... us) {
- return x + sum(us...);
-}
-
-} // namespace steel
-
-#pragma METAL internals : disable
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs
deleted file mode 100644
index b2785dd25..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub(crate) enum TransposeConfiguration {
- NN,
- NT,
- TN,
- TT,
-}
-
-impl TransposeConfiguration {
- pub fn as_str(&self) -> &'static str {
- match self {
- TransposeConfiguration::NN => "nn",
- TransposeConfiguration::NT => "nt",
- TransposeConfiguration::TN => "tn",
- TransposeConfiguration::TT => "tt",
- }
- }
-}
-
-pub(crate) fn transpose_configuration(
- a_t: bool,
- b_t: bool,
-) -> TransposeConfiguration {
- match (a_t, b_t) {
- (false, false) => TransposeConfiguration::NN,
- (false, true) => TransposeConfiguration::NT,
- (true, false) => TransposeConfiguration::TN,
- (true, true) => TransposeConfiguration::TT,
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
index 4b3585b17..0e562b3f8 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
@@ -1,63 +1,24 @@
-use super::{common::MatmulArguments, gemm, gemv, split_k};
+use super::gemm;
use crate::{
DataType,
- backends::metal::{context::MetalContext, error::MetalError},
+ backends::{
+ common::kernel::matmul::{MatmulArguments, MatmulDispatchDescriptor, gemv, split_k},
+ metal::{Metal, context::MetalContext, error::MetalError},
+ },
};
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum MatmulKernelVariant {
- Gemv,
- SplitK,
- Gemm,
-}
-
-impl MatmulKernelVariant {
- pub fn as_str(&self) -> &'static str {
- match self {
- MatmulKernelVariant::Gemv => "GEMV",
- MatmulKernelVariant::SplitK => "Split-K",
- MatmulKernelVariant::Gemm => "GEMM",
- }
- }
-}
-
-#[derive(Debug, Clone)]
-pub(crate) enum MatmulDispatchDescriptor {
- Gemv(gemv::DispatchDescriptor),
- SplitK(split_k::DispatchDescriptor),
- Gemm(gemm::DispatchDescriptor),
-}
-
-impl MatmulDispatchDescriptor {
- pub fn variant(&self) -> MatmulKernelVariant {
- match self {
- MatmulDispatchDescriptor::Gemv(_) => MatmulKernelVariant::Gemv,
- MatmulDispatchDescriptor::SplitK(_) => MatmulKernelVariant::SplitK,
- MatmulDispatchDescriptor::Gemm(_) => MatmulKernelVariant::Gemm,
- }
- }
-}
-
-pub(crate) fn choose_dispatch_descriptor(
+pub fn choose_dispatch_descriptor(
context: &MetalContext,
data_type: DataType,
- arguments: &MatmulArguments,
+ arguments: &MatmulArguments<Metal>,
) -> Result<MatmulDispatchDescriptor, MetalError> {
- if let Some(descriptor) = gemv::DispatchDescriptor::try_new(context, data_type, arguments)? {
+ if let Some(descriptor) = gemv::DispatchDescriptor::try_new::<Metal>(data_type, arguments)? {
return Ok(MatmulDispatchDescriptor::Gemv(descriptor));
}
- if let Some(descriptor) = split_k::DispatchDescriptor::try_new(context, data_type, arguments)? {
+ if let Some(descriptor) = split_k::DispatchDescriptor::try_new::<Metal>(data_type, arguments)? {
return Ok(MatmulDispatchDescriptor::SplitK(descriptor));
}
Ok(MatmulDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(context, data_type, arguments)?))
}
-
-pub fn determine_kernel_variant(
- context: &MetalContext,
- data_type: DataType,
- arguments: &MatmulArguments,
-) -> Result<MatmulKernelVariant, MetalError> {
- choose_dispatch_descriptor(context, data_type, arguments).map(|d| d.variant())
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
index 5765c3836..b0c2087c3 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
@@ -1,55 +1,35 @@
-use metal::MTLSize;
-
-use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::TileConfiguration};
+// Re-export the common DispatchDescriptor type — we add a constructor here
+pub use crate::backends::common::kernel::matmul::gemm::DispatchDescriptor;
use crate::{
DataType,
- backends::metal::{
- context::{DeviceClass, MetalContext},
- error::MetalError,
- kernel::matmul::common::{GEMMAddMMParams, GEMMParams, MatmulArguments},
+ backends::{
+ common::{
+ gpu_types::GEMMParams,
+ kernel::matmul::{GridSize, MatmulArguments, gemm::Specialization},
+ },
+ metal::{Metal, context::MetalContext, error::MetalError},
},
};
-#[derive(Debug, Clone)]
-pub(crate) struct DispatchDescriptor {
- pub(crate) pipeline_configuration: PipelineConfiguration,
- pub(crate) params: GEMMParams,
- pub(crate) addmm_params: Option<GEMMAddMMParams>,
- pub(crate) threadgroups: MTLSize,
- pub(crate) threads_per_threadgroup: MTLSize,
-}
-
impl DispatchDescriptor {
- pub(crate) fn new(
+ pub fn new(
context: &MetalContext,
data_type: DataType,
- arguments: &MatmulArguments,
+ arguments: &MatmulArguments<Metal>,
) -> Result<Self, MetalError> {
if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(MetalError::Generic(format!("Unsupported dtype for GEMM: {data_type:?}")));
}
- let tile = select_tile(context, data_type, arguments);
+ let config = Specialization::select(context, data_type, arguments);
let m = arguments.batch;
let n = arguments.output_dim;
let k = arguments.input_dim;
- let pipeline_configuration = PipelineConfiguration {
- tile,
- transpose_a: arguments.transpose_a,
- transpose_b: arguments.transpose_b,
- align_m: (m % tile.block_rows) == 0,
- align_n: (n % tile.block_cols) == 0,
- align_k: (k % tile.block_depth) == 0,
- has_batch: arguments.batch_count > 1,
- use_out_source: arguments.c.is_some(),
- do_axpby: arguments.c.is_some() && (arguments.alpha != 1.0 || arguments.beta != 0.0),
- };
-
- let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;
- let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;
- let swizzle_log = tile.swizzle_log2;
+ let tiles_n = (n + config.block_cols - 1) / config.block_cols;
+ let tiles_m = (m + config.block_rows - 1) / config.block_rows;
+ let swizzle_log = config.swizzle_log2;
let tile_swizzle = 1 << swizzle_log;
let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
@@ -76,104 +56,20 @@ impl DispatchDescriptor {
batch_stride_b: elements_per_matrix_b,
batch_stride_d: elements_per_matrix_d,
swizzle_log,
- gemm_k_iterations_aligned: k / tile.block_depth,
+ gemm_k_iterations_aligned: k / config.block_depth,
batch_ndim: 1,
};
- let addmm_params = if pipeline_configuration.use_out_source {
- let batch_stride_c = if arguments.batch_count > 1 {
- (arguments.ldd as i64) * (arguments.output_dim as i64)
- } else {
- 0
- };
- Some(GEMMAddMMParams {
- ldc: arguments.ldd,
- fdc: 1,
- batch_stride_c,
- alpha: arguments.alpha,
- beta: arguments.beta,
- })
- } else {
- None
+ let threadgroups = GridSize {
+ x: tn_swizzled as usize,
+ y: tm_swizzled as usize,
+ z: arguments.batch_count as usize,
};
- let threads_per_threadgroup = MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);
- let threadgroups = MTLSize::new(tn_swizzled as usize, tm_swizzled as usize, arguments.batch_count as usize);
-
Ok(Self {
- pipeline_configuration,
+ specialization: config,
params,
- addmm_params,
threadgroups,
- threads_per_threadgroup,
})
}
}
-
-fn select_tile(
- context: &MetalContext,
- data_type: DataType,
- arguments: &MatmulArguments,
-) -> TileConfiguration {
- let overall_work_elements =
- (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);
- let is_float32 = matches!(data_type, DataType::F32);
- let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();
-
- if context.is_nax_available() && prefer_half_or_tf32 {
- let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);
- let tile_rows = (arguments.batch + base_tile.block_rows - 1) / base_tile.block_rows;
- let swizzle_log2 = if tile_rows <= 3 {
- 0
- } else {
- 1
- };
- return TileConfiguration {
- swizzle_log2,
- ..base_tile
- };
- }
-
- match context.device_class() {
- DeviceClass::Integrated | DeviceClass::Phone | DeviceClass::Unknown(_) => {
- if prefer_half_or_tf32 {
- if !arguments.transpose_a && arguments.transpose_b {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- } else {
- TileConfiguration::new(64, 64, 16, 1, 2, 0)
- }
- } else if !arguments.transpose_a && arguments.transpose_b {
- TileConfiguration::new(32, 64, 16, 1, 2, 0)
- } else {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- }
- },
- DeviceClass::Desktop => {
- if overall_work_elements >= (1_i64 << 20) {
- if prefer_half_or_tf32 {
- if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {
- TileConfiguration::new(64, 64, 16, 2, 2, 0)
- } else if !arguments.transpose_a && arguments.transpose_b {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- } else {
- TileConfiguration::new(32, 64, 16, 1, 2, 0)
- }
- } else if !arguments.transpose_a && arguments.transpose_b {
- TileConfiguration::new(32, 64, 16, 1, 2, 0)
- } else {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- }
- } else if prefer_half_or_tf32 {
- if !arguments.transpose_a && arguments.transpose_b {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- } else {
- TileConfiguration::new(64, 64, 16, 1, 2, 0)
- }
- } else if !arguments.transpose_a && arguments.transpose_b {
- TileConfiguration::new(32, 64, 16, 1, 2, 0)
- } else {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- }
- },
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
deleted file mode 100644
index 493b03e46..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ /dev/null
@@ -1,170 +0,0 @@
-use std::collections::HashMap;
-
-use metal::{MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration, tile_configuration::TileConfiguration};
-use crate::{
- DataType,
- backends::metal::{
- context::MetalContext,
- error::MetalError,
- kernel::matmul::common::{MatmulArguments, transpose_configuration},
- metal_extensions::{ComputeEncoderSetValue, FunctionConstantValuesSetValue},
- },
-};
-
-pub struct Kernel {
- data_type: DataType,
- pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
-}
-
-impl Kernel {
- pub fn new(data_type: DataType) -> Result<Self, MetalError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MetalError::Generic(format!("Unsupported dtype for GEMM: {data_type:?}")));
- }
- Ok(Self {
- data_type,
- pipelines: HashMap::new(),
- })
- }
-
- #[allow(clippy::type_complexity)]
- pub fn precompile(
- &mut self,
- context: &MetalContext,
- ) -> Result<(), MetalError> {
- let tiles_and_alignments: &[(TileConfiguration, &[(bool, bool, bool)])] = match self.data_type {
- DataType::BF16 => &[
- (TileConfiguration::new(64, 32, 32, 2, 2, 0), &[(false, true, true), (true, true, true)]),
- (
- TileConfiguration::new(64, 64, 16, 2, 2, 0),
- &[(false, true, true), (true, false, true), (true, true, true)],
- ),
- (TileConfiguration::new(64, 64, 16, 1, 2, 0), &[(true, true, true)]),
- ],
- DataType::F16 => {
- &[(TileConfiguration::new(64, 64, 16, 2, 2, 0), &[(true, true, true), (false, true, true)])]
- },
- DataType::F32 => {
- &[(TileConfiguration::new(32, 64, 16, 1, 2, 0), &[(false, true, true), (true, true, true)])]
- },
- _ => return Ok(()),
- };
-
- for (tile, alignments) in tiles_and_alignments {
- for &(align_m, align_n, align_k) in *alignments {
- let config = PipelineConfiguration {
- tile: *tile,
- transpose_a: false,
- transpose_b: true,
- align_m,
- align_n,
- align_k,
- has_batch: false,
- use_out_source: false,
- do_axpby: false,
- };
- let _ = self.get_or_compile_pipeline(context, &config);
- }
- }
-
- Ok(())
- }
-
- fn type_name(&self) -> &'static str {
- match self.data_type {
- DataType::F16 => "float16",
- DataType::BF16 => "bfloat16",
- DataType::F32 => "float32",
- _ => unreachable!(),
- }
- }
-
- fn kernel_name(
- &self,
- configuration: &PipelineConfiguration,
- ) -> String {
- let type_name = self.type_name();
- let transpose_suffix = transpose_configuration(configuration.transpose_a, configuration.transpose_b);
- let prefix = if configuration.tile.is_nax() {
- "steel_gemm_nax"
- } else {
- "steel_gemm"
- };
- format!(
- "{}_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}",
- prefix,
- transpose_suffix.as_str(),
- type_name,
- type_name,
- configuration.tile.block_rows,
- configuration.tile.block_cols,
- configuration.tile.block_depth,
- configuration.tile.warps_per_row,
- configuration.tile.warps_per_col
- )
- }
-
- fn get_or_compile_pipeline(
- &mut self,
- context: &MetalContext,
- configuration: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {
- if !self.pipelines.contains_key(configuration) {
- let kernel_name = self.kernel_name(configuration);
- let function_constants = MTLFunctionConstantValues::new();
- function_constants.set_value(&configuration.has_batch, 10);
- function_constants.set_value(&configuration.use_out_source, 100);
- function_constants.set_value(&configuration.do_axpby, 110);
- function_constants.set_value(&configuration.align_m, 200);
- function_constants.set_value(&configuration.align_n, 201);
- function_constants.set_value(&configuration.align_k, 202);
-
- let cache_key = format!(
- "{}_am{}_an{}_ak{}_hb{}_uo{}_ax{}",
- kernel_name,
- configuration.align_m as u8,
- configuration.align_n as u8,
- configuration.align_k as u8,
- configuration.has_batch as u8,
- configuration.use_out_source as u8,
- configuration.do_axpby as u8
- );
- let pipeline_state =
- context.compute_pipeline_state_cached(&cache_key, &kernel_name, Some(&function_constants))?;
- self.pipelines.insert(configuration.clone(), pipeline_state);
- }
- Ok(self.pipelines.get(configuration).unwrap())
- }
-
- pub(crate) fn encode_descriptor(
- &mut self,
- context: &MetalContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MatmulArguments,
- descriptor: &DispatchDescriptor,
- ) -> Result<bool, MetalError> {
- let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;
- encoder.set_compute_pipeline_state(pipeline_state);
-
- encoder.set_buffer(Some(arguments.a), arguments.a_offset as usize, 0);
- encoder.set_buffer(Some(arguments.b), 0, 1);
- if descriptor.pipeline_configuration.use_out_source {
- if let Some(c_buffer) = arguments.c {
- encoder.set_buffer(Some(c_buffer), 0, 2);
- }
- }
- encoder.set_buffer(Some(arguments.d), 0, 3);
-
- encoder.set_value(&descriptor.params, 4);
-
- if let Some(addmm_params) = &descriptor.addmm_params {
- encoder.set_value(addmm_params, 5);
- }
-
- encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);
- Ok(false)
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
index 6b041d0fc..d9cb5df12 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -1,7 +1,4 @@
mod dispatch_descriptor;
-mod kernel;
-mod pipeline_configuration;
-mod tile_configuration;
+mod specialization;
-pub(crate) use dispatch_descriptor::DispatchDescriptor;
-pub use kernel::Kernel as GemmKernel;
+pub use dispatch_descriptor::DispatchDescriptor;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs
deleted file mode 100644
index 252792c91..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-use super::tile_configuration::TileConfiguration;
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct PipelineConfiguration {
- pub tile: TileConfiguration,
- pub transpose_a: bool,
- pub transpose_b: bool,
- pub align_m: bool,
- pub align_n: bool,
- pub align_k: bool,
- pub has_batch: bool,
- pub use_out_source: bool,
- pub do_axpby: bool,
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.h
deleted file mode 100644
index f201ef4af..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.h
+++ /dev/null
@@ -1,381 +0,0 @@
-
-
-using namespace steel;
-
-///////////////////////////////////////////////////////////////////////////////
-// GEMM kernels
-///////////////////////////////////////////////////////////////////////////////
-
-constant bool has_batch [[function_constant(10)]];
-
-constant bool use_out_source [[function_constant(100)]];
-constant bool do_axpby [[function_constant(110)]];
-
-constant bool align_M [[function_constant(200)]];
-constant bool align_N [[function_constant(201)]];
-constant bool align_K [[function_constant(202)]];
-
-// clang-format off
-template <
- typename T,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- typename AccumType = float>
-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gemm(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- const device T* C [[buffer(2), function_constant(use_out_source)]],
- device T* D [[buffer(3)]],
- const constant GEMMParams* params [[buffer(4)]],
- const constant GEMMAddMMParams* addmm_params [[buffer(5), function_constant(use_out_source)]],
- const constant int* batch_shape [[buffer(6), function_constant(has_batch)]],
- const constant int64_t* batch_strides [[buffer(7), function_constant(has_batch)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]]) { // clang-format on
- // Pacifying compiler
- (void)lid;
-
- using gemm_kernel = GEMMKernel<
- T,
- T,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- true,
- true,
- AccumType>;
-
- using loader_a_t = typename gemm_kernel::loader_a_t;
- using loader_b_t = typename gemm_kernel::loader_b_t;
- using mma_t = typename gemm_kernel::mma_t;
-
- // Find block
- const int tid_y = ((tid.y) << params->swizzle_log) +
- ((tid.x) & ((1 << params->swizzle_log) - 1));
- const int tid_x = (tid.x) >> params->swizzle_log;
-
- // Exit early if out of bounds
- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
- return;
- }
-
- // Adjust for batch
- if (has_batch) {
- const constant auto* A_bstrides = batch_strides;
- const constant auto* B_bstrides = batch_strides + params->batch_ndim;
-
- ulong2 batch_offsets = elem_to_loc_broadcast(
- tid.z,
- batch_shape,
- A_bstrides,
- B_bstrides,
- params->batch_ndim
- );
-
- A += batch_offsets.x;
- B += batch_offsets.y;
-
- if (use_out_source) {
- const constant auto* C_bstrides = B_bstrides + params->batch_ndim;
- C += elem_to_loc(tid.z, batch_shape, C_bstrides, params->batch_ndim);
- }
- } else {
- A += params->batch_stride_a * tid.z;
- B += params->batch_stride_b * tid.z;
-
- if (use_out_source) {
- C += addmm_params->batch_stride_c * tid.z;
- }
- }
-
- D += params->batch_stride_d * tid.z;
-
- // Prepare threadgroup memory
- threadgroup T As[gemm_kernel::tgp_mem_size_a];
- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];
-
- threadgroup_barrier(mem_flags::mem_none);
-
- // Find block in A, B, C
- const int c_row = tid_y * BM;
- const int c_col = tid_x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- A += transpose_a ? c_row_long : c_row_long * params->lda;
- B += transpose_b ? c_col_long * params->ldb : c_col_long;
- D += c_row_long * params->ldd + c_col_long;
-
- if (use_out_source) {
- C += c_row_long * addmm_params->ldc + c_col_long * addmm_params->fdc;
- }
-
- // Prepare threadgroup mma operation
- thread mma_t mma_op(simd_group_id, simd_lane_id);
-
- // Prepare threadgroup loading operations
- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);
-
- // Prepare threadgroup bounds
- const short tgp_bm = align_M ? BM : short(min(BM, params->M - c_row));
- const short tgp_bn = align_N ? BN : short(min(BN, params->N - c_col));
-
- // Prepare iterations
- int gemm_k_iterations = params->gemm_k_iterations_aligned;
-
- // Do unaligned K iterations first
- if (!align_K) {
- const int k_last = params->gemm_k_iterations_aligned * BK;
- const int k_remain = params->K - k_last;
- const size_t k_jump_a =
- transpose_a ? params->lda * size_t(k_last) : size_t(k_last);
- const size_t k_jump_b =
- transpose_b ? size_t(k_last) : params->ldb * size_t(k_last);
-
- // Move loader source ahead to end
- loader_a.src += k_jump_a;
- loader_b.src += k_jump_b;
-
- // Load tile
- const short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);
- const short2 tile_dims_B =
- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);
-
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Do matmul
- mma_op.mma(As, Bs);
-
- // Reset source back to start
- loader_a.src -= k_jump_a;
- loader_b.src -= k_jump_b;
- }
-
- const TransformAdd<AccumType, AccumType> epilogue_op_add(
- addmm_params->alpha,
- addmm_params->beta
- );
- const TransformAxpby<AccumType, AccumType> epilogue_op_axpby(
- addmm_params->alpha,
- addmm_params->beta
- );
-
- ///////////////////////////////////////////////////////////////////////////////
- // MNK aligned loop
- if (align_M && align_N) {
- // Do gemm
- for (int k = 0; k < gemm_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- // Load elements into threadgroup
- loader_a.load_unsafe();
- loader_b.load_unsafe();
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- // Do epilogue
- if (use_out_source) {
- if (do_axpby) {
- mma_op.apply_epilogue(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- epilogue_op_axpby
- );
- } else {
- mma_op.apply_epilogue(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- epilogue_op_add
- );
- }
- }
-
- // Store results to device memory
- return mma_op.store_result(D, params->ldd);
-
- }
- ///////////////////////////////////////////////////////////////////////////////
- // MN unaligned loop
- else { // Loop over K - unaligned case
- const int leftover_bk = 0;
-
- if ((align_M || tgp_bm == BM) && (align_N || tgp_bn == BN)) {
- // Do gemm
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<true, true, true>{}
- );
-
- // Do epilogue
- if (use_out_source) {
- if (do_axpby) {
- mma_op.apply_epilogue(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- epilogue_op_axpby
- );
- } else {
- mma_op.apply_epilogue(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- epilogue_op_add
- );
- }
- }
-
- // Store results to device memory
- return mma_op.store_result(D, params->ldd);
-
- } else if (align_N || tgp_bn == BN) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<false, true, true>{}
- );
-
- // Do epilogue
- if (use_out_source) {
- if (do_axpby) {
- mma_op.apply_epilogue_safe(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- short2(tgp_bn, tgp_bm),
- epilogue_op_axpby
- );
- } else {
- mma_op.apply_epilogue_safe(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- short2(tgp_bn, tgp_bm),
- epilogue_op_add
- );
- }
- }
-
- // Store results to device memory
- return mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
-
- } else if (align_M || tgp_bm == BM) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<true, false, true>{}
- );
-
- // Do epilogue
- if (use_out_source) {
- if (do_axpby) {
- mma_op.apply_epilogue_safe(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- short2(tgp_bn, tgp_bm),
- epilogue_op_axpby
- );
- } else {
- mma_op.apply_epilogue_safe(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- short2(tgp_bn, tgp_bm),
- epilogue_op_add
- );
- }
- }
-
- // Store results to device memory
- return mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
-
- } else {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<false, false, true>{}
- );
-
- // Do epilogue
- if (use_out_source) {
- if (do_axpby) {
- mma_op.apply_epilogue_safe(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- short2(tgp_bn, tgp_bm),
- epilogue_op_axpby
- );
- } else {
- mma_op.apply_epilogue_safe(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- short2(tgp_bn, tgp_bm),
- epilogue_op_add
- );
- }
- }
-
- // Store results to device memory
- return mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
index 34c164aa3..6a8becf1d 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
@@ -1,34 +1,232 @@
-
-
// clang-format off
#include "../../../common/utils.h"
+#include "../../../definitions.metal"
#include "../../common/steel/gemm/gemm.h"
-#include "steel_gemm.h"
-
-#define instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_kernel( \
- "steel_gemm_" #tname "_" #iname "_" #oname \
- "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn, \
- gemm, itype, bm, bn, bk, wm, wn, trans_a, trans_b, float)
-
-#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)
-
-#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 1, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 32, 32, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 64, 16, 1, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 32, 8, 4, 1)
-
-instantiate_gemm_shapes_helper(float16, half, float16, half);
-instantiate_gemm_shapes_helper(bfloat16, bfloat16_t, bfloat16, bfloat16_t);
-
-instantiate_gemm_shapes_helper(float32, float, float32, float);
-instantiate_gemm_shapes_helper(complex64, complex64_t, complex64, complex64_t);
+
+using namespace steel;
+
+// Upper bounds for threadgroup memory (in elements of T).
+// Max across all tile/type combos: 64x32x32 with half (padding=8).
+// tgp_mem_size_a = BM * (BK + padding) = 64 * (32 + 8) = 2560
+// tgp_mem_size_b = BN * (BK + padding) = 32 * (32 + 8) = 1280
+// For 64x64x16 with half: a=64*(16+8)=1536, b=64*(16+8)=1536
+// Max a = 2560, max b = 1536
+#define GEMM_MAX_TGP_A 2560
+#define GEMM_MAX_TGP_B 1536
+
+namespace uzu {
+namespace matmul {
+using GEMMParams = steel::GEMMParams;
+} // namespace matmul
+} // namespace uzu
+
+///////////////////////////////////////////////////////////////////////////////
+// GEMM implementation
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T, typename AccumType = float>
+METAL_FUNC void gemm_impl(
+ const device T* a,
+ const device T* b,
+ device T* d,
+ const constant GEMMParams* params,
+ const int BM,
+ const int BN,
+ const int BK,
+ const int WM,
+ const int WN,
+ const bool align_m,
+ const bool align_n,
+ const bool align_k,
+ threadgroup T* a_shared,
+ threadgroup T* b_shared,
+ uint simd_lane_id,
+ uint simd_group_id,
+ uint3 tid,
+ uint3 lid
+) {
+ (void)lid;
+
+ // Hardcoded: transpose_a = false, transpose_b = true
+ const bool transpose_a = false;
+ const bool transpose_b = true;
+
+ const short tgp_padding = 16 / sizeof(T);
+ const short lda_tgp = BK + tgp_padding; // transpose_a=false: BK + pad
+ const short ldb_tgp = BK + tgp_padding; // transpose_b=true: BK + pad
+ const short tgp_size = WM * WN * 32;
+
+ // Find block
+ const int tid_y = ((tid.y) << params->swizzle_log) +
+ ((tid.x) & ((1 << params->swizzle_log) - 1));
+ const int tid_x = (tid.x) >> params->swizzle_log;
+
+ if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
+ return;
+ }
+
+ // Batch offset (non-batched path only)
+ a += params->batch_stride_a * tid.z;
+ b += params->batch_stride_b * tid.z;
+ d += params->batch_stride_d * tid.z;
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ // Find block in a, b, d
+ const int c_row = tid_y * BM;
+ const int c_col = tid_x * BN;
+ const size_t c_row_long = size_t(c_row);
+ const size_t c_col_long = size_t(c_col);
+
+ // transpose_a=false: a += c_row * lda
+ a += c_row_long * params->lda;
+ // transpose_b=true: b += c_col * ldb
+ b += c_col_long * params->ldb;
+ d += c_row_long * params->ldd + c_col_long;
+
+ // Construct loader and MMA objects with runtime tile params
+ // transpose_a=false: loader_a BROWS=BM, BCOLS=BK, dst_ld=BK+pad, reduction_dim=1
+ thread BlockLoader<T> loader_a(
+ a, params->lda, a_shared,
+ simd_group_id, simd_lane_id,
+ BM, BK, lda_tgp, 1, tgp_size);
+
+ // transpose_b=true: loader_b BROWS=BN, BCOLS=BK, dst_ld=BK+pad, reduction_dim=1
+ thread BlockLoader<T> loader_b(
+ b, params->ldb, b_shared,
+ simd_group_id, simd_lane_id,
+ BN, BK, ldb_tgp, 1, tgp_size);
+
+ thread BlockMMA<T, T, AccumType> mma_op(
+ simd_group_id, simd_lane_id,
+ BM, BN, BK, WM, WN,
+ transpose_a, transpose_b,
+ lda_tgp, ldb_tgp);
+
+ const short tgp_bm = align_m ? BM : short(min(BM, params->M - c_row));
+ const short tgp_bn = align_n ? BN : short(min(BN, params->N - c_col));
+
+ int gemm_k_iterations = params->gemm_k_iterations_aligned;
+
+ // Do unaligned K iterations first
+ if (!align_k) {
+ const int k_last = params->gemm_k_iterations_aligned * BK;
+ const int k_remain = params->K - k_last;
+ // transpose_a=false: k_jump_a = k_last
+ const size_t k_jump_a = size_t(k_last);
+ // transpose_b=true: k_jump_b = k_last
+ const size_t k_jump_b = size_t(k_last);
+
+ loader_a.src += k_jump_a;
+ loader_b.src += k_jump_b;
+
+ // transpose_a=false: tile_dims_a = (k_remain, tgp_bm)
+ const short2 tile_dims_a = short2(k_remain, tgp_bm);
+ // transpose_b=true: tile_dims_b = (k_remain, tgp_bn)
+ const short2 tile_dims_b = short2(k_remain, tgp_bn);
+
+ loader_a.load_safe(tile_dims_a);
+ loader_b.load_safe(tile_dims_b);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(a_shared, b_shared);
+
+ loader_a.src -= k_jump_a;
+ loader_b.src -= k_jump_b;
+ }
+
+ // MNK aligned loop
+ if (align_m && align_n) {
+ for (int k = 0; k < gemm_k_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_a.load_unsafe();
+ loader_b.load_unsafe();
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(a_shared, b_shared);
+ loader_a.next();
+ loader_b.next();
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+ return mma_op.store_result(d, params->ldd);
+ } else {
+ const short leftover_bk = 0;
+
+ if ((align_m || tgp_bm == BM) && (align_n || tgp_bn == BN)) {
+ gemm_loop<T, T, AccumType>(
+ a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,
+ mma_op, tgp_bm, tgp_bn, leftover_bk,
+ BK, transpose_a, transpose_b, true, true, true);
+ return mma_op.store_result(d, params->ldd);
+ } else if (align_n || tgp_bn == BN) {
+ gemm_loop<T, T, AccumType>(
+ a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,
+ mma_op, tgp_bm, tgp_bn, leftover_bk,
+ BK, transpose_a, transpose_b, false, true, true);
+ return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));
+ } else if (align_m || tgp_bm == BM) {
+ gemm_loop<T, T, AccumType>(
+ a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,
+ mma_op, tgp_bm, tgp_bn, leftover_bk,
+ BK, transpose_a, transpose_b, true, false, true);
+ return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));
+ } else {
+ gemm_loop<T, T, AccumType>(
+ a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,
+ mma_op, tgp_bm, tgp_bn, leftover_bk,
+ BK, transpose_a, transpose_b, false, false, true);
+ return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));
+ }
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Unified DSL kernel
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(MatmulGemm)(
+ const device T* a,
+ const device T* b,
+ device T* d,
+ const constant uzu::matmul::GEMMParams* params,
+ const constant uint& group_count_x,
+ const constant uint& group_count_y,
+ const constant uint& group_count_z,
+ threadgroup T a_shared[GEMM_MAX_TGP_A],
+ threadgroup T b_shared[GEMM_MAX_TGP_B],
+ const uint block_rows SPECIALIZE,
+ const uint block_cols SPECIALIZE,
+ const uint block_depth SPECIALIZE,
+ const uint warps_per_row SPECIALIZE,
+ const uint warps_per_col SPECIALIZE,
+ const bool align_m SPECIALIZE,
+ const bool align_n SPECIALIZE,
+ const bool align_k SPECIALIZE,
+ const uint group_x GROUPS(group_count_x),
+ const uint group_y GROUPS(group_count_y),
+ const uint group_z GROUPS(group_count_z),
+ const uint thread_x THREADS(32),
+ const uint thread_y THREADS(2),
+ const uint thread_z THREADS(2),
+ const Simd simd
+) {
+ if (simd.group_idx >= warps_per_row * warps_per_col) {
+ return;
+ }
+
+ gemm_impl<T, float>(
+ a, b, d, params,
+ block_rows, block_cols, block_depth,
+ warps_per_row, warps_per_col,
+ align_m, align_n, align_k,
+ a_shared, b_shared,
+ simd.lane_idx, simd.group_idx,
+ uint3(group_x, group_y, group_z),
+ uint3(thread_x, thread_y, thread_z)
+ );
+}
+
// clang-format on
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.h
deleted file mode 100644
index 18338aa1a..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.h
+++ /dev/null
@@ -1,225 +0,0 @@
-
-
-using namespace steel;
-
-constant bool has_batch [[function_constant(10)]];
-
-constant bool use_out_source [[function_constant(100)]];
-constant bool do_axpby [[function_constant(110)]];
-
-constant bool align_M [[function_constant(200)]];
-constant bool align_N [[function_constant(201)]];
-constant bool align_K [[function_constant(202)]];
-
-// clang-format off
-template <
- bool kAlignedM,
- bool kAlignedN,
- typename NAXTile_t,
- typename T>
-void gemm_epilogue(
- thread NAXTile_t& Dtile,
- const device T* C,
- const constant GEMMParams* params,
- const constant GEMMAddMMParams* addmm_params,
- const short sgp_sm,
- const short sgp_sn) { // clang-format on
-
- (void)params;
-
- constexpr short UM = NAXTile_t::kSubTileRows;
- constexpr short UN = NAXTile_t::kSubTileCols;
- using CSubTile = NAXSubTile<T, UM, UN>;
-
- using V = typename NAXTile_t::elem_type;
-
- constexpr short TM = NAXTile_t::kTileRows;
- constexpr short TN = NAXTile_t::kTileCols;
- constexpr short kElemsPerSubTile = NAXTile_t::kElemsPerSubTile;
-
- STEEL_PRAGMA_UNROLL
- for (short mm = 0; mm < TM; mm++) {
- STEEL_PRAGMA_UNROLL
- for (short nn = 0; nn < TN; nn++) {
- const short m = mm * UM;
- const short n = nn * UN;
-
- CSubTile CTile;
-
- if constexpr (kAlignedM && kAlignedN) {
- CTile.load(C, addmm_params->ldc, addmm_params->fdc, m, n);
- } else {
- CTile.load_safe(
- C,
- addmm_params->ldc,
- addmm_params->fdc,
- sgp_sm,
- sgp_sn,
- m,
- n
- );
- }
-
- auto delems = Dtile.subtile_at(mm, nn).elems();
- auto celems = CTile.elems();
-
- STEEL_PRAGMA_UNROLL
- for (short i = 0; i < kElemsPerSubTile; i++) {
- if (do_axpby) {
- delems[i] = addmm_params->alpha * delems[i] +
- addmm_params->beta * static_cast<V>(celems[i]);
- } else {
- delems[i] += static_cast<V>(celems[i]);
- }
- }
- }
- }
-}
-
-// clang-format off
-template <
- typename T,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- typename AccumType = float>
-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gemm(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- const device T* C [[buffer(2), function_constant(use_out_source)]],
- device T* D [[buffer(3)]],
- const constant GEMMParams* params [[buffer(4)]],
- const constant GEMMAddMMParams* addmm_params [[buffer(5), function_constant(use_out_source)]],
- const constant int* batch_shape [[buffer(6), function_constant(has_batch)]],
- const constant int64_t* batch_strides [[buffer(7), function_constant(has_batch)]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]]) { // clang-format on
- // Find block
- const int tid_y = ((tid.y) << params->swizzle_log) +
- ((tid.x) & ((1 << params->swizzle_log) - 1));
- const int tid_x = (tid.x) >> params->swizzle_log;
-
- // Exit early if out of bounds
- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
- return;
- }
-
- // Adjust for batch
- if (has_batch) {
- const constant auto* A_bstrides = batch_strides;
- const constant auto* B_bstrides = batch_strides + params->batch_ndim;
-
- ulong2 batch_offsets = elem_to_loc_broadcast(
- tid.z,
- batch_shape,
- A_bstrides,
- B_bstrides,
- params->batch_ndim
- );
-
- A += batch_offsets.x;
- B += batch_offsets.y;
-
- if (use_out_source) {
- const constant auto* C_bstrides = B_bstrides + params->batch_ndim;
- C += elem_to_loc(tid.z, batch_shape, C_bstrides, params->batch_ndim);
- }
- } else {
- A += params->batch_stride_a * tid.z;
- B += params->batch_stride_b * tid.z;
-
- if (use_out_source) {
- C += addmm_params->batch_stride_c * tid.z;
- }
- }
-
- D += params->batch_stride_d * tid.z;
-
- // Prepare threadgroup memory
- threadgroup_barrier(mem_flags::mem_none);
-
- // Find block in A, B, C
- const int c_row = tid_y * BM;
- const int c_col = tid_x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- A += transpose_a ? c_row_long : c_row_long * params->lda;
- B += transpose_b ? c_col_long * params->ldb : c_col_long;
- D += c_row_long * params->ldd + c_col_long;
-
- if (use_out_source) {
- C += c_row_long * addmm_params->ldc + c_col_long * addmm_params->fdc;
- }
-
- constexpr short UM = 16;
- constexpr short UN = 32;
- constexpr short UK = 16;
- constexpr short SM = BM / WM;
- constexpr short SN = BN / WN;
- constexpr short SK = 32;
-
- constexpr short TM = SM / UM;
- constexpr short TN = SN / UN;
-
- const short tm = SM * (simd_group_id / WN);
- const short tn = SN * (simd_group_id % WN);
-
- const short sgp_sm = align_M ? SM : min(SM, short(params->M - (c_row + tm)));
- const bool is_unaligned_sm = align_M ? false : (sgp_sm != SM);
-
- const short sgp_sn = align_N ? SN : min(SN, short(params->N - (c_col + tn)));
- const bool is_unaligned_sn = align_N ? false : (sgp_sn != SN);
-
- A += transpose_a ? tm : (tm * params->lda);
- B += transpose_b ? (tn * params->ldb) : tn;
- D += tm * params->ldd + tn;
-
- if (use_out_source) {
- C += tm * addmm_params->ldc + tn * addmm_params->fdc;
- }
-
- using DSubTile = NAXSubTile<AccumType, UM, UN>;
- NAXTile<AccumType, TM, TN, DSubTile> Dtile;
-
- dispatch_bool(align_K, [&](auto kAlignedK) {
- dispatch_bool(align_M || !is_unaligned_sm, [&](auto kAlignedM) {
- dispatch_bool(align_N || !is_unaligned_sn, [&](auto kAlignedN) {
- Dtile = gemm_loop<
- T,
- SM,
- SN,
- SK,
- BK,
- transpose_a,
- transpose_b,
- kAlignedM.value,
- kAlignedN.value,
- kAlignedK.value,
- UM,
- UN,
- UK,
- AccumType>(A, B, params, sgp_sm, sgp_sn);
- if (use_out_source) {
- gemm_epilogue<kAlignedM.value, kAlignedN.value>(
- Dtile,
- C,
- params,
- addmm_params,
- sgp_sm,
- sgp_sn
- );
- }
- if constexpr (kAlignedM && kAlignedN) {
- Dtile.store(D, int(params->ldd));
- } else {
- Dtile.store_safe(D, int(params->ldd), short2(sgp_sn, sgp_sm));
- }
- });
- });
- });
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal
deleted file mode 100644
index 286b81152..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
-#include <metal_stdlib>
-
-#include "../../../common/utils.h"
-
-#include "../../common/steel/gemm/gemm_nax.h"
-#include "steel_gemm_nax.h"
-
-// clang-format off
-#define instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_kernel( \
- "steel_gemm_nax_" #tname "_" #iname "_" #oname \
- "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn, \
- gemm, itype, bm, bn, bk, wm, wn, trans_a, trans_b, float)
-
-#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)
-
-#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 64, 256, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 128, 128, 512, 4, 4)
-
-instantiate_gemm_shapes_helper(float16, half, float16, half);
-instantiate_gemm_shapes_helper(bfloat16, bfloat, bfloat16, bfloat);
-instantiate_gemm_shapes_helper(float32, float, float32, float);
-// clang-format on
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
new file mode 100644
index 000000000..d1ee12719
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -0,0 +1,93 @@
+use crate::{
+ DataType,
+ backends::{
+ common::kernel::matmul::{MatmulArguments, gemm::Specialization},
+ metal::{
+ Metal,
+ context::{DeviceClass, MetalContext},
+ },
+ },
+};
+
+impl Specialization {
+ pub fn select(
+ context: &MetalContext,
+ data_type: DataType,
+ arguments: &MatmulArguments<Metal>,
+ ) -> Self {
+ let overall_work_elements =
+ (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);
+ let is_float32 = matches!(data_type, DataType::F32);
+ let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();
+
+ let (block_rows, block_cols, block_depth, warps_per_row, warps_per_col, swizzle_log2) =
+ if context.is_nax_available() && prefer_half_or_tf32 {
+ let tile_rows = (arguments.batch + 128 - 1) / 128;
+ let swizzle_log2 = if tile_rows <= 3 {
+ 0
+ } else {
+ 1
+ };
+ (128, 128, 512, 4, 4, swizzle_log2)
+ } else {
+ match context.device_class() {
+ DeviceClass::Integrated | DeviceClass::Phone | DeviceClass::Unknown(_) => {
+ if prefer_half_or_tf32 {
+ if arguments.transpose_b {
+ (64, 32, 32, 2, 2, 0)
+ } else {
+ (64, 64, 16, 1, 2, 0)
+ }
+ } else if arguments.transpose_b {
+ (32, 64, 16, 1, 2, 0)
+ } else {
+ (64, 32, 32, 2, 2, 0)
+ }
+ },
+ DeviceClass::Desktop => {
+ if overall_work_elements >= (1_i64 << 20) {
+ if prefer_half_or_tf32 {
+ if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {
+ (64, 64, 16, 2, 2, 0)
+ } else if arguments.transpose_b {
+ (64, 32, 32, 2, 2, 0)
+ } else {
+ (32, 64, 16, 1, 2, 0)
+ }
+ } else if arguments.transpose_b {
+ (32, 64, 16, 1, 2, 0)
+ } else {
+ (64, 32, 32, 2, 2, 0)
+ }
+ } else if prefer_half_or_tf32 {
+ if arguments.transpose_b {
+ (64, 32, 32, 2, 2, 0)
+ } else {
+ (64, 64, 16, 1, 2, 0)
+ }
+ } else if arguments.transpose_b {
+ (32, 64, 16, 1, 2, 0)
+ } else {
+ (64, 32, 32, 2, 2, 0)
+ }
+ },
+ }
+ };
+
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ let k = arguments.input_dim;
+
+ Self {
+ block_rows,
+ block_cols,
+ block_depth,
+ warps_per_row,
+ warps_per_col,
+ swizzle_log2,
+ align_m: (m % block_rows) == 0,
+ align_n: (n % block_cols) == 0,
+ align_k: (k % block_depth) == 0,
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs
deleted file mode 100644
index 3f857b490..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct TileConfiguration {
- pub block_rows: i32,
- pub block_cols: i32,
- pub block_depth: i32,
- pub warps_per_row: u64,
- pub warps_per_col: u64,
- pub swizzle_log2: i32,
-}
-
-impl TileConfiguration {
- pub fn new(
- block_rows: i32,
- block_cols: i32,
- block_depth: i32,
- warps_per_row: u64,
- warps_per_col: u64,
- swizzle_log2: i32,
- ) -> Self {
- Self {
- block_rows,
- block_cols,
- block_depth,
- warps_per_row,
- warps_per_col,
- swizzle_log2,
- }
- }
-
- pub fn is_nax(&self) -> bool {
- self.block_depth >= 256
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs
deleted file mode 100644
index b72ad7cc0..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs
+++ /dev/null
@@ -1,213 +0,0 @@
-use std::sync::OnceLock;
-
-use metal::MTLSize;
-
-use super::pipeline_configuration::{PipelineConfiguration, select_configuration};
-use crate::{
- DataType,
- backends::metal::{context::MetalContext, error::MetalError, kernel::matmul::common::MatmulArguments},
-};
-
-const DEFAULT_GEMV_MAX_BATCH: i32 = 8;
-
-static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();
-
-fn max_gemv_batch_threshold() -> i32 {
- *GEMV_MAX_BATCH.get_or_init(|| {
- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
- })
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(crate) enum AxpbySource {
- None,
- Bias,
- C,
-}
-
-#[derive(Debug, Clone)]
-pub(crate) struct DispatchDescriptor {
- pub(crate) pipeline_configuration: PipelineConfiguration,
- pub(crate) matrix_is_rhs: bool,
- pub(crate) axpby_source: AxpbySource,
- pub(crate) input_dimension: i32,
- pub(crate) output_dimension: i32,
- pub(crate) matrix_leading_dim: i32,
- pub(crate) alpha: f32,
- pub(crate) beta: f32,
- pub(crate) batch_ndim: i32,
- pub(crate) batch_shape: [i32; 1],
- pub(crate) vector_batch_stride: [i64; 1],
- pub(crate) matrix_batch_stride: [i64; 1],
- pub(crate) bias_batch_stride: [i64; 1],
- pub(crate) bias_stride: i32,
- /// Number of batch rows (M dimension) - used for batched GEMV
- pub(crate) batch_rows: i32,
- /// Leading dimension of output (ldd)
- pub(crate) output_ld: i32,
- /// Leading dimension of input vector (lda)
- pub(crate) vector_ld: i32,
- pub(crate) threadgroups: MTLSize,
- pub(crate) threads_per_threadgroup: MTLSize,
-}
-
-impl DispatchDescriptor {
- pub(crate) fn try_new(
- _context: &MetalContext,
- data_type: DataType,
- arguments: &MatmulArguments,
- ) -> Result<Option<Self>, MetalError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MetalError::Generic(format!("Unsupported data type for GEMV: {data_type:?}")));
- }
-
- if arguments.transpose_a || !arguments.transpose_b {
- return Ok(None);
- }
-
- let m = arguments.batch;
- let n = arguments.output_dim;
-
- let max_gemv_batch = max_gemv_batch_threshold();
-
- if n == 1 {
- if m != 1 {
- return Ok(None);
- }
- } else if m > max_gemv_batch {
- return Ok(None);
- }
-
- let matrix_is_rhs = n != 1;
- let transpose_matrix = if matrix_is_rhs {
- !arguments.transpose_b
- } else {
- arguments.transpose_a
- };
-
- let axpby_source = if arguments.c.is_some() {
- AxpbySource::C
- } else if arguments.bias.is_some() {
- AxpbySource::Bias
- } else {
- AxpbySource::None
- };
-
- let (do_axpby, alpha, beta, bias_stride) = match axpby_source {
- AxpbySource::None => (false, 1.0f32, 0.0f32, 0),
- AxpbySource::Bias => (true, 1.0f32, 1.0f32, 1),
- AxpbySource::C => (true, arguments.alpha, arguments.beta, arguments.ldd),
- };
-
- let output_dimension = if matrix_is_rhs {
- arguments.output_dim
- } else {
- arguments.batch
- };
-
- let mut batch_pack = 1;
- if m == 4 && arguments.input_dim <= 2048 && (1536..=3072).contains(&output_dimension) {
- batch_pack = 2;
- } else if m <= 8 {
- batch_pack = if m >= 4 && m % 4 == 0 {
- 4
- } else if m >= 2 && m % 2 == 0 {
- 2
- } else {
- 1
- };
- }
-
- let pipeline_configuration = select_configuration(
- arguments.transpose_a,
- arguments.transpose_b,
- transpose_matrix,
- batch_pack as u32,
- arguments.input_dim,
- output_dimension,
- false,
- do_axpby,
- );
-
- let input_dimension = arguments.input_dim;
- let matrix_leading_dim = if matrix_is_rhs {
- arguments.ldb
- } else {
- arguments.lda
- };
-
- let batch_ndim = 1i32;
- let batch_groups = arguments.batch_count.max(1);
- let batch_shape = [if arguments.batch_count > 1 {
- arguments.batch_count
- } else {
- 1
- }];
-
- let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);
- let elements_per_matrix_b = if arguments.transpose_b {
- (arguments.output_dim as i64) * (arguments.ldb as i64)
- } else {
- (arguments.input_dim as i64) * (arguments.ldb as i64)
- };
-
- let vector_batch_stride = [if matrix_is_rhs {
- elements_per_matrix_a
- } else {
- elements_per_matrix_b
- }];
-
- let matrix_batch_stride = [if matrix_is_rhs {
- elements_per_matrix_b
- } else {
- elements_per_matrix_a
- }];
-
- let bias_batch_stride = [if arguments.batch_count > 1 {
- (output_dimension as i64) * (arguments.ldd as i64)
- } else {
- 0
- }];
-
- let output_elements_per_threadgroup = pipeline_configuration.output_elements_per_threadgroup();
- let threadgroup_count_x =
- ((output_dimension as u32 + output_elements_per_threadgroup - 1) / output_elements_per_threadgroup) as u64;
- let threadgroup_count_z = batch_groups.max(1) as u64;
-
- let batch_rows = arguments.batch;
- let threadgroup_count_y = (batch_rows / batch_pack).max(1) as u64;
-
- let threadgroups =
- MTLSize::new(threadgroup_count_x as usize, threadgroup_count_y as usize, threadgroup_count_z as usize);
- let threads_per_threadgroup = pipeline_configuration.threads_per_threadgroup();
-
- let output_ld = arguments.ldd;
- let vector_ld = arguments.lda;
-
- Ok(Some(Self {
- pipeline_configuration,
- matrix_is_rhs,
- axpby_source,
- input_dimension,
- output_dimension,
- matrix_leading_dim,
- alpha,
- beta,
- batch_ndim,
- batch_shape,
- vector_batch_stride,
- matrix_batch_stride,
- bias_batch_stride,
- bias_stride,
- batch_rows,
- output_ld,
- vector_ld,
- threadgroups,
- threads_per_threadgroup,
- }))
- }
-
- pub(crate) fn bias_is_fused(&self) -> bool {
- matches!(self.axpby_source, AxpbySource::Bias)
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
deleted file mode 100644
index 883b9ae69..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-use std::collections::HashMap;
-
-use metal::{MTLComputeCommandEncoder, MTLComputePipelineState};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::{
- dispatch_descriptor::{AxpbySource, DispatchDescriptor},
- pipeline_configuration::PipelineConfiguration,
-};
-use crate::{
- DataType,
- backends::metal::{
- context::MetalContext, error::MetalError, kernel::matmul::common::MatmulArguments,
- metal_extensions::ComputeEncoderSetValue,
- },
-};
-
-fn gemv_kernel_name(
- data_type: DataType,
- config: &PipelineConfiguration,
-) -> Result<String, MetalError> {
- let dtype_name = match data_type {
- DataType::F16 => "float16",
- DataType::BF16 => "bfloat16",
- DataType::F32 => "float32",
- _ => {
- return Err(MetalError::Generic(format!("Unsupported data type for GEMV: {:?}", data_type)));
- },
- };
-
- let prefix = if config.transpose_matrix {
- "gemv_t"
- } else if config.batch_pack > 1 {
- "gemv_bp"
- } else {
- "gemv"
- };
-
- let suffix = format!(
- "{prefix}_{dtype_name}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}_nc{}_axpby{}",
- config.threadgroup_rows,
- config.threadgroup_cols,
- config.threads_per_simdgroup_row,
- config.threads_per_simdgroup_col,
- config.elements_per_thread_row,
- config.elements_per_thread_col,
- config.non_contiguous_batch as u8,
- config.do_axpby as u8,
- );
-
- if config.batch_pack > 1 && !config.transpose_matrix {
- Ok(format!("{suffix}_bp{}", config.batch_pack))
- } else {
- Ok(suffix)
- }
-}
-
-pub struct Kernel {
- data_type: DataType,
- pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
-}
-
-impl Kernel {
- pub fn new(data_type: DataType) -> Result<Self, MetalError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MetalError::Generic(format!("Unsupported data type for GEMV: {:?}", data_type)));
- }
- Ok(Self {
- data_type,
- pipelines: HashMap::new(),
- })
- }
-
- pub fn precompile(
- &mut self,
- context: &MetalContext,
- ) -> Result<(), MetalError> {
- let configs: &[(u32, bool)] = match self.data_type {
- DataType::BF16 => &[(4, false), (4, true), (8, false), (8, true)],
- DataType::F16 => &[(8, false)],
- DataType::F32 => &[(8, false)],
- _ => return Ok(()),
- };
-
- for &(threadgroup_rows, do_axpby) in configs {
- let config = PipelineConfiguration {
- transpose_a: false,
- transpose_b: true,
- transpose_matrix: false,
- batch_pack: 1,
- threadgroup_rows,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- non_contiguous_batch: false,
- do_axpby,
- };
- let _ = self.get_pipeline(context, &config);
- }
-
- Ok(())
- }
-
- fn get_pipeline(
- &mut self,
- context: &MetalContext,
- config: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {
- if !self.pipelines.contains_key(config) {
- let kernel_name = gemv_kernel_name(self.data_type, config)?;
- let pipeline = context.compute_pipeline_state(&kernel_name, None)?;
- self.pipelines.insert(*config, pipeline);
- }
- Ok(self.pipelines.get(config).unwrap())
- }
-
- pub(crate) fn encode_descriptor(
- &mut self,
- context: &MetalContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MatmulArguments,
- descriptor: &DispatchDescriptor,
- ) -> Result<bool, MetalError> {
- let pipeline = self.get_pipeline(context, &descriptor.pipeline_configuration)?;
- encoder.set_compute_pipeline_state(pipeline);
-
- let (buf0, off0) = if descriptor.matrix_is_rhs {
- (arguments.b, 0)
- } else {
- (arguments.a, arguments.a_offset as usize)
- };
- encoder.set_buffer(Some(buf0), off0, 0);
-
- let (buf1, off1) = if descriptor.matrix_is_rhs {
- (arguments.a, arguments.a_offset as usize)
- } else {
- (arguments.b, 0)
- };
- encoder.set_buffer(Some(buf1), off1, 1);
-
- if descriptor.pipeline_configuration.do_axpby {
- match descriptor.axpby_source {
- AxpbySource::None => {
- return Err(MetalError::Generic(
- "GEMV descriptor mismatch: do_axpby=true but axpby_source=None".to_owned(),
- ));
- },
- AxpbySource::Bias => {
- let bias = arguments
- .bias
- .ok_or_else(|| MetalError::Generic("GEMV descriptor requires bias buffer".to_owned()))?;
- encoder.set_buffer(Some(bias), 0, 2);
- },
- AxpbySource::C => {
- let c_buffer = arguments
- .c
- .ok_or_else(|| MetalError::Generic("GEMV descriptor requires C buffer".to_owned()))?;
- encoder.set_buffer(Some(c_buffer), 0, 2);
- },
- }
- }
-
- encoder.set_buffer(Some(arguments.d), 0, 3);
-
- encoder.set_value(&descriptor.input_dimension, 4);
- encoder.set_value(&descriptor.output_dimension, 5);
- encoder.set_value(&descriptor.matrix_leading_dim, 6);
- encoder.set_value(&descriptor.alpha, 7);
- encoder.set_value(&descriptor.beta, 8);
- encoder.set_value(&descriptor.batch_ndim, 9);
- encoder.set_slice(&descriptor.batch_shape, 10);
- encoder.set_slice(&descriptor.vector_batch_stride, 11);
- encoder.set_slice(&descriptor.matrix_batch_stride, 12);
- encoder.set_slice(&descriptor.bias_batch_stride, 13);
- encoder.set_value(&descriptor.bias_stride, 14);
- encoder.set_value(&descriptor.batch_rows, 15);
- encoder.set_value(&descriptor.output_ld, 16);
- encoder.set_value(&descriptor.vector_ld, 17);
-
- encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);
-
- Ok(descriptor.bias_is_fused())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
index 5ee340249..889e3df5e 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -1,6 +1 @@
-mod dispatch_descriptor;
-mod kernel;
-mod pipeline_configuration;
-
-pub(crate) use dispatch_descriptor::DispatchDescriptor;
-pub use kernel::Kernel as GemvKernel;
+// Shader files only — Rust logic has moved to common/kernel/matmul/gemv/
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs
deleted file mode 100644
index 45b40814b..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs
+++ /dev/null
@@ -1,131 +0,0 @@
-use metal::MTLSize;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct PipelineConfiguration {
- pub transpose_a: bool,
- pub transpose_b: bool,
- pub transpose_matrix: bool,
- pub batch_pack: u32,
- pub threadgroup_rows: u32,
- pub threadgroup_cols: u32,
- pub threads_per_simdgroup_row: u32,
- pub threads_per_simdgroup_col: u32,
- pub elements_per_thread_row: u32,
- pub elements_per_thread_col: u32,
- pub non_contiguous_batch: bool,
- pub do_axpby: bool,
-}
-
-impl PipelineConfiguration {
- pub fn output_elements_per_threadgroup(&self) -> u32 {
- if self.transpose_matrix {
- self.threadgroup_cols * self.threads_per_simdgroup_col * self.elements_per_thread_col
- } else {
- self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row
- }
- }
-
- pub fn threads_per_threadgroup(&self) -> MTLSize {
- MTLSize::new(32, self.threadgroup_cols as usize, self.threadgroup_rows as usize)
- }
-}
-
-const FORCE_TILESET_SMALL_BATCH: i32 = 0;
-
-pub fn select_configuration(
- transpose_a: bool,
- transpose_b: bool,
- transpose_matrix: bool,
- batch_pack: u32,
- input_dimension: i32,
- output_dimension: i32,
- non_contiguous_batch: bool,
- do_axpby: bool,
-) -> PipelineConfiguration {
- let (threadgroup_rows, threadgroup_cols);
- let (threads_per_simdgroup_row, threads_per_simdgroup_col);
- let (elements_per_thread_row, elements_per_thread_col);
-
- if FORCE_TILESET_SMALL_BATCH == 1 && !transpose_matrix {
- threadgroup_rows = 4;
- threadgroup_cols = 1;
- threads_per_simdgroup_row = 1;
- threads_per_simdgroup_col = 32;
- elements_per_thread_row = 2;
- elements_per_thread_col = 4;
- } else if transpose_matrix {
- let mut sm = 8;
- let mut sn = 4;
- if input_dimension >= 8192 && output_dimension >= 2048 {
- sm = 4;
- sn = 8;
- }
-
- let bn = if output_dimension >= 2048 {
- 16
- } else if output_dimension >= 512 {
- 4
- } else {
- 2
- };
-
- let tn = if output_dimension < 4 {
- 1
- } else {
- 4
- };
-
- threadgroup_rows = 1;
- threadgroup_cols = bn;
- threads_per_simdgroup_row = sm;
- threads_per_simdgroup_col = sn;
- elements_per_thread_row = 4;
- elements_per_thread_col = tn;
- } else {
- let mut bm = if output_dimension >= 4096 {
- 8
- } else {
- 4
- };
- let mut sm = 1;
- let mut sn = 32;
- let mut bn = 1;
-
- if input_dimension <= 64 {
- bm = 1;
- sm = 8;
- sn = 4;
- } else if input_dimension >= 16 * output_dimension {
- bm = 1;
- bn = 8;
- }
-
- let tm = if output_dimension < 4 {
- 1
- } else {
- 4
- };
-
- threadgroup_rows = bm;
- threadgroup_cols = bn;
- threads_per_simdgroup_row = sm;
- threads_per_simdgroup_col = sn;
- elements_per_thread_row = tm;
- elements_per_thread_col = 4;
- }
-
- PipelineConfiguration {
- transpose_a,
- transpose_b,
- transpose_matrix,
- batch_pack,
- threadgroup_rows,
- threadgroup_cols,
- threads_per_simdgroup_row,
- threads_per_simdgroup_col,
- elements_per_thread_row,
- elements_per_thread_col,
- non_contiguous_batch,
- do_axpby,
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
index c71737961..fef07c3a7 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
@@ -1,1129 +1,328 @@
-#include <metal_simdgroup>
-#include <metal_stdlib>
-
#include "../../../common/utils.h"
+#include "../../../definitions.metal"
-#include "../../common/steel/utils.h"
+#include <metal_simdgroup>
using namespace metal;
-///////////////////////////////////////////////////////////////////////////////
-/// Matrix vector multiplication
-///////////////////////////////////////////////////////////////////////////////
-
-#define MTL_CONST static constant constexpr const
-
-template <typename U>
-struct DefaultAccT {
- using type = float;
-};
-template <>
-struct DefaultAccT<complex64_t> {
- using type = complex64_t;
-};
-
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const int TN, /* Thread cols (in elements) */
- const bool kDoAxpby, /* Do out = alpha * out + beta * bias */
- const int BP, /* Batch rows per threadgroup */
- typename AccT = typename DefaultAccT<T>::type>
-struct GEMVKernel {
- using acc_type = AccT;
-
- MTL_CONST int threadsM = BM * SM;
- MTL_CONST int threadsN = BN * SN;
-
- MTL_CONST int blockM = threadsM * TM;
- MTL_CONST int blockN = threadsN * TN;
+// Upper bound for threadgroup memory (in floats).
+// Max config: tg_simd_cols=16, output_rows_per_tg=16, thread_out_rows=4
+// → 16 * (16 + 4) = 320. Unused when tg_simd_cols == 1.
+#define GEMV_MAX_TG_MEMORY 320
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(MatmulGemv)(
+ const device T* matrix,
+ const device T* input_vector,
+ const device T* output_source OPTIONAL(apply_output_scale_and_accumulate),
+ device T* output_vector,
+ const constant int& input_dimension,
+ const constant int& output_dimension,
+ const constant int& matrix_leading_dimension,
+ const constant float& output_scale,
+ const constant float& output_accumulate_scale,
+ const constant int* batch_shape,
+ const constant int* vector_batch_stride,
+ const constant int* matrix_batch_stride,
+ const constant int* output_source_batch_stride,
+ const constant int& output_source_stride,
+ const constant int& batch_rows,
+ const constant int& output_rows_per_threadgroup,
+ threadgroup float threadgroup_memory[GEMV_MAX_TG_MEMORY],
+ const uint tg_simd_rows SPECIALIZE,
+ const uint tg_simd_cols SPECIALIZE,
+ const uint sg_thread_rows SPECIALIZE,
+ const uint sg_thread_cols SPECIALIZE,
+ const uint thread_out_rows SPECIALIZE,
+ const uint thread_out_cols SPECIALIZE,
+ const bool apply_output_scale_and_accumulate SPECIALIZE,
+ const uint threadgroup_index_x GROUPS((output_dimension + output_rows_per_threadgroup - 1) / output_rows_per_threadgroup),
+ const uint threadgroup_index_y GROUPS(batch_rows),
+ const uint threadgroup_index_z GROUPS(batch_shape[0]),
+ const uint thread_index_x THREADS(32),
+ const uint thread_index_y THREADS(16),
+ const uint thread_index_z THREADS(1),
+ const Simd simd
+) {
+ if (output_rows_per_threadgroup <= 0) {
+ return;
+ }
- static_assert(SM * SN == 32, "simdgroup can only have 32 threads");
+ // Simdgroup guard — kill excess simdgroups beyond what this tile uses
+ if (simd.group_idx >= tg_simd_rows * tg_simd_cols) {
+ return;
+ }
- static_assert(
- SN == 4 || SN == 8 || SN == 16 || SN == 32,
- "gemv block must have a width of 4, 8, 16, or 32"
- );
+ // Derived constants from SPECIALIZE params
+ const int threads_per_threadgroup_col = tg_simd_cols * sg_thread_cols;
+ const int input_columns_per_threadgroup =
+ threads_per_threadgroup_col * thread_out_cols;
- // - The matrix of size (M = out_vec_size, K = in_vec_size) is divided up
- // into blocks of (blockM, blockN) divided among threadgroups
- // - Every thread works on a block of (TM, TN)
- // - We assume each threadgroup has (threadsN, threadsM, 1) threads
- //
- // 1. A thread loads TN elements each from mat along TM rows
- // and the corresponding scalar from the vector
- // 2. The thread then multiplies and adds to accumulate its local result for
- // the block
- // 3. At the end, each thread has accumulated results over all blocks across
- // the rows. These are then summed up across the threadgroup
- // 4. Each threadgroup writes its accumulated blockM outputs
- //
- // Edge case handling:
- // - The threadgroup with the largest tid has blocks that exceed the matrix
- // * The blocks that start outside the matrix are never read (thread results
- // remain zero)
- // * The last thread that partially overlaps with the matrix is shifted
- // inwards such that the thread block fits exactly in the matrix
+ // Batch setup (from run_matmul_gemv_shape)
+ const uint3 threadgroup_position =
+ uint3(threadgroup_index_x, threadgroup_index_y, threadgroup_index_z);
- MTL_CONST short tgp_mem_size = BN > 1 ? BN*(blockM + TM) * BP : 0;
- MTL_CONST bool needs_tgp_reduction = BN > 1;
+ const int batch_row = static_cast<int>(threadgroup_position.y);
+ if (batch_row >= batch_rows) {
+ return;
+ }
- template <typename U = T>
- static METAL_FUNC void load_unsafe(
- const device T* src,
- thread U dst[TN],
- const int src_offset = 0
- ) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = static_cast<U>(src[src_offset + tn]);
- }
+ // Advance pointers by batch
+ const device T* batch_input_vector =
+ input_vector + threadgroup_position.z * vector_batch_stride[0];
+ const device T* batch_matrix =
+ matrix + threadgroup_position.z * matrix_batch_stride[0];
+ const device T* batch_output_source = output_source;
+ if (apply_output_scale_and_accumulate) {
+ batch_output_source +=
+ threadgroup_position.z * output_source_batch_stride[0];
}
+ device T* batch_output_vector =
+ output_vector + threadgroup_position.z * batch_rows * output_dimension;
+
+ // Thread local accumulation results
+ thread float accumulated_values[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+ thread T matrix_values[4];
+ thread float vector_coefficients[4];
+
+ const int thread_row_in_simdgroup =
+ sg_thread_cols != 32 ? int(simd.lane_idx) / int(sg_thread_cols) : 0;
+ const int thread_col_in_simdgroup =
+ sg_thread_cols != 32 ? int(simd.lane_idx) % int(sg_thread_cols)
+ : int(simd.lane_idx);
+
+ const int simdgroup_column_index =
+ tg_simd_cols != 1 ? int(simd.group_idx % tg_simd_cols) : 0;
+
+ const int simdgroup_row_thread_base =
+ tg_simd_cols != 1
+ ? int(sg_thread_rows) * int(simd.group_idx / tg_simd_cols)
+ : int(sg_thread_rows) * int(simd.group_idx);
+ const int simdgroup_col_thread_base =
+ tg_simd_cols != 1
+ ? int(sg_thread_cols) * int(simd.group_idx % tg_simd_cols)
+ : 0;
+
+ int output_block_row_offset =
+ (simdgroup_row_thread_base + thread_row_in_simdgroup) *
+ int(thread_out_rows);
+ int input_block_col_offset =
+ (simdgroup_col_thread_base + thread_col_in_simdgroup) *
+ int(thread_out_cols);
+
+ // Block position
+ int output_row_start =
+ int(threadgroup_position.x) * int(output_rows_per_threadgroup) +
+ output_block_row_offset;
+
+ // Exit simdgroup if rows out of bound
+ if (output_row_start >= output_dimension)
+ return;
- template <typename U = T>
- static METAL_FUNC void load_safe(
- const device T* src,
- thread U dst[TN],
- const int src_offset = 0,
- const int src_size = TN
- ) {
- if (src_offset + TN <= src_size) {
+ // Adjust tail simdgroup to ensure in bound reads
+ output_row_start = output_row_start + int(thread_out_rows) <= output_dimension
+ ? output_row_start
+ : output_dimension - int(thread_out_rows);
+
+ // Advance matrix
+ const device T* thread_matrix =
+ batch_matrix + output_row_start * matrix_leading_dimension;
+
+ const uniform<int> input_block_stride =
+ make_uniform(int(input_columns_per_threadgroup));
+ const uniform<int> input_vector_length = make_uniform(input_dimension);
+ const uniform<int> full_input_blocks =
+ input_vector_length / input_block_stride;
+ const uniform<int> remaining_input_columns =
+ input_vector_length - input_block_stride * full_input_blocks;
+
+ // Loop over input_vector in blocks of input_columns_per_threadgroup
+ for (int input_block_index = 0; input_block_index < full_input_blocks;
+ ++input_block_index) {
+ // Load vector coefficients (unchecked)
+ {
+ const device T* input_vector_row =
+ batch_input_vector + batch_row * input_dimension;
MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = static_cast<U>(src[src_offset + tn]);
- }
- } else { // Edgecase
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = src_offset + tn < src_size
- ? static_cast<U>(src[src_offset + tn])
- : U(0);
+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
+ input_col_offset++) {
+ vector_coefficients[input_col_offset] = static_cast<float>(
+ input_vector_row[input_block_col_offset + input_col_offset]
+ );
}
}
- }
-
- static METAL_FUNC void run(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- const device T* bias [[buffer(2)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& out_vec_size [[buffer(5)]],
- const constant int& matrix_ld [[buffer(6)]],
- const constant float& alpha [[buffer(7)]],
- const constant float& beta [[buffer(8)]],
- const constant int& bias_stride [[buffer(14)]],
- const int batch_row_base,
- threadgroup AccT* tgp_memory [[threadgroup(0)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
- ) {
- // Appease compiler
- (void)lid;
-
- // Thread local accumulation results
- thread AccT result[BP][TM] = {{0}};
- thread T inter[TN];
- thread AccT v_coeff[BP][TN];
-
- const int thrM = SN != 32 ? simd_lid / SN : 0;
- const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);
-
- const int sgN = BN != 1 ? (simd_gid % BN) : 0;
- const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);
- const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
-
- int bm = (simdM + thrM) * TM;
- int bn = (simdN + thrN) * TN;
-
- // Block position
- int out_row = tid.x * blockM + bm;
-
- // Exit simdgroup if rows out of bound
- if (out_row >= out_vec_size)
- return;
-
- // Adjust tail simdgroup to ensure in bound reads
- out_row = out_row + TM <= out_vec_size ? out_row : out_vec_size - TM;
-
- // Advance matrix
- mat += out_row * matrix_ld;
-
- constexpr const uniform<int> loop_stride = make_uniform(blockN);
- const uniform<int> in_size = make_uniform(in_vec_size);
- const uniform<int> n_iter = in_size / loop_stride;
- const uniform<int> last_iter = loop_stride * n_iter;
- const uniform<int> leftover = in_size - last_iter;
-
- // Loop over in_vec in blocks of blockN
- for (int i = 0; i < n_iter; ++i) {
+ // Per thread work loop
+ int matrix_row_offset = 0;
+ MTL_PRAGMA_UNROLL
+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
+ output_row_offset++) {
+ // Load matrix row (unchecked)
MTL_PRAGMA_UNROLL
- for (int bp = 0; bp < BP; bp++) {
- const int batch_row = batch_row_base + bp;
- const device T* vec_ptr = in_vec + batch_row * in_vec_size;
- load_unsafe<AccT>(vec_ptr, v_coeff[bp], bn);
+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
+ input_col_offset++) {
+ matrix_values[input_col_offset] = thread_matrix
+ [matrix_row_offset + input_block_col_offset + input_col_offset];
}
- // Per thread work loop
- int mat_offset = 0;
+ // Accumulate results
MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- // Load for the row
- load_unsafe(mat, inter, mat_offset + bn);
-
- // Accumulate results
- MTL_PRAGMA_UNROLL
- for (int bp = 0; bp < BP; bp++) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- result[bp][tm] += inter[tn] * v_coeff[bp][tn];
- }
- }
-
- mat_offset += matrix_ld;
+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
+ input_col_offset++) {
+ accumulated_values[output_row_offset] +=
+ static_cast<float>(matrix_values[input_col_offset]) *
+ vector_coefficients[input_col_offset];
}
- bn += blockN;
+ matrix_row_offset += matrix_leading_dimension;
}
- if (leftover > 0) {
- MTL_PRAGMA_UNROLL
- for (int bp = 0; bp < BP; bp++) {
- const int batch_row = batch_row_base + bp;
- const device T* vec_ptr = in_vec + batch_row * in_vec_size;
- load_safe<AccT>(vec_ptr, v_coeff[bp], bn, in_size);
- }
-
- // Per thread work loop
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- // Load for the row
- load_safe(&mat[tm * matrix_ld], inter, bn, in_size);
+ input_block_col_offset += input_columns_per_threadgroup;
+ }
- // Accumulate results
+ if (remaining_input_columns > 0) {
+ // Load vector coefficients (checked)
+ {
+ const device T* input_vector_row =
+ batch_input_vector + batch_row * input_dimension;
+ if (input_block_col_offset + int(thread_out_cols) <=
+ input_vector_length) {
MTL_PRAGMA_UNROLL
- for (int bp = 0; bp < BP; bp++) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- result[bp][tm] += inter[tn] * v_coeff[bp][tn];
- }
+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
+ input_col_offset++) {
+ vector_coefficients[input_col_offset] = static_cast<float>(
+ input_vector_row[input_block_col_offset + input_col_offset]
+ );
}
- }
- }
-
- // Simdgroup accumulations
- MTL_PRAGMA_UNROLL
- for (int bp = 0; bp < BP; bp++) {
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
+ } else {
MTL_PRAGMA_UNROLL
- for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {
- result[bp][tm] += simd_shuffle_down(result[bp][tm], sn);
+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
+ input_col_offset++) {
+ vector_coefficients[input_col_offset] =
+ input_block_col_offset + int(input_col_offset) <
+ input_vector_length
+ ? static_cast<float>(
+ input_vector_row
+ [input_block_col_offset + input_col_offset]
+ )
+ : 0.0f;
}
}
}
- // Threadgroup accumulation results
- if (needs_tgp_reduction) {
- if (thrN == 0) {
+ // Per thread work loop
+ MTL_PRAGMA_UNROLL
+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
+ output_row_offset++) {
+ // Load matrix row (checked)
+ if (input_block_col_offset + int(thread_out_cols) <=
+ input_vector_length) {
MTL_PRAGMA_UNROLL
- for (int bp = 0; bp < BP; bp++) {
- threadgroup AccT* tgp_results =
- tgp_memory + bp * (BN * (blockM + TM)) + sgN * (blockM + TM) + bm;
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- tgp_results[tm] = result[bp][tm];
- }
+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
+ input_col_offset++) {
+ matrix_values[input_col_offset] = thread_matrix
+ [output_row_offset * matrix_leading_dimension +
+ input_block_col_offset + input_col_offset];
}
-
- threadgroup_barrier(mem_flags::mem_none);
-
- if (sgN == 0) {
- MTL_PRAGMA_UNROLL
- for (int bp = 0; bp < BP; bp++) {
- threadgroup AccT* tgp_results =
- tgp_memory + bp * (BN * (blockM + TM)) + bm;
- MTL_PRAGMA_UNROLL
- for (int sgn = 1; sgn < BN; sgn++) {
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- result[bp][tm] += tgp_results[sgn * (blockM + TM) + tm];
- }
- }
- }
+ } else {
+ MTL_PRAGMA_UNROLL
+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
+ input_col_offset++) {
+ matrix_values[input_col_offset] =
+ input_block_col_offset + int(input_col_offset) <
+ input_vector_length
+ ? thread_matrix
+ [output_row_offset * matrix_leading_dimension +
+ input_block_col_offset + input_col_offset]
+ : T(0);
}
}
- }
- // Write outputs
- if (simdN == 0 && thrN == 0) {
+ // Accumulate results
MTL_PRAGMA_UNROLL
- for (int bp = 0; bp < BP; bp++) {
- const int batch_row = batch_row_base + bp;
- device T* out_row_ptr = out_vec + batch_row * out_vec_size;
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- if (kDoAxpby) {
- out_row_ptr[out_row + tm] =
- static_cast<T>(alpha) * static_cast<T>(result[bp][tm]) +
- static_cast<T>(beta) * bias[(out_row + tm) * bias_stride];
- } else {
- out_row_ptr[out_row + tm] = static_cast<T>(result[bp][tm]);
- }
- }
+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
+ input_col_offset++) {
+ accumulated_values[output_row_offset] +=
+ static_cast<float>(matrix_values[input_col_offset]) *
+ vector_coefficients[input_col_offset];
}
}
}
-};
-
-///////////////////////////////////////////////////////////////////////////////
-/// Vector matrix multiplication
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const int TN, /* Thread cols (in elements) */
- const bool kDoAxpby, /* Do out = alpha * out + beta * bias */
- typename AccT = typename DefaultAccT<T>::type>
-struct GEMVTKernel {
- using acc_type = AccT;
-
- MTL_CONST int threadsM = BM * SM;
- MTL_CONST int threadsN = BN * SN;
-
- MTL_CONST int blockM = threadsM * TM;
- MTL_CONST int blockN = threadsN * TN;
-
- static_assert(SM * SN == 32, "simdgroup can only have 32 threads");
-
- // - The matrix of size (M = in_vec_size, N = out_vec_size) is divided up
- // into blocks of (blockM, blockN) divided among threadgroups
- // - Every thread works on a block of (TM, TN)
- // - We assume each threadgroup has (threadsN, threadsM, 1) threads
- //
- // 1. A thread loads TN elements each from mat along TM contiguous rows
- // and the corresponding scalar from the vector
- // 2. The thread then accumulates its local result for the block
- // 3. At the end, each thread has accumulated results over all blocks across
- // the rows. These are then summed up across the threadgroup
- // 4. Each threadgroup writes its accumulated BN * TN outputs
- //
- // Edge case handling:
- // - The threadgroup with the largest tid has blocks that exceed the matrix
- // * The blocks that start outside the matrix are never read (thread results
- // remain zero)
- // * The last thread that partially overlaps with the matrix is shifted
- // inwards such that the thread block fits exactly in the matrix
-
- MTL_CONST short tgp_mem_size = BM > 1 ? BM*(blockN + TN) : 0;
- MTL_CONST bool needs_tgp_reduction = BM > 1;
-
- static METAL_FUNC void run(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- const device T* bias [[buffer(2)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& out_vec_size [[buffer(5)]],
- const constant int& marix_ld [[buffer(6)]],
- const constant float& alpha [[buffer(7)]],
- const constant float& beta [[buffer(8)]],
- const constant int& bias_stride [[buffer(14)]],
- threadgroup AccT* tgp_memory [[threadgroup(0)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
- ) {
- // Appease compiler
- (void)lid;
-
- // Thread local accumulation results
- AccT result[TN] = {0};
- T inter[TN];
- AccT v_coeff[TM];
- const int thrM = SN != 32 ? simd_lid / SN : 0;
- const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);
-
- const int sgM = BN != 1 ? (simd_gid / BN) : int(simd_gid);
- const int sgN = BN != 1 ? (simd_gid % BN) : 0;
-
- const int simdM = SM * sgM;
- const int simdN = SN * sgN;
-
- int cm = (simdM + thrM);
- int cn = (simdN + thrN);
-
- int bm = cm * TM;
- int bn = cn * TN;
- int out_col = tid.x * blockN + bn;
-
- constexpr const uniform<int> loop_stride = make_uniform(blockM);
- const uniform<int> in_size = make_uniform(in_vec_size);
- const uniform<int> n_iter = in_size / loop_stride;
- const uniform<int> last_iter = loop_stride * n_iter;
- const uniform<int> leftover = in_size - last_iter;
-
- // Edgecase handling
- if (out_col < out_vec_size) {
- out_col = out_col + TN < out_vec_size ? out_col : out_vec_size - TN;
-
- // Per thread accumulation main loop
- for (int i = 0; i < n_iter; ++i) {
- // Adding a threadgroup_barrier improves performance slightly
- // This is possibly it may help exploit cache better
- threadgroup_barrier(mem_flags::mem_none);
-
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- v_coeff[tm] = static_cast<AccT>(in_vec[bm + tm]);
- }
-
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- auto vc = static_cast<AccT>(v_coeff[tm]);
- for (int tn = 0; tn < TN; tn++) {
- inter[tn] = mat[(bm + tm) * marix_ld + out_col + tn];
- }
- for (int tn = 0; tn < TN; tn++) {
- result[tn] += vc * inter[tn];
- }
- }
-
- bm += blockM;
- }
-
- if (leftover > 0) {
- for (int tm = 0; tm < TM && bm + tm < in_vec_size; tm++) {
- v_coeff[tm] = static_cast<AccT>(in_vec[bm + tm]);
-
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- inter[tn] = mat[(bm + tm) * marix_ld + out_col + tn];
- }
-
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- result[tn] += v_coeff[tm] * inter[tn];
- }
- }
- }
+ // Simdgroup accumulations
+ MTL_PRAGMA_UNROLL
+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
+ output_row_offset++) {
+ MTL_PRAGMA_UNROLL
+ for (ushort simd_shuffle_offset = (sg_thread_cols / 2);
+ simd_shuffle_offset >= 1;
+ simd_shuffle_offset >>= 1) {
+ accumulated_values[output_row_offset] += simd_shuffle_down(
+ accumulated_values[output_row_offset],
+ simd_shuffle_offset
+ );
}
+ }
- // Simdgroup accumulations
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
+ // Threadgroup reduction (only when tg_simd_cols > 1)
+ if (tg_simd_cols > 1) {
+ const int computed_output_rows_per_tg =
+ int(tg_simd_rows) * int(sg_thread_rows) * int(thread_out_rows);
+
+ if (thread_col_in_simdgroup == 0) {
+ threadgroup float* threadgroup_partial_accumulations =
+ threadgroup_memory +
+ simdgroup_column_index *
+ (computed_output_rows_per_tg + int(thread_out_rows)) +
+ output_block_row_offset;
MTL_PRAGMA_UNROLL
- for (ushort sm = (SM / 2); sm >= 1; sm >>= 1) {
- result[tn] += simd_shuffle_down(result[tn], SN * sm);
+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
+ output_row_offset++) {
+ threadgroup_partial_accumulations[output_row_offset] =
+ accumulated_values[output_row_offset];
}
- }
-
- // Threadgroup accumulation results
- if (needs_tgp_reduction) {
- threadgroup AccT* tgp_results = tgp_memory + sgM * (blockN + TN) + bn;
- if (thrM == 0) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- tgp_results[tn] = result[tn];
- }
- threadgroup_barrier(mem_flags::mem_none);
+ threadgroup_barrier(mem_flags::mem_none);
- if (sgM == 0) {
+ if (simdgroup_column_index == 0) {
+ threadgroup float* base_partial =
+ threadgroup_memory + output_block_row_offset;
+ for (uint reduction_simdgroup_col = 1;
+ reduction_simdgroup_col < tg_simd_cols;
+ reduction_simdgroup_col++) {
MTL_PRAGMA_UNROLL
- for (int sgm = 1; sgm < BM; sgm++) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- result[tn] += tgp_results[sgm * (blockN + TN) + tn];
- }
+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
+ output_row_offset++) {
+ accumulated_values[output_row_offset] += base_partial
+ [reduction_simdgroup_col *
+ (computed_output_rows_per_tg + int(thread_out_rows)) +
+ output_row_offset];
}
}
}
}
-
- // Threadgroup accumulation and writing out results
- if (cm == 0 && out_col < out_vec_size) {
- MTL_PRAGMA_UNROLL
- for (int j = 0; j < TN; j++) {
- if (kDoAxpby) {
- out_vec[out_col + j] =
- static_cast<T>(alpha) * static_cast<T>(result[j]) +
- static_cast<T>(beta) * bias[(out_col + j) * bias_stride];
- } else {
- out_vec[out_col + j] = static_cast<T>(result[j]);
- }
- }
- }
- }
-};
-
-///////////////////////////////////////////////////////////////////////////////
-/// Matrix vector multiplication
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const int TN, /* Thread cols (in elements) */
- const bool kDoNCBatch, /* Batch ndim > 1 */
- const bool kDoAxpby> /* Do out = alpha * out + beta * bias */
-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- const device T* bias [[buffer(2)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& out_vec_size [[buffer(5)]],
- const constant int& marix_ld [[buffer(6)]],
- const constant float& alpha [[buffer(7)]],
- const constant float& beta [[buffer(8)]],
- const constant int& batch_ndim [[buffer(9)]],
- const constant int* batch_shape [[buffer(10)]],
- const constant int64_t* vector_batch_stride [[buffer(11)]],
- const constant int64_t* matrix_batch_stride [[buffer(12)]],
- const constant int64_t* bias_batch_stride [[buffer(13)]],
- const constant int& bias_stride [[buffer(14)]],
- const constant int& batch_rows [[buffer(15)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- using gemv_kernel = GEMVKernel<T, BM, BN, SM, SN, TM, TN, kDoAxpby, 1>;
- threadgroup typename gemv_kernel::acc_type tgp_memory
- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
-
- // Batched GEMV: tid.y is the batch row index
- const int batch_row = tid.y;
- if (batch_row >= batch_rows) {
- return;
- }
-
- // Update batch offsets
- if (kDoNCBatch) {
- in_vec += elem_to_loc(tid.z, batch_shape, vector_batch_stride, batch_ndim);
- mat += elem_to_loc(tid.z, batch_shape, matrix_batch_stride, batch_ndim);
-
- if (kDoAxpby) {
- bias += elem_to_loc(tid.z, batch_shape, bias_batch_stride, batch_ndim);
- }
-
- } else {
- in_vec += tid.z * vector_batch_stride[0];
- mat += tid.z * matrix_batch_stride[0];
-
- if (kDoAxpby) {
- bias += tid.z * bias_batch_stride[0];
- }
}
- const int batch_row_base = batch_row;
- out_vec += tid.z * batch_rows * out_vec_size;
-
- gemv_kernel::run(
- mat,
- in_vec,
- bias,
- out_vec,
- in_vec_size,
- out_vec_size,
- marix_ld,
- alpha,
- beta,
- bias_stride,
- batch_row_base,
- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
-}
-
-#define instantiate_gemv_bp_helper( \
- name, \
- itype, \
- bm, \
- bn, \
- sm, \
- sn, \
- tm, \
- nc, \
- axpby, \
- bp \
-) \
- instantiate_kernel( \
- "gemv_bp_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \
- "_tn4_nc" #nc "_axpby" #axpby "_bp" #bp, \
- gemv_bp, \
- itype, \
- bm, \
- bn, \
- sm, \
- sn, \
- tm, \
- nc, \
- axpby, \
- bp \
- )
-
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const bool kDoNCBatch, /* Batch ndim > 1 */
- const bool kDoAxpby, /* Do out = alpha * out + beta * bias */
- const int BP> /* Batch rows per threadgroup */
-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_bp(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- const device T* bias [[buffer(2)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& out_vec_size [[buffer(5)]],
- const constant int& marix_ld [[buffer(6)]],
- const constant float& alpha [[buffer(7)]],
- const constant float& beta [[buffer(8)]],
- const constant int& batch_ndim [[buffer(9)]],
- const constant int* batch_shape [[buffer(10)]],
- const constant int64_t* vector_batch_stride [[buffer(11)]],
- const constant int64_t* matrix_batch_stride [[buffer(12)]],
- const constant int64_t* bias_batch_stride [[buffer(13)]],
- const constant int& bias_stride [[buffer(14)]],
- const constant int& batch_rows [[buffer(15)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- using gemv_kernel = GEMVKernel<T, BM, BN, SM, SN, TM, 4, kDoAxpby, BP>;
- threadgroup typename gemv_kernel::acc_type tgp_memory
- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
-
- const int batch_row_base = int(tid.y) * BP;
-
- if (kDoNCBatch) {
- in_vec += elem_to_loc(tid.z, batch_shape, vector_batch_stride, batch_ndim);
- mat += elem_to_loc(tid.z, batch_shape, matrix_batch_stride, batch_ndim);
-
- if (kDoAxpby) {
- bias += elem_to_loc(tid.z, batch_shape, bias_batch_stride, batch_ndim);
- }
-
- } else {
- in_vec += tid.z * vector_batch_stride[0];
- mat += tid.z * matrix_batch_stride[0];
-
- if (kDoAxpby) {
- bias += tid.z * bias_batch_stride[0];
- }
- }
-
- out_vec += tid.z * batch_rows * out_vec_size;
-
- gemv_kernel::run(
- mat,
- in_vec,
- bias,
- out_vec,
- in_vec_size,
- out_vec_size,
- marix_ld,
- alpha,
- beta,
- bias_stride,
- batch_row_base,
- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
-}
-
-#define instantiate_gemv_helper( \
- name, \
- itype, \
- bm, \
- bn, \
- sm, \
- sn, \
- tm, \
- tn, \
- nc, \
- axpby \
-) \
- instantiate_kernel( \
- "gemv_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \
- "_tn" #tn "_nc" #nc "_axpby" #axpby, \
- gemv, \
- itype, \
- bm, \
- bn, \
- sm, \
- sn, \
- tm, \
- tn, \
- nc, \
- axpby \
- )
-
-// clang-format off
-#define instantiate_gemv(name, itype, bm, bn, sm, sn, tm, tn) \
- instantiate_gemv_helper(name, itype, bm, bn, sm, sn, tm, tn, 0, 0) \
- instantiate_gemv_helper(name, itype, bm, bn, sm, sn, tm, tn, 0, 1) \
- instantiate_gemv_helper(name, itype, bm, bn, sm, sn, tm, tn, 1, 0) \
- instantiate_gemv_helper(name, itype, bm, bn, sm, sn, tm, tn, 1, 1) // clang-format on
-
-#define instantiate_gemv_bp(name, itype, bm, bn, sm, sn, tm, bp) \
- instantiate_gemv_bp_helper( \
- name, \
- itype, \
- bm, \
- bn, \
- sm, \
- sn, \
- tm, \
- 0, \
- 0, \
- bp \
- ) instantiate_gemv_bp_helper(name, itype, bm, bn, sm, sn, tm, 0, 1, bp) \
- instantiate_gemv_bp_helper(name, itype, bm, bn, sm, sn, tm, 1, 0, bp) \
- instantiate_gemv_bp_helper( \
- name, \
- itype, \
- bm, \
- bn, \
- sm, \
- sn, \
- tm, \
- 1, \
- 1, \
- bp \
- ) // clang-format on
-
-// clang-format off
-#define instantiate_gemv_blocks(name, itype) \
- instantiate_gemv(name, itype, 1, 8, 1, 32, 4, 4) \
- instantiate_gemv(name, itype, 1, 8, 1, 32, 1, 4) \
- instantiate_gemv(name, itype, 1, 1, 8, 4, 4, 4) \
- instantiate_gemv(name, itype, 1, 1, 8, 4, 1, 4) \
- instantiate_gemv(name, itype, 4, 1, 1, 32, 1, 4) \
- instantiate_gemv(name, itype, 4, 1, 1, 32, 4, 4) \
- instantiate_gemv(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on
-
-#define instantiate_gemv_bp_blocks(name, itype, bp) \
- instantiate_gemv_bp(name, itype, 1, 8, 1, 32, 4, bp) \
- instantiate_gemv_bp(name, itype, 1, 8, 1, 32, 1, bp) \
- instantiate_gemv_bp(name, itype, 1, 1, 8, 4, 4, bp) \
- instantiate_gemv_bp(name, itype, 1, 1, 8, 4, 1, bp) \
- instantiate_gemv_bp(name, itype, 4, 1, 1, 32, 1, bp) \
- instantiate_gemv_bp(name, itype, 4, 1, 1, 32, 4, bp) \
- instantiate_gemv_bp( \
- name, \
- itype, \
- 8, \
- 1, \
- 1, \
- 32, \
- 4, \
- bp \
- ) // clang-format on
-
-instantiate_gemv_blocks(float32, float);
-instantiate_gemv_blocks(float16, half);
-instantiate_gemv_blocks(bfloat16, bfloat16_t);
-instantiate_gemv_blocks(complex64, complex64_t);
-instantiate_gemv_bp_blocks(float32, float, 2);
-instantiate_gemv_bp_blocks(float16, half, 2);
-instantiate_gemv_bp_blocks(bfloat16, bfloat16_t, 2);
-instantiate_gemv_bp_blocks(float32, float, 4);
-instantiate_gemv_bp_blocks(float16, half, 4);
-instantiate_gemv_bp_blocks(bfloat16, bfloat16_t, 4);
-
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const int TN> /* Thread cols (in elements) */
-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_gather(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- const device T* bias [[buffer(2)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& out_vec_size [[buffer(5)]],
- const constant int& marix_ld [[buffer(6)]],
- const constant float& alpha [[buffer(7)]],
- const constant float& beta [[buffer(8)]],
- const constant int& batch_ndim [[buffer(9)]],
- const constant int* batch_shape [[buffer(10)]],
- const constant int64_t* index_batch_strides [[buffer(11)]],
- const constant int& vector_batch_ndim [[buffer(12)]],
- const constant int* vector_batch_shape [[buffer(13)]],
- const constant int64_t* vector_batch_stride [[buffer(14)]],
- const constant int& matrix_batch_ndim [[buffer(15)]],
- const constant int* matrix_batch_shape [[buffer(16)]],
- const constant int64_t* matrix_batch_stride [[buffer(17)]],
- const constant uint32_t* vec_indices [[buffer(18)]],
- const constant uint32_t* mat_indices [[buffer(19)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- using gemv_kernel = GEMVKernel<T, BM, BN, SM, SN, TM, TN, false, 1>;
- threadgroup typename gemv_kernel::acc_type tgp_memory
- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
-
- uint32_t indx_vec;
- uint32_t indx_mat;
-
- // Update batch offsets
- if (batch_ndim > 1) {
- const constant auto* veci_bstrides = index_batch_strides;
- const constant auto* mati_bstrides = index_batch_strides + batch_ndim;
-
- ulong2 batch_offsets = elem_to_loc_broadcast(
- tid.z,
- batch_shape,
- veci_bstrides,
- mati_bstrides,
- batch_ndim
- );
-
- indx_vec = vec_indices[batch_offsets.x];
- indx_mat = mat_indices[batch_offsets.y];
-
- } else {
- indx_vec = vec_indices[index_batch_strides[0] * tid.z];
- indx_mat = mat_indices[index_batch_strides[batch_ndim] * tid.z];
- }
-
- if (vector_batch_ndim > 1) {
- in_vec += elem_to_loc(
- indx_vec,
- vector_batch_shape,
- vector_batch_stride,
- vector_batch_ndim
- );
- } else {
- in_vec += indx_vec * vector_batch_stride[0];
- }
-
- if (matrix_batch_ndim > 1) {
- mat += elem_to_loc(
- indx_mat,
- matrix_batch_shape,
- matrix_batch_stride,
- matrix_batch_ndim
- );
- } else {
- mat += indx_mat * matrix_batch_stride[0];
- }
-
- out_vec += tid.z * out_vec_size;
-
- gemv_kernel::run(
- mat,
- in_vec,
- bias,
- out_vec,
- in_vec_size,
- out_vec_size,
- marix_ld,
- alpha,
- beta,
- out_vec_size,
- 0,
- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
-}
-
-// clang-format off
-#define instantiate_gemv_bs_helper(nm, itype, bm, bn, sm, sn, tm, tn) \
- instantiate_kernel( \
- "gemv_gather_" #nm "_bm" #bm "_bn" #bn "_sm" #sm \
- "_sn" #sn "_tm" #tm "_tn" #tn, \
- gemv_gather, itype, bm, bn, sm, sn, tm, tn)
-
-#define instantiate_gemv_bs_blocks(name, itype) \
- instantiate_gemv_bs_helper(name, itype, 4, 1, 1, 32, 1, 4) \
- instantiate_gemv_bs_helper(name, itype, 4, 1, 1, 32, 4, 4) \
- instantiate_gemv_bs_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on
-
-instantiate_gemv_bs_blocks(float32, float);
-instantiate_gemv_bs_blocks(float16, half);
-instantiate_gemv_bs_blocks(bfloat16, bfloat16_t);
-instantiate_gemv_bs_blocks(complex64, complex64_t);
-
-///////////////////////////////////////////////////////////////////////////////
-/// Vector matrix multiplication
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const int TN, /* Thread cols (in elements) */
- const bool kDoNCBatch, /* Batch ndim > 1 */
- const bool kDoAxpby> /* Do out = alpha * out + beta * bias */
-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_t(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- const device T* bias [[buffer(2)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& out_vec_size [[buffer(5)]],
- const constant int& marix_ld [[buffer(6)]],
- const constant float& alpha [[buffer(7)]],
- const constant float& beta [[buffer(8)]],
- const constant int& batch_ndim [[buffer(9)]],
- const constant int* batch_shape [[buffer(10)]],
- const constant int64_t* vector_batch_stride [[buffer(11)]],
- const constant int64_t* matrix_batch_stride [[buffer(12)]],
- const constant int64_t* bias_batch_stride [[buffer(13)]],
- const constant int& bias_stride [[buffer(14)]],
- const constant int& batch_rows [[buffer(15)]],
- const constant int& output_ld [[buffer(16)]],
- const constant int& vector_ld [[buffer(17)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- using gemv_kernel = GEMVTKernel<T, BM, BN, SM, SN, TM, TN, kDoAxpby>;
- threadgroup typename gemv_kernel::acc_type tgp_memory
- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
-
- // Batched GEMV: tid.y is the batch row index
- const int batch_row = tid.y;
- if (batch_row >= batch_rows) {
- return;
- }
-
- // Update batch offsets
- if (kDoNCBatch) {
- in_vec += elem_to_loc(tid.z, batch_shape, vector_batch_stride, batch_ndim);
- mat += elem_to_loc(tid.z, batch_shape, matrix_batch_stride, batch_ndim);
-
- if (kDoAxpby) {
- bias += elem_to_loc(tid.z, batch_shape, bias_batch_stride, batch_ndim);
- }
-
- } else {
- in_vec += tid.z * vector_batch_stride[0];
- mat += tid.z * matrix_batch_stride[0];
-
- if (kDoAxpby) {
- bias += tid.z * bias_batch_stride[0];
+ // Write outputs
+ if (simdgroup_col_thread_base == 0 && thread_col_in_simdgroup == 0) {
+ device T* output_row_values =
+ batch_output_vector + batch_row * output_dimension;
+ MTL_PRAGMA_UNROLL
+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
+ output_row_offset++) {
+ if (apply_output_scale_and_accumulate) {
+ output_row_values[output_row_start + output_row_offset] =
+ static_cast<T>(output_scale) *
+ static_cast<T>(accumulated_values[output_row_offset]) +
+ static_cast<T>(output_accumulate_scale) *
+ batch_output_source
+ [(output_row_start + output_row_offset) *
+ output_source_stride];
+ } else {
+ output_row_values[output_row_start + output_row_offset] =
+ static_cast<T>(accumulated_values[output_row_offset]);
+ }
}
}
-
- // Offset by batch row (tid.y)
- in_vec += batch_row * vector_ld;
- out_vec += tid.z * batch_rows * output_ld + batch_row * output_ld;
-
- gemv_kernel::run(
- mat,
- in_vec,
- bias,
- out_vec,
- in_vec_size,
- out_vec_size,
- marix_ld,
- alpha,
- beta,
- bias_stride,
- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
}
-
-// clang-format off
-#define instantiate_gemv_t_helper( \
- name, itype, bm, bn, sm, sn, tm, tn, nc, axpby) \
- instantiate_kernel( \
- "gemv_t_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn \
- "_tm" #tm "_tn" #tn "_nc" #nc "_axpby" #axpby, \
- gemv_t, itype, bm, bn, sm, sn, tm, tn, nc, axpby)
-
-#define instantiate_gemv_t(name, itype, bm, bn, sm, sn, tm, tn) \
- instantiate_gemv_t_helper(name, itype, bm, bn, sm, sn, tm, tn, 0, 0) \
- instantiate_gemv_t_helper(name, itype, bm, bn, sm, sn, tm, tn, 0, 1) \
- instantiate_gemv_t_helper(name, itype, bm, bn, sm, sn, tm, tn, 1, 0) \
- instantiate_gemv_t_helper(name, itype, bm, bn, sm, sn, tm, tn, 1, 1) // clang-format on
-
-// clang-format off
-#define instantiate_gemv_t_blocks(name, itype) \
- instantiate_gemv_t(name, itype, 1, 2, 8, 4, 4, 1) \
- instantiate_gemv_t(name, itype, 1, 2, 8, 4, 4, 4) \
- instantiate_gemv_t(name, itype, 1, 4, 8, 4, 4, 4) \
- instantiate_gemv_t(name, itype, 1, 16, 8, 4, 4, 4) \
- instantiate_gemv_t(name, itype, 1, 16, 4, 8, 4, 4) // clang-format on
-
-// clang-format off
-instantiate_gemv_t_blocks(float32, float);
-instantiate_gemv_t_blocks(float16, half);
-instantiate_gemv_t_blocks(bfloat16, bfloat16_t);
-instantiate_gemv_t_blocks(complex64, complex64_t); // clang-format on
-
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const int TN> /* Thread cols (in elements) */
-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_t_gather(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- const device T* bias [[buffer(2)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& out_vec_size [[buffer(5)]],
- const constant int& marix_ld [[buffer(6)]],
- const constant float& alpha [[buffer(7)]],
- const constant float& beta [[buffer(8)]],
- const constant int& batch_ndim [[buffer(9)]],
- const constant int* batch_shape [[buffer(10)]],
- const constant int64_t* index_batch_strides [[buffer(11)]],
- const constant int& vector_batch_ndim [[buffer(12)]],
- const constant int* vector_batch_shape [[buffer(13)]],
- const constant int64_t* vector_batch_stride [[buffer(14)]],
- const constant int& matrix_batch_ndim [[buffer(15)]],
- const constant int* matrix_batch_shape [[buffer(16)]],
- const constant int64_t* matrix_batch_stride [[buffer(17)]],
- const constant uint32_t* vec_indices [[buffer(18)]],
- const constant uint32_t* mat_indices [[buffer(19)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- using gemv_kernel = GEMVTKernel<T, BM, BN, SM, SN, TM, TN, false>;
- threadgroup typename gemv_kernel::acc_type tgp_memory
- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
-
- uint32_t indx_vec;
- uint32_t indx_mat;
-
- // Update batch offsets
- if (batch_ndim > 1) {
- const constant auto* veci_bstrides = index_batch_strides;
- const constant auto* mati_bstrides = index_batch_strides + batch_ndim;
-
- ulong2 batch_offsets = elem_to_loc_broadcast(
- tid.z,
- batch_shape,
- veci_bstrides,
- mati_bstrides,
- batch_ndim
- );
-
- indx_vec = vec_indices[batch_offsets.x];
- indx_mat = mat_indices[batch_offsets.y];
-
- } else {
- indx_vec = vec_indices[index_batch_strides[0] * tid.z];
- indx_mat = mat_indices[index_batch_strides[batch_ndim] * tid.z];
- }
-
- if (vector_batch_ndim > 1) {
- in_vec += elem_to_loc(
- indx_vec,
- vector_batch_shape,
- vector_batch_stride,
- vector_batch_ndim
- );
- } else {
- in_vec += indx_vec * vector_batch_stride[0];
- }
-
- if (matrix_batch_ndim > 1) {
- mat += elem_to_loc(
- indx_mat,
- matrix_batch_shape,
- matrix_batch_stride,
- matrix_batch_ndim
- );
- } else {
- mat += indx_mat * matrix_batch_stride[0];
- }
-
- out_vec += tid.z * out_vec_size;
-
- gemv_kernel::run(
- mat,
- in_vec,
- bias,
- out_vec,
- in_vec_size,
- out_vec_size,
- marix_ld,
- alpha,
- beta,
- batch_ndim, // Not used,
- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
-}
-
-// clang-format off
-#define instantiate_gemv_t_bs_helper( \
- nm, itype, bm, bn, sm, sn, tm, tn) \
- instantiate_kernel( \
- "gemv_t_gather_" #nm "_bm" #bm "_bn" #bn "_sm" #sm \
- "_sn" #sn "_tm" #tm "_tn" #tn, \
- gemv_t_gather, itype, bm, bn, sm, sn, tm, tn)
-
-#define instantiate_gemv_t_bs_blocks(name, itype) \
- instantiate_gemv_t_bs_helper(name, itype, 1, 2, 8, 4, 4, 1) \
- instantiate_gemv_t_bs_helper(name, itype, 1, 2, 8, 4, 4, 4) \
- instantiate_gemv_t_bs_helper(name, itype, 1, 4, 8, 4, 4, 4) \
- instantiate_gemv_t_bs_helper(name, itype, 1, 16, 8, 4, 4, 4) \
- instantiate_gemv_t_bs_helper(name, itype, 1, 16, 4, 8, 4, 4) // clang-format on
-
-// clang-format off
-instantiate_gemv_t_bs_blocks(float32, float);
-instantiate_gemv_t_bs_blocks(float16, half);
-instantiate_gemv_t_bs_blocks(bfloat16, bfloat16_t);
-instantiate_gemv_t_bs_blocks(complex64, complex64_t); // clang-format on
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
deleted file mode 100644
index 4fdd13cb3..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ /dev/null
@@ -1,158 +0,0 @@
-use metal::{MTLBuffer, MTLComputeCommandEncoder};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::{
- common::MatmulArguments,
- dispatch_descriptor::{MatmulDispatchDescriptor, choose_dispatch_descriptor},
- gemm,
- gemv::GemvKernel,
- split_k::SplitKGemm,
-};
-use crate::{
- DataType,
- backends::{
- common::kernel::TensorAddBiasKernel,
- metal::{context::MetalContext, error::MetalError, kernel::dsl::TensorAddBiasMetalKernel},
- },
-};
-
-pub struct MatmulKernel {
- data_type: DataType,
- gemm: Option<gemm::GemmKernel>,
- gemv: Option<GemvKernel>,
- splitk: Option<SplitKGemm>,
- bias_add: Option<TensorAddBiasMetalKernel>,
-}
-
-impl MatmulKernel {
- pub fn new(data_type: DataType) -> Result<Self, MetalError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MetalError::Generic(format!("Unsupported dtype for MatmulKernel: {data_type:?}")));
- }
-
- Ok(Self {
- data_type,
- gemm: None,
- gemv: None,
- splitk: None,
- bias_add: None,
- })
- }
-
- pub fn precompile(
- &mut self,
- context: &MetalContext,
- ) -> Result<(), MetalError> {
- let gemm = self.get_or_create_gemm()?;
- gemm.precompile(context)?;
-
- let gemv = self.get_or_create_gemv()?;
- gemv.precompile(context)?;
-
- let splitk = self.get_or_create_splitk()?;
- splitk.precompile(context)?;
-
- Ok(())
- }
-
- fn get_or_create_gemm(&mut self) -> Result<&mut gemm::GemmKernel, MetalError> {
- if self.gemm.is_none() {
- self.gemm = Some(gemm::GemmKernel::new(self.data_type)?);
- }
- Ok(self.gemm.as_mut().unwrap())
- }
-
- fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel, MetalError> {
- if self.gemv.is_none() {
- self.gemv = Some(GemvKernel::new(self.data_type)?);
- }
- Ok(self.gemv.as_mut().unwrap())
- }
-
- fn get_or_create_splitk(&mut self) -> Result<&mut SplitKGemm, MetalError> {
- if self.splitk.is_none() {
- self.splitk = Some(SplitKGemm::new(self.data_type)?);
- }
- Ok(self.splitk.as_mut().unwrap())
- }
-
- fn encode_dispatch_descriptor(
- &mut self,
- context: &MetalContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MatmulArguments,
- descriptor: &MatmulDispatchDescriptor,
- ) -> Result<bool, MetalError> {
- match descriptor {
- MatmulDispatchDescriptor::Gemv(descriptor) => {
- let gemv = self.get_or_create_gemv()?;
- gemv.encode_descriptor(context, encoder, arguments, descriptor)
- },
- MatmulDispatchDescriptor::SplitK(descriptor) => {
- let splitk = self.get_or_create_splitk()?;
- splitk.encode_descriptor(context, encoder, arguments, descriptor)
- },
- MatmulDispatchDescriptor::Gemm(descriptor) => {
- let gemm = self.get_or_create_gemm()?;
- gemm.encode_descriptor(context, encoder, arguments, descriptor)
- },
- }
- }
-
- pub fn encode(
- &mut self,
- context: &MetalContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- mut arguments: MatmulArguments,
- ) -> Result<(), MetalError> {
- Self::apply_batch_collapse(&mut arguments);
-
- let descriptor = choose_dispatch_descriptor(context, self.data_type, &arguments)?;
-
- let bias_fused = self.encode_dispatch_descriptor(context, encoder, &arguments, &descriptor)?;
-
- if let Some(bias) = arguments.bias {
- if !bias_fused {
- self.apply_bias_add(context, encoder, &arguments, bias)?;
- }
- }
-
- Ok(())
- }
-
- fn apply_bias_add(
- &mut self,
- context: &MetalContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MatmulArguments,
- bias: &Retained<ProtocolObject<dyn MTLBuffer>>,
- ) -> Result<(), MetalError> {
- let m = arguments.batch as usize;
- let n = arguments.output_dim as usize;
- let batch_count = arguments.batch_count as usize;
- let total_len = m * n * batch_count;
- if total_len == 0 {
- return Ok(());
- }
-
- if self.bias_add.is_none() {
- self.bias_add = Some(TensorAddBiasMetalKernel::new(context, self.data_type)?);
- }
- let bias_add = self.bias_add.as_ref().unwrap();
- bias_add.encode(arguments.d, bias, arguments.d, n as u32, total_len as u32, encoder);
- Ok(())
- }
-
- fn apply_batch_collapse(arguments: &mut MatmulArguments) {
- if arguments.transpose_a {
- return;
- }
- if arguments.batch_count <= 1 {
- return;
- }
- if arguments.lda == arguments.input_dim && arguments.transpose_b {
- arguments.batch *= arguments.batch_count;
- arguments.batch_count = 1;
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index 050733c0a..d0464c58c 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,29 +1,21 @@
pub mod common;
mod dispatch_descriptor;
mod gemm;
-mod gemv;
-mod kernel;
-mod split_k;
-pub use common::MatmulArguments;
-pub use dispatch_descriptor::{MatmulKernelVariant, determine_kernel_variant};
-pub use gemv::GemvKernel;
-pub use kernel::MatmulKernel;
-use metal::MTLComputeCommandEncoder;
-use objc2::runtime::ProtocolObject;
-pub use split_k::SplitKGemm;
+pub use dispatch_descriptor::choose_dispatch_descriptor;
use crate::{
DataType,
backends::{
common::kernel::matmul::{
- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,
+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait, MatmulArguments,
+ MatmulKernel,
},
metal::{Metal, context::MetalContext, error::MetalError},
},
};
-impl FullPrecisionMatmulKernelTrait for MatmulKernel {
+impl FullPrecisionMatmulKernelTrait for MatmulKernel<Metal> {
type Backend = Metal;
fn new(
@@ -38,14 +30,13 @@ impl FullPrecisionMatmulKernelTrait for MatmulKernel {
fn encode(
&mut self,
context: &MetalContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ encoder: &<Metal as crate::backends::common::Backend>::ComputeEncoder,
arguments: FullPrecisionMatmulArguments<Metal>,
) {
- let matmul_arguments = MatmulArguments {
+ let mut matmul_arguments = MatmulArguments {
a: arguments.a,
a_offset: arguments.a_offset as u64,
b: arguments.b,
- c: None,
d: arguments.output,
bias: arguments.bias,
batch: arguments.batch as i32,
@@ -55,13 +46,15 @@ impl FullPrecisionMatmulKernelTrait for MatmulKernel {
ldb: arguments.input_dim as i32,
ldd: arguments.output_dim as i32,
batch_count: 1,
- alpha: 1.0,
- beta: 0.0,
- transpose_a: false,
transpose_b: true,
};
- MatmulKernel::encode(self, context, encoder, matmul_arguments)
+ MatmulKernel::<Metal>::apply_batch_collapse(&mut matmul_arguments);
+
+ let descriptor = choose_dispatch_descriptor(context, self.data_type, &matmul_arguments)
+ .expect("Failed to create dispatch descriptor for full precision matmul");
+
+ self.encode_with_descriptor(context, matmul_arguments, &descriptor, encoder)
.expect("Failed to encode full precision matmul kernel");
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
deleted file mode 100644
index 713481877..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
+++ /dev/null
@@ -1,241 +0,0 @@
-use std::collections::HashMap;
-
-use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};
-use crate::{
- DataType,
- backends::{
- common::Context,
- metal::{
- context::MetalContext,
- error::MetalError,
- kernel::matmul::common::{MatmulArguments, transpose_configuration},
- metal_extensions::ComputeEncoderSetValue,
- },
- },
-};
-
-pub struct Kernel {
- data_type: DataType,
- partial_pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- accum_pipeline: Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- accumulator_buffer_bytes: usize,
-}
-
-impl Kernel {
- pub fn new(data_type: DataType) -> Result<Self, MetalError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MetalError::Generic(format!("Unsupported dtype for Split-K: {:?}", data_type)));
- }
- Ok(Self {
- data_type,
- partial_pipelines: HashMap::new(),
- accum_pipeline: None,
- accumulator_buffer: None,
- accumulator_buffer_bytes: 0,
- })
- }
-
- pub fn precompile(
- &mut self,
- context: &MetalContext,
- ) -> Result<(), MetalError> {
- use super::tile_configuration::TileConfiguration;
-
- if !matches!(self.data_type, DataType::BF16) {
- return Ok(());
- }
-
- let tile_configs = [
- TileConfiguration {
- tile_rows: 16,
- tile_cols: 16,
- tile_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
- },
- TileConfiguration {
- tile_rows: 16,
- tile_cols: 32,
- tile_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
- },
- TileConfiguration {
- tile_rows: 32,
- tile_cols: 16,
- tile_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
- },
- TileConfiguration {
- tile_rows: 32,
- tile_cols: 32,
- tile_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
- },
- ];
-
- for tile in &tile_configs {
- for &mn_aligned in &[false, true] {
- let config = PipelineConfiguration {
- tile: *tile,
- transpose_a: false,
- transpose_b: true,
- mn_aligned,
- k_aligned: true,
- };
- let _ = self.get_partial_pipeline(context, &config);
- }
- }
-
- let _ = self.get_accum_pipeline(context);
-
- Ok(())
- }
-
- fn steel_type_name(&self) -> Result<&'static str, MetalError> {
- match self.data_type {
- DataType::F16 => Ok("float16"),
- DataType::BF16 => Ok("bfloat16"),
- DataType::F32 => Ok("float32"),
- _ => Err(MetalError::Generic(format!("Unsupported dtype for Split-K: {:?}", self.data_type))),
- }
- }
-
- fn splitk_partial_out_name(&self) -> &'static str {
- "float32"
- }
-
- fn partial_kernel_name(
- &self,
- config: &PipelineConfiguration,
- ) -> Result<String, MetalError> {
- let in_name = self.steel_type_name()?;
- let out_name = self.splitk_partial_out_name();
- let transpose_suffix = transpose_configuration(config.transpose_a, config.transpose_b).as_str();
- let mn_tag = if config.mn_aligned {
- "taligned"
- } else {
- "naligned"
- };
- let k_tag = if config.k_aligned {
- "taligned"
- } else {
- "naligned"
- };
- Ok(format!(
- "steel_gemm_splitk_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",
- transpose_suffix,
- in_name,
- out_name,
- config.tile.tile_rows,
- config.tile.tile_cols,
- config.tile.tile_depth,
- config.tile.warps_per_row,
- config.tile.warps_per_col,
- mn_tag,
- k_tag,
- ))
- }
-
- fn accum_kernel_name(&self) -> Result<String, MetalError> {
- let out_name = self.steel_type_name()?;
- Ok(format!("steel_gemm_splitk_accum_{}_{}", out_name, self.splitk_partial_out_name()))
- }
-
- fn get_partial_pipeline(
- &mut self,
- mtl: &MetalContext,
- config: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {
- if !self.partial_pipelines.contains_key(config) {
- let name = self.partial_kernel_name(config)?;
- let ps = mtl.compute_pipeline_state(&name, None)?;
- self.partial_pipelines.insert(config.clone(), ps);
- }
- Ok(self.partial_pipelines.get(config).unwrap())
- }
-
- fn get_accum_pipeline(
- &mut self,
- mtl: &MetalContext,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {
- if self.accum_pipeline.is_none() {
- let name = self.accum_kernel_name()?;
- let ps = mtl.compute_pipeline_state(&name, None)?;
- self.accum_pipeline = Some(ps);
- }
- Ok(self.accum_pipeline.as_ref().unwrap())
- }
-
- pub fn should_use_splitk(
- m: i32,
- n: i32,
- k: i32,
- batch_count: i32,
- ) -> bool {
- if batch_count != 1 {
- return false;
- }
- if m <= 0 || n <= 0 || k <= 0 {
- return false;
- }
- let m_tiles = m / 16;
- let n_tiles = n / 16;
- let k_tiles = k / 16;
- (m_tiles * n_tiles) <= 32 && k_tiles >= 8
- }
-
- pub(crate) fn encode_descriptor(
- &mut self,
- context: &MetalContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MatmulArguments,
- descriptor: &DispatchDescriptor,
- ) -> Result<bool, MetalError> {
- self.ensure_accumulator_buffer(context, descriptor.accumulator_bytes);
- let accumulator_buffer =
- self.accumulator_buffer.as_ref().cloned().expect("Accumulator buffer must be initialized");
-
- let partial_pipeline_state = self.get_partial_pipeline(context, &descriptor.pipeline_configuration)?;
-
- encoder.set_compute_pipeline_state(partial_pipeline_state);
- encoder.set_buffer(Some(arguments.a), arguments.a_offset as usize, 0);
- encoder.set_buffer(Some(arguments.b), 0, 1);
- encoder.set_buffer(Some(&accumulator_buffer), 0, 2);
- encoder.set_value(&descriptor.params, 3);
- encoder.dispatch_threadgroups(descriptor.partial_threadgroups, descriptor.partial_threads_per_threadgroup);
-
- let accum_pipeline_state = self.get_accum_pipeline(context)?;
- encoder.set_compute_pipeline_state(accum_pipeline_state);
- encoder.set_buffer(Some(&accumulator_buffer), 0, 0);
- encoder.set_buffer(Some(arguments.d), 0, 1);
-
- let partition_count = descriptor.partition_count;
- let output_elements_per_partition = descriptor.output_elements_per_partition;
- encoder.set_value(&partition_count, 2);
- encoder.set_value(&output_elements_per_partition, 3);
- encoder.set_value(&arguments.ldd, 4);
-
- encoder.dispatch_threads(descriptor.accum_total_threads, descriptor.accum_threads_per_threadgroup);
-
- Ok(false)
- }
-
- fn ensure_accumulator_buffer(
- &mut self,
- mtl: &MetalContext,
- required_bytes: usize,
- ) {
- if required_bytes <= self.accumulator_buffer_bytes && self.accumulator_buffer.is_some() {
- return;
- }
- self.accumulator_buffer = Some(mtl.create_buffer(required_bytes).expect("Failed to create accumulator buffer"));
- self.accumulator_buffer_bytes = required_bytes;
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
index 029fa0f90..b4607a3dd 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
@@ -1,7 +1 @@
-mod dispatch_descriptor;
-mod kernel;
-mod pipeline_configuration;
-mod tile_configuration;
-
-pub(crate) use dispatch_descriptor::DispatchDescriptor;
-pub use kernel::Kernel as SplitKGemm;
+// Shader files only — Rust logic has moved to common/kernel/matmul/split_k/
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h
deleted file mode 100644
index 60d238d98..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h
+++ /dev/null
@@ -1,235 +0,0 @@
-
-
-using namespace steel;
-
-///////////////////////////////////////////////////////////////////////////////
-// GEMM kernels
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- bool MN_aligned,
- bool K_aligned>
-[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void gemm_splitk(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- device U* C [[buffer(2)]],
- const constant GEMMSpiltKParams* params [[buffer(3)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]]
-) {
- (void)lid;
-
- using gemm_kernel = GEMMKernel<
- T,
- U,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- MN_aligned,
- K_aligned>;
- using loader_a_t = typename gemm_kernel::loader_a_t;
- using loader_b_t = typename gemm_kernel::loader_b_t;
- using mma_t = typename gemm_kernel::mma_t;
-
- threadgroup T As[gemm_kernel::tgp_mem_size_a];
- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];
-
- const int tid_x = tid.x;
- const int tid_y = tid.y;
- const int tid_z = tid.z;
-
- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
- return;
- }
-
- // Find block in A, B, C
- const int c_row = tid_y * BM;
- const int c_col = tid_x * BN;
- const int k_start = params->split_k_partition_size * tid_z;
-
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
- const size_t k_start_long = size_t(k_start);
-
- A += transpose_a ? (c_row_long + k_start_long * params->lda)
- : (k_start_long + c_row_long * params->lda);
- B += transpose_b ? (k_start_long + c_col_long * params->ldb)
- : (c_col_long + k_start_long * params->ldb);
- C += (size_t(params->split_k_partition_stride) * tid_z) +
- (c_row_long * params->ldc + c_col_long);
-
- // Prepare threadgroup loading operations
- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);
-
- // Prepare threadgroup mma operation
- thread mma_t mma_op(simd_group_id, simd_lane_id);
-
- int gemm_k_iterations = params->gemm_k_iterations_aligned;
-
- short tgp_bm = min(BM, params->M - c_row);
- short tgp_bn = min(BN, params->N - c_col);
- short leftover_bk = params->K % BK;
-
- if (MN_aligned || (tgp_bm == BM && tgp_bn == BN)) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<true, true, true>{}
- );
- } else if (tgp_bn == BN) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<false, true, true>{}
- );
- } else if (tgp_bm == BM) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<true, false, true>{}
- );
- } else {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<false, false, true>{}
- );
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if ((tid_z + 1) == (params->split_k_partitions)) {
- int gemm_k_iter_remaining =
- (params->K - (k_start + params->split_k_partition_size)) / BK;
- if (!K_aligned || gemm_k_iter_remaining > 0)
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iter_remaining,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk,
- LoopAlignment<false, false, K_aligned>{}
- );
- }
-
- if (MN_aligned || (tgp_bm == BM && tgp_bn == BN)) {
- mma_op.store_result(C, params->ldc);
- } else {
- mma_op.store_result_safe(C, params->ldc, short2(tgp_bn, tgp_bm));
- }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Split k accumulation kernel
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename AccT,
- typename OutT,
- typename Epilogue = TransformNone<OutT, AccT>>
-[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_accum(
- const device AccT* C_split [[buffer(0)]],
- device OutT* D [[buffer(1)]],
- const constant int& k_partitions [[buffer(2)]],
- const constant int& partition_stride [[buffer(3)]],
- const constant int& ldd [[buffer(4)]],
- uint2 gid [[thread_position_in_grid]]
-) {
- // Ajust D and C
- D += gid.x + gid.y * size_t(ldd);
- C_split += gid.x + gid.y * size_t(ldd);
-
- size_t offset = 0;
- AccT out = 0;
-
- for (int i = 0; i < k_partitions; i++) {
- out += C_split[offset];
- offset += partition_stride;
- }
-
- // Write output
- D[0] = Epilogue::apply(out);
-}
-
-template <
- typename AccT,
- typename OutT,
- typename Epilogue = TransformAxpby<OutT, AccT>>
-[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_accum_axpby(
- const device AccT* C_split [[buffer(0)]],
- device OutT* D [[buffer(1)]],
- const constant int& k_partitions [[buffer(2)]],
- const constant int& partition_stride [[buffer(3)]],
- const constant int& ldd [[buffer(4)]],
- const device OutT* C [[buffer(5)]],
- const constant int& ldc [[buffer(6)]],
- const constant int& fdc [[buffer(7)]],
- const constant float& alpha [[buffer(8)]],
- const constant float& beta [[buffer(9)]],
- uint2 gid [[thread_position_in_grid]]
-) {
- // Ajust D and C
- C += gid.x * size_t(fdc) + gid.y * size_t(ldc);
- D += gid.x + gid.y * size_t(ldd);
- C_split += gid.x + gid.y * size_t(ldd);
-
- size_t offset = 0;
- AccT out = 0;
-
- for (int i = 0; i < k_partitions; i++) {
- out += C_split[offset];
- offset += partition_stride;
- }
-
- // Write output
- Epilogue op(alpha, beta);
- D[0] = op.apply(out, *C);
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
index 34c61e201..806683221 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
@@ -1,76 +1,322 @@
-
-// clang-format off
#include "../../../common/utils.h"
+#include "../../../definitions.metal"
#include "../../common/steel/gemm/gemm.h"
-#include "steel_gemm_splitk.h"
-
-#define instantiate_gemm( \
- tname, \
- trans_a, \
- trans_b, \
- iname, \
- itype, \
- oname, \
- otype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- aname, \
- mn_aligned, \
- kname, \
- k_aligned) \
- instantiate_kernel( \
- "steel_gemm_splitk_" #tname "_" #iname "_" #oname \
- "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn \
- "_MN_" #aname "_K_" #kname, \
- gemm_splitk, \
- itype, \
- otype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- trans_a, \
- trans_b, \
- mn_aligned, \
- k_aligned)
-
-#define instantiate_gemm_aligned_helper(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, taligned, true) \
- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, naligned, false) \
- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, taligned, true) \
- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, naligned, false)
-
-#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm_aligned_helper(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm_aligned_helper(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm_aligned_helper(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm_aligned_helper(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)
-
-#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 16, 16, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 32, 16, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 16, 16, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)
-
-instantiate_gemm_shapes_helper(float16, half, float32, float);
-instantiate_gemm_shapes_helper(bfloat16, bfloat16_t, float32, float);
-instantiate_gemm_shapes_helper(float32, float, float32, float);
-instantiate_gemm_shapes_helper(complex64, complex64_t, complex64, complex64_t);
-
-#define instantiate_accum(oname, otype, aname, atype) \
- instantiate_kernel( \
- "steel_gemm_splitk_accum_" #oname "_" #aname, \
- gemm_splitk_accum, atype, otype) \
- instantiate_kernel( \
- "steel_gemm_splitk_accum_" #oname "_" #aname "_axbpy", \
- gemm_splitk_accum_axpby, atype, otype) \
-
-instantiate_accum(bfloat16, bfloat16_t, float32, float);
-instantiate_accum(float16, half, float32, float);
-instantiate_accum(float32, float, float32, float);
-instantiate_accum(complex64, complex64_t, complex64, complex64_t); // clang-format on
+
+using namespace steel;
+
+// Upper bounds for split-K threadgroup memory.
+// Split-K tile: BM=16, BN=32, BK=16 with bfloat16 (padding=8)
+// tgp_mem_size_a = BM * (BK + padding) = 16 * (16 + 8) = 384
+// tgp_mem_size_b = BN * (BK + padding) = 32 * (16 + 8) = 768
+#define SPLITK_MAX_TGP_A 384
+#define SPLITK_MAX_TGP_B 768
+
+namespace uzu {
+namespace matmul {
+using GEMMSpiltKParams = steel::GEMMSpiltKParams;
+} // namespace matmul
+} // namespace uzu
+
+///////////////////////////////////////////////////////////////////////////////
+// Split-K GEMM implementation
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T, typename U>
+METAL_FUNC void gemm_splitk_impl(
+ const device T* a,
+ const device T* b,
+ device U* c,
+ const constant GEMMSpiltKParams* params,
+ const int BM,
+ const int BN,
+ const int BK,
+ const int WM,
+ const int WN,
+ const bool align_m,
+ const bool align_k,
+ threadgroup T* a_shared,
+ threadgroup T* b_shared,
+ uint simd_lane_id,
+ uint simd_group_id,
+ uint3 tid,
+ uint3 lid
+) {
+ (void)lid;
+
+ const bool transpose_a = false;
+ const bool transpose_b = true;
+
+ const short tgp_padding = 16 / sizeof(T);
+ const short lda_tgp = BK + tgp_padding;
+ const short ldb_tgp = BK + tgp_padding;
+ const short tgp_size = WM * WN * 32;
+
+ const int tid_x = tid.x;
+ const int tid_y = tid.y;
+ const int tid_z = tid.z;
+
+ if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
+ return;
+ }
+
+ const int c_row = tid_y * BM;
+ const int c_col = tid_x * BN;
+ const int k_start = params->split_k_partition_size * tid_z;
+
+ const size_t c_row_long = size_t(c_row);
+ const size_t c_col_long = size_t(c_col);
+ const size_t k_start_long = size_t(k_start);
+
+ a += k_start_long + c_row_long * params->lda;
+ b += k_start_long + c_col_long * params->ldb;
+ c += (size_t(params->split_k_partition_stride) * tid_z) +
+ (c_row_long * params->ldc + c_col_long);
+
+ thread BlockLoader<T> loader_a(
+ a,
+ params->lda,
+ a_shared,
+ simd_group_id,
+ simd_lane_id,
+ BM,
+ BK,
+ lda_tgp,
+ 1,
+ tgp_size
+ );
+
+ thread BlockLoader<T> loader_b(
+ b,
+ params->ldb,
+ b_shared,
+ simd_group_id,
+ simd_lane_id,
+ BN,
+ BK,
+ ldb_tgp,
+ 1,
+ tgp_size
+ );
+
+ thread BlockMMA<T, U, float> mma_op(
+ simd_group_id,
+ simd_lane_id,
+ BM,
+ BN,
+ BK,
+ WM,
+ WN,
+ transpose_a,
+ transpose_b,
+ lda_tgp,
+ ldb_tgp
+ );
+
+ int gemm_k_iterations = params->gemm_k_iterations_aligned;
+
+ short tgp_bm = min(BM, params->M - c_row);
+ short tgp_bn = min(BN, params->N - c_col);
+ short leftover_bk = params->K % BK;
+
+ bool mn_aligned = align_m || (tgp_bm == BM && tgp_bn == BN);
+
+ if (mn_aligned) {
+ gemm_loop<T, U, float>(
+ a_shared,
+ b_shared,
+ gemm_k_iterations,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk,
+ BK,
+ transpose_a,
+ transpose_b,
+ true,
+ true,
+ true
+ );
+ } else if (tgp_bn == BN) {
+ gemm_loop<T, U, float>(
+ a_shared,
+ b_shared,
+ gemm_k_iterations,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk,
+ BK,
+ transpose_a,
+ transpose_b,
+ false,
+ true,
+ true
+ );
+ } else if (tgp_bm == BM) {
+ gemm_loop<T, U, float>(
+ a_shared,
+ b_shared,
+ gemm_k_iterations,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk,
+ BK,
+ transpose_a,
+ transpose_b,
+ true,
+ false,
+ true
+ );
+ } else {
+ gemm_loop<T, U, float>(
+ a_shared,
+ b_shared,
+ gemm_k_iterations,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk,
+ BK,
+ transpose_a,
+ transpose_b,
+ false,
+ false,
+ true
+ );
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if ((tid_z + 1) == (params->split_k_partitions)) {
+ int gemm_k_iter_remaining =
+ (params->K - (k_start + params->split_k_partition_size)) / BK;
+ if (!align_k || gemm_k_iter_remaining > 0)
+ gemm_loop<T, U, float>(
+ a_shared,
+ b_shared,
+ gemm_k_iter_remaining,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk,
+ BK,
+ transpose_a,
+ transpose_b,
+ false,
+ false,
+ align_k
+ );
+ }
+
+ if (mn_aligned) {
+ mma_op.store_result(c, params->ldc);
+ } else {
+ mma_op.store_result_safe(c, params->ldc, short2(tgp_bn, tgp_bm));
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Split-K accumulation
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename AccT,
+ typename OutT,
+ typename Epilogue = TransformNone<OutT, AccT>>
+METAL_FUNC void gemm_splitk_accum_impl(
+ const device AccT* c_split,
+ device OutT* d,
+ const constant int& k_partitions,
+ const constant int& partition_stride,
+ const constant int& ldd,
+ uint2 gid
+) {
+ d += gid.x + gid.y * size_t(ldd);
+ c_split += gid.x + gid.y * size_t(ldd);
+
+ size_t offset = 0;
+ AccT out = 0;
+
+ for (int i = 0; i < k_partitions; i++) {
+ out += c_split[offset];
+ offset += partition_stride;
+ }
+
+ d[0] = Epilogue::apply(out);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// DSL kernel entry points
+///////////////////////////////////////////////////////////////////////////////
+
+KERNEL(MatmulSplitKPartialBfloat16)(
+ const device bfloat16_t* a,
+ const device bfloat16_t* b,
+ device float* c,
+ const constant uzu::matmul::GEMMSpiltKParams* params,
+ const constant uint& partial_group_count_x,
+ const constant uint& partial_group_count_y,
+ const constant uint& partial_group_count_z,
+ const uint group_x GROUPS(partial_group_count_x),
+ const uint group_y GROUPS(partial_group_count_y),
+ const uint group_z GROUPS(partial_group_count_z),
+ const uint thread_x THREADS(32),
+ const uint thread_y THREADS(2),
+ const uint thread_z THREADS(2),
+ threadgroup bfloat16_t a_shared[SPLITK_MAX_TGP_A],
+ threadgroup bfloat16_t b_shared[SPLITK_MAX_TGP_B],
+ const Simd simd
+) {
+ gemm_splitk_impl<bfloat16_t, float>(
+ a,
+ b,
+ c,
+ params,
+ 16,
+ 32,
+ 16,
+ 2,
+ 2,
+ false,
+ true,
+ a_shared,
+ b_shared,
+ simd.lane_idx,
+ simd.group_idx,
+ uint3(group_x, group_y, group_z),
+ uint3(thread_x, thread_y, thread_z)
+ );
+}
+
+KERNEL(MatmulSplitKAccumBfloat16)(
+ const device float* c_split,
+ device bfloat16_t* d,
+ const constant int& k_partitions,
+ const constant int& partition_stride,
+ const constant int& ldd,
+ const constant uint& accum_total_threads_x,
+ const constant uint& accum_total_threads_y,
+ const uint gid_x AXIS(accum_total_threads_x, 16),
+ const uint gid_y AXIS(accum_total_threads_y, 16)
+) {
+ gemm_splitk_accum_impl<float, bfloat16_t>(
+ c_split,
+ d,
+ k_partitions,
+ partition_stride,
+ ldd,
+ uint2(gid_x, gid_y)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 8f0af55a2..021380535 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,5 +1,7 @@
-use crate::backends::common::kernel::matmul::MatmulKernels;
-
+use crate::backends::{
+ common::kernel::matmul::{MatmulKernel, MatmulKernels},
+ metal::Metal,
+};
pub mod dsl {
include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
}
@@ -7,8 +9,6 @@ pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default
pub mod matmul;
-pub use matmul::{MatmulArguments, MatmulKernel};
-
impl MatmulKernels for dsl::MetalKernels {
- type FullPrecisionMatmulKernel = MatmulKernel;
+ type FullPrecisionMatmulKernel = MatmulKernel<Metal>;
}
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 4982e0eca..8d0d232f6 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -7,11 +7,11 @@ use ndarray::Array2;
use uzu::{
DataType,
backends::{
- common::Context,
- metal::{
- MetalContext,
- kernel::{MatmulArguments, MatmulKernel},
+ common::{
+ Context,
+ kernel::matmul::{MatmulArguments, MatmulKernel},
},
+ metal::{Metal, MetalContext, kernel::matmul::choose_dispatch_descriptor},
},
};
@@ -43,33 +43,28 @@ fn run_metal_matmul(
n
};
- let mut kernel = MatmulKernel::new(DataType::BF16).expect("kernel");
+ let mut kernel = MatmulKernel::<Metal>::new(DataType::BF16).expect("kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
- let encode_result = kernel.encode(
- ctx,
- &enc,
- MatmulArguments {
- a: &a_buf,
- a_offset: 0,
- b: &b_buf,
- c: None,
- d: &d_buf,
- bias: None,
- batch: m as i32,
- input_dim: k as i32,
- output_dim: n as i32,
- lda: k as i32,
- ldb: ldb as i32,
- ldd: n as i32,
- batch_count: 1,
- alpha: 1.0,
- beta: 0.0,
- transpose_a: false,
- transpose_b,
- },
- );
+ let mut arguments = MatmulArguments {
+ a: &a_buf,
+ a_offset: 0,
+ b: &b_buf,
+ d: &d_buf,
+ bias: None,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: ldb as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ transpose_b,
+ };
+ MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);
+ let descriptor = choose_dispatch_descriptor(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");
+ let encode_result = kernel.encode_with_descriptor(ctx, arguments, &descriptor, &enc);
enc.end_encoding();
encode_result.expect("encode");
cb.commit();
@@ -129,87 +124,101 @@ struct TestCase {
tolerance: f32,
}
-fn test_cases() -> Vec<TestCase> {
- let base_tolerance = 0.01;
+const MATMUL_CORRECTNESS_FULL_ENV: &str = "UZU_MATMUL_CORRECTNESS_FULL";
+
+const QUICK_BATCH_SIZES: [usize; 2] = [1, 16];
+const FULL_BATCH_SIZES: [usize; 12] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];
- let batch_sizes: [usize; 12] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];
-
- let model_shapes: [(usize, usize); 51] = [
- // Qwen2.5-Coder-0.5B
- (896, 896),
- (896, 1152),
- (896, 4864),
- (4864, 896),
- // Qwen3-0.6B
- (1024, 1024),
- (1024, 3072),
- (1024, 4096),
- (3072, 1024),
- // Gemma-3-1B
- (1152, 1152),
- (1152, 1536),
- (1152, 6912),
- (6912, 1152),
- // Qwen2.5-Coder-1.5B
- (1536, 1536),
- (1536, 2048),
- (1536, 8960),
- (8960, 1536),
- // Llama-3.2-1B / Qwen3-1.7B / SmolLM2-1.7B
- (2048, 2048),
- (2048, 2560),
- (2048, 3072),
- (2048, 4096),
- (2048, 6144),
- (2048, 8192),
- (2048, 11008),
- (6144, 2048),
- (8192, 2048),
- (11008, 2048),
- // Qwen3-4B / Gemma-3-4B
- (2560, 2560),
- (2560, 4096),
- (2560, 6144),
- (2560, 9728),
- (2560, 10240),
- (9728, 2560),
- (10240, 2560),
- // Llama-3.2-3B
- (3072, 3072),
- (3072, 5120),
- (3072, 8192),
- (8192, 3072),
- // Qwen2.5-Coder-7B
- (3584, 3584),
- (3584, 4608),
- (3584, 18944),
- (18944, 3584),
- // Llama-3.1-8B / Qwen3-8B
- (4096, 4096),
- (4096, 6144),
- (4096, 12288),
- (4096, 14336),
- (12288, 4096),
- (14336, 4096),
- // Qwen3-14B
- (5120, 5120),
- (5120, 7168),
- (5120, 17408),
- (17408, 5120),
- ];
+const QUICK_MODEL_SHAPES: [(usize, usize); 8] =
+ [(896, 896), (896, 4864), (1024, 4096), (1152, 6912), (1536, 8960), (2048, 8192), (4096, 14336), (5120, 17408)];
+
+const FULL_MODEL_SHAPES: [(usize, usize); 51] = [
+ // Qwen2.5-Coder-0.5B
+ (896, 896),
+ (896, 1152),
+ (896, 4864),
+ (4864, 896),
+ // Qwen3-0.6B
+ (1024, 1024),
+ (1024, 3072),
+ (1024, 4096),
+ (3072, 1024),
+ // Gemma-3-1B
+ (1152, 1152),
+ (1152, 1536),
+ (1152, 6912),
+ (6912, 1152),
+ // Qwen2.5-Coder-1.5B
+ (1536, 1536),
+ (1536, 2048),
+ (1536, 8960),
+ (8960, 1536),
+ // Llama-3.2-1B / Qwen3-1.7B / SmolLM2-1.7B
+ (2048, 2048),
+ (2048, 2560),
+ (2048, 3072),
+ (2048, 4096),
+ (2048, 6144),
+ (2048, 8192),
+ (2048, 11008),
+ (6144, 2048),
+ (8192, 2048),
+ (11008, 2048),
+ // Qwen3-4B / Gemma-3-4B
+ (2560, 2560),
+ (2560, 4096),
+ (2560, 6144),
+ (2560, 9728),
+ (2560, 10240),
+ (9728, 2560),
+ (10240, 2560),
+ // Llama-3.2-3B
+ (3072, 3072),
+ (3072, 5120),
+ (3072, 8192),
+ (8192, 3072),
+ // Qwen2.5-Coder-7B
+ (3584, 3584),
+ (3584, 4608),
+ (3584, 18944),
+ (18944, 3584),
+ // Llama-3.1-8B / Qwen3-8B
+ (4096, 4096),
+ (4096, 6144),
+ (4096, 12288),
+ (4096, 14336),
+ (12288, 4096),
+ (14336, 4096),
+ // Qwen3-14B
+ (5120, 5120),
+ (5120, 7168),
+ (5120, 17408),
+ (17408, 5120),
+];
+
+fn run_full_case_matrix() -> bool {
+ std::env::var(MATMUL_CORRECTNESS_FULL_ENV)
+ .ok()
+ .map(|value| {
+ let normalized = value.trim().to_ascii_lowercase();
+ matches!(normalized.as_str(), "1" | "true" | "yes")
+ })
+ .unwrap_or(false)
+}
+
+fn test_cases(full_case_matrix: bool) -> Vec<TestCase> {
+ let base_tolerance = 0.01;
+ let (batch_sizes, model_shapes): (&[usize], &[(usize, usize)]) = if full_case_matrix {
+ (&FULL_BATCH_SIZES, &FULL_MODEL_SHAPES)
+ } else {
+ (&QUICK_BATCH_SIZES, &QUICK_MODEL_SHAPES)
+ };
let mut cases: Vec<(usize, usize, usize, bool)> =
model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n, true))).collect();
- // Non-transposed B (for attention AV matmul)
- for &(k, n) in &[(1024, 1024), (2048, 2048), (4096, 4096)] {
- for &m in &[1, 64, 128] {
- cases.push((m, k, n, false));
- }
- }
-
// Edge case: small matrix
- for &m in &batch_sizes {
+ for &m in batch_sizes {
cases.push((m, 128, 128, true));
}
@@ -270,7 +279,17 @@ fn matmul_correctness_comprehensive() {
return;
};
- let cases = test_cases();
+ let full_case_matrix = run_full_case_matrix();
+ let cases = test_cases(full_case_matrix);
+ eprintln!(
+ "Running {} matmul correctness case matrix (set {}=1 for exhaustive run)",
+ if full_case_matrix {
+ "exhaustive"
+ } else {
+ "quick"
+ },
+ MATMUL_CORRECTNESS_FULL_ENV
+ );
let mut passed = 0;
let mut failed = Vec::new();