A high-performance inference engine for AI models
dslify matmul
eugene10b86ceca51c
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex 92ff957d9..b7f677d03 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -52,7 +52,10 @@ pub fn bindgen(let entry_name = dynamic_mangle(kernel.name.as_ref(), variants_kernel_format);let base_index = specialize_indices.get(&kernel.name).copied();- let (specialize_args, specialize_setup): (Vec<TokenStream>, Vec<TokenStream>) = kernel+ let (specialize_args, (specialize_setup, specialize_arg_names)): (+ Vec<TokenStream>,+ (Vec<TokenStream>, Vec<Ident>),+ ) = kernel.arguments.iter().filter(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Specialize(_))))@@ -68,7 +71,7 @@ pub fn bindgen(let setup = quote! {function_constants.set_value(&#arg_name, #idx);};- (arg_def, setup)+ (arg_def, (setup, arg_name))}).unzip();@@ -86,6 +89,12 @@ pub fn bindgen(} else {quote! { None }};+ let cache_key = if has_specialize {+ let format_str = format!("{{}}{}", repeat_n("_{}", specialize_arg_names.len()).join(""));+ quote! { &format!(#format_str, &entry_name #(, #specialize_arg_names)*) }+ } else {+ quote! { &entry_name }+ };let mut arg_count: usize = 0;let mut indirect_flag = false;@@ -305,8 +314,9 @@ pub fn bindgen(type Backend = crate::backends::metal::Metal;fn new(context: &MetalContext #(, #variants_extra_arguments)* #(, #specialize_args)*) -> Result<Self, MetalError> {+ let entry_name = #entry_name;#function_constants_init- let pipeline = context.compute_pipeline_state(&#entry_name, #function_constants_arg)?;+ let pipeline = context.compute_pipeline_state(#cache_key, &entry_name, #function_constants_arg)?;Ok(Self { pipeline #(, #conditional_buffer_sets)* })}diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rsindex dbcc1d012..bd1fbd8e2 100644--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs@@ -42,33 +42,3 @@ pub struct GEMMSpiltKParams {pub split_k_partition_size: i32,pub gemm_k_iterations_aligned: i32,}--/// Split-K MLP Fused GEMM parameters.-#[repr(C)]-#[derive(Debug, Default, Copy, Clone)]-pub struct GEMMSpiltKMlpFusedParams {- pub M: i32,- pub N: i32,- pub K: i32,- pub lda: i32,- pub ldb: i32,- pub ldc: i32,- pub tiles_n: i32,- pub tiles_m: i32,- pub split_k_partitions: i32,- pub split_k_partition_stride: i32,- pub split_k_partition_size: i32,- pub gemm_k_iterations_aligned: i32,- pub hidden_dim: i32,-}--/// AddMM parameters (alpha * A @ B + beta * C).-#[repr(C)]-#[derive(Debug, Default, Copy, Clone)]-pub struct GEMMAddMMParams {- pub ldc: i32,- pub fdc: i32,- pub batch_stride_c: i64,- pub alpha: f32,- pub beta: f32,-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rsnew file mode 100644index 000000000..cb79a4815--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs@@ -0,0 +1,17 @@+use super::{gemm, gemv, split_k};++#[derive(Debug, Clone)]+pub enum MatmulDispatchDescriptor {+ Gemv(gemv::DispatchDescriptor),+ SplitK(split_k::DispatchDescriptor),+ Gemm(gemm::DispatchDescriptor),+}++impl MatmulDispatchDescriptor {+ pub fn bias_is_fused(&self) -> bool {+ match self {+ MatmulDispatchDescriptor::Gemv(d) => d.bias_is_fused(),+ MatmulDispatchDescriptor::SplitK(_) | MatmulDispatchDescriptor::Gemm(_) => false,+ }+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rsnew file mode 100644index 000000000..44e1d380b--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs@@ -0,0 +1,9 @@+use super::{super::grid_size::GridSize, specialization::Specialization};+use crate::backends::common::gpu_types::GEMMParams;++#[derive(Debug, Clone)]+pub struct DispatchDescriptor {+ pub specialization: Specialization,+ pub params: GEMMParams,+ pub threadgroups: GridSize,+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rsnew file mode 100644index 000000000..0d4b30556--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs@@ -0,0 +1,96 @@+use std::collections::HashMap;++use super::{+ super::matmul_arguments::MatmulArguments, dispatch_descriptor::DispatchDescriptor, specialization::Specialization,+};+use crate::{+ DataType,+ backends::common::{Backend, Kernels, kernel::MatmulGemmKernel},+};++pub struct GemmKernel<B: Backend> {+ data_type: DataType,+ pipelines: HashMap<Specialization, <B::Kernels as Kernels>::MatmulGemmKernel>,+}++impl<B: Backend> GemmKernel<B>+where+ B::Error: From<String>,+{+ pub fn new(data_type: DataType) -> Result<Self, B::Error> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(B::Error::from(format!("Unsupported dtype for GEMM: {data_type:?}")));+ }+ Ok(Self {+ data_type,+ pipelines: HashMap::new(),+ })+ }++ pub fn precompile(+ &mut self,+ context: &B::Context,+ ) -> Result<(), B::Error> {+ for &config in Specialization::precompile_configs(self.data_type) {+ self.get_or_create_kernel(context, config)?;+ }+ Ok(())+ }++ fn get_or_create_kernel(+ &mut self,+ context: &B::Context,+ config: Specialization,+ ) -> Result<&<B::Kernels as Kernels>::MatmulGemmKernel, B::Error> {+ if !self.pipelines.contains_key(&config) {+ let kernel = <B::Kernels as Kernels>::MatmulGemmKernel::new(+ context,+ self.data_type,+ config.block_rows as u32,+ config.block_cols as u32,+ config.block_depth as u32,+ config.warps_per_row as u32,+ config.warps_per_col as u32,+ config.align_m,+ config.align_n,+ config.align_k,+ )?;+ self.pipelines.insert(config, kernel);+ }+ Ok(self.pipelines.get(&config).unwrap())+ }++ pub fn encode(+ &mut self,+ context: &B::Context,+ arguments: &MatmulArguments<B>,+ dispatch_descriptor: &DispatchDescriptor,+ encoder: &B::ComputeEncoder,+ ) -> Result<(), B::Error> {+ let config = dispatch_descriptor.specialization;++ let group_count_x = u32::try_from(dispatch_descriptor.threadgroups.x).map_err(|_| {+ B::Error::from(format!("GEMM group count x overflows u32: {}", dispatch_descriptor.threadgroups.x))+ })?;+ let group_count_y = u32::try_from(dispatch_descriptor.threadgroups.y).map_err(|_| {+ B::Error::from(format!("GEMM group count y overflows u32: {}", dispatch_descriptor.threadgroups.y))+ })?;+ let group_count_z = u32::try_from(dispatch_descriptor.threadgroups.z).map_err(|_| {+ B::Error::from(format!("GEMM group count z overflows u32: {}", dispatch_descriptor.threadgroups.z))+ })?;++ let pipeline = self.get_or_create_kernel(context, config)?;+ pipeline.encode(+ (arguments.a, arguments.a_offset as usize),+ arguments.b,+ arguments.d,+ std::slice::from_ref(&dispatch_descriptor.params),+ group_count_x,+ group_count_y,+ group_count_z,+ encoder,+ );++ Ok(())+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rsnew file mode 100644index 000000000..a4cbe5d36--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs@@ -0,0 +1,7 @@+mod dispatch_descriptor;+mod kernel;+pub mod specialization;++pub use dispatch_descriptor::DispatchDescriptor;+pub use kernel::GemmKernel;+pub use specialization::Specialization;diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rsnew file mode 100644index 000000000..4c349a4c2--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs@@ -0,0 +1,138 @@+use crate::DataType;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct Specialization {+ pub block_rows: i32,+ pub block_cols: i32,+ pub block_depth: i32,+ pub warps_per_row: u64,+ pub warps_per_col: u64,+ pub swizzle_log2: i32,+ pub align_m: bool,+ pub align_n: bool,+ pub align_k: bool,+}++impl Specialization {+ pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {+ match data_type {+ DataType::BF16 => &[+ Self {+ block_rows: 64,+ block_cols: 32,+ block_depth: 32,+ warps_per_row: 2,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: false,+ align_n: true,+ align_k: true,+ },+ Self {+ block_rows: 64,+ block_cols: 32,+ block_depth: 32,+ warps_per_row: 2,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: true,+ align_n: true,+ align_k: true,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ block_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: false,+ align_n: true,+ align_k: true,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ block_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: true,+ align_n: false,+ align_k: true,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ block_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: true,+ align_n: true,+ align_k: true,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ block_depth: 16,+ warps_per_row: 1,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: true,+ align_n: true,+ align_k: true,+ },+ ],+ DataType::F16 => &[+ Self {+ block_rows: 64,+ block_cols: 64,+ block_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: true,+ align_n: true,+ align_k: true,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ block_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: false,+ align_n: true,+ align_k: true,+ },+ ],+ DataType::F32 => &[+ Self {+ block_rows: 32,+ block_cols: 64,+ block_depth: 16,+ warps_per_row: 1,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: false,+ align_n: true,+ align_k: true,+ },+ Self {+ block_rows: 32,+ block_cols: 64,+ block_depth: 16,+ warps_per_row: 1,+ warps_per_col: 2,+ swizzle_log2: 0,+ align_m: true,+ align_n: true,+ align_k: true,+ },+ ],+ _ => &[],+ }+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rsnew file mode 100644index 000000000..e458f0f87--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs@@ -0,0 +1,161 @@+use std::sync::OnceLock;++use super::{super::matmul_arguments::MatmulArguments, specialization::Specialization};+use crate::{DataType, backends::common::Backend};++const DEFAULT_GEMV_MAX_BATCH: i32 = 8;++static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();++fn max_gemv_batch_threshold() -> i32 {+ *GEMV_MAX_BATCH.get_or_init(|| {+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)+ })+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum OutputSource {+ None,+ Bias,+}++#[derive(Debug, Clone)]+pub struct DispatchDescriptor {+ pub specialization: Specialization,+ pub matrix_is_rhs: bool,+ pub output_source: OutputSource,+ pub input_dimension: i32,+ pub output_dimension: i32,+ pub matrix_leading_dim: i32,+ pub alpha: f32,+ pub beta: f32,+ pub batch_shape: [i32; 1],+ pub vector_batch_stride: [i64; 1],+ pub matrix_batch_stride: [i64; 1],+ pub bias_batch_stride: [i64; 1],+ pub bias_stride: i32,+ pub batch_rows: i32,+}++impl DispatchDescriptor {+ pub fn try_new<B: Backend>(+ data_type: DataType,+ arguments: &MatmulArguments<B>,+ ) -> Result<Option<Self>, B::Error>+ where+ B::Error: From<String>,+ {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(B::Error::from(format!("Unsupported data type for GEMV: {data_type:?}")));+ }++ if !arguments.transpose_b {+ return Ok(None);+ }++ let m = arguments.batch;+ let n = arguments.output_dim;++ let max_gemv_batch = max_gemv_batch_threshold();++ if n == 1 {+ if m != 1 {+ return Ok(None);+ }+ } else if m > max_gemv_batch {+ return Ok(None);+ }++ let matrix_is_rhs = n != 1;+ let transpose_matrix = if matrix_is_rhs {+ !arguments.transpose_b+ } else {+ false+ };++ let output_source = if arguments.bias.is_some() {+ OutputSource::Bias+ } else {+ OutputSource::None+ };++ let (apply_output_scale_and_accumulate, alpha, beta, bias_stride) = match output_source {+ OutputSource::None => (false, 1.0f32, 0.0f32, 0),+ OutputSource::Bias => (true, 1.0f32, 1.0f32, 1),+ };++ let output_dimension = if matrix_is_rhs {+ arguments.output_dim+ } else {+ arguments.batch+ };++ let specialization = Specialization::select(+ transpose_matrix,+ arguments.input_dim,+ output_dimension,+ apply_output_scale_and_accumulate,+ );++ let input_dimension = arguments.input_dim;+ let matrix_leading_dim = if matrix_is_rhs {+ arguments.ldb+ } else {+ arguments.lda+ };++ let batch_shape = [if arguments.batch_count > 1 {+ arguments.batch_count+ } else {+ 1+ }];++ let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);+ let elements_per_matrix_b = if arguments.transpose_b {+ (arguments.output_dim as i64) * (arguments.ldb as i64)+ } else {+ (arguments.input_dim as i64) * (arguments.ldb as i64)+ };++ let vector_batch_stride = [if matrix_is_rhs {+ elements_per_matrix_a+ } else {+ elements_per_matrix_b+ }];++ let matrix_batch_stride = [if matrix_is_rhs {+ elements_per_matrix_b+ } else {+ elements_per_matrix_a+ }];++ let bias_batch_stride = [if arguments.batch_count > 1 {+ (output_dimension as i64) * (arguments.ldd as i64)+ } else {+ 0+ }];++ let batch_rows = arguments.batch;++ Ok(Some(Self {+ specialization,+ matrix_is_rhs,+ output_source,+ input_dimension,+ output_dimension,+ matrix_leading_dim,+ alpha,+ beta,+ batch_shape,+ vector_batch_stride,+ matrix_batch_stride,+ bias_batch_stride,+ bias_stride,+ batch_rows,+ }))+ }++ pub fn bias_is_fused(&self) -> bool {+ matches!(self.output_source, OutputSource::Bias)+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rsnew file mode 100644index 000000000..950fc9980--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs@@ -0,0 +1,142 @@+use std::collections::HashMap;++use super::{+ super::matmul_arguments::MatmulArguments,+ dispatch_descriptor::{DispatchDescriptor, OutputSource},+ specialization::Specialization,+};+use crate::{+ DataType,+ backends::common::{Backend, Kernels, kernel::MatmulGemvKernel},+};++pub struct GemvKernel<B: Backend> {+ data_type: DataType,+ pipelines: HashMap<Specialization, <B::Kernels as Kernels>::MatmulGemvKernel>,+}++impl<B: Backend> GemvKernel<B>+where+ B::Error: From<String>,+{+ pub fn new(data_type: DataType) -> Result<Self, B::Error> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(B::Error::from(format!("Unsupported data type for GEMV: {data_type:?}")));+ }+ Ok(Self {+ data_type,+ pipelines: HashMap::new(),+ })+ }++ pub fn precompile(+ &mut self,+ context: &B::Context,+ ) -> Result<(), B::Error> {+ for &config in Specialization::precompile_configs(self.data_type) {+ self.get_or_create_kernel(context, config)?;+ }+ Ok(())+ }++ fn get_or_create_kernel(+ &mut self,+ context: &B::Context,+ config: Specialization,+ ) -> Result<&<B::Kernels as Kernels>::MatmulGemvKernel, B::Error> {+ if !self.pipelines.contains_key(&config) {+ let kernel = <B::Kernels as Kernels>::MatmulGemvKernel::new(+ context,+ self.data_type,+ config.threadgroup_rows,+ config.threadgroup_cols,+ config.threads_per_simdgroup_row,+ config.threads_per_simdgroup_col,+ config.elements_per_thread_row,+ config.elements_per_thread_col,+ config.apply_output_scale_and_accumulate,+ )?;+ self.pipelines.insert(config, kernel);+ }+ Ok(self.pipelines.get(&config).unwrap())+ }++ pub fn encode(+ &mut self,+ context: &B::Context,+ arguments: &MatmulArguments<B>,+ dispatch_descriptor: &DispatchDescriptor,+ encoder: &B::ComputeEncoder,+ ) -> Result<(), B::Error> {+ let config = dispatch_descriptor.specialization;+ let pipeline = self.get_or_create_kernel(context, config)?;++ let (matrix, matrix_offset) = if dispatch_descriptor.matrix_is_rhs {+ (arguments.b, 0usize)+ } else {+ (arguments.a, arguments.a_offset as usize)+ };+ let (input_vector, input_vector_offset) = if dispatch_descriptor.matrix_is_rhs {+ (arguments.a, arguments.a_offset as usize)+ } else {+ (arguments.b, 0usize)+ };++ let output_source = if config.apply_output_scale_and_accumulate {+ match dispatch_descriptor.output_source {+ OutputSource::None => {+ return Err(B::Error::from(+ "GEMV descriptor mismatch: apply_output_scale_and_accumulate=true but output_source=None"+ .to_owned(),+ ));+ },+ OutputSource::Bias => Some(+ arguments.bias.ok_or_else(|| B::Error::from("GEMV descriptor requires bias buffer".to_owned()))?,+ ),+ }+ } else {+ None+ };++ let vector_batch_stride = [i32::try_from(dispatch_descriptor.vector_batch_stride[0]).map_err(|_| {+ B::Error::from(format!(+ "GEMV path requires i32 vector_batch_stride but got {}",+ dispatch_descriptor.vector_batch_stride[0]+ ))+ })?];+ let matrix_batch_stride = [i32::try_from(dispatch_descriptor.matrix_batch_stride[0]).map_err(|_| {+ B::Error::from(format!(+ "GEMV path requires i32 matrix_batch_stride but got {}",+ dispatch_descriptor.matrix_batch_stride[0]+ ))+ })?];+ let output_source_batch_stride = [i32::try_from(dispatch_descriptor.bias_batch_stride[0]).map_err(|_| {+ B::Error::from(format!(+ "GEMV path requires i32 output_source_batch_stride but got {}",+ dispatch_descriptor.bias_batch_stride[0]+ ))+ })?];++ pipeline.encode(+ (matrix, matrix_offset),+ (input_vector, input_vector_offset),+ output_source.map(|buffer| (buffer, 0usize)),+ arguments.d,+ dispatch_descriptor.input_dimension,+ dispatch_descriptor.output_dimension,+ dispatch_descriptor.matrix_leading_dim,+ dispatch_descriptor.alpha,+ dispatch_descriptor.beta,+ &dispatch_descriptor.batch_shape,+ &vector_batch_stride,+ &matrix_batch_stride,+ &output_source_batch_stride,+ dispatch_descriptor.bias_stride,+ dispatch_descriptor.batch_rows,+ config.output_rows_per_threadgroup() as i32,+ encoder,+ );++ Ok(())+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rsnew file mode 100644index 000000000..eba171b69--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs@@ -0,0 +1,6 @@+mod dispatch_descriptor;+mod kernel;+mod specialization;++pub use dispatch_descriptor::DispatchDescriptor;+pub use kernel::GemvKernel;diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rsnew file mode 100644index 000000000..87fca97f1--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rs@@ -0,0 +1,161 @@+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct Specialization {+ pub threadgroup_rows: u32,+ pub threadgroup_cols: u32,+ pub threads_per_simdgroup_row: u32,+ pub threads_per_simdgroup_col: u32,+ pub elements_per_thread_row: u32,+ pub elements_per_thread_col: u32,+ pub apply_output_scale_and_accumulate: bool,+}++impl Specialization {+ pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {+ use crate::DataType;+ match data_type {+ DataType::BF16 => &[+ Self {+ threadgroup_rows: 4,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ apply_output_scale_and_accumulate: false,+ },+ Self {+ threadgroup_rows: 4,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ apply_output_scale_and_accumulate: true,+ },+ Self {+ threadgroup_rows: 8,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ apply_output_scale_and_accumulate: false,+ },+ Self {+ threadgroup_rows: 8,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ apply_output_scale_and_accumulate: true,+ },+ ],+ DataType::F16 => &[Self {+ threadgroup_rows: 8,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ apply_output_scale_and_accumulate: false,+ }],+ DataType::F32 => &[Self {+ threadgroup_rows: 8,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ apply_output_scale_and_accumulate: false,+ }],+ _ => &[],+ }+ }++ pub fn output_rows_per_threadgroup(&self) -> u32 {+ self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row+ }++ pub fn select(+ transpose_matrix: bool,+ input_dimension: i32,+ output_dimension: i32,+ apply_output_scale_and_accumulate: bool,+ ) -> Self {+ let (threadgroup_rows, threadgroup_cols);+ let (threads_per_simdgroup_row, threads_per_simdgroup_col);+ let (elements_per_thread_row, elements_per_thread_col);++ if transpose_matrix {+ let mut simdgroup_thread_rows = 8;+ let mut simdgroup_thread_cols = 4;+ if input_dimension >= 8192 && output_dimension >= 2048 {+ simdgroup_thread_rows = 4;+ simdgroup_thread_cols = 8;+ }++ let threadgroup_simd_cols = if output_dimension >= 2048 {+ 16+ } else if output_dimension >= 512 {+ 4+ } else {+ 2+ };++ let thread_output_cols = if output_dimension < 4 {+ 1+ } else {+ 4+ };++ threadgroup_rows = 1;+ threadgroup_cols = threadgroup_simd_cols;+ threads_per_simdgroup_row = simdgroup_thread_rows;+ threads_per_simdgroup_col = simdgroup_thread_cols;+ elements_per_thread_row = 4;+ elements_per_thread_col = thread_output_cols;+ } else {+ let threadgroup_simd_rows;+ let mut simdgroup_thread_rows = 1;+ let mut simdgroup_thread_cols = 32;+ let mut threadgroup_simd_cols = 1;++ if input_dimension <= 64 {+ threadgroup_simd_rows = 1;+ simdgroup_thread_rows = 8;+ simdgroup_thread_cols = 4;+ } else if input_dimension >= 16 * output_dimension {+ threadgroup_simd_rows = 1;+ threadgroup_simd_cols = 8;+ } else if output_dimension >= 4096 {+ threadgroup_simd_rows = 8;+ } else {+ threadgroup_simd_rows = 4;+ }++ let thread_output_rows = if output_dimension < 4 {+ 1+ } else {+ 4+ };++ threadgroup_rows = threadgroup_simd_rows;+ threadgroup_cols = threadgroup_simd_cols;+ threads_per_simdgroup_row = simdgroup_thread_rows;+ threads_per_simdgroup_col = simdgroup_thread_cols;+ elements_per_thread_row = thread_output_rows;+ elements_per_thread_col = 4;+ }++ Self {+ threadgroup_rows,+ threadgroup_cols,+ threads_per_simdgroup_row,+ threads_per_simdgroup_col,+ elements_per_thread_row,+ elements_per_thread_col,+ apply_output_scale_and_accumulate,+ }+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs b/crates/uzu/src/backends/common/kernel/matmul/grid_size.rsnew file mode 100644index 000000000..cee1e39fb--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs@@ -0,0 +1,6 @@+#[derive(Debug, Clone, Copy)]+pub struct GridSize {+ pub x: usize,+ pub y: usize,+ pub z: usize,+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/kernel.rsnew file mode 100644index 000000000..2473e601a--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs@@ -0,0 +1,146 @@+use super::{+ dispatch_descriptor::MatmulDispatchDescriptor, gemm::GemmKernel, gemv::GemvKernel,+ matmul_arguments::MatmulArguments, split_k::SplitKKernel,+};+use crate::{+ DataType,+ backends::common::{Backend, Kernels, kernel::TensorAddBiasKernel},+};++pub struct MatmulKernel<B: Backend> {+ pub(crate) data_type: DataType,+ gemm: Option<GemmKernel<B>>,+ gemv: Option<GemvKernel<B>>,+ splitk: Option<SplitKKernel<B>>,+ bias_add: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,+}++impl<B: Backend> MatmulKernel<B>+where+ B::Error: From<String>,+{+ pub fn new(data_type: DataType) -> Result<Self, B::Error> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(B::Error::from(format!("Unsupported dtype for MatmulKernel: {data_type:?}")));+ }++ Ok(Self {+ data_type,+ gemm: None,+ gemv: None,+ splitk: None,+ bias_add: None,+ })+ }++ pub fn precompile(+ &mut self,+ context: &B::Context,+ ) -> Result<(), B::Error> {+ let gemm = self.get_or_create_gemm()?;+ gemm.precompile(context)?;++ let gemv = self.get_or_create_gemv()?;+ gemv.precompile(context)?;++ let splitk = self.get_or_create_splitk()?;+ splitk.precompile(context)?;++ Ok(())+ }++ fn get_or_create_gemm(&mut self) -> Result<&mut GemmKernel<B>, B::Error> {+ if self.gemm.is_none() {+ self.gemm = Some(GemmKernel::<B>::new(self.data_type)?);+ }+ Ok(self.gemm.as_mut().unwrap())+ }++ fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel<B>, B::Error> {+ if self.gemv.is_none() {+ self.gemv = Some(GemvKernel::<B>::new(self.data_type)?);+ }+ Ok(self.gemv.as_mut().unwrap())+ }++ fn get_or_create_splitk(&mut self) -> Result<&mut SplitKKernel<B>, B::Error> {+ if self.splitk.is_none() {+ self.splitk = Some(SplitKKernel::<B>::new(self.data_type)?);+ }+ Ok(self.splitk.as_mut().unwrap())+ }++ fn encode_dispatch_descriptor(+ &mut self,+ context: &B::Context,+ arguments: &MatmulArguments<B>,+ dispatch_descriptor: &MatmulDispatchDescriptor,+ encoder: &B::ComputeEncoder,+ ) -> Result<(), B::Error> {+ match dispatch_descriptor {+ MatmulDispatchDescriptor::Gemv(d) => {+ let gemv = self.get_or_create_gemv()?;+ gemv.encode(context, arguments, d, encoder)+ },+ MatmulDispatchDescriptor::SplitK(d) => {+ let splitk = self.get_or_create_splitk()?;+ splitk.encode(context, arguments, d, encoder)+ },+ MatmulDispatchDescriptor::Gemm(d) => {+ let gemm = self.get_or_create_gemm()?;+ gemm.encode(context, arguments, d, encoder)+ },+ }+ }++ pub fn encode_with_descriptor(+ &mut self,+ context: &B::Context,+ arguments: MatmulArguments<B>,+ dispatch_descriptor: &MatmulDispatchDescriptor,+ encoder: &B::ComputeEncoder,+ ) -> Result<(), B::Error> {+ self.encode_dispatch_descriptor(context, &arguments, dispatch_descriptor, encoder)?;++ if let Some(bias) = arguments.bias {+ if !dispatch_descriptor.bias_is_fused() {+ self.apply_bias_add(context, &arguments, bias, encoder)?;+ }+ }++ Ok(())+ }++ fn apply_bias_add(+ &mut self,+ context: &B::Context,+ arguments: &MatmulArguments<B>,+ bias: &B::NativeBuffer,+ encoder: &B::ComputeEncoder,+ ) -> Result<(), B::Error> {+ let m = arguments.batch as usize;+ let n = arguments.output_dim as usize;+ let batch_count = arguments.batch_count as usize;+ let total_len = m * n * batch_count;+ if total_len == 0 {+ return Ok(());+ }++ if self.bias_add.is_none() {+ self.bias_add = Some(<B::Kernels as Kernels>::TensorAddBiasKernel::new(context, self.data_type)?);+ }+ let bias_add = self.bias_add.as_ref().unwrap();+ bias_add.encode(arguments.d, bias, arguments.d, n as u32, total_len as u32, encoder);+ Ok(())+ }++ pub fn apply_batch_collapse(arguments: &mut MatmulArguments<B>) {+ if arguments.batch_count <= 1 {+ return;+ }+ if arguments.lda == arguments.input_dim && arguments.transpose_b {+ arguments.batch *= arguments.batch_count;+ arguments.batch_count = 1;+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rssimilarity index 50%rename from crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rsrename to crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rsindex b01d3cd5a..22ac9571f 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs@@ -1,15 +1,13 @@-use metal::MTLBuffer;-use objc2::{rc::Retained, runtime::ProtocolObject};+use crate::backends::common::Backend;#[derive(Debug, Clone)]-pub struct MatmulArguments<'a> {- pub a: &'a Retained<ProtocolObject<dyn MTLBuffer>>,+pub struct MatmulArguments<'a, B: Backend> {+ pub a: &'a B::NativeBuffer,/// Byte offset into `a` (used for slicing the batch dimension).pub a_offset: u64,- pub b: &'a Retained<ProtocolObject<dyn MTLBuffer>>,- pub c: Option<&'a Retained<ProtocolObject<dyn MTLBuffer>>>,- pub d: &'a Retained<ProtocolObject<dyn MTLBuffer>>,- pub bias: Option<&'a Retained<ProtocolObject<dyn MTLBuffer>>>,+ pub b: &'a B::NativeBuffer,+ pub d: &'a B::NativeBuffer,+ pub bias: Option<&'a B::NativeBuffer>,/// M dimension - batch/number of tokens (rows of A, rows of D)pub batch: i32,/// K dimension - input_dim/reduction dimension (cols of A, rows of B)@@ -21,9 +19,5 @@ pub struct MatmulArguments<'a> {pub ldd: i32,/// Number of batched matrix multiplications (z-dimension)pub batch_count: i32,- /// Scaling factors for fused addmm (alpha * A @ B + beta * C)- pub alpha: f32,- pub beta: f32,- pub transpose_a: bool,pub transpose_b: bool,}diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rsindex bc496f181..41fe391d7 100644--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs@@ -1,6 +1,17 @@+mod dispatch_descriptor;mod full_precision;+pub mod gemm;+pub mod gemv;+mod grid_size;+mod kernel;+mod matmul_arguments;+pub mod split_k;+pub use dispatch_descriptor::MatmulDispatchDescriptor;pub use full_precision::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel};+pub use grid_size::GridSize;+pub use kernel::MatmulKernel;+pub use matmul_arguments::MatmulArguments;use super::Kernels;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rssimilarity index 59%rename from crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rsrename to crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rsindex 21c0e7523..11cd3d489 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs@@ -1,39 +1,29 @@-use metal::MTLSize;--use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};+use super::{+ super::{grid_size::GridSize, matmul_arguments::MatmulArguments},+ specialization::Specialization,+ tile_configuration::{TileConfiguration, select_tile_configuration},+};use crate::{DataType,- backends::metal::{- context::MetalContext,- error::MetalError,- kernel::matmul::common::{GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments},- },+ backends::common::{Backend, gpu_types::GEMMSpiltKParams as SplitKGEMMParams},};#[derive(Debug, Clone)]-pub(crate) struct DispatchDescriptor {- pub(crate) pipeline_configuration: PipelineConfiguration,- pub(crate) params: SplitKGEMMParams,- pub(crate) partition_count: i32,- pub(crate) output_elements_per_partition: i32,- pub(crate) accumulator_bytes: usize,- pub(crate) partial_threadgroups: MTLSize,- pub(crate) partial_threads_per_threadgroup: MTLSize,- pub(crate) accum_total_threads: MTLSize,- pub(crate) accum_threads_per_threadgroup: MTLSize,+pub struct DispatchDescriptor {+ pub params: SplitKGEMMParams,+ pub partition_count: i32,+ pub output_elements_per_partition: i32,+ pub accumulator_bytes: usize,+ pub partial_threadgroups: GridSize,+ pub accum_total_threads: GridSize,}impl DispatchDescriptor {- pub(crate) fn try_new(- _context: &MetalContext,+ pub fn try_new<B: Backend>(data_type: DataType,- arguments: &MatmulArguments,- ) -> Result<Option<Self>, MetalError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MetalError::Generic(format!("Unsupported dtype for Split-K: {:?}", data_type)));- }-- if arguments.c.is_some() {+ arguments: &MatmulArguments<B>,+ ) -> Result<Option<Self>, B::Error> {+ if !matches!(data_type, DataType::BF16) {return Ok(None);}@@ -51,14 +41,17 @@ impl DispatchDescriptor {let mn_aligned = (m % tile.tile_rows) == 0 && (n % tile.tile_cols) == 0;let k_aligned = (k % tile.tile_depth) == 0;- let pipeline_configuration = PipelineConfiguration {+ let specialization = Specialization {tile,- transpose_a: arguments.transpose_a,transpose_b: arguments.transpose_b,mn_aligned,k_aligned,};+ if !is_supported_specialization(&specialization) {+ return Ok(None);+ }+let tile_count_m = (m + tile.tile_rows - 1) / tile.tile_rows;let tile_count_n = (n + tile.tile_cols - 1) / tile.tile_cols;@@ -84,23 +77,25 @@ impl DispatchDescriptor {gemm_k_iterations_aligned: gemm_k_iterations,};- let partial_threads_per_threadgroup =- MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);- let partial_threadgroups = MTLSize::new(tile_count_n as usize, tile_count_m as usize, partition_count as usize);+ let partial_threadgroups = GridSize {+ x: tile_count_n as usize,+ y: tile_count_m as usize,+ z: partition_count as usize,+ };- let accum_total_threads = MTLSize::new(n as usize, m as usize, 1);- let accum_threads_per_threadgroup = MTLSize::new(16.min(n as usize), 16.min(m as usize), 1);+ let accum_total_threads = GridSize {+ x: n as usize,+ y: m as usize,+ z: 1,+ };Ok(Some(Self {- pipeline_configuration,params,partition_count,output_elements_per_partition,accumulator_bytes,partial_threadgroups,- partial_threads_per_threadgroup,accum_total_threads,- accum_threads_per_threadgroup,}))}}@@ -135,3 +130,15 @@ fn should_use_splitk(let k_tiles = k / 16;(m_tiles * n_tiles) <= 32 && k_tiles >= 8}++fn is_supported_specialization(config: &Specialization) -> bool {+ let supported_tile = TileConfiguration {+ tile_rows: 16,+ tile_cols: 32,+ tile_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ };++ config.tile == supported_tile && config.transpose_b && !config.mn_aligned && config.k_aligned+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rsnew file mode 100644index 000000000..33b31a032--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs@@ -0,0 +1,142 @@+use super::{super::matmul_arguments::MatmulArguments, dispatch_descriptor::DispatchDescriptor};+use crate::{+ DataType,+ backends::common::{+ Backend, Context, Kernels,+ kernel::{MatmulSplitKAccumBfloat16Kernel, MatmulSplitKPartialBfloat16Kernel},+ },+};++pub struct SplitKKernel<B: Backend> {+ data_type: DataType,+ partial_bfloat16: Option<<B::Kernels as Kernels>::MatmulSplitKPartialBfloat16Kernel>,+ accum_bfloat16: Option<<B::Kernels as Kernels>::MatmulSplitKAccumBfloat16Kernel>,+ accumulator_buffer: Option<B::NativeBuffer>,+ accumulator_buffer_bytes: usize,+}++impl<B: Backend> SplitKKernel<B>+where+ B::Error: From<String>,+{+ pub fn new(data_type: DataType) -> Result<Self, B::Error> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(B::Error::from(format!("Unsupported dtype for Split-K: {data_type:?}")));+ }+ Ok(Self {+ data_type,+ partial_bfloat16: None,+ accum_bfloat16: None,+ accumulator_buffer: None,+ accumulator_buffer_bytes: 0,+ })+ }++ pub fn precompile(+ &mut self,+ context: &B::Context,+ ) -> Result<(), B::Error> {+ if !matches!(self.data_type, DataType::BF16) {+ return Ok(());+ }+ self.ensure_kernels(context)+ }++ fn ensure_kernels(+ &mut self,+ context: &B::Context,+ ) -> Result<(), B::Error> {+ if !matches!(self.data_type, DataType::BF16) {+ return Err(B::Error::from(format!("Split-K currently supports BF16 only, got {:?}", self.data_type)));+ }+ if self.partial_bfloat16.is_none() {+ self.partial_bfloat16 = Some(<B::Kernels as Kernels>::MatmulSplitKPartialBfloat16Kernel::new(context)?);+ }+ if self.accum_bfloat16.is_none() {+ self.accum_bfloat16 = Some(<B::Kernels as Kernels>::MatmulSplitKAccumBfloat16Kernel::new(context)?);+ }+ Ok(())+ }++ pub fn encode(+ &mut self,+ context: &B::Context,+ arguments: &MatmulArguments<B>,+ dispatch_descriptor: &DispatchDescriptor,+ encoder: &B::ComputeEncoder,+ ) -> Result<(), B::Error> {+ self.ensure_kernels(context)?;+ self.ensure_accumulator_buffer(context, dispatch_descriptor.accumulator_bytes)?;+ let accumulator_buffer =+ self.accumulator_buffer.as_ref().cloned().expect("Accumulator buffer must be initialized");++ let partial_group_count_x = u32::try_from(dispatch_descriptor.partial_threadgroups.x).map_err(|_| {+ B::Error::from(format!(+ "Split-K partial group count x overflows u32: {}",+ dispatch_descriptor.partial_threadgroups.x+ ))+ })?;+ let partial_group_count_y = u32::try_from(dispatch_descriptor.partial_threadgroups.y).map_err(|_| {+ B::Error::from(format!(+ "Split-K partial group count y overflows u32: {}",+ dispatch_descriptor.partial_threadgroups.y+ ))+ })?;+ let partial_group_count_z = u32::try_from(dispatch_descriptor.partial_threadgroups.z).map_err(|_| {+ B::Error::from(format!(+ "Split-K partial group count z overflows u32: {}",+ dispatch_descriptor.partial_threadgroups.z+ ))+ })?;+ let partial = self.partial_bfloat16.as_ref().unwrap();+ partial.encode(+ (arguments.a, arguments.a_offset as usize),+ arguments.b,+ &accumulator_buffer,+ std::slice::from_ref(&dispatch_descriptor.params),+ partial_group_count_x,+ partial_group_count_y,+ partial_group_count_z,+ encoder,+ );++ let accum_total_threads_x = u32::try_from(dispatch_descriptor.accum_total_threads.x).map_err(|_| {+ B::Error::from(format!(+ "Split-K accum total threads x overflows u32: {}",+ dispatch_descriptor.accum_total_threads.x+ ))+ })?;+ let accum_total_threads_y = u32::try_from(dispatch_descriptor.accum_total_threads.y).map_err(|_| {+ B::Error::from(format!(+ "Split-K accum total threads y overflows u32: {}",+ dispatch_descriptor.accum_total_threads.y+ ))+ })?;+ let accum = self.accum_bfloat16.as_ref().unwrap();+ accum.encode(+ &accumulator_buffer,+ arguments.d,+ dispatch_descriptor.partition_count,+ dispatch_descriptor.output_elements_per_partition,+ arguments.ldd,+ accum_total_threads_x,+ accum_total_threads_y,+ encoder,+ );++ Ok(())+ }++ fn ensure_accumulator_buffer(+ &mut self,+ context: &B::Context,+ required_bytes: usize,+ ) -> Result<(), B::Error> {+ if required_bytes <= self.accumulator_buffer_bytes && self.accumulator_buffer.is_some() {+ return Ok(());+ }+ self.accumulator_buffer = Some(context.create_buffer(required_bytes)?);+ self.accumulator_buffer_bytes = required_bytes;+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rsnew file mode 100644index 000000000..0fcefe437--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs@@ -0,0 +1,7 @@+mod dispatch_descriptor;+mod kernel;+mod specialization;+mod tile_configuration;++pub use dispatch_descriptor::DispatchDescriptor;+pub use kernel::SplitKKernel;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/specialization.rssimilarity index 77%rename from crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rsrename to crates/uzu/src/backends/common/kernel/matmul/split_k/specialization.rsindex 1eab77254..dcd6b7e60 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/specialization.rs@@ -1,9 +1,8 @@use super::tile_configuration::TileConfiguration;#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct PipelineConfiguration {+pub struct Specialization {pub tile: TileConfiguration,- pub transpose_a: bool,pub transpose_b: bool,pub mn_aligned: bool,pub k_aligned: bool,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/tile_configuration.rssimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rsrename to crates/uzu/src/backends/common/kernel/matmul/split_k/tile_configuration.rsdiff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex 87dadbc6c..ca3e45905 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -156,18 +156,6 @@ impl MetalContext {}pub fn compute_pipeline_state(- &self,- function_name: &str,- constants: Option<&MTLFunctionConstantValues>,- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {- // Only cache pipelines without constants- if constants.is_some() {- return self.library.compute_pipeline_state(function_name, constants);- }- self.compute_pipeline_state_cached(function_name, function_name, None)- }-- pub fn compute_pipeline_state_cached(&self,cache_key: &str,function_name: &str,diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.hindex 0fe96df40..db0130600 100644--- a/crates/uzu/src/backends/metal/generated/matmul.h+++ b/crates/uzu/src/backends/metal/generated/matmul.h@@ -48,32 +48,6 @@ typedef struct {int32_t gemm_k_iterations_aligned;} GEMMSpiltKParams;-/**Split-K MLP Fused GEMM parameters. */-typedef struct {- int32_t M;- int32_t N;- int32_t K;- int32_t lda;- int32_t ldb;- int32_t ldc;- int32_t tiles_n;- int32_t tiles_m;- int32_t split_k_partitions;- int32_t split_k_partition_stride;- int32_t split_k_partition_size;- int32_t gemm_k_iterations_aligned;- int32_t hidden_dim;-} GEMMSpiltKMlpFusedParams;--/**AddMM parameters (alpha * A @ B + beta * C). */-typedef struct {- int32_t ldc;- int32_t fdc;- int64_t batch_stride_c;- float alpha;- float beta;-} GEMMAddMMParams;-#ifdef __METAL_VERSION__} // namespace matmul} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rsindex 6751a3863..3ce63cb23 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs@@ -1,7 +1 @@-mod matmul_arguments;-mod transpose_configuration;--pub use matmul_arguments::MatmulArguments;-pub(crate) use transpose_configuration::transpose_configuration;--pub use crate::backends::common::gpu_types::{GEMMAddMMParams, GEMMParams, GEMMSpiltKMlpFusedParams, GEMMSpiltKParams};+pub use crate::backends::common::gpu_types::{GEMMParams, GEMMSpiltKParams};diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.hindex bb8c8b8b2..40056c256 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h@@ -4,294 +4,74 @@#include "mma.h"#include "params.h"#include "transforms.h"-#include "../utils.h"using namespace metal;///////////////////////////////////////////////////////////////////////////////-// GEMM kernel class+// GEMM kernel helpers///////////////////////////////////////////////////////////////////////////////namespace steel {-template <bool M_aligned, bool N_aligned, bool K_aligned>-struct LoopAlignment {};-template <typename T,typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- bool MN_aligned,- bool K_aligned,- typename AccumType = typename AccumHelper<T>::accum_type,+ typename AccumType = float,typename Epilogue = TransformNone<U, AccumType>>-struct GEMMKernel {- STEEL_CONST short tgp_padding_a = 16 / sizeof(T);- STEEL_CONST short tgp_padding_b = 16 / sizeof(T);- STEEL_CONST short tgp_mem_size_a =- transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);- STEEL_CONST short tgp_mem_size_b =- transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b);- STEEL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;-- STEEL_CONST short tgp_size = WM * WN * 32;-- using loader_a_t = BlockLoader<- T,- transpose_a ? BK : BM,- transpose_a ? BM : BK,- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,- !transpose_a,- tgp_size>;- using loader_b_t = BlockLoader<- T,- transpose_b ? BN : BK,- transpose_b ? BK : BN,- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,- transpose_b,- tgp_size>;- using mma_t = BlockMMA<- T,- U,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,- AccumType,- Epilogue>;-- /* Main kernel function */- template <bool M_aligned, bool N_aligned, bool K_aligned_>- static METAL_FUNC void gemm_loop(- threadgroup T* As [[threadgroup(0)]],- threadgroup T* Bs [[threadgroup(1)]],- const int gemm_k_iterations,- thread loader_a_t& loader_a,- thread loader_b_t& loader_b,- thread mma_t& mma_op,- thread const short& tgp_bm,- thread const short& tgp_bn,- thread const short& lbk,- LoopAlignment<M_aligned, N_aligned, K_aligned_> l = {}- ) {- // Appease the compiler- (void)l;-- short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);-- short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);-- for (int k = 0; k < gemm_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- // Load elements into threadgroup- if (M_aligned) {- loader_a.load_unsafe();- } else {- loader_a.load_safe(tile_dims_A);- }-- if (N_aligned) {- loader_b.load_unsafe();- } else {- loader_b.load_safe(tile_dims_B);- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();+METAL_FUNC void gemm_loop(+ threadgroup T* As,+ threadgroup T* Bs,+ const int gemm_k_iterations,+ thread BlockLoader<T>& loader_a,+ thread BlockLoader<T>& loader_b,+ thread BlockMMA<T, U, AccumType, Epilogue>& mma_op,+ thread const short& tgp_bm,+ thread const short& tgp_bn,+ thread const short& lbk,+ const short BK,+ const bool transpose_a,+ const bool transpose_b,+ const bool align_m,+ const bool align_n,+ const bool align_k+) {+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);+ short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);++ for (int k = 0; k < gemm_k_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (align_m) {+ loader_a.load_unsafe();+ } else {+ loader_a.load_safe(tile_dims_A);}- if (!K_aligned_) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- short2 tile_dims_A_last =- transpose_a ? short2(tgp_bm, lbk) : short2(lbk, tgp_bm);- short2 tile_dims_B_last =- transpose_b ? short2(lbk, tgp_bn) : short2(tgp_bn, lbk);-- loader_a.load_safe(tile_dims_A_last);- loader_b.load_safe(tile_dims_B_last);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_op.mma(As, Bs);+ if (align_n) {+ loader_b.load_unsafe();+ } else {+ loader_b.load_safe(tile_dims_B);}- }-- /* Main kernel function */- static METAL_FUNC void run(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- device U* D [[buffer(2)]],- const constant GEMMParams* params [[buffer(3)]],- threadgroup T* As [[threadgroup(0)]],- threadgroup T* Bs [[threadgroup(1)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]]- ) {- // Pacifying compiler- (void)lid;-- const int tid_y = ((tid.y) << params->swizzle_log) +- ((tid.x) & ((1 << params->swizzle_log) - 1));- const int tid_x = (tid.x) >> params->swizzle_log;-- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {- return;- }-- threadgroup_barrier(mem_flags::mem_none);-- // Find block in A, B, C- const int c_row = tid_y * BM;- const int c_col = tid_x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);- A += transpose_a ? c_row_long : c_row_long * params->lda;- B += transpose_b ? c_col_long * params->ldb : c_col_long;- D += c_row_long * params->ldd + c_col_long;-- // Prepare threadgroup loading operations- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);-- // Prepare threadgroup mma operation- thread mma_t mma_op(simd_group_id, simd_lane_id);-- int gemm_k_iterations = params->gemm_k_iterations_aligned;-- ///////////////////////////////////////////////////////////////////////////////- // MNK aligned loop- if (MN_aligned) {- for (int k = 0; k < gemm_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- // Load elements into threadgroup- loader_a.load_unsafe();- loader_b.load_unsafe();-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }-- threadgroup_barrier(mem_flags::mem_none);-- // Loop tail- if (!K_aligned) {- int lbk = params->K - params->gemm_k_iterations_aligned * BK;- short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);- short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);-- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_op.mma(As, Bs);- }-- // Store results to device memory- mma_op.store_result(D, params->ldd);- return;-- }- ///////////////////////////////////////////////////////////////////////////////- // MN unaligned loop- else { // Loop over K - unaligned case- short tgp_bm = min(BM, params->M - c_row);- short tgp_bn = min(BN, params->N - c_col);- short leftover_bk = params->K - params->gemm_k_iterations_aligned * BK;-- if (tgp_bm == BM && tgp_bn == BN) {- gemm_loop<true, true, K_aligned>(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk- );-- mma_op.store_result(D, params->ldd);- return;-- } else if (tgp_bn == BN) {- gemm_loop<false, true, K_aligned>(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk- );-- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));- return;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(As, Bs);+ loader_a.next();+ loader_b.next();+ }- } else if (tgp_bm == BM) {- gemm_loop<true, false, K_aligned>(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk- );+ if (!align_k) {+ threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));- return;+ short2 tile_dims_A_last =+ transpose_a ? short2(tgp_bm, lbk) : short2(lbk, tgp_bm);+ short2 tile_dims_B_last =+ transpose_b ? short2(lbk, tgp_bn) : short2(tgp_bn, lbk);- } else {- gemm_loop<false, false, K_aligned>(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk- );+ loader_a.load_safe(tile_dims_A_last);+ loader_b.load_safe(tile_dims_B_last);- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));- return;- }- }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(As, Bs);}-};+}} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.hdeleted file mode 100644index 39f560249..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.h+++ /dev/null@@ -1,159 +0,0 @@---#pragma once--#include "nax.h"-#include "params.h"-#include "transforms.h"-#include "../utils.h"--using namespace metal;--namespace steel {--template <- typename T,- short SM,- short SN,- short SK,- short BK,- bool transpose_a,- bool transpose_b,- bool kAlignedM,- bool kAlignedN,- bool kAlignedK,- short UM,- short UN,- short UK,- typename AccumType = float>-auto gemm_loop(- const device T* A,- const device T* B,- const constant GEMMParams* params [[buffer(4)]],- const short sgp_sm,- const short sgp_sn-) {- constexpr short TM = SM / UM;- constexpr short TN = SN / UN;- constexpr short TK = SK / UK;-- constexpr int RA = transpose_a ? TK : TM;- constexpr int CA = transpose_a ? TM : TK;-- constexpr int RB = transpose_b ? TN : TK;- constexpr int CB = transpose_b ? TK : TN;-- using DSubTile = NAXSubTile<AccumType, UM, UN>;- using ASubTile =- NAXSubTile<T, (transpose_a ? UK : UM), (transpose_a ? UM : UK)>;- using BSubTile =- NAXSubTile<T, (transpose_b ? UN : UK), (transpose_b ? UK : UN)>;-- NAXTile<AccumType, TM, TN, DSubTile> Dtile;- Dtile.clear();-- int gemm_k_iterations_ = params->gemm_k_iterations_aligned;-- STEEL_PRAGMA_NO_UNROLL- for (int kk0 = 0; kk0 < gemm_k_iterations_; kk0++) {- threadgroup_barrier(mem_flags::mem_none);-- STEEL_PRAGMA_NO_UNROLL- for (int kk1 = 0; kk1 < BK; kk1 += SK) {- NAXTile<T, RA, CA, ASubTile> Atile;- NAXTile<T, RB, CB, BSubTile> Btile;- const int k = kk1;-- volatile int compiler_barrier;-- const int A_offset = transpose_a ? k * params->lda : k;- const int B_offset = transpose_b ? k : k * params->ldb;-- if constexpr (kAlignedM) {- Atile.load(A + A_offset, params->lda);- } else {- const short rmax = transpose_a ? SK : sgp_sm;- const short cmax = transpose_a ? sgp_sm : SK;- Atile.load_safe(A + A_offset, params->lda, short2(cmax, rmax));- }-- if constexpr (kAlignedN) {- Btile.load(B + B_offset, params->ldb);- } else {- const short rmax = transpose_b ? sgp_sn : SK;- const short cmax = transpose_b ? SK : sgp_sn;- Btile.load_safe(B + B_offset, params->ldb, short2(cmax, rmax));- }-- tile_matmad_nax(- Dtile,- Atile,- metal::bool_constant<transpose_a>{},- Btile,- metal::bool_constant<transpose_b>{}- );-- (void)compiler_barrier;- }-- A += transpose_a ? (BK * params->lda) : BK;- B += transpose_b ? BK : (BK * params->ldb);- }-- if constexpr (!kAlignedK) {- simdgroup_barrier(mem_flags::mem_none);-- const short rem_bk = params->K - gemm_k_iterations_ * BK;-- STEEL_PRAGMA_NO_UNROLL- for (int kk1 = 0; kk1 < rem_bk; kk1 += SK) {- NAXTile<T, 1, 1, ASubTile> Atile;- NAXTile<T, 1, 1, BSubTile> Btile;-- STEEL_PRAGMA_UNROLL- for (int mm = 0; mm < TM; mm++) {- STEEL_PRAGMA_UNROLL- for (int nn = 0; nn < TN; nn++) {- STEEL_PRAGMA_UNROLL- for (int kk = 0; kk < TK; kk++) {- const int m = mm * UM;- const int n = nn * UN;- const int k = kk1 + kk * UK;- const short psk = max(0, rem_bk - k);-- const int A_offset =- transpose_a ? (m + k * params->lda) : (m * params->lda + k);- const int B_offset =- transpose_b ? (k + n * params->ldb) : (k * params->ldb + n);-- {- const short psm = kAlignedM ? SM : max(0, sgp_sm - m);- const short rmax = transpose_a ? psk : psm;- const short cmax = transpose_a ? psm : psk;- Atile.load_safe(A + A_offset, params->lda, short2(cmax, rmax));- }-- {- const short psn = kAlignedN ? SN : max(0, sgp_sn - n);- const short rmax = transpose_b ? psn : psk;- const short cmax = transpose_b ? psk : psn;- Btile.load_safe(B + B_offset, params->ldb, short2(cmax, rmax));- }-- subtile_matmad_nax(- Dtile.subtile_at(mm, nn),- Atile.subtile_at(0, 0),- metal::bool_constant<transpose_a>{},- Btile.subtile_at(0, 0),- metal::bool_constant<transpose_b>{}- );- }- }- }- }- }-- return Dtile;-}--} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.hindex 7aab96e78..4aa7cc56c 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h@@ -10,27 +10,19 @@namespace steel {-template <- typename T,- short BROWS,- short BCOLS,- short dst_ld,- short reduction_dim,- short tgp_size,- short alignment = 1,- short n_reads = (BCOLS * BROWS) / (tgp_size),- short TCOLS = BCOLS / n_reads,- short TROWS = tgp_size / TCOLS>+template <typename T>struct BlockLoader {- STEEL_CONST short n_rows = (BROWS + TROWS - 1) / TROWS;- STEEL_CONST short vec_size = n_reads;+ // Tile params+ const short BROWS;+ const short dst_ld;+ const short vec_size;+ const short TROWS;// Leading dimension for srcconst int src_ld;const int tile_stride;// Thread location indices- const short thread_idx;const short bi;const short bj;@@ -38,41 +30,35 @@ struct BlockLoader {threadgroup T* dst;const device T* src;- struct alignas(alignment * sizeof(T)) ReadVector {- uint8_t v[sizeof(T) * vec_size];- };-/* Constructor */METAL_FUNC BlockLoader(const device T* src_,const int src_ld_,threadgroup T* dst_,- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]+ ushort simd_group_id,+ ushort simd_lane_id,+ short BROWS_,+ short BCOLS_,+ short dst_ld_,+ short reduction_dim,+ short tgp_size)- : src_ld(src_ld_), tile_stride(reduction_dim ? BCOLS : BROWS * src_ld),- thread_idx(simd_group_id * 32 + simd_lane_id), bi(thread_idx / TCOLS),- bj(vec_size * (thread_idx % TCOLS)), dst(dst_ + bi * dst_ld + bj),- src(src_ + bi * src_ld + bj) {}-- /* Apply operation to threadgroup without bound checking */- template <typename UnaryOp>- METAL_FUNC void apply_inplace_op(thread const UnaryOp& op) const {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < BROWS; i += TROWS) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- dst[i * dst_ld + j] = op.apply(dst[i * dst_ld + j]);- }- }- }+ : BROWS(BROWS_), dst_ld(dst_ld_), vec_size((BCOLS_ * BROWS_) / tgp_size),+ TROWS(tgp_size / (BCOLS_ / ((BCOLS_ * BROWS_) / tgp_size))),+ src_ld(src_ld_), tile_stride(reduction_dim ? BCOLS_ : BROWS_ * src_ld_),+ bi(short(simd_group_id * 32 + simd_lane_id) /+ short(BCOLS_ / ((BCOLS_ * BROWS_) / tgp_size))),+ bj(short((BCOLS_ * BROWS_) / tgp_size) *+ (short(simd_group_id * 32 + simd_lane_id) %+ short(BCOLS_ / ((BCOLS_ * BROWS_) / tgp_size)))),+ dst(dst_ + bi * dst_ld_ + bj), src(src_ + bi * src_ld_ + bj) {}/* Load from device memory into threadgroup memory - without bound checking */METAL_FUNC void load_unsafe() const {- STEEL_PRAGMA_UNROLLfor (short i = 0; i < BROWS; i += TROWS) {- *((threadgroup ReadVector*)(&dst[i * dst_ld])) =- *((const device ReadVector*)(&src[i * src_ld]));+ for (short j = 0; j < vec_size; j++) {+ dst[i * dst_ld + j] = src[i * src_ld + j];+ }}}@@ -80,11 +66,8 @@ struct BlockLoader {METAL_FUNC void load_safe(short2 src_tile_dim) const {src_tile_dim = src_tile_dim - short2(bj, bi);- // Skip loading if thread has no valid readsif (src_tile_dim.x <= 0 || src_tile_dim.y <= 0) {- STEEL_PRAGMA_UNROLLfor (short i = 0; i < BROWS; i += TROWS) {- STEEL_PRAGMA_UNROLLfor (short j = 0; j < vec_size; j++) {dst[i * dst_ld + j] = T(0);}@@ -92,34 +75,11 @@ struct BlockLoader {return;}- // Use fast thread memory for bound checks- bool tmp_idx[vec_size];- T tmp_val[vec_size];-- STEEL_PRAGMA_UNROLLfor (short i = 0; i < BROWS; i += TROWS) {- // Make sure tmp_idx only contains valid indices- STEEL_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- tmp_idx[j] = (i < src_tile_dim.y) && (j < src_tile_dim.x);- }-- // Read valid indices into tmp_val- STEEL_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- tmp_val[j] = src[(tmp_idx[j] ? i * src_ld + j : 0)];- }-- // Zero out unneeded values- STEEL_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- tmp_val[j] = tmp_idx[j] ? tmp_val[j] : T(0);- }-- // Copy values to threadgroup memory- STEEL_PRAGMA_UNROLLfor (short j = 0; j < vec_size; j++) {- dst[i * dst_ld + j] = tmp_val[j];+ bool valid = (i < src_tile_dim.y) && (j < src_tile_dim.x);+ T val = src[(valid ? i * src_ld + j : 0)];+ dst[i * dst_ld + j] = valid ? val : T(0);}}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.hindex 3bf58387a..989b22008 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h@@ -7,8 +7,8 @@#include <metal_stdlib>#include "../defines.h"+#include "../utils/type_traits.h"#include "transforms.h"-#include "../utils/integral_constant.h"using namespace metal;@@ -18,20 +18,16 @@ using namespace metal;namespace steel {-template <typename T, int kFragRows_, int kFragCols_>-struct BaseMMAFrag {- static_assert(- kFragRows_ == 8,- "Only 8 x 8 fragment matrices are currently supported"- );- static_assert(- kFragCols_ == 8,- "Only 8 x 8 fragment matrices are currently supported"- );-};+// Max tile dimensions across all configurations.+// GEMM: TM up to 8 (64/(8*1)), TN up to 4 (64/(8*2))+// Split-K: TM up to 1, TN up to 2+// tile_matmad K dim is always 1+#define STEEL_MAX_TM 8+#define STEEL_MAX_TN 4+#define STEEL_MAX_CTILE_FRAGS (STEEL_MAX_TM * STEEL_MAX_TN) // 32template <typename T>-struct BaseMMAFrag<T, 8, 8> {+struct BaseMMAFrag {STEEL_CONST int kFragRows = 8;STEEL_CONST int kFragCols = 8;@@ -40,11 +36,6 @@ struct BaseMMAFrag<T, 8, 8> {STEEL_CONST int kElemRows = 1;STEEL_CONST int kElemCols = 2;- static_assert(- kElemRows * kElemCols == kElemsPerFrag,- "MMAFrag shape is not consistent with MMAFrag size"- );-typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;typedef metal::vec<T, kElemsPerFrag> frag_type;@@ -57,12 +48,12 @@ struct BaseMMAFrag<T, 8, 8> {return short2{fn, fm};}- template <typename SrcPtrType, typename StrX, typename StrY>+ template <typename SrcPtrType>METAL_FUNC static constexpr void load(thread frag_type& dst,SrcPtrType src,- StrX str_x,- StrY str_y+ int str_x,+ int str_y) {STEEL_PRAGMA_UNROLLfor (short i = 0; i < kElemRows; i++) {@@ -73,23 +64,16 @@ struct BaseMMAFrag<T, 8, 8> {}}- template <- typename SrcPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX,- typename OffY>+ template <typename SrcPtrType>METAL_FUNC static constexpr void load_safe(thread frag_type& dst,SrcPtrType src,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = Int<0>{},- OffY off_y = Int<0>{}+ int str_x,+ int str_y,+ int lim_x,+ int lim_y,+ int off_x = 0,+ int off_y = 0) {STEEL_PRAGMA_UNROLLfor (short i = 0; i < kElemRows; i++) {@@ -105,12 +89,12 @@ struct BaseMMAFrag<T, 8, 8> {}}- template <typename DstPtrType, typename StrX, typename StrY>+ template <typename DstPtrType>METAL_FUNC static constexpr void store(const thread frag_type& src,DstPtrType dst,- StrX str_x,- StrY str_y+ int str_x,+ int str_y) {using U = pointer_element_t<DstPtrType>;@@ -123,23 +107,16 @@ struct BaseMMAFrag<T, 8, 8> {}}- template <- typename DstPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX,- typename OffY>+ template <typename DstPtrType>METAL_FUNC static constexpr void store_safe(const thread frag_type& src,DstPtrType dst,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = Int<0>{},- OffY off_y = Int<0>{}+ int str_x,+ int str_y,+ int lim_x,+ int lim_y,+ int off_x = 0,+ int off_y = 0) {using U = pointer_element_t<DstPtrType>;@@ -155,27 +132,18 @@ struct BaseMMAFrag<T, 8, 8> {}}- template <- typename DstPtrType,- typename StrX,- typename StrY,- typename StartX,- typename StopX,- typename StartY,- typename StopY,- typename OffX,- typename OffY>+ template <typename DstPtrType>METAL_FUNC static constexpr void store_slice(const thread frag_type& src,DstPtrType dst,- StrX str_x,- StrY str_y,- StartX start_x,- StopX stop_x,- StartY start_y,- StopY stop_y,- OffX off_x = Int<0>{},- OffY off_y = Int<0>{}+ int str_x,+ int str_y,+ int start_x,+ int stop_x,+ int start_y,+ int stop_y,+ int off_x = 0,+ int off_y = 0) {using U = pointer_element_t<DstPtrType>;@@ -222,37 +190,34 @@ struct BaseMMAFrag<T, 8, 8> {}};-template <- typename T,- int kTileRows_,- int kTileCols_,- class MMAFrag_ = BaseMMAFrag<T, 8, 8>>+template <typename T>struct MMATile {- using MMAFrag_t = MMAFrag_;+ using MMAFrag_t = BaseMMAFrag<T>;using elem_type = T;STEEL_CONST int kFragRows = MMAFrag_t::kFragRows;STEEL_CONST int kFragCols = MMAFrag_t::kFragCols;STEEL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;- STEEL_CONST int kTileRows = kTileRows_;- STEEL_CONST int kTileCols = kTileCols_;-- STEEL_CONST int kRows = kTileRows * kFragRows;- STEEL_CONST int kCols = kTileCols * kFragCols;-- STEEL_CONST int kNumFrags = kTileRows * kTileCols;- STEEL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;-typedef typename MMAFrag_t::mat_type mat_type;typedef typename MMAFrag_t::frag_type frag_type;- frag_type val_frags[kNumFrags] = {frag_type(0)};+ // Max-sized array — runtime kTileRows/kTileCols control how much is used+ frag_type val_frags[STEEL_MAX_CTILE_FRAGS] = {};- METAL_FUNC MMATile() thread {}+ short kTileRows;+ short kTileCols;+ short kNumFrags;++ METAL_FUNC MMATile() thread : kTileRows(0), kTileCols(0), kNumFrags(0) {}++ METAL_FUNC MMATile(short tile_rows, short tile_cols) thread+ : kTileRows(tile_rows),+ kTileCols(tile_cols),+ kNumFrags(tile_rows* tile_cols) {}METAL_FUNC constexpr void clear() {STEEL_PRAGMA_UNROLL- for (short i = 0; i < kNumFrags; ++i) {+ for (short i = 0; i < STEEL_MAX_CTILE_FRAGS; ++i) {val_frags[i] = frag_type(0);}}@@ -268,15 +233,6 @@ struct MMATile {return val_frags[i * kTileCols + j];}- METAL_FUNC mat_type mat_at(const short i, const short j) {- mat_type val_mat;- STEEL_PRAGMA_UNROLL- for (short ii = 0; ii < kElemsPerFrag; ++ii) {- val_mat.thread_elements()[ii] = frag_at(i, j)[ii];- }- return val_mat;- }-METAL_FUNC thread elem_type* elems() {return reinterpret_cast<thread elem_type*>(val_frags);}@@ -285,11 +241,17 @@ struct MMATile {return reinterpret_cast<const thread elem_type*>(val_frags);}- template <typename U, int w_x, int w_y, int str_x, int str_y>- METAL_FUNC void load(const threadgroup U* src) {- STEEL_PRAGMA_UNROLL+ METAL_FUNC short elems_per_tile() const { return kNumFrags * kElemsPerFrag; }++ template <typename U>+ METAL_FUNC void load(+ const threadgroup U* src,+ int w_x,+ int w_y,+ int str_x,+ int str_y+ ) {for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLLfor (short j = 0; j < kTileCols; ++j) {MMAFrag_t::load(frag_at(i, j),@@ -297,18 +259,22 @@ struct MMATile {src[(i * kFragRows) * w_x * str_x +(j * kFragCols) * w_y * str_y]),- Int<str_x>{},- Int<str_y>{}+ str_x,+ str_y);}}}- template <typename U, int w_x, int w_y, int str_x, int str_y>- METAL_FUNC void store(threadgroup U* dst) const {- STEEL_PRAGMA_UNROLL+ template <typename U>+ METAL_FUNC void store(+ threadgroup U* dst,+ int w_x,+ int w_y,+ int str_x,+ int str_y+ ) const {for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLLfor (short j = 0; j < kTileCols; ++j) {MMAFrag_t::store(frag_at(i, j),@@ -316,60 +282,56 @@ struct MMATile {dst[(i * kFragRows) * w_x * str_x +(j * kFragCols) * w_y * str_y]),- Int<str_x>{},- Int<str_y>{}+ str_x,+ str_y);}}}- template <typename U, int w_x, int w_y>- METAL_FUNC void load(const device U* src, const int ld) {- STEEL_PRAGMA_UNROLL+ template <typename U>+ METAL_FUNC void load(const device U* src, const int ld, int w_x, int w_y) {for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLLfor (short j = 0; j < kTileCols; ++j) {MMAFrag_t::load(frag_at(i, j),&(src[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),ld,- Int<1>{}+ 1);}}}- template <typename U, int w_x, int w_y>- METAL_FUNC void store(device U* dst, const int ld) const {- STEEL_PRAGMA_UNROLL+ template <typename U>+ METAL_FUNC void store(device U* dst, const int ld, int w_x, int w_y) const {for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLLfor (short j = 0; j < kTileCols; ++j) {MMAFrag_t::store(frag_at(i, j),&(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),ld,- Int<1>{}+ 1);}}}- template <typename U, int w_x, int w_y>+ template <typename U>METAL_FUNC void load_safe(const device U* src,const int ld,- const short2 src_tile_dims+ const short2 src_tile_dims,+ int w_x,+ int w_y) {- STEEL_PRAGMA_UNROLLfor (int i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLLfor (int j = 0; j < kTileCols; ++j) {MMAFrag_t::load_safe(frag_at(i, j),src,ld,- Int<1>{},+ 1,src_tile_dims.y,src_tile_dims.x,(i * kFragRows) * w_x,@@ -379,21 +341,21 @@ struct MMATile {}}- template <typename U, int w_x, int w_y>+ template <typename U>METAL_FUNC void store_safe(device U* dst,const int ld,- const short2 dst_tile_dims+ const short2 dst_tile_dims,+ int w_x,+ int w_y) const {- STEEL_PRAGMA_UNROLLfor (int i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLLfor (int j = 0; j < kTileCols; ++j) {MMAFrag_t::store_safe(frag_at(i, j),dst,ld,- Int<1>{},+ 1,dst_tile_dims.y,dst_tile_dims.x,(i * kFragRows) * w_x,@@ -403,22 +365,22 @@ struct MMATile {}}- template <typename U, int w_x, int w_y>+ template <typename U>METAL_FUNC void store_slice(device U* dst,const int ld,const short2 start,- const short2 stop+ const short2 stop,+ int w_x,+ int w_y) const {- STEEL_PRAGMA_UNROLLfor (int i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLLfor (int j = 0; j < kTileCols; ++j) {MMAFrag_t::store_slice(frag_at(i, j),dst,ld,- Int<1>{},+ 1,start.y,stop.y,start.x,@@ -431,21 +393,18 @@ struct MMATile {}};-template <typename T, typename U, int M, int N, int K>+template <typename T, typename U>METAL_FUNC void tile_matmad(- thread MMATile<T, M, N>& D,- thread MMATile<U, M, K>& A,- thread MMATile<U, K, N>& B,- thread MMATile<T, M, N>& C+ thread MMATile<T>& D,+ thread MMATile<U>& A,+ thread MMATile<U>& B,+ thread MMATile<T>& C) {- STEEL_PRAGMA_UNROLL- for (short m = 0; m < M; ++m) {- STEEL_PRAGMA_UNROLL- for (short n = 0; n < N; ++n) {- short n_serp = (m % 2) ? (N - 1 - n) : n;- STEEL_PRAGMA_UNROLL- for (short k = 0; k < K; ++k) {- MMATile<T, M, N>::MMAFrag_t::mma(+ for (short m = 0; m < A.kTileRows; ++m) {+ for (short n = 0; n < B.kTileCols; ++n) {+ short n_serp = (m % 2) ? (B.kTileCols - 1 - n) : n;+ for (short k = 0; k < A.kTileCols; ++k) {+ BaseMMAFrag<T>::mma(D.frag_at(m, n_serp),A.frag_at(m, k),B.frag_at(k, n_serp),@@ -465,62 +424,60 @@ struct TransformNone<complex64_t, InT> {template <typename T,typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- short lda_tgp,- short ldb_tgp,typename AccumType = float,typename Epilogue = TransformNone<U, AccumType>>struct BlockMMA {- // MMAFrag sizeSTEEL_CONST short kFragSize = 8;- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;+ using MMAFrag_acc_t = BaseMMAFrag<AccumType>;- // Warp tile simdgroup matrix strides along M- STEEL_CONST short TM_stride = kFragSize * WM;- // Warp tile simdgroup matrix strides along M- STEEL_CONST short TN_stride = kFragSize * WN;+ // Tile params (set at construction)+ short BM, BN, BK, WM, WN;+ bool transpose_a, transpose_b;+ short lda_tgp, ldb_tgp;- // Warp tile size along M- STEEL_CONST short TM = BM / (kFragSize * WM);- // Warp tile size along N- STEEL_CONST short TN = BN / (kFragSize * WN);-- // Threadgroup A strides- STEEL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M- STEEL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K-- // Threadgroup B strides- STEEL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K- STEEL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N-- // Threadgroup strides along K- STEEL_CONST short tile_stride_a = kFragSize * A_str_k;- STEEL_CONST short tile_stride_b = kFragSize * B_str_k;+ // Derived values+ short TM_stride, TN_stride;+ short TM, TN;+ short A_str_m, A_str_k;+ short B_str_k, B_str_n;+ short tile_stride_a, tile_stride_b;// Simdgroup matrices- MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;- MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;+ MMATile<AccumType> Atile;+ MMATile<AccumType> Btile;+ MMATile<AccumType> Ctile;// Offsets within threadgroupshort sm;short sn;-short As_offset;short Bs_offset;/* Constructor */METAL_FUNC BlockMMA(- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]- ) {- // Determine thread position in simdgroup matrix+ ushort simd_group_id,+ ushort simd_lane_id,+ short BM_,+ short BN_,+ short BK_,+ short WM_,+ short WN_,+ bool transpose_a_,+ bool transpose_b_,+ short lda_tgp_,+ short ldb_tgp_+ )+ : BM(BM_), BN(BN_), BK(BK_), WM(WM_), WN(WN_), transpose_a(transpose_a_),+ transpose_b(transpose_b_), lda_tgp(lda_tgp_), ldb_tgp(ldb_tgp_),+ TM_stride(kFragSize * WM_), TN_stride(kFragSize * WN_),+ TM(BM_ / (kFragSize * WM_)), TN(BN_ / (kFragSize * WN_)),+ A_str_m(transpose_a_ ? 1 : lda_tgp_),+ A_str_k(transpose_a_ ? lda_tgp_ : 1),+ B_str_k(transpose_b_ ? 1 : ldb_tgp_),+ B_str_n(transpose_b_ ? ldb_tgp_ : 1),+ tile_stride_a(kFragSize * (transpose_a_ ? lda_tgp_ : 1)),+ tile_stride_b(kFragSize * (transpose_b_ ? 1 : ldb_tgp_)), Atile(TM, 1),+ Btile(1, TN), Ctile(TM, TN) {short tm = kFragSize * (simd_group_id / WN);short tn = kFragSize * (simd_group_id % WN);@@ -528,9 +485,8 @@ struct BlockMMA {sm = simd_coord.y;sn = simd_coord.x;- // Determine thread and simdgroup offset- As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K- Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N+ As_offset = (tm + sm) * A_str_m + (sn)*A_str_k;+ Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n;sm += tm;sn += tn;@@ -538,26 +494,16 @@ struct BlockMMA {/* (BM, BK) X (BK, BN) multiply accumulate function */METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {- // Adjust for simdgroup and thread locationAs += As_offset;Bs += Bs_offset;- // Iterate over BK in blocks of kFragSize- STEEL_PRAGMA_UNROLLfor (short kk = 0; kk < BK; kk += kFragSize) {simdgroup_barrier(mem_flags::mem_none);-- Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);-+ Atile.load(As, WM, 1, A_str_m, A_str_k);simdgroup_barrier(mem_flags::mem_none);-- Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);-+ Btile.load(Bs, 1, WN, B_str_k, B_str_n);simdgroup_barrier(mem_flags::mem_none);-tile_matmad(Ctile, Atile, Btile, Ctile);-- // Progress to next simdgroup tileAs += tile_stride_a;Bs += tile_stride_b;}@@ -565,40 +511,11 @@ struct BlockMMA {/* Store results from simdgroup_matrix results into device memory */METAL_FUNC void store_result(device U* D, const int ldd) {- // Apply epilogue- STEEL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);- }-- // Adjust for simdgroup and thread location- D += sm * ldd + sn;-- Ctile.template store<U, WM, WN>(D, ldd);- }-- METAL_FUNC void store_result_slice(- device U* D,- const int ldd,- short2 start,- short2 stop- ) {- // Apply epilogue- STEEL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ for (short i = 0; i < Ctile.elems_per_tile(); i++) {Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);}-D += sm * ldd + sn;- start -= short2(sn, sm);- stop -= short2(sn, sm);-- // TODO: Check the start as well- if (stop.y <= 0 || stop.x <= 0) {- return;- }-- Ctile.template store_slice<U, WM, WN>(D, ldd, start, stop);+ Ctile.store(D, ldd, WM, WN);}METAL_FUNC void store_result_safe(@@ -606,30 +523,14 @@ struct BlockMMA {const int ldd,short2 dst_tile_dims) {- // Apply epilogue- STEEL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ for (short i = 0; i < Ctile.elems_per_tile(); i++) {Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);}-- // Adjust for simdgroup and thread locationD += sm * ldd + sn;dst_tile_dims -= short2(sn, sm);-if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)return;-- Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);- }-- /* Apply epilogue */- template <typename UnaryEpilogue>- METAL_FUNC void apply_epilogue(thread const UnaryEpilogue& epilogue_op) {- // Loop over all simdgroup tiles- STEEL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {- Ctile.elems()[i] = epilogue_op.apply(Ctile.elems()[i]);- }+ Ctile.store_safe(D, ldd, dst_tile_dims, WM, WN);}/* Apply epilogue */@@ -640,28 +541,24 @@ struct BlockMMA {const int fdc,thread const BinaryEpilogue& epilogue_op) {- // Adjust for simdgroup and thread locationC += (sm)*ldc + (sn)*fdc;- // Loop over all simdgroup tiles- STEEL_PRAGMA_UNROLLfor (short i = 0; i < TM; i++) {- STEEL_PRAGMA_UNROLLfor (short j = 0; j < TN; j++) {- // Get accumulated result and associated offset in Cthread auto& accum = Ctile.frag_at(i, j);int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;- // Apply epilogueSTEEL_PRAGMA_UNROLL- for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {+ for (short k = 0; k < kElemsPerFrag; k++) {accum[k] = epilogue_op.apply(accum[k], C[offset_c + k * fdc]);}}}}- /* Apply epilogue */+ STEEL_CONST short kElemsPerFrag = MMAFrag_acc_t::kElemsPerFrag;++ /* Apply epilogue safe */template <typename BinaryEpilogue>METAL_FUNC void apply_epilogue_safe(const device U* C,@@ -670,44 +567,35 @@ struct BlockMMA {short2 dst_tile_dims,thread const BinaryEpilogue& epilogue_op) {- // Adjust for simdgroup and thread locationC += (sm)*ldc + (sn)*fdc;dst_tile_dims -= short2(sn, sm);if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)return;- // Loop over all simdgroup tiles- STEEL_PRAGMA_UNROLLfor (short i = 0; i < TM; i++) {- STEEL_PRAGMA_UNROLLfor (short j = 0; j < TN; j++) {- // Get accumulated result and associated offset in Cthread auto& accum = Ctile.frag_at(i, j);int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;- constexpr short kelems = decltype(Ctile)::kElemsPerFrag;-- // Read C- U c_elems[kelems] = {0};+ U c_elems[2] = {0};STEEL_PRAGMA_UNROLL- for (short k = 0; k < kelems; k++) {+ for (short k = 0; k < kElemsPerFrag; k++) {if ((j * TN_stride + k) < dst_tile_dims.x) {c_elems[k] = C[offset_c + k * fdc];}}- // Apply epilogueSTEEL_PRAGMA_UNROLL- for (short k = 0; k < kelems; k++) {+ for (short k = 0; k < kElemsPerFrag; k++) {accum[k] = epilogue_op.apply(accum[k], c_elems[k]);}}}}- /* Store results from simdgroup_matrix results into device memory */+ /* Store results with epilogue from C source */METAL_FUNC void store_result(device U* D,const int ldd,@@ -716,25 +604,17 @@ struct BlockMMA {const int fdc,thread const Epilogue& epilogue_op) const {- // Adjust for simdgroup and thread locationC += (sm)*ldc + (sn)*fdc;D += (sm)*ldd + sn;- constexpr short kelems = decltype(Ctile)::kElemsPerFrag;-- // Loop over all simdgroup tiles- STEEL_PRAGMA_UNROLLfor (short i = 0; i < TM; i++) {- STEEL_PRAGMA_UNROLLfor (short j = 0; j < TN; j++) {- // Get accumulated result and associated offset in Cthread const auto& accum = Ctile.frag_at(i, j);int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;int offset_d = (i * TM_stride) * ldd + (j * TN_stride);- // Apply epilogueSTEEL_PRAGMA_UNROLL- for (short k = 0; k < kelems; k++) {+ for (short k = 0; k < kElemsPerFrag; k++) {D[offset_d + k] = epilogue_op.apply(accum[k], C[offset_c + k * fdc]);}}@@ -750,7 +630,6 @@ struct BlockMMA {short2 dst_tile_dims,thread const Epilogue& epilogue_op) const {- // Adjust for simdgroup and thread locationC += (sm)*ldc + (sn)*fdc;D += (sm)*ldd + sn;dst_tile_dims -= short2(sn, sm);@@ -758,21 +637,15 @@ struct BlockMMA {if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)return;- constexpr short kelems = decltype(Ctile)::kElemsPerFrag;-- STEEL_PRAGMA_UNROLLfor (int i = 0; i < TM; i++) {if (i * TM_stride < dst_tile_dims.y) {- STEEL_PRAGMA_UNROLLfor (int j = 0; j < TN; j++) {- // Get accumulated result and associated offset in Cthread const auto& accum = Ctile.frag_at(i, j);int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;int offset_d = (i * TM_stride) * ldd + (j * TN_stride);- // Apply epilogueSTEEL_PRAGMA_UNROLL- for (short k = 0; k < kelems; k++) {+ for (short k = 0; k < kElemsPerFrag; k++) {if ((j * TN_stride + k) < dst_tile_dims.x) {D[offset_d + k] =epilogue_op.apply(accum[k], C[offset_c + k * fdc]);@@ -784,426 +657,4 @@ struct BlockMMA {}};-template <- typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- short lda_tgp,- short ldb_tgp,- typename AccumType,- typename Epilogue>-struct BlockMMA<- complex64_t,- U,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- lda_tgp,- ldb_tgp,- AccumType,- Epilogue> {- static_assert(- metal::is_same_v<AccumType, float>,- "BlockMMA<complex64_t,...> expects float accumulators"- );- static_assert(- metal::is_same_v<U, complex64_t>,- "For complex BlockMMA, U must be complex64_t; use a different epilogue "- "for projections"- );- // MMAFrag size- STEEL_CONST short kFragSize = 8;- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;-- // Warp tile simdgroup matrix strides along M- STEEL_CONST short TM_stride = kFragSize * WM;- // Warp tile simdgroup matrix strides along M- STEEL_CONST short TN_stride = kFragSize * WN;-- // Warp tile size along M- STEEL_CONST short TM = BM / (kFragSize * WM);- // Warp tile size along N- STEEL_CONST short TN = BN / (kFragSize * WN);-- // Threadgroup A strides- STEEL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M- STEEL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K-- // Threadgroup B strides- STEEL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K- STEEL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N-- // Threadgroup strides along K- STEEL_CONST short tile_stride_a = kFragSize * A_str_k;- STEEL_CONST short tile_stride_b = kFragSize * B_str_k;-- // When indexing complex as float[2]- STEEL_CONST short A_str_m_f = A_str_m * 2;- STEEL_CONST short A_str_k_f = A_str_k * 2;- STEEL_CONST short B_str_k_f = B_str_k * 2;- STEEL_CONST short B_str_n_f = B_str_n * 2;- STEEL_CONST short tile_stride_a_f = tile_stride_a * 2;- STEEL_CONST short tile_stride_b_f = tile_stride_b * 2;-- // Accumulators (real/imag)- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile_r;- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile_i;-- // Offsets within threadgroup- short sm, sn;- short As_offset, Bs_offset;-- /* Constructor */- METAL_FUNC BlockMMA(- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]- ) {- // Determine thread position in simdgroup matrix- short tm = kFragSize * (simd_group_id / WN);- short tn = kFragSize * (simd_group_id % WN);-- short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);- sm = simd_coord.y;- sn = simd_coord.x;-- // Determine thread and simdgroup offset- As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // (M,K)- Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // (K,N)-- sm += tm;- sn += tn;- }-- /* Karatsuba MMA: 3 real MMAs per K-chunk */- METAL_FUNC void mma(- const threadgroup complex64_t* As,- const threadgroup complex64_t* Bs- ) {- // Adjust for simdgroup and thread location- As += As_offset;- Bs += Bs_offset;- threadgroup const float* As_f =- reinterpret_cast<threadgroup const float*>(As);- threadgroup const float* Bs_f =- reinterpret_cast<threadgroup const float*>(Bs);-- // Iterate over BK in blocks of kFragSize- STEEL_PRAGMA_UNROLL- for (short kk = 0; kk < BK; kk += kFragSize) {- simdgroup_barrier(mem_flags::mem_none);-- MMATile<AccumType, TM, 1, MMAFrag_acc_t> Ar, Ai;- Ar.template load<float, WM, 1, A_str_m_f, A_str_k_f>(As_f + 0);- Ai.template load<float, WM, 1, A_str_m_f, A_str_k_f>(As_f + 1);-- simdgroup_barrier(mem_flags::mem_none);-- MMATile<AccumType, 1, TN, MMAFrag_acc_t> Br, Bi;- Br.template load<float, 1, WN, B_str_k_f, B_str_n_f>(Bs_f + 0);- Bi.template load<float, 1, WN, B_str_k_f, B_str_n_f>(Bs_f + 1);-- simdgroup_barrier(mem_flags::mem_none);-- // P = Ar*Br ; Q = Ai*Bi ; R = (Ar+Ai)*(Br+Bi)- MMATile<AccumType, TM, TN, MMAFrag_acc_t> P, Q, R;-- tile_matmad(P, Ar, Br, P);- tile_matmad(Q, Ai, Bi, Q);-- STEEL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ar)::kElemsPerTile; ++i)- Ar.elems()[i] += Ai.elems()[i];- STEEL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Br)::kElemsPerTile; ++i)- Br.elems()[i] += Bi.elems()[i];-- tile_matmad(R, Ar, Br, R);-- // C_r += P - Q ; C_i -= Q- STEEL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile_r)::kElemsPerTile; ++i) {- const auto p = P.elems()[i];- const auto q = Q.elems()[i];- const auto r = R.elems()[i];- Ctile_r.elems()[i] += (p - q);- Ctile_i.elems()[i] += (r - p - q);- }-- // Progress to next simdgroup tile- As_f += tile_stride_a_f;- Bs_f += tile_stride_b_f;- }- }-- /* Store results from simdgroup_matrix results into device memory */- METAL_FUNC void store_result(device U* D, const int ldd) {- // Adjust for simdgroup and thread location- D += sm * ldd + sn;-- STEEL_PRAGMA_UNROLL- for (short i = 0; i < TM; i++) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < TN; j++) {- thread const auto& r = Ctile_r.frag_at(i, j);- thread const auto& im = Ctile_i.frag_at(i, j);- int off = (i * TM_stride) * ldd + (j * TN_stride);- STEEL_PRAGMA_UNROLL- for (short k = 0; k < decltype(Ctile_r)::kElemsPerFrag; k++) {- D[off + k] = Epilogue::apply(complex64_t(r[k], im[k]));- }- }- }- }-- METAL_FUNC void store_result_slice(- device U* D,- const int ldd,- short2 start,- short2 stop- ) {- D += sm * ldd + sn;- start -= short2(sn, sm);- stop -= short2(sn, sm);-- if (stop.y <= 0 || stop.x <= 0)- return;-- STEEL_PRAGMA_UNROLL- for (short i = 0; i < TM; ++i) {- const int row = i * TM_stride;- if (row >= start.y && row < stop.y) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < TN; ++j) {- const int off = row * ldd + (j * TN_stride);- thread const auto& r = Ctile_r.frag_at(i, j);- thread const auto& im = Ctile_i.frag_at(i, j);-- STEEL_PRAGMA_UNROLL- for (short k = 0; k < decltype(Ctile_r)::kElemsPerFrag; ++k) {- const int col = j * TN_stride + k;- if (col >= start.x && col < stop.x) {- D[off + k] = Epilogue::apply(complex64_t(r[k], im[k]));- }- }- }- }- }- }-- METAL_FUNC void store_result_safe(- device U* D,- const int ldd,- short2 dst_tile_dims- ) {- D += sm * ldd + sn;- dst_tile_dims -= short2(sn, sm);- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)- return;- STEEL_PRAGMA_UNROLL- for (short i = 0; i < TM; i++) {- if (i * TM_stride < dst_tile_dims.y) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < TN; j++) {- int off = (i * TM_stride) * ldd + (j * TN_stride);- thread const auto& r = Ctile_r.frag_at(i, j);- thread const auto& im = Ctile_i.frag_at(i, j);- STEEL_PRAGMA_UNROLL- for (short k = 0; k < decltype(Ctile_r)::kElemsPerFrag; k++) {- if ((j * TN_stride + k) < dst_tile_dims.x) {- D[off + k] = Epilogue::apply(complex64_t(r[k], im[k]));- }- }- }- }- }- }-- /* Apply epilogue */- template <typename UnaryEpilogue>- METAL_FUNC void apply_epilogue(thread const UnaryEpilogue& epilogue_op) {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile_r)::kElemsPerTile; i++) {- complex64_t out = epilogue_op.apply(- complex64_t(Ctile_r.elems()[i], Ctile_i.elems()[i])- );- Ctile_r.elems()[i] = out.real;- Ctile_i.elems()[i] = out.imag;- }- }-- /* Apply epilogue */- template <typename BinaryEpilogue>- METAL_FUNC void apply_epilogue(- const device U* C,- const int ldc,- const int fdc,- thread const BinaryEpilogue& epilogue_op- ) {- // Adjust for simdgroup and thread location- C += (sm)*ldc + (sn)*fdc;-- // Loop over all simdgroup tiles- STEEL_PRAGMA_UNROLL- for (short i = 0; i < TM; i++) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < TN; j++) {- // Get accumulated result and associated offset in Cr, Ci- thread auto& r = Ctile_r.frag_at(i, j);- thread auto& im = Ctile_i.frag_at(i, j);- int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;-- STEEL_PRAGMA_UNROLL- for (short k = 0; k < decltype(Ctile_r)::kElemsPerFrag; k++) {- complex64_t out = epilogue_op.apply(- complex64_t(r[k], im[k]),- C[offset_c + k * fdc]- );- r[k] = out.real;- im[k] = out.imag;- }- }- }- }-- /* Apply epilogue */- template <typename BinaryEpilogue>- METAL_FUNC void apply_epilogue_safe(- const device U* C,- const int ldc,- const int fdc,- short2 dst_tile_dims,- thread const BinaryEpilogue& epilogue_op- ) {- // Adjust for simdgroup and thread location- C += (sm)*ldc + (sn)*fdc;- dst_tile_dims -= short2(sn, sm);-- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)- return;-- // Loop over all simdgroup tiles- STEEL_PRAGMA_UNROLL- for (short i = 0; i < TM; i++) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < TN; j++) {- // Get accumulated result and associated offset in Cr, Ci- thread auto& r = Ctile_r.frag_at(i, j);- thread auto& im = Ctile_i.frag_at(i, j);- int offset_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;-- constexpr short kelems = decltype(Ctile_r)::kElemsPerFrag;- complex64_t tmp[kelems];-- STEEL_PRAGMA_UNROLL- for (short k = 0; k < kelems; k++) {- if ((j * TN_stride + k) < dst_tile_dims.x &&- (i * TM_stride) < dst_tile_dims.y) {- tmp[k] = C[offset_c + k * fdc];- } else {- tmp[k] = complex64_t(0.0f, 0.0f);- }- }-- // Apply epilogue- STEEL_PRAGMA_UNROLL- for (short k = 0; k < kelems; k++) {- complex64_t out = epilogue_op.apply(complex64_t(r[k], im[k]), tmp[k]);- r[k] = out.real;- im[k] = out.imag;- }- }- }- }-- /* Store results from simdgroup_matrix results into device memory */- METAL_FUNC void store_result(- device U* D,- const int ldd,- const device U* C,- const int ldc,- const int fdc,- thread const Epilogue& epilogue_op- ) const {- // Adjust for simdgroup and thread location- C += (sm)*ldc + (sn)*fdc;- D += (sm)*ldd + sn;-- constexpr short kelems = decltype(Ctile_r)::kElemsPerFrag;-- // Loop over all simdgroup tiles- STEEL_PRAGMA_UNROLL- for (short i = 0; i < TM; i++) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < TN; j++) {- // Get accumulated result and associated offset in Cr, Ci- thread const auto& r = Ctile_r.frag_at(i, j);- thread const auto& im = Ctile_i.frag_at(i, j);- int off_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;- int off_d = (i * TM_stride) * ldd + (j * TN_stride);-- // Apply epilogue- STEEL_PRAGMA_UNROLL- for (short k = 0; k < kelems; k++) {- D[off_d + k] =- epilogue_op.apply(complex64_t(r[k], im[k]), C[off_c + k * fdc]);- }- }- }- }-- METAL_FUNC void store_result_safe(- device U* D,- const int ldd,- const device U* C,- const int ldc,- const int fdc,- short2 dst_tile_dims,- thread const Epilogue& epilogue_op- ) const {- // Adjust for simdgroup and thread location- C += (sm)*ldc + (sn)*fdc;- D += (sm)*ldd + sn;- dst_tile_dims -= short2(sn, sm);-- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)- return;-- constexpr short kelems = decltype(Ctile_r)::kElemsPerFrag;-- STEEL_PRAGMA_UNROLL- for (int i = 0; i < TM; i++) {- if (i * TM_stride < dst_tile_dims.y) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < TN; j++) {- // Get accumulated result and associated offset in Cr, Ci- thread const auto& r = Ctile_r.frag_at(i, j);- thread const auto& im = Ctile_i.frag_at(i, j);- int off_c = (i * TM_stride) * ldc + (j * TN_stride) * fdc;- int off_d = (i * TM_stride) * ldd + (j * TN_stride);-- // Apply epilogue- STEEL_PRAGMA_UNROLL- for (short k = 0; k < kelems; k++) {- if ((j * TN_stride + k) < dst_tile_dims.x) {- D[off_d + k] = epilogue_op.apply(- complex64_t(r[k], im[k]),- C[off_c + k * fdc]- );- }- }- }- }- }- }-};-} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.hdeleted file mode 100644index e1d665fc5..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.h+++ /dev/null@@ -1,1144 +0,0 @@---#pragma once--#include <metal_simdgroup>-#include <metal_simdgroup_matrix>-#include <metal_stdlib>--#include "../defines.h"-#include "transforms.h"-#include "../utils/integral_constant.h"--#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>--using namespace metal;--///////////////////////////////////////////////////////////////////////////////-// MMA helper-///////////////////////////////////////////////////////////////////////////////--namespace steel {--///////////////////////////////////////////////////////////////////////////////-// NAX Steel with new tiles-///////////////////////////////////////////////////////////////////////////////--struct BaseNAXFrag {- STEEL_CONST short kFragRows = 16;- STEEL_CONST short kFragCols = 16;-- STEEL_CONST short kElemsPerFrag = (kFragRows * kFragCols) / 32;-- STEEL_CONST short kElemRows = 2;- STEEL_CONST short kElemCols = 4;-- STEEL_CONST short kElemRowsJump = 8;-- static_assert(- kElemRows * kElemCols == kElemsPerFrag,- "MMAFrag shape is not consistent with MMAFrag size"- );-- template <typename U>- using dtype_frag_t = typename metal::vec<U, kElemsPerFrag>;-- METAL_FUNC static short2 get_coord() {- const ushort simd_lane_id = __metal_get_thread_index_in_simdgroup(ushort());- const short qid = simd_lane_id >> 2;- const short fm = ((qid & 4) | ((simd_lane_id >> 1) & 3));- const short fn = ((qid & 2) | (simd_lane_id & 1)) * 4;- return short2{fn, fm};- }-- METAL_FUNC static short2 get_coord(short idx) {- const ushort simd_lane_id = __metal_get_thread_index_in_simdgroup(ushort());- const short qid = simd_lane_id >> 2;- const short fm = ((qid & 4) | ((simd_lane_id >> 1) & 3)) + (idx >> 2) * 8;- const short fn = ((qid & 2) | (simd_lane_id & 1)) * 4 + idx % 4;- return short2{fn, fm};- }-- template <- typename T,- typename SrcPtrType,- typename StrX,- typename StrY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC static constexpr void load(- thread dtype_frag_t<T>& dst,- SrcPtrType src,- StrX str_x,- StrY str_y,- OffX off_x = {},- OffY off_y = {}- ) {- const short2 sc = get_coord();- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- const auto r = off_x + i * kElemRowsJump + sc.y;- const auto c = off_y + sc.x;-- if constexpr (metal::is_same_v<StrY, Int<1>>) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[i * kElemCols + j] = static_cast<T>(src[r * str_x + c + j]);- }- } else {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[i * kElemCols + j] =- static_cast<T>(src[r * str_x + (c + j) * str_y]);- }- }- }- }-- template <- typename T,- typename SrcPtrType,- typename StrX,- typename StrY,- typename LimX,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC static constexpr void load_rows(- thread dtype_frag_t<T>& dst,- SrcPtrType src,- StrX str_x,- StrY str_y,- LimX lim_x,- OffX off_x = {},- OffY off_y = {}- ) {- const short2 sc = get_coord();- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- const auto r = off_x + i * kElemRowsJump + sc.y;- const auto c = off_y + sc.x;-- if (r < lim_x) {- if constexpr (metal::is_same_v<StrY, Int<1>>) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[i * kElemCols + j] = static_cast<T>(src[r * str_x + (c + j)]);- }- } else {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[i * kElemCols + j] =- static_cast<T>(src[r * str_x + (c + j) * str_y]);- }- }-- } else {- dst = dtype_frag_t<T>(0);- }- }- }-- template <- typename T,- typename SrcPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC static constexpr void load_safe(- thread dtype_frag_t<T>& dst,- SrcPtrType src,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = {},- OffY off_y = {}- ) {- const short2 sc = get_coord();- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- const auto r = off_x + i * kElemRowsJump + sc.y;- const auto c = off_y + sc.x;- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- if (r < lim_x && (c + j) < lim_y) {- dst[i * kElemCols + j] =- static_cast<T>(src[r * str_x + (c + j) * str_y]);- } else {- dst[i * kElemCols + j] = T(0);- }- }- }- }-- template <- typename T,- typename DstPtrType,- typename StrX,- typename StrY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC static constexpr void store(- const thread dtype_frag_t<T>& src,- DstPtrType dst,- StrX str_x,- StrY str_y,- OffX off_x = {},- OffY off_y = {}- ) {- using U = pointer_element_t<DstPtrType>;-- const short2 sc = get_coord();- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- const auto r = off_x + i * kElemRowsJump + sc.y;- const auto c = off_y + sc.x;-- if constexpr (metal::is_same_v<StrY, Int<1>>) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[r * str_x + c + j] = static_cast<U>(src[i * kElemCols + j]);- }- } else {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[r * str_x + (c + j) * str_y] =- static_cast<U>(src[i * kElemCols + j]);- }- }- }- }-- template <- typename T,- typename DstPtrType,- typename StrX,- typename StrY,- typename LimX,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC static constexpr void store_rows(- const thread dtype_frag_t<T>& src,- DstPtrType dst,- StrX str_x,- StrY str_y,- LimX lim_x,- OffX off_x = {},- OffY off_y = {}- ) {- using U = pointer_element_t<DstPtrType>;-- const short2 sc = get_coord();- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- const auto r = off_x + i * kElemRowsJump + sc.y;- const auto c = off_y + sc.x;-- if (r < lim_x) {- if constexpr (metal::is_same_v<StrY, Int<1>>) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[r * str_x + c + j] = static_cast<U>(src[i * kElemCols + j]);- }- } else {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[r * str_x + (c + j) * str_y] =- static_cast<U>(src[i * kElemCols + j]);- }- }- }- }- }-- template <- typename T,- typename DstPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC static constexpr void store_safe(- const thread dtype_frag_t<T>& src,- DstPtrType dst,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = {},- OffY off_y = {}- ) {- using U = pointer_element_t<DstPtrType>;-- const short2 sc = get_coord();- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- const auto r = off_x + i * kElemRowsJump + sc.y;- const auto c = off_y + sc.x;-- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- if (r < lim_x && (c + j) < lim_y) {- dst[r * str_x + (c + j) * str_y] =- static_cast<U>(src[i * kElemCols + j]);- }- }- }- }-- template <- typename T,- typename DstPtrType,- typename StrX,- typename StrY,- typename StartX,- typename StopX,- typename StartY,- typename StopY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC static constexpr void store_slice(- const thread dtype_frag_t<T>& src,- DstPtrType dst,- StrX str_x,- StrY str_y,- StartX start_x,- StopX stop_x,- StartY start_y,- StopY stop_y,- OffX off_x = Int<0>{},- OffY off_y = Int<0>{}- ) {- using U = pointer_element_t<DstPtrType>;-- const short2 sc = get_coord();-- const_for_loop<0, kElemRows, 1>([&](auto idx_row) {- const auto r = off_x + idx_row * Int<kElemRowsJump>{};- if (r >= stop_x - sc.y || r < start_x - sc.y) {- return;- }-- const_for_loop<0, kElemCols, 1>([&](auto idx_col) {- const auto c = off_y + idx_col;- if (c >= stop_y - sc.x || c < start_y - sc.x) {- return;- }-- const auto src_idx = idx_row * Int<kElemCols>{} + idx_col;- dst[(r + sc.y) * str_x + (c + sc.x) * str_y] =- static_cast<U>(src[src_idx]);- });- });- }-- template <typename Op, typename T>- METAL_FUNC static constexpr void row_reduce(- thread const dtype_frag_t<T>& inp_vals,- thread T* reduced_vals- ) {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- T thr_reduce = Op::apply(- Op::apply(inp_vals[i * kElemCols + 0], inp_vals[i * kElemCols + 1]),- Op::apply(inp_vals[i * kElemCols + 2], inp_vals[i * kElemCols + 3])- );-- T qgr_reduce = simd_shuffle_xor(thr_reduce, ushort(1));- qgr_reduce = Op::apply(thr_reduce, qgr_reduce);-- T sgr_reduce = simd_shuffle_xor(qgr_reduce, ushort(8));- sgr_reduce = Op::apply(qgr_reduce, sgr_reduce);-- reduced_vals[i] = Op::apply(reduced_vals[i], sgr_reduce);- }- }-- template <typename Op, typename T>- METAL_FUNC static constexpr void row_bin_op(- thread dtype_frag_t<T>& inp_vals,- thread T* row_vals- ) {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- inp_vals[i * kElemCols + j] =- Op::apply(inp_vals[i * kElemCols + j], row_vals[i]);- }- }- }-};--template <- typename T,- short kRows_,- short kCols_,- typename NAXFrag_t = BaseNAXFrag>-struct NAXSubTile {- STEEL_CONST short kRows = kRows_;- STEEL_CONST short kCols = kCols_;-- STEEL_CONST short kFragRows = NAXFrag_t::kFragRows;- STEEL_CONST short kFragCols = NAXFrag_t::kFragCols;- STEEL_CONST short kElemsPerFrag = NAXFrag_t::kElemsPerFrag;-- STEEL_CONST short kSubTileRows = kRows / kFragRows;- STEEL_CONST short kSubTileCols = kCols / kFragCols;-- STEEL_CONST short kNumFrags = kSubTileRows * kSubTileCols;- STEEL_CONST short kElemsPerSubTile = kNumFrags * kElemsPerFrag;-- STEEL_CONST int kRowsPerThread = kSubTileRows * NAXFrag_t::kElemRows;- STEEL_CONST int kColsPerThread = kSubTileCols * NAXFrag_t::kElemCols;-- STEEL_CONST short kFragThrRows = NAXFrag_t::kElemRows;- STEEL_CONST short kFragThrCols = NAXFrag_t::kElemCols;- STEEL_CONST short kFragRowsJump = NAXFrag_t::kElemRowsJump;-- using frag_type = typename NAXFrag_t::template dtype_frag_t<T>;-- frag_type val_frags[kNumFrags];-- METAL_FUNC constexpr void clear() {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kNumFrags; ++i) {- val_frags[i] = frag_type(0);- }- }-- METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {- return val_frags[i * kSubTileCols + j];- }-- METAL_FUNC constexpr const thread frag_type& frag_at(- const short i,- const short j- ) const {- return val_frags[i * kSubTileCols + j];- }-- template <int i, int j>- METAL_FUNC constexpr thread frag_type& frag_at() {- return val_frags[i * kSubTileCols + j];- }-- template <int i, int j>- METAL_FUNC constexpr const thread frag_type& frag_at() const {- return val_frags[i * kSubTileCols + j];- }-- METAL_FUNC thread T* elems() {- return reinterpret_cast<thread T*>(val_frags);- }-- METAL_FUNC const thread T* elems() const {- return reinterpret_cast<const thread T*>(val_frags);- }-- template <typename Op>- METAL_FUNC void row_reduce(thread metal::vec<T, kRowsPerThread>& vals) const {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kSubTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kSubTileCols; ++j) {- NAXFrag_t::template row_reduce<Op>(- frag_at(i, j),- &vals[i * kFragThrRows]- );- }- }- }-- template <typename Op>- METAL_FUNC void row_bin_op(thread metal::vec<T, kRowsPerThread>& vals) {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kSubTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kSubTileCols; ++j) {- NAXFrag_t::template row_bin_op<Op>(- frag_at(i, j),- &vals[i * kFragThrRows]- );- }- }- }-- template <- typename SrcPtrType,- typename StrX,- typename StrY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC constexpr void load(- SrcPtrType src,- StrX str_x,- StrY str_y,- OffX off_x = {},- OffY off_y = {}- ) {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kSubTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kSubTileCols; ++j) {- NAXFrag_t::load(- frag_at(i, j),- src,- str_x,- str_y,- off_x + i * kFragRows,- off_y + j * kFragCols- );- }- }- }-- template <- typename DstPtrType,- typename StrX,- typename StrY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC constexpr void store(- DstPtrType dst,- StrX str_x,- StrY str_y,- OffX off_x = {},- OffY off_y = {}- ) const {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kSubTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kSubTileCols; ++j) {- NAXFrag_t::store(- frag_at(i, j),- dst,- str_x,- str_y,- off_x + i * kFragRows,- off_y + j * kFragCols- );- }- }- }-- template <- typename SrcPtrType,- typename StrX,- typename StrY,- typename LimX,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC constexpr void load_rows(- SrcPtrType src,- StrX str_x,- StrY str_y,- LimX lim_x,- OffX off_x = {},- OffY off_y = {}- ) {- STEEL_PRAGMA_UNROLL- for (int i = 0; i < kSubTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < kSubTileCols; ++j) {- NAXFrag_t::load_rows(- frag_at(i, j),- src,- str_x,- str_y,- lim_x,- off_x + (i * kFragRows),- off_y + (j * kFragCols)- );- }- }- }-- template <- typename SrcPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC constexpr void load_safe(- SrcPtrType src,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = {},- OffY off_y = {}- ) {- STEEL_PRAGMA_UNROLL- for (int i = 0; i < kSubTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < kSubTileCols; ++j) {- NAXFrag_t::load_safe(- frag_at(i, j),- src,- str_x,- str_y,- lim_x,- lim_y,- off_x + (i * kFragRows),- off_y + (j * kFragCols)- );- }- }- }-- template <- typename DstPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC constexpr void store_safe(- DstPtrType dst,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = {},- OffY off_y = {}- ) const {- STEEL_PRAGMA_UNROLL- for (int i = 0; i < kSubTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < kSubTileCols; ++j) {- NAXFrag_t::store_safe(- frag_at(i, j),- dst,- str_x,- str_y,- lim_x,- lim_y,- off_x + (i * kFragRows),- off_y + (j * kFragCols)- );- }- }- }-- template <- typename DstPtrType,- typename StrX,- typename StrY,- typename LimX,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC constexpr void store_rows(- DstPtrType dst,- StrX str_x,- StrY str_y,- LimX lim_x,- OffX off_x = {},- OffY off_y = {}- ) const {- STEEL_PRAGMA_UNROLL- for (int i = 0; i < kSubTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < kSubTileCols; ++j) {- NAXFrag_t::store_safe(- frag_at(i, j),- dst,- str_x,- str_y,- lim_x,- off_x + (i * kFragRows),- off_y + (j * kFragCols)- );- }- }- }-- template <- typename DstPtrType,- typename StrX,- typename StrY,- typename StartX,- typename StopX,- typename StartY,- typename StopY,- typename OffX = Int<0>,- typename OffY = Int<0>>- METAL_FUNC constexpr void store_slice(- DstPtrType dst,- StrX str_x,- StrY str_y,- StartX start_x,- StopX stop_x,- StartY start_y,- StopY stop_y,- OffX off_x = Int<0>{},- OffY off_y = Int<0>{}- ) const {- const_for_loop<0, kSubTileRows, 1>([&](auto idx_row) {- const_for_loop<0, kSubTileCols, 1>([&](auto idx_col) {- NAXFrag_t::store_slice(- frag_at<idx_row.value, idx_col.value>(),- dst,- str_x,- str_y,- start_x,- stop_x,- start_y,- stop_y,- off_x + idx_row * Int<kFragRows>{},- off_y + idx_col * Int<kFragCols>{}- );- });- });- }-};--template <- short RC,- short CC,- short RA,- short CA,- short RB,- short CB,- typename CType,- typename AType,- typename BType,- bool transpose_a,- bool transpose_b,- typename NAXFrag_t = BaseNAXFrag>-METAL_FUNC void subtile_matmad_nax(- thread NAXSubTile<CType, RC, CC, NAXFrag_t>& C,- thread NAXSubTile<AType, RA, CA, NAXFrag_t>& A,- metal::bool_constant<transpose_a>,- thread NAXSubTile<BType, RB, CB, NAXFrag_t>& B,- metal::bool_constant<transpose_b>-) {- // Static checks- constexpr short FMa = transpose_a ? CA : RA;- constexpr short FMc = RC;- static_assert(FMa == FMc, "NAX matmul: M dimensions do not match");-- constexpr short FNb = transpose_b ? RB : CB;- constexpr short FNc = CC;- static_assert(FNb == FNc, "NAX matmul: N dimensions do not match");-- constexpr short FKa = transpose_a ? RA : CA;- constexpr short FKb = transpose_b ? CB : RB;- static_assert(FKa == FKb, "NAX matmul: N dimensions do not match");-- constexpr short FM = FMc;- constexpr short FN = FNc;- constexpr short FK = FKa;-- constexpr int TM = FM / 16;- constexpr int TN = FN / 16;- constexpr int TK = FK / 16;-- // Create Matmul descriptor- constexpr auto desc = mpp::tensor_ops::matmul2d_descriptor(- FM,- FN,- FK,- transpose_a,- transpose_b,- true,- mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate- );-- // Create matmul op- mpp::tensor_ops::matmul2d<desc, metal::execution_simdgroup> gemm_op;-- // Create matmul operands in registers- auto ct_a =- gemm_op.template get_left_input_cooperative_tensor<AType, BType, CType>();- auto ct_b =- gemm_op- .template get_right_input_cooperative_tensor<AType, BType, CType>();-- // Create matmul output in register- auto ct_c = gemm_op.template get_destination_cooperative_tensor<- decltype(ct_a),- decltype(ct_b),- CType>();-- // Load A in to left operand registers- STEEL_PRAGMA_UNROLL- for (short mm = 0; mm < TM; mm++) {- STEEL_PRAGMA_UNROLL- for (short kk = 0; kk < TK; kk++) {- const short fi = transpose_a ? kk : mm;- const short fj = transpose_a ? mm : kk;-- STEEL_PRAGMA_UNROLL- for (short i = 0; i < 8; i++) {- ct_a[(TK * mm + kk) * 8 + i] = A.frag_at(fi, fj)[i];- }- }- }-- // Load B into right operand registers- STEEL_PRAGMA_UNROLL- for (short nn = 0; nn < TN; nn++) {- STEEL_PRAGMA_UNROLL- for (short kk = 0; kk < TK; kk++) {- const short fi = transpose_b ? nn : kk;- const short fj = transpose_b ? kk : nn;-- STEEL_PRAGMA_UNROLL- for (short i = 0; i < 8; i++) {- ct_b[(TN * kk + nn) * 8 + i] = B.frag_at(fi, fj)[i];- }- }- }-- // Load C into output registers (op handles accumulation)- STEEL_PRAGMA_UNROLL- for (short i = 0; i < ct_c.get_capacity(); i++) {- ct_c[i] = C.elems()[i];- }-- // Do matmul- gemm_op.run(ct_a, ct_b, ct_c);-- // Copy out results- STEEL_PRAGMA_UNROLL- for (short i = 0; i < ct_c.get_capacity(); i++) {- C.elems()[i] = ct_c[i];- }-}--template <typename T, short kTileRows_, short kTileCols_, class NAXSubTile_>-struct NAXTile {- using NAXSubTile_t = NAXSubTile_;- using elem_type = T;- STEEL_CONST short kSubTileRows = NAXSubTile_t::kRows;- STEEL_CONST short kSubTileCols = NAXSubTile_t::kCols;- STEEL_CONST short kElemsPerSubTile = NAXSubTile_t::kElemsPerSubTile;-- STEEL_CONST short kTileRows = kTileRows_;- STEEL_CONST short kTileCols = kTileCols_;-- STEEL_CONST short kRows = kTileRows * kSubTileRows;- STEEL_CONST short kCols = kTileCols * kSubTileCols;-- STEEL_CONST short kSubTiles = kTileRows * kTileCols;- STEEL_CONST short kElemsPerTile = kSubTiles * kElemsPerSubTile;-- STEEL_CONST short kRowsPerThread = kTileRows * NAXSubTile_t::kRowsPerThread;- STEEL_CONST short kColsPerThread = kTileCols * NAXSubTile_t::kColsPerThread;-- STEEL_CONST short kSubTileThrRows = NAXSubTile_t::kRowsPerThread;- STEEL_CONST short kSubTileThrCols = NAXSubTile_t::kColsPerThread;-- NAXSubTile_t val_subtiles[kSubTiles];-- METAL_FUNC NAXTile() thread {}-- METAL_FUNC constexpr void clear() {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kSubTiles; ++i) {- val_subtiles[i].clear();- }- }-- METAL_FUNC constexpr thread NAXSubTile_t& subtile_at(- const short i,- const short j- ) {- return val_subtiles[i * kTileCols + j];- }-- METAL_FUNC constexpr const thread NAXSubTile_t& subtile_at(- const short i,- const short j- ) const {- return val_subtiles[i * kTileCols + j];- }-- template <int i, int j>- METAL_FUNC constexpr const thread NAXSubTile_t& subtile_at() const {- return val_subtiles[i * kTileCols + j];- }-- METAL_FUNC thread elem_type* elems() {- return reinterpret_cast<thread elem_type*>(val_subtiles[0].elems());- }-- METAL_FUNC const thread elem_type* elems() const {- return reinterpret_cast<const thread elem_type*>(val_subtiles[0].elems());- }-- template <typename Op>- METAL_FUNC void row_reduce(thread metal::vec<T, kRowsPerThread>& vals) const {- auto sub_rows = (thread metal::vec<T, kSubTileThrRows>*)(&vals);- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {- subtile_at(i, j).template row_reduce<Op>(sub_rows[i]);- }- }- }-- template <typename Op>- METAL_FUNC void row_bin_op(thread metal::vec<T, kRowsPerThread>& vals) {- auto sub_rows = (thread metal::vec<T, kSubTileThrRows>*)(&vals);- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {- subtile_at(i, j).template row_bin_op<Op>(sub_rows[i]);- }- }- }-- template <typename U, int str_x, int str_y>- METAL_FUNC void load(const threadgroup U* src) {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {- subtile_at(i, j).load(- src,- Int<str_x>{},- Int<str_y>{},- i * kSubTileRows,- j * kSubTileCols- );- }- }- }-- template <typename U, int str_x, int str_y>- METAL_FUNC void store(threadgroup U* dst) const {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {- subtile_at(i, j).store(- dst,- Int<str_x>{},- Int<str_y>{},- i * kSubTileRows,- j * kSubTileCols- );- }- }- }-- template <typename U>- METAL_FUNC void load(const device U* src, const int ld) {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {- subtile_at(i, j).load(- &src[(i * kSubTileRows * ld + j * kSubTileCols)],- ld,- Int<1>{}- );- }- }- }-- template <typename U>- METAL_FUNC void store(device U* dst, const int ld) const {- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {- subtile_at(i, j).store(- &dst[(i * kSubTileRows * ld + j * kSubTileCols)],- ld,- Int<1>{}- );- }- }- }-- template <typename U>- METAL_FUNC void load_rows(- const device U* src,- const int ld,- const short n_rows- ) {- STEEL_PRAGMA_UNROLL- for (int i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < kTileCols; ++j) {- subtile_at(i, j).load_rows(- &src[(i * kSubTileRows) * ld + (j * kSubTileCols)],- ld,- Int<1>{},- n_rows - i * kSubTileRows- );- }- }- }-- template <typename U>- METAL_FUNC void load_safe(- const device U* src,- const int ld,- const short2 src_tile_dims- ) {- STEEL_PRAGMA_UNROLL- for (int i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < kTileCols; ++j) {- subtile_at(i, j).load_safe(- src,- ld,- Int<1>{},- src_tile_dims.y,- src_tile_dims.x,- i * kSubTileRows,- j * kSubTileCols- );- }- }- }-- template <typename U>- METAL_FUNC void store_rows(- device U* dst,- const int ld,- const short n_rows- ) const {- STEEL_PRAGMA_UNROLL- for (int i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < kTileCols; ++j) {- subtile_at(i, j).store_rows(- &dst[(i * kSubTileRows) * ld + (j * kSubTileCols)],- ld,- Int<1>{},- n_rows - i * kSubTileRows- );- }- }- }-- template <typename U>- METAL_FUNC void store_safe(- device U* dst,- const int ld,- const short2 dst_tile_dims- ) const {- STEEL_PRAGMA_UNROLL- for (int i = 0; i < kTileRows; ++i) {- STEEL_PRAGMA_UNROLL- for (int j = 0; j < kTileCols; ++j) {- subtile_at(i, j).store_safe(- dst,- ld,- Int<1>{},- dst_tile_dims.y,- dst_tile_dims.x,- i * kSubTileRows,- j * kSubTileCols- );- }- }- }-- template <typename U>- METAL_FUNC void store_slice(- device U* dst,- const int ld,- const short2 start,- const short2 stop- ) const {- const_for_loop<0, kTileRows, 1>([&](auto idx_row) {- const_for_loop<0, kTileCols, 1>([&](auto idx_col) {- subtile_at<idx_row.value, idx_col.value>().store_slice(- dst,- ld,- Int<1>{},- start.y,- stop.y,- start.x,- stop.x,- idx_row * Int<kSubTileRows>{},- idx_col * Int<kSubTileCols>{}- );- });- });- }-};--template <- class CTile,- class ATile,- class BTile,- bool transpose_a,- bool transpose_b>-METAL_FUNC void tile_matmad_nax(- thread CTile& C,- thread ATile& A,- metal::bool_constant<transpose_a>,- thread BTile& B,- metal::bool_constant<transpose_b>-) {- // Static checks- constexpr short TMa = transpose_a ? ATile::kTileCols : ATile::kTileRows;- constexpr short TMc = CTile::kTileRows;- static_assert(TMa == TMc, "NAX tile matmul: M dimensions do not match");-- constexpr short FMa = transpose_a ? ATile::kSubTileCols : ATile::kSubTileRows;- constexpr short FMc = CTile::kSubTileRows;- static_assert(FMa == FMc, "NAX subtile matmul: M dimensions do not match");-- constexpr short TNb = transpose_b ? BTile::kTileRows : BTile::kTileCols;- constexpr short TNc = CTile::kTileCols;- static_assert(TNb == TNc, "NAX tile matmul: N dimensions do not match");-- constexpr short FNb = transpose_b ? BTile::kSubTileRows : BTile::kSubTileCols;- constexpr short FNc = CTile::kSubTileCols;- static_assert(FNb == FNc, "NAX subtile matmul: N dimensions do not match");-- constexpr short TKa = transpose_a ? ATile::kTileRows : ATile::kTileCols;- constexpr short TKb = transpose_b ? BTile::kTileCols : BTile::kTileRows;- static_assert(TKa == TKb, "NAX tile matmul: K dimensions do not match");-- constexpr short FKa = transpose_a ? ATile::kSubTileRows : ATile::kSubTileCols;- constexpr short FKb = transpose_b ? BTile::kSubTileCols : BTile::kSubTileRows;- static_assert(FKa == FKb, "NAX subtile matmul: K dimensions do not match");-- constexpr short TM = TMc;- constexpr short TN = TNc;- constexpr short TK = TKa;-- // Do matmul here- STEEL_PRAGMA_UNROLL- for (short i = 0; i < TM; ++i) {- STEEL_PRAGMA_UNROLL- for (short j = 0; j < TN; ++j) {- STEEL_PRAGMA_UNROLL- for (short k = 0; k < TK; ++k) {- const short ra = transpose_a ? k : i;- const short ca = transpose_a ? i : k;- const short rb = transpose_b ? j : k;- const short cb = transpose_b ? k : j;-- subtile_matmad_nax(- C.subtile_at(i, j),- A.subtile_at(ra, ca),- metal::bool_constant<transpose_a>{},- B.subtile_at(rb, cb),- metal::bool_constant<transpose_b>{}- );- }- }- }-}--} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.hindex e94df081f..fd8cb42ce 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h@@ -49,14 +49,4 @@ struct GEMMSpiltKParams {const int gemm_k_iterations_aligned;};-struct GEMMAddMMParams {- const int ldc;- const int fdc;-- const int64_t batch_stride_c;-- const float alpha;- const float beta;-};-} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.hindex 87246822c..f0cf5a745 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h@@ -2,8 +2,6 @@#pragma once-#include "../utils.h"-///////////////////////////////////////////////////////////////////////////////// Transforms and Epilogues///////////////////////////////////////////////////////////////////////////////diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.hdeleted file mode 100644index 78952e7b7..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.h+++ /dev/null@@ -1,42 +0,0 @@-#pragma once--#include <metal_stdlib>--METAL_FUNC ulong2 elem_to_loc_broadcast(- uint elem,- constant const int* shape,- constant const int64_t* a_strides,- constant const int64_t* b_strides,- int ndim-) {- ulong loc_a{0};- ulong loc_b{0};- for (int i = ndim - 1; i >= 0 && elem > 0; --i) {- int pos_in_dim = (elem % shape[i]);- elem /= shape[i];- loc_a += pos_in_dim * a_strides[i];- loc_b += pos_in_dim * b_strides[i];- }- return ulong2(loc_a, loc_b);-}--METAL_FUNC ulong3 elem_to_loc_broadcast(- uint elem,- constant const int* shape,- constant const int64_t* a_strides,- constant const int64_t* b_strides,- constant const int64_t* c_strides,- int ndim-) {- ulong loc_a{0};- ulong loc_b{0};- ulong loc_c{0};- for (int i = ndim - 1; i >= 0 && elem > 0; --i) {- int pos_in_dim = (elem % shape[i]);- elem /= shape[i];- loc_a += pos_in_dim * a_strides[i];- loc_b += pos_in_dim * b_strides[i];- loc_c += pos_in_dim * c_strides[i];- }- return ulong3(loc_a, loc_b, loc_c);-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.hdeleted file mode 100644index 5cf7e9645..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h+++ /dev/null@@ -1,131 +0,0 @@-#pragma once--#include <metal_stdlib>-#include "type_traits.h"-#include "../../../../definitions.metal"--#pragma METAL internals : enable--namespace steel {--///////////////////////////////////////////////////////////////////////////////-// Integral constant with casting-///////////////////////////////////////////////////////////////////////////////--template <typename T, T v>-struct integral_constant {- static constexpr constant T value = v;- using value_type = T;- using type = integral_constant;-- METAL_FUNC constexpr operator value_type() const noexcept { return value; }-- // METAL_FUNC constexpr value_type operator()() const noexcept {- // return value;- // }-};--template <bool B>-using bool_constant = integral_constant<bool, B>;-using true_type = bool_constant<true>;-using false_type = bool_constant<false>;--template <class T>-struct is_integral : bool_constant<metal::is_integral<T>::value> {};--template <class T, T v>-struct is_integral<integral_constant<T, v>>- : bool_constant<metal::is_integral<T>::value> {};--template <typename T>-constexpr constant bool is_integral_v = is_integral<T>::value;--template <int val>-using Int = integral_constant<int, val>;--///////////////////////////////////////////////////////////////////////////////-// Binary Operators on Integral constants-///////////////////////////////////////////////////////////////////////////////--#define integral_const_binop(__op__, __operator__) \- template <typename T, T tv, typename U, U uv> \- METAL_FUNC constexpr auto __operator__( \- integral_constant<T, tv>, \- integral_constant<U, uv> \- ) { \- constexpr auto res = tv __op__ uv; \- return integral_constant<decltype(res), res>{}; \- }--integral_const_binop(+, operator+);-integral_const_binop(-, operator-);-integral_const_binop(*, operator*);-integral_const_binop(/, operator/);--integral_const_binop(==, operator==);-integral_const_binop(!=, operator!=);-integral_const_binop(<, operator<);-integral_const_binop(>, operator>);-integral_const_binop(<=, operator<=);-integral_const_binop(>=, operator>=);--integral_const_binop(&&, operator&&);-integral_const_binop(||, operator||);--template <typename T, typename = metal::enable_if_t<!is_integral_v<T>>>-METAL_FUNC constexpr auto operator||(true_type, T) {- return true_type{};-}-template <typename T, typename = metal::enable_if_t<!is_integral_v<T>>>-METAL_FUNC constexpr auto operator||(T, true_type) {- return true_type{};-}--template <typename T, typename = metal::enable_if_t<!is_integral_v<T>>>-METAL_FUNC constexpr auto operator&&(false_type, T) {- return false_type{};-}--template <typename T, typename = metal::enable_if_t<!is_integral_v<T>>>-METAL_FUNC constexpr auto operator&&(T, false_type) {- return false_type{};-}--// Dispatch utilities-template <typename F>-void dispatch_bool(bool v, F f) {- if (v) {- f(true_type{});- } else {- f(false_type{});- }-}--template <int start, int stop, int step, typename F>-constexpr void const_for_loop(F f) {- IF_CONSTEXPR(start < stop) {- constexpr auto idx = Int<start>{};- f(idx);- const_for_loop<start + step, stop, step, F>(f);- }-}--#undef integral_const_binop--///////////////////////////////////////////////////////////////////////////////-// Reduction operators-///////////////////////////////////////////////////////////////////////////////--template <typename T>-METAL_FUNC constexpr T sum(T x) {- return x;-}--template <typename T, typename... Us>-METAL_FUNC constexpr auto sum(T x, Us... us) {- return x + sum(us...);-}--} // namespace steel--#pragma METAL internals : disablediff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rsdeleted file mode 100644index b2785dd25..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs+++ /dev/null@@ -1,30 +0,0 @@-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub(crate) enum TransposeConfiguration {- NN,- NT,- TN,- TT,-}--impl TransposeConfiguration {- pub fn as_str(&self) -> &'static str {- match self {- TransposeConfiguration::NN => "nn",- TransposeConfiguration::NT => "nt",- TransposeConfiguration::TN => "tn",- TransposeConfiguration::TT => "tt",- }- }-}--pub(crate) fn transpose_configuration(- a_t: bool,- b_t: bool,-) -> TransposeConfiguration {- match (a_t, b_t) {- (false, false) => TransposeConfiguration::NN,- (false, true) => TransposeConfiguration::NT,- (true, false) => TransposeConfiguration::TN,- (true, true) => TransposeConfiguration::TT,- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rsindex 4b3585b17..0e562b3f8 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs@@ -1,63 +1,24 @@-use super::{common::MatmulArguments, gemm, gemv, split_k};+use super::gemm;use crate::{DataType,- backends::metal::{context::MetalContext, error::MetalError},+ backends::{+ common::kernel::matmul::{MatmulArguments, MatmulDispatchDescriptor, gemv, split_k},+ metal::{Metal, context::MetalContext, error::MetalError},+ },};-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum MatmulKernelVariant {- Gemv,- SplitK,- Gemm,-}--impl MatmulKernelVariant {- pub fn as_str(&self) -> &'static str {- match self {- MatmulKernelVariant::Gemv => "GEMV",- MatmulKernelVariant::SplitK => "Split-K",- MatmulKernelVariant::Gemm => "GEMM",- }- }-}--#[derive(Debug, Clone)]-pub(crate) enum MatmulDispatchDescriptor {- Gemv(gemv::DispatchDescriptor),- SplitK(split_k::DispatchDescriptor),- Gemm(gemm::DispatchDescriptor),-}--impl MatmulDispatchDescriptor {- pub fn variant(&self) -> MatmulKernelVariant {- match self {- MatmulDispatchDescriptor::Gemv(_) => MatmulKernelVariant::Gemv,- MatmulDispatchDescriptor::SplitK(_) => MatmulKernelVariant::SplitK,- MatmulDispatchDescriptor::Gemm(_) => MatmulKernelVariant::Gemm,- }- }-}--pub(crate) fn choose_dispatch_descriptor(+pub fn choose_dispatch_descriptor(context: &MetalContext,data_type: DataType,- arguments: &MatmulArguments,+ arguments: &MatmulArguments<Metal>,) -> Result<MatmulDispatchDescriptor, MetalError> {- if let Some(descriptor) = gemv::DispatchDescriptor::try_new(context, data_type, arguments)? {+ if let Some(descriptor) = gemv::DispatchDescriptor::try_new::<Metal>(data_type, arguments)? {return Ok(MatmulDispatchDescriptor::Gemv(descriptor));}- if let Some(descriptor) = split_k::DispatchDescriptor::try_new(context, data_type, arguments)? {+ if let Some(descriptor) = split_k::DispatchDescriptor::try_new::<Metal>(data_type, arguments)? {return Ok(MatmulDispatchDescriptor::SplitK(descriptor));}Ok(MatmulDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(context, data_type, arguments)?))}--pub fn determine_kernel_variant(- context: &MetalContext,- data_type: DataType,- arguments: &MatmulArguments,-) -> Result<MatmulKernelVariant, MetalError> {- choose_dispatch_descriptor(context, data_type, arguments).map(|d| d.variant())-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rsindex 5765c3836..b0c2087c3 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs@@ -1,55 +1,35 @@-use metal::MTLSize;--use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::TileConfiguration};+// Re-export the common DispatchDescriptor type — we add a constructor here+pub use crate::backends::common::kernel::matmul::gemm::DispatchDescriptor;use crate::{DataType,- backends::metal::{- context::{DeviceClass, MetalContext},- error::MetalError,- kernel::matmul::common::{GEMMAddMMParams, GEMMParams, MatmulArguments},+ backends::{+ common::{+ gpu_types::GEMMParams,+ kernel::matmul::{GridSize, MatmulArguments, gemm::Specialization},+ },+ metal::{Metal, context::MetalContext, error::MetalError},},};-#[derive(Debug, Clone)]-pub(crate) struct DispatchDescriptor {- pub(crate) pipeline_configuration: PipelineConfiguration,- pub(crate) params: GEMMParams,- pub(crate) addmm_params: Option<GEMMAddMMParams>,- pub(crate) threadgroups: MTLSize,- pub(crate) threads_per_threadgroup: MTLSize,-}-impl DispatchDescriptor {- pub(crate) fn new(+ pub fn new(context: &MetalContext,data_type: DataType,- arguments: &MatmulArguments,+ arguments: &MatmulArguments<Metal>,) -> Result<Self, MetalError> {if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(MetalError::Generic(format!("Unsupported dtype for GEMM: {data_type:?}")));}- let tile = select_tile(context, data_type, arguments);+ let config = Specialization::select(context, data_type, arguments);let m = arguments.batch;let n = arguments.output_dim;let k = arguments.input_dim;- let pipeline_configuration = PipelineConfiguration {- tile,- transpose_a: arguments.transpose_a,- transpose_b: arguments.transpose_b,- align_m: (m % tile.block_rows) == 0,- align_n: (n % tile.block_cols) == 0,- align_k: (k % tile.block_depth) == 0,- has_batch: arguments.batch_count > 1,- use_out_source: arguments.c.is_some(),- do_axpby: arguments.c.is_some() && (arguments.alpha != 1.0 || arguments.beta != 0.0),- };-- let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;- let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;- let swizzle_log = tile.swizzle_log2;+ let tiles_n = (n + config.block_cols - 1) / config.block_cols;+ let tiles_m = (m + config.block_rows - 1) / config.block_rows;+ let swizzle_log = config.swizzle_log2;let tile_swizzle = 1 << swizzle_log;let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;@@ -76,104 +56,20 @@ impl DispatchDescriptor {batch_stride_b: elements_per_matrix_b,batch_stride_d: elements_per_matrix_d,swizzle_log,- gemm_k_iterations_aligned: k / tile.block_depth,+ gemm_k_iterations_aligned: k / config.block_depth,batch_ndim: 1,};- let addmm_params = if pipeline_configuration.use_out_source {- let batch_stride_c = if arguments.batch_count > 1 {- (arguments.ldd as i64) * (arguments.output_dim as i64)- } else {- 0- };- Some(GEMMAddMMParams {- ldc: arguments.ldd,- fdc: 1,- batch_stride_c,- alpha: arguments.alpha,- beta: arguments.beta,- })- } else {- None+ let threadgroups = GridSize {+ x: tn_swizzled as usize,+ y: tm_swizzled as usize,+ z: arguments.batch_count as usize,};- let threads_per_threadgroup = MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);- let threadgroups = MTLSize::new(tn_swizzled as usize, tm_swizzled as usize, arguments.batch_count as usize);-Ok(Self {- pipeline_configuration,+ specialization: config,params,- addmm_params,threadgroups,- threads_per_threadgroup,})}}--fn select_tile(- context: &MetalContext,- data_type: DataType,- arguments: &MatmulArguments,-) -> TileConfiguration {- let overall_work_elements =- (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);- let is_float32 = matches!(data_type, DataType::F32);- let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();-- if context.is_nax_available() && prefer_half_or_tf32 {- let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);- let tile_rows = (arguments.batch + base_tile.block_rows - 1) / base_tile.block_rows;- let swizzle_log2 = if tile_rows <= 3 {- 0- } else {- 1- };- return TileConfiguration {- swizzle_log2,- ..base_tile- };- }-- match context.device_class() {- DeviceClass::Integrated | DeviceClass::Phone | DeviceClass::Unknown(_) => {- if prefer_half_or_tf32 {- if !arguments.transpose_a && arguments.transpose_b {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- } else {- TileConfiguration::new(64, 64, 16, 1, 2, 0)- }- } else if !arguments.transpose_a && arguments.transpose_b {- TileConfiguration::new(32, 64, 16, 1, 2, 0)- } else {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- }- },- DeviceClass::Desktop => {- if overall_work_elements >= (1_i64 << 20) {- if prefer_half_or_tf32 {- if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {- TileConfiguration::new(64, 64, 16, 2, 2, 0)- } else if !arguments.transpose_a && arguments.transpose_b {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- } else {- TileConfiguration::new(32, 64, 16, 1, 2, 0)- }- } else if !arguments.transpose_a && arguments.transpose_b {- TileConfiguration::new(32, 64, 16, 1, 2, 0)- } else {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- }- } else if prefer_half_or_tf32 {- if !arguments.transpose_a && arguments.transpose_b {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- } else {- TileConfiguration::new(64, 64, 16, 1, 2, 0)- }- } else if !arguments.transpose_a && arguments.transpose_b {- TileConfiguration::new(32, 64, 16, 1, 2, 0)- } else {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- }- },- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsdeleted file mode 100644index 493b03e46..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ /dev/null@@ -1,170 +0,0 @@-use std::collections::HashMap;--use metal::{MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration, tile_configuration::TileConfiguration};-use crate::{- DataType,- backends::metal::{- context::MetalContext,- error::MetalError,- kernel::matmul::common::{MatmulArguments, transpose_configuration},- metal_extensions::{ComputeEncoderSetValue, FunctionConstantValuesSetValue},- },-};--pub struct Kernel {- data_type: DataType,- pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,-}--impl Kernel {- pub fn new(data_type: DataType) -> Result<Self, MetalError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MetalError::Generic(format!("Unsupported dtype for GEMM: {data_type:?}")));- }- Ok(Self {- data_type,- pipelines: HashMap::new(),- })- }-- #[allow(clippy::type_complexity)]- pub fn precompile(- &mut self,- context: &MetalContext,- ) -> Result<(), MetalError> {- let tiles_and_alignments: &[(TileConfiguration, &[(bool, bool, bool)])] = match self.data_type {- DataType::BF16 => &[- (TileConfiguration::new(64, 32, 32, 2, 2, 0), &[(false, true, true), (true, true, true)]),- (- TileConfiguration::new(64, 64, 16, 2, 2, 0),- &[(false, true, true), (true, false, true), (true, true, true)],- ),- (TileConfiguration::new(64, 64, 16, 1, 2, 0), &[(true, true, true)]),- ],- DataType::F16 => {- &[(TileConfiguration::new(64, 64, 16, 2, 2, 0), &[(true, true, true), (false, true, true)])]- },- DataType::F32 => {- &[(TileConfiguration::new(32, 64, 16, 1, 2, 0), &[(false, true, true), (true, true, true)])]- },- _ => return Ok(()),- };-- for (tile, alignments) in tiles_and_alignments {- for &(align_m, align_n, align_k) in *alignments {- let config = PipelineConfiguration {- tile: *tile,- transpose_a: false,- transpose_b: true,- align_m,- align_n,- align_k,- has_batch: false,- use_out_source: false,- do_axpby: false,- };- let _ = self.get_or_compile_pipeline(context, &config);- }- }-- Ok(())- }-- fn type_name(&self) -> &'static str {- match self.data_type {- DataType::F16 => "float16",- DataType::BF16 => "bfloat16",- DataType::F32 => "float32",- _ => unreachable!(),- }- }-- fn kernel_name(- &self,- configuration: &PipelineConfiguration,- ) -> String {- let type_name = self.type_name();- let transpose_suffix = transpose_configuration(configuration.transpose_a, configuration.transpose_b);- let prefix = if configuration.tile.is_nax() {- "steel_gemm_nax"- } else {- "steel_gemm"- };- format!(- "{}_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}",- prefix,- transpose_suffix.as_str(),- type_name,- type_name,- configuration.tile.block_rows,- configuration.tile.block_cols,- configuration.tile.block_depth,- configuration.tile.warps_per_row,- configuration.tile.warps_per_col- )- }-- fn get_or_compile_pipeline(- &mut self,- context: &MetalContext,- configuration: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {- if !self.pipelines.contains_key(configuration) {- let kernel_name = self.kernel_name(configuration);- let function_constants = MTLFunctionConstantValues::new();- function_constants.set_value(&configuration.has_batch, 10);- function_constants.set_value(&configuration.use_out_source, 100);- function_constants.set_value(&configuration.do_axpby, 110);- function_constants.set_value(&configuration.align_m, 200);- function_constants.set_value(&configuration.align_n, 201);- function_constants.set_value(&configuration.align_k, 202);-- let cache_key = format!(- "{}_am{}_an{}_ak{}_hb{}_uo{}_ax{}",- kernel_name,- configuration.align_m as u8,- configuration.align_n as u8,- configuration.align_k as u8,- configuration.has_batch as u8,- configuration.use_out_source as u8,- configuration.do_axpby as u8- );- let pipeline_state =- context.compute_pipeline_state_cached(&cache_key, &kernel_name, Some(&function_constants))?;- self.pipelines.insert(configuration.clone(), pipeline_state);- }- Ok(self.pipelines.get(configuration).unwrap())- }-- pub(crate) fn encode_descriptor(- &mut self,- context: &MetalContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MatmulArguments,- descriptor: &DispatchDescriptor,- ) -> Result<bool, MetalError> {- let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;- encoder.set_compute_pipeline_state(pipeline_state);-- encoder.set_buffer(Some(arguments.a), arguments.a_offset as usize, 0);- encoder.set_buffer(Some(arguments.b), 0, 1);- if descriptor.pipeline_configuration.use_out_source {- if let Some(c_buffer) = arguments.c {- encoder.set_buffer(Some(c_buffer), 0, 2);- }- }- encoder.set_buffer(Some(arguments.d), 0, 3);-- encoder.set_value(&descriptor.params, 4);-- if let Some(addmm_params) = &descriptor.addmm_params {- encoder.set_value(addmm_params, 5);- }-- encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);- Ok(false)- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rsindex 6b041d0fc..d9cb5df12 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -1,7 +1,4 @@mod dispatch_descriptor;-mod kernel;-mod pipeline_configuration;-mod tile_configuration;+mod specialization;-pub(crate) use dispatch_descriptor::DispatchDescriptor;-pub use kernel::Kernel as GemmKernel;+pub use dispatch_descriptor::DispatchDescriptor;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rsdeleted file mode 100644index 252792c91..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs+++ /dev/null@@ -1,14 +0,0 @@-use super::tile_configuration::TileConfiguration;--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct PipelineConfiguration {- pub tile: TileConfiguration,- pub transpose_a: bool,- pub transpose_b: bool,- pub align_m: bool,- pub align_n: bool,- pub align_k: bool,- pub has_batch: bool,- pub use_out_source: bool,- pub do_axpby: bool,-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.hdeleted file mode 100644index f201ef4af..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.h+++ /dev/null@@ -1,381 +0,0 @@---using namespace steel;--///////////////////////////////////////////////////////////////////////////////-// GEMM kernels-///////////////////////////////////////////////////////////////////////////////--constant bool has_batch [[function_constant(10)]];--constant bool use_out_source [[function_constant(100)]];-constant bool do_axpby [[function_constant(110)]];--constant bool align_M [[function_constant(200)]];-constant bool align_N [[function_constant(201)]];-constant bool align_K [[function_constant(202)]];--// clang-format off-template <- typename T,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- typename AccumType = float>-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gemm(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- const device T* C [[buffer(2), function_constant(use_out_source)]],- device T* D [[buffer(3)]],- const constant GEMMParams* params [[buffer(4)]],- const constant GEMMAddMMParams* addmm_params [[buffer(5), function_constant(use_out_source)]],- const constant int* batch_shape [[buffer(6), function_constant(has_batch)]],- const constant int64_t* batch_strides [[buffer(7), function_constant(has_batch)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]]) { // clang-format on- // Pacifying compiler- (void)lid;-- using gemm_kernel = GEMMKernel<- T,- T,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- true,- true,- AccumType>;-- using loader_a_t = typename gemm_kernel::loader_a_t;- using loader_b_t = typename gemm_kernel::loader_b_t;- using mma_t = typename gemm_kernel::mma_t;-- // Find block- const int tid_y = ((tid.y) << params->swizzle_log) +- ((tid.x) & ((1 << params->swizzle_log) - 1));- const int tid_x = (tid.x) >> params->swizzle_log;-- // Exit early if out of bounds- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {- return;- }-- // Adjust for batch- if (has_batch) {- const constant auto* A_bstrides = batch_strides;- const constant auto* B_bstrides = batch_strides + params->batch_ndim;-- ulong2 batch_offsets = elem_to_loc_broadcast(- tid.z,- batch_shape,- A_bstrides,- B_bstrides,- params->batch_ndim- );-- A += batch_offsets.x;- B += batch_offsets.y;-- if (use_out_source) {- const constant auto* C_bstrides = B_bstrides + params->batch_ndim;- C += elem_to_loc(tid.z, batch_shape, C_bstrides, params->batch_ndim);- }- } else {- A += params->batch_stride_a * tid.z;- B += params->batch_stride_b * tid.z;-- if (use_out_source) {- C += addmm_params->batch_stride_c * tid.z;- }- }-- D += params->batch_stride_d * tid.z;-- // Prepare threadgroup memory- threadgroup T As[gemm_kernel::tgp_mem_size_a];- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];-- threadgroup_barrier(mem_flags::mem_none);-- // Find block in A, B, C- const int c_row = tid_y * BM;- const int c_col = tid_x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- A += transpose_a ? c_row_long : c_row_long * params->lda;- B += transpose_b ? c_col_long * params->ldb : c_col_long;- D += c_row_long * params->ldd + c_col_long;-- if (use_out_source) {- C += c_row_long * addmm_params->ldc + c_col_long * addmm_params->fdc;- }-- // Prepare threadgroup mma operation- thread mma_t mma_op(simd_group_id, simd_lane_id);-- // Prepare threadgroup loading operations- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);-- // Prepare threadgroup bounds- const short tgp_bm = align_M ? BM : short(min(BM, params->M - c_row));- const short tgp_bn = align_N ? BN : short(min(BN, params->N - c_col));-- // Prepare iterations- int gemm_k_iterations = params->gemm_k_iterations_aligned;-- // Do unaligned K iterations first- if (!align_K) {- const int k_last = params->gemm_k_iterations_aligned * BK;- const int k_remain = params->K - k_last;- const size_t k_jump_a =- transpose_a ? params->lda * size_t(k_last) : size_t(k_last);- const size_t k_jump_b =- transpose_b ? size_t(k_last) : params->ldb * size_t(k_last);-- // Move loader source ahead to end- loader_a.src += k_jump_a;- loader_b.src += k_jump_b;-- // Load tile- const short2 tile_dims_A =- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);- const short2 tile_dims_B =- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);-- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Do matmul- mma_op.mma(As, Bs);-- // Reset source back to start- loader_a.src -= k_jump_a;- loader_b.src -= k_jump_b;- }-- const TransformAdd<AccumType, AccumType> epilogue_op_add(- addmm_params->alpha,- addmm_params->beta- );- const TransformAxpby<AccumType, AccumType> epilogue_op_axpby(- addmm_params->alpha,- addmm_params->beta- );-- ///////////////////////////////////////////////////////////////////////////////- // MNK aligned loop- if (align_M && align_N) {- // Do gemm- for (int k = 0; k < gemm_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- // Load elements into threadgroup- loader_a.load_unsafe();- loader_b.load_unsafe();-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }-- threadgroup_barrier(mem_flags::mem_none);-- // Do epilogue- if (use_out_source) {- if (do_axpby) {- mma_op.apply_epilogue(- C,- addmm_params->ldc,- addmm_params->fdc,- epilogue_op_axpby- );- } else {- mma_op.apply_epilogue(- C,- addmm_params->ldc,- addmm_params->fdc,- epilogue_op_add- );- }- }-- // Store results to device memory- return mma_op.store_result(D, params->ldd);-- }- ///////////////////////////////////////////////////////////////////////////////- // MN unaligned loop- else { // Loop over K - unaligned case- const int leftover_bk = 0;-- if ((align_M || tgp_bm == BM) && (align_N || tgp_bn == BN)) {- // Do gemm- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<true, true, true>{}- );-- // Do epilogue- if (use_out_source) {- if (do_axpby) {- mma_op.apply_epilogue(- C,- addmm_params->ldc,- addmm_params->fdc,- epilogue_op_axpby- );- } else {- mma_op.apply_epilogue(- C,- addmm_params->ldc,- addmm_params->fdc,- epilogue_op_add- );- }- }-- // Store results to device memory- return mma_op.store_result(D, params->ldd);-- } else if (align_N || tgp_bn == BN) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<false, true, true>{}- );-- // Do epilogue- if (use_out_source) {- if (do_axpby) {- mma_op.apply_epilogue_safe(- C,- addmm_params->ldc,- addmm_params->fdc,- short2(tgp_bn, tgp_bm),- epilogue_op_axpby- );- } else {- mma_op.apply_epilogue_safe(- C,- addmm_params->ldc,- addmm_params->fdc,- short2(tgp_bn, tgp_bm),- epilogue_op_add- );- }- }-- // Store results to device memory- return mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));-- } else if (align_M || tgp_bm == BM) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<true, false, true>{}- );-- // Do epilogue- if (use_out_source) {- if (do_axpby) {- mma_op.apply_epilogue_safe(- C,- addmm_params->ldc,- addmm_params->fdc,- short2(tgp_bn, tgp_bm),- epilogue_op_axpby- );- } else {- mma_op.apply_epilogue_safe(- C,- addmm_params->ldc,- addmm_params->fdc,- short2(tgp_bn, tgp_bm),- epilogue_op_add- );- }- }-- // Store results to device memory- return mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));-- } else {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<false, false, true>{}- );-- // Do epilogue- if (use_out_source) {- if (do_axpby) {- mma_op.apply_epilogue_safe(- C,- addmm_params->ldc,- addmm_params->fdc,- short2(tgp_bn, tgp_bm),- epilogue_op_axpby- );- } else {- mma_op.apply_epilogue_safe(- C,- addmm_params->ldc,- addmm_params->fdc,- short2(tgp_bn, tgp_bm),- epilogue_op_add- );- }- }-- // Store results to device memory- return mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metalindex 34c164aa3..6a8becf1d 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal@@ -1,34 +1,232 @@--// clang-format off#include "../../../common/utils.h"+#include "../../../definitions.metal"#include "../../common/steel/gemm/gemm.h"-#include "steel_gemm.h"--#define instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_kernel( \- "steel_gemm_" #tname "_" #iname "_" #oname \- "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn, \- gemm, itype, bm, bn, bk, wm, wn, trans_a, trans_b, float)--#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)--#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 1, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 32, 32, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 64, 16, 1, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 32, 8, 4, 1)--instantiate_gemm_shapes_helper(float16, half, float16, half);-instantiate_gemm_shapes_helper(bfloat16, bfloat16_t, bfloat16, bfloat16_t);--instantiate_gemm_shapes_helper(float32, float, float32, float);-instantiate_gemm_shapes_helper(complex64, complex64_t, complex64, complex64_t);++using namespace steel;++// Upper bounds for threadgroup memory (in elements of T).+// Max across all tile/type combos: 64x32x32 with half (padding=8).+// tgp_mem_size_a = BM * (BK + padding) = 64 * (32 + 8) = 2560+// tgp_mem_size_b = BN * (BK + padding) = 32 * (32 + 8) = 1280+// For 64x64x16 with half: a=64*(16+8)=1536, b=64*(16+8)=1536+// Max a = 2560, max b = 1536+#define GEMM_MAX_TGP_A 2560+#define GEMM_MAX_TGP_B 1536++namespace uzu {+namespace matmul {+using GEMMParams = steel::GEMMParams;+} // namespace matmul+} // namespace uzu++///////////////////////////////////////////////////////////////////////////////+// GEMM implementation+///////////////////////////////////////////////////////////////////////////////++template <typename T, typename AccumType = float>+METAL_FUNC void gemm_impl(+ const device T* a,+ const device T* b,+ device T* d,+ const constant GEMMParams* params,+ const int BM,+ const int BN,+ const int BK,+ const int WM,+ const int WN,+ const bool align_m,+ const bool align_n,+ const bool align_k,+ threadgroup T* a_shared,+ threadgroup T* b_shared,+ uint simd_lane_id,+ uint simd_group_id,+ uint3 tid,+ uint3 lid+) {+ (void)lid;++ // Hardcoded: transpose_a = false, transpose_b = true+ const bool transpose_a = false;+ const bool transpose_b = true;++ const short tgp_padding = 16 / sizeof(T);+ const short lda_tgp = BK + tgp_padding; // transpose_a=false: BK + pad+ const short ldb_tgp = BK + tgp_padding; // transpose_b=true: BK + pad+ const short tgp_size = WM * WN * 32;++ // Find block+ const int tid_y = ((tid.y) << params->swizzle_log) ++ ((tid.x) & ((1 << params->swizzle_log) - 1));+ const int tid_x = (tid.x) >> params->swizzle_log;++ if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {+ return;+ }++ // Batch offset (non-batched path only)+ a += params->batch_stride_a * tid.z;+ b += params->batch_stride_b * tid.z;+ d += params->batch_stride_d * tid.z;++ threadgroup_barrier(mem_flags::mem_none);++ // Find block in a, b, d+ const int c_row = tid_y * BM;+ const int c_col = tid_x * BN;+ const size_t c_row_long = size_t(c_row);+ const size_t c_col_long = size_t(c_col);++ // transpose_a=false: a += c_row * lda+ a += c_row_long * params->lda;+ // transpose_b=true: b += c_col * ldb+ b += c_col_long * params->ldb;+ d += c_row_long * params->ldd + c_col_long;++ // Construct loader and MMA objects with runtime tile params+ // transpose_a=false: loader_a BROWS=BM, BCOLS=BK, dst_ld=BK+pad, reduction_dim=1+ thread BlockLoader<T> loader_a(+ a, params->lda, a_shared,+ simd_group_id, simd_lane_id,+ BM, BK, lda_tgp, 1, tgp_size);++ // transpose_b=true: loader_b BROWS=BN, BCOLS=BK, dst_ld=BK+pad, reduction_dim=1+ thread BlockLoader<T> loader_b(+ b, params->ldb, b_shared,+ simd_group_id, simd_lane_id,+ BN, BK, ldb_tgp, 1, tgp_size);++ thread BlockMMA<T, T, AccumType> mma_op(+ simd_group_id, simd_lane_id,+ BM, BN, BK, WM, WN,+ transpose_a, transpose_b,+ lda_tgp, ldb_tgp);++ const short tgp_bm = align_m ? BM : short(min(BM, params->M - c_row));+ const short tgp_bn = align_n ? BN : short(min(BN, params->N - c_col));++ int gemm_k_iterations = params->gemm_k_iterations_aligned;++ // Do unaligned K iterations first+ if (!align_k) {+ const int k_last = params->gemm_k_iterations_aligned * BK;+ const int k_remain = params->K - k_last;+ // transpose_a=false: k_jump_a = k_last+ const size_t k_jump_a = size_t(k_last);+ // transpose_b=true: k_jump_b = k_last+ const size_t k_jump_b = size_t(k_last);++ loader_a.src += k_jump_a;+ loader_b.src += k_jump_b;++ // transpose_a=false: tile_dims_a = (k_remain, tgp_bm)+ const short2 tile_dims_a = short2(k_remain, tgp_bm);+ // transpose_b=true: tile_dims_b = (k_remain, tgp_bn)+ const short2 tile_dims_b = short2(k_remain, tgp_bn);++ loader_a.load_safe(tile_dims_a);+ loader_b.load_safe(tile_dims_b);++ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(a_shared, b_shared);++ loader_a.src -= k_jump_a;+ loader_b.src -= k_jump_b;+ }++ // MNK aligned loop+ if (align_m && align_n) {+ for (int k = 0; k < gemm_k_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_a.load_unsafe();+ loader_b.load_unsafe();+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(a_shared, b_shared);+ loader_a.next();+ loader_b.next();+ }++ threadgroup_barrier(mem_flags::mem_none);+ return mma_op.store_result(d, params->ldd);+ } else {+ const short leftover_bk = 0;++ if ((align_m || tgp_bm == BM) && (align_n || tgp_bn == BN)) {+ gemm_loop<T, T, AccumType>(+ a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,+ mma_op, tgp_bm, tgp_bn, leftover_bk,+ BK, transpose_a, transpose_b, true, true, true);+ return mma_op.store_result(d, params->ldd);+ } else if (align_n || tgp_bn == BN) {+ gemm_loop<T, T, AccumType>(+ a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,+ mma_op, tgp_bm, tgp_bn, leftover_bk,+ BK, transpose_a, transpose_b, false, true, true);+ return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));+ } else if (align_m || tgp_bm == BM) {+ gemm_loop<T, T, AccumType>(+ a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,+ mma_op, tgp_bm, tgp_bn, leftover_bk,+ BK, transpose_a, transpose_b, true, false, true);+ return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));+ } else {+ gemm_loop<T, T, AccumType>(+ a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,+ mma_op, tgp_bm, tgp_bn, leftover_bk,+ BK, transpose_a, transpose_b, false, false, true);+ return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));+ }+ }+}++///////////////////////////////////////////////////////////////////////////////+// Unified DSL kernel+///////////////////////////////////////////////////////////////////////////////++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(MatmulGemm)(+ const device T* a,+ const device T* b,+ device T* d,+ const constant uzu::matmul::GEMMParams* params,+ const constant uint& group_count_x,+ const constant uint& group_count_y,+ const constant uint& group_count_z,+ threadgroup T a_shared[GEMM_MAX_TGP_A],+ threadgroup T b_shared[GEMM_MAX_TGP_B],+ const uint block_rows SPECIALIZE,+ const uint block_cols SPECIALIZE,+ const uint block_depth SPECIALIZE,+ const uint warps_per_row SPECIALIZE,+ const uint warps_per_col SPECIALIZE,+ const bool align_m SPECIALIZE,+ const bool align_n SPECIALIZE,+ const bool align_k SPECIALIZE,+ const uint group_x GROUPS(group_count_x),+ const uint group_y GROUPS(group_count_y),+ const uint group_z GROUPS(group_count_z),+ const uint thread_x THREADS(32),+ const uint thread_y THREADS(2),+ const uint thread_z THREADS(2),+ const Simd simd+) {+ if (simd.group_idx >= warps_per_row * warps_per_col) {+ return;+ }++ gemm_impl<T, float>(+ a, b, d, params,+ block_rows, block_cols, block_depth,+ warps_per_row, warps_per_col,+ align_m, align_n, align_k,+ a_shared, b_shared,+ simd.lane_idx, simd.group_idx,+ uint3(group_x, group_y, group_z),+ uint3(thread_x, thread_y, thread_z)+ );+}+// clang-format ondiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.hdeleted file mode 100644index 18338aa1a..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.h+++ /dev/null@@ -1,225 +0,0 @@---using namespace steel;--constant bool has_batch [[function_constant(10)]];--constant bool use_out_source [[function_constant(100)]];-constant bool do_axpby [[function_constant(110)]];--constant bool align_M [[function_constant(200)]];-constant bool align_N [[function_constant(201)]];-constant bool align_K [[function_constant(202)]];--// clang-format off-template <- bool kAlignedM,- bool kAlignedN,- typename NAXTile_t,- typename T>-void gemm_epilogue(- thread NAXTile_t& Dtile,- const device T* C,- const constant GEMMParams* params,- const constant GEMMAddMMParams* addmm_params,- const short sgp_sm,- const short sgp_sn) { // clang-format on-- (void)params;-- constexpr short UM = NAXTile_t::kSubTileRows;- constexpr short UN = NAXTile_t::kSubTileCols;- using CSubTile = NAXSubTile<T, UM, UN>;-- using V = typename NAXTile_t::elem_type;-- constexpr short TM = NAXTile_t::kTileRows;- constexpr short TN = NAXTile_t::kTileCols;- constexpr short kElemsPerSubTile = NAXTile_t::kElemsPerSubTile;-- STEEL_PRAGMA_UNROLL- for (short mm = 0; mm < TM; mm++) {- STEEL_PRAGMA_UNROLL- for (short nn = 0; nn < TN; nn++) {- const short m = mm * UM;- const short n = nn * UN;-- CSubTile CTile;-- if constexpr (kAlignedM && kAlignedN) {- CTile.load(C, addmm_params->ldc, addmm_params->fdc, m, n);- } else {- CTile.load_safe(- C,- addmm_params->ldc,- addmm_params->fdc,- sgp_sm,- sgp_sn,- m,- n- );- }-- auto delems = Dtile.subtile_at(mm, nn).elems();- auto celems = CTile.elems();-- STEEL_PRAGMA_UNROLL- for (short i = 0; i < kElemsPerSubTile; i++) {- if (do_axpby) {- delems[i] = addmm_params->alpha * delems[i] +- addmm_params->beta * static_cast<V>(celems[i]);- } else {- delems[i] += static_cast<V>(celems[i]);- }- }- }- }-}--// clang-format off-template <- typename T,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- typename AccumType = float>-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gemm(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- const device T* C [[buffer(2), function_constant(use_out_source)]],- device T* D [[buffer(3)]],- const constant GEMMParams* params [[buffer(4)]],- const constant GEMMAddMMParams* addmm_params [[buffer(5), function_constant(use_out_source)]],- const constant int* batch_shape [[buffer(6), function_constant(has_batch)]],- const constant int64_t* batch_strides [[buffer(7), function_constant(has_batch)]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]]) { // clang-format on- // Find block- const int tid_y = ((tid.y) << params->swizzle_log) +- ((tid.x) & ((1 << params->swizzle_log) - 1));- const int tid_x = (tid.x) >> params->swizzle_log;-- // Exit early if out of bounds- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {- return;- }-- // Adjust for batch- if (has_batch) {- const constant auto* A_bstrides = batch_strides;- const constant auto* B_bstrides = batch_strides + params->batch_ndim;-- ulong2 batch_offsets = elem_to_loc_broadcast(- tid.z,- batch_shape,- A_bstrides,- B_bstrides,- params->batch_ndim- );-- A += batch_offsets.x;- B += batch_offsets.y;-- if (use_out_source) {- const constant auto* C_bstrides = B_bstrides + params->batch_ndim;- C += elem_to_loc(tid.z, batch_shape, C_bstrides, params->batch_ndim);- }- } else {- A += params->batch_stride_a * tid.z;- B += params->batch_stride_b * tid.z;-- if (use_out_source) {- C += addmm_params->batch_stride_c * tid.z;- }- }-- D += params->batch_stride_d * tid.z;-- // Prepare threadgroup memory- threadgroup_barrier(mem_flags::mem_none);-- // Find block in A, B, C- const int c_row = tid_y * BM;- const int c_col = tid_x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- A += transpose_a ? c_row_long : c_row_long * params->lda;- B += transpose_b ? c_col_long * params->ldb : c_col_long;- D += c_row_long * params->ldd + c_col_long;-- if (use_out_source) {- C += c_row_long * addmm_params->ldc + c_col_long * addmm_params->fdc;- }-- constexpr short UM = 16;- constexpr short UN = 32;- constexpr short UK = 16;- constexpr short SM = BM / WM;- constexpr short SN = BN / WN;- constexpr short SK = 32;-- constexpr short TM = SM / UM;- constexpr short TN = SN / UN;-- const short tm = SM * (simd_group_id / WN);- const short tn = SN * (simd_group_id % WN);-- const short sgp_sm = align_M ? SM : min(SM, short(params->M - (c_row + tm)));- const bool is_unaligned_sm = align_M ? false : (sgp_sm != SM);-- const short sgp_sn = align_N ? SN : min(SN, short(params->N - (c_col + tn)));- const bool is_unaligned_sn = align_N ? false : (sgp_sn != SN);-- A += transpose_a ? tm : (tm * params->lda);- B += transpose_b ? (tn * params->ldb) : tn;- D += tm * params->ldd + tn;-- if (use_out_source) {- C += tm * addmm_params->ldc + tn * addmm_params->fdc;- }-- using DSubTile = NAXSubTile<AccumType, UM, UN>;- NAXTile<AccumType, TM, TN, DSubTile> Dtile;-- dispatch_bool(align_K, [&](auto kAlignedK) {- dispatch_bool(align_M || !is_unaligned_sm, [&](auto kAlignedM) {- dispatch_bool(align_N || !is_unaligned_sn, [&](auto kAlignedN) {- Dtile = gemm_loop<- T,- SM,- SN,- SK,- BK,- transpose_a,- transpose_b,- kAlignedM.value,- kAlignedN.value,- kAlignedK.value,- UM,- UN,- UK,- AccumType>(A, B, params, sgp_sm, sgp_sn);- if (use_out_source) {- gemm_epilogue<kAlignedM.value, kAlignedN.value>(- Dtile,- C,- params,- addmm_params,- sgp_sm,- sgp_sn- );- }- if constexpr (kAlignedM && kAlignedN) {- Dtile.store(D, int(params->ldd));- } else {- Dtile.store_safe(D, int(params->ldd), short2(sgp_sn, sgp_sm));- }- });- });- });-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metaldeleted file mode 100644index 286b81152..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal+++ /dev/null@@ -1,30 +0,0 @@---#include <metal_stdlib>--#include "../../../common/utils.h"--#include "../../common/steel/gemm/gemm_nax.h"-#include "steel_gemm_nax.h"--// clang-format off-#define instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_kernel( \- "steel_gemm_nax_" #tname "_" #iname "_" #oname \- "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn, \- gemm, itype, bm, bn, bk, wm, wn, trans_a, trans_b, float)--#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)--#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 64, 64, 256, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 128, 128, 512, 4, 4)--instantiate_gemm_shapes_helper(float16, half, float16, half);-instantiate_gemm_shapes_helper(bfloat16, bfloat, bfloat16, bfloat);-instantiate_gemm_shapes_helper(float32, float, float32, float);-// clang-format ondiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsnew file mode 100644index 000000000..d1ee12719--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -0,0 +1,93 @@+use crate::{+ DataType,+ backends::{+ common::kernel::matmul::{MatmulArguments, gemm::Specialization},+ metal::{+ Metal,+ context::{DeviceClass, MetalContext},+ },+ },+};++impl Specialization {+ pub fn select(+ context: &MetalContext,+ data_type: DataType,+ arguments: &MatmulArguments<Metal>,+ ) -> Self {+ let overall_work_elements =+ (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);+ let is_float32 = matches!(data_type, DataType::F32);+ let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();++ let (block_rows, block_cols, block_depth, warps_per_row, warps_per_col, swizzle_log2) =+ if context.is_nax_available() && prefer_half_or_tf32 {+ let tile_rows = (arguments.batch + 128 - 1) / 128;+ let swizzle_log2 = if tile_rows <= 3 {+ 0+ } else {+ 1+ };+ (128, 128, 512, 4, 4, swizzle_log2)+ } else {+ match context.device_class() {+ DeviceClass::Integrated | DeviceClass::Phone | DeviceClass::Unknown(_) => {+ if prefer_half_or_tf32 {+ if arguments.transpose_b {+ (64, 32, 32, 2, 2, 0)+ } else {+ (64, 64, 16, 1, 2, 0)+ }+ } else if arguments.transpose_b {+ (32, 64, 16, 1, 2, 0)+ } else {+ (64, 32, 32, 2, 2, 0)+ }+ },+ DeviceClass::Desktop => {+ if overall_work_elements >= (1_i64 << 20) {+ if prefer_half_or_tf32 {+ if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {+ (64, 64, 16, 2, 2, 0)+ } else if arguments.transpose_b {+ (64, 32, 32, 2, 2, 0)+ } else {+ (32, 64, 16, 1, 2, 0)+ }+ } else if arguments.transpose_b {+ (32, 64, 16, 1, 2, 0)+ } else {+ (64, 32, 32, 2, 2, 0)+ }+ } else if prefer_half_or_tf32 {+ if arguments.transpose_b {+ (64, 32, 32, 2, 2, 0)+ } else {+ (64, 64, 16, 1, 2, 0)+ }+ } else if arguments.transpose_b {+ (32, 64, 16, 1, 2, 0)+ } else {+ (64, 32, 32, 2, 2, 0)+ }+ },+ }+ };++ let m = arguments.batch;+ let n = arguments.output_dim;+ let k = arguments.input_dim;++ Self {+ block_rows,+ block_cols,+ block_depth,+ warps_per_row,+ warps_per_col,+ swizzle_log2,+ align_m: (m % block_rows) == 0,+ align_n: (n % block_cols) == 0,+ align_k: (k % block_depth) == 0,+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rsdeleted file mode 100644index 3f857b490..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs+++ /dev/null@@ -1,33 +0,0 @@-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct TileConfiguration {- pub block_rows: i32,- pub block_cols: i32,- pub block_depth: i32,- pub warps_per_row: u64,- pub warps_per_col: u64,- pub swizzle_log2: i32,-}--impl TileConfiguration {- pub fn new(- block_rows: i32,- block_cols: i32,- block_depth: i32,- warps_per_row: u64,- warps_per_col: u64,- swizzle_log2: i32,- ) -> Self {- Self {- block_rows,- block_cols,- block_depth,- warps_per_row,- warps_per_col,- swizzle_log2,- }- }-- pub fn is_nax(&self) -> bool {- self.block_depth >= 256- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rsdeleted file mode 100644index b72ad7cc0..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs+++ /dev/null@@ -1,213 +0,0 @@-use std::sync::OnceLock;--use metal::MTLSize;--use super::pipeline_configuration::{PipelineConfiguration, select_configuration};-use crate::{- DataType,- backends::metal::{context::MetalContext, error::MetalError, kernel::matmul::common::MatmulArguments},-};--const DEFAULT_GEMV_MAX_BATCH: i32 = 8;--static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();--fn max_gemv_batch_threshold() -> i32 {- *GEMV_MAX_BATCH.get_or_init(|| {- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)- })-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub(crate) enum AxpbySource {- None,- Bias,- C,-}--#[derive(Debug, Clone)]-pub(crate) struct DispatchDescriptor {- pub(crate) pipeline_configuration: PipelineConfiguration,- pub(crate) matrix_is_rhs: bool,- pub(crate) axpby_source: AxpbySource,- pub(crate) input_dimension: i32,- pub(crate) output_dimension: i32,- pub(crate) matrix_leading_dim: i32,- pub(crate) alpha: f32,- pub(crate) beta: f32,- pub(crate) batch_ndim: i32,- pub(crate) batch_shape: [i32; 1],- pub(crate) vector_batch_stride: [i64; 1],- pub(crate) matrix_batch_stride: [i64; 1],- pub(crate) bias_batch_stride: [i64; 1],- pub(crate) bias_stride: i32,- /// Number of batch rows (M dimension) - used for batched GEMV- pub(crate) batch_rows: i32,- /// Leading dimension of output (ldd)- pub(crate) output_ld: i32,- /// Leading dimension of input vector (lda)- pub(crate) vector_ld: i32,- pub(crate) threadgroups: MTLSize,- pub(crate) threads_per_threadgroup: MTLSize,-}--impl DispatchDescriptor {- pub(crate) fn try_new(- _context: &MetalContext,- data_type: DataType,- arguments: &MatmulArguments,- ) -> Result<Option<Self>, MetalError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MetalError::Generic(format!("Unsupported data type for GEMV: {data_type:?}")));- }-- if arguments.transpose_a || !arguments.transpose_b {- return Ok(None);- }-- let m = arguments.batch;- let n = arguments.output_dim;-- let max_gemv_batch = max_gemv_batch_threshold();-- if n == 1 {- if m != 1 {- return Ok(None);- }- } else if m > max_gemv_batch {- return Ok(None);- }-- let matrix_is_rhs = n != 1;- let transpose_matrix = if matrix_is_rhs {- !arguments.transpose_b- } else {- arguments.transpose_a- };-- let axpby_source = if arguments.c.is_some() {- AxpbySource::C- } else if arguments.bias.is_some() {- AxpbySource::Bias- } else {- AxpbySource::None- };-- let (do_axpby, alpha, beta, bias_stride) = match axpby_source {- AxpbySource::None => (false, 1.0f32, 0.0f32, 0),- AxpbySource::Bias => (true, 1.0f32, 1.0f32, 1),- AxpbySource::C => (true, arguments.alpha, arguments.beta, arguments.ldd),- };-- let output_dimension = if matrix_is_rhs {- arguments.output_dim- } else {- arguments.batch- };-- let mut batch_pack = 1;- if m == 4 && arguments.input_dim <= 2048 && (1536..=3072).contains(&output_dimension) {- batch_pack = 2;- } else if m <= 8 {- batch_pack = if m >= 4 && m % 4 == 0 {- 4- } else if m >= 2 && m % 2 == 0 {- 2- } else {- 1- };- }-- let pipeline_configuration = select_configuration(- arguments.transpose_a,- arguments.transpose_b,- transpose_matrix,- batch_pack as u32,- arguments.input_dim,- output_dimension,- false,- do_axpby,- );-- let input_dimension = arguments.input_dim;- let matrix_leading_dim = if matrix_is_rhs {- arguments.ldb- } else {- arguments.lda- };-- let batch_ndim = 1i32;- let batch_groups = arguments.batch_count.max(1);- let batch_shape = [if arguments.batch_count > 1 {- arguments.batch_count- } else {- 1- }];-- let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);- let elements_per_matrix_b = if arguments.transpose_b {- (arguments.output_dim as i64) * (arguments.ldb as i64)- } else {- (arguments.input_dim as i64) * (arguments.ldb as i64)- };-- let vector_batch_stride = [if matrix_is_rhs {- elements_per_matrix_a- } else {- elements_per_matrix_b- }];-- let matrix_batch_stride = [if matrix_is_rhs {- elements_per_matrix_b- } else {- elements_per_matrix_a- }];-- let bias_batch_stride = [if arguments.batch_count > 1 {- (output_dimension as i64) * (arguments.ldd as i64)- } else {- 0- }];-- let output_elements_per_threadgroup = pipeline_configuration.output_elements_per_threadgroup();- let threadgroup_count_x =- ((output_dimension as u32 + output_elements_per_threadgroup - 1) / output_elements_per_threadgroup) as u64;- let threadgroup_count_z = batch_groups.max(1) as u64;-- let batch_rows = arguments.batch;- let threadgroup_count_y = (batch_rows / batch_pack).max(1) as u64;-- let threadgroups =- MTLSize::new(threadgroup_count_x as usize, threadgroup_count_y as usize, threadgroup_count_z as usize);- let threads_per_threadgroup = pipeline_configuration.threads_per_threadgroup();-- let output_ld = arguments.ldd;- let vector_ld = arguments.lda;-- Ok(Some(Self {- pipeline_configuration,- matrix_is_rhs,- axpby_source,- input_dimension,- output_dimension,- matrix_leading_dim,- alpha,- beta,- batch_ndim,- batch_shape,- vector_batch_stride,- matrix_batch_stride,- bias_batch_stride,- bias_stride,- batch_rows,- output_ld,- vector_ld,- threadgroups,- threads_per_threadgroup,- }))- }-- pub(crate) fn bias_is_fused(&self) -> bool {- matches!(self.axpby_source, AxpbySource::Bias)- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsdeleted file mode 100644index 883b9ae69..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ /dev/null@@ -1,186 +0,0 @@-use std::collections::HashMap;--use metal::{MTLComputeCommandEncoder, MTLComputePipelineState};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::{- dispatch_descriptor::{AxpbySource, DispatchDescriptor},- pipeline_configuration::PipelineConfiguration,-};-use crate::{- DataType,- backends::metal::{- context::MetalContext, error::MetalError, kernel::matmul::common::MatmulArguments,- metal_extensions::ComputeEncoderSetValue,- },-};--fn gemv_kernel_name(- data_type: DataType,- config: &PipelineConfiguration,-) -> Result<String, MetalError> {- let dtype_name = match data_type {- DataType::F16 => "float16",- DataType::BF16 => "bfloat16",- DataType::F32 => "float32",- _ => {- return Err(MetalError::Generic(format!("Unsupported data type for GEMV: {:?}", data_type)));- },- };-- let prefix = if config.transpose_matrix {- "gemv_t"- } else if config.batch_pack > 1 {- "gemv_bp"- } else {- "gemv"- };-- let suffix = format!(- "{prefix}_{dtype_name}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}_nc{}_axpby{}",- config.threadgroup_rows,- config.threadgroup_cols,- config.threads_per_simdgroup_row,- config.threads_per_simdgroup_col,- config.elements_per_thread_row,- config.elements_per_thread_col,- config.non_contiguous_batch as u8,- config.do_axpby as u8,- );-- if config.batch_pack > 1 && !config.transpose_matrix {- Ok(format!("{suffix}_bp{}", config.batch_pack))- } else {- Ok(suffix)- }-}--pub struct Kernel {- data_type: DataType,- pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,-}--impl Kernel {- pub fn new(data_type: DataType) -> Result<Self, MetalError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MetalError::Generic(format!("Unsupported data type for GEMV: {:?}", data_type)));- }- Ok(Self {- data_type,- pipelines: HashMap::new(),- })- }-- pub fn precompile(- &mut self,- context: &MetalContext,- ) -> Result<(), MetalError> {- let configs: &[(u32, bool)] = match self.data_type {- DataType::BF16 => &[(4, false), (4, true), (8, false), (8, true)],- DataType::F16 => &[(8, false)],- DataType::F32 => &[(8, false)],- _ => return Ok(()),- };-- for &(threadgroup_rows, do_axpby) in configs {- let config = PipelineConfiguration {- transpose_a: false,- transpose_b: true,- transpose_matrix: false,- batch_pack: 1,- threadgroup_rows,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- non_contiguous_batch: false,- do_axpby,- };- let _ = self.get_pipeline(context, &config);- }-- Ok(())- }-- fn get_pipeline(- &mut self,- context: &MetalContext,- config: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {- if !self.pipelines.contains_key(config) {- let kernel_name = gemv_kernel_name(self.data_type, config)?;- let pipeline = context.compute_pipeline_state(&kernel_name, None)?;- self.pipelines.insert(*config, pipeline);- }- Ok(self.pipelines.get(config).unwrap())- }-- pub(crate) fn encode_descriptor(- &mut self,- context: &MetalContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MatmulArguments,- descriptor: &DispatchDescriptor,- ) -> Result<bool, MetalError> {- let pipeline = self.get_pipeline(context, &descriptor.pipeline_configuration)?;- encoder.set_compute_pipeline_state(pipeline);-- let (buf0, off0) = if descriptor.matrix_is_rhs {- (arguments.b, 0)- } else {- (arguments.a, arguments.a_offset as usize)- };- encoder.set_buffer(Some(buf0), off0, 0);-- let (buf1, off1) = if descriptor.matrix_is_rhs {- (arguments.a, arguments.a_offset as usize)- } else {- (arguments.b, 0)- };- encoder.set_buffer(Some(buf1), off1, 1);-- if descriptor.pipeline_configuration.do_axpby {- match descriptor.axpby_source {- AxpbySource::None => {- return Err(MetalError::Generic(- "GEMV descriptor mismatch: do_axpby=true but axpby_source=None".to_owned(),- ));- },- AxpbySource::Bias => {- let bias = arguments- .bias- .ok_or_else(|| MetalError::Generic("GEMV descriptor requires bias buffer".to_owned()))?;- encoder.set_buffer(Some(bias), 0, 2);- },- AxpbySource::C => {- let c_buffer = arguments- .c- .ok_or_else(|| MetalError::Generic("GEMV descriptor requires C buffer".to_owned()))?;- encoder.set_buffer(Some(c_buffer), 0, 2);- },- }- }-- encoder.set_buffer(Some(arguments.d), 0, 3);-- encoder.set_value(&descriptor.input_dimension, 4);- encoder.set_value(&descriptor.output_dimension, 5);- encoder.set_value(&descriptor.matrix_leading_dim, 6);- encoder.set_value(&descriptor.alpha, 7);- encoder.set_value(&descriptor.beta, 8);- encoder.set_value(&descriptor.batch_ndim, 9);- encoder.set_slice(&descriptor.batch_shape, 10);- encoder.set_slice(&descriptor.vector_batch_stride, 11);- encoder.set_slice(&descriptor.matrix_batch_stride, 12);- encoder.set_slice(&descriptor.bias_batch_stride, 13);- encoder.set_value(&descriptor.bias_stride, 14);- encoder.set_value(&descriptor.batch_rows, 15);- encoder.set_value(&descriptor.output_ld, 16);- encoder.set_value(&descriptor.vector_ld, 17);-- encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);-- Ok(descriptor.bias_is_fused())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rsindex 5ee340249..889e3df5e 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -1,6 +1 @@-mod dispatch_descriptor;-mod kernel;-mod pipeline_configuration;--pub(crate) use dispatch_descriptor::DispatchDescriptor;-pub use kernel::Kernel as GemvKernel;+// Shader files only — Rust logic has moved to common/kernel/matmul/gemv/diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rsdeleted file mode 100644index 45b40814b..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs+++ /dev/null@@ -1,131 +0,0 @@-use metal::MTLSize;--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct PipelineConfiguration {- pub transpose_a: bool,- pub transpose_b: bool,- pub transpose_matrix: bool,- pub batch_pack: u32,- pub threadgroup_rows: u32,- pub threadgroup_cols: u32,- pub threads_per_simdgroup_row: u32,- pub threads_per_simdgroup_col: u32,- pub elements_per_thread_row: u32,- pub elements_per_thread_col: u32,- pub non_contiguous_batch: bool,- pub do_axpby: bool,-}--impl PipelineConfiguration {- pub fn output_elements_per_threadgroup(&self) -> u32 {- if self.transpose_matrix {- self.threadgroup_cols * self.threads_per_simdgroup_col * self.elements_per_thread_col- } else {- self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row- }- }-- pub fn threads_per_threadgroup(&self) -> MTLSize {- MTLSize::new(32, self.threadgroup_cols as usize, self.threadgroup_rows as usize)- }-}--const FORCE_TILESET_SMALL_BATCH: i32 = 0;--pub fn select_configuration(- transpose_a: bool,- transpose_b: bool,- transpose_matrix: bool,- batch_pack: u32,- input_dimension: i32,- output_dimension: i32,- non_contiguous_batch: bool,- do_axpby: bool,-) -> PipelineConfiguration {- let (threadgroup_rows, threadgroup_cols);- let (threads_per_simdgroup_row, threads_per_simdgroup_col);- let (elements_per_thread_row, elements_per_thread_col);-- if FORCE_TILESET_SMALL_BATCH == 1 && !transpose_matrix {- threadgroup_rows = 4;- threadgroup_cols = 1;- threads_per_simdgroup_row = 1;- threads_per_simdgroup_col = 32;- elements_per_thread_row = 2;- elements_per_thread_col = 4;- } else if transpose_matrix {- let mut sm = 8;- let mut sn = 4;- if input_dimension >= 8192 && output_dimension >= 2048 {- sm = 4;- sn = 8;- }-- let bn = if output_dimension >= 2048 {- 16- } else if output_dimension >= 512 {- 4- } else {- 2- };-- let tn = if output_dimension < 4 {- 1- } else {- 4- };-- threadgroup_rows = 1;- threadgroup_cols = bn;- threads_per_simdgroup_row = sm;- threads_per_simdgroup_col = sn;- elements_per_thread_row = 4;- elements_per_thread_col = tn;- } else {- let mut bm = if output_dimension >= 4096 {- 8- } else {- 4- };- let mut sm = 1;- let mut sn = 32;- let mut bn = 1;-- if input_dimension <= 64 {- bm = 1;- sm = 8;- sn = 4;- } else if input_dimension >= 16 * output_dimension {- bm = 1;- bn = 8;- }-- let tm = if output_dimension < 4 {- 1- } else {- 4- };-- threadgroup_rows = bm;- threadgroup_cols = bn;- threads_per_simdgroup_row = sm;- threads_per_simdgroup_col = sn;- elements_per_thread_row = tm;- elements_per_thread_col = 4;- }-- PipelineConfiguration {- transpose_a,- transpose_b,- transpose_matrix,- batch_pack,- threadgroup_rows,- threadgroup_cols,- threads_per_simdgroup_row,- threads_per_simdgroup_col,- elements_per_thread_row,- elements_per_thread_col,- non_contiguous_batch,- do_axpby,- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metalindex c71737961..fef07c3a7 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal@@ -1,1129 +1,328 @@-#include <metal_simdgroup>-#include <metal_stdlib>-#include "../../../common/utils.h"+#include "../../../definitions.metal"-#include "../../common/steel/utils.h"+#include <metal_simdgroup>using namespace metal;-///////////////////////////////////////////////////////////////////////////////-/// Matrix vector multiplication-///////////////////////////////////////////////////////////////////////////////--#define MTL_CONST static constant constexpr const--template <typename U>-struct DefaultAccT {- using type = float;-};-template <>-struct DefaultAccT<complex64_t> {- using type = complex64_t;-};--template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const int TN, /* Thread cols (in elements) */- const bool kDoAxpby, /* Do out = alpha * out + beta * bias */- const int BP, /* Batch rows per threadgroup */- typename AccT = typename DefaultAccT<T>::type>-struct GEMVKernel {- using acc_type = AccT;-- MTL_CONST int threadsM = BM * SM;- MTL_CONST int threadsN = BN * SN;-- MTL_CONST int blockM = threadsM * TM;- MTL_CONST int blockN = threadsN * TN;+// Upper bound for threadgroup memory (in floats).+// Max config: tg_simd_cols=16, output_rows_per_tg=16, thread_out_rows=4+// → 16 * (16 + 4) = 320. Unused when tg_simd_cols == 1.+#define GEMV_MAX_TG_MEMORY 320++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(MatmulGemv)(+ const device T* matrix,+ const device T* input_vector,+ const device T* output_source OPTIONAL(apply_output_scale_and_accumulate),+ device T* output_vector,+ const constant int& input_dimension,+ const constant int& output_dimension,+ const constant int& matrix_leading_dimension,+ const constant float& output_scale,+ const constant float& output_accumulate_scale,+ const constant int* batch_shape,+ const constant int* vector_batch_stride,+ const constant int* matrix_batch_stride,+ const constant int* output_source_batch_stride,+ const constant int& output_source_stride,+ const constant int& batch_rows,+ const constant int& output_rows_per_threadgroup,+ threadgroup float threadgroup_memory[GEMV_MAX_TG_MEMORY],+ const uint tg_simd_rows SPECIALIZE,+ const uint tg_simd_cols SPECIALIZE,+ const uint sg_thread_rows SPECIALIZE,+ const uint sg_thread_cols SPECIALIZE,+ const uint thread_out_rows SPECIALIZE,+ const uint thread_out_cols SPECIALIZE,+ const bool apply_output_scale_and_accumulate SPECIALIZE,+ const uint threadgroup_index_x GROUPS((output_dimension + output_rows_per_threadgroup - 1) / output_rows_per_threadgroup),+ const uint threadgroup_index_y GROUPS(batch_rows),+ const uint threadgroup_index_z GROUPS(batch_shape[0]),+ const uint thread_index_x THREADS(32),+ const uint thread_index_y THREADS(16),+ const uint thread_index_z THREADS(1),+ const Simd simd+) {+ if (output_rows_per_threadgroup <= 0) {+ return;+ }- static_assert(SM * SN == 32, "simdgroup can only have 32 threads");+ // Simdgroup guard — kill excess simdgroups beyond what this tile uses+ if (simd.group_idx >= tg_simd_rows * tg_simd_cols) {+ return;+ }- static_assert(- SN == 4 || SN == 8 || SN == 16 || SN == 32,- "gemv block must have a width of 4, 8, 16, or 32"- );+ // Derived constants from SPECIALIZE params+ const int threads_per_threadgroup_col = tg_simd_cols * sg_thread_cols;+ const int input_columns_per_threadgroup =+ threads_per_threadgroup_col * thread_out_cols;- // - The matrix of size (M = out_vec_size, K = in_vec_size) is divided up- // into blocks of (blockM, blockN) divided among threadgroups- // - Every thread works on a block of (TM, TN)- // - We assume each threadgroup has (threadsN, threadsM, 1) threads- //- // 1. A thread loads TN elements each from mat along TM rows- // and the corresponding scalar from the vector- // 2. The thread then multiplies and adds to accumulate its local result for- // the block- // 3. At the end, each thread has accumulated results over all blocks across- // the rows. These are then summed up across the threadgroup- // 4. Each threadgroup writes its accumulated blockM outputs- //- // Edge case handling:- // - The threadgroup with the largest tid has blocks that exceed the matrix- // * The blocks that start outside the matrix are never read (thread results- // remain zero)- // * The last thread that partially overlaps with the matrix is shifted- // inwards such that the thread block fits exactly in the matrix+ // Batch setup (from run_matmul_gemv_shape)+ const uint3 threadgroup_position =+ uint3(threadgroup_index_x, threadgroup_index_y, threadgroup_index_z);- MTL_CONST short tgp_mem_size = BN > 1 ? BN*(blockM + TM) * BP : 0;- MTL_CONST bool needs_tgp_reduction = BN > 1;+ const int batch_row = static_cast<int>(threadgroup_position.y);+ if (batch_row >= batch_rows) {+ return;+ }- template <typename U = T>- static METAL_FUNC void load_unsafe(- const device T* src,- thread U dst[TN],- const int src_offset = 0- ) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = static_cast<U>(src[src_offset + tn]);- }+ // Advance pointers by batch+ const device T* batch_input_vector =+ input_vector + threadgroup_position.z * vector_batch_stride[0];+ const device T* batch_matrix =+ matrix + threadgroup_position.z * matrix_batch_stride[0];+ const device T* batch_output_source = output_source;+ if (apply_output_scale_and_accumulate) {+ batch_output_source +=+ threadgroup_position.z * output_source_batch_stride[0];}+ device T* batch_output_vector =+ output_vector + threadgroup_position.z * batch_rows * output_dimension;++ // Thread local accumulation results+ thread float accumulated_values[4] = {0.0f, 0.0f, 0.0f, 0.0f};+ thread T matrix_values[4];+ thread float vector_coefficients[4];++ const int thread_row_in_simdgroup =+ sg_thread_cols != 32 ? int(simd.lane_idx) / int(sg_thread_cols) : 0;+ const int thread_col_in_simdgroup =+ sg_thread_cols != 32 ? int(simd.lane_idx) % int(sg_thread_cols)+ : int(simd.lane_idx);++ const int simdgroup_column_index =+ tg_simd_cols != 1 ? int(simd.group_idx % tg_simd_cols) : 0;++ const int simdgroup_row_thread_base =+ tg_simd_cols != 1+ ? int(sg_thread_rows) * int(simd.group_idx / tg_simd_cols)+ : int(sg_thread_rows) * int(simd.group_idx);+ const int simdgroup_col_thread_base =+ tg_simd_cols != 1+ ? int(sg_thread_cols) * int(simd.group_idx % tg_simd_cols)+ : 0;++ int output_block_row_offset =+ (simdgroup_row_thread_base + thread_row_in_simdgroup) *+ int(thread_out_rows);+ int input_block_col_offset =+ (simdgroup_col_thread_base + thread_col_in_simdgroup) *+ int(thread_out_cols);++ // Block position+ int output_row_start =+ int(threadgroup_position.x) * int(output_rows_per_threadgroup) ++ output_block_row_offset;++ // Exit simdgroup if rows out of bound+ if (output_row_start >= output_dimension)+ return;- template <typename U = T>- static METAL_FUNC void load_safe(- const device T* src,- thread U dst[TN],- const int src_offset = 0,- const int src_size = TN- ) {- if (src_offset + TN <= src_size) {+ // Adjust tail simdgroup to ensure in bound reads+ output_row_start = output_row_start + int(thread_out_rows) <= output_dimension+ ? output_row_start+ : output_dimension - int(thread_out_rows);++ // Advance matrix+ const device T* thread_matrix =+ batch_matrix + output_row_start * matrix_leading_dimension;++ const uniform<int> input_block_stride =+ make_uniform(int(input_columns_per_threadgroup));+ const uniform<int> input_vector_length = make_uniform(input_dimension);+ const uniform<int> full_input_blocks =+ input_vector_length / input_block_stride;+ const uniform<int> remaining_input_columns =+ input_vector_length - input_block_stride * full_input_blocks;++ // Loop over input_vector in blocks of input_columns_per_threadgroup+ for (int input_block_index = 0; input_block_index < full_input_blocks;+ ++input_block_index) {+ // Load vector coefficients (unchecked)+ {+ const device T* input_vector_row =+ batch_input_vector + batch_row * input_dimension;MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = static_cast<U>(src[src_offset + tn]);- }- } else { // Edgecase- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = src_offset + tn < src_size- ? static_cast<U>(src[src_offset + tn])- : U(0);+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;+ input_col_offset++) {+ vector_coefficients[input_col_offset] = static_cast<float>(+ input_vector_row[input_block_col_offset + input_col_offset]+ );}}- }-- static METAL_FUNC void run(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- const device T* bias [[buffer(2)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& out_vec_size [[buffer(5)]],- const constant int& matrix_ld [[buffer(6)]],- const constant float& alpha [[buffer(7)]],- const constant float& beta [[buffer(8)]],- const constant int& bias_stride [[buffer(14)]],- const int batch_row_base,- threadgroup AccT* tgp_memory [[threadgroup(0)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]- ) {- // Appease compiler- (void)lid;-- // Thread local accumulation results- thread AccT result[BP][TM] = {{0}};- thread T inter[TN];- thread AccT v_coeff[BP][TN];-- const int thrM = SN != 32 ? simd_lid / SN : 0;- const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);-- const int sgN = BN != 1 ? (simd_gid % BN) : 0;- const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);- const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;-- int bm = (simdM + thrM) * TM;- int bn = (simdN + thrN) * TN;-- // Block position- int out_row = tid.x * blockM + bm;-- // Exit simdgroup if rows out of bound- if (out_row >= out_vec_size)- return;-- // Adjust tail simdgroup to ensure in bound reads- out_row = out_row + TM <= out_vec_size ? out_row : out_vec_size - TM;-- // Advance matrix- mat += out_row * matrix_ld;-- constexpr const uniform<int> loop_stride = make_uniform(blockN);- const uniform<int> in_size = make_uniform(in_vec_size);- const uniform<int> n_iter = in_size / loop_stride;- const uniform<int> last_iter = loop_stride * n_iter;- const uniform<int> leftover = in_size - last_iter;-- // Loop over in_vec in blocks of blockN- for (int i = 0; i < n_iter; ++i) {+ // Per thread work loop+ int matrix_row_offset = 0;+ MTL_PRAGMA_UNROLL+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;+ output_row_offset++) {+ // Load matrix row (unchecked)MTL_PRAGMA_UNROLL- for (int bp = 0; bp < BP; bp++) {- const int batch_row = batch_row_base + bp;- const device T* vec_ptr = in_vec + batch_row * in_vec_size;- load_unsafe<AccT>(vec_ptr, v_coeff[bp], bn);+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;+ input_col_offset++) {+ matrix_values[input_col_offset] = thread_matrix+ [matrix_row_offset + input_block_col_offset + input_col_offset];}- // Per thread work loop- int mat_offset = 0;+ // Accumulate resultsMTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- // Load for the row- load_unsafe(mat, inter, mat_offset + bn);-- // Accumulate results- MTL_PRAGMA_UNROLL- for (int bp = 0; bp < BP; bp++) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- result[bp][tm] += inter[tn] * v_coeff[bp][tn];- }- }-- mat_offset += matrix_ld;+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;+ input_col_offset++) {+ accumulated_values[output_row_offset] +=+ static_cast<float>(matrix_values[input_col_offset]) *+ vector_coefficients[input_col_offset];}- bn += blockN;+ matrix_row_offset += matrix_leading_dimension;}- if (leftover > 0) {- MTL_PRAGMA_UNROLL- for (int bp = 0; bp < BP; bp++) {- const int batch_row = batch_row_base + bp;- const device T* vec_ptr = in_vec + batch_row * in_vec_size;- load_safe<AccT>(vec_ptr, v_coeff[bp], bn, in_size);- }-- // Per thread work loop- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- // Load for the row- load_safe(&mat[tm * matrix_ld], inter, bn, in_size);+ input_block_col_offset += input_columns_per_threadgroup;+ }- // Accumulate results+ if (remaining_input_columns > 0) {+ // Load vector coefficients (checked)+ {+ const device T* input_vector_row =+ batch_input_vector + batch_row * input_dimension;+ if (input_block_col_offset + int(thread_out_cols) <=+ input_vector_length) {MTL_PRAGMA_UNROLL- for (int bp = 0; bp < BP; bp++) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- result[bp][tm] += inter[tn] * v_coeff[bp][tn];- }+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;+ input_col_offset++) {+ vector_coefficients[input_col_offset] = static_cast<float>(+ input_vector_row[input_block_col_offset + input_col_offset]+ );}- }- }-- // Simdgroup accumulations- MTL_PRAGMA_UNROLL- for (int bp = 0; bp < BP; bp++) {- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {+ } else {MTL_PRAGMA_UNROLL- for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {- result[bp][tm] += simd_shuffle_down(result[bp][tm], sn);+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;+ input_col_offset++) {+ vector_coefficients[input_col_offset] =+ input_block_col_offset + int(input_col_offset) <+ input_vector_length+ ? static_cast<float>(+ input_vector_row+ [input_block_col_offset + input_col_offset]+ )+ : 0.0f;}}}- // Threadgroup accumulation results- if (needs_tgp_reduction) {- if (thrN == 0) {+ // Per thread work loop+ MTL_PRAGMA_UNROLL+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;+ output_row_offset++) {+ // Load matrix row (checked)+ if (input_block_col_offset + int(thread_out_cols) <=+ input_vector_length) {MTL_PRAGMA_UNROLL- for (int bp = 0; bp < BP; bp++) {- threadgroup AccT* tgp_results =- tgp_memory + bp * (BN * (blockM + TM)) + sgN * (blockM + TM) + bm;- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- tgp_results[tm] = result[bp][tm];- }+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;+ input_col_offset++) {+ matrix_values[input_col_offset] = thread_matrix+ [output_row_offset * matrix_leading_dimension ++ input_block_col_offset + input_col_offset];}-- threadgroup_barrier(mem_flags::mem_none);-- if (sgN == 0) {- MTL_PRAGMA_UNROLL- for (int bp = 0; bp < BP; bp++) {- threadgroup AccT* tgp_results =- tgp_memory + bp * (BN * (blockM + TM)) + bm;- MTL_PRAGMA_UNROLL- for (int sgn = 1; sgn < BN; sgn++) {- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- result[bp][tm] += tgp_results[sgn * (blockM + TM) + tm];- }- }- }+ } else {+ MTL_PRAGMA_UNROLL+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;+ input_col_offset++) {+ matrix_values[input_col_offset] =+ input_block_col_offset + int(input_col_offset) <+ input_vector_length+ ? thread_matrix+ [output_row_offset * matrix_leading_dimension ++ input_block_col_offset + input_col_offset]+ : T(0);}}- }- // Write outputs- if (simdN == 0 && thrN == 0) {+ // Accumulate resultsMTL_PRAGMA_UNROLL- for (int bp = 0; bp < BP; bp++) {- const int batch_row = batch_row_base + bp;- device T* out_row_ptr = out_vec + batch_row * out_vec_size;- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- if (kDoAxpby) {- out_row_ptr[out_row + tm] =- static_cast<T>(alpha) * static_cast<T>(result[bp][tm]) +- static_cast<T>(beta) * bias[(out_row + tm) * bias_stride];- } else {- out_row_ptr[out_row + tm] = static_cast<T>(result[bp][tm]);- }- }+ for (uint input_col_offset = 0; input_col_offset < thread_out_cols;+ input_col_offset++) {+ accumulated_values[output_row_offset] +=+ static_cast<float>(matrix_values[input_col_offset]) *+ vector_coefficients[input_col_offset];}}}-};--///////////////////////////////////////////////////////////////////////////////-/// Vector matrix multiplication-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const int TN, /* Thread cols (in elements) */- const bool kDoAxpby, /* Do out = alpha * out + beta * bias */- typename AccT = typename DefaultAccT<T>::type>-struct GEMVTKernel {- using acc_type = AccT;-- MTL_CONST int threadsM = BM * SM;- MTL_CONST int threadsN = BN * SN;-- MTL_CONST int blockM = threadsM * TM;- MTL_CONST int blockN = threadsN * TN;-- static_assert(SM * SN == 32, "simdgroup can only have 32 threads");-- // - The matrix of size (M = in_vec_size, N = out_vec_size) is divided up- // into blocks of (blockM, blockN) divided among threadgroups- // - Every thread works on a block of (TM, TN)- // - We assume each threadgroup has (threadsN, threadsM, 1) threads- //- // 1. A thread loads TN elements each from mat along TM contiguous rows- // and the corresponding scalar from the vector- // 2. The thread then accumulates its local result for the block- // 3. At the end, each thread has accumulated results over all blocks across- // the rows. These are then summed up across the threadgroup- // 4. Each threadgroup writes its accumulated BN * TN outputs- //- // Edge case handling:- // - The threadgroup with the largest tid has blocks that exceed the matrix- // * The blocks that start outside the matrix are never read (thread results- // remain zero)- // * The last thread that partially overlaps with the matrix is shifted- // inwards such that the thread block fits exactly in the matrix-- MTL_CONST short tgp_mem_size = BM > 1 ? BM*(blockN + TN) : 0;- MTL_CONST bool needs_tgp_reduction = BM > 1;-- static METAL_FUNC void run(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- const device T* bias [[buffer(2)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& out_vec_size [[buffer(5)]],- const constant int& marix_ld [[buffer(6)]],- const constant float& alpha [[buffer(7)]],- const constant float& beta [[buffer(8)]],- const constant int& bias_stride [[buffer(14)]],- threadgroup AccT* tgp_memory [[threadgroup(0)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]- ) {- // Appease compiler- (void)lid;-- // Thread local accumulation results- AccT result[TN] = {0};- T inter[TN];- AccT v_coeff[TM];- const int thrM = SN != 32 ? simd_lid / SN : 0;- const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);-- const int sgM = BN != 1 ? (simd_gid / BN) : int(simd_gid);- const int sgN = BN != 1 ? (simd_gid % BN) : 0;-- const int simdM = SM * sgM;- const int simdN = SN * sgN;-- int cm = (simdM + thrM);- int cn = (simdN + thrN);-- int bm = cm * TM;- int bn = cn * TN;- int out_col = tid.x * blockN + bn;-- constexpr const uniform<int> loop_stride = make_uniform(blockM);- const uniform<int> in_size = make_uniform(in_vec_size);- const uniform<int> n_iter = in_size / loop_stride;- const uniform<int> last_iter = loop_stride * n_iter;- const uniform<int> leftover = in_size - last_iter;-- // Edgecase handling- if (out_col < out_vec_size) {- out_col = out_col + TN < out_vec_size ? out_col : out_vec_size - TN;-- // Per thread accumulation main loop- for (int i = 0; i < n_iter; ++i) {- // Adding a threadgroup_barrier improves performance slightly- // This is possibly it may help exploit cache better- threadgroup_barrier(mem_flags::mem_none);-- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- v_coeff[tm] = static_cast<AccT>(in_vec[bm + tm]);- }-- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- auto vc = static_cast<AccT>(v_coeff[tm]);- for (int tn = 0; tn < TN; tn++) {- inter[tn] = mat[(bm + tm) * marix_ld + out_col + tn];- }- for (int tn = 0; tn < TN; tn++) {- result[tn] += vc * inter[tn];- }- }-- bm += blockM;- }-- if (leftover > 0) {- for (int tm = 0; tm < TM && bm + tm < in_vec_size; tm++) {- v_coeff[tm] = static_cast<AccT>(in_vec[bm + tm]);-- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- inter[tn] = mat[(bm + tm) * marix_ld + out_col + tn];- }-- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- result[tn] += v_coeff[tm] * inter[tn];- }- }- }+ // Simdgroup accumulations+ MTL_PRAGMA_UNROLL+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;+ output_row_offset++) {+ MTL_PRAGMA_UNROLL+ for (ushort simd_shuffle_offset = (sg_thread_cols / 2);+ simd_shuffle_offset >= 1;+ simd_shuffle_offset >>= 1) {+ accumulated_values[output_row_offset] += simd_shuffle_down(+ accumulated_values[output_row_offset],+ simd_shuffle_offset+ );}+ }- // Simdgroup accumulations- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {+ // Threadgroup reduction (only when tg_simd_cols > 1)+ if (tg_simd_cols > 1) {+ const int computed_output_rows_per_tg =+ int(tg_simd_rows) * int(sg_thread_rows) * int(thread_out_rows);++ if (thread_col_in_simdgroup == 0) {+ threadgroup float* threadgroup_partial_accumulations =+ threadgroup_memory ++ simdgroup_column_index *+ (computed_output_rows_per_tg + int(thread_out_rows)) ++ output_block_row_offset;MTL_PRAGMA_UNROLL- for (ushort sm = (SM / 2); sm >= 1; sm >>= 1) {- result[tn] += simd_shuffle_down(result[tn], SN * sm);+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;+ output_row_offset++) {+ threadgroup_partial_accumulations[output_row_offset] =+ accumulated_values[output_row_offset];}- }-- // Threadgroup accumulation results- if (needs_tgp_reduction) {- threadgroup AccT* tgp_results = tgp_memory + sgM * (blockN + TN) + bn;- if (thrM == 0) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- tgp_results[tn] = result[tn];- }- threadgroup_barrier(mem_flags::mem_none);+ threadgroup_barrier(mem_flags::mem_none);- if (sgM == 0) {+ if (simdgroup_column_index == 0) {+ threadgroup float* base_partial =+ threadgroup_memory + output_block_row_offset;+ for (uint reduction_simdgroup_col = 1;+ reduction_simdgroup_col < tg_simd_cols;+ reduction_simdgroup_col++) {MTL_PRAGMA_UNROLL- for (int sgm = 1; sgm < BM; sgm++) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- result[tn] += tgp_results[sgm * (blockN + TN) + tn];- }+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;+ output_row_offset++) {+ accumulated_values[output_row_offset] += base_partial+ [reduction_simdgroup_col *+ (computed_output_rows_per_tg + int(thread_out_rows)) ++ output_row_offset];}}}}-- // Threadgroup accumulation and writing out results- if (cm == 0 && out_col < out_vec_size) {- MTL_PRAGMA_UNROLL- for (int j = 0; j < TN; j++) {- if (kDoAxpby) {- out_vec[out_col + j] =- static_cast<T>(alpha) * static_cast<T>(result[j]) +- static_cast<T>(beta) * bias[(out_col + j) * bias_stride];- } else {- out_vec[out_col + j] = static_cast<T>(result[j]);- }- }- }- }-};--///////////////////////////////////////////////////////////////////////////////-/// Matrix vector multiplication-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const int TN, /* Thread cols (in elements) */- const bool kDoNCBatch, /* Batch ndim > 1 */- const bool kDoAxpby> /* Do out = alpha * out + beta * bias */-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- const device T* bias [[buffer(2)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& out_vec_size [[buffer(5)]],- const constant int& marix_ld [[buffer(6)]],- const constant float& alpha [[buffer(7)]],- const constant float& beta [[buffer(8)]],- const constant int& batch_ndim [[buffer(9)]],- const constant int* batch_shape [[buffer(10)]],- const constant int64_t* vector_batch_stride [[buffer(11)]],- const constant int64_t* matrix_batch_stride [[buffer(12)]],- const constant int64_t* bias_batch_stride [[buffer(13)]],- const constant int& bias_stride [[buffer(14)]],- const constant int& batch_rows [[buffer(15)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- using gemv_kernel = GEMVKernel<T, BM, BN, SM, SN, TM, TN, kDoAxpby, 1>;- threadgroup typename gemv_kernel::acc_type tgp_memory- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];-- // Batched GEMV: tid.y is the batch row index- const int batch_row = tid.y;- if (batch_row >= batch_rows) {- return;- }-- // Update batch offsets- if (kDoNCBatch) {- in_vec += elem_to_loc(tid.z, batch_shape, vector_batch_stride, batch_ndim);- mat += elem_to_loc(tid.z, batch_shape, matrix_batch_stride, batch_ndim);-- if (kDoAxpby) {- bias += elem_to_loc(tid.z, batch_shape, bias_batch_stride, batch_ndim);- }-- } else {- in_vec += tid.z * vector_batch_stride[0];- mat += tid.z * matrix_batch_stride[0];-- if (kDoAxpby) {- bias += tid.z * bias_batch_stride[0];- }}- const int batch_row_base = batch_row;- out_vec += tid.z * batch_rows * out_vec_size;-- gemv_kernel::run(- mat,- in_vec,- bias,- out_vec,- in_vec_size,- out_vec_size,- marix_ld,- alpha,- beta,- bias_stride,- batch_row_base,- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,- tid,- lid,- simd_gid,- simd_lid- );-}--#define instantiate_gemv_bp_helper( \- name, \- itype, \- bm, \- bn, \- sm, \- sn, \- tm, \- nc, \- axpby, \- bp \-) \- instantiate_kernel( \- "gemv_bp_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \- "_tn4_nc" #nc "_axpby" #axpby "_bp" #bp, \- gemv_bp, \- itype, \- bm, \- bn, \- sm, \- sn, \- tm, \- nc, \- axpby, \- bp \- )--template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const bool kDoNCBatch, /* Batch ndim > 1 */- const bool kDoAxpby, /* Do out = alpha * out + beta * bias */- const int BP> /* Batch rows per threadgroup */-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_bp(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- const device T* bias [[buffer(2)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& out_vec_size [[buffer(5)]],- const constant int& marix_ld [[buffer(6)]],- const constant float& alpha [[buffer(7)]],- const constant float& beta [[buffer(8)]],- const constant int& batch_ndim [[buffer(9)]],- const constant int* batch_shape [[buffer(10)]],- const constant int64_t* vector_batch_stride [[buffer(11)]],- const constant int64_t* matrix_batch_stride [[buffer(12)]],- const constant int64_t* bias_batch_stride [[buffer(13)]],- const constant int& bias_stride [[buffer(14)]],- const constant int& batch_rows [[buffer(15)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- using gemv_kernel = GEMVKernel<T, BM, BN, SM, SN, TM, 4, kDoAxpby, BP>;- threadgroup typename gemv_kernel::acc_type tgp_memory- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];-- const int batch_row_base = int(tid.y) * BP;-- if (kDoNCBatch) {- in_vec += elem_to_loc(tid.z, batch_shape, vector_batch_stride, batch_ndim);- mat += elem_to_loc(tid.z, batch_shape, matrix_batch_stride, batch_ndim);-- if (kDoAxpby) {- bias += elem_to_loc(tid.z, batch_shape, bias_batch_stride, batch_ndim);- }-- } else {- in_vec += tid.z * vector_batch_stride[0];- mat += tid.z * matrix_batch_stride[0];-- if (kDoAxpby) {- bias += tid.z * bias_batch_stride[0];- }- }-- out_vec += tid.z * batch_rows * out_vec_size;-- gemv_kernel::run(- mat,- in_vec,- bias,- out_vec,- in_vec_size,- out_vec_size,- marix_ld,- alpha,- beta,- bias_stride,- batch_row_base,- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,- tid,- lid,- simd_gid,- simd_lid- );-}--#define instantiate_gemv_helper( \- name, \- itype, \- bm, \- bn, \- sm, \- sn, \- tm, \- tn, \- nc, \- axpby \-) \- instantiate_kernel( \- "gemv_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \- "_tn" #tn "_nc" #nc "_axpby" #axpby, \- gemv, \- itype, \- bm, \- bn, \- sm, \- sn, \- tm, \- tn, \- nc, \- axpby \- )--// clang-format off-#define instantiate_gemv(name, itype, bm, bn, sm, sn, tm, tn) \- instantiate_gemv_helper(name, itype, bm, bn, sm, sn, tm, tn, 0, 0) \- instantiate_gemv_helper(name, itype, bm, bn, sm, sn, tm, tn, 0, 1) \- instantiate_gemv_helper(name, itype, bm, bn, sm, sn, tm, tn, 1, 0) \- instantiate_gemv_helper(name, itype, bm, bn, sm, sn, tm, tn, 1, 1) // clang-format on--#define instantiate_gemv_bp(name, itype, bm, bn, sm, sn, tm, bp) \- instantiate_gemv_bp_helper( \- name, \- itype, \- bm, \- bn, \- sm, \- sn, \- tm, \- 0, \- 0, \- bp \- ) instantiate_gemv_bp_helper(name, itype, bm, bn, sm, sn, tm, 0, 1, bp) \- instantiate_gemv_bp_helper(name, itype, bm, bn, sm, sn, tm, 1, 0, bp) \- instantiate_gemv_bp_helper( \- name, \- itype, \- bm, \- bn, \- sm, \- sn, \- tm, \- 1, \- 1, \- bp \- ) // clang-format on--// clang-format off-#define instantiate_gemv_blocks(name, itype) \- instantiate_gemv(name, itype, 1, 8, 1, 32, 4, 4) \- instantiate_gemv(name, itype, 1, 8, 1, 32, 1, 4) \- instantiate_gemv(name, itype, 1, 1, 8, 4, 4, 4) \- instantiate_gemv(name, itype, 1, 1, 8, 4, 1, 4) \- instantiate_gemv(name, itype, 4, 1, 1, 32, 1, 4) \- instantiate_gemv(name, itype, 4, 1, 1, 32, 4, 4) \- instantiate_gemv(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on--#define instantiate_gemv_bp_blocks(name, itype, bp) \- instantiate_gemv_bp(name, itype, 1, 8, 1, 32, 4, bp) \- instantiate_gemv_bp(name, itype, 1, 8, 1, 32, 1, bp) \- instantiate_gemv_bp(name, itype, 1, 1, 8, 4, 4, bp) \- instantiate_gemv_bp(name, itype, 1, 1, 8, 4, 1, bp) \- instantiate_gemv_bp(name, itype, 4, 1, 1, 32, 1, bp) \- instantiate_gemv_bp(name, itype, 4, 1, 1, 32, 4, bp) \- instantiate_gemv_bp( \- name, \- itype, \- 8, \- 1, \- 1, \- 32, \- 4, \- bp \- ) // clang-format on--instantiate_gemv_blocks(float32, float);-instantiate_gemv_blocks(float16, half);-instantiate_gemv_blocks(bfloat16, bfloat16_t);-instantiate_gemv_blocks(complex64, complex64_t);-instantiate_gemv_bp_blocks(float32, float, 2);-instantiate_gemv_bp_blocks(float16, half, 2);-instantiate_gemv_bp_blocks(bfloat16, bfloat16_t, 2);-instantiate_gemv_bp_blocks(float32, float, 4);-instantiate_gemv_bp_blocks(float16, half, 4);-instantiate_gemv_bp_blocks(bfloat16, bfloat16_t, 4);--template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const int TN> /* Thread cols (in elements) */-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_gather(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- const device T* bias [[buffer(2)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& out_vec_size [[buffer(5)]],- const constant int& marix_ld [[buffer(6)]],- const constant float& alpha [[buffer(7)]],- const constant float& beta [[buffer(8)]],- const constant int& batch_ndim [[buffer(9)]],- const constant int* batch_shape [[buffer(10)]],- const constant int64_t* index_batch_strides [[buffer(11)]],- const constant int& vector_batch_ndim [[buffer(12)]],- const constant int* vector_batch_shape [[buffer(13)]],- const constant int64_t* vector_batch_stride [[buffer(14)]],- const constant int& matrix_batch_ndim [[buffer(15)]],- const constant int* matrix_batch_shape [[buffer(16)]],- const constant int64_t* matrix_batch_stride [[buffer(17)]],- const constant uint32_t* vec_indices [[buffer(18)]],- const constant uint32_t* mat_indices [[buffer(19)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- using gemv_kernel = GEMVKernel<T, BM, BN, SM, SN, TM, TN, false, 1>;- threadgroup typename gemv_kernel::acc_type tgp_memory- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];-- uint32_t indx_vec;- uint32_t indx_mat;-- // Update batch offsets- if (batch_ndim > 1) {- const constant auto* veci_bstrides = index_batch_strides;- const constant auto* mati_bstrides = index_batch_strides + batch_ndim;-- ulong2 batch_offsets = elem_to_loc_broadcast(- tid.z,- batch_shape,- veci_bstrides,- mati_bstrides,- batch_ndim- );-- indx_vec = vec_indices[batch_offsets.x];- indx_mat = mat_indices[batch_offsets.y];-- } else {- indx_vec = vec_indices[index_batch_strides[0] * tid.z];- indx_mat = mat_indices[index_batch_strides[batch_ndim] * tid.z];- }-- if (vector_batch_ndim > 1) {- in_vec += elem_to_loc(- indx_vec,- vector_batch_shape,- vector_batch_stride,- vector_batch_ndim- );- } else {- in_vec += indx_vec * vector_batch_stride[0];- }-- if (matrix_batch_ndim > 1) {- mat += elem_to_loc(- indx_mat,- matrix_batch_shape,- matrix_batch_stride,- matrix_batch_ndim- );- } else {- mat += indx_mat * matrix_batch_stride[0];- }-- out_vec += tid.z * out_vec_size;-- gemv_kernel::run(- mat,- in_vec,- bias,- out_vec,- in_vec_size,- out_vec_size,- marix_ld,- alpha,- beta,- out_vec_size,- 0,- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,- tid,- lid,- simd_gid,- simd_lid- );-}--// clang-format off-#define instantiate_gemv_bs_helper(nm, itype, bm, bn, sm, sn, tm, tn) \- instantiate_kernel( \- "gemv_gather_" #nm "_bm" #bm "_bn" #bn "_sm" #sm \- "_sn" #sn "_tm" #tm "_tn" #tn, \- gemv_gather, itype, bm, bn, sm, sn, tm, tn)--#define instantiate_gemv_bs_blocks(name, itype) \- instantiate_gemv_bs_helper(name, itype, 4, 1, 1, 32, 1, 4) \- instantiate_gemv_bs_helper(name, itype, 4, 1, 1, 32, 4, 4) \- instantiate_gemv_bs_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on--instantiate_gemv_bs_blocks(float32, float);-instantiate_gemv_bs_blocks(float16, half);-instantiate_gemv_bs_blocks(bfloat16, bfloat16_t);-instantiate_gemv_bs_blocks(complex64, complex64_t);--///////////////////////////////////////////////////////////////////////////////-/// Vector matrix multiplication-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const int TN, /* Thread cols (in elements) */- const bool kDoNCBatch, /* Batch ndim > 1 */- const bool kDoAxpby> /* Do out = alpha * out + beta * bias */-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_t(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- const device T* bias [[buffer(2)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& out_vec_size [[buffer(5)]],- const constant int& marix_ld [[buffer(6)]],- const constant float& alpha [[buffer(7)]],- const constant float& beta [[buffer(8)]],- const constant int& batch_ndim [[buffer(9)]],- const constant int* batch_shape [[buffer(10)]],- const constant int64_t* vector_batch_stride [[buffer(11)]],- const constant int64_t* matrix_batch_stride [[buffer(12)]],- const constant int64_t* bias_batch_stride [[buffer(13)]],- const constant int& bias_stride [[buffer(14)]],- const constant int& batch_rows [[buffer(15)]],- const constant int& output_ld [[buffer(16)]],- const constant int& vector_ld [[buffer(17)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- using gemv_kernel = GEMVTKernel<T, BM, BN, SM, SN, TM, TN, kDoAxpby>;- threadgroup typename gemv_kernel::acc_type tgp_memory- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];-- // Batched GEMV: tid.y is the batch row index- const int batch_row = tid.y;- if (batch_row >= batch_rows) {- return;- }-- // Update batch offsets- if (kDoNCBatch) {- in_vec += elem_to_loc(tid.z, batch_shape, vector_batch_stride, batch_ndim);- mat += elem_to_loc(tid.z, batch_shape, matrix_batch_stride, batch_ndim);-- if (kDoAxpby) {- bias += elem_to_loc(tid.z, batch_shape, bias_batch_stride, batch_ndim);- }-- } else {- in_vec += tid.z * vector_batch_stride[0];- mat += tid.z * matrix_batch_stride[0];-- if (kDoAxpby) {- bias += tid.z * bias_batch_stride[0];+ // Write outputs+ if (simdgroup_col_thread_base == 0 && thread_col_in_simdgroup == 0) {+ device T* output_row_values =+ batch_output_vector + batch_row * output_dimension;+ MTL_PRAGMA_UNROLL+ for (uint output_row_offset = 0; output_row_offset < thread_out_rows;+ output_row_offset++) {+ if (apply_output_scale_and_accumulate) {+ output_row_values[output_row_start + output_row_offset] =+ static_cast<T>(output_scale) *+ static_cast<T>(accumulated_values[output_row_offset]) ++ static_cast<T>(output_accumulate_scale) *+ batch_output_source+ [(output_row_start + output_row_offset) *+ output_source_stride];+ } else {+ output_row_values[output_row_start + output_row_offset] =+ static_cast<T>(accumulated_values[output_row_offset]);+ }}}-- // Offset by batch row (tid.y)- in_vec += batch_row * vector_ld;- out_vec += tid.z * batch_rows * output_ld + batch_row * output_ld;-- gemv_kernel::run(- mat,- in_vec,- bias,- out_vec,- in_vec_size,- out_vec_size,- marix_ld,- alpha,- beta,- bias_stride,- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,- tid,- lid,- simd_gid,- simd_lid- );}--// clang-format off-#define instantiate_gemv_t_helper( \- name, itype, bm, bn, sm, sn, tm, tn, nc, axpby) \- instantiate_kernel( \- "gemv_t_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn \- "_tm" #tm "_tn" #tn "_nc" #nc "_axpby" #axpby, \- gemv_t, itype, bm, bn, sm, sn, tm, tn, nc, axpby)--#define instantiate_gemv_t(name, itype, bm, bn, sm, sn, tm, tn) \- instantiate_gemv_t_helper(name, itype, bm, bn, sm, sn, tm, tn, 0, 0) \- instantiate_gemv_t_helper(name, itype, bm, bn, sm, sn, tm, tn, 0, 1) \- instantiate_gemv_t_helper(name, itype, bm, bn, sm, sn, tm, tn, 1, 0) \- instantiate_gemv_t_helper(name, itype, bm, bn, sm, sn, tm, tn, 1, 1) // clang-format on--// clang-format off-#define instantiate_gemv_t_blocks(name, itype) \- instantiate_gemv_t(name, itype, 1, 2, 8, 4, 4, 1) \- instantiate_gemv_t(name, itype, 1, 2, 8, 4, 4, 4) \- instantiate_gemv_t(name, itype, 1, 4, 8, 4, 4, 4) \- instantiate_gemv_t(name, itype, 1, 16, 8, 4, 4, 4) \- instantiate_gemv_t(name, itype, 1, 16, 4, 8, 4, 4) // clang-format on--// clang-format off-instantiate_gemv_t_blocks(float32, float);-instantiate_gemv_t_blocks(float16, half);-instantiate_gemv_t_blocks(bfloat16, bfloat16_t);-instantiate_gemv_t_blocks(complex64, complex64_t); // clang-format on--template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const int TN> /* Thread cols (in elements) */-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_t_gather(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- const device T* bias [[buffer(2)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& out_vec_size [[buffer(5)]],- const constant int& marix_ld [[buffer(6)]],- const constant float& alpha [[buffer(7)]],- const constant float& beta [[buffer(8)]],- const constant int& batch_ndim [[buffer(9)]],- const constant int* batch_shape [[buffer(10)]],- const constant int64_t* index_batch_strides [[buffer(11)]],- const constant int& vector_batch_ndim [[buffer(12)]],- const constant int* vector_batch_shape [[buffer(13)]],- const constant int64_t* vector_batch_stride [[buffer(14)]],- const constant int& matrix_batch_ndim [[buffer(15)]],- const constant int* matrix_batch_shape [[buffer(16)]],- const constant int64_t* matrix_batch_stride [[buffer(17)]],- const constant uint32_t* vec_indices [[buffer(18)]],- const constant uint32_t* mat_indices [[buffer(19)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- using gemv_kernel = GEMVTKernel<T, BM, BN, SM, SN, TM, TN, false>;- threadgroup typename gemv_kernel::acc_type tgp_memory- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];-- uint32_t indx_vec;- uint32_t indx_mat;-- // Update batch offsets- if (batch_ndim > 1) {- const constant auto* veci_bstrides = index_batch_strides;- const constant auto* mati_bstrides = index_batch_strides + batch_ndim;-- ulong2 batch_offsets = elem_to_loc_broadcast(- tid.z,- batch_shape,- veci_bstrides,- mati_bstrides,- batch_ndim- );-- indx_vec = vec_indices[batch_offsets.x];- indx_mat = mat_indices[batch_offsets.y];-- } else {- indx_vec = vec_indices[index_batch_strides[0] * tid.z];- indx_mat = mat_indices[index_batch_strides[batch_ndim] * tid.z];- }-- if (vector_batch_ndim > 1) {- in_vec += elem_to_loc(- indx_vec,- vector_batch_shape,- vector_batch_stride,- vector_batch_ndim- );- } else {- in_vec += indx_vec * vector_batch_stride[0];- }-- if (matrix_batch_ndim > 1) {- mat += elem_to_loc(- indx_mat,- matrix_batch_shape,- matrix_batch_stride,- matrix_batch_ndim- );- } else {- mat += indx_mat * matrix_batch_stride[0];- }-- out_vec += tid.z * out_vec_size;-- gemv_kernel::run(- mat,- in_vec,- bias,- out_vec,- in_vec_size,- out_vec_size,- marix_ld,- alpha,- beta,- batch_ndim, // Not used,- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,- tid,- lid,- simd_gid,- simd_lid- );-}--// clang-format off-#define instantiate_gemv_t_bs_helper( \- nm, itype, bm, bn, sm, sn, tm, tn) \- instantiate_kernel( \- "gemv_t_gather_" #nm "_bm" #bm "_bn" #bn "_sm" #sm \- "_sn" #sn "_tm" #tm "_tn" #tn, \- gemv_t_gather, itype, bm, bn, sm, sn, tm, tn)--#define instantiate_gemv_t_bs_blocks(name, itype) \- instantiate_gemv_t_bs_helper(name, itype, 1, 2, 8, 4, 4, 1) \- instantiate_gemv_t_bs_helper(name, itype, 1, 2, 8, 4, 4, 4) \- instantiate_gemv_t_bs_helper(name, itype, 1, 4, 8, 4, 4, 4) \- instantiate_gemv_t_bs_helper(name, itype, 1, 16, 8, 4, 4, 4) \- instantiate_gemv_t_bs_helper(name, itype, 1, 16, 4, 8, 4, 4) // clang-format on--// clang-format off-instantiate_gemv_t_bs_blocks(float32, float);-instantiate_gemv_t_bs_blocks(float16, half);-instantiate_gemv_t_bs_blocks(bfloat16, bfloat16_t);-instantiate_gemv_t_bs_blocks(complex64, complex64_t); // clang-format ondiff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsdeleted file mode 100644index 4fdd13cb3..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ /dev/null@@ -1,158 +0,0 @@-use metal::{MTLBuffer, MTLComputeCommandEncoder};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::{- common::MatmulArguments,- dispatch_descriptor::{MatmulDispatchDescriptor, choose_dispatch_descriptor},- gemm,- gemv::GemvKernel,- split_k::SplitKGemm,-};-use crate::{- DataType,- backends::{- common::kernel::TensorAddBiasKernel,- metal::{context::MetalContext, error::MetalError, kernel::dsl::TensorAddBiasMetalKernel},- },-};--pub struct MatmulKernel {- data_type: DataType,- gemm: Option<gemm::GemmKernel>,- gemv: Option<GemvKernel>,- splitk: Option<SplitKGemm>,- bias_add: Option<TensorAddBiasMetalKernel>,-}--impl MatmulKernel {- pub fn new(data_type: DataType) -> Result<Self, MetalError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MetalError::Generic(format!("Unsupported dtype for MatmulKernel: {data_type:?}")));- }-- Ok(Self {- data_type,- gemm: None,- gemv: None,- splitk: None,- bias_add: None,- })- }-- pub fn precompile(- &mut self,- context: &MetalContext,- ) -> Result<(), MetalError> {- let gemm = self.get_or_create_gemm()?;- gemm.precompile(context)?;-- let gemv = self.get_or_create_gemv()?;- gemv.precompile(context)?;-- let splitk = self.get_or_create_splitk()?;- splitk.precompile(context)?;-- Ok(())- }-- fn get_or_create_gemm(&mut self) -> Result<&mut gemm::GemmKernel, MetalError> {- if self.gemm.is_none() {- self.gemm = Some(gemm::GemmKernel::new(self.data_type)?);- }- Ok(self.gemm.as_mut().unwrap())- }-- fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel, MetalError> {- if self.gemv.is_none() {- self.gemv = Some(GemvKernel::new(self.data_type)?);- }- Ok(self.gemv.as_mut().unwrap())- }-- fn get_or_create_splitk(&mut self) -> Result<&mut SplitKGemm, MetalError> {- if self.splitk.is_none() {- self.splitk = Some(SplitKGemm::new(self.data_type)?);- }- Ok(self.splitk.as_mut().unwrap())- }-- fn encode_dispatch_descriptor(- &mut self,- context: &MetalContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MatmulArguments,- descriptor: &MatmulDispatchDescriptor,- ) -> Result<bool, MetalError> {- match descriptor {- MatmulDispatchDescriptor::Gemv(descriptor) => {- let gemv = self.get_or_create_gemv()?;- gemv.encode_descriptor(context, encoder, arguments, descriptor)- },- MatmulDispatchDescriptor::SplitK(descriptor) => {- let splitk = self.get_or_create_splitk()?;- splitk.encode_descriptor(context, encoder, arguments, descriptor)- },- MatmulDispatchDescriptor::Gemm(descriptor) => {- let gemm = self.get_or_create_gemm()?;- gemm.encode_descriptor(context, encoder, arguments, descriptor)- },- }- }-- pub fn encode(- &mut self,- context: &MetalContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- mut arguments: MatmulArguments,- ) -> Result<(), MetalError> {- Self::apply_batch_collapse(&mut arguments);-- let descriptor = choose_dispatch_descriptor(context, self.data_type, &arguments)?;-- let bias_fused = self.encode_dispatch_descriptor(context, encoder, &arguments, &descriptor)?;-- if let Some(bias) = arguments.bias {- if !bias_fused {- self.apply_bias_add(context, encoder, &arguments, bias)?;- }- }-- Ok(())- }-- fn apply_bias_add(- &mut self,- context: &MetalContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MatmulArguments,- bias: &Retained<ProtocolObject<dyn MTLBuffer>>,- ) -> Result<(), MetalError> {- let m = arguments.batch as usize;- let n = arguments.output_dim as usize;- let batch_count = arguments.batch_count as usize;- let total_len = m * n * batch_count;- if total_len == 0 {- return Ok(());- }-- if self.bias_add.is_none() {- self.bias_add = Some(TensorAddBiasMetalKernel::new(context, self.data_type)?);- }- let bias_add = self.bias_add.as_ref().unwrap();- bias_add.encode(arguments.d, bias, arguments.d, n as u32, total_len as u32, encoder);- Ok(())- }-- fn apply_batch_collapse(arguments: &mut MatmulArguments) {- if arguments.transpose_a {- return;- }- if arguments.batch_count <= 1 {- return;- }- if arguments.lda == arguments.input_dim && arguments.transpose_b {- arguments.batch *= arguments.batch_count;- arguments.batch_count = 1;- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex 050733c0a..d0464c58c 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,29 +1,21 @@pub mod common;mod dispatch_descriptor;mod gemm;-mod gemv;-mod kernel;-mod split_k;-pub use common::MatmulArguments;-pub use dispatch_descriptor::{MatmulKernelVariant, determine_kernel_variant};-pub use gemv::GemvKernel;-pub use kernel::MatmulKernel;-use metal::MTLComputeCommandEncoder;-use objc2::runtime::ProtocolObject;-pub use split_k::SplitKGemm;+pub use dispatch_descriptor::choose_dispatch_descriptor;use crate::{DataType,backends::{common::kernel::matmul::{- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait, MatmulArguments,+ MatmulKernel,},metal::{Metal, context::MetalContext, error::MetalError},},};-impl FullPrecisionMatmulKernelTrait for MatmulKernel {+impl FullPrecisionMatmulKernelTrait for MatmulKernel<Metal> {type Backend = Metal;fn new(@@ -38,14 +30,13 @@ impl FullPrecisionMatmulKernelTrait for MatmulKernel {fn encode(&mut self,context: &MetalContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ encoder: &<Metal as crate::backends::common::Backend>::ComputeEncoder,arguments: FullPrecisionMatmulArguments<Metal>,) {- let matmul_arguments = MatmulArguments {+ let mut matmul_arguments = MatmulArguments {a: arguments.a,a_offset: arguments.a_offset as u64,b: arguments.b,- c: None,d: arguments.output,bias: arguments.bias,batch: arguments.batch as i32,@@ -55,13 +46,15 @@ impl FullPrecisionMatmulKernelTrait for MatmulKernel {ldb: arguments.input_dim as i32,ldd: arguments.output_dim as i32,batch_count: 1,- alpha: 1.0,- beta: 0.0,- transpose_a: false,transpose_b: true,};- MatmulKernel::encode(self, context, encoder, matmul_arguments)+ MatmulKernel::<Metal>::apply_batch_collapse(&mut matmul_arguments);++ let descriptor = choose_dispatch_descriptor(context, self.data_type, &matmul_arguments)+ .expect("Failed to create dispatch descriptor for full precision matmul");++ self.encode_with_descriptor(context, matmul_arguments, &descriptor, encoder).expect("Failed to encode full precision matmul kernel");}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rsdeleted file mode 100644index 713481877..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs+++ /dev/null@@ -1,241 +0,0 @@-use std::collections::HashMap;--use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};-use crate::{- DataType,- backends::{- common::Context,- metal::{- context::MetalContext,- error::MetalError,- kernel::matmul::common::{MatmulArguments, transpose_configuration},- metal_extensions::ComputeEncoderSetValue,- },- },-};--pub struct Kernel {- data_type: DataType,- partial_pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- accum_pipeline: Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- accumulator_buffer_bytes: usize,-}--impl Kernel {- pub fn new(data_type: DataType) -> Result<Self, MetalError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MetalError::Generic(format!("Unsupported dtype for Split-K: {:?}", data_type)));- }- Ok(Self {- data_type,- partial_pipelines: HashMap::new(),- accum_pipeline: None,- accumulator_buffer: None,- accumulator_buffer_bytes: 0,- })- }-- pub fn precompile(- &mut self,- context: &MetalContext,- ) -> Result<(), MetalError> {- use super::tile_configuration::TileConfiguration;-- if !matches!(self.data_type, DataType::BF16) {- return Ok(());- }-- let tile_configs = [- TileConfiguration {- tile_rows: 16,- tile_cols: 16,- tile_depth: 16,- warps_per_row: 2,- warps_per_col: 2,- },- TileConfiguration {- tile_rows: 16,- tile_cols: 32,- tile_depth: 16,- warps_per_row: 2,- warps_per_col: 2,- },- TileConfiguration {- tile_rows: 32,- tile_cols: 16,- tile_depth: 16,- warps_per_row: 2,- warps_per_col: 2,- },- TileConfiguration {- tile_rows: 32,- tile_cols: 32,- tile_depth: 16,- warps_per_row: 2,- warps_per_col: 2,- },- ];-- for tile in &tile_configs {- for &mn_aligned in &[false, true] {- let config = PipelineConfiguration {- tile: *tile,- transpose_a: false,- transpose_b: true,- mn_aligned,- k_aligned: true,- };- let _ = self.get_partial_pipeline(context, &config);- }- }-- let _ = self.get_accum_pipeline(context);-- Ok(())- }-- fn steel_type_name(&self) -> Result<&'static str, MetalError> {- match self.data_type {- DataType::F16 => Ok("float16"),- DataType::BF16 => Ok("bfloat16"),- DataType::F32 => Ok("float32"),- _ => Err(MetalError::Generic(format!("Unsupported dtype for Split-K: {:?}", self.data_type))),- }- }-- fn splitk_partial_out_name(&self) -> &'static str {- "float32"- }-- fn partial_kernel_name(- &self,- config: &PipelineConfiguration,- ) -> Result<String, MetalError> {- let in_name = self.steel_type_name()?;- let out_name = self.splitk_partial_out_name();- let transpose_suffix = transpose_configuration(config.transpose_a, config.transpose_b).as_str();- let mn_tag = if config.mn_aligned {- "taligned"- } else {- "naligned"- };- let k_tag = if config.k_aligned {- "taligned"- } else {- "naligned"- };- Ok(format!(- "steel_gemm_splitk_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",- transpose_suffix,- in_name,- out_name,- config.tile.tile_rows,- config.tile.tile_cols,- config.tile.tile_depth,- config.tile.warps_per_row,- config.tile.warps_per_col,- mn_tag,- k_tag,- ))- }-- fn accum_kernel_name(&self) -> Result<String, MetalError> {- let out_name = self.steel_type_name()?;- Ok(format!("steel_gemm_splitk_accum_{}_{}", out_name, self.splitk_partial_out_name()))- }-- fn get_partial_pipeline(- &mut self,- mtl: &MetalContext,- config: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {- if !self.partial_pipelines.contains_key(config) {- let name = self.partial_kernel_name(config)?;- let ps = mtl.compute_pipeline_state(&name, None)?;- self.partial_pipelines.insert(config.clone(), ps);- }- Ok(self.partial_pipelines.get(config).unwrap())- }-- fn get_accum_pipeline(- &mut self,- mtl: &MetalContext,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {- if self.accum_pipeline.is_none() {- let name = self.accum_kernel_name()?;- let ps = mtl.compute_pipeline_state(&name, None)?;- self.accum_pipeline = Some(ps);- }- Ok(self.accum_pipeline.as_ref().unwrap())- }-- pub fn should_use_splitk(- m: i32,- n: i32,- k: i32,- batch_count: i32,- ) -> bool {- if batch_count != 1 {- return false;- }- if m <= 0 || n <= 0 || k <= 0 {- return false;- }- let m_tiles = m / 16;- let n_tiles = n / 16;- let k_tiles = k / 16;- (m_tiles * n_tiles) <= 32 && k_tiles >= 8- }-- pub(crate) fn encode_descriptor(- &mut self,- context: &MetalContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MatmulArguments,- descriptor: &DispatchDescriptor,- ) -> Result<bool, MetalError> {- self.ensure_accumulator_buffer(context, descriptor.accumulator_bytes);- let accumulator_buffer =- self.accumulator_buffer.as_ref().cloned().expect("Accumulator buffer must be initialized");-- let partial_pipeline_state = self.get_partial_pipeline(context, &descriptor.pipeline_configuration)?;-- encoder.set_compute_pipeline_state(partial_pipeline_state);- encoder.set_buffer(Some(arguments.a), arguments.a_offset as usize, 0);- encoder.set_buffer(Some(arguments.b), 0, 1);- encoder.set_buffer(Some(&accumulator_buffer), 0, 2);- encoder.set_value(&descriptor.params, 3);- encoder.dispatch_threadgroups(descriptor.partial_threadgroups, descriptor.partial_threads_per_threadgroup);-- let accum_pipeline_state = self.get_accum_pipeline(context)?;- encoder.set_compute_pipeline_state(accum_pipeline_state);- encoder.set_buffer(Some(&accumulator_buffer), 0, 0);- encoder.set_buffer(Some(arguments.d), 0, 1);-- let partition_count = descriptor.partition_count;- let output_elements_per_partition = descriptor.output_elements_per_partition;- encoder.set_value(&partition_count, 2);- encoder.set_value(&output_elements_per_partition, 3);- encoder.set_value(&arguments.ldd, 4);-- encoder.dispatch_threads(descriptor.accum_total_threads, descriptor.accum_threads_per_threadgroup);-- Ok(false)- }-- fn ensure_accumulator_buffer(- &mut self,- mtl: &MetalContext,- required_bytes: usize,- ) {- if required_bytes <= self.accumulator_buffer_bytes && self.accumulator_buffer.is_some() {- return;- }- self.accumulator_buffer = Some(mtl.create_buffer(required_bytes).expect("Failed to create accumulator buffer"));- self.accumulator_buffer_bytes = required_bytes;- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rsindex 029fa0f90..b4607a3dd 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs@@ -1,7 +1 @@-mod dispatch_descriptor;-mod kernel;-mod pipeline_configuration;-mod tile_configuration;--pub(crate) use dispatch_descriptor::DispatchDescriptor;-pub use kernel::Kernel as SplitKGemm;+// Shader files only — Rust logic has moved to common/kernel/matmul/split_k/diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.hdeleted file mode 100644index 60d238d98..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h+++ /dev/null@@ -1,235 +0,0 @@---using namespace steel;--///////////////////////////////////////////////////////////////////////////////-// GEMM kernels-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- bool MN_aligned,- bool K_aligned>-[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void gemm_splitk(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- device U* C [[buffer(2)]],- const constant GEMMSpiltKParams* params [[buffer(3)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]]-) {- (void)lid;-- using gemm_kernel = GEMMKernel<- T,- U,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- MN_aligned,- K_aligned>;- using loader_a_t = typename gemm_kernel::loader_a_t;- using loader_b_t = typename gemm_kernel::loader_b_t;- using mma_t = typename gemm_kernel::mma_t;-- threadgroup T As[gemm_kernel::tgp_mem_size_a];- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];-- const int tid_x = tid.x;- const int tid_y = tid.y;- const int tid_z = tid.z;-- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {- return;- }-- // Find block in A, B, C- const int c_row = tid_y * BM;- const int c_col = tid_x * BN;- const int k_start = params->split_k_partition_size * tid_z;-- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);- const size_t k_start_long = size_t(k_start);-- A += transpose_a ? (c_row_long + k_start_long * params->lda)- : (k_start_long + c_row_long * params->lda);- B += transpose_b ? (k_start_long + c_col_long * params->ldb)- : (c_col_long + k_start_long * params->ldb);- C += (size_t(params->split_k_partition_stride) * tid_z) +- (c_row_long * params->ldc + c_col_long);-- // Prepare threadgroup loading operations- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);-- // Prepare threadgroup mma operation- thread mma_t mma_op(simd_group_id, simd_lane_id);-- int gemm_k_iterations = params->gemm_k_iterations_aligned;-- short tgp_bm = min(BM, params->M - c_row);- short tgp_bn = min(BN, params->N - c_col);- short leftover_bk = params->K % BK;-- if (MN_aligned || (tgp_bm == BM && tgp_bn == BN)) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<true, true, true>{}- );- } else if (tgp_bn == BN) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<false, true, true>{}- );- } else if (tgp_bm == BM) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<true, false, true>{}- );- } else {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<false, false, true>{}- );- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if ((tid_z + 1) == (params->split_k_partitions)) {- int gemm_k_iter_remaining =- (params->K - (k_start + params->split_k_partition_size)) / BK;- if (!K_aligned || gemm_k_iter_remaining > 0)- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iter_remaining,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk,- LoopAlignment<false, false, K_aligned>{}- );- }-- if (MN_aligned || (tgp_bm == BM && tgp_bn == BN)) {- mma_op.store_result(C, params->ldc);- } else {- mma_op.store_result_safe(C, params->ldc, short2(tgp_bn, tgp_bm));- }-}--///////////////////////////////////////////////////////////////////////////////-// Split k accumulation kernel-///////////////////////////////////////////////////////////////////////////////--template <- typename AccT,- typename OutT,- typename Epilogue = TransformNone<OutT, AccT>>-[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_accum(- const device AccT* C_split [[buffer(0)]],- device OutT* D [[buffer(1)]],- const constant int& k_partitions [[buffer(2)]],- const constant int& partition_stride [[buffer(3)]],- const constant int& ldd [[buffer(4)]],- uint2 gid [[thread_position_in_grid]]-) {- // Ajust D and C- D += gid.x + gid.y * size_t(ldd);- C_split += gid.x + gid.y * size_t(ldd);-- size_t offset = 0;- AccT out = 0;-- for (int i = 0; i < k_partitions; i++) {- out += C_split[offset];- offset += partition_stride;- }-- // Write output- D[0] = Epilogue::apply(out);-}--template <- typename AccT,- typename OutT,- typename Epilogue = TransformAxpby<OutT, AccT>>-[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_accum_axpby(- const device AccT* C_split [[buffer(0)]],- device OutT* D [[buffer(1)]],- const constant int& k_partitions [[buffer(2)]],- const constant int& partition_stride [[buffer(3)]],- const constant int& ldd [[buffer(4)]],- const device OutT* C [[buffer(5)]],- const constant int& ldc [[buffer(6)]],- const constant int& fdc [[buffer(7)]],- const constant float& alpha [[buffer(8)]],- const constant float& beta [[buffer(9)]],- uint2 gid [[thread_position_in_grid]]-) {- // Ajust D and C- C += gid.x * size_t(fdc) + gid.y * size_t(ldc);- D += gid.x + gid.y * size_t(ldd);- C_split += gid.x + gid.y * size_t(ldd);-- size_t offset = 0;- AccT out = 0;-- for (int i = 0; i < k_partitions; i++) {- out += C_split[offset];- offset += partition_stride;- }-- // Write output- Epilogue op(alpha, beta);- D[0] = op.apply(out, *C);-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metalindex 34c61e201..806683221 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal@@ -1,76 +1,322 @@--// clang-format off#include "../../../common/utils.h"+#include "../../../definitions.metal"#include "../../common/steel/gemm/gemm.h"-#include "steel_gemm_splitk.h"--#define instantiate_gemm( \- tname, \- trans_a, \- trans_b, \- iname, \- itype, \- oname, \- otype, \- bm, \- bn, \- bk, \- wm, \- wn, \- aname, \- mn_aligned, \- kname, \- k_aligned) \- instantiate_kernel( \- "steel_gemm_splitk_" #tname "_" #iname "_" #oname \- "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn \- "_MN_" #aname "_K_" #kname, \- gemm_splitk, \- itype, \- otype, \- bm, \- bn, \- bk, \- wm, \- wn, \- trans_a, \- trans_b, \- mn_aligned, \- k_aligned)--#define instantiate_gemm_aligned_helper(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, taligned, true) \- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, naligned, false) \- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, taligned, true) \- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, naligned, false)--#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm_aligned_helper(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm_aligned_helper(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm_aligned_helper(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm_aligned_helper(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)--#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 16, 16, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 32, 16, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 16, 16, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)--instantiate_gemm_shapes_helper(float16, half, float32, float);-instantiate_gemm_shapes_helper(bfloat16, bfloat16_t, float32, float);-instantiate_gemm_shapes_helper(float32, float, float32, float);-instantiate_gemm_shapes_helper(complex64, complex64_t, complex64, complex64_t);--#define instantiate_accum(oname, otype, aname, atype) \- instantiate_kernel( \- "steel_gemm_splitk_accum_" #oname "_" #aname, \- gemm_splitk_accum, atype, otype) \- instantiate_kernel( \- "steel_gemm_splitk_accum_" #oname "_" #aname "_axbpy", \- gemm_splitk_accum_axpby, atype, otype) \--instantiate_accum(bfloat16, bfloat16_t, float32, float);-instantiate_accum(float16, half, float32, float);-instantiate_accum(float32, float, float32, float);-instantiate_accum(complex64, complex64_t, complex64, complex64_t); // clang-format on++using namespace steel;++// Upper bounds for split-K threadgroup memory.+// Split-K tile: BM=16, BN=32, BK=16 with bfloat16 (padding=8)+// tgp_mem_size_a = BM * (BK + padding) = 16 * (16 + 8) = 384+// tgp_mem_size_b = BN * (BK + padding) = 32 * (16 + 8) = 768+#define SPLITK_MAX_TGP_A 384+#define SPLITK_MAX_TGP_B 768++namespace uzu {+namespace matmul {+using GEMMSpiltKParams = steel::GEMMSpiltKParams;+} // namespace matmul+} // namespace uzu++///////////////////////////////////////////////////////////////////////////////+// Split-K GEMM implementation+///////////////////////////////////////////////////////////////////////////////++template <typename T, typename U>+METAL_FUNC void gemm_splitk_impl(+ const device T* a,+ const device T* b,+ device U* c,+ const constant GEMMSpiltKParams* params,+ const int BM,+ const int BN,+ const int BK,+ const int WM,+ const int WN,+ const bool align_m,+ const bool align_k,+ threadgroup T* a_shared,+ threadgroup T* b_shared,+ uint simd_lane_id,+ uint simd_group_id,+ uint3 tid,+ uint3 lid+) {+ (void)lid;++ const bool transpose_a = false;+ const bool transpose_b = true;++ const short tgp_padding = 16 / sizeof(T);+ const short lda_tgp = BK + tgp_padding;+ const short ldb_tgp = BK + tgp_padding;+ const short tgp_size = WM * WN * 32;++ const int tid_x = tid.x;+ const int tid_y = tid.y;+ const int tid_z = tid.z;++ if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {+ return;+ }++ const int c_row = tid_y * BM;+ const int c_col = tid_x * BN;+ const int k_start = params->split_k_partition_size * tid_z;++ const size_t c_row_long = size_t(c_row);+ const size_t c_col_long = size_t(c_col);+ const size_t k_start_long = size_t(k_start);++ a += k_start_long + c_row_long * params->lda;+ b += k_start_long + c_col_long * params->ldb;+ c += (size_t(params->split_k_partition_stride) * tid_z) ++ (c_row_long * params->ldc + c_col_long);++ thread BlockLoader<T> loader_a(+ a,+ params->lda,+ a_shared,+ simd_group_id,+ simd_lane_id,+ BM,+ BK,+ lda_tgp,+ 1,+ tgp_size+ );++ thread BlockLoader<T> loader_b(+ b,+ params->ldb,+ b_shared,+ simd_group_id,+ simd_lane_id,+ BN,+ BK,+ ldb_tgp,+ 1,+ tgp_size+ );++ thread BlockMMA<T, U, float> mma_op(+ simd_group_id,+ simd_lane_id,+ BM,+ BN,+ BK,+ WM,+ WN,+ transpose_a,+ transpose_b,+ lda_tgp,+ ldb_tgp+ );++ int gemm_k_iterations = params->gemm_k_iterations_aligned;++ short tgp_bm = min(BM, params->M - c_row);+ short tgp_bn = min(BN, params->N - c_col);+ short leftover_bk = params->K % BK;++ bool mn_aligned = align_m || (tgp_bm == BM && tgp_bn == BN);++ if (mn_aligned) {+ gemm_loop<T, U, float>(+ a_shared,+ b_shared,+ gemm_k_iterations,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk,+ BK,+ transpose_a,+ transpose_b,+ true,+ true,+ true+ );+ } else if (tgp_bn == BN) {+ gemm_loop<T, U, float>(+ a_shared,+ b_shared,+ gemm_k_iterations,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk,+ BK,+ transpose_a,+ transpose_b,+ false,+ true,+ true+ );+ } else if (tgp_bm == BM) {+ gemm_loop<T, U, float>(+ a_shared,+ b_shared,+ gemm_k_iterations,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk,+ BK,+ transpose_a,+ transpose_b,+ true,+ false,+ true+ );+ } else {+ gemm_loop<T, U, float>(+ a_shared,+ b_shared,+ gemm_k_iterations,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk,+ BK,+ transpose_a,+ transpose_b,+ false,+ false,+ true+ );+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if ((tid_z + 1) == (params->split_k_partitions)) {+ int gemm_k_iter_remaining =+ (params->K - (k_start + params->split_k_partition_size)) / BK;+ if (!align_k || gemm_k_iter_remaining > 0)+ gemm_loop<T, U, float>(+ a_shared,+ b_shared,+ gemm_k_iter_remaining,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk,+ BK,+ transpose_a,+ transpose_b,+ false,+ false,+ align_k+ );+ }++ if (mn_aligned) {+ mma_op.store_result(c, params->ldc);+ } else {+ mma_op.store_result_safe(c, params->ldc, short2(tgp_bn, tgp_bm));+ }+}++///////////////////////////////////////////////////////////////////////////////+// Split-K accumulation+///////////////////////////////////////////////////////////////////////////////++template <+ typename AccT,+ typename OutT,+ typename Epilogue = TransformNone<OutT, AccT>>+METAL_FUNC void gemm_splitk_accum_impl(+ const device AccT* c_split,+ device OutT* d,+ const constant int& k_partitions,+ const constant int& partition_stride,+ const constant int& ldd,+ uint2 gid+) {+ d += gid.x + gid.y * size_t(ldd);+ c_split += gid.x + gid.y * size_t(ldd);++ size_t offset = 0;+ AccT out = 0;++ for (int i = 0; i < k_partitions; i++) {+ out += c_split[offset];+ offset += partition_stride;+ }++ d[0] = Epilogue::apply(out);+}++///////////////////////////////////////////////////////////////////////////////+// DSL kernel entry points+///////////////////////////////////////////////////////////////////////////////++KERNEL(MatmulSplitKPartialBfloat16)(+ const device bfloat16_t* a,+ const device bfloat16_t* b,+ device float* c,+ const constant uzu::matmul::GEMMSpiltKParams* params,+ const constant uint& partial_group_count_x,+ const constant uint& partial_group_count_y,+ const constant uint& partial_group_count_z,+ const uint group_x GROUPS(partial_group_count_x),+ const uint group_y GROUPS(partial_group_count_y),+ const uint group_z GROUPS(partial_group_count_z),+ const uint thread_x THREADS(32),+ const uint thread_y THREADS(2),+ const uint thread_z THREADS(2),+ threadgroup bfloat16_t a_shared[SPLITK_MAX_TGP_A],+ threadgroup bfloat16_t b_shared[SPLITK_MAX_TGP_B],+ const Simd simd+) {+ gemm_splitk_impl<bfloat16_t, float>(+ a,+ b,+ c,+ params,+ 16,+ 32,+ 16,+ 2,+ 2,+ false,+ true,+ a_shared,+ b_shared,+ simd.lane_idx,+ simd.group_idx,+ uint3(group_x, group_y, group_z),+ uint3(thread_x, thread_y, thread_z)+ );+}++KERNEL(MatmulSplitKAccumBfloat16)(+ const device float* c_split,+ device bfloat16_t* d,+ const constant int& k_partitions,+ const constant int& partition_stride,+ const constant int& ldd,+ const constant uint& accum_total_threads_x,+ const constant uint& accum_total_threads_y,+ const uint gid_x AXIS(accum_total_threads_x, 16),+ const uint gid_y AXIS(accum_total_threads_y, 16)+) {+ gemm_splitk_accum_impl<float, bfloat16_t>(+ c_split,+ d,+ k_partitions,+ partition_stride,+ ldd,+ uint2(gid_x, gid_y)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 8f0af55a2..021380535 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -1,5 +1,7 @@-use crate::backends::common::kernel::matmul::MatmulKernels;-+use crate::backends::{+ common::kernel::matmul::{MatmulKernel, MatmulKernels},+ metal::Metal,+};pub mod dsl {include!(concat!(env!("OUT_DIR"), "/dsl.rs"));}@@ -7,8 +9,6 @@ pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/defaultpub mod matmul;-pub use matmul::{MatmulArguments, MatmulKernel};-impl MatmulKernels for dsl::MetalKernels {- type FullPrecisionMatmulKernel = MatmulKernel;+ type FullPrecisionMatmulKernel = MatmulKernel<Metal>;}diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 4982e0eca..8d0d232f6 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -7,11 +7,11 @@ use ndarray::Array2;use uzu::{DataType,backends::{- common::Context,- metal::{- MetalContext,- kernel::{MatmulArguments, MatmulKernel},+ common::{+ Context,+ kernel::matmul::{MatmulArguments, MatmulKernel},},+ metal::{Metal, MetalContext, kernel::matmul::choose_dispatch_descriptor},},};@@ -43,33 +43,28 @@ fn run_metal_matmul(n};- let mut kernel = MatmulKernel::new(DataType::BF16).expect("kernel");+ let mut kernel = MatmulKernel::<Metal>::new(DataType::BF16).expect("kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");- let encode_result = kernel.encode(- ctx,- &enc,- MatmulArguments {- a: &a_buf,- a_offset: 0,- b: &b_buf,- c: None,- d: &d_buf,- bias: None,- batch: m as i32,- input_dim: k as i32,- output_dim: n as i32,- lda: k as i32,- ldb: ldb as i32,- ldd: n as i32,- batch_count: 1,- alpha: 1.0,- beta: 0.0,- transpose_a: false,- transpose_b,- },- );+ let mut arguments = MatmulArguments {+ a: &a_buf,+ a_offset: 0,+ b: &b_buf,+ d: &d_buf,+ bias: None,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: ldb as i32,+ ldd: n as i32,+ batch_count: 1,+ transpose_b,+ };+ MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);+ let descriptor = choose_dispatch_descriptor(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");+ let encode_result = kernel.encode_with_descriptor(ctx, arguments, &descriptor, &enc);enc.end_encoding();encode_result.expect("encode");cb.commit();@@ -129,87 +124,101 @@ struct TestCase {tolerance: f32,}-fn test_cases() -> Vec<TestCase> {- let base_tolerance = 0.01;+const MATMUL_CORRECTNESS_FULL_ENV: &str = "UZU_MATMUL_CORRECTNESS_FULL";++const QUICK_BATCH_SIZES: [usize; 2] = [1, 16];+const FULL_BATCH_SIZES: [usize; 12] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];- let batch_sizes: [usize; 12] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];-- let model_shapes: [(usize, usize); 51] = [- // Qwen2.5-Coder-0.5B- (896, 896),- (896, 1152),- (896, 4864),- (4864, 896),- // Qwen3-0.6B- (1024, 1024),- (1024, 3072),- (1024, 4096),- (3072, 1024),- // Gemma-3-1B- (1152, 1152),- (1152, 1536),- (1152, 6912),- (6912, 1152),- // Qwen2.5-Coder-1.5B- (1536, 1536),- (1536, 2048),- (1536, 8960),- (8960, 1536),- // Llama-3.2-1B / Qwen3-1.7B / SmolLM2-1.7B- (2048, 2048),- (2048, 2560),- (2048, 3072),- (2048, 4096),- (2048, 6144),- (2048, 8192),- (2048, 11008),- (6144, 2048),- (8192, 2048),- (11008, 2048),- // Qwen3-4B / Gemma-3-4B- (2560, 2560),- (2560, 4096),- (2560, 6144),- (2560, 9728),- (2560, 10240),- (9728, 2560),- (10240, 2560),- // Llama-3.2-3B- (3072, 3072),- (3072, 5120),- (3072, 8192),- (8192, 3072),- // Qwen2.5-Coder-7B- (3584, 3584),- (3584, 4608),- (3584, 18944),- (18944, 3584),- // Llama-3.1-8B / Qwen3-8B- (4096, 4096),- (4096, 6144),- (4096, 12288),- (4096, 14336),- (12288, 4096),- (14336, 4096),- // Qwen3-14B- (5120, 5120),- (5120, 7168),- (5120, 17408),- (17408, 5120),- ];+const QUICK_MODEL_SHAPES: [(usize, usize); 8] =+ [(896, 896), (896, 4864), (1024, 4096), (1152, 6912), (1536, 8960), (2048, 8192), (4096, 14336), (5120, 17408)];++const FULL_MODEL_SHAPES: [(usize, usize); 51] = [+ // Qwen2.5-Coder-0.5B+ (896, 896),+ (896, 1152),+ (896, 4864),+ (4864, 896),+ // Qwen3-0.6B+ (1024, 1024),+ (1024, 3072),+ (1024, 4096),+ (3072, 1024),+ // Gemma-3-1B+ (1152, 1152),+ (1152, 1536),+ (1152, 6912),+ (6912, 1152),+ // Qwen2.5-Coder-1.5B+ (1536, 1536),+ (1536, 2048),+ (1536, 8960),+ (8960, 1536),+ // Llama-3.2-1B / Qwen3-1.7B / SmolLM2-1.7B+ (2048, 2048),+ (2048, 2560),+ (2048, 3072),+ (2048, 4096),+ (2048, 6144),+ (2048, 8192),+ (2048, 11008),+ (6144, 2048),+ (8192, 2048),+ (11008, 2048),+ // Qwen3-4B / Gemma-3-4B+ (2560, 2560),+ (2560, 4096),+ (2560, 6144),+ (2560, 9728),+ (2560, 10240),+ (9728, 2560),+ (10240, 2560),+ // Llama-3.2-3B+ (3072, 3072),+ (3072, 5120),+ (3072, 8192),+ (8192, 3072),+ // Qwen2.5-Coder-7B+ (3584, 3584),+ (3584, 4608),+ (3584, 18944),+ (18944, 3584),+ // Llama-3.1-8B / Qwen3-8B+ (4096, 4096),+ (4096, 6144),+ (4096, 12288),+ (4096, 14336),+ (12288, 4096),+ (14336, 4096),+ // Qwen3-14B+ (5120, 5120),+ (5120, 7168),+ (5120, 17408),+ (17408, 5120),+];++fn run_full_case_matrix() -> bool {+ std::env::var(MATMUL_CORRECTNESS_FULL_ENV)+ .ok()+ .map(|value| {+ let normalized = value.trim().to_ascii_lowercase();+ matches!(normalized.as_str(), "1" | "true" | "yes")+ })+ .unwrap_or(false)+}++fn test_cases(full_case_matrix: bool) -> Vec<TestCase> {+ let base_tolerance = 0.01;+ let (batch_sizes, model_shapes): (&[usize], &[(usize, usize)]) = if full_case_matrix {+ (&FULL_BATCH_SIZES, &FULL_MODEL_SHAPES)+ } else {+ (&QUICK_BATCH_SIZES, &QUICK_MODEL_SHAPES)+ };let mut cases: Vec<(usize, usize, usize, bool)> =model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n, true))).collect();- // Non-transposed B (for attention AV matmul)- for &(k, n) in &[(1024, 1024), (2048, 2048), (4096, 4096)] {- for &m in &[1, 64, 128] {- cases.push((m, k, n, false));- }- }-// Edge case: small matrix- for &m in &batch_sizes {+ for &m in batch_sizes {cases.push((m, 128, 128, true));}@@ -270,7 +279,17 @@ fn matmul_correctness_comprehensive() {return;};- let cases = test_cases();+ let full_case_matrix = run_full_case_matrix();+ let cases = test_cases(full_case_matrix);+ eprintln!(+ "Running {} matmul correctness case matrix (set {}=1 for exhaustive run)",+ if full_case_matrix {+ "exhaustive"+ } else {+ "quick"+ },+ MATMUL_CORRECTNESS_FULL_ENV+ );let mut passed = 0;let mut failed = Vec::new();