A high-performance inference engine for AI models
move matmul impl details to metal from common (#262)
eugenee19fc3c2eb82
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex b5be8941d..528554320 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -315,6 +315,12 @@ pub fn bindgen((quote! {}, quote! {}, None)};+ let method_visibility = if kernel.public {+ quote! {}+ } else {+ quote! { pub(crate) }+ };+let kernel = quote! {pub struct #struct_name {pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,@@ -324,14 +330,14 @@ pub fn bindgen(impl #maybe_trait_impl #struct_name {#maybe_associate_backend- fn new(context: &MetalContext #(, #variants_extra_arguments)* #(, #specialize_args)*) -> Result<Self, MetalError> {+ #method_visibility fn new(context: &MetalContext #(, #variants_extra_arguments)* #(, #specialize_args)*) -> Result<Self, MetalError> {let entry_name = #entry_name;#function_constants_initlet pipeline = context.compute_pipeline_state(#cache_key, &entry_name, #function_constants_arg)?;Ok(Self { pipeline #(, #conditional_buffer_sets)* })}- fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#encode_args_defs, )* command_buffer: &'command_buffer mut MetalCommandBufferEncoding) {+ #method_visibility fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#encode_args_defs, )* command_buffer: &'command_buffer mut MetalCommandBufferEncoding) {let compute_encoder = command_buffer.ensure_compute();#empty_dispatch_guardscompute_encoder.set_compute_pipeline_state(&self.pipeline);diff --git a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rsdeleted file mode 100644index cde615975..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs+++ /dev/null@@ -1,35 +0,0 @@-use super::{MatmulArguments, MatmulError};-use crate::{- DataType,- backends::common::{- Backend,- kernel::matmul::{gemm::GemmDispatchDescriptor, gemv::GemvDispatchDescriptor},- },-};--#[derive(Debug, Clone)]-pub enum MatmulDispatchDescriptor {- Gemv(GemvDispatchDescriptor),- Gemm(GemmDispatchDescriptor),-}--impl MatmulDispatchDescriptor {- pub fn bias_is_fused(&self) -> bool {- match self {- MatmulDispatchDescriptor::Gemv(d) => d.bias_is_fused(),- MatmulDispatchDescriptor::Gemm(_) => false,- }- }-}--pub fn choose_matmul_dispatch_descriptor<B: Backend>(- context: &B::Context,- data_type: DataType,- arguments: &MatmulArguments<B>,-) -> Result<MatmulDispatchDescriptor, MatmulError<B>> {- if let Some(descriptor) = GemvDispatchDescriptor::try_new::<B>(data_type, arguments)? {- return Ok(MatmulDispatchDescriptor::Gemv(descriptor));- }-- Ok(MatmulDispatchDescriptor::Gemm(GemmDispatchDescriptor::try_new::<B>(context, data_type, arguments)?))-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rsdeleted file mode 100644index 5cf2ad669..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs+++ /dev/null@@ -1,77 +0,0 @@-use super::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels};-use crate::{- DataType,- backends::common::{- Backend, CommandBuffer, kernel::matmul::dispatch_descriptor::choose_matmul_dispatch_descriptor,- },-};--pub struct FullPrecisionMatmulArguments<'a, B: Backend> {- pub a: &'a B::Buffer,- pub a_offset: usize,- pub b: &'a B::Buffer,- pub output: &'a mut B::Buffer,- pub bias: Option<&'a B::Buffer>,- pub batch: usize,- pub input_dim: usize,- pub output_dim: usize,-}--pub trait FullPrecisionMatmulKernel: Sized {- type Backend: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>;-- fn new(- context: &<Self::Backend as Backend>::Context,- data_type: DataType,- ) -> Result<Self, MatmulError<Self::Backend>>;-- fn encode(- &mut self,- context: &<Self::Backend as Backend>::Context,- command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,- arguments: FullPrecisionMatmulArguments<Self::Backend>,- );-}--impl<B: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>> FullPrecisionMatmulKernel- for MatmulKernel<B>-{- type Backend = B;-- fn new(- context: &<Self::Backend as Backend>::Context,- data_type: DataType,- ) -> Result<Self, MatmulError<Self::Backend>> {- let mut kernel = MatmulKernel::new(data_type)?;- kernel.precompile(context)?;- Ok(kernel)- }-- fn encode(- &mut self,- context: &<Self::Backend as Backend>::Context,- command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,- arguments: FullPrecisionMatmulArguments<Self::Backend>,- ) {- let matmul_arguments = MatmulArguments {- a: arguments.a,- a_offset: arguments.a_offset as u64,- b: arguments.b,- d: arguments.output,- bias: arguments.bias,- batch: arguments.batch as i32,- input_dim: arguments.input_dim as i32,- output_dim: arguments.output_dim as i32,- leading_dimension_a: arguments.input_dim as i32,- leading_dimension_b: arguments.input_dim as i32,- leading_dimension_d: arguments.output_dim as i32,- transpose_b: true,- };-- let descriptor = choose_matmul_dispatch_descriptor(context, self.data_type, &matmul_arguments)- .expect("Failed to create dispatch descriptor for full precision matmul");-- self.encode_with_descriptor(context, matmul_arguments, &descriptor, command_buffer)- .expect("Failed to encode full precision matmul kernel");- }-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rsdeleted file mode 100644index 21ce06389..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs+++ /dev/null@@ -1,67 +0,0 @@-use super::{super::grid_size::GridSize, specialization::GemmSpecialization};-use crate::{- DataType,- backends::common::{- Backend,- gpu_types::GemmParams,- kernel::matmul::{MatmulArguments, MatmulError},- },-};--#[derive(Debug, Clone)]-pub struct GemmDispatchDescriptor {- pub specialization: GemmSpecialization,- pub params: GemmParams,- pub threadgroups: GridSize,-}--impl GemmDispatchDescriptor {- pub fn try_new<B: Backend>(- context: &B::Context,- data_type: DataType,- arguments: &MatmulArguments<B>,- ) -> Result<Self, MatmulError<B>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));- }-- let config = GemmSpecialization::select::<B>(context, data_type, arguments);-- let m = arguments.batch;- let n = arguments.output_dim;- let k = arguments.input_dim;-- let threadgroups_per_row = (n + config.block_cols - 1) / config.block_cols;- let threadgroups_per_column = (m + config.block_rows - 1) / config.block_rows;- let swizzle_log = config.swizzle_log2;-- let swizzle_stride = 1_i32 << swizzle_log;- let tm_swizzled = (threadgroups_per_column + swizzle_stride - 1) / swizzle_stride;- let tn_swizzled = threadgroups_per_row * swizzle_stride;-- let params = GemmParams {- M: m,- N: n,- K: k,- leading_dimension_a: arguments.leading_dimension_a,- leading_dimension_b: arguments.leading_dimension_b,- leading_dimension_d: arguments.leading_dimension_d,- threadgroups_per_row,- threadgroups_per_column,- swizzle_log,- aligned_inner_iterations: k / config.block_depth,- };-- let threadgroups = GridSize {- x: tn_swizzled as usize,- y: tm_swizzled as usize,- z: 1,- };-- Ok(Self {- specialization: config,- params,- threadgroups,- })- }-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rsdeleted file mode 100644index a1502a1bd..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs+++ /dev/null@@ -1,92 +0,0 @@-use std::{collections::HashMap, ops::DerefMut};--use super::{- super::matmul_arguments::MatmulArguments, dispatch_descriptor::GemmDispatchDescriptor,- specialization::GemmSpecialization,-};-use crate::{- DataType,- backends::common::{- Backend, CommandBuffer, Kernels,- kernel::{MatmulGemmKernel, matmul::MatmulError},- },-};--pub struct GemmKernel<B: Backend> {- data_type: DataType,- pipelines: HashMap<GemmSpecialization, <B::Kernels as Kernels>::MatmulGemmKernel>,-}--impl<B: Backend> GemmKernel<B> {- pub fn new(data_type: DataType) -> Result<Self, MatmulError<B>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));- }- Ok(Self {- data_type,- pipelines: HashMap::new(),- })- }-- pub fn precompile(- &mut self,- context: &B::Context,- ) -> Result<(), MatmulError<B>> {- for &config in GemmSpecialization::precompile_configs(self.data_type) {- self.get_or_create_kernel(context, config)?;- }- Ok(())- }-- fn get_or_create_kernel(- &mut self,- context: &B::Context,- config: GemmSpecialization,- ) -> Result<&<B::Kernels as Kernels>::MatmulGemmKernel, MatmulError<B>> {- if !self.pipelines.contains_key(&config) {- let kernel = <B::Kernels as Kernels>::MatmulGemmKernel::new(- context,- self.data_type,- config.block_rows as u32,- config.block_cols as u32,- config.block_depth as u32,- config.simdgroups_per_row as u32,- config.simdgroups_per_column as u32,- config.align_m,- config.align_n,- config.align_k,- )- .map_err(MatmulError::BackendError)?;- self.pipelines.insert(config, kernel);- }- Ok(self.pipelines.get(&config).unwrap())- }-- pub fn encode(- &mut self,- context: &B::Context,- arguments: &mut MatmulArguments<B>,- dispatch_descriptor: &GemmDispatchDescriptor,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), MatmulError<B>> {- let config = dispatch_descriptor.specialization;-- let group_count_x = u32::try_from(dispatch_descriptor.threadgroups.x)- .map_err(|_| MatmulError::<B>::ThreadgroupOverflow(dispatch_descriptor.threadgroups.x))?;- let group_count_y = u32::try_from(dispatch_descriptor.threadgroups.y)- .map_err(|_| MatmulError::<B>::ThreadgroupOverflow(dispatch_descriptor.threadgroups.y))?;-- let pipeline = self.get_or_create_kernel(context, config)?;- pipeline.encode(- (arguments.a, arguments.a_offset as usize),- arguments.b,- arguments.d.deref_mut(),- std::slice::from_ref(&dispatch_descriptor.params),- group_count_x,- group_count_y,- command_buffer,- );-- Ok(())- }-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rsdeleted file mode 100644index 9250445f6..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs+++ /dev/null@@ -1,7 +0,0 @@-mod dispatch_descriptor;-mod kernel;-pub mod specialization;--pub use dispatch_descriptor::GemmDispatchDescriptor;-pub use kernel::GemmKernel;-pub use specialization::GemmSpecialization;diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rsdeleted file mode 100644index e45ac57bf..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs+++ /dev/null@@ -1,122 +0,0 @@-use std::sync::OnceLock;--use super::{super::matmul_arguments::MatmulArguments, specialization::Specialization};-use crate::{- DataType,- backends::common::{Backend, kernel::matmul::MatmulError},-};--const DEFAULT_GEMV_MAX_BATCH: i32 = 8;--static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();--fn max_gemv_batch_threshold() -> i32 {- *GEMV_MAX_BATCH.get_or_init(|| {- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)- })-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum OutputSource {- None,- Bias,-}--#[derive(Debug, Clone)]-pub struct GemvDispatchDescriptor {- pub specialization: Specialization,- pub matrix_is_rhs: bool,- pub output_source: OutputSource,- pub input_dimension: i32,- pub output_dimension: i32,- pub matrix_leading_dim: i32,- pub alpha: f32,- pub beta: f32,- pub bias_stride: i32,- pub batch_rows: i32,-}--impl GemvDispatchDescriptor {- pub fn try_new<B: Backend>(- data_type: DataType,- arguments: &MatmulArguments<B>,- ) -> Result<Option<Self>, MatmulError<B>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));- }-- if !arguments.transpose_b {- return Ok(None);- }-- let m = arguments.batch;- let n = arguments.output_dim;-- let max_gemv_batch = max_gemv_batch_threshold();-- if n == 1 {- if m != 1 {- return Ok(None);- }- } else if m > max_gemv_batch {- return Ok(None);- }-- let matrix_is_rhs = n != 1;- let transpose_matrix = if matrix_is_rhs {- !arguments.transpose_b- } else {- false- };-- let output_source = if arguments.bias.is_some() {- OutputSource::Bias- } else {- OutputSource::None- };-- let (apply_output_scale_and_accumulate, alpha, beta, bias_stride) = match output_source {- OutputSource::None => (false, 1.0f32, 0.0f32, 0),- OutputSource::Bias => (true, 1.0f32, 1.0f32, 1),- };-- let output_dimension = if matrix_is_rhs {- arguments.output_dim- } else {- arguments.batch- };-- let specialization = Specialization::select(- transpose_matrix,- arguments.input_dim,- output_dimension,- apply_output_scale_and_accumulate,- );-- let input_dimension = arguments.input_dim;- let matrix_leading_dim = if matrix_is_rhs {- arguments.leading_dimension_b- } else {- arguments.leading_dimension_a- };-- let batch_rows = arguments.batch;-- Ok(Some(Self {- specialization,- matrix_is_rhs,- output_source,- input_dimension,- output_dimension,- matrix_leading_dim,- alpha,- beta,- bias_stride,- batch_rows,- }))- }-- pub fn bias_is_fused(&self) -> bool {- matches!(self.output_source, OutputSource::Bias)- }-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rsdeleted file mode 100644index 92a59669e..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs+++ /dev/null@@ -1,115 +0,0 @@-use std::{collections::HashMap, ops::DerefMut};--use super::{- super::matmul_arguments::MatmulArguments,- dispatch_descriptor::{GemvDispatchDescriptor, OutputSource},- specialization::Specialization,-};-use crate::{- DataType,- backends::common::{- Backend, CommandBuffer, Kernels,- kernel::{MatmulGemvKernel, matmul::MatmulError},- },-};--pub struct GemvKernel<B: Backend> {- data_type: DataType,- pipelines: HashMap<Specialization, <B::Kernels as Kernels>::MatmulGemvKernel>,-}--impl<B: Backend> GemvKernel<B> {- pub fn new(data_type: DataType) -> Result<Self, MatmulError<B>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));- }- Ok(Self {- data_type,- pipelines: HashMap::new(),- })- }-- pub fn precompile(- &mut self,- context: &B::Context,- ) -> Result<(), MatmulError<B>> {- for &config in Specialization::precompile_configs(self.data_type) {- self.get_or_create_kernel(context, config)?;- }- Ok(())- }-- fn get_or_create_kernel(- &mut self,- context: &B::Context,- config: Specialization,- ) -> Result<&<B::Kernels as Kernels>::MatmulGemvKernel, MatmulError<B>> {- if !self.pipelines.contains_key(&config) {- let kernel = <B::Kernels as Kernels>::MatmulGemvKernel::new(- context,- self.data_type,- config.threadgroup_rows,- config.threadgroup_cols,- config.threads_per_simdgroup_row,- config.threads_per_simdgroup_col,- config.elements_per_thread_row,- config.elements_per_thread_col,- config.apply_output_scale_and_accumulate,- )- .map_err(MatmulError::BackendError)?;- self.pipelines.insert(config, kernel);- }- Ok(self.pipelines.get(&config).unwrap())- }-- pub fn encode(- &mut self,- context: &B::Context,- arguments: &mut MatmulArguments<B>,- dispatch_descriptor: &GemvDispatchDescriptor,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), MatmulError<B>> {- let config = dispatch_descriptor.specialization;- let pipeline = self.get_or_create_kernel(context, config)?;-- let (matrix, matrix_offset) = if dispatch_descriptor.matrix_is_rhs {- (arguments.b, 0usize)- } else {- (arguments.a, arguments.a_offset as usize)- };- let (input_vector, input_vector_offset) = if dispatch_descriptor.matrix_is_rhs {- (arguments.a, arguments.a_offset as usize)- } else {- (arguments.b, 0usize)- };-- let output_source = if config.apply_output_scale_and_accumulate {- match dispatch_descriptor.output_source {- OutputSource::None => {- return Err(MatmulError::GemvOutputSourceMismatch);- },- OutputSource::Bias => Some(arguments.bias.ok_or(MatmulError::<B>::GemvMissingBias)?),- }- } else {- None- };-- pipeline.encode(- (matrix, matrix_offset),- (input_vector, input_vector_offset),- output_source.map(|buffer| (buffer, 0usize)),- arguments.d.deref_mut(),- dispatch_descriptor.input_dimension,- dispatch_descriptor.output_dimension,- dispatch_descriptor.matrix_leading_dim,- dispatch_descriptor.alpha,- dispatch_descriptor.beta,- dispatch_descriptor.bias_stride,- dispatch_descriptor.batch_rows,- config.output_rows_per_threadgroup() as i32,- command_buffer,- );-- Ok(())- }-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rsdeleted file mode 100644index 4fe67a542..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs+++ /dev/null@@ -1,6 +0,0 @@-mod dispatch_descriptor;-mod kernel;-mod specialization;--pub use dispatch_descriptor::GemvDispatchDescriptor;-pub use kernel::GemvKernel;diff --git a/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs b/crates/uzu/src/backends/common/kernel/matmul/grid_size.rsdeleted file mode 100644index cee1e39fb..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs+++ /dev/null@@ -1,6 +0,0 @@-#[derive(Debug, Clone, Copy)]-pub struct GridSize {- pub x: usize,- pub y: usize,- pub z: usize,-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/kernel.rsdeleted file mode 100644index e414f506d..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs+++ /dev/null@@ -1,121 +0,0 @@-use std::ops::DerefMut;--use super::{- MatmulError, dispatch_descriptor::MatmulDispatchDescriptor, gemm::GemmKernel, gemv::GemvKernel,- matmul_arguments::MatmulArguments,-};-use crate::{- DataType,- backends::common::{Backend, CommandBuffer, Kernels, kernel::TensorAddBiasKernel},-};--pub struct MatmulKernel<B: Backend> {- pub(crate) data_type: DataType,- gemm: Option<GemmKernel<B>>,- gemv: Option<GemvKernel<B>>,- bias_add: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,-}--impl<B: Backend> MatmulKernel<B> {- pub fn new(data_type: DataType) -> Result<Self, MatmulError<B>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));- }-- Ok(Self {- data_type,- gemm: None,- gemv: None,- bias_add: None,- })- }-- pub fn precompile(- &mut self,- context: &B::Context,- ) -> Result<(), MatmulError<B>> {- let gemm = self.get_or_create_gemm()?;- gemm.precompile(context)?;-- let gemv = self.get_or_create_gemv()?;- gemv.precompile(context)?;-- Ok(())- }-- fn get_or_create_gemm(&mut self) -> Result<&mut GemmKernel<B>, MatmulError<B>> {- if self.gemm.is_none() {- self.gemm = Some(GemmKernel::<B>::new(self.data_type)?);- }- Ok(self.gemm.as_mut().unwrap())- }-- fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel<B>, MatmulError<B>> {- if self.gemv.is_none() {- self.gemv = Some(GemvKernel::<B>::new(self.data_type)?);- }- Ok(self.gemv.as_mut().unwrap())- }-- fn encode_dispatch_descriptor(- &mut self,- context: &B::Context,- arguments: &mut MatmulArguments<B>,- dispatch_descriptor: &MatmulDispatchDescriptor,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), MatmulError<B>> {- match dispatch_descriptor {- MatmulDispatchDescriptor::Gemv(d) => {- let gemv = self.get_or_create_gemv()?;- gemv.encode(context, arguments, d, command_buffer)- },- MatmulDispatchDescriptor::Gemm(d) => {- let gemm = self.get_or_create_gemm()?;- gemm.encode(context, arguments, d, command_buffer)- },- }- }-- pub fn encode_with_descriptor(- &mut self,- context: &B::Context,- mut arguments: MatmulArguments<B>,- dispatch_descriptor: &MatmulDispatchDescriptor,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), MatmulError<B>> {- self.encode_dispatch_descriptor(context, &mut arguments, dispatch_descriptor, command_buffer)?;-- if let Some(bias) = arguments.bias {- if !dispatch_descriptor.bias_is_fused() {- self.apply_bias_add(context, &mut arguments, bias, command_buffer)?;- }- }-- Ok(())- }-- fn apply_bias_add(- &mut self,- context: &B::Context,- arguments: &mut MatmulArguments<B>,- bias: &B::Buffer,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), MatmulError<B>> {- let m = arguments.batch as usize;- let n = arguments.output_dim as usize;- let total_len = m * n;- if total_len == 0 {- return Ok(());- }-- if self.bias_add.is_none() {- self.bias_add = Some(- <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, self.data_type, true)- .map_err(MatmulError::BackendError)?,- );- }- let bias_add = self.bias_add.as_ref().unwrap();- bias_add.encode(None::<&B::Buffer>, bias, arguments.d.deref_mut(), n as u32, total_len as u32, command_buffer);- Ok(())- }-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rsindex 97318dd7b..f1c793475 100644--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs@@ -1,35 +1,40 @@-mod dispatch_descriptor;-mod full_precision;-pub mod gemm;-pub mod gemv;-mod grid_size;-mod kernel;mod matmul_arguments;-pub use dispatch_descriptor::{MatmulDispatchDescriptor, choose_matmul_dispatch_descriptor};-pub use full_precision::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel};-pub use grid_size::GridSize;-pub use kernel::MatmulKernel;pub use matmul_arguments::MatmulArguments;use thiserror::Error;use super::Kernels;-use crate::backends::common::Backend;+use crate::{+ DataType,+ backends::common::{Backend, CommandBuffer},+};pub trait MatmulKernels: Kernels {- type FullPrecisionMatmulKernel: FullPrecisionMatmulKernel<Backend = Self::Backend>;+ type MatmulKernel: MatmulKernel<Backend = Self::Backend>;+}++pub trait MatmulKernel: Sized {+ type Backend: Backend<Kernels: MatmulKernels<MatmulKernel = Self>>;++ fn new(+ context: &<Self::Backend as Backend>::Context,+ data_type: DataType,+ ) -> Result<Self, MatmulError<Self::Backend>>;++ fn encode(+ &mut self,+ context: &<Self::Backend as Backend>::Context,+ arguments: MatmulArguments<Self::Backend>,+ command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,+ );}#[derive(Debug, Error)]pub enum MatmulError<B: Backend> {#[error("Unsupported data type: {0:?}")]- UnsupportedDataType(crate::DataType),+ UnsupportedDataType(DataType),#[error("Threadgroup dimension overflows u32: {0}")]ThreadgroupOverflow(usize),- #[error("GEMV descriptor mismatch: apply_output_scale_and_accumulate=true but output_source=None")]- GemvOutputSourceMismatch,- #[error("GEMV descriptor requires bias buffer")]- GemvMissingBias,#[error("Backend error: {0}")]BackendError(#[source] B::Error),}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rsindex 302d77480..f8cc265a3 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs@@ -1,5 +1,3 @@-use dsl::kernel;-use half::{bf16, f16};use num_traits::Float;use crate::{@@ -32,7 +30,7 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(}}-fn matmul_gemm_impl<T: ArrayElement + Float>(+pub fn matmul_gemm_impl<T: ArrayElement + Float>(a: *const T,b: *const T,d: *mut T,@@ -84,53 +82,3 @@ fn matmul_gemm_impl<T: ArrayElement + Float>(});}}--#[kernel(MatmulGemm)]-#[variants(T, f32, f16, bf16)]-pub fn matmul_gemm<T: ArrayElement + Float>(- a: *const T,- b: *const T,- d: *mut T,- params: &[crate::backends::common::gpu_types::matmul::GemmParams],- #[allow(unused)] group_count_x: u32,- #[allow(unused)] group_count_y: u32,- #[allow(unused)]- #[specialize]- block_rows: u32,- #[allow(unused)]- #[specialize]- block_cols: u32,- #[allow(unused)]- #[specialize]- block_depth: u32,- #[allow(unused)]- #[specialize]- simdgroups_per_row: u32,- #[allow(unused)]- #[specialize]- simdgroups_per_column: u32,- #[allow(unused)]- #[specialize]- align_m: bool,- #[allow(unused)]- #[specialize]- align_n: bool,- #[allow(unused)]- #[specialize]- align_k: bool,-) {- unsafe {- let p = &*params.as_ptr();- matmul_gemm_impl::<T>(- a,- b,- d,- p.M as usize,- p.N as usize,- p.K as usize,- p.leading_dimension_a as usize,- p.leading_dimension_b as usize,- p.leading_dimension_d as usize,- );- }-}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rsindex fd406d0cd..22e4ec542 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs@@ -1,5 +1,3 @@-use dsl::kernel;-use half::{bf16, f16};use num_traits::Float;use crate::{@@ -43,7 +41,7 @@ unsafe fn compute_gemv_rows<T: ArrayElement + Float>(}}-fn matmul_gemv_impl<T: ArrayElement + Float>(+pub fn matmul_gemv_impl<T: ArrayElement + Float>(matrix: *const T,input_vector: *const T,output_source: Option<*const T>,@@ -103,54 +101,3 @@ fn matmul_gemv_impl<T: ArrayElement + Float>(}}}--#[kernel(MatmulGemv)]-#[variants(T, f32, f16, bf16)]-pub fn matmul_gemv<T: ArrayElement + Float>(- matrix: *const T,- input_vector: *const T,- #[optional(apply_output_scale_and_accumulate)] output_source: Option<*const T>,- output_vector: *mut T,- input_dimension: i32,- output_dimension: i32,- matrix_leading_dimension: i32,- output_scale: f32,- output_accumulate_scale: f32,- output_source_stride: i32,- batch_rows: i32,- #[allow(unused)] output_rows_per_threadgroup: i32,- #[allow(unused)]- #[specialize]- tg_simd_rows: u32,- #[allow(unused)]- #[specialize]- tg_simd_cols: u32,- #[allow(unused)]- #[specialize]- sg_thread_rows: u32,- #[allow(unused)]- #[specialize]- sg_thread_cols: u32,- #[allow(unused)]- #[specialize]- thread_out_rows: u32,- #[allow(unused)]- #[specialize]- thread_out_cols: u32,- #[specialize] apply_output_scale_and_accumulate: bool,-) {- matmul_gemv_impl::<T>(- matrix,- input_vector,- output_source,- output_vector,- input_dimension as usize,- output_dimension as usize,- matrix_leading_dimension as usize,- output_scale,- output_accumulate_scale,- output_source_stride as usize,- batch_rows as usize,- apply_output_scale_and_accumulate,- );-}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rsnew file mode 100644index 000000000..0e9cb49d5--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -0,0 +1,118 @@+use half::{bf16, f16};++use crate::{+ DataType,+ backends::{+ common::kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},+ cpu::{Cpu, command_buffer::CpuCommandBuffer, context::CpuContext},+ },+};++pub struct MatmulCpuKernel {+ data_type: DataType,+}++impl MatmulKernel for MatmulCpuKernel {+ type Backend = Cpu;++ fn new(+ _context: &CpuContext,+ data_type: DataType,+ ) -> Result<Self, MatmulError<Cpu>> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MatmulError::UnsupportedDataType(data_type));+ }+ Ok(Self {+ data_type,+ })+ }++ fn encode(+ &mut self,+ _context: &CpuContext,+ arguments: MatmulArguments<Cpu>,+ command_buffer: &mut CpuCommandBuffer,+ ) {+ let m = arguments.batch as usize;+ let n = arguments.output_dim as usize;+ let k = arguments.input_dim as usize;+ let lda = arguments.leading_dimension_a as usize;+ let ldb = arguments.leading_dimension_b as usize;+ let ldd = arguments.leading_dimension_d as usize;+ let a_offset = arguments.a_offset as usize;+ let data_type = self.data_type;++ let a_ptr = arguments.a.as_ptr().wrapping_byte_add(a_offset);+ let b_ptr = arguments.b.as_ptr();+ let d_ptr = arguments.d.as_mut_ptr();+ let bias_ptr = arguments.bias.map(|b| b.as_ptr());++ command_buffer.push_command(move || match data_type {+ DataType::F32 => {+ super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f32>(+ a_ptr as *const f32,+ b_ptr as *const f32,+ d_ptr as *mut f32,+ m,+ n,+ k,+ lda,+ ldb,+ ldd,+ );+ if let Some(bias) = bias_ptr {+ apply_bias::<f32>(d_ptr as *mut f32, bias as *const f32, m, n);+ }+ },+ DataType::F16 => {+ super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f16>(+ a_ptr as *const f16,+ b_ptr as *const f16,+ d_ptr as *mut f16,+ m,+ n,+ k,+ lda,+ ldb,+ ldd,+ );+ if let Some(bias) = bias_ptr {+ apply_bias::<f16>(d_ptr as *mut f16, bias as *const f16, m, n);+ }+ },+ DataType::BF16 => {+ super::gemm::shaders::steel_gemm::matmul_gemm_impl::<bf16>(+ a_ptr as *const bf16,+ b_ptr as *const bf16,+ d_ptr as *mut bf16,+ m,+ n,+ k,+ lda,+ ldb,+ ldd,+ );+ if let Some(bias) = bias_ptr {+ apply_bias::<bf16>(d_ptr as *mut bf16, bias as *const bf16, m, n);+ }+ },+ _ => unreachable!(),+ });+ }+}++fn apply_bias<T: num_traits::Float + Copy>(+ d: *mut T,+ bias: *const T,+ m: usize,+ n: usize,+) {+ unsafe {+ for row in 0..m {+ for col in 0..n {+ let idx = row * n + col;+ *d.add(idx) = *d.add(idx) + *bias.add(col);+ }+ }+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rsindex f2ab16625..23d8c2f63 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs@@ -1,11 +1,11 @@-use crate::backends::{- common::kernel::matmul::{MatmulKernel, MatmulKernels},- cpu::{Cpu, kernel::CpuKernels},-};-pub mod gemm;pub mod gemv;+mod matmul;++pub use matmul::MatmulCpuKernel;++use crate::backends::{common::kernel::matmul::MatmulKernels, cpu::kernel::CpuKernels};impl MatmulKernels for CpuKernels {- type FullPrecisionMatmulKernel = MatmulKernel<Cpu>;+ type MatmulKernel = MatmulCpuKernel;}diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex fbe1a533b..57009d9ae 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -58,7 +58,7 @@ PUBLIC KERNEL(AttentionGemm)(const bool has_sinks SPECIALIZE,threadgroup T q_smem[BLOCK_QUERY_ROWS * (BD + 16 / sizeof(T))],threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],- const ThreadContext simd,+ const ThreadContext thread_context,const uint tgid_x GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),const uint tgid_y GROUPS(num_heads),const uint tgid_z GROUPS(1),@@ -139,22 +139,22 @@ PUBLIC KERNEL(AttentionGemm)(q,query_source_stride,query_shared,- simd.threadgroup_index,- simd.simdgroup_index+ thread_context.threadgroup_index,+ thread_context.simdgroup_index);thread KeyLoader key_loader(k,key_source_stride,key_shared,- simd.threadgroup_index,- simd.simdgroup_index+ thread_context.threadgroup_index,+ thread_context.simdgroup_index);thread ValueLoader value_loader(v,value_source_stride,value_shared,- simd.threadgroup_index,- simd.simdgroup_index+ thread_context.threadgroup_index,+ thread_context.simdgroup_index);TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));@@ -220,13 +220,13 @@ PUBLIC KERNEL(AttentionGemm)(// Lane coordinates and pointer offsetsconst short2 lane_coordinates =SimdgroupMultiplyAccumulateType::get_lane_coordinates(- simd.simdgroup_index+ thread_context.simdgroup_index);const short lane_row = lane_coordinates.y;const short lane_col = lane_coordinates.x;- const short simdgroup_row_base =- SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS * short(simd.threadgroup_index);+ const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS *+ short(thread_context.threadgroup_index);const short query_shared_offset =(simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex 1a09cb6be..bec72e50c 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -35,7 +35,7 @@ PUBLIC KERNEL(AttentionSinglePass)(const bool has_mask SPECIALIZE,const bool has_sinks SPECIALIZE,const bool do_causal SPECIALIZE,- const ThreadContext simd,+ const ThreadContext thread_context,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)@@ -60,22 +60,23 @@ PUBLIC KERNEL(AttentionSinglePass)(const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx: head_idx * tpg.y + q_seq_idx;- queries +=- q_offset * HEAD_DIM + simd.simdgroup_index * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride + simd.threadgroup_index * k_seq_stride +- simd.simdgroup_index * qk_elements_per_thread;+ queries += q_offset * HEAD_DIM ++ thread_context.simdgroup_index * qk_elements_per_thread;+ keys += kv_head_idx * k_head_stride ++ thread_context.threadgroup_index * k_seq_stride ++ thread_context.simdgroup_index * qk_elements_per_thread;values += kv_head_idx * v_head_stride +- simd.threadgroup_index * v_seq_stride +- simd.simdgroup_index * value_elements_per_thread;+ thread_context.threadgroup_index * v_seq_stride ++ thread_context.simdgroup_index * value_elements_per_thread;if (float_mask) {fmask += head_idx * mask_head_stride +- simd.threadgroup_index * mask_kv_seq_stride ++ thread_context.threadgroup_index * mask_kv_seq_stride +q_seq_idx * mask_q_seq_stride;}- out +=- o_offset * value_dim + simd.threadgroup_index * value_elements_per_thread;+ out += o_offset * value_dim ++ thread_context.threadgroup_index * value_elements_per_thread;// Read the query and 0 the output accumulatorfor (int i = 0; i < qk_elements_per_thread; i++) {@@ -87,7 +88,7 @@ PUBLIC KERNEL(AttentionSinglePass)(U max_score = -INFINITY;U sum_exp_score = 0;- if (has_sinks && simd.threadgroup_index == 0) {+ if (has_sinks && thread_context.threadgroup_index == 0) {const int num_q_heads = static_cast<int>(tpg.x);int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);@@ -95,7 +96,7 @@ PUBLIC KERNEL(AttentionSinglePass)(}// For each key- for (uint i = simd.threadgroup_index; i < sequence_length;+ for (uint i = thread_context.threadgroup_index; i < sequence_length;i += SEQUENCE_BLOCK_SIZE) {bool use_key = true;if (do_causal) {@@ -141,27 +142,27 @@ PUBLIC KERNEL(AttentionSinglePass)(}// Each thread has a partial part of the output so we need to combine them.- if (simd.simdgroup_index == 0) {- shared_max_scores[simd.threadgroup_index] = max_score;- shared_sum_exp_scores[simd.threadgroup_index] = sum_exp_score;+ if (thread_context.simdgroup_index == 0) {+ shared_max_scores[thread_context.threadgroup_index] = max_score;+ shared_sum_exp_scores[thread_context.threadgroup_index] = sum_exp_score;}threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = shared_max_scores[simd.simdgroup_index];+ max_score = shared_max_scores[thread_context.simdgroup_index];U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);sum_exp_score =- simd_sum(shared_sum_exp_scores[simd.simdgroup_index] * factor);+ simd_sum(shared_sum_exp_scores[thread_context.simdgroup_index] * factor);// Now we need to aggregate all the outputsfor (int i = 0; i < value_elements_per_thread; i++) {shared_outputs- [simd.simdgroup_index * HEAD_BLOCK_SIZE + simd.threadgroup_index] =- o[i];+ [thread_context.simdgroup_index * HEAD_BLOCK_SIZE ++ thread_context.threadgroup_index] = o[i];threadgroup_barrier(mem_flags::mem_threadgroup);o[i] = simd_sum(shared_outputs- [simd.threadgroup_index * HEAD_BLOCK_SIZE +- simd.simdgroup_index] *+ [thread_context.threadgroup_index * HEAD_BLOCK_SIZE ++ thread_context.simdgroup_index] *factor) /sum_exp_score;@@ -169,7 +170,7 @@ PUBLIC KERNEL(AttentionSinglePass)(}// And write the output- if (simd.simdgroup_index == 0) {+ if (thread_context.simdgroup_index == 0) {for (int i = 0; i < value_elements_per_thread; i++) {out[i] = static_cast<T>(o[i]);}diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex a47849b78..51d1b18dd 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -39,7 +39,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],- const ThreadContext simd,+ const ThreadContext thread_context,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),@@ -67,23 +67,26 @@ PUBLIC KERNEL(AttentionTwoPass1)(: head_idx * tpg.y + q_seq_idx; // Consistent with single-passconst uint kv_head_idx = head_idx / gqa_factor;- queries +=- q_offset * HEAD_DIM + simd.simdgroup_index * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *- k_seq_stride +- simd.simdgroup_index * qk_elements_per_thread;- values += kv_head_idx * v_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *- v_seq_stride +- simd.simdgroup_index * value_elements_per_thread;+ queries += q_offset * HEAD_DIM ++ thread_context.simdgroup_index * qk_elements_per_thread;+ keys +=+ kv_head_idx * k_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *+ k_seq_stride ++ thread_context.simdgroup_index * qk_elements_per_thread;+ values +=+ kv_head_idx * v_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *+ v_seq_stride ++ thread_context.simdgroup_index * value_elements_per_thread;out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +- simd.simdgroup_index * value_elements_per_thread;+ thread_context.simdgroup_index * value_elements_per_thread;if (float_mask) {- fmask += head_idx * mask_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *- mask_kv_seq_stride +- q_seq_idx * mask_q_seq_stride;+ fmask +=+ head_idx * mask_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *+ mask_kv_seq_stride ++ q_seq_idx * mask_q_seq_stride;}sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;@@ -98,7 +101,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(U max_score = -1e9;U sum_exp_score = 0;- if (has_sinks && block_idx == 0 && simd.threadgroup_index == 0) {+ if (has_sinks && block_idx == 0 && thread_context.threadgroup_index == 0) {const uint num_q_heads = tpg.x;int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);@@ -106,7 +109,8 @@ PUBLIC KERNEL(AttentionTwoPass1)(}// For each key- for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index;+ for (uint i =+ block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index;i < sequence_length;i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {bool use_key = true;@@ -153,23 +157,23 @@ PUBLIC KERNEL(AttentionTwoPass1)(}// Each thread has a partial part of the output so we need to combine them.- if (simd.simdgroup_index == 0) {- shared_max_scores[simd.threadgroup_index] = max_score;- shared_sum_exp_scores[simd.threadgroup_index] = sum_exp_score;+ if (thread_context.simdgroup_index == 0) {+ shared_max_scores[thread_context.threadgroup_index] = max_score;+ shared_sum_exp_scores[thread_context.threadgroup_index] = sum_exp_score;}threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = (simd.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)- ? shared_max_scores[simd.simdgroup_index]+ max_score = (thread_context.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)+ ? shared_max_scores[thread_context.simdgroup_index]: -1e9;U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- sum_exp_score = (simd.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)- ? shared_sum_exp_scores[simd.simdgroup_index]+ sum_exp_score = (thread_context.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)+ ? shared_sum_exp_scores[thread_context.simdgroup_index]: 0;sum_exp_score = simd_sum(sum_exp_score * factor);// Write the sum and new max- if (simd.threadgroup_index == 0) {+ if (thread_context.threadgroup_index == 0) {sums[0] = sum_exp_score;maxs[0] = new_max;}@@ -177,18 +181,21 @@ PUBLIC KERNEL(AttentionTwoPass1)(// Now we need to aggregate all the outputsfor (uint i = 0; i < value_elements_per_thread; i++) {shared_outputs- [simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 +- simd.threadgroup_index] =+ [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 ++ thread_context.threadgroup_index] =o[i] *- fast::exp(shared_max_scores[simd.threadgroup_index] - new_max);+ fast::exp(+ shared_max_scores[thread_context.threadgroup_index] - new_max+ );threadgroup_barrier(mem_flags::mem_threadgroup);// And write the output- if (simd.threadgroup_index == 0) {- U output = shared_outputs[simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1];+ if (thread_context.threadgroup_index == 0) {+ U output = shared_outputs+ [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1];for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {- output +=- shared_outputs[simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 + j];+ output += shared_outputs+ [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 + j];}out[i] = static_cast<T>(output);}@@ -207,7 +214,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(const constant uint& num_heads,const constant uint& suffix_length,threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],- const ThreadContext simd,+ const ThreadContext thread_context,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)@@ -221,19 +228,19 @@ PUBLIC KERNEL(AttentionTwoPass2)(const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layoutpartials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +- simd.threadgroup_index * HEAD_DIM +- simd.simdgroup_index * elements_per_thread;+ thread_context.threadgroup_index * HEAD_DIM ++ thread_context.simdgroup_index * elements_per_thread;sums += o_offset * TOTAL_BLOCKS_COUNT;maxs += o_offset * TOTAL_BLOCKS_COUNT;out +=- o_offset * HEAD_DIM +- simd.threadgroup_index * elements_per_thread; // Our custom output layout+ o_offset * HEAD_DIM + thread_context.threadgroup_index *+ elements_per_thread; // Our custom output layout// First everybody reads the max and sum_exp- U max_score = maxs[simd.simdgroup_index];+ U max_score = maxs[thread_context.simdgroup_index];U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- U sum_exp_score = simd_sum(sums[simd.simdgroup_index] * factor);+ U sum_exp_score = simd_sum(sums[thread_context.simdgroup_index] * factor);// Now read the block into registers and then use shared memory to transpose// it@@ -242,13 +249,13 @@ PUBLIC KERNEL(AttentionTwoPass2)(}for (uint i = 0; i < elements_per_thread; i++) {shared_outputs- [simd.simdgroup_index * HEAD_BLOCK_SIZE + simd.threadgroup_index] =- o[i];+ [thread_context.simdgroup_index * HEAD_BLOCK_SIZE ++ thread_context.threadgroup_index] = o[i];threadgroup_barrier(mem_flags::mem_threadgroup);o[i] = simd_sum(shared_outputs- [simd.threadgroup_index * HEAD_BLOCK_SIZE +- simd.simdgroup_index] *+ [thread_context.threadgroup_index * HEAD_BLOCK_SIZE ++ thread_context.simdgroup_index] *factor) /sum_exp_score;@@ -256,7 +263,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(}// And write the output- if (simd.simdgroup_index == 0) {+ if (thread_context.simdgroup_index == 0) {for (uint i = 0; i < elements_per_thread; i++) {out[i] = static_cast<T>(o[i]);}diff --git a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.hindex 2438e37a7..4475cc46d 100644--- a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h+++ b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h@@ -138,19 +138,20 @@ static T threadgroup_cooperative_reduce_sum(T value,threadgroup T* shared,const ushort lid,- const thread ThreadContext& simd+ const thread ThreadContext& thread_context) {T local_sum = simd_sum(value);- if (simd.simdgroup_index == 0) {- shared[simd.threadgroup_index] = local_sum;+ if (thread_context.simdgroup_index == 0) {+ shared[thread_context.threadgroup_index] = local_sum;}threadgroup_barrier(mem_flags::mem_threadgroup);T total_sum = T(0);const ushort num_simd_groups =- (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size;+ (BLOCK_SIZE + thread_context.simdgroup_size - 1) /+ thread_context.simdgroup_size;if (lid < num_simd_groups) {total_sum = shared[lid];}@@ -171,20 +172,20 @@ static T threadgroup_cooperative_reduce_max(T value,threadgroup T* shared,const ushort lid,- const thread ThreadContext& simd+ const thread ThreadContext& thread_context) {T local_max = simd_max(value);- if (simd.simdgroup_index == 0) {- shared[simd.threadgroup_index] = local_max;+ if (thread_context.simdgroup_index == 0) {+ shared[thread_context.threadgroup_index] = local_max;}threadgroup_barrier(mem_flags::mem_threadgroup);- T total_max =- lid < (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size- ? shared[lid]- : T(-INFINITY);+ T total_max = lid < (BLOCK_SIZE + thread_context.simdgroup_size - 1) /+ thread_context.simdgroup_size+ ? shared[lid]+ : T(-INFINITY);total_max = simd_max(total_max);if (lid == 0) {@@ -202,20 +203,20 @@ static T threadgroup_cooperative_reduce_min(T value,threadgroup T* shared,const ushort lid,- const thread ThreadContext& simd+ const thread ThreadContext& thread_context) {T local_min = simd_min(value);- if (simd.simdgroup_index == 0) {- shared[simd.threadgroup_index] = local_min;+ if (thread_context.simdgroup_index == 0) {+ shared[thread_context.threadgroup_index] = local_min;}threadgroup_barrier(mem_flags::mem_threadgroup);- T total_min =- lid < (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size- ? shared[lid]- : T(INFINITY);+ T total_min = lid < (BLOCK_SIZE + thread_context.simdgroup_size - 1) /+ thread_context.simdgroup_size+ ? shared[lid]+ : T(INFINITY);total_min = simd_min(total_min);if (lid == 0) {diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalindex c6e54cee0..08758f523 100644--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -20,7 +20,7 @@ void layer_norm_core(threadgroup AccumT* shared_variance,uint thread_in_row,bool full_layer,- const thread ThreadContext& simd+ const thread ThreadContext& thread_context) {AccumT partial_sum = static_cast<AccumT>(0.0f);@@ -40,7 +40,7 @@ void layer_norm_core(partial_sum,shared_mean,thread_in_row,- simd+ thread_context);AccumT mean = total_sum / static_cast<AccumT>(element_count);@@ -62,7 +62,7 @@ void layer_norm_core(partial_var_sum,shared_variance,thread_in_row,- simd+ thread_context);AccumT variance = total_var / static_cast<AccumT>(element_count);AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));@@ -132,7 +132,7 @@ PUBLIC KERNEL(LayerNorm) (const bool in_place SPECIALIZE,threadgroup ACC shared_mean[METAL_SIMD_SIZE],threadgroup ACC shared_variance[METAL_SIMD_SIZE],- const ThreadContext simd,+ const ThreadContext thread_context,uint batch_idx GROUPS(batch_size),uint thread_in_row THREADS(BLOCK_SIZE)) {@@ -151,6 +151,6 @@ PUBLIC KERNEL(LayerNorm) (shared_variance,thread_in_row,full_layer,- simd+ thread_context);}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rsnew file mode 100644index 000000000..14f9855ce--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -0,0 +1,3 @@+mod specialization;++pub use specialization::GemmSpecialization;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metalsimilarity index 81%rename from crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metalrename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metalindex 6608d1cb7..d020bced4 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal@@ -29,14 +29,14 @@ METAL_FUNC void gemm_dispatch(simdgroups_per_row == SIMDGROUPS_PER_ROW && simdgroups_per_column == SIMDGROUPS_PER_COLUMN) { \if (align_m && align_n) { \if (align_k) \- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, true>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, true>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \else \- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, false>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, false>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \} else { \if (align_k) \- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \else \- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, false>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, false>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \} \return; \}@@ -46,7 +46,7 @@ METAL_FUNC void gemm_dispatch(template <typename T>VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MatmulGemm)(+KERNEL(MatmulGemm)(const device T* a,const device T* b,device T* d,@@ -68,7 +68,7 @@ PUBLIC KERNEL(MatmulGemm)(const uint thread_x THREADS(32),const uint thread_y THREADS(2),const uint thread_z THREADS(2),- const ThreadContext simd+ const ThreadContext thread_context) {GEMM_DISPATCH(T, 64, 64, 16, 2, 2)GEMM_DISPATCH(T, 64, 64, 16, 1, 2)diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rssimilarity index 95%rename from crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rsrename to crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 42e60d822..36e9db557 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -1,6 +1,9 @@use crate::{DataType,- backends::common::{Backend, Context, kernel::matmul::MatmulArguments},+ backends::{+ common::{Context, kernel::matmul::MatmulArguments},+ metal::{Metal, context::MetalContext},+ },};#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -139,14 +142,14 @@ impl GemmSpecialization {}}- pub fn select<B: Backend>(- context: &B::Context,+ pub fn select(+ context: &MetalContext,data_type: DataType,- arguments: &MatmulArguments<B>,+ arguments: &MatmulArguments<Metal>,) -> Self {let overall_work_elements = (arguments.batch as i64) * (arguments.output_dim as i64);let is_float32 = matches!(data_type, DataType::F32);- let prefer_half_or_tf32 = !is_float32 || B::Context::tf32_enabled();+ let prefer_half_or_tf32 = !is_float32 || MetalContext::tf32_enabled();let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column, swizzle_log2) =if context.is_high_performance() && overall_work_elements >= (1_i64 << 20) {diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rsnew file mode 100644index 000000000..1d2748972--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -0,0 +1,3 @@+mod specialization;++pub use specialization::Specialization;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metalindex ca4aeecf7..169715ce6 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal@@ -13,7 +13,7 @@ using namespace metal;template <typename T>VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MatmulGemv)(+KERNEL(MatmulGemv)(const device T* matrix,const device T* input_vector,const device T* output_source OPTIONAL(apply_output_scale_and_accumulate),@@ -39,14 +39,14 @@ PUBLIC KERNEL(MatmulGemv)(const uint thread_index_x THREADS(32),const uint thread_index_y THREADS(16),const uint thread_index_z THREADS(1),- const ThreadContext simd+ const ThreadContext thread_context) {if (output_rows_per_threadgroup <= 0) {return;}// Simdgroup guard — kill excess simdgroups beyond what this tile uses- if (simd.threadgroup_index >= tg_simd_rows * tg_simd_cols) {+ if (thread_context.threadgroup_index >= tg_simd_rows * tg_simd_cols) {return;}@@ -66,22 +66,27 @@ PUBLIC KERNEL(MatmulGemv)(thread float vector_coefficients[4];const int thread_row_in_simdgroup =- sg_thread_cols != 32 ? int(simd.simdgroup_index) / int(sg_thread_cols)- : 0;+ sg_thread_cols != 32+ ? int(thread_context.simdgroup_index) / int(sg_thread_cols)+ : 0;const int thread_col_in_simdgroup =- sg_thread_cols != 32 ? int(simd.simdgroup_index) % int(sg_thread_cols)- : int(simd.simdgroup_index);+ sg_thread_cols != 32+ ? int(thread_context.simdgroup_index) % int(sg_thread_cols)+ : int(thread_context.simdgroup_index);const int simdgroup_column_index =- tg_simd_cols != 1 ? int(simd.threadgroup_index % tg_simd_cols) : 0;+ tg_simd_cols != 1 ? int(thread_context.threadgroup_index % tg_simd_cols)+ : 0;const int simdgroup_row_thread_base =tg_simd_cols != 1- ? int(sg_thread_rows) * int(simd.threadgroup_index / tg_simd_cols)- : int(sg_thread_rows) * int(simd.threadgroup_index);+ ? int(sg_thread_rows) *+ int(thread_context.threadgroup_index / tg_simd_cols)+ : int(sg_thread_rows) * int(thread_context.threadgroup_index);const int simdgroup_col_thread_base =tg_simd_cols != 1- ? int(sg_thread_cols) * int(simd.threadgroup_index % tg_simd_cols)+ ? int(sg_thread_cols) *+ int(thread_context.threadgroup_index % tg_simd_cols): 0;int output_block_row_offset =diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rssimilarity index 100%rename from crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rsrename to crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rsdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rsnew file mode 100644index 000000000..f0dada6f4--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs@@ -0,0 +1,338 @@+use std::{+ collections::{HashMap, hash_map::Entry},+ sync::OnceLock,+};++use super::{+ super::dsl::{MatmulGemmMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel},+ gemm, gemv,+};+use crate::{+ DataType,+ backends::{+ common::{+ gpu_types::GemmParams,+ kernel::{+ TensorAddBiasKernel,+ matmul::{MatmulArguments, MatmulError, MatmulKernel},+ },+ },+ metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext},+ },+};++pub struct MatmulMetalKernel {+ data_type: DataType,+ gemv_kernels: HashMap<gemv::Specialization, MatmulGemvMetalKernel>,+ gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,+ bias_add: Option<TensorAddBiasMetalKernel>,+}++const DEFAULT_GEMV_MAX_BATCH: i32 = 8;++static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();++fn max_gemv_batch_threshold() -> i32 {+ *GEMV_MAX_BATCH.get_or_init(|| {+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)+ })+}++impl MatmulMetalKernel {+ fn is_gemv_eligible(arguments: &MatmulArguments<Metal>) -> bool {+ let m = arguments.batch;+ let n = arguments.output_dim;+ let max_batch = max_gemv_batch_threshold();++ if !arguments.transpose_b {+ return false;+ }++ if n == 1 {+ m == 1+ } else {+ m <= max_batch+ }+ }++ fn get_or_create_gemv(+ &mut self,+ context: &MetalContext,+ specialization: gemv::Specialization,+ ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {+ match self.gemv_kernels.entry(specialization) {+ Entry::Occupied(entry) => Ok(entry.into_mut()),+ Entry::Vacant(entry) => {+ let kernel = MatmulGemvMetalKernel::new(+ context,+ self.data_type,+ specialization.threadgroup_rows,+ specialization.threadgroup_cols,+ specialization.threads_per_simdgroup_row,+ specialization.threads_per_simdgroup_col,+ specialization.elements_per_thread_row,+ specialization.elements_per_thread_col,+ specialization.apply_output_scale_and_accumulate,+ )+ .map_err(MatmulError::BackendError)?;+ Ok(entry.insert(kernel))+ },+ }+ }++ fn get_or_create_gemm(+ &mut self,+ context: &MetalContext,+ specialization: gemm::GemmSpecialization,+ ) -> Result<&MatmulGemmMetalKernel, MatmulError<Metal>> {+ match self.gemm_kernels.entry(specialization) {+ Entry::Occupied(entry) => Ok(entry.into_mut()),+ Entry::Vacant(entry) => {+ let kernel = MatmulGemmMetalKernel::new(+ context,+ self.data_type,+ specialization.block_rows as u32,+ specialization.block_cols as u32,+ specialization.block_depth as u32,+ specialization.simdgroups_per_row as u32,+ specialization.simdgroups_per_column as u32,+ specialization.align_m,+ specialization.align_n,+ specialization.align_k,+ )+ .map_err(MatmulError::BackendError)?;+ Ok(entry.insert(kernel))+ },+ }+ }++ fn encode_gemv(+ &mut self,+ context: &MetalContext,+ command_buffer: &mut MetalCommandBufferEncoding,+ arguments: MatmulArguments<Metal>,+ ) -> Result<(), MatmulError<Metal>> {+ let m = arguments.batch;+ let n = arguments.output_dim;++ let matrix_is_rhs = n != 1;+ let has_bias = arguments.bias.is_some();+ let apply_output_scale_and_accumulate = has_bias;++ let effective_output_dimension = if matrix_is_rhs {+ n+ } else {+ m+ };++ let transpose_matrix = if matrix_is_rhs {+ !arguments.transpose_b+ } else {+ false+ };++ let specialization = gemv::Specialization::select(+ transpose_matrix,+ arguments.input_dim,+ effective_output_dimension,+ apply_output_scale_and_accumulate,+ );++ let (alpha, beta, bias_stride) = if apply_output_scale_and_accumulate {+ (1.0f32, 1.0f32, 1i32)+ } else {+ (1.0f32, 0.0f32, 0i32)+ };++ let matrix_leading_dim = if matrix_is_rhs {+ arguments.leading_dimension_b+ } else {+ arguments.leading_dimension_a+ };++ let (matrix, matrix_offset) = if matrix_is_rhs {+ (arguments.b, 0usize)+ } else {+ (arguments.a, arguments.a_offset as usize)+ };+ let (input_vector, input_vector_offset) = if matrix_is_rhs {+ (arguments.a, arguments.a_offset as usize)+ } else {+ (arguments.b, 0usize)+ };++ let output_source = if apply_output_scale_and_accumulate {+ arguments.bias+ } else {+ None+ };++ let kernel = self.get_or_create_gemv(context, specialization)?;++ kernel.encode(+ (matrix, matrix_offset),+ (input_vector, input_vector_offset),+ output_source.map(|buffer| (buffer, 0usize)),+ &mut *arguments.d,+ arguments.input_dim,+ effective_output_dimension,+ matrix_leading_dim,+ alpha,+ beta,+ bias_stride,+ arguments.batch,+ specialization.output_rows_per_threadgroup() as i32,+ command_buffer,+ );++ let bias_is_fused = apply_output_scale_and_accumulate && has_bias;+ if !bias_is_fused {+ self.encode_bias_add(+ context,+ arguments.bias,+ arguments.d,+ arguments.batch as usize,+ arguments.output_dim as usize,+ command_buffer,+ )?;+ }++ Ok(())+ }++ fn encode_gemm(+ &mut self,+ context: &MetalContext,+ command_buffer: &mut MetalCommandBufferEncoding,+ arguments: MatmulArguments<Metal>,+ ) -> Result<(), MatmulError<Metal>> {+ let m = arguments.batch;+ let n = arguments.output_dim;+ let k = arguments.input_dim;++ let specialization = gemm::GemmSpecialization::select(context, self.data_type, &arguments);++ let threadgroups_per_row = (n + specialization.block_cols - 1) / specialization.block_cols;+ let threadgroups_per_column = (m + specialization.block_rows - 1) / specialization.block_rows;+ let swizzle_log = specialization.swizzle_log2;++ let swizzle_stride = 1_i32 << swizzle_log;+ let tm_swizzled = (threadgroups_per_column + swizzle_stride - 1) / swizzle_stride;+ let tn_swizzled = threadgroups_per_row * swizzle_stride;++ let params = GemmParams {+ M: m,+ N: n,+ K: k,+ leading_dimension_a: arguments.leading_dimension_a,+ leading_dimension_b: arguments.leading_dimension_b,+ leading_dimension_d: arguments.leading_dimension_d,+ threadgroups_per_row,+ threadgroups_per_column,+ swizzle_log,+ aligned_inner_iterations: k / specialization.block_depth,+ };++ let group_count_x =+ u32::try_from(tn_swizzled).map_err(|_| MatmulError::<Metal>::ThreadgroupOverflow(tn_swizzled as usize))?;+ let group_count_y =+ u32::try_from(tm_swizzled).map_err(|_| MatmulError::<Metal>::ThreadgroupOverflow(tm_swizzled as usize))?;++ let kernel = self.get_or_create_gemm(context, specialization)?;++ kernel.encode(+ (arguments.a, arguments.a_offset as usize),+ arguments.b,+ &mut *arguments.d,+ std::slice::from_ref(¶ms),+ group_count_x,+ group_count_y,+ command_buffer,+ );++ self.encode_bias_add(+ context,+ arguments.bias,+ arguments.d,+ arguments.batch as usize,+ arguments.output_dim as usize,+ command_buffer,+ )?;++ Ok(())+ }++ fn encode_bias_add(+ &mut self,+ context: &MetalContext,+ bias: Option<&<Metal as crate::backends::common::Backend>::Buffer>,+ output: &mut <Metal as crate::backends::common::Backend>::Buffer,+ batch: usize,+ output_dim: usize,+ command_buffer: &mut MetalCommandBufferEncoding,+ ) -> Result<(), MatmulError<Metal>> {+ let Some(bias) = bias else {+ return Ok(());+ };+ let total_length = batch * output_dim;+ if total_length == 0 {+ return Ok(());+ }+ if self.bias_add.is_none() {+ self.bias_add =+ Some(TensorAddBiasMetalKernel::new(context, self.data_type, true).map_err(MatmulError::BackendError)?);+ }+ let bias_add = self.bias_add.as_ref().unwrap();+ bias_add.encode(+ None::<&<Metal as crate::backends::common::Backend>::Buffer>,+ bias,+ output,+ output_dim as u32,+ total_length as u32,+ command_buffer,+ );+ Ok(())+ }+}++impl MatmulKernel for MatmulMetalKernel {+ type Backend = Metal;++ fn new(+ context: &MetalContext,+ data_type: DataType,+ ) -> Result<Self, MatmulError<Metal>> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MatmulError::UnsupportedDataType(data_type));+ }++ let mut kernel = Self {+ data_type,+ gemv_kernels: HashMap::new(),+ gemm_kernels: HashMap::new(),+ bias_add: None,+ };++ for &config in gemv::Specialization::precompile_configs(data_type) {+ kernel.get_or_create_gemv(context, config)?;+ }+ for &config in gemm::GemmSpecialization::precompile_configs(data_type) {+ kernel.get_or_create_gemm(context, config)?;+ }++ Ok(kernel)+ }++ fn encode(+ &mut self,+ context: &MetalContext,+ arguments: MatmulArguments<Metal>,+ command_buffer: &mut MetalCommandBufferEncoding,+ ) {+ if Self::is_gemv_eligible(&arguments) {+ self.encode_gemv(context, command_buffer, arguments).expect("Failed to encode GEMV kernel");+ } else {+ self.encode_gemm(context, command_buffer, arguments).expect("Failed to encode GEMM kernel");+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsnew file mode 100644index 000000000..27c0a5c14--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -0,0 +1,5 @@+pub mod gemm;+pub mod gemv;+mod matmul;++pub use matmul::MatmulMetalKernel;diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 429f084ad..dedaa9318 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -1,12 +1,12 @@-use crate::backends::{- common::kernel::matmul::{MatmulKernel, MatmulKernels},- metal::Metal,-};+pub mod matmul;++use crate::backends::common::kernel::matmul::MatmulKernels;+pub mod dsl {include!(concat!(env!("OUT_DIR"), "/dsl.rs"));}pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));impl MatmulKernels for dsl::MetalKernels {- type FullPrecisionMatmulKernel = MatmulKernel<Metal>;+ type MatmulKernel = matmul::MatmulMetalKernel;}diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex 1281a2bd8..571d408ce 100644--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -25,7 +25,7 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(threadgroup uintcounts_shared[BLOCK_SIZE], // Cache counts in threadgroup memorythreadgroup uint& carry,- const ThreadContext simd,+ const ThreadContext thread_context,const uint lid THREADS(128)) {if (e_input == 0) {@@ -104,7 +104,7 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(v,reduce_shared,(ushort)lid,- simd+ thread_context);if (lid == 0) {carry += block_sum;diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metalindex 70320f217..03cf07d3d 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -27,7 +27,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(constant float& up_clip_min,constant float& up_clip_max,const uint gating_sel SPECIALIZE, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU- const ThreadContext simd,+ const ThreadContext thread_context,const uint h_block_idx GROUPS(d_ff.div_ceil(4)),const uint k_slot GROUPS(k),const uint tid THREADS(128)@@ -37,7 +37,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(return;const uint expert_u = uint(expert_id);- const uint h_idx = h_block_idx * 4 + simd.threadgroup_index;+ const uint h_idx = h_block_idx * 4 + thread_context.threadgroup_index;if (h_idx >= d_ff)return;@@ -55,7 +55,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(const uint vec_iters = d_model / 128;for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd.simdgroup_index * 4;+ uint base_idx = i * 128 + thread_context.simdgroup_index * 4;device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);device const T* w_up_vec2 =@@ -75,7 +75,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(}}- uint leftover_start = vec_iters * 128 + simd.simdgroup_index;+ uint leftover_start = vec_iters * 128 + thread_context.simdgroup_index;for (uint idx = leftover_start; idx < d_model; idx += 32) {float xv = float(x[idx]);acc_up += xv * float(w_up_row[idx]);@@ -89,7 +89,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(acc_gate = simd_sum(acc_gate);}- if (simd.simdgroup_index == 0) {+ if (thread_context.simdgroup_index == 0) {float up_val = clamp(acc_up + float(biases[bias_base + h_idx]),up_clip_min,@@ -133,11 +133,11 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(constant uint& d_model,constant uint& d_ff,constant uint& k_input,- const ThreadContext simd,+ const ThreadContext thread_context,const uint d_block GROUPS(d_model.div_ceil(8)),const uint tid THREADS(256)) {- const uint my_col = d_block * 8 + simd.threadgroup_index;+ const uint my_col = d_block * 8 + thread_context.threadgroup_index;if (my_col >= d_model)return;@@ -159,7 +159,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(// Vectorized reductionfor (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd.simdgroup_index * 4;+ uint base_idx = i * 128 + thread_context.simdgroup_index * 4;float4 h_vec =*reinterpret_cast<device const float4*>(hidden_ptr + base_idx);@@ -172,20 +172,21 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(}// Remainder- for (uint idx = vec_iters * 128 + simd.simdgroup_index; idx < d_ff;+ for (uint idx = vec_iters * 128 + thread_context.simdgroup_index;+ idx < d_ff;idx += 32) {acc += hidden_ptr[idx] * float(w2_ptr[idx]);}// Simdgroup reduction and accumulatefloat result = simd_sum(acc);- if (simd.simdgroup_index == 0) {+ if (thread_context.simdgroup_index == 0) {result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);final_acc += prob * result;}}- if (simd.simdgroup_index == 0) {+ if (thread_context.simdgroup_index == 0) {y[my_col] = T(final_acc);}}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex deab8e40c..07bf1e12c 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -27,7 +27,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(constant float& silu_alpha,device const uint* tile_map,const uint gating_sel SPECIALIZE,- const ThreadContext simd,+ const ThreadContext thread_context,const uint tile_idx GROUPS(INDIRECT),const uint tid THREADS(128)) {@@ -44,7 +44,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(return;// Each simdgroup outputs one hidden element- const uint h_idx = h_block_idx * 4 + simd.threadgroup_index;+ const uint h_idx = h_block_idx * 4 + thread_context.threadgroup_index;if (h_idx >= d_ff)return;@@ -66,7 +66,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(const uint vec_iters = d_model / 128;for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd.simdgroup_index * 4;+ uint base_idx = i * 128 + thread_context.simdgroup_index * 4;device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);device const T* w_up_vec =@@ -87,7 +87,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(}// Handle leftover elements- uint leftover_start = vec_iters * 128 + simd.simdgroup_index;+ uint leftover_start = vec_iters * 128 + thread_context.simdgroup_index;for (uint idx = leftover_start; idx < d_model; idx += 32) {float xv = float(x_ptr[idx]);acc_up += xv * float(w_up_row[idx]);@@ -103,7 +103,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(}// Lane 0 applies activation and writes result- if (simd.simdgroup_index == 0) {+ if (thread_context.simdgroup_index == 0) {float up_val = acc_up + float(up_biases[bias_base + h_idx]);up_val = clamp(up_val, up_clip_min, up_clip_max);@@ -142,7 +142,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(constant uint& d_model,constant uint& d_ff,constant uint& e,- const ThreadContext simd,+ const ThreadContext thread_context,const uint tgpig_x GROUPS(d_model.div_ceil(SIMDGROUPS_PER_TG)),const uint tgpig_y GROUPS(total_rows),const uint tid THREADS(256)@@ -150,7 +150,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(const uint row_idx = tgpig_y;// Each simdgroup computes one output column- const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd.threadgroup_index;+ const uint my_col =+ tgpig_x * SIMDGROUPS_PER_TG + thread_context.threadgroup_index;if (my_col >= d_model)return;@@ -171,7 +172,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(const uint k_vec_iters = k_iters / 8;for (uint iter = 0; iter < k_vec_iters; ++iter) {- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd.simdgroup_index;+ const uint k_base =+ iter * (8 * THREADS_PER_SIMD) + thread_context.simdgroup_index;// hidden: stride-32 per lane, already f32 from Pass Aconst AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];@@ -217,7 +219,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Handle remaining full iterationsfor (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {- const uint k = iter * THREADS_PER_SIMD + simd.simdgroup_index;+ const uint k = iter * THREADS_PER_SIMD + thread_context.simdgroup_index;acc =fma(AccumT(hidden[hidden_base + k]),AccumT(w2_all[w2_col_base + k]),@@ -226,8 +228,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Handle leftover elements (d_ff % 32)const uint leftover_start = k_iters * THREADS_PER_SIMD;- if (leftover_start + simd.simdgroup_index < d_ff) {- const uint k = leftover_start + simd.simdgroup_index;+ if (leftover_start + thread_context.simdgroup_index < d_ff) {+ const uint k = leftover_start + thread_context.simdgroup_index;acc =fma(AccumT(hidden[hidden_base + k]),AccumT(w2_all[w2_col_base + k]),@@ -238,7 +240,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(AccumT result = simd_sum(acc);// Lane 0 writes result- if (simd.simdgroup_index == 0) {+ if (thread_context.simdgroup_index == 0) {const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;result += AccumT(down_biases[bias_idx]);diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex f91b44dbc..76176d392 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -60,7 +60,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(const uint n_tile_idx GROUPS(INDIRECT),const uint row_tile_idx GROUPS(INDIRECT),const uint local_tid_x THREADS(128),- const ThreadContext simd+ const ThreadContext thread_context) {const uint base = row_tile_idx * 3u;const uint expert_idx = tile_map[base + 0u];@@ -106,15 +106,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(// simdgroup tile mappingconst uint sg_col_count = Bn / SgBn;- const uint row_sg = simd.threadgroup_index / sg_col_count;- const uint col_sg = simd.threadgroup_index % sg_col_count;+ const uint row_sg = thread_context.threadgroup_index / sg_col_count;+ const uint col_sg = thread_context.threadgroup_index % sg_col_count;const uint row_sg_off = row_sg * SgBm;const uint col_sg_off = col_sg * SgBn;// Guard against misconfigured TG sizesconstexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);- if (simd.threadgroup_index >= SG_EXPECTED)+ if (thread_context.threadgroup_index >= SG_EXPECTED)return;// partial accumulators (8x8 tiles per simdgroup)@@ -295,7 +295,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Epilogue: activation/gating → hidden_out (f32) ----- const uint sg_lane_start = simd.threadgroup_index * 32;+ const uint sg_lane_start = thread_context.threadgroup_index * 32;const uint sg_lane_end = sg_lane_start + 32;if (lin < sg_lane_start || lin >= sg_lane_end)return;@@ -404,7 +404,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)],threadgroup T Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)],threadgroup float bias_tile[PASSB_BN],- const ThreadContext simd,+ const ThreadContext thread_context,const uint n_tile_idx GROUPS(INDIRECT),const uint row_tile_idx GROUPS(INDIRECT),const uint lin THREADS(128)@@ -446,14 +446,14 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(const ulong bias_base = (ulong)expert_idx * (ulong)d_model;// 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)- const uint row_sg = simd.threadgroup_index / 2; // 0-1 → rows- const uint col_sg = simd.threadgroup_index % 2; // 0-1 → cols+ const uint row_sg = thread_context.threadgroup_index / 2; // 0-1 → rows+ const uint col_sg = thread_context.threadgroup_index % 2; // 0-1 → colsconst uint row_sg_off = row_sg * SgBm;const uint col_sg_off = col_sg * SgBn;// Guard against excessive simdgroupsconstexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);- if (simd.threadgroup_index >= SG_EXPECTED)+ if (thread_context.threadgroup_index >= SG_EXPECTED)return;// 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles@@ -561,10 +561,11 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Writeout with fragment extraction ----- const uint lane_qid = simd.simdgroup_index >> 2;- const uint lane_row = (lane_qid & 4u) + ((simd.simdgroup_index >> 1) & 3u);+ const uint lane_qid = thread_context.simdgroup_index >> 2;+ const uint lane_row =+ (lane_qid & 4u) + ((thread_context.simdgroup_index >> 1) & 3u);const uint lane_col_base =- ((lane_qid & 2u) << 1) + ((simd.simdgroup_index & 1u) << 1);+ ((lane_qid & 2u) << 1) + ((thread_context.simdgroup_index & 1u) << 1);const uint local_row = row_sg_off + lane_row;if (local_row >= m_rows)diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metalindex ad16c6b40..f88fecc4c 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -30,7 +30,7 @@ PUBLIC KERNEL(MoeRouterTopK)(threadgroup uint reduce_tmp_u[THREADS_PER_TG],threadgroup uint shared_best_idx[1],threadgroup float shared_best_val[1],- const ThreadContext simd,+ const ThreadContext thread_context,const uint tgpig_x GROUPS(1),const uint token_idx GROUPS(t),const uint lid THREADS(256)@@ -52,12 +52,12 @@ PUBLIC KERNEL(MoeRouterTopK)(}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint row = simd.threadgroup_index; row < e;- row += simd.simdgroups_per_threadgroup) {+ for (uint row = thread_context.threadgroup_index; row < e;+ row += thread_context.simdgroups_per_threadgroup) {const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;float4 accum4 = float4(0.0f);- for (uint c = simd.simdgroup_index; c < vecs; c += 32u) {+ for (uint c = thread_context.simdgroup_index; c < vecs; c += 32u) {const float4 wv = float4(w_vec[c * 4 + 0],w_vec[c * 4 + 1],@@ -100,7 +100,7 @@ PUBLIC KERNEL(MoeRouterTopK)(local_best,reduce_tmp,lid,- simd+ thread_context);uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;@@ -108,7 +108,7 @@ PUBLIC KERNEL(MoeRouterTopK)(candidate_id,reduce_tmp_u,lid,- simd+ thread_context);if (lid == 0) {diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex 6cc4b4701..3eade2617 100644--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -30,7 +30,7 @@ inline void moe_scatter_buckets_impl(uint3 tgpig,threadgroup _atomic<uint>* sg_counts,threadgroup uint* sg_base,- const thread ThreadContext& simd+ const thread ThreadContext& thread_context) {if (E == 0 || T == 0 || K == 0)return;@@ -65,7 +65,7 @@ inline void moe_scatter_buckets_impl(if (ue >= e0 && ue < e0 + tile_e) {const uint te = ue - e0;atomic_fetch_add_explicit(- &sg_counts[simd.threadgroup_index * TILE_E + te],+ &sg_counts[thread_context.threadgroup_index * TILE_E + te],1u,memory_order_relaxed);@@ -103,8 +103,8 @@ inline void moe_scatter_buckets_impl(threadgroup_barrier(mem_flags::mem_threadgroup);// Phase 3: deterministic write with 32-lane sequencing, no device atomics- for (uint step = 0; step < simd.simdgroup_size; ++step) {- if (simd.simdgroup_index == step) {+ for (uint step = 0; step < thread_context.simdgroup_size; ++step) {+ if (thread_context.simdgroup_index == step) {const uint t = t_start + lid;if (t < t_end) {const uint base = t * K;@@ -117,7 +117,8 @@ inline void moe_scatter_buckets_impl(const uint tile_id = e0 / TILE_E;const uint base_idx_block =(block_id * num_tiles + tile_id) * TILE_E + te;- const uint idx_sg_te = simd.threadgroup_index * TILE_E + te;+ const uint idx_sg_te =+ thread_context.threadgroup_index * TILE_E + te;const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te],1u,@@ -202,7 +203,7 @@ PUBLIC KERNEL(MoeScatterBuckets)(constant uint& num_tiles,threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],threadgroup uint sg_base[NUM_SG * TILE_E],- const ThreadContext simd,+ const ThreadContext thread_context,const uint tgpig_x GROUPS(num_blocks),const uint lid THREADS(256)) {@@ -228,7 +229,7 @@ PUBLIC KERNEL(MoeScatterBuckets)(tgpig,sg_counts,sg_base,- simd+ thread_context);}@@ -251,7 +252,7 @@ PUBLIC KERNEL(MoeScatterBucketsMap)(device int* tok2row,threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],threadgroup uint sg_base[NUM_SG * TILE_E],- const ThreadContext simd,+ const ThreadContext thread_context,const uint tgpig_x GROUPS(num_blocks),const uint lid THREADS(256)) {@@ -277,6 +278,6 @@ PUBLIC KERNEL(MoeScatterBucketsMap)(tgpig,sg_counts,sg_base,- simd+ thread_context);}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex eb0d04ac4..d08730f49 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -25,7 +25,7 @@ PUBLIC KERNEL(RMSNorm)(constant bool& full_layer,const bool in_place SPECIALIZE,threadgroup AccumT shared_sum[METAL_SIMD_SIZE],- const ThreadContext simd,+ const ThreadContext thread_context,const uint batch_idx GROUPS(batch_size),const uint thread_in_row THREADS(1024)) {@@ -60,7 +60,7 @@ PUBLIC KERNEL(RMSNorm)(partial_sum,shared_sum,thread_in_row,- simd+ thread_context);// Compute RMS norm factordiff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metalindex 86939cf03..497ffa642 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -27,7 +27,7 @@ static ArgmaxPair threadgroup_cooperative_argmax(ArgmaxPair value,threadgroup ArgmaxPair* shared,const ushort lid,- const thread ThreadContext& simd+ const thread ThreadContext& thread_context) {// Reduce within simdgroup using manual shuffle operationsArgmaxPair local_result = value;@@ -40,8 +40,8 @@ static ArgmaxPair threadgroup_cooperative_argmax(}// First thread in each simdgroup writes to shared memory- if (simd.simdgroup_index == 0) {- shared[simd.threadgroup_index] = local_result;+ if (thread_context.simdgroup_index == 0) {+ shared[thread_context.threadgroup_index] = local_result;}// Synchronize across the threadgroup@@ -203,7 +203,7 @@ PUBLIC KERNEL(ArgmaxMain)(constant uint& batch_size,constant uint& vocab_size,threadgroup ArgmaxPair shared[BLOCK_SIZE],- const ThreadContext simd,+ const ThreadContext thread_context,uint batch_idx GROUPS(batch_size),uint vocab_group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),ushort local_id THREADS(BLOCK_SIZE)@@ -230,7 +230,7 @@ PUBLIC KERNEL(ArgmaxMain)(thread_result,shared,local_id,- simd+ thread_context);if (local_id == 0) {@@ -247,7 +247,7 @@ PUBLIC KERNEL(ArgmaxFinal)(constant uint& batch_size,constant uint& vocab_size,threadgroup ArgmaxPair shared[BLOCK_SIZE],- const ThreadContext simd,+ const ThreadContext thread_context,uint batch_idx GROUPS(batch_size),ushort local_id THREADS(BLOCK_SIZE)) {@@ -271,7 +271,7 @@ PUBLIC KERNEL(ArgmaxFinal)(thread_result,shared,local_id,- simd+ thread_context);if (local_id == 0) {diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metalindex 121a1cca3..9b3962188 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal@@ -15,7 +15,7 @@ PUBLIC KERNEL(MinP) (constant uint& batch_size,constant uint& vocab_size,constant float& min_p,- const ThreadContext simd,+ const ThreadContext thread_context,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE),const bool in_place SPECIALIZE@@ -37,7 +37,7 @@ PUBLIC KERNEL(MinP) (local_max,shared_reduce_buffer,thread_idx,- simd+ thread_context);// Then the threshold is just max_logit + log(min_p), mask everything strictlydiff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex d921385b2..c4c08b604 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -16,7 +16,7 @@ PUBLIC KERNEL(TopK)(constant uint& batch_size,constant uint& vocab_size,constant uint& top_k,- const ThreadContext simd,+ const ThreadContext thread_context,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE),const bool in_place SPECIALIZE@@ -44,13 +44,13 @@ PUBLIC KERNEL(TopK)(local_max,shared_reduce_buffer,thread_idx,- simd+ thread_context);float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min,shared_reduce_buffer,thread_idx,- simd+ thread_context);// Do the binary search on the thresholdfloat low = min_logit;@@ -69,7 +69,7 @@ PUBLIC KERNEL(TopK)(local_num_above_threshold,(threadgroup uint*)shared_reduce_buffer,thread_idx,- simd+ thread_context);// Update binary searchdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metalindex cd4f01774..1ab64c2b7 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -17,7 +17,7 @@ PUBLIC KERNEL(TopP) (constant uint& batch_size,constant uint& vocab_size,constant float& top_p,- const ThreadContext simd,+ const ThreadContext thread_context,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE),const bool in_place SPECIALIZE@@ -45,13 +45,13 @@ PUBLIC KERNEL(TopP) (local_max,shared_reduce_buffer,thread_idx,- simd+ thread_context);float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min,shared_reduce_buffer,thread_idx,- simd+ thread_context);// Find denominator for softmax@@ -69,7 +69,7 @@ PUBLIC KERNEL(TopP) (local_sum,shared_reduce_buffer,thread_idx,- simd+ thread_context);// Do the binary search on the threshold@@ -98,13 +98,13 @@ PUBLIC KERNEL(TopP) (local_sum_above_threshold,shared_reduce_buffer,thread_idx,- simd+ thread_context);float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min_above_threshold,shared_reduce_buffer,thread_idx,- simd+ thread_context);// Early exitdiff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex f84606b78..d9b4001f3 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -28,7 +28,7 @@ PUBLIC KERNEL(SSDPrefill64)(constant const uint* state_strides,constant const uint& num_heads,constant const uint& head_dim,- const ThreadContext simd,+ const ThreadContext thread_context,const uint pair_idx GROUPS(num_heads * head_dim),const uint lane_idx THREADS(32)) {@@ -56,7 +56,7 @@ PUBLIC KERNEL(SSDPrefill64)(const float d_scalar = float(d[h_idx]);const uint idx0 = lane_idx;- const uint idx1 = lane_idx + simd.simdgroup_size;+ const uint idx1 = lane_idx + thread_context.simdgroup_size;float state0 = 0.0f;float state1 = 0.0f;@@ -126,7 +126,7 @@ PUBLIC KERNEL(SSDPrefill)(constant const uint* state_strides,constant const uint& num_heads,constant const uint& head_dim,- const ThreadContext simd,+ const ThreadContext thread_context,const uint pair_idx GROUPS(num_heads * head_dim),const uint lane_idx THREADS(32)) {@@ -154,9 +154,9 @@ PUBLIC KERNEL(SSDPrefill)(const uint cb_group_base = group_idx * cb_group_stride;const float d_scalar = float(d[h_idx]);- const int max_chunks = SSM_PREFILL_MAX_STATE / simd.simdgroup_size;- const int chunk_count =- (state_dim + int(simd.simdgroup_size) - 1) / int(simd.simdgroup_size);+ const int max_chunks = SSM_PREFILL_MAX_STATE / thread_context.simdgroup_size;+ const int chunk_count = (state_dim + int(thread_context.simdgroup_size) - 1) /+ int(thread_context.simdgroup_size);if (state_dim <= 0 || chunk_count > max_chunks) {return;}@@ -165,7 +165,7 @@ PUBLIC KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);+ const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);const uint state_idx = state_base + idx * state_inner_stride;lane_states[chunk] = float(state[state_idx]);}@@ -183,7 +183,8 @@ PUBLIC KERNEL(SSDPrefill)(float contrib_sum = 0.0f;#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);+ const int idx =+ chunk * int(thread_context.simdgroup_size) + int(lane_idx);const uint cb_idx =cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;const float new_state =@@ -200,7 +201,7 @@ PUBLIC KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);+ const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);const uint state_idx = state_base + uint(idx) * state_inner_stride;state[state_idx] = static_cast<T>(lane_states[chunk]);}diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex 4e6deb496..a630ffefb 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -11,7 +11,7 @@ use crate::{Backend, CommandBuffer, Kernels,kernel::{FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,- matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},+ matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,QuantizedMatmulKernelEncodable, QuantizedMatmulType,@@ -47,7 +47,7 @@ enum TiedEmbeddingType<B: Backend> {FullPrecision {weights: B::Buffer,lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,- readout: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,+ readout: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,},Quantized {weights: B::Buffer,@@ -74,7 +74,7 @@ enum UntiedEmbeddingLookupType<B: Backend> {enum UntiedEmbeddingReadoutType<B: Backend> {FullPrecision {weights: B::Buffer,- readout: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,+ readout: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,},Quantized {weights: B::Buffer,@@ -153,8 +153,7 @@ impl<B: Backend> Embedding<B> {let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;- let readout =- RefCell::new(<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?);+ let readout = RefCell::new(<B::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)?);EmbeddingTying::Tied {ty: TiedEmbeddingType::FullPrecision {@@ -180,8 +179,7 @@ impl<B: Backend> Embedding<B> {let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;- let readout =- RefCell::new(<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?);+ let readout = RefCell::new(<B::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)?);EmbeddingTying::Untied {input_ty: UntiedEmbeddingLookupType::FullPrecision {@@ -553,19 +551,25 @@ impl<B: Backend> Embedding<B> {readout,},} => {+ let input_dim = self.model_dim as usize;+ let output_dim = self.vocab_size as usize;readout.borrow_mut().encode(state.context(),- command_buffer,- FullPrecisionMatmulArguments {+ MatmulArguments {a: input,- a_offset: input_offset,+ a_offset: input_offset as u64,b: weights,- output,+ d: output,bias: None,- batch: batch_size,- input_dim: self.model_dim as usize,- output_dim: self.vocab_size as usize,+ batch: batch_size as i32,+ input_dim: input_dim as i32,+ output_dim: output_dim as i32,+ leading_dimension_a: input_dim as i32,+ leading_dimension_b: input_dim as i32,+ leading_dimension_d: output_dim as i32,+ transpose_b: true,},+ command_buffer,);},EmbeddingTying::Tied {diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rsindex 2416b1bb2..e54049ec7 100644--- a/crates/uzu/src/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs@@ -11,7 +11,7 @@ use crate::{DataType,backends::common::{Backend, CommandBuffer,- kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},},forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},@@ -49,7 +49,7 @@ pub enum FullPrecisionLinearError<B: Backend> {}pub struct FullPrecisionLinear<B: Backend> {- kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,+ kernel: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,bias_buffer: Option<Rc<RefCell<B::Buffer>>>,weights_buffer: Rc<RefCell<B::Buffer>>,input_dim: usize,@@ -111,7 +111,7 @@ impl<B: Backend> FullPrecisionLinear<B> {Err(_) => None,};- let kernel = <B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, precision)?;+ let kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(context, precision)?;Ok(Self {kernel: RefCell::new(kernel),@@ -139,17 +139,21 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());self.kernel.borrow_mut().encode(state.context(),- command_buffer,- FullPrecisionMatmulArguments {+ MatmulArguments {a: input_array.buffer().borrow().deref(),a_offset: 0,b: self.weights_buffer.borrow().deref(),- output: output_array.buffer().borrow_mut().deref_mut(),+ d: output_array.buffer().borrow_mut().deref_mut(),bias: bias_borrow.as_deref(),- batch: batch_size,- input_dim: self.input_dim,- output_dim: self.output_dim,+ batch: batch_size as i32,+ input_dim: self.input_dim as i32,+ output_dim: self.output_dim as i32,+ leading_dimension_a: self.input_dim as i32,+ leading_dimension_b: self.input_dim as i32,+ leading_dimension_d: self.output_dim as i32,+ transpose_b: true,},+ command_buffer,);Ok(())}diff --git a/crates/uzu/tests/kernel/matmul/gemv_test.rs b/crates/uzu/tests/kernel/matmul/gemv_test.rsindex 2381dfe8f..0b3160835 100644--- a/crates/uzu/tests/kernel/matmul/gemv_test.rs+++ b/crates/uzu/tests/kernel/matmul/gemv_test.rs@@ -12,10 +12,7 @@ use uzu::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context,- kernel::matmul::{- MatmulArguments,- gemv::{GemvDispatchDescriptor, GemvKernel},- },+ kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},},cpu::Cpu,},@@ -69,29 +66,28 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let d_buf = d_array.buffer();let mut d_ref = d_buf.borrow_mut();- let mut arguments = MatmulArguments {- a: a_ref.deref(),- a_offset: 0,- b: b_ref.deref(),- d: d_ref.deref_mut(),- bias: None,- batch: m,- input_dim: k,- output_dim: n,- leading_dimension_a: k,- leading_dimension_b: k,- leading_dimension_d: n,- transpose_b: true,- };-- let descriptor = GemvDispatchDescriptor::try_new::<B>(T::data_type(), &arguments)- .expect("Failed to create descriptor")- .expect("GEMV not applicable for these dimensions");-- let mut kernel = GemvKernel::<B>::new(T::data_type()).expect("Failed to create GemvKernel");+ let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())+ .expect("Failed to create MatmulKernel");let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel.encode(&context, &mut arguments, &descriptor, &mut command_buffer).expect("Failed to encode");+ kernel.encode(+ &context,+ MatmulArguments {+ a: a_ref.deref(),+ a_offset: 0,+ b: b_ref.deref(),+ d: d_ref.deref_mut(),+ bias: None,+ batch: m,+ input_dim: k,+ output_dim: n,+ leading_dimension_a: k,+ leading_dimension_b: k,+ leading_dimension_d: n,+ transpose_b: true,+ },+ &mut command_buffer,+ );command_buffer.end_encoding().submit().wait_until_completed().unwrap();drop(d_ref);@@ -111,7 +107,6 @@ fn test<T: ArrayElement + Float + Debug + Display>(});}-// M=1 (true vector-matrix multiply)#[test]fn test_f32_m1() {test::<f32>(1, 128, 64, 0.01);@@ -127,7 +122,6 @@ fn test_bf16_m1() {test::<bf16>(1, 128, 64, 0.1);}-// Small batch (M <= 8, batched GEMV)#[test]fn test_f32_batched() {test::<f32>(4, 128, 64, 0.01);@@ -143,7 +137,6 @@ fn test_bf16_batched() {test::<bf16>(4, 128, 64, 0.1);}-// Max batch (M=8)#[test]fn test_f32_max_batch() {test::<f32>(8, 128, 64, 0.01);@@ -159,7 +152,6 @@ fn test_bf16_max_batch() {test::<bf16>(8, 128, 64, 0.1);}-// Unaligned K dimension#[test]fn test_f32_unaligned_k() {test::<f32>(1, 33, 64, 0.01);@@ -175,7 +167,6 @@ fn test_bf16_unaligned_k() {test::<bf16>(1, 33, 64, 0.1);}-// Unaligned N dimension#[test]fn test_f32_unaligned_n() {test::<f32>(1, 128, 11, 0.01);@@ -191,7 +182,6 @@ fn test_bf16_unaligned_n() {test::<bf16>(1, 128, 11, 0.1);}-// Large dimensions#[test]fn test_f32_large() {test::<f32>(1, 4096, 2048, 0.05);@@ -207,7 +197,6 @@ fn test_bf16_large() {test::<bf16>(1, 4096, 2048, 1.0);}-// Small N (< 4, triggers elements_per_thread_col=1 path)#[test]fn test_f32_small_n() {test::<f32>(1, 128, 3, 0.01);diff --git a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rsindex 70c4d305f..2b73062f5 100644--- a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs+++ b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs@@ -12,11 +12,7 @@ use uzu::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context,- gpu_types::GemmParams,- kernel::matmul::{- GridSize, MatmulArguments,- gemm::{GemmDispatchDescriptor, GemmKernel, GemmSpecialization},- },+ kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},},cpu::Cpu,},@@ -63,52 +59,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");let d_array = context.create_array_uninitialized(&[input.m, input.n], T::data_type(), "");- // Pick a precompile config matching alignment requirements- let configs = GemmSpecialization::precompile_configs(T::data_type());- let base = configs[0];- let align_m = (m % base.block_rows) == 0;- let align_n = (n % base.block_cols) == 0;- let align_k = (k % base.block_depth) == 0;- // Find a config that matches alignment, or fall back to overriding the first one- let config = configs- .iter()- .find(|c| c.align_m == align_m && c.align_n == align_n && c.align_k == align_k)- .copied()- .unwrap_or(GemmSpecialization {- align_m,- align_n,- align_k,- ..base- });-- let threadgroups_per_row = (n + config.block_cols - 1) / config.block_cols;- let threadgroups_per_column = (m + config.block_rows - 1) / config.block_rows;-- let params = GemmParams {- M: m,- N: n,- K: k,- leading_dimension_a: k,- leading_dimension_b: k,- leading_dimension_d: n,- threadgroups_per_row,- threadgroups_per_column,- swizzle_log: 0,- aligned_inner_iterations: k / config.block_depth,- };-- let descriptor = GemmDispatchDescriptor {- specialization: config,- params,- threadgroups: GridSize {- x: threadgroups_per_row as usize,- y: threadgroups_per_column as usize,- z: 1,- },- };-- let mut kernel = GemmKernel::<B>::new(T::data_type()).expect("Failed to create GemmKernel");-let a_buf = a_array.buffer();let a_ref = a_buf.borrow();let b_buf = b_array.buffer();@@ -116,22 +66,28 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let d_buf = d_array.buffer();let mut d_ref = d_buf.borrow_mut();+ let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())+ .expect("Failed to create MatmulKernel");+let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- let mut arguments = MatmulArguments {- a: a_ref.deref(),- a_offset: 0,- b: b_ref.deref(),- d: d_ref.deref_mut(),- bias: None,- batch: m,- input_dim: k,- output_dim: n,- leading_dimension_a: k,- leading_dimension_b: k,- leading_dimension_d: n,- transpose_b: true,- };- kernel.encode(&context, &mut arguments, &descriptor, &mut command_buffer).expect("Failed to encode");+ kernel.encode(+ &context,+ MatmulArguments {+ a: a_ref.deref(),+ a_offset: 0,+ b: b_ref.deref(),+ d: d_ref.deref_mut(),+ bias: None,+ batch: m,+ input_dim: k,+ output_dim: n,+ leading_dimension_a: k,+ leading_dimension_b: k,+ leading_dimension_d: n,+ transpose_b: true,+ },+ &mut command_buffer,+ );command_buffer.end_encoding().submit().wait_until_completed().unwrap();drop(d_ref);@@ -151,7 +107,6 @@ fn test<T: ArrayElement + Float + Debug + Display>(});}-// Aligned dimensions (divisible by common block sizes)#[test]fn test_f32_aligned() {test::<f32>(64, 64, 64, 0.01);@@ -167,7 +122,6 @@ fn test_bf16_aligned() {test::<bf16>(64, 64, 64, 0.1);}-// Unaligned dimensions#[test]fn test_f32_unaligned() {test::<f32>(7, 33, 11, 0.01);@@ -183,7 +137,6 @@ fn test_bf16_unaligned() {test::<bf16>(7, 33, 11, 0.1);}-// Larger matrix#[test]fn test_f32_large() {test::<f32>(16, 128, 256, 0.01);diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 802873345..1608ba0ed 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -10,7 +10,7 @@ use uzu::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context,- kernel::matmul::{MatmulArguments, MatmulKernel, choose_matmul_dispatch_descriptor},+ kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},},metal::Metal,},@@ -44,26 +44,28 @@ fn run_metal_matmul(n};- let mut kernel = MatmulKernel::<Metal>::new(DataType::BF16).expect("kernel");+ let mut kernel =+ <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();- let arguments = MatmulArguments {- a: &a_buf,- a_offset: 0,- b: &b_buf,- d: &mut d_buf,- bias: None,- batch: m as i32,- input_dim: k as i32,- output_dim: n as i32,- leading_dimension_a: k as i32,- leading_dimension_b: leading_dim_b as i32,- leading_dimension_d: n as i32,- transpose_b,- };- let descriptor =- choose_matmul_dispatch_descriptor::<Metal>(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");- kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut command_buffer).expect("encode");+ kernel.encode(+ ctx,+ MatmulArguments {+ a: &a_buf,+ a_offset: 0,+ b: &b_buf,+ d: &mut d_buf,+ bias: None,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ leading_dimension_a: k as i32,+ leading_dimension_b: leading_dim_b as i32,+ leading_dimension_d: n as i32,+ transpose_b,+ },+ &mut command_buffer,+ );command_buffer.end_encoding().submit().wait_until_completed().unwrap();unsafe {diff --git a/crates/uzu/tests/matmul_perf_test/bench.rs b/crates/uzu/tests/matmul_perf_test/bench.rsindex 460f3ed83..224a9aeb2 100644--- a/crates/uzu/tests/matmul_perf_test/bench.rs+++ b/crates/uzu/tests/matmul_perf_test/bench.rs@@ -6,7 +6,7 @@ use uzu::{common::{Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,CommandBufferPending, Context,- kernel::matmul::{MatmulArguments, MatmulDispatchDescriptor, MatmulKernel},+ kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},},metal::Metal,},@@ -31,44 +31,34 @@ fn fill_buffer_random(}}-pub fn make_arguments<'a>(- a_buffer: &'a Buf,- b_buffer: &'a Buf,- d_buffer: &'a mut Buf,- shape: &TestShape,-) -> MatmulArguments<'a, Metal> {- MatmulArguments {- a: a_buffer,- a_offset: 0,- b: b_buffer,- d: d_buffer,- bias: None,- batch: shape.batch as i32,- input_dim: shape.input_dim as i32,- output_dim: shape.output_dim as i32,- leading_dimension_a: shape.input_dim as i32,- leading_dimension_b: shape.input_dim as i32,- leading_dimension_d: shape.output_dim as i32,- transpose_b: true,- }-}-fn encode_and_run(context: &Ctx,- kernel: &mut MatmulKernel<Metal>,+ kernel: &mut <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel,shape: &TestShape,a_buffer: &Buf,b_buffer: &Buf,d_buffer: &mut Buf,- dispatch_descriptor: &MatmulDispatchDescriptor,) -> Result<f64, BenchError> {let mut command_buffer = context.create_command_buffer().map_err(|_| BenchError::CommandBuffer)?.start_encoding();- let arguments = make_arguments(a_buffer, b_buffer, d_buffer, shape);-- kernel- .encode_with_descriptor(context, arguments, dispatch_descriptor, &mut command_buffer)- .map_err(|e| BenchError::Encode(e.to_string()))?;+ kernel.encode(+ context,+ MatmulArguments {+ a: a_buffer,+ a_offset: 0,+ b: b_buffer,+ d: d_buffer,+ bias: None,+ batch: shape.batch as i32,+ input_dim: shape.input_dim as i32,+ output_dim: shape.output_dim as i32,+ leading_dimension_a: shape.input_dim as i32,+ leading_dimension_b: shape.input_dim as i32,+ leading_dimension_d: shape.output_dim as i32,+ transpose_b: true,+ },+ &mut command_buffer,+ );let completed =command_buffer.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;@@ -80,9 +70,9 @@ fn run_benchmark(context: &Ctx,data_type: DataType,shape: &TestShape,- dispatch_descriptor: &MatmulDispatchDescriptor,) -> Result<f64, BenchError> {- let mut kernel = MatmulKernel::<Metal>::new(data_type).map_err(|e| BenchError::Kernel(e.to_string()))?;+ let mut kernel = <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)+ .map_err(|e| BenchError::Kernel(e.to_string()))?;let elem_size = data_type.size_in_bytes();let a_byte_count = shape.batch * shape.input_dim * elem_size;@@ -102,22 +92,23 @@ fn run_benchmark(fill_buffer_random(&b_buffer, b_byte_count);for iteration in 0..WARMUP_ITERATIONS {- encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer, dispatch_descriptor).map_err(- |source| BenchError::Warmup {+ encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer).map_err(|source| {+ BenchError::Warmup {iteration,source: Box::new(source),- },- )?;+ }+ })?;}let mut gpu_time_total_ms = 0.0;for iteration in 0..BENCHMARK_ITERATIONS {let gpu_ms =- encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer, dispatch_descriptor)- .map_err(|source| BenchError::Benchmark {+ encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer).map_err(|source| {+ BenchError::Benchmark {iteration,source: Box::new(source),- })?;+ }+ })?;gpu_time_total_ms += gpu_ms;}@@ -128,17 +119,15 @@ pub fn benchmark_single(context: &Ctx,data_type: DataType,shape: &TestShape,- path_name: &str,- dispatch_descriptor: &MatmulDispatchDescriptor,) -> PerfResult {- match run_benchmark(context, data_type, shape, dispatch_descriptor) {+ match run_benchmark(context, data_type, shape) {Ok(duration_ms) => {let flops = 2.0 * shape.batch as f64 * shape.input_dim as f64 * shape.output_dim as f64;let gflops = flops / (duration_ms / 1000.0) / 1e9;PerfResult {combo: format!("{data_type:?}"),shape: format!("{shape}"),- dispatch_path: path_name.to_owned(),+ dispatch_path: "auto".to_owned(),duration_ms,gflops,status: "ok".into(),@@ -148,7 +137,7 @@ pub fn benchmark_single(Err(error) => PerfResult {combo: format!("{data_type:?}"),shape: format!("{shape}"),- dispatch_path: path_name.to_owned(),+ dispatch_path: "auto".to_owned(),duration_ms: 0.0,gflops: 0.0,status: "error".into(),diff --git a/crates/uzu/tests/matmul_perf_test/error.rs b/crates/uzu/tests/matmul_perf_test/error.rsindex 3d11b6d69..dfd690cfe 100644--- a/crates/uzu/tests/matmul_perf_test/error.rs+++ b/crates/uzu/tests/matmul_perf_test/error.rs@@ -4,8 +4,6 @@ use thiserror::Error;pub enum BenchError {#[error("failed to create command buffer")]CommandBuffer,- #[error("encode failed: {0}")]- Encode(String),#[error("GPU timestamps unavailable")]GpuTimestamps,#[error("kernel creation failed: {0}")]diff --git a/crates/uzu/tests/matmul_perf_test/main.rs b/crates/uzu/tests/matmul_perf_test/main.rsindex c12f4d959..9bfd1fffb 100644--- a/crates/uzu/tests/matmul_perf_test/main.rs+++ b/crates/uzu/tests/matmul_perf_test/main.rs@@ -6,16 +6,13 @@ mod output;mod shapes;use indicatif::{ProgressBar, ProgressStyle};-use metal::{MTLDeviceExt, MTLResourceOptions};+use metal::MTLDeviceExt;use output::print_results_table;use shapes::test_shapes;use uzu::{DataType,backends::{- common::{- Backend, Context,- kernel::matmul::{MatmulDispatchDescriptor, choose_matmul_dispatch_descriptor},- },+ common::{Backend, Context},metal::Metal,},};@@ -56,47 +53,9 @@ fn matmul_perf() {let mut results = Vec::new();for &data_type in &data_types {- let elem_size = data_type.size_in_bytes();-for shape in &test_shapes {- let a_byte_count = shape.batch * shape.input_dim * elem_size;- let b_byte_count = shape.output_dim * shape.input_dim * elem_size;- let d_byte_count = shape.batch * shape.output_dim * elem_size;-- let (a_buffer, b_buffer, mut d_buffer) = match (- context.device.new_buffer(a_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),- context.device.new_buffer(b_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),- context.device.new_buffer(d_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),- ) {- (Some(a), Some(b), Some(d)) => (a, b, d),- _ => continue,- };-- let arguments = bench::make_arguments(&a_buffer, &b_buffer, &mut d_buffer, shape);-- let descriptor = match choose_matmul_dispatch_descriptor::<Metal>(&context, data_type, &arguments) {- Ok(d) => d,- Err(e) => {- results.push(output::PerfResult {- combo: format!("{data_type:?}"),- shape: format!("{shape}"),- dispatch_path: "auto".into(),- duration_ms: 0.0,- gflops: 0.0,- status: "error".into(),- error: Some(format!("dispatch: {e}")),- });- continue;- },- };-- let path_name = match &descriptor {- MatmulDispatchDescriptor::Gemv(_) => "Gemv",- MatmulDispatchDescriptor::Gemm(_) => "Gemm",- };-- progress_bar.set_message(format!("{data_type:?} {shape} {path_name}"));- let result = bench::benchmark_single(&context, data_type, shape, path_name, &descriptor);+ progress_bar.set_message(format!("{data_type:?} {shape}"));+ let result = bench::benchmark_single(&context, data_type, shape);results.push(result);progress_bar.inc(1);}