A high-performance inference engine for AI models
Generalize and parallelize matmul kernel (#253)
Aleksandr Golokoz9e6fe067b2dd
diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rsindex d4fb3fd05..264f8ed4e 100644--- a/crates/uzu/src/backends/common/context.rs+++ b/crates/uzu/src/backends/common/context.rs@@ -1,4 +1,4 @@-use std::{path::Path, rc::Rc};+use std::{env, path::Path, rc::Rc};use super::Backend;use crate::backends::common::CommandBuffer;@@ -18,6 +18,21 @@ impl DeviceClass {}}+/// Device performance class based on the last character of architecture name.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DeviceType {+ Phone, // 'p' - iPhone/iPad integrated+ Integrated, // 'g' - Mac integrated GPU+ Desktop, // 'd' - Mac Pro/Max discrete-class GPU+ Unknown(char),+}++impl DeviceType {+ pub fn is_high_performance(&self) -> bool {+ matches!(self, Self::Desktop)+ }+}+pub trait Context: Sized {type Backend: Backend<Context = Self>;@@ -25,6 +40,8 @@ pub trait Context: Sized {fn device_class(&self) -> DeviceClass;+ fn device_type(&self) -> DeviceType;+fn debug_active(&self) -> bool;fn create_command_buffer(@@ -46,4 +63,8 @@ pub trait Context: Sized {) -> Result<(), <Self::Backend as Backend>::Error>;fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;++ fn tf32_enabled() -> bool {+ env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)+ }}diff --git a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rsindex cb79a4815..5d6b3fbb5 100644--- a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs@@ -1,10 +1,19 @@-use super::{gemm, gemv, split_k};+use super::{MatmulArguments, MatmulError};+use crate::{+ DataType,+ backends::common::{+ Backend,+ kernel::matmul::{+ gemm::GemmDispatchDescriptor, gemv::GemvDispatchDescriptor, split_k::SplitKDispatchDescriptor,+ },+ },+};#[derive(Debug, Clone)]pub enum MatmulDispatchDescriptor {- Gemv(gemv::DispatchDescriptor),- SplitK(split_k::DispatchDescriptor),- Gemm(gemm::DispatchDescriptor),+ Gemv(GemvDispatchDescriptor),+ SplitK(SplitKDispatchDescriptor),+ Gemm(GemmDispatchDescriptor),}impl MatmulDispatchDescriptor {@@ -15,3 +24,19 @@ impl MatmulDispatchDescriptor {}}}++pub fn choose_matmul_dispatch_descriptor<B: Backend>(+ context: &B::Context,+ data_type: DataType,+ arguments: &MatmulArguments<B>,+) -> Result<MatmulDispatchDescriptor, MatmulError<B>> {+ if let Some(descriptor) = GemvDispatchDescriptor::try_new::<B>(data_type, arguments)? {+ return Ok(MatmulDispatchDescriptor::Gemv(descriptor));+ }++ if let Some(descriptor) = SplitKDispatchDescriptor::try_new::<B>(data_type, arguments)? {+ return Ok(MatmulDispatchDescriptor::SplitK(descriptor));+ }++ Ok(MatmulDispatchDescriptor::Gemm(GemmDispatchDescriptor::try_new::<B>(context, data_type, arguments)?))+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rsindex ec91a6d71..af3a17afb 100644--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs@@ -1,7 +1,9 @@-use super::{MatmulError, MatmulKernels};+use super::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels};use crate::{DataType,- backends::common::{Backend, CommandBuffer},+ backends::common::{+ Backend, CommandBuffer, kernel::matmul::dispatch_descriptor::choose_matmul_dispatch_descriptor,+ },};pub struct FullPrecisionMatmulArguments<'a, B: Backend> {@@ -30,3 +32,49 @@ pub trait FullPrecisionMatmulKernel: Sized {arguments: FullPrecisionMatmulArguments<Self::Backend>,);}++impl<B: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>> FullPrecisionMatmulKernel+ for MatmulKernel<B>+{+ type Backend = B;++ fn new(+ context: &<Self::Backend as Backend>::Context,+ data_type: DataType,+ ) -> Result<Self, MatmulError<Self::Backend>> {+ let mut kernel = MatmulKernel::new(data_type)?;+ kernel.precompile(context)?;+ Ok(kernel)+ }++ fn encode(+ &mut self,+ context: &<Self::Backend as Backend>::Context,+ command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,+ arguments: FullPrecisionMatmulArguments<Self::Backend>,+ ) {+ let mut matmul_arguments = MatmulArguments {+ a: arguments.a,+ a_offset: arguments.a_offset as u64,+ b: arguments.b,+ d: arguments.output,+ bias: arguments.bias,+ batch: arguments.batch as i32,+ input_dim: arguments.input_dim as i32,+ output_dim: arguments.output_dim as i32,+ lda: arguments.input_dim as i32,+ ldb: arguments.input_dim as i32,+ ldd: arguments.output_dim as i32,+ batch_count: 1,+ transpose_b: true,+ };++ MatmulKernel::<B>::apply_batch_collapse(&mut matmul_arguments);++ let descriptor = choose_matmul_dispatch_descriptor(context, self.data_type, &matmul_arguments)+ .expect("Failed to create dispatch descriptor for full precision matmul");++ self.encode_with_descriptor(context, matmul_arguments, &descriptor, command_buffer)+ .expect("Failed to encode full precision matmul kernel");+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rsindex 44e1d380b..2355e4d90 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs@@ -1,9 +1,79 @@-use super::{super::grid_size::GridSize, specialization::Specialization};-use crate::backends::common::gpu_types::GEMMParams;+use super::{super::grid_size::GridSize, specialization::GemmSpecialization};+use crate::{+ DataType,+ backends::common::{+ Backend,+ gpu_types::GEMMParams,+ kernel::matmul::{MatmulArguments, MatmulError},+ },+};#[derive(Debug, Clone)]-pub struct DispatchDescriptor {- pub specialization: Specialization,+pub struct GemmDispatchDescriptor {+ pub specialization: GemmSpecialization,pub params: GEMMParams,pub threadgroups: GridSize,}++impl GemmDispatchDescriptor {+ pub fn try_new<B: Backend>(+ context: &B::Context,+ data_type: DataType,+ arguments: &MatmulArguments<B>,+ ) -> Result<Self, MatmulError<B>> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MatmulError::UnsupportedDataType(data_type));+ }++ let config = GemmSpecialization::select::<B>(context, data_type, arguments);++ let m = arguments.batch;+ let n = arguments.output_dim;+ let k = arguments.input_dim;++ let tiles_n = (n + config.block_cols - 1) / config.block_cols;+ let tiles_m = (m + config.block_rows - 1) / config.block_rows;+ let swizzle_log = config.swizzle_log2;++ let tile_swizzle = 1 << swizzle_log;+ let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;+ let tn_swizzled = tiles_n * tile_swizzle;++ let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);+ let elements_per_matrix_b = if arguments.transpose_b {+ (arguments.output_dim as i64) * (arguments.ldb as i64)+ } else {+ (arguments.input_dim as i64) * (arguments.ldb as i64)+ };+ let elements_per_matrix_d = (arguments.batch as i64) * (arguments.ldd as i64);++ let params = GEMMParams {+ M: m,+ N: n,+ K: k,+ lda: arguments.lda,+ ldb: arguments.ldb,+ ldd: arguments.ldd,+ tiles_n,+ tiles_m,+ batch_stride_a: elements_per_matrix_a,+ batch_stride_b: elements_per_matrix_b,+ batch_stride_d: elements_per_matrix_d,+ swizzle_log,+ gemm_k_iterations_aligned: k / config.block_depth,+ batch_ndim: 1,+ };++ let threadgroups = GridSize {+ x: tn_swizzled as usize,+ y: tm_swizzled as usize,+ z: arguments.batch_count as usize,+ };++ Ok(Self {+ specialization: config,+ params,+ threadgroups,+ })+ }+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rsindex ca3df3b48..6ee8898eb 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs@@ -1,7 +1,8 @@use std::{collections::HashMap, ops::DerefMut};use super::{- super::matmul_arguments::MatmulArguments, dispatch_descriptor::DispatchDescriptor, specialization::Specialization,+ super::matmul_arguments::MatmulArguments, dispatch_descriptor::GemmDispatchDescriptor,+ specialization::GemmSpecialization,};use crate::{DataType,@@ -13,7 +14,7 @@ use crate::{pub struct GemmKernel<B: Backend> {data_type: DataType,- pipelines: HashMap<Specialization, <B::Kernels as Kernels>::MatmulGemmKernel>,+ pipelines: HashMap<GemmSpecialization, <B::Kernels as Kernels>::MatmulGemmKernel>,}impl<B: Backend> GemmKernel<B> {@@ -31,7 +32,7 @@ impl<B: Backend> GemmKernel<B> {&mut self,context: &B::Context,) -> Result<(), MatmulError<B>> {- for &config in Specialization::precompile_configs(self.data_type) {+ for &config in GemmSpecialization::precompile_configs(self.data_type) {self.get_or_create_kernel(context, config)?;}Ok(())@@ -40,7 +41,7 @@ impl<B: Backend> GemmKernel<B> {fn get_or_create_kernel(&mut self,context: &B::Context,- config: Specialization,+ config: GemmSpecialization,) -> Result<&<B::Kernels as Kernels>::MatmulGemmKernel, MatmulError<B>> {if !self.pipelines.contains_key(&config) {let kernel = <B::Kernels as Kernels>::MatmulGemmKernel::new(@@ -65,7 +66,7 @@ impl<B: Backend> GemmKernel<B> {&mut self,context: &B::Context,arguments: &mut MatmulArguments<B>,- dispatch_descriptor: &DispatchDescriptor,+ dispatch_descriptor: &GemmDispatchDescriptor,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {let config = dispatch_descriptor.specialization;diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rsindex a4cbe5d36..9250445f6 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs@@ -2,6 +2,6 @@ mod dispatch_descriptor;mod kernel;pub mod specialization;-pub use dispatch_descriptor::DispatchDescriptor;+pub use dispatch_descriptor::GemmDispatchDescriptor;pub use kernel::GemmKernel;-pub use specialization::Specialization;+pub use specialization::GemmSpecialization;diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rsindex 40051d5db..4e0249017 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs@@ -1,7 +1,10 @@-use crate::DataType;+use crate::{+ DataType,+ backends::common::{Backend, Context, context::DeviceType, kernel::matmul::MatmulArguments},+};#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct Specialization {+pub struct GemmSpecialization {pub block_rows: i32,pub block_cols: i32,pub block_depth: i32,@@ -13,8 +16,8 @@ pub struct Specialization {pub align_k: bool,}-impl Specialization {- pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {+impl GemmSpecialization {+ pub fn precompile_configs(data_type: DataType) -> &'static [Self] {match data_type {DataType::BF16 => &[Self {@@ -135,4 +138,75 @@ impl Specialization {_ => &[],}}++ pub fn select<B: Backend>(+ context: &B::Context,+ data_type: DataType,+ arguments: &MatmulArguments<B>,+ ) -> Self {+ let overall_work_elements =+ (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);+ let is_float32 = matches!(data_type, DataType::F32);+ let prefer_half_or_tf32 = !is_float32 || B::Context::tf32_enabled();++ let (block_rows, block_cols, block_depth, warps_per_row, warps_per_col, swizzle_log2) =+ match context.device_type() {+ DeviceType::Integrated | DeviceType::Phone | DeviceType::Unknown(_) => {+ if prefer_half_or_tf32 {+ if arguments.transpose_b {+ (64, 32, 32, 2, 2, 0)+ } else {+ (64, 64, 16, 2, 2, 0)+ }+ } else if arguments.transpose_b {+ (32, 64, 16, 2, 2, 0)+ } else {+ (64, 32, 32, 2, 2, 0)+ }+ },+ DeviceType::Desktop => {+ if overall_work_elements >= (1_i64 << 20) {+ if prefer_half_or_tf32 {+ if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {+ (64, 64, 16, 2, 2, 0)+ } else if arguments.transpose_b {+ (64, 32, 32, 2, 2, 0)+ } else {+ (32, 64, 16, 2, 2, 0)+ }+ } else if arguments.transpose_b {+ (32, 64, 16, 2, 2, 0)+ } else {+ (64, 32, 32, 2, 2, 0)+ }+ } else if prefer_half_or_tf32 {+ if arguments.transpose_b {+ (64, 32, 32, 2, 2, 0)+ } else {+ (64, 64, 16, 2, 2, 0)+ }+ } else if arguments.transpose_b {+ (32, 64, 16, 2, 2, 0)+ } else {+ (64, 32, 32, 2, 2, 0)+ }+ },+ };++ let m = arguments.batch;+ let n = arguments.output_dim;+ let k = arguments.input_dim;++ Self {+ block_rows,+ block_cols,+ block_depth,+ warps_per_row,+ warps_per_col,+ swizzle_log2,+ align_m: (m % block_rows) == 0,+ align_n: (n % block_cols) == 0,+ align_k: (k % block_depth) == 0,+ }+ }}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rsindex 58132550c..d8e350f83 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs@@ -23,7 +23,7 @@ pub enum OutputSource {}#[derive(Debug, Clone)]-pub struct DispatchDescriptor {+pub struct GemvDispatchDescriptor {pub specialization: Specialization,pub matrix_is_rhs: bool,pub output_source: OutputSource,@@ -40,7 +40,7 @@ pub struct DispatchDescriptor {pub batch_rows: i32,}-impl DispatchDescriptor {+impl GemvDispatchDescriptor {pub fn try_new<B: Backend>(data_type: DataType,arguments: &MatmulArguments<B>,diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rsindex e4046f267..b7f28209e 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs@@ -2,7 +2,7 @@ use std::{collections::HashMap, ops::DerefMut};use super::{super::matmul_arguments::MatmulArguments,- dispatch_descriptor::{DispatchDescriptor, OutputSource},+ dispatch_descriptor::{GemvDispatchDescriptor, OutputSource},specialization::Specialization,};use crate::{@@ -66,7 +66,7 @@ impl<B: Backend> GemvKernel<B> {&mut self,context: &B::Context,arguments: &mut MatmulArguments<B>,- dispatch_descriptor: &DispatchDescriptor,+ dispatch_descriptor: &GemvDispatchDescriptor,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {let config = dispatch_descriptor.specialization;diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rsindex eba171b69..4fe67a542 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs@@ -2,5 +2,5 @@ mod dispatch_descriptor;mod kernel;mod specialization;-pub use dispatch_descriptor::DispatchDescriptor;+pub use dispatch_descriptor::GemvDispatchDescriptor;pub use kernel::GemvKernel;diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rsindex 267ea32a5..47baac2e7 100644--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs@@ -7,7 +7,7 @@ mod kernel;mod matmul_arguments;pub mod split_k;-pub use dispatch_descriptor::MatmulDispatchDescriptor;+pub use dispatch_descriptor::{MatmulDispatchDescriptor, choose_matmul_dispatch_descriptor};pub use full_precision::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel};pub use grid_size::GridSize;pub use kernel::MatmulKernel;diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rsindex db0641fb9..664fbf10a 100644--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs@@ -9,7 +9,7 @@ use crate::{};#[derive(Debug, Clone)]-pub struct DispatchDescriptor {+pub struct SplitKDispatchDescriptor {pub params: SplitKGEMMParams,pub partition_count: i32,pub output_elements_per_partition: i32,@@ -18,7 +18,7 @@ pub struct DispatchDescriptor {pub accum_total_threads: GridSize,}-impl DispatchDescriptor {+impl SplitKDispatchDescriptor {pub fn try_new<B: Backend>(data_type: DataType,arguments: &MatmulArguments<B>,diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rsindex 08cd9b9d2..4763402bb 100644--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs@@ -1,6 +1,6 @@use std::ops::{Deref, DerefMut};-use super::{super::matmul_arguments::MatmulArguments, dispatch_descriptor::DispatchDescriptor};+use super::{super::matmul_arguments::MatmulArguments, dispatch_descriptor::SplitKDispatchDescriptor};use crate::{DataType,backends::common::{@@ -67,7 +67,7 @@ impl<B: Backend> SplitKKernel<B> {&mut self,context: &B::Context,arguments: &mut MatmulArguments<B>,- dispatch_descriptor: &DispatchDescriptor,+ dispatch_descriptor: &SplitKDispatchDescriptor,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {self.ensure_kernels(context)?;diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rsindex 0fcefe437..8c1b0f372 100644--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs@@ -3,5 +3,5 @@ mod kernel;mod specialization;mod tile_configuration;-pub use dispatch_descriptor::DispatchDescriptor;+pub use dispatch_descriptor::SplitKDispatchDescriptor;pub use kernel::SplitKKernel;diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rsindex b98c4bdfa..eee34aca0 100644--- a/crates/uzu/src/backends/common/mod.rs+++ b/crates/uzu/src/backends/common/mod.rs@@ -12,6 +12,6 @@ pub use command_buffer::{CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,CommandBufferPending,};-pub use context::{Context, DeviceClass};+pub use context::{Context, DeviceClass, DeviceType};pub use event::Event;pub use kernel::Kernels;diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rsindex 79c450bfc..1fd61d943 100644--- a/crates/uzu/src/backends/cpu/context.rs+++ b/crates/uzu/src/backends/cpu/context.rs@@ -1,7 +1,7 @@use std::{cell::RefCell, rc::Rc};use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};-use crate::backends::common::{Context, DeviceClass};+use crate::backends::common::{Context, DeviceClass, DeviceType};pub struct CpuContext;@@ -47,4 +47,8 @@ impl Context for CpuContext {fn stop_capture(&self) -> Result<(), CpuError> {Err(CpuError::NotSupported)}++ fn device_type(&self) -> DeviceType {+ DeviceType::Integrated+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rsindex 0c9f8585e..453477ad5 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs@@ -2,7 +2,125 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{+ ArrayElement,+ pointers::{SendPtr, SendPtrMut},+};++#[inline(always)]+unsafe fn matmul_gemm_multi_thread_compute_row<T: ArrayElement + Float>(+ a: SendPtr<T>,+ b: SendPtr<T>,+ d: SendPtrMut<T>,+ row: usize,+ n: usize,+ k: usize,+ lda: usize,+ ldb: usize,+ ldd: usize,+) {+ unsafe {+ for col in 0..n {+ let mut acc: f32 = 0.0;+ for i in 0..k {+ let a_val = *a.0.add(row * lda + i);+ let b_val = *b.0.add(col * ldb + i);+ acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();+ }+ *d.0.add(row * ldd + col) = T::from(acc).unwrap();+ }+ }+}++fn matmul_gemm_multi_thread<T: ArrayElement + Float>(+ a: *const T,+ b: *const T,+ d: *mut T,+ batches_count: usize,+ m: usize,+ n: usize,+ k: usize,+ batch_stride_a: usize,+ batch_stride_b: usize,+ batch_stride_d: usize,+ lda: usize,+ ldb: usize,+ ldd: usize,+) {+ let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);+ unsafe {+ for batch in 0..batches_count {+ let a_batch = SendPtr(a.add(batch * batch_stride_a));+ let b_batch = SendPtr(b.add(batch * batch_stride_b));+ let d_batch = SendPtrMut(d.add(batch * batch_stride_d));++ std::thread::scope(|s| {+ let rows_per_thread = m / num_threads;+ let rows_per_thread_remainder = m % num_threads;+ let mut start = 0;++ for t in 0..num_threads {+ let chunk_add = if t < rows_per_thread_remainder {+ 1+ } else {+ 0+ };+ let chunk = rows_per_thread + chunk_add;++ let end = start + chunk;+ let ab = a_batch;+ let bb = b_batch;+ let db = d_batch;+ s.spawn(move || unsafe {+ for row in start..end {+ matmul_gemm_multi_thread_compute_row(ab, bb, db, row, n, k, lda, ldb, ldd);+ }+ });++ start = end;+ }+ });+ }+ }+}++fn matmul_gemm_single_thread<T: ArrayElement + Float>(+ a: *const T,+ b: *const T,+ d: *mut T,+ batches_count: usize,+ m: usize,+ n: usize,+ k: usize,+ batch_stride_a: usize,+ batch_stride_b: usize,+ batch_stride_d: usize,+ lda: usize,+ ldb: usize,+ ldd: usize,+) {+ unsafe {+ for batch in 0..batches_count {+ let a_batch = a.add(batch * batch_stride_a);+ let b_batch = b.add(batch * batch_stride_b);+ let d_batch = d.add(batch * batch_stride_d);++ for row in 0..m {+ for col in 0..n {+ let mut acc: f32 = 0.0;+ for i in 0..k {+ // A[row, i]: row-major with leading dimension lda+ let a_val = *a_batch.add(row * lda + i);+ // B[col, i]: B is transposed, so B is [N, K] row-major with leading dimension ldb+ let b_val = *b_batch.add(col * ldb + i);+ acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();+ }+ *d_batch.add(row * ldd + col) = T::from(acc).unwrap();+ }+ }+ }+ }+}#[kernel(MatmulGemm)]#[variants(T, f32, f16, bf16)]@@ -39,30 +157,23 @@ pub fn matmul_gemm<T: ArrayElement + Float>(#[specialize]align_k: bool,) {+ let batches_count = group_count_z as usize;unsafe {let p = &*params.as_ptr();- let lda = p.lda as usize;- let ldb = p.ldb as usize;- let ldd = p.ldd as usize;-- for batch in 0..group_count_z as usize {- let a_batch = a.add(batch * (p.batch_stride_a as usize));- let b_batch = b.add(batch * (p.batch_stride_b as usize));- let d_batch = d.add(batch * (p.batch_stride_d as usize));-- for row in 0..p.M as usize {- for col in 0..p.N as usize {- let mut acc: f32 = 0.0;- for i in 0..p.K as usize {- // A[row, i]: row-major with leading dimension lda- let a_val = *a_batch.add(row * lda + i);- // B[col, i]: B is transposed, so B is [N, K] row-major with leading dimension ldb- let b_val = *b_batch.add(col * ldb + i);- acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();- }- *d_batch.add(row * ldd + col) = T::from(acc).unwrap();- }- }- }+ matmul_gemm_multi_thread::<T>(+ a,+ b,+ d,+ group_count_z as usize,+ p.M as usize,+ p.N as usize,+ p.K as usize,+ p.batch_stride_a as usize,+ p.batch_stride_b as usize,+ p.batch_stride_d as usize,+ p.lda as usize,+ p.ldb as usize,+ p.ldd as usize,+ );}}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rsindex cf071f264..2febb91d5 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs@@ -2,7 +2,167 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{+ ArrayElement,+ pointers::{SendPtr, SendPtrMut},+};++#[inline(always)]+unsafe fn compute_gemv_rows<T: ArrayElement + Float>(+ mat: SendPtr<T>,+ vec_row: SendPtr<T>,+ out_row: SendPtrMut<T>,+ src: Option<SendPtr<T>>,+ out_start: usize,+ out_end: usize,+ in_dim: usize,+ mat_ld: usize,+ output_scale: f32,+ output_accumulate_scale: f32,+ output_source_stride: usize,+ apply_output_scale_and_accumulate: bool,+) {+ unsafe {+ for out_idx in out_start..out_end {+ let mat_row = mat.0.add(out_idx * mat_ld);++ let mut acc: f32 = 0.0;+ for in_idx in 0..in_dim {+ acc += (*mat_row.add(in_idx)).to_f32().unwrap() * (*vec_row.0.add(in_idx)).to_f32().unwrap();+ }++ if apply_output_scale_and_accumulate {+ if let Some(s) = src {+ let src_val = (*s.0.add(out_idx * output_source_stride)).to_f32().unwrap();+ acc = output_scale * acc + output_accumulate_scale * src_val;+ }+ }++ *out_row.0.add(out_idx) = T::from(acc).unwrap();+ }+ }+}++fn matmul_gemv_multi_thread<T: ArrayElement + Float>(+ matrix: *const T,+ input_vector: *const T,+ output_source: Option<*const T>,+ output_vector: *mut T,+ input_dimension: usize,+ output_dimension: usize,+ matrix_leading_dimension: usize,+ output_scale: f32,+ output_accumulate_scale: f32,+ batches_count: usize,+ vector_batch_stride: &[i32],+ matrix_batch_stride: &[i32],+ output_source_batch_stride: &[i32],+ output_source_stride: usize,+ batch_rows: usize,+ apply_output_scale_and_accumulate: bool,+) {+ let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);+ unsafe {+ for outer_batch in 0..batches_count as usize {+ let batch_vec = input_vector.add(outer_batch * vector_batch_stride[0] as usize);+ let batch_mat = SendPtr(matrix.add(outer_batch * matrix_batch_stride[0] as usize));+ let batch_out = SendPtrMut(output_vector.add(outer_batch * batch_rows * output_dimension));+ let batch_src =+ output_source.map(|src| SendPtr(src.add(outer_batch * output_source_batch_stride[0] as usize)));++ for batch_row in 0..batch_rows {+ let vec_row = SendPtr(batch_vec.add(batch_row * input_dimension));+ let out_row = SendPtrMut(batch_out.0.add(batch_row * output_dimension));++ std::thread::scope(|s| {+ let rows_per_thread = output_dimension / num_threads;+ let rows_per_thread_remainder = output_dimension % num_threads;+ let mut start = 0;++ for t in 0..num_threads {+ let chunk = rows_per_thread+ + if t < rows_per_thread_remainder {+ 1+ } else {+ 0+ };+ let end = start + chunk;++ s.spawn(move || unsafe {+ compute_gemv_rows(+ batch_mat,+ vec_row,+ out_row,+ batch_src,+ start,+ end,+ input_dimension,+ matrix_leading_dimension,+ output_scale,+ output_accumulate_scale,+ output_source_stride,+ apply_output_scale_and_accumulate,+ );+ });++ start = end;+ }+ });+ }+ }+ }+}++fn matmul_gemv_single_thread<T: ArrayElement + Float>(+ matrix: *const T,+ input_vector: *const T,+ output_source: Option<*const T>,+ output_vector: *mut T,+ input_dimension: usize,+ output_dimension: usize,+ matrix_leading_dimension: usize,+ output_scale: f32,+ output_accumulate_scale: f32,+ batches_count: usize,+ vector_batch_stride: &[i32],+ matrix_batch_stride: &[i32],+ output_source_batch_stride: &[i32],+ output_source_stride: usize,+ batch_rows: usize,+ apply_output_scale_and_accumulate: bool,+) {+ unsafe {+ for outer_batch in 0..batches_count {+ let batch_vec = input_vector.add(outer_batch * vector_batch_stride[0] as usize);+ let batch_mat = matrix.add(outer_batch * matrix_batch_stride[0] as usize);+ let batch_out = output_vector.add(outer_batch * batch_rows * output_dimension);+ let batch_src = output_source.map(|src| src.add(outer_batch * output_source_batch_stride[0] as usize));++ for batch_row in 0..batch_rows {+ let vec_row = batch_vec.add(batch_row * input_dimension);+ let out_row = batch_out.add(batch_row * output_dimension);++ for out_idx in 0..output_dimension {+ let mat_row = batch_mat.add(out_idx * matrix_leading_dimension);++ let mut acc: f32 = 0.0;+ for in_idx in 0..input_dimension {+ acc += (*mat_row.add(in_idx)).to_f32().unwrap() * (*vec_row.add(in_idx)).to_f32().unwrap();+ }++ if apply_output_scale_and_accumulate {+ if let Some(src) = batch_src {+ let src_val = (*src.add(out_idx * output_source_stride as usize)).to_f32().unwrap();+ acc = output_scale * acc + output_accumulate_scale * src_val;+ }+ }++ *out_row.add(out_idx) = T::from(acc).unwrap();+ }+ }+ }+ }+}#[kernel(MatmulGemv)]#[variants(T, f32, f16, bf16)]@@ -43,41 +203,22 @@ pub fn matmul_gemv<T: ArrayElement + Float>(thread_out_cols: u32,#[specialize] apply_output_scale_and_accumulate: bool,) {- let in_dim = input_dimension as usize;- let out_dim = output_dimension as usize;- let mat_ld = matrix_leading_dimension as usize;- let num_batch_rows = batch_rows as usize;-- unsafe {- for outer_batch in 0..batch_shape[0] as usize {- let batch_vec = input_vector.add(outer_batch * vector_batch_stride[0] as usize);- let batch_mat = matrix.add(outer_batch * matrix_batch_stride[0] as usize);- let batch_out = output_vector.add(outer_batch * num_batch_rows * out_dim);-- let batch_src = output_source.map(|src| src.add(outer_batch * output_source_batch_stride[0] as usize));-- for batch_row in 0..num_batch_rows {- let vec_row = batch_vec.add(batch_row * in_dim);- let out_row = batch_out.add(batch_row * out_dim);-- for out_idx in 0..out_dim {- let mat_row = batch_mat.add(out_idx * mat_ld);-- let mut acc: f32 = 0.0;- for in_idx in 0..in_dim {- acc += (*mat_row.add(in_idx)).to_f32().unwrap() * (*vec_row.add(in_idx)).to_f32().unwrap();- }-- if apply_output_scale_and_accumulate {- if let Some(src) = batch_src {- let src_val = (*src.add(out_idx * output_source_stride as usize)).to_f32().unwrap();- acc = output_scale * acc + output_accumulate_scale * src_val;- }- }-- *out_row.add(out_idx) = T::from(acc).unwrap();- }- }- }- }+ matmul_gemv_multi_thread::<T>(+ matrix,+ input_vector,+ output_source,+ output_vector,+ input_dimension as usize,+ output_dimension as usize,+ matrix_leading_dimension as usize,+ output_scale,+ output_accumulate_scale,+ batch_shape[0] as usize,+ vector_batch_stride,+ matrix_batch_stride,+ output_source_batch_stride,+ output_source_stride as usize,+ batch_rows as usize,+ apply_output_scale_and_accumulate,+ );}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rsindex 26abe007d..bc6efa06f 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs@@ -1,3 +1,12 @@+use crate::backends::{+ common::kernel::matmul::{MatmulKernel, MatmulKernels},+ cpu::{Cpu, kernel::CpuKernels},+};+pub mod gemm;pub mod gemv;pub mod split_k;++impl MatmulKernels for CpuKernels {+ type FullPrecisionMatmulKernel = MatmulKernel<Cpu>;+}diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rsindex 49d1e7de5..287488d31 100644--- a/crates/uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs@@ -23,40 +23,3 @@ mod tensor_copy;mod token_copy;include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));--use crate::{- DataType,- backends::{- common::kernel::matmul::{- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait, MatmulError,- MatmulKernels,- },- cpu::{Cpu, command_buffer::CpuCommandBuffer, context::CpuContext},- },-};--pub struct FullPrecisionMatmulCpuKernel;--impl FullPrecisionMatmulKernelTrait for FullPrecisionMatmulCpuKernel {- type Backend = Cpu;-- fn new(- #[allow(unused)] context: &CpuContext,- #[allow(unused)] data_type: DataType,- ) -> Result<Self, MatmulError<Cpu>> {- Ok(Self)- }-- fn encode(- &mut self,- #[allow(unused)] context: &CpuContext,- #[allow(unused)] encoder: &mut CpuCommandBuffer,- #[allow(unused)] arguments: FullPrecisionMatmulArguments<Cpu>,- ) {- encoder.push_command(move || todo!());- }-}--impl MatmulKernels for CpuKernels {- type FullPrecisionMatmulKernel = FullPrecisionMatmulCpuKernel;-}diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex 69e10c1ed..2625b7390 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -1,4 +1,4 @@-use std::{cell::RefCell, collections::HashMap, env, rc::Rc};+use std::{cell::RefCell, collections::HashMap, rc::Rc};use metal::{MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue, MTLCommandQueueExt,@@ -9,7 +9,7 @@ use objc2::{rc::Retained, runtime::ProtocolObject};use super::{Metal, error::MetalError, kernel, metal_extensions::LibraryPipelineExtensions};use crate::backends::{- common::{Context, DeviceClass as CommonDeviceClass},+ common::{Context, DeviceClass as CommonDeviceClass, DeviceType},metal::command_buffer::MetalCommandBufferInitial,};@@ -40,26 +40,11 @@ impl DeviceGeneration {}}-/// Device performance class based on the last character of architecture name.-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum DeviceClass {- Phone, // 'p' - iPhone/iPad integrated- Integrated, // 'g' - Mac integrated GPU- Desktop, // 'd' - Mac Pro/Max discrete-class GPU- Unknown(char),-}--impl DeviceClass {- pub fn is_high_performance(&self) -> bool {- matches!(self, Self::Desktop)- }-}-/// Complete device architecture information.#[derive(Debug, Clone)]pub struct DeviceArchitecture {pub generation: DeviceGeneration,- pub device_class: DeviceClass,+ pub device_class: DeviceType,#[allow(dead_code)]pub arch_string: String,}@@ -79,7 +64,7 @@ impl DeviceArchitecture {}}- fn parse_architecture_info(arch: &str) -> (DeviceGeneration, DeviceClass) {+ fn parse_architecture_info(arch: &str) -> (DeviceGeneration, DeviceType) {// Parse generation and class from device name (e.g. "Apple M3 Pro")let generation = if arch.contains("M5") {DeviceGeneration::Gen18@@ -100,11 +85,11 @@ impl DeviceArchitecture {};let device_class = if arch.contains("Max") || arch.contains("Ultra") || arch.contains("Pro") {- DeviceClass::Desktop+ DeviceType::Desktop} else if arch.contains("iPhone") || arch.contains("iPad") {- DeviceClass::Phone+ DeviceType::Phone} else {- DeviceClass::Integrated+ DeviceType::Integrated};(generation, device_class)@@ -148,16 +133,6 @@ impl MetalContext {self.architecture.generation}- /// Returns the device class for performance tuning.- pub fn device_class(&self) -> DeviceClass {- self.architecture.device_class- }-- /// TF32 toggle via UZU_TF32 environment variable.- pub fn tf32_enabled(&self) -> bool {- env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)- }-pub fn compute_pipeline_state(&self,cache_key: &str,@@ -217,6 +192,11 @@ impl Context for MetalContext {}}+ /// Returns the device class for performance tuning.+ fn device_type(&self) -> DeviceType {+ self.architecture.device_class+ }+fn debug_active(&self) -> bool {let upper = std::env::var("METAL_DEVICE_WRAPPER_TYPE").unwrap_or_default().to_ascii_uppercase();matches!(upper.as_str(), "1" | "YES" | "TRUE")diff --git a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rsdeleted file mode 100644index 25a1e42c8..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs+++ /dev/null@@ -1,24 +0,0 @@-use super::gemm;-use crate::{- DataType,- backends::{- common::kernel::matmul::{MatmulArguments, MatmulDispatchDescriptor, MatmulError, gemv, split_k},- metal::{Metal, context::MetalContext},- },-};--pub fn choose_dispatch_descriptor(- context: &MetalContext,- data_type: DataType,- arguments: &MatmulArguments<Metal>,-) -> Result<MatmulDispatchDescriptor, MatmulError<Metal>> {- if let Some(descriptor) = gemv::DispatchDescriptor::try_new::<Metal>(data_type, arguments)? {- return Ok(MatmulDispatchDescriptor::Gemv(descriptor));- }-- if let Some(descriptor) = split_k::DispatchDescriptor::try_new::<Metal>(data_type, arguments)? {- return Ok(MatmulDispatchDescriptor::SplitK(descriptor));- }-- Ok(MatmulDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(context, data_type, arguments)?))-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rsdeleted file mode 100644index 009c1808d..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs+++ /dev/null@@ -1,75 +0,0 @@-// Re-export the common DispatchDescriptor type — we add a constructor here-pub use crate::backends::common::kernel::matmul::gemm::DispatchDescriptor;-use crate::{- DataType,- backends::{- common::{- gpu_types::GEMMParams,- kernel::matmul::{GridSize, MatmulArguments, MatmulError, gemm::Specialization},- },- metal::{Metal, context::MetalContext},- },-};--impl DispatchDescriptor {- pub fn new(- context: &MetalContext,- data_type: DataType,- arguments: &MatmulArguments<Metal>,- ) -> Result<Self, MatmulError<Metal>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));- }-- let config = Specialization::select(context, data_type, arguments);-- let m = arguments.batch;- let n = arguments.output_dim;- let k = arguments.input_dim;-- let tiles_n = (n + config.block_cols - 1) / config.block_cols;- let tiles_m = (m + config.block_rows - 1) / config.block_rows;- let swizzle_log = config.swizzle_log2;-- let tile_swizzle = 1 << swizzle_log;- let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;- let tn_swizzled = tiles_n * tile_swizzle;-- let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);- let elements_per_matrix_b = if arguments.transpose_b {- (arguments.output_dim as i64) * (arguments.ldb as i64)- } else {- (arguments.input_dim as i64) * (arguments.ldb as i64)- };- let elements_per_matrix_d = (arguments.batch as i64) * (arguments.ldd as i64);-- let params = GEMMParams {- M: m,- N: n,- K: k,- lda: arguments.lda,- ldb: arguments.ldb,- ldd: arguments.ldd,- tiles_n,- tiles_m,- batch_stride_a: elements_per_matrix_a,- batch_stride_b: elements_per_matrix_b,- batch_stride_d: elements_per_matrix_d,- swizzle_log,- gemm_k_iterations_aligned: k / config.block_depth,- batch_ndim: 1,- };-- let threadgroups = GridSize {- x: tn_swizzled as usize,- y: tm_swizzled as usize,- z: arguments.batch_count as usize,- };-- Ok(Self {- specialization: config,- params,- threadgroups,- })- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rsdeleted file mode 100644index d9cb5df12..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs+++ /dev/null@@ -1,4 +0,0 @@-mod dispatch_descriptor;-mod specialization;--pub use dispatch_descriptor::DispatchDescriptor;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsdeleted file mode 100644index a7ffc547a..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ /dev/null@@ -1,83 +0,0 @@-use crate::{- DataType,- backends::{- common::kernel::matmul::{MatmulArguments, gemm::Specialization},- metal::{- Metal,- context::{DeviceClass, MetalContext},- },- },-};--impl Specialization {- pub fn select(- context: &MetalContext,- data_type: DataType,- arguments: &MatmulArguments<Metal>,- ) -> Self {- let overall_work_elements =- (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);- let is_float32 = matches!(data_type, DataType::F32);- let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();-- let (block_rows, block_cols, block_depth, warps_per_row, warps_per_col, swizzle_log2) =- match context.device_class() {- DeviceClass::Integrated | DeviceClass::Phone | DeviceClass::Unknown(_) => {- if prefer_half_or_tf32 {- if arguments.transpose_b {- (64, 32, 32, 2, 2, 0)- } else {- (64, 64, 16, 2, 2, 0)- }- } else if arguments.transpose_b {- (32, 64, 16, 2, 2, 0)- } else {- (64, 32, 32, 2, 2, 0)- }- },- DeviceClass::Desktop => {- if overall_work_elements >= (1_i64 << 20) {- if prefer_half_or_tf32 {- if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {- (64, 64, 16, 2, 2, 0)- } else if arguments.transpose_b {- (64, 32, 32, 2, 2, 0)- } else {- (32, 64, 16, 2, 2, 0)- }- } else if arguments.transpose_b {- (32, 64, 16, 2, 2, 0)- } else {- (64, 32, 32, 2, 2, 0)- }- } else if prefer_half_or_tf32 {- if arguments.transpose_b {- (64, 32, 32, 2, 2, 0)- } else {- (64, 64, 16, 2, 2, 0)- }- } else if arguments.transpose_b {- (32, 64, 16, 2, 2, 0)- } else {- (64, 32, 32, 2, 2, 0)- }- },- };-- let m = arguments.batch;- let n = arguments.output_dim;- let k = arguments.input_dim;-- Self {- block_rows,- block_cols,- block_depth,- warps_per_row,- warps_per_col,- swizzle_log2,- align_m: (m % block_rows) == 0,- align_n: (n % block_cols) == 0,- align_k: (k % block_depth) == 0,- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsdeleted file mode 100644index f39fd382c..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ /dev/null@@ -1,62 +0,0 @@-mod dispatch_descriptor;-mod gemm;--pub use dispatch_descriptor::choose_dispatch_descriptor;--use crate::{- DataType,- backends::{- common::{- Backend, CommandBuffer,- kernel::matmul::{- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,- MatmulArguments, MatmulError, MatmulKernel,- },- },- metal::{Metal, context::MetalContext},- },-};--impl FullPrecisionMatmulKernelTrait for MatmulKernel<Metal> {- type Backend = Metal;-- fn new(- context: &MetalContext,- data_type: DataType,- ) -> Result<Self, MatmulError<Metal>> {- let mut kernel = MatmulKernel::new(data_type)?;- kernel.precompile(context)?;- Ok(kernel)- }-- fn encode(- &mut self,- context: &MetalContext,- encoder: &mut <<Metal as Backend>::CommandBuffer as CommandBuffer>::Encoding,- arguments: FullPrecisionMatmulArguments<Metal>,- ) {- let mut matmul_arguments = MatmulArguments {- a: arguments.a,- a_offset: arguments.a_offset as u64,- b: arguments.b,- d: arguments.output,- bias: arguments.bias,- batch: arguments.batch as i32,- input_dim: arguments.input_dim as i32,- output_dim: arguments.output_dim as i32,- lda: arguments.input_dim as i32,- ldb: arguments.input_dim as i32,- ldd: arguments.output_dim as i32,- batch_count: 1,- transpose_b: true,- };-- MatmulKernel::<Metal>::apply_batch_collapse(&mut matmul_arguments);-- let descriptor = choose_dispatch_descriptor(context, self.data_type, &matmul_arguments)- .expect("Failed to create dispatch descriptor for full precision matmul");-- self.encode_with_descriptor(context, matmul_arguments, &descriptor, encoder)- .expect("Failed to encode full precision matmul kernel");- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 021380535..429f084ad 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -7,8 +7,6 @@ pub mod dsl {}pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));-pub mod matmul;-impl MatmulKernels for dsl::MetalKernels {type FullPrecisionMatmulKernel = MatmulKernel<Metal>;}diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 2a7ef3c41..47b00fc47 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -8,4 +8,3 @@ mod kernel;mod metal_extensions;pub use backend::Metal;-pub use kernel::matmul::choose_dispatch_descriptor;diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rsindex c18a5712d..ad393c6b5 100644--- a/crates/uzu/src/utils/mod.rs+++ b/crates/uzu/src/utils/mod.rs@@ -1,6 +1,7 @@pub mod attention;pub mod env_utils;pub mod model_size;+pub mod pointers;pub mod storage;pub mod version;diff --git a/crates/uzu/src/utils/pointers.rs b/crates/uzu/src/utils/pointers.rsnew file mode 100644index 000000000..9f623c59d--- /dev/null+++ b/crates/uzu/src/utils/pointers.rs@@ -0,0 +1,19 @@+pub struct SendPtr<T>(pub *const T);+unsafe impl<T> Send for SendPtr<T> {}+unsafe impl<T> Sync for SendPtr<T> {}+impl<T> Clone for SendPtr<T> {+ fn clone(&self) -> Self {+ *self+ }+}+impl<T> Copy for SendPtr<T> {}++pub struct SendPtrMut<T>(pub *mut T);+unsafe impl<T> Send for SendPtrMut<T> {}+unsafe impl<T> Sync for SendPtrMut<T> {}+impl<T> Clone for SendPtrMut<T> {+ fn clone(&self) -> Self {+ *self+ }+}+impl<T> Copy for SendPtrMut<T> {}diff --git a/crates/uzu/tests/kernel/matmul/gemv_test.rs b/crates/uzu/tests/kernel/matmul/gemv_test.rsindex cdd7a4f79..617b2da41 100644--- a/crates/uzu/tests/kernel/matmul/gemv_test.rs+++ b/crates/uzu/tests/kernel/matmul/gemv_test.rs@@ -12,7 +12,10 @@ use uzu::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context,- kernel::matmul::{MatmulArguments, gemv},+ kernel::matmul::{+ MatmulArguments,+ gemv::{GemvDispatchDescriptor, GemvKernel},+ },},cpu::Cpu,},@@ -82,11 +85,11 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {transpose_b: true,};- let descriptor = gemv::DispatchDescriptor::try_new::<B>(T::data_type(), &arguments)+ let descriptor = GemvDispatchDescriptor::try_new::<B>(T::data_type(), &arguments).expect("Failed to create descriptor").expect("GEMV not applicable for these dimensions");- let mut kernel = gemv::GemvKernel::<B>::new(T::data_type()).expect("Failed to create GemvKernel");+ let mut kernel = GemvKernel::<B>::new(T::data_type()).expect("Failed to create GemvKernel");let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel.encode(&context, &mut arguments, &descriptor, &mut command_buffer).expect("Failed to encode");diff --git a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rsindex 30e7e2a39..e12e3edbd 100644--- a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs+++ b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs@@ -13,7 +13,10 @@ use uzu::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context,gpu_types::GEMMParams,- kernel::matmul::{GridSize, MatmulArguments, gemm},+ kernel::matmul::{+ GridSize, MatmulArguments,+ gemm::{GemmDispatchDescriptor, GemmKernel, GemmSpecialization},+ },},cpu::Cpu,},@@ -61,7 +64,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let d_array = context.create_array_uninitialized(&[input.m, input.n], T::data_type(), "");// Pick a precompile config matching alignment requirements- let configs = gemm::Specialization::precompile_configs(T::data_type());+ let configs = GemmSpecialization::precompile_configs(T::data_type());let base = configs[0];let align_m = (m % base.block_rows) == 0;let align_n = (n % base.block_cols) == 0;@@ -71,7 +74,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.iter().find(|c| c.align_m == align_m && c.align_n == align_n && c.align_k == align_k).copied()- .unwrap_or(gemm::Specialization {+ .unwrap_or(GemmSpecialization {align_m,align_n,align_k,@@ -98,7 +101,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {batch_ndim: 1,};- let descriptor = gemm::DispatchDescriptor {+ let descriptor = GemmDispatchDescriptor {specialization: config,params,threadgroups: GridSize {@@ -108,7 +111,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {},};- let mut kernel = gemm::GemmKernel::<B>::new(T::data_type()).expect("Failed to create GemmKernel");+ let mut kernel = GemmKernel::<B>::new(T::data_type()).expect("Failed to create GemmKernel");let a_buf = a_array.buffer();let a_ref = a_buf.borrow();diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 0e1723233..a6da65c83 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -10,9 +10,9 @@ use uzu::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context,- kernel::matmul::{MatmulArguments, MatmulKernel},+ kernel::matmul::{MatmulArguments, MatmulKernel, choose_matmul_dispatch_descriptor},},- metal::{Metal, choose_dispatch_descriptor},+ metal::Metal,},};@@ -63,7 +63,8 @@ fn run_metal_matmul(transpose_b,};MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);- let descriptor = choose_dispatch_descriptor(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");+ let descriptor =+ choose_matmul_dispatch_descriptor::<Metal>(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut command_buffer).expect("encode");command_buffer.end_encoding().submit().wait_until_completed().unwrap();