Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Generalize and parallelize matmul kernel (#253)

Aleksandr Golokoz9e6fe067b2dd
diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rs
index d4fb3fd05..264f8ed4e 100644
--- a/crates/uzu/src/backends/common/context.rs
+++ b/crates/uzu/src/backends/common/context.rs
@@ -1,4 +1,4 @@
-use std::{path::Path, rc::Rc};
+use std::{env, path::Path, rc::Rc};
use super::Backend;
use crate::backends::common::CommandBuffer;
@@ -18,6 +18,21 @@ impl DeviceClass {
}
}
+/// Device performance class based on the last character of architecture name.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DeviceType {
+ Phone, // 'p' - iPhone/iPad integrated
+ Integrated, // 'g' - Mac integrated GPU
+ Desktop, // 'd' - Mac Pro/Max discrete-class GPU
+ Unknown(char),
+}
+
+impl DeviceType {
+ pub fn is_high_performance(&self) -> bool {
+ matches!(self, Self::Desktop)
+ }
+}
+
pub trait Context: Sized {
type Backend: Backend<Context = Self>;
@@ -25,6 +40,8 @@ pub trait Context: Sized {
fn device_class(&self) -> DeviceClass;
+ fn device_type(&self) -> DeviceType;
+
fn debug_active(&self) -> bool;
fn create_command_buffer(
@@ -46,4 +63,8 @@ pub trait Context: Sized {
) -> Result<(), <Self::Backend as Backend>::Error>;
fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;
+
+ fn tf32_enabled() -> bool {
+ env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)
+ }
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs
index cb79a4815..5d6b3fbb5 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs
@@ -1,10 +1,19 @@
-use super::{gemm, gemv, split_k};
+use super::{MatmulArguments, MatmulError};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend,
+ kernel::matmul::{
+ gemm::GemmDispatchDescriptor, gemv::GemvDispatchDescriptor, split_k::SplitKDispatchDescriptor,
+ },
+ },
+};
#[derive(Debug, Clone)]
pub enum MatmulDispatchDescriptor {
- Gemv(gemv::DispatchDescriptor),
- SplitK(split_k::DispatchDescriptor),
- Gemm(gemm::DispatchDescriptor),
+ Gemv(GemvDispatchDescriptor),
+ SplitK(SplitKDispatchDescriptor),
+ Gemm(GemmDispatchDescriptor),
}
impl MatmulDispatchDescriptor {
@@ -15,3 +24,19 @@ impl MatmulDispatchDescriptor {
}
}
}
+
+pub fn choose_matmul_dispatch_descriptor<B: Backend>(
+ context: &B::Context,
+ data_type: DataType,
+ arguments: &MatmulArguments<B>,
+) -> Result<MatmulDispatchDescriptor, MatmulError<B>> {
+ if let Some(descriptor) = GemvDispatchDescriptor::try_new::<B>(data_type, arguments)? {
+ return Ok(MatmulDispatchDescriptor::Gemv(descriptor));
+ }
+
+ if let Some(descriptor) = SplitKDispatchDescriptor::try_new::<B>(data_type, arguments)? {
+ return Ok(MatmulDispatchDescriptor::SplitK(descriptor));
+ }
+
+ Ok(MatmulDispatchDescriptor::Gemm(GemmDispatchDescriptor::try_new::<B>(context, data_type, arguments)?))
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
index ec91a6d71..af3a17afb 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
@@ -1,7 +1,9 @@
-use super::{MatmulError, MatmulKernels};
+use super::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels};
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer},
+ backends::common::{
+ Backend, CommandBuffer, kernel::matmul::dispatch_descriptor::choose_matmul_dispatch_descriptor,
+ },
};
pub struct FullPrecisionMatmulArguments<'a, B: Backend> {
@@ -30,3 +32,49 @@ pub trait FullPrecisionMatmulKernel: Sized {
arguments: FullPrecisionMatmulArguments<Self::Backend>,
);
}
+
+impl<B: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>> FullPrecisionMatmulKernel
+ for MatmulKernel<B>
+{
+ type Backend = B;
+
+ fn new(
+ context: &<Self::Backend as Backend>::Context,
+ data_type: DataType,
+ ) -> Result<Self, MatmulError<Self::Backend>> {
+ let mut kernel = MatmulKernel::new(data_type)?;
+ kernel.precompile(context)?;
+ Ok(kernel)
+ }
+
+ fn encode(
+ &mut self,
+ context: &<Self::Backend as Backend>::Context,
+ command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
+ arguments: FullPrecisionMatmulArguments<Self::Backend>,
+ ) {
+ let mut matmul_arguments = MatmulArguments {
+ a: arguments.a,
+ a_offset: arguments.a_offset as u64,
+ b: arguments.b,
+ d: arguments.output,
+ bias: arguments.bias,
+ batch: arguments.batch as i32,
+ input_dim: arguments.input_dim as i32,
+ output_dim: arguments.output_dim as i32,
+ lda: arguments.input_dim as i32,
+ ldb: arguments.input_dim as i32,
+ ldd: arguments.output_dim as i32,
+ batch_count: 1,
+ transpose_b: true,
+ };
+
+ MatmulKernel::<B>::apply_batch_collapse(&mut matmul_arguments);
+
+ let descriptor = choose_matmul_dispatch_descriptor(context, self.data_type, &matmul_arguments)
+ .expect("Failed to create dispatch descriptor for full precision matmul");
+
+ self.encode_with_descriptor(context, matmul_arguments, &descriptor, command_buffer)
+ .expect("Failed to encode full precision matmul kernel");
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
index 44e1d380b..2355e4d90 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
@@ -1,9 +1,79 @@
-use super::{super::grid_size::GridSize, specialization::Specialization};
-use crate::backends::common::gpu_types::GEMMParams;
+use super::{super::grid_size::GridSize, specialization::GemmSpecialization};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend,
+ gpu_types::GEMMParams,
+ kernel::matmul::{MatmulArguments, MatmulError},
+ },
+};
#[derive(Debug, Clone)]
-pub struct DispatchDescriptor {
- pub specialization: Specialization,
+pub struct GemmDispatchDescriptor {
+ pub specialization: GemmSpecialization,
pub params: GEMMParams,
pub threadgroups: GridSize,
}
+
+impl GemmDispatchDescriptor {
+ pub fn try_new<B: Backend>(
+ context: &B::Context,
+ data_type: DataType,
+ arguments: &MatmulArguments<B>,
+ ) -> Result<Self, MatmulError<B>> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MatmulError::UnsupportedDataType(data_type));
+ }
+
+ let config = GemmSpecialization::select::<B>(context, data_type, arguments);
+
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ let k = arguments.input_dim;
+
+ let tiles_n = (n + config.block_cols - 1) / config.block_cols;
+ let tiles_m = (m + config.block_rows - 1) / config.block_rows;
+ let swizzle_log = config.swizzle_log2;
+
+ let tile_swizzle = 1 << swizzle_log;
+ let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
+ let tn_swizzled = tiles_n * tile_swizzle;
+
+ let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);
+ let elements_per_matrix_b = if arguments.transpose_b {
+ (arguments.output_dim as i64) * (arguments.ldb as i64)
+ } else {
+ (arguments.input_dim as i64) * (arguments.ldb as i64)
+ };
+ let elements_per_matrix_d = (arguments.batch as i64) * (arguments.ldd as i64);
+
+ let params = GEMMParams {
+ M: m,
+ N: n,
+ K: k,
+ lda: arguments.lda,
+ ldb: arguments.ldb,
+ ldd: arguments.ldd,
+ tiles_n,
+ tiles_m,
+ batch_stride_a: elements_per_matrix_a,
+ batch_stride_b: elements_per_matrix_b,
+ batch_stride_d: elements_per_matrix_d,
+ swizzle_log,
+ gemm_k_iterations_aligned: k / config.block_depth,
+ batch_ndim: 1,
+ };
+
+ let threadgroups = GridSize {
+ x: tn_swizzled as usize,
+ y: tm_swizzled as usize,
+ z: arguments.batch_count as usize,
+ };
+
+ Ok(Self {
+ specialization: config,
+ params,
+ threadgroups,
+ })
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
index ca3df3b48..6ee8898eb 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
@@ -1,7 +1,8 @@
use std::{collections::HashMap, ops::DerefMut};
use super::{
- super::matmul_arguments::MatmulArguments, dispatch_descriptor::DispatchDescriptor, specialization::Specialization,
+ super::matmul_arguments::MatmulArguments, dispatch_descriptor::GemmDispatchDescriptor,
+ specialization::GemmSpecialization,
};
use crate::{
DataType,
@@ -13,7 +14,7 @@ use crate::{
pub struct GemmKernel<B: Backend> {
data_type: DataType,
- pipelines: HashMap<Specialization, <B::Kernels as Kernels>::MatmulGemmKernel>,
+ pipelines: HashMap<GemmSpecialization, <B::Kernels as Kernels>::MatmulGemmKernel>,
}
impl<B: Backend> GemmKernel<B> {
@@ -31,7 +32,7 @@ impl<B: Backend> GemmKernel<B> {
&mut self,
context: &B::Context,
) -> Result<(), MatmulError<B>> {
- for &config in Specialization::precompile_configs(self.data_type) {
+ for &config in GemmSpecialization::precompile_configs(self.data_type) {
self.get_or_create_kernel(context, config)?;
}
Ok(())
@@ -40,7 +41,7 @@ impl<B: Backend> GemmKernel<B> {
fn get_or_create_kernel(
&mut self,
context: &B::Context,
- config: Specialization,
+ config: GemmSpecialization,
) -> Result<&<B::Kernels as Kernels>::MatmulGemmKernel, MatmulError<B>> {
if !self.pipelines.contains_key(&config) {
let kernel = <B::Kernels as Kernels>::MatmulGemmKernel::new(
@@ -65,7 +66,7 @@ impl<B: Backend> GemmKernel<B> {
&mut self,
context: &B::Context,
arguments: &mut MatmulArguments<B>,
- dispatch_descriptor: &DispatchDescriptor,
+ dispatch_descriptor: &GemmDispatchDescriptor,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
let config = dispatch_descriptor.specialization;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs
index a4cbe5d36..9250445f6 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs
@@ -2,6 +2,6 @@ mod dispatch_descriptor;
mod kernel;
pub mod specialization;
-pub use dispatch_descriptor::DispatchDescriptor;
+pub use dispatch_descriptor::GemmDispatchDescriptor;
pub use kernel::GemmKernel;
-pub use specialization::Specialization;
+pub use specialization::GemmSpecialization;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
index 40051d5db..4e0249017 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
@@ -1,7 +1,10 @@
-use crate::DataType;
+use crate::{
+ DataType,
+ backends::common::{Backend, Context, context::DeviceType, kernel::matmul::MatmulArguments},
+};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct Specialization {
+pub struct GemmSpecialization {
pub block_rows: i32,
pub block_cols: i32,
pub block_depth: i32,
@@ -13,8 +16,8 @@ pub struct Specialization {
pub align_k: bool,
}
-impl Specialization {
- pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
+impl GemmSpecialization {
+ pub fn precompile_configs(data_type: DataType) -> &'static [Self] {
match data_type {
DataType::BF16 => &[
Self {
@@ -135,4 +138,75 @@ impl Specialization {
_ => &[],
}
}
+
+ pub fn select<B: Backend>(
+ context: &B::Context,
+ data_type: DataType,
+ arguments: &MatmulArguments<B>,
+ ) -> Self {
+ let overall_work_elements =
+ (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);
+ let is_float32 = matches!(data_type, DataType::F32);
+ let prefer_half_or_tf32 = !is_float32 || B::Context::tf32_enabled();
+
+ let (block_rows, block_cols, block_depth, warps_per_row, warps_per_col, swizzle_log2) =
+ match context.device_type() {
+ DeviceType::Integrated | DeviceType::Phone | DeviceType::Unknown(_) => {
+ if prefer_half_or_tf32 {
+ if arguments.transpose_b {
+ (64, 32, 32, 2, 2, 0)
+ } else {
+ (64, 64, 16, 2, 2, 0)
+ }
+ } else if arguments.transpose_b {
+ (32, 64, 16, 2, 2, 0)
+ } else {
+ (64, 32, 32, 2, 2, 0)
+ }
+ },
+ DeviceType::Desktop => {
+ if overall_work_elements >= (1_i64 << 20) {
+ if prefer_half_or_tf32 {
+ if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {
+ (64, 64, 16, 2, 2, 0)
+ } else if arguments.transpose_b {
+ (64, 32, 32, 2, 2, 0)
+ } else {
+ (32, 64, 16, 2, 2, 0)
+ }
+ } else if arguments.transpose_b {
+ (32, 64, 16, 2, 2, 0)
+ } else {
+ (64, 32, 32, 2, 2, 0)
+ }
+ } else if prefer_half_or_tf32 {
+ if arguments.transpose_b {
+ (64, 32, 32, 2, 2, 0)
+ } else {
+ (64, 64, 16, 2, 2, 0)
+ }
+ } else if arguments.transpose_b {
+ (32, 64, 16, 2, 2, 0)
+ } else {
+ (64, 32, 32, 2, 2, 0)
+ }
+ },
+ };
+
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ let k = arguments.input_dim;
+
+ Self {
+ block_rows,
+ block_cols,
+ block_depth,
+ warps_per_row,
+ warps_per_col,
+ swizzle_log2,
+ align_m: (m % block_rows) == 0,
+ align_n: (n % block_cols) == 0,
+ align_k: (k % block_depth) == 0,
+ }
+ }
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
index 58132550c..d8e350f83 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
@@ -23,7 +23,7 @@ pub enum OutputSource {
}
#[derive(Debug, Clone)]
-pub struct DispatchDescriptor {
+pub struct GemvDispatchDescriptor {
pub specialization: Specialization,
pub matrix_is_rhs: bool,
pub output_source: OutputSource,
@@ -40,7 +40,7 @@ pub struct DispatchDescriptor {
pub batch_rows: i32,
}
-impl DispatchDescriptor {
+impl GemvDispatchDescriptor {
pub fn try_new<B: Backend>(
data_type: DataType,
arguments: &MatmulArguments<B>,
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
index e4046f267..b7f28209e 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
@@ -2,7 +2,7 @@ use std::{collections::HashMap, ops::DerefMut};
use super::{
super::matmul_arguments::MatmulArguments,
- dispatch_descriptor::{DispatchDescriptor, OutputSource},
+ dispatch_descriptor::{GemvDispatchDescriptor, OutputSource},
specialization::Specialization,
};
use crate::{
@@ -66,7 +66,7 @@ impl<B: Backend> GemvKernel<B> {
&mut self,
context: &B::Context,
arguments: &mut MatmulArguments<B>,
- dispatch_descriptor: &DispatchDescriptor,
+ dispatch_descriptor: &GemvDispatchDescriptor,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
let config = dispatch_descriptor.specialization;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs
index eba171b69..4fe67a542 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs
@@ -2,5 +2,5 @@ mod dispatch_descriptor;
mod kernel;
mod specialization;
-pub use dispatch_descriptor::DispatchDescriptor;
+pub use dispatch_descriptor::GemvDispatchDescriptor;
pub use kernel::GemvKernel;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
index 267ea32a5..47baac2e7 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
@@ -7,7 +7,7 @@ mod kernel;
mod matmul_arguments;
pub mod split_k;
-pub use dispatch_descriptor::MatmulDispatchDescriptor;
+pub use dispatch_descriptor::{MatmulDispatchDescriptor, choose_matmul_dispatch_descriptor};
pub use full_precision::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel};
pub use grid_size::GridSize;
pub use kernel::MatmulKernel;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs
index db0641fb9..664fbf10a 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/dispatch_descriptor.rs
@@ -9,7 +9,7 @@ use crate::{
};
#[derive(Debug, Clone)]
-pub struct DispatchDescriptor {
+pub struct SplitKDispatchDescriptor {
pub params: SplitKGEMMParams,
pub partition_count: i32,
pub output_elements_per_partition: i32,
@@ -18,7 +18,7 @@ pub struct DispatchDescriptor {
pub accum_total_threads: GridSize,
}
-impl DispatchDescriptor {
+impl SplitKDispatchDescriptor {
pub fn try_new<B: Backend>(
data_type: DataType,
arguments: &MatmulArguments<B>,
diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
index 08cd9b9d2..4763402bb 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
@@ -1,6 +1,6 @@
use std::ops::{Deref, DerefMut};
-use super::{super::matmul_arguments::MatmulArguments, dispatch_descriptor::DispatchDescriptor};
+use super::{super::matmul_arguments::MatmulArguments, dispatch_descriptor::SplitKDispatchDescriptor};
use crate::{
DataType,
backends::common::{
@@ -67,7 +67,7 @@ impl<B: Backend> SplitKKernel<B> {
&mut self,
context: &B::Context,
arguments: &mut MatmulArguments<B>,
- dispatch_descriptor: &DispatchDescriptor,
+ dispatch_descriptor: &SplitKDispatchDescriptor,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
self.ensure_kernels(context)?;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs
index 0fcefe437..8c1b0f372 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/mod.rs
@@ -3,5 +3,5 @@ mod kernel;
mod specialization;
mod tile_configuration;
-pub use dispatch_descriptor::DispatchDescriptor;
+pub use dispatch_descriptor::SplitKDispatchDescriptor;
pub use kernel::SplitKKernel;
diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rs
index b98c4bdfa..eee34aca0 100644
--- a/crates/uzu/src/backends/common/mod.rs
+++ b/crates/uzu/src/backends/common/mod.rs
@@ -12,6 +12,6 @@ pub use command_buffer::{
CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
CommandBufferPending,
};
-pub use context::{Context, DeviceClass};
+pub use context::{Context, DeviceClass, DeviceType};
pub use event::Event;
pub use kernel::Kernels;
diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rs
index 79c450bfc..1fd61d943 100644
--- a/crates/uzu/src/backends/cpu/context.rs
+++ b/crates/uzu/src/backends/cpu/context.rs
@@ -1,7 +1,7 @@
use std::{cell::RefCell, rc::Rc};
use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};
-use crate::backends::common::{Context, DeviceClass};
+use crate::backends::common::{Context, DeviceClass, DeviceType};
pub struct CpuContext;
@@ -47,4 +47,8 @@ impl Context for CpuContext {
fn stop_capture(&self) -> Result<(), CpuError> {
Err(CpuError::NotSupported)
}
+
+ fn device_type(&self) -> DeviceType {
+ DeviceType::Integrated
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
index 0c9f8585e..453477ad5 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
@@ -2,7 +2,125 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{
+ ArrayElement,
+ pointers::{SendPtr, SendPtrMut},
+};
+
+#[inline(always)]
+unsafe fn matmul_gemm_multi_thread_compute_row<T: ArrayElement + Float>(
+ a: SendPtr<T>,
+ b: SendPtr<T>,
+ d: SendPtrMut<T>,
+ row: usize,
+ n: usize,
+ k: usize,
+ lda: usize,
+ ldb: usize,
+ ldd: usize,
+) {
+ unsafe {
+ for col in 0..n {
+ let mut acc: f32 = 0.0;
+ for i in 0..k {
+ let a_val = *a.0.add(row * lda + i);
+ let b_val = *b.0.add(col * ldb + i);
+ acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();
+ }
+ *d.0.add(row * ldd + col) = T::from(acc).unwrap();
+ }
+ }
+}
+
+fn matmul_gemm_multi_thread<T: ArrayElement + Float>(
+ a: *const T,
+ b: *const T,
+ d: *mut T,
+ batches_count: usize,
+ m: usize,
+ n: usize,
+ k: usize,
+ batch_stride_a: usize,
+ batch_stride_b: usize,
+ batch_stride_d: usize,
+ lda: usize,
+ ldb: usize,
+ ldd: usize,
+) {
+ let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
+ unsafe {
+ for batch in 0..batches_count {
+ let a_batch = SendPtr(a.add(batch * batch_stride_a));
+ let b_batch = SendPtr(b.add(batch * batch_stride_b));
+ let d_batch = SendPtrMut(d.add(batch * batch_stride_d));
+
+ std::thread::scope(|s| {
+ let rows_per_thread = m / num_threads;
+ let rows_per_thread_remainder = m % num_threads;
+ let mut start = 0;
+
+ for t in 0..num_threads {
+ let chunk_add = if t < rows_per_thread_remainder {
+ 1
+ } else {
+ 0
+ };
+ let chunk = rows_per_thread + chunk_add;
+
+ let end = start + chunk;
+ let ab = a_batch;
+ let bb = b_batch;
+ let db = d_batch;
+ s.spawn(move || unsafe {
+ for row in start..end {
+ matmul_gemm_multi_thread_compute_row(ab, bb, db, row, n, k, lda, ldb, ldd);
+ }
+ });
+
+ start = end;
+ }
+ });
+ }
+ }
+}
+
+fn matmul_gemm_single_thread<T: ArrayElement + Float>(
+ a: *const T,
+ b: *const T,
+ d: *mut T,
+ batches_count: usize,
+ m: usize,
+ n: usize,
+ k: usize,
+ batch_stride_a: usize,
+ batch_stride_b: usize,
+ batch_stride_d: usize,
+ lda: usize,
+ ldb: usize,
+ ldd: usize,
+) {
+ unsafe {
+ for batch in 0..batches_count {
+ let a_batch = a.add(batch * batch_stride_a);
+ let b_batch = b.add(batch * batch_stride_b);
+ let d_batch = d.add(batch * batch_stride_d);
+
+ for row in 0..m {
+ for col in 0..n {
+ let mut acc: f32 = 0.0;
+ for i in 0..k {
+ // A[row, i]: row-major with leading dimension lda
+ let a_val = *a_batch.add(row * lda + i);
+ // B[col, i]: B is transposed, so B is [N, K] row-major with leading dimension ldb
+ let b_val = *b_batch.add(col * ldb + i);
+ acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();
+ }
+ *d_batch.add(row * ldd + col) = T::from(acc).unwrap();
+ }
+ }
+ }
+ }
+}
#[kernel(MatmulGemm)]
#[variants(T, f32, f16, bf16)]
@@ -39,30 +157,23 @@ pub fn matmul_gemm<T: ArrayElement + Float>(
#[specialize]
align_k: bool,
) {
+ let batches_count = group_count_z as usize;
unsafe {
let p = &*params.as_ptr();
- let lda = p.lda as usize;
- let ldb = p.ldb as usize;
- let ldd = p.ldd as usize;
-
- for batch in 0..group_count_z as usize {
- let a_batch = a.add(batch * (p.batch_stride_a as usize));
- let b_batch = b.add(batch * (p.batch_stride_b as usize));
- let d_batch = d.add(batch * (p.batch_stride_d as usize));
-
- for row in 0..p.M as usize {
- for col in 0..p.N as usize {
- let mut acc: f32 = 0.0;
- for i in 0..p.K as usize {
- // A[row, i]: row-major with leading dimension lda
- let a_val = *a_batch.add(row * lda + i);
- // B[col, i]: B is transposed, so B is [N, K] row-major with leading dimension ldb
- let b_val = *b_batch.add(col * ldb + i);
- acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();
- }
- *d_batch.add(row * ldd + col) = T::from(acc).unwrap();
- }
- }
- }
+ matmul_gemm_multi_thread::<T>(
+ a,
+ b,
+ d,
+ group_count_z as usize,
+ p.M as usize,
+ p.N as usize,
+ p.K as usize,
+ p.batch_stride_a as usize,
+ p.batch_stride_b as usize,
+ p.batch_stride_d as usize,
+ p.lda as usize,
+ p.ldb as usize,
+ p.ldd as usize,
+ );
}
}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs
index cf071f264..2febb91d5 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs
@@ -2,7 +2,167 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{
+ ArrayElement,
+ pointers::{SendPtr, SendPtrMut},
+};
+
+#[inline(always)]
+unsafe fn compute_gemv_rows<T: ArrayElement + Float>(
+ mat: SendPtr<T>,
+ vec_row: SendPtr<T>,
+ out_row: SendPtrMut<T>,
+ src: Option<SendPtr<T>>,
+ out_start: usize,
+ out_end: usize,
+ in_dim: usize,
+ mat_ld: usize,
+ output_scale: f32,
+ output_accumulate_scale: f32,
+ output_source_stride: usize,
+ apply_output_scale_and_accumulate: bool,
+) {
+ unsafe {
+ for out_idx in out_start..out_end {
+ let mat_row = mat.0.add(out_idx * mat_ld);
+
+ let mut acc: f32 = 0.0;
+ for in_idx in 0..in_dim {
+ acc += (*mat_row.add(in_idx)).to_f32().unwrap() * (*vec_row.0.add(in_idx)).to_f32().unwrap();
+ }
+
+ if apply_output_scale_and_accumulate {
+ if let Some(s) = src {
+ let src_val = (*s.0.add(out_idx * output_source_stride)).to_f32().unwrap();
+ acc = output_scale * acc + output_accumulate_scale * src_val;
+ }
+ }
+
+ *out_row.0.add(out_idx) = T::from(acc).unwrap();
+ }
+ }
+}
+
+fn matmul_gemv_multi_thread<T: ArrayElement + Float>(
+ matrix: *const T,
+ input_vector: *const T,
+ output_source: Option<*const T>,
+ output_vector: *mut T,
+ input_dimension: usize,
+ output_dimension: usize,
+ matrix_leading_dimension: usize,
+ output_scale: f32,
+ output_accumulate_scale: f32,
+ batches_count: usize,
+ vector_batch_stride: &[i32],
+ matrix_batch_stride: &[i32],
+ output_source_batch_stride: &[i32],
+ output_source_stride: usize,
+ batch_rows: usize,
+ apply_output_scale_and_accumulate: bool,
+) {
+ let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
+ unsafe {
+ for outer_batch in 0..batches_count as usize {
+ let batch_vec = input_vector.add(outer_batch * vector_batch_stride[0] as usize);
+ let batch_mat = SendPtr(matrix.add(outer_batch * matrix_batch_stride[0] as usize));
+ let batch_out = SendPtrMut(output_vector.add(outer_batch * batch_rows * output_dimension));
+ let batch_src =
+ output_source.map(|src| SendPtr(src.add(outer_batch * output_source_batch_stride[0] as usize)));
+
+ for batch_row in 0..batch_rows {
+ let vec_row = SendPtr(batch_vec.add(batch_row * input_dimension));
+ let out_row = SendPtrMut(batch_out.0.add(batch_row * output_dimension));
+
+ std::thread::scope(|s| {
+ let rows_per_thread = output_dimension / num_threads;
+ let rows_per_thread_remainder = output_dimension % num_threads;
+ let mut start = 0;
+
+ for t in 0..num_threads {
+ let chunk = rows_per_thread
+ + if t < rows_per_thread_remainder {
+ 1
+ } else {
+ 0
+ };
+ let end = start + chunk;
+
+ s.spawn(move || unsafe {
+ compute_gemv_rows(
+ batch_mat,
+ vec_row,
+ out_row,
+ batch_src,
+ start,
+ end,
+ input_dimension,
+ matrix_leading_dimension,
+ output_scale,
+ output_accumulate_scale,
+ output_source_stride,
+ apply_output_scale_and_accumulate,
+ );
+ });
+
+ start = end;
+ }
+ });
+ }
+ }
+ }
+}
+
+fn matmul_gemv_single_thread<T: ArrayElement + Float>(
+ matrix: *const T,
+ input_vector: *const T,
+ output_source: Option<*const T>,
+ output_vector: *mut T,
+ input_dimension: usize,
+ output_dimension: usize,
+ matrix_leading_dimension: usize,
+ output_scale: f32,
+ output_accumulate_scale: f32,
+ batches_count: usize,
+ vector_batch_stride: &[i32],
+ matrix_batch_stride: &[i32],
+ output_source_batch_stride: &[i32],
+ output_source_stride: usize,
+ batch_rows: usize,
+ apply_output_scale_and_accumulate: bool,
+) {
+ unsafe {
+ for outer_batch in 0..batches_count {
+ let batch_vec = input_vector.add(outer_batch * vector_batch_stride[0] as usize);
+ let batch_mat = matrix.add(outer_batch * matrix_batch_stride[0] as usize);
+ let batch_out = output_vector.add(outer_batch * batch_rows * output_dimension);
+ let batch_src = output_source.map(|src| src.add(outer_batch * output_source_batch_stride[0] as usize));
+
+ for batch_row in 0..batch_rows {
+ let vec_row = batch_vec.add(batch_row * input_dimension);
+ let out_row = batch_out.add(batch_row * output_dimension);
+
+ for out_idx in 0..output_dimension {
+ let mat_row = batch_mat.add(out_idx * matrix_leading_dimension);
+
+ let mut acc: f32 = 0.0;
+ for in_idx in 0..input_dimension {
+ acc += (*mat_row.add(in_idx)).to_f32().unwrap() * (*vec_row.add(in_idx)).to_f32().unwrap();
+ }
+
+ if apply_output_scale_and_accumulate {
+ if let Some(src) = batch_src {
+ let src_val = (*src.add(out_idx * output_source_stride as usize)).to_f32().unwrap();
+ acc = output_scale * acc + output_accumulate_scale * src_val;
+ }
+ }
+
+ *out_row.add(out_idx) = T::from(acc).unwrap();
+ }
+ }
+ }
+ }
+}
#[kernel(MatmulGemv)]
#[variants(T, f32, f16, bf16)]
@@ -43,41 +203,22 @@ pub fn matmul_gemv<T: ArrayElement + Float>(
thread_out_cols: u32,
#[specialize] apply_output_scale_and_accumulate: bool,
) {
- let in_dim = input_dimension as usize;
- let out_dim = output_dimension as usize;
- let mat_ld = matrix_leading_dimension as usize;
- let num_batch_rows = batch_rows as usize;
-
- unsafe {
- for outer_batch in 0..batch_shape[0] as usize {
- let batch_vec = input_vector.add(outer_batch * vector_batch_stride[0] as usize);
- let batch_mat = matrix.add(outer_batch * matrix_batch_stride[0] as usize);
- let batch_out = output_vector.add(outer_batch * num_batch_rows * out_dim);
-
- let batch_src = output_source.map(|src| src.add(outer_batch * output_source_batch_stride[0] as usize));
-
- for batch_row in 0..num_batch_rows {
- let vec_row = batch_vec.add(batch_row * in_dim);
- let out_row = batch_out.add(batch_row * out_dim);
-
- for out_idx in 0..out_dim {
- let mat_row = batch_mat.add(out_idx * mat_ld);
-
- let mut acc: f32 = 0.0;
- for in_idx in 0..in_dim {
- acc += (*mat_row.add(in_idx)).to_f32().unwrap() * (*vec_row.add(in_idx)).to_f32().unwrap();
- }
-
- if apply_output_scale_and_accumulate {
- if let Some(src) = batch_src {
- let src_val = (*src.add(out_idx * output_source_stride as usize)).to_f32().unwrap();
- acc = output_scale * acc + output_accumulate_scale * src_val;
- }
- }
-
- *out_row.add(out_idx) = T::from(acc).unwrap();
- }
- }
- }
- }
+ matmul_gemv_multi_thread::<T>(
+ matrix,
+ input_vector,
+ output_source,
+ output_vector,
+ input_dimension as usize,
+ output_dimension as usize,
+ matrix_leading_dimension as usize,
+ output_scale,
+ output_accumulate_scale,
+ batch_shape[0] as usize,
+ vector_batch_stride,
+ matrix_batch_stride,
+ output_source_batch_stride,
+ output_source_stride as usize,
+ batch_rows as usize,
+ apply_output_scale_and_accumulate,
+ );
}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
index 26abe007d..bc6efa06f 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
@@ -1,3 +1,12 @@
+use crate::backends::{
+ common::kernel::matmul::{MatmulKernel, MatmulKernels},
+ cpu::{Cpu, kernel::CpuKernels},
+};
+
pub mod gemm;
pub mod gemv;
pub mod split_k;
+
+impl MatmulKernels for CpuKernels {
+ type FullPrecisionMatmulKernel = MatmulKernel<Cpu>;
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rs
index 49d1e7de5..287488d31 100644
--- a/crates/uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs
@@ -23,40 +23,3 @@ mod tensor_copy;
mod token_copy;
include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));
-
-use crate::{
- DataType,
- backends::{
- common::kernel::matmul::{
- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait, MatmulError,
- MatmulKernels,
- },
- cpu::{Cpu, command_buffer::CpuCommandBuffer, context::CpuContext},
- },
-};
-
-pub struct FullPrecisionMatmulCpuKernel;
-
-impl FullPrecisionMatmulKernelTrait for FullPrecisionMatmulCpuKernel {
- type Backend = Cpu;
-
- fn new(
- #[allow(unused)] context: &CpuContext,
- #[allow(unused)] data_type: DataType,
- ) -> Result<Self, MatmulError<Cpu>> {
- Ok(Self)
- }
-
- fn encode(
- &mut self,
- #[allow(unused)] context: &CpuContext,
- #[allow(unused)] encoder: &mut CpuCommandBuffer,
- #[allow(unused)] arguments: FullPrecisionMatmulArguments<Cpu>,
- ) {
- encoder.push_command(move || todo!());
- }
-}
-
-impl MatmulKernels for CpuKernels {
- type FullPrecisionMatmulKernel = FullPrecisionMatmulCpuKernel;
-}
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index 69e10c1ed..2625b7390 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -1,4 +1,4 @@
-use std::{cell::RefCell, collections::HashMap, env, rc::Rc};
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
use metal::{
MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue, MTLCommandQueueExt,
@@ -9,7 +9,7 @@ use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{Metal, error::MetalError, kernel, metal_extensions::LibraryPipelineExtensions};
use crate::backends::{
- common::{Context, DeviceClass as CommonDeviceClass},
+ common::{Context, DeviceClass as CommonDeviceClass, DeviceType},
metal::command_buffer::MetalCommandBufferInitial,
};
@@ -40,26 +40,11 @@ impl DeviceGeneration {
}
}
-/// Device performance class based on the last character of architecture name.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DeviceClass {
- Phone, // 'p' - iPhone/iPad integrated
- Integrated, // 'g' - Mac integrated GPU
- Desktop, // 'd' - Mac Pro/Max discrete-class GPU
- Unknown(char),
-}
-
-impl DeviceClass {
- pub fn is_high_performance(&self) -> bool {
- matches!(self, Self::Desktop)
- }
-}
-
/// Complete device architecture information.
#[derive(Debug, Clone)]
pub struct DeviceArchitecture {
pub generation: DeviceGeneration,
- pub device_class: DeviceClass,
+ pub device_class: DeviceType,
#[allow(dead_code)]
pub arch_string: String,
}
@@ -79,7 +64,7 @@ impl DeviceArchitecture {
}
}
- fn parse_architecture_info(arch: &str) -> (DeviceGeneration, DeviceClass) {
+ fn parse_architecture_info(arch: &str) -> (DeviceGeneration, DeviceType) {
// Parse generation and class from device name (e.g. "Apple M3 Pro")
let generation = if arch.contains("M5") {
DeviceGeneration::Gen18
@@ -100,11 +85,11 @@ impl DeviceArchitecture {
};
let device_class = if arch.contains("Max") || arch.contains("Ultra") || arch.contains("Pro") {
- DeviceClass::Desktop
+ DeviceType::Desktop
} else if arch.contains("iPhone") || arch.contains("iPad") {
- DeviceClass::Phone
+ DeviceType::Phone
} else {
- DeviceClass::Integrated
+ DeviceType::Integrated
};
(generation, device_class)
@@ -148,16 +133,6 @@ impl MetalContext {
self.architecture.generation
}
- /// Returns the device class for performance tuning.
- pub fn device_class(&self) -> DeviceClass {
- self.architecture.device_class
- }
-
- /// TF32 toggle via UZU_TF32 environment variable.
- pub fn tf32_enabled(&self) -> bool {
- env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)
- }
-
pub fn compute_pipeline_state(
&self,
cache_key: &str,
@@ -217,6 +192,11 @@ impl Context for MetalContext {
}
}
+ /// Returns the device class for performance tuning.
+ fn device_type(&self) -> DeviceType {
+ self.architecture.device_class
+ }
+
fn debug_active(&self) -> bool {
let upper = std::env::var("METAL_DEVICE_WRAPPER_TYPE").unwrap_or_default().to_ascii_uppercase();
matches!(upper.as_str(), "1" | "YES" | "TRUE")
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
deleted file mode 100644
index 25a1e42c8..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
+++ /dev/null
@@ -1,24 +0,0 @@
-use super::gemm;
-use crate::{
- DataType,
- backends::{
- common::kernel::matmul::{MatmulArguments, MatmulDispatchDescriptor, MatmulError, gemv, split_k},
- metal::{Metal, context::MetalContext},
- },
-};
-
-pub fn choose_dispatch_descriptor(
- context: &MetalContext,
- data_type: DataType,
- arguments: &MatmulArguments<Metal>,
-) -> Result<MatmulDispatchDescriptor, MatmulError<Metal>> {
- if let Some(descriptor) = gemv::DispatchDescriptor::try_new::<Metal>(data_type, arguments)? {
- return Ok(MatmulDispatchDescriptor::Gemv(descriptor));
- }
-
- if let Some(descriptor) = split_k::DispatchDescriptor::try_new::<Metal>(data_type, arguments)? {
- return Ok(MatmulDispatchDescriptor::SplitK(descriptor));
- }
-
- Ok(MatmulDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(context, data_type, arguments)?))
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
deleted file mode 100644
index 009c1808d..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
+++ /dev/null
@@ -1,75 +0,0 @@
-// Re-export the common DispatchDescriptor type — we add a constructor here
-pub use crate::backends::common::kernel::matmul::gemm::DispatchDescriptor;
-use crate::{
- DataType,
- backends::{
- common::{
- gpu_types::GEMMParams,
- kernel::matmul::{GridSize, MatmulArguments, MatmulError, gemm::Specialization},
- },
- metal::{Metal, context::MetalContext},
- },
-};
-
-impl DispatchDescriptor {
- pub fn new(
- context: &MetalContext,
- data_type: DataType,
- arguments: &MatmulArguments<Metal>,
- ) -> Result<Self, MatmulError<Metal>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
- }
-
- let config = Specialization::select(context, data_type, arguments);
-
- let m = arguments.batch;
- let n = arguments.output_dim;
- let k = arguments.input_dim;
-
- let tiles_n = (n + config.block_cols - 1) / config.block_cols;
- let tiles_m = (m + config.block_rows - 1) / config.block_rows;
- let swizzle_log = config.swizzle_log2;
-
- let tile_swizzle = 1 << swizzle_log;
- let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
- let tn_swizzled = tiles_n * tile_swizzle;
-
- let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);
- let elements_per_matrix_b = if arguments.transpose_b {
- (arguments.output_dim as i64) * (arguments.ldb as i64)
- } else {
- (arguments.input_dim as i64) * (arguments.ldb as i64)
- };
- let elements_per_matrix_d = (arguments.batch as i64) * (arguments.ldd as i64);
-
- let params = GEMMParams {
- M: m,
- N: n,
- K: k,
- lda: arguments.lda,
- ldb: arguments.ldb,
- ldd: arguments.ldd,
- tiles_n,
- tiles_m,
- batch_stride_a: elements_per_matrix_a,
- batch_stride_b: elements_per_matrix_b,
- batch_stride_d: elements_per_matrix_d,
- swizzle_log,
- gemm_k_iterations_aligned: k / config.block_depth,
- batch_ndim: 1,
- };
-
- let threadgroups = GridSize {
- x: tn_swizzled as usize,
- y: tm_swizzled as usize,
- z: arguments.batch_count as usize,
- };
-
- Ok(Self {
- specialization: config,
- params,
- threadgroups,
- })
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
deleted file mode 100644
index d9cb5df12..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-mod dispatch_descriptor;
-mod specialization;
-
-pub use dispatch_descriptor::DispatchDescriptor;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
deleted file mode 100644
index a7ffc547a..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ /dev/null
@@ -1,83 +0,0 @@
-use crate::{
- DataType,
- backends::{
- common::kernel::matmul::{MatmulArguments, gemm::Specialization},
- metal::{
- Metal,
- context::{DeviceClass, MetalContext},
- },
- },
-};
-
-impl Specialization {
- pub fn select(
- context: &MetalContext,
- data_type: DataType,
- arguments: &MatmulArguments<Metal>,
- ) -> Self {
- let overall_work_elements =
- (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);
- let is_float32 = matches!(data_type, DataType::F32);
- let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();
-
- let (block_rows, block_cols, block_depth, warps_per_row, warps_per_col, swizzle_log2) =
- match context.device_class() {
- DeviceClass::Integrated | DeviceClass::Phone | DeviceClass::Unknown(_) => {
- if prefer_half_or_tf32 {
- if arguments.transpose_b {
- (64, 32, 32, 2, 2, 0)
- } else {
- (64, 64, 16, 2, 2, 0)
- }
- } else if arguments.transpose_b {
- (32, 64, 16, 2, 2, 0)
- } else {
- (64, 32, 32, 2, 2, 0)
- }
- },
- DeviceClass::Desktop => {
- if overall_work_elements >= (1_i64 << 20) {
- if prefer_half_or_tf32 {
- if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {
- (64, 64, 16, 2, 2, 0)
- } else if arguments.transpose_b {
- (64, 32, 32, 2, 2, 0)
- } else {
- (32, 64, 16, 2, 2, 0)
- }
- } else if arguments.transpose_b {
- (32, 64, 16, 2, 2, 0)
- } else {
- (64, 32, 32, 2, 2, 0)
- }
- } else if prefer_half_or_tf32 {
- if arguments.transpose_b {
- (64, 32, 32, 2, 2, 0)
- } else {
- (64, 64, 16, 2, 2, 0)
- }
- } else if arguments.transpose_b {
- (32, 64, 16, 2, 2, 0)
- } else {
- (64, 32, 32, 2, 2, 0)
- }
- },
- };
-
- let m = arguments.batch;
- let n = arguments.output_dim;
- let k = arguments.input_dim;
-
- Self {
- block_rows,
- block_cols,
- block_depth,
- warps_per_row,
- warps_per_col,
- swizzle_log2,
- align_m: (m % block_rows) == 0,
- align_n: (n % block_cols) == 0,
- align_k: (k % block_depth) == 0,
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
deleted file mode 100644
index f39fd382c..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ /dev/null
@@ -1,62 +0,0 @@
-mod dispatch_descriptor;
-mod gemm;
-
-pub use dispatch_descriptor::choose_dispatch_descriptor;
-
-use crate::{
- DataType,
- backends::{
- common::{
- Backend, CommandBuffer,
- kernel::matmul::{
- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,
- MatmulArguments, MatmulError, MatmulKernel,
- },
- },
- metal::{Metal, context::MetalContext},
- },
-};
-
-impl FullPrecisionMatmulKernelTrait for MatmulKernel<Metal> {
- type Backend = Metal;
-
- fn new(
- context: &MetalContext,
- data_type: DataType,
- ) -> Result<Self, MatmulError<Metal>> {
- let mut kernel = MatmulKernel::new(data_type)?;
- kernel.precompile(context)?;
- Ok(kernel)
- }
-
- fn encode(
- &mut self,
- context: &MetalContext,
- encoder: &mut <<Metal as Backend>::CommandBuffer as CommandBuffer>::Encoding,
- arguments: FullPrecisionMatmulArguments<Metal>,
- ) {
- let mut matmul_arguments = MatmulArguments {
- a: arguments.a,
- a_offset: arguments.a_offset as u64,
- b: arguments.b,
- d: arguments.output,
- bias: arguments.bias,
- batch: arguments.batch as i32,
- input_dim: arguments.input_dim as i32,
- output_dim: arguments.output_dim as i32,
- lda: arguments.input_dim as i32,
- ldb: arguments.input_dim as i32,
- ldd: arguments.output_dim as i32,
- batch_count: 1,
- transpose_b: true,
- };
-
- MatmulKernel::<Metal>::apply_batch_collapse(&mut matmul_arguments);
-
- let descriptor = choose_dispatch_descriptor(context, self.data_type, &matmul_arguments)
- .expect("Failed to create dispatch descriptor for full precision matmul");
-
- self.encode_with_descriptor(context, matmul_arguments, &descriptor, encoder)
- .expect("Failed to encode full precision matmul kernel");
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 021380535..429f084ad 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -7,8 +7,6 @@ pub mod dsl {
}
pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
-pub mod matmul;
-
impl MatmulKernels for dsl::MetalKernels {
type FullPrecisionMatmulKernel = MatmulKernel<Metal>;
}
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 2a7ef3c41..47b00fc47 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -8,4 +8,3 @@ mod kernel;
mod metal_extensions;
pub use backend::Metal;
-pub use kernel::matmul::choose_dispatch_descriptor;
diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rs
index c18a5712d..ad393c6b5 100644
--- a/crates/uzu/src/utils/mod.rs
+++ b/crates/uzu/src/utils/mod.rs
@@ -1,6 +1,7 @@
pub mod attention;
pub mod env_utils;
pub mod model_size;
+pub mod pointers;
pub mod storage;
pub mod version;
diff --git a/crates/uzu/src/utils/pointers.rs b/crates/uzu/src/utils/pointers.rs
new file mode 100644
index 000000000..9f623c59d
--- /dev/null
+++ b/crates/uzu/src/utils/pointers.rs
@@ -0,0 +1,19 @@
+pub struct SendPtr<T>(pub *const T);
+unsafe impl<T> Send for SendPtr<T> {}
+unsafe impl<T> Sync for SendPtr<T> {}
+impl<T> Clone for SendPtr<T> {
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+impl<T> Copy for SendPtr<T> {}
+
+pub struct SendPtrMut<T>(pub *mut T);
+unsafe impl<T> Send for SendPtrMut<T> {}
+unsafe impl<T> Sync for SendPtrMut<T> {}
+impl<T> Clone for SendPtrMut<T> {
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+impl<T> Copy for SendPtrMut<T> {}
diff --git a/crates/uzu/tests/kernel/matmul/gemv_test.rs b/crates/uzu/tests/kernel/matmul/gemv_test.rs
index cdd7a4f79..617b2da41 100644
--- a/crates/uzu/tests/kernel/matmul/gemv_test.rs
+++ b/crates/uzu/tests/kernel/matmul/gemv_test.rs
@@ -12,7 +12,10 @@ use uzu::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context,
- kernel::matmul::{MatmulArguments, gemv},
+ kernel::matmul::{
+ MatmulArguments,
+ gemv::{GemvDispatchDescriptor, GemvKernel},
+ },
},
cpu::Cpu,
},
@@ -82,11 +85,11 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
transpose_b: true,
};
- let descriptor = gemv::DispatchDescriptor::try_new::<B>(T::data_type(), &arguments)
+ let descriptor = GemvDispatchDescriptor::try_new::<B>(T::data_type(), &arguments)
.expect("Failed to create descriptor")
.expect("GEMV not applicable for these dimensions");
- let mut kernel = gemv::GemvKernel::<B>::new(T::data_type()).expect("Failed to create GemvKernel");
+ let mut kernel = GemvKernel::<B>::new(T::data_type()).expect("Failed to create GemvKernel");
let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel.encode(&context, &mut arguments, &descriptor, &mut command_buffer).expect("Failed to encode");
diff --git a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
index 30e7e2a39..e12e3edbd 100644
--- a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
+++ b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
@@ -13,7 +13,10 @@ use uzu::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context,
gpu_types::GEMMParams,
- kernel::matmul::{GridSize, MatmulArguments, gemm},
+ kernel::matmul::{
+ GridSize, MatmulArguments,
+ gemm::{GemmDispatchDescriptor, GemmKernel, GemmSpecialization},
+ },
},
cpu::Cpu,
},
@@ -61,7 +64,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let d_array = context.create_array_uninitialized(&[input.m, input.n], T::data_type(), "");
// Pick a precompile config matching alignment requirements
- let configs = gemm::Specialization::precompile_configs(T::data_type());
+ let configs = GemmSpecialization::precompile_configs(T::data_type());
let base = configs[0];
let align_m = (m % base.block_rows) == 0;
let align_n = (n % base.block_cols) == 0;
@@ -71,7 +74,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.iter()
.find(|c| c.align_m == align_m && c.align_n == align_n && c.align_k == align_k)
.copied()
- .unwrap_or(gemm::Specialization {
+ .unwrap_or(GemmSpecialization {
align_m,
align_n,
align_k,
@@ -98,7 +101,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
batch_ndim: 1,
};
- let descriptor = gemm::DispatchDescriptor {
+ let descriptor = GemmDispatchDescriptor {
specialization: config,
params,
threadgroups: GridSize {
@@ -108,7 +111,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
},
};
- let mut kernel = gemm::GemmKernel::<B>::new(T::data_type()).expect("Failed to create GemmKernel");
+ let mut kernel = GemmKernel::<B>::new(T::data_type()).expect("Failed to create GemmKernel");
let a_buf = a_array.buffer();
let a_ref = a_buf.borrow();
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 0e1723233..a6da65c83 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -10,9 +10,9 @@ use uzu::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context,
- kernel::matmul::{MatmulArguments, MatmulKernel},
+ kernel::matmul::{MatmulArguments, MatmulKernel, choose_matmul_dispatch_descriptor},
},
- metal::{Metal, choose_dispatch_descriptor},
+ metal::Metal,
},
};
@@ -63,7 +63,8 @@ fn run_metal_matmul(
transpose_b,
};
MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);
- let descriptor = choose_dispatch_descriptor(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");
+ let descriptor =
+ choose_matmul_dispatch_descriptor::<Metal>(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");
kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut command_buffer).expect("encode");
command_buffer.end_encoding().submit().wait_until_completed().unwrap();