Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

move matmul impl details to metal from common (#262)

eugenee19fc3c2eb82
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index b5be8941d..528554320 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -315,6 +315,12 @@ pub fn bindgen(
(quote! {}, quote! {}, None)
};
+ let method_visibility = if kernel.public {
+ quote! {}
+ } else {
+ quote! { pub(crate) }
+ };
+
let kernel = quote! {
pub struct #struct_name {
pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
@@ -324,14 +330,14 @@ pub fn bindgen(
impl #maybe_trait_impl #struct_name {
#maybe_associate_backend
- fn new(context: &MetalContext #(, #variants_extra_arguments)* #(, #specialize_args)*) -> Result<Self, MetalError> {
+ #method_visibility fn new(context: &MetalContext #(, #variants_extra_arguments)* #(, #specialize_args)*) -> Result<Self, MetalError> {
let entry_name = #entry_name;
#function_constants_init
let pipeline = context.compute_pipeline_state(#cache_key, &entry_name, #function_constants_arg)?;
Ok(Self { pipeline #(, #conditional_buffer_sets)* })
}
- fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#encode_args_defs, )* command_buffer: &'command_buffer mut MetalCommandBufferEncoding) {
+ #method_visibility fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#encode_args_defs, )* command_buffer: &'command_buffer mut MetalCommandBufferEncoding) {
let compute_encoder = command_buffer.ensure_compute();
#empty_dispatch_guards
compute_encoder.set_compute_pipeline_state(&self.pipeline);
diff --git a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs
deleted file mode 100644
index cde615975..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/dispatch_descriptor.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-use super::{MatmulArguments, MatmulError};
-use crate::{
- DataType,
- backends::common::{
- Backend,
- kernel::matmul::{gemm::GemmDispatchDescriptor, gemv::GemvDispatchDescriptor},
- },
-};
-
-#[derive(Debug, Clone)]
-pub enum MatmulDispatchDescriptor {
- Gemv(GemvDispatchDescriptor),
- Gemm(GemmDispatchDescriptor),
-}
-
-impl MatmulDispatchDescriptor {
- pub fn bias_is_fused(&self) -> bool {
- match self {
- MatmulDispatchDescriptor::Gemv(d) => d.bias_is_fused(),
- MatmulDispatchDescriptor::Gemm(_) => false,
- }
- }
-}
-
-pub fn choose_matmul_dispatch_descriptor<B: Backend>(
- context: &B::Context,
- data_type: DataType,
- arguments: &MatmulArguments<B>,
-) -> Result<MatmulDispatchDescriptor, MatmulError<B>> {
- if let Some(descriptor) = GemvDispatchDescriptor::try_new::<B>(data_type, arguments)? {
- return Ok(MatmulDispatchDescriptor::Gemv(descriptor));
- }
-
- Ok(MatmulDispatchDescriptor::Gemm(GemmDispatchDescriptor::try_new::<B>(context, data_type, arguments)?))
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
deleted file mode 100644
index 5cf2ad669..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use super::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels};
-use crate::{
- DataType,
- backends::common::{
- Backend, CommandBuffer, kernel::matmul::dispatch_descriptor::choose_matmul_dispatch_descriptor,
- },
-};
-
-pub struct FullPrecisionMatmulArguments<'a, B: Backend> {
- pub a: &'a B::Buffer,
- pub a_offset: usize,
- pub b: &'a B::Buffer,
- pub output: &'a mut B::Buffer,
- pub bias: Option<&'a B::Buffer>,
- pub batch: usize,
- pub input_dim: usize,
- pub output_dim: usize,
-}
-
-pub trait FullPrecisionMatmulKernel: Sized {
- type Backend: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>;
-
- fn new(
- context: &<Self::Backend as Backend>::Context,
- data_type: DataType,
- ) -> Result<Self, MatmulError<Self::Backend>>;
-
- fn encode(
- &mut self,
- context: &<Self::Backend as Backend>::Context,
- command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
- arguments: FullPrecisionMatmulArguments<Self::Backend>,
- );
-}
-
-impl<B: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>> FullPrecisionMatmulKernel
- for MatmulKernel<B>
-{
- type Backend = B;
-
- fn new(
- context: &<Self::Backend as Backend>::Context,
- data_type: DataType,
- ) -> Result<Self, MatmulError<Self::Backend>> {
- let mut kernel = MatmulKernel::new(data_type)?;
- kernel.precompile(context)?;
- Ok(kernel)
- }
-
- fn encode(
- &mut self,
- context: &<Self::Backend as Backend>::Context,
- command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
- arguments: FullPrecisionMatmulArguments<Self::Backend>,
- ) {
- let matmul_arguments = MatmulArguments {
- a: arguments.a,
- a_offset: arguments.a_offset as u64,
- b: arguments.b,
- d: arguments.output,
- bias: arguments.bias,
- batch: arguments.batch as i32,
- input_dim: arguments.input_dim as i32,
- output_dim: arguments.output_dim as i32,
- leading_dimension_a: arguments.input_dim as i32,
- leading_dimension_b: arguments.input_dim as i32,
- leading_dimension_d: arguments.output_dim as i32,
- transpose_b: true,
- };
-
- let descriptor = choose_matmul_dispatch_descriptor(context, self.data_type, &matmul_arguments)
- .expect("Failed to create dispatch descriptor for full precision matmul");
-
- self.encode_with_descriptor(context, matmul_arguments, &descriptor, command_buffer)
- .expect("Failed to encode full precision matmul kernel");
- }
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
deleted file mode 100644
index 21ce06389..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-use super::{super::grid_size::GridSize, specialization::GemmSpecialization};
-use crate::{
- DataType,
- backends::common::{
- Backend,
- gpu_types::GemmParams,
- kernel::matmul::{MatmulArguments, MatmulError},
- },
-};
-
-#[derive(Debug, Clone)]
-pub struct GemmDispatchDescriptor {
- pub specialization: GemmSpecialization,
- pub params: GemmParams,
- pub threadgroups: GridSize,
-}
-
-impl GemmDispatchDescriptor {
- pub fn try_new<B: Backend>(
- context: &B::Context,
- data_type: DataType,
- arguments: &MatmulArguments<B>,
- ) -> Result<Self, MatmulError<B>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
- }
-
- let config = GemmSpecialization::select::<B>(context, data_type, arguments);
-
- let m = arguments.batch;
- let n = arguments.output_dim;
- let k = arguments.input_dim;
-
- let threadgroups_per_row = (n + config.block_cols - 1) / config.block_cols;
- let threadgroups_per_column = (m + config.block_rows - 1) / config.block_rows;
- let swizzle_log = config.swizzle_log2;
-
- let swizzle_stride = 1_i32 << swizzle_log;
- let tm_swizzled = (threadgroups_per_column + swizzle_stride - 1) / swizzle_stride;
- let tn_swizzled = threadgroups_per_row * swizzle_stride;
-
- let params = GemmParams {
- M: m,
- N: n,
- K: k,
- leading_dimension_a: arguments.leading_dimension_a,
- leading_dimension_b: arguments.leading_dimension_b,
- leading_dimension_d: arguments.leading_dimension_d,
- threadgroups_per_row,
- threadgroups_per_column,
- swizzle_log,
- aligned_inner_iterations: k / config.block_depth,
- };
-
- let threadgroups = GridSize {
- x: tn_swizzled as usize,
- y: tm_swizzled as usize,
- z: 1,
- };
-
- Ok(Self {
- specialization: config,
- params,
- threadgroups,
- })
- }
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
deleted file mode 100644
index a1502a1bd..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
+++ /dev/null
@@ -1,92 +0,0 @@
-use std::{collections::HashMap, ops::DerefMut};
-
-use super::{
- super::matmul_arguments::MatmulArguments, dispatch_descriptor::GemmDispatchDescriptor,
- specialization::GemmSpecialization,
-};
-use crate::{
- DataType,
- backends::common::{
- Backend, CommandBuffer, Kernels,
- kernel::{MatmulGemmKernel, matmul::MatmulError},
- },
-};
-
-pub struct GemmKernel<B: Backend> {
- data_type: DataType,
- pipelines: HashMap<GemmSpecialization, <B::Kernels as Kernels>::MatmulGemmKernel>,
-}
-
-impl<B: Backend> GemmKernel<B> {
- pub fn new(data_type: DataType) -> Result<Self, MatmulError<B>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
- }
- Ok(Self {
- data_type,
- pipelines: HashMap::new(),
- })
- }
-
- pub fn precompile(
- &mut self,
- context: &B::Context,
- ) -> Result<(), MatmulError<B>> {
- for &config in GemmSpecialization::precompile_configs(self.data_type) {
- self.get_or_create_kernel(context, config)?;
- }
- Ok(())
- }
-
- fn get_or_create_kernel(
- &mut self,
- context: &B::Context,
- config: GemmSpecialization,
- ) -> Result<&<B::Kernels as Kernels>::MatmulGemmKernel, MatmulError<B>> {
- if !self.pipelines.contains_key(&config) {
- let kernel = <B::Kernels as Kernels>::MatmulGemmKernel::new(
- context,
- self.data_type,
- config.block_rows as u32,
- config.block_cols as u32,
- config.block_depth as u32,
- config.simdgroups_per_row as u32,
- config.simdgroups_per_column as u32,
- config.align_m,
- config.align_n,
- config.align_k,
- )
- .map_err(MatmulError::BackendError)?;
- self.pipelines.insert(config, kernel);
- }
- Ok(self.pipelines.get(&config).unwrap())
- }
-
- pub fn encode(
- &mut self,
- context: &B::Context,
- arguments: &mut MatmulArguments<B>,
- dispatch_descriptor: &GemmDispatchDescriptor,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), MatmulError<B>> {
- let config = dispatch_descriptor.specialization;
-
- let group_count_x = u32::try_from(dispatch_descriptor.threadgroups.x)
- .map_err(|_| MatmulError::<B>::ThreadgroupOverflow(dispatch_descriptor.threadgroups.x))?;
- let group_count_y = u32::try_from(dispatch_descriptor.threadgroups.y)
- .map_err(|_| MatmulError::<B>::ThreadgroupOverflow(dispatch_descriptor.threadgroups.y))?;
-
- let pipeline = self.get_or_create_kernel(context, config)?;
- pipeline.encode(
- (arguments.a, arguments.a_offset as usize),
- arguments.b,
- arguments.d.deref_mut(),
- std::slice::from_ref(&dispatch_descriptor.params),
- group_count_x,
- group_count_y,
- command_buffer,
- );
-
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs
deleted file mode 100644
index 9250445f6..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/mod.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-mod dispatch_descriptor;
-mod kernel;
-pub mod specialization;
-
-pub use dispatch_descriptor::GemmDispatchDescriptor;
-pub use kernel::GemmKernel;
-pub use specialization::GemmSpecialization;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
deleted file mode 100644
index e45ac57bf..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-use std::sync::OnceLock;
-
-use super::{super::matmul_arguments::MatmulArguments, specialization::Specialization};
-use crate::{
- DataType,
- backends::common::{Backend, kernel::matmul::MatmulError},
-};
-
-const DEFAULT_GEMV_MAX_BATCH: i32 = 8;
-
-static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();
-
-fn max_gemv_batch_threshold() -> i32 {
- *GEMV_MAX_BATCH.get_or_init(|| {
- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
- })
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum OutputSource {
- None,
- Bias,
-}
-
-#[derive(Debug, Clone)]
-pub struct GemvDispatchDescriptor {
- pub specialization: Specialization,
- pub matrix_is_rhs: bool,
- pub output_source: OutputSource,
- pub input_dimension: i32,
- pub output_dimension: i32,
- pub matrix_leading_dim: i32,
- pub alpha: f32,
- pub beta: f32,
- pub bias_stride: i32,
- pub batch_rows: i32,
-}
-
-impl GemvDispatchDescriptor {
- pub fn try_new<B: Backend>(
- data_type: DataType,
- arguments: &MatmulArguments<B>,
- ) -> Result<Option<Self>, MatmulError<B>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
- }
-
- if !arguments.transpose_b {
- return Ok(None);
- }
-
- let m = arguments.batch;
- let n = arguments.output_dim;
-
- let max_gemv_batch = max_gemv_batch_threshold();
-
- if n == 1 {
- if m != 1 {
- return Ok(None);
- }
- } else if m > max_gemv_batch {
- return Ok(None);
- }
-
- let matrix_is_rhs = n != 1;
- let transpose_matrix = if matrix_is_rhs {
- !arguments.transpose_b
- } else {
- false
- };
-
- let output_source = if arguments.bias.is_some() {
- OutputSource::Bias
- } else {
- OutputSource::None
- };
-
- let (apply_output_scale_and_accumulate, alpha, beta, bias_stride) = match output_source {
- OutputSource::None => (false, 1.0f32, 0.0f32, 0),
- OutputSource::Bias => (true, 1.0f32, 1.0f32, 1),
- };
-
- let output_dimension = if matrix_is_rhs {
- arguments.output_dim
- } else {
- arguments.batch
- };
-
- let specialization = Specialization::select(
- transpose_matrix,
- arguments.input_dim,
- output_dimension,
- apply_output_scale_and_accumulate,
- );
-
- let input_dimension = arguments.input_dim;
- let matrix_leading_dim = if matrix_is_rhs {
- arguments.leading_dimension_b
- } else {
- arguments.leading_dimension_a
- };
-
- let batch_rows = arguments.batch;
-
- Ok(Some(Self {
- specialization,
- matrix_is_rhs,
- output_source,
- input_dimension,
- output_dimension,
- matrix_leading_dim,
- alpha,
- beta,
- bias_stride,
- batch_rows,
- }))
- }
-
- pub fn bias_is_fused(&self) -> bool {
- matches!(self.output_source, OutputSource::Bias)
- }
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
deleted file mode 100644
index 92a59669e..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
+++ /dev/null
@@ -1,115 +0,0 @@
-use std::{collections::HashMap, ops::DerefMut};
-
-use super::{
- super::matmul_arguments::MatmulArguments,
- dispatch_descriptor::{GemvDispatchDescriptor, OutputSource},
- specialization::Specialization,
-};
-use crate::{
- DataType,
- backends::common::{
- Backend, CommandBuffer, Kernels,
- kernel::{MatmulGemvKernel, matmul::MatmulError},
- },
-};
-
-pub struct GemvKernel<B: Backend> {
- data_type: DataType,
- pipelines: HashMap<Specialization, <B::Kernels as Kernels>::MatmulGemvKernel>,
-}
-
-impl<B: Backend> GemvKernel<B> {
- pub fn new(data_type: DataType) -> Result<Self, MatmulError<B>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
- }
- Ok(Self {
- data_type,
- pipelines: HashMap::new(),
- })
- }
-
- pub fn precompile(
- &mut self,
- context: &B::Context,
- ) -> Result<(), MatmulError<B>> {
- for &config in Specialization::precompile_configs(self.data_type) {
- self.get_or_create_kernel(context, config)?;
- }
- Ok(())
- }
-
- fn get_or_create_kernel(
- &mut self,
- context: &B::Context,
- config: Specialization,
- ) -> Result<&<B::Kernels as Kernels>::MatmulGemvKernel, MatmulError<B>> {
- if !self.pipelines.contains_key(&config) {
- let kernel = <B::Kernels as Kernels>::MatmulGemvKernel::new(
- context,
- self.data_type,
- config.threadgroup_rows,
- config.threadgroup_cols,
- config.threads_per_simdgroup_row,
- config.threads_per_simdgroup_col,
- config.elements_per_thread_row,
- config.elements_per_thread_col,
- config.apply_output_scale_and_accumulate,
- )
- .map_err(MatmulError::BackendError)?;
- self.pipelines.insert(config, kernel);
- }
- Ok(self.pipelines.get(&config).unwrap())
- }
-
- pub fn encode(
- &mut self,
- context: &B::Context,
- arguments: &mut MatmulArguments<B>,
- dispatch_descriptor: &GemvDispatchDescriptor,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), MatmulError<B>> {
- let config = dispatch_descriptor.specialization;
- let pipeline = self.get_or_create_kernel(context, config)?;
-
- let (matrix, matrix_offset) = if dispatch_descriptor.matrix_is_rhs {
- (arguments.b, 0usize)
- } else {
- (arguments.a, arguments.a_offset as usize)
- };
- let (input_vector, input_vector_offset) = if dispatch_descriptor.matrix_is_rhs {
- (arguments.a, arguments.a_offset as usize)
- } else {
- (arguments.b, 0usize)
- };
-
- let output_source = if config.apply_output_scale_and_accumulate {
- match dispatch_descriptor.output_source {
- OutputSource::None => {
- return Err(MatmulError::GemvOutputSourceMismatch);
- },
- OutputSource::Bias => Some(arguments.bias.ok_or(MatmulError::<B>::GemvMissingBias)?),
- }
- } else {
- None
- };
-
- pipeline.encode(
- (matrix, matrix_offset),
- (input_vector, input_vector_offset),
- output_source.map(|buffer| (buffer, 0usize)),
- arguments.d.deref_mut(),
- dispatch_descriptor.input_dimension,
- dispatch_descriptor.output_dimension,
- dispatch_descriptor.matrix_leading_dim,
- dispatch_descriptor.alpha,
- dispatch_descriptor.beta,
- dispatch_descriptor.bias_stride,
- dispatch_descriptor.batch_rows,
- config.output_rows_per_threadgroup() as i32,
- command_buffer,
- );
-
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs
deleted file mode 100644
index 4fe67a542..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/mod.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-mod dispatch_descriptor;
-mod kernel;
-mod specialization;
-
-pub use dispatch_descriptor::GemvDispatchDescriptor;
-pub use kernel::GemvKernel;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs b/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs
deleted file mode 100644
index cee1e39fb..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/grid_size.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-#[derive(Debug, Clone, Copy)]
-pub struct GridSize {
- pub x: usize,
- pub y: usize,
- pub z: usize,
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
deleted file mode 100644
index e414f506d..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-use std::ops::DerefMut;
-
-use super::{
- MatmulError, dispatch_descriptor::MatmulDispatchDescriptor, gemm::GemmKernel, gemv::GemvKernel,
- matmul_arguments::MatmulArguments,
-};
-use crate::{
- DataType,
- backends::common::{Backend, CommandBuffer, Kernels, kernel::TensorAddBiasKernel},
-};
-
-pub struct MatmulKernel<B: Backend> {
- pub(crate) data_type: DataType,
- gemm: Option<GemmKernel<B>>,
- gemv: Option<GemvKernel<B>>,
- bias_add: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
-}
-
-impl<B: Backend> MatmulKernel<B> {
- pub fn new(data_type: DataType) -> Result<Self, MatmulError<B>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
- }
-
- Ok(Self {
- data_type,
- gemm: None,
- gemv: None,
- bias_add: None,
- })
- }
-
- pub fn precompile(
- &mut self,
- context: &B::Context,
- ) -> Result<(), MatmulError<B>> {
- let gemm = self.get_or_create_gemm()?;
- gemm.precompile(context)?;
-
- let gemv = self.get_or_create_gemv()?;
- gemv.precompile(context)?;
-
- Ok(())
- }
-
- fn get_or_create_gemm(&mut self) -> Result<&mut GemmKernel<B>, MatmulError<B>> {
- if self.gemm.is_none() {
- self.gemm = Some(GemmKernel::<B>::new(self.data_type)?);
- }
- Ok(self.gemm.as_mut().unwrap())
- }
-
- fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel<B>, MatmulError<B>> {
- if self.gemv.is_none() {
- self.gemv = Some(GemvKernel::<B>::new(self.data_type)?);
- }
- Ok(self.gemv.as_mut().unwrap())
- }
-
- fn encode_dispatch_descriptor(
- &mut self,
- context: &B::Context,
- arguments: &mut MatmulArguments<B>,
- dispatch_descriptor: &MatmulDispatchDescriptor,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), MatmulError<B>> {
- match dispatch_descriptor {
- MatmulDispatchDescriptor::Gemv(d) => {
- let gemv = self.get_or_create_gemv()?;
- gemv.encode(context, arguments, d, command_buffer)
- },
- MatmulDispatchDescriptor::Gemm(d) => {
- let gemm = self.get_or_create_gemm()?;
- gemm.encode(context, arguments, d, command_buffer)
- },
- }
- }
-
- pub fn encode_with_descriptor(
- &mut self,
- context: &B::Context,
- mut arguments: MatmulArguments<B>,
- dispatch_descriptor: &MatmulDispatchDescriptor,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), MatmulError<B>> {
- self.encode_dispatch_descriptor(context, &mut arguments, dispatch_descriptor, command_buffer)?;
-
- if let Some(bias) = arguments.bias {
- if !dispatch_descriptor.bias_is_fused() {
- self.apply_bias_add(context, &mut arguments, bias, command_buffer)?;
- }
- }
-
- Ok(())
- }
-
- fn apply_bias_add(
- &mut self,
- context: &B::Context,
- arguments: &mut MatmulArguments<B>,
- bias: &B::Buffer,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), MatmulError<B>> {
- let m = arguments.batch as usize;
- let n = arguments.output_dim as usize;
- let total_len = m * n;
- if total_len == 0 {
- return Ok(());
- }
-
- if self.bias_add.is_none() {
- self.bias_add = Some(
- <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, self.data_type, true)
- .map_err(MatmulError::BackendError)?,
- );
- }
- let bias_add = self.bias_add.as_ref().unwrap();
- bias_add.encode(None::<&B::Buffer>, bias, arguments.d.deref_mut(), n as u32, total_len as u32, command_buffer);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
index 97318dd7b..f1c793475 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
@@ -1,35 +1,40 @@
-mod dispatch_descriptor;
-mod full_precision;
-pub mod gemm;
-pub mod gemv;
-mod grid_size;
-mod kernel;
mod matmul_arguments;
-pub use dispatch_descriptor::{MatmulDispatchDescriptor, choose_matmul_dispatch_descriptor};
-pub use full_precision::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel};
-pub use grid_size::GridSize;
-pub use kernel::MatmulKernel;
pub use matmul_arguments::MatmulArguments;
use thiserror::Error;
use super::Kernels;
-use crate::backends::common::Backend;
+use crate::{
+ DataType,
+ backends::common::{Backend, CommandBuffer},
+};
pub trait MatmulKernels: Kernels {
- type FullPrecisionMatmulKernel: FullPrecisionMatmulKernel<Backend = Self::Backend>;
+ type MatmulKernel: MatmulKernel<Backend = Self::Backend>;
+}
+
+pub trait MatmulKernel: Sized {
+ type Backend: Backend<Kernels: MatmulKernels<MatmulKernel = Self>>;
+
+ fn new(
+ context: &<Self::Backend as Backend>::Context,
+ data_type: DataType,
+ ) -> Result<Self, MatmulError<Self::Backend>>;
+
+ fn encode(
+ &mut self,
+ context: &<Self::Backend as Backend>::Context,
+ arguments: MatmulArguments<Self::Backend>,
+ command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
+ );
}
#[derive(Debug, Error)]
pub enum MatmulError<B: Backend> {
#[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(crate::DataType),
+ UnsupportedDataType(DataType),
#[error("Threadgroup dimension overflows u32: {0}")]
ThreadgroupOverflow(usize),
- #[error("GEMV descriptor mismatch: apply_output_scale_and_accumulate=true but output_source=None")]
- GemvOutputSourceMismatch,
- #[error("GEMV descriptor requires bias buffer")]
- GemvMissingBias,
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
index 302d77480..f8cc265a3 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
@@ -1,5 +1,3 @@
-use dsl::kernel;
-use half::{bf16, f16};
use num_traits::Float;
use crate::{
@@ -32,7 +30,7 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
}
}
-fn matmul_gemm_impl<T: ArrayElement + Float>(
+pub fn matmul_gemm_impl<T: ArrayElement + Float>(
a: *const T,
b: *const T,
d: *mut T,
@@ -84,53 +82,3 @@ fn matmul_gemm_impl<T: ArrayElement + Float>(
});
}
}
-
-#[kernel(MatmulGemm)]
-#[variants(T, f32, f16, bf16)]
-pub fn matmul_gemm<T: ArrayElement + Float>(
- a: *const T,
- b: *const T,
- d: *mut T,
- params: &[crate::backends::common::gpu_types::matmul::GemmParams],
- #[allow(unused)] group_count_x: u32,
- #[allow(unused)] group_count_y: u32,
- #[allow(unused)]
- #[specialize]
- block_rows: u32,
- #[allow(unused)]
- #[specialize]
- block_cols: u32,
- #[allow(unused)]
- #[specialize]
- block_depth: u32,
- #[allow(unused)]
- #[specialize]
- simdgroups_per_row: u32,
- #[allow(unused)]
- #[specialize]
- simdgroups_per_column: u32,
- #[allow(unused)]
- #[specialize]
- align_m: bool,
- #[allow(unused)]
- #[specialize]
- align_n: bool,
- #[allow(unused)]
- #[specialize]
- align_k: bool,
-) {
- unsafe {
- let p = &*params.as_ptr();
- matmul_gemm_impl::<T>(
- a,
- b,
- d,
- p.M as usize,
- p.N as usize,
- p.K as usize,
- p.leading_dimension_a as usize,
- p.leading_dimension_b as usize,
- p.leading_dimension_d as usize,
- );
- }
-}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs
index fd406d0cd..22e4ec542 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs
@@ -1,5 +1,3 @@
-use dsl::kernel;
-use half::{bf16, f16};
use num_traits::Float;
use crate::{
@@ -43,7 +41,7 @@ unsafe fn compute_gemv_rows<T: ArrayElement + Float>(
}
}
-fn matmul_gemv_impl<T: ArrayElement + Float>(
+pub fn matmul_gemv_impl<T: ArrayElement + Float>(
matrix: *const T,
input_vector: *const T,
output_source: Option<*const T>,
@@ -103,54 +101,3 @@ fn matmul_gemv_impl<T: ArrayElement + Float>(
}
}
}
-
-#[kernel(MatmulGemv)]
-#[variants(T, f32, f16, bf16)]
-pub fn matmul_gemv<T: ArrayElement + Float>(
- matrix: *const T,
- input_vector: *const T,
- #[optional(apply_output_scale_and_accumulate)] output_source: Option<*const T>,
- output_vector: *mut T,
- input_dimension: i32,
- output_dimension: i32,
- matrix_leading_dimension: i32,
- output_scale: f32,
- output_accumulate_scale: f32,
- output_source_stride: i32,
- batch_rows: i32,
- #[allow(unused)] output_rows_per_threadgroup: i32,
- #[allow(unused)]
- #[specialize]
- tg_simd_rows: u32,
- #[allow(unused)]
- #[specialize]
- tg_simd_cols: u32,
- #[allow(unused)]
- #[specialize]
- sg_thread_rows: u32,
- #[allow(unused)]
- #[specialize]
- sg_thread_cols: u32,
- #[allow(unused)]
- #[specialize]
- thread_out_rows: u32,
- #[allow(unused)]
- #[specialize]
- thread_out_cols: u32,
- #[specialize] apply_output_scale_and_accumulate: bool,
-) {
- matmul_gemv_impl::<T>(
- matrix,
- input_vector,
- output_source,
- output_vector,
- input_dimension as usize,
- output_dimension as usize,
- matrix_leading_dimension as usize,
- output_scale,
- output_accumulate_scale,
- output_source_stride as usize,
- batch_rows as usize,
- apply_output_scale_and_accumulate,
- );
-}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
new file mode 100644
index 000000000..0e9cb49d5
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -0,0 +1,118 @@
+use half::{bf16, f16};
+
+use crate::{
+ DataType,
+ backends::{
+ common::kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},
+ cpu::{Cpu, command_buffer::CpuCommandBuffer, context::CpuContext},
+ },
+};
+
+pub struct MatmulCpuKernel {
+ data_type: DataType,
+}
+
+impl MatmulKernel for MatmulCpuKernel {
+ type Backend = Cpu;
+
+ fn new(
+ _context: &CpuContext,
+ data_type: DataType,
+ ) -> Result<Self, MatmulError<Cpu>> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MatmulError::UnsupportedDataType(data_type));
+ }
+ Ok(Self {
+ data_type,
+ })
+ }
+
+ fn encode(
+ &mut self,
+ _context: &CpuContext,
+ arguments: MatmulArguments<Cpu>,
+ command_buffer: &mut CpuCommandBuffer,
+ ) {
+ let m = arguments.batch as usize;
+ let n = arguments.output_dim as usize;
+ let k = arguments.input_dim as usize;
+ let lda = arguments.leading_dimension_a as usize;
+ let ldb = arguments.leading_dimension_b as usize;
+ let ldd = arguments.leading_dimension_d as usize;
+ let a_offset = arguments.a_offset as usize;
+ let data_type = self.data_type;
+
+ let a_ptr = arguments.a.as_ptr().wrapping_byte_add(a_offset);
+ let b_ptr = arguments.b.as_ptr();
+ let d_ptr = arguments.d.as_mut_ptr();
+ let bias_ptr = arguments.bias.map(|b| b.as_ptr());
+
+ command_buffer.push_command(move || match data_type {
+ DataType::F32 => {
+ super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f32>(
+ a_ptr as *const f32,
+ b_ptr as *const f32,
+ d_ptr as *mut f32,
+ m,
+ n,
+ k,
+ lda,
+ ldb,
+ ldd,
+ );
+ if let Some(bias) = bias_ptr {
+ apply_bias::<f32>(d_ptr as *mut f32, bias as *const f32, m, n);
+ }
+ },
+ DataType::F16 => {
+ super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f16>(
+ a_ptr as *const f16,
+ b_ptr as *const f16,
+ d_ptr as *mut f16,
+ m,
+ n,
+ k,
+ lda,
+ ldb,
+ ldd,
+ );
+ if let Some(bias) = bias_ptr {
+ apply_bias::<f16>(d_ptr as *mut f16, bias as *const f16, m, n);
+ }
+ },
+ DataType::BF16 => {
+ super::gemm::shaders::steel_gemm::matmul_gemm_impl::<bf16>(
+ a_ptr as *const bf16,
+ b_ptr as *const bf16,
+ d_ptr as *mut bf16,
+ m,
+ n,
+ k,
+ lda,
+ ldb,
+ ldd,
+ );
+ if let Some(bias) = bias_ptr {
+ apply_bias::<bf16>(d_ptr as *mut bf16, bias as *const bf16, m, n);
+ }
+ },
+ _ => unreachable!(),
+ });
+ }
+}
+
+fn apply_bias<T: num_traits::Float + Copy>(
+ d: *mut T,
+ bias: *const T,
+ m: usize,
+ n: usize,
+) {
+ unsafe {
+ for row in 0..m {
+ for col in 0..n {
+ let idx = row * n + col;
+ *d.add(idx) = *d.add(idx) + *bias.add(col);
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
index f2ab16625..23d8c2f63 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
@@ -1,11 +1,11 @@
-use crate::backends::{
- common::kernel::matmul::{MatmulKernel, MatmulKernels},
- cpu::{Cpu, kernel::CpuKernels},
-};
-
pub mod gemm;
pub mod gemv;
+mod matmul;
+
+pub use matmul::MatmulCpuKernel;
+
+use crate::backends::{common::kernel::matmul::MatmulKernels, cpu::kernel::CpuKernels};
impl MatmulKernels for CpuKernels {
- type FullPrecisionMatmulKernel = MatmulKernel<Cpu>;
+ type MatmulKernel = MatmulCpuKernel;
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index fbe1a533b..57009d9ae 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -58,7 +58,7 @@ PUBLIC KERNEL(AttentionGemm)(
const bool has_sinks SPECIALIZE,
threadgroup T q_smem[BLOCK_QUERY_ROWS * (BD + 16 / sizeof(T))],
threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint tgid_x GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),
const uint tgid_y GROUPS(num_heads),
const uint tgid_z GROUPS(1),
@@ -139,22 +139,22 @@ PUBLIC KERNEL(AttentionGemm)(
q,
query_source_stride,
query_shared,
- simd.threadgroup_index,
- simd.simdgroup_index
+ thread_context.threadgroup_index,
+ thread_context.simdgroup_index
);
thread KeyLoader key_loader(
k,
key_source_stride,
key_shared,
- simd.threadgroup_index,
- simd.simdgroup_index
+ thread_context.threadgroup_index,
+ thread_context.simdgroup_index
);
thread ValueLoader value_loader(
v,
value_source_stride,
value_shared,
- simd.threadgroup_index,
- simd.simdgroup_index
+ thread_context.threadgroup_index,
+ thread_context.simdgroup_index
);
TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));
@@ -220,13 +220,13 @@ PUBLIC KERNEL(AttentionGemm)(
// Lane coordinates and pointer offsets
const short2 lane_coordinates =
SimdgroupMultiplyAccumulateType::get_lane_coordinates(
- simd.simdgroup_index
+ thread_context.simdgroup_index
);
const short lane_row = lane_coordinates.y;
const short lane_col = lane_coordinates.x;
- const short simdgroup_row_base =
- SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS * short(simd.threadgroup_index);
+ const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS *
+ short(thread_context.threadgroup_index);
const short query_shared_offset =
(simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index 1a09cb6be..bec72e50c 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -35,7 +35,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
const bool has_mask SPECIALIZE,
const bool has_sinks SPECIALIZE,
const bool do_causal SPECIALIZE,
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
@@ -60,22 +60,23 @@ PUBLIC KERNEL(AttentionSinglePass)(
const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx
: head_idx * tpg.y + q_seq_idx;
- queries +=
- q_offset * HEAD_DIM + simd.simdgroup_index * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride + simd.threadgroup_index * k_seq_stride +
- simd.simdgroup_index * qk_elements_per_thread;
+ queries += q_offset * HEAD_DIM +
+ thread_context.simdgroup_index * qk_elements_per_thread;
+ keys += kv_head_idx * k_head_stride +
+ thread_context.threadgroup_index * k_seq_stride +
+ thread_context.simdgroup_index * qk_elements_per_thread;
values += kv_head_idx * v_head_stride +
- simd.threadgroup_index * v_seq_stride +
- simd.simdgroup_index * value_elements_per_thread;
+ thread_context.threadgroup_index * v_seq_stride +
+ thread_context.simdgroup_index * value_elements_per_thread;
if (float_mask) {
fmask += head_idx * mask_head_stride +
- simd.threadgroup_index * mask_kv_seq_stride +
+ thread_context.threadgroup_index * mask_kv_seq_stride +
q_seq_idx * mask_q_seq_stride;
}
- out +=
- o_offset * value_dim + simd.threadgroup_index * value_elements_per_thread;
+ out += o_offset * value_dim +
+ thread_context.threadgroup_index * value_elements_per_thread;
// Read the query and 0 the output accumulator
for (int i = 0; i < qk_elements_per_thread; i++) {
@@ -87,7 +88,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
U max_score = -INFINITY;
U sum_exp_score = 0;
- if (has_sinks && simd.threadgroup_index == 0) {
+ if (has_sinks && thread_context.threadgroup_index == 0) {
const int num_q_heads = static_cast<int>(tpg.x);
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
@@ -95,7 +96,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// For each key
- for (uint i = simd.threadgroup_index; i < sequence_length;
+ for (uint i = thread_context.threadgroup_index; i < sequence_length;
i += SEQUENCE_BLOCK_SIZE) {
bool use_key = true;
if (do_causal) {
@@ -141,27 +142,27 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// Each thread has a partial part of the output so we need to combine them.
- if (simd.simdgroup_index == 0) {
- shared_max_scores[simd.threadgroup_index] = max_score;
- shared_sum_exp_scores[simd.threadgroup_index] = sum_exp_score;
+ if (thread_context.simdgroup_index == 0) {
+ shared_max_scores[thread_context.threadgroup_index] = max_score;
+ shared_sum_exp_scores[thread_context.threadgroup_index] = sum_exp_score;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = shared_max_scores[simd.simdgroup_index];
+ max_score = shared_max_scores[thread_context.simdgroup_index];
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
sum_exp_score =
- simd_sum(shared_sum_exp_scores[simd.simdgroup_index] * factor);
+ simd_sum(shared_sum_exp_scores[thread_context.simdgroup_index] * factor);
// Now we need to aggregate all the outputs
for (int i = 0; i < value_elements_per_thread; i++) {
shared_outputs
- [simd.simdgroup_index * HEAD_BLOCK_SIZE + simd.threadgroup_index] =
- o[i];
+ [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +
+ thread_context.threadgroup_index] = o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
o[i] = simd_sum(
shared_outputs
- [simd.threadgroup_index * HEAD_BLOCK_SIZE +
- simd.simdgroup_index] *
+ [thread_context.threadgroup_index * HEAD_BLOCK_SIZE +
+ thread_context.simdgroup_index] *
factor
) /
sum_exp_score;
@@ -169,7 +170,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// And write the output
- if (simd.simdgroup_index == 0) {
+ if (thread_context.simdgroup_index == 0) {
for (int i = 0; i < value_elements_per_thread; i++) {
out[i] = static_cast<T>(o[i]);
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index a47849b78..51d1b18dd 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -39,7 +39,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],
threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],
threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),
@@ -67,23 +67,26 @@ PUBLIC KERNEL(AttentionTwoPass1)(
: head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
const uint kv_head_idx = head_idx / gqa_factor;
- queries +=
- q_offset * HEAD_DIM + simd.simdgroup_index * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *
- k_seq_stride +
- simd.simdgroup_index * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *
- v_seq_stride +
- simd.simdgroup_index * value_elements_per_thread;
+ queries += q_offset * HEAD_DIM +
+ thread_context.simdgroup_index * qk_elements_per_thread;
+ keys +=
+ kv_head_idx * k_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *
+ k_seq_stride +
+ thread_context.simdgroup_index * qk_elements_per_thread;
+ values +=
+ kv_head_idx * v_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *
+ v_seq_stride +
+ thread_context.simdgroup_index * value_elements_per_thread;
out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +
- simd.simdgroup_index * value_elements_per_thread;
+ thread_context.simdgroup_index * value_elements_per_thread;
if (float_mask) {
- fmask += head_idx * mask_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *
- mask_kv_seq_stride +
- q_seq_idx * mask_q_seq_stride;
+ fmask +=
+ head_idx * mask_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *
+ mask_kv_seq_stride +
+ q_seq_idx * mask_q_seq_stride;
}
sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
@@ -98,7 +101,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
U max_score = -1e9;
U sum_exp_score = 0;
- if (has_sinks && block_idx == 0 && simd.threadgroup_index == 0) {
+ if (has_sinks && block_idx == 0 && thread_context.threadgroup_index == 0) {
const uint num_q_heads = tpg.x;
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
@@ -106,7 +109,8 @@ PUBLIC KERNEL(AttentionTwoPass1)(
}
// For each key
- for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index;
+ for (uint i =
+ block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index;
i < sequence_length;
i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {
bool use_key = true;
@@ -153,23 +157,23 @@ PUBLIC KERNEL(AttentionTwoPass1)(
}
// Each thread has a partial part of the output so we need to combine them.
- if (simd.simdgroup_index == 0) {
- shared_max_scores[simd.threadgroup_index] = max_score;
- shared_sum_exp_scores[simd.threadgroup_index] = sum_exp_score;
+ if (thread_context.simdgroup_index == 0) {
+ shared_max_scores[thread_context.threadgroup_index] = max_score;
+ shared_sum_exp_scores[thread_context.threadgroup_index] = sum_exp_score;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = (simd.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)
- ? shared_max_scores[simd.simdgroup_index]
+ max_score = (thread_context.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)
+ ? shared_max_scores[thread_context.simdgroup_index]
: -1e9;
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- sum_exp_score = (simd.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)
- ? shared_sum_exp_scores[simd.simdgroup_index]
+ sum_exp_score = (thread_context.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)
+ ? shared_sum_exp_scores[thread_context.simdgroup_index]
: 0;
sum_exp_score = simd_sum(sum_exp_score * factor);
// Write the sum and new max
- if (simd.threadgroup_index == 0) {
+ if (thread_context.threadgroup_index == 0) {
sums[0] = sum_exp_score;
maxs[0] = new_max;
}
@@ -177,18 +181,21 @@ PUBLIC KERNEL(AttentionTwoPass1)(
// Now we need to aggregate all the outputs
for (uint i = 0; i < value_elements_per_thread; i++) {
shared_outputs
- [simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 +
- simd.threadgroup_index] =
+ [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 +
+ thread_context.threadgroup_index] =
o[i] *
- fast::exp(shared_max_scores[simd.threadgroup_index] - new_max);
+ fast::exp(
+ shared_max_scores[thread_context.threadgroup_index] - new_max
+ );
threadgroup_barrier(mem_flags::mem_threadgroup);
// And write the output
- if (simd.threadgroup_index == 0) {
- U output = shared_outputs[simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1];
+ if (thread_context.threadgroup_index == 0) {
+ U output = shared_outputs
+ [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1];
for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {
- output +=
- shared_outputs[simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 + j];
+ output += shared_outputs
+ [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 + j];
}
out[i] = static_cast<T>(output);
}
@@ -207,7 +214,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(
const constant uint& num_heads,
const constant uint& suffix_length,
threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
@@ -221,19 +228,19 @@ PUBLIC KERNEL(AttentionTwoPass2)(
const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
partials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +
- simd.threadgroup_index * HEAD_DIM +
- simd.simdgroup_index * elements_per_thread;
+ thread_context.threadgroup_index * HEAD_DIM +
+ thread_context.simdgroup_index * elements_per_thread;
sums += o_offset * TOTAL_BLOCKS_COUNT;
maxs += o_offset * TOTAL_BLOCKS_COUNT;
out +=
- o_offset * HEAD_DIM +
- simd.threadgroup_index * elements_per_thread; // Our custom output layout
+ o_offset * HEAD_DIM + thread_context.threadgroup_index *
+ elements_per_thread; // Our custom output layout
// First everybody reads the max and sum_exp
- U max_score = maxs[simd.simdgroup_index];
+ U max_score = maxs[thread_context.simdgroup_index];
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- U sum_exp_score = simd_sum(sums[simd.simdgroup_index] * factor);
+ U sum_exp_score = simd_sum(sums[thread_context.simdgroup_index] * factor);
// Now read the block into registers and then use shared memory to transpose
// it
@@ -242,13 +249,13 @@ PUBLIC KERNEL(AttentionTwoPass2)(
}
for (uint i = 0; i < elements_per_thread; i++) {
shared_outputs
- [simd.simdgroup_index * HEAD_BLOCK_SIZE + simd.threadgroup_index] =
- o[i];
+ [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +
+ thread_context.threadgroup_index] = o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
o[i] = simd_sum(
shared_outputs
- [simd.threadgroup_index * HEAD_BLOCK_SIZE +
- simd.simdgroup_index] *
+ [thread_context.threadgroup_index * HEAD_BLOCK_SIZE +
+ thread_context.simdgroup_index] *
factor
) /
sum_exp_score;
@@ -256,7 +263,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(
}
// And write the output
- if (simd.simdgroup_index == 0) {
+ if (thread_context.simdgroup_index == 0) {
for (uint i = 0; i < elements_per_thread; i++) {
out[i] = static_cast<T>(o[i]);
}
diff --git a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
index 2438e37a7..4475cc46d 100644
--- a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
+++ b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
@@ -138,19 +138,20 @@ static T threadgroup_cooperative_reduce_sum(
T value,
threadgroup T* shared,
const ushort lid,
- const thread ThreadContext& simd
+ const thread ThreadContext& thread_context
) {
T local_sum = simd_sum(value);
- if (simd.simdgroup_index == 0) {
- shared[simd.threadgroup_index] = local_sum;
+ if (thread_context.simdgroup_index == 0) {
+ shared[thread_context.threadgroup_index] = local_sum;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
T total_sum = T(0);
const ushort num_simd_groups =
- (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size;
+ (BLOCK_SIZE + thread_context.simdgroup_size - 1) /
+ thread_context.simdgroup_size;
if (lid < num_simd_groups) {
total_sum = shared[lid];
}
@@ -171,20 +172,20 @@ static T threadgroup_cooperative_reduce_max(
T value,
threadgroup T* shared,
const ushort lid,
- const thread ThreadContext& simd
+ const thread ThreadContext& thread_context
) {
T local_max = simd_max(value);
- if (simd.simdgroup_index == 0) {
- shared[simd.threadgroup_index] = local_max;
+ if (thread_context.simdgroup_index == 0) {
+ shared[thread_context.threadgroup_index] = local_max;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- T total_max =
- lid < (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size
- ? shared[lid]
- : T(-INFINITY);
+ T total_max = lid < (BLOCK_SIZE + thread_context.simdgroup_size - 1) /
+ thread_context.simdgroup_size
+ ? shared[lid]
+ : T(-INFINITY);
total_max = simd_max(total_max);
if (lid == 0) {
@@ -202,20 +203,20 @@ static T threadgroup_cooperative_reduce_min(
T value,
threadgroup T* shared,
const ushort lid,
- const thread ThreadContext& simd
+ const thread ThreadContext& thread_context
) {
T local_min = simd_min(value);
- if (simd.simdgroup_index == 0) {
- shared[simd.threadgroup_index] = local_min;
+ if (thread_context.simdgroup_index == 0) {
+ shared[thread_context.threadgroup_index] = local_min;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- T total_min =
- lid < (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size
- ? shared[lid]
- : T(INFINITY);
+ T total_min = lid < (BLOCK_SIZE + thread_context.simdgroup_size - 1) /
+ thread_context.simdgroup_size
+ ? shared[lid]
+ : T(INFINITY);
total_min = simd_min(total_min);
if (lid == 0) {
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
index c6e54cee0..08758f523 100644
--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -20,7 +20,7 @@ void layer_norm_core(
threadgroup AccumT* shared_variance,
uint thread_in_row,
bool full_layer,
- const thread ThreadContext& simd
+ const thread ThreadContext& thread_context
) {
AccumT partial_sum = static_cast<AccumT>(0.0f);
@@ -40,7 +40,7 @@ void layer_norm_core(
partial_sum,
shared_mean,
thread_in_row,
- simd
+ thread_context
);
AccumT mean = total_sum / static_cast<AccumT>(element_count);
@@ -62,7 +62,7 @@ void layer_norm_core(
partial_var_sum,
shared_variance,
thread_in_row,
- simd
+ thread_context
);
AccumT variance = total_var / static_cast<AccumT>(element_count);
AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));
@@ -132,7 +132,7 @@ PUBLIC KERNEL(LayerNorm) (
const bool in_place SPECIALIZE,
threadgroup ACC shared_mean[METAL_SIMD_SIZE],
threadgroup ACC shared_variance[METAL_SIMD_SIZE],
- const ThreadContext simd,
+ const ThreadContext thread_context,
uint batch_idx GROUPS(batch_size),
uint thread_in_row THREADS(BLOCK_SIZE)
) {
@@ -151,6 +151,6 @@ PUBLIC KERNEL(LayerNorm) (
shared_variance,
thread_in_row,
full_layer,
- simd
+ thread_context
);
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
new file mode 100644
index 000000000..14f9855ce
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -0,0 +1,3 @@
+mod specialization;
+
+pub use specialization::GemmSpecialization;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal
similarity index 81%
rename from crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
rename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal
index 6608d1cb7..d020bced4 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal
@@ -29,14 +29,14 @@ METAL_FUNC void gemm_dispatch(
simdgroups_per_row == SIMDGROUPS_PER_ROW && simdgroups_per_column == SIMDGROUPS_PER_COLUMN) { \
if (align_m && align_n) { \
if (align_k) \
- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, true>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, true>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
else \
- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, false>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, false>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
} else { \
if (align_k) \
- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
else \
- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, false>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, false>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
} \
return; \
}
@@ -46,7 +46,7 @@ METAL_FUNC void gemm_dispatch(
template <typename T>
VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MatmulGemm)(
+KERNEL(MatmulGemm)(
const device T* a,
const device T* b,
device T* d,
@@ -68,7 +68,7 @@ PUBLIC KERNEL(MatmulGemm)(
const uint thread_x THREADS(32),
const uint thread_y THREADS(2),
const uint thread_z THREADS(2),
- const ThreadContext simd
+ const ThreadContext thread_context
) {
GEMM_DISPATCH(T, 64, 64, 16, 2, 2)
GEMM_DISPATCH(T, 64, 64, 16, 1, 2)
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
similarity index 95%
rename from crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 42e60d822..36e9db557 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -1,6 +1,9 @@
use crate::{
DataType,
- backends::common::{Backend, Context, kernel::matmul::MatmulArguments},
+ backends::{
+ common::{Context, kernel::matmul::MatmulArguments},
+ metal::{Metal, context::MetalContext},
+ },
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -139,14 +142,14 @@ impl GemmSpecialization {
}
}
- pub fn select<B: Backend>(
- context: &B::Context,
+ pub fn select(
+ context: &MetalContext,
data_type: DataType,
- arguments: &MatmulArguments<B>,
+ arguments: &MatmulArguments<Metal>,
) -> Self {
let overall_work_elements = (arguments.batch as i64) * (arguments.output_dim as i64);
let is_float32 = matches!(data_type, DataType::F32);
- let prefer_half_or_tf32 = !is_float32 || B::Context::tf32_enabled();
+ let prefer_half_or_tf32 = !is_float32 || MetalContext::tf32_enabled();
let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column, swizzle_log2) =
if context.is_high_performance() && overall_work_elements >= (1_i64 << 20) {
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
new file mode 100644
index 000000000..1d2748972
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -0,0 +1,3 @@
+mod specialization;
+
+pub use specialization::Specialization;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
index ca4aeecf7..169715ce6 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
@@ -13,7 +13,7 @@ using namespace metal;
template <typename T>
VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MatmulGemv)(
+KERNEL(MatmulGemv)(
const device T* matrix,
const device T* input_vector,
const device T* output_source OPTIONAL(apply_output_scale_and_accumulate),
@@ -39,14 +39,14 @@ PUBLIC KERNEL(MatmulGemv)(
const uint thread_index_x THREADS(32),
const uint thread_index_y THREADS(16),
const uint thread_index_z THREADS(1),
- const ThreadContext simd
+ const ThreadContext thread_context
) {
if (output_rows_per_threadgroup <= 0) {
return;
}
// Simdgroup guard — kill excess simdgroups beyond what this tile uses
- if (simd.threadgroup_index >= tg_simd_rows * tg_simd_cols) {
+ if (thread_context.threadgroup_index >= tg_simd_rows * tg_simd_cols) {
return;
}
@@ -66,22 +66,27 @@ PUBLIC KERNEL(MatmulGemv)(
thread float vector_coefficients[4];
const int thread_row_in_simdgroup =
- sg_thread_cols != 32 ? int(simd.simdgroup_index) / int(sg_thread_cols)
- : 0;
+ sg_thread_cols != 32
+ ? int(thread_context.simdgroup_index) / int(sg_thread_cols)
+ : 0;
const int thread_col_in_simdgroup =
- sg_thread_cols != 32 ? int(simd.simdgroup_index) % int(sg_thread_cols)
- : int(simd.simdgroup_index);
+ sg_thread_cols != 32
+ ? int(thread_context.simdgroup_index) % int(sg_thread_cols)
+ : int(thread_context.simdgroup_index);
const int simdgroup_column_index =
- tg_simd_cols != 1 ? int(simd.threadgroup_index % tg_simd_cols) : 0;
+ tg_simd_cols != 1 ? int(thread_context.threadgroup_index % tg_simd_cols)
+ : 0;
const int simdgroup_row_thread_base =
tg_simd_cols != 1
- ? int(sg_thread_rows) * int(simd.threadgroup_index / tg_simd_cols)
- : int(sg_thread_rows) * int(simd.threadgroup_index);
+ ? int(sg_thread_rows) *
+ int(thread_context.threadgroup_index / tg_simd_cols)
+ : int(sg_thread_rows) * int(thread_context.threadgroup_index);
const int simdgroup_col_thread_base =
tg_simd_cols != 1
- ? int(sg_thread_cols) * int(simd.threadgroup_index % tg_simd_cols)
+ ? int(sg_thread_cols) *
+ int(thread_context.threadgroup_index % tg_simd_cols)
: 0;
int output_block_row_offset =
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rs
similarity index 100%
rename from crates/uzu/src/backends/common/kernel/matmul/gemv/specialization.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rs
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs
new file mode 100644
index 000000000..f0dada6f4
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs
@@ -0,0 +1,338 @@
+use std::{
+ collections::{HashMap, hash_map::Entry},
+ sync::OnceLock,
+};
+
+use super::{
+ super::dsl::{MatmulGemmMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel},
+ gemm, gemv,
+};
+use crate::{
+ DataType,
+ backends::{
+ common::{
+ gpu_types::GemmParams,
+ kernel::{
+ TensorAddBiasKernel,
+ matmul::{MatmulArguments, MatmulError, MatmulKernel},
+ },
+ },
+ metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext},
+ },
+};
+
+pub struct MatmulMetalKernel {
+ data_type: DataType,
+ gemv_kernels: HashMap<gemv::Specialization, MatmulGemvMetalKernel>,
+ gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,
+ bias_add: Option<TensorAddBiasMetalKernel>,
+}
+
+const DEFAULT_GEMV_MAX_BATCH: i32 = 8;
+
+static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();
+
+fn max_gemv_batch_threshold() -> i32 {
+ *GEMV_MAX_BATCH.get_or_init(|| {
+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
+ })
+}
+
+impl MatmulMetalKernel {
+ fn is_gemv_eligible(arguments: &MatmulArguments<Metal>) -> bool {
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ let max_batch = max_gemv_batch_threshold();
+
+ if !arguments.transpose_b {
+ return false;
+ }
+
+ if n == 1 {
+ m == 1
+ } else {
+ m <= max_batch
+ }
+ }
+
+ fn get_or_create_gemv(
+ &mut self,
+ context: &MetalContext,
+ specialization: gemv::Specialization,
+ ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {
+ match self.gemv_kernels.entry(specialization) {
+ Entry::Occupied(entry) => Ok(entry.into_mut()),
+ Entry::Vacant(entry) => {
+ let kernel = MatmulGemvMetalKernel::new(
+ context,
+ self.data_type,
+ specialization.threadgroup_rows,
+ specialization.threadgroup_cols,
+ specialization.threads_per_simdgroup_row,
+ specialization.threads_per_simdgroup_col,
+ specialization.elements_per_thread_row,
+ specialization.elements_per_thread_col,
+ specialization.apply_output_scale_and_accumulate,
+ )
+ .map_err(MatmulError::BackendError)?;
+ Ok(entry.insert(kernel))
+ },
+ }
+ }
+
+ fn get_or_create_gemm(
+ &mut self,
+ context: &MetalContext,
+ specialization: gemm::GemmSpecialization,
+ ) -> Result<&MatmulGemmMetalKernel, MatmulError<Metal>> {
+ match self.gemm_kernels.entry(specialization) {
+ Entry::Occupied(entry) => Ok(entry.into_mut()),
+ Entry::Vacant(entry) => {
+ let kernel = MatmulGemmMetalKernel::new(
+ context,
+ self.data_type,
+ specialization.block_rows as u32,
+ specialization.block_cols as u32,
+ specialization.block_depth as u32,
+ specialization.simdgroups_per_row as u32,
+ specialization.simdgroups_per_column as u32,
+ specialization.align_m,
+ specialization.align_n,
+ specialization.align_k,
+ )
+ .map_err(MatmulError::BackendError)?;
+ Ok(entry.insert(kernel))
+ },
+ }
+ }
+
+ fn encode_gemv(
+ &mut self,
+ context: &MetalContext,
+ command_buffer: &mut MetalCommandBufferEncoding,
+ arguments: MatmulArguments<Metal>,
+ ) -> Result<(), MatmulError<Metal>> {
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+
+ let matrix_is_rhs = n != 1;
+ let has_bias = arguments.bias.is_some();
+ let apply_output_scale_and_accumulate = has_bias;
+
+ let effective_output_dimension = if matrix_is_rhs {
+ n
+ } else {
+ m
+ };
+
+ let transpose_matrix = if matrix_is_rhs {
+ !arguments.transpose_b
+ } else {
+ false
+ };
+
+ let specialization = gemv::Specialization::select(
+ transpose_matrix,
+ arguments.input_dim,
+ effective_output_dimension,
+ apply_output_scale_and_accumulate,
+ );
+
+ let (alpha, beta, bias_stride) = if apply_output_scale_and_accumulate {
+ (1.0f32, 1.0f32, 1i32)
+ } else {
+ (1.0f32, 0.0f32, 0i32)
+ };
+
+ let matrix_leading_dim = if matrix_is_rhs {
+ arguments.leading_dimension_b
+ } else {
+ arguments.leading_dimension_a
+ };
+
+ let (matrix, matrix_offset) = if matrix_is_rhs {
+ (arguments.b, 0usize)
+ } else {
+ (arguments.a, arguments.a_offset as usize)
+ };
+ let (input_vector, input_vector_offset) = if matrix_is_rhs {
+ (arguments.a, arguments.a_offset as usize)
+ } else {
+ (arguments.b, 0usize)
+ };
+
+ let output_source = if apply_output_scale_and_accumulate {
+ arguments.bias
+ } else {
+ None
+ };
+
+ let kernel = self.get_or_create_gemv(context, specialization)?;
+
+ kernel.encode(
+ (matrix, matrix_offset),
+ (input_vector, input_vector_offset),
+ output_source.map(|buffer| (buffer, 0usize)),
+ &mut *arguments.d,
+ arguments.input_dim,
+ effective_output_dimension,
+ matrix_leading_dim,
+ alpha,
+ beta,
+ bias_stride,
+ arguments.batch,
+ specialization.output_rows_per_threadgroup() as i32,
+ command_buffer,
+ );
+
+ let bias_is_fused = apply_output_scale_and_accumulate && has_bias;
+ if !bias_is_fused {
+ self.encode_bias_add(
+ context,
+ arguments.bias,
+ arguments.d,
+ arguments.batch as usize,
+ arguments.output_dim as usize,
+ command_buffer,
+ )?;
+ }
+
+ Ok(())
+ }
+
+ fn encode_gemm(
+ &mut self,
+ context: &MetalContext,
+ command_buffer: &mut MetalCommandBufferEncoding,
+ arguments: MatmulArguments<Metal>,
+ ) -> Result<(), MatmulError<Metal>> {
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ let k = arguments.input_dim;
+
+ let specialization = gemm::GemmSpecialization::select(context, self.data_type, &arguments);
+
+ let threadgroups_per_row = (n + specialization.block_cols - 1) / specialization.block_cols;
+ let threadgroups_per_column = (m + specialization.block_rows - 1) / specialization.block_rows;
+ let swizzle_log = specialization.swizzle_log2;
+
+ let swizzle_stride = 1_i32 << swizzle_log;
+ let tm_swizzled = (threadgroups_per_column + swizzle_stride - 1) / swizzle_stride;
+ let tn_swizzled = threadgroups_per_row * swizzle_stride;
+
+ let params = GemmParams {
+ M: m,
+ N: n,
+ K: k,
+ leading_dimension_a: arguments.leading_dimension_a,
+ leading_dimension_b: arguments.leading_dimension_b,
+ leading_dimension_d: arguments.leading_dimension_d,
+ threadgroups_per_row,
+ threadgroups_per_column,
+ swizzle_log,
+ aligned_inner_iterations: k / specialization.block_depth,
+ };
+
+ let group_count_x =
+ u32::try_from(tn_swizzled).map_err(|_| MatmulError::<Metal>::ThreadgroupOverflow(tn_swizzled as usize))?;
+ let group_count_y =
+ u32::try_from(tm_swizzled).map_err(|_| MatmulError::<Metal>::ThreadgroupOverflow(tm_swizzled as usize))?;
+
+ let kernel = self.get_or_create_gemm(context, specialization)?;
+
+ kernel.encode(
+ (arguments.a, arguments.a_offset as usize),
+ arguments.b,
+ &mut *arguments.d,
+ std::slice::from_ref(&params),
+ group_count_x,
+ group_count_y,
+ command_buffer,
+ );
+
+ self.encode_bias_add(
+ context,
+ arguments.bias,
+ arguments.d,
+ arguments.batch as usize,
+ arguments.output_dim as usize,
+ command_buffer,
+ )?;
+
+ Ok(())
+ }
+
+ fn encode_bias_add(
+ &mut self,
+ context: &MetalContext,
+ bias: Option<&<Metal as crate::backends::common::Backend>::Buffer>,
+ output: &mut <Metal as crate::backends::common::Backend>::Buffer,
+ batch: usize,
+ output_dim: usize,
+ command_buffer: &mut MetalCommandBufferEncoding,
+ ) -> Result<(), MatmulError<Metal>> {
+ let Some(bias) = bias else {
+ return Ok(());
+ };
+ let total_length = batch * output_dim;
+ if total_length == 0 {
+ return Ok(());
+ }
+ if self.bias_add.is_none() {
+ self.bias_add =
+ Some(TensorAddBiasMetalKernel::new(context, self.data_type, true).map_err(MatmulError::BackendError)?);
+ }
+ let bias_add = self.bias_add.as_ref().unwrap();
+ bias_add.encode(
+ None::<&<Metal as crate::backends::common::Backend>::Buffer>,
+ bias,
+ output,
+ output_dim as u32,
+ total_length as u32,
+ command_buffer,
+ );
+ Ok(())
+ }
+}
+
+impl MatmulKernel for MatmulMetalKernel {
+ type Backend = Metal;
+
+ fn new(
+ context: &MetalContext,
+ data_type: DataType,
+ ) -> Result<Self, MatmulError<Metal>> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MatmulError::UnsupportedDataType(data_type));
+ }
+
+ let mut kernel = Self {
+ data_type,
+ gemv_kernels: HashMap::new(),
+ gemm_kernels: HashMap::new(),
+ bias_add: None,
+ };
+
+ for &config in gemv::Specialization::precompile_configs(data_type) {
+ kernel.get_or_create_gemv(context, config)?;
+ }
+ for &config in gemm::GemmSpecialization::precompile_configs(data_type) {
+ kernel.get_or_create_gemm(context, config)?;
+ }
+
+ Ok(kernel)
+ }
+
+ fn encode(
+ &mut self,
+ context: &MetalContext,
+ arguments: MatmulArguments<Metal>,
+ command_buffer: &mut MetalCommandBufferEncoding,
+ ) {
+ if Self::is_gemv_eligible(&arguments) {
+ self.encode_gemv(context, command_buffer, arguments).expect("Failed to encode GEMV kernel");
+ } else {
+ self.encode_gemm(context, command_buffer, arguments).expect("Failed to encode GEMM kernel");
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
new file mode 100644
index 000000000..27c0a5c14
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -0,0 +1,5 @@
+pub mod gemm;
+pub mod gemv;
+mod matmul;
+
+pub use matmul::MatmulMetalKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 429f084ad..dedaa9318 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,12 +1,12 @@
-use crate::backends::{
- common::kernel::matmul::{MatmulKernel, MatmulKernels},
- metal::Metal,
-};
+pub mod matmul;
+
+use crate::backends::common::kernel::matmul::MatmulKernels;
+
pub mod dsl {
include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
}
pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
impl MatmulKernels for dsl::MetalKernels {
- type FullPrecisionMatmulKernel = MatmulKernel<Metal>;
+ type MatmulKernel = matmul::MatmulMetalKernel;
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index 1281a2bd8..571d408ce 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -25,7 +25,7 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(
threadgroup uint
counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory
threadgroup uint& carry,
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint lid THREADS(128)
) {
if (e_input == 0) {
@@ -104,7 +104,7 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(
v,
reduce_shared,
(ushort)lid,
- simd
+ thread_context
);
if (lid == 0) {
carry += block_sum;
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
index 70320f217..03cf07d3d 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -27,7 +27,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
constant float& up_clip_min,
constant float& up_clip_max,
const uint gating_sel SPECIALIZE, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint h_block_idx GROUPS(d_ff.div_ceil(4)),
const uint k_slot GROUPS(k),
const uint tid THREADS(128)
@@ -37,7 +37,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
return;
const uint expert_u = uint(expert_id);
- const uint h_idx = h_block_idx * 4 + simd.threadgroup_index;
+ const uint h_idx = h_block_idx * 4 + thread_context.threadgroup_index;
if (h_idx >= d_ff)
return;
@@ -55,7 +55,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
const uint vec_iters = d_model / 128;
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd.simdgroup_index * 4;
+ uint base_idx = i * 128 + thread_context.simdgroup_index * 4;
device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);
device const T* w_up_vec2 =
@@ -75,7 +75,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
}
}
- uint leftover_start = vec_iters * 128 + simd.simdgroup_index;
+ uint leftover_start = vec_iters * 128 + thread_context.simdgroup_index;
for (uint idx = leftover_start; idx < d_model; idx += 32) {
float xv = float(x[idx]);
acc_up += xv * float(w_up_row[idx]);
@@ -89,7 +89,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
acc_gate = simd_sum(acc_gate);
}
- if (simd.simdgroup_index == 0) {
+ if (thread_context.simdgroup_index == 0) {
float up_val = clamp(
acc_up + float(biases[bias_base + h_idx]),
up_clip_min,
@@ -133,11 +133,11 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
constant uint& d_model,
constant uint& d_ff,
constant uint& k_input,
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint d_block GROUPS(d_model.div_ceil(8)),
const uint tid THREADS(256)
) {
- const uint my_col = d_block * 8 + simd.threadgroup_index;
+ const uint my_col = d_block * 8 + thread_context.threadgroup_index;
if (my_col >= d_model)
return;
@@ -159,7 +159,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
// Vectorized reduction
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd.simdgroup_index * 4;
+ uint base_idx = i * 128 + thread_context.simdgroup_index * 4;
float4 h_vec =
*reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
@@ -172,20 +172,21 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
}
// Remainder
- for (uint idx = vec_iters * 128 + simd.simdgroup_index; idx < d_ff;
+ for (uint idx = vec_iters * 128 + thread_context.simdgroup_index;
+ idx < d_ff;
idx += 32) {
acc += hidden_ptr[idx] * float(w2_ptr[idx]);
}
// Simdgroup reduction and accumulate
float result = simd_sum(acc);
- if (simd.simdgroup_index == 0) {
+ if (thread_context.simdgroup_index == 0) {
result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);
final_acc += prob * result;
}
}
- if (simd.simdgroup_index == 0) {
+ if (thread_context.simdgroup_index == 0) {
y[my_col] = T(final_acc);
}
}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index deab8e40c..07bf1e12c 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -27,7 +27,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
constant float& silu_alpha,
device const uint* tile_map,
const uint gating_sel SPECIALIZE,
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint tile_idx GROUPS(INDIRECT),
const uint tid THREADS(128)
) {
@@ -44,7 +44,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
return;
// Each simdgroup outputs one hidden element
- const uint h_idx = h_block_idx * 4 + simd.threadgroup_index;
+ const uint h_idx = h_block_idx * 4 + thread_context.threadgroup_index;
if (h_idx >= d_ff)
return;
@@ -66,7 +66,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
const uint vec_iters = d_model / 128;
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd.simdgroup_index * 4;
+ uint base_idx = i * 128 + thread_context.simdgroup_index * 4;
device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);
device const T* w_up_vec =
@@ -87,7 +87,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
}
// Handle leftover elements
- uint leftover_start = vec_iters * 128 + simd.simdgroup_index;
+ uint leftover_start = vec_iters * 128 + thread_context.simdgroup_index;
for (uint idx = leftover_start; idx < d_model; idx += 32) {
float xv = float(x_ptr[idx]);
acc_up += xv * float(w_up_row[idx]);
@@ -103,7 +103,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
}
// Lane 0 applies activation and writes result
- if (simd.simdgroup_index == 0) {
+ if (thread_context.simdgroup_index == 0) {
float up_val = acc_up + float(up_biases[bias_base + h_idx]);
up_val = clamp(up_val, up_clip_min, up_clip_max);
@@ -142,7 +142,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
constant uint& d_model,
constant uint& d_ff,
constant uint& e,
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint tgpig_x GROUPS(d_model.div_ceil(SIMDGROUPS_PER_TG)),
const uint tgpig_y GROUPS(total_rows),
const uint tid THREADS(256)
@@ -150,7 +150,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
const uint row_idx = tgpig_y;
// Each simdgroup computes one output column
- const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd.threadgroup_index;
+ const uint my_col =
+ tgpig_x * SIMDGROUPS_PER_TG + thread_context.threadgroup_index;
if (my_col >= d_model)
return;
@@ -171,7 +172,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
const uint k_vec_iters = k_iters / 8;
for (uint iter = 0; iter < k_vec_iters; ++iter) {
- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd.simdgroup_index;
+ const uint k_base =
+ iter * (8 * THREADS_PER_SIMD) + thread_context.simdgroup_index;
// hidden: stride-32 per lane, already f32 from Pass A
const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
@@ -217,7 +219,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle remaining full iterations
for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
- const uint k = iter * THREADS_PER_SIMD + simd.simdgroup_index;
+ const uint k = iter * THREADS_PER_SIMD + thread_context.simdgroup_index;
acc =
fma(AccumT(hidden[hidden_base + k]),
AccumT(w2_all[w2_col_base + k]),
@@ -226,8 +228,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle leftover elements (d_ff % 32)
const uint leftover_start = k_iters * THREADS_PER_SIMD;
- if (leftover_start + simd.simdgroup_index < d_ff) {
- const uint k = leftover_start + simd.simdgroup_index;
+ if (leftover_start + thread_context.simdgroup_index < d_ff) {
+ const uint k = leftover_start + thread_context.simdgroup_index;
acc =
fma(AccumT(hidden[hidden_base + k]),
AccumT(w2_all[w2_col_base + k]),
@@ -238,7 +240,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
AccumT result = simd_sum(acc);
// Lane 0 writes result
- if (simd.simdgroup_index == 0) {
+ if (thread_context.simdgroup_index == 0) {
const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
result += AccumT(down_biases[bias_idx]);
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index f91b44dbc..76176d392 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -60,7 +60,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
const uint n_tile_idx GROUPS(INDIRECT),
const uint row_tile_idx GROUPS(INDIRECT),
const uint local_tid_x THREADS(128),
- const ThreadContext simd
+ const ThreadContext thread_context
) {
const uint base = row_tile_idx * 3u;
const uint expert_idx = tile_map[base + 0u];
@@ -106,15 +106,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
// simdgroup tile mapping
const uint sg_col_count = Bn / SgBn;
- const uint row_sg = simd.threadgroup_index / sg_col_count;
- const uint col_sg = simd.threadgroup_index % sg_col_count;
+ const uint row_sg = thread_context.threadgroup_index / sg_col_count;
+ const uint col_sg = thread_context.threadgroup_index % sg_col_count;
const uint row_sg_off = row_sg * SgBm;
const uint col_sg_off = col_sg * SgBn;
// Guard against misconfigured TG sizes
constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);
- if (simd.threadgroup_index >= SG_EXPECTED)
+ if (thread_context.threadgroup_index >= SG_EXPECTED)
return;
// partial accumulators (8x8 tiles per simdgroup)
@@ -295,7 +295,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Epilogue: activation/gating → hidden_out (f32) ----
- const uint sg_lane_start = simd.threadgroup_index * 32;
+ const uint sg_lane_start = thread_context.threadgroup_index * 32;
const uint sg_lane_end = sg_lane_start + 32;
if (lin < sg_lane_start || lin >= sg_lane_end)
return;
@@ -404,7 +404,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)],
threadgroup T Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)],
threadgroup float bias_tile[PASSB_BN],
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint n_tile_idx GROUPS(INDIRECT),
const uint row_tile_idx GROUPS(INDIRECT),
const uint lin THREADS(128)
@@ -446,14 +446,14 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
// 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)
- const uint row_sg = simd.threadgroup_index / 2; // 0-1 → rows
- const uint col_sg = simd.threadgroup_index % 2; // 0-1 → cols
+ const uint row_sg = thread_context.threadgroup_index / 2; // 0-1 → rows
+ const uint col_sg = thread_context.threadgroup_index % 2; // 0-1 → cols
const uint row_sg_off = row_sg * SgBm;
const uint col_sg_off = col_sg * SgBn;
// Guard against excessive simdgroups
constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);
- if (simd.threadgroup_index >= SG_EXPECTED)
+ if (thread_context.threadgroup_index >= SG_EXPECTED)
return;
// 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles
@@ -561,10 +561,11 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Writeout with fragment extraction ----
- const uint lane_qid = simd.simdgroup_index >> 2;
- const uint lane_row = (lane_qid & 4u) + ((simd.simdgroup_index >> 1) & 3u);
+ const uint lane_qid = thread_context.simdgroup_index >> 2;
+ const uint lane_row =
+ (lane_qid & 4u) + ((thread_context.simdgroup_index >> 1) & 3u);
const uint lane_col_base =
- ((lane_qid & 2u) << 1) + ((simd.simdgroup_index & 1u) << 1);
+ ((lane_qid & 2u) << 1) + ((thread_context.simdgroup_index & 1u) << 1);
const uint local_row = row_sg_off + lane_row;
if (local_row >= m_rows)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
index ad16c6b40..f88fecc4c 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -30,7 +30,7 @@ PUBLIC KERNEL(MoeRouterTopK)(
threadgroup uint reduce_tmp_u[THREADS_PER_TG],
threadgroup uint shared_best_idx[1],
threadgroup float shared_best_val[1],
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint tgpig_x GROUPS(1),
const uint token_idx GROUPS(t),
const uint lid THREADS(256)
@@ -52,12 +52,12 @@ PUBLIC KERNEL(MoeRouterTopK)(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint row = simd.threadgroup_index; row < e;
- row += simd.simdgroups_per_threadgroup) {
+ for (uint row = thread_context.threadgroup_index; row < e;
+ row += thread_context.simdgroups_per_threadgroup) {
const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;
float4 accum4 = float4(0.0f);
- for (uint c = simd.simdgroup_index; c < vecs; c += 32u) {
+ for (uint c = thread_context.simdgroup_index; c < vecs; c += 32u) {
const float4 wv = float4(
w_vec[c * 4 + 0],
w_vec[c * 4 + 1],
@@ -100,7 +100,7 @@ PUBLIC KERNEL(MoeRouterTopK)(
local_best,
reduce_tmp,
lid,
- simd
+ thread_context
);
uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;
@@ -108,7 +108,7 @@ PUBLIC KERNEL(MoeRouterTopK)(
candidate_id,
reduce_tmp_u,
lid,
- simd
+ thread_context
);
if (lid == 0) {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index 6cc4b4701..3eade2617 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -30,7 +30,7 @@ inline void moe_scatter_buckets_impl(
uint3 tgpig,
threadgroup _atomic<uint>* sg_counts,
threadgroup uint* sg_base,
- const thread ThreadContext& simd
+ const thread ThreadContext& thread_context
) {
if (E == 0 || T == 0 || K == 0)
return;
@@ -65,7 +65,7 @@ inline void moe_scatter_buckets_impl(
if (ue >= e0 && ue < e0 + tile_e) {
const uint te = ue - e0;
atomic_fetch_add_explicit(
- &sg_counts[simd.threadgroup_index * TILE_E + te],
+ &sg_counts[thread_context.threadgroup_index * TILE_E + te],
1u,
memory_order_relaxed
);
@@ -103,8 +103,8 @@ inline void moe_scatter_buckets_impl(
threadgroup_barrier(mem_flags::mem_threadgroup);
// Phase 3: deterministic write with 32-lane sequencing, no device atomics
- for (uint step = 0; step < simd.simdgroup_size; ++step) {
- if (simd.simdgroup_index == step) {
+ for (uint step = 0; step < thread_context.simdgroup_size; ++step) {
+ if (thread_context.simdgroup_index == step) {
const uint t = t_start + lid;
if (t < t_end) {
const uint base = t * K;
@@ -117,7 +117,8 @@ inline void moe_scatter_buckets_impl(
const uint tile_id = e0 / TILE_E;
const uint base_idx_block =
(block_id * num_tiles + tile_id) * TILE_E + te;
- const uint idx_sg_te = simd.threadgroup_index * TILE_E + te;
+ const uint idx_sg_te =
+ thread_context.threadgroup_index * TILE_E + te;
const uint local = atomic_fetch_add_explicit(
&sg_counts[idx_sg_te],
1u,
@@ -202,7 +203,7 @@ PUBLIC KERNEL(MoeScatterBuckets)(
constant uint& num_tiles,
threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
threadgroup uint sg_base[NUM_SG * TILE_E],
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint tgpig_x GROUPS(num_blocks),
const uint lid THREADS(256)
) {
@@ -228,7 +229,7 @@ PUBLIC KERNEL(MoeScatterBuckets)(
tgpig,
sg_counts,
sg_base,
- simd
+ thread_context
);
}
@@ -251,7 +252,7 @@ PUBLIC KERNEL(MoeScatterBucketsMap)(
device int* tok2row,
threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
threadgroup uint sg_base[NUM_SG * TILE_E],
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint tgpig_x GROUPS(num_blocks),
const uint lid THREADS(256)
) {
@@ -277,6 +278,6 @@ PUBLIC KERNEL(MoeScatterBucketsMap)(
tgpig,
sg_counts,
sg_base,
- simd
+ thread_context
);
}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index eb0d04ac4..d08730f49 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -25,7 +25,7 @@ PUBLIC KERNEL(RMSNorm)(
constant bool& full_layer,
const bool in_place SPECIALIZE,
threadgroup AccumT shared_sum[METAL_SIMD_SIZE],
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint batch_idx GROUPS(batch_size),
const uint thread_in_row THREADS(1024)
) {
@@ -60,7 +60,7 @@ PUBLIC KERNEL(RMSNorm)(
partial_sum,
shared_sum,
thread_in_row,
- simd
+ thread_context
);
// Compute RMS norm factor
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index 86939cf03..497ffa642 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -27,7 +27,7 @@ static ArgmaxPair threadgroup_cooperative_argmax(
ArgmaxPair value,
threadgroup ArgmaxPair* shared,
const ushort lid,
- const thread ThreadContext& simd
+ const thread ThreadContext& thread_context
) {
// Reduce within simdgroup using manual shuffle operations
ArgmaxPair local_result = value;
@@ -40,8 +40,8 @@ static ArgmaxPair threadgroup_cooperative_argmax(
}
// First thread in each simdgroup writes to shared memory
- if (simd.simdgroup_index == 0) {
- shared[simd.threadgroup_index] = local_result;
+ if (thread_context.simdgroup_index == 0) {
+ shared[thread_context.threadgroup_index] = local_result;
}
// Synchronize across the threadgroup
@@ -203,7 +203,7 @@ PUBLIC KERNEL(ArgmaxMain)(
constant uint& batch_size,
constant uint& vocab_size,
threadgroup ArgmaxPair shared[BLOCK_SIZE],
- const ThreadContext simd,
+ const ThreadContext thread_context,
uint batch_idx GROUPS(batch_size),
uint vocab_group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),
ushort local_id THREADS(BLOCK_SIZE)
@@ -230,7 +230,7 @@ PUBLIC KERNEL(ArgmaxMain)(
thread_result,
shared,
local_id,
- simd
+ thread_context
);
if (local_id == 0) {
@@ -247,7 +247,7 @@ PUBLIC KERNEL(ArgmaxFinal)(
constant uint& batch_size,
constant uint& vocab_size,
threadgroup ArgmaxPair shared[BLOCK_SIZE],
- const ThreadContext simd,
+ const ThreadContext thread_context,
uint batch_idx GROUPS(batch_size),
ushort local_id THREADS(BLOCK_SIZE)
) {
@@ -271,7 +271,7 @@ PUBLIC KERNEL(ArgmaxFinal)(
thread_result,
shared,
local_id,
- simd
+ thread_context
);
if (local_id == 0) {
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
index 121a1cca3..9b3962188 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
@@ -15,7 +15,7 @@ PUBLIC KERNEL(MinP) (
constant uint& batch_size,
constant uint& vocab_size,
constant float& min_p,
- const ThreadContext simd,
+ const ThreadContext thread_context,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE),
const bool in_place SPECIALIZE
@@ -37,7 +37,7 @@ PUBLIC KERNEL(MinP) (
local_max,
shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
// Then the threshold is just max_logit + log(min_p), mask everything strictly
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index d921385b2..c4c08b604 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -16,7 +16,7 @@ PUBLIC KERNEL(TopK)(
constant uint& batch_size,
constant uint& vocab_size,
constant uint& top_k,
- const ThreadContext simd,
+ const ThreadContext thread_context,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE),
const bool in_place SPECIALIZE
@@ -44,13 +44,13 @@ PUBLIC KERNEL(TopK)(
local_max,
shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
local_min,
shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
// Do the binary search on the threshold
float low = min_logit;
@@ -69,7 +69,7 @@ PUBLIC KERNEL(TopK)(
local_num_above_threshold,
(threadgroup uint*)shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
// Update binary search
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index cd4f01774..1ab64c2b7 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -17,7 +17,7 @@ PUBLIC KERNEL(TopP) (
constant uint& batch_size,
constant uint& vocab_size,
constant float& top_p,
- const ThreadContext simd,
+ const ThreadContext thread_context,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE),
const bool in_place SPECIALIZE
@@ -45,13 +45,13 @@ PUBLIC KERNEL(TopP) (
local_max,
shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
local_min,
shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
// Find denominator for softmax
@@ -69,7 +69,7 @@ PUBLIC KERNEL(TopP) (
local_sum,
shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
// Do the binary search on the threshold
@@ -98,13 +98,13 @@ PUBLIC KERNEL(TopP) (
local_sum_above_threshold,
shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
local_min_above_threshold,
shared_reduce_buffer,
thread_idx,
- simd
+ thread_context
);
// Early exit
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index f84606b78..d9b4001f3 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -28,7 +28,7 @@ PUBLIC KERNEL(SSDPrefill64)(
constant const uint* state_strides,
constant const uint& num_heads,
constant const uint& head_dim,
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint pair_idx GROUPS(num_heads * head_dim),
const uint lane_idx THREADS(32)
) {
@@ -56,7 +56,7 @@ PUBLIC KERNEL(SSDPrefill64)(
const float d_scalar = float(d[h_idx]);
const uint idx0 = lane_idx;
- const uint idx1 = lane_idx + simd.simdgroup_size;
+ const uint idx1 = lane_idx + thread_context.simdgroup_size;
float state0 = 0.0f;
float state1 = 0.0f;
@@ -126,7 +126,7 @@ PUBLIC KERNEL(SSDPrefill)(
constant const uint* state_strides,
constant const uint& num_heads,
constant const uint& head_dim,
- const ThreadContext simd,
+ const ThreadContext thread_context,
const uint pair_idx GROUPS(num_heads * head_dim),
const uint lane_idx THREADS(32)
) {
@@ -154,9 +154,9 @@ PUBLIC KERNEL(SSDPrefill)(
const uint cb_group_base = group_idx * cb_group_stride;
const float d_scalar = float(d[h_idx]);
- const int max_chunks = SSM_PREFILL_MAX_STATE / simd.simdgroup_size;
- const int chunk_count =
- (state_dim + int(simd.simdgroup_size) - 1) / int(simd.simdgroup_size);
+ const int max_chunks = SSM_PREFILL_MAX_STATE / thread_context.simdgroup_size;
+ const int chunk_count = (state_dim + int(thread_context.simdgroup_size) - 1) /
+ int(thread_context.simdgroup_size);
if (state_dim <= 0 || chunk_count > max_chunks) {
return;
}
@@ -165,7 +165,7 @@ PUBLIC KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);
+ const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);
const uint state_idx = state_base + idx * state_inner_stride;
lane_states[chunk] = float(state[state_idx]);
}
@@ -183,7 +183,8 @@ PUBLIC KERNEL(SSDPrefill)(
float contrib_sum = 0.0f;
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);
+ const int idx =
+ chunk * int(thread_context.simdgroup_size) + int(lane_idx);
const uint cb_idx =
cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;
const float new_state =
@@ -200,7 +201,7 @@ PUBLIC KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);
+ const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);
const uint state_idx = state_base + uint(idx) * state_inner_stride;
state[state_idx] = static_cast<T>(lane_states[chunk]);
}
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index 4e6deb496..a630ffefb 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -11,7 +11,7 @@ use crate::{
Backend, CommandBuffer, Kernels,
kernel::{
FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,
- matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},
+ matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},
quant_matmul::{
QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
QuantizedMatmulKernelEncodable, QuantizedMatmulType,
@@ -47,7 +47,7 @@ enum TiedEmbeddingType<B: Backend> {
FullPrecision {
weights: B::Buffer,
lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
- readout: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
+ readout: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,
},
Quantized {
weights: B::Buffer,
@@ -74,7 +74,7 @@ enum UntiedEmbeddingLookupType<B: Backend> {
enum UntiedEmbeddingReadoutType<B: Backend> {
FullPrecision {
weights: B::Buffer,
- readout: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
+ readout: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,
},
Quantized {
weights: B::Buffer,
@@ -153,8 +153,7 @@ impl<B: Backend> Embedding<B> {
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
- let readout =
- RefCell::new(<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?);
+ let readout = RefCell::new(<B::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)?);
EmbeddingTying::Tied {
ty: TiedEmbeddingType::FullPrecision {
@@ -180,8 +179,7 @@ impl<B: Backend> Embedding<B> {
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
- let readout =
- RefCell::new(<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?);
+ let readout = RefCell::new(<B::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)?);
EmbeddingTying::Untied {
input_ty: UntiedEmbeddingLookupType::FullPrecision {
@@ -553,19 +551,25 @@ impl<B: Backend> Embedding<B> {
readout,
},
} => {
+ let input_dim = self.model_dim as usize;
+ let output_dim = self.vocab_size as usize;
readout.borrow_mut().encode(
state.context(),
- command_buffer,
- FullPrecisionMatmulArguments {
+ MatmulArguments {
a: input,
- a_offset: input_offset,
+ a_offset: input_offset as u64,
b: weights,
- output,
+ d: output,
bias: None,
- batch: batch_size,
- input_dim: self.model_dim as usize,
- output_dim: self.vocab_size as usize,
+ batch: batch_size as i32,
+ input_dim: input_dim as i32,
+ output_dim: output_dim as i32,
+ leading_dimension_a: input_dim as i32,
+ leading_dimension_b: input_dim as i32,
+ leading_dimension_d: output_dim as i32,
+ transpose_b: true,
},
+ command_buffer,
);
},
EmbeddingTying::Tied {
diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rs
index 2416b1bb2..e54049ec7 100644
--- a/crates/uzu/src/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs
@@ -11,7 +11,7 @@ use crate::{
DataType,
backends::common::{
Backend, CommandBuffer,
- kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},
+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
@@ -49,7 +49,7 @@ pub enum FullPrecisionLinearError<B: Backend> {
}
pub struct FullPrecisionLinear<B: Backend> {
- kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
+ kernel: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,
bias_buffer: Option<Rc<RefCell<B::Buffer>>>,
weights_buffer: Rc<RefCell<B::Buffer>>,
input_dim: usize,
@@ -111,7 +111,7 @@ impl<B: Backend> FullPrecisionLinear<B> {
Err(_) => None,
};
- let kernel = <B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, precision)?;
+ let kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(context, precision)?;
Ok(Self {
kernel: RefCell::new(kernel),
@@ -139,17 +139,21 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());
self.kernel.borrow_mut().encode(
state.context(),
- command_buffer,
- FullPrecisionMatmulArguments {
+ MatmulArguments {
a: input_array.buffer().borrow().deref(),
a_offset: 0,
b: self.weights_buffer.borrow().deref(),
- output: output_array.buffer().borrow_mut().deref_mut(),
+ d: output_array.buffer().borrow_mut().deref_mut(),
bias: bias_borrow.as_deref(),
- batch: batch_size,
- input_dim: self.input_dim,
- output_dim: self.output_dim,
+ batch: batch_size as i32,
+ input_dim: self.input_dim as i32,
+ output_dim: self.output_dim as i32,
+ leading_dimension_a: self.input_dim as i32,
+ leading_dimension_b: self.input_dim as i32,
+ leading_dimension_d: self.output_dim as i32,
+ transpose_b: true,
},
+ command_buffer,
);
Ok(())
}
diff --git a/crates/uzu/tests/kernel/matmul/gemv_test.rs b/crates/uzu/tests/kernel/matmul/gemv_test.rs
index 2381dfe8f..0b3160835 100644
--- a/crates/uzu/tests/kernel/matmul/gemv_test.rs
+++ b/crates/uzu/tests/kernel/matmul/gemv_test.rs
@@ -12,10 +12,7 @@ use uzu::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context,
- kernel::matmul::{
- MatmulArguments,
- gemv::{GemvDispatchDescriptor, GemvKernel},
- },
+ kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
},
cpu::Cpu,
},
@@ -69,29 +66,28 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let d_buf = d_array.buffer();
let mut d_ref = d_buf.borrow_mut();
- let mut arguments = MatmulArguments {
- a: a_ref.deref(),
- a_offset: 0,
- b: b_ref.deref(),
- d: d_ref.deref_mut(),
- bias: None,
- batch: m,
- input_dim: k,
- output_dim: n,
- leading_dimension_a: k,
- leading_dimension_b: k,
- leading_dimension_d: n,
- transpose_b: true,
- };
-
- let descriptor = GemvDispatchDescriptor::try_new::<B>(T::data_type(), &arguments)
- .expect("Failed to create descriptor")
- .expect("GEMV not applicable for these dimensions");
-
- let mut kernel = GemvKernel::<B>::new(T::data_type()).expect("Failed to create GemvKernel");
+ let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())
+ .expect("Failed to create MatmulKernel");
let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel.encode(&context, &mut arguments, &descriptor, &mut command_buffer).expect("Failed to encode");
+ kernel.encode(
+ &context,
+ MatmulArguments {
+ a: a_ref.deref(),
+ a_offset: 0,
+ b: b_ref.deref(),
+ d: d_ref.deref_mut(),
+ bias: None,
+ batch: m,
+ input_dim: k,
+ output_dim: n,
+ leading_dimension_a: k,
+ leading_dimension_b: k,
+ leading_dimension_d: n,
+ transpose_b: true,
+ },
+ &mut command_buffer,
+ );
command_buffer.end_encoding().submit().wait_until_completed().unwrap();
drop(d_ref);
@@ -111,7 +107,6 @@ fn test<T: ArrayElement + Float + Debug + Display>(
});
}
-// M=1 (true vector-matrix multiply)
#[test]
fn test_f32_m1() {
test::<f32>(1, 128, 64, 0.01);
@@ -127,7 +122,6 @@ fn test_bf16_m1() {
test::<bf16>(1, 128, 64, 0.1);
}
-// Small batch (M <= 8, batched GEMV)
#[test]
fn test_f32_batched() {
test::<f32>(4, 128, 64, 0.01);
@@ -143,7 +137,6 @@ fn test_bf16_batched() {
test::<bf16>(4, 128, 64, 0.1);
}
-// Max batch (M=8)
#[test]
fn test_f32_max_batch() {
test::<f32>(8, 128, 64, 0.01);
@@ -159,7 +152,6 @@ fn test_bf16_max_batch() {
test::<bf16>(8, 128, 64, 0.1);
}
-// Unaligned K dimension
#[test]
fn test_f32_unaligned_k() {
test::<f32>(1, 33, 64, 0.01);
@@ -175,7 +167,6 @@ fn test_bf16_unaligned_k() {
test::<bf16>(1, 33, 64, 0.1);
}
-// Unaligned N dimension
#[test]
fn test_f32_unaligned_n() {
test::<f32>(1, 128, 11, 0.01);
@@ -191,7 +182,6 @@ fn test_bf16_unaligned_n() {
test::<bf16>(1, 128, 11, 0.1);
}
-// Large dimensions
#[test]
fn test_f32_large() {
test::<f32>(1, 4096, 2048, 0.05);
@@ -207,7 +197,6 @@ fn test_bf16_large() {
test::<bf16>(1, 4096, 2048, 1.0);
}
-// Small N (< 4, triggers elements_per_thread_col=1 path)
#[test]
fn test_f32_small_n() {
test::<f32>(1, 128, 3, 0.01);
diff --git a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
index 70c4d305f..2b73062f5 100644
--- a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
+++ b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
@@ -12,11 +12,7 @@ use uzu::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context,
- gpu_types::GemmParams,
- kernel::matmul::{
- GridSize, MatmulArguments,
- gemm::{GemmDispatchDescriptor, GemmKernel, GemmSpecialization},
- },
+ kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
},
cpu::Cpu,
},
@@ -63,52 +59,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");
let d_array = context.create_array_uninitialized(&[input.m, input.n], T::data_type(), "");
- // Pick a precompile config matching alignment requirements
- let configs = GemmSpecialization::precompile_configs(T::data_type());
- let base = configs[0];
- let align_m = (m % base.block_rows) == 0;
- let align_n = (n % base.block_cols) == 0;
- let align_k = (k % base.block_depth) == 0;
- // Find a config that matches alignment, or fall back to overriding the first one
- let config = configs
- .iter()
- .find(|c| c.align_m == align_m && c.align_n == align_n && c.align_k == align_k)
- .copied()
- .unwrap_or(GemmSpecialization {
- align_m,
- align_n,
- align_k,
- ..base
- });
-
- let threadgroups_per_row = (n + config.block_cols - 1) / config.block_cols;
- let threadgroups_per_column = (m + config.block_rows - 1) / config.block_rows;
-
- let params = GemmParams {
- M: m,
- N: n,
- K: k,
- leading_dimension_a: k,
- leading_dimension_b: k,
- leading_dimension_d: n,
- threadgroups_per_row,
- threadgroups_per_column,
- swizzle_log: 0,
- aligned_inner_iterations: k / config.block_depth,
- };
-
- let descriptor = GemmDispatchDescriptor {
- specialization: config,
- params,
- threadgroups: GridSize {
- x: threadgroups_per_row as usize,
- y: threadgroups_per_column as usize,
- z: 1,
- },
- };
-
- let mut kernel = GemmKernel::<B>::new(T::data_type()).expect("Failed to create GemmKernel");
-
let a_buf = a_array.buffer();
let a_ref = a_buf.borrow();
let b_buf = b_array.buffer();
@@ -116,22 +66,28 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let d_buf = d_array.buffer();
let mut d_ref = d_buf.borrow_mut();
+ let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())
+ .expect("Failed to create MatmulKernel");
+
let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- let mut arguments = MatmulArguments {
- a: a_ref.deref(),
- a_offset: 0,
- b: b_ref.deref(),
- d: d_ref.deref_mut(),
- bias: None,
- batch: m,
- input_dim: k,
- output_dim: n,
- leading_dimension_a: k,
- leading_dimension_b: k,
- leading_dimension_d: n,
- transpose_b: true,
- };
- kernel.encode(&context, &mut arguments, &descriptor, &mut command_buffer).expect("Failed to encode");
+ kernel.encode(
+ &context,
+ MatmulArguments {
+ a: a_ref.deref(),
+ a_offset: 0,
+ b: b_ref.deref(),
+ d: d_ref.deref_mut(),
+ bias: None,
+ batch: m,
+ input_dim: k,
+ output_dim: n,
+ leading_dimension_a: k,
+ leading_dimension_b: k,
+ leading_dimension_d: n,
+ transpose_b: true,
+ },
+ &mut command_buffer,
+ );
command_buffer.end_encoding().submit().wait_until_completed().unwrap();
drop(d_ref);
@@ -151,7 +107,6 @@ fn test<T: ArrayElement + Float + Debug + Display>(
});
}
-// Aligned dimensions (divisible by common block sizes)
#[test]
fn test_f32_aligned() {
test::<f32>(64, 64, 64, 0.01);
@@ -167,7 +122,6 @@ fn test_bf16_aligned() {
test::<bf16>(64, 64, 64, 0.1);
}
-// Unaligned dimensions
#[test]
fn test_f32_unaligned() {
test::<f32>(7, 33, 11, 0.01);
@@ -183,7 +137,6 @@ fn test_bf16_unaligned() {
test::<bf16>(7, 33, 11, 0.1);
}
-// Larger matrix
#[test]
fn test_f32_large() {
test::<f32>(16, 128, 256, 0.01);
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 802873345..1608ba0ed 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -10,7 +10,7 @@ use uzu::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context,
- kernel::matmul::{MatmulArguments, MatmulKernel, choose_matmul_dispatch_descriptor},
+ kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
},
metal::Metal,
},
@@ -44,26 +44,28 @@ fn run_metal_matmul(
n
};
- let mut kernel = MatmulKernel::<Metal>::new(DataType::BF16).expect("kernel");
+ let mut kernel =
+ <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");
let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
- let arguments = MatmulArguments {
- a: &a_buf,
- a_offset: 0,
- b: &b_buf,
- d: &mut d_buf,
- bias: None,
- batch: m as i32,
- input_dim: k as i32,
- output_dim: n as i32,
- leading_dimension_a: k as i32,
- leading_dimension_b: leading_dim_b as i32,
- leading_dimension_d: n as i32,
- transpose_b,
- };
- let descriptor =
- choose_matmul_dispatch_descriptor::<Metal>(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");
- kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut command_buffer).expect("encode");
+ kernel.encode(
+ ctx,
+ MatmulArguments {
+ a: &a_buf,
+ a_offset: 0,
+ b: &b_buf,
+ d: &mut d_buf,
+ bias: None,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ leading_dimension_a: k as i32,
+ leading_dimension_b: leading_dim_b as i32,
+ leading_dimension_d: n as i32,
+ transpose_b,
+ },
+ &mut command_buffer,
+ );
command_buffer.end_encoding().submit().wait_until_completed().unwrap();
unsafe {
diff --git a/crates/uzu/tests/matmul_perf_test/bench.rs b/crates/uzu/tests/matmul_perf_test/bench.rs
index 460f3ed83..224a9aeb2 100644
--- a/crates/uzu/tests/matmul_perf_test/bench.rs
+++ b/crates/uzu/tests/matmul_perf_test/bench.rs
@@ -6,7 +6,7 @@ use uzu::{
common::{
Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
CommandBufferPending, Context,
- kernel::matmul::{MatmulArguments, MatmulDispatchDescriptor, MatmulKernel},
+ kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
},
metal::Metal,
},
@@ -31,44 +31,34 @@ fn fill_buffer_random(
}
}
-pub fn make_arguments<'a>(
- a_buffer: &'a Buf,
- b_buffer: &'a Buf,
- d_buffer: &'a mut Buf,
- shape: &TestShape,
-) -> MatmulArguments<'a, Metal> {
- MatmulArguments {
- a: a_buffer,
- a_offset: 0,
- b: b_buffer,
- d: d_buffer,
- bias: None,
- batch: shape.batch as i32,
- input_dim: shape.input_dim as i32,
- output_dim: shape.output_dim as i32,
- leading_dimension_a: shape.input_dim as i32,
- leading_dimension_b: shape.input_dim as i32,
- leading_dimension_d: shape.output_dim as i32,
- transpose_b: true,
- }
-}
-
fn encode_and_run(
context: &Ctx,
- kernel: &mut MatmulKernel<Metal>,
+ kernel: &mut <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel,
shape: &TestShape,
a_buffer: &Buf,
b_buffer: &Buf,
d_buffer: &mut Buf,
- dispatch_descriptor: &MatmulDispatchDescriptor,
) -> Result<f64, BenchError> {
let mut command_buffer = context.create_command_buffer().map_err(|_| BenchError::CommandBuffer)?.start_encoding();
- let arguments = make_arguments(a_buffer, b_buffer, d_buffer, shape);
-
- kernel
- .encode_with_descriptor(context, arguments, dispatch_descriptor, &mut command_buffer)
- .map_err(|e| BenchError::Encode(e.to_string()))?;
+ kernel.encode(
+ context,
+ MatmulArguments {
+ a: a_buffer,
+ a_offset: 0,
+ b: b_buffer,
+ d: d_buffer,
+ bias: None,
+ batch: shape.batch as i32,
+ input_dim: shape.input_dim as i32,
+ output_dim: shape.output_dim as i32,
+ leading_dimension_a: shape.input_dim as i32,
+ leading_dimension_b: shape.input_dim as i32,
+ leading_dimension_d: shape.output_dim as i32,
+ transpose_b: true,
+ },
+ &mut command_buffer,
+ );
let completed =
command_buffer.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;
@@ -80,9 +70,9 @@ fn run_benchmark(
context: &Ctx,
data_type: DataType,
shape: &TestShape,
- dispatch_descriptor: &MatmulDispatchDescriptor,
) -> Result<f64, BenchError> {
- let mut kernel = MatmulKernel::<Metal>::new(data_type).map_err(|e| BenchError::Kernel(e.to_string()))?;
+ let mut kernel = <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)
+ .map_err(|e| BenchError::Kernel(e.to_string()))?;
let elem_size = data_type.size_in_bytes();
let a_byte_count = shape.batch * shape.input_dim * elem_size;
@@ -102,22 +92,23 @@ fn run_benchmark(
fill_buffer_random(&b_buffer, b_byte_count);
for iteration in 0..WARMUP_ITERATIONS {
- encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer, dispatch_descriptor).map_err(
- |source| BenchError::Warmup {
+ encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer).map_err(|source| {
+ BenchError::Warmup {
iteration,
source: Box::new(source),
- },
- )?;
+ }
+ })?;
}
let mut gpu_time_total_ms = 0.0;
for iteration in 0..BENCHMARK_ITERATIONS {
let gpu_ms =
- encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer, dispatch_descriptor)
- .map_err(|source| BenchError::Benchmark {
+ encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer).map_err(|source| {
+ BenchError::Benchmark {
iteration,
source: Box::new(source),
- })?;
+ }
+ })?;
gpu_time_total_ms += gpu_ms;
}
@@ -128,17 +119,15 @@ pub fn benchmark_single(
context: &Ctx,
data_type: DataType,
shape: &TestShape,
- path_name: &str,
- dispatch_descriptor: &MatmulDispatchDescriptor,
) -> PerfResult {
- match run_benchmark(context, data_type, shape, dispatch_descriptor) {
+ match run_benchmark(context, data_type, shape) {
Ok(duration_ms) => {
let flops = 2.0 * shape.batch as f64 * shape.input_dim as f64 * shape.output_dim as f64;
let gflops = flops / (duration_ms / 1000.0) / 1e9;
PerfResult {
combo: format!("{data_type:?}"),
shape: format!("{shape}"),
- dispatch_path: path_name.to_owned(),
+ dispatch_path: "auto".to_owned(),
duration_ms,
gflops,
status: "ok".into(),
@@ -148,7 +137,7 @@ pub fn benchmark_single(
Err(error) => PerfResult {
combo: format!("{data_type:?}"),
shape: format!("{shape}"),
- dispatch_path: path_name.to_owned(),
+ dispatch_path: "auto".to_owned(),
duration_ms: 0.0,
gflops: 0.0,
status: "error".into(),
diff --git a/crates/uzu/tests/matmul_perf_test/error.rs b/crates/uzu/tests/matmul_perf_test/error.rs
index 3d11b6d69..dfd690cfe 100644
--- a/crates/uzu/tests/matmul_perf_test/error.rs
+++ b/crates/uzu/tests/matmul_perf_test/error.rs
@@ -4,8 +4,6 @@ use thiserror::Error;
pub enum BenchError {
#[error("failed to create command buffer")]
CommandBuffer,
- #[error("encode failed: {0}")]
- Encode(String),
#[error("GPU timestamps unavailable")]
GpuTimestamps,
#[error("kernel creation failed: {0}")]
diff --git a/crates/uzu/tests/matmul_perf_test/main.rs b/crates/uzu/tests/matmul_perf_test/main.rs
index c12f4d959..9bfd1fffb 100644
--- a/crates/uzu/tests/matmul_perf_test/main.rs
+++ b/crates/uzu/tests/matmul_perf_test/main.rs
@@ -6,16 +6,13 @@ mod output;
mod shapes;
use indicatif::{ProgressBar, ProgressStyle};
-use metal::{MTLDeviceExt, MTLResourceOptions};
+use metal::MTLDeviceExt;
use output::print_results_table;
use shapes::test_shapes;
use uzu::{
DataType,
backends::{
- common::{
- Backend, Context,
- kernel::matmul::{MatmulDispatchDescriptor, choose_matmul_dispatch_descriptor},
- },
+ common::{Backend, Context},
metal::Metal,
},
};
@@ -56,47 +53,9 @@ fn matmul_perf() {
let mut results = Vec::new();
for &data_type in &data_types {
- let elem_size = data_type.size_in_bytes();
-
for shape in &test_shapes {
- let a_byte_count = shape.batch * shape.input_dim * elem_size;
- let b_byte_count = shape.output_dim * shape.input_dim * elem_size;
- let d_byte_count = shape.batch * shape.output_dim * elem_size;
-
- let (a_buffer, b_buffer, mut d_buffer) = match (
- context.device.new_buffer(a_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
- context.device.new_buffer(b_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
- context.device.new_buffer(d_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
- ) {
- (Some(a), Some(b), Some(d)) => (a, b, d),
- _ => continue,
- };
-
- let arguments = bench::make_arguments(&a_buffer, &b_buffer, &mut d_buffer, shape);
-
- let descriptor = match choose_matmul_dispatch_descriptor::<Metal>(&context, data_type, &arguments) {
- Ok(d) => d,
- Err(e) => {
- results.push(output::PerfResult {
- combo: format!("{data_type:?}"),
- shape: format!("{shape}"),
- dispatch_path: "auto".into(),
- duration_ms: 0.0,
- gflops: 0.0,
- status: "error".into(),
- error: Some(format!("dispatch: {e}")),
- });
- continue;
- },
- };
-
- let path_name = match &descriptor {
- MatmulDispatchDescriptor::Gemv(_) => "Gemv",
- MatmulDispatchDescriptor::Gemm(_) => "Gemm",
- };
-
- progress_bar.set_message(format!("{data_type:?} {shape} {path_name}"));
- let result = bench::benchmark_single(&context, data_type, shape, path_name, &descriptor);
+ progress_bar.set_message(format!("{data_type:?} {shape}"));
+ let result = bench::benchmark_single(&context, data_type, shape);
results.push(result);
progress_bar.inc(1);
}