A high-performance inference engine for AI models
mpp
eugenebb72fd42e321
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 7ed6c78f8..f916a75b6 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -59,7 +59,7 @@ walkdir.workspace = truedepfile.workspace = truetempfile.workspace = trueserde.workspace = true-serde_json.workspace = true+serde_json = { workspace = true, features = ["unbounded_depth"] }futures.workspace = truetokio.workspace = trueblake3.workspace = truediff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rsindex b5262ab8a..241fcdcde 100644--- a/crates/uzu/build/metal/toolchain.rs+++ b/crates/uzu/build/metal/toolchain.rs@@ -7,6 +7,7 @@ use std::{};use anyhow::{Context, bail};+use serde::Deserialize;use tempfile::NamedTempFile;use tokio::{io::AsyncWriteExt, process::Command};@@ -180,8 +181,11 @@ impl MetalToolchain {bail!("metal analyzer failed: {}", String::from_utf8_lossy(&analyze_output.stderr));}- let ast_root =- serde_json::from_slice::<MetalAstNode>(&analyze_output.stdout).context("cannot deserialize ast dump")?;+ let ast_root = {+ let mut deserializer = serde_json::Deserializer::from_slice(&analyze_output.stdout);+ deserializer.disable_recursion_limit();+ MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?+ };if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {bail!(diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rsindex a9c496065..26089436f 100644--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs@@ -13,4 +13,5 @@ pub struct GemmParams {pub threadgroups_per_row: u32,pub swizzle_log: u32,pub aligned_inner_iterations: u32,+ pub use_morton: bool,}diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.hindex c3fdf827e..07525ecdb 100644--- a/crates/uzu/src/backends/metal/generated/matmul.h+++ b/crates/uzu/src/backends/metal/generated/matmul.h@@ -25,6 +25,7 @@ typedef struct {uint32_t threadgroups_per_row;uint32_t swizzle_log;uint32_t aligned_inner_iterations;+ bool use_morton;} GemmParams;#ifdef __METAL_VERSION__diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.hnew file mode 100644index 000000000..c4e885f9f--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h@@ -0,0 +1,362 @@+#pragma once++#include "defines.h"+#include "loader.h"+#include "../../../generated/matmul.h"++#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>++using namespace metal;++namespace uzu {+namespace matmul {++template <+ typename T,+ short BLOCK_ROWS,+ short BLOCK_COLS,+ short SIMDGROUPS_PER_ROW,+ short SIMDGROUPS_PER_COLUMN,+ short SUBTILE_ROWS = 16,+ short SUBTILE_COLS = 32,+ short MATMUL_K_STEP = 16,+ // Number of cooperative tensor elements owned by each thread.+ // Determined by the MPP compiler intrinsic get_capacity().+ // Empirically: SUBTILE_ROWS * SUBTILE_COLS / SIMD_SIZE = 16 * 32 / 32 = 16.+ short ACCUMULATOR_CAPACITY = 16>+struct ThreadgroupGemmMpp {+ METAL_CONST short PREFETCH_K = 32;+ METAL_CONST short THREADGROUP_PADDING = short(16 / sizeof(T));+ METAL_CONST short THREADGROUP_LD = PREFETCH_K + THREADGROUP_PADDING;+ METAL_CONST short THREADGROUP_SIZE =+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;+ METAL_CONST short SIMDGROUP_BLOCK_M = BLOCK_ROWS / SIMDGROUPS_PER_ROW;+ METAL_CONST short SIMDGROUP_BLOCK_N = BLOCK_COLS / SIMDGROUPS_PER_COLUMN;+ METAL_CONST short TILES_M = SIMDGROUP_BLOCK_M / SUBTILE_ROWS;+ METAL_CONST short TILES_N = SIMDGROUP_BLOCK_N / SUBTILE_COLS;+ METAL_CONST short INNER_K_STEPS = PREFETCH_K / MATMUL_K_STEP;++ using AccumulatorType = float;++ template <+ typename MatmulOperation,+ typename LeftTensor,+ typename RightTensor,+ typename AccumulatorTensor>+ static METAL_FUNC void accumulate_tiles(+ const short k_steps,+ const ushort tile_row_offset,+ const ushort tile_col_offset,+ threadgroup T* left_shared,+ threadgroup T* right_shared,+ thread MatmulOperation& matmul_operation,+ thread LeftTensor& left_tensor,+ thread RightTensor& right_tensor,+ thread AccumulatorTensor& accumulator_tensor,+ thread AccumulatorType+ accumulator_storage[TILES_M * TILES_N][ACCUMULATOR_CAPACITY]+ ) {+ METAL_PRAGMA_UNROLL+ for (short tile_m = 0; tile_m < TILES_M; tile_m++) {+ METAL_PRAGMA_UNROLL+ for (short tile_n = 0; tile_n < TILES_N; tile_n++) {+ const short subtile_index = tile_m * TILES_N + tile_n;+ const short left_row_base = tile_row_offset + tile_m * SUBTILE_ROWS;+ const short right_col_base = tile_col_offset + tile_n * SUBTILE_COLS;++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < ACCUMULATOR_CAPACITY; i++) {+ accumulator_tensor[i] = accumulator_storage[subtile_index][i];+ }++ for (short k_step = 0; k_step < k_steps; k_step++) {+ const short k_offset = k_step * MATMUL_K_STEP;++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < left_tensor.get_capacity(); i++) {+ auto coord = left_tensor.get_multidimensional_index(i);+ left_tensor[i] = left_shared+ [(left_row_base + coord[1]) * THREADGROUP_LD + coord[0] ++ k_offset];+ }++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < right_tensor.get_capacity(); i++) {+ auto coord = right_tensor.get_multidimensional_index(i);+ right_tensor[i] = right_shared+ [(right_col_base + coord[1]) * THREADGROUP_LD + coord[0] ++ k_offset];+ }++ matmul_operation.run(left_tensor, right_tensor, accumulator_tensor);+ }++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < ACCUMULATOR_CAPACITY; i++) {+ accumulator_storage[subtile_index][i] = accumulator_tensor[i];+ }+ }+ }+ }++ static METAL_FUNC void run(+ const device T* left_matrix,+ const device T* right_matrix,+ device T* output_matrix,+ const constant GemmParams* params,+ const bool align_m,+ const bool align_n,+ const bool is_accumulate,+ const float ab_scale,+ threadgroup T* left_shared,+ threadgroup T* right_shared,+ uint simd_lane_id,+ uint simd_group_id,+ uint2 threadgroup_position+ ) {+ uint tile_id_x, tile_id_y;+ if (params->use_morton) {+ uint linear_id = threadgroup_position.x;+ uint morton_x = linear_id;+ uint morton_y = linear_id >> 1;+ morton_x &= 0x55555555u;+ morton_x = (morton_x | (morton_x >> 1)) & 0x33333333u;+ morton_x = (morton_x | (morton_x >> 2)) & 0x0F0F0F0Fu;+ morton_x = (morton_x | (morton_x >> 4)) & 0x00FF00FFu;+ morton_x = (morton_x | (morton_x >> 8)) & 0x0000FFFFu;+ morton_y &= 0x55555555u;+ morton_y = (morton_y | (morton_y >> 1)) & 0x33333333u;+ morton_y = (morton_y | (morton_y >> 2)) & 0x0F0F0F0Fu;+ morton_y = (morton_y | (morton_y >> 4)) & 0x00FF00FFu;+ morton_y = (morton_y | (morton_y >> 8)) & 0x0000FFFFu;+ tile_id_x = morton_x;+ tile_id_y = morton_y;+ } else {+ tile_id_x = threadgroup_position.x;+ tile_id_y = threadgroup_position.y;+ }++ if (tile_id_x >= params->threadgroups_per_row ||+ tile_id_y >= params->threadgroups_per_column) {+ return;+ }++ const uint block_row_start = tile_id_y * BLOCK_ROWS;+ const uint block_col_start = tile_id_x * BLOCK_COLS;+ const size_t block_row_start_long = size_t(block_row_start);+ const size_t block_col_start_long = size_t(block_col_start);++ const device T* left_block_ptr =+ left_matrix + block_row_start_long * params->leading_dimension_a;+ const device T* right_block_ptr =+ right_matrix + block_col_start_long * params->leading_dimension_b;++ ThreadgroupLoader<+ T,+ BLOCK_ROWS,+ PREFETCH_K,+ THREADGROUP_LD,+ 1,+ THREADGROUP_SIZE>+ left_loader(+ left_block_ptr,+ params->leading_dimension_a,+ left_shared,+ ushort(simd_group_id),+ ushort(simd_lane_id)+ );+ ThreadgroupLoader<+ T,+ BLOCK_COLS,+ PREFETCH_K,+ THREADGROUP_LD,+ 1,+ THREADGROUP_SIZE>+ right_loader(+ right_block_ptr,+ params->leading_dimension_b,+ right_shared,+ ushort(simd_group_id),+ ushort(simd_lane_id)+ );++ const ushort tile_row_offset =+ SIMDGROUP_BLOCK_M * (simd_group_id / SIMDGROUPS_PER_COLUMN);+ const ushort tile_col_offset =+ SIMDGROUP_BLOCK_N * (simd_group_id % SIMDGROUPS_PER_COLUMN);++ device T* output_ptr =+ output_matrix + block_row_start_long * params->leading_dimension_d ++ block_col_start_long + tile_row_offset * params->leading_dimension_d ++ tile_col_offset;++ const short simdgroup_limit_m =+ align_m+ ? SIMDGROUP_BLOCK_M+ : short(+ min(int(SIMDGROUP_BLOCK_M),+ int(params->M) - int(block_row_start + tile_row_offset))+ );+ const short simdgroup_limit_n =+ align_n+ ? SIMDGROUP_BLOCK_N+ : short(+ min(int(SIMDGROUP_BLOCK_N),+ int(params->N) - int(block_col_start + tile_col_offset))+ );++ constexpr auto matmul_descriptor = mpp::tensor_ops::matmul2d_descriptor(+ SUBTILE_ROWS,+ SUBTILE_COLS,+ MATMUL_K_STEP,+ false,+ true,+ false,+ mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate+ );++ mpp::tensor_ops::matmul2d<matmul_descriptor, metal::execution_simdgroup>+ matmul_operation;++ auto left_tensor =+ matmul_operation.template get_left_input_cooperative_tensor<+ T,+ T,+ AccumulatorType>();+ auto right_tensor =+ matmul_operation.template get_right_input_cooperative_tensor<+ T,+ T,+ AccumulatorType>();+ auto accumulator_tensor =+ matmul_operation.template get_destination_cooperative_tensor<+ decltype(left_tensor),+ decltype(right_tensor),+ AccumulatorType>();++ const uint leading_dimension_d = params->leading_dimension_d;++ AccumulatorType accumulator_storage[TILES_M * TILES_N]+ [ACCUMULATOR_CAPACITY] = {{0}};++ const uint full_prefetch_iterations = params->K / PREFETCH_K;+ const uint k_remainder = params->K - full_prefetch_iterations * PREFETCH_K;++ const ushort actual_block_rows =+ align_m ? BLOCK_ROWS+ : ushort(min(uint(BLOCK_ROWS), params->M - block_row_start));+ const ushort actual_block_cols =+ align_n ? BLOCK_COLS+ : ushort(min(uint(BLOCK_COLS), params->N - block_col_start));++ // Main loop+ for (uint outer_k = 0; outer_k < full_prefetch_iterations; outer_k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (align_m) {+ left_loader.load_unsafe();+ } else {+ left_loader.load_safe(short2(PREFETCH_K, actual_block_rows));+ }+ if (align_n) {+ right_loader.load_unsafe();+ } else {+ right_loader.load_safe(short2(PREFETCH_K, actual_block_cols));+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ accumulate_tiles(+ INNER_K_STEPS,+ tile_row_offset,+ tile_col_offset,+ left_shared,+ right_shared,+ matmul_operation,+ left_tensor,+ right_tensor,+ accumulator_tensor,+ accumulator_storage+ );++ left_loader.next();+ right_loader.next();+ }++ // Remainder loop+ if (k_remainder > 0) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ left_loader.load_safe(short2(k_remainder, actual_block_rows));+ right_loader.load_safe(short2(k_remainder, actual_block_cols));+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const short remainder_steps =+ short((k_remainder + MATMUL_K_STEP - 1) / MATMUL_K_STEP);++ accumulate_tiles(+ remainder_steps,+ tile_row_offset,+ tile_col_offset,+ left_shared,+ right_shared,+ matmul_operation,+ left_tensor,+ right_tensor,+ accumulator_tensor,+ accumulator_storage+ );+ }++ // Store results+ METAL_PRAGMA_UNROLL+ for (short tile_m = 0; tile_m < TILES_M; tile_m++) {+ METAL_PRAGMA_UNROLL+ for (short tile_n = 0; tile_n < TILES_N; tile_n++) {+ const short row_offset = tile_m * SUBTILE_ROWS;+ const short col_offset = tile_n * SUBTILE_COLS;+ const short m_limit =+ align_m ? SUBTILE_ROWS+ : short(max(0, int(simdgroup_limit_m) - row_offset));+ const short n_limit =+ align_n ? SUBTILE_COLS+ : short(max(0, int(simdgroup_limit_n) - col_offset));+ if (m_limit <= 0 || n_limit <= 0)+ continue;++ device T* output_tile_ptr =+ output_ptr + row_offset * leading_dimension_d + col_offset;++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < short(ACCUMULATOR_CAPACITY); i++) {+ auto coord = accumulator_tensor.get_multidimensional_index(i);+ const bool is_valid_element = accumulator_tensor.is_valid_element(i);+ AccumulatorType accumulated_value =+ accumulator_storage[tile_m * TILES_N + tile_n][i] *+ AccumulatorType(ab_scale);+ if (align_m && align_n) {+ if (is_valid_element) {+ if (is_accumulate) {+ accumulated_value += AccumulatorType(+ output_tile_ptr[coord[1] * leading_dimension_d + coord[0]]+ );+ }+ output_tile_ptr[coord[1] * leading_dimension_d + coord[0]] =+ T(accumulated_value);+ }+ } else {+ if (is_valid_element && coord[1] < m_limit && coord[0] < n_limit) {+ if (is_accumulate) {+ accumulated_value += AccumulatorType(+ output_tile_ptr[coord[1] * leading_dimension_d + coord[0]]+ );+ }+ output_tile_ptr[coord[1] * leading_dimension_d + coord[0]] =+ T(accumulated_value);+ }+ }+ }+ }+ }+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.metalnew file mode 100644index 000000000..e0a2a8b43--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal@@ -0,0 +1,56 @@+#include "../common/dsl.h"+#include "../common/thread_context.h"++#include "common/gemm_mpp_core.h"++using namespace uzu::matmul;++#define GEMM_MPP_THREADGROUP_TILE_SIZE (128 * (32 + (short)(16 / sizeof(T))))++template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN>+VARIANTS(T, float, half, bfloat)+VARIANTS(BLOCK_ROWS, 32, 64)+VARIANTS(BLOCK_COLS, 32, 64)+VARIANTS(SIMDGROUPS_PER_ROW, 2, 4)+VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)+CONSTRAINT(BLOCK_ROWS >= 16 * SIMDGROUPS_PER_ROW && BLOCK_COLS >= 32 * SIMDGROUPS_PER_COLUMN)+KERNEL(MatmulGemmMpp)(+ const device T* left_matrix,+ const device T* right_matrix,+ device T* output_matrix,+ const constant uzu::matmul::GemmParams* params,+ const constant uint& group_count_x,+ const constant uint& group_count_y,+ const constant float& ab_scale,+ const bool align_m SPECIALIZE,+ const bool align_n SPECIALIZE,+ const bool is_accumulate SPECIALIZE,+ threadgroup T left_shared[GEMM_MPP_THREADGROUP_TILE_SIZE],+ threadgroup T right_shared[GEMM_MPP_THREADGROUP_TILE_SIZE],+ const uint group_x GROUPS(group_count_x),+ const uint group_y GROUPS(group_count_y),+ const uint thread_x THREADS(32),+ const uint thread_y THREADS(2),+ const uint thread_z THREADS(2),+ const ThreadContext thread_context+) {+ ThreadgroupGemmMpp<+ T,+ BLOCK_ROWS,+ BLOCK_COLS,+ SIMDGROUPS_PER_ROW,+ SIMDGROUPS_PER_COLUMN>::+ run(left_matrix,+ right_matrix,+ output_matrix,+ params,+ align_m,+ align_n,+ is_accumulate,+ ab_scale,+ left_shared,+ right_shared,+ thread_context.simdgroup_index,+ thread_context.threadgroup_index,+ uint2(group_x, group_y));+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.rsnew file mode 100644index 000000000..03510f4e6--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs@@ -0,0 +1,189 @@+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct GemmMppSpecialization {+ pub block_rows: u32,+ pub block_cols: u32,+ pub simdgroups_per_row: u32,+ pub simdgroups_per_column: u32,+ pub align_m: bool,+ pub align_n: bool,+ pub is_accumulate: bool,+}++impl GemmMppSpecialization {+ pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {+ if !matches!(data_type, crate::DataType::F16 | crate::DataType::BF16) {+ return &[];+ }+ &[+ Self {+ block_rows: 64,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: true,+ align_n: true,+ is_accumulate: false,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: false,+ align_n: true,+ is_accumulate: false,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: true,+ align_n: false,+ is_accumulate: false,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: false,+ align_n: false,+ is_accumulate: false,+ },+ Self {+ block_rows: 32,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: true,+ align_n: true,+ is_accumulate: false,+ },+ Self {+ block_rows: 32,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: false,+ align_n: true,+ is_accumulate: false,+ },+ Self {+ block_rows: 64,+ block_cols: 32,+ simdgroups_per_row: 4,+ simdgroups_per_column: 1,+ align_m: true,+ align_n: true,+ is_accumulate: false,+ },+ Self {+ block_rows: 64,+ block_cols: 32,+ simdgroups_per_row: 4,+ simdgroups_per_column: 1,+ align_m: true,+ align_n: false,+ is_accumulate: false,+ },+ // Accumulate variants+ Self {+ block_rows: 64,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: true,+ align_n: true,+ is_accumulate: true,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: false,+ align_n: true,+ is_accumulate: true,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: true,+ align_n: false,+ is_accumulate: true,+ },+ Self {+ block_rows: 64,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: false,+ align_n: false,+ is_accumulate: true,+ },+ Self {+ block_rows: 32,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: true,+ align_n: true,+ is_accumulate: true,+ },+ Self {+ block_rows: 32,+ block_cols: 64,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,+ align_m: false,+ align_n: true,+ is_accumulate: true,+ },+ Self {+ block_rows: 64,+ block_cols: 32,+ simdgroups_per_row: 4,+ simdgroups_per_column: 1,+ align_m: true,+ align_n: true,+ is_accumulate: true,+ },+ Self {+ block_rows: 64,+ block_cols: 32,+ simdgroups_per_row: 4,+ simdgroups_per_column: 1,+ align_m: true,+ align_n: false,+ is_accumulate: true,+ },+ ]+ }++ pub fn select(+ m: u32,+ n: u32,+ is_accumulate: bool,+ ) -> Self {+ let (block_rows, block_cols, simdgroups_per_row, simdgroups_per_column) = if n < 64 {+ (64, 32, 4u32, 1u32)+ } else if m < 64 {+ (32, 64, 2u32, 2u32)+ } else {+ (64, 64, 2u32, 2u32)+ };++ Self {+ block_rows,+ block_cols,+ simdgroups_per_row,+ simdgroups_per_column,+ align_m: (m % block_rows) == 0,+ align_n: (n % block_cols) == 0,+ is_accumulate,+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex 9ebb64163..f8f3a852a 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -17,18 +17,23 @@ use crate::{metal::{Metal,context::MetalContext,- kernel::{MatmulGemmMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel},+ kernel::{+ MatmulGemmMetalKernel, MatmulGemmMppMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel,+ },+ metal_extensions::DeviceExt,},},};mod gemm;+mod gemm_mpp;mod gemv;pub struct MatmulMetalKernel {data_type: DataType,gemv_kernels: HashMap<gemv::GemvSpecialization, MatmulGemvMetalKernel>,gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,+ gemm_mpp_kernels: HashMap<gemm_mpp::GemmMppSpecialization, MatmulGemmMppMetalKernel>,bias_add: TensorAddBiasMetalKernel,}@@ -43,6 +48,13 @@ fn max_gemv_batch_threshold() -> u32 {}impl MatmulMetalKernel {+ fn is_mpp_eligible(+ &self,+ context: &MetalContext,+ ) -> bool {+ context.device.supports_mxu() && matches!(self.data_type, DataType::F16 | DataType::BF16)+ }+fn get_or_create_gemv(&mut self,context: &MetalContext,@@ -95,6 +107,31 @@ impl MatmulMetalKernel {}}+ fn get_or_create_gemm_mpp(+ &mut self,+ context: &MetalContext,+ specialization: gemm_mpp::GemmMppSpecialization,+ ) -> Result<&MatmulGemmMppMetalKernel, MatmulError<Metal>> {+ match self.gemm_mpp_kernels.entry(specialization) {+ Entry::Occupied(entry) => Ok(entry.into_mut()),+ Entry::Vacant(entry) => {+ let kernel = MatmulGemmMppMetalKernel::new(+ context,+ self.data_type,+ specialization.block_rows,+ specialization.block_cols,+ specialization.simdgroups_per_row,+ specialization.simdgroups_per_column,+ specialization.align_m,+ specialization.align_n,+ specialization.is_accumulate,+ )+ .map_err(MatmulError::BackendError)?;+ Ok(entry.insert(kernel))+ },+ }+ }+fn encode_gemv(&mut self,context: &MetalContext,@@ -153,6 +190,7 @@ impl MatmulMetalKernel {threadgroups_per_column,swizzle_log: 0,aligned_inner_iterations: arguments.input_dim / specialization.block_depth,+ use_morton: false,};let kernel = self.get_or_create_gemm(context, specialization)?;@@ -181,6 +219,102 @@ impl MatmulMetalKernel {Ok(())}++ fn encode_gemm_mpp(+ &mut self,+ context: &MetalContext,+ encoder: &mut Encoder<Metal>,+ arguments: MatmulArguments<Metal>,+ ) -> Result<(), MatmulError<Metal>> {+ let is_accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);+ let specialization =+ gemm_mpp::GemmMppSpecialization::select(arguments.batch_dim, arguments.output_dim, is_accumulate);++ let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);+ let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);++ let max_threadgroups_per_dim = threadgroups_per_row.max(threadgroups_per_column);+ let min_threadgroups_per_dim = threadgroups_per_row.min(threadgroups_per_column);+ let morton_dim = max_threadgroups_per_dim.next_power_of_two();+ let morton_total = morton_dim.saturating_mul(morton_dim);+ let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);+ // Morton ordering improves cache locality but requires padding to a square power-of-two grid.+ // Allow up to 4x overhead (idle threadgroups) before the padding cost outweighs the benefit.+ let use_morton = min_threadgroups_per_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);++ let params = GemmParams {+ M: arguments.batch_dim,+ N: arguments.output_dim,+ K: arguments.input_dim,+ leading_dimension_a: arguments.input_dim,+ leading_dimension_b: arguments.input_dim,+ leading_dimension_d: arguments.output_dim,+ threadgroups_per_row,+ threadgroups_per_column,+ swizzle_log: 0,+ aligned_inner_iterations: 0,+ use_morton,+ };++ let (group_count_x, group_count_y) = if use_morton {+ (morton_total, 1_u32)+ } else {+ (threadgroups_per_row, threadgroups_per_column)+ };++ let kernel = self.get_or_create_gemm_mpp(context, specialization)?;+ kernel.encode(+ (arguments.a, arguments.a_offset as usize),+ arguments.b,+ &mut *arguments.d,+ std::slice::from_ref(¶ms),+ group_count_x,+ group_count_y,+ arguments.ab_scale,+ encoder,+ );++ if let MatmulArgumentC::Bias(bias) = arguments.c {+ self.bias_add.encode(+ None::<&<Metal as crate::backends::common::Backend>::Buffer>,+ bias,+ arguments.d,+ arguments.output_dim,+ arguments.batch_dim * arguments.output_dim,+ encoder,+ );+ }++ Ok(())+ }+}++/// Explicit dispatch paths for testing individual kernels independent of production routing.+#[derive(Debug, Clone, Copy)]+pub enum MatmulDispatchPath {+ Auto,+ Gemv,+ Gemm,+ GemmMpp,+}++impl MatmulMetalKernel {+ pub fn encode_with_path(+ &mut self,+ context: &MetalContext,+ arguments: MatmulArguments<Metal>,+ encoder: &mut Encoder<Metal>,+ path: MatmulDispatchPath,+ ) {+ match path {+ MatmulDispatchPath::Auto => self.encode(context, arguments, encoder),+ MatmulDispatchPath::Gemv => self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV"),+ MatmulDispatchPath::Gemm => self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM"),+ MatmulDispatchPath::GemmMpp => {+ self.encode_gemm_mpp(context, encoder, arguments).expect("Failed to encode GEMM MPP")+ },+ }+ }}impl MatmulKernel for MatmulMetalKernel {@@ -200,6 +334,7 @@ impl MatmulKernel for MatmulMetalKernel {data_type,gemv_kernels: HashMap::new(),gemm_kernels: HashMap::new(),+ gemm_mpp_kernels: HashMap::new(),bias_add,};@@ -209,6 +344,11 @@ impl MatmulKernel for MatmulMetalKernel {for &config in gemm::GemmSpecialization::precompile_configs(data_type) {kernel.get_or_create_gemm(context, config)?;}+ if context.device.supports_mxu() {+ for &config in gemm_mpp::GemmMppSpecialization::precompile_configs(data_type) {+ kernel.get_or_create_gemm_mpp(context, config)?;+ }+ }Ok(kernel)}@@ -221,6 +361,8 @@ impl MatmulKernel for MatmulMetalKernel {) {if arguments.batch_dim <= max_gemv_batch_threshold() {self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");+ } else if self.is_mpp_eligible(context) {+ self.encode_gemm_mpp(context, encoder, arguments).expect("Failed to encode GEMM MPP kernel");} else {self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");}diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex 90cc84f97..2c46971e0 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -304,7 +304,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(// layout used above.const uint lane_qid = sg_lin >> 2;const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);+ const uint lane_col_base = ((lane_qid & 2u) * 2u) + ((sg_lin & 1u) * 2u);METAL_PRAGMA_UNROLLfor (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {@@ -567,7 +567,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(const uint lane_row =(lane_qid & 4u) + ((thread_context.simdgroup_index >> 1) & 3u);const uint lane_col_base =- ((lane_qid & 2u) << 1) + ((thread_context.simdgroup_index & 1u) << 1);+ ((lane_qid & 2u) * 2u) + ((thread_context.simdgroup_index & 1u) * 2u);const uint local_row = row_sg_off + lane_row;if (local_row >= m_rows)diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metalindex a6489393a..c9ee593df 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal@@ -35,7 +35,7 @@ PUBLIC KERNEL(MoeTileScan)(// Inclusive scan in-place over scratch// Simple O(log E) scan; E is small- for (uint offset = 1u; offset < e; offset <<= 1u) {+ for (uint offset = 1u; offset < e; offset *= 2u) {uint val = 0u;if (tid < e) {val = scratch[tid];@@ -105,4 +105,4 @@ PUBLIC KERNEL(MoeWriteDispatchArgs)(dispatch_args[0] = num_tiles_n; // xdispatch_args[1] = total_rows; // ydispatch_args[2] = 1u; // z-}\ No newline at end of file+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.hindex a27c349e9..c0122c0cb 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h@@ -139,9 +139,9 @@ inline U qdot_zero_point(if (bits == 4) {const device uint16_t* ws = (const device uint16_t*)w;const uint16_t zp0 = static_cast<uint16_t>(zero_point);- const uint16_t zp1 = static_cast<uint16_t>(zero_point) << 4;- const uint16_t zp2 = static_cast<uint16_t>(zero_point) << 8;- const uint16_t zp3 = static_cast<uint16_t>(zero_point) << 12;+ const uint16_t zp1 = static_cast<uint16_t>(zero_point) * 16;+ const uint16_t zp2 = static_cast<uint16_t>(zero_point) * 256;+ const uint16_t zp3 = static_cast<uint16_t>(zero_point) * 4096;for (int i = 0; i < (values_per_thread / 4); i++) {uint16_t word = ws[i];diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex f747f319c..d97a298fe 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -11,4 +11,5 @@ mod metal_extensions;pub use backend::Metal;pub use context::MetalContext;pub use device_capabilities::MetalDeviceCapabilities;-pub use metal_extensions::DeviceGeneration;+pub use kernel::matmul::MatmulDispatchPath;+pub use metal_extensions::{DeviceExt, DeviceGeneration};diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rsindex b988e2c96..915816913 100644--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs@@ -578,8 +578,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let mut mask = vec![0_u32; row_words];for token_index in 0..limit {let word = token_index / 32;- let bit = token_index % 32;- mask[word] |= 1_u32 << bit;+ mask[word] |= 2_u32.pow((token_index % 32) as u32);}self.single_token_vocab_masks.insert(limit, mask.into_boxed_slice());Ok(())@@ -605,8 +604,7 @@ impl<B: Backend> TokenDecoderRunner<B> {];for token_index in 0..limit {let word = token_index / 32;- let bit = token_index % 32;- mask[row_words + word] |= 1_u32 << bit;+ mask[row_words + word] |= 2_u32.pow((token_index % 32) as u32);}self.two_token_vocab_masks.insert(limit, mask.into_boxed_slice());Ok(())@@ -852,8 +850,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let mut mask = vec![0_u32; row_words];for token_index in 0..limit {let word = token_index / 32;- let bit = token_index % 32;- mask[word] |= 1_u32 << bit;+ mask[word] |= 2_u32.pow((token_index % 32) as u32);}Some(mask.into_boxed_slice())}@@ -870,8 +867,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let mut mask = vec![0_u32; total_words];for token_index in 0..limit {let word = token_index / 32;- let bit = token_index % 32;- mask[sampling_start * row_words + word] |= 1_u32 << bit;+ mask[sampling_start * row_words + word] |= 2_u32.pow((token_index % 32) as u32);}Some(mask.into_boxed_slice())}diff --git a/crates/uzu/src/session/tts_session/decoder_support.rs b/crates/uzu/src/session/tts_session/decoder_support.rsindex 773bf7e39..827fe299c 100644--- a/crates/uzu/src/session/tts_session/decoder_support.rs+++ b/crates/uzu/src/session/tts_session/decoder_support.rs@@ -551,13 +551,11 @@ pub(super) fn build_semantic_sampling_mask_row(for token_index in semantic_begin..=semantic_end {let token = usize::try_from(token_index).map_err(|_| Error::UnableToLoadConfig)?;let word = token / 32;- let bit = token % 32;- mask[word] |= 1_u32 << bit;+ mask[word] |= 2_u32.pow((token % 32) as u32);}let im_end_token = usize::try_from(im_end).map_err(|_| Error::UnableToLoadConfig)?;let word = im_end_token / 32;- let bit = im_end_token % 32;- mask[word] |= 1_u32 << bit;+ mask[word] |= 2_u32.pow((im_end_token % 32) as u32);Ok(mask.into_boxed_slice())}@@ -570,11 +568,10 @@ pub(super) fn clear_token_in_sampling_mask(}let token = usize::try_from(token).map_err(|_| Error::UnableToLoadConfig)?;let word = token / 32;- let bit = token % 32;if word >= mask.len() {return Err(Error::UnableToLoadConfig);}- mask[word] &= !(1_u32 << bit);+ mask[word] &= !2_u32.pow((token % 32) as u32);Ok(())}diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rsindex ebb324b74..3121a5f82 100644--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs@@ -22,9 +22,9 @@ fn pack_u4_weights(values: &[u8]) -> Vec<u8> {let mut out = Vec::with_capacity((values.len() + 3) / 4 * 2);for chunk in values.chunks(4) {let w0 = (*chunk.get(0).unwrap_or(&0) as u16) & 0x0F;- let w1 = ((*chunk.get(1).unwrap_or(&0) as u16) & 0x0F) << 4;- let w2 = ((*chunk.get(2).unwrap_or(&0) as u16) & 0x0F) << 8;- let w3 = ((*chunk.get(3).unwrap_or(&0) as u16) & 0x0F) << 12;+ let w1 = ((*chunk.get(1).unwrap_or(&0) as u16) & 0x0F) * 16;+ let w2 = ((*chunk.get(2).unwrap_or(&0) as u16) & 0x0F) * 256;+ let w3 = ((*chunk.get(3).unwrap_or(&0) as u16) & 0x0F) * 4096;let word: u16 = w0 | w1 | w2 | w3;out.push(word as u8);@@ -99,7 +99,7 @@ fn get_4bit_value(let byte_idx = word_idx * 2;let word = if byte_idx + 1 < data.len() {- data[byte_idx] as u16 | ((data[byte_idx + 1] as u16) << 8)+ data[byte_idx] as u16 | ((data[byte_idx + 1] as u16) * 256)} else {0};diff --git a/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rsindex e77fa3c87..d1b31c51e 100644--- a/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs+++ b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs@@ -42,7 +42,7 @@ fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Inpulet packing_divisor = quant_mode.packing_divisor() as u32;let weights_stride = model_dim / packing_divisor;let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize)- .map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) << 4))+ .map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) * 16)).collect();let num_groups = (model_dim + group_size - 1) / group_size;diff --git a/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rsnew file mode 100644index 000000000..d2fb5eff9--- /dev/null+++ b/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs@@ -0,0 +1,330 @@+#![cfg(metal_backend)]++use std::{ops::{Deref, DerefMut}, rc::Rc};++use criterion::{BenchmarkId, Criterion, Throughput};+use half::bf16;+use uzu::{+ ArrayContextExt, ArrayElement,+ backends::{+ common::{+ Backend, Context, Encoder,+ kernel::{+ ManualKernels,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ },+ },+ cpu::Cpu,+ metal::{DeviceExt, MatmulDispatchPath, Metal, MetalContext},+ },+};++use crate::{+ common::{assert::assert_eq_float, type_short_name},+ uzu_bench, uzu_test,+};++struct Input {+ left: Box<[bf16]>,+ right: Box<[bf16]>,+ destination_prefill: Option<Box<[bf16]>>,+ batch_dim: usize,+ input_dim: usize,+ output_dim: usize,+ ab_scale: f32,+}++fn get_test_data(+ batch_dim: usize,+ input_dim: usize,+ output_dim: usize,+ ab_scale: f32,+ accumulate: bool,+) -> Input {+ let left: Vec<bf16> = (0..batch_dim * input_dim)+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.1 - 0.6))+ .collect();+ let right: Vec<bf16> = (0..output_dim * input_dim)+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.1 - 0.8))+ .collect();+ let destination_prefill = if accumulate {+ Some(+ (0..batch_dim * output_dim)+ .map(|i| bf16::from_f32(((i % 7) as f32) * 0.03 - 0.09))+ .collect(),+ )+ } else {+ None+ };++ Input {+ left: left.into_boxed_slice(),+ right: right.into_boxed_slice(),+ destination_prefill,+ batch_dim,+ input_dim,+ output_dim,+ ab_scale,+ }+}++fn get_cpu_output(input: &Input) -> Vec<bf16> {+ let cpu_context = <Cpu as Backend>::Context::new().expect("CPU context");++ let left_array = cpu_context.create_array_from(&[input.batch_dim, input.input_dim], &input.left, "");+ let right_array = cpu_context.create_array_from(&[input.output_dim, input.input_dim], &input.right, "");+ let destination_array = if let Some(ref prefill) = input.destination_prefill {+ cpu_context.create_array_from(&[input.batch_dim, input.output_dim], prefill, "")+ } else {+ cpu_context.create_array_uninitialized(&[input.batch_dim, input.output_dim], bf16::data_type(), "")+ };++ let left_buffer = left_array.buffer();+ let left_ref = left_buffer.borrow();+ let right_buffer = right_array.buffer();+ let right_ref = right_buffer.borrow();+ let destination_buffer = destination_array.buffer();+ let mut destination_ref = destination_buffer.borrow_mut();++ let argument_c = if input.destination_prefill.is_some() {+ MatmulArgumentC::Accumulate+ } else {+ MatmulArgumentC::None+ };++ let mut matmul_kernel =+ <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&cpu_context, bf16::data_type())+ .expect("CPU MatmulKernel");++ let mut encoder = Encoder::new(cpu_context.as_ref()).expect("encoder");+ matmul_kernel.encode(+ &cpu_context,+ MatmulArguments {+ a: left_ref.deref(),+ a_offset: 0,+ b: right_ref.deref(),+ ab_scale: input.ab_scale,+ c: argument_c,+ d: destination_ref.deref_mut(),+ batch_dim: input.batch_dim as u32,+ input_dim: input.input_dim as u32,+ output_dim: input.output_dim as u32,+ },+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ drop(destination_ref);+ destination_array.as_slice().to_vec()+}++fn get_mpp_output(+ metal_context: &MetalContext,+ matmul_kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,+ input: &Input,+) -> Vec<bf16> {+ let left_array = metal_context.create_array_from(&[input.batch_dim, input.input_dim], &input.left, "");+ let right_array = metal_context.create_array_from(&[input.output_dim, input.input_dim], &input.right, "");+ let destination_array = if let Some(ref prefill) = input.destination_prefill {+ metal_context.create_array_from(&[input.batch_dim, input.output_dim], prefill, "")+ } else {+ metal_context.create_array_uninitialized(&[input.batch_dim, input.output_dim], bf16::data_type(), "")+ };++ let left_buffer = left_array.buffer();+ let left_ref = left_buffer.borrow();+ let right_buffer = right_array.buffer();+ let right_ref = right_buffer.borrow();+ let destination_buffer = destination_array.buffer();+ let mut destination_ref = destination_buffer.borrow_mut();++ let argument_c = if input.destination_prefill.is_some() {+ MatmulArgumentC::Accumulate+ } else {+ MatmulArgumentC::None+ };++ let mut encoder = Encoder::new(metal_context).expect("encoder");+ matmul_kernel.encode_with_path(+ metal_context,+ MatmulArguments {+ a: left_ref.deref(),+ a_offset: 0,+ b: right_ref.deref(),+ ab_scale: input.ab_scale,+ c: argument_c,+ d: destination_ref.deref_mut(),+ batch_dim: input.batch_dim as u32,+ input_dim: input.input_dim as u32,+ output_dim: input.output_dim as u32,+ },+ &mut encoder,+ MatmulDispatchPath::GemmMpp,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ drop(destination_ref);+ destination_array.as_slice().to_vec()+}++fn create_metal_context_and_matmul_kernel()+ -> (Rc<MetalContext>, <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel)+{+ let metal_context = MetalContext::new().expect("Metal context");+ let matmul_kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &metal_context,+ bf16::data_type(),+ )+ .expect("MatmulKernel");+ (metal_context, matmul_kernel)+}++fn test_mpp_correctness(+ batch_dim: usize,+ input_dim: usize,+ output_dim: usize,+ ab_scale: f32,+ accumulate: bool,+ tolerance: f32,+) {+ let (metal_context, mut matmul_kernel) = create_metal_context_and_matmul_kernel();+ if !metal_context.device.supports_mxu() {+ eprintln!("Skipping MPP test: device does not support MXU");+ return;+ }++ let input = get_test_data(batch_dim, input_dim, output_dim, ab_scale, accumulate);+ let expected = get_cpu_output(&input);+ let actual = get_mpp_output(&metal_context, &mut matmul_kernel, &input);+ assert_eq_float(&expected, &actual, tolerance, "GEMM_MPP vs CPU reference");+}++// Basic correctness — explicitly dispatched to MPP kernel++#[uzu_test]+fn test_bf16_aligned() {+ test_mpp_correctness(64, 2048, 2048, 1.0, false, 1.0);+}++#[uzu_test]+fn test_bf16_unaligned_m() {+ test_mpp_correctness(33, 2048, 2048, 1.0, false, 1.0);+}++#[uzu_test]+fn test_bf16_unaligned_n() {+ test_mpp_correctness(64, 2048, 33, 1.0, false, 1.0);+}++#[uzu_test]+fn test_bf16_large() {+ test_mpp_correctness(128, 4096, 14336, 1.0, false, 2.0);+}++#[uzu_test]+fn test_bf16_small_batch() {+ test_mpp_correctness(1, 2048, 2048, 1.0, false, 1.0);+}++// ab_scale++#[uzu_test]+fn test_bf16_ab_scale() {+ test_mpp_correctness(64, 2048, 2048, 0.5, false, 1.0);+}++// accumulate++#[uzu_test]+fn test_bf16_accumulate() {+ test_mpp_correctness(64, 2048, 2048, 1.0, true, 1.0);+}++// scale + accumulate combined++#[uzu_test]+fn test_bf16_scale_and_accumulate() {+ test_mpp_correctness(64, 2048, 2048, 0.5, true, 1.0);+}++// Benchmarks++const BENCHMARK_SHAPES: &[(usize, usize, usize)] = &[+ (128, 2048, 8192),+ (128, 4096, 14336),+ (256, 4096, 4096),+ (512, 8192, 2048),+];++#[uzu_bench]+fn bench_gemm_mpp(criterion: &mut Criterion) {+ let metal_context = MetalContext::new().unwrap();+ if !metal_context.device.supports_mxu() {+ return;+ }++ let mut matmul_kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &metal_context,+ bf16::data_type(),+ )+ .expect("MatmulKernel");++ let mut benchmark_group = criterion.benchmark_group(+ format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()),+ );++ for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {+ let left_array =+ metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");+ let right_array =+ metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");+ let destination_array =+ metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");++ let floating_point_operations = 2 * batch_dim * input_dim * output_dim;+ benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));++ benchmark_group.bench_function(+ BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),+ |bencher| {+ let left_buffer = left_array.buffer();+ let left_ref = left_buffer.borrow();+ let right_buffer = right_array.buffer();+ let right_ref = right_buffer.borrow();+ let destination_buffer = destination_array.buffer();++ bencher.iter_custom(|iteration_count| {+ let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();++ for _ in 0..iteration_count {+ let mut destination_ref = destination_buffer.borrow_mut();+ matmul_kernel.encode_with_path(+ &metal_context,+ MatmulArguments {+ a: left_ref.deref(),+ a_offset: 0,+ b: right_ref.deref(),+ ab_scale: 1.0,+ c: MatmulArgumentC::None,+ d: destination_ref.deref_mut(),+ batch_dim: batch_dim as u32,+ input_dim: input_dim as u32,+ output_dim: output_dim as u32,+ },+ &mut encoder,+ MatmulDispatchPath::GemmMpp,+ );+ }++ encoder+ .end_encoding()+ .submit()+ .wait_until_completed()+ .unwrap()+ .gpu_execution_time()+ .unwrap()+ })+ },+ );+ }+}diff --git a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemm_test.rssimilarity index 80%rename from crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rsrename to crates/uzu/tests/unit/kernel/matmul/gemm_test.rsindex d803ec381..c29779868 100644--- a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/gemm_test.rs@@ -45,11 +45,11 @@ fn get_test_data<T: ArrayElement + Float>(n,};- let expected = get_output::<T, Cpu>(&input);+ let expected = get_output::<T, Cpu>(&input, 1.0);(input, expected)}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>, ab_scale: f32) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let m = input.m as u32;@@ -77,7 +77,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {a: a_ref.deref(),a_offset: 0,b: b_ref.deref(),- ab_scale: 1.0,+ ab_scale,c: MatmulArgumentC::None,d: d_ref.deref_mut(),batch_dim: m,@@ -100,11 +100,26 @@ fn test<T: ArrayElement + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(m, k, n);for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input);+ let output = get_output::<T, B>(&input, 1.0);assert_eq_float(&expected, &output, eps, &format!("backend {}", std::any::type_name::<B>()));});}+fn test_with_scale<T: ArrayElement + Float + Debug + Display>(+ m: usize,+ k: usize,+ n: usize,+ ab_scale: f32,+ eps: f32,+) {+ let (input, _) = get_test_data::<T>(m, k, n);+ let expected = get_output::<T, Cpu>(&input, ab_scale);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input, ab_scale);+ assert_eq_float(&expected, &output, eps, &format!("backend {} ab_scale={ab_scale}", std::any::type_name::<B>()));+ });+}+#[uzu_test]fn test_f32_aligned() {test::<f32>(64, 64, 64, 0.01);@@ -149,3 +164,15 @@ fn test_f16_large() {fn test_bf16_large() {test::<bf16>(16, 128, 256, 0.1);}++// ab_scale tests++#[uzu_test]+fn test_f32_ab_scale() {+ test_with_scale::<f32>(16, 128, 256, 0.5, 0.01);+}++#[uzu_test]+fn test_bf16_ab_scale() {+ test_with_scale::<bf16>(16, 128, 256, 0.5, 0.1);+}diff --git a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rsdeleted file mode 100644index 11406a698..000000000--- a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs+++ /dev/null@@ -1,311 +0,0 @@-//! Correctness tests comparing Metal matmul kernels against ndarray--#![cfg(metal_backend)]-use bytemuck;-use half::bf16;-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};-use ndarray::Array2;-use uzu::{- DataType,- backends::{- common::{- Backend, Context, Encoder,- kernel::{- ManualKernels,- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},- },- },- metal::Metal,- },-};--use crate::uzu_test;--fn run_metal_matmul(- ctx: &<Metal as Backend>::Context,- a_data: &[bf16],- b_data: &[bf16],- m: usize,- k: usize,- n: usize,-) -> Vec<bf16> {- let a_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(a_data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let b_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(b_data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let mut d_buf = ctx- .device- .new_buffer(m * n * core::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut kernel =- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");-- let mut encoder = Encoder::new(ctx).unwrap();- kernel.encode(- ctx,- MatmulArguments {- a: &a_buf,- a_offset: 0,- b: &b_buf,- ab_scale: 1.0,- c: MatmulArgumentC::None,- d: &mut d_buf,- batch_dim: m as u32,- input_dim: k as u32,- output_dim: n as u32,- },- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- unsafe {- let ptr = d_buf.contents().as_ptr() as *const bf16;- std::slice::from_raw_parts(ptr, m * n).to_vec()- }-}--fn run_ndarray_matmul(- a_data: &[bf16],- b_data: &[bf16],- m: usize,- k: usize,- n: usize,-) -> Vec<bf16> {- let a_f32: Vec<f32> = a_data.iter().map(|x| x.to_f32()).collect();- let b_f32: Vec<f32> = b_data.iter().map(|x| x.to_f32()).collect();-- let a_arr = Array2::from_shape_vec((m, k), a_f32).expect("A shape");- let b_arr = Array2::from_shape_vec((n, k), b_f32).expect("B shape");-- let result = a_arr.dot(&b_arr.t());-- result.iter().map(|&x| bf16::from_f32(x)).collect()-}--fn generate_test_data(- m: usize,- k: usize,- n: usize,-) -> (Vec<bf16>, Vec<bf16>) {- let a: Vec<bf16> = (0..(m * k)).map(|i| bf16::from_f32(((i % 13) as f32) * 0.01 - 0.06)).collect();- let b_size = n * k;- let b: Vec<bf16> = (0..b_size).map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.15)).collect();- (a, b)-}--struct TestCase {- m: usize,- k: usize,- n: usize,- tolerance: f32,-}--const MATMUL_CORRECTNESS_FULL_ENV: &str = "UZU_MATMUL_CORRECTNESS_FULL";--const QUICK_BATCH_SIZES: [usize; 2] = [1, 16];-const FULL_BATCH_SIZES: [usize; 12] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];--const QUICK_MODEL_SHAPES: [(usize, usize); 8] =- [(896, 896), (896, 4864), (1024, 4096), (1152, 6912), (1536, 8960), (2048, 8192), (4096, 14336), (5120, 17408)];--const FULL_MODEL_SHAPES: [(usize, usize); 51] = [- // Qwen2.5-Coder-0.5B- (896, 896),- (896, 1152),- (896, 4864),- (4864, 896),- // Qwen3-0.6B- (1024, 1024),- (1024, 3072),- (1024, 4096),- (3072, 1024),- // Gemma-3-1B- (1152, 1152),- (1152, 1536),- (1152, 6912),- (6912, 1152),- // Qwen2.5-Coder-1.5B- (1536, 1536),- (1536, 2048),- (1536, 8960),- (8960, 1536),- // Llama-3.2-1B / Qwen3-1.7B / SmolLM2-1.7B- (2048, 2048),- (2048, 2560),- (2048, 3072),- (2048, 4096),- (2048, 6144),- (2048, 8192),- (2048, 11008),- (6144, 2048),- (8192, 2048),- (11008, 2048),- // Qwen3-4B / Gemma-3-4B- (2560, 2560),- (2560, 4096),- (2560, 6144),- (2560, 9728),- (2560, 10240),- (9728, 2560),- (10240, 2560),- // Llama-3.2-3B- (3072, 3072),- (3072, 5120),- (3072, 8192),- (8192, 3072),- // Qwen2.5-Coder-7B- (3584, 3584),- (3584, 4608),- (3584, 18944),- (18944, 3584),- // Llama-3.1-8B / Qwen3-8B- (4096, 4096),- (4096, 6144),- (4096, 12288),- (4096, 14336),- (12288, 4096),- (14336, 4096),- // Qwen3-14B- (5120, 5120),- (5120, 7168),- (5120, 17408),- (17408, 5120),-];--fn run_full_case_matrix() -> bool {- std::env::var(MATMUL_CORRECTNESS_FULL_ENV)- .ok()- .map(|value| {- let normalized = value.trim().to_ascii_lowercase();- matches!(normalized.as_str(), "1" | "true" | "yes")- })- .unwrap_or(false)-}--fn test_cases(full_case_matrix: bool) -> Vec<TestCase> {- let base_tolerance = 0.01;- let (batch_sizes, model_shapes): (&[usize], &[(usize, usize)]) = if full_case_matrix {- (&FULL_BATCH_SIZES, &FULL_MODEL_SHAPES)- } else {- (&QUICK_BATCH_SIZES, &QUICK_MODEL_SHAPES)- };-- let mut cases: Vec<(usize, usize, usize)> =- model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n))).collect();-- // Edge case: small matrix- for &m in batch_sizes {- cases.push((m, 128, 128));- }-- cases- .into_iter()- .map(|(m, k, n)| {- let tolerance = base_tolerance * (k as f32 / 1024.0).sqrt();- let tolerance = tolerance * (1.0 + (m as f32).log2() * 0.02);- TestCase {- m,- k,- n,- tolerance,- }- })- .collect()-}--fn compare_results(- metal: &[bf16],- reference: &[bf16],- tolerance: f32,-) -> Result<(), (f32, usize, usize)> {- if metal.len() != reference.len() {- return Err((f32::INFINITY, 0, metal.len()));- }-- let mut max_diff: f32 = 0.0;- let mut max_diff_idx = 0;- let mut diff_count = 0;-- for (i, (&m_val, &r_val)) in metal.iter().zip(reference.iter()).enumerate() {- let mf = m_val.to_f32();- let rf = r_val.to_f32();- let diff = (mf - rf).abs();- if diff > max_diff {- max_diff = diff;- max_diff_idx = i;- }- if diff > tolerance {- diff_count += 1;- }- }-- if max_diff <= tolerance {- Ok(())- } else {- Err((max_diff, max_diff_idx, diff_count))- }-}--#[uzu_test]-#[ignore]-fn matmul_correctness_comprehensive() {- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {- eprintln!("No Metal device available, skipping test");- return;- };-- let full_case_matrix = run_full_case_matrix();- let cases = test_cases(full_case_matrix);- eprintln!(- "Running {} matmul correctness case matrix (set {}=1 for exhaustive run)",- if full_case_matrix {- "exhaustive"- } else {- "quick"- },- MATMUL_CORRECTNESS_FULL_ENV- );- let mut passed = 0;- let mut failed = Vec::new();-- for case in &cases {- let (a, b) = generate_test_data(case.m, case.k, case.n);- let metal_result = run_metal_matmul(&ctx, &a, &b, case.m, case.k, case.n);- let reference = run_ndarray_matmul(&a, &b, case.m, case.k, case.n);-- let trans_str = "T";-- match compare_results(&metal_result, &reference, case.tolerance) {- Ok(()) => {- passed += 1;- eprintln!("✓ m={} k={} n={} B={}", case.m, case.k, case.n, trans_str);- },- Err((max_diff, idx, count)) => {- failed.push((case, max_diff, idx, count));- eprintln!(- "✗ m={} k={} n={} B={} max_diff={:.6} at idx {} ({} exceed tol {})",- case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance- );- },- }- }-- eprintln!("\n{}/{} tests passed", passed, cases.len());-- if !failed.is_empty() {- eprintln!("\nFailed tests:");- for (case, max_diff, idx, count) in &failed {- let trans_str = "T";- eprintln!(- " m={} k={} n={} B={}: max_diff={:.6} at idx {}, {} exceed tol {}",- case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance- );- }- panic!("{} tests failed", failed.len());- }-}diff --git a/crates/uzu/tests/unit/kernel/matmul/mod.rs b/crates/uzu/tests/unit/kernel/matmul/mod.rsindex 346c2e7ba..e7089955d 100644--- a/crates/uzu/tests/unit/kernel/matmul/mod.rs+++ b/crates/uzu/tests/unit/kernel/matmul/mod.rs@@ -1,2 +1,3 @@+mod gemm_mpp_test;+mod gemm_test;mod gemv_test;-mod steel_gemm_test;