Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

mpp

eugenebb72fd42e321
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 7ed6c78f8..f916a75b6 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -59,7 +59,7 @@ walkdir.workspace = true
depfile.workspace = true
tempfile.workspace = true
serde.workspace = true
-serde_json.workspace = true
+serde_json = { workspace = true, features = ["unbounded_depth"] }
futures.workspace = true
tokio.workspace = true
blake3.workspace = true
diff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rs
index b5262ab8a..241fcdcde 100644
--- a/crates/uzu/build/metal/toolchain.rs
+++ b/crates/uzu/build/metal/toolchain.rs
@@ -7,6 +7,7 @@ use std::{
};
use anyhow::{Context, bail};
+use serde::Deserialize;
use tempfile::NamedTempFile;
use tokio::{io::AsyncWriteExt, process::Command};
@@ -180,8 +181,11 @@ impl MetalToolchain {
bail!("metal analyzer failed: {}", String::from_utf8_lossy(&analyze_output.stderr));
}
- let ast_root =
- serde_json::from_slice::<MetalAstNode>(&analyze_output.stdout).context("cannot deserialize ast dump")?;
+ let ast_root = {
+ let mut deserializer = serde_json::Deserializer::from_slice(&analyze_output.stdout);
+ deserializer.disable_recursion_limit();
+ MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?
+ };
if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {
bail!(
diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rs
index a9c496065..26089436f 100644
--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs
+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs
@@ -13,4 +13,5 @@ pub struct GemmParams {
pub threadgroups_per_row: u32,
pub swizzle_log: u32,
pub aligned_inner_iterations: u32,
+ pub use_morton: bool,
}
diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.h
index c3fdf827e..07525ecdb 100644
--- a/crates/uzu/src/backends/metal/generated/matmul.h
+++ b/crates/uzu/src/backends/metal/generated/matmul.h
@@ -25,6 +25,7 @@ typedef struct {
uint32_t threadgroups_per_row;
uint32_t swizzle_log;
uint32_t aligned_inner_iterations;
+ bool use_morton;
} GemmParams;
#ifdef __METAL_VERSION__
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
new file mode 100644
index 000000000..c4e885f9f
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
@@ -0,0 +1,362 @@
+#pragma once
+
+#include "defines.h"
+#include "loader.h"
+#include "../../../generated/matmul.h"
+
+#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+template <
+ typename T,
+ short BLOCK_ROWS,
+ short BLOCK_COLS,
+ short SIMDGROUPS_PER_ROW,
+ short SIMDGROUPS_PER_COLUMN,
+ short SUBTILE_ROWS = 16,
+ short SUBTILE_COLS = 32,
+ short MATMUL_K_STEP = 16,
+ // Number of cooperative tensor elements owned by each thread.
+ // Determined by the MPP compiler intrinsic get_capacity().
+ // Empirically: SUBTILE_ROWS * SUBTILE_COLS / SIMD_SIZE = 16 * 32 / 32 = 16.
+ short ACCUMULATOR_CAPACITY = 16>
+struct ThreadgroupGemmMpp {
+ METAL_CONST short PREFETCH_K = 32;
+ METAL_CONST short THREADGROUP_PADDING = short(16 / sizeof(T));
+ METAL_CONST short THREADGROUP_LD = PREFETCH_K + THREADGROUP_PADDING;
+ METAL_CONST short THREADGROUP_SIZE =
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
+ METAL_CONST short SIMDGROUP_BLOCK_M = BLOCK_ROWS / SIMDGROUPS_PER_ROW;
+ METAL_CONST short SIMDGROUP_BLOCK_N = BLOCK_COLS / SIMDGROUPS_PER_COLUMN;
+ METAL_CONST short TILES_M = SIMDGROUP_BLOCK_M / SUBTILE_ROWS;
+ METAL_CONST short TILES_N = SIMDGROUP_BLOCK_N / SUBTILE_COLS;
+ METAL_CONST short INNER_K_STEPS = PREFETCH_K / MATMUL_K_STEP;
+
+ using AccumulatorType = float;
+
+ template <
+ typename MatmulOperation,
+ typename LeftTensor,
+ typename RightTensor,
+ typename AccumulatorTensor>
+ static METAL_FUNC void accumulate_tiles(
+ const short k_steps,
+ const ushort tile_row_offset,
+ const ushort tile_col_offset,
+ threadgroup T* left_shared,
+ threadgroup T* right_shared,
+ thread MatmulOperation& matmul_operation,
+ thread LeftTensor& left_tensor,
+ thread RightTensor& right_tensor,
+ thread AccumulatorTensor& accumulator_tensor,
+ thread AccumulatorType
+ accumulator_storage[TILES_M * TILES_N][ACCUMULATOR_CAPACITY]
+ ) {
+ METAL_PRAGMA_UNROLL
+ for (short tile_m = 0; tile_m < TILES_M; tile_m++) {
+ METAL_PRAGMA_UNROLL
+ for (short tile_n = 0; tile_n < TILES_N; tile_n++) {
+ const short subtile_index = tile_m * TILES_N + tile_n;
+ const short left_row_base = tile_row_offset + tile_m * SUBTILE_ROWS;
+ const short right_col_base = tile_col_offset + tile_n * SUBTILE_COLS;
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < ACCUMULATOR_CAPACITY; i++) {
+ accumulator_tensor[i] = accumulator_storage[subtile_index][i];
+ }
+
+ for (short k_step = 0; k_step < k_steps; k_step++) {
+ const short k_offset = k_step * MATMUL_K_STEP;
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < left_tensor.get_capacity(); i++) {
+ auto coord = left_tensor.get_multidimensional_index(i);
+ left_tensor[i] = left_shared
+ [(left_row_base + coord[1]) * THREADGROUP_LD + coord[0] +
+ k_offset];
+ }
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < right_tensor.get_capacity(); i++) {
+ auto coord = right_tensor.get_multidimensional_index(i);
+ right_tensor[i] = right_shared
+ [(right_col_base + coord[1]) * THREADGROUP_LD + coord[0] +
+ k_offset];
+ }
+
+ matmul_operation.run(left_tensor, right_tensor, accumulator_tensor);
+ }
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < ACCUMULATOR_CAPACITY; i++) {
+ accumulator_storage[subtile_index][i] = accumulator_tensor[i];
+ }
+ }
+ }
+ }
+
+ static METAL_FUNC void run(
+ const device T* left_matrix,
+ const device T* right_matrix,
+ device T* output_matrix,
+ const constant GemmParams* params,
+ const bool align_m,
+ const bool align_n,
+ const bool is_accumulate,
+ const float ab_scale,
+ threadgroup T* left_shared,
+ threadgroup T* right_shared,
+ uint simd_lane_id,
+ uint simd_group_id,
+ uint2 threadgroup_position
+ ) {
+ uint tile_id_x, tile_id_y;
+ if (params->use_morton) {
+ uint linear_id = threadgroup_position.x;
+ uint morton_x = linear_id;
+ uint morton_y = linear_id >> 1;
+ morton_x &= 0x55555555u;
+ morton_x = (morton_x | (morton_x >> 1)) & 0x33333333u;
+ morton_x = (morton_x | (morton_x >> 2)) & 0x0F0F0F0Fu;
+ morton_x = (morton_x | (morton_x >> 4)) & 0x00FF00FFu;
+ morton_x = (morton_x | (morton_x >> 8)) & 0x0000FFFFu;
+ morton_y &= 0x55555555u;
+ morton_y = (morton_y | (morton_y >> 1)) & 0x33333333u;
+ morton_y = (morton_y | (morton_y >> 2)) & 0x0F0F0F0Fu;
+ morton_y = (morton_y | (morton_y >> 4)) & 0x00FF00FFu;
+ morton_y = (morton_y | (morton_y >> 8)) & 0x0000FFFFu;
+ tile_id_x = morton_x;
+ tile_id_y = morton_y;
+ } else {
+ tile_id_x = threadgroup_position.x;
+ tile_id_y = threadgroup_position.y;
+ }
+
+ if (tile_id_x >= params->threadgroups_per_row ||
+ tile_id_y >= params->threadgroups_per_column) {
+ return;
+ }
+
+ const uint block_row_start = tile_id_y * BLOCK_ROWS;
+ const uint block_col_start = tile_id_x * BLOCK_COLS;
+ const size_t block_row_start_long = size_t(block_row_start);
+ const size_t block_col_start_long = size_t(block_col_start);
+
+ const device T* left_block_ptr =
+ left_matrix + block_row_start_long * params->leading_dimension_a;
+ const device T* right_block_ptr =
+ right_matrix + block_col_start_long * params->leading_dimension_b;
+
+ ThreadgroupLoader<
+ T,
+ BLOCK_ROWS,
+ PREFETCH_K,
+ THREADGROUP_LD,
+ 1,
+ THREADGROUP_SIZE>
+ left_loader(
+ left_block_ptr,
+ params->leading_dimension_a,
+ left_shared,
+ ushort(simd_group_id),
+ ushort(simd_lane_id)
+ );
+ ThreadgroupLoader<
+ T,
+ BLOCK_COLS,
+ PREFETCH_K,
+ THREADGROUP_LD,
+ 1,
+ THREADGROUP_SIZE>
+ right_loader(
+ right_block_ptr,
+ params->leading_dimension_b,
+ right_shared,
+ ushort(simd_group_id),
+ ushort(simd_lane_id)
+ );
+
+ const ushort tile_row_offset =
+ SIMDGROUP_BLOCK_M * (simd_group_id / SIMDGROUPS_PER_COLUMN);
+ const ushort tile_col_offset =
+ SIMDGROUP_BLOCK_N * (simd_group_id % SIMDGROUPS_PER_COLUMN);
+
+ device T* output_ptr =
+ output_matrix + block_row_start_long * params->leading_dimension_d +
+ block_col_start_long + tile_row_offset * params->leading_dimension_d +
+ tile_col_offset;
+
+ const short simdgroup_limit_m =
+ align_m
+ ? SIMDGROUP_BLOCK_M
+ : short(
+ min(int(SIMDGROUP_BLOCK_M),
+ int(params->M) - int(block_row_start + tile_row_offset))
+ );
+ const short simdgroup_limit_n =
+ align_n
+ ? SIMDGROUP_BLOCK_N
+ : short(
+ min(int(SIMDGROUP_BLOCK_N),
+ int(params->N) - int(block_col_start + tile_col_offset))
+ );
+
+ constexpr auto matmul_descriptor = mpp::tensor_ops::matmul2d_descriptor(
+ SUBTILE_ROWS,
+ SUBTILE_COLS,
+ MATMUL_K_STEP,
+ false,
+ true,
+ false,
+ mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate
+ );
+
+ mpp::tensor_ops::matmul2d<matmul_descriptor, metal::execution_simdgroup>
+ matmul_operation;
+
+ auto left_tensor =
+ matmul_operation.template get_left_input_cooperative_tensor<
+ T,
+ T,
+ AccumulatorType>();
+ auto right_tensor =
+ matmul_operation.template get_right_input_cooperative_tensor<
+ T,
+ T,
+ AccumulatorType>();
+ auto accumulator_tensor =
+ matmul_operation.template get_destination_cooperative_tensor<
+ decltype(left_tensor),
+ decltype(right_tensor),
+ AccumulatorType>();
+
+ const uint leading_dimension_d = params->leading_dimension_d;
+
+ AccumulatorType accumulator_storage[TILES_M * TILES_N]
+ [ACCUMULATOR_CAPACITY] = {{0}};
+
+ const uint full_prefetch_iterations = params->K / PREFETCH_K;
+ const uint k_remainder = params->K - full_prefetch_iterations * PREFETCH_K;
+
+ const ushort actual_block_rows =
+ align_m ? BLOCK_ROWS
+ : ushort(min(uint(BLOCK_ROWS), params->M - block_row_start));
+ const ushort actual_block_cols =
+ align_n ? BLOCK_COLS
+ : ushort(min(uint(BLOCK_COLS), params->N - block_col_start));
+
+ // Main loop
+ for (uint outer_k = 0; outer_k < full_prefetch_iterations; outer_k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (align_m) {
+ left_loader.load_unsafe();
+ } else {
+ left_loader.load_safe(short2(PREFETCH_K, actual_block_rows));
+ }
+ if (align_n) {
+ right_loader.load_unsafe();
+ } else {
+ right_loader.load_safe(short2(PREFETCH_K, actual_block_cols));
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ accumulate_tiles(
+ INNER_K_STEPS,
+ tile_row_offset,
+ tile_col_offset,
+ left_shared,
+ right_shared,
+ matmul_operation,
+ left_tensor,
+ right_tensor,
+ accumulator_tensor,
+ accumulator_storage
+ );
+
+ left_loader.next();
+ right_loader.next();
+ }
+
+ // Remainder loop
+ if (k_remainder > 0) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ left_loader.load_safe(short2(k_remainder, actual_block_rows));
+ right_loader.load_safe(short2(k_remainder, actual_block_cols));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const short remainder_steps =
+ short((k_remainder + MATMUL_K_STEP - 1) / MATMUL_K_STEP);
+
+ accumulate_tiles(
+ remainder_steps,
+ tile_row_offset,
+ tile_col_offset,
+ left_shared,
+ right_shared,
+ matmul_operation,
+ left_tensor,
+ right_tensor,
+ accumulator_tensor,
+ accumulator_storage
+ );
+ }
+
+ // Store results
+ METAL_PRAGMA_UNROLL
+ for (short tile_m = 0; tile_m < TILES_M; tile_m++) {
+ METAL_PRAGMA_UNROLL
+ for (short tile_n = 0; tile_n < TILES_N; tile_n++) {
+ const short row_offset = tile_m * SUBTILE_ROWS;
+ const short col_offset = tile_n * SUBTILE_COLS;
+ const short m_limit =
+ align_m ? SUBTILE_ROWS
+ : short(max(0, int(simdgroup_limit_m) - row_offset));
+ const short n_limit =
+ align_n ? SUBTILE_COLS
+ : short(max(0, int(simdgroup_limit_n) - col_offset));
+ if (m_limit <= 0 || n_limit <= 0)
+ continue;
+
+ device T* output_tile_ptr =
+ output_ptr + row_offset * leading_dimension_d + col_offset;
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < short(ACCUMULATOR_CAPACITY); i++) {
+ auto coord = accumulator_tensor.get_multidimensional_index(i);
+ const bool is_valid_element = accumulator_tensor.is_valid_element(i);
+ AccumulatorType accumulated_value =
+ accumulator_storage[tile_m * TILES_N + tile_n][i] *
+ AccumulatorType(ab_scale);
+ if (align_m && align_n) {
+ if (is_valid_element) {
+ if (is_accumulate) {
+ accumulated_value += AccumulatorType(
+ output_tile_ptr[coord[1] * leading_dimension_d + coord[0]]
+ );
+ }
+ output_tile_ptr[coord[1] * leading_dimension_d + coord[0]] =
+ T(accumulated_value);
+ }
+ } else {
+ if (is_valid_element && coord[1] < m_limit && coord[0] < n_limit) {
+ if (is_accumulate) {
+ accumulated_value += AccumulatorType(
+ output_tile_ptr[coord[1] * leading_dimension_d + coord[0]]
+ );
+ }
+ output_tile_ptr[coord[1] * leading_dimension_d + coord[0]] =
+ T(accumulated_value);
+ }
+ }
+ }
+ }
+ }
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
new file mode 100644
index 000000000..e0a2a8b43
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
@@ -0,0 +1,56 @@
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+
+#include "common/gemm_mpp_core.h"
+
+using namespace uzu::matmul;
+
+#define GEMM_MPP_THREADGROUP_TILE_SIZE (128 * (32 + (short)(16 / sizeof(T))))
+
+template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(BLOCK_ROWS, 32, 64)
+VARIANTS(BLOCK_COLS, 32, 64)
+VARIANTS(SIMDGROUPS_PER_ROW, 2, 4)
+VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)
+CONSTRAINT(BLOCK_ROWS >= 16 * SIMDGROUPS_PER_ROW && BLOCK_COLS >= 32 * SIMDGROUPS_PER_COLUMN)
+KERNEL(MatmulGemmMpp)(
+ const device T* left_matrix,
+ const device T* right_matrix,
+ device T* output_matrix,
+ const constant uzu::matmul::GemmParams* params,
+ const constant uint& group_count_x,
+ const constant uint& group_count_y,
+ const constant float& ab_scale,
+ const bool align_m SPECIALIZE,
+ const bool align_n SPECIALIZE,
+ const bool is_accumulate SPECIALIZE,
+ threadgroup T left_shared[GEMM_MPP_THREADGROUP_TILE_SIZE],
+ threadgroup T right_shared[GEMM_MPP_THREADGROUP_TILE_SIZE],
+ const uint group_x GROUPS(group_count_x),
+ const uint group_y GROUPS(group_count_y),
+ const uint thread_x THREADS(32),
+ const uint thread_y THREADS(2),
+ const uint thread_z THREADS(2),
+ const ThreadContext thread_context
+) {
+ ThreadgroupGemmMpp<
+ T,
+ BLOCK_ROWS,
+ BLOCK_COLS,
+ SIMDGROUPS_PER_ROW,
+ SIMDGROUPS_PER_COLUMN>::
+ run(left_matrix,
+ right_matrix,
+ output_matrix,
+ params,
+ align_m,
+ align_n,
+ is_accumulate,
+ ab_scale,
+ left_shared,
+ right_shared,
+ thread_context.simdgroup_index,
+ thread_context.threadgroup_index,
+ uint2(group_x, group_y));
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs
new file mode 100644
index 000000000..03510f4e6
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs
@@ -0,0 +1,189 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct GemmMppSpecialization {
+ pub block_rows: u32,
+ pub block_cols: u32,
+ pub simdgroups_per_row: u32,
+ pub simdgroups_per_column: u32,
+ pub align_m: bool,
+ pub align_n: bool,
+ pub is_accumulate: bool,
+}
+
+impl GemmMppSpecialization {
+ pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
+ if !matches!(data_type, crate::DataType::F16 | crate::DataType::BF16) {
+ return &[];
+ }
+ &[
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: true,
+ align_n: true,
+ is_accumulate: false,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: false,
+ align_n: true,
+ is_accumulate: false,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: true,
+ align_n: false,
+ is_accumulate: false,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: false,
+ align_n: false,
+ is_accumulate: false,
+ },
+ Self {
+ block_rows: 32,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: true,
+ align_n: true,
+ is_accumulate: false,
+ },
+ Self {
+ block_rows: 32,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: false,
+ align_n: true,
+ is_accumulate: false,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 32,
+ simdgroups_per_row: 4,
+ simdgroups_per_column: 1,
+ align_m: true,
+ align_n: true,
+ is_accumulate: false,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 32,
+ simdgroups_per_row: 4,
+ simdgroups_per_column: 1,
+ align_m: true,
+ align_n: false,
+ is_accumulate: false,
+ },
+ // Accumulate variants
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: true,
+ align_n: true,
+ is_accumulate: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: false,
+ align_n: true,
+ is_accumulate: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: true,
+ align_n: false,
+ is_accumulate: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: false,
+ align_n: false,
+ is_accumulate: true,
+ },
+ Self {
+ block_rows: 32,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: true,
+ align_n: true,
+ is_accumulate: true,
+ },
+ Self {
+ block_rows: 32,
+ block_cols: 64,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
+ align_m: false,
+ align_n: true,
+ is_accumulate: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 32,
+ simdgroups_per_row: 4,
+ simdgroups_per_column: 1,
+ align_m: true,
+ align_n: true,
+ is_accumulate: true,
+ },
+ Self {
+ block_rows: 64,
+ block_cols: 32,
+ simdgroups_per_row: 4,
+ simdgroups_per_column: 1,
+ align_m: true,
+ align_n: false,
+ is_accumulate: true,
+ },
+ ]
+ }
+
+ pub fn select(
+ m: u32,
+ n: u32,
+ is_accumulate: bool,
+ ) -> Self {
+ let (block_rows, block_cols, simdgroups_per_row, simdgroups_per_column) = if n < 64 {
+ (64, 32, 4u32, 1u32)
+ } else if m < 64 {
+ (32, 64, 2u32, 2u32)
+ } else {
+ (64, 64, 2u32, 2u32)
+ };
+
+ Self {
+ block_rows,
+ block_cols,
+ simdgroups_per_row,
+ simdgroups_per_column,
+ align_m: (m % block_rows) == 0,
+ align_n: (n % block_cols) == 0,
+ is_accumulate,
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index 9ebb64163..f8f3a852a 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -17,18 +17,23 @@ use crate::{
metal::{
Metal,
context::MetalContext,
- kernel::{MatmulGemmMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel},
+ kernel::{
+ MatmulGemmMetalKernel, MatmulGemmMppMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel,
+ },
+ metal_extensions::DeviceExt,
},
},
};
mod gemm;
+mod gemm_mpp;
mod gemv;
pub struct MatmulMetalKernel {
data_type: DataType,
gemv_kernels: HashMap<gemv::GemvSpecialization, MatmulGemvMetalKernel>,
gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,
+ gemm_mpp_kernels: HashMap<gemm_mpp::GemmMppSpecialization, MatmulGemmMppMetalKernel>,
bias_add: TensorAddBiasMetalKernel,
}
@@ -43,6 +48,13 @@ fn max_gemv_batch_threshold() -> u32 {
}
impl MatmulMetalKernel {
+ fn is_mpp_eligible(
+ &self,
+ context: &MetalContext,
+ ) -> bool {
+ context.device.supports_mxu() && matches!(self.data_type, DataType::F16 | DataType::BF16)
+ }
+
fn get_or_create_gemv(
&mut self,
context: &MetalContext,
@@ -95,6 +107,31 @@ impl MatmulMetalKernel {
}
}
+ fn get_or_create_gemm_mpp(
+ &mut self,
+ context: &MetalContext,
+ specialization: gemm_mpp::GemmMppSpecialization,
+ ) -> Result<&MatmulGemmMppMetalKernel, MatmulError<Metal>> {
+ match self.gemm_mpp_kernels.entry(specialization) {
+ Entry::Occupied(entry) => Ok(entry.into_mut()),
+ Entry::Vacant(entry) => {
+ let kernel = MatmulGemmMppMetalKernel::new(
+ context,
+ self.data_type,
+ specialization.block_rows,
+ specialization.block_cols,
+ specialization.simdgroups_per_row,
+ specialization.simdgroups_per_column,
+ specialization.align_m,
+ specialization.align_n,
+ specialization.is_accumulate,
+ )
+ .map_err(MatmulError::BackendError)?;
+ Ok(entry.insert(kernel))
+ },
+ }
+ }
+
fn encode_gemv(
&mut self,
context: &MetalContext,
@@ -153,6 +190,7 @@ impl MatmulMetalKernel {
threadgroups_per_column,
swizzle_log: 0,
aligned_inner_iterations: arguments.input_dim / specialization.block_depth,
+ use_morton: false,
};
let kernel = self.get_or_create_gemm(context, specialization)?;
@@ -181,6 +219,102 @@ impl MatmulMetalKernel {
Ok(())
}
+
+ fn encode_gemm_mpp(
+ &mut self,
+ context: &MetalContext,
+ encoder: &mut Encoder<Metal>,
+ arguments: MatmulArguments<Metal>,
+ ) -> Result<(), MatmulError<Metal>> {
+ let is_accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);
+ let specialization =
+ gemm_mpp::GemmMppSpecialization::select(arguments.batch_dim, arguments.output_dim, is_accumulate);
+
+ let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);
+ let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);
+
+ let max_threadgroups_per_dim = threadgroups_per_row.max(threadgroups_per_column);
+ let min_threadgroups_per_dim = threadgroups_per_row.min(threadgroups_per_column);
+ let morton_dim = max_threadgroups_per_dim.next_power_of_two();
+ let morton_total = morton_dim.saturating_mul(morton_dim);
+ let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);
+ // Morton ordering improves cache locality but requires padding to a square power-of-two grid.
+ // Allow up to 4x overhead (idle threadgroups) before the padding cost outweighs the benefit.
+ let use_morton = min_threadgroups_per_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);
+
+ let params = GemmParams {
+ M: arguments.batch_dim,
+ N: arguments.output_dim,
+ K: arguments.input_dim,
+ leading_dimension_a: arguments.input_dim,
+ leading_dimension_b: arguments.input_dim,
+ leading_dimension_d: arguments.output_dim,
+ threadgroups_per_row,
+ threadgroups_per_column,
+ swizzle_log: 0,
+ aligned_inner_iterations: 0,
+ use_morton,
+ };
+
+ let (group_count_x, group_count_y) = if use_morton {
+ (morton_total, 1_u32)
+ } else {
+ (threadgroups_per_row, threadgroups_per_column)
+ };
+
+ let kernel = self.get_or_create_gemm_mpp(context, specialization)?;
+ kernel.encode(
+ (arguments.a, arguments.a_offset as usize),
+ arguments.b,
+ &mut *arguments.d,
+ std::slice::from_ref(&params),
+ group_count_x,
+ group_count_y,
+ arguments.ab_scale,
+ encoder,
+ );
+
+ if let MatmulArgumentC::Bias(bias) = arguments.c {
+ self.bias_add.encode(
+ None::<&<Metal as crate::backends::common::Backend>::Buffer>,
+ bias,
+ arguments.d,
+ arguments.output_dim,
+ arguments.batch_dim * arguments.output_dim,
+ encoder,
+ );
+ }
+
+ Ok(())
+ }
+}
+
+/// Explicit dispatch paths for testing individual kernels independent of production routing.
+#[derive(Debug, Clone, Copy)]
+pub enum MatmulDispatchPath {
+ Auto,
+ Gemv,
+ Gemm,
+ GemmMpp,
+}
+
+impl MatmulMetalKernel {
+ pub fn encode_with_path(
+ &mut self,
+ context: &MetalContext,
+ arguments: MatmulArguments<Metal>,
+ encoder: &mut Encoder<Metal>,
+ path: MatmulDispatchPath,
+ ) {
+ match path {
+ MatmulDispatchPath::Auto => self.encode(context, arguments, encoder),
+ MatmulDispatchPath::Gemv => self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV"),
+ MatmulDispatchPath::Gemm => self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM"),
+ MatmulDispatchPath::GemmMpp => {
+ self.encode_gemm_mpp(context, encoder, arguments).expect("Failed to encode GEMM MPP")
+ },
+ }
+ }
}
impl MatmulKernel for MatmulMetalKernel {
@@ -200,6 +334,7 @@ impl MatmulKernel for MatmulMetalKernel {
data_type,
gemv_kernels: HashMap::new(),
gemm_kernels: HashMap::new(),
+ gemm_mpp_kernels: HashMap::new(),
bias_add,
};
@@ -209,6 +344,11 @@ impl MatmulKernel for MatmulMetalKernel {
for &config in gemm::GemmSpecialization::precompile_configs(data_type) {
kernel.get_or_create_gemm(context, config)?;
}
+ if context.device.supports_mxu() {
+ for &config in gemm_mpp::GemmMppSpecialization::precompile_configs(data_type) {
+ kernel.get_or_create_gemm_mpp(context, config)?;
+ }
+ }
Ok(kernel)
}
@@ -221,6 +361,8 @@ impl MatmulKernel for MatmulMetalKernel {
) {
if arguments.batch_dim <= max_gemv_batch_threshold() {
self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");
+ } else if self.is_mpp_eligible(context) {
+ self.encode_gemm_mpp(context, encoder, arguments).expect("Failed to encode GEMM MPP kernel");
} else {
self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index 90cc84f97..2c46971e0 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -304,7 +304,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
// layout used above.
const uint lane_qid = sg_lin >> 2;
const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
+ const uint lane_col_base = ((lane_qid & 2u) * 2u) + ((sg_lin & 1u) * 2u);
METAL_PRAGMA_UNROLL
for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
@@ -567,7 +567,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
const uint lane_row =
(lane_qid & 4u) + ((thread_context.simdgroup_index >> 1) & 3u);
const uint lane_col_base =
- ((lane_qid & 2u) << 1) + ((thread_context.simdgroup_index & 1u) << 1);
+ ((lane_qid & 2u) * 2u) + ((thread_context.simdgroup_index & 1u) * 2u);
const uint local_row = row_sg_off + lane_row;
if (local_row >= m_rows)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
index a6489393a..c9ee593df 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
@@ -35,7 +35,7 @@ PUBLIC KERNEL(MoeTileScan)(
// Inclusive scan in-place over scratch
// Simple O(log E) scan; E is small
- for (uint offset = 1u; offset < e; offset <<= 1u) {
+ for (uint offset = 1u; offset < e; offset *= 2u) {
uint val = 0u;
if (tid < e) {
val = scratch[tid];
@@ -105,4 +105,4 @@ PUBLIC KERNEL(MoeWriteDispatchArgs)(
dispatch_args[0] = num_tiles_n; // x
dispatch_args[1] = total_rows; // y
dispatch_args[2] = 1u; // z
-}
\ No newline at end of file
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
index a27c349e9..c0122c0cb 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
@@ -139,9 +139,9 @@ inline U qdot_zero_point(
if (bits == 4) {
const device uint16_t* ws = (const device uint16_t*)w;
const uint16_t zp0 = static_cast<uint16_t>(zero_point);
- const uint16_t zp1 = static_cast<uint16_t>(zero_point) << 4;
- const uint16_t zp2 = static_cast<uint16_t>(zero_point) << 8;
- const uint16_t zp3 = static_cast<uint16_t>(zero_point) << 12;
+ const uint16_t zp1 = static_cast<uint16_t>(zero_point) * 16;
+ const uint16_t zp2 = static_cast<uint16_t>(zero_point) * 256;
+ const uint16_t zp3 = static_cast<uint16_t>(zero_point) * 4096;
for (int i = 0; i < (values_per_thread / 4); i++) {
uint16_t word = ws[i];
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index f747f319c..d97a298fe 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -11,4 +11,5 @@ mod metal_extensions;
pub use backend::Metal;
pub use context::MetalContext;
pub use device_capabilities::MetalDeviceCapabilities;
-pub use metal_extensions::DeviceGeneration;
+pub use kernel::matmul::MatmulDispatchPath;
+pub use metal_extensions::{DeviceExt, DeviceGeneration};
diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rs
index b988e2c96..915816913 100644
--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs
@@ -578,8 +578,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let mut mask = vec![0_u32; row_words];
for token_index in 0..limit {
let word = token_index / 32;
- let bit = token_index % 32;
- mask[word] |= 1_u32 << bit;
+ mask[word] |= 2_u32.pow((token_index % 32) as u32);
}
self.single_token_vocab_masks.insert(limit, mask.into_boxed_slice());
Ok(())
@@ -605,8 +604,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
];
for token_index in 0..limit {
let word = token_index / 32;
- let bit = token_index % 32;
- mask[row_words + word] |= 1_u32 << bit;
+ mask[row_words + word] |= 2_u32.pow((token_index % 32) as u32);
}
self.two_token_vocab_masks.insert(limit, mask.into_boxed_slice());
Ok(())
@@ -852,8 +850,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let mut mask = vec![0_u32; row_words];
for token_index in 0..limit {
let word = token_index / 32;
- let bit = token_index % 32;
- mask[word] |= 1_u32 << bit;
+ mask[word] |= 2_u32.pow((token_index % 32) as u32);
}
Some(mask.into_boxed_slice())
}
@@ -870,8 +867,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let mut mask = vec![0_u32; total_words];
for token_index in 0..limit {
let word = token_index / 32;
- let bit = token_index % 32;
- mask[sampling_start * row_words + word] |= 1_u32 << bit;
+ mask[sampling_start * row_words + word] |= 2_u32.pow((token_index % 32) as u32);
}
Some(mask.into_boxed_slice())
}
diff --git a/crates/uzu/src/session/tts_session/decoder_support.rs b/crates/uzu/src/session/tts_session/decoder_support.rs
index 773bf7e39..827fe299c 100644
--- a/crates/uzu/src/session/tts_session/decoder_support.rs
+++ b/crates/uzu/src/session/tts_session/decoder_support.rs
@@ -551,13 +551,11 @@ pub(super) fn build_semantic_sampling_mask_row(
for token_index in semantic_begin..=semantic_end {
let token = usize::try_from(token_index).map_err(|_| Error::UnableToLoadConfig)?;
let word = token / 32;
- let bit = token % 32;
- mask[word] |= 1_u32 << bit;
+ mask[word] |= 2_u32.pow((token % 32) as u32);
}
let im_end_token = usize::try_from(im_end).map_err(|_| Error::UnableToLoadConfig)?;
let word = im_end_token / 32;
- let bit = im_end_token % 32;
- mask[word] |= 1_u32 << bit;
+ mask[word] |= 2_u32.pow((im_end_token % 32) as u32);
Ok(mask.into_boxed_slice())
}
@@ -570,11 +568,10 @@ pub(super) fn clear_token_in_sampling_mask(
}
let token = usize::try_from(token).map_err(|_| Error::UnableToLoadConfig)?;
let word = token / 32;
- let bit = token % 32;
if word >= mask.len() {
return Err(Error::UnableToLoadConfig);
}
- mask[word] &= !(1_u32 << bit);
+ mask[word] &= !2_u32.pow((token % 32) as u32);
Ok(())
}
diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
index ebb324b74..3121a5f82 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
@@ -22,9 +22,9 @@ fn pack_u4_weights(values: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity((values.len() + 3) / 4 * 2);
for chunk in values.chunks(4) {
let w0 = (*chunk.get(0).unwrap_or(&0) as u16) & 0x0F;
- let w1 = ((*chunk.get(1).unwrap_or(&0) as u16) & 0x0F) << 4;
- let w2 = ((*chunk.get(2).unwrap_or(&0) as u16) & 0x0F) << 8;
- let w3 = ((*chunk.get(3).unwrap_or(&0) as u16) & 0x0F) << 12;
+ let w1 = ((*chunk.get(1).unwrap_or(&0) as u16) & 0x0F) * 16;
+ let w2 = ((*chunk.get(2).unwrap_or(&0) as u16) & 0x0F) * 256;
+ let w3 = ((*chunk.get(3).unwrap_or(&0) as u16) & 0x0F) * 4096;
let word: u16 = w0 | w1 | w2 | w3;
out.push(word as u8);
@@ -99,7 +99,7 @@ fn get_4bit_value(
let byte_idx = word_idx * 2;
let word = if byte_idx + 1 < data.len() {
- data[byte_idx] as u16 | ((data[byte_idx + 1] as u16) << 8)
+ data[byte_idx] as u16 | ((data[byte_idx + 1] as u16) * 256)
} else {
0
};
diff --git a/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
index e77fa3c87..d1b31c51e 100644
--- a/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
+++ b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
@@ -42,7 +42,7 @@ fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Inpu
let packing_divisor = quant_mode.packing_divisor() as u32;
let weights_stride = model_dim / packing_divisor;
let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize)
- .map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) << 4))
+ .map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) * 16))
.collect();
let num_groups = (model_dim + group_size - 1) / group_size;
diff --git a/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
new file mode 100644
index 000000000..d2fb5eff9
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
@@ -0,0 +1,330 @@
+#![cfg(metal_backend)]
+
+use std::{ops::{Deref, DerefMut}, rc::Rc};
+
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::bf16;
+use uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::{
+ common::{
+ Backend, Context, Encoder,
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ },
+ },
+ cpu::Cpu,
+ metal::{DeviceExt, MatmulDispatchPath, Metal, MetalContext},
+ },
+};
+
+use crate::{
+ common::{assert::assert_eq_float, type_short_name},
+ uzu_bench, uzu_test,
+};
+
+struct Input {
+ left: Box<[bf16]>,
+ right: Box<[bf16]>,
+ destination_prefill: Option<Box<[bf16]>>,
+ batch_dim: usize,
+ input_dim: usize,
+ output_dim: usize,
+ ab_scale: f32,
+}
+
+fn get_test_data(
+ batch_dim: usize,
+ input_dim: usize,
+ output_dim: usize,
+ ab_scale: f32,
+ accumulate: bool,
+) -> Input {
+ let left: Vec<bf16> = (0..batch_dim * input_dim)
+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.1 - 0.6))
+ .collect();
+ let right: Vec<bf16> = (0..output_dim * input_dim)
+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.1 - 0.8))
+ .collect();
+ let destination_prefill = if accumulate {
+ Some(
+ (0..batch_dim * output_dim)
+ .map(|i| bf16::from_f32(((i % 7) as f32) * 0.03 - 0.09))
+ .collect(),
+ )
+ } else {
+ None
+ };
+
+ Input {
+ left: left.into_boxed_slice(),
+ right: right.into_boxed_slice(),
+ destination_prefill,
+ batch_dim,
+ input_dim,
+ output_dim,
+ ab_scale,
+ }
+}
+
+fn get_cpu_output(input: &Input) -> Vec<bf16> {
+ let cpu_context = <Cpu as Backend>::Context::new().expect("CPU context");
+
+ let left_array = cpu_context.create_array_from(&[input.batch_dim, input.input_dim], &input.left, "");
+ let right_array = cpu_context.create_array_from(&[input.output_dim, input.input_dim], &input.right, "");
+ let destination_array = if let Some(ref prefill) = input.destination_prefill {
+ cpu_context.create_array_from(&[input.batch_dim, input.output_dim], prefill, "")
+ } else {
+ cpu_context.create_array_uninitialized(&[input.batch_dim, input.output_dim], bf16::data_type(), "")
+ };
+
+ let left_buffer = left_array.buffer();
+ let left_ref = left_buffer.borrow();
+ let right_buffer = right_array.buffer();
+ let right_ref = right_buffer.borrow();
+ let destination_buffer = destination_array.buffer();
+ let mut destination_ref = destination_buffer.borrow_mut();
+
+ let argument_c = if input.destination_prefill.is_some() {
+ MatmulArgumentC::Accumulate
+ } else {
+ MatmulArgumentC::None
+ };
+
+ let mut matmul_kernel =
+ <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&cpu_context, bf16::data_type())
+ .expect("CPU MatmulKernel");
+
+ let mut encoder = Encoder::new(cpu_context.as_ref()).expect("encoder");
+ matmul_kernel.encode(
+ &cpu_context,
+ MatmulArguments {
+ a: left_ref.deref(),
+ a_offset: 0,
+ b: right_ref.deref(),
+ ab_scale: input.ab_scale,
+ c: argument_c,
+ d: destination_ref.deref_mut(),
+ batch_dim: input.batch_dim as u32,
+ input_dim: input.input_dim as u32,
+ output_dim: input.output_dim as u32,
+ },
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ drop(destination_ref);
+ destination_array.as_slice().to_vec()
+}
+
+fn get_mpp_output(
+ metal_context: &MetalContext,
+ matmul_kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,
+ input: &Input,
+) -> Vec<bf16> {
+ let left_array = metal_context.create_array_from(&[input.batch_dim, input.input_dim], &input.left, "");
+ let right_array = metal_context.create_array_from(&[input.output_dim, input.input_dim], &input.right, "");
+ let destination_array = if let Some(ref prefill) = input.destination_prefill {
+ metal_context.create_array_from(&[input.batch_dim, input.output_dim], prefill, "")
+ } else {
+ metal_context.create_array_uninitialized(&[input.batch_dim, input.output_dim], bf16::data_type(), "")
+ };
+
+ let left_buffer = left_array.buffer();
+ let left_ref = left_buffer.borrow();
+ let right_buffer = right_array.buffer();
+ let right_ref = right_buffer.borrow();
+ let destination_buffer = destination_array.buffer();
+ let mut destination_ref = destination_buffer.borrow_mut();
+
+ let argument_c = if input.destination_prefill.is_some() {
+ MatmulArgumentC::Accumulate
+ } else {
+ MatmulArgumentC::None
+ };
+
+ let mut encoder = Encoder::new(metal_context).expect("encoder");
+ matmul_kernel.encode_with_path(
+ metal_context,
+ MatmulArguments {
+ a: left_ref.deref(),
+ a_offset: 0,
+ b: right_ref.deref(),
+ ab_scale: input.ab_scale,
+ c: argument_c,
+ d: destination_ref.deref_mut(),
+ batch_dim: input.batch_dim as u32,
+ input_dim: input.input_dim as u32,
+ output_dim: input.output_dim as u32,
+ },
+ &mut encoder,
+ MatmulDispatchPath::GemmMpp,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ drop(destination_ref);
+ destination_array.as_slice().to_vec()
+}
+
+fn create_metal_context_and_matmul_kernel()
+ -> (Rc<MetalContext>, <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel)
+{
+ let metal_context = MetalContext::new().expect("Metal context");
+ let matmul_kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &metal_context,
+ bf16::data_type(),
+ )
+ .expect("MatmulKernel");
+ (metal_context, matmul_kernel)
+}
+
+fn test_mpp_correctness(
+ batch_dim: usize,
+ input_dim: usize,
+ output_dim: usize,
+ ab_scale: f32,
+ accumulate: bool,
+ tolerance: f32,
+) {
+ let (metal_context, mut matmul_kernel) = create_metal_context_and_matmul_kernel();
+ if !metal_context.device.supports_mxu() {
+ eprintln!("Skipping MPP test: device does not support MXU");
+ return;
+ }
+
+ let input = get_test_data(batch_dim, input_dim, output_dim, ab_scale, accumulate);
+ let expected = get_cpu_output(&input);
+ let actual = get_mpp_output(&metal_context, &mut matmul_kernel, &input);
+ assert_eq_float(&expected, &actual, tolerance, "GEMM_MPP vs CPU reference");
+}
+
+// Basic correctness — explicitly dispatched to MPP kernel
+
+#[uzu_test]
+fn test_bf16_aligned() {
+ test_mpp_correctness(64, 2048, 2048, 1.0, false, 1.0);
+}
+
+#[uzu_test]
+fn test_bf16_unaligned_m() {
+ test_mpp_correctness(33, 2048, 2048, 1.0, false, 1.0);
+}
+
+#[uzu_test]
+fn test_bf16_unaligned_n() {
+ test_mpp_correctness(64, 2048, 33, 1.0, false, 1.0);
+}
+
+#[uzu_test]
+fn test_bf16_large() {
+ test_mpp_correctness(128, 4096, 14336, 1.0, false, 2.0);
+}
+
+#[uzu_test]
+fn test_bf16_small_batch() {
+ test_mpp_correctness(1, 2048, 2048, 1.0, false, 1.0);
+}
+
+// ab_scale
+
+#[uzu_test]
+fn test_bf16_ab_scale() {
+ test_mpp_correctness(64, 2048, 2048, 0.5, false, 1.0);
+}
+
+// accumulate
+
+#[uzu_test]
+fn test_bf16_accumulate() {
+ test_mpp_correctness(64, 2048, 2048, 1.0, true, 1.0);
+}
+
+// scale + accumulate combined
+
+#[uzu_test]
+fn test_bf16_scale_and_accumulate() {
+ test_mpp_correctness(64, 2048, 2048, 0.5, true, 1.0);
+}
+
+// Benchmarks
+
+const BENCHMARK_SHAPES: &[(usize, usize, usize)] = &[
+ (128, 2048, 8192),
+ (128, 4096, 14336),
+ (256, 4096, 4096),
+ (512, 8192, 2048),
+];
+
+#[uzu_bench]
+fn bench_gemm_mpp(criterion: &mut Criterion) {
+ let metal_context = MetalContext::new().unwrap();
+ if !metal_context.device.supports_mxu() {
+ return;
+ }
+
+ let mut matmul_kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &metal_context,
+ bf16::data_type(),
+ )
+ .expect("MatmulKernel");
+
+ let mut benchmark_group = criterion.benchmark_group(
+ format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()),
+ );
+
+ for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {
+ let left_array =
+ metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");
+ let right_array =
+ metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");
+ let destination_array =
+ metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");
+
+ let floating_point_operations = 2 * batch_dim * input_dim * output_dim;
+ benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));
+
+ benchmark_group.bench_function(
+ BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),
+ |bencher| {
+ let left_buffer = left_array.buffer();
+ let left_ref = left_buffer.borrow();
+ let right_buffer = right_array.buffer();
+ let right_ref = right_buffer.borrow();
+ let destination_buffer = destination_array.buffer();
+
+ bencher.iter_custom(|iteration_count| {
+ let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();
+
+ for _ in 0..iteration_count {
+ let mut destination_ref = destination_buffer.borrow_mut();
+ matmul_kernel.encode_with_path(
+ &metal_context,
+ MatmulArguments {
+ a: left_ref.deref(),
+ a_offset: 0,
+ b: right_ref.deref(),
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
+ d: destination_ref.deref_mut(),
+ batch_dim: batch_dim as u32,
+ input_dim: input_dim as u32,
+ output_dim: output_dim as u32,
+ },
+ &mut encoder,
+ MatmulDispatchPath::GemmMpp,
+ );
+ }
+
+ encoder
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .unwrap()
+ .gpu_execution_time()
+ .unwrap()
+ })
+ },
+ );
+ }
+}
diff --git a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemm_test.rs
similarity index 80%
rename from crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs
rename to crates/uzu/tests/unit/kernel/matmul/gemm_test.rs
index d803ec381..c29779868 100644
--- a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/gemm_test.rs
@@ -45,11 +45,11 @@ fn get_test_data<T: ArrayElement + Float>(
n,
};
- let expected = get_output::<T, Cpu>(&input);
+ let expected = get_output::<T, Cpu>(&input, 1.0);
(input, expected)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>, ab_scale: f32) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let m = input.m as u32;
@@ -77,7 +77,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
a: a_ref.deref(),
a_offset: 0,
b: b_ref.deref(),
- ab_scale: 1.0,
+ ab_scale,
c: MatmulArgumentC::None,
d: d_ref.deref_mut(),
batch_dim: m,
@@ -100,11 +100,26 @@ fn test<T: ArrayElement + Float + Debug + Display>(
) {
let (input, expected) = get_test_data::<T>(m, k, n);
for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input);
+ let output = get_output::<T, B>(&input, 1.0);
assert_eq_float(&expected, &output, eps, &format!("backend {}", std::any::type_name::<B>()));
});
}
+fn test_with_scale<T: ArrayElement + Float + Debug + Display>(
+ m: usize,
+ k: usize,
+ n: usize,
+ ab_scale: f32,
+ eps: f32,
+) {
+ let (input, _) = get_test_data::<T>(m, k, n);
+ let expected = get_output::<T, Cpu>(&input, ab_scale);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input, ab_scale);
+ assert_eq_float(&expected, &output, eps, &format!("backend {} ab_scale={ab_scale}", std::any::type_name::<B>()));
+ });
+}
+
#[uzu_test]
fn test_f32_aligned() {
test::<f32>(64, 64, 64, 0.01);
@@ -149,3 +164,15 @@ fn test_f16_large() {
fn test_bf16_large() {
test::<bf16>(16, 128, 256, 0.1);
}
+
+// ab_scale tests
+
+#[uzu_test]
+fn test_f32_ab_scale() {
+ test_with_scale::<f32>(16, 128, 256, 0.5, 0.01);
+}
+
+#[uzu_test]
+fn test_bf16_ab_scale() {
+ test_with_scale::<bf16>(16, 128, 256, 0.5, 0.1);
+}
diff --git a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs
deleted file mode 100644
index 11406a698..000000000
--- a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs
+++ /dev/null
@@ -1,311 +0,0 @@
-//! Correctness tests comparing Metal matmul kernels against ndarray
-
-#![cfg(metal_backend)]
-use bytemuck;
-use half::bf16;
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
-use ndarray::Array2;
-use uzu::{
- DataType,
- backends::{
- common::{
- Backend, Context, Encoder,
- kernel::{
- ManualKernels,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
- },
- },
- metal::Metal,
- },
-};
-
-use crate::uzu_test;
-
-fn run_metal_matmul(
- ctx: &<Metal as Backend>::Context,
- a_data: &[bf16],
- b_data: &[bf16],
- m: usize,
- k: usize,
- n: usize,
-) -> Vec<bf16> {
- let a_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(a_data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let b_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(b_data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let mut d_buf = ctx
- .device
- .new_buffer(m * n * core::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut kernel =
- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");
-
- let mut encoder = Encoder::new(ctx).unwrap();
- kernel.encode(
- ctx,
- MatmulArguments {
- a: &a_buf,
- a_offset: 0,
- b: &b_buf,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
- d: &mut d_buf,
- batch_dim: m as u32,
- input_dim: k as u32,
- output_dim: n as u32,
- },
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- unsafe {
- let ptr = d_buf.contents().as_ptr() as *const bf16;
- std::slice::from_raw_parts(ptr, m * n).to_vec()
- }
-}
-
-fn run_ndarray_matmul(
- a_data: &[bf16],
- b_data: &[bf16],
- m: usize,
- k: usize,
- n: usize,
-) -> Vec<bf16> {
- let a_f32: Vec<f32> = a_data.iter().map(|x| x.to_f32()).collect();
- let b_f32: Vec<f32> = b_data.iter().map(|x| x.to_f32()).collect();
-
- let a_arr = Array2::from_shape_vec((m, k), a_f32).expect("A shape");
- let b_arr = Array2::from_shape_vec((n, k), b_f32).expect("B shape");
-
- let result = a_arr.dot(&b_arr.t());
-
- result.iter().map(|&x| bf16::from_f32(x)).collect()
-}
-
-fn generate_test_data(
- m: usize,
- k: usize,
- n: usize,
-) -> (Vec<bf16>, Vec<bf16>) {
- let a: Vec<bf16> = (0..(m * k)).map(|i| bf16::from_f32(((i % 13) as f32) * 0.01 - 0.06)).collect();
- let b_size = n * k;
- let b: Vec<bf16> = (0..b_size).map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.15)).collect();
- (a, b)
-}
-
-struct TestCase {
- m: usize,
- k: usize,
- n: usize,
- tolerance: f32,
-}
-
-const MATMUL_CORRECTNESS_FULL_ENV: &str = "UZU_MATMUL_CORRECTNESS_FULL";
-
-const QUICK_BATCH_SIZES: [usize; 2] = [1, 16];
-const FULL_BATCH_SIZES: [usize; 12] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];
-
-const QUICK_MODEL_SHAPES: [(usize, usize); 8] =
- [(896, 896), (896, 4864), (1024, 4096), (1152, 6912), (1536, 8960), (2048, 8192), (4096, 14336), (5120, 17408)];
-
-const FULL_MODEL_SHAPES: [(usize, usize); 51] = [
- // Qwen2.5-Coder-0.5B
- (896, 896),
- (896, 1152),
- (896, 4864),
- (4864, 896),
- // Qwen3-0.6B
- (1024, 1024),
- (1024, 3072),
- (1024, 4096),
- (3072, 1024),
- // Gemma-3-1B
- (1152, 1152),
- (1152, 1536),
- (1152, 6912),
- (6912, 1152),
- // Qwen2.5-Coder-1.5B
- (1536, 1536),
- (1536, 2048),
- (1536, 8960),
- (8960, 1536),
- // Llama-3.2-1B / Qwen3-1.7B / SmolLM2-1.7B
- (2048, 2048),
- (2048, 2560),
- (2048, 3072),
- (2048, 4096),
- (2048, 6144),
- (2048, 8192),
- (2048, 11008),
- (6144, 2048),
- (8192, 2048),
- (11008, 2048),
- // Qwen3-4B / Gemma-3-4B
- (2560, 2560),
- (2560, 4096),
- (2560, 6144),
- (2560, 9728),
- (2560, 10240),
- (9728, 2560),
- (10240, 2560),
- // Llama-3.2-3B
- (3072, 3072),
- (3072, 5120),
- (3072, 8192),
- (8192, 3072),
- // Qwen2.5-Coder-7B
- (3584, 3584),
- (3584, 4608),
- (3584, 18944),
- (18944, 3584),
- // Llama-3.1-8B / Qwen3-8B
- (4096, 4096),
- (4096, 6144),
- (4096, 12288),
- (4096, 14336),
- (12288, 4096),
- (14336, 4096),
- // Qwen3-14B
- (5120, 5120),
- (5120, 7168),
- (5120, 17408),
- (17408, 5120),
-];
-
-fn run_full_case_matrix() -> bool {
- std::env::var(MATMUL_CORRECTNESS_FULL_ENV)
- .ok()
- .map(|value| {
- let normalized = value.trim().to_ascii_lowercase();
- matches!(normalized.as_str(), "1" | "true" | "yes")
- })
- .unwrap_or(false)
-}
-
-fn test_cases(full_case_matrix: bool) -> Vec<TestCase> {
- let base_tolerance = 0.01;
- let (batch_sizes, model_shapes): (&[usize], &[(usize, usize)]) = if full_case_matrix {
- (&FULL_BATCH_SIZES, &FULL_MODEL_SHAPES)
- } else {
- (&QUICK_BATCH_SIZES, &QUICK_MODEL_SHAPES)
- };
-
- let mut cases: Vec<(usize, usize, usize)> =
- model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n))).collect();
-
- // Edge case: small matrix
- for &m in batch_sizes {
- cases.push((m, 128, 128));
- }
-
- cases
- .into_iter()
- .map(|(m, k, n)| {
- let tolerance = base_tolerance * (k as f32 / 1024.0).sqrt();
- let tolerance = tolerance * (1.0 + (m as f32).log2() * 0.02);
- TestCase {
- m,
- k,
- n,
- tolerance,
- }
- })
- .collect()
-}
-
-fn compare_results(
- metal: &[bf16],
- reference: &[bf16],
- tolerance: f32,
-) -> Result<(), (f32, usize, usize)> {
- if metal.len() != reference.len() {
- return Err((f32::INFINITY, 0, metal.len()));
- }
-
- let mut max_diff: f32 = 0.0;
- let mut max_diff_idx = 0;
- let mut diff_count = 0;
-
- for (i, (&m_val, &r_val)) in metal.iter().zip(reference.iter()).enumerate() {
- let mf = m_val.to_f32();
- let rf = r_val.to_f32();
- let diff = (mf - rf).abs();
- if diff > max_diff {
- max_diff = diff;
- max_diff_idx = i;
- }
- if diff > tolerance {
- diff_count += 1;
- }
- }
-
- if max_diff <= tolerance {
- Ok(())
- } else {
- Err((max_diff, max_diff_idx, diff_count))
- }
-}
-
-#[uzu_test]
-#[ignore]
-fn matmul_correctness_comprehensive() {
- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
- eprintln!("No Metal device available, skipping test");
- return;
- };
-
- let full_case_matrix = run_full_case_matrix();
- let cases = test_cases(full_case_matrix);
- eprintln!(
- "Running {} matmul correctness case matrix (set {}=1 for exhaustive run)",
- if full_case_matrix {
- "exhaustive"
- } else {
- "quick"
- },
- MATMUL_CORRECTNESS_FULL_ENV
- );
- let mut passed = 0;
- let mut failed = Vec::new();
-
- for case in &cases {
- let (a, b) = generate_test_data(case.m, case.k, case.n);
- let metal_result = run_metal_matmul(&ctx, &a, &b, case.m, case.k, case.n);
- let reference = run_ndarray_matmul(&a, &b, case.m, case.k, case.n);
-
- let trans_str = "T";
-
- match compare_results(&metal_result, &reference, case.tolerance) {
- Ok(()) => {
- passed += 1;
- eprintln!("✓ m={} k={} n={} B={}", case.m, case.k, case.n, trans_str);
- },
- Err((max_diff, idx, count)) => {
- failed.push((case, max_diff, idx, count));
- eprintln!(
- "✗ m={} k={} n={} B={} max_diff={:.6} at idx {} ({} exceed tol {})",
- case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance
- );
- },
- }
- }
-
- eprintln!("\n{}/{} tests passed", passed, cases.len());
-
- if !failed.is_empty() {
- eprintln!("\nFailed tests:");
- for (case, max_diff, idx, count) in &failed {
- let trans_str = "T";
- eprintln!(
- " m={} k={} n={} B={}: max_diff={:.6} at idx {}, {} exceed tol {}",
- case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance
- );
- }
- panic!("{} tests failed", failed.len());
- }
-}
diff --git a/crates/uzu/tests/unit/kernel/matmul/mod.rs b/crates/uzu/tests/unit/kernel/matmul/mod.rs
index 346c2e7ba..e7089955d 100644
--- a/crates/uzu/tests/unit/kernel/matmul/mod.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/mod.rs
@@ -1,2 +1,3 @@
+mod gemm_mpp_test;
+mod gemm_test;
mod gemv_test;
-mod steel_gemm_test;