Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

fix matmul perf

eugenefff0a55290d0
diff --git a/Cargo.lock b/Cargo.lock
index 5b411e5c5..e53cf9531 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4445,10 +4445,12 @@ dependencies = [
"cbindgen",
"clang-ast",
"clap",
+ "comfy-table",
"depfile",
"dsl",
"futures",
"half",
+ "indicatif 0.18.3",
"is_close",
"itertools 0.14.0",
"libc",
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 33d592042..2e581cd82 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -49,6 +49,8 @@ is_close.workspace = true
serde_json5.workspace = true
schemars.workspace = true
mpsgraph.workspace = true
+comfy-table.workspace = true
+indicatif.workspace = true
[build-dependencies]
anyhow.workspace = true
@@ -94,3 +96,7 @@ path = "tests/kernel/mod.rs"
[[test]]
name = "moe"
path = "tests/moe/mod.rs"
+
+[[test]]
+name = "matmul_perf_test"
+path = "tests/matmul_perf_test/main.rs"
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
index 39a4fc783..2c6cf9213 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
@@ -204,7 +204,7 @@ struct GEMMKernel {
// Loop tail
if (!K_aligned) {
- int lbk = params->input_dim - params->gemm_k_iterations_aligned * BK;
+ int lbk = params->K - params->gemm_k_iterations_aligned * BK;
short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);
short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);
@@ -223,10 +223,9 @@ struct GEMMKernel {
///////////////////////////////////////////////////////////////////////////
// MN unaligned loop
else { // Loop over K - unaligned case
- short tgp_bm = min(BM, params->batch - c_row);
- short tgp_bn = min(BN, params->output_dim - c_col);
- short leftover_bk =
- params->input_dim - params->gemm_k_iterations_aligned * BK;
+ short tgp_bm = min(BM, params->M - c_row);
+ short tgp_bn = min(BN, params->N - c_col);
+ short leftover_bk = params->K - params->gemm_k_iterations_aligned * BK;
if (tgp_bm == BM && tgp_bn == BN) {
gemm_loop<true, true, K_aligned>(
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h
new file mode 100644
index 000000000..7386d512f
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "steel/gemm/params.h"
+
+namespace uzu {
+namespace matmul {
+using GEMMParams = steel::GEMMParams;
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
index bbbe12039..018c284a4 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
@@ -2,188 +2,46 @@
#include "../../../common/utils.h"
#include "../../../definitions.metal"
-#include "../../common/steel/gemm/gemm.h"
+#include "../../common/gemm.h"
-using namespace steel;
+using namespace uzu::matmul;
-// Upper bounds for threadgroup memory (in elements of T).
-// Max across all tile/type combos: 64x32x32 with half (padding=8).
-// tgp_mem_size_a = BM * (BK + padding) = 64 * (32 + 8) = 2560
-// tgp_mem_size_b = BN * (BK + padding) = 32 * (32 + 8) = 1280
-// For 64x64x16 with half: a=64*(16+8)=1536, b=64*(16+8)=1536
-// Max a = 2560, max b = 1536
-#define GEMM_MAX_TGP_A 2560
-#define GEMM_MAX_TGP_B 1536
-
-namespace uzu {
-namespace matmul {
-using GEMMParams = steel::GEMMParams;
-} // namespace matmul
-} // namespace uzu
-
-///////////////////////////////////////////////////////////////////////////////
-// GEMM implementation
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename T, typename AccumType = float>
-METAL_FUNC void gemm_impl(
+template <typename T, int BM, int BN, int BK, int WM, int WN, bool MN_aligned, bool K_aligned>
+METAL_FUNC void gemm_dispatch(
const device T* a,
const device T* b,
device T* d,
const constant GEMMParams* params,
- const int BM,
- const int BN,
- const int BK,
- const int WM,
- const int WN,
- const bool align_m,
- const bool align_n,
- const bool align_k,
- threadgroup T* a_shared,
- threadgroup T* b_shared,
+ threadgroup T* As,
+ threadgroup T* Bs,
uint simd_lane_id,
uint simd_group_id,
uint3 tid,
uint3 lid
) {
- (void)lid;
-
- // Hardcoded: transpose_a = false, transpose_b = true
- const bool transpose_a = false;
- const bool transpose_b = true;
-
- const short tgp_padding = 16 / sizeof(T);
- const short lda_tgp = BK + tgp_padding; // transpose_a=false: BK + pad
- const short ldb_tgp = BK + tgp_padding; // transpose_b=true: BK + pad
- const short tgp_size = WM * WN * 32;
-
- // Find block
- const int tid_y = ((tid.y) << params->swizzle_log) +
- ((tid.x) & ((1 << params->swizzle_log) - 1));
- const int tid_x = (tid.x) >> params->swizzle_log;
-
- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
- return;
- }
-
- // Batch offset (non-batched path only)
- a += params->batch_stride_a * tid.z;
- b += params->batch_stride_b * tid.z;
- d += params->batch_stride_d * tid.z;
-
- threadgroup_barrier(mem_flags::mem_none);
-
- // Find block in a, b, d
- const int c_row = tid_y * BM;
- const int c_col = tid_x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- // transpose_a=false: a += c_row * lda
- a += c_row_long * params->lda;
- // transpose_b=true: b += c_col * ldb
- b += c_col_long * params->ldb;
- d += c_row_long * params->ldd + c_col_long;
-
- // Construct loader and MMA objects with runtime tile params
- // transpose_a=false: loader_a BROWS=BM, BCOLS=BK, dst_ld=BK+pad, reduction_dim=1
- thread BlockLoader<T> loader_a(
- a, params->lda, a_shared,
- simd_group_id, simd_lane_id,
- BM, BK, lda_tgp, 1, tgp_size);
-
- // transpose_b=true: loader_b BROWS=BN, BCOLS=BK, dst_ld=BK+pad, reduction_dim=1
- thread BlockLoader<T> loader_b(
- b, params->ldb, b_shared,
- simd_group_id, simd_lane_id,
- BN, BK, ldb_tgp, 1, tgp_size);
-
- thread BlockMMA<T, T, AccumType> mma_op(
- simd_group_id, simd_lane_id,
- BM, BN, BK, WM, WN,
- transpose_a, transpose_b,
- lda_tgp, ldb_tgp);
-
- const short tgp_bm = align_m ? BM : short(min(BM, params->M - c_row));
- const short tgp_bn = align_n ? BN : short(min(BN, params->N - c_col));
-
- int gemm_k_iterations = params->gemm_k_iterations_aligned;
-
- // Do unaligned K iterations first
- if (!align_k) {
- const int k_last = params->gemm_k_iterations_aligned * BK;
- const int k_remain = params->K - k_last;
- // transpose_a=false: k_jump_a = k_last
- const size_t k_jump_a = size_t(k_last);
- // transpose_b=true: k_jump_b = k_last
- const size_t k_jump_b = size_t(k_last);
-
- loader_a.src += k_jump_a;
- loader_b.src += k_jump_b;
-
- // transpose_a=false: tile_dims_a = (k_remain, tgp_bm)
- const short2 tile_dims_a = short2(k_remain, tgp_bm);
- // transpose_b=true: tile_dims_b = (k_remain, tgp_bn)
- const short2 tile_dims_b = short2(k_remain, tgp_bn);
-
- loader_a.load_safe(tile_dims_a);
- loader_b.load_safe(tile_dims_b);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(a_shared, b_shared);
-
- loader_a.src -= k_jump_a;
- loader_b.src -= k_jump_b;
- }
-
- // MNK aligned loop
- if (align_m && align_n) {
- for (int k = 0; k < gemm_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_a.load_unsafe();
- loader_b.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(a_shared, b_shared);
- loader_a.next();
- loader_b.next();
- }
-
- threadgroup_barrier(mem_flags::mem_none);
- return mma_op.store_result(d, params->ldd);
- } else {
- const short leftover_bk = 0;
+ using gemm_kernel = GEMMKernel<T, T, BM, BN, BK, WM, WN, false, true, MN_aligned, K_aligned, float>;
+ gemm_kernel::run(a, b, d, params, As, Bs, simd_lane_id, simd_group_id, tid, lid);
+}
- if ((align_m || tgp_bm == BM) && (align_n || tgp_bn == BN)) {
- gemm_loop<T, T, AccumType>(
- a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,
- mma_op, tgp_bm, tgp_bn, leftover_bk,
- BK, transpose_a, transpose_b, true, true, true);
- return mma_op.store_result(d, params->ldd);
- } else if (align_n || tgp_bn == BN) {
- gemm_loop<T, T, AccumType>(
- a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,
- mma_op, tgp_bm, tgp_bn, leftover_bk,
- BK, transpose_a, transpose_b, false, true, true);
- return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));
- } else if (align_m || tgp_bm == BM) {
- gemm_loop<T, T, AccumType>(
- a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,
- mma_op, tgp_bm, tgp_bn, leftover_bk,
- BK, transpose_a, transpose_b, true, false, true);
- return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));
- } else {
- gemm_loop<T, T, AccumType>(
- a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,
- mma_op, tgp_bm, tgp_bn, leftover_bk,
- BK, transpose_a, transpose_b, false, false, true);
- return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));
- }
+#define GEMM_DISPATCH(T, BM, BN, BK, WM, WN) \
+ if (block_rows == BM && block_cols == BN && block_depth == BK && \
+ warps_per_row == WM && warps_per_col == WN) { \
+ if (align_m && align_n) { \
+ if (align_k) \
+ gemm_dispatch<T, BM, BN, BK, WM, WN, true, true>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint3(group_x, group_y, group_z), uint3(thread_x, thread_y, thread_z)); \
+ else \
+ gemm_dispatch<T, BM, BN, BK, WM, WN, true, false>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint3(group_x, group_y, group_z), uint3(thread_x, thread_y, thread_z)); \
+ } else { \
+ if (align_k) \
+ gemm_dispatch<T, BM, BN, BK, WM, WN, false, true>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint3(group_x, group_y, group_z), uint3(thread_x, thread_y, thread_z)); \
+ else \
+ gemm_dispatch<T, BM, BN, BK, WM, WN, false, false>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint3(group_x, group_y, group_z), uint3(thread_x, thread_y, thread_z)); \
+ } \
+ return; \
}
-}
-///////////////////////////////////////////////////////////////////////////////
-// Unified DSL kernel
-///////////////////////////////////////////////////////////////////////////////
+#define GEMM_MAX_TGP_A 2560
+#define GEMM_MAX_TGP_B 1536
template <typename T>
VARIANTS(T, float, half, bfloat)
@@ -213,16 +71,13 @@ PUBLIC KERNEL(MatmulGemm)(
const uint thread_z THREADS(2),
const Simd simd
) {
- gemm_impl<T, float>(
- a, b, d, params,
- block_rows, block_cols, block_depth,
- warps_per_row, warps_per_col,
- align_m, align_n, align_k,
- a_shared, b_shared,
- simd.lane_idx, simd.group_idx,
- uint3(group_x, group_y, group_z),
- uint3(thread_x, thread_y, thread_z)
- );
+ GEMM_DISPATCH(T, 64, 64, 16, 2, 2)
+ GEMM_DISPATCH(T, 64, 64, 16, 1, 2)
+ GEMM_DISPATCH(T, 64, 32, 32, 2, 2)
+ GEMM_DISPATCH(T, 32, 64, 16, 1, 2)
+ GEMM_DISPATCH(T, 32, 64, 16, 2, 2)
+ GEMM_DISPATCH(T, 32, 32, 16, 2, 2)
+ GEMM_DISPATCH(T, 64, 32, 8, 4, 1)
}
// clang-format on
diff --git a/crates/uzu/tests/matmul_perf_test/bench.rs b/crates/uzu/tests/matmul_perf_test/bench.rs
new file mode 100644
index 000000000..87adf905e
--- /dev/null
+++ b/crates/uzu/tests/matmul_perf_test/bench.rs
@@ -0,0 +1,159 @@
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
+use objc2::{rc::Retained, runtime::ProtocolObject};
+use uzu::{
+ DataType,
+ backends::{
+ common::{
+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending, Context,
+ kernel::matmul::{MatmulArguments, MatmulDispatchDescriptor, MatmulKernel},
+ },
+ metal::Metal,
+ },
+};
+
+use super::{error::BenchError, output::PerfResult, shapes::TestShape};
+
+type Ctx = <Metal as Backend>::Context;
+type Buf = Retained<ProtocolObject<dyn MTLBuffer>>;
+
+const WARMUP_ITERATIONS: usize = 3;
+const BENCHMARK_ITERATIONS: usize = 10;
+
+fn fill_buffer_random(
+ buffer: &ProtocolObject<dyn MTLBuffer>,
+ byte_count: usize,
+) {
+ let pointer = buffer.contents().as_ptr() as *mut u8;
+ let slice = unsafe { std::slice::from_raw_parts_mut(pointer, byte_count) };
+ for (i, byte) in slice.iter_mut().enumerate() {
+ *byte = (i % 251) as u8;
+ }
+}
+
+pub fn make_arguments<'a>(
+ a_buffer: &'a Buf,
+ b_buffer: &'a Buf,
+ d_buffer: &'a mut Buf,
+ shape: &TestShape,
+) -> MatmulArguments<'a, Metal> {
+ MatmulArguments {
+ a: a_buffer,
+ a_offset: 0,
+ b: b_buffer,
+ d: d_buffer,
+ bias: None,
+ batch: shape.batch as i32,
+ input_dim: shape.input_dim as i32,
+ output_dim: shape.output_dim as i32,
+ lda: shape.input_dim as i32,
+ ldb: shape.input_dim as i32,
+ ldd: shape.output_dim as i32,
+ batch_count: 1,
+ transpose_b: true,
+ }
+}
+
+fn encode_and_run(
+ context: &Ctx,
+ kernel: &mut MatmulKernel<Metal>,
+ shape: &TestShape,
+ a_buffer: &Buf,
+ b_buffer: &Buf,
+ d_buffer: &mut Buf,
+ dispatch_descriptor: &MatmulDispatchDescriptor,
+) -> Result<f64, BenchError> {
+ let mut command_buffer = context.create_command_buffer().map_err(|_| BenchError::CommandBuffer)?.start_encoding();
+
+ let arguments = make_arguments(a_buffer, b_buffer, d_buffer, shape);
+
+ kernel
+ .encode_with_descriptor(context, arguments, dispatch_descriptor, &mut command_buffer)
+ .map_err(|e| BenchError::Encode(e.to_string()))?;
+
+ let completed =
+ command_buffer.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;
+
+ completed.gpu_execution_time().map(|duration| duration.as_secs_f64() * 1000.0).ok_or(BenchError::GpuTimestamps)
+}
+
+fn run_benchmark(
+ context: &Ctx,
+ data_type: DataType,
+ shape: &TestShape,
+ dispatch_descriptor: &MatmulDispatchDescriptor,
+) -> Result<f64, BenchError> {
+ let mut kernel = MatmulKernel::<Metal>::new(data_type).map_err(|e| BenchError::Kernel(e.to_string()))?;
+
+ let elem_size = data_type.size_in_bytes();
+ let a_byte_count = shape.batch * shape.input_dim * elem_size;
+ let b_byte_count = shape.output_dim * shape.input_dim * elem_size;
+ let d_byte_count = shape.batch * shape.output_dim * elem_size;
+
+ let (a_buffer, b_buffer, mut d_buffer) = match (
+ context.device.new_buffer(a_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
+ context.device.new_buffer(b_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
+ context.device.new_buffer(d_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
+ ) {
+ (Some(a), Some(b), Some(d)) => (a, b, d),
+ _ => return Err(BenchError::BufferAllocation),
+ };
+
+ fill_buffer_random(&a_buffer, a_byte_count);
+ fill_buffer_random(&b_buffer, b_byte_count);
+
+ for iteration in 0..WARMUP_ITERATIONS {
+ encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer, dispatch_descriptor).map_err(
+ |source| BenchError::Warmup {
+ iteration,
+ source: Box::new(source),
+ },
+ )?;
+ }
+
+ let mut gpu_time_total_ms = 0.0;
+ for iteration in 0..BENCHMARK_ITERATIONS {
+ let gpu_ms =
+ encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer, dispatch_descriptor)
+ .map_err(|source| BenchError::Benchmark {
+ iteration,
+ source: Box::new(source),
+ })?;
+ gpu_time_total_ms += gpu_ms;
+ }
+
+ Ok(gpu_time_total_ms / BENCHMARK_ITERATIONS as f64)
+}
+
+pub fn benchmark_single(
+ context: &Ctx,
+ data_type: DataType,
+ shape: &TestShape,
+ path_name: &str,
+ dispatch_descriptor: &MatmulDispatchDescriptor,
+) -> PerfResult {
+ match run_benchmark(context, data_type, shape, dispatch_descriptor) {
+ Ok(duration_ms) => {
+ let flops = 2.0 * shape.batch as f64 * shape.input_dim as f64 * shape.output_dim as f64;
+ let gflops = flops / (duration_ms / 1000.0) / 1e9;
+ PerfResult {
+ combo: format!("{data_type:?}"),
+ shape: format!("{shape}"),
+ dispatch_path: path_name.to_owned(),
+ duration_ms,
+ gflops,
+ status: "ok".into(),
+ error: None,
+ }
+ },
+ Err(error) => PerfResult {
+ combo: format!("{data_type:?}"),
+ shape: format!("{shape}"),
+ dispatch_path: path_name.to_owned(),
+ duration_ms: 0.0,
+ gflops: 0.0,
+ status: "error".into(),
+ error: Some(error.to_string()),
+ },
+ }
+}
diff --git a/crates/uzu/tests/matmul_perf_test/error.rs b/crates/uzu/tests/matmul_perf_test/error.rs
new file mode 100644
index 000000000..3d11b6d69
--- /dev/null
+++ b/crates/uzu/tests/matmul_perf_test/error.rs
@@ -0,0 +1,25 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum BenchError {
+ #[error("failed to create command buffer")]
+ CommandBuffer,
+ #[error("encode failed: {0}")]
+ Encode(String),
+ #[error("GPU timestamps unavailable")]
+ GpuTimestamps,
+ #[error("kernel creation failed: {0}")]
+ Kernel(String),
+ #[error("buffer allocation failed")]
+ BufferAllocation,
+ #[error("warmup iteration {iteration}: {source}")]
+ Warmup {
+ iteration: usize,
+ source: Box<BenchError>,
+ },
+ #[error("benchmark iteration {iteration}: {source}")]
+ Benchmark {
+ iteration: usize,
+ source: Box<BenchError>,
+ },
+}
diff --git a/crates/uzu/tests/matmul_perf_test/main.rs b/crates/uzu/tests/matmul_perf_test/main.rs
new file mode 100644
index 000000000..d384c9ddd
--- /dev/null
+++ b/crates/uzu/tests/matmul_perf_test/main.rs
@@ -0,0 +1,115 @@
+#![cfg(all(target_os = "macos", feature = "metal"))]
+
+mod bench;
+mod error;
+mod output;
+mod shapes;
+
+use indicatif::{ProgressBar, ProgressStyle};
+use metal::{MTLDeviceExt, MTLResourceOptions};
+use output::print_results_table;
+use shapes::test_shapes;
+use uzu::{
+ DataType,
+ backends::{
+ common::{
+ Backend, Context,
+ kernel::matmul::{MatmulDispatchDescriptor, MatmulKernel, choose_matmul_dispatch_descriptor},
+ },
+ metal::Metal,
+ },
+};
+
+type Ctx = <Metal as Backend>::Context;
+
+fn write_json_results<T: serde::Serialize>(
+ test_name: &str,
+ device: &str,
+ results: &[T],
+) {
+ if let Ok(dir) = std::env::var("UZU_TEST_RESULTS_DIR") {
+ let path = std::path::Path::new(&dir);
+ std::fs::create_dir_all(path).expect("create results dir");
+ let file = path.join(format!("{test_name}.json"));
+ let wrapper = serde_json::json!({ "device": device, "results": results });
+ let json = serde_json::to_string_pretty(&wrapper).expect("serialize");
+ std::fs::write(&file, json).expect("write results");
+ eprintln!("Results written to {}", file.display());
+ }
+}
+
+#[test]
+#[ignore]
+fn matmul_perf() {
+ let context = Ctx::new().expect("Metal context required");
+
+ let data_types = [DataType::BF16, DataType::F16];
+ let test_shapes = test_shapes();
+
+ eprintln!("Matmul perf: {} dtypes x {} shapes", data_types.len(), test_shapes.len(),);
+
+ let progress_bar = ProgressBar::new_spinner();
+ progress_bar.set_style(
+ ProgressStyle::with_template("{spinner} {pos} benchmarks [{elapsed_precise}] {msg}").expect("progress style"),
+ );
+
+ let mut results = Vec::new();
+
+ for &data_type in &data_types {
+ let elem_size = data_type.size_in_bytes();
+
+ for shape in &test_shapes {
+ let a_byte_count = shape.batch * shape.input_dim * elem_size;
+ let b_byte_count = shape.output_dim * shape.input_dim * elem_size;
+ let d_byte_count = shape.batch * shape.output_dim * elem_size;
+
+ let (a_buffer, b_buffer, mut d_buffer) = match (
+ context.device.new_buffer(a_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
+ context.device.new_buffer(b_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
+ context.device.new_buffer(d_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
+ ) {
+ (Some(a), Some(b), Some(d)) => (a, b, d),
+ _ => continue,
+ };
+
+ let mut arguments = bench::make_arguments(&a_buffer, &b_buffer, &mut d_buffer, shape);
+ MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);
+
+ let descriptor = match choose_matmul_dispatch_descriptor::<Metal>(&context, data_type, &arguments) {
+ Ok(d) => d,
+ Err(e) => {
+ results.push(output::PerfResult {
+ combo: format!("{data_type:?}"),
+ shape: format!("{shape}"),
+ dispatch_path: "auto".into(),
+ duration_ms: 0.0,
+ gflops: 0.0,
+ status: "error".into(),
+ error: Some(format!("dispatch: {e}")),
+ });
+ continue;
+ },
+ };
+
+ let path_name = match &descriptor {
+ MatmulDispatchDescriptor::Gemv(_) => "Gemv",
+ MatmulDispatchDescriptor::SplitK(_) => "SplitK",
+ MatmulDispatchDescriptor::Gemm(_) => "Gemm",
+ };
+
+ progress_bar.set_message(format!("{data_type:?} {shape} {path_name}"));
+ let result = bench::benchmark_single(&context, data_type, shape, path_name, &descriptor);
+ results.push(result);
+ progress_bar.inc(1);
+ }
+ }
+
+ progress_bar.finish_with_message("done");
+ print_results_table(&results);
+ write_json_results("matmul_perf", &context.device.name(), &results);
+
+ let error_results: Vec<_> = results.iter().filter(|r| r.status != "ok").collect();
+ if !error_results.is_empty() {
+ eprintln!("{} / {} cases had errors", error_results.len(), results.len());
+ }
+}
diff --git a/crates/uzu/tests/matmul_perf_test/output.rs b/crates/uzu/tests/matmul_perf_test/output.rs
new file mode 100644
index 000000000..cf2611518
--- /dev/null
+++ b/crates/uzu/tests/matmul_perf_test/output.rs
@@ -0,0 +1,40 @@
+use comfy_table::{CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL};
+use serde::Serialize;
+
+#[derive(Serialize)]
+pub struct PerfResult {
+ pub combo: String,
+ pub shape: String,
+ pub dispatch_path: String,
+ pub duration_ms: f64,
+ pub gflops: f64,
+ pub status: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub error: Option<String>,
+}
+
+pub fn print_results_table(results: &[PerfResult]) {
+ let mut table = Table::new();
+ table
+ .load_preset(UTF8_FULL)
+ .apply_modifier(UTF8_ROUND_CORNERS)
+ .set_content_arrangement(ContentArrangement::Dynamic)
+ .set_header(vec!["Dtype", "Shape (MxKxN)", "Dispatch", "GFLOPS", "ms/iter", "Status"]);
+
+ for r in results {
+ let (gflops_str, ms_str, status_str) = if r.status == "ok" {
+ (format!("{:.1}", r.gflops), format!("{:.3}", r.duration_ms), "ok".into())
+ } else {
+ ("-".into(), "-".into(), format!("ERR: {}", r.error.as_deref().unwrap_or("?")))
+ };
+ table.add_row(vec![r.combo.clone(), r.shape.clone(), r.dispatch_path.clone(), gflops_str, ms_str, status_str]);
+ }
+
+ for col in [3, 4] {
+ if let Some(column) = table.column_mut(col) {
+ column.set_cell_alignment(CellAlignment::Right);
+ }
+ }
+
+ println!("{table}");
+}
diff --git a/crates/uzu/tests/matmul_perf_test/shapes.rs b/crates/uzu/tests/matmul_perf_test/shapes.rs
new file mode 100644
index 000000000..d26cfebea
--- /dev/null
+++ b/crates/uzu/tests/matmul_perf_test/shapes.rs
@@ -0,0 +1,41 @@
+pub struct TestShape {
+ pub batch: usize,
+ pub input_dim: usize,
+ pub output_dim: usize,
+}
+
+impl std::fmt::Display for TestShape {
+ fn fmt(
+ &self,
+ f: &mut std::fmt::Formatter<'_>,
+ ) -> std::fmt::Result {
+ write!(f, "{}x{}x{}", self.batch, self.input_dim, self.output_dim)
+ }
+}
+
+const BATCH_SIZES: &[usize] = &[1, 4, 16, 64, 128, 256, 512];
+
+const MODEL_DIMS: &[(usize, usize)] = &[
+ (896, 896),
+ (896, 4864),
+ (4864, 896),
+ (2048, 2048),
+ (2048, 8192),
+ (8192, 2048),
+ (4096, 4096),
+ (4096, 14336),
+ (14336, 4096),
+];
+
+pub fn test_shapes() -> Vec<TestShape> {
+ MODEL_DIMS
+ .iter()
+ .flat_map(|&(input_dim, output_dim)| {
+ BATCH_SIZES.iter().map(move |&batch| TestShape {
+ batch,
+ input_dim,
+ output_dim,
+ })
+ })
+ .collect()
+}