A high-performance inference engine for AI models
fix matmul perf
eugenefff0a55290d0
diff --git a/Cargo.lock b/Cargo.lockindex 5b411e5c5..e53cf9531 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4445,10 +4445,12 @@ dependencies = ["cbindgen","clang-ast","clap",+ "comfy-table","depfile","dsl","futures","half",+ "indicatif 0.18.3","is_close","itertools 0.14.0","libc",diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 33d592042..2e581cd82 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -49,6 +49,8 @@ is_close.workspace = trueserde_json5.workspace = trueschemars.workspace = truempsgraph.workspace = true+comfy-table.workspace = true+indicatif.workspace = true[build-dependencies]anyhow.workspace = true@@ -94,3 +96,7 @@ path = "tests/kernel/mod.rs"[[test]]name = "moe"path = "tests/moe/mod.rs"++[[test]]+name = "matmul_perf_test"+path = "tests/matmul_perf_test/main.rs"diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.hindex 39a4fc783..2c6cf9213 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h@@ -204,7 +204,7 @@ struct GEMMKernel {// Loop tailif (!K_aligned) {- int lbk = params->input_dim - params->gemm_k_iterations_aligned * BK;+ int lbk = params->K - params->gemm_k_iterations_aligned * BK;short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);@@ -223,10 +223,9 @@ struct GEMMKernel {///////////////////////////////////////////////////////////////////////////// MN unaligned loopelse { // Loop over K - unaligned case- short tgp_bm = min(BM, params->batch - c_row);- short tgp_bn = min(BN, params->output_dim - c_col);- short leftover_bk =- params->input_dim - params->gemm_k_iterations_aligned * BK;+ short tgp_bm = min(BM, params->M - c_row);+ short tgp_bn = min(BN, params->N - c_col);+ short leftover_bk = params->K - params->gemm_k_iterations_aligned * BK;if (tgp_bm == BM && tgp_bn == BN) {gemm_loop<true, true, K_aligned>(diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.hnew file mode 100644index 000000000..7386d512f--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h@@ -0,0 +1,9 @@+#pragma once++#include "steel/gemm/params.h"++namespace uzu {+namespace matmul {+using GEMMParams = steel::GEMMParams;+} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metalindex bbbe12039..018c284a4 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal@@ -2,188 +2,46 @@#include "../../../common/utils.h"#include "../../../definitions.metal"-#include "../../common/steel/gemm/gemm.h"+#include "../../common/gemm.h"-using namespace steel;+using namespace uzu::matmul;-// Upper bounds for threadgroup memory (in elements of T).-// Max across all tile/type combos: 64x32x32 with half (padding=8).-// tgp_mem_size_a = BM * (BK + padding) = 64 * (32 + 8) = 2560-// tgp_mem_size_b = BN * (BK + padding) = 32 * (32 + 8) = 1280-// For 64x64x16 with half: a=64*(16+8)=1536, b=64*(16+8)=1536-// Max a = 2560, max b = 1536-#define GEMM_MAX_TGP_A 2560-#define GEMM_MAX_TGP_B 1536--namespace uzu {-namespace matmul {-using GEMMParams = steel::GEMMParams;-} // namespace matmul-} // namespace uzu--///////////////////////////////////////////////////////////////////////////////-// GEMM implementation-///////////////////////////////////////////////////////////////////////////////--template <typename T, typename AccumType = float>-METAL_FUNC void gemm_impl(+template <typename T, int BM, int BN, int BK, int WM, int WN, bool MN_aligned, bool K_aligned>+METAL_FUNC void gemm_dispatch(const device T* a,const device T* b,device T* d,const constant GEMMParams* params,- const int BM,- const int BN,- const int BK,- const int WM,- const int WN,- const bool align_m,- const bool align_n,- const bool align_k,- threadgroup T* a_shared,- threadgroup T* b_shared,+ threadgroup T* As,+ threadgroup T* Bs,uint simd_lane_id,uint simd_group_id,uint3 tid,uint3 lid) {- (void)lid;-- // Hardcoded: transpose_a = false, transpose_b = true- const bool transpose_a = false;- const bool transpose_b = true;-- const short tgp_padding = 16 / sizeof(T);- const short lda_tgp = BK + tgp_padding; // transpose_a=false: BK + pad- const short ldb_tgp = BK + tgp_padding; // transpose_b=true: BK + pad- const short tgp_size = WM * WN * 32;-- // Find block- const int tid_y = ((tid.y) << params->swizzle_log) +- ((tid.x) & ((1 << params->swizzle_log) - 1));- const int tid_x = (tid.x) >> params->swizzle_log;-- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {- return;- }-- // Batch offset (non-batched path only)- a += params->batch_stride_a * tid.z;- b += params->batch_stride_b * tid.z;- d += params->batch_stride_d * tid.z;-- threadgroup_barrier(mem_flags::mem_none);-- // Find block in a, b, d- const int c_row = tid_y * BM;- const int c_col = tid_x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- // transpose_a=false: a += c_row * lda- a += c_row_long * params->lda;- // transpose_b=true: b += c_col * ldb- b += c_col_long * params->ldb;- d += c_row_long * params->ldd + c_col_long;-- // Construct loader and MMA objects with runtime tile params- // transpose_a=false: loader_a BROWS=BM, BCOLS=BK, dst_ld=BK+pad, reduction_dim=1- thread BlockLoader<T> loader_a(- a, params->lda, a_shared,- simd_group_id, simd_lane_id,- BM, BK, lda_tgp, 1, tgp_size);-- // transpose_b=true: loader_b BROWS=BN, BCOLS=BK, dst_ld=BK+pad, reduction_dim=1- thread BlockLoader<T> loader_b(- b, params->ldb, b_shared,- simd_group_id, simd_lane_id,- BN, BK, ldb_tgp, 1, tgp_size);-- thread BlockMMA<T, T, AccumType> mma_op(- simd_group_id, simd_lane_id,- BM, BN, BK, WM, WN,- transpose_a, transpose_b,- lda_tgp, ldb_tgp);-- const short tgp_bm = align_m ? BM : short(min(BM, params->M - c_row));- const short tgp_bn = align_n ? BN : short(min(BN, params->N - c_col));-- int gemm_k_iterations = params->gemm_k_iterations_aligned;-- // Do unaligned K iterations first- if (!align_k) {- const int k_last = params->gemm_k_iterations_aligned * BK;- const int k_remain = params->K - k_last;- // transpose_a=false: k_jump_a = k_last- const size_t k_jump_a = size_t(k_last);- // transpose_b=true: k_jump_b = k_last- const size_t k_jump_b = size_t(k_last);-- loader_a.src += k_jump_a;- loader_b.src += k_jump_b;-- // transpose_a=false: tile_dims_a = (k_remain, tgp_bm)- const short2 tile_dims_a = short2(k_remain, tgp_bm);- // transpose_b=true: tile_dims_b = (k_remain, tgp_bn)- const short2 tile_dims_b = short2(k_remain, tgp_bn);-- loader_a.load_safe(tile_dims_a);- loader_b.load_safe(tile_dims_b);-- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(a_shared, b_shared);-- loader_a.src -= k_jump_a;- loader_b.src -= k_jump_b;- }-- // MNK aligned loop- if (align_m && align_n) {- for (int k = 0; k < gemm_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_a.load_unsafe();- loader_b.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(a_shared, b_shared);- loader_a.next();- loader_b.next();- }-- threadgroup_barrier(mem_flags::mem_none);- return mma_op.store_result(d, params->ldd);- } else {- const short leftover_bk = 0;+ using gemm_kernel = GEMMKernel<T, T, BM, BN, BK, WM, WN, false, true, MN_aligned, K_aligned, float>;+ gemm_kernel::run(a, b, d, params, As, Bs, simd_lane_id, simd_group_id, tid, lid);+}- if ((align_m || tgp_bm == BM) && (align_n || tgp_bn == BN)) {- gemm_loop<T, T, AccumType>(- a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,- mma_op, tgp_bm, tgp_bn, leftover_bk,- BK, transpose_a, transpose_b, true, true, true);- return mma_op.store_result(d, params->ldd);- } else if (align_n || tgp_bn == BN) {- gemm_loop<T, T, AccumType>(- a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,- mma_op, tgp_bm, tgp_bn, leftover_bk,- BK, transpose_a, transpose_b, false, true, true);- return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));- } else if (align_m || tgp_bm == BM) {- gemm_loop<T, T, AccumType>(- a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,- mma_op, tgp_bm, tgp_bn, leftover_bk,- BK, transpose_a, transpose_b, true, false, true);- return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));- } else {- gemm_loop<T, T, AccumType>(- a_shared, b_shared, gemm_k_iterations, loader_a, loader_b,- mma_op, tgp_bm, tgp_bn, leftover_bk,- BK, transpose_a, transpose_b, false, false, true);- return mma_op.store_result_safe(d, params->ldd, short2(tgp_bn, tgp_bm));- }+#define GEMM_DISPATCH(T, BM, BN, BK, WM, WN) \+ if (block_rows == BM && block_cols == BN && block_depth == BK && \+ warps_per_row == WM && warps_per_col == WN) { \+ if (align_m && align_n) { \+ if (align_k) \+ gemm_dispatch<T, BM, BN, BK, WM, WN, true, true>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint3(group_x, group_y, group_z), uint3(thread_x, thread_y, thread_z)); \+ else \+ gemm_dispatch<T, BM, BN, BK, WM, WN, true, false>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint3(group_x, group_y, group_z), uint3(thread_x, thread_y, thread_z)); \+ } else { \+ if (align_k) \+ gemm_dispatch<T, BM, BN, BK, WM, WN, false, true>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint3(group_x, group_y, group_z), uint3(thread_x, thread_y, thread_z)); \+ else \+ gemm_dispatch<T, BM, BN, BK, WM, WN, false, false>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint3(group_x, group_y, group_z), uint3(thread_x, thread_y, thread_z)); \+ } \+ return; \}-}-///////////////////////////////////////////////////////////////////////////////-// Unified DSL kernel-///////////////////////////////////////////////////////////////////////////////+#define GEMM_MAX_TGP_A 2560+#define GEMM_MAX_TGP_B 1536template <typename T>VARIANTS(T, float, half, bfloat)@@ -213,16 +71,13 @@ PUBLIC KERNEL(MatmulGemm)(const uint thread_z THREADS(2),const Simd simd) {- gemm_impl<T, float>(- a, b, d, params,- block_rows, block_cols, block_depth,- warps_per_row, warps_per_col,- align_m, align_n, align_k,- a_shared, b_shared,- simd.lane_idx, simd.group_idx,- uint3(group_x, group_y, group_z),- uint3(thread_x, thread_y, thread_z)- );+ GEMM_DISPATCH(T, 64, 64, 16, 2, 2)+ GEMM_DISPATCH(T, 64, 64, 16, 1, 2)+ GEMM_DISPATCH(T, 64, 32, 32, 2, 2)+ GEMM_DISPATCH(T, 32, 64, 16, 1, 2)+ GEMM_DISPATCH(T, 32, 64, 16, 2, 2)+ GEMM_DISPATCH(T, 32, 32, 16, 2, 2)+ GEMM_DISPATCH(T, 64, 32, 8, 4, 1)}// clang-format ondiff --git a/crates/uzu/tests/matmul_perf_test/bench.rs b/crates/uzu/tests/matmul_perf_test/bench.rsnew file mode 100644index 000000000..87adf905e--- /dev/null+++ b/crates/uzu/tests/matmul_perf_test/bench.rs@@ -0,0 +1,159 @@+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};+use objc2::{rc::Retained, runtime::ProtocolObject};+use uzu::{+ DataType,+ backends::{+ common::{+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending, Context,+ kernel::matmul::{MatmulArguments, MatmulDispatchDescriptor, MatmulKernel},+ },+ metal::Metal,+ },+};++use super::{error::BenchError, output::PerfResult, shapes::TestShape};++type Ctx = <Metal as Backend>::Context;+type Buf = Retained<ProtocolObject<dyn MTLBuffer>>;++const WARMUP_ITERATIONS: usize = 3;+const BENCHMARK_ITERATIONS: usize = 10;++fn fill_buffer_random(+ buffer: &ProtocolObject<dyn MTLBuffer>,+ byte_count: usize,+) {+ let pointer = buffer.contents().as_ptr() as *mut u8;+ let slice = unsafe { std::slice::from_raw_parts_mut(pointer, byte_count) };+ for (i, byte) in slice.iter_mut().enumerate() {+ *byte = (i % 251) as u8;+ }+}++pub fn make_arguments<'a>(+ a_buffer: &'a Buf,+ b_buffer: &'a Buf,+ d_buffer: &'a mut Buf,+ shape: &TestShape,+) -> MatmulArguments<'a, Metal> {+ MatmulArguments {+ a: a_buffer,+ a_offset: 0,+ b: b_buffer,+ d: d_buffer,+ bias: None,+ batch: shape.batch as i32,+ input_dim: shape.input_dim as i32,+ output_dim: shape.output_dim as i32,+ lda: shape.input_dim as i32,+ ldb: shape.input_dim as i32,+ ldd: shape.output_dim as i32,+ batch_count: 1,+ transpose_b: true,+ }+}++fn encode_and_run(+ context: &Ctx,+ kernel: &mut MatmulKernel<Metal>,+ shape: &TestShape,+ a_buffer: &Buf,+ b_buffer: &Buf,+ d_buffer: &mut Buf,+ dispatch_descriptor: &MatmulDispatchDescriptor,+) -> Result<f64, BenchError> {+ let mut command_buffer = context.create_command_buffer().map_err(|_| BenchError::CommandBuffer)?.start_encoding();++ let arguments = make_arguments(a_buffer, b_buffer, d_buffer, shape);++ kernel+ .encode_with_descriptor(context, arguments, dispatch_descriptor, &mut command_buffer)+ .map_err(|e| BenchError::Encode(e.to_string()))?;++ let completed =+ command_buffer.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;++ completed.gpu_execution_time().map(|duration| duration.as_secs_f64() * 1000.0).ok_or(BenchError::GpuTimestamps)+}++fn run_benchmark(+ context: &Ctx,+ data_type: DataType,+ shape: &TestShape,+ dispatch_descriptor: &MatmulDispatchDescriptor,+) -> Result<f64, BenchError> {+ let mut kernel = MatmulKernel::<Metal>::new(data_type).map_err(|e| BenchError::Kernel(e.to_string()))?;++ let elem_size = data_type.size_in_bytes();+ let a_byte_count = shape.batch * shape.input_dim * elem_size;+ let b_byte_count = shape.output_dim * shape.input_dim * elem_size;+ let d_byte_count = shape.batch * shape.output_dim * elem_size;++ let (a_buffer, b_buffer, mut d_buffer) = match (+ context.device.new_buffer(a_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),+ context.device.new_buffer(b_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),+ context.device.new_buffer(d_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),+ ) {+ (Some(a), Some(b), Some(d)) => (a, b, d),+ _ => return Err(BenchError::BufferAllocation),+ };++ fill_buffer_random(&a_buffer, a_byte_count);+ fill_buffer_random(&b_buffer, b_byte_count);++ for iteration in 0..WARMUP_ITERATIONS {+ encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer, dispatch_descriptor).map_err(+ |source| BenchError::Warmup {+ iteration,+ source: Box::new(source),+ },+ )?;+ }++ let mut gpu_time_total_ms = 0.0;+ for iteration in 0..BENCHMARK_ITERATIONS {+ let gpu_ms =+ encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer, dispatch_descriptor)+ .map_err(|source| BenchError::Benchmark {+ iteration,+ source: Box::new(source),+ })?;+ gpu_time_total_ms += gpu_ms;+ }++ Ok(gpu_time_total_ms / BENCHMARK_ITERATIONS as f64)+}++pub fn benchmark_single(+ context: &Ctx,+ data_type: DataType,+ shape: &TestShape,+ path_name: &str,+ dispatch_descriptor: &MatmulDispatchDescriptor,+) -> PerfResult {+ match run_benchmark(context, data_type, shape, dispatch_descriptor) {+ Ok(duration_ms) => {+ let flops = 2.0 * shape.batch as f64 * shape.input_dim as f64 * shape.output_dim as f64;+ let gflops = flops / (duration_ms / 1000.0) / 1e9;+ PerfResult {+ combo: format!("{data_type:?}"),+ shape: format!("{shape}"),+ dispatch_path: path_name.to_owned(),+ duration_ms,+ gflops,+ status: "ok".into(),+ error: None,+ }+ },+ Err(error) => PerfResult {+ combo: format!("{data_type:?}"),+ shape: format!("{shape}"),+ dispatch_path: path_name.to_owned(),+ duration_ms: 0.0,+ gflops: 0.0,+ status: "error".into(),+ error: Some(error.to_string()),+ },+ }+}diff --git a/crates/uzu/tests/matmul_perf_test/error.rs b/crates/uzu/tests/matmul_perf_test/error.rsnew file mode 100644index 000000000..3d11b6d69--- /dev/null+++ b/crates/uzu/tests/matmul_perf_test/error.rs@@ -0,0 +1,25 @@+use thiserror::Error;++#[derive(Debug, Error)]+pub enum BenchError {+ #[error("failed to create command buffer")]+ CommandBuffer,+ #[error("encode failed: {0}")]+ Encode(String),+ #[error("GPU timestamps unavailable")]+ GpuTimestamps,+ #[error("kernel creation failed: {0}")]+ Kernel(String),+ #[error("buffer allocation failed")]+ BufferAllocation,+ #[error("warmup iteration {iteration}: {source}")]+ Warmup {+ iteration: usize,+ source: Box<BenchError>,+ },+ #[error("benchmark iteration {iteration}: {source}")]+ Benchmark {+ iteration: usize,+ source: Box<BenchError>,+ },+}diff --git a/crates/uzu/tests/matmul_perf_test/main.rs b/crates/uzu/tests/matmul_perf_test/main.rsnew file mode 100644index 000000000..d384c9ddd--- /dev/null+++ b/crates/uzu/tests/matmul_perf_test/main.rs@@ -0,0 +1,115 @@+#![cfg(all(target_os = "macos", feature = "metal"))]++mod bench;+mod error;+mod output;+mod shapes;++use indicatif::{ProgressBar, ProgressStyle};+use metal::{MTLDeviceExt, MTLResourceOptions};+use output::print_results_table;+use shapes::test_shapes;+use uzu::{+ DataType,+ backends::{+ common::{+ Backend, Context,+ kernel::matmul::{MatmulDispatchDescriptor, MatmulKernel, choose_matmul_dispatch_descriptor},+ },+ metal::Metal,+ },+};++type Ctx = <Metal as Backend>::Context;++fn write_json_results<T: serde::Serialize>(+ test_name: &str,+ device: &str,+ results: &[T],+) {+ if let Ok(dir) = std::env::var("UZU_TEST_RESULTS_DIR") {+ let path = std::path::Path::new(&dir);+ std::fs::create_dir_all(path).expect("create results dir");+ let file = path.join(format!("{test_name}.json"));+ let wrapper = serde_json::json!({ "device": device, "results": results });+ let json = serde_json::to_string_pretty(&wrapper).expect("serialize");+ std::fs::write(&file, json).expect("write results");+ eprintln!("Results written to {}", file.display());+ }+}++#[test]+#[ignore]+fn matmul_perf() {+ let context = Ctx::new().expect("Metal context required");++ let data_types = [DataType::BF16, DataType::F16];+ let test_shapes = test_shapes();++ eprintln!("Matmul perf: {} dtypes x {} shapes", data_types.len(), test_shapes.len(),);++ let progress_bar = ProgressBar::new_spinner();+ progress_bar.set_style(+ ProgressStyle::with_template("{spinner} {pos} benchmarks [{elapsed_precise}] {msg}").expect("progress style"),+ );++ let mut results = Vec::new();++ for &data_type in &data_types {+ let elem_size = data_type.size_in_bytes();++ for shape in &test_shapes {+ let a_byte_count = shape.batch * shape.input_dim * elem_size;+ let b_byte_count = shape.output_dim * shape.input_dim * elem_size;+ let d_byte_count = shape.batch * shape.output_dim * elem_size;++ let (a_buffer, b_buffer, mut d_buffer) = match (+ context.device.new_buffer(a_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),+ context.device.new_buffer(b_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),+ context.device.new_buffer(d_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),+ ) {+ (Some(a), Some(b), Some(d)) => (a, b, d),+ _ => continue,+ };++ let mut arguments = bench::make_arguments(&a_buffer, &b_buffer, &mut d_buffer, shape);+ MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);++ let descriptor = match choose_matmul_dispatch_descriptor::<Metal>(&context, data_type, &arguments) {+ Ok(d) => d,+ Err(e) => {+ results.push(output::PerfResult {+ combo: format!("{data_type:?}"),+ shape: format!("{shape}"),+ dispatch_path: "auto".into(),+ duration_ms: 0.0,+ gflops: 0.0,+ status: "error".into(),+ error: Some(format!("dispatch: {e}")),+ });+ continue;+ },+ };++ let path_name = match &descriptor {+ MatmulDispatchDescriptor::Gemv(_) => "Gemv",+ MatmulDispatchDescriptor::SplitK(_) => "SplitK",+ MatmulDispatchDescriptor::Gemm(_) => "Gemm",+ };++ progress_bar.set_message(format!("{data_type:?} {shape} {path_name}"));+ let result = bench::benchmark_single(&context, data_type, shape, path_name, &descriptor);+ results.push(result);+ progress_bar.inc(1);+ }+ }++ progress_bar.finish_with_message("done");+ print_results_table(&results);+ write_json_results("matmul_perf", &context.device.name(), &results);++ let error_results: Vec<_> = results.iter().filter(|r| r.status != "ok").collect();+ if !error_results.is_empty() {+ eprintln!("{} / {} cases had errors", error_results.len(), results.len());+ }+}diff --git a/crates/uzu/tests/matmul_perf_test/output.rs b/crates/uzu/tests/matmul_perf_test/output.rsnew file mode 100644index 000000000..cf2611518--- /dev/null+++ b/crates/uzu/tests/matmul_perf_test/output.rs@@ -0,0 +1,40 @@+use comfy_table::{CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL};+use serde::Serialize;++#[derive(Serialize)]+pub struct PerfResult {+ pub combo: String,+ pub shape: String,+ pub dispatch_path: String,+ pub duration_ms: f64,+ pub gflops: f64,+ pub status: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error: Option<String>,+}++pub fn print_results_table(results: &[PerfResult]) {+ let mut table = Table::new();+ table+ .load_preset(UTF8_FULL)+ .apply_modifier(UTF8_ROUND_CORNERS)+ .set_content_arrangement(ContentArrangement::Dynamic)+ .set_header(vec!["Dtype", "Shape (MxKxN)", "Dispatch", "GFLOPS", "ms/iter", "Status"]);++ for r in results {+ let (gflops_str, ms_str, status_str) = if r.status == "ok" {+ (format!("{:.1}", r.gflops), format!("{:.3}", r.duration_ms), "ok".into())+ } else {+ ("-".into(), "-".into(), format!("ERR: {}", r.error.as_deref().unwrap_or("?")))+ };+ table.add_row(vec![r.combo.clone(), r.shape.clone(), r.dispatch_path.clone(), gflops_str, ms_str, status_str]);+ }++ for col in [3, 4] {+ if let Some(column) = table.column_mut(col) {+ column.set_cell_alignment(CellAlignment::Right);+ }+ }++ println!("{table}");+}diff --git a/crates/uzu/tests/matmul_perf_test/shapes.rs b/crates/uzu/tests/matmul_perf_test/shapes.rsnew file mode 100644index 000000000..d26cfebea--- /dev/null+++ b/crates/uzu/tests/matmul_perf_test/shapes.rs@@ -0,0 +1,41 @@+pub struct TestShape {+ pub batch: usize,+ pub input_dim: usize,+ pub output_dim: usize,+}++impl std::fmt::Display for TestShape {+ fn fmt(+ &self,+ f: &mut std::fmt::Formatter<'_>,+ ) -> std::fmt::Result {+ write!(f, "{}x{}x{}", self.batch, self.input_dim, self.output_dim)+ }+}++const BATCH_SIZES: &[usize] = &[1, 4, 16, 64, 128, 256, 512];++const MODEL_DIMS: &[(usize, usize)] = &[+ (896, 896),+ (896, 4864),+ (4864, 896),+ (2048, 2048),+ (2048, 8192),+ (8192, 2048),+ (4096, 4096),+ (4096, 14336),+ (14336, 4096),+];++pub fn test_shapes() -> Vec<TestShape> {+ MODEL_DIMS+ .iter()+ .flat_map(|&(input_dim, output_dim)| {+ BATCH_SIZES.iter().map(move |&batch| TestShape {+ batch,+ input_dim,+ output_dim,+ })+ })+ .collect()+}