Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

update matmul tests

Eugene Bokhanb3314e3ece4e
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 397b05072..ed9a04c94 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -1,14 +1,8 @@
-//! Correctness tests comparing Metal matmul kernels against MPSGraph
-
-use std::collections::HashMap;
+//! Correctness tests comparing Metal matmul kernels against ndarray
use half::bf16;
use metal::{Device, MTLResourceOptions};
-use mpsgraph::{
- CommandBuffer, Device as MPSDevice, ExecutableExecutionDescriptor, Graph,
- ShapedType, TensorData,
-};
-use objc2::rc::autoreleasepool;
+use ndarray::Array2;
use uzu::{
DataType,
backends::metal::{
@@ -90,8 +84,7 @@ fn run_metal_matmul(
}
}
-fn run_mps_matmul(
- device: &Device,
+fn run_ndarray_matmul(
a_data: &[bf16],
b_data: &[bf16],
m: usize,
@@ -99,116 +92,20 @@ fn run_mps_matmul(
n: usize,
transpose_b: bool,
) -> Vec<bf16> {
- autoreleasepool(|_| {
- let graph = Graph::new();
-
- let a_shape = [m as isize, k as isize];
- let b_shape_for_placeholder = if transpose_b {
- [n as isize, k as isize]
- } else {
- [k as isize, n as isize]
- };
-
- let a_placeholder = graph.placeholder(
- Some(&a_shape),
- mpsgraph::DataType::BFloat16,
- Some("A"),
- );
- let b_placeholder = graph.placeholder(
- Some(&b_shape_for_placeholder),
- mpsgraph::DataType::BFloat16,
- Some("B"),
- );
-
- let result = if transpose_b {
- let b_transposed =
- graph.transpose(&b_placeholder, &[1, 0], Some("B_T"));
- graph.matrix_multiplication(&a_placeholder, &b_transposed, None)
- } else {
- graph.matrix_multiplication(&a_placeholder, &b_placeholder, None)
- };
-
- let a_shaped_type = ShapedType::new_with_shape_data_type(
- Some(&a_shape),
- mpsgraph::DataType::BFloat16,
- );
- let b_shaped_type = ShapedType::new_with_shape_data_type(
- Some(&b_shape_for_placeholder),
- mpsgraph::DataType::BFloat16,
- );
-
- let feeds = HashMap::from([
- (&*a_placeholder, &*a_shaped_type),
- (&*b_placeholder, &*b_shaped_type),
- ]);
-
- let mps_device = MPSDevice::with_device(device);
- let executable =
- graph.compile(&mps_device, &feeds, &[&result], None, None);
-
- let a_buf = device.new_buffer_with_data(
- a_data.as_ptr() as *const _,
- (a_data.len() * core::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let b_buf = device.new_buffer_with_data(
- b_data.as_ptr() as *const _,
- (b_data.len() * core::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let d_buf = device.new_buffer(
- (m * n * core::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
-
- let b_usize_shape: Vec<usize> = if transpose_b {
- vec![n, k]
- } else {
- vec![k, n]
- };
-
- let a_tensor = TensorData::new_with_mtl_buffer(
- &a_buf,
- &[m, k],
- mpsgraph::DataType::BFloat16,
- None,
- );
- let b_tensor = TensorData::new_with_mtl_buffer(
- &b_buf,
- &b_usize_shape,
- mpsgraph::DataType::BFloat16,
- None,
- );
- let d_tensor = TensorData::new_with_mtl_buffer(
- &d_buf,
- &[m, n],
- mpsgraph::DataType::BFloat16,
- None,
- );
+ let a_f32: Vec<f32> = a_data.iter().map(|x| x.to_f32()).collect();
+ let b_f32: Vec<f32> = b_data.iter().map(|x| x.to_f32()).collect();
- let inputs: &[&TensorData] = &[&a_tensor, &b_tensor];
- let outputs: &[&TensorData] = &[&d_tensor];
+ let a_arr = Array2::from_shape_vec((m, k), a_f32).expect("A shape");
- let command_queue = device.new_command_queue();
- let exec_desc = ExecutableExecutionDescriptor::new();
- exec_desc.set_enable_commit_and_continue(true);
-
- let command_buffer = CommandBuffer::from_command_queue(&command_queue);
- let root_cb = command_buffer.root_command_buffer().to_owned();
- let _ = executable.encode_to_command_buffer(
- &command_buffer,
- inputs,
- Some(outputs),
- Some(&exec_desc),
- );
- root_cb.commit();
- root_cb.wait_until_completed();
+ let result = if transpose_b {
+ let b_arr = Array2::from_shape_vec((n, k), b_f32).expect("B shape");
+ a_arr.dot(&b_arr.t())
+ } else {
+ let b_arr = Array2::from_shape_vec((k, n), b_f32).expect("B shape");
+ a_arr.dot(&b_arr)
+ };
- unsafe {
- let ptr = d_buf.contents() as *const bf16;
- std::slice::from_raw_parts(ptr, m * n).to_vec()
- }
- })
+ result.iter().map(|&x| bf16::from_f32(x)).collect()
}
fn generate_test_data(
@@ -231,29 +128,280 @@ fn generate_test_data(
(a, b)
}
+struct TestCase {
+ name: &'static str,
+ m: usize,
+ k: usize,
+ n: usize,
+ transpose_b: bool,
+ tolerance: f32,
+}
+
+fn test_cases() -> Vec<TestCase> {
+ vec![
+ // GEMV shapes (m=1)
+ TestCase {
+ name: "gemv_small",
+ m: 1,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.01,
+ },
+ TestCase {
+ name: "gemv_medium",
+ m: 1,
+ k: 4096,
+ n: 4096,
+ transpose_b: true,
+ tolerance: 0.01,
+ },
+ TestCase {
+ name: "gemv_large",
+ m: 1,
+ k: 8192,
+ n: 8192,
+ transpose_b: true,
+ tolerance: 0.02,
+ },
+ // MLP shapes
+ TestCase {
+ name: "mlp_up",
+ m: 1,
+ k: 2048,
+ n: 8192,
+ transpose_b: true,
+ tolerance: 0.01,
+ },
+ TestCase {
+ name: "mlp_down",
+ m: 1,
+ k: 8192,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.02,
+ },
+ // Small batch sizes
+ TestCase {
+ name: "batch_8",
+ m: 8,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.02,
+ },
+ TestCase {
+ name: "batch_16",
+ m: 16,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.03,
+ },
+ TestCase {
+ name: "batch_32",
+ m: 32,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.05,
+ },
+ TestCase {
+ name: "batch_64",
+ m: 64,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.08,
+ },
+ TestCase {
+ name: "batch_128",
+ m: 128,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ // Prefill shapes
+ TestCase {
+ name: "prefill_256",
+ m: 256,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.15,
+ },
+ TestCase {
+ name: "prefill_512",
+ m: 512,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.2,
+ },
+ TestCase {
+ name: "prefill_1024",
+ m: 1024,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.25,
+ },
+ // Square matrices
+ TestCase {
+ name: "square_128",
+ m: 128,
+ k: 128,
+ n: 128,
+ transpose_b: true,
+ tolerance: 0.05,
+ },
+ TestCase {
+ name: "square_256",
+ m: 256,
+ k: 256,
+ n: 256,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ TestCase {
+ name: "square_512",
+ m: 512,
+ k: 512,
+ n: 512,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ TestCase {
+ name: "square_1024",
+ m: 1024,
+ k: 1024,
+ n: 1024,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ // Attention shapes
+ TestCase {
+ name: "attention_qk",
+ m: 512,
+ k: 64,
+ n: 512,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ TestCase {
+ name: "attention_av",
+ m: 512,
+ k: 512,
+ n: 64,
+ transpose_b: false,
+ tolerance: 0.1,
+ },
+ // Non-transposed B
+ TestCase {
+ name: "non_transposed_small",
+ m: 64,
+ k: 128,
+ n: 64,
+ transpose_b: false,
+ tolerance: 0.15,
+ },
+ TestCase {
+ name: "non_transposed_medium",
+ m: 128,
+ k: 256,
+ n: 128,
+ transpose_b: false,
+ tolerance: 0.15,
+ },
+ // Non-aligned sizes
+ TestCase {
+ name: "unaligned_m",
+ m: 17,
+ k: 2048,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.05,
+ },
+ TestCase {
+ name: "unaligned_n",
+ m: 32,
+ k: 2048,
+ n: 1537,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ TestCase {
+ name: "unaligned_k",
+ m: 32,
+ k: 1999,
+ n: 2048,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ TestCase {
+ name: "unaligned_all",
+ m: 33,
+ k: 127,
+ n: 65,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ // Edge cases
+ TestCase {
+ name: "thin_m",
+ m: 2,
+ k: 4096,
+ n: 4096,
+ transpose_b: true,
+ tolerance: 0.02,
+ },
+ TestCase {
+ name: "thin_n",
+ m: 128,
+ k: 2048,
+ n: 1,
+ transpose_b: true,
+ tolerance: 0.02,
+ },
+ TestCase {
+ name: "small_k",
+ m: 64,
+ k: 32,
+ n: 64,
+ transpose_b: true,
+ tolerance: 0.05,
+ },
+ // Large K (tests SplitK path)
+ TestCase {
+ name: "large_k_small_mn",
+ m: 4,
+ k: 16384,
+ n: 4,
+ transpose_b: true,
+ tolerance: 0.1,
+ },
+ ]
+}
+
fn compare_results(
metal: &[bf16],
- mps: &[bf16],
+ reference: &[bf16],
tolerance: f32,
- test_name: &str,
-) {
- assert_eq!(
- metal.len(),
- mps.len(),
- "{}: length mismatch {} vs {}",
- test_name,
- metal.len(),
- mps.len()
- );
+) -> Result<(), (f32, usize, usize)> {
+ if metal.len() != reference.len() {
+ return Err((f32::INFINITY, 0, metal.len()));
+ }
let mut max_diff: f32 = 0.0;
let mut max_diff_idx = 0;
let mut diff_count = 0;
- for (i, (&m_val, &p_val)) in metal.iter().zip(mps.iter()).enumerate() {
+ for (i, (&m_val, &r_val)) in metal.iter().zip(reference.iter()).enumerate()
+ {
let mf = m_val.to_f32();
- let pf = p_val.to_f32();
- let diff = (mf - pf).abs();
+ let rf = r_val.to_f32();
+ let diff = (mf - rf).abs();
if diff > max_diff {
max_diff = diff;
max_diff_idx = i;
@@ -263,237 +411,89 @@ fn compare_results(
}
}
- assert!(
- max_diff <= tolerance,
- "{}: max diff {} at idx {} exceeds tolerance {} ({} elements differ)",
- test_name,
- max_diff,
- max_diff_idx,
- tolerance,
- diff_count
- );
-}
-
-#[test]
-fn matmul_correctness_gemv_small() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (1, 2048, 2048);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.01, "GEMV small");
-}
-
-#[test]
-fn matmul_correctness_gemv_medium() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (1, 4096, 4096);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.01, "GEMV medium");
-}
-
-#[test]
-fn matmul_correctness_gemv_large() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (1, 8192, 8192);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.02, "GEMV large");
-}
-
-#[test]
-fn matmul_correctness_small_batch() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (8, 2048, 2048);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.02, "Small batch");
-}
-
-#[test]
-fn matmul_correctness_medium_batch() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (32, 2048, 2048);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.05, "Medium batch");
-}
-
-#[test]
-fn matmul_correctness_large_batch() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (128, 2048, 2048);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.1, "Large batch");
-}
-
-#[test]
-fn matmul_correctness_prefill_small() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (256, 2048, 2048);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.15, "Prefill small");
-}
-
-#[test]
-fn matmul_correctness_prefill_medium() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (512, 2048, 2048);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.2, "Prefill medium");
-}
-
-#[test]
-fn matmul_correctness_mlp_up() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (1, 2048, 8192);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.01, "MLP up");
-}
-
-#[test]
-fn matmul_correctness_mlp_down() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (1, 8192, 2048);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.02, "MLP down");
-}
-
-#[test]
-fn matmul_correctness_square_small() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (256, 256, 256);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.1, "Square small");
-}
-
-#[test]
-fn matmul_correctness_square_medium() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (512, 512, 512);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.1, "Square medium");
-}
-
-#[test]
-fn matmul_correctness_square_large() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (1024, 1024, 1024);
- let (a, b) = generate_test_data(m, k, n, true);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.1, "Square large");
-}
-
-#[test]
-fn matmul_correctness_non_transposed() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (64, 128, 64);
- let (a, b) = generate_test_data(m, k, n, false);
-
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, false);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, false);
-
- compare_results(&metal_result, &mps_result, 0.15, "Non-transposed");
+ if max_diff <= tolerance {
+ Ok(())
+ } else {
+ Err((max_diff, max_diff_idx, diff_count))
+ }
}
#[test]
-fn matmul_correctness_attention_qk() {
+fn matmul_correctness_comprehensive() {
let Some(ctx) = create_test_context() else {
+ eprintln!("No Metal device available, skipping test");
return;
};
- let (m, k, n) = (512, 64, 512);
- let (a, b) = generate_test_data(m, k, n, true);
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, true);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, true);
-
- compare_results(&metal_result, &mps_result, 0.1, "Attention Q*K^T");
-}
-
-#[test]
-fn matmul_correctness_attention_av() {
- let Some(ctx) = create_test_context() else {
- return;
- };
- let (m, k, n) = (512, 512, 64);
- let (a, b) = generate_test_data(m, k, n, false);
+ let cases = test_cases();
+ let mut passed = 0;
+ let mut failed = Vec::new();
+
+ for case in &cases {
+ let (a, b) =
+ generate_test_data(case.m, case.k, case.n, case.transpose_b);
+ let metal_result = run_metal_matmul(
+ &ctx,
+ &a,
+ &b,
+ case.m,
+ case.k,
+ case.n,
+ case.transpose_b,
+ );
+ let reference = run_ndarray_matmul(
+ &a,
+ &b,
+ case.m,
+ case.k,
+ case.n,
+ case.transpose_b,
+ );
- let metal_result = run_metal_matmul(&ctx, &a, &b, m, k, n, false);
- let mps_result = run_mps_matmul(&ctx.device, &a, &b, m, k, n, false);
+ match compare_results(&metal_result, &reference, case.tolerance) {
+ Ok(()) => {
+ passed += 1;
+ eprintln!(
+ "✓ {} (m={}, k={}, n={}, transpose_b={})",
+ case.name, case.m, case.k, case.n, case.transpose_b
+ );
+ },
+ Err((max_diff, idx, count)) => {
+ failed.push((case, max_diff, idx, count));
+ eprintln!(
+ "✗ {} (m={}, k={}, n={}, transpose_b={}) - max_diff={:.6} at idx {} ({} elements exceed tolerance {})",
+ case.name,
+ case.m,
+ case.k,
+ case.n,
+ case.transpose_b,
+ max_diff,
+ idx,
+ count,
+ case.tolerance
+ );
+ },
+ }
+ }
- compare_results(&metal_result, &mps_result, 0.1, "Attention A*V");
+ eprintln!("\n{}/{} tests passed", passed, cases.len());
+
+ if !failed.is_empty() {
+ eprintln!("\nFailed tests:");
+ for (case, max_diff, idx, count) in &failed {
+ eprintln!(
+ " {} ({}x{}x{}, transpose_b={}): max_diff={:.6} at idx {}, {} elements exceed tolerance {}",
+ case.name,
+ case.m,
+ case.k,
+ case.n,
+ case.transpose_b,
+ max_diff,
+ idx,
+ count,
+ case.tolerance
+ );
+ }
+ panic!("{} tests failed", failed.len());
+ }
}