A high-performance inference engine for AI models
add matmul benchmark
Eugene Bokhan10c656a7c42e
diff --git a/Cargo.lock b/Cargo.lockindex c3df3983a..1be372c1f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2043,6 +2043,24 @@ dependencies = ["regex-automata 0.1.10",]+[[package]]+name = "matmul-benchmark"+version = "0.1.0"+dependencies = [+ "autocxx",+ "autocxx-build",+ "cmake",+ "cxx",+ "half",+ "metal",+ "mpsgraph",+ "objc2",+ "serde",+ "serde_json",+ "uzu",+ "walkdir",+]+[[package]]name = "matrixmultiply"version = "0.3.10"diff --git a/Cargo.toml b/Cargo.tomlindex 46dccc8f4..e6b72f308 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,5 +1,10 @@[workspace]-members = ["crates/cli", "crates/uzu", "crates/benchmarks"]+members = [+ "crates/cli",+ "crates/uzu",+ "crates/benchmarks",+ "crates/matmul-benchmark",+]resolver = "3"[workspace.package]diff --git a/crates/matmul-benchmark/Cargo.toml b/crates/matmul-benchmark/Cargo.tomlnew file mode 100644index 000000000..98d7dbd3c--- /dev/null+++ b/crates/matmul-benchmark/Cargo.toml@@ -0,0 +1,26 @@+[package]+name = "matmul-benchmark"+version = "0.1.0"+edition = "2024"+build = "build.rs"+links = "mlx"++[[bin]]+name = "matmul-benchmark"+path = "src/main.rs"++[dependencies]+autocxx = "0.30.0"+cxx = "1.0.187"+uzu = { path = "../uzu" }+metal = "0.32.0"+mpsgraph = "0.2.0"+half = "2.4"+objc2 = "0.6"+serde = { version = "1.0", features = ["derive"] }+serde_json = "1.0"++[build-dependencies]+walkdir = "2.5.0"+cmake = "0.1.57"+autocxx-build = "0.30.0"diff --git a/crates/matmul-benchmark/build.rs b/crates/matmul-benchmark/build.rsnew file mode 100644index 000000000..ff1e8c453--- /dev/null+++ b/crates/matmul-benchmark/build.rs@@ -0,0 +1,524 @@+use std::{+ env,+ fs::{self, create_dir_all},+ path::{Path, PathBuf},+ process::Command,+};++use cmake::Config as CMakeConfig;+use walkdir::WalkDir;++const DEFAULT_MLX_GIT_URL: &str = "https://github.com/ml-explore/mlx.git";+const DEFAULT_MLX_GIT_REF: &str = "v0.30.1";++fn abs_path<P: AsRef<Path>>(p: P) -> PathBuf {+ if p.as_ref().is_absolute() {+ p.as_ref().to_path_buf()+ } else {+ env::current_dir().expect("current_dir failed").join(p)+ }+}++fn looks_like_mlx_repo_root(dir: &Path) -> bool {+ dir.join("CMakeLists.txt").exists()+ && dir.join("mlx").exists()+ && dir.join("mlx/mlx.h").exists()+}++fn is_truthy_env(name: &str) -> bool {+ let Ok(v) = env::var(name) else {+ return false;+ };+ matches!(+ v.trim().to_ascii_lowercase().as_str(),+ "1" | "true" | "yes" | "on"+ )+}++fn cargo_offline() -> bool {+ is_truthy_env("CARGO_NET_OFFLINE") || is_truthy_env("MLX_RS_OFFLINE")+}++fn cache_dir(out_dir: &Path) -> PathBuf {+ if let Ok(p) = env::var("MLX_RS_CACHE_DIR") {+ return abs_path(p);+ }++ if let Ok(p) = env::var("CARGO_HOME") {+ return abs_path(p).join("mlx-rs-cache");+ }++ if let Ok(p) = env::var("HOME") {+ return PathBuf::from(p).join(".cache/mlx-rs");+ }+ if let Ok(p) = env::var("LOCALAPPDATA") {+ return PathBuf::from(p).join("mlx-rs");+ }++ out_dir.join("mlx-rs-cache")+}++fn run_checked(+ mut cmd: Command,+ what: &str,+) {+ let output = cmd.output().unwrap_or_else(|e| {+ panic!("Failed to run {}: {}", what, e);+ });+ if !output.status.success() {+ let stdout = String::from_utf8_lossy(&output.stdout);+ let stderr = String::from_utf8_lossy(&output.stderr);+ panic!(+ "{} failed (exit={:?})\n--- stdout ---\n{}\n--- stderr ---\n{}\n",+ what,+ output.status.code(),+ stdout,+ stderr+ );+ }+}++fn pins_toml_path(manifest_dir: &Path) -> PathBuf {+ if let Ok(p) = env::var("MLX_RS_PINS_TOML") {+ return abs_path(p);+ }+ manifest_dir.join("mlx-pins.toml")+}++#[derive(Debug, Clone)]+struct Pins {+ repo_url: Option<String>,+ repo_ref: Option<String>,+}++fn parse_pins(pins_path: &Path) -> Pins {+ let contents = fs::read_to_string(pins_path).unwrap_or_else(|e| {+ panic!(+ "Failed to read pins file at {}: {}. \+ Update the pins file or set MLX_RS_PINS_TOML to a valid path.",+ pins_path.display(),+ e+ )+ });++ #[derive(Debug)]+ enum Section {+ None,+ Repo,+ }++ let mut section = Section::None;+ let mut pins = Pins {+ repo_url: None,+ repo_ref: None,+ };++ for raw in contents.lines() {+ let line = raw.trim();+ if line.is_empty() || line.starts_with('#') {+ continue;+ }+ if line.starts_with('[') && line.ends_with(']') {+ let header = &line[1..line.len() - 1];+ if header == "repo" {+ section = Section::Repo;+ } else {+ section = Section::None;+ }+ continue;+ }++ let Some((k, v)) = line.split_once('=') else {+ continue;+ };+ let key = k.trim();+ let mut val = v.trim();+ if val.starts_with('"') && val.ends_with('"') && val.len() >= 2 {+ val = &val[1..val.len() - 1];+ }++ match &mut section {+ Section::Repo => match key {+ "url" => pins.repo_url = Some(val.to_string()),+ "ref" | "rev" => pins.repo_ref = Some(val.to_string()),+ _ => {},+ },+ Section::None => {},+ }+ }++ pins+}++fn pinned_mlx_git(pins: &Pins) -> (String, String) {+ let url = env::var("MLX_GIT_URL")+ .ok()+ .or_else(|| pins.repo_url.clone())+ .unwrap_or_else(|| DEFAULT_MLX_GIT_URL.to_string());+ let rev = env::var("MLX_GIT_REF")+ .ok()+ .or_else(|| pins.repo_ref.clone())+ .unwrap_or_else(|| DEFAULT_MLX_GIT_REF.to_string());+ (url, rev)+}++fn maybe_clear_cmake_build_dir(+ build_dir: &Path,+ source_dir: &Path,+) {+ let cache = build_dir.join("CMakeCache.txt");+ let Ok(contents) = fs::read_to_string(&cache) else {+ return;+ };+ let src =+ source_dir.canonicalize().unwrap_or_else(|_| source_dir.to_path_buf());+ for line in contents.lines() {+ if !line.starts_with("CMAKE_HOME_DIRECTORY") {+ continue;+ }+ let needs_cleanup = match line.split('=').next_back() {+ Some(cmake_home) => fs::canonicalize(cmake_home)+ .ok()+ .map(|cmake_home| cmake_home != src)+ .unwrap_or(true),+ None => true,+ };+ if needs_cleanup {+ let _ = fs::remove_dir_all(build_dir);+ }+ break;+ }+}++fn ensure_git_checkout_cached(+ name: &str,+ url: &str,+ rev: &str,+ cache_dir: &Path,+) -> PathBuf {+ let checkout_dir = cache_dir.join(format!("{}-{}", name, rev));+ let marker = checkout_dir.join(".mlx_rs_fetched");+ if marker.exists() {+ return checkout_dir;+ }++ if checkout_dir.exists() {+ let _ = fs::remove_dir_all(&checkout_dir);+ }+ create_dir_all(cache_dir).expect("Failed to create cache dir");++ run_checked(+ {+ let mut c = Command::new("git");+ c.arg("clone").arg(url).arg(&checkout_dir);+ c+ },+ &format!("git clone {} into cache", name),+ );+ run_checked(+ {+ let mut c = Command::new("git");+ c.arg("-C").arg(&checkout_dir).arg("checkout").arg(rev);+ c+ },+ &format!("git checkout {}@{}", name, rev),+ );++ let _ = fs::write(&marker, rev);+ checkout_dir+}++fn ensure_mlx_repo(+ out_dir: &Path,+ manifest_dir: &Path,+ repo_url: &str,+ repo_ref: &str,+) -> PathBuf {+ // First, check for explicit MLX_SRC_DIR environment variable+ if let Ok(p) = env::var("MLX_SRC_DIR") {+ let p = abs_path(p);+ if !looks_like_mlx_repo_root(&p) {+ panic!(+ "MLX_SRC_DIR={} does not look like an MLX repo root \+ (expected CMakeLists.txt + mlx/ + mlx/mlx.h)",+ p.display()+ );+ }+ return p;+ }++ // Second, check for local external/mlx in the workspace+ let workspace_root = manifest_dir.parent().and_then(|p| p.parent());+ if let Some(root) = workspace_root {+ let local_mlx = root.join("external/mlx");+ if looks_like_mlx_repo_root(&local_mlx) {+ println!(+ "cargo:warning=mlx-rs: using local MLX at {}",+ local_mlx.display()+ );+ return local_mlx;+ }+ }++ if cargo_offline() {+ panic!(+ "MLX sources not found locally and Cargo is offline. \+ Set MLX_SRC_DIR to a checked-out MLX repo or build with network access."+ );+ }++ let cache = cache_dir(out_dir);+ println!(+ "cargo:warning=mlx-rs: fetching MLX {}@{} into {}",+ repo_url,+ repo_ref,+ cache.display()+ );+ ensure_git_checkout_cached("mlx", repo_url, repo_ref, &cache)+}++fn find_mlx_lib_dir(root: &Path) -> Option<PathBuf> {+ let static_candidates = ["libmlx.a", "mlx.lib"];++ for entry in+ WalkDir::new(root).max_depth(6).into_iter().filter_map(Result::ok)+ {+ if !entry.file_type().is_file() {+ continue;+ }++ let name = entry.file_name().to_string_lossy();+ if static_candidates.iter().any(|c| name == *c) {+ return entry.path().parent().map(|p| p.to_path_buf());+ }+ }++ None+}++#[derive(Debug, Clone)]+struct BuildContext {+ #[allow(dead_code)]+ manifest_dir: PathBuf,+ mlx_src_dir: PathBuf,+ out_dir: PathBuf,+ src_include_dir: PathBuf,+ target: String,+}++fn collect_build_context() -> BuildContext {+ println!("cargo:rerun-if-env-changed=MLX_SRC_DIR");+ println!("cargo:rerun-if-env-changed=MLX_RS_PINS_TOML");+ println!("cargo:rerun-if-env-changed=MLX_GIT_URL");+ println!("cargo:rerun-if-env-changed=MLX_GIT_REF");+ println!("cargo:rerun-if-env-changed=MLX_RS_CACHE_DIR");+ println!("cargo:rerun-if-env-changed=MLX_RS_OFFLINE");+ println!("cargo:rerun-if-env-changed=CARGO_NET_OFFLINE");+ println!("cargo:rerun-if-env-changed=CARGO_HOME");++ let manifest_dir = abs_path(+ env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"),+ );+ let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set"));++ let pins_path = pins_toml_path(&manifest_dir);+ println!("cargo:rerun-if-changed={}", pins_path.display());+ let pins = parse_pins(&pins_path);++ let (repo_url, repo_ref) = pinned_mlx_git(&pins);+ let mlx_src_dir =+ ensure_mlx_repo(&out_dir, &manifest_dir, &repo_url, &repo_ref);++ println!("cargo:rerun-if-changed={}", mlx_src_dir.join("mlx").display());+ println!(+ "cargo:rerun-if-changed={}",+ mlx_src_dir.join("CMakeLists.txt").display()+ );++ let src_include_dir = manifest_dir.join("src");+ let target = env::var("TARGET").unwrap_or_default();++ BuildContext {+ manifest_dir,+ mlx_src_dir,+ out_dir,+ src_include_dir,+ target,+ }+}++fn build_mlx_cmake(ctx: &BuildContext) -> PathBuf {+ let cmake_build_dir = ctx.out_dir.join("build");+ maybe_clear_cmake_build_dir(&cmake_build_dir, &ctx.mlx_src_dir);+ create_dir_all(&cmake_build_dir).ok();++ let mut cmake_config = CMakeConfig::new(&ctx.mlx_src_dir);+ cmake_config.out_dir(&ctx.out_dir);++ // Disable features we don't need+ cmake_config.define("MLX_BUILD_TESTS", "OFF");+ cmake_config.define("MLX_BUILD_EXAMPLES", "OFF");+ cmake_config.define("MLX_BUILD_BENCHMARKS", "OFF");+ cmake_config.define("MLX_BUILD_PYTHON_BINDINGS", "OFF");++ // Enable Metal backend only (CPU backend has compatibility issues with some Xcode versions)+ cmake_config.define("MLX_BUILD_METAL", "ON");+ cmake_config.define("MLX_BUILD_CPU", "OFF");++ // Build static library+ cmake_config.define("BUILD_SHARED_LIBS", "OFF");++ // C++ settings+ cmake_config.define("CMAKE_CXX_STANDARD", "17");+ cmake_config.define("CMAKE_CXX_STANDARD_REQUIRED", "ON");+ cmake_config.define("CMAKE_CXX_EXTENSIONS", "OFF");++ // Disable LTO to avoid linking issues+ cmake_config.define("CMAKE_INTERPROCEDURAL_OPTIMIZATION", "OFF");++ let is_msvc = ctx.target.contains("msvc");+ if !is_msvc {+ cmake_config.cflag("-fno-lto");+ cmake_config.cxxflag("-fno-lto");+ }++ let build_profile =+ match env::var("PROFILE").unwrap_or_else(|_| "release".into()).as_str()+ {+ "debug" => "Debug",+ "release" => "Release",+ _ => "RelWithDebInfo",+ };+ cmake_config.profile(build_profile);++ // macOS specific settings+ if ctx.target.contains("apple-darwin") {+ let arch = if ctx.target.contains("aarch64") {+ "arm64"+ } else {+ "x86_64"+ };+ cmake_config.define("CMAKE_OSX_ARCHITECTURES", arch);+ // Force deployment target to avoid cmake crate confusing SDK version with deployment target+ let dep_target = env::var("MACOSX_DEPLOYMENT_TARGET")+ .unwrap_or_else(|_| "14.0".into());+ cmake_config.define("CMAKE_OSX_DEPLOYMENT_TARGET", &dep_target);+ // Also set the env var so cmake-rs doesn't override with bad values+ unsafe { env::set_var("MACOSX_DEPLOYMENT_TARGET", &dep_target) };+ }++ cmake_config.build_target("mlx").build()+}++fn link_mlx_static(+ ctx: &BuildContext,+ destination_path: &Path,+) {+ let cmake_build_dir = ctx.out_dir.join("build");+ let lib_search_dir = find_mlx_lib_dir(&cmake_build_dir)+ .or_else(|| find_mlx_lib_dir(destination_path))+ .unwrap_or_else(|| destination_path.join("lib"));+ println!("cargo:rustc-link-search=native={}", lib_search_dir.display());+ println!("cargo:rustc-link-lib=static=mlx");++ // Link Apple frameworks required by MLX+ println!("cargo:rustc-link-lib=framework=Metal");+ println!("cargo:rustc-link-lib=framework=Foundation");+ println!("cargo:rustc-link-lib=framework=QuartzCore");+ println!("cargo:rustc-link-lib=framework=Accelerate");++ // Link C++ standard library+ println!("cargo:rustc-link-lib=c++");++ // Set minimum macOS version for linking to match what MLX was built with+ if ctx.target.contains("apple") {+ let dep_target = env::var("MACOSX_DEPLOYMENT_TARGET")+ .unwrap_or_else(|_| "14.0".into());+ println!("cargo:rustc-link-arg=-mmacosx-version-min={}", dep_target);+ // Link clang runtime for __isPlatformVersionAtLeast and other builtins+ // Find clang resource directory for runtime libraries+ if let Ok(output) =+ Command::new("clang").args(["--print-resource-dir"]).output()+ {+ if output.status.success() {+ let resource_dir =+ String::from_utf8_lossy(&output.stdout).trim().to_string();+ let lib_dir = format!("{}/lib/darwin", resource_dir);+ println!("cargo:rustc-link-search=native={}", lib_dir);+ println!("cargo:rustc-link-lib=static=clang_rt.osx");+ }+ }+ }+}++fn build_autocxx_bridge(+ ctx: &BuildContext,+ metal_cpp_dir: &Path,+) {+ println!("cargo:rerun-if-changed=src/lib.rs");+ println!("cargo:rerun-if-changed=src/cxx_utils.hpp");+ println!("cargo:rerun-if-changed=src/cxx_utils/matmul.hpp");++ let extra_clang_args = vec!["-std=c++17".to_string()];+ let extra_clang_args_refs: Vec<&str> =+ extra_clang_args.iter().map(|s| s.as_str()).collect();++ let mut autocxx_builder = autocxx_build::Builder::new(+ "src/lib.rs",+ &[&ctx.src_include_dir, &ctx.mlx_src_dir, metal_cpp_dir],+ )+ .extra_clang_args(&extra_clang_args_refs)+ .build()+ .expect("autocxx build failed");++ autocxx_builder+ .flag_if_supported("-std=c++17")+ .flag_if_supported("-Wno-unused-parameter")+ .flag_if_supported("-Wno-deprecated-copy")+ .include(&ctx.src_include_dir)+ .include(&ctx.mlx_src_dir)+ .include(metal_cpp_dir);++ autocxx_builder.compile("mlx_rs_bridge");+}++fn find_metal_cpp_dir(out_dir: &Path) -> PathBuf {+ // metal-cpp is fetched by MLX's CMake into the build directory+ for entry in+ WalkDir::new(out_dir).max_depth(6).into_iter().filter_map(Result::ok)+ {+ let path = entry.path();+ if path.ends_with("Metal/Metal.hpp") {+ if let Some(parent) = path.parent().and_then(|p| p.parent()) {+ return parent.to_path_buf();+ }+ }+ }+ // Fallback+ out_dir.join("build/_deps/metal_cpp-src")+}++fn main() {+ // Set MACOSX_DEPLOYMENT_TARGET early to ensure all compilation uses consistent version+ // MLX requires macOS 14.0++ let target = env::var("TARGET").unwrap_or_default();+ if target.contains("apple") {+ let dep_target = env::var("MACOSX_DEPLOYMENT_TARGET")+ .unwrap_or_else(|_| "14.0".into());+ // Ensure minimum 14.0 for MLX compatibility+ let version: f32 = dep_target.parse().unwrap_or(14.0);+ let final_target = if version < 14.0 {+ "14.0"+ } else {+ &dep_target+ };+ unsafe { env::set_var("MACOSX_DEPLOYMENT_TARGET", final_target) };+ }++ let ctx = collect_build_context();+ let destination_path = build_mlx_cmake(&ctx);+ link_mlx_static(&ctx, &destination_path);++ let metal_cpp_dir = find_metal_cpp_dir(&ctx.out_dir);+ build_autocxx_bridge(&ctx, &metal_cpp_dir);+}diff --git a/crates/matmul-benchmark/mlx-pins.toml b/crates/matmul-benchmark/mlx-pins.tomlnew file mode 100644index 000000000..2b19c03f8--- /dev/null+++ b/crates/matmul-benchmark/mlx-pins.toml@@ -0,0 +1,6 @@+# Pins for MLX repo used by build.rs+# Similar to xgrammar-rs pattern++[repo]+url = "https://github.com/ml-explore/mlx.git"+ref = "v0.30.1"diff --git a/crates/matmul-benchmark/src/cxx_utils.hpp b/crates/matmul-benchmark/src/cxx_utils.hppnew file mode 100644index 000000000..01793910e--- /dev/null+++ b/crates/matmul-benchmark/src/cxx_utils.hpp@@ -0,0 +1,6 @@+#ifndef MLX_RS_CXX_UTILS_H_+#define MLX_RS_CXX_UTILS_H_++#include "cxx_utils/matmul.hpp"++#endif // MLX_RS_CXX_UTILS_H_diff --git a/crates/matmul-benchmark/src/cxx_utils/matmul.hpp b/crates/matmul-benchmark/src/cxx_utils/matmul.hppnew file mode 100644index 000000000..4a0cef9d0--- /dev/null+++ b/crates/matmul-benchmark/src/cxx_utils/matmul.hpp@@ -0,0 +1,200 @@+#ifndef MLX_RS_CXX_UTILS_MATMUL_H_+#define MLX_RS_CXX_UTILS_MATMUL_H_++#include <chrono>+#include <cstring>+#include <vector>+#include "mlx/mlx.h"++namespace cxx_utils {++namespace mx = mlx::core;++struct BenchmarkResult {+ double avg_time_ms;+ double gflops;+};++// Compute matmul and copy result to output buffer for accuracy testing+// Input: A[M,K] row-major, B[K,N] row-major (but we'll transpose B for consistency with UZU)+// UZU expects: A[M,K], B^T[N,K] (transpose_b=true means B is stored as [N,K])+// So we need to pass B as [N,K] to MLX too, then transpose it+inline void matmul_bf16_with_output(+ const uint16_t* a_data, // [M, K] row-major bf16 as raw bytes+ const uint16_t* b_data, // [N, K] row-major bf16 (transposed storage)+ uint16_t* output_data, // [M, N] output+ int M, int K, int N+) {+ // Create MLX arrays from raw data+ // A is [M, K], B is stored as [N, K] (transposed)+ mx::Shape a_shape = {M, K};+ mx::Shape b_shape = {N, K};++ auto a = mx::array(+ reinterpret_cast<const mx::bfloat16_t*>(a_data),+ a_shape,+ mx::bfloat16+ );++ auto b_transposed = mx::array(+ reinterpret_cast<const mx::bfloat16_t*>(b_data),+ b_shape,+ mx::bfloat16+ );++ // Transpose B back to [K, N] for standard matmul+ auto b = mx::transpose(b_transposed);+ mx::eval(a, b);++ // Compute matmul: [M, K] @ [K, N] -> [M, N]+ auto result = mx::matmul(a, b);+ mx::eval(result);++ // Copy result to output buffer+ const auto* result_ptr = result.data<mx::bfloat16_t>();+ std::memcpy(output_data, result_ptr, M * N * sizeof(uint16_t));+}++inline void matmul_f16_with_output(+ const uint16_t* a_data,+ const uint16_t* b_data,+ uint16_t* output_data,+ int M, int K, int N+) {+ mx::Shape a_shape = {M, K};+ mx::Shape b_shape = {N, K};++ auto a = mx::array(+ reinterpret_cast<const mx::float16_t*>(a_data),+ a_shape,+ mx::float16+ );++ auto b_transposed = mx::array(+ reinterpret_cast<const mx::float16_t*>(b_data),+ b_shape,+ mx::float16+ );++ auto b = mx::transpose(b_transposed);+ mx::eval(a, b);++ auto result = mx::matmul(a, b);+ mx::eval(result);++ const auto* result_ptr = result.data<mx::float16_t>();+ std::memcpy(output_data, result_ptr, M * N * sizeof(uint16_t));+}++inline void matmul_f32_with_output(+ const float* a_data,+ const float* b_data,+ float* output_data,+ int M, int K, int N+) {+ mx::Shape a_shape = {M, K};+ mx::Shape b_shape = {N, K};++ auto a = mx::array(a_data, a_shape, mx::float32);+ auto b_transposed = mx::array(b_data, b_shape, mx::float32);++ auto b = mx::transpose(b_transposed);+ mx::eval(a, b);++ auto result = mx::matmul(a, b);+ mx::eval(result);++ const auto* result_ptr = result.data<float>();+ std::memcpy(output_data, result_ptr, M * N * sizeof(float));+}++inline BenchmarkResult benchmark_matmul_f16(+ int M, int N, int K,+ int warmup_iters,+ int bench_iters+) {+ auto a = mx::random::uniform({M, K}, mx::float16);+ // Match UZU weight layout: B is stored as [N, K] and viewed as [K, N] via transpose+ auto b_transposed = mx::random::uniform({N, K}, mx::float16);+ auto b = mx::transpose(b_transposed);+ mx::eval(a, b);++ for (int i = 0; i < warmup_iters; ++i) {+ mx::eval(mx::matmul(a, b));+ }++ auto start = std::chrono::high_resolution_clock::now();+ for (int i = 0; i < bench_iters; ++i) {+ mx::eval(mx::matmul(a, b));+ }+ auto end = std::chrono::high_resolution_clock::now();++ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();+ double avg_ms = total_ms / bench_iters;+ double flops = 2.0 * M * N * K;+ double gflops = (flops / (avg_ms / 1000.0)) / 1e9;++ return BenchmarkResult{avg_ms, gflops};+}++inline BenchmarkResult benchmark_matmul_f32(+ int M, int N, int K,+ int warmup_iters,+ int bench_iters+) {+ auto a = mx::random::uniform({M, K}, mx::float32);+ // Match UZU weight layout: B is stored as [N, K] and viewed as [K, N] via transpose+ auto b_transposed = mx::random::uniform({N, K}, mx::float32);+ auto b = mx::transpose(b_transposed);+ mx::eval(a, b);++ for (int i = 0; i < warmup_iters; ++i) {+ mx::eval(mx::matmul(a, b));+ }++ auto start = std::chrono::high_resolution_clock::now();+ for (int i = 0; i < bench_iters; ++i) {+ mx::eval(mx::matmul(a, b));+ }+ auto end = std::chrono::high_resolution_clock::now();++ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();+ double avg_ms = total_ms / bench_iters;+ double flops = 2.0 * M * N * K;+ double gflops = (flops / (avg_ms / 1000.0)) / 1e9;++ return BenchmarkResult{avg_ms, gflops};+}++inline BenchmarkResult benchmark_matmul_bf16(+ int M, int N, int K,+ int warmup_iters,+ int bench_iters+) {+ auto a = mx::random::uniform({M, K}, mx::bfloat16);+ // Match UZU weight layout: B is stored as [N, K] and viewed as [K, N] via transpose+ auto b_transposed = mx::random::uniform({N, K}, mx::bfloat16);+ auto b = mx::transpose(b_transposed);+ mx::eval(a, b);++ for (int i = 0; i < warmup_iters; ++i) {+ mx::eval(mx::matmul(a, b));+ }++ auto start = std::chrono::high_resolution_clock::now();+ for (int i = 0; i < bench_iters; ++i) {+ mx::eval(mx::matmul(a, b));+ }+ auto end = std::chrono::high_resolution_clock::now();++ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();+ double avg_ms = total_ms / bench_iters;+ double flops = 2.0 * M * N * K;+ double gflops = (flops / (avg_ms / 1000.0)) / 1e9;++ return BenchmarkResult{avg_ms, gflops};+}++} // namespace cxx_utils++#endif // MLX_RS_CXX_UTILS_MATMUL_H_diff --git a/crates/matmul-benchmark/src/lib.rs b/crates/matmul-benchmark/src/lib.rsnew file mode 100644index 000000000..6dcfd7eff--- /dev/null+++ b/crates/matmul-benchmark/src/lib.rs@@ -0,0 +1,132 @@+#![allow(unsafe_op_in_unsafe_fn)]+#![allow(unused_imports)]++use autocxx::prelude::*;++include_cpp! {+ #include "mlx/mlx.h"+ #include "cxx_utils.hpp"+ safety!(unsafe_ffi)++ // Benchmark utilities+ generate_pod!("cxx_utils::BenchmarkResult")+ generate!("cxx_utils::benchmark_matmul_f16")+ generate!("cxx_utils::benchmark_matmul_f32")+ generate!("cxx_utils::benchmark_matmul_bf16")+ generate!("cxx_utils::matmul_f16_with_output")+ generate!("cxx_utils::matmul_f32_with_output")+ generate!("cxx_utils::matmul_bf16_with_output")+}++pub use ffi::cxx_utils::BenchmarkResult;++pub fn benchmark_matmul_f16(+ m: i32,+ n: i32,+ k: i32,+ warmup: i32,+ iters: i32,+) -> BenchmarkResult {+ ffi::cxx_utils::benchmark_matmul_f16(+ c_int(m),+ c_int(n),+ c_int(k),+ c_int(warmup),+ c_int(iters),+ )+}++pub fn benchmark_matmul_f32(+ m: i32,+ n: i32,+ k: i32,+ warmup: i32,+ iters: i32,+) -> BenchmarkResult {+ ffi::cxx_utils::benchmark_matmul_f32(+ c_int(m),+ c_int(n),+ c_int(k),+ c_int(warmup),+ c_int(iters),+ )+}++pub fn benchmark_matmul_bf16(+ m: i32,+ n: i32,+ k: i32,+ warmup: i32,+ iters: i32,+) -> BenchmarkResult {+ ffi::cxx_utils::benchmark_matmul_bf16(+ c_int(m),+ c_int(n),+ c_int(k),+ c_int(warmup),+ c_int(iters),+ )+}++/// Compute matmul using MLX and return result for accuracy comparison+/// A: [m, k] row-major, B: [n, k] row-major (transposed storage)+/// Output: [m, n]+pub fn matmul_f16_with_output(+ a: &[u16],+ b: &[u16],+ output: &mut [u16],+ m: i32,+ k: i32,+ n: i32,+) {+ unsafe {+ ffi::cxx_utils::matmul_f16_with_output(+ a.as_ptr(),+ b.as_ptr(),+ output.as_mut_ptr(),+ c_int(m),+ c_int(k),+ c_int(n),+ );+ }+}++pub fn matmul_f32_with_output(+ a: &[f32],+ b: &[f32],+ output: &mut [f32],+ m: i32,+ k: i32,+ n: i32,+) {+ unsafe {+ ffi::cxx_utils::matmul_f32_with_output(+ a.as_ptr(),+ b.as_ptr(),+ output.as_mut_ptr(),+ c_int(m),+ c_int(k),+ c_int(n),+ );+ }+}++pub fn matmul_bf16_with_output(+ a: &[u16],+ b: &[u16],+ output: &mut [u16],+ m: i32,+ k: i32,+ n: i32,+) {+ unsafe {+ ffi::cxx_utils::matmul_bf16_with_output(+ a.as_ptr(),+ b.as_ptr(),+ output.as_mut_ptr(),+ c_int(m),+ c_int(k),+ c_int(n),+ );+ }+}diff --git a/crates/matmul-benchmark/src/main.rs b/crates/matmul-benchmark/src/main.rsnew file mode 100644index 000000000..c6c3c1d20--- /dev/null+++ b/crates/matmul-benchmark/src/main.rs@@ -0,0 +1,1194 @@+//! Matmul benchmark comparing MLX, UZU, and MPSGraph backends+//!+//! Run with: cargo run --release -p matmul-benchmark+use std::{+ collections::HashMap,+ fmt,+ fs::{self, File},+ io::Read,+ path::{Path, PathBuf},+ time::Instant,+};++use half::{bf16, f16};+use matmul_benchmark::{+ benchmark_matmul_bf16 as mlx_bf16, benchmark_matmul_f16 as mlx_f16,+ benchmark_matmul_f32 as mlx_f32, matmul_bf16_with_output,+ matmul_f16_with_output, matmul_f32_with_output,+};+use metal::{Device, MTLResourceOptions};+use mpsgraph::{+ CommandBuffer, Device as MPSDevice, ExecutableExecutionDescriptor, Graph,+ ShapedType, TensorData,+};+use objc2::rc::autoreleasepool;+use serde::Deserialize;+use uzu::{+ DataType,+ backends::metal::{+ MTLContext,+ kernel::{MatmulArguments, MatmulKernel},+ },+};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum DType {+ F16,+ F32,+ BF16,+}++impl fmt::Display for DType {+ fn fmt(+ &self,+ f: &mut fmt::Formatter<'_>,+ ) -> fmt::Result {+ match self {+ DType::F16 => write!(f, " f16"),+ DType::F32 => write!(f, " f32"),+ DType::BF16 => write!(f, "bf16"),+ }+ }+}++impl DType {+ fn to_uzu(self) -> DataType {+ match self {+ DType::F16 => DataType::F16,+ DType::F32 => DataType::F32,+ DType::BF16 => DataType::BF16,+ }+ }++ fn to_mps(self) -> mpsgraph::DataType {+ match self {+ DType::F16 => mpsgraph::DataType::Float16,+ DType::F32 => mpsgraph::DataType::Float32,+ DType::BF16 => mpsgraph::DataType::BFloat16,+ }+ }++ fn element_size(self) -> usize {+ match self {+ DType::F16 | DType::BF16 => 2,+ DType::F32 => 4,+ }+ }+}++#[derive(Debug, Clone)]+struct BenchmarkResult {+ mlx_ms: f64,+ uzu_ms: f64,+ mps_ms: f64,+}++#[derive(Debug, Clone)]+struct AccuracyResult {+ max_abs_error: f32,+ mean_abs_error: f32,+ max_rel_error: f32,+}++#[derive(Debug, Clone)]+struct BenchCase {+ name: String,+ m: usize,+ k: usize,+ n: usize,+ dtype: DType,+}++impl BenchCase {+ fn b_bytes(&self) -> usize {+ self.n * self.k * self.dtype.element_size()+ }+}++// ============================================================================+// LLM SHAPES (derived from performance-benchmarks model.safetensors)+// ============================================================================++#[derive(Debug, Clone, Deserialize)]+struct ModelConfigFile {+ repo: String,+ model_config: ModelConfigOuter,+}++#[derive(Debug, Clone, Deserialize)]+struct ModelConfigOuter {+ model_config: ModelConfigInner,+}++#[derive(Debug, Clone, Deserialize)]+struct ModelConfigInner {+ embedding_config: EmbeddingConfig,+}++#[derive(Debug, Clone, Deserialize)]+#[serde(tag = "type")]+enum EmbeddingConfig {+ TiedEmbeddingConfig {+ precision: String,+ },+ MLXQuantizedTiedEmbeddingConfig {+ activation_precision: String,+ },+ #[serde(other)]+ Unknown,+}++#[derive(Debug, Clone, Deserialize)]+struct SafetensorsTensorInfo {+ dtype: String,+ shape: Vec<usize>,+}++#[derive(Debug, Clone, Deserialize)]+struct PerfBenchRun {+ task: PerfBenchTask,+ tokens_count_input: usize,+}++#[derive(Debug, Clone, Deserialize)]+struct PerfBenchTask {+ repo_id: String,+}++fn workspace_root() -> PathBuf {+ Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("../..")+ .canonicalize()+ .unwrap_or_else(|_| Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."))+}++fn model_code(model_dir_name: &str) -> String {+ if model_dir_name.starts_with("Llama-3.2-1B") {+ "L3.2-1B".to_string()+ } else if model_dir_name.starts_with("Qwen3-0.6B") {+ "Q3-0.6B".to_string()+ } else if model_dir_name.starts_with("Qwen3-4B-AWQ") {+ "Q3-4B-AWQ".to_string()+ } else if model_dir_name.starts_with("Qwen3-4B-MLX-4bit") {+ "Q3-4B-4b".to_string()+ } else {+ // Best-effort fallback: keep within table width (<= 9 chars)+ model_dir_name.chars().take(9).collect()+ }+}++fn dtype_from_precision_string(precision: &str) -> Option<DType> {+ match precision {+ "float16" => Some(DType::F16),+ "bfloat16" => Some(DType::BF16),+ "float32" => Some(DType::F32),+ _ => None,+ }+}++fn model_activation_dtype(cfg: &ModelConfigFile) -> Option<DType> {+ match &cfg.model_config.model_config.embedding_config {+ EmbeddingConfig::TiedEmbeddingConfig {+ precision,+ } => dtype_from_precision_string(precision),+ EmbeddingConfig::MLXQuantizedTiedEmbeddingConfig {+ activation_precision,+ } => dtype_from_precision_string(activation_precision),+ EmbeddingConfig::Unknown => None,+ }+}++fn read_safetensors_header(+ path: &Path+) -> Result<HashMap<String, SafetensorsTensorInfo>, Box<dyn std::error::Error>>+{+ let mut file = File::open(path)?;+ let mut header_len_bytes = [0u8; 8];+ file.read_exact(&mut header_len_bytes)?;+ let header_len = u64::from_le_bytes(header_len_bytes) as usize;++ let mut header_bytes = vec![0u8; header_len];+ file.read_exact(&mut header_bytes)?;++ let raw: HashMap<String, serde_json::Value> =+ serde_json::from_slice(&header_bytes)?;++ raw.into_iter()+ .filter(|(k, _)| k != "__metadata__")+ .map(|(k, v)| {+ let info: SafetensorsTensorInfo = serde_json::from_value(v)?;+ Ok((k, info))+ })+ .collect()+}++fn weight_dims_nk(+ header: &HashMap<String, SafetensorsTensorInfo>,+ key: &str,+) -> Option<(usize, usize)> {+ let info = header.get(key)?;+ let (n, packed_k) = match info.shape.as_slice() {+ [n, k] => (*n, *k),+ _ => return None,+ };++ let k = match info.dtype.as_str() {+ // 4-bit packed (2 weights per byte)+ "U8" => packed_k.checked_mul(2)?,+ _ => packed_k,+ };++ Some((n, k))+}++fn collect_json_files(+ dir: &Path,+ out: &mut Vec<PathBuf>,+) -> std::io::Result<()> {+ for entry in fs::read_dir(dir)? {+ let entry = entry?;+ let path = entry.path();+ if path.is_dir() {+ collect_json_files(&path, out)?;+ } else if path.extension().is_some_and(|ext| ext == "json") {+ out.push(path);+ }+ }+ Ok(())+}++fn prefill_tokens_from_perf_results(+ json_files: &[PathBuf],+ repo_id: &str,+) -> Option<usize> {+ for path in json_files {+ let file = File::open(path).ok()?;+ let runs: Vec<PerfBenchRun> = serde_json::from_reader(file).ok()?;+ for run in runs {+ if run.task.repo_id == repo_id {+ return Some(run.tokens_count_input);+ }+ }+ }+ None+}++fn default_benchmark_cases() -> Vec<BenchCase> {+ let shapes: &[(&str, usize, usize, usize)] = &[+ ("GEMV 4K", 1, 4096, 4096),+ ("GEMV 8K", 1, 8192, 8192),+ ("Batch 4", 4, 4096, 4096),+ ("Batch 16", 16, 4096, 4096),+ ("Prefill 128", 128, 4096, 4096),+ ("Prefill 512", 512, 4096, 4096),+ ("Square 1K", 1024, 1024, 1024),+ ("Square 2K", 2048, 2048, 2048),+ ];++ let dtypes = [DType::F16, DType::BF16, DType::F32];+ shapes+ .iter()+ .flat_map(|(name, m, k, n)| {+ dtypes.iter().map(|dtype| BenchCase {+ name: (*name).to_string(),+ m: *m,+ k: *k,+ n: *n,+ dtype: *dtype,+ })+ })+ .collect()+}++fn llm_benchmark_cases() -> Vec<BenchCase> {+ match try_llm_benchmark_cases() {+ Ok(cases) if !cases.is_empty() => cases,+ _ => default_benchmark_cases(),+ }+}++fn try_llm_benchmark_cases()+-> Result<Vec<BenchCase>, Box<dyn std::error::Error>> {+ let root = workspace_root();+ let models_dir =+ root.join("external/performance-benchmarks/workspace/models/0.1.7");+ if !models_dir.is_dir() {+ return Ok(Vec::new());+ }++ let results_dir =+ root.join("external/performance-benchmarks/workspace/results");+ let json_files = if results_dir.is_dir() {+ let mut files = Vec::new();+ collect_json_files(&results_dir, &mut files)?;+ files+ } else {+ Vec::new()+ };++ let mut model_dirs: Vec<PathBuf> = fs::read_dir(&models_dir)?+ .filter_map(|e| e.ok())+ .map(|e| e.path())+ .filter(|p| p.is_dir())+ .collect();+ model_dirs.sort();++ let ops: [(&str, &str); 4] = [+ ("QKV", "transformer.layers.0.mixer.qkv_projection.weights"),+ ("O", "transformer.layers.0.mixer.out_projection.weights"),+ ("MLPu", "transformer.layers.0.mlp.up_projection.weights"),+ ("MLPd", "transformer.layers.0.mlp.down_projection.weights"),+ ];++ let mut cases = Vec::new();+ for model_dir in model_dirs {+ let dir_name =+ model_dir.file_name().and_then(|s| s.to_str()).unwrap_or_default();+ let code = model_code(dir_name);++ let cfg: ModelConfigFile = {+ let file = File::open(model_dir.join("config.json"))?;+ serde_json::from_reader(file)?+ };++ let dtype = model_activation_dtype(&cfg).unwrap_or(DType::BF16);++ let prefill_m =+ prefill_tokens_from_perf_results(&json_files, &cfg.repo)+ .unwrap_or(1024);+ let ms = if prefill_m == 1 {+ vec![1usize]+ } else {+ vec![1usize, prefill_m]+ };++ let header =+ read_safetensors_header(&model_dir.join("model.safetensors"))?;++ for (op_code, key) in ops {+ let Some((n, k)) = weight_dims_nk(&header, key) else {+ continue;+ };++ for m in &ms {+ cases.push(BenchCase {+ name: format!("{code} {op_code}"),+ m: *m,+ k,+ n,+ dtype,+ });+ }+ }+ }++ Ok(cases)+}++fn compute_gflops(+ m: usize,+ k: usize,+ n: usize,+ time_ms: f64,+) -> f64 {+ let flops = 2.0 * (m as f64) * (k as f64) * (n as f64);+ flops / (time_ms * 1e6)+}++/// Generate deterministic test data+fn generate_test_data_f16(+ m: usize,+ k: usize,+ n: usize,+) -> (Vec<f16>, Vec<f16>) {+ let a: Vec<f16> =+ (0..(m * k)).map(|i| f16::from_f32(((i % 13) as f32) * 0.01)).collect();+ let b: Vec<f16> = (0..(n * k))+ .map(|i| f16::from_f32(((i % 17) as f32) * 0.02 - 0.1))+ .collect();+ (a, b)+}++fn generate_test_data_f32(+ m: usize,+ k: usize,+ n: usize,+) -> (Vec<f32>, Vec<f32>) {+ let a: Vec<f32> = (0..(m * k)).map(|i| ((i % 13) as f32) * 0.01).collect();+ let b: Vec<f32> =+ (0..(n * k)).map(|i| ((i % 17) as f32) * 0.02 - 0.1).collect();+ (a, b)+}++fn generate_test_data_bf16(+ m: usize,+ k: usize,+ n: usize,+) -> (Vec<bf16>, Vec<bf16>) {+ let a: Vec<bf16> = (0..(m * k))+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))+ .collect();+ let b: Vec<bf16> = (0..(n * k))+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))+ .collect();+ (a, b)+}++fn benchmark_uzu(+ ctx: &MTLContext,+ dtype: DType,+ m: usize,+ k: usize,+ n: usize,+ warmup_iters: usize,+ bench_iters: usize,+) -> f64 {+ let elem_size = dtype.element_size();+ let a_bytes = m * k * elem_size;+ let b_bytes = n * k * elem_size;+ let d_bytes = m * n * elem_size;++ let a_buf = ctx+ .device+ .new_buffer(a_bytes as u64, MTLResourceOptions::StorageModeShared);+ let b_buf = ctx+ .device+ .new_buffer(b_bytes as u64, MTLResourceOptions::StorageModeShared);+ let d_buf = ctx+ .device+ .new_buffer(d_bytes as u64, MTLResourceOptions::StorageModeShared);++ // Initialize with test data+ match dtype {+ DType::F16 => {+ let (a, b) = generate_test_data_f16(m, k, n);+ unsafe {+ std::ptr::copy_nonoverlapping(+ a.as_ptr(),+ a_buf.contents() as *mut f16,+ a.len(),+ );+ std::ptr::copy_nonoverlapping(+ b.as_ptr(),+ b_buf.contents() as *mut f16,+ b.len(),+ );+ }+ },+ DType::F32 => {+ let (a, b) = generate_test_data_f32(m, k, n);+ unsafe {+ std::ptr::copy_nonoverlapping(+ a.as_ptr(),+ a_buf.contents() as *mut f32,+ a.len(),+ );+ std::ptr::copy_nonoverlapping(+ b.as_ptr(),+ b_buf.contents() as *mut f32,+ b.len(),+ );+ }+ },+ DType::BF16 => {+ let (a, b) = generate_test_data_bf16(m, k, n);+ unsafe {+ std::ptr::copy_nonoverlapping(+ a.as_ptr(),+ a_buf.contents() as *mut bf16,+ a.len(),+ );+ std::ptr::copy_nonoverlapping(+ b.as_ptr(),+ b_buf.contents() as *mut bf16,+ b.len(),+ );+ }+ },+ }++ let mut kernel = MatmulKernel::new(ctx, dtype.to_uzu(), false, true)+ .expect("kernel new");++ for _ in 0..warmup_iters {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ c: None,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: n as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }++ let start = Instant::now();+ for _ in 0..bench_iters {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ c: None,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: n as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }+ let elapsed = start.elapsed();+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64+}++fn benchmark_mpsgraph(+ device: &metal::Device,+ dtype: DType,+ m: usize,+ k: usize,+ n: usize,+ warmup_iters: usize,+ bench_iters: usize,+) -> f64 {+ autoreleasepool(|_| {+ let graph = Graph::new();+ let mps_dtype = dtype.to_mps();++ let a_shape = [m as isize, k as isize];+ let b_shape = [k as isize, n as isize];++ let a_placeholder =+ graph.placeholder(Some(&a_shape), mps_dtype, Some("a"));+ let b_placeholder =+ graph.placeholder(Some(&b_shape), mps_dtype, Some("b"));++ let result =+ graph.matrix_multiplication(&a_placeholder, &b_placeholder, None);++ let a_shaped_type =+ ShapedType::new_with_shape_data_type(Some(&a_shape), mps_dtype);+ let b_shaped_type =+ ShapedType::new_with_shape_data_type(Some(&b_shape), mps_dtype);++ let feeds = HashMap::from([+ (&*a_placeholder, &*a_shaped_type),+ (&*b_placeholder, &*b_shaped_type),+ ]);++ let mps_device = MPSDevice::with_device(device);+ let executable =+ graph.compile(&mps_device, &feeds, &[&result], None, None);++ let execution_descriptor = ExecutableExecutionDescriptor::new();++ let elem_size = dtype.element_size();+ let a_buf = device.new_buffer(+ (m * k * elem_size) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = device.new_buffer(+ (k * n * elem_size) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let output_buf = device.new_buffer(+ (m * n * elem_size) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let command_queue = device.new_command_queue();++ let a_tensor_data =+ TensorData::new_with_mtl_buffer(&a_buf, &[m, k], mps_dtype, None);+ let b_tensor_data =+ TensorData::new_with_mtl_buffer(&b_buf, &[k, n], mps_dtype, None);+ let output_tensor_data = TensorData::new_with_mtl_buffer(+ &output_buf,+ &[m, n],+ mps_dtype,+ None,+ );++ let inputs: &[&TensorData] = &[&a_tensor_data, &b_tensor_data];+ let outputs: &[&TensorData] = &[&output_tensor_data];++ execution_descriptor.set_enable_commit_and_continue(true);+ for _ in 0..warmup_iters {+ let command_buffer =+ CommandBuffer::from_command_queue(&command_queue);+ let root_cb = command_buffer.root_command_buffer().to_owned();+ let _ = executable.encode_to_command_buffer(+ &command_buffer,+ inputs,+ Some(outputs),+ Some(&execution_descriptor),+ );+ command_buffer.commit_and_continue();+ root_cb.wait_until_completed();+ }++ let start = Instant::now();+ for _ in 0..bench_iters {+ let command_buffer =+ CommandBuffer::from_command_queue(&command_queue);+ let root_cb = command_buffer.root_command_buffer().to_owned();+ let _ = executable.encode_to_command_buffer(+ &command_buffer,+ inputs,+ Some(outputs),+ Some(&execution_descriptor),+ );+ command_buffer.commit_and_continue();+ root_cb.wait_until_completed();+ }+ let elapsed = start.elapsed();+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64+ })+}++fn benchmark_mlx(+ dtype: DType,+ m: usize,+ k: usize,+ n: usize,+ warmup: i32,+ iters: i32,+) -> f64 {+ match dtype {+ DType::F16 => {+ mlx_f16(m as i32, n as i32, k as i32, warmup, iters).avg_time_ms+ },+ DType::F32 => {+ mlx_f32(m as i32, n as i32, k as i32, warmup, iters).avg_time_ms+ },+ DType::BF16 => {+ mlx_bf16(m as i32, n as i32, k as i32, warmup, iters).avg_time_ms+ },+ }+}++fn run_benchmark(+ ctx: &MTLContext,+ dtype: DType,+ m: usize,+ k: usize,+ n: usize,+ warmup: usize,+ iters: usize,+) -> BenchmarkResult {+ let mlx_ms = benchmark_mlx(dtype, m, k, n, warmup as i32, iters as i32);+ let uzu_ms = benchmark_uzu(ctx, dtype, m, k, n, warmup, iters);+ let mps_ms = benchmark_mpsgraph(&ctx.device, dtype, m, k, n, warmup, iters);++ BenchmarkResult {+ mlx_ms,+ uzu_ms,+ mps_ms,+ }+}++fn format_speedup(+ base: f64,+ other: f64,+) -> String {+ let ratio = other / base;+ format!("{:.2}x", ratio)+}++/// Run accuracy test for a given shape and dtype+fn run_accuracy_test(+ ctx: &MTLContext,+ dtype: DType,+ m: usize,+ k: usize,+ n: usize,+) -> AccuracyResult {+ match dtype {+ DType::F16 => run_accuracy_test_f16(ctx, m, k, n),+ DType::F32 => run_accuracy_test_f32(ctx, m, k, n),+ DType::BF16 => run_accuracy_test_bf16(ctx, m, k, n),+ }+}++fn run_accuracy_test_f16(+ ctx: &MTLContext,+ m: usize,+ k: usize,+ n: usize,+) -> AccuracyResult {+ let (a, b) = generate_test_data_f16(m, k, n);++ // MLX+ let a_u16: Vec<u16> = a.iter().map(|x| x.to_bits()).collect();+ let b_u16: Vec<u16> = b.iter().map(|x| x.to_bits()).collect();+ let mut mlx_out_u16 = vec![0u16; m * n];+ matmul_f16_with_output(+ &a_u16,+ &b_u16,+ &mut mlx_out_u16,+ m as i32,+ k as i32,+ n as i32,+ );+ let mlx_out: Vec<f16> =+ mlx_out_u16.iter().map(|&x| f16::from_bits(x)).collect();++ // UZU+ let a_buf = ctx.device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * 2) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * 2) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx+ .device+ .new_buffer((m * n * 2) as u64, MTLResourceOptions::StorageModeShared);++ let mut kernel =+ MatmulKernel::new(ctx, DataType::F16, false, true).unwrap();+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ c: None,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: n as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();++ let uzu_out: Vec<f16> = unsafe {+ std::slice::from_raw_parts(d_buf.contents() as *const f16, m * n)+ .to_vec()+ };++ compare_outputs_f16(&mlx_out, &uzu_out)+}++fn run_accuracy_test_f32(+ ctx: &MTLContext,+ m: usize,+ k: usize,+ n: usize,+) -> AccuracyResult {+ let (a, b) = generate_test_data_f32(m, k, n);++ // MLX+ let mut mlx_out = vec![0f32; m * n];+ matmul_f32_with_output(&a, &b, &mut mlx_out, m as i32, k as i32, n as i32);++ // UZU+ let a_buf = ctx.device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * 4) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * 4) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx+ .device+ .new_buffer((m * n * 4) as u64, MTLResourceOptions::StorageModeShared);++ let mut kernel =+ MatmulKernel::new(ctx, DataType::F32, false, true).unwrap();+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ c: None,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: n as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();++ let uzu_out: Vec<f32> = unsafe {+ std::slice::from_raw_parts(d_buf.contents() as *const f32, m * n)+ .to_vec()+ };++ compare_outputs_f32(&mlx_out, &uzu_out)+}++fn run_accuracy_test_bf16(+ ctx: &MTLContext,+ m: usize,+ k: usize,+ n: usize,+) -> AccuracyResult {+ let (a, b) = generate_test_data_bf16(m, k, n);++ // MLX+ let a_u16: Vec<u16> = a.iter().map(|x| x.to_bits()).collect();+ let b_u16: Vec<u16> = b.iter().map(|x| x.to_bits()).collect();+ let mut mlx_out_u16 = vec![0u16; m * n];+ matmul_bf16_with_output(+ &a_u16,+ &b_u16,+ &mut mlx_out_u16,+ m as i32,+ k as i32,+ n as i32,+ );+ let mlx_out: Vec<bf16> =+ mlx_out_u16.iter().map(|&x| bf16::from_bits(x)).collect();++ // UZU+ let a_buf = ctx.device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * 2) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * 2) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx+ .device+ .new_buffer((m * n * 2) as u64, MTLResourceOptions::StorageModeShared);++ let mut kernel =+ MatmulKernel::new(ctx, DataType::BF16, false, true).unwrap();+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ c: None,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: n as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();++ let uzu_out: Vec<bf16> = unsafe {+ std::slice::from_raw_parts(d_buf.contents() as *const bf16, m * n)+ .to_vec()+ };++ compare_outputs_bf16(&mlx_out, &uzu_out)+}++fn compare_outputs_f16(+ reference: &[f16],+ test: &[f16],+) -> AccuracyResult {+ let mut max_abs_error: f32 = 0.0;+ let mut sum_abs_error: f32 = 0.0;+ let mut max_rel_error: f32 = 0.0;++ for (r, t) in reference.iter().zip(test.iter()) {+ let r_f32 = r.to_f32();+ let t_f32 = t.to_f32();+ let abs_error = (r_f32 - t_f32).abs();+ sum_abs_error += abs_error;+ max_abs_error = max_abs_error.max(abs_error);+ if r_f32.abs() > 1e-6 {+ max_rel_error = max_rel_error.max(abs_error / r_f32.abs());+ }+ }++ AccuracyResult {+ max_abs_error,+ mean_abs_error: sum_abs_error / reference.len() as f32,+ max_rel_error,+ }+}++fn compare_outputs_f32(+ reference: &[f32],+ test: &[f32],+) -> AccuracyResult {+ let mut max_abs_error: f32 = 0.0;+ let mut sum_abs_error: f32 = 0.0;+ let mut max_rel_error: f32 = 0.0;++ for (r, t) in reference.iter().zip(test.iter()) {+ let abs_error = (r - t).abs();+ sum_abs_error += abs_error;+ max_abs_error = max_abs_error.max(abs_error);+ if r.abs() > 1e-6 {+ max_rel_error = max_rel_error.max(abs_error / r.abs());+ }+ }++ AccuracyResult {+ max_abs_error,+ mean_abs_error: sum_abs_error / reference.len() as f32,+ max_rel_error,+ }+}++fn compare_outputs_bf16(+ reference: &[bf16],+ test: &[bf16],+) -> AccuracyResult {+ let mut max_abs_error: f32 = 0.0;+ let mut sum_abs_error: f32 = 0.0;+ let mut max_rel_error: f32 = 0.0;++ for (r, t) in reference.iter().zip(test.iter()) {+ let r_f32 = r.to_f32();+ let t_f32 = t.to_f32();+ let abs_error = (r_f32 - t_f32).abs();+ sum_abs_error += abs_error;+ max_abs_error = max_abs_error.max(abs_error);+ if r_f32.abs() > 1e-6 {+ max_rel_error = max_rel_error.max(abs_error / r_f32.abs());+ }+ }++ AccuracyResult {+ max_abs_error,+ mean_abs_error: sum_abs_error / reference.len() as f32,+ max_rel_error,+ }+}++fn main() {+ let device = Device::system_default().expect("No Metal device found");+ let command_queue = device.new_command_queue();+ let ctx = MTLContext::new(device.clone(), command_queue)+ .expect("Failed to create MTLContext");++ let cases = llm_benchmark_cases();++ // =========================================================================+ // ACCURACY TESTS+ // =========================================================================+ println!();+ println!(+ "╔═══════════════════════════════════════════════════════════════════════════════╗"+ );+ println!(+ "║ ACCURACY TESTS: UZU vs MLX ║"+ );+ println!(+ "╠═══════════════════════════════════════════════════════════════════════════════╣"+ );+ println!("║ Device: {:<69}║", device.name());+ println!(+ "╚═══════════════════════════════════════════════════════════════════════════════╝"+ );+ println!();++ println!(+ "┌────────────────┬───────┬───────┬───────┬──────┬───────────┬───────────┬────────────┐"+ );+ println!(+ "│ Shape │ M │ K │ N │ Type │ Max Abs │ Mean Abs │ Max Rel │"+ );+ println!(+ "├────────────────┼───────┼───────┼───────┼──────┼───────────┼───────────┼────────────┤"+ );++ let mut all_passed = true;+ let tolerance = 0.01;++ // Keep accuracy checks reasonably fast: only small(ish) B buffers and M=1+ let accuracy_cases: Vec<&BenchCase> = cases+ .iter()+ .filter(|c| c.m == 1 && c.b_bytes() <= 32 * 1024 * 1024)+ .collect();++ for case in &accuracy_cases {+ let result =+ run_accuracy_test(&ctx, case.dtype, case.m, case.k, case.n);+ let passed = result.max_rel_error < tolerance;+ if !passed {+ all_passed = false;+ }++ println!(+ "│ {:<14} │ {:>5} │ {:>5} │ {:>5} │ {:>4} │ {:>9.2e} │ {:>9.2e} │ {:>9.2e} │",+ case.name,+ case.m,+ case.k,+ case.n,+ case.dtype,+ result.max_abs_error,+ result.mean_abs_error,+ result.max_rel_error+ );+ }++ println!(+ "└────────────────┴───────┴───────┴───────┴──────┴───────────┴───────────┴────────────┘"+ );+ println!();++ if all_passed {+ println!(+ "✓ All accuracy tests PASSED (max relative error < {:.0}%)",+ tolerance * 100.0+ );+ } else {+ println!(+ "✗ Some accuracy tests FAILED (max relative error >= {:.0}%)",+ tolerance * 100.0+ );+ }+ println!();++ // =========================================================================+ // PERFORMANCE BENCHMARKS+ // =========================================================================+ println!();+ println!(+ "╔══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╗"+ );+ println!(+ "║ MATMUL BENCHMARK: MLX vs UZU vs MPSGraph ║"+ );+ println!(+ "╠══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╣"+ );+ println!("║ Device: {:<103}║", device.name());+ println!(+ "╚══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝"+ );+ println!();++ let warmup = 10;+ let iters = 100;++ println!(+ "┌────────────────┬───────┬───────┬───────┬──────┬────────────────────────────┬────────────────────────────┬─────────────────────┐"+ );+ println!(+ "│ │ │ │ │ │ Time (ms) │ GFLOPS │ UZU Speedup vs │"+ );+ println!(+ "│ Shape │ M │ K │ N │ Type ├────────┬────────┬──────────┼────────┬────────┬──────────┼──────────┬──────────┤"+ );+ println!(+ "│ │ │ │ │ │ MLX │ UZU │ MPS │ MLX │ UZU │ MPS │ MLX │ MPS │"+ );+ println!(+ "├────────────────┼───────┼───────┼───────┼──────┼────────┼────────┼──────────┼────────┼────────┼──────────┼──────────┼──────────┤"+ );++ for case in &cases {+ let result = run_benchmark(+ &ctx, case.dtype, case.m, case.k, case.n, warmup, iters,+ );++ let mlx_gflops = compute_gflops(case.m, case.k, case.n, result.mlx_ms);+ let uzu_gflops = compute_gflops(case.m, case.k, case.n, result.uzu_ms);+ let mps_gflops = compute_gflops(case.m, case.k, case.n, result.mps_ms);++ let uzu_vs_mlx = format_speedup(result.uzu_ms, result.mlx_ms);+ let uzu_vs_mps = format_speedup(result.uzu_ms, result.mps_ms);++ println!(+ "│ {:<14} │ {:>5} │ {:>5} │ {:>5} │ {:>4} │ {:>6.3} │ {:>6.3} │ {:>8.3} │ {:>6.0} │ {:>6.0} │ {:>8.0} │ {:>8} │ {:>8} │",+ case.name,+ case.m,+ case.k,+ case.n,+ case.dtype,+ result.mlx_ms,+ result.uzu_ms,+ result.mps_ms,+ mlx_gflops,+ uzu_gflops,+ mps_gflops,+ uzu_vs_mlx,+ uzu_vs_mps+ );+ }++ println!(+ "└────────────────┴───────┴───────┴───────┴──────┴────────┴────────┴──────────┴────────┴────────┴──────────┴──────────┴──────────┘"+ );+ println!();+ println!("Legend:");+ println!(" - UZU Speedup > 1.0x means UZU is faster");+ println!(" - UZU Speedup < 1.0x means the other backend is faster");+ println!();+}