Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add matmul benchmark

Eugene Bokhan10c656a7c42e
diff --git a/Cargo.lock b/Cargo.lock
index c3df3983a..1be372c1f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2043,6 +2043,24 @@ dependencies = [
"regex-automata 0.1.10",
]
+[[package]]
+name = "matmul-benchmark"
+version = "0.1.0"
+dependencies = [
+ "autocxx",
+ "autocxx-build",
+ "cmake",
+ "cxx",
+ "half",
+ "metal",
+ "mpsgraph",
+ "objc2",
+ "serde",
+ "serde_json",
+ "uzu",
+ "walkdir",
+]
+
[[package]]
name = "matrixmultiply"
version = "0.3.10"
diff --git a/Cargo.toml b/Cargo.toml
index 46dccc8f4..e6b72f308 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,10 @@
[workspace]
-members = ["crates/cli", "crates/uzu", "crates/benchmarks"]
+members = [
+ "crates/cli",
+ "crates/uzu",
+ "crates/benchmarks",
+ "crates/matmul-benchmark",
+]
resolver = "3"
[workspace.package]
diff --git a/crates/matmul-benchmark/Cargo.toml b/crates/matmul-benchmark/Cargo.toml
new file mode 100644
index 000000000..98d7dbd3c
--- /dev/null
+++ b/crates/matmul-benchmark/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "matmul-benchmark"
+version = "0.1.0"
+edition = "2024"
+build = "build.rs"
+links = "mlx"
+
+[[bin]]
+name = "matmul-benchmark"
+path = "src/main.rs"
+
+[dependencies]
+autocxx = "0.30.0"
+cxx = "1.0.187"
+uzu = { path = "../uzu" }
+metal = "0.32.0"
+mpsgraph = "0.2.0"
+half = "2.4"
+objc2 = "0.6"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+[build-dependencies]
+walkdir = "2.5.0"
+cmake = "0.1.57"
+autocxx-build = "0.30.0"
diff --git a/crates/matmul-benchmark/build.rs b/crates/matmul-benchmark/build.rs
new file mode 100644
index 000000000..ff1e8c453
--- /dev/null
+++ b/crates/matmul-benchmark/build.rs
@@ -0,0 +1,524 @@
+use std::{
+ env,
+ fs::{self, create_dir_all},
+ path::{Path, PathBuf},
+ process::Command,
+};
+
+use cmake::Config as CMakeConfig;
+use walkdir::WalkDir;
+
+const DEFAULT_MLX_GIT_URL: &str = "https://github.com/ml-explore/mlx.git";
+const DEFAULT_MLX_GIT_REF: &str = "v0.30.1";
+
+fn abs_path<P: AsRef<Path>>(p: P) -> PathBuf {
+ if p.as_ref().is_absolute() {
+ p.as_ref().to_path_buf()
+ } else {
+ env::current_dir().expect("current_dir failed").join(p)
+ }
+}
+
+fn looks_like_mlx_repo_root(dir: &Path) -> bool {
+ dir.join("CMakeLists.txt").exists()
+ && dir.join("mlx").exists()
+ && dir.join("mlx/mlx.h").exists()
+}
+
+fn is_truthy_env(name: &str) -> bool {
+ let Ok(v) = env::var(name) else {
+ return false;
+ };
+ matches!(
+ v.trim().to_ascii_lowercase().as_str(),
+ "1" | "true" | "yes" | "on"
+ )
+}
+
+fn cargo_offline() -> bool {
+ is_truthy_env("CARGO_NET_OFFLINE") || is_truthy_env("MLX_RS_OFFLINE")
+}
+
+fn cache_dir(out_dir: &Path) -> PathBuf {
+ if let Ok(p) = env::var("MLX_RS_CACHE_DIR") {
+ return abs_path(p);
+ }
+
+ if let Ok(p) = env::var("CARGO_HOME") {
+ return abs_path(p).join("mlx-rs-cache");
+ }
+
+ if let Ok(p) = env::var("HOME") {
+ return PathBuf::from(p).join(".cache/mlx-rs");
+ }
+ if let Ok(p) = env::var("LOCALAPPDATA") {
+ return PathBuf::from(p).join("mlx-rs");
+ }
+
+ out_dir.join("mlx-rs-cache")
+}
+
+fn run_checked(
+ mut cmd: Command,
+ what: &str,
+) {
+ let output = cmd.output().unwrap_or_else(|e| {
+ panic!("Failed to run {}: {}", what, e);
+ });
+ if !output.status.success() {
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ let stderr = String::from_utf8_lossy(&output.stderr);
+ panic!(
+ "{} failed (exit={:?})\n--- stdout ---\n{}\n--- stderr ---\n{}\n",
+ what,
+ output.status.code(),
+ stdout,
+ stderr
+ );
+ }
+}
+
+fn pins_toml_path(manifest_dir: &Path) -> PathBuf {
+ if let Ok(p) = env::var("MLX_RS_PINS_TOML") {
+ return abs_path(p);
+ }
+ manifest_dir.join("mlx-pins.toml")
+}
+
+#[derive(Debug, Clone)]
+struct Pins {
+ repo_url: Option<String>,
+ repo_ref: Option<String>,
+}
+
+fn parse_pins(pins_path: &Path) -> Pins {
+ let contents = fs::read_to_string(pins_path).unwrap_or_else(|e| {
+ panic!(
+ "Failed to read pins file at {}: {}. \
+ Update the pins file or set MLX_RS_PINS_TOML to a valid path.",
+ pins_path.display(),
+ e
+ )
+ });
+
+ #[derive(Debug)]
+ enum Section {
+ None,
+ Repo,
+ }
+
+ let mut section = Section::None;
+ let mut pins = Pins {
+ repo_url: None,
+ repo_ref: None,
+ };
+
+ for raw in contents.lines() {
+ let line = raw.trim();
+ if line.is_empty() || line.starts_with('#') {
+ continue;
+ }
+ if line.starts_with('[') && line.ends_with(']') {
+ let header = &line[1..line.len() - 1];
+ if header == "repo" {
+ section = Section::Repo;
+ } else {
+ section = Section::None;
+ }
+ continue;
+ }
+
+ let Some((k, v)) = line.split_once('=') else {
+ continue;
+ };
+ let key = k.trim();
+ let mut val = v.trim();
+ if val.starts_with('"') && val.ends_with('"') && val.len() >= 2 {
+ val = &val[1..val.len() - 1];
+ }
+
+ match &mut section {
+ Section::Repo => match key {
+ "url" => pins.repo_url = Some(val.to_string()),
+ "ref" | "rev" => pins.repo_ref = Some(val.to_string()),
+ _ => {},
+ },
+ Section::None => {},
+ }
+ }
+
+ pins
+}
+
+fn pinned_mlx_git(pins: &Pins) -> (String, String) {
+ let url = env::var("MLX_GIT_URL")
+ .ok()
+ .or_else(|| pins.repo_url.clone())
+ .unwrap_or_else(|| DEFAULT_MLX_GIT_URL.to_string());
+ let rev = env::var("MLX_GIT_REF")
+ .ok()
+ .or_else(|| pins.repo_ref.clone())
+ .unwrap_or_else(|| DEFAULT_MLX_GIT_REF.to_string());
+ (url, rev)
+}
+
+fn maybe_clear_cmake_build_dir(
+ build_dir: &Path,
+ source_dir: &Path,
+) {
+ let cache = build_dir.join("CMakeCache.txt");
+ let Ok(contents) = fs::read_to_string(&cache) else {
+ return;
+ };
+ let src =
+ source_dir.canonicalize().unwrap_or_else(|_| source_dir.to_path_buf());
+ for line in contents.lines() {
+ if !line.starts_with("CMAKE_HOME_DIRECTORY") {
+ continue;
+ }
+ let needs_cleanup = match line.split('=').next_back() {
+ Some(cmake_home) => fs::canonicalize(cmake_home)
+ .ok()
+ .map(|cmake_home| cmake_home != src)
+ .unwrap_or(true),
+ None => true,
+ };
+ if needs_cleanup {
+ let _ = fs::remove_dir_all(build_dir);
+ }
+ break;
+ }
+}
+
+fn ensure_git_checkout_cached(
+ name: &str,
+ url: &str,
+ rev: &str,
+ cache_dir: &Path,
+) -> PathBuf {
+ let checkout_dir = cache_dir.join(format!("{}-{}", name, rev));
+ let marker = checkout_dir.join(".mlx_rs_fetched");
+ if marker.exists() {
+ return checkout_dir;
+ }
+
+ if checkout_dir.exists() {
+ let _ = fs::remove_dir_all(&checkout_dir);
+ }
+ create_dir_all(cache_dir).expect("Failed to create cache dir");
+
+ run_checked(
+ {
+ let mut c = Command::new("git");
+ c.arg("clone").arg(url).arg(&checkout_dir);
+ c
+ },
+ &format!("git clone {} into cache", name),
+ );
+ run_checked(
+ {
+ let mut c = Command::new("git");
+ c.arg("-C").arg(&checkout_dir).arg("checkout").arg(rev);
+ c
+ },
+ &format!("git checkout {}@{}", name, rev),
+ );
+
+ let _ = fs::write(&marker, rev);
+ checkout_dir
+}
+
+fn ensure_mlx_repo(
+ out_dir: &Path,
+ manifest_dir: &Path,
+ repo_url: &str,
+ repo_ref: &str,
+) -> PathBuf {
+ // First, check for explicit MLX_SRC_DIR environment variable
+ if let Ok(p) = env::var("MLX_SRC_DIR") {
+ let p = abs_path(p);
+ if !looks_like_mlx_repo_root(&p) {
+ panic!(
+ "MLX_SRC_DIR={} does not look like an MLX repo root \
+ (expected CMakeLists.txt + mlx/ + mlx/mlx.h)",
+ p.display()
+ );
+ }
+ return p;
+ }
+
+ // Second, check for local external/mlx in the workspace
+ let workspace_root = manifest_dir.parent().and_then(|p| p.parent());
+ if let Some(root) = workspace_root {
+ let local_mlx = root.join("external/mlx");
+ if looks_like_mlx_repo_root(&local_mlx) {
+ println!(
+ "cargo:warning=mlx-rs: using local MLX at {}",
+ local_mlx.display()
+ );
+ return local_mlx;
+ }
+ }
+
+ if cargo_offline() {
+ panic!(
+ "MLX sources not found locally and Cargo is offline. \
+ Set MLX_SRC_DIR to a checked-out MLX repo or build with network access."
+ );
+ }
+
+ let cache = cache_dir(out_dir);
+ println!(
+ "cargo:warning=mlx-rs: fetching MLX {}@{} into {}",
+ repo_url,
+ repo_ref,
+ cache.display()
+ );
+ ensure_git_checkout_cached("mlx", repo_url, repo_ref, &cache)
+}
+
+fn find_mlx_lib_dir(root: &Path) -> Option<PathBuf> {
+ let static_candidates = ["libmlx.a", "mlx.lib"];
+
+ for entry in
+ WalkDir::new(root).max_depth(6).into_iter().filter_map(Result::ok)
+ {
+ if !entry.file_type().is_file() {
+ continue;
+ }
+
+ let name = entry.file_name().to_string_lossy();
+ if static_candidates.iter().any(|c| name == *c) {
+ return entry.path().parent().map(|p| p.to_path_buf());
+ }
+ }
+
+ None
+}
+
+#[derive(Debug, Clone)]
+struct BuildContext {
+ #[allow(dead_code)]
+ manifest_dir: PathBuf,
+ mlx_src_dir: PathBuf,
+ out_dir: PathBuf,
+ src_include_dir: PathBuf,
+ target: String,
+}
+
+fn collect_build_context() -> BuildContext {
+ println!("cargo:rerun-if-env-changed=MLX_SRC_DIR");
+ println!("cargo:rerun-if-env-changed=MLX_RS_PINS_TOML");
+ println!("cargo:rerun-if-env-changed=MLX_GIT_URL");
+ println!("cargo:rerun-if-env-changed=MLX_GIT_REF");
+ println!("cargo:rerun-if-env-changed=MLX_RS_CACHE_DIR");
+ println!("cargo:rerun-if-env-changed=MLX_RS_OFFLINE");
+ println!("cargo:rerun-if-env-changed=CARGO_NET_OFFLINE");
+ println!("cargo:rerun-if-env-changed=CARGO_HOME");
+
+ let manifest_dir = abs_path(
+ env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"),
+ );
+ let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set"));
+
+ let pins_path = pins_toml_path(&manifest_dir);
+ println!("cargo:rerun-if-changed={}", pins_path.display());
+ let pins = parse_pins(&pins_path);
+
+ let (repo_url, repo_ref) = pinned_mlx_git(&pins);
+ let mlx_src_dir =
+ ensure_mlx_repo(&out_dir, &manifest_dir, &repo_url, &repo_ref);
+
+ println!("cargo:rerun-if-changed={}", mlx_src_dir.join("mlx").display());
+ println!(
+ "cargo:rerun-if-changed={}",
+ mlx_src_dir.join("CMakeLists.txt").display()
+ );
+
+ let src_include_dir = manifest_dir.join("src");
+ let target = env::var("TARGET").unwrap_or_default();
+
+ BuildContext {
+ manifest_dir,
+ mlx_src_dir,
+ out_dir,
+ src_include_dir,
+ target,
+ }
+}
+
+fn build_mlx_cmake(ctx: &BuildContext) -> PathBuf {
+ let cmake_build_dir = ctx.out_dir.join("build");
+ maybe_clear_cmake_build_dir(&cmake_build_dir, &ctx.mlx_src_dir);
+ create_dir_all(&cmake_build_dir).ok();
+
+ let mut cmake_config = CMakeConfig::new(&ctx.mlx_src_dir);
+ cmake_config.out_dir(&ctx.out_dir);
+
+ // Disable features we don't need
+ cmake_config.define("MLX_BUILD_TESTS", "OFF");
+ cmake_config.define("MLX_BUILD_EXAMPLES", "OFF");
+ cmake_config.define("MLX_BUILD_BENCHMARKS", "OFF");
+ cmake_config.define("MLX_BUILD_PYTHON_BINDINGS", "OFF");
+
+ // Enable Metal backend only (CPU backend has compatibility issues with some Xcode versions)
+ cmake_config.define("MLX_BUILD_METAL", "ON");
+ cmake_config.define("MLX_BUILD_CPU", "OFF");
+
+ // Build static library
+ cmake_config.define("BUILD_SHARED_LIBS", "OFF");
+
+ // C++ settings
+ cmake_config.define("CMAKE_CXX_STANDARD", "17");
+ cmake_config.define("CMAKE_CXX_STANDARD_REQUIRED", "ON");
+ cmake_config.define("CMAKE_CXX_EXTENSIONS", "OFF");
+
+ // Disable LTO to avoid linking issues
+ cmake_config.define("CMAKE_INTERPROCEDURAL_OPTIMIZATION", "OFF");
+
+ let is_msvc = ctx.target.contains("msvc");
+ if !is_msvc {
+ cmake_config.cflag("-fno-lto");
+ cmake_config.cxxflag("-fno-lto");
+ }
+
+ let build_profile =
+ match env::var("PROFILE").unwrap_or_else(|_| "release".into()).as_str()
+ {
+ "debug" => "Debug",
+ "release" => "Release",
+ _ => "RelWithDebInfo",
+ };
+ cmake_config.profile(build_profile);
+
+ // macOS specific settings
+ if ctx.target.contains("apple-darwin") {
+ let arch = if ctx.target.contains("aarch64") {
+ "arm64"
+ } else {
+ "x86_64"
+ };
+ cmake_config.define("CMAKE_OSX_ARCHITECTURES", arch);
+ // Force deployment target to avoid cmake crate confusing SDK version with deployment target
+ let dep_target = env::var("MACOSX_DEPLOYMENT_TARGET")
+ .unwrap_or_else(|_| "14.0".into());
+ cmake_config.define("CMAKE_OSX_DEPLOYMENT_TARGET", &dep_target);
+ // Also set the env var so cmake-rs doesn't override with bad values
+ unsafe { env::set_var("MACOSX_DEPLOYMENT_TARGET", &dep_target) };
+ }
+
+ cmake_config.build_target("mlx").build()
+}
+
+fn link_mlx_static(
+ ctx: &BuildContext,
+ destination_path: &Path,
+) {
+ let cmake_build_dir = ctx.out_dir.join("build");
+ let lib_search_dir = find_mlx_lib_dir(&cmake_build_dir)
+ .or_else(|| find_mlx_lib_dir(destination_path))
+ .unwrap_or_else(|| destination_path.join("lib"));
+ println!("cargo:rustc-link-search=native={}", lib_search_dir.display());
+ println!("cargo:rustc-link-lib=static=mlx");
+
+ // Link Apple frameworks required by MLX
+ println!("cargo:rustc-link-lib=framework=Metal");
+ println!("cargo:rustc-link-lib=framework=Foundation");
+ println!("cargo:rustc-link-lib=framework=QuartzCore");
+ println!("cargo:rustc-link-lib=framework=Accelerate");
+
+ // Link C++ standard library
+ println!("cargo:rustc-link-lib=c++");
+
+ // Set minimum macOS version for linking to match what MLX was built with
+ if ctx.target.contains("apple") {
+ let dep_target = env::var("MACOSX_DEPLOYMENT_TARGET")
+ .unwrap_or_else(|_| "14.0".into());
+ println!("cargo:rustc-link-arg=-mmacosx-version-min={}", dep_target);
+ // Link clang runtime for __isPlatformVersionAtLeast and other builtins
+ // Find clang resource directory for runtime libraries
+ if let Ok(output) =
+ Command::new("clang").args(["--print-resource-dir"]).output()
+ {
+ if output.status.success() {
+ let resource_dir =
+ String::from_utf8_lossy(&output.stdout).trim().to_string();
+ let lib_dir = format!("{}/lib/darwin", resource_dir);
+ println!("cargo:rustc-link-search=native={}", lib_dir);
+ println!("cargo:rustc-link-lib=static=clang_rt.osx");
+ }
+ }
+ }
+}
+
+fn build_autocxx_bridge(
+ ctx: &BuildContext,
+ metal_cpp_dir: &Path,
+) {
+ println!("cargo:rerun-if-changed=src/lib.rs");
+ println!("cargo:rerun-if-changed=src/cxx_utils.hpp");
+ println!("cargo:rerun-if-changed=src/cxx_utils/matmul.hpp");
+
+ let extra_clang_args = vec!["-std=c++17".to_string()];
+ let extra_clang_args_refs: Vec<&str> =
+ extra_clang_args.iter().map(|s| s.as_str()).collect();
+
+ let mut autocxx_builder = autocxx_build::Builder::new(
+ "src/lib.rs",
+ &[&ctx.src_include_dir, &ctx.mlx_src_dir, metal_cpp_dir],
+ )
+ .extra_clang_args(&extra_clang_args_refs)
+ .build()
+ .expect("autocxx build failed");
+
+ autocxx_builder
+ .flag_if_supported("-std=c++17")
+ .flag_if_supported("-Wno-unused-parameter")
+ .flag_if_supported("-Wno-deprecated-copy")
+ .include(&ctx.src_include_dir)
+ .include(&ctx.mlx_src_dir)
+ .include(metal_cpp_dir);
+
+ autocxx_builder.compile("mlx_rs_bridge");
+}
+
+fn find_metal_cpp_dir(out_dir: &Path) -> PathBuf {
+ // metal-cpp is fetched by MLX's CMake into the build directory
+ for entry in
+ WalkDir::new(out_dir).max_depth(6).into_iter().filter_map(Result::ok)
+ {
+ let path = entry.path();
+ if path.ends_with("Metal/Metal.hpp") {
+ if let Some(parent) = path.parent().and_then(|p| p.parent()) {
+ return parent.to_path_buf();
+ }
+ }
+ }
+ // Fallback
+ out_dir.join("build/_deps/metal_cpp-src")
+}
+
+fn main() {
+ // Set MACOSX_DEPLOYMENT_TARGET early to ensure all compilation uses consistent version
+ // MLX requires macOS 14.0+
+ let target = env::var("TARGET").unwrap_or_default();
+ if target.contains("apple") {
+ let dep_target = env::var("MACOSX_DEPLOYMENT_TARGET")
+ .unwrap_or_else(|_| "14.0".into());
+ // Ensure minimum 14.0 for MLX compatibility
+ let version: f32 = dep_target.parse().unwrap_or(14.0);
+ let final_target = if version < 14.0 {
+ "14.0"
+ } else {
+ &dep_target
+ };
+ unsafe { env::set_var("MACOSX_DEPLOYMENT_TARGET", final_target) };
+ }
+
+ let ctx = collect_build_context();
+ let destination_path = build_mlx_cmake(&ctx);
+ link_mlx_static(&ctx, &destination_path);
+
+ let metal_cpp_dir = find_metal_cpp_dir(&ctx.out_dir);
+ build_autocxx_bridge(&ctx, &metal_cpp_dir);
+}
diff --git a/crates/matmul-benchmark/mlx-pins.toml b/crates/matmul-benchmark/mlx-pins.toml
new file mode 100644
index 000000000..2b19c03f8
--- /dev/null
+++ b/crates/matmul-benchmark/mlx-pins.toml
@@ -0,0 +1,6 @@
+# Pins for MLX repo used by build.rs
+# Similar to xgrammar-rs pattern
+
+[repo]
+url = "https://github.com/ml-explore/mlx.git"
+ref = "v0.30.1"
diff --git a/crates/matmul-benchmark/src/cxx_utils.hpp b/crates/matmul-benchmark/src/cxx_utils.hpp
new file mode 100644
index 000000000..01793910e
--- /dev/null
+++ b/crates/matmul-benchmark/src/cxx_utils.hpp
@@ -0,0 +1,6 @@
+#ifndef MLX_RS_CXX_UTILS_H_
+#define MLX_RS_CXX_UTILS_H_
+
+#include "cxx_utils/matmul.hpp"
+
+#endif // MLX_RS_CXX_UTILS_H_
diff --git a/crates/matmul-benchmark/src/cxx_utils/matmul.hpp b/crates/matmul-benchmark/src/cxx_utils/matmul.hpp
new file mode 100644
index 000000000..4a0cef9d0
--- /dev/null
+++ b/crates/matmul-benchmark/src/cxx_utils/matmul.hpp
@@ -0,0 +1,200 @@
+#ifndef MLX_RS_CXX_UTILS_MATMUL_H_
+#define MLX_RS_CXX_UTILS_MATMUL_H_
+
+#include <chrono>
+#include <cstring>
+#include <vector>
+#include "mlx/mlx.h"
+
+namespace cxx_utils {
+
+namespace mx = mlx::core;
+
+struct BenchmarkResult {
+ double avg_time_ms;
+ double gflops;
+};
+
+// Compute matmul and copy result to output buffer for accuracy testing
+// Input: A[M,K] row-major, B[K,N] row-major (but we'll transpose B for consistency with UZU)
+// UZU expects: A[M,K], B^T[N,K] (transpose_b=true means B is stored as [N,K])
+// So we need to pass B as [N,K] to MLX too, then transpose it
+inline void matmul_bf16_with_output(
+ const uint16_t* a_data, // [M, K] row-major bf16 as raw bytes
+ const uint16_t* b_data, // [N, K] row-major bf16 (transposed storage)
+ uint16_t* output_data, // [M, N] output
+ int M, int K, int N
+) {
+ // Create MLX arrays from raw data
+ // A is [M, K], B is stored as [N, K] (transposed)
+ mx::Shape a_shape = {M, K};
+ mx::Shape b_shape = {N, K};
+
+ auto a = mx::array(
+ reinterpret_cast<const mx::bfloat16_t*>(a_data),
+ a_shape,
+ mx::bfloat16
+ );
+
+ auto b_transposed = mx::array(
+ reinterpret_cast<const mx::bfloat16_t*>(b_data),
+ b_shape,
+ mx::bfloat16
+ );
+
+ // Transpose B back to [K, N] for standard matmul
+ auto b = mx::transpose(b_transposed);
+ mx::eval(a, b);
+
+ // Compute matmul: [M, K] @ [K, N] -> [M, N]
+ auto result = mx::matmul(a, b);
+ mx::eval(result);
+
+ // Copy result to output buffer
+ const auto* result_ptr = result.data<mx::bfloat16_t>();
+ std::memcpy(output_data, result_ptr, M * N * sizeof(uint16_t));
+}
+
+inline void matmul_f16_with_output(
+ const uint16_t* a_data,
+ const uint16_t* b_data,
+ uint16_t* output_data,
+ int M, int K, int N
+) {
+ mx::Shape a_shape = {M, K};
+ mx::Shape b_shape = {N, K};
+
+ auto a = mx::array(
+ reinterpret_cast<const mx::float16_t*>(a_data),
+ a_shape,
+ mx::float16
+ );
+
+ auto b_transposed = mx::array(
+ reinterpret_cast<const mx::float16_t*>(b_data),
+ b_shape,
+ mx::float16
+ );
+
+ auto b = mx::transpose(b_transposed);
+ mx::eval(a, b);
+
+ auto result = mx::matmul(a, b);
+ mx::eval(result);
+
+ const auto* result_ptr = result.data<mx::float16_t>();
+ std::memcpy(output_data, result_ptr, M * N * sizeof(uint16_t));
+}
+
+inline void matmul_f32_with_output(
+ const float* a_data,
+ const float* b_data,
+ float* output_data,
+ int M, int K, int N
+) {
+ mx::Shape a_shape = {M, K};
+ mx::Shape b_shape = {N, K};
+
+ auto a = mx::array(a_data, a_shape, mx::float32);
+ auto b_transposed = mx::array(b_data, b_shape, mx::float32);
+
+ auto b = mx::transpose(b_transposed);
+ mx::eval(a, b);
+
+ auto result = mx::matmul(a, b);
+ mx::eval(result);
+
+ const auto* result_ptr = result.data<float>();
+ std::memcpy(output_data, result_ptr, M * N * sizeof(float));
+}
+
+inline BenchmarkResult benchmark_matmul_f16(
+ int M, int N, int K,
+ int warmup_iters,
+ int bench_iters
+) {
+ auto a = mx::random::uniform({M, K}, mx::float16);
+ // Match UZU weight layout: B is stored as [N, K] and viewed as [K, N] via transpose
+ auto b_transposed = mx::random::uniform({N, K}, mx::float16);
+ auto b = mx::transpose(b_transposed);
+ mx::eval(a, b);
+
+ for (int i = 0; i < warmup_iters; ++i) {
+ mx::eval(mx::matmul(a, b));
+ }
+
+ auto start = std::chrono::high_resolution_clock::now();
+ for (int i = 0; i < bench_iters; ++i) {
+ mx::eval(mx::matmul(a, b));
+ }
+ auto end = std::chrono::high_resolution_clock::now();
+
+ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+ double avg_ms = total_ms / bench_iters;
+ double flops = 2.0 * M * N * K;
+ double gflops = (flops / (avg_ms / 1000.0)) / 1e9;
+
+ return BenchmarkResult{avg_ms, gflops};
+}
+
+inline BenchmarkResult benchmark_matmul_f32(
+ int M, int N, int K,
+ int warmup_iters,
+ int bench_iters
+) {
+ auto a = mx::random::uniform({M, K}, mx::float32);
+ // Match UZU weight layout: B is stored as [N, K] and viewed as [K, N] via transpose
+ auto b_transposed = mx::random::uniform({N, K}, mx::float32);
+ auto b = mx::transpose(b_transposed);
+ mx::eval(a, b);
+
+ for (int i = 0; i < warmup_iters; ++i) {
+ mx::eval(mx::matmul(a, b));
+ }
+
+ auto start = std::chrono::high_resolution_clock::now();
+ for (int i = 0; i < bench_iters; ++i) {
+ mx::eval(mx::matmul(a, b));
+ }
+ auto end = std::chrono::high_resolution_clock::now();
+
+ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+ double avg_ms = total_ms / bench_iters;
+ double flops = 2.0 * M * N * K;
+ double gflops = (flops / (avg_ms / 1000.0)) / 1e9;
+
+ return BenchmarkResult{avg_ms, gflops};
+}
+
+inline BenchmarkResult benchmark_matmul_bf16(
+ int M, int N, int K,
+ int warmup_iters,
+ int bench_iters
+) {
+ auto a = mx::random::uniform({M, K}, mx::bfloat16);
+ // Match UZU weight layout: B is stored as [N, K] and viewed as [K, N] via transpose
+ auto b_transposed = mx::random::uniform({N, K}, mx::bfloat16);
+ auto b = mx::transpose(b_transposed);
+ mx::eval(a, b);
+
+ for (int i = 0; i < warmup_iters; ++i) {
+ mx::eval(mx::matmul(a, b));
+ }
+
+ auto start = std::chrono::high_resolution_clock::now();
+ for (int i = 0; i < bench_iters; ++i) {
+ mx::eval(mx::matmul(a, b));
+ }
+ auto end = std::chrono::high_resolution_clock::now();
+
+ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+ double avg_ms = total_ms / bench_iters;
+ double flops = 2.0 * M * N * K;
+ double gflops = (flops / (avg_ms / 1000.0)) / 1e9;
+
+ return BenchmarkResult{avg_ms, gflops};
+}
+
+} // namespace cxx_utils
+
+#endif // MLX_RS_CXX_UTILS_MATMUL_H_
diff --git a/crates/matmul-benchmark/src/lib.rs b/crates/matmul-benchmark/src/lib.rs
new file mode 100644
index 000000000..6dcfd7eff
--- /dev/null
+++ b/crates/matmul-benchmark/src/lib.rs
@@ -0,0 +1,132 @@
+#![allow(unsafe_op_in_unsafe_fn)]
+#![allow(unused_imports)]
+
+use autocxx::prelude::*;
+
+include_cpp! {
+ #include "mlx/mlx.h"
+ #include "cxx_utils.hpp"
+ safety!(unsafe_ffi)
+
+ // Benchmark utilities
+ generate_pod!("cxx_utils::BenchmarkResult")
+ generate!("cxx_utils::benchmark_matmul_f16")
+ generate!("cxx_utils::benchmark_matmul_f32")
+ generate!("cxx_utils::benchmark_matmul_bf16")
+ generate!("cxx_utils::matmul_f16_with_output")
+ generate!("cxx_utils::matmul_f32_with_output")
+ generate!("cxx_utils::matmul_bf16_with_output")
+}
+
+pub use ffi::cxx_utils::BenchmarkResult;
+
+pub fn benchmark_matmul_f16(
+ m: i32,
+ n: i32,
+ k: i32,
+ warmup: i32,
+ iters: i32,
+) -> BenchmarkResult {
+ ffi::cxx_utils::benchmark_matmul_f16(
+ c_int(m),
+ c_int(n),
+ c_int(k),
+ c_int(warmup),
+ c_int(iters),
+ )
+}
+
+pub fn benchmark_matmul_f32(
+ m: i32,
+ n: i32,
+ k: i32,
+ warmup: i32,
+ iters: i32,
+) -> BenchmarkResult {
+ ffi::cxx_utils::benchmark_matmul_f32(
+ c_int(m),
+ c_int(n),
+ c_int(k),
+ c_int(warmup),
+ c_int(iters),
+ )
+}
+
+pub fn benchmark_matmul_bf16(
+ m: i32,
+ n: i32,
+ k: i32,
+ warmup: i32,
+ iters: i32,
+) -> BenchmarkResult {
+ ffi::cxx_utils::benchmark_matmul_bf16(
+ c_int(m),
+ c_int(n),
+ c_int(k),
+ c_int(warmup),
+ c_int(iters),
+ )
+}
+
+/// Compute matmul using MLX and return result for accuracy comparison
+/// A: [m, k] row-major, B: [n, k] row-major (transposed storage)
+/// Output: [m, n]
+pub fn matmul_f16_with_output(
+ a: &[u16],
+ b: &[u16],
+ output: &mut [u16],
+ m: i32,
+ k: i32,
+ n: i32,
+) {
+ unsafe {
+ ffi::cxx_utils::matmul_f16_with_output(
+ a.as_ptr(),
+ b.as_ptr(),
+ output.as_mut_ptr(),
+ c_int(m),
+ c_int(k),
+ c_int(n),
+ );
+ }
+}
+
+pub fn matmul_f32_with_output(
+ a: &[f32],
+ b: &[f32],
+ output: &mut [f32],
+ m: i32,
+ k: i32,
+ n: i32,
+) {
+ unsafe {
+ ffi::cxx_utils::matmul_f32_with_output(
+ a.as_ptr(),
+ b.as_ptr(),
+ output.as_mut_ptr(),
+ c_int(m),
+ c_int(k),
+ c_int(n),
+ );
+ }
+}
+
+pub fn matmul_bf16_with_output(
+ a: &[u16],
+ b: &[u16],
+ output: &mut [u16],
+ m: i32,
+ k: i32,
+ n: i32,
+) {
+ unsafe {
+ ffi::cxx_utils::matmul_bf16_with_output(
+ a.as_ptr(),
+ b.as_ptr(),
+ output.as_mut_ptr(),
+ c_int(m),
+ c_int(k),
+ c_int(n),
+ );
+ }
+}
diff --git a/crates/matmul-benchmark/src/main.rs b/crates/matmul-benchmark/src/main.rs
new file mode 100644
index 000000000..c6c3c1d20
--- /dev/null
+++ b/crates/matmul-benchmark/src/main.rs
@@ -0,0 +1,1194 @@
+//! Matmul benchmark comparing MLX, UZU, and MPSGraph backends
+//!
+//! Run with: cargo run --release -p matmul-benchmark
+use std::{
+ collections::HashMap,
+ fmt,
+ fs::{self, File},
+ io::Read,
+ path::{Path, PathBuf},
+ time::Instant,
+};
+
+use half::{bf16, f16};
+use matmul_benchmark::{
+ benchmark_matmul_bf16 as mlx_bf16, benchmark_matmul_f16 as mlx_f16,
+ benchmark_matmul_f32 as mlx_f32, matmul_bf16_with_output,
+ matmul_f16_with_output, matmul_f32_with_output,
+};
+use metal::{Device, MTLResourceOptions};
+use mpsgraph::{
+ CommandBuffer, Device as MPSDevice, ExecutableExecutionDescriptor, Graph,
+ ShapedType, TensorData,
+};
+use objc2::rc::autoreleasepool;
+use serde::Deserialize;
+use uzu::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ kernel::{MatmulArguments, MatmulKernel},
+ },
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum DType {
+ F16,
+ F32,
+ BF16,
+}
+
+impl fmt::Display for DType {
+ fn fmt(
+ &self,
+ f: &mut fmt::Formatter<'_>,
+ ) -> fmt::Result {
+ match self {
+ DType::F16 => write!(f, " f16"),
+ DType::F32 => write!(f, " f32"),
+ DType::BF16 => write!(f, "bf16"),
+ }
+ }
+}
+
+impl DType {
+ fn to_uzu(self) -> DataType {
+ match self {
+ DType::F16 => DataType::F16,
+ DType::F32 => DataType::F32,
+ DType::BF16 => DataType::BF16,
+ }
+ }
+
+ fn to_mps(self) -> mpsgraph::DataType {
+ match self {
+ DType::F16 => mpsgraph::DataType::Float16,
+ DType::F32 => mpsgraph::DataType::Float32,
+ DType::BF16 => mpsgraph::DataType::BFloat16,
+ }
+ }
+
+ fn element_size(self) -> usize {
+ match self {
+ DType::F16 | DType::BF16 => 2,
+ DType::F32 => 4,
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+struct BenchmarkResult {
+ mlx_ms: f64,
+ uzu_ms: f64,
+ mps_ms: f64,
+}
+
+#[derive(Debug, Clone)]
+struct AccuracyResult {
+ max_abs_error: f32,
+ mean_abs_error: f32,
+ max_rel_error: f32,
+}
+
+#[derive(Debug, Clone)]
+struct BenchCase {
+ name: String,
+ m: usize,
+ k: usize,
+ n: usize,
+ dtype: DType,
+}
+
+impl BenchCase {
+ fn b_bytes(&self) -> usize {
+ self.n * self.k * self.dtype.element_size()
+ }
+}
+
+// ============================================================================
+// LLM SHAPES (derived from performance-benchmarks model.safetensors)
+// ============================================================================
+
+#[derive(Debug, Clone, Deserialize)]
+struct ModelConfigFile {
+ repo: String,
+ model_config: ModelConfigOuter,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct ModelConfigOuter {
+ model_config: ModelConfigInner,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct ModelConfigInner {
+ embedding_config: EmbeddingConfig,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(tag = "type")]
+enum EmbeddingConfig {
+ TiedEmbeddingConfig {
+ precision: String,
+ },
+ MLXQuantizedTiedEmbeddingConfig {
+ activation_precision: String,
+ },
+ #[serde(other)]
+ Unknown,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct SafetensorsTensorInfo {
+ dtype: String,
+ shape: Vec<usize>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct PerfBenchRun {
+ task: PerfBenchTask,
+ tokens_count_input: usize,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct PerfBenchTask {
+ repo_id: String,
+}
+
+fn workspace_root() -> PathBuf {
+ Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("../..")
+ .canonicalize()
+ .unwrap_or_else(|_| Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."))
+}
+
+fn model_code(model_dir_name: &str) -> String {
+ if model_dir_name.starts_with("Llama-3.2-1B") {
+ "L3.2-1B".to_string()
+ } else if model_dir_name.starts_with("Qwen3-0.6B") {
+ "Q3-0.6B".to_string()
+ } else if model_dir_name.starts_with("Qwen3-4B-AWQ") {
+ "Q3-4B-AWQ".to_string()
+ } else if model_dir_name.starts_with("Qwen3-4B-MLX-4bit") {
+ "Q3-4B-4b".to_string()
+ } else {
+ // Best-effort fallback: keep within table width (<= 9 chars)
+ model_dir_name.chars().take(9).collect()
+ }
+}
+
+fn dtype_from_precision_string(precision: &str) -> Option<DType> {
+ match precision {
+ "float16" => Some(DType::F16),
+ "bfloat16" => Some(DType::BF16),
+ "float32" => Some(DType::F32),
+ _ => None,
+ }
+}
+
+fn model_activation_dtype(cfg: &ModelConfigFile) -> Option<DType> {
+ match &cfg.model_config.model_config.embedding_config {
+ EmbeddingConfig::TiedEmbeddingConfig {
+ precision,
+ } => dtype_from_precision_string(precision),
+ EmbeddingConfig::MLXQuantizedTiedEmbeddingConfig {
+ activation_precision,
+ } => dtype_from_precision_string(activation_precision),
+ EmbeddingConfig::Unknown => None,
+ }
+}
+
+fn read_safetensors_header(
+ path: &Path
+) -> Result<HashMap<String, SafetensorsTensorInfo>, Box<dyn std::error::Error>>
+{
+ let mut file = File::open(path)?;
+ let mut header_len_bytes = [0u8; 8];
+ file.read_exact(&mut header_len_bytes)?;
+ let header_len = u64::from_le_bytes(header_len_bytes) as usize;
+
+ let mut header_bytes = vec![0u8; header_len];
+ file.read_exact(&mut header_bytes)?;
+
+ let raw: HashMap<String, serde_json::Value> =
+ serde_json::from_slice(&header_bytes)?;
+
+ raw.into_iter()
+ .filter(|(k, _)| k != "__metadata__")
+ .map(|(k, v)| {
+ let info: SafetensorsTensorInfo = serde_json::from_value(v)?;
+ Ok((k, info))
+ })
+ .collect()
+}
+
+fn weight_dims_nk(
+ header: &HashMap<String, SafetensorsTensorInfo>,
+ key: &str,
+) -> Option<(usize, usize)> {
+ let info = header.get(key)?;
+ let (n, packed_k) = match info.shape.as_slice() {
+ [n, k] => (*n, *k),
+ _ => return None,
+ };
+
+ let k = match info.dtype.as_str() {
+ // 4-bit packed (2 weights per byte)
+ "U8" => packed_k.checked_mul(2)?,
+ _ => packed_k,
+ };
+
+ Some((n, k))
+}
+
+fn collect_json_files(
+ dir: &Path,
+ out: &mut Vec<PathBuf>,
+) -> std::io::Result<()> {
+ for entry in fs::read_dir(dir)? {
+ let entry = entry?;
+ let path = entry.path();
+ if path.is_dir() {
+ collect_json_files(&path, out)?;
+ } else if path.extension().is_some_and(|ext| ext == "json") {
+ out.push(path);
+ }
+ }
+ Ok(())
+}
+
+fn prefill_tokens_from_perf_results(
+ json_files: &[PathBuf],
+ repo_id: &str,
+) -> Option<usize> {
+ for path in json_files {
+ let file = File::open(path).ok()?;
+ let runs: Vec<PerfBenchRun> = serde_json::from_reader(file).ok()?;
+ for run in runs {
+ if run.task.repo_id == repo_id {
+ return Some(run.tokens_count_input);
+ }
+ }
+ }
+ None
+}
+
+fn default_benchmark_cases() -> Vec<BenchCase> {
+ let shapes: &[(&str, usize, usize, usize)] = &[
+ ("GEMV 4K", 1, 4096, 4096),
+ ("GEMV 8K", 1, 8192, 8192),
+ ("Batch 4", 4, 4096, 4096),
+ ("Batch 16", 16, 4096, 4096),
+ ("Prefill 128", 128, 4096, 4096),
+ ("Prefill 512", 512, 4096, 4096),
+ ("Square 1K", 1024, 1024, 1024),
+ ("Square 2K", 2048, 2048, 2048),
+ ];
+
+ let dtypes = [DType::F16, DType::BF16, DType::F32];
+ shapes
+ .iter()
+ .flat_map(|(name, m, k, n)| {
+ dtypes.iter().map(|dtype| BenchCase {
+ name: (*name).to_string(),
+ m: *m,
+ k: *k,
+ n: *n,
+ dtype: *dtype,
+ })
+ })
+ .collect()
+}
+
+fn llm_benchmark_cases() -> Vec<BenchCase> {
+ match try_llm_benchmark_cases() {
+ Ok(cases) if !cases.is_empty() => cases,
+ _ => default_benchmark_cases(),
+ }
+}
+
+fn try_llm_benchmark_cases()
+-> Result<Vec<BenchCase>, Box<dyn std::error::Error>> {
+ let root = workspace_root();
+ let models_dir =
+ root.join("external/performance-benchmarks/workspace/models/0.1.7");
+ if !models_dir.is_dir() {
+ return Ok(Vec::new());
+ }
+
+ let results_dir =
+ root.join("external/performance-benchmarks/workspace/results");
+ let json_files = if results_dir.is_dir() {
+ let mut files = Vec::new();
+ collect_json_files(&results_dir, &mut files)?;
+ files
+ } else {
+ Vec::new()
+ };
+
+ let mut model_dirs: Vec<PathBuf> = fs::read_dir(&models_dir)?
+ .filter_map(|e| e.ok())
+ .map(|e| e.path())
+ .filter(|p| p.is_dir())
+ .collect();
+ model_dirs.sort();
+
+ let ops: [(&str, &str); 4] = [
+ ("QKV", "transformer.layers.0.mixer.qkv_projection.weights"),
+ ("O", "transformer.layers.0.mixer.out_projection.weights"),
+ ("MLPu", "transformer.layers.0.mlp.up_projection.weights"),
+ ("MLPd", "transformer.layers.0.mlp.down_projection.weights"),
+ ];
+
+ let mut cases = Vec::new();
+ for model_dir in model_dirs {
+ let dir_name =
+ model_dir.file_name().and_then(|s| s.to_str()).unwrap_or_default();
+ let code = model_code(dir_name);
+
+ let cfg: ModelConfigFile = {
+ let file = File::open(model_dir.join("config.json"))?;
+ serde_json::from_reader(file)?
+ };
+
+ let dtype = model_activation_dtype(&cfg).unwrap_or(DType::BF16);
+
+ let prefill_m =
+ prefill_tokens_from_perf_results(&json_files, &cfg.repo)
+ .unwrap_or(1024);
+ let ms = if prefill_m == 1 {
+ vec![1usize]
+ } else {
+ vec![1usize, prefill_m]
+ };
+
+ let header =
+ read_safetensors_header(&model_dir.join("model.safetensors"))?;
+
+ for (op_code, key) in ops {
+ let Some((n, k)) = weight_dims_nk(&header, key) else {
+ continue;
+ };
+
+ for m in &ms {
+ cases.push(BenchCase {
+ name: format!("{code} {op_code}"),
+ m: *m,
+ k,
+ n,
+ dtype,
+ });
+ }
+ }
+ }
+
+ Ok(cases)
+}
+
+fn compute_gflops(
+ m: usize,
+ k: usize,
+ n: usize,
+ time_ms: f64,
+) -> f64 {
+ let flops = 2.0 * (m as f64) * (k as f64) * (n as f64);
+ flops / (time_ms * 1e6)
+}
+
+/// Generate deterministic test data
+fn generate_test_data_f16(
+ m: usize,
+ k: usize,
+ n: usize,
+) -> (Vec<f16>, Vec<f16>) {
+ let a: Vec<f16> =
+ (0..(m * k)).map(|i| f16::from_f32(((i % 13) as f32) * 0.01)).collect();
+ let b: Vec<f16> = (0..(n * k))
+ .map(|i| f16::from_f32(((i % 17) as f32) * 0.02 - 0.1))
+ .collect();
+ (a, b)
+}
+
+fn generate_test_data_f32(
+ m: usize,
+ k: usize,
+ n: usize,
+) -> (Vec<f32>, Vec<f32>) {
+ let a: Vec<f32> = (0..(m * k)).map(|i| ((i % 13) as f32) * 0.01).collect();
+ let b: Vec<f32> =
+ (0..(n * k)).map(|i| ((i % 17) as f32) * 0.02 - 0.1).collect();
+ (a, b)
+}
+
+fn generate_test_data_bf16(
+ m: usize,
+ k: usize,
+ n: usize,
+) -> (Vec<bf16>, Vec<bf16>) {
+ let a: Vec<bf16> = (0..(m * k))
+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))
+ .collect();
+ let b: Vec<bf16> = (0..(n * k))
+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))
+ .collect();
+ (a, b)
+}
+
+fn benchmark_uzu(
+ ctx: &MTLContext,
+ dtype: DType,
+ m: usize,
+ k: usize,
+ n: usize,
+ warmup_iters: usize,
+ bench_iters: usize,
+) -> f64 {
+ let elem_size = dtype.element_size();
+ let a_bytes = m * k * elem_size;
+ let b_bytes = n * k * elem_size;
+ let d_bytes = m * n * elem_size;
+
+ let a_buf = ctx
+ .device
+ .new_buffer(a_bytes as u64, MTLResourceOptions::StorageModeShared);
+ let b_buf = ctx
+ .device
+ .new_buffer(b_bytes as u64, MTLResourceOptions::StorageModeShared);
+ let d_buf = ctx
+ .device
+ .new_buffer(d_bytes as u64, MTLResourceOptions::StorageModeShared);
+
+ // Initialize with test data
+ match dtype {
+ DType::F16 => {
+ let (a, b) = generate_test_data_f16(m, k, n);
+ unsafe {
+ std::ptr::copy_nonoverlapping(
+ a.as_ptr(),
+ a_buf.contents() as *mut f16,
+ a.len(),
+ );
+ std::ptr::copy_nonoverlapping(
+ b.as_ptr(),
+ b_buf.contents() as *mut f16,
+ b.len(),
+ );
+ }
+ },
+ DType::F32 => {
+ let (a, b) = generate_test_data_f32(m, k, n);
+ unsafe {
+ std::ptr::copy_nonoverlapping(
+ a.as_ptr(),
+ a_buf.contents() as *mut f32,
+ a.len(),
+ );
+ std::ptr::copy_nonoverlapping(
+ b.as_ptr(),
+ b_buf.contents() as *mut f32,
+ b.len(),
+ );
+ }
+ },
+ DType::BF16 => {
+ let (a, b) = generate_test_data_bf16(m, k, n);
+ unsafe {
+ std::ptr::copy_nonoverlapping(
+ a.as_ptr(),
+ a_buf.contents() as *mut bf16,
+ a.len(),
+ );
+ std::ptr::copy_nonoverlapping(
+ b.as_ptr(),
+ b_buf.contents() as *mut bf16,
+ b.len(),
+ );
+ }
+ },
+ }
+
+ let mut kernel = MatmulKernel::new(ctx, dtype.to_uzu(), false, true)
+ .expect("kernel new");
+
+ for _ in 0..warmup_iters {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ c: None,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ let start = Instant::now();
+ for _ in 0..bench_iters {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ c: None,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+ let elapsed = start.elapsed();
+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64
+}
+
+fn benchmark_mpsgraph(
+ device: &metal::Device,
+ dtype: DType,
+ m: usize,
+ k: usize,
+ n: usize,
+ warmup_iters: usize,
+ bench_iters: usize,
+) -> f64 {
+ autoreleasepool(|_| {
+ let graph = Graph::new();
+ let mps_dtype = dtype.to_mps();
+
+ let a_shape = [m as isize, k as isize];
+ let b_shape = [k as isize, n as isize];
+
+ let a_placeholder =
+ graph.placeholder(Some(&a_shape), mps_dtype, Some("a"));
+ let b_placeholder =
+ graph.placeholder(Some(&b_shape), mps_dtype, Some("b"));
+
+ let result =
+ graph.matrix_multiplication(&a_placeholder, &b_placeholder, None);
+
+ let a_shaped_type =
+ ShapedType::new_with_shape_data_type(Some(&a_shape), mps_dtype);
+ let b_shaped_type =
+ ShapedType::new_with_shape_data_type(Some(&b_shape), mps_dtype);
+
+ let feeds = HashMap::from([
+ (&*a_placeholder, &*a_shaped_type),
+ (&*b_placeholder, &*b_shaped_type),
+ ]);
+
+ let mps_device = MPSDevice::with_device(device);
+ let executable =
+ graph.compile(&mps_device, &feeds, &[&result], None, None);
+
+ let execution_descriptor = ExecutableExecutionDescriptor::new();
+
+ let elem_size = dtype.element_size();
+ let a_buf = device.new_buffer(
+ (m * k * elem_size) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = device.new_buffer(
+ (k * n * elem_size) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let output_buf = device.new_buffer(
+ (m * n * elem_size) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let command_queue = device.new_command_queue();
+
+ let a_tensor_data =
+ TensorData::new_with_mtl_buffer(&a_buf, &[m, k], mps_dtype, None);
+ let b_tensor_data =
+ TensorData::new_with_mtl_buffer(&b_buf, &[k, n], mps_dtype, None);
+ let output_tensor_data = TensorData::new_with_mtl_buffer(
+ &output_buf,
+ &[m, n],
+ mps_dtype,
+ None,
+ );
+
+ let inputs: &[&TensorData] = &[&a_tensor_data, &b_tensor_data];
+ let outputs: &[&TensorData] = &[&output_tensor_data];
+
+ execution_descriptor.set_enable_commit_and_continue(true);
+ for _ in 0..warmup_iters {
+ let command_buffer =
+ CommandBuffer::from_command_queue(&command_queue);
+ let root_cb = command_buffer.root_command_buffer().to_owned();
+ let _ = executable.encode_to_command_buffer(
+ &command_buffer,
+ inputs,
+ Some(outputs),
+ Some(&execution_descriptor),
+ );
+ command_buffer.commit_and_continue();
+ root_cb.wait_until_completed();
+ }
+
+ let start = Instant::now();
+ for _ in 0..bench_iters {
+ let command_buffer =
+ CommandBuffer::from_command_queue(&command_queue);
+ let root_cb = command_buffer.root_command_buffer().to_owned();
+ let _ = executable.encode_to_command_buffer(
+ &command_buffer,
+ inputs,
+ Some(outputs),
+ Some(&execution_descriptor),
+ );
+ command_buffer.commit_and_continue();
+ root_cb.wait_until_completed();
+ }
+ let elapsed = start.elapsed();
+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64
+ })
+}
+
+fn benchmark_mlx(
+ dtype: DType,
+ m: usize,
+ k: usize,
+ n: usize,
+ warmup: i32,
+ iters: i32,
+) -> f64 {
+ match dtype {
+ DType::F16 => {
+ mlx_f16(m as i32, n as i32, k as i32, warmup, iters).avg_time_ms
+ },
+ DType::F32 => {
+ mlx_f32(m as i32, n as i32, k as i32, warmup, iters).avg_time_ms
+ },
+ DType::BF16 => {
+ mlx_bf16(m as i32, n as i32, k as i32, warmup, iters).avg_time_ms
+ },
+ }
+}
+
+fn run_benchmark(
+ ctx: &MTLContext,
+ dtype: DType,
+ m: usize,
+ k: usize,
+ n: usize,
+ warmup: usize,
+ iters: usize,
+) -> BenchmarkResult {
+ let mlx_ms = benchmark_mlx(dtype, m, k, n, warmup as i32, iters as i32);
+ let uzu_ms = benchmark_uzu(ctx, dtype, m, k, n, warmup, iters);
+ let mps_ms = benchmark_mpsgraph(&ctx.device, dtype, m, k, n, warmup, iters);
+
+ BenchmarkResult {
+ mlx_ms,
+ uzu_ms,
+ mps_ms,
+ }
+}
+
+fn format_speedup(
+ base: f64,
+ other: f64,
+) -> String {
+ let ratio = other / base;
+ format!("{:.2}x", ratio)
+}
+
+/// Run accuracy test for a given shape and dtype
+fn run_accuracy_test(
+ ctx: &MTLContext,
+ dtype: DType,
+ m: usize,
+ k: usize,
+ n: usize,
+) -> AccuracyResult {
+ match dtype {
+ DType::F16 => run_accuracy_test_f16(ctx, m, k, n),
+ DType::F32 => run_accuracy_test_f32(ctx, m, k, n),
+ DType::BF16 => run_accuracy_test_bf16(ctx, m, k, n),
+ }
+}
+
+fn run_accuracy_test_f16(
+ ctx: &MTLContext,
+ m: usize,
+ k: usize,
+ n: usize,
+) -> AccuracyResult {
+ let (a, b) = generate_test_data_f16(m, k, n);
+
+ // MLX
+ let a_u16: Vec<u16> = a.iter().map(|x| x.to_bits()).collect();
+ let b_u16: Vec<u16> = b.iter().map(|x| x.to_bits()).collect();
+ let mut mlx_out_u16 = vec![0u16; m * n];
+ matmul_f16_with_output(
+ &a_u16,
+ &b_u16,
+ &mut mlx_out_u16,
+ m as i32,
+ k as i32,
+ n as i32,
+ );
+ let mlx_out: Vec<f16> =
+ mlx_out_u16.iter().map(|&x| f16::from_bits(x)).collect();
+
+ // UZU
+ let a_buf = ctx.device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * 2) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * 2) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx
+ .device
+ .new_buffer((m * n * 2) as u64, MTLResourceOptions::StorageModeShared);
+
+ let mut kernel =
+ MatmulKernel::new(ctx, DataType::F16, false, true).unwrap();
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ c: None,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+
+ let uzu_out: Vec<f16> = unsafe {
+ std::slice::from_raw_parts(d_buf.contents() as *const f16, m * n)
+ .to_vec()
+ };
+
+ compare_outputs_f16(&mlx_out, &uzu_out)
+}
+
+fn run_accuracy_test_f32(
+ ctx: &MTLContext,
+ m: usize,
+ k: usize,
+ n: usize,
+) -> AccuracyResult {
+ let (a, b) = generate_test_data_f32(m, k, n);
+
+ // MLX
+ let mut mlx_out = vec![0f32; m * n];
+ matmul_f32_with_output(&a, &b, &mut mlx_out, m as i32, k as i32, n as i32);
+
+ // UZU
+ let a_buf = ctx.device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * 4) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * 4) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx
+ .device
+ .new_buffer((m * n * 4) as u64, MTLResourceOptions::StorageModeShared);
+
+ let mut kernel =
+ MatmulKernel::new(ctx, DataType::F32, false, true).unwrap();
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ c: None,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+
+ let uzu_out: Vec<f32> = unsafe {
+ std::slice::from_raw_parts(d_buf.contents() as *const f32, m * n)
+ .to_vec()
+ };
+
+ compare_outputs_f32(&mlx_out, &uzu_out)
+}
+
+fn run_accuracy_test_bf16(
+ ctx: &MTLContext,
+ m: usize,
+ k: usize,
+ n: usize,
+) -> AccuracyResult {
+ let (a, b) = generate_test_data_bf16(m, k, n);
+
+ // MLX
+ let a_u16: Vec<u16> = a.iter().map(|x| x.to_bits()).collect();
+ let b_u16: Vec<u16> = b.iter().map(|x| x.to_bits()).collect();
+ let mut mlx_out_u16 = vec![0u16; m * n];
+ matmul_bf16_with_output(
+ &a_u16,
+ &b_u16,
+ &mut mlx_out_u16,
+ m as i32,
+ k as i32,
+ n as i32,
+ );
+ let mlx_out: Vec<bf16> =
+ mlx_out_u16.iter().map(|&x| bf16::from_bits(x)).collect();
+
+ // UZU
+ let a_buf = ctx.device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * 2) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * 2) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx
+ .device
+ .new_buffer((m * n * 2) as u64, MTLResourceOptions::StorageModeShared);
+
+ let mut kernel =
+ MatmulKernel::new(ctx, DataType::BF16, false, true).unwrap();
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ c: None,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+
+ let uzu_out: Vec<bf16> = unsafe {
+ std::slice::from_raw_parts(d_buf.contents() as *const bf16, m * n)
+ .to_vec()
+ };
+
+ compare_outputs_bf16(&mlx_out, &uzu_out)
+}
+
+fn compare_outputs_f16(
+ reference: &[f16],
+ test: &[f16],
+) -> AccuracyResult {
+ let mut max_abs_error: f32 = 0.0;
+ let mut sum_abs_error: f32 = 0.0;
+ let mut max_rel_error: f32 = 0.0;
+
+ for (r, t) in reference.iter().zip(test.iter()) {
+ let r_f32 = r.to_f32();
+ let t_f32 = t.to_f32();
+ let abs_error = (r_f32 - t_f32).abs();
+ sum_abs_error += abs_error;
+ max_abs_error = max_abs_error.max(abs_error);
+ if r_f32.abs() > 1e-6 {
+ max_rel_error = max_rel_error.max(abs_error / r_f32.abs());
+ }
+ }
+
+ AccuracyResult {
+ max_abs_error,
+ mean_abs_error: sum_abs_error / reference.len() as f32,
+ max_rel_error,
+ }
+}
+
+fn compare_outputs_f32(
+ reference: &[f32],
+ test: &[f32],
+) -> AccuracyResult {
+ let mut max_abs_error: f32 = 0.0;
+ let mut sum_abs_error: f32 = 0.0;
+ let mut max_rel_error: f32 = 0.0;
+
+ for (r, t) in reference.iter().zip(test.iter()) {
+ let abs_error = (r - t).abs();
+ sum_abs_error += abs_error;
+ max_abs_error = max_abs_error.max(abs_error);
+ if r.abs() > 1e-6 {
+ max_rel_error = max_rel_error.max(abs_error / r.abs());
+ }
+ }
+
+ AccuracyResult {
+ max_abs_error,
+ mean_abs_error: sum_abs_error / reference.len() as f32,
+ max_rel_error,
+ }
+}
+
+fn compare_outputs_bf16(
+ reference: &[bf16],
+ test: &[bf16],
+) -> AccuracyResult {
+ let mut max_abs_error: f32 = 0.0;
+ let mut sum_abs_error: f32 = 0.0;
+ let mut max_rel_error: f32 = 0.0;
+
+ for (r, t) in reference.iter().zip(test.iter()) {
+ let r_f32 = r.to_f32();
+ let t_f32 = t.to_f32();
+ let abs_error = (r_f32 - t_f32).abs();
+ sum_abs_error += abs_error;
+ max_abs_error = max_abs_error.max(abs_error);
+ if r_f32.abs() > 1e-6 {
+ max_rel_error = max_rel_error.max(abs_error / r_f32.abs());
+ }
+ }
+
+ AccuracyResult {
+ max_abs_error,
+ mean_abs_error: sum_abs_error / reference.len() as f32,
+ max_rel_error,
+ }
+}
+
+fn main() {
+ let device = Device::system_default().expect("No Metal device found");
+ let command_queue = device.new_command_queue();
+ let ctx = MTLContext::new(device.clone(), command_queue)
+ .expect("Failed to create MTLContext");
+
+ let cases = llm_benchmark_cases();
+
+ // =========================================================================
+ // ACCURACY TESTS
+ // =========================================================================
+ println!();
+ println!(
+ "╔═══════════════════════════════════════════════════════════════════════════════╗"
+ );
+ println!(
+ "║ ACCURACY TESTS: UZU vs MLX ║"
+ );
+ println!(
+ "╠═══════════════════════════════════════════════════════════════════════════════╣"
+ );
+ println!("║ Device: {:<69}║", device.name());
+ println!(
+ "╚═══════════════════════════════════════════════════════════════════════════════╝"
+ );
+ println!();
+
+ println!(
+ "┌────────────────┬───────┬───────┬───────┬──────┬───────────┬───────────┬────────────┐"
+ );
+ println!(
+ "│ Shape │ M │ K │ N │ Type │ Max Abs │ Mean Abs │ Max Rel │"
+ );
+ println!(
+ "├────────────────┼───────┼───────┼───────┼──────┼───────────┼───────────┼────────────┤"
+ );
+
+ let mut all_passed = true;
+ let tolerance = 0.01;
+
+ // Keep accuracy checks reasonably fast: only small(ish) B buffers and M=1
+ let accuracy_cases: Vec<&BenchCase> = cases
+ .iter()
+ .filter(|c| c.m == 1 && c.b_bytes() <= 32 * 1024 * 1024)
+ .collect();
+
+ for case in &accuracy_cases {
+ let result =
+ run_accuracy_test(&ctx, case.dtype, case.m, case.k, case.n);
+ let passed = result.max_rel_error < tolerance;
+ if !passed {
+ all_passed = false;
+ }
+
+ println!(
+ "│ {:<14} │ {:>5} │ {:>5} │ {:>5} │ {:>4} │ {:>9.2e} │ {:>9.2e} │ {:>9.2e} │",
+ case.name,
+ case.m,
+ case.k,
+ case.n,
+ case.dtype,
+ result.max_abs_error,
+ result.mean_abs_error,
+ result.max_rel_error
+ );
+ }
+
+ println!(
+ "└────────────────┴───────┴───────┴───────┴──────┴───────────┴───────────┴────────────┘"
+ );
+ println!();
+
+ if all_passed {
+ println!(
+ "✓ All accuracy tests PASSED (max relative error < {:.0}%)",
+ tolerance * 100.0
+ );
+ } else {
+ println!(
+ "✗ Some accuracy tests FAILED (max relative error >= {:.0}%)",
+ tolerance * 100.0
+ );
+ }
+ println!();
+
+ // =========================================================================
+ // PERFORMANCE BENCHMARKS
+ // =========================================================================
+ println!();
+ println!(
+ "╔══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╗"
+ );
+ println!(
+ "║ MATMUL BENCHMARK: MLX vs UZU vs MPSGraph ║"
+ );
+ println!(
+ "╠══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╣"
+ );
+ println!("║ Device: {:<103}║", device.name());
+ println!(
+ "╚══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝"
+ );
+ println!();
+
+ let warmup = 10;
+ let iters = 100;
+
+ println!(
+ "┌────────────────┬───────┬───────┬───────┬──────┬────────────────────────────┬────────────────────────────┬─────────────────────┐"
+ );
+ println!(
+ "│ │ │ │ │ │ Time (ms) │ GFLOPS │ UZU Speedup vs │"
+ );
+ println!(
+ "│ Shape │ M │ K │ N │ Type ├────────┬────────┬──────────┼────────┬────────┬──────────┼──────────┬──────────┤"
+ );
+ println!(
+ "│ │ │ │ │ │ MLX │ UZU │ MPS │ MLX │ UZU │ MPS │ MLX │ MPS │"
+ );
+ println!(
+ "├────────────────┼───────┼───────┼───────┼──────┼────────┼────────┼──────────┼────────┼────────┼──────────┼──────────┼──────────┤"
+ );
+
+ for case in &cases {
+ let result = run_benchmark(
+ &ctx, case.dtype, case.m, case.k, case.n, warmup, iters,
+ );
+
+ let mlx_gflops = compute_gflops(case.m, case.k, case.n, result.mlx_ms);
+ let uzu_gflops = compute_gflops(case.m, case.k, case.n, result.uzu_ms);
+ let mps_gflops = compute_gflops(case.m, case.k, case.n, result.mps_ms);
+
+ let uzu_vs_mlx = format_speedup(result.uzu_ms, result.mlx_ms);
+ let uzu_vs_mps = format_speedup(result.uzu_ms, result.mps_ms);
+
+ println!(
+ "│ {:<14} │ {:>5} │ {:>5} │ {:>5} │ {:>4} │ {:>6.3} │ {:>6.3} │ {:>8.3} │ {:>6.0} │ {:>6.0} │ {:>8.0} │ {:>8} │ {:>8} │",
+ case.name,
+ case.m,
+ case.k,
+ case.n,
+ case.dtype,
+ result.mlx_ms,
+ result.uzu_ms,
+ result.mps_ms,
+ mlx_gflops,
+ uzu_gflops,
+ mps_gflops,
+ uzu_vs_mlx,
+ uzu_vs_mps
+ );
+ }
+
+ println!(
+ "└────────────────┴───────┴───────┴───────┴──────┴────────┴────────┴──────────┴────────┴────────┴──────────┴──────────┴──────────┘"
+ );
+ println!();
+ println!("Legend:");
+ println!(" - UZU Speedup > 1.0x means UZU is faster");
+ println!(" - UZU Speedup < 1.0x means the other backend is faster");
+ println!();
+}