A high-performance inference engine for AI models
dsl improvements (#81)
uuuvnb0a1f7040880
diff --git a/Cargo.lock b/Cargo.lockindex d5f40d829..d27189172 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -136,9 +136,9 @@ dependencies = [[[package]]name = "async-trait"-version = "0.1.88"+version = "0.1.89"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"+checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"dependencies = ["proc-macro2","quote",@@ -4188,6 +4188,7 @@ name = "uzu"version = "0.1.7"dependencies = ["anyhow",+ "async-trait","bindgen","bitflags 2.9.4","blake3",diff --git a/Cargo.toml b/Cargo.tomlindex 69e611aca..7152d1ec5 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -45,6 +45,7 @@ schemars = { version = "1.0.5", features = ["derive"] }is_close = "0.1.3"serde_json5 = "0.2.1"anyhow = "1.0.100"+async-trait = "0.1.89"bindgen = "0.72.1"walkdir = "2.5.0"depfile = "0.1.1"@@ -70,5 +71,9 @@ cxx = "1.0.187"cmake = "0.1.57"autocxx-build = "0.30.0"+# optimize the build script in debug builds+[profile.dev.build-override]+opt-level = 3+[profile.release]lto = "thin"diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 7f22f6758..e68a2ee77 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -38,6 +38,7 @@ mpsgraph.workspace = true[build-dependencies]anyhow.workspace = true+async-trait.workspace = truebindgen.workspace = truewalkdir.workspace = truedepfile.workspace = true@@ -55,11 +56,13 @@ prettyplease.workspace = trueitertools.workspace = true[features]-build-debug = []-build-clean = []-tracing = []+default = ["metal"]++metal = []metal-nax = []+tracing = []+[[test]]name = "moe"path = "tests/moe/mod.rs"diff --git a/crates/uzu/build.rs b/crates/uzu/build.rsdeleted file mode 100644index 8f6ce42b6..000000000--- a/crates/uzu/build.rs+++ /dev/null@@ -1,572 +0,0 @@-use std::{- collections::HashSet,- env, fs,- io::Write,- path::{Path, PathBuf},- process::Command,- time::SystemTime,-};--fn main() {- // Generate Rust bindings from shared Metal/C headers- generate_metal_bindings();-- if cfg!(feature = "metal-shaders") {- println!("cargo:rerun-if-env-changed=MY_API_LEVEL");- compile_metal_shaders();- } else {- write_empty_metallib();- }-}--/// Generate Rust bindings from shared C headers using bindgen.-/// These headers are also used by Metal shaders, ensuring type consistency.-fn generate_metal_bindings() {- let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());- let matmul_common_dir =- manifest_dir.join("src/backends/metal/kernel/matmul/common");- let types_header = matmul_common_dir.join("shared_types.h");-- // Only regenerate if header changed- println!("cargo:rerun-if-changed={}", types_header.display());-- let bindings = bindgen::Builder::default()- .header(types_header.to_string_lossy())- // Generate for C mode (no Metal-specific stuff)- .clang_arg("-x")- .clang_arg("c")- // Allow uppercase field names to match the C header without warnings- .raw_line("#![allow(non_snake_case)]")- // Derive common traits- .derive_default(true)- .derive_copy(true)- // Only generate types we care about- .allowlist_type("GEMMParams")- .allowlist_type("GEMMAddMMParams")- .allowlist_type("GEMMSpiltKParams")- .allowlist_type("GEMMSpiltKMlpFusedParams")- // Use core types- .use_core()- .generate()- .expect("Failed to generate bindings from shared_types.h");-- // Write bindings to the source directory so it can be checked in- let bindings_path = matmul_common_dir.join("shared_types.rs");- bindings- .write_to_file(&bindings_path)- .expect("Failed to write shared_types.rs");-- // ---------------------------------------------------------------------- // Gemm attention shared types- // ---------------------------------------------------------------------- let attention_dir =- manifest_dir.join("src/backends/metal/kernel/attention");- let attn_types_header = attention_dir.join("gemm_types.h");-- println!("cargo:rerun-if-changed={}", attn_types_header.display());-- let attn_bindings = bindgen::Builder::default()- .header(attn_types_header.to_string_lossy())- // Generate for C mode (no Metal-specific stuff)- .clang_arg("-x")- .clang_arg("c")- // Derive common traits- .derive_default(true)- .derive_copy(true)- // Only generate types we care about- .allowlist_type("AttnParams")- .allowlist_type("AttnMaskParams")- // Use core types- .use_core()- .generate()- .expect("Failed to generate bindings from gemm_types.h");-- let attn_bindings_path = attention_dir.join("gemm_types.rs");- attn_bindings- .write_to_file(&attn_bindings_path)- .expect("Failed to write gemm_types.rs");-}--fn compile_metal_shaders() {- // Determine target platform (only proceed on Apple platforms)- let target = env::var("TARGET").unwrap_or_default();- let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();- let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default();-- // Choose appropriate SDK: macosx, iphoneos, iphonesimulator, or maccatalyst- let sdk = if target_os == "ios" {- if target.contains("macabi") || target_env == "macabi" {- "maccatalyst"- } else if target.contains("ios")- && (target.contains("86_64") || target_env == "sim")- {- // Use iPhoneSimulator SDK for simulator targets (x86_64 or explicit 'sim')- "iphonesimulator"- } else {- "iphoneos"- }- } else if target_os == "macos" {- "macosx"- } else {- println!(- "cargo:warning=Not an Apple platform, skipping Metal shader compilation"- );- return;- };-- println!("cargo:info=Compiling Metal shaders for SDK: {}", sdk);-- // Re-run build script if the host OS version changes (affects Metal feature availability)- if cfg!(target_os = "macos") {- println!(- "cargo:rerun-if-changed=/System/Library/CoreServices/SystemVersion.plist"- );- }-- // Gather all .metal files in the project (src directory and subdirs)- let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());- let src_dir = manifest_dir.join("src");- let metal_files = find_metal_files(&src_dir);- if metal_files.is_empty() {- println!("cargo:warning=No .metal files found; nothing to compile");- return;- }- println!("cargo:info=Found {} Metal shader files", metal_files.len());-- // Prepare output directory for metallib- let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());- let metallib_dir = out_dir.join("metallib");- fs::create_dir_all(&metallib_dir).unwrap();-- // Require xcrun (should be present on macOS with Xcode)- if !Command::new("xcrun")- .arg("--version")- .status()- .map(|s| s.success())- .unwrap_or(false)- {- panic!(- "xcrun not found. Please install Xcode command-line tools to compile Metal shaders."- );- }-- // Platform-specific metallib path (separate for macos vs ios for caching)- let platform_suffix = if sdk == "macosx" || sdk == "maccatalyst" {- "macos"- } else {- "ios"- };- let metallib_path =- metallib_dir.join(format!("shaders_{}.metallib", platform_suffix));-- let nax_enabled = should_enable_nax(&target_os, sdk);- if nax_enabled {- println!("cargo:info=Metal NAX kernels enabled (Metal 4, OS 26+)");- } else {- println!("cargo:info=Metal NAX kernels disabled");- }-- let metal_files_to_compile: Vec<PathBuf> = if nax_enabled {- metal_files.clone()- } else {- metal_files.iter().filter(|p| !is_nax_kernel(p)).cloned().collect()- };-- // Decide whether to compile (if metallib missing or any source changed)- let should_compile = needs_compilation(- &metal_files_to_compile,- &metallib_path,- &target,- nax_enabled,- );- if should_compile {- compile_metal_files(- &metal_files_to_compile,- &metallib_dir,- sdk,- &src_dir,- nax_enabled,- );- // Record the target triple for which we built the metallib- let _ = fs::write(metallib_path.with_extension("target"), &target);- // Record whether we built with NAX kernels enabled- let _ = fs::write(- metallib_path.with_extension("nax"),- if nax_enabled {- "1"- } else {- "0"- },- );- } else {- println!(- "cargo:info=Metal shaders are up-to-date; skipping compilation"- );- }-- if !metallib_path.exists() {- panic!("Metal library not found at {}", metallib_path.display());- }-- // Generate a tiny Rust source file that uses include_bytes! to embed the- // compiled metallib. This avoids generating a massive Rust file containing- // a hex-encoded byte array (which is slow to write/compile).- write_metallib_include(platform_suffix, &out_dir.join("metal_lib.rs"));-- // Tell Cargo when to re-run this build script: all shader files, plus build.rs and any FFI headers- let mut sorted_files: Vec<&PathBuf> = metal_files.iter().collect();- sorted_files.sort();- for file in sorted_files {- println!("cargo:rerun-if-changed={}", file.display());- }- println!("cargo:rerun-if-changed=build.rs");-}--fn is_nax_kernel(path: &Path) -> bool {- path.file_name()- .and_then(|s| s.to_str())- .map(|s| s.contains("_nax"))- .unwrap_or(false)-}--const NAX_MIN_OS_MAJOR: u32 = 26;-const NAX_MIN_SDK_VERSION: (u32, u32) = (26, 2);--fn host_macos_major_version() -> Option<u32> {- if !cfg!(target_os = "macos") {- return None;- }- let output =- Command::new("sw_vers").arg("-productVersion").output().ok()?;- if !output.status.success() {- return None;- }- let version = String::from_utf8_lossy(&output.stdout);- let major = version.trim().split('.').next()?.parse::<u32>().ok()?;- Some(major)-}--fn macos_sdk_version(sdk: &str) -> Option<(u32, u32)> {- let output = Command::new("xcrun")- .args(["--sdk", sdk, "--show-sdk-version"])- .output()- .ok()?;- if !output.status.success() {- return None;- }- let version = String::from_utf8_lossy(&output.stdout);- let version = version.trim();- let mut parts = version.split('.');- let major = parts.next()?.parse::<u32>().ok()?;- let minor = parts.next().and_then(|s| s.parse::<u32>().ok()).unwrap_or(0);- Some((major, minor))-}--fn should_enable_nax(- target_os: &str,- sdk: &str,-) -> bool {- if target_os != "macos" && target_os != "ios" {- return false;- }- // Conservative: don't enable NAX for simulator builds- if sdk == "iphonesimulator" {- return false;- }- // Check host OS version- let os_ok = host_macos_major_version()- .is_some_and(|major| major >= NAX_MIN_OS_MAJOR);- if !os_ok {- return false;- }- // Check SDK version (NAX APIs require SDK >= 26.2)- macos_sdk_version(sdk)- .is_some_and(|(major, minor)| (major, minor) >= NAX_MIN_SDK_VERSION)-}--// Recursively find all .metal files in the given directory-fn find_metal_files(dir: &Path) -> Vec<PathBuf> {- let mut files = Vec::new();- if let Ok(entries) = fs::read_dir(dir) {- for entry in entries.flatten() {- let path = entry.path();- if path.is_dir() {- files.extend(find_metal_files(&path));- } else if path.extension().and_then(|s| s.to_str()) == Some("metal")- {- files.push(path);- }- }- }- files-}--// Check timestamps and target to determine if recompilation is needed-fn needs_compilation(- metal_files: &[PathBuf],- metallib_path: &Path,- target: &str,- nax_enabled: bool,-) -> bool {- // If output metallib doesn't exist, need to compile- if !metallib_path.exists() {- return true;- }- // If the previously built target differs (target changed, e.g. arch or OS)- if let Ok(prev_target) =- fs::read_to_string(metallib_path.with_extension("target"))- {- if prev_target != target {- println!(- "cargo:warning=Target changed from {} to {}, recompiling shaders",- prev_target, target- );- return true;- }- }- // If the previous NAX mode differs, force a rebuild (affects metallib compatibility)- let desired_nax = if nax_enabled {- "1"- } else {- "0"- };- match fs::read_to_string(metallib_path.with_extension("nax")) {- Ok(prev_nax) if prev_nax.trim() == desired_nax => {},- _ => return true,- }- // If any source file is newer than the metallib, recompile- let metallib_mtime = fs::metadata(metallib_path)- .and_then(|m| m.modified())- .unwrap_or(SystemTime::UNIX_EPOCH);- for metal_file in metal_files {- if let Ok(src_mtime) =- fs::metadata(metal_file).and_then(|m| m.modified())- {- if src_mtime > metallib_mtime {- return true;- }- } else {- // If any file's metadata can't be read, assume we should rebuild- return true;- }- }- false-}--fn compile_metal_files(- metal_files: &[PathBuf],- out_dir: &Path,- sdk: &str,- src_dir: &Path,- nax_enabled: bool,-) {- // Create temp directory for .air files- let air_dir = out_dir.join("air");- fs::create_dir_all(&air_dir).unwrap();- // Keep clang's module cache inside the build output to avoid writing to- // system-wide cache directories that may not be writable.- let module_cache_dir = out_dir.join("clang-module-cache");- fs::create_dir_all(&module_cache_dir).unwrap();-- // Collect include directories (parents of each metal file + src dir)- let mut include_dirs = HashSet::new();- include_dirs.insert(src_dir.to_path_buf());-- // Add kernel root and common/matmul subdirectories to satisfy kernel includes- let kernel_dir = src_dir.join("backends/metal/kernel");- if kernel_dir.exists() {- include_dirs.insert(kernel_dir.clone());- let common_dir = kernel_dir.join("common");- if common_dir.exists() {- include_dirs.insert(common_dir);- }-- let matmul_dir = kernel_dir.join("matmul");- if matmul_dir.exists() {- include_dirs.insert(matmul_dir.clone());-- let matmul_common_dir = matmul_dir.join("common");- if matmul_common_dir.exists() {- include_dirs.insert(matmul_common_dir);- }-- let gemv_dir = matmul_dir.join("gemv");- if gemv_dir.exists() {- include_dirs.insert(gemv_dir);- }-- let steel_dir = matmul_dir.join("steel");- if steel_dir.exists() {- include_dirs.insert(steel_dir.clone());-- let steel_gemm_dir = steel_dir.join("gemm");- if steel_gemm_dir.exists() {- include_dirs.insert(steel_gemm_dir.clone());- let steel_gemm_kernels_dir = steel_gemm_dir.join("kernels");- if steel_gemm_kernels_dir.exists() {- include_dirs.insert(steel_gemm_kernels_dir);- }- }-- let steel_utils_dir = steel_dir.join("utils");- if steel_utils_dir.exists() {- include_dirs.insert(steel_utils_dir);- }- }- }- }-- for metal_file in metal_files {- if let Some(parent) = metal_file.parent() {- include_dirs.insert(parent.to_path_buf());- if let Some(grand) = parent.parent() {- include_dirs.insert(grand.to_path_buf());- }- }- }-- // Optimization level: Metal supports up to -O2- let opt = env::var("OPT_LEVEL").unwrap_or_else(|_| "0".into());- let metal_opt_flag = match opt.as_str() {- "0" => "-O1", // NOTE TODO FIXME WORKAROUND: matmul kernels compiled with -O0 are broken and require a reboot to unfreeze the os- "1" => "-O1",- _ => "-O2", // treat levels 2,3,s,z as O2 for metal- };-- let metal_std = if nax_enabled {- "metal4.0"- } else {- "metal3.1"- };-- // Platform-specific min version flags.- // - Metal 3.1 requires macOS 14 / iOS 17- // - NAX kernels require macOS/iOS 26+ (Metal 4)- let platform_flags = if nax_enabled {- if sdk == "macosx" || sdk == "maccatalyst" {- vec!["-mmacosx-version-min=26.0"]- } else {- vec!["-mios-version-min=26.0"]- }- } else if sdk == "macosx" || sdk == "maccatalyst" {- vec!["-mmacosx-version-min=14.0"]- } else {- vec!["-mios-version-min=17.0"]- };-- // Compile each .metal source to an intermediate .air file- let mut air_files = Vec::new();- for metal_file in metal_files {- let name = metal_file.file_stem().unwrap().to_str().unwrap();- // Name .air with platform prefix to avoid collisions- let prefix = if sdk == "macosx" || sdk == "maccatalyst" {- "macos_"- } else {- "ios_"- };- let air_path = air_dir.join(format!("{}{}.air", prefix, name));- air_files.push(air_path.clone());-- // Invoke Metal compiler- let mut cmd = Command::new("xcrun");- cmd.args(&["-sdk", sdk, "metal", metal_opt_flag]);- cmd.arg(format!("-std={}", metal_std));- cmd.arg(format!(- "-fmodules-cache-path={}",- module_cache_dir.to_string_lossy()- ));- for flag in &platform_flags {- cmd.arg(flag);- }- for inc in &include_dirs {- cmd.args(&["-I", inc.to_str().unwrap()]);- }- cmd.env("CLANG_MODULE_CACHE_PATH", &module_cache_dir);- if opt == "0" {- cmd.arg("-gline-tables-only");- if sdk == "macosx" {- cmd.args(&["-frecord-sources", "-fpreserve-invariance"]);- }- }- cmd.args(&[- "-c",- metal_file.to_str().unwrap(),- "-o",- air_path.to_str().unwrap(),- ]);-- println!(- "cargo:info=Running Metal compiler: {}",- cmd.get_args()- .map(|a| a.to_string_lossy())- .collect::<Vec<_>>()- .join(" ")- );- if !cmd.status().expect("Failed to run metal compiler").success() {- panic!(- "Metal shader compilation failed for {}",- metal_file.display()- );- }- }-- // Link all .air files into a single .metallib library- let metallib_path =- out_dir.join(if sdk == "macosx" || sdk == "maccatalyst" {- "shaders_macos.metallib"- } else {- "shaders_ios.metallib"- });- let mut lib_cmd = Command::new("xcrun");- lib_cmd.args(&["-sdk", sdk, "metallib"]);- for air in &air_files {- lib_cmd.arg(air);- }- lib_cmd.args(&["-o", metallib_path.to_str().unwrap()]);- println!(- "cargo:info=Linking Metal libraries: {}",- lib_cmd- .get_args()- .map(|a| a.to_string_lossy())- .collect::<Vec<_>>()- .join(" ")- );- if !lib_cmd.status().expect("Failed to run metallib linker").success() {- panic!("Failed to link Metal shader libraries");- }-- // Clean up intermediate .air files- for air in air_files {- let _ = fs::remove_file(air);- }- println!(- "cargo:info=Metal shaders compiled successfully to {}",- metallib_path.display()- );-}--fn write_metallib_include(- platform_suffix: &str,- out_path: &Path,-) {- let mut f = fs::File::create(out_path).expect("Cannot create metal_lib.rs");- let metallib_file_name = format!("shaders_{}.metallib", platform_suffix);- writeln!(f, "// AUTO-GENERATED Metal library include").unwrap();- writeln!(- f,- "pub const METAL_LIBRARY_DATA: &[u8] = include_bytes!(concat!(env!(\"OUT_DIR\"), \"/metallib/{}\"));",- metallib_file_name- )- .unwrap();-}--fn write_empty_metallib() {- let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());- let metal_lib_rs = out_dir.join("metal_lib.rs");-- if let Some(parent) = metal_lib_rs.parent() {- let _ = fs::create_dir_all(parent);- }- fs::write(metal_lib_rs, b"pub const METAL_LIBRARY_DATA: &[u8] = &[];\n")- .expect("Cannot create stub metal_lib.rs");-}diff --git a/crates/uzu/build/common/build_debug.rs b/crates/uzu/build/common/build_debug.rsdeleted file mode 100644index b3d0f3315..000000000--- a/crates/uzu/build/common/build_debug.rs+++ /dev/null@@ -1,19 +0,0 @@-use std::sync::OnceLock;-use std::time::Instant;--static START: OnceLock<Instant> = OnceLock::new();--pub fn log(args: std::fmt::Arguments) {- let start = START.get_or_init(Instant::now);- let elapsed_ms = start.elapsed().as_millis();- println!("cargo::warning=(build-debug) [{elapsed_ms}ms] {args}");-}--#[macro_export]-macro_rules! debug_log {- ($($arg:tt)*) => {{- if cfg!(feature = "build-debug") {- $crate::common::build_debug::log(format_args!($($arg)*));- }- }};-}diff --git a/crates/uzu/build/common/caching.rs b/crates/uzu/build/common/caching.rsnew file mode 100644index 000000000..84e1b498f--- /dev/null+++ b/crates/uzu/build/common/caching.rs@@ -0,0 +1,20 @@+use anyhow::Context;+use std::{fs, sync::OnceLock};++static BUILD_SYSTEM_HASH: OnceLock<blake3::Hash> = OnceLock::new();++pub fn build_system_hash() -> anyhow::Result<&'static blake3::Hash> {+ if let Some(bsh) = BUILD_SYSTEM_HASH.get() {+ Ok(bsh)+ } else {+ let bsh = blake3::hash(+ &fs::read(+ &std::env::current_exe()+ .context("cannot get build system executable")?,+ )+ .context("cannot read build system executable")?,+ );++ Ok(BUILD_SYSTEM_HASH.get_or_init(|| bsh))+ }+}diff --git a/crates/uzu/build/common/compiler.rs b/crates/uzu/build/common/compiler.rsnew file mode 100644index 000000000..5f402a2ed--- /dev/null+++ b/crates/uzu/build/common/compiler.rs@@ -0,0 +1,6 @@+use async_trait::async_trait;++#[async_trait]+pub trait Compiler {+ async fn build(&self) -> anyhow::Result<()>;+}diff --git a/crates/uzu/build/common/envs.rs b/crates/uzu/build/common/envs.rsnew file mode 100644index 000000000..927349201--- /dev/null+++ b/crates/uzu/build/common/envs.rs@@ -0,0 +1,22 @@+use std::sync::OnceLock;++static BUILD_DEBUG: OnceLock<bool> = OnceLock::new();+static BUILD_ALWAYS: OnceLock<bool> = OnceLock::new();+static BUILD_CLEAN: OnceLock<bool> = OnceLock::new();++fn env_flag(name: &str) -> bool {+ println!("cargo::rerun-if-env-changed={name}");+ std::env::var(name).is_ok()+}++pub fn build_debug() -> bool {+ *BUILD_DEBUG.get_or_init(|| env_flag("BUILD_DEBUG"))+}++pub fn build_always() -> bool {+ *BUILD_ALWAYS.get_or_init(|| env_flag("BUILD_ALWAYS"))+}++pub fn build_clean() -> bool {+ *BUILD_CLEAN.get_or_init(|| env_flag("BUILD_CLEAN"))+}diff --git a/crates/uzu/build/common/logging.rs b/crates/uzu/build/common/logging.rsnew file mode 100644index 000000000..2e25f91f9--- /dev/null+++ b/crates/uzu/build/common/logging.rs@@ -0,0 +1,23 @@+use std::sync::OnceLock;+use std::time::Instant;++use crate::common::envs;++static START: OnceLock<Instant> = OnceLock::new();++pub fn elapsed_ms() -> u128 {+ START.get_or_init(Instant::now).elapsed().as_millis()+}++pub fn _debug_log(args: std::fmt::Arguments) {+ if envs::build_debug() {+ println!("cargo::warning=(build-debug) [{}ms] {}", elapsed_ms(), args);+ }+}++#[macro_export]+macro_rules! debug_log {+ ($($arg:tt)*) => {{+ $crate::common::logging::_debug_log(format_args!($($arg)*));+ }};+}diff --git a/crates/uzu/build/common/mod.rs b/crates/uzu/build/common/mod.rsindex 78c5959e9..3b41dfd7f 100644--- a/crates/uzu/build/common/mod.rs+++ b/crates/uzu/build/common/mod.rs@@ -1 +1,4 @@-pub(crate) mod build_debug;+pub mod caching;+pub mod compiler;+pub mod envs;+pub mod logging;diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rsindex 6e86b0025..918d1228e 100644--- a/crates/uzu/build/main.rs+++ b/crates/uzu/build/main.rs@@ -1,17 +1,28 @@use std::{env, fs, path::PathBuf};use anyhow::Context;-use tokio::try_join;mod common;mod metal;mod shared_types;+use common::compiler::Compiler;+use common::envs;+use futures::future::try_join_all;+#[tokio::main(flavor = "current_thread")]async fn main() -> anyhow::Result<()> {+ println!("cargo::rerun-if-changed=.");++ if envs::build_always() {+ println!(+ "cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun"+ );+ }+debug_log!("build script started");- if cfg!(feature = "build-clean") {+ if envs::build_clean() {let out_dir =PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);if out_dir.exists() {@@ -25,7 +36,14 @@ async fn main() -> anyhow::Result<()> {debug_log!("cleaned caches");}- try_join!(metal::main(), shared_types::main())?;+ let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();++ if cfg!(feature = "metal") {+ compilers.push(Box::new(metal::MetalCompiler::new()?));+ compilers.push(Box::new(shared_types::SharedTypesCompiler::new()?));+ }++ try_join_all(compilers.iter().map(|c| c.build())).await?;debug_log!("build script ended");diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rsindex 8e448b740..8e2bdb401 100644--- a/crates/uzu/build/metal/compiler.rs+++ b/crates/uzu/build/metal/compiler.rs@@ -1,26 +1,43 @@use std::collections::HashMap;use std::iter::{once, zip};+use std::path::Path;use std::{env, fs, path::PathBuf};use anyhow::{Context, bail};use futures::future::try_join_all;+use futures::{StreamExt, TryStreamExt, stream};+use proc_macro2::TokenStream;+use quote::quote;use serde::{Deserialize, Serialize};use tokio::io::AsyncReadExt;use tokio::task::spawn_blocking;+use walkdir::WalkDir;+use async_trait::async_trait;++use crate::common::compiler::Compiler;+use crate::common::{caching, envs};use crate::debug_log;use super::ast::{MetalArgumentType, MetalKernelInfo};use super::toolchain::MetalToolchain;#[derive(Serialize, Deserialize, Debug)]-pub struct ObjectInfo {- pub object_path: PathBuf,- pub kernels: Box<[MetalKernelInfo]>,- pub dependency_hashes: HashMap<Box<str>, [u8; blake3::OUT_LEN]>,+struct ObjectInfo {+ object_path: PathBuf,+ kernels: Box<[MetalKernelInfo]>,+ buildsystem_hash: [u8; blake3::OUT_LEN],+ dependency_hashes: HashMap<Box<str>, [u8; blake3::OUT_LEN]>,+}++fn is_nax_source(path: &Path) -> bool {+ path.file_name()+ .and_then(|s| s.to_str())+ .map(|s| s.contains("_nax"))+ .unwrap_or(false)}-pub fn wrappers(kernel: &MetalKernelInfo) -> anyhow::Result<Box<[Box<str>]>> {+fn wrappers(kernel: &MetalKernelInfo) -> anyhow::Result<Box<[Box<str>]>> {let mut kernel_wrappers = Vec::new();for specialization_variant in kernel@@ -190,15 +207,47 @@ async fn hash_dependencies(try_join_all(futures).await.map(|v| v.into_iter().collect())}+fn objects_hash<'a>(+ objects: impl IntoIterator<Item = &'a ObjectInfo>+) -> anyhow::Result<blake3::Hash> {+ let mut hasher = blake3::Hasher::new();+ hasher.update(+ caching::build_system_hash()+ .context("cannot get build system hash")?+ .as_bytes(),+ );+ let mut paths: Vec<_> =+ objects.into_iter().map(|o| &o.object_path).collect();+ paths.sort();+ for path in paths {+ let path_bytes = path.to_string_lossy();+ let path_bytes = path_bytes.as_bytes();+ hasher.update(&(path_bytes.len() as u32).to_le_bytes());+ hasher.update(path_bytes);+ let contents = fs::read(path)+ .with_context(|| format!("cannot read {}", path.display()))?;+ hasher.update(&(contents.len() as u32).to_le_bytes());+ hasher.update(&contents);+ }+ Ok(hasher.finalize())+}+#[derive(Debug)]pub struct MetalCompiler {- out_dir: PathBuf,+ src_dir: PathBuf,build_dir: PathBuf,+ out_dir: PathBuf,toolchain: MetalToolchain,}impl MetalCompiler {pub fn new() -> anyhow::Result<Self> {+ let src_dir = PathBuf::from(+ env::var("CARGO_MANIFEST_DIR")+ .context("missing CARGO_MANIFEST_DIR")?,+ )+ .join("src");+let out_dir =PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);let build_dir = out_dir.join("metal");@@ -210,16 +259,22 @@ impl MetalCompiler {MetalToolchain::from_env().context("cannot create toolchain")?;Ok(Self {- out_dir,+ src_dir,build_dir,+ out_dir,toolchain,})}- pub async fn compile(+ async fn compile(&self,source_path: PathBuf,- ) -> anyhow::Result<(ObjectInfo, bool)> {+ ) -> anyhow::Result<ObjectInfo> {+ let buildsystem_hash = caching::build_system_hash()+ .context("cannot get build system cache")?+ .as_bytes()+ .clone();+let source_path_pretty =source_path.file_name().unwrap().to_str().unwrap();debug_log!("compile start: {source_path_pretty}");@@ -242,12 +297,13 @@ impl MetalCompiler {if let Ok(contents) = tokio::fs::read(&objectinfo_path).await&& let Ok(cached) =serde_json::from_slice::<ObjectInfo>(&contents)- && let Ok(current_hashes) =+ && buildsystem_hash == cached.buildsystem_hash+ && let Ok(dependency_hashes) =hash_dependencies(cached.dependency_hashes.keys().cloned()).await- && current_hashes == cached.dependency_hashes+ && dependency_hashes == cached.dependency_hashes{- if cfg!(feature = "build-debug") {+ if envs::build_debug() {let kernel_list = cached.kernels.iter()@@ -262,7 +318,7 @@ impl MetalCompiler {);}}- return Ok((cached, true));+ return Ok(cached);}fs::remove_dir_all(&build_dir).ok();}@@ -317,6 +373,7 @@ impl MetalCompiler {let object_info = ObjectInfo {object_path,kernels: kernel_infos.into_boxed_slice(),+ buildsystem_hash,dependency_hashes,};@@ -330,7 +387,7 @@ impl MetalCompiler {format!("cannot write object info {}", objectinfo_path.display())})?;- if cfg!(feature = "build-debug") {+ if envs::build_debug() {let kernel_list = object_info.kernels.iter()@@ -346,16 +403,26 @@ impl MetalCompiler {}}- Ok((object_info, false))+ Ok(object_info)}- pub async fn link(+ async fn link<'a>(&self,- objects: impl IntoIterator<Item = &ObjectInfo>,+ objects: impl IntoIterator<Item = &'a ObjectInfo> + Clone,) -> anyhow::Result<PathBuf> {- debug_log!("linking start");-let library_path = self.out_dir.join("default.metallib");+ let hash_path = self.out_dir.join("default.metallib.hash");++ let hash = objects_hash(objects.clone())?;++ if let Ok(cached_hash) = fs::read_to_string(&hash_path)+ && cached_hash == hash.to_string()+ {+ debug_log!("linking cached");+ return Ok(library_path);+ }++ debug_log!("linking start");let link_output = self.toolchain@@ -369,8 +436,93 @@ impl MetalCompiler {}}+ fs::write(&hash_path, hash.to_string()).with_context(|| {+ format!("cannot write hash file {}", hash_path.display())+ })?;+debug_log!("linking end");Ok(library_path)}++ fn bindgen<'a>(+ &self,+ objects: impl IntoIterator<Item = &'a ObjectInfo> + Clone,+ ) -> anyhow::Result<()> {+ let out_path = self.out_dir.join("dsl.rs");+ let hash_path = self.out_dir.join("dsl.rs.hash");++ let hash = objects_hash(objects.clone())?;++ if let Ok(cached_hash) = fs::read_to_string(&hash_path)+ && cached_hash == hash.to_string()+ {+ debug_log!("bindgen cached");+ return Ok(());+ }++ debug_log!("bindgen start");++ let bindings = objects+ .into_iter()+ .flat_map(|o| &o.kernels)+ .map(|k| {+ super::bindgen::bindgen(k).with_context(|| {+ format!("cannot generate bindings for {}", k.name)+ })+ })+ .collect::<anyhow::Result<Vec<TokenStream>>>()?;++ let tokens = quote! { #(#bindings)* };++ let parsed =+ syn::parse2(tokens).context("cannot parse generated bindings")?;+ fs::write(&out_path, prettyplease::unparse(&parsed)).with_context(+ || format!("cannot write bindings file {}", out_path.display()),+ )?;++ fs::write(&hash_path, hash.to_string()).with_context(|| {+ format!("cannot write hash file {}", hash_path.display())+ })?;++ debug_log!("bindgen end");++ Ok(())+ }+}++#[async_trait]+impl Compiler for MetalCompiler {+ async fn build(&self) -> anyhow::Result<()> {+ let nax_enabled = cfg!(feature = "metal-nax");+ debug_log!("metal nax {}", if nax_enabled { "enabled" } else { "disabled" });++ let metal_sources: Vec<PathBuf> = WalkDir::new(&self.src_dir)+ .into_iter()+ .filter_map(|e| e.ok())+ .filter(|e| {+ e.file_type().is_file()+ && e.path().extension().and_then(|s| s.to_str())+ == Some("metal")+ })+ .map(|e| e.into_path())+ .filter(|p| nax_enabled || !is_nax_source(p))+ .collect();++ let num_concurrent_compiles =+ std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4)+ * 2;++ let objects: Vec<ObjectInfo> = stream::iter(metal_sources)+ .map(|p| self.compile(p))+ .buffer_unordered(num_concurrent_compiles)+ .try_collect()+ .await+ .context("cannot compile metal sources")?;++ self.link(&objects).await.context("cannot link objects")?;+ self.bindgen(&objects).context("cannot generate bindings")?;++ Ok(())+ }}diff --git a/crates/uzu/build/metal/mod.rs b/crates/uzu/build/metal/mod.rsindex b4abe8ed2..ce9ce685c 100644--- a/crates/uzu/build/metal/mod.rs+++ b/crates/uzu/build/metal/mod.rs@@ -1,107 +1,6 @@-use std::{- env, fs,- path::{Path, PathBuf},-};--use anyhow::Context;-use compiler::{MetalCompiler, ObjectInfo};-use futures::{StreamExt, TryStreamExt, stream};-use proc_macro2::TokenStream;-use quote::quote;-use walkdir::WalkDir;--use crate::debug_log;-mod ast;mod bindgen;mod compiler;mod toolchain;-fn is_nax_source(path: &Path) -> bool {- path.file_name()- .and_then(|s| s.to_str())- .map(|s| s.contains("_nax"))- .unwrap_or(false)-}--pub async fn main() -> anyhow::Result<()> {- let out_dir =- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);-- let nax_enabled = cfg!(feature = "metal-nax");- debug_log!("metal nax enabled: {nax_enabled}");-- let nax_mode_path = out_dir.join("metal_nax_mode");- let desired_nax_mode = if nax_enabled { "1" } else { "0" };- let prev_nax_mode =- fs::read_to_string(&nax_mode_path).ok().map(|s| s.trim().to_owned());- let nax_mode_changed =- prev_nax_mode.as_deref() != Some(desired_nax_mode);-- let dsl = MetalCompiler::new().context("cannot create metal dsl")?;-- let src_dir = PathBuf::from(- env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?,- )- .join("src");-- let metal_sources: Vec<PathBuf> = WalkDir::new(&src_dir)- .into_iter()- .filter_map(|e| e.ok())- .filter(|e| {- e.file_type().is_file()- && e.path().extension().and_then(|s| s.to_str())- == Some("metal")- })- .map(|e| e.into_path())- .filter(|p| nax_enabled || !is_nax_source(p))- .collect();-- let num_concurrent_compiles =- std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4) * 2;-- let results: Vec<(ObjectInfo, bool)> = stream::iter(metal_sources)- .map(|p| dsl.compile(p))- .buffer_unordered(num_concurrent_compiles)- .try_collect()- .await- .context("cannot compile metal sources")?;-- let all_cached = results.iter().all(|(_, cached)| *cached);- let kernels: Vec<ObjectInfo> =- results.into_iter().map(|(o, _)| o).collect();-- if !all_cached || nax_mode_changed {- dsl.link(&kernels).await.context("cannot link objects")?;-- let bindings = kernels- .iter()- .flat_map(|k| &k.kernels)- .map(|k| {- bindgen::bindgen(k).with_context(|| {- format!("cannot generate bindings for {}", k.name)- })- })- .collect::<anyhow::Result<Vec<TokenStream>>>()?;-- let tokens = quote! { #(#bindings)* };-- let out_path =- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?)- .join("dsl.rs");- let parsed =- syn::parse2(tokens).context("cannot parse generated bindings")?;- fs::write(&out_path, prettyplease::unparse(&parsed)).with_context(- || format!("cannot write bindings file {}", out_path.display()),- )?;-- fs::write(&nax_mode_path, desired_nax_mode).with_context(|| {- format!(- "cannot write nax mode file {}",- nax_mode_path.display()- )- })?;- }-- Ok(())-}+pub use compiler::MetalCompiler;diff --git a/crates/uzu/build/shared_types/mod.rs b/crates/uzu/build/shared_types/mod.rsindex da478d473..7921cf6f3 100644--- a/crates/uzu/build/shared_types/mod.rs+++ b/crates/uzu/build/shared_types/mod.rs@@ -1,9 +1,13 @@use std::{env, fs, path::PathBuf};-use crate::debug_log;use anyhow::Context;use tokio::{task::spawn_blocking, try_join};+use async_trait::async_trait;++use crate::common::compiler::Compiler;+use crate::debug_log;+struct BindgenConfig {name: &'static str,kernel_subdir: &'static str,@@ -36,79 +40,130 @@ const ATTENTION_CONFIG: BindgenConfig = BindgenConfig {raw_lines: &[],};-fn autogen_shared_types(config: &BindgenConfig) -> anyhow::Result<()> {- let out_dir = env::var("OUT_DIR").context("missing OUT_DIR")?;- let build_dir = PathBuf::from(out_dir).join("shared_types");- fs::create_dir_all(&build_dir).with_context(|| {- format!("cannot create shared types build dir {}", build_dir.display())- })?;-- let manifest_dir = PathBuf::from(- env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?,- );- let kernel_dir = manifest_dir- .join(format!("src/backends/metal/kernel/{}", config.kernel_subdir));- let types_header = kernel_dir.join(config.header_file);- let types_header_hash: [u8; blake3::OUT_LEN] =- blake3::hash(&fs::read(types_header.as_path()).with_context(|| {- format!("cannot read {}", types_header.display())- })?)- .into();- let cached_types_header_hash =- build_dir.join(format!("{}.hash", config.header_file));-- if let Some(cached_hash) = fs::read(&cached_types_header_hash).ok()- && cached_hash == types_header_hash- {- debug_log!("{} bindgen cached", config.name);- return Ok(());- }- debug_log!("{} bindgen started", config.name);-- let mut builder = bindgen::Builder::default()- .header(types_header.to_string_lossy())- .clang_arg("-x")- .clang_arg("c")- .derive_default(true)- .derive_copy(true)- .use_core();-- for raw_line in config.raw_lines {- builder = builder.raw_line(*raw_line);- }-- for ty in config.allowlist_types {- builder = builder.allowlist_type(*ty);- }+#[derive(Debug)]+pub struct SharedTypesCompiler {+ src_dir: PathBuf,+ build_dir: PathBuf,+}- let bindings = builder.generate().map_err(|e| {- anyhow::anyhow!(- "failed to generate bindings from {}: {e}",- config.header_file+impl SharedTypesCompiler {+ pub fn new() -> anyhow::Result<Self> {+ let src_dir = PathBuf::from(+ env::var("CARGO_MANIFEST_DIR")+ .context("missing CARGO_MANIFEST_DIR")?,)- })?;-- let bindings_path = kernel_dir.join(config.output_file);- bindings.write_to_file(&bindings_path).with_context(|| {- format!("failed to write {}", bindings_path.display())- })?;-- fs::write(&cached_types_header_hash, &types_header_hash).with_context(- || format!("failed to write {}", cached_types_header_hash.display()),- )?;- debug_log!("{} bindgen done", config.name);+ .join("src");++ let build_dir =+ PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?)+ .join("shared_types");+ fs::create_dir_all(&build_dir).with_context(|| {+ format!("cannot create {}", build_dir.display())+ })?;++ Ok(Self {+ src_dir,+ build_dir,+ })+ }- Ok(())+ fn generate(+ &self,+ config: &BindgenConfig,+ ) -> anyhow::Result<()> {+ let kernel_dir = self+ .src_dir+ .join(format!("backends/metal/kernel/{}", config.kernel_subdir));+ let types_header = kernel_dir.join(config.header_file);+ let types_header_hash: [u8; blake3::OUT_LEN] =+ blake3::hash(&fs::read(&types_header).with_context(|| {+ format!("cannot read {}", types_header.display())+ })?)+ .into();+ let cached_types_header_hash =+ self.build_dir.join(format!("{}.hash", config.header_file));++ if let Some(cached_hash) = fs::read(&cached_types_header_hash).ok()+ && cached_hash == types_header_hash+ {+ debug_log!("{} bindgen cached", config.name);+ return Ok(());+ }+ debug_log!("{} bindgen started", config.name);++ let mut builder = bindgen::Builder::default()+ .header(types_header.to_string_lossy())+ .clang_arg("-x")+ .clang_arg("c")+ .derive_default(true)+ .derive_copy(true)+ .use_core();++ for raw_line in config.raw_lines {+ builder = builder.raw_line(*raw_line);+ }++ for ty in config.allowlist_types {+ builder = builder.allowlist_type(*ty);+ }++ let bindings = builder.generate().map_err(|e| {+ anyhow::anyhow!(+ "failed to generate bindings from {}: {e}",+ config.header_file+ )+ })?;++ let bindings_path = kernel_dir.join(config.output_file);+ bindings.write_to_file(&bindings_path).with_context(|| {+ format!("failed to write {}", bindings_path.display())+ })?;++ fs::write(&cached_types_header_hash, &types_header_hash).with_context(+ || {+ format!(+ "failed to write {}",+ cached_types_header_hash.display()+ )+ },+ )?;+ debug_log!("{} bindgen done", config.name);++ Ok(())+ }}-pub async fn main() -> anyhow::Result<()> {- let (matmul_result, attn_result) = try_join!(- spawn_blocking(|| autogen_shared_types(&MATMUL_CONFIG)),- spawn_blocking(|| autogen_shared_types(&ATTENTION_CONFIG))- )?;-- matmul_result?;- attn_result?;-- Ok(())+#[async_trait]+impl Compiler for SharedTypesCompiler {+ async fn build(&self) -> anyhow::Result<()> {+ let (matmul_result, attn_result) = try_join!(+ spawn_blocking({+ let src_dir = self.src_dir.clone();+ let build_dir = self.build_dir.clone();+ move || {+ SharedTypesCompiler {+ src_dir,+ build_dir,+ }+ .generate(&MATMUL_CONFIG)+ }+ }),+ spawn_blocking({+ let src_dir = self.src_dir.clone();+ let build_dir = self.build_dir.clone();+ move || {+ SharedTypesCompiler {+ src_dir,+ build_dir,+ }+ .generate(&ATTENTION_CONFIG)+ }+ })+ )?;++ matmul_result?;+ attn_result?;++ Ok(())+ }}diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rsindex 70ad744f0..4c80a9a53 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs@@ -307,8 +307,12 @@ impl RMSNormKernel {// We only ever normalize within [Q] and/or [K].let (head_offset, head_count): (u32, u32) = match args.target {QKNormTarget::QueryHeads => (0, args.num_q_heads as u32),- QKNormTarget::KeyHeads => (args.num_q_heads as u32, args.num_kv_heads as u32),- QKNormTarget::Both => (0, (args.num_q_heads + args.num_kv_heads) as u32),+ QKNormTarget::KeyHeads => {+ (args.num_q_heads as u32, args.num_kv_heads as u32)+ },+ QKNormTarget::Both => {+ (0, (args.num_q_heads + args.num_kv_heads) as u32)+ },};if args.batch_size <= 0 || args.head_dim <= 0 || head_count == 0 {@@ -329,12 +333,14 @@ impl RMSNormKernel {// One SIMD-group per head, multiple heads per threadgroup.let simd_width = self.pipeline.thread_execution_width() as u64;- let max_threads = self.pipeline.max_total_threads_per_threadgroup() as u64;+ let max_threads =+ self.pipeline.max_total_threads_per_threadgroup() as u64;let max_heads_per_threadgroup = (max_threads / simd_width).max(1);let heads_per_threadgroup =(head_count as u64).min(max_heads_per_threadgroup).max(1);- let num_head_tiles = (head_count as u64).div_ceil(heads_per_threadgroup);+ let num_head_tiles =+ (head_count as u64).div_ceil(heads_per_threadgroup);let threadgroups_per_grid = MTLSize {width: args.batch_size as u64,