Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dsl improvements (#81)

uuuvnb0a1f7040880
diff --git a/Cargo.lock b/Cargo.lock
index d5f40d829..d27189172 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -136,9 +136,9 @@ dependencies = [
[[package]]
name = "async-trait"
-version = "0.1.88"
+version = "0.1.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"
+checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
@@ -4188,6 +4188,7 @@ name = "uzu"
version = "0.1.7"
dependencies = [
"anyhow",
+ "async-trait",
"bindgen",
"bitflags 2.9.4",
"blake3",
diff --git a/Cargo.toml b/Cargo.toml
index 69e611aca..7152d1ec5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,6 +45,7 @@ schemars = { version = "1.0.5", features = ["derive"] }
is_close = "0.1.3"
serde_json5 = "0.2.1"
anyhow = "1.0.100"
+async-trait = "0.1.89"
bindgen = "0.72.1"
walkdir = "2.5.0"
depfile = "0.1.1"
@@ -70,5 +71,9 @@ cxx = "1.0.187"
cmake = "0.1.57"
autocxx-build = "0.30.0"
+# optimize the build script in debug builds
+[profile.dev.build-override]
+opt-level = 3
+
[profile.release]
lto = "thin"
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 7f22f6758..e68a2ee77 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -38,6 +38,7 @@ mpsgraph.workspace = true
[build-dependencies]
anyhow.workspace = true
+async-trait.workspace = true
bindgen.workspace = true
walkdir.workspace = true
depfile.workspace = true
@@ -55,11 +56,13 @@ prettyplease.workspace = true
itertools.workspace = true
[features]
-build-debug = []
-build-clean = []
-tracing = []
+default = ["metal"]
+
+metal = []
metal-nax = []
+tracing = []
+
[[test]]
name = "moe"
path = "tests/moe/mod.rs"
diff --git a/crates/uzu/build.rs b/crates/uzu/build.rs
deleted file mode 100644
index 8f6ce42b6..000000000
--- a/crates/uzu/build.rs
+++ /dev/null
@@ -1,572 +0,0 @@
-use std::{
- collections::HashSet,
- env, fs,
- io::Write,
- path::{Path, PathBuf},
- process::Command,
- time::SystemTime,
-};
-
-fn main() {
- // Generate Rust bindings from shared Metal/C headers
- generate_metal_bindings();
-
- if cfg!(feature = "metal-shaders") {
- println!("cargo:rerun-if-env-changed=MY_API_LEVEL");
- compile_metal_shaders();
- } else {
- write_empty_metallib();
- }
-}
-
-/// Generate Rust bindings from shared C headers using bindgen.
-/// These headers are also used by Metal shaders, ensuring type consistency.
-fn generate_metal_bindings() {
- let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
- let matmul_common_dir =
- manifest_dir.join("src/backends/metal/kernel/matmul/common");
- let types_header = matmul_common_dir.join("shared_types.h");
-
- // Only regenerate if header changed
- println!("cargo:rerun-if-changed={}", types_header.display());
-
- let bindings = bindgen::Builder::default()
- .header(types_header.to_string_lossy())
- // Generate for C mode (no Metal-specific stuff)
- .clang_arg("-x")
- .clang_arg("c")
- // Allow uppercase field names to match the C header without warnings
- .raw_line("#![allow(non_snake_case)]")
- // Derive common traits
- .derive_default(true)
- .derive_copy(true)
- // Only generate types we care about
- .allowlist_type("GEMMParams")
- .allowlist_type("GEMMAddMMParams")
- .allowlist_type("GEMMSpiltKParams")
- .allowlist_type("GEMMSpiltKMlpFusedParams")
- // Use core types
- .use_core()
- .generate()
- .expect("Failed to generate bindings from shared_types.h");
-
- // Write bindings to the source directory so it can be checked in
- let bindings_path = matmul_common_dir.join("shared_types.rs");
- bindings
- .write_to_file(&bindings_path)
- .expect("Failed to write shared_types.rs");
-
- // ---------------------------------------------------------------------
- // Gemm attention shared types
- // ---------------------------------------------------------------------
- let attention_dir =
- manifest_dir.join("src/backends/metal/kernel/attention");
- let attn_types_header = attention_dir.join("gemm_types.h");
-
- println!("cargo:rerun-if-changed={}", attn_types_header.display());
-
- let attn_bindings = bindgen::Builder::default()
- .header(attn_types_header.to_string_lossy())
- // Generate for C mode (no Metal-specific stuff)
- .clang_arg("-x")
- .clang_arg("c")
- // Derive common traits
- .derive_default(true)
- .derive_copy(true)
- // Only generate types we care about
- .allowlist_type("AttnParams")
- .allowlist_type("AttnMaskParams")
- // Use core types
- .use_core()
- .generate()
- .expect("Failed to generate bindings from gemm_types.h");
-
- let attn_bindings_path = attention_dir.join("gemm_types.rs");
- attn_bindings
- .write_to_file(&attn_bindings_path)
- .expect("Failed to write gemm_types.rs");
-}
-
-fn compile_metal_shaders() {
- // Determine target platform (only proceed on Apple platforms)
- let target = env::var("TARGET").unwrap_or_default();
- let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();
- let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default();
-
- // Choose appropriate SDK: macosx, iphoneos, iphonesimulator, or maccatalyst
- let sdk = if target_os == "ios" {
- if target.contains("macabi") || target_env == "macabi" {
- "maccatalyst"
- } else if target.contains("ios")
- && (target.contains("86_64") || target_env == "sim")
- {
- // Use iPhoneSimulator SDK for simulator targets (x86_64 or explicit 'sim')
- "iphonesimulator"
- } else {
- "iphoneos"
- }
- } else if target_os == "macos" {
- "macosx"
- } else {
- println!(
- "cargo:warning=Not an Apple platform, skipping Metal shader compilation"
- );
- return;
- };
-
- println!("cargo:info=Compiling Metal shaders for SDK: {}", sdk);
-
- // Re-run build script if the host OS version changes (affects Metal feature availability)
- if cfg!(target_os = "macos") {
- println!(
- "cargo:rerun-if-changed=/System/Library/CoreServices/SystemVersion.plist"
- );
- }
-
- // Gather all .metal files in the project (src directory and subdirs)
- let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
- let src_dir = manifest_dir.join("src");
- let metal_files = find_metal_files(&src_dir);
- if metal_files.is_empty() {
- println!("cargo:warning=No .metal files found; nothing to compile");
- return;
- }
- println!("cargo:info=Found {} Metal shader files", metal_files.len());
-
- // Prepare output directory for metallib
- let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
- let metallib_dir = out_dir.join("metallib");
- fs::create_dir_all(&metallib_dir).unwrap();
-
- // Require xcrun (should be present on macOS with Xcode)
- if !Command::new("xcrun")
- .arg("--version")
- .status()
- .map(|s| s.success())
- .unwrap_or(false)
- {
- panic!(
- "xcrun not found. Please install Xcode command-line tools to compile Metal shaders."
- );
- }
-
- // Platform-specific metallib path (separate for macos vs ios for caching)
- let platform_suffix = if sdk == "macosx" || sdk == "maccatalyst" {
- "macos"
- } else {
- "ios"
- };
- let metallib_path =
- metallib_dir.join(format!("shaders_{}.metallib", platform_suffix));
-
- let nax_enabled = should_enable_nax(&target_os, sdk);
- if nax_enabled {
- println!("cargo:info=Metal NAX kernels enabled (Metal 4, OS 26+)");
- } else {
- println!("cargo:info=Metal NAX kernels disabled");
- }
-
- let metal_files_to_compile: Vec<PathBuf> = if nax_enabled {
- metal_files.clone()
- } else {
- metal_files.iter().filter(|p| !is_nax_kernel(p)).cloned().collect()
- };
-
- // Decide whether to compile (if metallib missing or any source changed)
- let should_compile = needs_compilation(
- &metal_files_to_compile,
- &metallib_path,
- &target,
- nax_enabled,
- );
- if should_compile {
- compile_metal_files(
- &metal_files_to_compile,
- &metallib_dir,
- sdk,
- &src_dir,
- nax_enabled,
- );
- // Record the target triple for which we built the metallib
- let _ = fs::write(metallib_path.with_extension("target"), &target);
- // Record whether we built with NAX kernels enabled
- let _ = fs::write(
- metallib_path.with_extension("nax"),
- if nax_enabled {
- "1"
- } else {
- "0"
- },
- );
- } else {
- println!(
- "cargo:info=Metal shaders are up-to-date; skipping compilation"
- );
- }
-
- if !metallib_path.exists() {
- panic!("Metal library not found at {}", metallib_path.display());
- }
-
- // Generate a tiny Rust source file that uses include_bytes! to embed the
- // compiled metallib. This avoids generating a massive Rust file containing
- // a hex-encoded byte array (which is slow to write/compile).
- write_metallib_include(platform_suffix, &out_dir.join("metal_lib.rs"));
-
- // Tell Cargo when to re-run this build script: all shader files, plus build.rs and any FFI headers
- let mut sorted_files: Vec<&PathBuf> = metal_files.iter().collect();
- sorted_files.sort();
- for file in sorted_files {
- println!("cargo:rerun-if-changed={}", file.display());
- }
- println!("cargo:rerun-if-changed=build.rs");
-}
-
-fn is_nax_kernel(path: &Path) -> bool {
- path.file_name()
- .and_then(|s| s.to_str())
- .map(|s| s.contains("_nax"))
- .unwrap_or(false)
-}
-
-const NAX_MIN_OS_MAJOR: u32 = 26;
-const NAX_MIN_SDK_VERSION: (u32, u32) = (26, 2);
-
-fn host_macos_major_version() -> Option<u32> {
- if !cfg!(target_os = "macos") {
- return None;
- }
- let output =
- Command::new("sw_vers").arg("-productVersion").output().ok()?;
- if !output.status.success() {
- return None;
- }
- let version = String::from_utf8_lossy(&output.stdout);
- let major = version.trim().split('.').next()?.parse::<u32>().ok()?;
- Some(major)
-}
-
-fn macos_sdk_version(sdk: &str) -> Option<(u32, u32)> {
- let output = Command::new("xcrun")
- .args(["--sdk", sdk, "--show-sdk-version"])
- .output()
- .ok()?;
- if !output.status.success() {
- return None;
- }
- let version = String::from_utf8_lossy(&output.stdout);
- let version = version.trim();
- let mut parts = version.split('.');
- let major = parts.next()?.parse::<u32>().ok()?;
- let minor = parts.next().and_then(|s| s.parse::<u32>().ok()).unwrap_or(0);
- Some((major, minor))
-}
-
-fn should_enable_nax(
- target_os: &str,
- sdk: &str,
-) -> bool {
- if target_os != "macos" && target_os != "ios" {
- return false;
- }
- // Conservative: don't enable NAX for simulator builds
- if sdk == "iphonesimulator" {
- return false;
- }
- // Check host OS version
- let os_ok = host_macos_major_version()
- .is_some_and(|major| major >= NAX_MIN_OS_MAJOR);
- if !os_ok {
- return false;
- }
- // Check SDK version (NAX APIs require SDK >= 26.2)
- macos_sdk_version(sdk)
- .is_some_and(|(major, minor)| (major, minor) >= NAX_MIN_SDK_VERSION)
-}
-
-// Recursively find all .metal files in the given directory
-fn find_metal_files(dir: &Path) -> Vec<PathBuf> {
- let mut files = Vec::new();
- if let Ok(entries) = fs::read_dir(dir) {
- for entry in entries.flatten() {
- let path = entry.path();
- if path.is_dir() {
- files.extend(find_metal_files(&path));
- } else if path.extension().and_then(|s| s.to_str()) == Some("metal")
- {
- files.push(path);
- }
- }
- }
- files
-}
-
-// Check timestamps and target to determine if recompilation is needed
-fn needs_compilation(
- metal_files: &[PathBuf],
- metallib_path: &Path,
- target: &str,
- nax_enabled: bool,
-) -> bool {
- // If output metallib doesn't exist, need to compile
- if !metallib_path.exists() {
- return true;
- }
- // If the previously built target differs (target changed, e.g. arch or OS)
- if let Ok(prev_target) =
- fs::read_to_string(metallib_path.with_extension("target"))
- {
- if prev_target != target {
- println!(
- "cargo:warning=Target changed from {} to {}, recompiling shaders",
- prev_target, target
- );
- return true;
- }
- }
- // If the previous NAX mode differs, force a rebuild (affects metallib compatibility)
- let desired_nax = if nax_enabled {
- "1"
- } else {
- "0"
- };
- match fs::read_to_string(metallib_path.with_extension("nax")) {
- Ok(prev_nax) if prev_nax.trim() == desired_nax => {},
- _ => return true,
- }
- // If any source file is newer than the metallib, recompile
- let metallib_mtime = fs::metadata(metallib_path)
- .and_then(|m| m.modified())
- .unwrap_or(SystemTime::UNIX_EPOCH);
- for metal_file in metal_files {
- if let Ok(src_mtime) =
- fs::metadata(metal_file).and_then(|m| m.modified())
- {
- if src_mtime > metallib_mtime {
- return true;
- }
- } else {
- // If any file's metadata can't be read, assume we should rebuild
- return true;
- }
- }
- false
-}
-
-fn compile_metal_files(
- metal_files: &[PathBuf],
- out_dir: &Path,
- sdk: &str,
- src_dir: &Path,
- nax_enabled: bool,
-) {
- // Create temp directory for .air files
- let air_dir = out_dir.join("air");
- fs::create_dir_all(&air_dir).unwrap();
- // Keep clang's module cache inside the build output to avoid writing to
- // system-wide cache directories that may not be writable.
- let module_cache_dir = out_dir.join("clang-module-cache");
- fs::create_dir_all(&module_cache_dir).unwrap();
-
- // Collect include directories (parents of each metal file + src dir)
- let mut include_dirs = HashSet::new();
- include_dirs.insert(src_dir.to_path_buf());
-
- // Add kernel root and common/matmul subdirectories to satisfy kernel includes
- let kernel_dir = src_dir.join("backends/metal/kernel");
- if kernel_dir.exists() {
- include_dirs.insert(kernel_dir.clone());
- let common_dir = kernel_dir.join("common");
- if common_dir.exists() {
- include_dirs.insert(common_dir);
- }
-
- let matmul_dir = kernel_dir.join("matmul");
- if matmul_dir.exists() {
- include_dirs.insert(matmul_dir.clone());
-
- let matmul_common_dir = matmul_dir.join("common");
- if matmul_common_dir.exists() {
- include_dirs.insert(matmul_common_dir);
- }
-
- let gemv_dir = matmul_dir.join("gemv");
- if gemv_dir.exists() {
- include_dirs.insert(gemv_dir);
- }
-
- let steel_dir = matmul_dir.join("steel");
- if steel_dir.exists() {
- include_dirs.insert(steel_dir.clone());
-
- let steel_gemm_dir = steel_dir.join("gemm");
- if steel_gemm_dir.exists() {
- include_dirs.insert(steel_gemm_dir.clone());
- let steel_gemm_kernels_dir = steel_gemm_dir.join("kernels");
- if steel_gemm_kernels_dir.exists() {
- include_dirs.insert(steel_gemm_kernels_dir);
- }
- }
-
- let steel_utils_dir = steel_dir.join("utils");
- if steel_utils_dir.exists() {
- include_dirs.insert(steel_utils_dir);
- }
- }
- }
- }
-
- for metal_file in metal_files {
- if let Some(parent) = metal_file.parent() {
- include_dirs.insert(parent.to_path_buf());
- if let Some(grand) = parent.parent() {
- include_dirs.insert(grand.to_path_buf());
- }
- }
- }
-
- // Optimization level: Metal supports up to -O2
- let opt = env::var("OPT_LEVEL").unwrap_or_else(|_| "0".into());
- let metal_opt_flag = match opt.as_str() {
- "0" => "-O1", // NOTE TODO FIXME WORKAROUND: matmul kernels compiled with -O0 are broken and require a reboot to unfreeze the os
- "1" => "-O1",
- _ => "-O2", // treat levels 2,3,s,z as O2 for metal
- };
-
- let metal_std = if nax_enabled {
- "metal4.0"
- } else {
- "metal3.1"
- };
-
- // Platform-specific min version flags.
- // - Metal 3.1 requires macOS 14 / iOS 17
- // - NAX kernels require macOS/iOS 26+ (Metal 4)
- let platform_flags = if nax_enabled {
- if sdk == "macosx" || sdk == "maccatalyst" {
- vec!["-mmacosx-version-min=26.0"]
- } else {
- vec!["-mios-version-min=26.0"]
- }
- } else if sdk == "macosx" || sdk == "maccatalyst" {
- vec!["-mmacosx-version-min=14.0"]
- } else {
- vec!["-mios-version-min=17.0"]
- };
-
- // Compile each .metal source to an intermediate .air file
- let mut air_files = Vec::new();
- for metal_file in metal_files {
- let name = metal_file.file_stem().unwrap().to_str().unwrap();
- // Name .air with platform prefix to avoid collisions
- let prefix = if sdk == "macosx" || sdk == "maccatalyst" {
- "macos_"
- } else {
- "ios_"
- };
- let air_path = air_dir.join(format!("{}{}.air", prefix, name));
- air_files.push(air_path.clone());
-
- // Invoke Metal compiler
- let mut cmd = Command::new("xcrun");
- cmd.args(&["-sdk", sdk, "metal", metal_opt_flag]);
- cmd.arg(format!("-std={}", metal_std));
- cmd.arg(format!(
- "-fmodules-cache-path={}",
- module_cache_dir.to_string_lossy()
- ));
- for flag in &platform_flags {
- cmd.arg(flag);
- }
- for inc in &include_dirs {
- cmd.args(&["-I", inc.to_str().unwrap()]);
- }
- cmd.env("CLANG_MODULE_CACHE_PATH", &module_cache_dir);
- if opt == "0" {
- cmd.arg("-gline-tables-only");
- if sdk == "macosx" {
- cmd.args(&["-frecord-sources", "-fpreserve-invariance"]);
- }
- }
- cmd.args(&[
- "-c",
- metal_file.to_str().unwrap(),
- "-o",
- air_path.to_str().unwrap(),
- ]);
-
- println!(
- "cargo:info=Running Metal compiler: {}",
- cmd.get_args()
- .map(|a| a.to_string_lossy())
- .collect::<Vec<_>>()
- .join(" ")
- );
- if !cmd.status().expect("Failed to run metal compiler").success() {
- panic!(
- "Metal shader compilation failed for {}",
- metal_file.display()
- );
- }
- }
-
- // Link all .air files into a single .metallib library
- let metallib_path =
- out_dir.join(if sdk == "macosx" || sdk == "maccatalyst" {
- "shaders_macos.metallib"
- } else {
- "shaders_ios.metallib"
- });
- let mut lib_cmd = Command::new("xcrun");
- lib_cmd.args(&["-sdk", sdk, "metallib"]);
- for air in &air_files {
- lib_cmd.arg(air);
- }
- lib_cmd.args(&["-o", metallib_path.to_str().unwrap()]);
- println!(
- "cargo:info=Linking Metal libraries: {}",
- lib_cmd
- .get_args()
- .map(|a| a.to_string_lossy())
- .collect::<Vec<_>>()
- .join(" ")
- );
- if !lib_cmd.status().expect("Failed to run metallib linker").success() {
- panic!("Failed to link Metal shader libraries");
- }
-
- // Clean up intermediate .air files
- for air in air_files {
- let _ = fs::remove_file(air);
- }
- println!(
- "cargo:info=Metal shaders compiled successfully to {}",
- metallib_path.display()
- );
-}
-
-fn write_metallib_include(
- platform_suffix: &str,
- out_path: &Path,
-) {
- let mut f = fs::File::create(out_path).expect("Cannot create metal_lib.rs");
- let metallib_file_name = format!("shaders_{}.metallib", platform_suffix);
- writeln!(f, "// AUTO-GENERATED Metal library include").unwrap();
- writeln!(
- f,
- "pub const METAL_LIBRARY_DATA: &[u8] = include_bytes!(concat!(env!(\"OUT_DIR\"), \"/metallib/{}\"));",
- metallib_file_name
- )
- .unwrap();
-}
-
-fn write_empty_metallib() {
- let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
- let metal_lib_rs = out_dir.join("metal_lib.rs");
-
- if let Some(parent) = metal_lib_rs.parent() {
- let _ = fs::create_dir_all(parent);
- }
- fs::write(metal_lib_rs, b"pub const METAL_LIBRARY_DATA: &[u8] = &[];\n")
- .expect("Cannot create stub metal_lib.rs");
-}
diff --git a/crates/uzu/build/common/build_debug.rs b/crates/uzu/build/common/build_debug.rs
deleted file mode 100644
index b3d0f3315..000000000
--- a/crates/uzu/build/common/build_debug.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-use std::sync::OnceLock;
-use std::time::Instant;
-
-static START: OnceLock<Instant> = OnceLock::new();
-
-pub fn log(args: std::fmt::Arguments) {
- let start = START.get_or_init(Instant::now);
- let elapsed_ms = start.elapsed().as_millis();
- println!("cargo::warning=(build-debug) [{elapsed_ms}ms] {args}");
-}
-
-#[macro_export]
-macro_rules! debug_log {
- ($($arg:tt)*) => {{
- if cfg!(feature = "build-debug") {
- $crate::common::build_debug::log(format_args!($($arg)*));
- }
- }};
-}
diff --git a/crates/uzu/build/common/caching.rs b/crates/uzu/build/common/caching.rs
new file mode 100644
index 000000000..84e1b498f
--- /dev/null
+++ b/crates/uzu/build/common/caching.rs
@@ -0,0 +1,20 @@
+use anyhow::Context;
+use std::{fs, sync::OnceLock};
+
+static BUILD_SYSTEM_HASH: OnceLock<blake3::Hash> = OnceLock::new();
+
+pub fn build_system_hash() -> anyhow::Result<&'static blake3::Hash> {
+ if let Some(bsh) = BUILD_SYSTEM_HASH.get() {
+ Ok(bsh)
+ } else {
+ let bsh = blake3::hash(
+ &fs::read(
+ &std::env::current_exe()
+ .context("cannot get build system executable")?,
+ )
+ .context("cannot read build system executable")?,
+ );
+
+ Ok(BUILD_SYSTEM_HASH.get_or_init(|| bsh))
+ }
+}
diff --git a/crates/uzu/build/common/compiler.rs b/crates/uzu/build/common/compiler.rs
new file mode 100644
index 000000000..5f402a2ed
--- /dev/null
+++ b/crates/uzu/build/common/compiler.rs
@@ -0,0 +1,6 @@
+use async_trait::async_trait;
+
+#[async_trait]
+pub trait Compiler {
+ async fn build(&self) -> anyhow::Result<()>;
+}
diff --git a/crates/uzu/build/common/envs.rs b/crates/uzu/build/common/envs.rs
new file mode 100644
index 000000000..927349201
--- /dev/null
+++ b/crates/uzu/build/common/envs.rs
@@ -0,0 +1,22 @@
+use std::sync::OnceLock;
+
+static BUILD_DEBUG: OnceLock<bool> = OnceLock::new();
+static BUILD_ALWAYS: OnceLock<bool> = OnceLock::new();
+static BUILD_CLEAN: OnceLock<bool> = OnceLock::new();
+
+fn env_flag(name: &str) -> bool {
+ println!("cargo::rerun-if-env-changed={name}");
+ std::env::var(name).is_ok()
+}
+
+pub fn build_debug() -> bool {
+ *BUILD_DEBUG.get_or_init(|| env_flag("BUILD_DEBUG"))
+}
+
+pub fn build_always() -> bool {
+ *BUILD_ALWAYS.get_or_init(|| env_flag("BUILD_ALWAYS"))
+}
+
+pub fn build_clean() -> bool {
+ *BUILD_CLEAN.get_or_init(|| env_flag("BUILD_CLEAN"))
+}
diff --git a/crates/uzu/build/common/logging.rs b/crates/uzu/build/common/logging.rs
new file mode 100644
index 000000000..2e25f91f9
--- /dev/null
+++ b/crates/uzu/build/common/logging.rs
@@ -0,0 +1,23 @@
+use std::sync::OnceLock;
+use std::time::Instant;
+
+use crate::common::envs;
+
+static START: OnceLock<Instant> = OnceLock::new();
+
+pub fn elapsed_ms() -> u128 {
+ START.get_or_init(Instant::now).elapsed().as_millis()
+}
+
+pub fn _debug_log(args: std::fmt::Arguments) {
+ if envs::build_debug() {
+ println!("cargo::warning=(build-debug) [{}ms] {}", elapsed_ms(), args);
+ }
+}
+
+#[macro_export]
+macro_rules! debug_log {
+ ($($arg:tt)*) => {{
+ $crate::common::logging::_debug_log(format_args!($($arg)*));
+ }};
+}
diff --git a/crates/uzu/build/common/mod.rs b/crates/uzu/build/common/mod.rs
index 78c5959e9..3b41dfd7f 100644
--- a/crates/uzu/build/common/mod.rs
+++ b/crates/uzu/build/common/mod.rs
@@ -1 +1,4 @@
-pub(crate) mod build_debug;
+pub mod caching;
+pub mod compiler;
+pub mod envs;
+pub mod logging;
diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rs
index 6e86b0025..918d1228e 100644
--- a/crates/uzu/build/main.rs
+++ b/crates/uzu/build/main.rs
@@ -1,17 +1,28 @@
use std::{env, fs, path::PathBuf};
use anyhow::Context;
-use tokio::try_join;
mod common;
mod metal;
mod shared_types;
+use common::compiler::Compiler;
+use common::envs;
+use futures::future::try_join_all;
+
#[tokio::main(flavor = "current_thread")]
async fn main() -> anyhow::Result<()> {
+ println!("cargo::rerun-if-changed=.");
+
+ if envs::build_always() {
+ println!(
+ "cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun"
+ );
+ }
+
debug_log!("build script started");
- if cfg!(feature = "build-clean") {
+ if envs::build_clean() {
let out_dir =
PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
if out_dir.exists() {
@@ -25,7 +36,14 @@ async fn main() -> anyhow::Result<()> {
debug_log!("cleaned caches");
}
- try_join!(metal::main(), shared_types::main())?;
+ let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();
+
+ if cfg!(feature = "metal") {
+ compilers.push(Box::new(metal::MetalCompiler::new()?));
+ compilers.push(Box::new(shared_types::SharedTypesCompiler::new()?));
+ }
+
+ try_join_all(compilers.iter().map(|c| c.build())).await?;
debug_log!("build script ended");
diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rs
index 8e448b740..8e2bdb401 100644
--- a/crates/uzu/build/metal/compiler.rs
+++ b/crates/uzu/build/metal/compiler.rs
@@ -1,26 +1,43 @@
use std::collections::HashMap;
use std::iter::{once, zip};
+use std::path::Path;
use std::{env, fs, path::PathBuf};
use anyhow::{Context, bail};
use futures::future::try_join_all;
+use futures::{StreamExt, TryStreamExt, stream};
+use proc_macro2::TokenStream;
+use quote::quote;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncReadExt;
use tokio::task::spawn_blocking;
+use walkdir::WalkDir;
+use async_trait::async_trait;
+
+use crate::common::compiler::Compiler;
+use crate::common::{caching, envs};
use crate::debug_log;
use super::ast::{MetalArgumentType, MetalKernelInfo};
use super::toolchain::MetalToolchain;
#[derive(Serialize, Deserialize, Debug)]
-pub struct ObjectInfo {
- pub object_path: PathBuf,
- pub kernels: Box<[MetalKernelInfo]>,
- pub dependency_hashes: HashMap<Box<str>, [u8; blake3::OUT_LEN]>,
+struct ObjectInfo {
+ object_path: PathBuf,
+ kernels: Box<[MetalKernelInfo]>,
+ buildsystem_hash: [u8; blake3::OUT_LEN],
+ dependency_hashes: HashMap<Box<str>, [u8; blake3::OUT_LEN]>,
+}
+
+fn is_nax_source(path: &Path) -> bool {
+ path.file_name()
+ .and_then(|s| s.to_str())
+ .map(|s| s.contains("_nax"))
+ .unwrap_or(false)
}
-pub fn wrappers(kernel: &MetalKernelInfo) -> anyhow::Result<Box<[Box<str>]>> {
+fn wrappers(kernel: &MetalKernelInfo) -> anyhow::Result<Box<[Box<str>]>> {
let mut kernel_wrappers = Vec::new();
for specialization_variant in kernel
@@ -190,15 +207,47 @@ async fn hash_dependencies(
try_join_all(futures).await.map(|v| v.into_iter().collect())
}
+fn objects_hash<'a>(
+ objects: impl IntoIterator<Item = &'a ObjectInfo>
+) -> anyhow::Result<blake3::Hash> {
+ let mut hasher = blake3::Hasher::new();
+ hasher.update(
+ caching::build_system_hash()
+ .context("cannot get build system hash")?
+ .as_bytes(),
+ );
+ let mut paths: Vec<_> =
+ objects.into_iter().map(|o| &o.object_path).collect();
+ paths.sort();
+ for path in paths {
+ let path_bytes = path.to_string_lossy();
+ let path_bytes = path_bytes.as_bytes();
+ hasher.update(&(path_bytes.len() as u32).to_le_bytes());
+ hasher.update(path_bytes);
+ let contents = fs::read(path)
+ .with_context(|| format!("cannot read {}", path.display()))?;
+ hasher.update(&(contents.len() as u32).to_le_bytes());
+ hasher.update(&contents);
+ }
+ Ok(hasher.finalize())
+}
+
#[derive(Debug)]
pub struct MetalCompiler {
- out_dir: PathBuf,
+ src_dir: PathBuf,
build_dir: PathBuf,
+ out_dir: PathBuf,
toolchain: MetalToolchain,
}
impl MetalCompiler {
pub fn new() -> anyhow::Result<Self> {
+ let src_dir = PathBuf::from(
+ env::var("CARGO_MANIFEST_DIR")
+ .context("missing CARGO_MANIFEST_DIR")?,
+ )
+ .join("src");
+
let out_dir =
PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
let build_dir = out_dir.join("metal");
@@ -210,16 +259,22 @@ impl MetalCompiler {
MetalToolchain::from_env().context("cannot create toolchain")?;
Ok(Self {
- out_dir,
+ src_dir,
build_dir,
+ out_dir,
toolchain,
})
}
- pub async fn compile(
+ async fn compile(
&self,
source_path: PathBuf,
- ) -> anyhow::Result<(ObjectInfo, bool)> {
+ ) -> anyhow::Result<ObjectInfo> {
+ let buildsystem_hash = caching::build_system_hash()
+ .context("cannot get build system cache")?
+ .as_bytes()
+ .clone();
+
let source_path_pretty =
source_path.file_name().unwrap().to_str().unwrap();
debug_log!("compile start: {source_path_pretty}");
@@ -242,12 +297,13 @@ impl MetalCompiler {
if let Ok(contents) = tokio::fs::read(&objectinfo_path).await
&& let Ok(cached) =
serde_json::from_slice::<ObjectInfo>(&contents)
- && let Ok(current_hashes) =
+ && buildsystem_hash == cached.buildsystem_hash
+ && let Ok(dependency_hashes) =
hash_dependencies(cached.dependency_hashes.keys().cloned())
.await
- && current_hashes == cached.dependency_hashes
+ && dependency_hashes == cached.dependency_hashes
{
- if cfg!(feature = "build-debug") {
+ if envs::build_debug() {
let kernel_list = cached
.kernels
.iter()
@@ -262,7 +318,7 @@ impl MetalCompiler {
);
}
}
- return Ok((cached, true));
+ return Ok(cached);
}
fs::remove_dir_all(&build_dir).ok();
}
@@ -317,6 +373,7 @@ impl MetalCompiler {
let object_info = ObjectInfo {
object_path,
kernels: kernel_infos.into_boxed_slice(),
+ buildsystem_hash,
dependency_hashes,
};
@@ -330,7 +387,7 @@ impl MetalCompiler {
format!("cannot write object info {}", objectinfo_path.display())
})?;
- if cfg!(feature = "build-debug") {
+ if envs::build_debug() {
let kernel_list = object_info
.kernels
.iter()
@@ -346,16 +403,26 @@ impl MetalCompiler {
}
}
- Ok((object_info, false))
+ Ok(object_info)
}
- pub async fn link(
+ async fn link<'a>(
&self,
- objects: impl IntoIterator<Item = &ObjectInfo>,
+ objects: impl IntoIterator<Item = &'a ObjectInfo> + Clone,
) -> anyhow::Result<PathBuf> {
- debug_log!("linking start");
-
let library_path = self.out_dir.join("default.metallib");
+ let hash_path = self.out_dir.join("default.metallib.hash");
+
+ let hash = objects_hash(objects.clone())?;
+
+ if let Ok(cached_hash) = fs::read_to_string(&hash_path)
+ && cached_hash == hash.to_string()
+ {
+ debug_log!("linking cached");
+ return Ok(library_path);
+ }
+
+ debug_log!("linking start");
let link_output = self
.toolchain
@@ -369,8 +436,93 @@ impl MetalCompiler {
}
}
+ fs::write(&hash_path, hash.to_string()).with_context(|| {
+ format!("cannot write hash file {}", hash_path.display())
+ })?;
+
debug_log!("linking end");
Ok(library_path)
}
+
+ fn bindgen<'a>(
+ &self,
+ objects: impl IntoIterator<Item = &'a ObjectInfo> + Clone,
+ ) -> anyhow::Result<()> {
+ let out_path = self.out_dir.join("dsl.rs");
+ let hash_path = self.out_dir.join("dsl.rs.hash");
+
+ let hash = objects_hash(objects.clone())?;
+
+ if let Ok(cached_hash) = fs::read_to_string(&hash_path)
+ && cached_hash == hash.to_string()
+ {
+ debug_log!("bindgen cached");
+ return Ok(());
+ }
+
+ debug_log!("bindgen start");
+
+ let bindings = objects
+ .into_iter()
+ .flat_map(|o| &o.kernels)
+ .map(|k| {
+ super::bindgen::bindgen(k).with_context(|| {
+ format!("cannot generate bindings for {}", k.name)
+ })
+ })
+ .collect::<anyhow::Result<Vec<TokenStream>>>()?;
+
+ let tokens = quote! { #(#bindings)* };
+
+ let parsed =
+ syn::parse2(tokens).context("cannot parse generated bindings")?;
+ fs::write(&out_path, prettyplease::unparse(&parsed)).with_context(
+ || format!("cannot write bindings file {}", out_path.display()),
+ )?;
+
+ fs::write(&hash_path, hash.to_string()).with_context(|| {
+ format!("cannot write hash file {}", hash_path.display())
+ })?;
+
+ debug_log!("bindgen end");
+
+ Ok(())
+ }
+}
+
+#[async_trait]
+impl Compiler for MetalCompiler {
+ async fn build(&self) -> anyhow::Result<()> {
+ let nax_enabled = cfg!(feature = "metal-nax");
+ debug_log!("metal nax {}", if nax_enabled { "enabled" } else { "disabled" });
+
+ let metal_sources: Vec<PathBuf> = WalkDir::new(&self.src_dir)
+ .into_iter()
+ .filter_map(|e| e.ok())
+ .filter(|e| {
+ e.file_type().is_file()
+ && e.path().extension().and_then(|s| s.to_str())
+ == Some("metal")
+ })
+ .map(|e| e.into_path())
+ .filter(|p| nax_enabled || !is_nax_source(p))
+ .collect();
+
+ let num_concurrent_compiles =
+ std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4)
+ * 2;
+
+ let objects: Vec<ObjectInfo> = stream::iter(metal_sources)
+ .map(|p| self.compile(p))
+ .buffer_unordered(num_concurrent_compiles)
+ .try_collect()
+ .await
+ .context("cannot compile metal sources")?;
+
+ self.link(&objects).await.context("cannot link objects")?;
+ self.bindgen(&objects).context("cannot generate bindings")?;
+
+ Ok(())
+ }
}
diff --git a/crates/uzu/build/metal/mod.rs b/crates/uzu/build/metal/mod.rs
index b4abe8ed2..ce9ce685c 100644
--- a/crates/uzu/build/metal/mod.rs
+++ b/crates/uzu/build/metal/mod.rs
@@ -1,107 +1,6 @@
-use std::{
- env, fs,
- path::{Path, PathBuf},
-};
-
-use anyhow::Context;
-use compiler::{MetalCompiler, ObjectInfo};
-use futures::{StreamExt, TryStreamExt, stream};
-use proc_macro2::TokenStream;
-use quote::quote;
-use walkdir::WalkDir;
-
-use crate::debug_log;
-
mod ast;
mod bindgen;
mod compiler;
mod toolchain;
-fn is_nax_source(path: &Path) -> bool {
- path.file_name()
- .and_then(|s| s.to_str())
- .map(|s| s.contains("_nax"))
- .unwrap_or(false)
-}
-
-pub async fn main() -> anyhow::Result<()> {
- let out_dir =
- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
-
- let nax_enabled = cfg!(feature = "metal-nax");
- debug_log!("metal nax enabled: {nax_enabled}");
-
- let nax_mode_path = out_dir.join("metal_nax_mode");
- let desired_nax_mode = if nax_enabled { "1" } else { "0" };
- let prev_nax_mode =
- fs::read_to_string(&nax_mode_path).ok().map(|s| s.trim().to_owned());
- let nax_mode_changed =
- prev_nax_mode.as_deref() != Some(desired_nax_mode);
-
- let dsl = MetalCompiler::new().context("cannot create metal dsl")?;
-
- let src_dir = PathBuf::from(
- env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?,
- )
- .join("src");
-
- let metal_sources: Vec<PathBuf> = WalkDir::new(&src_dir)
- .into_iter()
- .filter_map(|e| e.ok())
- .filter(|e| {
- e.file_type().is_file()
- && e.path().extension().and_then(|s| s.to_str())
- == Some("metal")
- })
- .map(|e| e.into_path())
- .filter(|p| nax_enabled || !is_nax_source(p))
- .collect();
-
- let num_concurrent_compiles =
- std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4) * 2;
-
- let results: Vec<(ObjectInfo, bool)> = stream::iter(metal_sources)
- .map(|p| dsl.compile(p))
- .buffer_unordered(num_concurrent_compiles)
- .try_collect()
- .await
- .context("cannot compile metal sources")?;
-
- let all_cached = results.iter().all(|(_, cached)| *cached);
- let kernels: Vec<ObjectInfo> =
- results.into_iter().map(|(o, _)| o).collect();
-
- if !all_cached || nax_mode_changed {
- dsl.link(&kernels).await.context("cannot link objects")?;
-
- let bindings = kernels
- .iter()
- .flat_map(|k| &k.kernels)
- .map(|k| {
- bindgen::bindgen(k).with_context(|| {
- format!("cannot generate bindings for {}", k.name)
- })
- })
- .collect::<anyhow::Result<Vec<TokenStream>>>()?;
-
- let tokens = quote! { #(#bindings)* };
-
- let out_path =
- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?)
- .join("dsl.rs");
- let parsed =
- syn::parse2(tokens).context("cannot parse generated bindings")?;
- fs::write(&out_path, prettyplease::unparse(&parsed)).with_context(
- || format!("cannot write bindings file {}", out_path.display()),
- )?;
-
- fs::write(&nax_mode_path, desired_nax_mode).with_context(|| {
- format!(
- "cannot write nax mode file {}",
- nax_mode_path.display()
- )
- })?;
- }
-
- Ok(())
-}
+pub use compiler::MetalCompiler;
diff --git a/crates/uzu/build/shared_types/mod.rs b/crates/uzu/build/shared_types/mod.rs
index da478d473..7921cf6f3 100644
--- a/crates/uzu/build/shared_types/mod.rs
+++ b/crates/uzu/build/shared_types/mod.rs
@@ -1,9 +1,13 @@
use std::{env, fs, path::PathBuf};
-use crate::debug_log;
use anyhow::Context;
use tokio::{task::spawn_blocking, try_join};
+use async_trait::async_trait;
+
+use crate::common::compiler::Compiler;
+use crate::debug_log;
+
struct BindgenConfig {
name: &'static str,
kernel_subdir: &'static str,
@@ -36,79 +40,130 @@ const ATTENTION_CONFIG: BindgenConfig = BindgenConfig {
raw_lines: &[],
};
-fn autogen_shared_types(config: &BindgenConfig) -> anyhow::Result<()> {
- let out_dir = env::var("OUT_DIR").context("missing OUT_DIR")?;
- let build_dir = PathBuf::from(out_dir).join("shared_types");
- fs::create_dir_all(&build_dir).with_context(|| {
- format!("cannot create shared types build dir {}", build_dir.display())
- })?;
-
- let manifest_dir = PathBuf::from(
- env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?,
- );
- let kernel_dir = manifest_dir
- .join(format!("src/backends/metal/kernel/{}", config.kernel_subdir));
- let types_header = kernel_dir.join(config.header_file);
- let types_header_hash: [u8; blake3::OUT_LEN] =
- blake3::hash(&fs::read(types_header.as_path()).with_context(|| {
- format!("cannot read {}", types_header.display())
- })?)
- .into();
- let cached_types_header_hash =
- build_dir.join(format!("{}.hash", config.header_file));
-
- if let Some(cached_hash) = fs::read(&cached_types_header_hash).ok()
- && cached_hash == types_header_hash
- {
- debug_log!("{} bindgen cached", config.name);
- return Ok(());
- }
- debug_log!("{} bindgen started", config.name);
-
- let mut builder = bindgen::Builder::default()
- .header(types_header.to_string_lossy())
- .clang_arg("-x")
- .clang_arg("c")
- .derive_default(true)
- .derive_copy(true)
- .use_core();
-
- for raw_line in config.raw_lines {
- builder = builder.raw_line(*raw_line);
- }
-
- for ty in config.allowlist_types {
- builder = builder.allowlist_type(*ty);
- }
+#[derive(Debug)]
+pub struct SharedTypesCompiler {
+ src_dir: PathBuf,
+ build_dir: PathBuf,
+}
- let bindings = builder.generate().map_err(|e| {
- anyhow::anyhow!(
- "failed to generate bindings from {}: {e}",
- config.header_file
+impl SharedTypesCompiler {
+ pub fn new() -> anyhow::Result<Self> {
+ let src_dir = PathBuf::from(
+ env::var("CARGO_MANIFEST_DIR")
+ .context("missing CARGO_MANIFEST_DIR")?,
)
- })?;
-
- let bindings_path = kernel_dir.join(config.output_file);
- bindings.write_to_file(&bindings_path).with_context(|| {
- format!("failed to write {}", bindings_path.display())
- })?;
-
- fs::write(&cached_types_header_hash, &types_header_hash).with_context(
- || format!("failed to write {}", cached_types_header_hash.display()),
- )?;
- debug_log!("{} bindgen done", config.name);
+ .join("src");
+
+ let build_dir =
+ PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?)
+ .join("shared_types");
+ fs::create_dir_all(&build_dir).with_context(|| {
+ format!("cannot create {}", build_dir.display())
+ })?;
+
+ Ok(Self {
+ src_dir,
+ build_dir,
+ })
+ }
- Ok(())
+ fn generate(
+ &self,
+ config: &BindgenConfig,
+ ) -> anyhow::Result<()> {
+ let kernel_dir = self
+ .src_dir
+ .join(format!("backends/metal/kernel/{}", config.kernel_subdir));
+ let types_header = kernel_dir.join(config.header_file);
+ let types_header_hash: [u8; blake3::OUT_LEN] =
+ blake3::hash(&fs::read(&types_header).with_context(|| {
+ format!("cannot read {}", types_header.display())
+ })?)
+ .into();
+ let cached_types_header_hash =
+ self.build_dir.join(format!("{}.hash", config.header_file));
+
+ if let Some(cached_hash) = fs::read(&cached_types_header_hash).ok()
+ && cached_hash == types_header_hash
+ {
+ debug_log!("{} bindgen cached", config.name);
+ return Ok(());
+ }
+ debug_log!("{} bindgen started", config.name);
+
+ let mut builder = bindgen::Builder::default()
+ .header(types_header.to_string_lossy())
+ .clang_arg("-x")
+ .clang_arg("c")
+ .derive_default(true)
+ .derive_copy(true)
+ .use_core();
+
+ for raw_line in config.raw_lines {
+ builder = builder.raw_line(*raw_line);
+ }
+
+ for ty in config.allowlist_types {
+ builder = builder.allowlist_type(*ty);
+ }
+
+ let bindings = builder.generate().map_err(|e| {
+ anyhow::anyhow!(
+ "failed to generate bindings from {}: {e}",
+ config.header_file
+ )
+ })?;
+
+ let bindings_path = kernel_dir.join(config.output_file);
+ bindings.write_to_file(&bindings_path).with_context(|| {
+ format!("failed to write {}", bindings_path.display())
+ })?;
+
+ fs::write(&cached_types_header_hash, &types_header_hash).with_context(
+ || {
+ format!(
+ "failed to write {}",
+ cached_types_header_hash.display()
+ )
+ },
+ )?;
+ debug_log!("{} bindgen done", config.name);
+
+ Ok(())
+ }
}
-pub async fn main() -> anyhow::Result<()> {
- let (matmul_result, attn_result) = try_join!(
- spawn_blocking(|| autogen_shared_types(&MATMUL_CONFIG)),
- spawn_blocking(|| autogen_shared_types(&ATTENTION_CONFIG))
- )?;
-
- matmul_result?;
- attn_result?;
-
- Ok(())
+#[async_trait]
+impl Compiler for SharedTypesCompiler {
+ async fn build(&self) -> anyhow::Result<()> {
+ let (matmul_result, attn_result) = try_join!(
+ spawn_blocking({
+ let src_dir = self.src_dir.clone();
+ let build_dir = self.build_dir.clone();
+ move || {
+ SharedTypesCompiler {
+ src_dir,
+ build_dir,
+ }
+ .generate(&MATMUL_CONFIG)
+ }
+ }),
+ spawn_blocking({
+ let src_dir = self.src_dir.clone();
+ let build_dir = self.build_dir.clone();
+ move || {
+ SharedTypesCompiler {
+ src_dir,
+ build_dir,
+ }
+ .generate(&ATTENTION_CONFIG)
+ }
+ })
+ )?;
+
+ matmul_result?;
+ attn_result?;
+
+ Ok(())
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
index 70ad744f0..4c80a9a53 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
@@ -307,8 +307,12 @@ impl RMSNormKernel {
// We only ever normalize within [Q] and/or [K].
let (head_offset, head_count): (u32, u32) = match args.target {
QKNormTarget::QueryHeads => (0, args.num_q_heads as u32),
- QKNormTarget::KeyHeads => (args.num_q_heads as u32, args.num_kv_heads as u32),
- QKNormTarget::Both => (0, (args.num_q_heads + args.num_kv_heads) as u32),
+ QKNormTarget::KeyHeads => {
+ (args.num_q_heads as u32, args.num_kv_heads as u32)
+ },
+ QKNormTarget::Both => {
+ (0, (args.num_q_heads + args.num_kv_heads) as u32)
+ },
};
if args.batch_size <= 0 || args.head_dim <= 0 || head_count == 0 {
@@ -329,12 +333,14 @@ impl RMSNormKernel {
// One SIMD-group per head, multiple heads per threadgroup.
let simd_width = self.pipeline.thread_execution_width() as u64;
- let max_threads = self.pipeline.max_total_threads_per_threadgroup() as u64;
+ let max_threads =
+ self.pipeline.max_total_threads_per_threadgroup() as u64;
let max_heads_per_threadgroup = (max_threads / simd_width).max(1);
let heads_per_threadgroup =
(head_count as u64).min(max_heads_per_threadgroup).max(1);
- let num_head_tiles = (head_count as u64).div_ceil(heads_per_threadgroup);
+ let num_head_tiles =
+ (head_count as u64).div_ceil(heads_per_threadgroup);
let threadgroups_per_grid = MTLSize {
width: args.batch_size as u64,