A high-performance inference engine for AI models
Shard metal compilation (#758)
uuuvn0cf3b02fe0f6
diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.tomlindex ad3f8484e..d5ccdb8a1 100644--- a/crates/backend-uzu/Cargo.toml+++ b/crates/backend-uzu/Cargo.toml@@ -23,6 +23,7 @@ metal = ["dep:serde_json","dep:objc2","dep:obfstr",+ "dep:xxhash-rust","dep:zstd","dep:metal",]@@ -51,6 +52,7 @@ rhai.workspace = trueclang-ast = { workspace = true, optional = true }tempfile = { workspace = true, optional = true }serde_json = { workspace = true, features = ["unbounded_depth"], optional = true }+xxhash-rust = { workspace = true, features = ["xxh3"], optional = true }zstd = { workspace = true, optional = true }[dependencies]@@ -66,7 +68,6 @@ thiserror.workspace = truetokenizers.workspace = truerangemap.workspace = truebytemuck = { workspace = true, features = ["derive"] }-xxhash-rust = { workspace = true, features = ["xxh3"] }memmap2.workspace = truenum-traits.workspace = truelibm.workspace = true@@ -80,6 +81,7 @@ tokio-util.workspace = true[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]objc2 = { workspace = true, optional = true }obfstr = { workspace = true, optional = true }+xxhash-rust = { workspace = true, features = ["xxh3"], optional = true }zstd = { workspace = true, optional = true }metal = { workspace = true, optional = true }diff --git a/crates/backend-uzu/build/metal/bindgen/mod.rs b/crates/backend-uzu/build/metal/bindgen/mod.rsindex dda895ef4..1633fbd0e 100644--- a/crates/backend-uzu/build/metal/bindgen/mod.rs+++ b/crates/backend-uzu/build/metal/bindgen/mod.rs@@ -18,6 +18,7 @@ pub fn bindgen(specialize_indices: &SpecializeBaseIndices,enum_paths: &EnumPaths,library_const: &proc_macro2::Ident,+ num_shards: usize,library_compressed: bool,) -> Result<(TokenStream, Option<TokenStream>)> {let kernel_name = kernel.name.as_ref();@@ -69,6 +70,13 @@ pub fn bindgen(let dispatch_code = &dispatch_emission.dispatch_code;let empty_dispatch_guards = &dispatch_emission.empty_dispatch_guards;+ let library_data = if num_shards == 1 {+ quote! { #library_const[0] }+ } else {+ let num_shards = num_shards as u64;+ quote! { #library_const[(xxhash_rust::xxh3::xxh3_64(entry_name.as_bytes()) % #num_shards) as usize] }+ };+let trait_implementation_for = &trait_wiring.trait_implementation_for;let associate_backend = &trait_wiring.associate_backend;let method_visibility = &trait_wiring.method_visibility;@@ -97,7 +105,7 @@ pub fn bindgen() -> Result<Self, MetalError> {let entry_name = #entry_name;#function_constants_initialization- let pipeline = context.compute_pipeline_state(#library_const, #library_compressed, #cache_key, &entry_name, #function_constants_argument)?;+ let pipeline = context.compute_pipeline_state(#library_data, #library_compressed, #cache_key, &entry_name, #function_constants_argument)?;Ok(Self {pipeline#(, #conditional_buffer_initializers)*diff --git a/crates/backend-uzu/build/metal/compiler.rs b/crates/backend-uzu/build/metal/compiler.rsindex cad996ede..b732f9110 100644--- a/crates/backend-uzu/build/metal/compiler.rs+++ b/crates/backend-uzu/build/metal/compiler.rs@@ -6,12 +6,19 @@ use std::{use anyhow::Context;use async_trait::async_trait;-use futures::{StreamExt, TryStreamExt, stream};+use futures::{StreamExt, TryStreamExt, future::try_join_all, stream};+use itertools::{Itertools, izip};use quote::{format_ident, quote};use serde::{Deserialize, Serialize};use walkdir::WalkDir;+use xxhash_rust::xxh3::xxh3_64;-use super::{ast::MetalKernelInfo, bindgen::bindgen_global, toolchain::MetalToolchain, wrapper::wrappers};+use super::{+ ast::MetalKernelInfo,+ bindgen::bindgen_global,+ toolchain::MetalToolchain,+ wrapper::{KernelWrappers, VariantWrapper, wrappers},+};use crate::{common::{caching, codegen::write_tokens, compiler::Compiler, enum_paths::EnumPaths, gpu_types::GpuTypes,@@ -21,12 +28,54 @@ use crate::{metal::gpu_types::gpu_type_gen,};+const MIN_VARIANTS_PER_SHARD: usize = 8;+const MAX_VARIANTS_PER_SHARD: usize = 64;++fn shard_footers(kernel_wrappers: &[KernelWrappers]) -> Vec<String> {+ let total_variants: usize = kernel_wrappers.iter().map(|kernel| kernel.variants.len()).sum();+ let min_shards = total_variants.div_ceil(MAX_VARIANTS_PER_SHARD);+ let ncpu = std::thread::available_parallelism().map(|x| x.get()).unwrap_or(1);+ let num_shards = total_variants.div_ceil(MIN_VARIANTS_PER_SHARD).clamp(min_shards, min_shards.max(ncpu));+ let num_shards = if num_shards >= ncpu {+ num_shards.div_ceil(ncpu) * ncpu+ } else {+ num_shards+ };++ let mut footers = vec![String::new(); num_shards];+ for kernel in kernel_wrappers {+ let mut emit = |index: usize, variants: Vec<&VariantWrapper>| {+ let footer = &mut footers[index];+ footer.push_str(kernel.header.as_deref().unwrap_or(""));+ footer.push_str(&variants.iter().map(|variant| variant.source.as_ref()).join(""));+ footer.push_str(kernel.footer.as_deref().unwrap_or(""));+ };++ if num_shards == 1 {+ emit(0, kernel.variants.iter().collect());+ } else {+ for (index, variants) in kernel+ .variants+ .iter()+ .into_group_map_by(|variant| (xxh3_64(variant.name.as_bytes()) % num_shards as u64) as usize)+ .into_iter()+ .sorted_unstable_by_key(|(index, _)| *index)+ {+ emit(index, variants);+ }+ }+ }+ footers+}+#[derive(Serialize, Deserialize)]struct Cached {cache_key: [u8; blake3::OUT_LEN],dependency_hashes: HashMap<Box<str>, [u8; blake3::OUT_LEN]>,public_kernels: Box<[Kernel]>,has_kernels: bool,+ num_variants: usize,+ num_shards: usize,}#[derive(Debug)]@@ -110,13 +159,6 @@ impl MetalCompiler {fs::create_dir_all(output_base_path.parent().context("cannot get output directory")?).context("cannot create output directory")?;- let object_file = output_base_path.with_extension("air");- let metallib_file = output_base_path.with_extension("metallib");- let metallib_maybe_compressed_file = if self.metallib_compressed {- metallib_file.with_added_extension("zst")- } else {- metallib_file.clone()- };let bindgen_file = output_base_path.with_extension("rs");let cached_file = output_base_path.with_extension("cached");@@ -130,7 +172,12 @@ impl MetalCompiler {for path in cached.dependency_hashes.keys() {self.emit_rerun_if_changed_for_dependency(path);}- debug_log!("compile cached: {source_path_relative_str}");+ let sharding = if cached.has_kernels {+ format!(" ({} variants / {} shards)", cached.num_variants, cached.num_shards)+ } else {+ Default::default()+ };+ debug_log!("compile cached: {source_path_relative_str}{sharding}");return Ok((kernel_path, cached.public_kernels, cached.has_kernels));}@@ -153,56 +200,70 @@ impl MetalCompiler {.collect::<anyhow::Result<HashMap<Box<str>, [u8; blake3::OUT_LEN]>>>().context("cannot hash dependencies")?;+ let mut num_variants = 0;+ let mut num_shards = 0;if !kernel_infos.is_empty() {- let (wrapper_strs, specialize_indices) =+ let (kernel_wrappers, specialize_indices) =wrappers(&kernel_infos, enum_paths).context("cannot generate kernel wrappers")?;- let mut footer = String::new();- for wrapper in wrapper_strs.iter() {- footer.push_str(wrapper);- }-- let compile_output = self- .toolchain- .compile(&source_path, &footer, &object_file)- .await- .with_context(|| format!("cannot compile {source_path_relative_str}"))?;+ num_variants = kernel_wrappers.iter().map(|kernel| kernel.variants.len()).sum();+ let footers = shard_footers(&kernel_wrappers);+ num_shards = footers.len();- if let Some(warnings) = &compile_output {- for line in warnings.lines() {- println!("cargo::warning={line}");- }- }+ let metallib_files: Vec<PathBuf> = match num_shards {+ 1 => vec![output_base_path.with_extension("metallib")],+ num_shards => {+ (0..num_shards).map(|i| output_base_path.with_extension(format!("shard{i}.metallib"))).collect()+ },+ };- let link_output = self- .toolchain- .link(&object_file, &metallib_file)- .await- .with_context(|| format!("cannot link {source_path_relative_str}"))?;+ let metallib_maybe_compressed_files: Vec<PathBuf> = metallib_files+ .iter()+ .map(|file| {+ if self.metallib_compressed {+ file.with_added_extension("zst")+ } else {+ file.clone()+ }+ })+ .collect();++ let compile_outputs = try_join_all(izip!(&footers, &metallib_files, &metallib_maybe_compressed_files).map(+ |(footer, metallib_file, compressed_file)| {+ let source_path = &source_path;+ async move {+ let warnings = self.toolchain.compile(source_path, footer, metallib_file).await?;++ if self.metallib_compressed {+ let metallib_file = metallib_file.clone();+ let compressed_file = compressed_file.clone();+ tokio::task::spawn_blocking(move || -> anyhow::Result<()> {+ let metallib_source = fs::read(metallib_file)?;+ fs::write(compressed_file, zstd::encode_all(metallib_source.as_slice(), 22)?)?;+ Ok(())+ })+ .await??;+ }++ anyhow::Ok(warnings)+ }+ },+ ))+ .await+ .with_context(|| format!("cannot compile {source_path_relative_str}"))?;- if let Some(warnings) = &link_output {+ for warnings in compile_outputs.into_iter().flatten() {for line in warnings.lines() {println!("cargo::warning={line}");}}- if self.metallib_compressed {- let metallib_file = metallib_file.clone();- let metallib_compressed_file = metallib_maybe_compressed_file.clone();-- tokio::task::spawn_blocking(move || -> anyhow::Result<()> {- let metallib_source = fs::read(&metallib_file)?;- let metallib_compressed = zstd::encode_all(metallib_source.as_slice(), 22)?;- fs::write(&metallib_compressed_file, &metallib_compressed)?;- Ok(())- })- .await??;- };-let library_const =format_ident!("MTLB_{}", blake3::hash(source_path_relative_str.as_bytes()).to_hex().to_uppercase());- let metallib_maybe_compressed_file_str =- metallib_maybe_compressed_file.to_str().context("metallib path is not utf-8")?;+ let metallib_file_strs = metallib_maybe_compressed_files+ .iter()+ .map(|file| file.to_str().context("metallib path is not utf-8"))+ .collect::<anyhow::Result<Vec<_>>>()?;let bindings = kernel_infos.iter()@@ -212,6 +273,7 @@ impl MetalCompiler {&specialize_indices,enum_paths,&library_const,+ num_shards,self.metallib_compressed,).with_context(|| format!("cannot generate bindings for {}", kernel.name))@@ -220,7 +282,7 @@ impl MetalCompiler {.collect::<anyhow::Result<Vec<_>>>()?;let tokens = quote! {- const #library_const: &[u8] = include_bytes!(#metallib_maybe_compressed_file_str);+ const #library_const: [&[u8]; #num_shards] = [#(include_bytes!(#metallib_file_strs)),*];#(#bindings)*};@@ -236,11 +298,18 @@ impl MetalCompiler {dependency_hashes,public_kernels: public_kernels.clone(),has_kernels,+ num_variants,+ num_shards,};fs::write(&cached_file, serde_json::to_vec_pretty(&cached).context("cannot serialize cache")?).context("cannot write cache file")?;- debug_log!("compile end: {source_path_relative_str}");+ let sharding = if has_kernels {+ format!(" ({num_variants} variants / {num_shards} shards)")+ } else {+ Default::default()+ };+ debug_log!("compile end: {source_path_relative_str}{sharding}");Ok((kernel_path, public_kernels, has_kernels))}diff --git a/crates/backend-uzu/build/metal/toolchain.rs b/crates/backend-uzu/build/metal/toolchain.rsindex c135f8208..be8274809 100644--- a/crates/backend-uzu/build/metal/toolchain.rs+++ b/crates/backend-uzu/build/metal/toolchain.rs@@ -215,28 +215,28 @@ impl MetalToolchain {bail!("metal analyzer failed: {}", String::from_utf8_lossy(&analyze_output.stderr));}- let ast_root = {+ let source_contents = fs::read_to_string(path).context("cannot read source file")?;++ let kernel_infos = tokio::task::spawn_blocking(move || {let mut deserializer = serde_json::Deserializer::from_slice(&analyze_output.stdout);deserializer.disable_recursion_limit();- MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?- };+ let ast_root = MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?;- if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {- bail!(- "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",- ast_root.kind- );- }-- let source_contents = fs::read_to_string(path).context("cannot read source file")?;+ if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {+ bail!(+ "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",+ ast_root.kind+ );+ }- let kernel_infos = ast_root- .inner- .into_iter()- .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())- .collect::<anyhow::Result<Vec<_>>>()- .context("cannot parse kernel infos from AST")?- .into_iter();+ ast_root+ .inner+ .into_iter()+ .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())+ .collect::<anyhow::Result<Vec<_>>>()+ .context("cannot parse kernel infos from AST")+ })+ .await??;let depfile_contents = fs::read_to_string(depfile_path.path()).context("cannot read depfile")?;@@ -248,7 +248,7 @@ impl MetalToolchain {.collect::<Vec<_>>().into_iter();- Ok((kernel_infos, dependencies))+ Ok((kernel_infos.into_iter(), dependencies))}pub async fn compile(@@ -259,7 +259,6 @@ impl MetalToolchain {) -> anyhow::Result<Option<Box<str>>> {let mut cmd = self.xcrun();cmd.arg("metal")- .arg("-c").args(["-x", "metal"]).arg(format!("-std={}", self.std.to_str())).args(self.extra_options.as_ref())@@ -301,34 +300,4 @@ impl MetalToolchain {Ok(warnings)}-- pub async fn link(- &self,- object: impl AsRef<Path>,- output: impl AsRef<Path>,- ) -> anyhow::Result<Option<Box<str>>> {- let link_output = self- .xcrun()- .arg("metallib")- .arg(object.as_ref())- .arg("-o")- .arg(output.as_ref())- .output()- .await- .context("cannot execute metal linker")?;-- let stderr = String::from_utf8_lossy(&link_output.stderr).into_owned().into_boxed_str();-- if !link_output.status.success() {- bail!("metal linker failed: {stderr}");- }-- let warnings = if !stderr.is_empty() {- Some(stderr)- } else {- None- };-- Ok(warnings)- }}diff --git a/crates/backend-uzu/build/metal/wrapper.rs b/crates/backend-uzu/build/metal/wrapper.rsindex 9b4acc338..0f6876a93 100644--- a/crates/backend-uzu/build/metal/wrapper.rs+++ b/crates/backend-uzu/build/metal/wrapper.rs@@ -11,10 +11,21 @@ use crate::common::{enum_paths::EnumPaths, identifiers::KernelName, mangling::stpub type SpecializeBaseIndices = HashMap<KernelName, usize>;+pub struct VariantWrapper {+ pub name: Box<str>,+ pub source: Box<str>,+}++pub struct KernelWrappers {+ pub header: Option<Box<str>>,+ pub variants: Box<[VariantWrapper]>,+ pub footer: Option<Box<str>>,+}+pub fn wrappers(kernels: &[MetalKernelInfo],enum_paths: &EnumPaths,-) -> anyhow::Result<(Box<[Box<str>]>, SpecializeBaseIndices)> {+) -> anyhow::Result<(Box<[KernelWrappers]>, SpecializeBaseIndices)> {let mut all_wrappers = Vec::new();let mut base_indices = SpecializeBaseIndices::new();let mut next_index = 0usize;@@ -28,8 +39,7 @@ pub fn wrappers(next_index += specialize_count;}- let kernel_wrappers = kernel_wrappers(kernel, base_indices.get(&kernel.name), enum_paths)?;- all_wrappers.extend(kernel_wrappers.into_vec());+ all_wrappers.push(kernel_wrappers(kernel, base_indices.get(&kernel.name), enum_paths)?);}Ok((all_wrappers.into_boxed_slice(), base_indices))@@ -129,13 +139,11 @@ fn kernel_wrappers(kernel: &MetalKernelInfo,base_index: Option<&usize>,enum_paths: &EnumPaths,-) -> anyhow::Result<Box<[Box<str>]>> {- let mut kernel_wrappers = Vec::new();+) -> anyhow::Result<KernelWrappers> {+ let mut variants = Vec::new();let bindings = specialize_bindings(kernel, enum_paths);- if let Some(&base) = base_index {- kernel_wrappers.push(kernel_header(&bindings, base).into_boxed_str());- }+ let header = base_index.map(|&base| kernel_header(&bindings, base).into_boxed_str());let evaluator = crate::common::constraints::Evaluator::new(kernel.variants.as_deref().into_iter().flatten().flat_map(|tp| tp.variants.iter().map(|v| v.as_ref())),@@ -289,19 +297,22 @@ fn kernel_wrappers(let condition_definitions = condition_definitions.into_iter().flatten().join("\n");let undefs = undefs.join("");- kernel_wrappers.push(- format!(+ variants.push(VariantWrapper {+ source: format!("{defs}\n{condition_definitions}\n[[kernel, max_total_threads_per_threadgroup({max_total_threads_per_threadgroup})]] void {wrapper_name}({wrapper_arguments}) {{\n{wrapper_body}}}\n{undefs}").into(),- );+ name: wrapper_name.into_boxed_str(),+ });}- if base_index.is_some() {- kernel_wrappers.push(kernel_footer(&bindings).into_boxed_str());- }+ let footer = base_index.map(|_| kernel_footer(&bindings).into_boxed_str());- Ok(kernel_wrappers.into())+ Ok(KernelWrappers {+ header,+ variants: variants.into(),+ footer,+ })}fn optional_shared_arguments(kernel: &MetalKernelInfo) -> impl Iterator<Item = &MetalArgument> {