Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Shard metal compilation (#758)

uuuvn0cf3b02fe0f6
diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.toml
index ad3f8484e..d5ccdb8a1 100644
--- a/crates/backend-uzu/Cargo.toml
+++ b/crates/backend-uzu/Cargo.toml
@@ -23,6 +23,7 @@ metal = [
"dep:serde_json",
"dep:objc2",
"dep:obfstr",
+ "dep:xxhash-rust",
"dep:zstd",
"dep:metal",
]
@@ -51,6 +52,7 @@ rhai.workspace = true
clang-ast = { workspace = true, optional = true }
tempfile = { workspace = true, optional = true }
serde_json = { workspace = true, features = ["unbounded_depth"], optional = true }
+xxhash-rust = { workspace = true, features = ["xxh3"], optional = true }
zstd = { workspace = true, optional = true }
[dependencies]
@@ -66,7 +68,6 @@ thiserror.workspace = true
tokenizers.workspace = true
rangemap.workspace = true
bytemuck = { workspace = true, features = ["derive"] }
-xxhash-rust = { workspace = true, features = ["xxh3"] }
memmap2.workspace = true
num-traits.workspace = true
libm.workspace = true
@@ -80,6 +81,7 @@ tokio-util.workspace = true
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]
objc2 = { workspace = true, optional = true }
obfstr = { workspace = true, optional = true }
+xxhash-rust = { workspace = true, features = ["xxh3"], optional = true }
zstd = { workspace = true, optional = true }
metal = { workspace = true, optional = true }
diff --git a/crates/backend-uzu/build/metal/bindgen/mod.rs b/crates/backend-uzu/build/metal/bindgen/mod.rs
index dda895ef4..1633fbd0e 100644
--- a/crates/backend-uzu/build/metal/bindgen/mod.rs
+++ b/crates/backend-uzu/build/metal/bindgen/mod.rs
@@ -18,6 +18,7 @@ pub fn bindgen(
specialize_indices: &SpecializeBaseIndices,
enum_paths: &EnumPaths,
library_const: &proc_macro2::Ident,
+ num_shards: usize,
library_compressed: bool,
) -> Result<(TokenStream, Option<TokenStream>)> {
let kernel_name = kernel.name.as_ref();
@@ -69,6 +70,13 @@ pub fn bindgen(
let dispatch_code = &dispatch_emission.dispatch_code;
let empty_dispatch_guards = &dispatch_emission.empty_dispatch_guards;
+ let library_data = if num_shards == 1 {
+ quote! { #library_const[0] }
+ } else {
+ let num_shards = num_shards as u64;
+ quote! { #library_const[(xxhash_rust::xxh3::xxh3_64(entry_name.as_bytes()) % #num_shards) as usize] }
+ };
+
let trait_implementation_for = &trait_wiring.trait_implementation_for;
let associate_backend = &trait_wiring.associate_backend;
let method_visibility = &trait_wiring.method_visibility;
@@ -97,7 +105,7 @@ pub fn bindgen(
) -> Result<Self, MetalError> {
let entry_name = #entry_name;
#function_constants_initialization
- let pipeline = context.compute_pipeline_state(#library_const, #library_compressed, #cache_key, &entry_name, #function_constants_argument)?;
+ let pipeline = context.compute_pipeline_state(#library_data, #library_compressed, #cache_key, &entry_name, #function_constants_argument)?;
Ok(Self {
pipeline
#(, #conditional_buffer_initializers)*
diff --git a/crates/backend-uzu/build/metal/compiler.rs b/crates/backend-uzu/build/metal/compiler.rs
index cad996ede..b732f9110 100644
--- a/crates/backend-uzu/build/metal/compiler.rs
+++ b/crates/backend-uzu/build/metal/compiler.rs
@@ -6,12 +6,19 @@ use std::{
use anyhow::Context;
use async_trait::async_trait;
-use futures::{StreamExt, TryStreamExt, stream};
+use futures::{StreamExt, TryStreamExt, future::try_join_all, stream};
+use itertools::{Itertools, izip};
use quote::{format_ident, quote};
use serde::{Deserialize, Serialize};
use walkdir::WalkDir;
+use xxhash_rust::xxh3::xxh3_64;
-use super::{ast::MetalKernelInfo, bindgen::bindgen_global, toolchain::MetalToolchain, wrapper::wrappers};
+use super::{
+ ast::MetalKernelInfo,
+ bindgen::bindgen_global,
+ toolchain::MetalToolchain,
+ wrapper::{KernelWrappers, VariantWrapper, wrappers},
+};
use crate::{
common::{
caching, codegen::write_tokens, compiler::Compiler, enum_paths::EnumPaths, gpu_types::GpuTypes,
@@ -21,12 +28,54 @@ use crate::{
metal::gpu_types::gpu_type_gen,
};
+const MIN_VARIANTS_PER_SHARD: usize = 8;
+const MAX_VARIANTS_PER_SHARD: usize = 64;
+
+fn shard_footers(kernel_wrappers: &[KernelWrappers]) -> Vec<String> {
+ let total_variants: usize = kernel_wrappers.iter().map(|kernel| kernel.variants.len()).sum();
+ let min_shards = total_variants.div_ceil(MAX_VARIANTS_PER_SHARD);
+ let ncpu = std::thread::available_parallelism().map(|x| x.get()).unwrap_or(1);
+ let num_shards = total_variants.div_ceil(MIN_VARIANTS_PER_SHARD).clamp(min_shards, min_shards.max(ncpu));
+ let num_shards = if num_shards >= ncpu {
+ num_shards.div_ceil(ncpu) * ncpu
+ } else {
+ num_shards
+ };
+
+ let mut footers = vec![String::new(); num_shards];
+ for kernel in kernel_wrappers {
+ let mut emit = |index: usize, variants: Vec<&VariantWrapper>| {
+ let footer = &mut footers[index];
+ footer.push_str(kernel.header.as_deref().unwrap_or(""));
+ footer.push_str(&variants.iter().map(|variant| variant.source.as_ref()).join(""));
+ footer.push_str(kernel.footer.as_deref().unwrap_or(""));
+ };
+
+ if num_shards == 1 {
+ emit(0, kernel.variants.iter().collect());
+ } else {
+ for (index, variants) in kernel
+ .variants
+ .iter()
+ .into_group_map_by(|variant| (xxh3_64(variant.name.as_bytes()) % num_shards as u64) as usize)
+ .into_iter()
+ .sorted_unstable_by_key(|(index, _)| *index)
+ {
+ emit(index, variants);
+ }
+ }
+ }
+ footers
+}
+
#[derive(Serialize, Deserialize)]
struct Cached {
cache_key: [u8; blake3::OUT_LEN],
dependency_hashes: HashMap<Box<str>, [u8; blake3::OUT_LEN]>,
public_kernels: Box<[Kernel]>,
has_kernels: bool,
+ num_variants: usize,
+ num_shards: usize,
}
#[derive(Debug)]
@@ -110,13 +159,6 @@ impl MetalCompiler {
fs::create_dir_all(output_base_path.parent().context("cannot get output directory")?)
.context("cannot create output directory")?;
- let object_file = output_base_path.with_extension("air");
- let metallib_file = output_base_path.with_extension("metallib");
- let metallib_maybe_compressed_file = if self.metallib_compressed {
- metallib_file.with_added_extension("zst")
- } else {
- metallib_file.clone()
- };
let bindgen_file = output_base_path.with_extension("rs");
let cached_file = output_base_path.with_extension("cached");
@@ -130,7 +172,12 @@ impl MetalCompiler {
for path in cached.dependency_hashes.keys() {
self.emit_rerun_if_changed_for_dependency(path);
}
- debug_log!("compile cached: {source_path_relative_str}");
+ let sharding = if cached.has_kernels {
+ format!(" ({} variants / {} shards)", cached.num_variants, cached.num_shards)
+ } else {
+ Default::default()
+ };
+ debug_log!("compile cached: {source_path_relative_str}{sharding}");
return Ok((kernel_path, cached.public_kernels, cached.has_kernels));
}
@@ -153,56 +200,70 @@ impl MetalCompiler {
.collect::<anyhow::Result<HashMap<Box<str>, [u8; blake3::OUT_LEN]>>>()
.context("cannot hash dependencies")?;
+ let mut num_variants = 0;
+ let mut num_shards = 0;
if !kernel_infos.is_empty() {
- let (wrapper_strs, specialize_indices) =
+ let (kernel_wrappers, specialize_indices) =
wrappers(&kernel_infos, enum_paths).context("cannot generate kernel wrappers")?;
- let mut footer = String::new();
- for wrapper in wrapper_strs.iter() {
- footer.push_str(wrapper);
- }
-
- let compile_output = self
- .toolchain
- .compile(&source_path, &footer, &object_file)
- .await
- .with_context(|| format!("cannot compile {source_path_relative_str}"))?;
+ num_variants = kernel_wrappers.iter().map(|kernel| kernel.variants.len()).sum();
+ let footers = shard_footers(&kernel_wrappers);
+ num_shards = footers.len();
- if let Some(warnings) = &compile_output {
- for line in warnings.lines() {
- println!("cargo::warning={line}");
- }
- }
+ let metallib_files: Vec<PathBuf> = match num_shards {
+ 1 => vec![output_base_path.with_extension("metallib")],
+ num_shards => {
+ (0..num_shards).map(|i| output_base_path.with_extension(format!("shard{i}.metallib"))).collect()
+ },
+ };
- let link_output = self
- .toolchain
- .link(&object_file, &metallib_file)
- .await
- .with_context(|| format!("cannot link {source_path_relative_str}"))?;
+ let metallib_maybe_compressed_files: Vec<PathBuf> = metallib_files
+ .iter()
+ .map(|file| {
+ if self.metallib_compressed {
+ file.with_added_extension("zst")
+ } else {
+ file.clone()
+ }
+ })
+ .collect();
+
+ let compile_outputs = try_join_all(izip!(&footers, &metallib_files, &metallib_maybe_compressed_files).map(
+ |(footer, metallib_file, compressed_file)| {
+ let source_path = &source_path;
+ async move {
+ let warnings = self.toolchain.compile(source_path, footer, metallib_file).await?;
+
+ if self.metallib_compressed {
+ let metallib_file = metallib_file.clone();
+ let compressed_file = compressed_file.clone();
+ tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+ let metallib_source = fs::read(metallib_file)?;
+ fs::write(compressed_file, zstd::encode_all(metallib_source.as_slice(), 22)?)?;
+ Ok(())
+ })
+ .await??;
+ }
+
+ anyhow::Ok(warnings)
+ }
+ },
+ ))
+ .await
+ .with_context(|| format!("cannot compile {source_path_relative_str}"))?;
- if let Some(warnings) = &link_output {
+ for warnings in compile_outputs.into_iter().flatten() {
for line in warnings.lines() {
println!("cargo::warning={line}");
}
}
- if self.metallib_compressed {
- let metallib_file = metallib_file.clone();
- let metallib_compressed_file = metallib_maybe_compressed_file.clone();
-
- tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- let metallib_source = fs::read(&metallib_file)?;
- let metallib_compressed = zstd::encode_all(metallib_source.as_slice(), 22)?;
- fs::write(&metallib_compressed_file, &metallib_compressed)?;
- Ok(())
- })
- .await??;
- };
-
let library_const =
format_ident!("MTLB_{}", blake3::hash(source_path_relative_str.as_bytes()).to_hex().to_uppercase());
- let metallib_maybe_compressed_file_str =
- metallib_maybe_compressed_file.to_str().context("metallib path is not utf-8")?;
+ let metallib_file_strs = metallib_maybe_compressed_files
+ .iter()
+ .map(|file| file.to_str().context("metallib path is not utf-8"))
+ .collect::<anyhow::Result<Vec<_>>>()?;
let bindings = kernel_infos
.iter()
@@ -212,6 +273,7 @@ impl MetalCompiler {
&specialize_indices,
enum_paths,
&library_const,
+ num_shards,
self.metallib_compressed,
)
.with_context(|| format!("cannot generate bindings for {}", kernel.name))
@@ -220,7 +282,7 @@ impl MetalCompiler {
.collect::<anyhow::Result<Vec<_>>>()?;
let tokens = quote! {
- const #library_const: &[u8] = include_bytes!(#metallib_maybe_compressed_file_str);
+ const #library_const: [&[u8]; #num_shards] = [#(include_bytes!(#metallib_file_strs)),*];
#(#bindings)*
};
@@ -236,11 +298,18 @@ impl MetalCompiler {
dependency_hashes,
public_kernels: public_kernels.clone(),
has_kernels,
+ num_variants,
+ num_shards,
};
fs::write(&cached_file, serde_json::to_vec_pretty(&cached).context("cannot serialize cache")?)
.context("cannot write cache file")?;
- debug_log!("compile end: {source_path_relative_str}");
+ let sharding = if has_kernels {
+ format!(" ({num_variants} variants / {num_shards} shards)")
+ } else {
+ Default::default()
+ };
+ debug_log!("compile end: {source_path_relative_str}{sharding}");
Ok((kernel_path, public_kernels, has_kernels))
}
diff --git a/crates/backend-uzu/build/metal/toolchain.rs b/crates/backend-uzu/build/metal/toolchain.rs
index c135f8208..be8274809 100644
--- a/crates/backend-uzu/build/metal/toolchain.rs
+++ b/crates/backend-uzu/build/metal/toolchain.rs
@@ -215,28 +215,28 @@ impl MetalToolchain {
bail!("metal analyzer failed: {}", String::from_utf8_lossy(&analyze_output.stderr));
}
- let ast_root = {
+ let source_contents = fs::read_to_string(path).context("cannot read source file")?;
+
+ let kernel_infos = tokio::task::spawn_blocking(move || {
let mut deserializer = serde_json::Deserializer::from_slice(&analyze_output.stdout);
deserializer.disable_recursion_limit();
- MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?
- };
+ let ast_root = MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?;
- if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {
- bail!(
- "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",
- ast_root.kind
- );
- }
-
- let source_contents = fs::read_to_string(path).context("cannot read source file")?;
+ if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {
+ bail!(
+ "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",
+ ast_root.kind
+ );
+ }
- let kernel_infos = ast_root
- .inner
- .into_iter()
- .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())
- .collect::<anyhow::Result<Vec<_>>>()
- .context("cannot parse kernel infos from AST")?
- .into_iter();
+ ast_root
+ .inner
+ .into_iter()
+ .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())
+ .collect::<anyhow::Result<Vec<_>>>()
+ .context("cannot parse kernel infos from AST")
+ })
+ .await??;
let depfile_contents = fs::read_to_string(depfile_path.path()).context("cannot read depfile")?;
@@ -248,7 +248,7 @@ impl MetalToolchain {
.collect::<Vec<_>>()
.into_iter();
- Ok((kernel_infos, dependencies))
+ Ok((kernel_infos.into_iter(), dependencies))
}
pub async fn compile(
@@ -259,7 +259,6 @@ impl MetalToolchain {
) -> anyhow::Result<Option<Box<str>>> {
let mut cmd = self.xcrun();
cmd.arg("metal")
- .arg("-c")
.args(["-x", "metal"])
.arg(format!("-std={}", self.std.to_str()))
.args(self.extra_options.as_ref())
@@ -301,34 +300,4 @@ impl MetalToolchain {
Ok(warnings)
}
-
- pub async fn link(
- &self,
- object: impl AsRef<Path>,
- output: impl AsRef<Path>,
- ) -> anyhow::Result<Option<Box<str>>> {
- let link_output = self
- .xcrun()
- .arg("metallib")
- .arg(object.as_ref())
- .arg("-o")
- .arg(output.as_ref())
- .output()
- .await
- .context("cannot execute metal linker")?;
-
- let stderr = String::from_utf8_lossy(&link_output.stderr).into_owned().into_boxed_str();
-
- if !link_output.status.success() {
- bail!("metal linker failed: {stderr}");
- }
-
- let warnings = if !stderr.is_empty() {
- Some(stderr)
- } else {
- None
- };
-
- Ok(warnings)
- }
}
diff --git a/crates/backend-uzu/build/metal/wrapper.rs b/crates/backend-uzu/build/metal/wrapper.rs
index 9b4acc338..0f6876a93 100644
--- a/crates/backend-uzu/build/metal/wrapper.rs
+++ b/crates/backend-uzu/build/metal/wrapper.rs
@@ -11,10 +11,21 @@ use crate::common::{enum_paths::EnumPaths, identifiers::KernelName, mangling::st
pub type SpecializeBaseIndices = HashMap<KernelName, usize>;
+pub struct VariantWrapper {
+ pub name: Box<str>,
+ pub source: Box<str>,
+}
+
+pub struct KernelWrappers {
+ pub header: Option<Box<str>>,
+ pub variants: Box<[VariantWrapper]>,
+ pub footer: Option<Box<str>>,
+}
+
pub fn wrappers(
kernels: &[MetalKernelInfo],
enum_paths: &EnumPaths,
-) -> anyhow::Result<(Box<[Box<str>]>, SpecializeBaseIndices)> {
+) -> anyhow::Result<(Box<[KernelWrappers]>, SpecializeBaseIndices)> {
let mut all_wrappers = Vec::new();
let mut base_indices = SpecializeBaseIndices::new();
let mut next_index = 0usize;
@@ -28,8 +39,7 @@ pub fn wrappers(
next_index += specialize_count;
}
- let kernel_wrappers = kernel_wrappers(kernel, base_indices.get(&kernel.name), enum_paths)?;
- all_wrappers.extend(kernel_wrappers.into_vec());
+ all_wrappers.push(kernel_wrappers(kernel, base_indices.get(&kernel.name), enum_paths)?);
}
Ok((all_wrappers.into_boxed_slice(), base_indices))
@@ -129,13 +139,11 @@ fn kernel_wrappers(
kernel: &MetalKernelInfo,
base_index: Option<&usize>,
enum_paths: &EnumPaths,
-) -> anyhow::Result<Box<[Box<str>]>> {
- let mut kernel_wrappers = Vec::new();
+) -> anyhow::Result<KernelWrappers> {
+ let mut variants = Vec::new();
let bindings = specialize_bindings(kernel, enum_paths);
- if let Some(&base) = base_index {
- kernel_wrappers.push(kernel_header(&bindings, base).into_boxed_str());
- }
+ let header = base_index.map(|&base| kernel_header(&bindings, base).into_boxed_str());
let evaluator = crate::common::constraints::Evaluator::new(
kernel.variants.as_deref().into_iter().flatten().flat_map(|tp| tp.variants.iter().map(|v| v.as_ref())),
@@ -289,19 +297,22 @@ fn kernel_wrappers(
let condition_definitions = condition_definitions.into_iter().flatten().join("\n");
let undefs = undefs.join("");
- kernel_wrappers.push(
- format!(
+ variants.push(VariantWrapper {
+ source: format!(
"{defs}\n{condition_definitions}\n[[kernel, max_total_threads_per_threadgroup({max_total_threads_per_threadgroup})]] void {wrapper_name}({wrapper_arguments}) {{\n{wrapper_body}}}\n{undefs}"
)
.into(),
- );
+ name: wrapper_name.into_boxed_str(),
+ });
}
- if base_index.is_some() {
- kernel_wrappers.push(kernel_footer(&bindings).into_boxed_str());
- }
+ let footer = base_index.map(|_| kernel_footer(&bindings).into_boxed_str());
- Ok(kernel_wrappers.into())
+ Ok(KernelWrappers {
+ header,
+ variants: variants.into(),
+ footer,
+ })
}
fn optional_shared_arguments(kernel: &MetalKernelInfo) -> impl Iterator<Item = &MetalArgument> {