Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Compress metallib in release builds (#662)

uuuvnba3a8357ca85
diff --git a/Cargo.lock b/Cargo.lock
index 063b3031c..291a76a63 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -515,6 +515,7 @@ dependencies = [
"walkdir",
"xgrammar-rs",
"xxhash-rust",
+ "zstd",
]
[[package]]
@@ -8517,6 +8518,34 @@ version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
[[package]]
name = "zune-core"
version = "0.5.1"
diff --git a/Cargo.toml b/Cargo.toml
index 7aef02c1d..4d2af9345 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -78,6 +78,7 @@ objc2-core-foundation = "0.3.2"
objc2-io-kit = { version = "0.3.2", default-features = false, features = ["libc", "ps"] }
objc2-security = "0.3.2"
obfstr = "0.4.4"
+zstd = "0.13.3"
metal = { package = "mtl-rs", version = "0.1.14" }
block2 = "0.6"
bytemuck = "1.23"
diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.toml
index ac989bad2..f195a116a 100644
--- a/crates/backend-uzu/Cargo.toml
+++ b/crates/backend-uzu/Cargo.toml
@@ -22,8 +22,9 @@ metal = [
"dep:tempfile",
"dep:serde_json",
"dep:objc2",
- "dep:metal",
"dep:obfstr",
+ "dep:zstd",
+ "dep:metal",
]
grammar = ["dep:xgrammar", "dep:schemars"]
@@ -50,6 +51,7 @@ rhai.workspace = true
clang-ast = { workspace = true, optional = true }
tempfile = { workspace = true, optional = true }
serde_json = { workspace = true, features = ["unbounded_depth"], optional = true }
+zstd = { workspace = true, optional = true }
[dependencies]
proc-macros.workspace = true
@@ -77,8 +79,9 @@ tokio-util.workspace = true
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]
objc2 = { workspace = true, optional = true }
-metal = { workspace = true, optional = true }
obfstr = { workspace = true, optional = true }
+zstd = { workspace = true, optional = true }
+metal = { workspace = true, optional = true }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
xgrammar = { workspace = true, optional = true }
diff --git a/crates/backend-uzu/build/metal/bindgen/mod.rs b/crates/backend-uzu/build/metal/bindgen/mod.rs
index 737135003..2e64a608f 100644
--- a/crates/backend-uzu/build/metal/bindgen/mod.rs
+++ b/crates/backend-uzu/build/metal/bindgen/mod.rs
@@ -18,6 +18,7 @@ pub fn bindgen(
specialize_indices: &SpecializeBaseIndices,
enum_paths: &EnumPaths,
library_const: &proc_macro2::Ident,
+ library_compressed: bool,
) -> Result<(TokenStream, Option<TokenStream>)> {
let kernel_name = kernel.name.as_ref();
let trait_name = format_ident!("{}Kernel", kernel_name);
@@ -96,7 +97,7 @@ pub fn bindgen(
) -> Result<Self, MetalError> {
let entry_name = #entry_name;
#function_constants_initialization
- let pipeline = context.compute_pipeline_state(#library_const, #cache_key, &entry_name, #function_constants_argument)?;
+ let pipeline = context.compute_pipeline_state(#library_const, #library_compressed, #cache_key, &entry_name, #function_constants_argument)?;
Ok(Self {
pipeline
#(, #conditional_buffer_initializers)*
diff --git a/crates/backend-uzu/build/metal/compiler.rs b/crates/backend-uzu/build/metal/compiler.rs
index d690b57f4..e121da175 100644
--- a/crates/backend-uzu/build/metal/compiler.rs
+++ b/crates/backend-uzu/build/metal/compiler.rs
@@ -30,6 +30,7 @@ pub struct MetalCompiler {
source_directory: PathBuf,
gpu_types_directory: PathBuf,
output_directory: PathBuf,
+ metallib_compressed: bool,
toolchain: MetalToolchain,
}
@@ -44,6 +45,11 @@ impl MetalCompiler {
fs::create_dir_all(&output_directory)
.with_context(|| format!("cannot create {}", output_directory.display()))?;
+ let metallib_compressed = match env::var("OPT_LEVEL").context("missing OPT_LEVEL")?.as_str() {
+ "0" | "1" | "2" => false, // treat opt-level 0/1/2 as debug/test build where size doesn't matter
+ _ => true, // treat everything else (3,s,z) as release build where size matters
+ };
+
let toolchain = MetalToolchain::from_env_with_include_dir(Some(gpu_types_directory.clone()))
.context("cannot create toolchain")?;
@@ -51,6 +57,7 @@ impl MetalCompiler {
source_directory,
gpu_types_directory,
output_directory,
+ metallib_compressed,
toolchain,
})
}
@@ -77,6 +84,11 @@ impl MetalCompiler {
let object_file = output_base_path.with_extension("air");
let metallib_file = output_base_path.with_extension("metallib");
+ let metallib_maybe_compressed_file = if self.metallib_compressed {
+ metallib_file.with_added_extension("zst")
+ } else {
+ metallib_file.clone()
+ };
let bindgen_file = output_base_path.with_extension("rs");
let cached_file = output_base_path.with_extension("cached");
@@ -144,21 +156,41 @@ impl MetalCompiler {
}
}
+ if self.metallib_compressed {
+ let metallib_file = metallib_file.clone();
+ let metallib_compressed_file = metallib_maybe_compressed_file.clone();
+
+ tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+ let metallib_source = fs::read(&metallib_file)?;
+ let metallib_compressed = zstd::encode_all(metallib_source.as_slice(), 22)?;
+ fs::write(&metallib_compressed_file, &metallib_compressed)?;
+ Ok(())
+ })
+ .await??;
+ };
+
let library_const =
format_ident!("MTLB_{}", blake3::hash(source_path_relative_str.as_bytes()).to_hex().to_uppercase());
- let metallib_file_str = metallib_file.to_str().context("metallib path is not utf-8")?;
+ let metallib_maybe_compressed_file_str =
+ metallib_maybe_compressed_file.to_str().context("metallib path is not utf-8")?;
let bindings = kernel_infos
.iter()
.map(|kernel| {
- super::bindgen::bindgen(kernel, &specialize_indices, enum_paths, &library_const)
- .with_context(|| format!("cannot generate bindings for {}", kernel.name))
- .map(|(tokens, _associated_type)| tokens)
+ super::bindgen::bindgen(
+ kernel,
+ &specialize_indices,
+ enum_paths,
+ &library_const,
+ self.metallib_compressed,
+ )
+ .with_context(|| format!("cannot generate bindings for {}", kernel.name))
+ .map(|(tokens, _associated_type)| tokens)
})
.collect::<anyhow::Result<Vec<_>>>()?;
let tokens = quote! {
- const #library_const: &[u8] = include_bytes!(#metallib_file_str);
+ const #library_const: &[u8] = include_bytes!(#metallib_maybe_compressed_file_str);
#(#bindings)*
};
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index 9a645a946..75d35ab4d 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -64,6 +64,7 @@ impl MetalContext {
fn library(
&self,
data: &'static [u8],
+ compressed: bool,
) -> Result<Retained<ProtocolObject<dyn MTLLibrary>>, MetalError> {
// `data` always comes from an `include_bytes!` constant, so its address is a stable, unique key.
let key = data.as_ptr() as usize;
@@ -71,6 +72,15 @@ impl MetalContext {
return Ok(library.clone());
}
+ let maybe_uncompressed_data_owned;
+ let data = if compressed {
+ maybe_uncompressed_data_owned = zstd::decode_all(data).map_err(MetalError::CannotDecompressLibrary)?;
+
+ &maybe_uncompressed_data_owned
+ } else {
+ data
+ };
+
let library = self
.device
.new_library_with_data(data)
@@ -83,6 +93,7 @@ impl MetalContext {
pub fn compute_pipeline_state(
&self,
library_data: &'static [u8],
+ library_compressed: bool,
cache_key: &str,
function_name: &str,
constants: Option<&MTLFunctionConstantValues>,
@@ -91,7 +102,8 @@ impl MetalContext {
return Ok(pipeline.clone());
}
- let pipeline = self.library(library_data)?.compute_pipeline_state(function_name, constants)?;
+ let pipeline =
+ self.library(library_data, library_compressed)?.compute_pipeline_state(function_name, constants)?;
self.pipeline_cache.lock().insert(cache_key.to_string(), pipeline.clone());
Ok(pipeline)
diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rs
index 5a59d16da..560ae48f7 100644
--- a/crates/backend-uzu/src/backends/metal/error.rs
+++ b/crates/backend-uzu/src/backends/metal/error.rs
@@ -15,12 +15,12 @@ pub enum MetalError {
CannotStartGpuCapture(String),
#[error("Cannot create library: {0}")]
CannotCreateLibrary(String),
+ #[error("Cannot decompress library: {0}")]
+ CannotDecompressLibrary(#[source] std::io::Error),
#[error("Cannot create command queue")]
CannotCreateCommandQueue,
#[error("Cannot create command Metal 4 queue")]
CannotCreateCommandQueueMtl4,
- #[error("Cannot read model weights metadata: {0}")]
- CannotReadModelWeightsMetadata(#[from] std::io::Error),
#[error("Cannot create buffer")]
CannotCreateBuffer,
#[error("Cannot create command buffer")]