A high-performance inference engine for AI models
Compress metallib in release builds (#662)
uuuvnba3a8357ca85
diff --git a/Cargo.lock b/Cargo.lockindex 063b3031c..291a76a63 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -515,6 +515,7 @@ dependencies = ["walkdir","xgrammar-rs","xxhash-rust",+ "zstd",][[package]]@@ -8517,6 +8518,34 @@ version = "1.0.21"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"+[[package]]+name = "zstd"+version = "0.13.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"+dependencies = [+ "zstd-safe",+]++[[package]]+name = "zstd-safe"+version = "7.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"+dependencies = [+ "zstd-sys",+]++[[package]]+name = "zstd-sys"+version = "2.0.16+zstd.1.5.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"+dependencies = [+ "cc",+ "pkg-config",+]+[[package]]name = "zune-core"version = "0.5.1"diff --git a/Cargo.toml b/Cargo.tomlindex 7aef02c1d..4d2af9345 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -78,6 +78,7 @@ objc2-core-foundation = "0.3.2"objc2-io-kit = { version = "0.3.2", default-features = false, features = ["libc", "ps"] }objc2-security = "0.3.2"obfstr = "0.4.4"+zstd = "0.13.3"metal = { package = "mtl-rs", version = "0.1.14" }block2 = "0.6"bytemuck = "1.23"diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.tomlindex ac989bad2..f195a116a 100644--- a/crates/backend-uzu/Cargo.toml+++ b/crates/backend-uzu/Cargo.toml@@ -22,8 +22,9 @@ metal = ["dep:tempfile","dep:serde_json","dep:objc2",- "dep:metal","dep:obfstr",+ "dep:zstd",+ "dep:metal",]grammar = ["dep:xgrammar", "dep:schemars"]@@ -50,6 +51,7 @@ rhai.workspace = trueclang-ast = { workspace = true, optional = true }tempfile = { workspace = true, optional = true }serde_json = { workspace = true, features = ["unbounded_depth"], optional = true }+zstd = { workspace = true, optional = true }[dependencies]proc-macros.workspace = true@@ -77,8 +79,9 @@ tokio-util.workspace = true[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]objc2 = { workspace = true, optional = true }-metal = { workspace = true, optional = true }obfstr = { workspace = true, optional = true }+zstd = { workspace = true, optional = true }+metal = { workspace = true, optional = true }[target.'cfg(not(target_arch = "wasm32"))'.dependencies]xgrammar = { workspace = true, optional = true }diff --git a/crates/backend-uzu/build/metal/bindgen/mod.rs b/crates/backend-uzu/build/metal/bindgen/mod.rsindex 737135003..2e64a608f 100644--- a/crates/backend-uzu/build/metal/bindgen/mod.rs+++ b/crates/backend-uzu/build/metal/bindgen/mod.rs@@ -18,6 +18,7 @@ pub fn bindgen(specialize_indices: &SpecializeBaseIndices,enum_paths: &EnumPaths,library_const: &proc_macro2::Ident,+ library_compressed: bool,) -> Result<(TokenStream, Option<TokenStream>)> {let kernel_name = kernel.name.as_ref();let trait_name = format_ident!("{}Kernel", kernel_name);@@ -96,7 +97,7 @@ pub fn bindgen() -> Result<Self, MetalError> {let entry_name = #entry_name;#function_constants_initialization- let pipeline = context.compute_pipeline_state(#library_const, #cache_key, &entry_name, #function_constants_argument)?;+ let pipeline = context.compute_pipeline_state(#library_const, #library_compressed, #cache_key, &entry_name, #function_constants_argument)?;Ok(Self {pipeline#(, #conditional_buffer_initializers)*diff --git a/crates/backend-uzu/build/metal/compiler.rs b/crates/backend-uzu/build/metal/compiler.rsindex d690b57f4..e121da175 100644--- a/crates/backend-uzu/build/metal/compiler.rs+++ b/crates/backend-uzu/build/metal/compiler.rs@@ -30,6 +30,7 @@ pub struct MetalCompiler {source_directory: PathBuf,gpu_types_directory: PathBuf,output_directory: PathBuf,+ metallib_compressed: bool,toolchain: MetalToolchain,}@@ -44,6 +45,11 @@ impl MetalCompiler {fs::create_dir_all(&output_directory).with_context(|| format!("cannot create {}", output_directory.display()))?;+ let metallib_compressed = match env::var("OPT_LEVEL").context("missing OPT_LEVEL")?.as_str() {+ "0" | "1" | "2" => false, // treat opt-level 0/1/2 as debug/test build where size doesn't matter+ _ => true, // treat everything else (3,s,z) as release build where size matters+ };+let toolchain = MetalToolchain::from_env_with_include_dir(Some(gpu_types_directory.clone())).context("cannot create toolchain")?;@@ -51,6 +57,7 @@ impl MetalCompiler {source_directory,gpu_types_directory,output_directory,+ metallib_compressed,toolchain,})}@@ -77,6 +84,11 @@ impl MetalCompiler {let object_file = output_base_path.with_extension("air");let metallib_file = output_base_path.with_extension("metallib");+ let metallib_maybe_compressed_file = if self.metallib_compressed {+ metallib_file.with_added_extension("zst")+ } else {+ metallib_file.clone()+ };let bindgen_file = output_base_path.with_extension("rs");let cached_file = output_base_path.with_extension("cached");@@ -144,21 +156,41 @@ impl MetalCompiler {}}+ if self.metallib_compressed {+ let metallib_file = metallib_file.clone();+ let metallib_compressed_file = metallib_maybe_compressed_file.clone();++ tokio::task::spawn_blocking(move || -> anyhow::Result<()> {+ let metallib_source = fs::read(&metallib_file)?;+ let metallib_compressed = zstd::encode_all(metallib_source.as_slice(), 22)?;+ fs::write(&metallib_compressed_file, &metallib_compressed)?;+ Ok(())+ })+ .await??;+ };+let library_const =format_ident!("MTLB_{}", blake3::hash(source_path_relative_str.as_bytes()).to_hex().to_uppercase());- let metallib_file_str = metallib_file.to_str().context("metallib path is not utf-8")?;+ let metallib_maybe_compressed_file_str =+ metallib_maybe_compressed_file.to_str().context("metallib path is not utf-8")?;let bindings = kernel_infos.iter().map(|kernel| {- super::bindgen::bindgen(kernel, &specialize_indices, enum_paths, &library_const)- .with_context(|| format!("cannot generate bindings for {}", kernel.name))- .map(|(tokens, _associated_type)| tokens)+ super::bindgen::bindgen(+ kernel,+ &specialize_indices,+ enum_paths,+ &library_const,+ self.metallib_compressed,+ )+ .with_context(|| format!("cannot generate bindings for {}", kernel.name))+ .map(|(tokens, _associated_type)| tokens)}).collect::<anyhow::Result<Vec<_>>>()?;let tokens = quote! {- const #library_const: &[u8] = include_bytes!(#metallib_file_str);+ const #library_const: &[u8] = include_bytes!(#metallib_maybe_compressed_file_str);#(#bindings)*};diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex 9a645a946..75d35ab4d 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -64,6 +64,7 @@ impl MetalContext {fn library(&self,data: &'static [u8],+ compressed: bool,) -> Result<Retained<ProtocolObject<dyn MTLLibrary>>, MetalError> {// `data` always comes from an `include_bytes!` constant, so its address is a stable, unique key.let key = data.as_ptr() as usize;@@ -71,6 +72,15 @@ impl MetalContext {return Ok(library.clone());}+ let maybe_uncompressed_data_owned;+ let data = if compressed {+ maybe_uncompressed_data_owned = zstd::decode_all(data).map_err(MetalError::CannotDecompressLibrary)?;++ &maybe_uncompressed_data_owned+ } else {+ data+ };+let library = self.device.new_library_with_data(data)@@ -83,6 +93,7 @@ impl MetalContext {pub fn compute_pipeline_state(&self,library_data: &'static [u8],+ library_compressed: bool,cache_key: &str,function_name: &str,constants: Option<&MTLFunctionConstantValues>,@@ -91,7 +102,8 @@ impl MetalContext {return Ok(pipeline.clone());}- let pipeline = self.library(library_data)?.compute_pipeline_state(function_name, constants)?;+ let pipeline =+ self.library(library_data, library_compressed)?.compute_pipeline_state(function_name, constants)?;self.pipeline_cache.lock().insert(cache_key.to_string(), pipeline.clone());Ok(pipeline)diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rsindex 5a59d16da..560ae48f7 100644--- a/crates/backend-uzu/src/backends/metal/error.rs+++ b/crates/backend-uzu/src/backends/metal/error.rs@@ -15,12 +15,12 @@ pub enum MetalError {CannotStartGpuCapture(String),#[error("Cannot create library: {0}")]CannotCreateLibrary(String),+ #[error("Cannot decompress library: {0}")]+ CannotDecompressLibrary(#[source] std::io::Error),#[error("Cannot create command queue")]CannotCreateCommandQueue,#[error("Cannot create command Metal 4 queue")]CannotCreateCommandQueueMtl4,- #[error("Cannot read model weights metadata: {0}")]- CannotReadModelWeightsMetadata(#[from] std::io::Error),#[error("Cannot create buffer")]CannotCreateBuffer,#[error("Cannot create command buffer")]