A high-performance inference engine for AI models
Objc2 dependency only for metal (#260)
Aleksandr Golokoz5c4a025180ca
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.ymlindex 8e98fee84..0892eb8fa 100644--- a/.github/actions/setup/action.yml+++ b/.github/actions/setup/action.yml@@ -23,6 +23,9 @@ runs:using: compositesteps:- uses: dtolnay/rust-toolchain@stable+ - uses: dtolnay/rust-toolchain@nightly+ with:+ components: rustfmt- if: inputs.nextest == 'true'uses: taiki-e/install-action@nextest- if: inputs.linters == 'true'diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 571d035fc..43fb5416d 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -22,7 +22,7 @@ defaults:jobs:linters:- name: Linters+ name: lintersruns-on: [self-hosted, macos, x64, mac, m3max, runner2]steps:@@ -32,12 +32,6 @@ jobs:with:linters: true- - name: Install nightly rustfmt- if: ${{ always() && steps.setup.outcome == 'success' }}- uses: dtolnay/rust-toolchain@nightly- with:- components: rustfmt-- name: Check Rust formattingif: ${{ always() && steps.setup.outcome == 'success' }}run: cargo +nightly fmt --check@@ -53,12 +47,13 @@ jobs:git diff --exit-code -- Cargo.locktests:- name: Tests${{ matrix.tracing && ' (tracing)' || '' }}+ name: ${{ matrix.backend }} ${{ matrix.tracing && ' (tracing)' || '' }}runs-on: [self-hosted, macos, x64, mac, m3max, runner1]if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}strategy:matrix:+ backend: ['cpu', 'metal']tracing: [false, true]env:@@ -68,14 +63,14 @@ jobs:- uses: actions/checkout@v6- uses: ./.github/actions/setupwith:- metal: true+ metal: ${{ matrix.backend == 'metal' }}test_model: truenextest: true- name: Build cratesrun: |- cargo build ${{ matrix.tracing && '--features tracing' || '' }}+ cargo +stable build --no-default-features ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} ${{ matrix.tracing && '--features tracing' || '' }}- name: Run testsrun: |- cargo nextest run --profile=ci ${{ matrix.tracing && '--features tracing' || '' }}+ cargo +stable nextest run --profile=ci --no-default-features ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} ${{ matrix.tracing && '--features tracing' || '' }}diff --git a/Cargo.lock b/Cargo.lockindex 37289b08c..99d5be237 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -309,13 +309,8 @@ checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"name = "benchmarks"version = "0.1.8"dependencies = [- "half",- "libc","mach2",- "mtl-rs",- "objc2","serde",- "serde_json","sysinfo","uzu",]@@ -381,12 +376,6 @@ dependencies = ["cpufeatures 0.2.17",]-[[package]]-name = "block"-version = "0.1.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"-[[package]]name = "block-buffer"version = "0.10.4"@@ -589,7 +578,6 @@ dependencies = ["comfy-table","console 0.16.3","ctrlc",- "futures","indicatif 0.18.4","inquire","rocket",@@ -729,17 +717,6 @@ version = "0.8.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"-[[package]]-name = "core-graphics-types"-version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb"-dependencies = [- "bitflags",- "core-foundation 0.10.1",- "libc",-]-[[package]]name = "cpufeatures"version = "0.2.17"@@ -1266,28 +1243,7 @@ version = "0.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"dependencies = [- "foreign-types-shared 0.1.1",-]--[[package]]-name = "foreign-types"-version = "0.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"-dependencies = [- "foreign-types-macros",- "foreign-types-shared 0.3.1",-]--[[package]]-name = "foreign-types-macros"-version = "0.2.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.117",+ "foreign-types-shared",][[package]]@@ -1296,12 +1252,6 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"-[[package]]-name = "foreign-types-shared"-version = "0.3.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"-[[package]]name = "form_urlencoded"version = "1.2.2"@@ -2206,15 +2156,6 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"-[[package]]-name = "malloc_buf"-version = "0.0.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"-dependencies = [- "libc",-]-[[package]]name = "matchers"version = "0.2.0"@@ -2249,21 +2190,6 @@ dependencies = ["libc",]-[[package]]-name = "metal"-version = "0.32.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "00c15a6f673ff72ddcc22394663290f870fb224c1bfce55734a75c414150e605"-dependencies = [- "bitflags",- "block",- "core-graphics-types",- "foreign-types 0.5.0",- "log",- "objc",- "paste",-]-[[package]]name = "miette"version = "5.10.0"@@ -2371,19 +2297,6 @@ dependencies = ["cxx",]-[[package]]-name = "mpsgraph"-version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b3d4f4ee48894ff2d83808e8a0333f00eb7576375a5a77e8132f10ebb2a1f86"-dependencies = [- "bitflags",- "block2",- "metal",- "objc2",- "objc2-foundation",-]-[[package]]name = "mtl-rs"version = "0.1.7"@@ -2540,15 +2453,6 @@ version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"-[[package]]-name = "objc"-version = "0.2.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1"-dependencies = [- "malloc_buf",-]-[[package]]name = "objc2"version = "0.6.4"@@ -2655,7 +2559,7 @@ checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"dependencies = ["bitflags","cfg-if",- "foreign-types 0.3.2",+ "foreign-types","libc","once_cell","openssl-macros",@@ -4429,14 +4333,11 @@ dependencies = ["anyhow","async-trait","bindgen",- "bitflags","blake3",- "block2","bytemuck","bytesize","cbindgen","clang-ast",- "clap","comfy-table","depfile","dsl",@@ -4449,7 +4350,6 @@ dependencies = ["memmap2","minijinja","minijinja-contrib",- "mpsgraph","mtl-rs","ndarray","num-traits",diff --git a/Cargo.toml b/Cargo.tomlindex b19869d93..e754234c3 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -13,7 +13,7 @@ repository = "https://github.com/trymirai/uzu"[workspace.dependencies]benchmarks = { path = "crates/benchmarks" }dsl = { path = "crates/dsl" }-uzu = { path = "crates/uzu" }+uzu = { path = "crates/uzu", default-features = false }xgrammar = { package = "xgrammar-rs", version = "0.1", features = ["hf"] }half = { version = "2.7", features = ["bytemuck", "num-traits"] }ndarray = "0.17"@@ -21,7 +21,6 @@ serde = { version = "1.0", features = ["derive"] }serde_json = "1.0"thiserror = "2.0"tokenizers = "0.22"-mpsgraph = "0.2"objc2 = "0.6"objc2-foundation = "0.3"metal = { package = "mtl-rs", version = "0.1" }@@ -34,7 +33,6 @@ memmap2 = "0.9.8"num-traits = "0.2"regex = "1.12"libc = "0.2"-bitflags = "2.11"minijinja = "2.17"minijinja-contrib = { version = "2.17", features = ["pycompat"] }clap = { version = "4.6", features = ["derive"] }diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.tomlindex 7501b6c94..c8ef2bcba 100644--- a/crates/benchmarks/Cargo.toml+++ b/crates/benchmarks/Cargo.toml@@ -4,12 +4,14 @@ version.workspace = trueedition.workspace = true[dependencies]-metal.workspace = trueuzu.workspace = trueserde.workspace = true-serde_json.workspace = truesysinfo.workspace = true-libc.workspace = true++[target.'cfg(target_os = "macos")'.dependencies]mach2.workspace = true-half.workspace = true-objc2.workspace = true++[features]+default = ["metal"]++metal = ["uzu/metal"]diff --git a/crates/benchmarks/src/runner/helpers/memory_usage.rs b/crates/benchmarks/src/runner/helpers/memory_usage.rsindex e93ebf88c..822703223 100644--- a/crates/benchmarks/src/runner/helpers/memory_usage.rs+++ b/crates/benchmarks/src/runner/helpers/memory_usage.rs@@ -1,15 +1,16 @@-use std::mem;+#[cfg(target_os = "macos")]+pub fn get_memory_usage() -> Option<u64> {+ use std::mem;-use mach2::{- kern_return::KERN_SUCCESS,- mach_types::task_t,- message::mach_msg_type_number_t,- task::task_info,- task_info::{TASK_BASIC_INFO, task_basic_info},- traps::mach_task_self,-};+ use mach2::{+ kern_return::KERN_SUCCESS,+ mach_types::task_t,+ message::mach_msg_type_number_t,+ task::task_info,+ task_info::{TASK_BASIC_INFO, task_basic_info},+ traps::mach_task_self,+ };-pub fn get_memory_usage() -> Option<u64> {unsafe {let task: task_t = mach_task_self();@@ -33,3 +34,8 @@ pub fn get_memory_usage() -> Option<u64> {}}}++#[cfg(not(target_os = "macos"))]+pub fn get_memory_usage() -> Option<u64> {+ unimplemented!()+}diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.tomlindex 95f7bf3ba..b58b8aec6 100644--- a/crates/cli/Cargo.toml+++ b/crates/cli/Cargo.toml@@ -21,4 +21,8 @@ serde.workspace = trueserde_json.workspace = trueuuid.workspace = truecomfy-table.workspace = true-futures.workspace = true++[features]+default = ["metal"]++metal = ["uzu/metal"]diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 414aa97a3..ad78a8c06 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -13,7 +13,6 @@ serde.workspace = trueserde_json.workspace = truethiserror.workspace = truetokenizers.workspace = true-mpsgraph.workspace = truebytemuck.workspace = truezerocopy = { workspace = true, features = ["derive"] }xxhash-rust = { workspace = true, features = ["xxh3"] }@@ -21,27 +20,20 @@ memmap2.workspace = truenum-traits.workspace = trueregex.workspace = truelibc.workspace = true-bitflags.workspace = trueminijinja.workspace = trueminijinja-contrib.workspace = true-clap.workspace = truerand.workspace = truesysinfo.workspace = trueitertools.workspace = trueschemars.workspace = true-# metal feature requires objc2, block2 (for callbacks) and metal-bytesize = { workspace = true, optional = true }+# metal on macos requires objc2, bytesize, metal+[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]objc2 = { workspace = true, optional = true }objc2-foundation = { workspace = true, optional = true }-block2 = { workspace = true, optional = true }+bytesize = { workspace = true, optional = true }metal = { workspace = true, optional = true }-# macos requires objc2 even on non-metal (but shouldn't)-[target.'cfg(target_os = "macos")'.dependencies]-objc2.workspace = true-objc2-foundation.workspace = true-[dev-dependencies]test-tag.workspace = trueproptest.workspace = true@@ -49,7 +41,6 @@ rand_distr.workspace = trueis_close.workspace = trueserde_json5.workspace = trueschemars.workspace = true-mpsgraph.workspace = truecomfy-table.workspace = trueindicatif.workspace = true@@ -72,18 +63,18 @@ prettyplease.workspace = trueitertools.workspace = truecbindgen.workspace = true-# metal feature requires clang-ast+# metal on macos requires clang-ast for dsl build+[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.build-dependencies]clang-ast = { workspace = true, optional = true }[features]default = ["metal"]metal = [- "dep:bytesize","dep:clang-ast","dep:objc2","dep:objc2-foundation",- "dep:block2",+ "dep:bytesize","dep:metal",]tracing = []diff --git a/crates/uzu/build/common/caching.rs b/crates/uzu/build/common/caching.rsindex d4939e48b..c65dd27d0 100644--- a/crates/uzu/build/common/caching.rs+++ b/crates/uzu/build/common/caching.rs@@ -1,3 +1,5 @@+#![cfg(all(feature = "metal", target_os = "macos"))]+use std::{fs, sync::OnceLock};use anyhow::Context;diff --git a/crates/uzu/build/common/mangling.rs b/crates/uzu/build/common/mangling.rsindex b017e4f5b..2b256af80 100644--- a/crates/uzu/build/common/mangling.rs+++ b/crates/uzu/build/common/mangling.rs@@ -1,3 +1,5 @@+#![cfg(all(feature = "metal", target_os = "macos"))]+use std::iter::repeat_n;use itertools::Itertools;diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rsindex 9062be932..30826ed30 100644--- a/crates/uzu/build/gpu_types/mod.rs+++ b/crates/uzu/build/gpu_types/mod.rs@@ -34,6 +34,7 @@ impl GpuTypesCompiler {})}+ #[cfg(all(feature = "metal", target_os = "macos"))]pub fn generated_header_dir(&self) -> &PathBuf {&self.generated_dir}diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rsindex ae9dd59f2..a09bd59e8 100644--- a/crates/uzu/build/main.rs+++ b/crates/uzu/build/main.rs@@ -8,7 +8,7 @@ mod gpu_types;mod cpu;-#[cfg(feature = "metal")]+#[cfg(all(feature = "metal", target_os = "macos"))]mod metal;use common::{compiler::Compiler, envs, traitgen::traitgen_all};@@ -21,6 +21,14 @@ async fn main() -> anyhow::Result<()> {println!("cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun");}+ let metal_backend = cfg!(feature = "metal")+ && matches!(env::var("CARGO_CFG_TARGET_OS").unwrap().as_ref(), "macos" | "ios" | "tvos" | "visionos");++ println!("cargo::rustc-check-cfg=cfg(metal_backend)");+ if metal_backend {+ println!("cargo::rustc-cfg=metal_backend");+ }+debug_log!("build script started");if envs::build_clean() {@@ -33,7 +41,6 @@ async fn main() -> anyhow::Result<()> {}let gpu_types_compiler = gpu_types::GpuTypesCompiler::new()?;- let generated_header_dir = gpu_types_compiler.generated_header_dir().clone();gpu_types_compiler.build().await?;debug_log!("gpu_types build done");@@ -41,8 +48,11 @@ async fn main() -> anyhow::Result<()> {compilers.push(Box::new(cpu::CpuCompiler::new()?));- #[cfg(feature = "metal")]- compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));+ #[cfg(all(feature = "metal", target_os = "macos"))]+ if metal_backend {+ let generated_header_dir = gpu_types_compiler.generated_header_dir().clone();+ compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));+ }let backends_kernels = try_join_all(compilers.iter().map(|c| c.build())).await?;diff --git a/crates/uzu/examples/quick_start_chat.rs b/crates/uzu/examples/quick_start_chat.rsindex d75619c9a..2c4792126 100644--- a/crates/uzu/examples/quick_start_chat.rs+++ b/crates/uzu/examples/quick_start_chat.rs@@ -7,8 +7,10 @@ use uzu::session::{};fn main() -> Result<(), Box<dyn std::error::Error>> {- let model_path = PathBuf::from("MODEL_PATH");- let mut session = ChatSession::new(model_path, DecodingConfig::default())?;+ let model_path = std::env::var("UZU_MODEL_PATH")+ .map_err(|_| -> Box<dyn std::error::Error> { "UZU_MODEL_PATH environment variable is not set.".into() })?;+ let model_path_buf = PathBuf::from(model_path);+ let mut session = ChatSession::new(model_path_buf, DecodingConfig::default())?;let input = Input::Text(String::from("Tell about London"));let output = session.run(diff --git a/crates/uzu/src/backends/mod.rs b/crates/uzu/src/backends/mod.rsindex d123fda17..dbf208ad2 100644--- a/crates/uzu/src/backends/mod.rs+++ b/crates/uzu/src/backends/mod.rs@@ -1,17 +1,22 @@pub mod common;pub mod cpu;-#[cfg(feature = "metal")]+#[cfg(metal_backend)]pub mod metal;macro_rules! select_backend {($expr:expr, $unk:expr) => {{- match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or("metal") {+ let mut default = "cpu";+ if cfg!(metal_backend) {+ default = "metal";+ }++ match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or(default) {"cpu" => {type B = crate::backends::cpu::Cpu;$expr},- #[cfg(feature = "metal")]+ #[cfg(metal_backend)]"metal" => {type B = crate::backends::metal::Metal;$exprdiff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 0dd5bf55a..1b16fe499 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -2,8 +2,6 @@use std::{cell::RefCell, rc::Rc};use std::{collections::HashMap, path::Path, time::Instant};-use objc2::rc::autoreleasepool;-#[cfg(feature = "tracing")]use super::ActivationTrace;use super::{ClassificationOutput, ClassificationStats, ClassifierContext};@@ -37,42 +35,38 @@ impl<B: Backend> ClassifierTrait for Classifier<B> {) -> Result<ClassificationOutput, Error> {let run_start = Instant::now();- autoreleasepool(|_pool| {- let forward_start = Instant::now();-- #[cfg(feature = "tracing")]- let (logits, _traces) = self.forward_pass_with_traces(&token_ids, &token_positions)?;-- #[cfg(not(feature = "tracing"))]- let logits = self.forward_pass(&token_ids, &token_positions)?;-- let forward_duration = forward_start.elapsed().as_secs_f64();-- let postprocessing_start = Instant::now();- let probabilities = self.logits_to_probabilities(&logits)?;- let postprocessing_duration = postprocessing_start.elapsed().as_secs_f64();-- let (predicted_label, confidence) = probabilities- .iter()- .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())- .map(|(label, prob)| (label.clone(), *prob))- .unwrap_or((String::from("unknown"), 0.0));-- let stats = ClassificationStats::new(- 0.0,- forward_duration,- postprocessing_duration,- run_start.elapsed().as_secs_f64(),- token_ids.len() as u64,- predicted_label,- confidence,- );-- Ok(ClassificationOutput {- logits,- probabilities,- stats,- })+ #[cfg(feature = "tracing")]+ let (logits, _traces) = self.forward_pass_with_traces(&token_ids, &token_positions)?;++ #[cfg(not(feature = "tracing"))]+ let logits = self.forward_pass(&token_ids, &token_positions)?;++ let forward_duration = run_start.elapsed().as_secs_f64();++ let postprocessing_start = Instant::now();+ let probabilities = self.logits_to_probabilities(&logits)?;+ let postprocessing_duration = postprocessing_start.elapsed().as_secs_f64();++ let (predicted_label, confidence) = probabilities+ .iter()+ .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())+ .map(|(label, prob)| (label.clone(), *prob))+ .unwrap_or((String::from("unknown"), 0.0));++ let stats = ClassificationStats::new(+ 0.0,+ forward_duration,+ postprocessing_duration,+ run_start.elapsed().as_secs_f64(),+ token_ids.len() as u64,+ predicted_label,+ confidence,+ );++ Ok(ClassificationOutput {+ logits,+ probabilities,+ stats,})}}@@ -100,78 +94,73 @@ impl<B: Backend> Classifier<B> {token_ids: &[u64],token_positions: &[usize],) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {- autoreleasepool(|_| {- let num_labels = self.context.model_config.model_config.num_labels;- let mut state = ForwardPassState::new_classifier(- self.context.context.clone(),- &self.context.model_shape,- &self.context.scratch_buffers,- self.context.shared_buffers.clone(),- token_ids,- token_positions,- true,- num_labels,- );-- let encoding_params = EncodingParameters::new();-- let mut command_buffer = self- .context- .context- .create_command_buffer()- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?- .start_encoding();-- self.context- .embed- .encode_lookup(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .embedding_norm- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- #[cfg(feature = "tracing")]- {- let traces = state.traces().clone();- state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().embedding_norm().clone());- }-- for layer in self.context.layers.iter() {- layer- .encode(&mut state, &encoding_params, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- }- self.context- .output_norm- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- #[cfg(feature = "tracing")]- {- let traces = state.traces().clone();- state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());- }-- self.context- .pooling- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ let num_labels = self.context.model_config.model_config.num_labels;+ let mut state = ForwardPassState::new_classifier(+ self.context.context.clone(),+ &self.context.model_shape,+ &self.context.scratch_buffers,+ self.context.shared_buffers.clone(),+ token_ids,+ token_positions,+ true,+ num_labels,+ );++ let encoding_params = EncodingParameters::new();++ let mut command_buffer = self+ .context+ .context+ .create_command_buffer()+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?+ .start_encoding();++ self.context+ .embed+ .encode_lookup(&mut state, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context+ .embedding_norm+ .encode(&mut state, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ #[cfg(feature = "tracing")]+ {+ let traces = state.traces().clone();+ state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().embedding_norm().clone());+ }- self.context- .prediction_head- .encode(&mut state, &mut command_buffer)+ for layer in self.context.layers.iter() {+ layer+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ }+ self.context+ .output_norm+ .encode(&mut state, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ #[cfg(feature = "tracing")]+ {+ let traces = state.traces().clone();+ state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());+ }- command_buffer- .end_encoding()- .submit()- .wait_until_completed()- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ self.context.pooling.encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- let logits = self.copy_logits_from_state(&state)?;+ self.context+ .prediction_head+ .encode(&mut state, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- let traces = state.traces().clone();- Ok((logits, traces))- })+ command_buffer+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;++ let logits = self.copy_logits_from_state(&state)?;++ let traces = state.traces().clone();+ Ok((logits, traces))}#[cfg(not(feature = "tracing"))]@@ -180,64 +169,59 @@ impl<B: Backend> Classifier<B> {token_ids: &[u64],token_positions: &[usize],) -> Result<Box<[f32]>, Error> {- autoreleasepool(|_| {- let num_labels = self.context.model_config.model_config.num_labels;- let mut state = ForwardPassState::new_classifier(- self.context.context.clone(),- &self.context.model_shape,- &self.context.scratch_buffers,- self.context.shared_buffers.clone(),- token_ids,- token_positions,- true,- num_labels,- );-- let encoding_params = EncodingParameters::new();-- let mut command_buffer = self- .context- .context- .create_command_buffer()- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?- .start_encoding();-- self.context- .embed- .encode_lookup(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .embedding_norm- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- for layer in self.context.layers.iter() {- layer- .encode(&mut state, &encoding_params, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- }- self.context- .output_norm- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .pooling- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .prediction_head- .encode(&mut state, &mut command_buffer)+ let num_labels = self.context.model_config.model_config.num_labels;+ let mut state = ForwardPassState::new_classifier(+ self.context.context.clone(),+ &self.context.model_shape,+ &self.context.scratch_buffers,+ self.context.shared_buffers.clone(),+ token_ids,+ token_positions,+ true,+ num_labels,+ );++ let encoding_params = EncodingParameters::new();++ let mut command_buffer = self+ .context+ .context+ .create_command_buffer()+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?+ .start_encoding();++ self.context+ .embed+ .encode_lookup(&mut state, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context+ .embedding_norm+ .encode(&mut state, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ for layer in self.context.layers.iter() {+ layer+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;-- command_buffer- .end_encoding()- .submit()- .wait_until_completed()- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;-- let logits = self.copy_logits_from_state(&state)?;-- Ok(logits)- })+ }+ self.context+ .output_norm+ .encode(&mut state, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.pooling.encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context+ .prediction_head+ .encode(&mut state, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;++ command_buffer+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;++ let logits = self.copy_logits_from_state(&state)?;++ Ok(logits)}fn copy_logits_from_state(diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex 9c1d60ac5..658d6de62 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -1,7 +1,5 @@use std::rc::Rc;-use objc2::rc::autoreleasepool;-use crate::{DataType,backends::common::{Backend, CommandBuffer},@@ -46,84 +44,19 @@ impl<B: Backend> ClassifierLayer<B> {layer_loader: &ParameterTree<B::Context>,rope: Rc<Rope<B>>,) -> Self {- autoreleasepool(|_| {- let ctx = context.as_ref(); // Reference for functions expecting &B::Context- let attention_config =- layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");- let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();-- let copy_main_to_shortcut_mixer = TensorCopy::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap();-- let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {- if layer_loader.subtree("pre_mixer_norm").is_ok() {- Some(- Normalization::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &layer_loader.subtree("pre_mixer_norm").unwrap(),- )- .expect("Failed to create pre-attention norm kernel"),- )- } else {- None- }- } else {- None- };+ let ctx = context.as_ref(); // Reference for functions expecting &B::Context+ let attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");+ let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();- let qkv_projection = <dyn Linear<B>>::new(- &attention_config.qkv_projection_config,- attention_config.has_qkv_biases,- model_dim,- [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],- ctx,- &layer_loader.subtree("mixer.qkv_projection").unwrap(),- ArrayId::Main,- ArrayId::QKV,- )- .expect("Failed to create qkv projection");-- let qk_norm = if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some()- {- match QKNorm::new(- ctx,- intermediate_data_type,- attention_config.query_norm_config.clone(),- attention_config.key_norm_config.clone(),- ArrayId::QKV,- &layer_loader.subtree("mixer").unwrap(),- num_heads,- num_groups,- head_dim,- ) {- Ok(norm) => Some(norm),- Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),- }- } else {- None- };+ let copy_main_to_shortcut_mixer = TensorCopy::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap();- let out_projection = <dyn Linear<B>>::new(- &attention_config.out_projection_config,- attention_config.has_out_biases,- num_heads * head_dim,- [model_dim],- ctx,- &layer_loader.subtree("mixer.out_projection").unwrap(),- ArrayId::AttentionOutput,- ArrayId::Main,- )- .expect("Failed to create out projection");-- let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {+ let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {+ if layer_loader.subtree("pre_mixer_norm").is_ok() {Some(Normalization::new(ctx,@@ -131,99 +64,160 @@ impl<B: Backend> ClassifierLayer<B> {norm_config.clone(),ArrayId::Main,ArrayId::Main,- &layer_loader.subtree("post_mixer_norm").unwrap(),+ &layer_loader.subtree("pre_mixer_norm").unwrap(),)- .expect("Failed to create post-attention norm kernel"),+ .expect("Failed to create pre-attention norm kernel"),)} else {None- };+ }+ } else {+ None+ };- let mixer_residual_add = TensorAddSwap::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),- )- .unwrap();+ let qkv_projection = <dyn Linear<B>>::new(+ &attention_config.qkv_projection_config,+ attention_config.has_qkv_biases,+ model_dim,+ [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],+ ctx,+ &layer_loader.subtree("mixer.qkv_projection").unwrap(),+ ArrayId::Main,+ ArrayId::QKV,+ )+ .expect("Failed to create qkv projection");- let copy_main_to_shortcut_mlp = TensorCopy::<B>::new(+ let qk_norm = if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {+ match QKNorm::new(ctx,intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap();+ attention_config.query_norm_config.clone(),+ attention_config.key_norm_config.clone(),+ ArrayId::QKV,+ &layer_loader.subtree("mixer").unwrap(),+ num_heads,+ num_groups,+ head_dim,+ ) {+ Ok(norm) => Some(norm),+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),+ }+ } else {+ None+ };- let pre_mlp_norm = Normalization::new(- ctx,- intermediate_data_type,- layer_config.pre_mlp_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &layer_loader.subtree("pre_mlp_norm").unwrap(),- )- .expect("Failed to create pre-MLP norm kernel");+ let out_projection = <dyn Linear<B>>::new(+ &attention_config.out_projection_config,+ attention_config.has_out_biases,+ num_heads * head_dim,+ [model_dim],+ ctx,+ &layer_loader.subtree("mixer.out_projection").unwrap(),+ ArrayId::AttentionOutput,+ ArrayId::Main,+ )+ .expect("Failed to create out projection");- let mlp = <dyn Mlp<B>>::new(- &layer_config.mlp_config,- model_dim,- hidden_dim,- context.as_ref(),- &layer_loader.subtree("mlp").unwrap(),+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {+ Some(+ Normalization::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("post_mixer_norm").unwrap(),+ )+ .expect("Failed to create post-attention norm kernel"),)- .expect("Failed to create mlp block");+ } else {+ None+ };- let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {- Some(- Normalization::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &layer_loader.subtree("post_mlp_norm").unwrap(),- )- .expect("Failed to create post-MLP norm kernel"),- )- } else {- None- };+ let mixer_residual_add = TensorAddSwap::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap();- let attention = Attention::new(- ctx,- intermediate_data_type,- layer_index,- attention_scale,- attention_config.has_sinks,- false,- attention_config.sliding_window_size,- )- .expect("Failed to create attention kernel");+ let copy_main_to_shortcut_mlp = TensorCopy::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap();- let mlp_residual_add = TensorAddSwap::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ let pre_mlp_norm = Normalization::new(+ ctx,+ intermediate_data_type,+ layer_config.pre_mlp_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("pre_mlp_norm").unwrap(),+ )+ .expect("Failed to create pre-MLP norm kernel");++ let mlp = <dyn Mlp<B>>::new(+ &layer_config.mlp_config,+ model_dim,+ hidden_dim,+ context.as_ref(),+ &layer_loader.subtree("mlp").unwrap(),+ )+ .expect("Failed to create mlp block");++ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {+ Some(+ Normalization::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("post_mlp_norm").unwrap(),+ )+ .expect("Failed to create post-MLP norm kernel"),)- .unwrap();+ } else {+ None+ };- Self {- layer_index,- copy_main_to_shortcut_mixer,- pre_attention_norm,- qkv_projection,- qk_norm,- rope,- attention,- out_projection,- post_attention_norm,- mixer_residual_add,- copy_main_to_shortcut_mlp,- pre_mlp_norm,- mlp,- post_mlp_norm,- mlp_residual_add,- }- })+ let attention = Attention::new(+ ctx,+ intermediate_data_type,+ layer_index,+ attention_scale,+ attention_config.has_sinks,+ false,+ attention_config.sliding_window_size,+ )+ .expect("Failed to create attention kernel");++ let mlp_residual_add = TensorAddSwap::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap();++ Self {+ layer_index,+ copy_main_to_shortcut_mixer,+ pre_attention_norm,+ qkv_projection,+ qk_norm,+ rope,+ attention,+ out_projection,+ post_attention_norm,+ mixer_residual_add,+ copy_main_to_shortcut_mlp,+ pre_mlp_norm,+ mlp,+ post_mlp_norm,+ mlp_residual_add,+ }}pub fn encode(diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex 0c4b4aed5..1d2671500 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -2,8 +2,6 @@use std::rc::Rc;-use objc2::rc::autoreleasepool;-use super::MixerExecutables;use crate::{DataType, DecoderLayerConfig,@@ -46,198 +44,196 @@ impl<B: Backend> LayerExecutables<B> {decoder_layer_loader: &ParameterTree<B::Context>,rope: Option<Rc<Rope<B>>>,) -> Self {- autoreleasepool(|_| {- let ctx = context.as_ref(); // Reference for functions expecting &B::Context- let intermediate_data_type: DataType = match &layer_config.mixer_config {- MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),- MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),- MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),- };- let copy_main_to_shortcut = TensorCopy::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap();+ let ctx = context.as_ref(); // Reference for functions expecting &B::Context+ let intermediate_data_type: DataType = match &layer_config.mixer_config {+ MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),+ MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),+ MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),+ };+ let copy_main_to_shortcut = TensorCopy::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap();- let pre_attention_norm = RMSNorm::new(- ctx,- intermediate_data_type,- layer_config.pre_attention_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel");+ let pre_attention_norm = RMSNorm::new(+ ctx,+ intermediate_data_type,+ layer_config.pre_attention_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel");- let mixer = match &layer_config.mixer_config {- MixerConfig::Attention(attention_config) => {- let rope_block = rope.expect("RoPE encoder missing for attention layer");+ let mixer = match &layer_config.mixer_config {+ MixerConfig::Attention(attention_config) => {+ let rope_block = rope.expect("RoPE encoder missing for attention layer");- let qkv_projection = <dyn Linear<B>>::new(- &attention_config.qkv_projection_config,- attention_config.has_qkv_biases,- model_dim,- [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],- ctx,- &decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),- ArrayId::Main,- ArrayId::QKV,- )- .expect("Failed to create qkv projection");+ let qkv_projection = <dyn Linear<B>>::new(+ &attention_config.qkv_projection_config,+ attention_config.has_qkv_biases,+ model_dim,+ [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],+ ctx,+ &decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),+ ArrayId::Main,+ ArrayId::QKV,+ )+ .expect("Failed to create qkv projection");- let qk_norm =- if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {- match QKNorm::new(- ctx,- intermediate_data_type,- attention_config.query_norm_config.clone(),- attention_config.key_norm_config.clone(),- ArrayId::QKV,- &decoder_layer_loader.subtree("mixer").unwrap(),- num_heads,- num_groups,- head_dim,- ) {- Ok(qk_norm) => Some(qk_norm),- Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),- }- } else {- None- };+ let qk_norm =+ if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {+ match QKNorm::new(+ ctx,+ intermediate_data_type,+ attention_config.query_norm_config.clone(),+ attention_config.key_norm_config.clone(),+ ArrayId::QKV,+ &decoder_layer_loader.subtree("mixer").unwrap(),+ num_heads,+ num_groups,+ head_dim,+ ) {+ Ok(qk_norm) => Some(qk_norm),+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),+ }+ } else {+ None+ };- let out_projection = <dyn Linear<B>>::new(- &attention_config.out_projection_config,- attention_config.has_out_biases,- num_heads * head_dim,- [model_dim],- ctx,- &decoder_layer_loader.subtree("mixer.out_projection").unwrap(),- ArrayId::AttentionOutput,- ArrayId::Main,- )- .expect("Failed to create out projection");+ let out_projection = <dyn Linear<B>>::new(+ &attention_config.out_projection_config,+ attention_config.has_out_biases,+ num_heads * head_dim,+ [model_dim],+ ctx,+ &decoder_layer_loader.subtree("mixer.out_projection").unwrap(),+ ArrayId::AttentionOutput,+ ArrayId::Main,+ )+ .expect("Failed to create out projection");- let attention = Attention::new(- ctx,- intermediate_data_type,- layer_index,- attention_scale,- attention_config.has_sinks,- attention_config.is_causal.unwrap_or(true),- attention_config.sliding_window_size,- )- .expect("Failed to create AttentionWrapper kernel");+ let attention = Attention::new(+ ctx,+ intermediate_data_type,+ layer_index,+ attention_scale,+ attention_config.has_sinks,+ attention_config.is_causal.unwrap_or(true),+ attention_config.sliding_window_size,+ )+ .expect("Failed to create AttentionWrapper kernel");- MixerExecutables::Attention {- qkv_projection,- qk_norm,- rope: rope_block,- attention,- out_projection,- }- },- MixerConfig::Mamba(mamba_config) => {- let mixer = MambaMixer::new(- ctx,- layer_type.clone(),- mamba_config.clone(),- layer_index,- model_dim,- num_heads,- head_dim,- num_groups,- decoder_layer_loader,- );- MixerExecutables::StateSpace {- mixer,- }- },- MixerConfig::ShortConv(short_conv_config) => {- let mixer = ShortConvMixer::new(- ctx,- layer_type.clone(),- short_conv_config.clone(),- layer_index,- model_dim,- decoder_layer_loader,- );- MixerExecutables::ShortConv {- mixer,- }- },- };+ MixerExecutables::Attention {+ qkv_projection,+ qk_norm,+ rope: rope_block,+ attention,+ out_projection,+ }+ },+ MixerConfig::Mamba(mamba_config) => {+ let mixer = MambaMixer::new(+ ctx,+ layer_type.clone(),+ mamba_config.clone(),+ layer_index,+ model_dim,+ num_heads,+ head_dim,+ num_groups,+ decoder_layer_loader,+ );+ MixerExecutables::StateSpace {+ mixer,+ }+ },+ MixerConfig::ShortConv(short_conv_config) => {+ let mixer = ShortConvMixer::new(+ ctx,+ layer_type.clone(),+ short_conv_config.clone(),+ layer_index,+ model_dim,+ decoder_layer_loader,+ );+ MixerExecutables::ShortConv {+ mixer,+ }+ },+ };- let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {- Some(- RMSNorm::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {+ Some(+ RMSNorm::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),)- } else {- None- };-- let main_shortcut_add_swap = TensorAddSwap::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ .expect("Failed to create RMS norm kernel"),)- .unwrap();+ } else {+ None+ };- let pre_mlp_norm = RMSNorm::new(- ctx,- intermediate_data_type,- layer_config.pre_mlp_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel");+ let main_shortcut_add_swap = TensorAddSwap::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap();- let mlp = <dyn Mlp<B>>::new(- &layer_config.mlp_config,- model_dim,- hidden_dim,- context.as_ref(),- &decoder_layer_loader.subtree("mlp").unwrap(),- )- .expect("Failed to create mlp block");+ let pre_mlp_norm = RMSNorm::new(+ ctx,+ intermediate_data_type,+ layer_config.pre_mlp_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel");- let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {- Some(- RMSNorm::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),+ let mlp = <dyn Mlp<B>>::new(+ &layer_config.mlp_config,+ model_dim,+ hidden_dim,+ context.as_ref(),+ &decoder_layer_loader.subtree("mlp").unwrap(),+ )+ .expect("Failed to create mlp block");++ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {+ Some(+ RMSNorm::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),)- } else {- None- };+ .expect("Failed to create RMS norm kernel"),+ )+ } else {+ None+ };- Self {- layer_index,- copy_main_to_shortcut,- pre_attention_norm,- mixer,- post_attention_norm,- main_shortcut_add_swap,- pre_mlp_norm,- mlp,- post_mlp_norm,- }- })+ Self {+ layer_index,+ copy_main_to_shortcut,+ pre_attention_norm,+ mixer,+ post_attention_norm,+ main_shortcut_add_swap,+ pre_mlp_norm,+ mlp,+ post_mlp_norm,+ }}pub fn encode(diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex f2655a3e0..ac08f8599 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -237,9 +237,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let _ = self.gpu_capture.start_capture(&self.context.context, "prefill");}- objc2::rc::autoreleasepool(|_pool| {- let _ = last_state.take();- });+ let _ = last_state.take();let task = Task {token_ids: &step_token_ids,@@ -612,9 +610,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {}fn clear_cache(&mut self) {- objc2::rc::autoreleasepool(|_pool| {- self.pre_encoded_task = None;- });+ self.pre_encoded_task = None;}fn reset_state(&mut self) {@@ -742,88 +738,86 @@ impl<B: Backend> LanguageModelGenerator<B> {sampling_method: SamplingMethod,should_fill_attention_bias: bool,) -> Result<(ForwardPassState<B>, f64), Error> {- objc2::rc::autoreleasepool(|_pool| {- let run_start = Instant::now();-- let mut state = ForwardPassState::new_llm(- self.context.context.clone(),- &self.context.decoder_config,- &self.context.model_shape,- &self.context.scratch_buffers,- self.context.cache_layers.clone(),- self.context.shared_buffers.clone(),- task.token_ids,- task.token_positions,- task.token_bitmask,- task.token_seeds,- task.active_suffix_length,- task.sampling_start,- task.sampling_length,- task.is_prefilling,- None,- false,- should_fill_attention_bias,- None,- None,- );+ let run_start = Instant::now();- if let Some(method) = state.sampling_method_mut() {- *method = Some(sampling_method);- }+ let mut state = ForwardPassState::new_llm(+ self.context.context.clone(),+ &self.context.decoder_config,+ &self.context.model_shape,+ &self.context.scratch_buffers,+ self.context.cache_layers.clone(),+ self.context.shared_buffers.clone(),+ task.token_ids,+ task.token_positions,+ task.token_bitmask,+ task.token_seeds,+ task.active_suffix_length,+ task.sampling_start,+ task.sampling_length,+ task.is_prefilling,+ None,+ false,+ should_fill_attention_bias,+ None,+ None,+ );- let encoding_key = TaskEncodingKey {- context_len: self.tokens.len(),- batch_size: task.token_ids.len(),- expected_number_of_new_tokens: task.expected_number_of_new_tokens,- active_suffix_len: task.active_suffix_length,- sampling_method,- sampling_start: task.sampling_start,- sampling_len: task.sampling_length,- has_bitmask: task.token_bitmask.is_some(),- is_prefilling: task.is_prefilling,- };+ if let Some(method) = state.sampling_method_mut() {+ *method = Some(sampling_method);+ }- let is_first_decode = task.token_ids.len() == 1;- let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);+ let encoding_key = TaskEncodingKey {+ context_len: self.tokens.len(),+ batch_size: task.token_ids.len(),+ expected_number_of_new_tokens: task.expected_number_of_new_tokens,+ active_suffix_len: task.active_suffix_length,+ sampling_method,+ sampling_start: task.sampling_start,+ sampling_len: task.sampling_length,+ has_bitmask: task.token_bitmask.is_some(),+ is_prefilling: task.is_prefilling,+ };- if should_capture {- self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;- self.pre_encoded_task = None;- }+ let is_first_decode = task.token_ids.len() == 1;+ let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);- let sample = !task.is_prefilling;+ if should_capture {+ self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;+ self.pre_encoded_task = None;+ }- let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()- && pre_encoded_key == encoding_key- {- pre_encoded_executable- } else {- self.encode_forward_pass(&mut state, &EncodingParameters::new(), sample)?- };+ let sample = !task.is_prefilling;- let pending = executable.submit();+ let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()+ && pre_encoded_key == encoding_key+ {+ pre_encoded_executable+ } else {+ self.encode_forward_pass(&mut state, &EncodingParameters::new(), sample)?+ };- if allow_pre_encode {- let mut next_encoding_key = encoding_key;+ let pending = executable.submit();- next_encoding_key.context_len += 1;+ if allow_pre_encode {+ let mut next_encoding_key = encoding_key;- let next_executable =- self.encode_forward_pass(&mut state, &EncodingParameters::new().with_projection(1), sample)?;+ next_encoding_key.context_len += 1;- self.pre_encoded_task = Some((next_encoding_key, next_executable));- }+ let next_executable =+ self.encode_forward_pass(&mut state, &EncodingParameters::new().with_projection(1), sample)?;- pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ self.pre_encoded_task = Some((next_encoding_key, next_executable));+ }- let run_time = run_start.elapsed().as_secs_f64();+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;- if should_capture {- self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;- }+ let run_time = run_start.elapsed().as_secs_f64();- Ok((state, run_time))- })+ if should_capture {+ self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;+ }++ Ok((state, run_time))}fn encode_forward_pass(diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rsindex a6b2cc3ab..bf17cd263 100644--- a/crates/uzu/src/prelude.rs+++ b/crates/uzu/src/prelude.rs@@ -21,5 +21,4 @@ pub use crate::{},},speculators::{empty_speculator::EmptySpeculator, speculator::Speculator},- storage_path,};diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 0e4eef3f5..d8031c18a 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -11,11 +11,6 @@ use std::{time::Instant,};-fn nanos_to_secs(nanos: u64) -> f64 {- nanos as f64 / 1_000_000_000.0-}--use objc2::rc::autoreleasepool;use tokenizers::Tokenizer;use xgrammar::TokenizerInfo;@@ -35,6 +30,10 @@ use crate::{},};+fn nanos_to_secs(nanos: u64) -> f64 {+ nanos as f64 / 1_000_000_000.0+}+struct RunContext {eos_tokens: Vec<u64>,context_length: usize,@@ -696,8 +695,6 @@ impl ChatSession {impl Drop for ChatSession {fn drop(&mut self) {- autoreleasepool(|_| {- self.llm = None;- });+ self.llm = None}}diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rsindex ad393c6b5..4df84d5ba 100644--- a/crates/uzu/src/utils/mod.rs+++ b/crates/uzu/src/utils/mod.rs@@ -2,10 +2,8 @@ pub mod attention;pub mod env_utils;pub mod model_size;pub mod pointers;-pub mod storage;pub mod version;pub use env_utils::*;pub use model_size::ModelSize;-pub use storage::*;pub use version::VERSION;diff --git a/crates/uzu/src/utils/storage.rs b/crates/uzu/src/utils/storage.rsdeleted file mode 100644index cde74aeff..000000000--- a/crates/uzu/src/utils/storage.rs+++ /dev/null@@ -1,85 +0,0 @@-use std::path::PathBuf;--use objc2::rc::{Retained, autoreleasepool};-use objc2_foundation::{NSHomeDirectory, NSSearchPathDirectory, NSString};--const STORAGE_DIR_NAME: &str = "com.mirai.sdk.storage";--/// Returns the *user-domain* path for a given directory without any-/// application-specific suffixes.-pub fn user_domain_path(dir: NSSearchPathDirectory) -> PathBuf {- #[cfg(any(target_os = "macos", target_os = "ios"))]- {- use objc2_foundation::{NSFileManager, NSSearchPathDomainMask};-- if let Some(path) = autoreleasepool(|_| {- let fm = NSFileManager::defaultManager();- let urls = fm.URLsForDirectory_inDomains(dir, NSSearchPathDomainMask::UserDomainMask);-- urls.firstObject().map(|url| {- let ns_path: Retained<NSString> = url.path().expect("URL should have a path");- PathBuf::from(ns_path.to_string())- })- }) {- return path;- }- }-- let home = std::env::var("HOME").unwrap_or_else(|_| String::from("."));- if dir == NSSearchPathDirectory::CachesDirectory {- #[cfg(target_os = "macos")]- {- PathBuf::from(&home).join("Library").join("Caches")- }- #[cfg(not(target_os = "macos"))]- {- PathBuf::from(&home).join(".cache")- }- } else if dir == NSSearchPathDirectory::DownloadsDirectory {- PathBuf::from(&home).join("Downloads")- } else if dir == NSSearchPathDirectory::DocumentDirectory {- PathBuf::from(&home).join("Documents")- } else {- PathBuf::from(home)- }-}--/// Platform-specific directory the SDK uses for persistent storage.-///-/// * macOS: `~/Library/Caches/<STORAGE_DIR_NAME>/<VERSION>/`-/// * iOS: `~/Documents/<STORAGE_DIR_NAME>/<VERSION>/`-/// * other: `$HOME/.cache/<STORAGE_DIR_NAME>/<VERSION>/`-pub fn storage_path() -> PathBuf {- #[cfg(target_os = "macos")]- let base = {- // In a sandboxed macOS application, NSHomeDirectory() returns- // `~/Library/Containers/<bundle-id>/Data`. Persist model files inside- // the app-scoped caches directory to comply with sandbox rules:- // `~/Library/Containers/<bundle-id>/Data/Library/Caches/…`.- let home = NSHomeDirectory().to_string();- PathBuf::from(&home).join("Library").join("Caches")- };-- #[cfg(target_os = "ios")]- let base = user_domain_path(NSSearchPathDirectory::DocumentDirectory);-- #[cfg(not(any(target_os = "macos", target_os = "ios")))]- let base = {- let home = std::env::var("HOME").unwrap_or_else(|_| String::from("."));- PathBuf::from(home).join(".cache")- };-- let full = base.join(STORAGE_DIR_NAME).join(crate::VERSION);- let _ = std::fs::create_dir_all(&full);- full-}--/// Deprecated: use `user_domain_path` or `storage_path` instead. Retained for-/// compatibility with existing code paths.-pub fn root_dir(location: NSSearchPathDirectory) -> PathBuf {- if location == NSSearchPathDirectory::CachesDirectory {- storage_path()- } else {- user_domain_path(location)- }-}diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex ad2d1907c..bc311814b 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]mod common;diff --git a/crates/uzu/tests/audio_codec_test.rs b/crates/uzu/tests/audio_codec_test.rsindex 8e5348c2d..d545ef699 100644--- a/crates/uzu/tests/audio_codec_test.rs+++ b/crates/uzu/tests/audio_codec_test.rs@@ -1,4 +1,4 @@-#![cfg(all(feature = "audio-runtime", target_os = "macos"))]+#![cfg(all(feature = "audio-runtime", metal_backend))]use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};use uzu::{diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex 688235123..90b95e1b7 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -89,7 +89,7 @@ macro_rules! for_each_backend {type $B = uzu::backends::cpu::Cpu;$body}- #[cfg(feature = "metal")]+ #[cfg(metal_backend)]{type $B = uzu::backends::metal::Metal;$body@@ -100,7 +100,7 @@ macro_rules! for_each_backend {#[macro_export]macro_rules! for_each_non_cpu_backend {(|$B:ident| $body:expr) => {{- #[cfg(feature = "metal")]+ #[cfg(metal_backend)]{type $B = uzu::backends::metal::Metal;$bodydiff --git a/crates/uzu/tests/common/proptest.rs b/crates/uzu/tests/common/proptest.rsindex 4236e303c..282048327 100644--- a/crates/uzu/tests/common/proptest.rs+++ b/crates/uzu/tests/common/proptest.rs@@ -1,7 +1,7 @@use std::rc::Rc;use proptest::prelude::*;-#[cfg(feature = "metal")]+#[cfg(metal_backend)]use uzu::backends::metal::Metal;use uzu::{DataType,@@ -17,7 +17,7 @@ pub fn kernel_data_type() -> impl Strategy<Value = DataType> {pub struct TestContextes {pub cpu: Rc<<Cpu as Backend>::Context>,- #[cfg(feature = "metal")]+ #[cfg(metal_backend)]pub metal: Rc<<Metal as Backend>::Context>,}@@ -25,7 +25,7 @@ impl TestContextes {pub fn new() -> TestContextes {TestContextes {cpu: <Cpu as Backend>::Context::new().expect("Failed to create Cpu context"),- #[cfg(feature = "metal")]+ #[cfg(metal_backend)]metal: <Metal as Backend>::Context::new().expect("Failed to create Metal context"),}}@@ -33,7 +33,7 @@ impl TestContextes {pub struct TestResults<T> {pub cpu: T,- #[cfg(feature = "metal")]+ #[cfg(metal_backend)]pub metal: T,}@@ -46,7 +46,7 @@ macro_rules! for_each_context {let $CONTEXT_NAME = $CONTEXTES.cpu.as_ref();$body})?,- #[cfg(feature = "metal")]+ #[cfg(metal_backend)]metal: ({type $CONTEXT_TYPE = <uzu::backends::metal::Metal as uzu::backends::common::Backend>::Context;let $CONTEXT_NAME = $CONTEXTES.metal.as_ref();@@ -66,7 +66,7 @@ pub trait ComparableTestResults {impl<T: ComparableTestResults> TestResults<T> {pub fn compare_results(&self) -> Result<(), TestCaseError> {- #[cfg(feature = "metal")]+ #[cfg(metal_backend)]T::compare("metal", &self.metal, &self.cpu)?;Ok(())diff --git a/crates/uzu/tests/context_mode_test.rs b/crates/uzu/tests/context_mode_test.rsindex a18ef59fa..0815def03 100644--- a/crates/uzu/tests/context_mode_test.rs+++ b/crates/uzu/tests/context_mode_test.rs@@ -1,4 +1,7 @@+#![cfg(metal_backend)]+mod common;+use std::path::PathBuf;use test_tag::tag;diff --git a/crates/uzu/tests/grammar_json_schema_test.rs b/crates/uzu/tests/grammar_json_schema_test.rsindex 45cd0f53e..fbab1248b 100644--- a/crates/uzu/tests/grammar_json_schema_test.rs+++ b/crates/uzu/tests/grammar_json_schema_test.rs@@ -1,4 +1,4 @@-#![cfg(target_os = "macos")]+#![cfg(metal_backend)]use std::sync::Arc;diff --git a/crates/uzu/tests/kernel/rms_norm_test.rs b/crates/uzu/tests/kernel/rms_norm_test.rsindex be4345970..b74d69830 100644--- a/crates/uzu/tests/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/kernel/rms_norm_test.rs@@ -7,8 +7,6 @@ use std::{use half::{bf16, f16};use num_traits::Float;use rand::{RngExt, SeedableRng, prelude::StdRng};-#[cfg(feature = "metal")]-use uzu::backends::metal::Metal;use uzu::{ArrayElement, DataType,array::ArrayContextExt,@@ -440,16 +438,16 @@ fn test_edge_f16_f16_f16_f16() {}// performance tests+#[cfg(metal_backend)]#[test]fn test_perf_8k() {- if cfg!(feature = "metal") {- test_performance::<Metal>(8, 8192); // Large model like LLaMA-70B- }+ use uzu::backends::metal::Metal;+ test_performance::<Metal>(8, 8192); // Large model like LLaMA-70B}+#[cfg(metal_backend)]#[test]fn test_perf_16k() {- if cfg!(feature = "metal") {- test_performance::<Metal>(16, 16384); // Huge model dimension- }+ use uzu::backends::metal::Metal;+ test_performance::<Metal>(16, 16384); // Huge model dimension}diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex e73dc063b..f176b52f7 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use uzu::{DataType,diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rsindex da5c15980..8f704c09d 100644--- a/crates/uzu/tests/kv_cache_update_test.rs+++ b/crates/uzu/tests/kv_cache_update_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use std::{cell::RefCell, rc::Rc};diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 1608ba0ed..c868918d6 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -1,5 +1,7 @@//! Correctness tests comparing Metal matmul kernels against ndarray+#![cfg(metal_backend)]+use bytemuck;use half::bf16;use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};diff --git a/crates/uzu/tests/matmul_perf_test/main.rs b/crates/uzu/tests/matmul_perf_test/main.rsindex 9bfd1fffb..fd7eee941 100644--- a/crates/uzu/tests/matmul_perf_test/main.rs+++ b/crates/uzu/tests/matmul_perf_test/main.rs@@ -1,4 +1,4 @@-#![cfg(all(target_os = "macos", feature = "metal"))]+#![cfg(all(target_os = "macos", metal_backend))]mod bench;mod error;diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rsindex e79359560..ce4c5badd 100644--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use half::bf16;use metal::MTLBuffer;diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex a30ea799f..8b038f7f0 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -1,3 +1,5 @@+#![cfg(metal_backend)]+use half::bf16;use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rsindex 8fd419e07..b7c5377da 100644--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs@@ -1,3 +1,5 @@+#![cfg(metal_backend)]+use std::time::Instant;use half::bf16;diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rsindex 0afb09350..701784cb9 100644--- a/crates/uzu/tests/moe/moe_experts_test.rs+++ b/crates/uzu/tests/moe/moe_experts_test.rs@@ -6,7 +6,7 @@//! - Intermediate buffer correctness (row maps, tiles, dispatch args)//! - Numerical correctness against CPU reference-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use half::bf16;use metal::MTLBuffer;diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rsindex 3677d3a5c..583b5e376 100644--- a/crates/uzu/tests/moe/moe_finalize_test.rs+++ b/crates/uzu/tests/moe/moe_finalize_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use half::bf16;use metal::MTLBuffer;diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rsindex f9cf5a01a..b38c936ef 100644--- a/crates/uzu/tests/moe/moe_gather_test.rs+++ b/crates/uzu/tests/moe/moe_gather_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use half::bf16;use metal::MTLBuffer;diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex 1e734200c..e87707aba 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -1,3 +1,5 @@+#![cfg(metal_backend)]+use std::time::Instant;use half::bf16;diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rsindex cb869b65b..75100831f 100644--- a/crates/uzu/tests/moe/moe_router_topk_test.rs+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use half::bf16;use metal::MTLBuffer;diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rsindex a00356825..e532d764a 100644--- a/crates/uzu/tests/moe/moe_scatter_test.rs+++ b/crates/uzu/tests/moe/moe_scatter_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use half::bf16;use metal::MTLBuffer;diff --git a/crates/uzu/tests/moe/moe_test_utils.rs b/crates/uzu/tests/moe/moe_test_utils.rsindex eb6217eb5..51881427f 100644--- a/crates/uzu/tests/moe/moe_test_utils.rs+++ b/crates/uzu/tests/moe/moe_test_utils.rs@@ -3,7 +3,7 @@//! Contains common helper functions, CPU ground truth implementations,//! buffer allocation helpers, and test fixtures used across all MoE test files.-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use std::rc::Rc;diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rsindex 9ab1ffc3e..ab7eac9f1 100644--- a/crates/uzu/tests/moe/moe_tiles_test.rs+++ b/crates/uzu/tests/moe/moe_tiles_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos", target_os = "ios"))]+#![cfg(metal_backend)]use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rsindex accf5c52a..94e564709 100644--- a/crates/uzu/tests/parameters_loader_test.rs+++ b/crates/uzu/tests/parameters_loader_test.rs@@ -1,3 +1,4 @@+#![cfg(metal_backend)]mod common;// New integration test for ParameterLoaderdiff --git a/crates/uzu/tests/person_schema_metal_bitmask_test.rs b/crates/uzu/tests/person_schema_metal_bitmask_test.rsindex 4e4e1fcc6..92ca87fbc 100644--- a/crates/uzu/tests/person_schema_metal_bitmask_test.rs+++ b/crates/uzu/tests/person_schema_metal_bitmask_test.rs@@ -1,4 +1,4 @@-#![cfg(target_os = "macos")]+#![cfg(metal_backend)]use std::{cell::RefCell, path::PathBuf, rc::Rc};diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rsindex acedd70ea..306d24b57 100644--- a/crates/uzu/tests/quant_matmul_test.rs+++ b/crates/uzu/tests/quant_matmul_test.rs@@ -1,3 +1,5 @@+#![cfg(metal_backend)]+use std::time::Instant;use bytemuck;diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex 9dfcd9b59..85ba5a08a 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -1,4 +1,7 @@+#![cfg(metal_backend)]+mod common;+use bytemuck;use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use rand::seq::SliceRandom;diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rsindex 70ca79769..c1c0ac69e 100644--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs@@ -1,4 +1,4 @@-#![cfg(target_os = "macos")]+#![cfg(metal_backend)]use std::mem::size_of;diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rsindex 9567e0434..b057715b8 100644--- a/crates/uzu/tests/ssm_update_test.rs+++ b/crates/uzu/tests/ssm_update_test.rs@@ -1,4 +1,4 @@-#![cfg(any(target_os = "macos"))]+#![cfg(metal_backend)]use bytemuck;use half::bf16;diff --git a/crates/uzu/tests/text_session_test.rs b/crates/uzu/tests/text_session_test.rsindex 2fe1b0e17..ef46240f9 100644--- a/crates/uzu/tests/text_session_test.rs+++ b/crates/uzu/tests/text_session_test.rs@@ -1,4 +1,7 @@+#![cfg(metal_backend)]+mod common;+use std::path::PathBuf;use test_tag::tag;diff --git a/crates/uzu/tests/tracer_test.rs b/crates/uzu/tests/tracer_test.rsindex d78365e79..ec6999ce4 100644--- a/crates/uzu/tests/tracer_test.rs+++ b/crates/uzu/tests/tracer_test.rs@@ -1,4 +1,4 @@-#![cfg(feature = "tracing")]+#![cfg(all(metal_backend, feature = "tracing"))]mod common;use std::path::PathBuf;diff --git a/rust-toolchain.toml b/rust-toolchain.tomlindex 6ed70738f..f1b1d8f97 100644--- a/rust-toolchain.toml+++ b/rust-toolchain.toml@@ -2,4 +2,4 @@channel = "stable"profile = "minimal"components = ["rustfmt", "clippy"]-targets = ["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin"]+targets = ["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "aarch64-linux-android", "x86_64-unknown-linux-gnu"]diff --git a/scripts/download_test_model.sh b/scripts/download_test_model.shindex cab4a5875..6bfa4d337 100755--- a/scripts/download_test_model.sh+++ b/scripts/download_test_model.sh@@ -2,11 +2,6 @@set -euo pipefail-if [[ "$(uname)" != "Darwin" ]]; then- echo "Non-macOS platform detected (\"$(uname)\"); skipping test-model download."- exit 0-fi-ROOT_PATH="$(cd "$(dirname "$0")/.." && pwd)"HELPERS_PATH="$ROOT_PATH/tools/helpers"