Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Objc2 dependency only for metal (#260)

Aleksandr Golokoz5c4a025180ca
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
index 8e98fee84..0892eb8fa 100644
--- a/.github/actions/setup/action.yml
+++ b/.github/actions/setup/action.yml
@@ -23,6 +23,9 @@ runs:
using: composite
steps:
- uses: dtolnay/rust-toolchain@stable
+ - uses: dtolnay/rust-toolchain@nightly
+ with:
+ components: rustfmt
- if: inputs.nextest == 'true'
uses: taiki-e/install-action@nextest
- if: inputs.linters == 'true'
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 571d035fc..43fb5416d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -22,7 +22,7 @@ defaults:
jobs:
linters:
- name: Linters
+ name: linters
runs-on: [self-hosted, macos, x64, mac, m3max, runner2]
steps:
@@ -32,12 +32,6 @@ jobs:
with:
linters: true
- - name: Install nightly rustfmt
- if: ${{ always() && steps.setup.outcome == 'success' }}
- uses: dtolnay/rust-toolchain@nightly
- with:
- components: rustfmt
-
- name: Check Rust formatting
if: ${{ always() && steps.setup.outcome == 'success' }}
run: cargo +nightly fmt --check
@@ -53,12 +47,13 @@ jobs:
git diff --exit-code -- Cargo.lock
tests:
- name: Tests${{ matrix.tracing && ' (tracing)' || '' }}
+ name: ${{ matrix.backend }} ${{ matrix.tracing && ' (tracing)' || '' }}
runs-on: [self-hosted, macos, x64, mac, m3max, runner1]
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
strategy:
matrix:
+ backend: ['cpu', 'metal']
tracing: [false, true]
env:
@@ -68,14 +63,14 @@ jobs:
- uses: actions/checkout@v6
- uses: ./.github/actions/setup
with:
- metal: true
+ metal: ${{ matrix.backend == 'metal' }}
test_model: true
nextest: true
- name: Build crates
run: |
- cargo build ${{ matrix.tracing && '--features tracing' || '' }}
+ cargo +stable build --no-default-features ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} ${{ matrix.tracing && '--features tracing' || '' }}
- name: Run tests
run: |
- cargo nextest run --profile=ci ${{ matrix.tracing && '--features tracing' || '' }}
+ cargo +stable nextest run --profile=ci --no-default-features ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} ${{ matrix.tracing && '--features tracing' || '' }}
diff --git a/Cargo.lock b/Cargo.lock
index 37289b08c..99d5be237 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -309,13 +309,8 @@ checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
name = "benchmarks"
version = "0.1.8"
dependencies = [
- "half",
- "libc",
"mach2",
- "mtl-rs",
- "objc2",
"serde",
- "serde_json",
"sysinfo",
"uzu",
]
@@ -381,12 +376,6 @@ dependencies = [
"cpufeatures 0.2.17",
]
-[[package]]
-name = "block"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
-
[[package]]
name = "block-buffer"
version = "0.10.4"
@@ -589,7 +578,6 @@ dependencies = [
"comfy-table",
"console 0.16.3",
"ctrlc",
- "futures",
"indicatif 0.18.4",
"inquire",
"rocket",
@@ -729,17 +717,6 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
-[[package]]
-name = "core-graphics-types"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb"
-dependencies = [
- "bitflags",
- "core-foundation 0.10.1",
- "libc",
-]
-
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -1266,28 +1243,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
- "foreign-types-shared 0.1.1",
-]
-
-[[package]]
-name = "foreign-types"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
-dependencies = [
- "foreign-types-macros",
- "foreign-types-shared 0.3.1",
-]
-
-[[package]]
-name = "foreign-types-macros"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.117",
+ "foreign-types-shared",
]
[[package]]
@@ -1296,12 +1252,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-[[package]]
-name = "foreign-types-shared"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"
-
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -2206,15 +2156,6 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
-[[package]]
-name = "malloc_buf"
-version = "0.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "matchers"
version = "0.2.0"
@@ -2249,21 +2190,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "metal"
-version = "0.32.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00c15a6f673ff72ddcc22394663290f870fb224c1bfce55734a75c414150e605"
-dependencies = [
- "bitflags",
- "block",
- "core-graphics-types",
- "foreign-types 0.5.0",
- "log",
- "objc",
- "paste",
-]
-
[[package]]
name = "miette"
version = "5.10.0"
@@ -2371,19 +2297,6 @@ dependencies = [
"cxx",
]
-[[package]]
-name = "mpsgraph"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b3d4f4ee48894ff2d83808e8a0333f00eb7576375a5a77e8132f10ebb2a1f86"
-dependencies = [
- "bitflags",
- "block2",
- "metal",
- "objc2",
- "objc2-foundation",
-]
-
[[package]]
name = "mtl-rs"
version = "0.1.7"
@@ -2540,15 +2453,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
-[[package]]
-name = "objc"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1"
-dependencies = [
- "malloc_buf",
-]
-
[[package]]
name = "objc2"
version = "0.6.4"
@@ -2655,7 +2559,7 @@ checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
dependencies = [
"bitflags",
"cfg-if",
- "foreign-types 0.3.2",
+ "foreign-types",
"libc",
"once_cell",
"openssl-macros",
@@ -4429,14 +4333,11 @@ dependencies = [
"anyhow",
"async-trait",
"bindgen",
- "bitflags",
"blake3",
- "block2",
"bytemuck",
"bytesize",
"cbindgen",
"clang-ast",
- "clap",
"comfy-table",
"depfile",
"dsl",
@@ -4449,7 +4350,6 @@ dependencies = [
"memmap2",
"minijinja",
"minijinja-contrib",
- "mpsgraph",
"mtl-rs",
"ndarray",
"num-traits",
diff --git a/Cargo.toml b/Cargo.toml
index b19869d93..e754234c3 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@ repository = "https://github.com/trymirai/uzu"
[workspace.dependencies]
benchmarks = { path = "crates/benchmarks" }
dsl = { path = "crates/dsl" }
-uzu = { path = "crates/uzu" }
+uzu = { path = "crates/uzu", default-features = false }
xgrammar = { package = "xgrammar-rs", version = "0.1", features = ["hf"] }
half = { version = "2.7", features = ["bytemuck", "num-traits"] }
ndarray = "0.17"
@@ -21,7 +21,6 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
thiserror = "2.0"
tokenizers = "0.22"
-mpsgraph = "0.2"
objc2 = "0.6"
objc2-foundation = "0.3"
metal = { package = "mtl-rs", version = "0.1" }
@@ -34,7 +33,6 @@ memmap2 = "0.9.8"
num-traits = "0.2"
regex = "1.12"
libc = "0.2"
-bitflags = "2.11"
minijinja = "2.17"
minijinja-contrib = { version = "2.17", features = ["pycompat"] }
clap = { version = "4.6", features = ["derive"] }
diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.toml
index 7501b6c94..c8ef2bcba 100644
--- a/crates/benchmarks/Cargo.toml
+++ b/crates/benchmarks/Cargo.toml
@@ -4,12 +4,14 @@ version.workspace = true
edition.workspace = true
[dependencies]
-metal.workspace = true
uzu.workspace = true
serde.workspace = true
-serde_json.workspace = true
sysinfo.workspace = true
-libc.workspace = true
+
+[target.'cfg(target_os = "macos")'.dependencies]
mach2.workspace = true
-half.workspace = true
-objc2.workspace = true
+
+[features]
+default = ["metal"]
+
+metal = ["uzu/metal"]
diff --git a/crates/benchmarks/src/runner/helpers/memory_usage.rs b/crates/benchmarks/src/runner/helpers/memory_usage.rs
index e93ebf88c..822703223 100644
--- a/crates/benchmarks/src/runner/helpers/memory_usage.rs
+++ b/crates/benchmarks/src/runner/helpers/memory_usage.rs
@@ -1,15 +1,16 @@
-use std::mem;
+#[cfg(target_os = "macos")]
+pub fn get_memory_usage() -> Option<u64> {
+ use std::mem;
-use mach2::{
- kern_return::KERN_SUCCESS,
- mach_types::task_t,
- message::mach_msg_type_number_t,
- task::task_info,
- task_info::{TASK_BASIC_INFO, task_basic_info},
- traps::mach_task_self,
-};
+ use mach2::{
+ kern_return::KERN_SUCCESS,
+ mach_types::task_t,
+ message::mach_msg_type_number_t,
+ task::task_info,
+ task_info::{TASK_BASIC_INFO, task_basic_info},
+ traps::mach_task_self,
+ };
-pub fn get_memory_usage() -> Option<u64> {
unsafe {
let task: task_t = mach_task_self();
@@ -33,3 +34,8 @@ pub fn get_memory_usage() -> Option<u64> {
}
}
}
+
+#[cfg(not(target_os = "macos"))]
+pub fn get_memory_usage() -> Option<u64> {
+ unimplemented!()
+}
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
index 95f7bf3ba..b58b8aec6 100644
--- a/crates/cli/Cargo.toml
+++ b/crates/cli/Cargo.toml
@@ -21,4 +21,8 @@ serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
comfy-table.workspace = true
-futures.workspace = true
+
+[features]
+default = ["metal"]
+
+metal = ["uzu/metal"]
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 414aa97a3..ad78a8c06 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -13,7 +13,6 @@ serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tokenizers.workspace = true
-mpsgraph.workspace = true
bytemuck.workspace = true
zerocopy = { workspace = true, features = ["derive"] }
xxhash-rust = { workspace = true, features = ["xxh3"] }
@@ -21,27 +20,20 @@ memmap2.workspace = true
num-traits.workspace = true
regex.workspace = true
libc.workspace = true
-bitflags.workspace = true
minijinja.workspace = true
minijinja-contrib.workspace = true
-clap.workspace = true
rand.workspace = true
sysinfo.workspace = true
itertools.workspace = true
schemars.workspace = true
-# metal feature requires objc2, block2 (for callbacks) and metal
-bytesize = { workspace = true, optional = true }
+# metal on macos requires objc2, bytesize, metal
+[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]
objc2 = { workspace = true, optional = true }
objc2-foundation = { workspace = true, optional = true }
-block2 = { workspace = true, optional = true }
+bytesize = { workspace = true, optional = true }
metal = { workspace = true, optional = true }
-# macos requires objc2 even on non-metal (but shouldn't)
-[target.'cfg(target_os = "macos")'.dependencies]
-objc2.workspace = true
-objc2-foundation.workspace = true
-
[dev-dependencies]
test-tag.workspace = true
proptest.workspace = true
@@ -49,7 +41,6 @@ rand_distr.workspace = true
is_close.workspace = true
serde_json5.workspace = true
schemars.workspace = true
-mpsgraph.workspace = true
comfy-table.workspace = true
indicatif.workspace = true
@@ -72,18 +63,18 @@ prettyplease.workspace = true
itertools.workspace = true
cbindgen.workspace = true
-# metal feature requires clang-ast
+# metal on macos requires clang-ast for dsl build
+[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.build-dependencies]
clang-ast = { workspace = true, optional = true }
[features]
default = ["metal"]
metal = [
- "dep:bytesize",
"dep:clang-ast",
"dep:objc2",
"dep:objc2-foundation",
- "dep:block2",
+ "dep:bytesize",
"dep:metal",
]
tracing = []
diff --git a/crates/uzu/build/common/caching.rs b/crates/uzu/build/common/caching.rs
index d4939e48b..c65dd27d0 100644
--- a/crates/uzu/build/common/caching.rs
+++ b/crates/uzu/build/common/caching.rs
@@ -1,3 +1,5 @@
+#![cfg(all(feature = "metal", target_os = "macos"))]
+
use std::{fs, sync::OnceLock};
use anyhow::Context;
diff --git a/crates/uzu/build/common/mangling.rs b/crates/uzu/build/common/mangling.rs
index b017e4f5b..2b256af80 100644
--- a/crates/uzu/build/common/mangling.rs
+++ b/crates/uzu/build/common/mangling.rs
@@ -1,3 +1,5 @@
+#![cfg(all(feature = "metal", target_os = "macos"))]
+
use std::iter::repeat_n;
use itertools::Itertools;
diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rs
index 9062be932..30826ed30 100644
--- a/crates/uzu/build/gpu_types/mod.rs
+++ b/crates/uzu/build/gpu_types/mod.rs
@@ -34,6 +34,7 @@ impl GpuTypesCompiler {
})
}
+ #[cfg(all(feature = "metal", target_os = "macos"))]
pub fn generated_header_dir(&self) -> &PathBuf {
&self.generated_dir
}
diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rs
index ae9dd59f2..a09bd59e8 100644
--- a/crates/uzu/build/main.rs
+++ b/crates/uzu/build/main.rs
@@ -8,7 +8,7 @@ mod gpu_types;
mod cpu;
-#[cfg(feature = "metal")]
+#[cfg(all(feature = "metal", target_os = "macos"))]
mod metal;
use common::{compiler::Compiler, envs, traitgen::traitgen_all};
@@ -21,6 +21,14 @@ async fn main() -> anyhow::Result<()> {
println!("cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun");
}
+ let metal_backend = cfg!(feature = "metal")
+ && matches!(env::var("CARGO_CFG_TARGET_OS").unwrap().as_ref(), "macos" | "ios" | "tvos" | "visionos");
+
+ println!("cargo::rustc-check-cfg=cfg(metal_backend)");
+ if metal_backend {
+ println!("cargo::rustc-cfg=metal_backend");
+ }
+
debug_log!("build script started");
if envs::build_clean() {
@@ -33,7 +41,6 @@ async fn main() -> anyhow::Result<()> {
}
let gpu_types_compiler = gpu_types::GpuTypesCompiler::new()?;
- let generated_header_dir = gpu_types_compiler.generated_header_dir().clone();
gpu_types_compiler.build().await?;
debug_log!("gpu_types build done");
@@ -41,8 +48,11 @@ async fn main() -> anyhow::Result<()> {
compilers.push(Box::new(cpu::CpuCompiler::new()?));
- #[cfg(feature = "metal")]
- compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));
+ #[cfg(all(feature = "metal", target_os = "macos"))]
+ if metal_backend {
+ let generated_header_dir = gpu_types_compiler.generated_header_dir().clone();
+ compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));
+ }
let backends_kernels = try_join_all(compilers.iter().map(|c| c.build())).await?;
diff --git a/crates/uzu/examples/quick_start_chat.rs b/crates/uzu/examples/quick_start_chat.rs
index d75619c9a..2c4792126 100644
--- a/crates/uzu/examples/quick_start_chat.rs
+++ b/crates/uzu/examples/quick_start_chat.rs
@@ -7,8 +7,10 @@ use uzu::session::{
};
fn main() -> Result<(), Box<dyn std::error::Error>> {
- let model_path = PathBuf::from("MODEL_PATH");
- let mut session = ChatSession::new(model_path, DecodingConfig::default())?;
+ let model_path = std::env::var("UZU_MODEL_PATH")
+ .map_err(|_| -> Box<dyn std::error::Error> { "UZU_MODEL_PATH environment variable is not set.".into() })?;
+ let model_path_buf = PathBuf::from(model_path);
+ let mut session = ChatSession::new(model_path_buf, DecodingConfig::default())?;
let input = Input::Text(String::from("Tell about London"));
let output = session.run(
diff --git a/crates/uzu/src/backends/mod.rs b/crates/uzu/src/backends/mod.rs
index d123fda17..dbf208ad2 100644
--- a/crates/uzu/src/backends/mod.rs
+++ b/crates/uzu/src/backends/mod.rs
@@ -1,17 +1,22 @@
pub mod common;
pub mod cpu;
-#[cfg(feature = "metal")]
+#[cfg(metal_backend)]
pub mod metal;
macro_rules! select_backend {
($expr:expr, $unk:expr) => {{
- match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or("metal") {
+ let mut default = "cpu";
+ if cfg!(metal_backend) {
+ default = "metal";
+ }
+
+ match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or(default) {
"cpu" => {
type B = crate::backends::cpu::Cpu;
$expr
},
- #[cfg(feature = "metal")]
+ #[cfg(metal_backend)]
"metal" => {
type B = crate::backends::metal::Metal;
$expr
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 0dd5bf55a..1b16fe499 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -2,8 +2,6 @@
use std::{cell::RefCell, rc::Rc};
use std::{collections::HashMap, path::Path, time::Instant};
-use objc2::rc::autoreleasepool;
-
#[cfg(feature = "tracing")]
use super::ActivationTrace;
use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
@@ -37,42 +35,38 @@ impl<B: Backend> ClassifierTrait for Classifier<B> {
) -> Result<ClassificationOutput, Error> {
let run_start = Instant::now();
- autoreleasepool(|_pool| {
- let forward_start = Instant::now();
-
- #[cfg(feature = "tracing")]
- let (logits, _traces) = self.forward_pass_with_traces(&token_ids, &token_positions)?;
-
- #[cfg(not(feature = "tracing"))]
- let logits = self.forward_pass(&token_ids, &token_positions)?;
-
- let forward_duration = forward_start.elapsed().as_secs_f64();
-
- let postprocessing_start = Instant::now();
- let probabilities = self.logits_to_probabilities(&logits)?;
- let postprocessing_duration = postprocessing_start.elapsed().as_secs_f64();
-
- let (predicted_label, confidence) = probabilities
- .iter()
- .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
- .map(|(label, prob)| (label.clone(), *prob))
- .unwrap_or((String::from("unknown"), 0.0));
-
- let stats = ClassificationStats::new(
- 0.0,
- forward_duration,
- postprocessing_duration,
- run_start.elapsed().as_secs_f64(),
- token_ids.len() as u64,
- predicted_label,
- confidence,
- );
-
- Ok(ClassificationOutput {
- logits,
- probabilities,
- stats,
- })
+ #[cfg(feature = "tracing")]
+ let (logits, _traces) = self.forward_pass_with_traces(&token_ids, &token_positions)?;
+
+ #[cfg(not(feature = "tracing"))]
+ let logits = self.forward_pass(&token_ids, &token_positions)?;
+
+ let forward_duration = run_start.elapsed().as_secs_f64();
+
+ let postprocessing_start = Instant::now();
+ let probabilities = self.logits_to_probabilities(&logits)?;
+ let postprocessing_duration = postprocessing_start.elapsed().as_secs_f64();
+
+ let (predicted_label, confidence) = probabilities
+ .iter()
+ .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
+ .map(|(label, prob)| (label.clone(), *prob))
+ .unwrap_or((String::from("unknown"), 0.0));
+
+ let stats = ClassificationStats::new(
+ 0.0,
+ forward_duration,
+ postprocessing_duration,
+ run_start.elapsed().as_secs_f64(),
+ token_ids.len() as u64,
+ predicted_label,
+ confidence,
+ );
+
+ Ok(ClassificationOutput {
+ logits,
+ probabilities,
+ stats,
})
}
}
@@ -100,78 +94,73 @@ impl<B: Backend> Classifier<B> {
token_ids: &[u64],
token_positions: &[usize],
) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {
- autoreleasepool(|_| {
- let num_labels = self.context.model_config.model_config.num_labels;
- let mut state = ForwardPassState::new_classifier(
- self.context.context.clone(),
- &self.context.model_shape,
- &self.context.scratch_buffers,
- self.context.shared_buffers.clone(),
- token_ids,
- token_positions,
- true,
- num_labels,
- );
-
- let encoding_params = EncodingParameters::new();
-
- let mut command_buffer = self
- .context
- .context
- .create_command_buffer()
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
- .start_encoding();
-
- self.context
- .embed
- .encode_lookup(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .embedding_norm
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- #[cfg(feature = "tracing")]
- {
- let traces = state.traces().clone();
- state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().embedding_norm().clone());
- }
-
- for layer in self.context.layers.iter() {
- layer
- .encode(&mut state, &encoding_params, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- }
- self.context
- .output_norm
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- #[cfg(feature = "tracing")]
- {
- let traces = state.traces().clone();
- state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());
- }
-
- self.context
- .pooling
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ let num_labels = self.context.model_config.model_config.num_labels;
+ let mut state = ForwardPassState::new_classifier(
+ self.context.context.clone(),
+ &self.context.model_shape,
+ &self.context.scratch_buffers,
+ self.context.shared_buffers.clone(),
+ token_ids,
+ token_positions,
+ true,
+ num_labels,
+ );
+
+ let encoding_params = EncodingParameters::new();
+
+ let mut command_buffer = self
+ .context
+ .context
+ .create_command_buffer()
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
+ .start_encoding();
+
+ self.context
+ .embed
+ .encode_lookup(&mut state, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context
+ .embedding_norm
+ .encode(&mut state, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ #[cfg(feature = "tracing")]
+ {
+ let traces = state.traces().clone();
+ state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().embedding_norm().clone());
+ }
- self.context
- .prediction_head
- .encode(&mut state, &mut command_buffer)
+ for layer in self.context.layers.iter() {
+ layer
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ }
+ self.context
+ .output_norm
+ .encode(&mut state, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ #[cfg(feature = "tracing")]
+ {
+ let traces = state.traces().clone();
+ state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());
+ }
- command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ self.context.pooling.encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- let logits = self.copy_logits_from_state(&state)?;
+ self.context
+ .prediction_head
+ .encode(&mut state, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- let traces = state.traces().clone();
- Ok((logits, traces))
- })
+ command_buffer
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+
+ let logits = self.copy_logits_from_state(&state)?;
+
+ let traces = state.traces().clone();
+ Ok((logits, traces))
}
#[cfg(not(feature = "tracing"))]
@@ -180,64 +169,59 @@ impl<B: Backend> Classifier<B> {
token_ids: &[u64],
token_positions: &[usize],
) -> Result<Box<[f32]>, Error> {
- autoreleasepool(|_| {
- let num_labels = self.context.model_config.model_config.num_labels;
- let mut state = ForwardPassState::new_classifier(
- self.context.context.clone(),
- &self.context.model_shape,
- &self.context.scratch_buffers,
- self.context.shared_buffers.clone(),
- token_ids,
- token_positions,
- true,
- num_labels,
- );
-
- let encoding_params = EncodingParameters::new();
-
- let mut command_buffer = self
- .context
- .context
- .create_command_buffer()
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
- .start_encoding();
-
- self.context
- .embed
- .encode_lookup(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .embedding_norm
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- for layer in self.context.layers.iter() {
- layer
- .encode(&mut state, &encoding_params, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- }
- self.context
- .output_norm
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .pooling
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .prediction_head
- .encode(&mut state, &mut command_buffer)
+ let num_labels = self.context.model_config.model_config.num_labels;
+ let mut state = ForwardPassState::new_classifier(
+ self.context.context.clone(),
+ &self.context.model_shape,
+ &self.context.scratch_buffers,
+ self.context.shared_buffers.clone(),
+ token_ids,
+ token_positions,
+ true,
+ num_labels,
+ );
+
+ let encoding_params = EncodingParameters::new();
+
+ let mut command_buffer = self
+ .context
+ .context
+ .create_command_buffer()
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
+ .start_encoding();
+
+ self.context
+ .embed
+ .encode_lookup(&mut state, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context
+ .embedding_norm
+ .encode(&mut state, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ for layer in self.context.layers.iter() {
+ layer
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
-
- command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
-
- let logits = self.copy_logits_from_state(&state)?;
-
- Ok(logits)
- })
+ }
+ self.context
+ .output_norm
+ .encode(&mut state, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.pooling.encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context
+ .prediction_head
+ .encode(&mut state, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+
+ command_buffer
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+
+ let logits = self.copy_logits_from_state(&state)?;
+
+ Ok(logits)
}
fn copy_logits_from_state(
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index 9c1d60ac5..658d6de62 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -1,7 +1,5 @@
use std::rc::Rc;
-use objc2::rc::autoreleasepool;
-
use crate::{
DataType,
backends::common::{Backend, CommandBuffer},
@@ -46,84 +44,19 @@ impl<B: Backend> ClassifierLayer<B> {
layer_loader: &ParameterTree<B::Context>,
rope: Rc<Rope<B>>,
) -> Self {
- autoreleasepool(|_| {
- let ctx = context.as_ref(); // Reference for functions expecting &B::Context
- let attention_config =
- layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
- let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
-
- let copy_main_to_shortcut_mixer = TensorCopy::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap();
-
- let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {
- if layer_loader.subtree("pre_mixer_norm").is_ok() {
- Some(
- Normalization::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &layer_loader.subtree("pre_mixer_norm").unwrap(),
- )
- .expect("Failed to create pre-attention norm kernel"),
- )
- } else {
- None
- }
- } else {
- None
- };
+ let ctx = context.as_ref(); // Reference for functions expecting &B::Context
+ let attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
+ let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
- let qkv_projection = <dyn Linear<B>>::new(
- &attention_config.qkv_projection_config,
- attention_config.has_qkv_biases,
- model_dim,
- [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
- ctx,
- &layer_loader.subtree("mixer.qkv_projection").unwrap(),
- ArrayId::Main,
- ArrayId::QKV,
- )
- .expect("Failed to create qkv projection");
-
- let qk_norm = if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some()
- {
- match QKNorm::new(
- ctx,
- intermediate_data_type,
- attention_config.query_norm_config.clone(),
- attention_config.key_norm_config.clone(),
- ArrayId::QKV,
- &layer_loader.subtree("mixer").unwrap(),
- num_heads,
- num_groups,
- head_dim,
- ) {
- Ok(norm) => Some(norm),
- Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
- }
- } else {
- None
- };
+ let copy_main_to_shortcut_mixer = TensorCopy::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap();
- let out_projection = <dyn Linear<B>>::new(
- &attention_config.out_projection_config,
- attention_config.has_out_biases,
- num_heads * head_dim,
- [model_dim],
- ctx,
- &layer_loader.subtree("mixer.out_projection").unwrap(),
- ArrayId::AttentionOutput,
- ArrayId::Main,
- )
- .expect("Failed to create out projection");
-
- let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
+ let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {
+ if layer_loader.subtree("pre_mixer_norm").is_ok() {
Some(
Normalization::new(
ctx,
@@ -131,99 +64,160 @@ impl<B: Backend> ClassifierLayer<B> {
norm_config.clone(),
ArrayId::Main,
ArrayId::Main,
- &layer_loader.subtree("post_mixer_norm").unwrap(),
+ &layer_loader.subtree("pre_mixer_norm").unwrap(),
)
- .expect("Failed to create post-attention norm kernel"),
+ .expect("Failed to create pre-attention norm kernel"),
)
} else {
None
- };
+ }
+ } else {
+ None
+ };
- let mixer_residual_add = TensorAddSwap::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
- )
- .unwrap();
+ let qkv_projection = <dyn Linear<B>>::new(
+ &attention_config.qkv_projection_config,
+ attention_config.has_qkv_biases,
+ model_dim,
+ [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
+ ctx,
+ &layer_loader.subtree("mixer.qkv_projection").unwrap(),
+ ArrayId::Main,
+ ArrayId::QKV,
+ )
+ .expect("Failed to create qkv projection");
- let copy_main_to_shortcut_mlp = TensorCopy::<B>::new(
+ let qk_norm = if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
+ match QKNorm::new(
ctx,
intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap();
+ attention_config.query_norm_config.clone(),
+ attention_config.key_norm_config.clone(),
+ ArrayId::QKV,
+ &layer_loader.subtree("mixer").unwrap(),
+ num_heads,
+ num_groups,
+ head_dim,
+ ) {
+ Ok(norm) => Some(norm),
+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
+ }
+ } else {
+ None
+ };
- let pre_mlp_norm = Normalization::new(
- ctx,
- intermediate_data_type,
- layer_config.pre_mlp_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &layer_loader.subtree("pre_mlp_norm").unwrap(),
- )
- .expect("Failed to create pre-MLP norm kernel");
+ let out_projection = <dyn Linear<B>>::new(
+ &attention_config.out_projection_config,
+ attention_config.has_out_biases,
+ num_heads * head_dim,
+ [model_dim],
+ ctx,
+ &layer_loader.subtree("mixer.out_projection").unwrap(),
+ ArrayId::AttentionOutput,
+ ArrayId::Main,
+ )
+ .expect("Failed to create out projection");
- let mlp = <dyn Mlp<B>>::new(
- &layer_config.mlp_config,
- model_dim,
- hidden_dim,
- context.as_ref(),
- &layer_loader.subtree("mlp").unwrap(),
+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
+ Some(
+ Normalization::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("post_mixer_norm").unwrap(),
+ )
+ .expect("Failed to create post-attention norm kernel"),
)
- .expect("Failed to create mlp block");
+ } else {
+ None
+ };
- let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
- Some(
- Normalization::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &layer_loader.subtree("post_mlp_norm").unwrap(),
- )
- .expect("Failed to create post-MLP norm kernel"),
- )
- } else {
- None
- };
+ let mixer_residual_add = TensorAddSwap::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap();
- let attention = Attention::new(
- ctx,
- intermediate_data_type,
- layer_index,
- attention_scale,
- attention_config.has_sinks,
- false,
- attention_config.sliding_window_size,
- )
- .expect("Failed to create attention kernel");
+ let copy_main_to_shortcut_mlp = TensorCopy::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap();
- let mlp_residual_add = TensorAddSwap::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ let pre_mlp_norm = Normalization::new(
+ ctx,
+ intermediate_data_type,
+ layer_config.pre_mlp_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("pre_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create pre-MLP norm kernel");
+
+ let mlp = <dyn Mlp<B>>::new(
+ &layer_config.mlp_config,
+ model_dim,
+ hidden_dim,
+ context.as_ref(),
+ &layer_loader.subtree("mlp").unwrap(),
+ )
+ .expect("Failed to create mlp block");
+
+ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
+ Some(
+ Normalization::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("post_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create post-MLP norm kernel"),
)
- .unwrap();
+ } else {
+ None
+ };
- Self {
- layer_index,
- copy_main_to_shortcut_mixer,
- pre_attention_norm,
- qkv_projection,
- qk_norm,
- rope,
- attention,
- out_projection,
- post_attention_norm,
- mixer_residual_add,
- copy_main_to_shortcut_mlp,
- pre_mlp_norm,
- mlp,
- post_mlp_norm,
- mlp_residual_add,
- }
- })
+ let attention = Attention::new(
+ ctx,
+ intermediate_data_type,
+ layer_index,
+ attention_scale,
+ attention_config.has_sinks,
+ false,
+ attention_config.sliding_window_size,
+ )
+ .expect("Failed to create attention kernel");
+
+ let mlp_residual_add = TensorAddSwap::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap();
+
+ Self {
+ layer_index,
+ copy_main_to_shortcut_mixer,
+ pre_attention_norm,
+ qkv_projection,
+ qk_norm,
+ rope,
+ attention,
+ out_projection,
+ post_attention_norm,
+ mixer_residual_add,
+ copy_main_to_shortcut_mlp,
+ pre_mlp_norm,
+ mlp,
+ post_mlp_norm,
+ mlp_residual_add,
+ }
}
pub fn encode(
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index 0c4b4aed5..1d2671500 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -2,8 +2,6 @@
use std::rc::Rc;
-use objc2::rc::autoreleasepool;
-
use super::MixerExecutables;
use crate::{
DataType, DecoderLayerConfig,
@@ -46,198 +44,196 @@ impl<B: Backend> LayerExecutables<B> {
decoder_layer_loader: &ParameterTree<B::Context>,
rope: Option<Rc<Rope<B>>>,
) -> Self {
- autoreleasepool(|_| {
- let ctx = context.as_ref(); // Reference for functions expecting &B::Context
- let intermediate_data_type: DataType = match &layer_config.mixer_config {
- MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),
- MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),
- MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
- };
- let copy_main_to_shortcut = TensorCopy::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap();
+ let ctx = context.as_ref(); // Reference for functions expecting &B::Context
+ let intermediate_data_type: DataType = match &layer_config.mixer_config {
+ MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),
+ MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),
+ MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
+ };
+ let copy_main_to_shortcut = TensorCopy::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap();
- let pre_attention_norm = RMSNorm::new(
- ctx,
- intermediate_data_type,
- layer_config.pre_attention_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel");
+ let pre_attention_norm = RMSNorm::new(
+ ctx,
+ intermediate_data_type,
+ layer_config.pre_attention_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel");
- let mixer = match &layer_config.mixer_config {
- MixerConfig::Attention(attention_config) => {
- let rope_block = rope.expect("RoPE encoder missing for attention layer");
+ let mixer = match &layer_config.mixer_config {
+ MixerConfig::Attention(attention_config) => {
+ let rope_block = rope.expect("RoPE encoder missing for attention layer");
- let qkv_projection = <dyn Linear<B>>::new(
- &attention_config.qkv_projection_config,
- attention_config.has_qkv_biases,
- model_dim,
- [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
- ctx,
- &decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),
- ArrayId::Main,
- ArrayId::QKV,
- )
- .expect("Failed to create qkv projection");
+ let qkv_projection = <dyn Linear<B>>::new(
+ &attention_config.qkv_projection_config,
+ attention_config.has_qkv_biases,
+ model_dim,
+ [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
+ ctx,
+ &decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),
+ ArrayId::Main,
+ ArrayId::QKV,
+ )
+ .expect("Failed to create qkv projection");
- let qk_norm =
- if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
- match QKNorm::new(
- ctx,
- intermediate_data_type,
- attention_config.query_norm_config.clone(),
- attention_config.key_norm_config.clone(),
- ArrayId::QKV,
- &decoder_layer_loader.subtree("mixer").unwrap(),
- num_heads,
- num_groups,
- head_dim,
- ) {
- Ok(qk_norm) => Some(qk_norm),
- Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
- }
- } else {
- None
- };
+ let qk_norm =
+ if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
+ match QKNorm::new(
+ ctx,
+ intermediate_data_type,
+ attention_config.query_norm_config.clone(),
+ attention_config.key_norm_config.clone(),
+ ArrayId::QKV,
+ &decoder_layer_loader.subtree("mixer").unwrap(),
+ num_heads,
+ num_groups,
+ head_dim,
+ ) {
+ Ok(qk_norm) => Some(qk_norm),
+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
+ }
+ } else {
+ None
+ };
- let out_projection = <dyn Linear<B>>::new(
- &attention_config.out_projection_config,
- attention_config.has_out_biases,
- num_heads * head_dim,
- [model_dim],
- ctx,
- &decoder_layer_loader.subtree("mixer.out_projection").unwrap(),
- ArrayId::AttentionOutput,
- ArrayId::Main,
- )
- .expect("Failed to create out projection");
+ let out_projection = <dyn Linear<B>>::new(
+ &attention_config.out_projection_config,
+ attention_config.has_out_biases,
+ num_heads * head_dim,
+ [model_dim],
+ ctx,
+ &decoder_layer_loader.subtree("mixer.out_projection").unwrap(),
+ ArrayId::AttentionOutput,
+ ArrayId::Main,
+ )
+ .expect("Failed to create out projection");
- let attention = Attention::new(
- ctx,
- intermediate_data_type,
- layer_index,
- attention_scale,
- attention_config.has_sinks,
- attention_config.is_causal.unwrap_or(true),
- attention_config.sliding_window_size,
- )
- .expect("Failed to create AttentionWrapper kernel");
+ let attention = Attention::new(
+ ctx,
+ intermediate_data_type,
+ layer_index,
+ attention_scale,
+ attention_config.has_sinks,
+ attention_config.is_causal.unwrap_or(true),
+ attention_config.sliding_window_size,
+ )
+ .expect("Failed to create AttentionWrapper kernel");
- MixerExecutables::Attention {
- qkv_projection,
- qk_norm,
- rope: rope_block,
- attention,
- out_projection,
- }
- },
- MixerConfig::Mamba(mamba_config) => {
- let mixer = MambaMixer::new(
- ctx,
- layer_type.clone(),
- mamba_config.clone(),
- layer_index,
- model_dim,
- num_heads,
- head_dim,
- num_groups,
- decoder_layer_loader,
- );
- MixerExecutables::StateSpace {
- mixer,
- }
- },
- MixerConfig::ShortConv(short_conv_config) => {
- let mixer = ShortConvMixer::new(
- ctx,
- layer_type.clone(),
- short_conv_config.clone(),
- layer_index,
- model_dim,
- decoder_layer_loader,
- );
- MixerExecutables::ShortConv {
- mixer,
- }
- },
- };
+ MixerExecutables::Attention {
+ qkv_projection,
+ qk_norm,
+ rope: rope_block,
+ attention,
+ out_projection,
+ }
+ },
+ MixerConfig::Mamba(mamba_config) => {
+ let mixer = MambaMixer::new(
+ ctx,
+ layer_type.clone(),
+ mamba_config.clone(),
+ layer_index,
+ model_dim,
+ num_heads,
+ head_dim,
+ num_groups,
+ decoder_layer_loader,
+ );
+ MixerExecutables::StateSpace {
+ mixer,
+ }
+ },
+ MixerConfig::ShortConv(short_conv_config) => {
+ let mixer = ShortConvMixer::new(
+ ctx,
+ layer_type.clone(),
+ short_conv_config.clone(),
+ layer_index,
+ model_dim,
+ decoder_layer_loader,
+ );
+ MixerExecutables::ShortConv {
+ mixer,
+ }
+ },
+ };
- let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
- Some(
- RMSNorm::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
+ Some(
+ RMSNorm::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
)
- } else {
- None
- };
-
- let main_shortcut_add_swap = TensorAddSwap::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ .expect("Failed to create RMS norm kernel"),
)
- .unwrap();
+ } else {
+ None
+ };
- let pre_mlp_norm = RMSNorm::new(
- ctx,
- intermediate_data_type,
- layer_config.pre_mlp_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel");
+ let main_shortcut_add_swap = TensorAddSwap::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap();
- let mlp = <dyn Mlp<B>>::new(
- &layer_config.mlp_config,
- model_dim,
- hidden_dim,
- context.as_ref(),
- &decoder_layer_loader.subtree("mlp").unwrap(),
- )
- .expect("Failed to create mlp block");
+ let pre_mlp_norm = RMSNorm::new(
+ ctx,
+ intermediate_data_type,
+ layer_config.pre_mlp_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel");
- let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
- Some(
- RMSNorm::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
+ let mlp = <dyn Mlp<B>>::new(
+ &layer_config.mlp_config,
+ model_dim,
+ hidden_dim,
+ context.as_ref(),
+ &decoder_layer_loader.subtree("mlp").unwrap(),
+ )
+ .expect("Failed to create mlp block");
+
+ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
+ Some(
+ RMSNorm::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
)
- } else {
- None
- };
+ .expect("Failed to create RMS norm kernel"),
+ )
+ } else {
+ None
+ };
- Self {
- layer_index,
- copy_main_to_shortcut,
- pre_attention_norm,
- mixer,
- post_attention_norm,
- main_shortcut_add_swap,
- pre_mlp_norm,
- mlp,
- post_mlp_norm,
- }
- })
+ Self {
+ layer_index,
+ copy_main_to_shortcut,
+ pre_attention_norm,
+ mixer,
+ post_attention_norm,
+ main_shortcut_add_swap,
+ pre_mlp_norm,
+ mlp,
+ post_mlp_norm,
+ }
}
pub fn encode(
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index f2655a3e0..ac08f8599 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -237,9 +237,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let _ = self.gpu_capture.start_capture(&self.context.context, "prefill");
}
- objc2::rc::autoreleasepool(|_pool| {
- let _ = last_state.take();
- });
+ let _ = last_state.take();
let task = Task {
token_ids: &step_token_ids,
@@ -612,9 +610,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
}
fn clear_cache(&mut self) {
- objc2::rc::autoreleasepool(|_pool| {
- self.pre_encoded_task = None;
- });
+ self.pre_encoded_task = None;
}
fn reset_state(&mut self) {
@@ -742,88 +738,86 @@ impl<B: Backend> LanguageModelGenerator<B> {
sampling_method: SamplingMethod,
should_fill_attention_bias: bool,
) -> Result<(ForwardPassState<B>, f64), Error> {
- objc2::rc::autoreleasepool(|_pool| {
- let run_start = Instant::now();
-
- let mut state = ForwardPassState::new_llm(
- self.context.context.clone(),
- &self.context.decoder_config,
- &self.context.model_shape,
- &self.context.scratch_buffers,
- self.context.cache_layers.clone(),
- self.context.shared_buffers.clone(),
- task.token_ids,
- task.token_positions,
- task.token_bitmask,
- task.token_seeds,
- task.active_suffix_length,
- task.sampling_start,
- task.sampling_length,
- task.is_prefilling,
- None,
- false,
- should_fill_attention_bias,
- None,
- None,
- );
+ let run_start = Instant::now();
- if let Some(method) = state.sampling_method_mut() {
- *method = Some(sampling_method);
- }
+ let mut state = ForwardPassState::new_llm(
+ self.context.context.clone(),
+ &self.context.decoder_config,
+ &self.context.model_shape,
+ &self.context.scratch_buffers,
+ self.context.cache_layers.clone(),
+ self.context.shared_buffers.clone(),
+ task.token_ids,
+ task.token_positions,
+ task.token_bitmask,
+ task.token_seeds,
+ task.active_suffix_length,
+ task.sampling_start,
+ task.sampling_length,
+ task.is_prefilling,
+ None,
+ false,
+ should_fill_attention_bias,
+ None,
+ None,
+ );
- let encoding_key = TaskEncodingKey {
- context_len: self.tokens.len(),
- batch_size: task.token_ids.len(),
- expected_number_of_new_tokens: task.expected_number_of_new_tokens,
- active_suffix_len: task.active_suffix_length,
- sampling_method,
- sampling_start: task.sampling_start,
- sampling_len: task.sampling_length,
- has_bitmask: task.token_bitmask.is_some(),
- is_prefilling: task.is_prefilling,
- };
+ if let Some(method) = state.sampling_method_mut() {
+ *method = Some(sampling_method);
+ }
- let is_first_decode = task.token_ids.len() == 1;
- let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
+ let encoding_key = TaskEncodingKey {
+ context_len: self.tokens.len(),
+ batch_size: task.token_ids.len(),
+ expected_number_of_new_tokens: task.expected_number_of_new_tokens,
+ active_suffix_len: task.active_suffix_length,
+ sampling_method,
+ sampling_start: task.sampling_start,
+ sampling_len: task.sampling_length,
+ has_bitmask: task.token_bitmask.is_some(),
+ is_prefilling: task.is_prefilling,
+ };
- if should_capture {
- self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
- self.pre_encoded_task = None;
- }
+ let is_first_decode = task.token_ids.len() == 1;
+ let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
- let sample = !task.is_prefilling;
+ if should_capture {
+ self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
+ self.pre_encoded_task = None;
+ }
- let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()
- && pre_encoded_key == encoding_key
- {
- pre_encoded_executable
- } else {
- self.encode_forward_pass(&mut state, &EncodingParameters::new(), sample)?
- };
+ let sample = !task.is_prefilling;
- let pending = executable.submit();
+ let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()
+ && pre_encoded_key == encoding_key
+ {
+ pre_encoded_executable
+ } else {
+ self.encode_forward_pass(&mut state, &EncodingParameters::new(), sample)?
+ };
- if allow_pre_encode {
- let mut next_encoding_key = encoding_key;
+ let pending = executable.submit();
- next_encoding_key.context_len += 1;
+ if allow_pre_encode {
+ let mut next_encoding_key = encoding_key;
- let next_executable =
- self.encode_forward_pass(&mut state, &EncodingParameters::new().with_projection(1), sample)?;
+ next_encoding_key.context_len += 1;
- self.pre_encoded_task = Some((next_encoding_key, next_executable));
- }
+ let next_executable =
+ self.encode_forward_pass(&mut state, &EncodingParameters::new().with_projection(1), sample)?;
- pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ self.pre_encoded_task = Some((next_encoding_key, next_executable));
+ }
- let run_time = run_start.elapsed().as_secs_f64();
+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
- if should_capture {
- self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
- }
+ let run_time = run_start.elapsed().as_secs_f64();
- Ok((state, run_time))
- })
+ if should_capture {
+ self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
+ }
+
+ Ok((state, run_time))
}
fn encode_forward_pass(
diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rs
index a6b2cc3ab..bf17cd263 100644
--- a/crates/uzu/src/prelude.rs
+++ b/crates/uzu/src/prelude.rs
@@ -21,5 +21,4 @@ pub use crate::{
},
},
speculators::{empty_speculator::EmptySpeculator, speculator::Speculator},
- storage_path,
};
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 0e4eef3f5..d8031c18a 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -11,11 +11,6 @@ use std::{
time::Instant,
};
-fn nanos_to_secs(nanos: u64) -> f64 {
- nanos as f64 / 1_000_000_000.0
-}
-
-use objc2::rc::autoreleasepool;
use tokenizers::Tokenizer;
use xgrammar::TokenizerInfo;
@@ -35,6 +30,10 @@ use crate::{
},
};
+fn nanos_to_secs(nanos: u64) -> f64 {
+ nanos as f64 / 1_000_000_000.0
+}
+
struct RunContext {
eos_tokens: Vec<u64>,
context_length: usize,
@@ -696,8 +695,6 @@ impl ChatSession {
impl Drop for ChatSession {
fn drop(&mut self) {
- autoreleasepool(|_| {
- self.llm = None;
- });
+ self.llm = None
}
}
diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rs
index ad393c6b5..4df84d5ba 100644
--- a/crates/uzu/src/utils/mod.rs
+++ b/crates/uzu/src/utils/mod.rs
@@ -2,10 +2,8 @@ pub mod attention;
pub mod env_utils;
pub mod model_size;
pub mod pointers;
-pub mod storage;
pub mod version;
pub use env_utils::*;
pub use model_size::ModelSize;
-pub use storage::*;
pub use version::VERSION;
diff --git a/crates/uzu/src/utils/storage.rs b/crates/uzu/src/utils/storage.rs
deleted file mode 100644
index cde74aeff..000000000
--- a/crates/uzu/src/utils/storage.rs
+++ /dev/null
@@ -1,85 +0,0 @@
-use std::path::PathBuf;
-
-use objc2::rc::{Retained, autoreleasepool};
-use objc2_foundation::{NSHomeDirectory, NSSearchPathDirectory, NSString};
-
-const STORAGE_DIR_NAME: &str = "com.mirai.sdk.storage";
-
-/// Returns the *user-domain* path for a given directory without any
-/// application-specific suffixes.
-pub fn user_domain_path(dir: NSSearchPathDirectory) -> PathBuf {
- #[cfg(any(target_os = "macos", target_os = "ios"))]
- {
- use objc2_foundation::{NSFileManager, NSSearchPathDomainMask};
-
- if let Some(path) = autoreleasepool(|_| {
- let fm = NSFileManager::defaultManager();
- let urls = fm.URLsForDirectory_inDomains(dir, NSSearchPathDomainMask::UserDomainMask);
-
- urls.firstObject().map(|url| {
- let ns_path: Retained<NSString> = url.path().expect("URL should have a path");
- PathBuf::from(ns_path.to_string())
- })
- }) {
- return path;
- }
- }
-
- let home = std::env::var("HOME").unwrap_or_else(|_| String::from("."));
- if dir == NSSearchPathDirectory::CachesDirectory {
- #[cfg(target_os = "macos")]
- {
- PathBuf::from(&home).join("Library").join("Caches")
- }
- #[cfg(not(target_os = "macos"))]
- {
- PathBuf::from(&home).join(".cache")
- }
- } else if dir == NSSearchPathDirectory::DownloadsDirectory {
- PathBuf::from(&home).join("Downloads")
- } else if dir == NSSearchPathDirectory::DocumentDirectory {
- PathBuf::from(&home).join("Documents")
- } else {
- PathBuf::from(home)
- }
-}
-
-/// Platform-specific directory the SDK uses for persistent storage.
-///
-/// * macOS: `~/Library/Caches/<STORAGE_DIR_NAME>/<VERSION>/`
-/// * iOS: `~/Documents/<STORAGE_DIR_NAME>/<VERSION>/`
-/// * other: `$HOME/.cache/<STORAGE_DIR_NAME>/<VERSION>/`
-pub fn storage_path() -> PathBuf {
- #[cfg(target_os = "macos")]
- let base = {
- // In a sandboxed macOS application, NSHomeDirectory() returns
- // `~/Library/Containers/<bundle-id>/Data`. Persist model files inside
- // the app-scoped caches directory to comply with sandbox rules:
- // `~/Library/Containers/<bundle-id>/Data/Library/Caches/…`.
- let home = NSHomeDirectory().to_string();
- PathBuf::from(&home).join("Library").join("Caches")
- };
-
- #[cfg(target_os = "ios")]
- let base = user_domain_path(NSSearchPathDirectory::DocumentDirectory);
-
- #[cfg(not(any(target_os = "macos", target_os = "ios")))]
- let base = {
- let home = std::env::var("HOME").unwrap_or_else(|_| String::from("."));
- PathBuf::from(home).join(".cache")
- };
-
- let full = base.join(STORAGE_DIR_NAME).join(crate::VERSION);
- let _ = std::fs::create_dir_all(&full);
- full
-}
-
-/// Deprecated: use `user_domain_path` or `storage_path` instead. Retained for
-/// compatibility with existing code paths.
-pub fn root_dir(location: NSSearchPathDirectory) -> PathBuf {
- if location == NSSearchPathDirectory::CachesDirectory {
- storage_path()
- } else {
- user_domain_path(location)
- }
-}
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index ad2d1907c..bc311814b 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
mod common;
diff --git a/crates/uzu/tests/audio_codec_test.rs b/crates/uzu/tests/audio_codec_test.rs
index 8e5348c2d..d545ef699 100644
--- a/crates/uzu/tests/audio_codec_test.rs
+++ b/crates/uzu/tests/audio_codec_test.rs
@@ -1,4 +1,4 @@
-#![cfg(all(feature = "audio-runtime", target_os = "macos"))]
+#![cfg(all(feature = "audio-runtime", metal_backend))]
use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use uzu::{
diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rs
index 688235123..90b95e1b7 100644
--- a/crates/uzu/tests/common/mod.rs
+++ b/crates/uzu/tests/common/mod.rs
@@ -89,7 +89,7 @@ macro_rules! for_each_backend {
type $B = uzu::backends::cpu::Cpu;
$body
}
- #[cfg(feature = "metal")]
+ #[cfg(metal_backend)]
{
type $B = uzu::backends::metal::Metal;
$body
@@ -100,7 +100,7 @@ macro_rules! for_each_backend {
#[macro_export]
macro_rules! for_each_non_cpu_backend {
(|$B:ident| $body:expr) => {{
- #[cfg(feature = "metal")]
+ #[cfg(metal_backend)]
{
type $B = uzu::backends::metal::Metal;
$body
diff --git a/crates/uzu/tests/common/proptest.rs b/crates/uzu/tests/common/proptest.rs
index 4236e303c..282048327 100644
--- a/crates/uzu/tests/common/proptest.rs
+++ b/crates/uzu/tests/common/proptest.rs
@@ -1,7 +1,7 @@
use std::rc::Rc;
use proptest::prelude::*;
-#[cfg(feature = "metal")]
+#[cfg(metal_backend)]
use uzu::backends::metal::Metal;
use uzu::{
DataType,
@@ -17,7 +17,7 @@ pub fn kernel_data_type() -> impl Strategy<Value = DataType> {
pub struct TestContextes {
pub cpu: Rc<<Cpu as Backend>::Context>,
- #[cfg(feature = "metal")]
+ #[cfg(metal_backend)]
pub metal: Rc<<Metal as Backend>::Context>,
}
@@ -25,7 +25,7 @@ impl TestContextes {
pub fn new() -> TestContextes {
TestContextes {
cpu: <Cpu as Backend>::Context::new().expect("Failed to create Cpu context"),
- #[cfg(feature = "metal")]
+ #[cfg(metal_backend)]
metal: <Metal as Backend>::Context::new().expect("Failed to create Metal context"),
}
}
@@ -33,7 +33,7 @@ impl TestContextes {
pub struct TestResults<T> {
pub cpu: T,
- #[cfg(feature = "metal")]
+ #[cfg(metal_backend)]
pub metal: T,
}
@@ -46,7 +46,7 @@ macro_rules! for_each_context {
let $CONTEXT_NAME = $CONTEXTES.cpu.as_ref();
$body
})?,
- #[cfg(feature = "metal")]
+ #[cfg(metal_backend)]
metal: ({
type $CONTEXT_TYPE = <uzu::backends::metal::Metal as uzu::backends::common::Backend>::Context;
let $CONTEXT_NAME = $CONTEXTES.metal.as_ref();
@@ -66,7 +66,7 @@ pub trait ComparableTestResults {
impl<T: ComparableTestResults> TestResults<T> {
pub fn compare_results(&self) -> Result<(), TestCaseError> {
- #[cfg(feature = "metal")]
+ #[cfg(metal_backend)]
T::compare("metal", &self.metal, &self.cpu)?;
Ok(())
diff --git a/crates/uzu/tests/context_mode_test.rs b/crates/uzu/tests/context_mode_test.rs
index a18ef59fa..0815def03 100644
--- a/crates/uzu/tests/context_mode_test.rs
+++ b/crates/uzu/tests/context_mode_test.rs
@@ -1,4 +1,7 @@
+#![cfg(metal_backend)]
+
mod common;
+
use std::path::PathBuf;
use test_tag::tag;
diff --git a/crates/uzu/tests/grammar_json_schema_test.rs b/crates/uzu/tests/grammar_json_schema_test.rs
index 45cd0f53e..fbab1248b 100644
--- a/crates/uzu/tests/grammar_json_schema_test.rs
+++ b/crates/uzu/tests/grammar_json_schema_test.rs
@@ -1,4 +1,4 @@
-#![cfg(target_os = "macos")]
+#![cfg(metal_backend)]
use std::sync::Arc;
diff --git a/crates/uzu/tests/kernel/rms_norm_test.rs b/crates/uzu/tests/kernel/rms_norm_test.rs
index be4345970..b74d69830 100644
--- a/crates/uzu/tests/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/kernel/rms_norm_test.rs
@@ -7,8 +7,6 @@ use std::{
use half::{bf16, f16};
use num_traits::Float;
use rand::{RngExt, SeedableRng, prelude::StdRng};
-#[cfg(feature = "metal")]
-use uzu::backends::metal::Metal;
use uzu::{
ArrayElement, DataType,
array::ArrayContextExt,
@@ -440,16 +438,16 @@ fn test_edge_f16_f16_f16_f16() {
}
// performance tests
+#[cfg(metal_backend)]
#[test]
fn test_perf_8k() {
- if cfg!(feature = "metal") {
- test_performance::<Metal>(8, 8192); // Large model like LLaMA-70B
- }
+ use uzu::backends::metal::Metal;
+ test_performance::<Metal>(8, 8192); // Large model like LLaMA-70B
}
+#[cfg(metal_backend)]
#[test]
fn test_perf_16k() {
- if cfg!(feature = "metal") {
- test_performance::<Metal>(16, 16384); // Huge model dimension
- }
+ use uzu::backends::metal::Metal;
+ test_performance::<Metal>(16, 16384); // Huge model dimension
}
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index e73dc063b..f176b52f7 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use uzu::{
DataType,
diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rs
index da5c15980..8f704c09d 100644
--- a/crates/uzu/tests/kv_cache_update_test.rs
+++ b/crates/uzu/tests/kv_cache_update_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use std::{cell::RefCell, rc::Rc};
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 1608ba0ed..c868918d6 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -1,5 +1,7 @@
//! Correctness tests comparing Metal matmul kernels against ndarray
+#![cfg(metal_backend)]
+
use bytemuck;
use half::bf16;
use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
diff --git a/crates/uzu/tests/matmul_perf_test/main.rs b/crates/uzu/tests/matmul_perf_test/main.rs
index 9bfd1fffb..fd7eee941 100644
--- a/crates/uzu/tests/matmul_perf_test/main.rs
+++ b/crates/uzu/tests/matmul_perf_test/main.rs
@@ -1,4 +1,4 @@
-#![cfg(all(target_os = "macos", feature = "metal"))]
+#![cfg(all(target_os = "macos", metal_backend))]
mod bench;
mod error;
diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
index e79359560..ce4c5badd 100644
--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs
+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use half::bf16;
use metal::MTLBuffer;
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index a30ea799f..8b038f7f0 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -1,3 +1,5 @@
+#![cfg(metal_backend)]
+
use half::bf16;
use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index 8fd419e07..b7c5377da 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -1,3 +1,5 @@
+#![cfg(metal_backend)]
+
use std::time::Instant;
use half::bf16;
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index 0afb09350..701784cb9 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -6,7 +6,7 @@
//! - Intermediate buffer correctness (row maps, tiles, dispatch args)
//! - Numerical correctness against CPU reference
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use half::bf16;
use metal::MTLBuffer;
diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rs
index 3677d3a5c..583b5e376 100644
--- a/crates/uzu/tests/moe/moe_finalize_test.rs
+++ b/crates/uzu/tests/moe/moe_finalize_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use half::bf16;
use metal::MTLBuffer;
diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rs
index f9cf5a01a..b38c936ef 100644
--- a/crates/uzu/tests/moe/moe_gather_test.rs
+++ b/crates/uzu/tests/moe/moe_gather_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use half::bf16;
use metal::MTLBuffer;
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index 1e734200c..e87707aba 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -1,3 +1,5 @@
+#![cfg(metal_backend)]
+
use std::time::Instant;
use half::bf16;
diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rs
index cb869b65b..75100831f 100644
--- a/crates/uzu/tests/moe/moe_router_topk_test.rs
+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use half::bf16;
use metal::MTLBuffer;
diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rs
index a00356825..e532d764a 100644
--- a/crates/uzu/tests/moe/moe_scatter_test.rs
+++ b/crates/uzu/tests/moe/moe_scatter_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use half::bf16;
use metal::MTLBuffer;
diff --git a/crates/uzu/tests/moe/moe_test_utils.rs b/crates/uzu/tests/moe/moe_test_utils.rs
index eb6217eb5..51881427f 100644
--- a/crates/uzu/tests/moe/moe_test_utils.rs
+++ b/crates/uzu/tests/moe/moe_test_utils.rs
@@ -3,7 +3,7 @@
//! Contains common helper functions, CPU ground truth implementations,
//! buffer allocation helpers, and test fixtures used across all MoE test files.
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use std::rc::Rc;
diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rs
index 9ab1ffc3e..ab7eac9f1 100644
--- a/crates/uzu/tests/moe/moe_tiles_test.rs
+++ b/crates/uzu/tests/moe/moe_tiles_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos", target_os = "ios"))]
+#![cfg(metal_backend)]
use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rs
index accf5c52a..94e564709 100644
--- a/crates/uzu/tests/parameters_loader_test.rs
+++ b/crates/uzu/tests/parameters_loader_test.rs
@@ -1,3 +1,4 @@
+#![cfg(metal_backend)]
mod common;
// New integration test for ParameterLoader
diff --git a/crates/uzu/tests/person_schema_metal_bitmask_test.rs b/crates/uzu/tests/person_schema_metal_bitmask_test.rs
index 4e4e1fcc6..92ca87fbc 100644
--- a/crates/uzu/tests/person_schema_metal_bitmask_test.rs
+++ b/crates/uzu/tests/person_schema_metal_bitmask_test.rs
@@ -1,4 +1,4 @@
-#![cfg(target_os = "macos")]
+#![cfg(metal_backend)]
use std::{cell::RefCell, path::PathBuf, rc::Rc};
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
index acedd70ea..306d24b57 100644
--- a/crates/uzu/tests/quant_matmul_test.rs
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -1,3 +1,5 @@
+#![cfg(metal_backend)]
+
use std::time::Instant;
use bytemuck;
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index 9dfcd9b59..85ba5a08a 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -1,4 +1,7 @@
+#![cfg(metal_backend)]
+
mod common;
+
use bytemuck;
use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use rand::seq::SliceRandom;
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index 70ca79769..c1c0ac69e 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -1,4 +1,4 @@
-#![cfg(target_os = "macos")]
+#![cfg(metal_backend)]
use std::mem::size_of;
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index 9567e0434..b057715b8 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -1,4 +1,4 @@
-#![cfg(any(target_os = "macos"))]
+#![cfg(metal_backend)]
use bytemuck;
use half::bf16;
diff --git a/crates/uzu/tests/text_session_test.rs b/crates/uzu/tests/text_session_test.rs
index 2fe1b0e17..ef46240f9 100644
--- a/crates/uzu/tests/text_session_test.rs
+++ b/crates/uzu/tests/text_session_test.rs
@@ -1,4 +1,7 @@
+#![cfg(metal_backend)]
+
mod common;
+
use std::path::PathBuf;
use test_tag::tag;
diff --git a/crates/uzu/tests/tracer_test.rs b/crates/uzu/tests/tracer_test.rs
index d78365e79..ec6999ce4 100644
--- a/crates/uzu/tests/tracer_test.rs
+++ b/crates/uzu/tests/tracer_test.rs
@@ -1,4 +1,4 @@
-#![cfg(feature = "tracing")]
+#![cfg(all(metal_backend, feature = "tracing"))]
mod common;
use std::path::PathBuf;
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 6ed70738f..f1b1d8f97 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -2,4 +2,4 @@
channel = "stable"
profile = "minimal"
components = ["rustfmt", "clippy"]
-targets = ["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin"]
+targets = ["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "aarch64-linux-android", "x86_64-unknown-linux-gnu"]
diff --git a/scripts/download_test_model.sh b/scripts/download_test_model.sh
index cab4a5875..6bfa4d337 100755
--- a/scripts/download_test_model.sh
+++ b/scripts/download_test_model.sh
@@ -2,11 +2,6 @@
set -euo pipefail
-if [[ "$(uname)" != "Darwin" ]]; then
- echo "Non-macOS platform detected (\"$(uname)\"); skipping test-model download."
- exit 0
-fi
-
ROOT_PATH="$(cd "$(dirname "$0")/.." && pwd)"
HELPERS_PATH="$ROOT_PATH/tools/helpers"