A high-performance inference engine for AI models
Infrastructure for kernel benchmarks (#324)
uuuvn23f80fa779ec
diff --git a/Cargo.lock b/Cargo.lockindex 6b09e1746..9bcac21d7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -32,6 +32,21 @@ dependencies = ["memchr",]+[[package]]+name = "alloca"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"+dependencies = [+ "cc",+]++[[package]]+name = "anes"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"+[[package]]name = "anstream"version = "1.0.0"@@ -445,6 +460,12 @@ version = "2.3.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3"+[[package]]+name = "cast"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"+[[package]]name = "castaway"version = "0.2.4"@@ -515,6 +536,33 @@ dependencies = ["rand_core 0.10.0",]+[[package]]+name = "ciborium"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"+dependencies = [+ "ciborium-io",+ "ciborium-ll",+ "serde",+]++[[package]]+name = "ciborium-io"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"++[[package]]+name = "ciborium-ll"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"+dependencies = [+ "ciborium-io",+ "half",+]+[[package]]name = "clang-ast"version = "0.1.35"@@ -776,6 +824,51 @@ dependencies = ["cfg-if",]+[[package]]+name = "criterion"+version = "0.8.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"+dependencies = [+ "alloca",+ "anes",+ "cast",+ "ciborium",+ "clap",+ "criterion-plot",+ "itertools 0.13.0",+ "num-traits",+ "oorandom",+ "page_size",+ "plotters",+ "rayon",+ "regex",+ "serde",+ "serde_json",+ "tinytemplate",+ "walkdir",+]++[[package]]+name = "criterion-macro"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "288a8f36b28a19d7dbd572c76006c0a0eba1f3bf912a254dda211c6f665e7ffc"+dependencies = [+ "proc-macro2",+ "quote",+]++[[package]]+name = "criterion-plot"+version = "0.8.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"+dependencies = [+ "cast",+ "itertools 0.13.0",+]+[[package]]name = "crossbeam-deque"version = "0.8.6"@@ -2584,6 +2677,12 @@ dependencies = ["pkg-config",]+[[package]]+name = "oorandom"+version = "11.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"+[[package]]name = "openssl"version = "0.10.76"@@ -2634,6 +2733,16 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"+[[package]]+name = "page_size"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"+dependencies = [+ "libc",+ "winapi",+]+[[package]]name = "parking_lot"version = "0.12.5"@@ -2762,6 +2871,34 @@ version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"+[[package]]+name = "plotters"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"+dependencies = [+ "num-traits",+ "plotters-backend",+ "plotters-svg",+ "wasm-bindgen",+ "web-sys",+]++[[package]]+name = "plotters-backend"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"++[[package]]+name = "plotters-svg"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"+dependencies = [+ "plotters-backend",+]+[[package]]name = "portable-atomic"version = "1.13.1"@@ -3981,6 +4118,16 @@ dependencies = ["zerovec",]+[[package]]+name = "tinytemplate"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"+dependencies = [+ "serde",+ "serde_json",+]+[[package]]name = "tokenizers"version = "0.22.2"@@ -4463,6 +4610,8 @@ dependencies = ["cbindgen","clang-ast","comfy-table",+ "criterion",+ "criterion-macro","depfile","dsl","futures",diff --git a/Cargo.toml b/Cargo.tomlindex 4894e1bce..470149702 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -74,6 +74,8 @@ cmake = "0.1"autocxx-build = "0.30"test-tag = "0.1"proptest = "1.10"+criterion = "0.8.2"+criterion-macro = "0.4.0"# optimize the build script in debug builds[profile.dev.build-override]diff --git a/crates/dsl/src/lib.rs b/crates/dsl/src/lib.rsindex 1d84d30b8..9c6a90b38 100644--- a/crates/dsl/src/lib.rs+++ b/crates/dsl/src/lib.rs@@ -28,3 +28,45 @@ pub fn kernel(}.into()}++#[proc_macro_attribute]+pub fn __internal_uzu_test(+ _args: TokenStream,+ input: TokenStream,+) -> TokenStream {+ let func = parse_macro_input!(input as ItemFn);++ quote! {+ #[test]+ #func+ }+ .into()+}++#[proc_macro_attribute]+pub fn __internal_uzu_bench(+ _args: TokenStream,+ input: TokenStream,+) -> TokenStream {+ let func = parse_macro_input!(input as ItemFn);++ quote! {+ #[::criterion_macro::criterion]+ #func+ }+ .into()+}++#[proc_macro_attribute]+pub fn __internal_uzu_ignored(+ _args: TokenStream,+ input: TokenStream,+) -> TokenStream {+ let func = parse_macro_input!(input as ItemFn);++ quote! {+ #[allow(unused)]+ #func+ }+ .into()+}diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex cfe806582..7ed6c78f8 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -42,6 +42,8 @@ metal = { workspace = true, optional = true }[dev-dependencies]test-tag.workspace = trueproptest.workspace = true+criterion.workspace = true+criterion-macro.workspace = truerand_distr.workspace = trueis_close.workspace = trueserde_json5.workspace = true@@ -93,7 +95,11 @@ path = "tests/integration/mod.rs"[[test]]name = "kernel"-path = "tests/unit/kernel/mod.rs"+path = "tests/unit/kernel/test_entry.rs"++[[bench]]+name = "kernel"+path = "tests/unit/kernel/bench_entry.rs"[[test]]name = "performance"diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex 7cb07bca9..1f00f9c5b 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -1,4 +1,4 @@-#![allow(dead_code)]+#![allow(dead_code, unused_imports, unused_macros)]pub mod assert;pub mod audio;@@ -7,8 +7,13 @@ pub mod path;pub mod perf;pub mod proptest;+pub(crate) use proptest::{dispatch_dtype, for_each_context};++pub fn type_short_name<T>() -> &'static str {+ std::any::type_name::<T>().rsplit("::").next().unwrap()+}+/// Invokes `$body` once per available backend, with `$B` bound to each backend type.-#[macro_export]macro_rules! for_each_backend {(|$B:ident| $body:expr) => {{{@@ -22,8 +27,8 @@ macro_rules! for_each_backend {}}};}+pub(crate) use for_each_backend;-#[macro_export]macro_rules! for_each_non_cpu_backend {(|$B:ident| $body:expr) => {{#[cfg(metal_backend)]@@ -39,3 +44,4 @@ macro_rules! for_each_non_cpu_backend {}}};}+pub(crate) use for_each_non_cpu_backend;diff --git a/crates/uzu/tests/common/proptest.rs b/crates/uzu/tests/common/proptest.rsindex 282048327..5cc5804bc 100644--- a/crates/uzu/tests/common/proptest.rs+++ b/crates/uzu/tests/common/proptest.rs@@ -37,7 +37,6 @@ pub struct TestResults<T> {pub metal: T,}-#[macro_export]macro_rules! for_each_context {($CONTEXTES:ident, |$CONTEXT_NAME:ident: $CONTEXT_TYPE:ident| $body:expr) => {crate::common::proptest::TestResults {@@ -55,6 +54,7 @@ macro_rules! for_each_context {}};}+pub(crate) use for_each_context;pub trait ComparableTestResults {fn compare(@@ -73,7 +73,6 @@ impl<T: ComparableTestResults> TestResults<T> {}}-#[macro_export]macro_rules! dispatch_dtype {(|()| $body:expr) => { $body };(|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {@@ -85,3 +84,4 @@ macro_rules! dispatch_dtype {}};}+pub(crate) use dispatch_dtype;diff --git a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rsindex 2904de720..5c2bd7799 100644--- a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs@@ -15,9 +15,12 @@ use crate::{ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},- for_each_non_cpu_backend,};+#[macro_use]+#[path = "../../../../common/mod.rs"]+mod common;+fn write_buffer<B: Backend>(buf: &B::Buffer,data: &[f32],diff --git a/crates/uzu/tests/unit/kernel/activation_mul_test.rs b/crates/uzu/tests/unit/kernel/activation_mul_test.rsindex f55d292e5..e00b64974 100644--- a/crates/uzu/tests/unit/kernel/activation_mul_test.rs+++ b/crates/uzu/tests/unit/kernel/activation_mul_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {fused_up: Box<[T]>,@@ -84,32 +84,32 @@ fn test<T: ArrayElement + Float + Debug + Display>(act_type: ActivationType) {});}-#[test]+#[uzu_test]fn test_silu_f32() {test::<f32>(ActivationType::SILU);}-#[test]+#[uzu_test]fn test_silu_f16() {test::<f16>(ActivationType::SILU);}-#[test]+#[uzu_test]fn test_silu_bf16() {test::<bf16>(ActivationType::SILU);}-#[test]+#[uzu_test]fn test_gelu_f32() {test::<f32>(ActivationType::GELU);}-#[test]+#[uzu_test]fn test_gelu_f16() {test::<f16>(ActivationType::GELU);}-#[test]+#[uzu_test]fn test_gelu_bf16() {test::<bf16>(ActivationType::GELU);}diff --git a/crates/uzu/tests/unit/kernel/activation_test.rs b/crates/uzu/tests/unit/kernel/activation_test.rsindex a06798046..317a7708b 100644--- a/crates/uzu/tests/unit/kernel/activation_test.rs+++ b/crates/uzu/tests/unit/kernel/activation_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {data: Box<[T]>,@@ -153,108 +153,108 @@ fn test_activation_large<T: ArrayElement + Float + Debug + Display>(}// SILU out-of-place tests-#[test]+#[uzu_test]fn test_silu_f32() {test_activation::<f32>(ActivationType::SILU, false);}-#[test]+#[uzu_test]fn test_silu_f16() {test_activation::<f16>(ActivationType::SILU, false);}-#[test]+#[uzu_test]fn test_silu_bf16() {test_activation::<bf16>(ActivationType::SILU, false);}// SILU in-place tests-#[test]+#[uzu_test]fn test_silu_in_place_f32() {test_activation::<f32>(ActivationType::SILU, true);}-#[test]+#[uzu_test]fn test_silu_in_place_f16() {test_activation::<f16>(ActivationType::SILU, true);}-#[test]+#[uzu_test]fn test_silu_in_place_bf16() {test_activation::<bf16>(ActivationType::SILU, true);}// GELU out-of-place tests-#[test]+#[uzu_test]fn test_gelu_f32() {test_activation::<f32>(ActivationType::GELU, false);}-#[test]+#[uzu_test]fn test_gelu_f16() {test_activation::<f16>(ActivationType::GELU, false);}-#[test]+#[uzu_test]fn test_gelu_bf16() {test_activation::<bf16>(ActivationType::GELU, false);}// GELU in-place tests-#[test]+#[uzu_test]fn test_gelu_in_place_f32() {test_activation::<f32>(ActivationType::GELU, true);}-#[test]+#[uzu_test]fn test_gelu_in_place_f16() {test_activation::<f16>(ActivationType::GELU, true);}-#[test]+#[uzu_test]fn test_gelu_in_place_bf16() {test_activation::<bf16>(ActivationType::GELU, true);}// Large SILU tests-#[test]+#[uzu_test]fn test_silu_large_f32() {test_activation_large::<f32>(ActivationType::SILU, false);}-#[test]+#[uzu_test]fn test_silu_large_f16() {test_activation_large::<f16>(ActivationType::SILU, false);}-#[test]+#[uzu_test]fn test_silu_large_bf16() {test_activation_large::<bf16>(ActivationType::SILU, false);}// Large GELU tests-#[test]+#[uzu_test]fn test_gelu_large_f32() {test_activation_large::<f32>(ActivationType::GELU, false);}-#[test]+#[uzu_test]fn test_gelu_large_f16() {test_activation_large::<f16>(ActivationType::GELU, false);}-#[test]+#[uzu_test]fn test_gelu_large_bf16() {test_activation_large::<bf16>(ActivationType::GELU, false);}// Large in-place tests-#[test]+#[uzu_test]fn test_silu_in_place_large_f32() {test_activation_large::<f32>(ActivationType::SILU, true);}-#[test]+#[uzu_test]fn test_gelu_in_place_large_f32() {test_activation_large::<f32>(ActivationType::GELU, true);}diff --git a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rsindex 946684835..16b3207c1 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {queries: Box<[T]>,@@ -200,81 +200,81 @@ fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {}// Basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// Causal tests-#[test]+#[uzu_test]fn test_causal_f32() {test_causal::<f32>();}-#[test]+#[uzu_test]fn test_causal_f16() {test_causal::<f16>();}-#[test]+#[uzu_test]fn test_causal_bf16() {test_causal::<bf16>();}// GQA tests-#[test]+#[uzu_test]fn test_gqa_f32() {test_gqa::<f32>();}-#[test]+#[uzu_test]fn test_gqa_f16() {test_gqa::<f16>();}-#[test]+#[uzu_test]fn test_gqa_bf16() {test_gqa::<bf16>();}// Head dim 128-#[test]+#[uzu_test]fn test_head_dim_128_f32() {test_head_dim::<f32>(128);}-#[test]+#[uzu_test]fn test_head_dim_128_f16() {test_head_dim::<f16>(128);}-#[test]+#[uzu_test]fn test_head_dim_128_bf16() {test_head_dim::<bf16>(128);}// Unaligned tests-#[test]+#[uzu_test]fn test_unaligned_f32() {test_unaligned::<f32>();}-#[test]+#[uzu_test]fn test_unaligned_f16() {test_unaligned::<f16>();}-#[test]+#[uzu_test]fn test_unaligned_bf16() {test_unaligned::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rsindex ffc99f37b..41a553e90 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {queries: Box<[T]>,@@ -191,65 +191,65 @@ fn test_head_dim_128<T: ArrayElement + Float + Debug + Display>() {}// Basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// Causal tests-#[test]+#[uzu_test]fn test_causal_f32() {test_causal::<f32>();}-#[test]+#[uzu_test]fn test_causal_f16() {test_causal::<f16>();}-#[test]+#[uzu_test]fn test_causal_bf16() {test_causal::<bf16>();}// GQA tests-#[test]+#[uzu_test]fn test_gqa_f32() {test_gqa::<f32>();}-#[test]+#[uzu_test]fn test_gqa_f16() {test_gqa::<f16>();}-#[test]+#[uzu_test]fn test_gqa_bf16() {test_gqa::<bf16>();}// Head dim 128-#[test]+#[uzu_test]fn test_head_dim_128_f32() {test_head_dim_128::<f32>();}-#[test]+#[uzu_test]fn test_head_dim_128_f16() {test_head_dim_128::<f16>();}-#[test]+#[uzu_test]fn test_head_dim_128_bf16() {test_head_dim_128::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rsindex b6cec168c..3f4ffa49a 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs@@ -16,7 +16,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};const TOTAL_BLOCKS_COUNT: u32 = 32;@@ -329,94 +329,94 @@ fn test_second_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {// --- First pass test entries ----#[test]+#[uzu_test]fn test_first_pass_basic_f32() {test_first_pass_basic::<f32>();}-#[test]+#[uzu_test]fn test_first_pass_basic_f16() {test_first_pass_basic::<f16>();}-#[test]+#[uzu_test]fn test_first_pass_basic_bf16() {test_first_pass_basic::<bf16>();}-#[test]+#[uzu_test]fn test_first_pass_causal_f32() {test_first_pass_causal::<f32>();}-#[test]+#[uzu_test]fn test_first_pass_causal_f16() {test_first_pass_causal::<f16>();}-#[test]+#[uzu_test]fn test_first_pass_causal_bf16() {test_first_pass_causal::<bf16>();}-#[test]+#[uzu_test]fn test_first_pass_gqa_f32() {test_first_pass_gqa::<f32>();}-#[test]+#[uzu_test]fn test_first_pass_gqa_f16() {test_first_pass_gqa::<f16>();}-#[test]+#[uzu_test]fn test_first_pass_gqa_bf16() {test_first_pass_gqa::<bf16>();}-#[test]+#[uzu_test]fn test_first_pass_head_dim_128_f32() {test_first_pass_head_dim_128::<f32>();}-#[test]+#[uzu_test]fn test_first_pass_head_dim_128_f16() {test_first_pass_head_dim_128::<f16>();}-#[test]+#[uzu_test]fn test_first_pass_head_dim_128_bf16() {test_first_pass_head_dim_128::<bf16>();}// --- Second pass test entries ----#[test]+#[uzu_test]fn test_second_pass_basic_f32() {test_second_pass_basic::<f32>();}-#[test]+#[uzu_test]fn test_second_pass_basic_f16() {test_second_pass_basic::<f16>();}-#[test]+#[uzu_test]fn test_second_pass_basic_bf16() {test_second_pass_basic::<bf16>();}-#[test]+#[uzu_test]fn test_second_pass_head_dim_128_f32() {test_second_pass_head_dim_128::<f32>();}-#[test]+#[uzu_test]fn test_second_pass_head_dim_128_f16() {test_second_pass_head_dim_128::<f16>();}-#[test]+#[uzu_test]fn test_second_pass_head_dim_128_bf16() {test_second_pass_head_dim_128::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rsindex e16d1d23d..2f5fff81d 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {rotated_keys: Option<Box<[T]>>,@@ -259,49 +259,49 @@ fn test_prefix_zero<T: ArrayElement + Float + Debug + Display>() {}// Basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// Single token tests (typical decode step)-#[test]+#[uzu_test]fn test_single_token_f32() {test_single_token::<f32>();}-#[test]+#[uzu_test]fn test_single_token_f16() {test_single_token::<f16>();}-#[test]+#[uzu_test]fn test_single_token_bf16() {test_single_token::<bf16>();}// Prefix zero (first fill of empty cache)-#[test]+#[uzu_test]fn test_prefix_zero_f32() {test_prefix_zero::<f32>();}-#[test]+#[uzu_test]fn test_prefix_zero_f16() {test_prefix_zero::<f16>();}-#[test]+#[uzu_test]fn test_prefix_zero_bf16() {test_prefix_zero::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rsindex 25f3f9d6d..e04741bd0 100644--- a/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs@@ -10,6 +10,8 @@ use uzu::{},};+use crate::uzu_test;+struct Config {num_heads: u32,head_dim: u32,@@ -81,7 +83,7 @@ fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {});}-#[test]+#[uzu_test]fn test_sigmoid_gate_f32() {run_test::<f32>(&Config {num_heads: 8,@@ -100,7 +102,7 @@ fn test_sigmoid_gate_f32() {});}-#[test]+#[uzu_test]fn test_sigmoid_gate_f16() {run_test::<f16>(&Config {num_heads: 8,@@ -114,7 +116,7 @@ fn test_sigmoid_gate_f16() {});}-#[test]+#[uzu_test]fn test_sigmoid_gate_bf16() {run_test::<bf16>(&Config {num_heads: 8,diff --git a/crates/uzu/tests/unit/kernel/audio/audio_test.rs b/crates/uzu/tests/unit/kernel/audio/audio_test.rsindex 514421e4c..4821f830b 100644--- a/crates/uzu/tests/unit/kernel/audio/audio_test.rs+++ b/crates/uzu/tests/unit/kernel/audio/audio_test.rs@@ -24,6 +24,7 @@ use super::super::common::audio::{conv1d_reference, half_snake_reference,},};+use crate::uzu_test;macro_rules! borrow_array_buffer {($name:ident = $array:expr) => {@@ -52,7 +53,7 @@ fn run_command_buffer(encoder.end_encoding().submit().wait_until_completed().expect("command buffer completed");}-#[test]+#[uzu_test]fn audio_conv1d_replicate_matches_reference_f32() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioConv1dKernel::new(&context, DataType::F32)@@ -149,7 +150,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_causal_conv1d_matches_reference_f32() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dKernel::new(&context, DataType::F32)@@ -235,7 +236,7 @@ fn audio_causal_conv1d_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_causal_conv_transpose1d_matches_reference_f32() {let context = create_test_context();let kernel =@@ -334,7 +335,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(@@ -453,7 +454,7 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_leaky_relu_matches_reference_f32() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, DataType::F32)@@ -500,7 +501,7 @@ fn audio_leaky_relu_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_tanh_matches_reference_f32() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, DataType::F32, false)@@ -528,7 +529,7 @@ fn audio_tanh_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_add_matches_reference_f32() {let context = create_test_context();let add_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioAddKernel::new(&context, DataType::F32)@@ -567,7 +568,7 @@ fn audio_add_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_half_snake_matches_reference_f32() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, DataType::F32)@@ -635,7 +636,7 @@ fn audio_half_snake_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_norm_ncs_matches_reference_f32() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioNormNcsKernel::new(&context, DataType::F32)@@ -744,7 +745,7 @@ fn audio_norm_ncs_matches_reference_f32() {}}-#[test]+#[uzu_test]fn audio_fsq_decode_matches_reference() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32)@@ -801,7 +802,7 @@ fn audio_fsq_decode_matches_reference() {}}-#[test]+#[uzu_test]fn audio_fsq_encode_matches_reference() {let context = create_test_context();let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32)diff --git a/crates/uzu/tests/unit/kernel/bench_entry.rs b/crates/uzu/tests/unit/kernel/bench_entry.rsnew file mode 100644index 000000000..c00d5a135--- /dev/null+++ b/crates/uzu/tests/unit/kernel/bench_entry.rs@@ -0,0 +1,6 @@+#![feature(custom_test_frameworks)]+#![test_runner(criterion::runner)]++use dsl::{__internal_uzu_bench as uzu_bench, __internal_uzu_ignored as uzu_test};++include!("mod.rs");diff --git a/crates/uzu/tests/unit/kernel/delta_net_test.rs b/crates/uzu/tests/unit/kernel/delta_net_test.rsindex 599e10d59..9dfbafec9 100644--- a/crates/uzu/tests/unit/kernel/delta_net_test.rs+++ b/crates/uzu/tests/unit/kernel/delta_net_test.rs@@ -17,6 +17,8 @@ use uzu::{},};+use crate::uzu_test;+fn run_conv_update<B: Backend>(in_proj: &[f32],w: &[f32],@@ -118,7 +120,7 @@ fn assert_close(// DeltaNetConvUpdate-#[test]+#[uzu_test]fn test_delta_net_conv_update_small() {let conv_dim = 32;let kernel_size = 4;@@ -140,7 +142,7 @@ fn test_delta_net_conv_update_small() {// DeltaNetConvScan-#[test]+#[uzu_test]fn test_delta_net_conv_scan() {let conv_dim = 32;let kernel_size: usize = 4;@@ -280,7 +282,7 @@ fn test_delta_net_update_impl(assert_close(&cpu_state, &gpu_state, 1e-4, 1e-3, &format!("{label} state"));}-#[test]+#[uzu_test]fn test_delta_net_update_qwen35_shapes() {test_delta_net_update_impl(48, 16, 128, 128, "DeltaNetUpdate Qwen3.5");}@@ -443,12 +445,12 @@ fn test_prefill_norm_gate_impl(assert_close(&ref_state, &gpu_state, 1e-3, 1e-2, &format!("{label} state"));}-#[test]+#[uzu_test]fn test_delta_net_prefill_qwen35_shapes() {test_prefill_norm_gate_impl(48, 16, 128, 128, 32, "Prefill+NormGate Qwen3.5");}-#[test]+#[uzu_test]fn test_delta_net_prefill_prep() {let num_v_heads = 48usize;let num_k_heads = 16usize;@@ -551,7 +553,7 @@ fn test_delta_net_prefill_prep() {assert_close(gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");}-#[test]+#[uzu_test]#[ignore]fn bench_delta_net_prefill() {use crate::common::perf::run_perf_with_warmup;diff --git a/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs b/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rsindex d3cb57570..9065a96d4 100644--- a/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs+++ b/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {token_indices: Box<[u32]>,@@ -181,49 +181,49 @@ fn test_oob<T: ArrayElement + Float + Debug + Display>() {}// basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// edge tests-#[test]+#[uzu_test]fn test_edge_f32() {test_edge::<f32>();}-#[test]+#[uzu_test]fn test_edge_f16() {test_edge::<f16>();}-#[test]+#[uzu_test]fn test_edge_bf16() {test_edge::<bf16>();}// out-of-bounds tests-#[test]+#[uzu_test]fn test_oob_f32() {test_oob::<f32>();}-#[test]+#[uzu_test]fn test_oob_f16() {test_oob::<f16>();}-#[test]+#[uzu_test]fn test_oob_bf16() {test_oob::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs b/crates/uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rsindex d5e279fa1..3b6fdc76d 100644--- a/crates/uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs+++ b/crates/uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs@@ -10,6 +10,8 @@ use uzu::{backends::common::{Backend, Context, Encoder, Kernels, kernel::FullPrecisionEmbeddingLookupKernel},};+use crate::uzu_test;+struct Input<T: ArrayElement + Float> {token_ids: Box<[u64]>,weights: Box<[T]>,@@ -83,17 +85,17 @@ fn test<T: ArrayElement + Float + Debug>() {});}-#[test]+#[uzu_test]fn test_f32() {test::<f32>()}-#[test]+#[uzu_test]fn test_f16() {test::<f16>()}-#[test]+#[uzu_test]fn test_bf16() {test::<bf16>()}diff --git a/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rsindex 40c667400..e77fa3c87 100644--- a/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs+++ b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs@@ -15,7 +15,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {token_ids: Box<[u64]>,@@ -182,65 +182,65 @@ fn test_oob<T: ArrayElement + Float + Debug + Display>() {}// UINT4 tests-#[test]+#[uzu_test]fn test_uint4_f32() {test_quant_mode::<f32>(QuantizationMode::UINT4);}-#[test]+#[uzu_test]fn test_uint4_f16() {test_quant_mode::<f16>(QuantizationMode::UINT4);}-#[test]+#[uzu_test]fn test_uint4_bf16() {test_quant_mode::<bf16>(QuantizationMode::UINT4);}// INT8 tests-#[test]+#[uzu_test]fn test_int8_f32() {test_quant_mode::<f32>(QuantizationMode::INT8);}-#[test]+#[uzu_test]fn test_int8_f16() {test_quant_mode::<f16>(QuantizationMode::INT8);}-#[test]+#[uzu_test]fn test_int8_bf16() {test_quant_mode::<bf16>(QuantizationMode::INT8);}// UINT8 tests-#[test]+#[uzu_test]fn test_uint8_f32() {test_quant_mode::<f32>(QuantizationMode::UINT8);}-#[test]+#[uzu_test]fn test_uint8_f16() {test_quant_mode::<f16>(QuantizationMode::UINT8);}-#[test]+#[uzu_test]fn test_uint8_bf16() {test_quant_mode::<bf16>(QuantizationMode::UINT8);}// OOB tests-#[test]+#[uzu_test]fn test_oob_f32() {test_oob::<f32>();}-#[test]+#[uzu_test]fn test_oob_f16() {test_oob::<f16>();}-#[test]+#[uzu_test]fn test_oob_bf16() {test_oob::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/kv_cache_update_test.rs b/crates/uzu/tests/unit/kernel/kv_cache_update_test.rsindex 670344b39..87b5a2c6c 100644--- a/crates/uzu/tests/unit/kernel/kv_cache_update_test.rs+++ b/crates/uzu/tests/unit/kernel/kv_cache_update_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {keys: Box<[T]>,@@ -218,65 +218,65 @@ fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {}// Single swap tests-#[test]+#[uzu_test]fn test_single_swap_f32() {test_single_swap_internal::<f32>();}-#[test]+#[uzu_test]fn test_single_swap_f16() {test_single_swap_internal::<f16>();}-#[test]+#[uzu_test]fn test_single_swap_bf16() {test_single_swap_internal::<bf16>();}// Multi swap tests-#[test]+#[uzu_test]fn test_multi_swap_f32() {test_multi_swap_internal::<f32>();}-#[test]+#[uzu_test]fn test_multi_swap_f16() {test_multi_swap_internal::<f16>();}-#[test]+#[uzu_test]fn test_multi_swap_bf16() {test_multi_swap_internal::<bf16>();}// No swap tests-#[test]+#[uzu_test]fn test_no_swap_f32() {test_no_swap_internal::<f32>();}-#[test]+#[uzu_test]fn test_no_swap_f16() {test_no_swap_internal::<f16>();}-#[test]+#[uzu_test]fn test_no_swap_bf16() {test_no_swap_internal::<bf16>();}// Large dimension tests-#[test]+#[uzu_test]fn test_large_f32() {test_large_internal::<f32>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large_internal::<f16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large_internal::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/layer_norm_test.rs b/crates/uzu/tests/unit/kernel/layer_norm_test.rsindex f0af5edfb..570793061 100644--- a/crates/uzu/tests/unit/kernel/layer_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/layer_norm_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};static BOOL_ALL: &[bool] = &[true, false];static BOOL_FALSE: &[bool] = &[false];@@ -226,89 +226,89 @@ fn test_edge<}// basic tests - AccumT f32-#[test]+#[uzu_test]fn test_f32_f32_f32_f32() {test_basic::<f32, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_f16_f32_f32_f32() {test_basic::<f16, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_f32_f16_f32_f32() {test_basic::<f32, f16, f32, f32>();}-#[test]+#[uzu_test]fn test_f32_f32_f16_f32() {test_basic::<f32, f32, f16, f32>();}-#[test]+#[uzu_test]fn test_f16_f16_f32_f32() {test_basic::<f16, f16, f32, f32>();}-#[test]+#[uzu_test]fn test_f16_f32_f16_f32() {test_basic::<f16, f32, f16, f32>();}-#[test]+#[uzu_test]fn test_f32_f16_f16_f32() {test_basic::<f32, f16, f16, f32>();}-#[test]+#[uzu_test]fn test_f16_f16_f16_f32() {test_basic::<f16, f16, f16, f32>();}// basic tests - AccumT f32 + bf16-#[test]+#[uzu_test]fn test_bf16_f32_f32_f32() {test_basic::<bf16, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_f32_bf16_f32_f32() {test_basic::<f32, bf16, f32, f32>();}-#[test]+#[uzu_test]fn test_f32_f32_bf16_f32() {test_basic::<f32, f32, bf16, f32>();}-#[test]+#[uzu_test]fn test_bf16_bf16_f32_f32() {test_basic::<bf16, bf16, f32, f32>();}-#[test]+#[uzu_test]fn test_bf16_f32_bf16_f32() {test_basic::<bf16, f32, bf16, f32>();}-#[test]+#[uzu_test]fn test_bf16_bf16_bf16_f32() {test_basic::<bf16, bf16, bf16, f32>();}// edge tests-#[test]+#[uzu_test]fn test_edge_f32_f32_f32_f32() {test_edge::<f32, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_edge_f32_f16_f16_f32() {test_edge::<f32, f16, f16, f32>();}-#[test]+#[uzu_test]fn test_edge_f16_f16_f16_f32() {test_edge::<f16, f16, f16, f32>();}diff --git a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rsindex 67604f30b..ac1c8d42f 100644--- a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs@@ -19,7 +19,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {a: Box<[T]>,@@ -105,107 +105,107 @@ fn test<T: ArrayElement + Float + Debug + Display>(});}-#[test]+#[uzu_test]fn test_f32_m1() {test::<f32>(1, 128, 64, 0.01);}-#[test]+#[uzu_test]fn test_f16_m1() {test::<f16>(1, 128, 64, 0.01);}-#[test]+#[uzu_test]fn test_bf16_m1() {test::<bf16>(1, 128, 64, 0.1);}-#[test]+#[uzu_test]fn test_f32_batched() {test::<f32>(4, 128, 64, 0.01);}-#[test]+#[uzu_test]fn test_f16_batched() {test::<f16>(4, 128, 64, 0.01);}-#[test]+#[uzu_test]fn test_bf16_batched() {test::<bf16>(4, 128, 64, 0.1);}-#[test]+#[uzu_test]fn test_f32_max_batch() {test::<f32>(8, 128, 64, 0.01);}-#[test]+#[uzu_test]fn test_f16_max_batch() {test::<f16>(8, 128, 64, 0.01);}-#[test]+#[uzu_test]fn test_bf16_max_batch() {test::<bf16>(8, 128, 64, 0.1);}-#[test]+#[uzu_test]fn test_f32_unaligned_k() {test::<f32>(1, 33, 64, 0.01);}-#[test]+#[uzu_test]fn test_f16_unaligned_k() {test::<f16>(1, 33, 64, 0.01);}-#[test]+#[uzu_test]fn test_bf16_unaligned_k() {test::<bf16>(1, 33, 64, 0.1);}-#[test]+#[uzu_test]fn test_f32_unaligned_n() {test::<f32>(1, 128, 11, 0.01);}-#[test]+#[uzu_test]fn test_f16_unaligned_n() {test::<f16>(1, 128, 11, 0.01);}-#[test]+#[uzu_test]fn test_bf16_unaligned_n() {test::<bf16>(1, 128, 11, 0.1);}-#[test]+#[uzu_test]fn test_f32_large() {test::<f32>(1, 4096, 2048, 0.05);}-#[test]+#[uzu_test]fn test_f16_large() {test::<f16>(1, 4096, 2048, 0.5);}-#[test]+#[uzu_test]fn test_bf16_large() {test::<bf16>(1, 4096, 2048, 1.0);}-#[test]+#[uzu_test]fn test_f32_small_n() {test::<f32>(1, 128, 3, 0.01);}-#[test]+#[uzu_test]fn test_f16_small_n() {test::<f16>(1, 128, 3, 0.01);}-#[test]+#[uzu_test]fn test_bf16_small_n() {test::<bf16>(1, 128, 3, 0.1);}diff --git a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rsindex 842642aa7..11406a698 100644--- a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs@@ -1,7 +1,6 @@//! Correctness tests comparing Metal matmul kernels against ndarray#![cfg(metal_backend)]-use bytemuck;use half::bf16;use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};@@ -20,6 +19,8 @@ use uzu::{},};+use crate::uzu_test;+fn run_metal_matmul(ctx: &<Metal as Backend>::Context,a_data: &[bf16],@@ -250,7 +251,7 @@ fn compare_results(}}-#[test]+#[uzu_test]#[ignore]fn matmul_correctness_comprehensive() {let Some(ctx) = <Metal as Backend>::Context::new().ok() else {diff --git a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rsindex 7b1966953..d803ec381 100644--- a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs@@ -19,7 +19,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {a: Box<[T]>,@@ -105,47 +105,47 @@ fn test<T: ArrayElement + Float + Debug + Display>(});}-#[test]+#[uzu_test]fn test_f32_aligned() {test::<f32>(64, 64, 64, 0.01);}-#[test]+#[uzu_test]fn test_f16_aligned() {test::<f16>(64, 64, 64, 0.01);}-#[test]+#[uzu_test]fn test_bf16_aligned() {test::<bf16>(64, 64, 64, 0.1);}-#[test]+#[uzu_test]fn test_f32_unaligned() {test::<f32>(7, 33, 11, 0.01);}-#[test]+#[uzu_test]fn test_f16_unaligned() {test::<f16>(7, 33, 11, 0.01);}-#[test]+#[uzu_test]fn test_bf16_unaligned() {test::<bf16>(7, 33, 11, 0.1);}-#[test]+#[uzu_test]fn test_f32_large() {test::<f32>(16, 128, 256, 0.01);}-#[test]+#[uzu_test]fn test_f16_large() {test::<f16>(16, 128, 256, 0.01);}-#[test]+#[uzu_test]fn test_bf16_large() {test::<bf16>(16, 128, 256, 0.1);}diff --git a/crates/uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rsindex ebd6ec032..d41c2d1b7 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs@@ -9,6 +9,8 @@ use uzu::{},};+use crate::uzu_test;+fn gen_random_topk_ids(t: usize,e: usize,@@ -67,7 +69,7 @@ fn get_output<B: Backend>((offsets, sum_k, partials)}-#[test]+#[uzu_test]fn test_counts_offsets_fused_parity_random() {let shapes = vec![(1usize, 4usize), (7, 16), (64, 64), (1024, 128)];let ks = vec![1usize, 2usize, 4usize];@@ -91,7 +93,7 @@ fn test_counts_offsets_fused_parity_random() {}}-#[test]+#[uzu_test]fn test_counts_offsets_fused_all_tokens_one_expert() {let (t, e, k) = (16usize, 8usize, 2usize);let topk_ids: Vec<i32> = vec![3i32; t * k];@@ -107,7 +109,7 @@ fn test_counts_offsets_fused_all_tokens_one_expert() {});}-#[test]+#[uzu_test]fn test_counts_offsets_fused_zero_tokens() {let (t, e, k) = (0usize, 8usize, 2usize);let topk_ids: Vec<i32> = vec![];@@ -120,7 +122,7 @@ fn test_counts_offsets_fused_zero_tokens() {});}-#[test]+#[uzu_test]fn test_counts_offsets_fused_negative_ids_ignored() {let (t, e, k) = (8usize, 4usize, 2usize);// Mix of valid IDs and -1 (invalid/padding)@@ -143,7 +145,7 @@ fn test_counts_offsets_fused_negative_ids_ignored() {});}-#[test]+#[uzu_test]fn test_counts_offsets_fused_single_token() {let (t, e, k) = (1usize, 16usize, 1usize);let topk_ids = vec![7i32];@@ -160,7 +162,7 @@ fn test_counts_offsets_fused_single_token() {});}-#[test]+#[uzu_test]fn test_counts_offsets_fused_uniform_distribution() {// Each expert gets exactly 1 tokenlet e = 8usize;@@ -177,7 +179,7 @@ fn test_counts_offsets_fused_uniform_distribution() {});}-#[test]+#[uzu_test]fn test_counts_offsets_fused_large_t() {// Stress test with many tokenslet (t, e, k) = (2048usize, 128usize, 4usize);@@ -193,7 +195,7 @@ fn test_counts_offsets_fused_large_t() {});}-#[test]+#[uzu_test]fn test_counts_offsets_fused_offsets_monotonic() {// Verify offsets are monotonically non-decreasing for various shapeslet shapes = vec![(10, 8, 2), (100, 32, 4), (5, 128, 1)];diff --git a/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rsindex 810834fad..ebefa6f05 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs@@ -11,7 +11,10 @@ use uzu::{},};-use crate::common::{assert::assert_eq_float, helpers::create_context};+use crate::{+ common::{assert::assert_eq_float, helpers::create_context},+ uzu_test,+};struct Input<T: ArrayElement + Float> {tok2row: Box<[i32]>,@@ -94,22 +97,22 @@ fn test_finalize_internal(});}-#[test]+#[uzu_test]fn test_finalize_single_token() {test_finalize_internal(1, 2, 64, 2)}-#[test]+#[uzu_test]fn test_finalize_small_batch() {test_finalize_internal(4, 2, 128, 8)}-#[test]+#[uzu_test]fn test_finalize_medium() {test_finalize_internal(8, 4, 256, 32)}-#[test]+#[uzu_test]fn test_finalize_large() {test_finalize_internal(16, 2, 512, 32)}diff --git a/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rsindex ebe1395b6..a5bd8543a 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs@@ -14,7 +14,10 @@ use uzu::{},};-use crate::common::{assert::assert_eq_float, helpers::create_context};+use crate::{+ common::{assert::assert_eq_float, helpers::create_context},+ uzu_test,+};struct Input<T: ArrayElement + Float> {x: Box<[T]>,@@ -77,22 +80,22 @@ fn test_gather_internal(})}-#[test]+#[uzu_test]fn test_gather_single_token() {test_gather_internal(1, 2, 64);}-#[test]+#[uzu_test]fn test_gather_small_batch() {test_gather_internal(4, 8, 128);}-#[test]+#[uzu_test]fn test_gather_medium_batch() {test_gather_internal(16, 32, 256)}-#[test]+#[uzu_test]fn test_gather_large_batch() {test_gather_internal(64, 128, 512)}diff --git a/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rsindex 6610b2604..1660a4925 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs@@ -14,7 +14,7 @@ use uzu::{},};-use crate::common::helpers::create_context;+use crate::{common::helpers::create_context, uzu_test};fn get_output<B: Backend, T: ArrayElement + Float>(input: &[T],@@ -107,7 +107,7 @@ fn run_router_topk_once<B: Backend, T: ArrayElement + Debug + Float>(}}-#[test]+#[uzu_test]fn test_router_topk_fused_matches_reference() {let configs =[(1usize, 64usize, 32usize, 4usize), (2, 128, 64, 8), (4, 256, 128, 16), (8, 256, 256, 32), (1, 512, 512, 64)];diff --git a/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rsindex 65924f595..f865760c7 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs@@ -17,7 +17,7 @@ use uzu::{},};-use crate::common::helpers::create_context;+use crate::{common::helpers::create_context, uzu_test};fn cpu_expert_buckets<T: ArrayElement + Float>(topk_ids: &[i32],@@ -224,17 +224,17 @@ fn test_scatter_internal<B: Backend, T: ArrayElement + Float>(}}-#[test]+#[uzu_test]fn test_scatter_buckets_small() {for_each_non_cpu_backend!(|B| test_scatter_internal::<B, bf16>(1, 64, 4, 1));}-#[test]+#[uzu_test]fn test_scatter_buckets_medium() {for_each_non_cpu_backend!(|B| test_scatter_internal::<B, bf16>(7, 64, 16, 2));}-#[test]+#[uzu_test]fn test_scatter_buckets_big() {for_each_non_cpu_backend!(|B| test_scatter_internal::<B, bf16>(128, 64, 64, 2));}diff --git a/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rs b/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rsindex 248d9c87f..057a5189c 100644--- a/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rs+++ b/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {input: Box<[T]>,@@ -87,71 +87,71 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// Basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_internal::<f32>(2, 16, 64);}-#[test]+#[uzu_test]fn test_basic_f16() {test_internal::<f16>(2, 16, 64);}-#[test]+#[uzu_test]fn test_basic_bf16() {test_internal::<bf16>(2, 16, 64);}// Single batch-#[test]+#[uzu_test]fn test_single_batch_f32() {test_internal::<f32>(1, 32, 128);}-#[test]+#[uzu_test]fn test_single_batch_f16() {test_internal::<f16>(1, 32, 128);}-#[test]+#[uzu_test]fn test_single_batch_bf16() {test_internal::<bf16>(1, 32, 128);}// Single token sequence-#[test]+#[uzu_test]fn test_single_token_f32() {test_internal::<f32>(2, 1, 64);}-#[test]+#[uzu_test]fn test_single_token_f16() {test_internal::<f16>(2, 1, 64);}-#[test]+#[uzu_test]fn test_single_token_bf16() {test_internal::<bf16>(2, 1, 64);}// Large hidden dim-#[test]+#[uzu_test]fn test_large_hidden_dim_f32() {test_internal::<f32>(2, 8, 512);}// Non-aligned hidden dim-#[test]+#[uzu_test]fn test_non_aligned_f32() {test_internal::<f32>(3, 10, 100);}-#[test]+#[uzu_test]fn test_non_aligned_f16() {test_internal::<f16>(3, 10, 100);}-#[test]+#[uzu_test]fn test_non_aligned_bf16() {test_internal::<bf16>(3, 10, 100);}diff --git a/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rs b/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rsindex 526045fca..feb4da725 100644--- a/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rs+++ b/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {input: Box<[T]>,@@ -87,77 +87,77 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// Basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_internal::<f32>(2, 16, 64);}-#[test]+#[uzu_test]fn test_basic_f16() {test_internal::<f16>(2, 16, 64);}-#[test]+#[uzu_test]fn test_basic_bf16() {test_internal::<bf16>(2, 16, 64);}// Single batch-#[test]+#[uzu_test]fn test_single_batch_f32() {test_internal::<f32>(1, 32, 128);}-#[test]+#[uzu_test]fn test_single_batch_f16() {test_internal::<f16>(1, 32, 128);}-#[test]+#[uzu_test]fn test_single_batch_bf16() {test_internal::<bf16>(1, 32, 128);}// Single token sequence — mean of one element equals that element-#[test]+#[uzu_test]fn test_single_token_f32() {test_internal::<f32>(2, 1, 64);}-#[test]+#[uzu_test]fn test_single_token_f16() {test_internal::<f16>(2, 1, 64);}-#[test]+#[uzu_test]fn test_single_token_bf16() {test_internal::<bf16>(2, 1, 64);}// Large hidden dim-#[test]+#[uzu_test]fn test_large_hidden_dim_f32() {test_internal::<f32>(2, 8, 512);}// Long sequence — more accumulation, tests precision-#[test]+#[uzu_test]fn test_long_seq_f32() {test_internal::<f32>(1, 256, 64);}// Non-aligned hidden dim-#[test]+#[uzu_test]fn test_non_aligned_f32() {test_internal::<f32>(3, 10, 100);}-#[test]+#[uzu_test]fn test_non_aligned_f16() {test_internal::<f16>(3, 10, 100);}-#[test]+#[uzu_test]fn test_non_aligned_bf16() {test_internal::<bf16>(3, 10, 100);}diff --git a/crates/uzu/tests/unit/kernel/qk_norm_test.rs b/crates/uzu/tests/unit/kernel/qk_norm_test.rsindex ebf13d876..2af9c8d50 100644--- a/crates/uzu/tests/unit/kernel/qk_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/qk_norm_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<InputT: ArrayElement + Float, ScaleT: ArrayElement + Float, OutputT: ArrayElement + Float> {qkv: Box<[OutputT]>,@@ -254,64 +254,64 @@ fn test_addressing<}// Q norm tests-#[test]+#[uzu_test]fn test_q_norm_f32_f32_f32_f32() {test_q_norm::<f32, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_q_norm_f16_f16_f16_f32() {test_q_norm::<f16, f16, f16, f32>();}-#[test]+#[uzu_test]fn test_q_norm_f16_f16_f16_f16() {test_q_norm::<f16, f16, f16, f16>();}-#[test]+#[uzu_test]fn test_q_norm_bf16_bf16_bf16_f32() {test_q_norm::<bf16, bf16, bf16, f32>();}-#[test]+#[uzu_test]fn test_q_norm_f32_f16_f32_f32() {test_q_norm::<f32, f16, f32, f32>();}-#[test]+#[uzu_test]fn test_q_norm_f16_f32_f16_f32() {test_q_norm::<f16, f32, f16, f32>();}// K norm tests-#[test]+#[uzu_test]fn test_k_norm_f32_f32_f32_f32() {test_k_norm::<f32, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_k_norm_f16_f16_f16_f32() {test_k_norm::<f16, f16, f16, f32>();}-#[test]+#[uzu_test]fn test_k_norm_f16_f16_f16_f16() {test_k_norm::<f16, f16, f16, f16>();}-#[test]+#[uzu_test]fn test_k_norm_bf16_bf16_bf16_f32() {test_k_norm::<bf16, bf16, bf16, f32>();}// Addressing tests (Q norm should not touch K/V)-#[test]+#[uzu_test]fn test_addressing_f32_f32_f32_f32() {test_addressing::<f32, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_addressing_f16_f16_f16_f32() {test_addressing::<f16, f16, f16, f32>();}diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rsindex fb779c3cc..05bf22344 100644--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs@@ -16,7 +16,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};static BOOL_ALL: &[bool] = &[true, false];static BOOL_FALSE: &[bool] = &[false];@@ -307,145 +307,145 @@ fn test_performance<B: Backend>(}// AccumT f32-#[test]+#[uzu_test]fn test_f32_f32_f32_f32() {test_basic::<f32, f32, f32, f32>();}// AccumT f32 + f16-#[test]+#[uzu_test]fn test_f16_f32_f32_f32() {test_basic::<f16, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_f32_f16_f32_f32() {test_basic::<f32, f16, f32, f32>();}-#[test]+#[uzu_test]fn test_f32_f32_f16_f32() {test_basic::<f32, f32, f16, f32>();}-#[test]+#[uzu_test]fn test_f16_f16_f32_f32() {test_basic::<f16, f16, f32, f32>();}-#[test]+#[uzu_test]fn test_f16_f32_f16_f32() {test_basic::<f16, f32, f16, f32>();}-#[test]+#[uzu_test]fn test_f32_f16_f16_f32() {test_basic::<f32, f16, f16, f32>();}-#[test]+#[uzu_test]fn test_f16_f16_f16_f32() {test_basic::<f16, f16, f16, f32>();}// AccumT f32 + bf16-#[test]+#[uzu_test]fn test_bf16_f32_f32_f32() {test_basic::<bf16, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_f32_bf16_f32_f32() {test_basic::<f32, bf16, f32, f32>();}-#[test]+#[uzu_test]fn test_f32_f32_bf16_f32() {test_basic::<f32, f32, bf16, f32>();}-#[test]+#[uzu_test]fn test_bf16_bf16_f32_f32() {test_basic::<bf16, bf16, f32, f32>();}-#[test]+#[uzu_test]fn test_bf16_f32_bf16_f32() {test_basic::<bf16, f32, bf16, f32>();}-#[test]+#[uzu_test]fn test_bf16_bf16_bf16_f32() {test_basic::<bf16, bf16, bf16, f32>();}// AccumT f16-#[test]+#[uzu_test]fn test_f32_f32_f32_f16() {test_basic::<f32, f32, f32, f16>();}-#[test]+#[uzu_test]fn test_f16_f32_f32_f16() {test_basic::<f16, f32, f32, f16>();}-#[test]+#[uzu_test]fn test_f32_f16_f32_f16() {test_basic::<f32, f16, f32, f16>();}-#[test]+#[uzu_test]fn test_f32_f32_f16_f16() {test_basic::<f32, f32, f16, f16>();}-#[test]+#[uzu_test]fn test_f16_f16_f32_f16() {test_basic::<f16, f16, f32, f16>();}-#[test]+#[uzu_test]fn test_f16_f32_f16_f16() {test_basic::<f16, f32, f16, f16>();}-#[test]+#[uzu_test]fn test_f32_f16_f16_f16() {test_basic::<f32, f16, f16, f16>();}-#[test]+#[uzu_test]fn test_f16_f16_f16_f16() {test_basic::<f16, f16, f16, f16>();}// edge cases-#[test]+#[uzu_test]fn test_edge_f32_f32_f32_f32() {test_edge::<f32, f32, f32, f32>();}-#[test]+#[uzu_test]fn test_edge_f32_f16_f16_f32() {test_edge::<f32, f16, f16, f32>();}-#[test]+#[uzu_test]fn test_edge_f16_f16_f16_f16() {test_edge::<f16, f16, f16, f16>();}// performance tests#[cfg(metal_backend)]-#[test]+#[uzu_test]fn test_perf_8k() {use uzu::backends::metal::Metal;test_performance::<Metal>(8, 8192); // Large model like LLaMA-70B}#[cfg(metal_backend)]-#[test]+#[uzu_test]fn test_perf_16k() {use uzu::backends::metal::Metal;test_performance::<Metal>(16, 16384); // Huge model dimensiondiff --git a/crates/uzu/tests/unit/kernel/rope_test.rs b/crates/uzu/tests/unit/kernel/rope_test.rsindex 24a306243..1ddad3464 100644--- a/crates/uzu/tests/unit/kernel/rope_test.rs+++ b/crates/uzu/tests/unit/kernel/rope_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {qkv: Box<[T]>,@@ -171,79 +171,79 @@ fn test_nonzero_positions<T: ArrayElement + Float + Debug + Display>() {}// f32 tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_mha_f32() {test_mha::<f32>();}-#[test]+#[uzu_test]fn test_single_token_f32() {test_single_token::<f32>();}-#[test]+#[uzu_test]fn test_small_f32() {test_small::<f32>();}-#[test]+#[uzu_test]fn test_nonzero_positions_f32() {test_nonzero_positions::<f32>();}// f16 tests-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_mha_f16() {test_mha::<f16>();}-#[test]+#[uzu_test]fn test_single_token_f16() {test_single_token::<f16>();}-#[test]+#[uzu_test]fn test_small_f16() {test_small::<f16>();}// bf16 tests-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}-#[test]+#[uzu_test]fn test_mha_bf16() {test_mha::<bf16>();}-#[test]+#[uzu_test]fn test_single_token_bf16() {test_single_token::<bf16>();}-#[test]+#[uzu_test]fn test_small_bf16() {test_small::<bf16>();}-#[test]+#[uzu_test]fn test_partial_rope_basic_f32() {test_partial_rope_basic::<f32>();}-#[test]+#[uzu_test]fn test_partial_rope_small_f32() {test_partial_rope_small::<f32>();}diff --git a/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs b/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rsindex 35c3f92cf..e1799ba67 100644--- a/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs+++ b/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs@@ -15,6 +15,8 @@ use uzu::{},};+use crate::uzu_test;+struct Input<T: ArrayElement + Float> {logits: Box<[T]>,batch_size: u32,@@ -149,56 +151,56 @@ fn test_two_pass<T: ArrayElement + Float + Display>(}// Single-pass tests-#[test]+#[uzu_test]fn test_single_pass_f32() {test_single_pass::<f32>(4, 1024);}-#[test]+#[uzu_test]fn test_single_pass_f16() {test_single_pass::<f16>(4, 1024);}-#[test]+#[uzu_test]fn test_single_pass_bf16() {test_single_pass::<bf16>(4, 1024);}// Two-pass tests-#[test]+#[uzu_test]fn test_two_pass_f32() {test_two_pass::<f32>(4, 1024);}-#[test]+#[uzu_test]fn test_two_pass_f16() {test_two_pass::<f16>(4, 1024);}-#[test]+#[uzu_test]fn test_two_pass_bf16() {test_two_pass::<bf16>(4, 1024);}// Large vocab tests (exercises multi-group reduction)-#[test]+#[uzu_test]fn test_single_pass_large_vocab_f32() {test_single_pass::<f32>(4, 128 * 1024);}-#[test]+#[uzu_test]fn test_two_pass_large_vocab_f32() {test_two_pass::<f32>(4, 128 * 1024);}// Edge cases-#[test]+#[uzu_test]fn test_single_batch_f32() {test_single_pass::<f32>(1, 1024);test_two_pass::<f32>(1, 1024);}-#[test]+#[uzu_test]fn test_small_vocab_f32() {test_single_pass::<f32>(4, 4);test_two_pass::<f32>(4, 4);diff --git a/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rs b/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rsindex 71de2c260..5dc8ffad5 100644--- a/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {logits: Box<[T]>,@@ -219,81 +219,81 @@ fn test_non_aligned<T: ArrayElement + Float + Debug + Display>() {}// Out-of-place basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>(false);}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>(false);}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>(false);}// In-place basic tests-#[test]+#[uzu_test]fn test_in_place_f32() {test_basic::<f32>(true);}-#[test]+#[uzu_test]fn test_in_place_f16() {test_basic::<f16>(true);}-#[test]+#[uzu_test]fn test_in_place_bf16() {test_basic::<bf16>(true);}// All-allowed tests-#[test]+#[uzu_test]fn test_all_allowed_f32() {test_all_allowed::<f32>();}-#[test]+#[uzu_test]fn test_all_allowed_f16() {test_all_allowed::<f16>();}-#[test]+#[uzu_test]fn test_all_allowed_bf16() {test_all_allowed::<bf16>();}// None-allowed tests-#[test]+#[uzu_test]fn test_none_allowed_f32() {test_none_allowed::<f32>();}-#[test]+#[uzu_test]fn test_none_allowed_f16() {test_none_allowed::<f16>();}-#[test]+#[uzu_test]fn test_none_allowed_bf16() {test_none_allowed::<bf16>();}// Non-aligned vocab_size tests-#[test]+#[uzu_test]fn test_non_aligned_f32() {test_non_aligned::<f32>();}-#[test]+#[uzu_test]fn test_non_aligned_f16() {test_non_aligned::<f16>();}-#[test]+#[uzu_test]fn test_non_aligned_bf16() {test_non_aligned::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs b/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rsindex 5e293cd2a..4df6c87be 100644--- a/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs@@ -14,6 +14,8 @@ use uzu::{gumbel_float, revidx,};+use crate::uzu_test;+struct Input<T: ArrayElement + Float> {logits: Box<[T]>,seeds: Box<[u64]>,@@ -129,32 +131,32 @@ fn test_internal<T: ArrayElement + Float + Display>(});}-#[test]+#[uzu_test]fn test_determinism_f32() {test_determinism::<f32>(4, 128);}-#[test]+#[uzu_test]fn test_determinism_f16() {test_determinism::<f16>(4, 128);}-#[test]+#[uzu_test]fn test_determinism_bf16() {test_determinism::<bf16>(4, 128);}-#[test]+#[uzu_test]fn test_f32() {test_internal::<f32>(7, 16 * 1024 * 64);}-#[test]+#[uzu_test]fn test_f16() {test_internal::<f16>(7, 16 * 1024 * 64);}-#[test]+#[uzu_test]fn test_bf16() {test_internal::<bf16>(7, 16 * 1024 * 64);}diff --git a/crates/uzu/tests/unit/kernel/sampling/min_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/min_p_test.rsindex 1ad9a74aa..6622f7d45 100644--- a/crates/uzu/tests/unit/kernel/sampling/min_p_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/min_p_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {logits: Box<[T]>,@@ -96,77 +96,77 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// Out-of-place tests-#[test]+#[uzu_test]fn test_f32() {test_internal::<f32>(4, 128, 0.1, false);}-#[test]+#[uzu_test]fn test_f16() {test_internal::<f16>(4, 128, 0.1, false);}-#[test]+#[uzu_test]fn test_bf16() {test_internal::<bf16>(4, 128, 0.1, false);}// In-place tests-#[test]+#[uzu_test]fn test_in_place_f32() {test_internal::<f32>(4, 128, 0.1, true);}-#[test]+#[uzu_test]fn test_in_place_f16() {test_internal::<f16>(4, 128, 0.1, true);}-#[test]+#[uzu_test]fn test_in_place_bf16() {test_internal::<bf16>(4, 128, 0.1, true);}// High min_p — filters most tokens-#[test]+#[uzu_test]fn test_high_min_p_f32() {test_internal::<f32>(2, 128, 0.9, false);}-#[test]+#[uzu_test]fn test_high_min_p_f16() {test_internal::<f16>(2, 128, 0.9, false);}-#[test]+#[uzu_test]fn test_high_min_p_bf16() {test_internal::<bf16>(2, 128, 0.9, false);}// Low min_p — keeps most tokens-#[test]+#[uzu_test]fn test_low_min_p_f32() {test_internal::<f32>(2, 128, 0.01, false);}-#[test]+#[uzu_test]fn test_low_min_p_f16() {test_internal::<f16>(2, 128, 0.01, false);}-#[test]+#[uzu_test]fn test_low_min_p_bf16() {test_internal::<bf16>(2, 128, 0.01, false);}// Single batch-#[test]+#[uzu_test]fn test_single_batch_f32() {test_internal::<f32>(1, 256, 0.1, false);}// Large vocab-#[test]+#[uzu_test]fn test_large_vocab_f32() {test_internal::<f32>(2, 2048, 0.1, false);}diff --git a/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs b/crates/uzu/tests/unit/kernel/sampling/temperature_test.rsindex d8d502c23..9b60aeb51 100644--- a/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {logits: Box<[T]>,@@ -101,54 +101,54 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// Out-of-place tests-#[test]+#[uzu_test]fn test_f32() {test_internal::<f32>(4, 128, 0.7, false);}-#[test]+#[uzu_test]fn test_f16() {test_internal::<f16>(4, 128, 0.7, false);}-#[test]+#[uzu_test]fn test_bf16() {test_internal::<bf16>(4, 128, 0.7, false);}// In-place tests-#[test]+#[uzu_test]fn test_in_place_f32() {test_internal::<f32>(4, 128, 0.7, true);}-#[test]+#[uzu_test]fn test_in_place_f16() {test_internal::<f16>(4, 128, 0.7, true);}-#[test]+#[uzu_test]fn test_in_place_bf16() {test_internal::<bf16>(4, 128, 0.7, true);}// Edge cases-#[test]+#[uzu_test]fn test_temperature_1_f32() {test_internal::<f32>(2, 64, 1.0, false);}-#[test]+#[uzu_test]fn test_high_temperature_f32() {test_internal::<f32>(2, 64, 100.0, false);}-#[test]+#[uzu_test]fn test_low_temperature_f32() {test_internal::<f32>(2, 64, 0.1, false);}-#[test]+#[uzu_test]fn test_single_batch_f32() {test_internal::<f32>(1, 256, 0.7, false);}diff --git a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rsindex 993664f96..d8b17834a 100644--- a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs+++ b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs@@ -1,5 +1,6 @@use std::mem::MaybeUninit;+use criterion::{BenchmarkId, Criterion, Throughput};use num_traits::{Float, NumCast};use proptest::prelude::*;use rand::{RngExt, SeedableRng, rngs::SmallRng};@@ -14,8 +15,12 @@ use uzu::{};use crate::{- common::proptest::{ComparableTestResults, TestContextes, kernel_data_type},- dispatch_dtype, for_each_context,+ common::{+ dispatch_dtype, for_each_context,+ proptest::{ComparableTestResults, TestContextes, kernel_data_type},+ type_short_name,+ },+ uzu_bench, uzu_test,};struct ArgmaxTestResults(Box<[u32]>);@@ -91,34 +96,118 @@ fn do_argmax_backend<B: Backend, T: ArrayElement + Float>(Ok(ArgmaxTestResults(single_output))}-fn do_argmax<T: ArrayElement + Float>(- contextes: &TestContextes,+fn get_argmax_data<T: ArrayElement + Float>(seed: u64,batch_size: usize,vocab_size: usize,-) -> Result<(), TestCaseError> {+) -> Box<[T]> {let mut rng = SmallRng::seed_from_u64(seed);let distribution = Normal::new(0.0, 32.0).unwrap();let mut logits: Box<[MaybeUninit<T>]> = Box::new_uninit_slice(batch_size * vocab_size);for logit in logits.iter_mut() {logit.write(<T as NumCast>::from(rng.sample(distribution)).unwrap());}- let logits = unsafe { logits.assume_init() };-- for_each_context!(contextes, |context: C| do_argmax_backend::<<C as Context>::Backend, T>(- context,- logits.as_ref(),- batch_size,- vocab_size- ))- .compare_results()+ unsafe { logits.assume_init() }}-#[test]+#[uzu_test]fn test_argmax_prop() {let contextes = TestContextes::new();proptest!(|(data_type in kernel_data_type(), seed in any::<u64>().no_shrink(), batch_size in (1usize..=7usize), vocab_size in (1usize..=123456usize))| {- dispatch_dtype!(|(T: data_type)| do_argmax::<T>(&contextes, seed, batch_size, vocab_size)?);+ dispatch_dtype!(|(T: data_type)| {+ let logits = get_argmax_data(seed, batch_size, vocab_size);++ for_each_context!(contextes, |context: C| do_argmax_backend::<<C as Context>::Backend, T>(+ context,+ logits.as_ref(),+ batch_size,+ vocab_size+ ))+ .compare_results()?+ });+ });+}++#[uzu_bench]+fn bench_argmax(c: &mut Criterion) {+ type T = f32;+ let batch_size = 1;++ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();++ let single_kernel =+ <<B as Backend>::Kernels as Kernels>::ArgmaxSingleKernel::new(&context, T::data_type()).unwrap();+ let twopass_kernel_main =+ <<B as Backend>::Kernels as Kernels>::ArgmaxMainKernel::new(&context, T::data_type()).unwrap();+ let twopass_kernel_final = <<B as Backend>::Kernels as Kernels>::ArgmaxFinalKernel::new(&context).unwrap();++ let mut output_buffer = context.create_buffer(batch_size * std::mem::size_of::<u32>()).unwrap();++ let mut group = c.benchmark_group(format!("{}/Kernel/Sampling/Argmax", type_short_name::<B>()));++ for vocab_size in [+ 65536, // LFM2-350M+ 128256, // Llama-3.2-1B+ 151936, // Qwen3-1.7B+ 262144, // Gemma-3-1b+ ] {+ let logits_data = get_argmax_data::<T>(1337, batch_size, vocab_size);+ let logits_buffer = context.create_buffer(logits_data.len() * std::mem::size_of::<T>()).unwrap();+ unsafe {+ std::slice::from_raw_parts_mut::<T>(logits_buffer.cpu_ptr().as_ptr() as *mut T, logits_data.len())+ .copy_from_slice(logits_data.as_ref());+ }++ let mut twopass_partial_results_buffer = context+ .create_buffer(batch_size * vocab_size.div_ceil(4096) * std::mem::size_of::<ArgmaxPair>())+ .unwrap();++ group.throughput(Throughput::Elements(vocab_size as u64));++ group.bench_function(BenchmarkId::new("SinglePass", format!("Vocab[{vocab_size}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(&context).unwrap();++ for _ in 0..n_iters {+ single_kernel.encode(+ &logits_buffer,+ &mut output_buffer,+ batch_size as u32,+ vocab_size as u32,+ &mut encoder,+ );+ }++ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()+ })+ });++ group.bench_function(BenchmarkId::new("TwoPass", format!("Vocab[{vocab_size}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(&context).unwrap();++ for _ in 0..n_iters {+ twopass_kernel_main.encode(+ &logits_buffer,+ &mut twopass_partial_results_buffer,+ batch_size as u32,+ vocab_size as u32,+ &mut encoder,+ );+ twopass_kernel_final.encode(+ &twopass_partial_results_buffer,+ &mut output_buffer,+ batch_size as u32,+ vocab_size as u32,+ &mut encoder,+ );+ }++ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()+ })+ });+ }});}diff --git a/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_k_test.rsindex b8ef4d709..5c1854915 100644--- a/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs@@ -14,6 +14,8 @@ use uzu::{},};+use crate::uzu_test;+struct Input<T: ArrayElement + Float> {logits: Box<[T]>,batch_size: u32,@@ -140,49 +142,49 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// Out-of-place tests (same data as sampling_test::test_topk_gpu_cpu_match)-#[test]+#[uzu_test]fn test_f32() {test_internal::<f32>(4, 1024, 16, false);}-#[test]+#[uzu_test]fn test_f16() {test_internal::<f16>(4, 1024, 16, false);}-#[test]+#[uzu_test]fn test_bf16() {test_internal::<bf16>(4, 1024, 16, false);}// In-place tests-#[test]+#[uzu_test]fn test_in_place_f32() {test_internal::<f32>(4, 1024, 16, true);}-#[test]+#[uzu_test]fn test_in_place_f16() {test_internal::<f16>(4, 1024, 16, true);}-#[test]+#[uzu_test]fn test_in_place_bf16() {test_internal::<bf16>(4, 1024, 16, true);}// Edge cases-#[test]+#[uzu_test]fn test_single_batch_f32() {test_internal::<f32>(1, 1024, 16, false);}-#[test]+#[uzu_test]fn test_top_k_1_f32() {test_internal::<f32>(4, 1024, 1, false);}-#[test]+#[uzu_test]fn test_top_k_equals_vocab_f32() {test_internal::<f32>(4, 1024, 1024, false);}diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rsindex 0bd3c6b46..deaf6745d 100644--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs@@ -19,6 +19,8 @@ use uzu::{session::parameter::{SamplingMethod, SamplingProcessingOrder},};+use crate::uzu_test;+const TEST_SAMPLING_SEED: u64 = 42;struct Input<T: ArrayElement + Float> {@@ -142,49 +144,49 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// Out-of-place tests (same data as sampling_test::test_topp_gpu_cpu_match)-#[test]+#[uzu_test]fn test_f32() {test_internal::<f32>(4, 1024, 0.9, false);}-#[test]+#[uzu_test]fn test_f16() {test_internal::<f16>(4, 1024, 0.9, false);}-#[test]+#[uzu_test]fn test_bf16() {test_internal::<bf16>(4, 1024, 0.9, false);}// In-place tests-#[test]+#[uzu_test]fn test_in_place_f32() {test_internal::<f32>(4, 1024, 0.9, true);}-#[test]+#[uzu_test]fn test_in_place_f16() {test_internal::<f16>(4, 1024, 0.9, true);}-#[test]+#[uzu_test]fn test_in_place_bf16() {test_internal::<bf16>(4, 1024, 0.9, true);}// Edge cases-#[test]+#[uzu_test]fn test_single_batch_f32() {test_internal::<f32>(1, 1024, 0.9, false);}-#[test]+#[uzu_test]fn test_top_p_very_small() {test_internal::<f32>(4, 1024, 0.01, false);}-#[test]+#[uzu_test]fn test_top_p_near_1() {test_internal::<f32>(4, 1024, 0.99, false);}@@ -333,21 +335,21 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>();}-#[test]+#[uzu_test]fn test_topp_sampling_match_small() {for_each_non_cpu_backend!(|B| {test_topp_sampling_from_prob_exact_match_internal::<B>(8, 10, 1024);});}-#[test]+#[uzu_test]fn test_topp_sampling_match_large() {for_each_non_cpu_backend!(|B| {test_topp_sampling_from_prob_exact_match_internal::<B>(32, 50, 4096);});}-#[test]+#[uzu_test]fn test_topp_sampling_statistical_large() {for_each_non_cpu_backend!(|B| {const BATCH: usize = 32;@@ -437,7 +439,7 @@ fn test_topp_sampling_statistical_large() {});}-#[test]+#[uzu_test]fn perf_topp_128k_vocab() {for_each_non_cpu_backend!(|B| {const BATCH: usize = 8;diff --git a/crates/uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs b/crates/uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rsindex c3c478fd5..7bab1a1a1 100644--- a/crates/uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs+++ b/crates/uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {in_proj: Box<[T]>,@@ -295,65 +295,65 @@ fn test_edge_kernel<T: ArrayElement + Float + Debug + Display>() {}// basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// large tests-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}// edge: small dimensions-#[test]+#[uzu_test]fn test_edge_small_f32() {test_edge_small::<f32>();}-#[test]+#[uzu_test]fn test_edge_small_f16() {test_edge_small::<f16>();}-#[test]+#[uzu_test]fn test_edge_small_bf16() {test_edge_small::<bf16>();}// edge: kernel_size=1 (no state taps)-#[test]+#[uzu_test]fn test_edge_kernel_f32() {test_edge_kernel::<f32>();}-#[test]+#[uzu_test]fn test_edge_kernel_f16() {test_edge_kernel::<f16>();}-#[test]+#[uzu_test]fn test_edge_kernel_bf16() {test_edge_kernel::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs b/crates/uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rsindex c443bb71c..c4c9ae247 100644--- a/crates/uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs+++ b/crates/uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {state_in: Box<[T]>,@@ -174,65 +174,65 @@ fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {}// basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// large tests-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}// edge tests: single token-#[test]+#[uzu_test]fn test_edge_single_token_f32() {test_edge_single_token::<f32>();}-#[test]+#[uzu_test]fn test_edge_single_token_f16() {test_edge_single_token::<f16>();}-#[test]+#[uzu_test]fn test_edge_single_token_bf16() {test_edge_single_token::<bf16>();}// edge tests: no state (state_stride = 0)-#[test]+#[uzu_test]fn test_edge_small_f32() {test_edge_small::<f32>();}-#[test]+#[uzu_test]fn test_edge_small_f16() {test_edge_small::<f16>();}-#[test]+#[uzu_test]fn test_edge_small_bf16() {test_edge_small::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs b/crates/uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rsindex 3dda9c8b2..cd8c6f89e 100644--- a/crates/uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs+++ b/crates/uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {padded: Box<[T]>,@@ -256,65 +256,65 @@ fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {}// basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// large tests-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}// edge: single token-#[test]+#[uzu_test]fn test_edge_single_token_f32() {test_edge_single_token::<f32>();}-#[test]+#[uzu_test]fn test_edge_single_token_f16() {test_edge_single_token::<f16>();}-#[test]+#[uzu_test]fn test_edge_single_token_bf16() {test_edge_single_token::<bf16>();}// edge: small-#[test]+#[uzu_test]fn test_edge_small_f32() {test_edge_small::<f32>();}-#[test]+#[uzu_test]fn test_edge_small_f16() {test_edge_small::<f16>();}-#[test]+#[uzu_test]fn test_edge_small_bf16() {test_edge_small::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs b/crates/uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rsindex d0a8d6eb1..1bcb2f5c4 100644--- a/crates/uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs+++ b/crates/uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {in_proj: Box<[T]>,@@ -358,81 +358,81 @@ fn test_edge_kernel<T: ArrayElement + Float + Debug + Display>() {}// basic tests (linear chain)-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// large tests-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}// branching trie tests-#[test]+#[uzu_test]fn test_branching_f32() {test_branching::<f32>();}-#[test]+#[uzu_test]fn test_branching_f16() {test_branching::<f16>();}-#[test]+#[uzu_test]fn test_branching_bf16() {test_branching::<bf16>();}// edge: small dimensions-#[test]+#[uzu_test]fn test_edge_small_f32() {test_edge_small::<f32>();}-#[test]+#[uzu_test]fn test_edge_small_f16() {test_edge_small::<f16>();}-#[test]+#[uzu_test]fn test_edge_small_bf16() {test_edge_small::<bf16>();}// edge: kernel_size=1 (no state taps)-#[test]+#[uzu_test]fn test_edge_kernel_f32() {test_edge_kernel::<f32>();}-#[test]+#[uzu_test]fn test_edge_kernel_f16() {test_edge_kernel::<f16>();}-#[test]+#[uzu_test]fn test_edge_kernel_bf16() {test_edge_kernel::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rsindex 40210ca72..2ecbd452d 100644--- a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {x: Box<[T]>,@@ -285,64 +285,64 @@ fn test_tanh_activation<T: ArrayElement + Float + Debug + Display>() {}// f32-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_small_kernel_f32() {test_small_kernel::<f32>();}-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_identity_activation_f32() {test_identity_activation::<f32>();}-#[test]+#[uzu_test]fn test_gelu_activation_f32() {test_gelu_activation::<f32>();}-#[test]+#[uzu_test]fn test_tanh_activation_f32() {test_tanh_activation::<f32>();}// f16-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_small_kernel_f16() {test_small_kernel::<f16>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}// bf16-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}-#[test]+#[uzu_test]fn test_small_kernel_bf16() {test_small_kernel::<bf16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rsindex 29d026dcd..d22412644 100644--- a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {state_in: Box<[T]>,@@ -136,69 +136,69 @@ fn test_channels_lt_row_stride<T: ArrayElement + Float + Debug + Display>() {}// f32-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_single_token_f32() {test_single_token::<f32>();}-#[test]+#[uzu_test]fn test_many_tokens_f32() {test_many_tokens::<f32>();}-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_channels_lt_row_stride_f32() {test_channels_lt_row_stride::<f32>();}// f16-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_single_token_f16() {test_single_token::<f16>();}-#[test]+#[uzu_test]fn test_many_tokens_f16() {test_many_tokens::<f16>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}// bf16-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}-#[test]+#[uzu_test]fn test_single_token_bf16() {test_single_token::<bf16>();}-#[test]+#[uzu_test]fn test_many_tokens_bf16() {test_many_tokens::<bf16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rsindex 9d249dd38..cab75550e 100644--- a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {padded: Box<[T]>,@@ -238,94 +238,94 @@ fn test_tanh_activation<T: ArrayElement + Float + Debug + Display>() {}// f32-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_small_kernel_f32() {test_small_kernel::<f32>();}-#[test]+#[uzu_test]fn test_single_token_f32() {test_single_token::<f32>();}-#[test]+#[uzu_test]fn test_many_tokens_f32() {test_many_tokens::<f32>();}-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_identity_activation_f32() {test_identity_activation::<f32>();}-#[test]+#[uzu_test]fn test_gelu_activation_f32() {test_gelu_activation::<f32>();}-#[test]+#[uzu_test]fn test_tanh_activation_f32() {test_tanh_activation::<f32>();}// f16-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_small_kernel_f16() {test_small_kernel::<f16>();}-#[test]+#[uzu_test]fn test_single_token_f16() {test_single_token::<f16>();}-#[test]+#[uzu_test]fn test_many_tokens_f16() {test_many_tokens::<f16>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}// bf16-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}-#[test]+#[uzu_test]fn test_small_kernel_bf16() {test_small_kernel::<bf16>();}-#[test]+#[uzu_test]fn test_single_token_bf16() {test_single_token::<bf16>();}-#[test]+#[uzu_test]fn test_many_tokens_bf16() {test_many_tokens::<bf16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rsindex dc2b25316..7c22abd1f 100644--- a/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs@@ -16,7 +16,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {x: Box<[T]>,@@ -238,53 +238,53 @@ fn test_shape(}// --- Sequential ----#[test]+#[uzu_test]fn test_sequential_basic() {test_shape(&KernelType::Sequential, 512, 32, 64, 64, 1, "sequential_basic");}-#[test]+#[uzu_test]fn test_sequential_small() {test_shape(&KernelType::Sequential, 4, 4, 4, 8, 1, "sequential_small");}-#[test]+#[uzu_test]fn test_sequential_minimal() {test_shape(&KernelType::Sequential, 1, 1, 1, 1, 1, "sequential_minimal");}-#[test]+#[uzu_test]fn test_sequential_multi_group() {test_shape(&KernelType::Sequential, 8, 8, 4, 16, 4, "sequential_multi_group");}-#[test]+#[uzu_test]fn test_sequential_group_per_head() {test_shape(&KernelType::Sequential, 8, 4, 4, 8, 1, "sequential_group_per_head");}// --- Prefill ----#[test]+#[uzu_test]fn test_prefill_basic() {test_shape(&KernelType::Prefill, 512, 32, 64, 64, 1, "prefill_basic");}-#[test]+#[uzu_test]fn test_prefill_small() {test_shape(&KernelType::Prefill, 4, 4, 4, 8, 1, "prefill_small");}-#[test]+#[uzu_test]fn test_prefill_minimal() {test_shape(&KernelType::Prefill, 1, 1, 1, 1, 1, "prefill_minimal");}-#[test]+#[uzu_test]fn test_prefill_multi_group() {test_shape(&KernelType::Prefill, 8, 8, 4, 16, 4, "prefill_multi_group");}-#[test]+#[uzu_test]fn test_prefill_group_per_head() {test_shape(&KernelType::Prefill, 8, 4, 4, 8, 1, "prefill_group_per_head");}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rsindex e7ed1d3a1..27a69dc06 100644--- a/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {input: Box<[T]>,@@ -172,69 +172,69 @@ fn test_minimal<T: ArrayElement + Float + Debug + Display>() {}// f32-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_single_token_f32() {test_single_token::<f32>();}-#[test]+#[uzu_test]fn test_many_tokens_f32() {test_many_tokens::<f32>();}-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_minimal_f32() {test_minimal::<f32>();}// f16-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_single_token_f16() {test_single_token::<f16>();}-#[test]+#[uzu_test]fn test_many_tokens_f16() {test_many_tokens::<f16>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}// bf16-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}-#[test]+#[uzu_test]fn test_single_token_bf16() {test_single_token::<bf16>();}-#[test]+#[uzu_test]fn test_many_tokens_bf16() {test_many_tokens::<bf16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rsindex 2cd1c4cd8..8b6e3c1d9 100644--- a/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rs+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {x: Box<[T]>,@@ -252,74 +252,74 @@ fn test_group_per_head<T: ArrayElement + Float + Debug + Display>() {}// f32-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_multi_batch_f32() {test_multi_batch::<f32>();}-#[test]+#[uzu_test]fn test_state_in_place_f32() {test_state_in_place::<f32>();}-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_minimal_f32() {test_minimal::<f32>();}-#[test]+#[uzu_test]fn test_group_per_head_f32() {test_group_per_head::<f32>();}// f16-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_multi_batch_f16() {test_multi_batch::<f16>();}-#[test]+#[uzu_test]fn test_state_in_place_f16() {test_state_in_place::<f16>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}// bf16-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}-#[test]+#[uzu_test]fn test_multi_batch_bf16() {test_multi_batch::<bf16>();}-#[test]+#[uzu_test]fn test_state_in_place_bf16() {test_state_in_place::<bf16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rsindex 0b90c6239..fb8e3b6b8 100644--- a/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs+++ b/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs@@ -10,6 +10,8 @@ use uzu::{backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},};+use crate::uzu_test;+struct Input<T: ArrayElement + Float> {input: Box<[T]>,bias: Box<[T]>,@@ -96,32 +98,32 @@ fn test<T: ArrayElement + Float + Debug>(in_place: bool) {});}-#[test]+#[uzu_test]fn test_f32() {test::<f32>(false);}-#[test]+#[uzu_test]fn test_f32_in_place() {test::<f32>(true);}-#[test]+#[uzu_test]fn test_f16() {test::<f16>(false);}-#[test]+#[uzu_test]fn test_f16_in_place() {test::<f16>(true);}-#[test]+#[uzu_test]fn test_bf16() {test::<bf16>(false);}-#[test]+#[uzu_test]fn test_bf16_in_place() {test::<bf16>(true);}diff --git a/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rsindex 9ebcbe6eb..b047f0c60 100644--- a/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rs+++ b/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rs@@ -13,7 +13,7 @@ use uzu::{},};-use crate::common::assert::assert_eq_float;+use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {input: Box<[T]>,@@ -168,49 +168,49 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>() {}// basic tests-#[test]+#[uzu_test]fn test_basic_f32() {test_basic::<f32>();}-#[test]+#[uzu_test]fn test_basic_f16() {test_basic::<f16>();}-#[test]+#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}// large tests-#[test]+#[uzu_test]fn test_large_f32() {test_large::<f32>();}-#[test]+#[uzu_test]fn test_large_f16() {test_large::<f16>();}-#[test]+#[uzu_test]fn test_large_bf16() {test_large::<bf16>();}// edge tests-#[test]+#[uzu_test]fn test_edge_f32() {test_edge::<f32>();}-#[test]+#[uzu_test]fn test_edge_f16() {test_edge::<f16>();}-#[test]+#[uzu_test]fn test_edge_bf16() {test_edge::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rsindex f1179f9e3..8b5aaf620 100644--- a/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs+++ b/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs@@ -7,6 +7,8 @@ use uzu::{backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddSwapKernel},};+use crate::uzu_test;+struct Input<T: ArrayElement + Float> {skip_buffer: Box<[T]>,main_buffer: Box<[T]>,@@ -76,17 +78,17 @@ fn test<T: ArrayElement + Float + Debug>() {});}-#[test]+#[uzu_test]fn test_f32() {test::<f32>();}-#[test]+#[uzu_test]fn test_f16() {test::<f16>();}-#[test]+#[uzu_test]fn test_bf16() {test::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/tensor_copy_test.rs b/crates/uzu/tests/unit/kernel/tensor_copy_test.rsindex 5fb070f58..7c95092cc 100644--- a/crates/uzu/tests/unit/kernel/tensor_copy_test.rs+++ b/crates/uzu/tests/unit/kernel/tensor_copy_test.rs@@ -10,6 +10,8 @@ use uzu::{backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorCopyKernel},};+use crate::uzu_test;+struct Input<T: ArrayElement + Float> {src: Box<[T]>,length: u32,@@ -64,17 +66,17 @@ fn test<T: ArrayElement + Float + Debug>() {});}-#[test]+#[uzu_test]fn test_f32() {test::<f32>();}-#[test]+#[uzu_test]fn test_f16() {test::<f16>();}-#[test]+#[uzu_test]fn test_bf16() {test::<bf16>();}diff --git a/crates/uzu/tests/unit/kernel/test_entry.rs b/crates/uzu/tests/unit/kernel/test_entry.rsnew file mode 100644index 000000000..104ea63cb--- /dev/null+++ b/crates/uzu/tests/unit/kernel/test_entry.rs@@ -0,0 +1,3 @@+use dsl::{__internal_uzu_ignored as uzu_bench, __internal_uzu_test as uzu_test};++include!("mod.rs");diff --git a/crates/uzu/tests/unit/kernel/token_copy_test.rs b/crates/uzu/tests/unit/kernel/token_copy_test.rsindex 7ef483573..f1e9d8c19 100644--- a/crates/uzu/tests/unit/kernel/token_copy_test.rs+++ b/crates/uzu/tests/unit/kernel/token_copy_test.rs@@ -8,6 +8,8 @@ use uzu::{},};+use crate::uzu_test;+fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {let context = B::Context::new().expect("Failed to create Context");@@ -42,7 +44,7 @@ fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());}-#[test]+#[uzu_test]fn test_token_copy_sampled() {for_each_backend!(|B| {test_token_copy_sampled_impl::<B>(0);@@ -51,7 +53,7 @@ fn test_token_copy_sampled() {});}-#[test]+#[uzu_test]fn test_token_copy_to_results() {for_each_backend!(|B| {test_token_copy_to_results_impl::<B>(0);