A high-performance inference engine for AI models
rustfmt-max-width (#164)
Aleksandr Golokoz7bc54256a447
diff --git a/crates/benchmarks/src/runner/helpers/memory_usage.rs b/crates/benchmarks/src/runner/helpers/memory_usage.rsindex 751f9e9e2..e93ebf88c 100644--- a/crates/benchmarks/src/runner/helpers/memory_usage.rs+++ b/crates/benchmarks/src/runner/helpers/memory_usage.rs@@ -22,15 +22,9 @@ pub fn get_memory_usage() -> Option<u64> {suspend_count: 0,};- let mut count: mach_msg_type_number_t =- (mem::size_of::<task_basic_info>() / mem::size_of::<u32>()) as u32;+ let mut count: mach_msg_type_number_t = (mem::size_of::<task_basic_info>() / mem::size_of::<u32>()) as u32;- let result = task_info(- task,- TASK_BASIC_INFO,- &mut info as *mut _ as *mut i32,- &mut count,- );+ let result = task_info(task, TASK_BASIC_INFO, &mut info as *mut _ as *mut i32, &mut count);if result == KERN_SUCCESS {Some(info.resident_size as u64)diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rsindex 9d4232674..a6b0f382d 100644--- a/crates/benchmarks/src/runner/runner.rs+++ b/crates/benchmarks/src/runner/runner.rs@@ -9,9 +9,7 @@ use uzu::{session::{ChatSession,config::{DecodingConfig, RunConfig},- parameter::{- ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy,- },+ parameter::{ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy},types::{Input, Output},},};@@ -40,14 +38,9 @@ impl Runner {}}- fn minimal_context_length(- &self- ) -> Result<usize, Box<dyn std::error::Error>> {+ fn minimal_context_length(&self) -> Result<usize, Box<dyn std::error::Error>> {let input = Input::Messages(self.task.messages.clone());- let mut session = ChatSession::new(- PathBuf::from(self.model_path.clone()),- DecodingConfig::default(),- )?;+ let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()), DecodingConfig::default())?;let run_config = RunConfig::default().tokens_limit(1);let output = session.run(input.clone(),@@ -56,8 +49,7 @@ impl Runner {return true;}),)?;- return Ok(output.stats.total_stats.tokens_count_input as usize- + self.task.tokens_limit as usize);+ return Ok(output.stats.total_stats.tokens_count_input as usize + self.task.tokens_limit as usize);}pub fn run<F>(@@ -69,29 +61,18 @@ impl Runner {{let context_length = self.minimal_context_length()?;- let mut decoding_config = DecodingConfig::default()- .with_context_length(ContextLength::Custom(context_length));+ let mut decoding_config = DecodingConfig::default().with_context_length(ContextLength::Custom(context_length));if let Some(prefill_step_size) = self.prefill_step_size {- decoding_config = decoding_config.with_prefill_step_size(- PrefillStepSize::Custom(prefill_step_size),- );+ decoding_config = decoding_config.with_prefill_step_size(PrefillStepSize::Custom(prefill_step_size));}- let mut session = ChatSession::new(- PathBuf::from(self.model_path.clone()),- decoding_config,- )?;+ let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()), decoding_config)?;- let precision =- session.model_metadata.model_config.as_language_model().map(- |config| {- config- .model_config- .transformer_config- .output_norm_config- .scale_precision- },- );+ let precision = session+ .model_metadata+ .model_config+ .as_language_model()+ .map(|config| config.model_config.transformer_config.output_norm_config.scale_precision);let device = self.get_device_info();@@ -99,16 +80,13 @@ impl Runner {let mut results: Vec<TaskResult> = Vec::new();for run_index in 0..self.task.number_of_runs {- let timestamp =- SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();+ let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();- let mut run_config =- RunConfig::default().tokens_limit(self.task.tokens_limit);+ let mut run_config = RunConfig::default().tokens_limit(self.task.tokens_limit);if self.task.greedy {- run_config =- run_config.sampling_policy(SamplingPolicy::Custom {- value: SamplingMethod::Greedy,- });+ run_config = run_config.sampling_policy(SamplingPolicy::Custom {+ value: SamplingMethod::Greedy,+ });}let output = session.run(input.clone(),@@ -128,27 +106,16 @@ impl Runner {precision,memory_used,tokens_count_input: output.stats.total_stats.tokens_count_input,- tokens_count_output: output- .stats- .total_stats- .tokens_count_output,+ tokens_count_output: output.stats.total_stats.tokens_count_output,time_to_first_token: output.stats.prefill_stats.duration,- prompt_tokens_per_second: output- .stats- .prefill_stats- .processed_tokens_per_second,- generate_tokens_per_second: output- .stats- .generate_stats- .map(|stats| stats.tokens_per_second),+ prompt_tokens_per_second: output.stats.prefill_stats.processed_tokens_per_second,+ generate_tokens_per_second: output.stats.generate_stats.map(|stats| stats.tokens_per_second),text: output.text.original,};results.push(result);if let Some(progress) = progress.as_mut() {- progress(- (run_index + 1) as f64 / self.task.number_of_runs as f64,- );+ progress((run_index + 1) as f64 / self.task.number_of_runs as f64);}}@@ -162,8 +129,7 @@ impl Runner {system_info.refresh_all();let os_name = System::long_os_version();- let cpu_name =- system_info.cpus().first().map(|cpu| cpu.brand().to_string());+ let cpu_name = system_info.cpus().first().map(|cpu| cpu.brand().to_string());let memory_total = system_info.total_memory();let device = Device {diff --git a/crates/cli/src/handlers/bench.rs b/crates/cli/src/handlers/bench.rsindex 4c420383e..f46a7b707 100644--- a/crates/cli/src/handlers/bench.rs+++ b/crates/cli/src/handlers/bench.rs@@ -1,10 +1,7 @@use std::fs;use benchmarks::{runner::Runner, types::Task};-use comfy_table::{- CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS,- presets::UTF8_FULL,-};+use comfy_table::{CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL};use indicatif::ProgressBar;pub fn handle_bench(@@ -27,8 +24,7 @@ pub fn handle_bench(let results_data = serde_json::to_string_pretty(&results)?;fs::write(output_path, results_data)?;- let tokens_count_input =- results.first().expect("No results").tokens_count_input;+ let tokens_count_input = results.first().expect("No results").tokens_count_input;println!("Input tokens count: {}", tokens_count_input);let memory_used_list = results@@ -36,25 +32,16 @@ pub fn handle_bench(.filter_map(|result| result.memory_used).map(|memory_used| (memory_used as f64) / 1024.0 / 1024.0 / 1024.0).collect::<Vec<f64>>();- let time_to_first_token_list = results- .iter()- .map(|result| result.time_to_first_token)- .collect::<Vec<f64>>();- let prompt_tokens_per_second_list = results- .iter()- .map(|result| result.prompt_tokens_per_second)- .collect::<Vec<f64>>();- let generate_tokens_per_second_list = results- .iter()- .filter_map(|result| result.generate_tokens_per_second)- .collect::<Vec<f64>>();+ let time_to_first_token_list = results.iter().map(|result| result.time_to_first_token).collect::<Vec<f64>>();+ let prompt_tokens_per_second_list =+ results.iter().map(|result| result.prompt_tokens_per_second).collect::<Vec<f64>>();+ let generate_tokens_per_second_list =+ results.iter().filter_map(|result| result.generate_tokens_per_second).collect::<Vec<f64>>();let memory_used_metric = calculate_metric(memory_used_list);let time_to_first_token_metric = calculate_metric(time_to_first_token_list);- let prompt_tokens_per_second_metric =- calculate_metric(prompt_tokens_per_second_list);- let generate_tokens_per_second_metric =- calculate_metric(generate_tokens_per_second_list);+ let prompt_tokens_per_second_metric = calculate_metric(prompt_tokens_per_second_list);+ let generate_tokens_per_second_metric = calculate_metric(generate_tokens_per_second_list);let mut table = Table::new();table@@ -65,10 +52,7 @@ pub fn handle_bench(.add_row(vec!["Used memory, GB", memory_used_metric.as_str()]).add_row(vec!["TTFT, s", time_to_first_token_metric.as_str()]).add_row(vec!["Prompt, t/s", prompt_tokens_per_second_metric.as_str()])- .add_row(vec![- "Generate, t/s",- generate_tokens_per_second_metric.as_str(),- ]);+ .add_row(vec!["Generate, t/s", generate_tokens_per_second_metric.as_str()]);let column = table.column_mut(1).expect("Column not found");column.set_cell_alignment(CellAlignment::Right);println!("{table}");diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rsindex 651a8d4c5..feee0c1d5 100644--- a/crates/cli/src/handlers/run.rs+++ b/crates/cli/src/handlers/run.rs@@ -16,18 +16,14 @@ use crate::server::load_session;fn format_output(output: Output) -> String {let stats = &output.stats;- let tokens_per_second = if let Some(generate_stats) = &stats.generate_stats- {+ let tokens_per_second = if let Some(generate_stats) = &stats.generate_stats {generate_stats.tokens_per_second} else {stats.prefill_stats.tokens_per_second};let style_stats = Style::new().bold();- let stats_info = style_stats.apply_to(format!(- "{:.3}s, {:.3}t/s",- stats.total_stats.duration, tokens_per_second,- ));+ let stats_info = style_stats.apply_to(format!("{:.3}s, {:.3}t/s", stats.total_stats.duration, tokens_per_second,));let result = format!("{}\n\n{}", output.text.original, stats_info,);result@@ -76,11 +72,7 @@ pub fn handle_run(let progress_bar_message_limit: usize = 1024;let progress_bar = ProgressBar::new_spinner();progress_bar.enable_steady_tick(std::time::Duration::from_millis(100));- progress_bar.set_style(- ProgressStyle::default_spinner()- .template("{spinner:.green} {msg}")- .unwrap(),- );+ progress_bar.set_style(ProgressStyle::default_spinner().template("{spinner:.green} {msg}").unwrap());let progress_bar_for_progress = progress_bar.clone();let is_model_running_for_progress = is_model_running.clone();@@ -103,12 +95,7 @@ pub fn handle_run(let session_output = match session.run(Input::Text(input.to_string()),- RunConfig::new(- tokens_limit as u64,- true,- SamplingPolicy::Default,- None,- ),+ RunConfig::new(tokens_limit as u64, true, SamplingPolicy::Default, None),Some(session_progress),) {Ok(output) => output,@@ -121,9 +108,7 @@ pub fn handle_run(};let result = format_output(session_output);- progress_bar.set_style(- ProgressStyle::default_spinner().template("{msg}").unwrap(),- );+ progress_bar.set_style(ProgressStyle::default_spinner().template("{msg}").unwrap());progress_bar.finish_and_clear();println!("{}", result);is_model_running.store(false, Ordering::SeqCst);diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rsindex 6aea93b25..3972aaa5a 100644--- a/crates/cli/src/server/chat_completions.rs+++ b/crates/cli/src/server/chat_completions.rs@@ -6,10 +6,7 @@ use uuid::Uuid;use uzu::session::{config::RunConfig,parameter::SamplingPolicy,- types::{- FinishReason, Input, Message, Output, Role, RunStats, Stats, StepStats,- TotalStats,- },+ types::{FinishReason, Input, Message, Output, Role, RunStats, Stats, StepStats, TotalStats},};use crate::server::SessionState;@@ -53,10 +50,7 @@ pub fn handle_chat_completions(println!("π¨ [{}] Incoming chat completion request:", id);println!(" Messages: {} message(s)", request.messages.len());- println!(- " Max tokens: {}",- request.max_completion_tokens.unwrap_or(2048)- );+ println!(" Max tokens: {}", request.max_completion_tokens.unwrap_or(2048));for (i, msg) in request.messages.iter().enumerate() {let content_preview = if msg.content.len() > 10000 {@@ -79,58 +73,53 @@ pub fn handle_chat_completions(}).collect();let input = Input::Messages(messages);- let run_config =- RunConfig::new(tokens_limit, true, SamplingPolicy::Default, None);+ let run_config = RunConfig::new(tokens_limit, true, SamplingPolicy::Default, None);let start_time = std::time::Instant::now();let mut session = state.session_wrapper.lock();- let output =- match session.run(input, run_config, None::<fn(Output) -> bool>) {- Ok(output) => output,- Err(e) => {- eprintln!("β [{}] Session run error: {}", id, e);- // Return error response- let error_response = ChatCompletionResponse {- id: id.clone(),- object: "chat.completion".to_string(),- created: SystemTime::now()- .duration_since(UNIX_EPOCH)- .unwrap()- .as_secs() as i64,- model: model_name.clone(),- choices: vec![ChatCompletionChoice {- index: 0,- message: ChatMessage {- content: format!("Error: {}", e),- role: Role::Assistant,- },- finish_reason: "error".to_string(),- }],- stats: Stats {- prefill_stats: StepStats {- duration: 0.0,- suffix_length: 0,- tokens_count: 0,- tokens_per_second: 0.0,- processed_tokens_per_second: 0.0,- model_run: RunStats {- count: 0,- average_duration: 0.0,- },- run: None,- },- generate_stats: None,- total_stats: TotalStats {- duration: 0.0,- tokens_count_input: 0,- tokens_count_output: 0,+ let output = match session.run(input, run_config, None::<fn(Output) -> bool>) {+ Ok(output) => output,+ Err(e) => {+ eprintln!("β [{}] Session run error: {}", id, e);+ // Return error response+ let error_response = ChatCompletionResponse {+ id: id.clone(),+ object: "chat.completion".to_string(),+ created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64,+ model: model_name.clone(),+ choices: vec![ChatCompletionChoice {+ index: 0,+ message: ChatMessage {+ content: format!("Error: {}", e),+ role: Role::Assistant,+ },+ finish_reason: "error".to_string(),+ }],+ stats: Stats {+ prefill_stats: StepStats {+ duration: 0.0,+ suffix_length: 0,+ tokens_count: 0,+ tokens_per_second: 0.0,+ processed_tokens_per_second: 0.0,+ model_run: RunStats {+ count: 0,+ average_duration: 0.0,},+ run: None,},- };- return Json(error_response);- },- };+ generate_stats: None,+ total_stats: TotalStats {+ duration: 0.0,+ tokens_count_input: 0,+ tokens_count_output: 0,+ },+ },+ };+ return Json(error_response);+ },+ };let processing_time = start_time.elapsed();@@ -149,14 +138,7 @@ pub fn handle_chat_completions(println!(" Response length: {} chars", text.original.len());println!(" Finish reason: {:?}", finish_reason_val);println!(" Processing time: {:.3}s", processing_time.as_secs_f64());- println!(- " Tokens/second: {:.1}",- stats- .generate_stats- .as_ref()- .map(|s| s.tokens_per_second)- .unwrap_or(0.0)- );+ println!(" Tokens/second: {:.1}", stats.generate_stats.as_ref().map(|s| s.tokens_per_second).unwrap_or(0.0));print!(" Response preview: ");let text = text.original.clone();@@ -171,8 +153,7 @@ pub fn handle_chat_completions(let response = ChatCompletionResponse {id: id.clone(),object: "chat.completion".to_string(),- created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()- as i64,+ created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64,model: model_name,choices: vec![ChatCompletionChoice {index: 0,diff --git a/crates/cli/src/server/main.rs b/crates/cli/src/server/main.rsindex 3a2bc8ba0..6a94ccfdb 100644--- a/crates/cli/src/server/main.rs+++ b/crates/cli/src/server/main.rs@@ -3,9 +3,7 @@ use std::path::PathBuf;use log::LevelFilter;use rocket::{Config, config::LogLevel, log::private as log, routes};-use crate::server::{- SessionState, SessionWrapper, handle_chat_completions, load_session,-};+use crate::server::{SessionState, SessionWrapper, handle_chat_completions, load_session};struct SilentLogger;static SILENT_LOGGER: SilentLogger = SilentLogger;@@ -30,8 +28,7 @@ pub async fn run_server(prefill_step_size: Option<usize>,) {// Install the silent logger **before** Rocket initializes its own logger.- let _ = log::set_logger(&SILENT_LOGGER)- .map(|_| log::set_max_level(LevelFilter::Off));+ let _ = log::set_logger(&SILENT_LOGGER).map(|_| log::set_max_level(LevelFilter::Off));let config = Config {workers: 1,@@ -39,22 +36,12 @@ pub async fn run_server(..Config::default()};- let model_name = PathBuf::from(model_path.clone())- .file_name()- .unwrap()- .to_str()- .unwrap()- .to_string();+ let model_name = PathBuf::from(model_path.clone()).file_name().unwrap().to_str().unwrap().to_string();println!("π Starting server with model: {}", model_name);println!("π Model path: {}", model_path);- println!(- "π Server will be available at: http://localhost:{}",- config.port- );- println!(- "π Endpoints:\n POST /chat/completions - Chat completions API\n"- );+ println!("π Server will be available at: http://localhost:{}", config.port);+ println!("π Endpoints:\n POST /chat/completions - Chat completions API\n");let session = load_session(model_path, prefill_step_size, None);let state = SessionState {@@ -62,9 +49,5 @@ pub async fn run_server(session_wrapper: SessionWrapper::new(session),};- let _ = rocket::custom(config)- .manage(state)- .mount("/", routes![handle_chat_completions])- .launch()- .await;+ let _ = rocket::custom(config).manage(state).mount("/", routes![handle_chat_completions]).launch().await;}diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rsindex 53717e3bc..117abcae2 100644--- a/crates/cli/src/server/state.rs+++ b/crates/cli/src/server/state.rs@@ -36,19 +36,11 @@ pub fn load_session(let style_bold = Style::new().bold();let model_path_buf = PathBuf::from(model_path);- let model_name = style_bold- .apply_to(- model_path_buf.file_name().unwrap().to_str().unwrap().to_string(),- )- .to_string();+ let model_name = style_bold.apply_to(model_path_buf.file_name().unwrap().to_str().unwrap().to_string()).to_string();let progress_bar = ProgressBar::new_spinner();progress_bar.enable_steady_tick(std::time::Duration::from_millis(100));- progress_bar.set_style(- ProgressStyle::default_spinner()- .template("{spinner:.green} Loading: {msg}")- .unwrap(),- );+ progress_bar.set_style(ProgressStyle::default_spinner().template("{spinner:.green} Loading: {msg}").unwrap());progress_bar.set_message(model_name.clone());let prefill_step_size_config: PrefillStepSize;@@ -58,18 +50,14 @@ pub fn load_session(prefill_step_size_config = PrefillStepSize::Default;}- let decoding_config = DecodingConfig::default()- .with_prefill_step_size(prefill_step_size_config)- .with_sampling_seed(match seed {+ let decoding_config =+ DecodingConfig::default().with_prefill_step_size(prefill_step_size_config).with_sampling_seed(match seed {Some(seed) => SamplingSeed::Custom(seed),None => SamplingSeed::Default,});- let session = Session::new(model_path_buf, decoding_config)- .expect("Failed to create session");+ let session = Session::new(model_path_buf, decoding_config).expect("Failed to create session");- progress_bar.set_style(- ProgressStyle::default_spinner().template("Loaded: {msg}").unwrap(),- );+ progress_bar.set_style(ProgressStyle::default_spinner().template("Loaded: {msg}").unwrap());progress_bar.finish_with_message(model_name.clone());sessiondiff --git a/crates/uzu/build/common/caching.rs b/crates/uzu/build/common/caching.rsindex 84e1b498f..e992ae939 100644--- a/crates/uzu/build/common/caching.rs+++ b/crates/uzu/build/common/caching.rs@@ -8,11 +8,8 @@ pub fn build_system_hash() -> anyhow::Result<&'static blake3::Hash> {Ok(bsh)} else {let bsh = blake3::hash(- &fs::read(- &std::env::current_exe()- .context("cannot get build system executable")?,- )- .context("cannot read build system executable")?,+ &fs::read(&std::env::current_exe().context("cannot get build system executable")?)+ .context("cannot read build system executable")?,);Ok(BUILD_SYSTEM_HASH.get_or_init(|| bsh))diff --git a/crates/uzu/build/common/codegen.rs b/crates/uzu/build/common/codegen.rsindex da55239a6..5b8a97cd2 100644--- a/crates/uzu/build/common/codegen.rs+++ b/crates/uzu/build/common/codegen.rs@@ -10,15 +10,11 @@ pub fn write_tokens(let tokens = tokens.into();let file = file.as_ref();- let parsed = syn::parse2(tokens.into())- .context("cannot parse generated bindings")?;+ let parsed = syn::parse2(tokens.into()).context("cannot parse generated bindings")?;fs::write(&file, prettyplease::unparse(&parsed)).with_context(|| format!("cannot write file {}", file.display()))?;- std::process::Command::new("rustfmt")- .arg(&file)- .status()- .context("rustfmt failed")?;+ std::process::Command::new("rustfmt").arg(&file).status().context("rustfmt failed")?;Ok(())}diff --git a/crates/uzu/build/common/compiler.rs b/crates/uzu/build/common/compiler.rsindex 109c81b17..9ffc5b0a7 100644--- a/crates/uzu/build/common/compiler.rs+++ b/crates/uzu/build/common/compiler.rs@@ -6,7 +6,5 @@ use super::kernel::Kernel;#[async_trait]pub trait Compiler {- async fn build(- &self- ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>;+ async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>;}diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rsindex a1291fc4f..29a89a5bb 100644--- a/crates/uzu/build/common/traitgen.rs+++ b/crates/uzu/build/common/traitgen.rs@@ -56,8 +56,7 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {}).collect::<(Vec<_>, Vec<_>)>();- let encode_generics =- encode_generics.into_iter().flatten().collect::<Vec<_>>();+ let encode_generics = encode_generics.into_iter().flatten().collect::<Vec<_>>();let kernel_trait = quote! {pub trait #trait_name: Sized {@@ -70,17 +69,13 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {}};- let kernel_type =- quote! { type #trait_name: #trait_name<Backend = Self::Backend>; };+ let kernel_type = quote! { type #trait_name: #trait_name<Backend = Self::Backend>; };(kernel_trait, kernel_type)}-pub fn traitgen_all(- backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>-) -> anyhow::Result<()> {- let out_dir =- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);+pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>) -> anyhow::Result<()> {+ let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);let mut kernels: HashMap<Box<[Box<str>]>, Box<[Kernel]>> = HashMap::new();@@ -99,9 +94,7 @@ pub fn traitgen_all(let mut kernel_traits = Vec::new();let mut kernel_types = Vec::new();- for (_file_path, file_kernels) in- kernels.into_iter().sorted_by_key(|(p, _k)| p.join("::"))- {+ for (_file_path, file_kernels) in kernels.into_iter().sorted_by_key(|(p, _k)| p.join("::")) {for (tr, ty) in file_kernels.iter().map(traitgen) {kernel_traits.push(tr);kernel_types.push(ty);@@ -137,8 +130,7 @@ pub fn traitgen_all(}};- write_tokens(kernel_traits, out_dir.join("traits.rs"))- .context("cannot write kernel traits")?;+ write_tokens(kernel_traits, out_dir.join("traits.rs")).context("cannot write kernel traits")?;Ok(())}diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rsindex 19baafcdc..080e2b45a 100644--- a/crates/uzu/build/gpu_types/mod.rs+++ b/crates/uzu/build/gpu_types/mod.rs@@ -2,10 +2,7 @@ use std::{collections::HashMap, env, fs, path::PathBuf};use anyhow::Context;use async_trait::async_trait;-use syn::{- Expr, ExprLit, Fields, Item, ItemStruct, Lit, Meta, Type, TypeArray,- TypePath,-};+use syn::{Expr, ExprLit, Fields, Item, ItemStruct, Lit, Meta, Type, TypeArray, TypePath};use tokio::task::spawn_blocking;use crate::common::{compiler::Compiler, kernel::Kernel};@@ -20,22 +17,13 @@ pub struct GpuTypesCompiler {impl GpuTypesCompiler {pub fn new() -> anyhow::Result<Self> {- let crate_dir = PathBuf::from(- env::var("CARGO_MANIFEST_DIR")- .context("missing CARGO_MANIFEST_DIR")?,- );+ let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?);let generated_dir = crate_dir.join("src/backends/metal/generated");- fs::create_dir_all(&generated_dir).with_context(|| {- format!("cannot create {}", generated_dir.display())- })?;+ fs::create_dir_all(&generated_dir).with_context(|| format!("cannot create {}", generated_dir.display()))?;- let build_dir =- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?)- .join("gpu_types");- fs::create_dir_all(&build_dir).with_context(|| {- format!("cannot create {}", build_dir.display())- })?;+ let build_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?).join("gpu_types");+ fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;Ok(Self {crate_dir,@@ -49,25 +37,20 @@ impl GpuTypesCompiler {}fn generate(&self) -> anyhow::Result<()> {- let gpu_types_dir =- self.crate_dir.join("src/backends/common/gpu_types");+ let gpu_types_dir = self.crate_dir.join("src/backends/common/gpu_types");let source_files: Vec<PathBuf> = fs::read_dir(&gpu_types_dir)- .with_context(|| {- format!("cannot read {}", gpu_types_dir.display())- })?+ .with_context(|| format!("cannot read {}", gpu_types_dir.display()))?.filter_map(|entry| entry.ok()).map(|entry| entry.path()).filter(|path| {- path.extension().map(|ext| ext == "rs").unwrap_or(false)- && path.file_stem() != Some("mod".as_ref())+ path.extension().map(|ext| ext == "rs").unwrap_or(false) && path.file_stem() != Some("mod".as_ref())}).collect();let mut hasher = blake3::Hasher::new();for path in &source_files {- let content = fs::read(path)- .with_context(|| format!("cannot read {}", path.display()))?;+ let content = fs::read(path).with_context(|| format!("cannot read {}", path.display()))?;hasher.update(&content);}@@ -87,9 +70,8 @@ impl GpuTypesCompiler {self.generate_header_for_file(src_path)?;}- fs::write(&cached_hash_path, &source_hash).with_context(|| {- format!("cannot write {}", cached_hash_path.display())- })?;+ fs::write(&cached_hash_path, &source_hash)+ .with_context(|| format!("cannot write {}", cached_hash_path.display()))?;debug_log!("gpu_types generation done");@@ -100,24 +82,18 @@ impl GpuTypesCompiler {&self,src_path: &PathBuf,) -> anyhow::Result<()> {- let module_name = src_path- .file_stem()- .and_then(|s| s.to_str())- .context("invalid source file name")?;+ let module_name = src_path.file_stem().and_then(|s| s.to_str()).context("invalid source file name")?;- let source = fs::read_to_string(src_path)- .with_context(|| format!("cannot read {}", src_path.display()))?;+ let source = fs::read_to_string(src_path).with_context(|| format!("cannot read {}", src_path.display()))?;- let syntax = syn::parse_file(&source)- .with_context(|| format!("cannot parse {}", src_path.display()))?;+ let syntax = syn::parse_file(&source).with_context(|| format!("cannot parse {}", src_path.display()))?;let mut repr_c_structs: Vec<ItemStruct> = Vec::new();for item in syntax.items {if let Item::Struct(item_struct) = item {let has_repr_c = item_struct.attrs.iter().any(|attribute| {if attribute.path().is_ident("repr") {- if let Ok(ident) = attribute.parse_args::<syn::Ident>()- {+ if let Ok(ident) = attribute.parse_args::<syn::Ident>() {return ident == "C";}}@@ -156,18 +132,12 @@ impl GpuTypesCompiler {if let Fields::Named(named_fields) = &rust_struct.fields {for field in &named_fields.named {- let field_name = field- .ident- .as_ref()- .map(|ident| ident.to_string())- .unwrap_or_default();+ let field_name = field.ident.as_ref().map(|ident| ident.to_string()).unwrap_or_default();let c_type = rust_type_to_c(&field.ty);for attribute in &field.attrs {if attribute.path().is_ident("doc") {- if let Meta::NameValue(meta_name_value) =- &attribute.meta- {+ if let Meta::NameValue(meta_name_value) = &attribute.meta {if let Expr::Lit(ExprLit {lit: Lit::Str(literal_string),..@@ -177,8 +147,7 @@ impl GpuTypesCompiler {let doc_comment = doc_comment.trim();if !doc_comment.is_empty() {c_struct_definitions.push_str(" /**");- c_struct_definitions- .push_str(doc_comment);+ c_struct_definitions.push_str(doc_comment);c_struct_definitions.push_str(" */\n");}}@@ -186,15 +155,10 @@ impl GpuTypesCompiler {}}- if let Some((base_type, array_size)) =- c_type.split_once('[')- {- c_struct_definitions.push_str(&format!(- " {base_type} {field_name}[{array_size};\n"- ));+ if let Some((base_type, array_size)) = c_type.split_once('[') {+ c_struct_definitions.push_str(&format!(" {base_type} {field_name}[{array_size};\n"));} else {- c_struct_definitions- .push_str(&format!(" {c_type} {field_name};\n"));+ c_struct_definitions.push_str(&format!(" {c_type} {field_name};\n"));}}}@@ -244,9 +208,7 @@ namespace {module_name} {{);let output_path = self.generated_dir.join(format!("{module_name}.h"));- fs::write(&output_path, &header_content).with_context(|| {- format!("cannot write {}", output_path.display())- })?;+ fs::write(&output_path, &header_content).with_context(|| format!("cannot write {}", output_path.display()))?;debug_log!(" -> {}", output_path.display());@@ -303,9 +265,7 @@ fn rust_type_to_c(rust_type: &Type) -> String {#[async_trait]impl Compiler for GpuTypesCompiler {- async fn build(- &self- ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {+ async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {let crate_dir = self.crate_dir.clone();let generated_dir = self.generated_dir.clone();let build_dir = self.build_dir.clone();diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rsindex 8eab62ce7..bc25ca1bd 100644--- a/crates/uzu/build/main.rs+++ b/crates/uzu/build/main.rs@@ -16,42 +16,31 @@ async fn main() -> anyhow::Result<()> {println!("cargo::rerun-if-changed=.");if envs::build_always() {- println!(- "cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun"- );+ println!("cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun");}debug_log!("build script started");if envs::build_clean() {- let out_dir =- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);+ let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);if out_dir.exists() {- fs::remove_dir_all(&out_dir).with_context(|| {- format!("cannot clean {}", out_dir.display())- })?;- fs::create_dir_all(&out_dir).with_context(|| {- format!("cannot recreate {}", out_dir.display())- })?;+ fs::remove_dir_all(&out_dir).with_context(|| format!("cannot clean {}", out_dir.display()))?;+ fs::create_dir_all(&out_dir).with_context(|| format!("cannot recreate {}", out_dir.display()))?;}debug_log!("cleaned caches");}let gpu_types_compiler = gpu_types::GpuTypesCompiler::new()?;- let generated_header_dir =- gpu_types_compiler.generated_header_dir().clone();+ let generated_header_dir = gpu_types_compiler.generated_header_dir().clone();gpu_types_compiler.build().await?;debug_log!("gpu_types build done");let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();#[cfg(feature = "metal")]- compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(- generated_header_dir,- )?));+ compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));- let backends_kernels =- try_join_all(compilers.iter().map(|c| c.build())).await?;+ let backends_kernels = try_join_all(compilers.iter().map(|c| c.build())).await?;debug_log!("backend build end");diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rsindex 84d2f09fa..f61fc77d5 100644--- a/crates/uzu/build/metal/ast.rs+++ b/crates/uzu/build/metal/ast.rs@@ -1,10 +1,7 @@use anyhow::{Context, bail};use serde::{Deserialize, Serialize};-use crate::common::kernel::{- Kernel, KernelArgument, KernelArgumentType, KernelParameter,- KernelParameterType,-};+use crate::common::kernel::{Kernel, KernelArgument, KernelArgumentType, KernelParameter, KernelParameterType};pub type MetalAstNode = clang_ast::Node<MetalAstKind>;@@ -43,9 +40,7 @@ pub enum MetalAstKind {Other,}-fn annotation_from_ast_node(- annotation_node: MetalAstNode-) -> anyhow::Result<Box<[Box<str>]>> {+fn annotation_from_ast_node(annotation_node: MetalAstNode) -> anyhow::Result<Box<[Box<str>]>> {if !matches!(annotation_node.kind, MetalAstKind::AnnotateAttr) {bail!("unexpected kind of root node: MetalAstKind::AnnotateAttr expected, but {:?} found",@@ -62,26 +57,17 @@ fn annotation_from_ast_node(};if constant_expr.inner.len() != 1 {- bail!(- "ConstantExpr must have exactly one child, found {}",- constant_expr.inner.len()- );+ bail!("ConstantExpr must have exactly one child, found {}", constant_expr.inner.len());}let mut implicit_cast_expr = constant_expr.inner.pop().unwrap();let MetalAstKind::ImplicitCastExpr = implicit_cast_expr.kind else {- bail!(- "expected ImplicitCastExpr, found {:?}",- implicit_cast_expr.kind- );+ bail!("expected ImplicitCastExpr, found {:?}", implicit_cast_expr.kind);};if implicit_cast_expr.inner.len() != 1 {- bail!(- "ImplicitCastExpr must have exactly one child, found {}",- implicit_cast_expr.inner.len()- );+ bail!("ImplicitCastExpr must have exactly one child, found {}", implicit_cast_expr.inner.len());}let string_literal = implicit_cast_expr.inner.pop().unwrap();@@ -90,15 +76,11 @@ fn annotation_from_ast_node(value,} = string_literal.kindelse {- bail!(- "expected StringLiteral, found {:?}",- string_literal.kind- );+ bail!("expected StringLiteral, found {:?}", string_literal.kind);};// NOTE: string literal includes "" (and is probably not parsed?), using json parse here for now- Ok(serde_json::from_str(&value)- .context("failed to parse string literal")?)+ Ok(serde_json::from_str(&value).context("failed to parse string literal")?)}).collect()}@@ -145,9 +127,9 @@ impl MetalArgument {["uint"] | ["uint32_t"] | ["unsigned", "int"] => Ok("u32".into()),["int"] | ["int32_t"] => Ok("i32".into()),["float"] => Ok("f32".into()),- [vpath] if vpath.starts_with("uzu::") => Ok(vpath- .replacen("uzu::", "crate::backends::common::gpu_types::", 1)- .into()),+ [vpath] if vpath.starts_with("uzu::") => {+ Ok(vpath.replacen("uzu::", "crate::backends::common::gpu_types::", 1).into())+ },_ => bail!("unknown scalar type: {c_type}"),}}@@ -159,11 +141,10 @@ impl MetalArgument {let MetalAstKind::ParmVarDecl {name,range,- ty:- ParmVarDeclType {- qual_type,- desugared_qual_type,- },+ ty: ParmVarDeclType {+ qual_type,+ desugared_qual_type,+ },} = argument_node.kindelse {bail!("argument isn't ParmVarDecl: {:?}", argument_node.kind);@@ -177,27 +158,16 @@ impl MetalArgument {bail!("more than one annotation on argument ast node");}- let annotation =- if let Some(annotation_node) = argument_node.inner.first() {- Some(annotation_from_ast_node(annotation_node.clone())?)- } else {- None- };+ let annotation = if let Some(annotation_node) = argument_node.inner.first() {+ Some(annotation_from_ast_node(annotation_node.clone())?)+ } else {+ None+ };- let start_offset = range- .begin- .spelling_loc- .context("no start location in source range")?- .offset;- let end_offset = range- .end- .spelling_loc- .context("no end location in source range")?- .offset;+ let start_offset = range.begin.spelling_loc.context("no start location in source range")?.offset;+ let end_offset = range.end.spelling_loc.context("no end location in source range")?.offset;let source =- str::from_utf8(&source.as_bytes()[start_offset..=end_offset])- .context("source range is not utf-8")?- .into();+ str::from_utf8(&source.as_bytes()[start_offset..=end_offset]).context("source range is not utf-8")?.into();Ok(Self {name,@@ -219,58 +189,36 @@ impl MetalArgument {match &*annotation_key {"dsl.specialize" => {if !annotation.is_empty() {- bail!(- "dsl.specialize takes no arguments, got {}",- annotation.len()- );+ bail!("dsl.specialize takes no arguments, got {}", annotation.len());}let rust_type = Self::scalar_type_to_rust(&self.c_type)?;Ok(MetalArgumentType::Specialize(rust_type.into()))},"dsl.axis" => {if annotation.len() != 2 {- bail!(- "dsl.axis requires 2 arguments, got {}",- annotation.len()- );+ bail!("dsl.axis requires 2 arguments, got {}", annotation.len());}- Ok(MetalArgumentType::Axis(- annotation.remove(0),- annotation.remove(0),- ))+ Ok(MetalArgumentType::Axis(annotation.remove(0), annotation.remove(0)))},"dsl.groups" => {if annotation.len() != 1 {- bail!(- "dsl.groups requires 1 argument, got {}",- annotation.len()- );+ bail!("dsl.groups requires 1 argument, got {}", annotation.len());}let dim = annotation.remove(0);match dim.as_ref() {- "INDIRECT" => Ok(MetalArgumentType::Groups(- MetalGroupsType::Indirect,- )),- _ => Ok(MetalArgumentType::Groups(- MetalGroupsType::Direct(dim),- )),+ "INDIRECT" => Ok(MetalArgumentType::Groups(MetalGroupsType::Indirect)),+ _ => Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(dim))),}},"dsl.threads" => {if annotation.len() != 1 {- bail!(- "dsl.threads requires 1 argument, got {}",- annotation.len()- );+ bail!("dsl.threads requires 1 argument, got {}", annotation.len());}Ok(MetalArgumentType::Threads(annotation.remove(0)))},_ => bail!("unknown annotation: {annotation_key}"),}- } else if self.c_type.contains("device")- && self.c_type.contains('*')- && !self.c_type.contains('&')- {+ } else if self.c_type.contains("device") && self.c_type.contains('*') && !self.c_type.contains('&') {Ok(MetalArgumentType::Buffer)} else if let ["const", "constant", c_type_scalar, "&"] =self.c_type.split_whitespace().collect::<Vec<_>>().as_slice()@@ -286,23 +234,12 @@ impl MetalArgument {Self::scalar_type_to_rust(c_type_scalar)?.into(),MetalConstantType::Array,)))- } else if self.c_type.contains("threadgroup")- && self.c_type.contains('*')- {- let lbracket = self- .source- .rfind('[')- .context("threadgroup missing size bracket")?- + 1;- let rbracket = self- .source- .rfind(']')- .context("threadgroup missing size bracket")?;+ } else if self.c_type.contains("threadgroup") && self.c_type.contains('*') {+ let lbracket = self.source.rfind('[').context("threadgroup missing size bracket")? + 1;+ let rbracket = self.source.rfind(']').context("threadgroup missing size bracket")?;let size_expr = &self.source[lbracket..rbracket];Ok(MetalArgumentType::Shared(Some(size_expr.into())))- } else if self.c_type.contains("threadgroup")- && self.c_type.contains('&')- {+ } else if self.c_type.contains("threadgroup") && self.c_type.contains('&') {Ok(MetalArgumentType::Shared(None))} else {bail!("cannot parse c type: {}", self.c_type);@@ -344,39 +281,27 @@ pub struct MetalKernelInfo {impl MetalKernelInfo {pub fn has_axis(&self) -> bool {- self.arguments.iter().any(|a| {- matches!(a.argument_type(), Ok(MetalArgumentType::Axis(..)))- })+ self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Axis(..))))}pub fn has_groups(&self) -> bool {- self.arguments.iter().any(|a| {- matches!(a.argument_type(), Ok(MetalArgumentType::Groups(_)))- })+ self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Groups(_))))}pub fn has_groups_direct(&self) -> bool {- self.arguments.iter().any(|a| {- matches!(- a.argument_type(),- Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(_)))- )- })+ self.arguments+ .iter()+ .any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(_)))))}pub fn has_groups_indirect(&self) -> bool {- self.arguments.iter().any(|a| {- matches!(- a.argument_type(),- Ok(MetalArgumentType::Groups(MetalGroupsType::Indirect))- )- })+ self.arguments+ .iter()+ .any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Groups(MetalGroupsType::Indirect))))}pub fn has_threads(&self) -> bool {- self.arguments.iter().any(|a| {- matches!(a.argument_type(), Ok(MetalArgumentType::Threads(_)))- })+ self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Threads(_))))}pub fn to_kernel(&self) -> Kernel {@@ -400,26 +325,18 @@ impl MetalKernelInfo {name: a.name.clone(),ty: KernelArgumentType::Buffer,}),- Ok(MetalArgumentType::Groups(- MetalGroupsType::Indirect,- )) if !indirect_flag => {+ Ok(MetalArgumentType::Groups(MetalGroupsType::Indirect)) if !indirect_flag => {indirect_flag = true;Some(KernelArgument {name: "__dsl_indirect_dispatch_buffer".into(),ty: KernelArgumentType::Buffer,})},- Ok(MetalArgumentType::Constant((- ty,- MetalConstantType::Scalar,- ))) => Some(KernelArgument {+ Ok(MetalArgumentType::Constant((ty, MetalConstantType::Scalar))) => Some(KernelArgument {name: a.name.clone(),ty: KernelArgumentType::Scalar(ty),}),- Ok(MetalArgumentType::Constant((- ty,- MetalConstantType::Array,- ))) => Some(KernelArgument {+ Ok(MetalArgumentType::Constant((ty, MetalConstantType::Array))) => Some(KernelArgument {name: a.name.clone(),ty: KernelArgumentType::Constant(ty),}),@@ -435,48 +352,41 @@ impl MetalKernelInfo {node: MetalAstNode,source: &str,) -> anyhow::Result<Option<Self>> {- let (is_template, template_parameters, node) =- if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {- let mut template_parameters = Vec::new();- let mut function_node = None;-- for child in node.inner {- match child.kind {- MetalAstKind::TemplateTypeParmDecl {- name,- } => {- let name = name- .context("template parameter missing name")?;- template_parameters.push(name);- },- MetalAstKind::FunctionDecl {- name: _,- } => {- function_node = Some(child);- },- _ => (),- }+ let (is_template, template_parameters, node) = if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {+ let mut template_parameters = Vec::new();+ let mut function_node = None;++ for child in node.inner {+ match child.kind {+ MetalAstKind::TemplateTypeParmDecl {+ name,+ } => {+ let name = name.context("template parameter missing name")?;+ template_parameters.push(name);+ },+ MetalAstKind::FunctionDecl {+ name: _,+ } => {+ function_node = Some(child);+ },+ _ => (),}+ }- let node = function_node.context(- "unexpected kind of root node: template without function",- )?;+ let node = function_node.context("unexpected kind of root node: template without function")?;- (true, template_parameters, node)- } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {- (false, Vec::new(), node)- } else {- return Ok(None);- };+ (true, template_parameters, node)+ } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {+ (false, Vec::new(), node)+ } else {+ return Ok(None);+ };let MetalAstKind::FunctionDecl {name,} = node.kindelse {- bail!(- "unexpected kind of root node: function expected, but {:?} found",- node.kind- );+ bail!("unexpected kind of root node: function expected, but {:?} found", node.kind);};let mut arg_nodes = Vec::new();@@ -489,9 +399,7 @@ impl MetalKernelInfo {range: _,ty: _,} => arg_nodes.push(node),- MetalAstKind::AnnotateAttr => {- annotations.push(annotation_from_ast_node(node)?)- },+ MetalAstKind::AnnotateAttr => annotations.push(annotation_from_ast_node(node)?),_ => (),}}@@ -520,10 +428,7 @@ impl MetalKernelInfo {bail!("malformed dsl.variants annotation");};- let variants = variant_values- .split(',')- .map(|v| v.trim().into())- .collect::<Box<[Box<str>]>>();+ let variants = variant_values.split(',').map(|v| v.trim().into()).collect::<Box<[Box<str>]>>();Ok(MetalTypeParameter {name: typename.clone(),@@ -537,20 +442,10 @@ impl MetalKernelInfo {}if is_template {- let template_names = template_parameters- .iter()- .map(|name| name.as_ref())- .collect::<Vec<_>>();- let variant_names = variants- .iter()- .map(|variant| variant.name.as_ref())- .collect::<Vec<_>>();+ let template_names = template_parameters.iter().map(|name| name.as_ref()).collect::<Vec<_>>();+ let variant_names = variants.iter().map(|variant| variant.name.as_ref()).collect::<Vec<_>>();if template_names != variant_names {- bail!(- "template parameters {:?} do not match dsl.variants order {:?}",- template_names,- variant_names- );+ bail!("template parameters {:?} do not match dsl.variants order {:?}", template_names, variant_names);}}@@ -574,25 +469,24 @@ impl MetalKernelInfo {}pub fn validate_raw_kernel(node: &MetalAstNode) -> anyhow::Result<()> {- let (node, is_template) =- if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {- let inner_fn = node.inner.iter().find(|c| {- matches!(- c.kind,- MetalAstKind::FunctionDecl {- name: _- }- )- });- match inner_fn {- Some(n) => (n, true),- None => return Ok(()),- }- } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {- (node, false)- } else {- return Ok(());- };+ let (node, is_template) = if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {+ let inner_fn = node.inner.iter().find(|c| {+ matches!(+ c.kind,+ MetalAstKind::FunctionDecl {+ name: _+ }+ )+ });+ match inner_fn {+ Some(n) => (n, true),+ None => return Ok(()),+ }+ } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {+ (node, false)+ } else {+ return Ok(());+ };let MetalAstKind::FunctionDecl {name,@@ -608,15 +502,10 @@ pub fn validate_raw_kernel(node: &MetalAstNode) -> anyhow::Result<()> {for child in &node.inner {match &child.kind {MetalAstKind::MetalKernelAttr => has_metal_kernel_attr = true,- MetalAstKind::MetalMaxTotalThreadsPerThreadGroupAttr => {- has_max_threads_attr = true- },+ MetalAstKind::MetalMaxTotalThreadsPerThreadGroupAttr => has_max_threads_attr = true,MetalAstKind::AnnotateAttr => {- if let Ok(annotation) = annotation_from_ast_node(child.clone())- {- if annotation.first().map(|s| s.as_ref())- == Some("dsl.kernel")- {+ if let Ok(annotation) = annotation_from_ast_node(child.clone()) {+ if annotation.first().map(|s| s.as_ref()) == Some("dsl.kernel") {has_dsl_kernel = true;}}diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex 9b556c8da..2f9e34446 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -22,33 +22,18 @@ pub fn bindgen(let struct_name = format_ident!("{kernel_name}MetalKernel");let parse_expr = |expr: &Box<str>| -> anyhow::Result<TokenStream> {- syn::parse_str(expr.as_ref()).with_context(|| {- format!(- "cannot parse rust expression `{}` in kernel `{}`",- expr, kernel_name- )- })+ syn::parse_str(expr.as_ref())+ .with_context(|| format!("cannot parse rust expression `{}` in kernel `{}`", expr, kernel_name))};- let (variants_extra_arguments, variants_kernel_format) = if let Some(- variants,- ) =- &kernel.variants- {- let variant_names = variants- .iter()- .map(|type_parameter| {- format_ident!("{}", type_parameter.name.as_ref())- })- .collect::<Vec<_>>();+ let (variants_extra_arguments, variants_kernel_format) = if let Some(variants) = &kernel.variants {+ let variant_names =+ variants.iter().map(|type_parameter| format_ident!("{}", type_parameter.name.as_ref())).collect::<Vec<_>>();let kernel_format = repeat_n("{}", variant_names.len() + 1).join("_");(- variant_names- .iter()- .map(|name| quote! { #[allow(non_snake_case)] #name: crate::DataType })- .collect(),+ variant_names.iter().map(|name| quote! { #[allow(non_snake_case)] #name: crate::DataType }).collect(),quote! { &format!(#kernel_format, #kernel_name #(, KernelDataType::from(#variant_names).function_name_suffix())*) },)} else {@@ -56,15 +41,10 @@ pub fn bindgen(};let base_index = specialize_indices.get(&kernel.name).copied();- let (specialize_args, specialize_setup): (- Vec<TokenStream>,- Vec<TokenStream>,- ) = kernel+ let (specialize_args, specialize_setup): (Vec<TokenStream>, Vec<TokenStream>) = kernel.arguments.iter()- .filter(|a| {- matches!(a.argument_type(), Ok(MetalArgumentType::Specialize(_)))- })+ .filter(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Specialize(_)))).enumerate().map(|(i, a)| {let arg_name = format_ident!("{}", a.name.as_ref());@@ -166,8 +146,7 @@ pub fn bindgen(}).multiunzip();- let encode_generics =- encode_generics.into_iter().flatten().collect::<Vec<_>>();+ let encode_generics = encode_generics.into_iter().flatten().collect::<Vec<_>>();let (dispatch, elements) = if kernel.has_axis() {if kernel.has_groups() || kernel.has_threads() {@@ -178,10 +157,9 @@ pub fn bindgen(.arguments.iter().filter_map(|a| match a.argument_type() {- Ok(MetalArgumentType::Axis(- threads_rexprs,- threads_per_group_rexprs,- )) => Some((threads_rexprs, threads_per_group_rexprs)),+ Ok(MetalArgumentType::Axis(threads_rexprs, threads_per_group_rexprs)) => {+ Some((threads_rexprs, threads_per_group_rexprs))+ },_ => None,}).map(|(threads_rexprs, threads_per_group_rexprs)| {@@ -192,8 +170,7 @@ pub fn bindgen(.collect::<anyhow::Result<Vec<(TokenStream, TokenStream)>>>()?;axis.extend(repeat_n((quote! {1}, quote! {1}), 3 - axis.len()));- let (threads, threads_per_group): (Vec<TokenStream>, Vec<TokenStream>) =- axis.into_iter().unzip();+ let (threads, threads_per_group): (Vec<TokenStream>, Vec<TokenStream>) = axis.into_iter().unzip();(quote! {@@ -209,9 +186,7 @@ pub fn bindgen(.arguments.iter().filter_map(|a| {- if let Ok(MetalArgumentType::Threads(rexprs)) =- a.argument_type()- {+ if let Ok(MetalArgumentType::Threads(rexprs)) = a.argument_type() {Some(parse_expr(&rexprs))} else {None@@ -241,10 +216,7 @@ pub fn bindgen(.arguments.iter().filter_map(|a| {- if let Ok(MetalArgumentType::Groups(- MetalGroupsType::Direct(rexprs),- )) = a.argument_type()- {+ if let Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(rexprs))) = a.argument_type() {Some(parse_expr(&rexprs))} else {Nonediff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rsindex cf232c231..45ec2ff0a 100644--- a/crates/uzu/build/metal/compiler.rs+++ b/crates/uzu/build/metal/compiler.rs@@ -19,10 +19,7 @@ use super::{wrapper::{SpecializeBaseIndices, wrappers},};use crate::{- common::{- caching, codegen::write_tokens, compiler::Compiler, envs,- kernel::Kernel,- },+ common::{caching, codegen::write_tokens, compiler::Compiler, envs, kernel::Kernel},debug_log,};@@ -55,10 +52,7 @@ impl ObjectInfo {}fn is_nax_source(path: &Path) -> bool {- path.file_name()- .and_then(|s| s.to_str())- .map(|s| s.contains("_nax"))- .unwrap_or(false)+ path.file_name().and_then(|s| s.to_str()).map(|s| s.contains("_nax")).unwrap_or(false)}async fn hash_dependencies(@@ -66,41 +60,28 @@ async fn hash_dependencies() -> anyhow::Result<HashMap<Box<str>, [u8; blake3::OUT_LEN]>> {let futures: Vec<_> = dependencies.map(|path| async move {- let mut file = tokio::fs::File::open(path.as_ref())- .await- .with_context(|| format!("cannot open file {}", path))?;+ let mut file =+ tokio::fs::File::open(path.as_ref()).await.with_context(|| format!("cannot open file {}", path))?;let mut contents = Vec::new();- file.read_to_end(&mut contents)- .await- .with_context(|| format!("cannot read file {}", path))?;- let hash = spawn_blocking(move || blake3::hash(&contents))- .await- .context("hash task panicked")?;+ file.read_to_end(&mut contents).await.with_context(|| format!("cannot read file {}", path))?;+ let hash = spawn_blocking(move || blake3::hash(&contents)).await.context("hash task panicked")?;Ok::<_, anyhow::Error>((path, hash.into()))}).collect();try_join_all(futures).await.map(|v| v.into_iter().collect())}-fn objects_hash<'a>(- objects: impl IntoIterator<Item = &'a ObjectInfo>-) -> anyhow::Result<blake3::Hash> {+fn objects_hash<'a>(objects: impl IntoIterator<Item = &'a ObjectInfo>) -> anyhow::Result<blake3::Hash> {let mut hasher = blake3::Hasher::new();- hasher.update(- caching::build_system_hash()- .context("cannot get build system hash")?- .as_bytes(),- );- let mut paths: Vec<_> =- objects.into_iter().map(|o| &o.object_path).collect();+ hasher.update(caching::build_system_hash().context("cannot get build system hash")?.as_bytes());+ let mut paths: Vec<_> = objects.into_iter().map(|o| &o.object_path).collect();paths.sort();for path in paths {let path_bytes = path.to_string_lossy();let path_bytes = path_bytes.as_bytes();hasher.update(&(path_bytes.len() as u32).to_le_bytes());hasher.update(path_bytes);- let contents = fs::read(path)- .with_context(|| format!("cannot read {}", path.display()))?;+ let contents = fs::read(path).with_context(|| format!("cannot read {}", path.display()))?;hasher.update(&(contents.len() as u32).to_le_bytes());hasher.update(&contents);}@@ -117,22 +98,15 @@ pub struct MetalCompiler {impl MetalCompiler {pub fn new_with_include_dir(include_dir: PathBuf) -> anyhow::Result<Self> {- let src_dir = PathBuf::from(- env::var("CARGO_MANIFEST_DIR")- .context("missing CARGO_MANIFEST_DIR")?,- )- .join("src/backends/metal/kernel");+ let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?)+ .join("src/backends/metal/kernel");- let out_dir =- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);+ let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);let build_dir = out_dir.join("metal");- fs::create_dir_all(&build_dir).with_context(|| {- format!("cannot create {}", build_dir.display())- })?;+ fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;let toolchain =- MetalToolchain::from_env_with_include_dir(Some(include_dir))- .context("cannot create toolchain")?;+ MetalToolchain::from_env_with_include_dir(Some(include_dir)).context("cannot create toolchain")?;Ok(Self {src_dir,@@ -146,55 +120,35 @@ impl MetalCompiler {&self,source_path: PathBuf,) -> anyhow::Result<ObjectInfo> {- let buildsystem_hash = caching::build_system_hash()- .context("cannot get build system cache")?- .as_bytes()- .clone();+ let buildsystem_hash =+ caching::build_system_hash().context("cannot get build system cache")?.as_bytes().clone();- let source_path_pretty =- source_path.file_name().unwrap().to_str().unwrap();+ let source_path_pretty = source_path.file_name().unwrap().to_str().unwrap();debug_log!("compile start: {source_path_pretty}");let source_path_display = source_path.display().to_string();- let source_path_hash =- blake3::hash(source_path.to_string_lossy().as_bytes());- let source_file_name = source_path- .with_extension("")- .file_name()- .context("source path has no file name")?- .to_os_string();+ let source_path_hash = blake3::hash(source_path.to_string_lossy().as_bytes());+ let source_file_name =+ source_path.with_extension("").file_name().context("source path has no file name")?.to_os_string();- let source_path_rel = source_path- .strip_prefix(&self.src_dir)- .context("source is not in src_dir")?;+ let source_path_rel = source_path.strip_prefix(&self.src_dir).context("source is not in src_dir")?;let build_dir = self.build_dir.join(source_path_hash.to_string());- let objectinfo_path =- build_dir.join(&source_file_name).with_extension("objectinfo");+ let objectinfo_path = build_dir.join(&source_file_name).with_extension("objectinfo");if build_dir.exists() {if let Ok(contents) = tokio::fs::read(&objectinfo_path).await- && let Ok(cached) =- serde_json::from_slice::<ObjectInfo>(&contents)+ && let Ok(cached) = serde_json::from_slice::<ObjectInfo>(&contents)&& buildsystem_hash == cached.buildsystem_hash- && let Ok(dependency_hashes) =- hash_dependencies(cached.dependency_hashes.keys().cloned())- .await+ && let Ok(dependency_hashes) = hash_dependencies(cached.dependency_hashes.keys().cloned()).await&& dependency_hashes == cached.dependency_hashes{if envs::build_debug() {- let kernel_list = cached- .kernels- .iter()- .map(|k| k.name.as_ref())- .collect::<Vec<_>>()- .join(", ");+ let kernel_list = cached.kernels.iter().map(|k| k.name.as_ref()).collect::<Vec<_>>().join(", ");if kernel_list.is_empty() {debug_log!("compile cached: {source_path_pretty}");} else {- debug_log!(- "compile cached: {source_path_pretty} (kernels: [{kernel_list}])"- );+ debug_log!("compile cached: {source_path_pretty} (kernels: [{kernel_list}])");}}return Ok(cached);@@ -202,35 +156,31 @@ impl MetalCompiler {fs::remove_dir_all(&build_dir).ok();}- fs::create_dir_all(&build_dir).with_context(|| {- format!("cannot create build directory {}", build_dir.display())- })?;+ fs::create_dir_all(&build_dir)+ .with_context(|| format!("cannot create build directory {}", build_dir.display()))?;- let (metal_kernel_infos, dependencies) =- self.toolchain.analyze(&source_path).await.with_context(|| {- format!("cannot analyze {}", source_path_display)- })?;+ let (metal_kernel_infos, dependencies) = self+ .toolchain+ .analyze(&source_path)+ .await+ .with_context(|| format!("cannot analyze {}", source_path_display))?;let kernel_infos: Vec<MetalKernelInfo> = metal_kernel_infos.collect();- let (wrapper_strs, specialize_indices) = wrappers(&kernel_infos)- .context("cannot generate kernel wrappers")?;+ let (wrapper_strs, specialize_indices) = wrappers(&kernel_infos).context("cannot generate kernel wrappers")?;let mut footer = String::new();for wrapper in wrapper_strs.iter() {footer.push_str(wrapper);}- let object_path =- build_dir.join(&source_file_name).with_extension("air");+ let object_path = build_dir.join(&source_file_name).with_extension("air");let compile_output = self.toolchain.compile(&source_path, &footer, &object_path).await- .with_context(|| {- format!("cannot compile {}", source_path_display)- })?;+ .with_context(|| format!("cannot compile {}", source_path_display))?;if let Some(warnings) = &compile_output {for line in warnings.lines() {@@ -238,9 +188,7 @@ impl MetalCompiler {}}- let dependency_hashes = hash_dependencies(dependencies)- .await- .context("cannot hash dependencies")?;+ let dependency_hashes = hash_dependencies(dependencies).await.context("cannot hash dependencies")?;let object_info = ObjectInfo {src_rel_path: source_path_rel.into(),@@ -253,27 +201,16 @@ impl MetalCompiler {fs::write(&objectinfo_path,- serde_json::to_string_pretty(&object_info)- .context("failed to serialize object info")?- .as_bytes(),+ serde_json::to_string_pretty(&object_info).context("failed to serialize object info")?.as_bytes(),)- .with_context(|| {- format!("cannot write object info {}", objectinfo_path.display())- })?;+ .with_context(|| format!("cannot write object info {}", objectinfo_path.display()))?;if envs::build_debug() {- let kernel_list = object_info- .kernels- .iter()- .map(|k| k.name.as_ref())- .collect::<Vec<_>>()- .join(", ");+ let kernel_list = object_info.kernels.iter().map(|k| k.name.as_ref()).collect::<Vec<_>>().join(", ");if kernel_list.is_empty() {debug_log!("compile end: {source_path_pretty}");} else {- debug_log!(- "compile end: {source_path_pretty} (kernels: [{kernel_list}])"- );+ debug_log!("compile end: {source_path_pretty} (kernels: [{kernel_list}])");}}@@ -310,9 +247,8 @@ impl MetalCompiler {}}- fs::write(&hash_path, hash.to_string()).with_context(|| {- format!("cannot write hash file {}", hash_path.display())- })?;+ fs::write(&hash_path, hash.to_string())+ .with_context(|| format!("cannot write hash file {}", hash_path.display()))?;debug_log!("linking end");@@ -341,12 +277,10 @@ impl MetalCompiler {.into_iter().flat_map(|o| o.kernels.iter().map(|k| (k, &o.specialize_indices))).map(|(k, specialize_indices)| {- super::bindgen::bindgen(k, specialize_indices).with_context(- || format!("cannot generate bindings for {}", k.name),- )+ super::bindgen::bindgen(k, specialize_indices)+ .with_context(|| format!("cannot generate bindings for {}", k.name))})- .collect::<anyhow::Result<(Vec<TokenStream>, Vec<TokenStream>)>>(- )?;+ .collect::<anyhow::Result<(Vec<TokenStream>, Vec<TokenStream>)>>()?;let tokens = quote! {use crate::backends::metal::{@@ -376,9 +310,8 @@ impl MetalCompiler {write_tokens(tokens, &out_path).context("cannot write bindings")?;- fs::write(&hash_path, hash.to_string()).with_context(|| {- format!("cannot write hash file {}", hash_path.display())- })?;+ fs::write(&hash_path, hash.to_string())+ .with_context(|| format!("cannot write hash file {}", hash_path.display()))?;debug_log!("bindgen end");@@ -388,9 +321,7 @@ impl MetalCompiler {#[async_trait]impl Compiler for MetalCompiler {- async fn build(- &self- ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {+ async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {let nax_enabled = cfg!(feature = "metal-nax");debug_log!("metal nax {}",@@ -404,18 +335,12 @@ impl Compiler for MetalCompiler {let metal_sources: Vec<PathBuf> = WalkDir::new(&self.src_dir).into_iter().filter_map(|e| e.ok())- .filter(|e| {- e.file_type().is_file()- && e.path().extension().and_then(|s| s.to_str())- == Some("metal")- })+ .filter(|e| e.file_type().is_file() && e.path().extension().and_then(|s| s.to_str()) == Some("metal")).map(|e| e.into_path()).filter(|p| nax_enabled || !is_nax_source(p)).collect();- let num_concurrent_compiles =- std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4)- * 2;+ let num_concurrent_compiles = std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4) * 2;let objects: Vec<ObjectInfo> = stream::iter(metal_sources).map(|p| self.compile(p))diff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rsindex 0c3d540cb..7c4743f10 100644--- a/crates/uzu/build/metal/toolchain.rs+++ b/crates/uzu/build/metal/toolchain.rs@@ -10,9 +10,7 @@ use anyhow::{Context, bail};use tempfile::NamedTempFile;use tokio::{io::AsyncWriteExt, process::Command};-use super::ast::{- MetalAstKind, MetalAstNode, MetalKernelInfo, validate_raw_kernel,-};+use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo, validate_raw_kernel};#[derive(Debug)]pub enum MetalSdk {@@ -31,9 +29,7 @@ impl MetalSdk {if target_os == "ios" {if target.contains("macabi") || target_env == "macabi" {Ok(Self::MacCatalyst)- } else if target.contains("ios")- && (target.contains("86_64") || target_env == "sim")- {+ } else if target.contains("ios") && (target.contains("86_64") || target_env == "sim") {Ok(Self::IPhoneSimulator)} else {Ok(Self::IPhoneOS)@@ -41,18 +37,14 @@ impl MetalSdk {} else if target_os == "macos" {Ok(Self::MacOSX)} else {- bail!(- "cannot find matching metal sdk for ({target:?}, {target_os:?}, {target_env:?})"- );+ bail!("cannot find matching metal sdk for ({target:?}, {target_os:?}, {target_env:?})");}}pub fn from_env() -> anyhow::Result<Self> {let target = env::var("TARGET").context("missing TARGET")?;- let target_os = env::var("CARGO_CFG_TARGET_OS")- .context("missing CARGO_CFG_TARGET_OS")?;- let target_env = env::var("CARGO_CFG_TARGET_ENV")- .context("missing CARGO_CFG_TARGET_ENV")?;+ let target_os = env::var("CARGO_CFG_TARGET_OS").context("missing CARGO_CFG_TARGET_OS")?;+ let target_env = env::var("CARGO_CFG_TARGET_ENV").context("missing CARGO_CFG_TARGET_ENV")?;let sdk = Self::from_parts(&target, &target_os, &target_env)?;@@ -118,21 +110,16 @@ pub struct MetalToolchain {}impl MetalToolchain {- pub fn from_env_with_include_dir(- include_dir: Option<PathBuf>- ) -> anyhow::Result<Self> {+ pub fn from_env_with_include_dir(include_dir: Option<PathBuf>) -> anyhow::Result<Self> {let sdk = MetalSdk::from_env().context("cannot get sdk")?;let std = MetalStd::default();- let opt_flags = match env::var("OPT_LEVEL")- .context("missing OPT_LEVEL")?- .as_str()- {+ let opt_flags = match env::var("OPT_LEVEL").context("missing OPT_LEVEL")?.as_str() {"0" => {[OsString::from("-O1"), // matmul kernels compiled with -O0 are broken and require a reboot to unfreeze the osOsString::from("-gline-tables-only"), // debug with line tables only- OsString::from("-frecord-sources"), // include source code+ OsString::from("-frecord-sources"), // include source code].into()},@@ -141,11 +128,7 @@ impl MetalToolchain {};let extra_options = Box::new([- OsString::from(format!(- "-m{}-version-min={}",- sdk.os(),- std.min_os()- )),+ OsString::from(format!("-m{}-version-min={}", sdk.os(), std.min_os())),// NOTE: Previous build.rs script didn't forward metal compiler warnings to cargo warnings, the new one does.// This is temporary to avoid warning spam without modifying unrelated things in the same pull request as introducing the dsl and converting kernelsOsString::from("-Wno-sign-compare"),@@ -182,14 +165,10 @@ impl MetalToolchain {pub async fn analyze(&self,path: impl AsRef<Path>,- ) -> anyhow::Result<(- impl Iterator<Item = MetalKernelInfo>,- impl Iterator<Item = Box<str>>,- )> {+ ) -> anyhow::Result<(impl Iterator<Item = MetalKernelInfo>, impl Iterator<Item = Box<str>>)> {let path = path.as_ref();- let depfile_path =- NamedTempFile::new().context("cannot create temporary file")?;+ let depfile_path = NamedTempFile::new().context("cannot create temporary file")?;let mut cmd = self.xcrun();cmd.arg("metal")@@ -208,19 +187,14 @@ impl MetalToolchain {.stdout(Stdio::piped()).stderr(Stdio::piped());- let analyze_output =- cmd.output().await.context("cannot execute metal analyzer")?;+ let analyze_output = cmd.output().await.context("cannot execute metal analyzer")?;if !analyze_output.status.success() {- bail!(- "metal analyzer failed: {}",- String::from_utf8_lossy(&analyze_output.stderr)- );+ bail!("metal analyzer failed: {}", String::from_utf8_lossy(&analyze_output.stderr));}let ast_root =- serde_json::from_slice::<MetalAstNode>(&analyze_output.stdout)- .context("cannot deserialize ast dump")?;+ serde_json::from_slice::<MetalAstNode>(&analyze_output.stdout).context("cannot deserialize ast dump")?;if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {bail!(@@ -229,30 +203,21 @@ impl MetalToolchain {);}- let source_contents =- fs::read_to_string(path).context("cannot read source file")?;+ let source_contents = fs::read_to_string(path).context("cannot read source file")?;for node in &ast_root.inner {- validate_raw_kernel(node)- .context("validation of legacy (non-dsl) kernel failed")?;+ validate_raw_kernel(node).context("validation of legacy (non-dsl) kernel failed")?;}let kernel_infos = ast_root.inner.into_iter()- .filter_map(|node| {- MetalKernelInfo::from_ast_node_and_source(- node,- &source_contents,- )- .transpose()- })+ .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose()).collect::<anyhow::Result<Vec<_>>>().context("cannot parse kernel infos from AST")?.into_iter();- let depfile_contents = fs::read_to_string(depfile_path.path())- .context("cannot read depfile")?;+ let depfile_contents = fs::read_to_string(depfile_path.path()).context("cannot read depfile")?;let dependencies = depfile::parse(&depfile_contents).map_err(|e| anyhow::anyhow!("cannot parse depfile: {e}"))?@@ -289,8 +254,7 @@ impl MetalToolchain {.stdin(Stdio::piped()).stderr(Stdio::piped());- let mut compile_child =- cmd.spawn().context("cannot execute metal compiler")?;+ let mut compile_child = cmd.spawn().context("cannot execute metal compiler")?;compile_child.stdin@@ -300,14 +264,9 @@ impl MetalToolchain {.await.context("cannot write to metal compiler stdin")?;- let compile_output = compile_child- .wait_with_output()- .await- .context("cannot wait on metal compiler")?;+ let compile_output = compile_child.wait_with_output().await.context("cannot wait on metal compiler")?;- let stderr = String::from_utf8_lossy(&compile_output.stderr)- .into_owned()- .into_boxed_str();+ let stderr = String::from_utf8_lossy(&compile_output.stderr).into_owned().into_boxed_str();if !compile_output.status.success() {bail!("metal compiler failed: {stderr}");@@ -330,18 +289,14 @@ impl MetalToolchain {let link_output = self.xcrun().arg("metallib")- .args(- objects.into_iter().map(|p| p.as_ref().as_os_str().to_owned()),- )+ .args(objects.into_iter().map(|p| p.as_ref().as_os_str().to_owned())).arg("-o").arg(output.as_ref()).output().await.context("cannot execute metal linker")?;- let stderr = String::from_utf8_lossy(&link_output.stderr)- .into_owned()- .into_boxed_str();+ let stderr = String::from_utf8_lossy(&link_output.stderr).into_owned().into_boxed_str();if !link_output.status.success() {bail!("metal linker failed: {stderr}");diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rsindex 6348506a1..9e0825467 100644--- a/crates/uzu/build/metal/wrapper.rs+++ b/crates/uzu/build/metal/wrapper.rs@@ -7,9 +7,7 @@ use super::ast::{MetalArgumentType, MetalKernelInfo};pub type SpecializeBaseIndices = HashMap<Box<str>, usize>;-pub fn wrappers(- kernels: &[MetalKernelInfo]-) -> anyhow::Result<(Box<[Box<str>]>, SpecializeBaseIndices)> {+pub fn wrappers(kernels: &[MetalKernelInfo]) -> anyhow::Result<(Box<[Box<str>]>, SpecializeBaseIndices)> {let mut all_wrappers = Vec::new();let mut base_indices = SpecializeBaseIndices::new();let mut next_index = 0usize;@@ -18,12 +16,7 @@ pub fn wrappers(let specialize_count = kernel.arguments.iter()- .filter(|a| {- matches!(- a.argument_type(),- Ok(MetalArgumentType::Specialize(_))- )- })+ .filter(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Specialize(_)))).count();if specialize_count > 0 {@@ -31,8 +24,7 @@ pub fn wrappers(next_index += specialize_count;}- let kernel_wrappers =- kernel_wrappers(kernel, base_indices.get(&kernel.name))?;+ let kernel_wrappers = kernel_wrappers(kernel, base_indices.get(&kernel.name))?;all_wrappers.extend(kernel_wrappers.into_vec());}@@ -54,10 +46,7 @@ fn kernel_wrappers(.map(|(i, a)| {let c_type = a.c_type.trim_start_matches("const ");let idx = base + i;- format!(- "constant {c_type} __dsl_specialize_{}_{} [[function_constant({idx})]];\n",- kernel.name, a.name- )+ format!("constant {c_type} __dsl_specialize_{}_{} [[function_constant({idx})]];\n", kernel.name, a.name)}).collect::<String>()} else {@@ -82,52 +71,37 @@ fn kernel_wrappers(} else {vec![None]} {- let (wrapper_name, underlying_name) =- if let Some(type_variant) = &type_variant {- (- format!(- "{}_{}",- kernel.name,- type_variant.iter().map(|(_k, v)| v).join("_")- ),- format!(- "{}<{}>",- kernel.name,- type_variant.iter().map(|(_k, v)| v).join(", ")- ),- )- } else {- (kernel.name.to_string(), kernel.name.to_string())- };+ let (wrapper_name, underlying_name) = if let Some(type_variant) = &type_variant {+ (+ format!("{}_{}", kernel.name, type_variant.iter().map(|(_k, v)| v).join("_")),+ format!("{}<{}>", kernel.name, type_variant.iter().map(|(_k, v)| v).join(", ")),+ )+ } else {+ (kernel.name.to_string(), kernel.name.to_string())+ };let max_total_threads_per_threadgroup = kernel.arguments.iter().filter_map(|a| match a.argument_type() {- Ok(MetalArgumentType::Axis(_, l))- | Ok(MetalArgumentType::Threads(l)) => Some(format!("({l})")),+ Ok(MetalArgumentType::Axis(_, l)) | Ok(MetalArgumentType::Threads(l)) => Some(format!("({l})")),_ => None,}).collect::<Vec<_>>();- let max_total_threads_per_threadgroup =- if !max_total_threads_per_threadgroup.is_empty() {- max_total_threads_per_threadgroup.join(" * ")- } else {- "1".to_string()- };+ let max_total_threads_per_threadgroup = if !max_total_threads_per_threadgroup.is_empty() {+ max_total_threads_per_threadgroup.join(" * ")+ } else {+ "1".to_string()+ };- let wrapper_argument_replace = type_variant- .as_ref()- .map(|tv| tv.iter().cloned().collect::<HashMap<_, _>>());+ let wrapper_argument_replace = type_variant.as_ref().map(|tv| tv.iter().cloned().collect::<HashMap<_, _>>());let apply_replace = |s: &str| {s.split_whitespace().map(|token| {- if let Some(wrapper_argument_replace) =- &wrapper_argument_replace- && let Some(replacement) =- wrapper_argument_replace.get(token)+ if let Some(wrapper_argument_replace) = &wrapper_argument_replace+ && let Some(replacement) = wrapper_argument_replace.get(token){replacement} else {@@ -141,22 +115,12 @@ fn kernel_wrappers(let mut wrapper_arguments = kernel.arguments.iter()- .filter(|a| {- matches!(- a.argument_type(),- Ok(MetalArgumentType::Buffer)- | Ok(MetalArgumentType::Constant(_))- )- })+ .filter(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Buffer) | Ok(MetalArgumentType::Constant(_)))).enumerate().map(|(i, a)| match a.argument_type() {- Ok(MetalArgumentType::Buffer)- | Ok(MetalArgumentType::Constant(_)) => format!(- "{} {} [[buffer({})]]",- apply_replace(&a.c_type),- a.name,- i- ),+ Ok(MetalArgumentType::Buffer) | Ok(MetalArgumentType::Constant(_)) => {+ format!("{} {} [[buffer({})]]", apply_replace(&a.c_type), a.name, i)+ },_ => unreachable!(),}).collect::<Vec<_>>();@@ -166,41 +130,28 @@ fn kernel_wrappers(bail!("mixing groups/threads and axis is not supported");}- wrapper_arguments.push(- "uint3 __dsl_axis_idx [[thread_position_in_grid]]".into(),- );+ wrapper_arguments.push("uint3 __dsl_axis_idx [[thread_position_in_grid]]".into());}if kernel.has_groups() {- wrapper_arguments.push(- "uint3 __dsl_group_idx [[threadgroup_position_in_grid]]".into(),- );+ wrapper_arguments.push("uint3 __dsl_group_idx [[threadgroup_position_in_grid]]".into());}if kernel.has_threads() {- wrapper_arguments.push(- "uint3 __dsl_thread_idx [[thread_position_in_threadgroup]]"- .into(),- );+ wrapper_arguments.push("uint3 __dsl_thread_idx [[thread_position_in_threadgroup]]".into());}let wrapper_arguments = wrapper_arguments.join(", ");- let shared_definitions =- kernel.arguments.iter().filter_map(|a| match a.argument_type() {- Ok(MetalArgumentType::Shared(Some(len))) => Some(format!(- "{} {}[{}]",- apply_replace(&a.c_type.replace('*', "")),- a.name,- len.as_ref(),- )),- Ok(MetalArgumentType::Shared(None)) => Some(format!(- "{} {}",- apply_replace(&a.c_type.replace('&', "")),- a.name- )),- _ => None,- });+ let shared_definitions = kernel.arguments.iter().filter_map(|a| match a.argument_type() {+ Ok(MetalArgumentType::Shared(Some(len))) => {+ Some(format!("{} {}[{}]", apply_replace(&a.c_type.replace('*', "")), a.name, len.as_ref(),))+ },+ Ok(MetalArgumentType::Shared(None)) => {+ Some(format!("{} {}", apply_replace(&a.c_type.replace('&', "")), a.name))+ },+ _ => None,+ });let underlying_arguments = {let mut group_axis_letters = ["x", "y", "z"].iter();@@ -210,43 +161,30 @@ fn kernel_wrappers(.arguments.iter().map(|a| match a.argument_type().unwrap() {- MetalArgumentType::Buffer- | MetalArgumentType::Constant(_)- | MetalArgumentType::Shared(_) => a.name.to_string(),+ MetalArgumentType::Buffer | MetalArgumentType::Constant(_) | MetalArgumentType::Shared(_) => {+ a.name.to_string()+ },MetalArgumentType::Specialize(_) => {format!("__dsl_specialize_{}_{}", kernel.name, a.name)},MetalArgumentType::Axis(..) => {- format!(- "__dsl_axis_idx.{}",- group_axis_letters.next().unwrap()- )+ format!("__dsl_axis_idx.{}", group_axis_letters.next().unwrap())},MetalArgumentType::Groups(_) => {- format!(- "__dsl_group_idx.{}",- group_axis_letters.next().unwrap()- )+ format!("__dsl_group_idx.{}", group_axis_letters.next().unwrap())},MetalArgumentType::Threads(_) => {- format!(- "__dsl_thread_idx.{}",- thread_axis_letters.next().unwrap()- )+ format!("__dsl_thread_idx.{}", thread_axis_letters.next().unwrap())},}).collect::<Vec<_>>().join(", ")};- let underlying_call =- format!("{underlying_name}({underlying_arguments})");+ let underlying_call = format!("{underlying_name}({underlying_arguments})");- let wrapper_body = shared_definitions- .chain(once(underlying_call))- .map(|l| format!(" {l};\n"))- .collect::<Vec<_>>()- .join("");+ let wrapper_body =+ shared_definitions.chain(once(underlying_call)).map(|l| format!(" {l};\n")).collect::<Vec<_>>().join("");kernel_wrappers.push(format!(diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rsindex 07524801f..67ac64b41 100644--- a/crates/uzu/src/array.rs+++ b/crates/uzu/src/array.rs@@ -61,14 +61,7 @@ impl<B: Backend> Array<B> {&self,shape: &[usize],) -> Self {- unsafe {- Self::from_parts(- self.buffer.clone(),- self.offset,- shape,- self.data_type,- )- }+ unsafe { Self::from_parts(self.buffer.clone(), self.offset, shape, self.data_type) }}// Getters@@ -112,21 +105,11 @@ impl<B: Backend> Array<B> {}pub fn as_bytes(&self) -> &[u8] {- unsafe {- std::slice::from_raw_parts(- self.cpu_ptr().as_ptr() as *const u8,- self.size(),- )- }+ unsafe { std::slice::from_raw_parts(self.cpu_ptr().as_ptr() as *const u8, self.size()) }}pub fn as_bytes_mut(&mut self) -> &mut [u8] {- unsafe {- std::slice::from_raw_parts_mut(- self.cpu_ptr().as_ptr() as *mut u8,- self.size(),- )- }+ unsafe { std::slice::from_raw_parts_mut(self.cpu_ptr().as_ptr() as *mut u8, self.size()) }}pub fn as_slice<T: ArrayElement>(&self) -> &[T] {@@ -140,8 +123,7 @@ impl<B: Backend> Array<B> {}pub fn as_view<T: ArrayElement>(&self) -> ArrayView<'_, T, IxDyn> {- ArrayView::from_shape(IxDyn(self.shape()), self.as_slice::<T>())- .expect("Failed to create array view")+ ArrayView::from_shape(IxDyn(self.shape()), self.as_slice::<T>()).expect("Failed to create array view")}pub fn copy_from_array<C: Backend>(@@ -168,17 +150,12 @@ impl<B: Backend> Array<B> {assert_eq!(a, b, "Shapes must match on all non-sliced axes");}}- assert_eq!(- self.data_type(),- source.data_type(),- "Arrays must have the same data type"- );+ assert_eq!(self.data_type(), source.data_type(), "Arrays must have the same data type");let elem_size = self.data_type().size_in_bytes();// The number of contiguous elements to copy for each slice operation- let block_size_elems =- self.shape().iter().skip(axis + 1).product::<usize>();+ let block_size_elems = self.shape().iter().skip(axis + 1).product::<usize>();let block_size_bytes = block_size_elems * elem_size;// The total number of blocks to copy@@ -199,8 +176,7 @@ impl<B: Backend> Array<B> {let src_block_start = i * src_stride_bytes;let dst_block_start = i * dst_stride_bytes;- let src_start =- src_block_start + src_range.start * block_size_bytes;+ let src_start = src_block_start + src_range.start * block_size_bytes;let dst_start = dst_block_start + dst_offset * block_size_bytes;let src_slice = &src_buf[src_start..src_start + copy_bytes];@@ -267,8 +243,7 @@ pub trait ArrayContextExt {data_type: DataType,label: &str,) -> Array<Self::Backend> {- let mut array =- self.create_array_uninitialized(shape, data_type, label);+ let mut array = self.create_array_uninitialized(shape, data_type, label);array.as_bytes_mut().fill(0);array}@@ -285,9 +260,7 @@ impl<C: Context> ArrayContextExt for C {) -> Array<Self::Backend> {let buffer_size_bytes = size_for_shape(shape, data_type);- let buffer = self- .create_buffer(buffer_size_bytes)- .expect("Failed to create buffer");+ let buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");buffer.set_label(Some(label));unsafe { Array::from_parts(buffer, 0, shape, data_type) }diff --git a/crates/uzu/src/backends/common/allocator/allocator.rs b/crates/uzu/src/backends/common/allocator/allocator.rsindex 57e1ad196..66bf849b1 100644--- a/crates/uzu/src/backends/common/allocator/allocator.rs+++ b/crates/uzu/src/backends/common/allocator/allocator.rs@@ -7,9 +7,7 @@ use super::{AllocError,scratch_pool::{ScratchPool, align_size},};-use crate::backends::common::{- Backend, Buffer, BufferLifetime, Context, NativeBuffer,-};+use crate::backends::common::{Backend, Buffer, BufferLifetime, Context, NativeBuffer};pub struct Allocator<B: Backend> {context: Weak<B::Context>,@@ -36,11 +34,9 @@ impl<B: Backend> Allocator<B> {&self,size: usize,) -> Result<B::NativeBuffer, AllocError> {- let context = self.context.upgrade().ok_or_else(|| {- AllocError::AllocationFailed {- size,- reason: "context has been dropped".to_string(),- }+ let context = self.context.upgrade().ok_or_else(|| AllocError::AllocationFailed {+ size,+ reason: "context has been dropped".to_string(),})?;context.create_buffer(size).map_err(|e| AllocError::AllocationFailed {size,@@ -141,9 +137,7 @@ impl<B: Backend> Allocator<B> {self.track_allocation(aligned_size);(buffer, false)},- BufferLifetime::Scratch => {- (self.alloc_scratch_internal(aligned_size)?, true)- },+ BufferLifetime::Scratch => (self.alloc_scratch_internal(aligned_size)?, true),};Ok(self.wrap_buffer(native, is_scratch))diff --git a/crates/uzu/src/backends/common/allocator/error.rs b/crates/uzu/src/backends/common/allocator/error.rsindex f6c4c949c..bb109b172 100644--- a/crates/uzu/src/backends/common/allocator/error.rs+++ b/crates/uzu/src/backends/common/allocator/error.rs@@ -22,11 +22,7 @@ impl fmt::Display for AllocError {requested,available,} => {- write!(- f,- "Out of memory: requested {} bytes, {} available",- requested, available- )+ write!(f, "Out of memory: requested {} bytes, {} available", requested, available)},AllocError::AllocationFailed {size,diff --git a/crates/uzu/src/backends/common/allocator/scratch_pool.rs b/crates/uzu/src/backends/common/allocator/scratch_pool.rsindex 1c62f71f9..963deb3d2 100644--- a/crates/uzu/src/backends/common/allocator/scratch_pool.rs+++ b/crates/uzu/src/backends/common/allocator/scratch_pool.rs@@ -157,11 +157,10 @@ impl<B> ScratchPool<B> {}}- let (key, slot, bucket_idx) =- match (found_key, found_slot, found_bucket_idx) {- (Some(k), Some(s), Some(i)) => (k, s, i),- _ => return None,- };+ let (key, slot, bucket_idx) = match (found_key, found_slot, found_bucket_idx) {+ (Some(k), Some(s), Some(i)) => (k, s, i),+ _ => return None,+ };self.remove_slot_from_bucket(key, slot, bucket_idx);self.remove_from_lru(slot);@@ -248,8 +247,7 @@ impl<B> ScratchPool<B> {self.size_index.clear();self.lru_head = None;self.lru_tail = None;- self.total_allocated =- self.total_allocated.saturating_sub(self.pool_size);+ self.total_allocated = self.total_allocated.saturating_sub(self.pool_size);self.pool_size = 0;countdiff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rsindex cb674cda9..240289230 100644--- a/crates/uzu/src/backends/common/context.rs+++ b/crates/uzu/src/backends/common/context.rs@@ -12,15 +12,9 @@ pub trait Context: Sized {fn create_buffer(&self,size: usize,- ) -> Result<- <Self::Backend as Backend>::NativeBuffer,- <Self::Backend as Backend>::Error,- >;+ ) -> Result<<Self::Backend as Backend>::NativeBuffer, <Self::Backend as Backend>::Error>;fn create_command_buffer(&self- ) -> Result<- <Self::Backend as Backend>::CommandBuffer,- <Self::Backend as Backend>::Error,- >;+ ) -> Result<<Self::Backend as Backend>::CommandBuffer, <Self::Backend as Backend>::Error>;}diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rsindex ab6af972f..50cc11aff 100644--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs@@ -2,10 +2,7 @@ use thiserror::Error;use crate::{DataType,- backends::common::{- Backend, CommandBuffer, Kernels, gpu_types::Swap,- kernel::KVCacheUpdateKernel,- },+ backends::common::{Backend, CommandBuffer, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},};pub struct KVLayerData<B: Backend> {@@ -44,10 +41,8 @@ impl<B: Backend> KVCacheUpdate<B> {data_type: DataType,max_sequence_length: usize,) -> Result<Self, KVCacheUpdateError<B>> {- let kernel = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(- context, data_type,- )- .map_err(KVCacheUpdateError::BackendError)?;+ let kernel = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(context, data_type)+ .map_err(KVCacheUpdateError::BackendError)?;Ok(Self {kernel,@@ -63,12 +58,7 @@ impl<B: Backend> KVCacheUpdate<B> {command_buffer: &B::CommandBuffer,) -> Result<(), KVCacheUpdateError<B>> {command_buffer.with_compute_encoder(|encoder| {- self.encode_with_encoder(- in_place_data,- source_indices,- destination_indices,- encoder,- )+ self.encode_with_encoder(in_place_data, source_indices, destination_indices, encoder)})}@@ -92,18 +82,14 @@ impl<B: Backend> KVCacheUpdate<B> {// For small swap counts, use set_bytes to avoid shared buffer race in async mode.// Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.let use_inline_bytes = swaps.len() <= 32;- assert!(- use_inline_bytes,- "non-inline is not supported yet (and is broken anyways due to a data race)"- );+ assert!(use_inline_bytes, "non-inline is not supported yet (and is broken anyways due to a data race)");for layer_data in in_place_data {if layer_data.key_shape != layer_data.value_shape {return Err(KVCacheUpdateError::ShapeMismatch);}- let [num_heads, max_sequence_length, head_dim] =- layer_data.key_shape;+ let [num_heads, max_sequence_length, head_dim] = layer_data.key_shape;self.kernel.encode(&layer_data.key_buffer,diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rsindex 82bef77ba..189b49013 100644--- a/crates/uzu/src/backends/common/kernel/sampling.rs+++ b/crates/uzu/src/backends/common/kernel/sampling.rs@@ -7,9 +7,8 @@ use crate::{backends::common::{Context,kernel::{- ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel,- BitmaskKernel, GumbelKernel, MinPKernel, TemperatureKernel,- TopKKernel, TopPKernel,+ ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel, BitmaskKernel, GumbelKernel, MinPKernel,+ TemperatureKernel, TopKKernel, TopPKernel,},},session::parameter::SamplingMethod,@@ -83,13 +82,7 @@ impl<B: Backend> SamplingKernel<B> {max_batch_size: usize,max_vocab_size: usize,) -> Result<Self, SamplingError<B>> {- Self::new_with_strategy(- context,- data_type,- max_batch_size,- max_vocab_size,- ArgmaxStrategy::TwoPass,- )+ Self::new_with_strategy(context, data_type, max_batch_size, max_vocab_size, ArgmaxStrategy::TwoPass)}pub fn new_with_strategy(@@ -100,56 +93,39 @@ impl<B: Backend> SamplingKernel<B> {argmax_strategy: ArgmaxStrategy,) -> Result<Self, SamplingError<B>> {let bitmask =- <B::Kernels as Kernels>::BitmaskKernel::new(context, data_type)- .map_err(SamplingError::BackendError)?;+ <B::Kernels as Kernels>::BitmaskKernel::new(context, data_type).map_err(SamplingError::BackendError)?;let temperature =- <B::Kernels as Kernels>::TemperatureKernel::new(context, data_type)- .map_err(SamplingError::BackendError)?;- let topk = <B::Kernels as Kernels>::TopKKernel::new(context, data_type)- .map_err(SamplingError::BackendError)?;- let topp = <B::Kernels as Kernels>::TopPKernel::new(context, data_type)- .map_err(SamplingError::BackendError)?;- let minp = <B::Kernels as Kernels>::MinPKernel::new(context, data_type)- .map_err(SamplingError::BackendError)?;+ <B::Kernels as Kernels>::TemperatureKernel::new(context, data_type).map_err(SamplingError::BackendError)?;+ let topk = <B::Kernels as Kernels>::TopKKernel::new(context, data_type).map_err(SamplingError::BackendError)?;+ let topp = <B::Kernels as Kernels>::TopPKernel::new(context, data_type).map_err(SamplingError::BackendError)?;+ let minp = <B::Kernels as Kernels>::MinPKernel::new(context, data_type).map_err(SamplingError::BackendError)?;let gumbel =- <B::Kernels as Kernels>::GumbelKernel::new(context, data_type)- .map_err(SamplingError::BackendError)?;+ <B::Kernels as Kernels>::GumbelKernel::new(context, data_type).map_err(SamplingError::BackendError)?;let argmax_implementation = match argmax_strategy {ArgmaxStrategy::SinglePass => {- let kernel = <B::Kernels as Kernels>::ArgmaxSingleKernel::new(- context, data_type,- )- .map_err(SamplingError::BackendError)?;+ let kernel = <B::Kernels as Kernels>::ArgmaxSingleKernel::new(context, data_type)+ .map_err(SamplingError::BackendError)?;ArgmaxImplementation::SinglePass {kernel,}},ArgmaxStrategy::TwoPass => {- let main_kernel =- <B::Kernels as Kernels>::ArgmaxMainKernel::new(- context, data_type,- )+ let main_kernel = <B::Kernels as Kernels>::ArgmaxMainKernel::new(context, data_type).map_err(SamplingError::BackendError)?;let final_kernel =- <B::Kernels as Kernels>::ArgmaxFinalKernel::new(context)- .map_err(SamplingError::BackendError)?;+ <B::Kernels as Kernels>::ArgmaxFinalKernel::new(context).map_err(SamplingError::BackendError)?;let block_size = 1024;let grain_size = 4;let elements_per_group = block_size * grain_size;- let max_vocab_groups_per_batch =- (max_vocab_size + elements_per_group - 1)- / elements_per_group;- let max_partial_results =- max_batch_size * max_vocab_groups_per_batch;+ let max_vocab_groups_per_batch = (max_vocab_size + elements_per_group - 1) / elements_per_group;+ let max_partial_results = max_batch_size * max_vocab_groups_per_batch;let partial_results_buffer = context- .create_buffer(- max_partial_results * size_of::<ArgmaxPair>(),- )+ .create_buffer(max_partial_results * size_of::<ArgmaxPair>()).expect("Failed to create partial results buffer");ArgmaxImplementation::TwoPass {@@ -187,16 +163,10 @@ impl<B: Backend> SamplingKernel<B> {compute_encoder: &B::ComputeEncoder,) -> Result<(), SamplingError<B>> {if batch_size > self.max_batch_size {- return Err(SamplingError::BatchSizeExceeded(- batch_size,- self.max_batch_size,- ));+ return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));}if vocab_size > self.max_vocab_size {- return Err(SamplingError::VocabSizeExceeded(- vocab_size,- self.max_vocab_size,- ));+ return Err(SamplingError::VocabSizeExceeded(vocab_size, self.max_vocab_size));}if let Some(bitmask_buffer) = bitmask_buffer {@@ -263,10 +233,7 @@ impl<B: Backend> SamplingKernel<B> {self.gumbel.encode(logits_buffer,- (- seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,- seeds_offset,- ),+ (seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?, seeds_offset),logits_buffer,batch_size as u32,vocab_size as u32,diff --git a/crates/uzu/src/backends/cpu/argmax.rs b/crates/uzu/src/backends/cpu/argmax.rsindex 61cf89d7c..9e6212780 100644--- a/crates/uzu/src/backends/cpu/argmax.rs+++ b/crates/uzu/src/backends/cpu/argmax.rs@@ -24,8 +24,7 @@ pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {for chunk in 0..4 {let base = i + chunk * 8;for j in 0..8 {- let val: f32 = NumCast::from(*ptr.add(base + j))- .unwrap_or(f32::NEG_INFINITY);+ let val: f32 = NumCast::from(*ptr.add(base + j)).unwrap_or(f32::NEG_INFINITY);if val > max_val || (val == max_val && base + j < max_idx) {max_val = val;max_idx = base + j;@@ -38,8 +37,7 @@ pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {// Handle remaining 16x chunkswhile i + 16 <= len {for j in 0..16 {- let val: f32 =- NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);+ let val: f32 = NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);if val > max_val || (val == max_val && i + j < max_idx) {max_val = val;max_idx = i + j;@@ -51,8 +49,7 @@ pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {// Handle remaining 8x chunkswhile i + 8 <= len {for j in 0..8 {- let val: f32 =- NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);+ let val: f32 = NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);if val > max_val || (val == max_val && i + j < max_idx) {max_val = val;max_idx = i + j;@@ -63,8 +60,7 @@ pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {// Handle remaining elementswhile i < len {- let val: f32 =- NumCast::from(*ptr.add(i)).unwrap_or(f32::NEG_INFINITY);+ let val: f32 = NumCast::from(*ptr.add(i)).unwrap_or(f32::NEG_INFINITY);if val > max_val || (val == max_val && i < max_idx) {max_val = val;max_idx = i;@@ -80,20 +76,14 @@ pub fn simple_argmax<T: ArrayElement>(input: &[T]) -> usize {input.iter().enumerate()- .fold(- (0, f32::NEG_INFINITY),- |(best_index, best_value), (index, &value)| {- let value_f32 =- NumCast::from(value).unwrap_or(f32::NEG_INFINITY);- if value_f32 > best_value- || (value_f32 == best_value && index < best_index)- {- (index, value_f32)- } else {- (best_index, best_value)- }- },- )+ .fold((0, f32::NEG_INFINITY), |(best_index, best_value), (index, &value)| {+ let value_f32 = NumCast::from(value).unwrap_or(f32::NEG_INFINITY);+ if value_f32 > best_value || (value_f32 == best_value && index < best_index) {+ (index, value_f32)+ } else {+ (best_index, best_value)+ }+ }).0}@@ -150,8 +140,7 @@ pub unsafe fn neon_argmax_f32(input: &[f32]) -> usize {for j in 1..4 {if max_vals_array[j] > global_max- || (max_vals_array[j] == global_max- && (max_indices_array[j] as usize) < global_max_idx)+ || (max_vals_array[j] == global_max && (max_indices_array[j] as usize) < global_max_idx){global_max = max_vals_array[j];global_max_idx = max_indices_array[j] as usize;@@ -182,10 +171,7 @@ pub fn neon_optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {// For f32, use direct NEON implementationif std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>() {unsafe {- let f32_slice = std::slice::from_raw_parts(- input.as_ptr() as *const f32,- input.len(),- );+ let f32_slice = std::slice::from_raw_parts(input.as_ptr() as *const f32, input.len());return neon_argmax_f32(f32_slice);}}@@ -225,11 +211,7 @@ mod tests {// Compare NEON result to expectedlet neon_result = neon_optimized_argmax(&data);- assert_eq!(- expected, neon_result,- "64k array test failed: expected {}, got {}",- expected, neon_result- );+ assert_eq!(expected, neon_result, "64k array test failed: expected {}, got {}", expected, neon_result);assert_eq!(expected, 42_000, "Expected max should be at index 42_000");}@@ -280,34 +262,15 @@ mod tests {if batch_idx == 0 {println!("128k argmax performance comparison:");- println!(- "NEON: {:.3} ms",- neon_duration.as_secs_f64() * 1000.0- );- println!(- "Simple: {:.3} ms",- simple_duration.as_secs_f64() * 1000.0- );- println!(- "Optimized: {:.3} ms",- optimized_duration.as_secs_f64() * 1000.0- );-- if simple_duration.as_nanos() > 0- && neon_duration.as_nanos() > 0- {- let speedup_simple = simple_duration.as_secs_f64()- / neon_duration.as_secs_f64();- let speedup_optimized = optimized_duration.as_secs_f64()- / neon_duration.as_secs_f64();- println!(- "NEON vs Simple speedup: {:.2}x",- speedup_simple- );- println!(- "NEON vs Optimized speedup: {:.2}x",- speedup_optimized- );+ println!("NEON: {:.3} ms", neon_duration.as_secs_f64() * 1000.0);+ println!("Simple: {:.3} ms", simple_duration.as_secs_f64() * 1000.0);+ println!("Optimized: {:.3} ms", optimized_duration.as_secs_f64() * 1000.0);++ if simple_duration.as_nanos() > 0 && neon_duration.as_nanos() > 0 {+ let speedup_simple = simple_duration.as_secs_f64() / neon_duration.as_secs_f64();+ let speedup_optimized = optimized_duration.as_secs_f64() / neon_duration.as_secs_f64();+ println!("NEON vs Simple speedup: {:.2}x", speedup_simple);+ println!("NEON vs Optimized speedup: {:.2}x", speedup_optimized);}}diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rsindex 763324f34..559046842 100644--- a/crates/uzu/src/backends/metal/backend.rs+++ b/crates/uzu/src/backends/metal/backend.rs@@ -1,7 +1,4 @@-use metal::{- MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,- MTLComputeCommandEncoder,-};+use metal::{MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder};use objc2::{rc::Retained, runtime::ProtocolObject};use super::{MTLContext, MTLError, MetalKernels};diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsindex f1ccb6e94..6bc5a4d0a 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -2,22 +2,17 @@ use crate::backends::metal::Metal;use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};use super::{- KernelDataType, MTLCommandBuffer, MTLCommandQueue, MTLContext,- ProtocolObject, Retained,+ KernelDataType, MTLCommandBuffer, MTLCommandQueue, MTLContext, ProtocolObject, Retained,compilation_parameters::CompilationConfig,encodable_block::{- Activation, ClassifierLayer, ClassifierPredictionHead, Normalization,- Pooling, Rope,+ Activation, ClassifierLayer, ClassifierPredictionHead, Normalization, Pooling, Rope,transformer_layer::{embed_block, linear_block},},kernel::dsl::SigmoidMetalKernel,};use crate::{DataType,- backends::{- common::Context, common::kernel::SigmoidKernel,- metal::error::ClassifierError,- },+ backends::{common::Context, common::kernel::SigmoidKernel, metal::error::ClassifierError},config::{ClassifierModelConfig, ModelMetadata},encodable_block::EncodableBlock,forward_pass::{@@ -54,34 +49,22 @@ pub struct ClassifierContext {impl ClassifierContext {pub fn new(model_path: &Path) -> Result<Self, Error> {- let context =- MTLContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;+ let context = MTLContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;- let command_buffer = context- .create_command_buffer()- .map_err(|_| Error::UnableToCreateMetalContext)?;+ let command_buffer = context.create_command_buffer().map_err(|_| Error::UnableToCreateMetalContext)?;let config_path = model_path.join("config.json");if !config_path.exists() {return Err(Error::UnableToLoadConfig);}- let config_file =- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;let model_metadata: ModelMetadata =- serde_json::from_reader(BufReader::new(config_file))- .map_err(|_| Error::UnableToLoadConfig)?;+ serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;- let classifier_model_config = model_metadata- .model_config- .as_classifier()- .ok_or(Error::UnableToLoadConfig)?;-- let decoder_config = Rc::new(- classifier_model_config- .model_config- .to_decoder_config()- .map_err(|_| Error::UnableToLoadConfig)?,- );+ let classifier_model_config = model_metadata.model_config.as_classifier().ok_or(Error::UnableToLoadConfig)?;++ let decoder_config =+ Rc::new(classifier_model_config.model_config.to_decoder_config().map_err(|_| Error::UnableToLoadConfig)?);let model_shape = ModelShape::from_decoder_config(&decoder_config);let compilation_config = Rc::new(CompilationConfig::default());@@ -89,23 +72,14 @@ impl ClassifierContext {if !weights_path.exists() {return Err(Error::UnableToLoadWeights);}- let weights_file = File::open(&weights_path)- .map_err(|_| Error::UnableToLoadWeights)?;- let loader = ParameterLoader::new(&weights_file, context.as_ref())- .map_err(|_| Error::UnableToLoadWeights)?;+ let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let root_loader_view = loader.tree();- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(- context.as_ref(),- &decoder_config,- &model_shape,- )));- let transformer_tree =- root_loader_view.subtree("transformer").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound(- "transformer".to_string(),- ))- })?;+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));+ let transformer_tree = root_loader_view+ .subtree("transformer")+ .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("transformer".to_string())))?;{let mut shared_bufs = shared_buffers.borrow_mut();@@ -128,17 +102,13 @@ impl ClassifierContext {let embed = embed_block(&decoder_config, &context, &root_loader_view);- let global_rope =- Self::create_rope_block(&context, data_type, RopeType::Global)- .map_err(Error::Classifier)?;+ let global_rope = Self::create_rope_block(&context, data_type, RopeType::Global).map_err(Error::Classifier)?;let local_rope = classifier_model_config.model_config.transformer_config.local_rope_config.as_ref()- .map(|_| {- Self::create_rope_block(&context, data_type, RopeType::Local)- })+ .map(|_| Self::create_rope_block(&context, data_type, RopeType::Local)).transpose().map_err(Error::Classifier)?;@@ -150,9 +120,7 @@ impl ClassifierContext {.enumerate().map(|(layer_index, layer_config)| {let mut rope = global_rope.clone();- let attn = layer_config- .attention_config()- .ok_or(ClassifierError::NonAttentionMixer)?;+ let attn = layer_config.attention_config().ok_or(ClassifierError::NonAttentionMixer)?;if attn.sliding_window_size.is_some() {if let Some(local_rope_block) = local_rope.clone() {@@ -161,32 +129,18 @@ impl ClassifierContext {}let num_heads = attn.num_heads.ok_or_else(|| {- ClassifierError::MissingConfigField(format!(- "num_heads in layer {}",- layer_index- ))- })?;- let head_dim = attn.head_dim.ok_or_else(|| {- ClassifierError::MissingConfigField(format!(- "head_dim in layer {}",- layer_index- ))+ ClassifierError::MissingConfigField(format!("num_heads in layer {}", layer_index))})?;+ let head_dim = attn+ .head_dim+ .ok_or_else(|| ClassifierError::MissingConfigField(format!("head_dim in layer {}", layer_index)))?;let num_groups = attn.num_groups.ok_or_else(|| {- ClassifierError::MissingConfigField(format!(- "num_groups in layer {}",- layer_index- ))+ ClassifierError::MissingConfigField(format!("num_groups in layer {}", layer_index))})?;let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_index))- .map_err(|_| {- ClassifierError::WeightSubtreeNotFound(format!(- "layers.{}",- layer_index- ))- })?;+ .map_err(|_| ClassifierError::WeightSubtreeNotFound(format!("layers.{}", layer_index)))?;Ok(ClassifierLayer::new(context.clone(),@@ -194,10 +148,7 @@ impl ClassifierContext {compilation_config.clone(),layer_index,classifier_model_config.model_config.model_dim,- classifier_model_config- .model_config- .transformer_config- .hidden_dim,+ classifier_model_config.model_config.transformer_config.hidden_dim,num_heads,head_dim,num_groups,@@ -210,47 +161,26 @@ impl ClassifierContext {.map_err(Error::Classifier)?.into_boxed_slice();- let output_norm_tree =- transformer_tree.subtree("output_norm").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound(- "output_norm".to_string(),- ))- })?;+ let output_norm_tree = transformer_tree+ .subtree("output_norm")+ .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("output_norm".to_string())))?;let output_norm = Normalization::new(&context,data_type,- classifier_model_config- .model_config- .transformer_config- .output_norm_config- .clone(),+ classifier_model_config.model_config.transformer_config.output_norm_config.clone(),ArrayId::Main,ArrayId::Main,&output_norm_tree,)- .map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(format!(- "output norm: {:?}",- e- )))- })?;+ .map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("output norm: {:?}", e))))?;- let context_length =- classifier_model_config.model_config.context_length;- let scratch_buffers = ScratchBuffers::new(- context.as_ref(),- &decoder_config,- &model_shape,- context_length,- context_length,- );+ let context_length = classifier_model_config.model_config.context_length;+ let scratch_buffers =+ ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, context_length, context_length);- let embedding_norm_tree =- root_loader_view.subtree("embedding_norm").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound(- "embedding_norm".to_string(),- ))- })?;+ let embedding_norm_tree = root_loader_view+ .subtree("embedding_norm")+ .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("embedding_norm".to_string())))?;let embedding_norm = Normalization::new(&context,data_type,@@ -259,33 +189,20 @@ impl ClassifierContext {ArrayId::Main,&embedding_norm_tree,)- .map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(format!(- "embedding norm: {:?}",- e- )))- })?;+ .map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("embedding norm: {:?}", e))))?;let model_dim = classifier_model_config.model_config.model_dim;let num_labels = classifier_model_config.model_config.num_labels;- let prediction_head_config =- &classifier_model_config.model_config.prediction_head_config;- let prediction_head_tree =- root_loader_view.subtree("prediction_head").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound(- "prediction_head".to_string(),- ))- })?;+ let prediction_head_config = &classifier_model_config.model_config.prediction_head_config;+ let prediction_head_tree = root_loader_view+ .subtree("prediction_head")+ .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head".to_string())))?;- let prediction_head_data_type: DataType =- prediction_head_config.dense_config.activation_precision().into();+ let prediction_head_data_type: DataType = prediction_head_config.dense_config.activation_precision().into();- let prediction_head_dense_tree =- prediction_head_tree.subtree("dense").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound(- "prediction_head.dense".to_string(),- ))- })?;+ let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))+ })?;let prediction_head_dense = linear_block::<1>(&prediction_head_config.dense_config,prediction_head_config.use_dense_bias,@@ -306,18 +223,13 @@ impl ClassifierContext {ArrayId::ClassifierPredictionHeadDense,).map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(- format!("prediction head activation: {:?}", e),- ))+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))})?,);- let prediction_head_norm_tree =- prediction_head_tree.subtree("norm").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound(- "prediction_head.norm".to_string(),- ))- })?;+ let prediction_head_norm_tree = prediction_head_tree.subtree("norm").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.norm".to_string()))+ })?;let prediction_head_norm = Normalization::new(&context,prediction_head_data_type,@@ -327,18 +239,12 @@ impl ClassifierContext {&prediction_head_norm_tree,).map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(format!(- "prediction head norm: {:?}",- e- )))+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head norm: {:?}", e)))})?;- let prediction_head_readout_tree =- prediction_head_tree.subtree("readout").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound(- "prediction_head.readout".to_string(),- ))- })?;+ let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))+ })?;let prediction_head_final_linear = linear_block::<1>(&prediction_head_config.readout_config,true,@@ -350,19 +256,13 @@ impl ClassifierContext {ArrayId::ClassifierPredictionHeadLogits,).map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(format!(- "prediction head readout: {:?}",- e- )))+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head readout: {:?}", e)))})?;- let sigmoid_kernel =- SigmoidMetalKernel::new(&context, data_type.into()).map_err(- |e| {- eprintln!("Failed to create sigmoid kernel: {:?}", e);- Error::UnableToCreateMetalContext- },- )?;+ let sigmoid_kernel = SigmoidMetalKernel::new(&context, data_type.into()).map_err(|e| {+ eprintln!("Failed to create sigmoid kernel: {:?}", e);+ Error::UnableToCreateMetalContext+ })?;let pooling = Box::new(Pooling::new(@@ -379,9 +279,7 @@ impl ClassifierContext {let prediction_head = Box::new(ClassifierPredictionHead::new(prediction_head_dense.map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(- format!("prediction head dense: {:?}", e),- ))+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))})?,prediction_head_activation,Box::new(prediction_head_norm),@@ -416,24 +314,14 @@ impl ClassifierContext {let kernel_data_type: KernelDataType = data_type.into();let rotation: Box<dyn EncodableBlock<Metal>> = Box::new(- Rope::new(mtl_context, kernel_data_type, rope_type).map_err(- |e| {- ClassifierError::KernelCreationFailed(format!(- "RoPE: {:?}",- e- ))- },- )?,+ Rope::new(mtl_context, kernel_data_type, rope_type)+ .map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?,);Ok(Rc::new(rotation))}pub fn reset_command_buffer(&mut self) {- self.command_buffer = self- .mtl_context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();+ self.command_buffer =+ self.mtl_context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();}}diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex 5741b68a9..3f03a900b 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -10,13 +10,9 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {fn with_compute_encoder<T>(&self,- callback: impl FnOnce(- &<Self::Backend as crate::backends::common::Backend>::ComputeEncoder,- ) -> T,+ callback: impl FnOnce(&<Self::Backend as crate::backends::common::Backend>::ComputeEncoder) -> T,) -> T {- let encoder = self- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = self.new_compute_command_encoder().expect("Failed to create compute command encoder");let ret = callback(&encoder);@@ -27,13 +23,9 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {fn with_copy_encoder<T>(&self,- callback: impl FnOnce(- &<Self::Backend as crate::backends::common::Backend>::CopyEncoder,- ) -> T,+ callback: impl FnOnce(&<Self::Backend as crate::backends::common::Backend>::CopyEncoder) -> T,) -> T {- let encoder = self- .new_blit_command_encoder()- .expect("Failed to create blit command encoder");+ let encoder = self.new_blit_command_encoder().expect("Failed to create blit command encoder");let ret = callback(&encoder);diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex be16f6d3b..40169a5af 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -3,14 +3,12 @@ use std::{cell::RefCell, collections::HashMap, env, rc::Rc};use metal::{MTLBuffer, MTLCommandBuffer};use objc2::{rc::Retained, runtime::ProtocolObject};-use super::{- Metal, error::MTLError, kernel, metal_extensions::LibraryPipelineExtensions,-};+use super::{Metal, error::MTLError, kernel, metal_extensions::LibraryPipelineExtensions};use crate::backends::{common::{Allocator, Context},metal::{- MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt,- MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions,+ MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLFunctionConstantValues, MTLLibrary,+ MTLResourceOptions,},};@@ -70,8 +68,7 @@ impl DeviceArchitecture {// Extract generation from architecture name// Format is typically like "Apple M3 Pro" or the GPU name "applegpu_g15d"- let (generation, device_class) =- Self::parse_architecture_info(&arch_string);+ let (generation, device_class) = Self::parse_architecture_info(&arch_string);Self {generation,@@ -100,10 +97,7 @@ impl DeviceArchitecture {DeviceGeneration::Unknown(0)};- let device_class = if arch.contains("Max")- || arch.contains("Ultra")- || arch.contains("Pro")- {+ let device_class = if arch.contains("Max") || arch.contains("Ultra") || arch.contains("Pro") {DeviceClass::Desktop} else if arch.contains("iPhone") || arch.contains("iPad") {DeviceClass::Phone@@ -133,9 +127,7 @@ pub struct MTLContext {pub command_queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,pub architecture: DeviceArchitecture,pub library: Retained<ProtocolObject<dyn MTLLibrary>>,- pipeline_cache: RefCell<- HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- >,+ pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,allocator: Allocator<Metal>,}@@ -162,22 +154,17 @@ impl MTLContext {/// TF32 toggle via UZU_TF32 environment variable.pub fn tf32_enabled(&self) -> bool {- env::var("UZU_TF32")- .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))- .unwrap_or(false)+ env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)}pub fn compute_pipeline_state(&self,function_name: &str,constants: Option<&MTLFunctionConstantValues>,- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {// Only cache pipelines without constantsif constants.is_some() {- return self- .library- .compute_pipeline_state(function_name, constants);+ return self.library.compute_pipeline_state(function_name, constants);}self.compute_pipeline_state_cached(function_name, function_name, None)}@@ -187,18 +174,14 @@ impl MTLContext {cache_key: &str,function_name: &str,constants: Option<&MTLFunctionConstantValues>,- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if let Some(pipeline) = self.pipeline_cache.borrow().get(cache_key) {return Ok(pipeline.clone());}- let pipeline =- self.library.compute_pipeline_state(function_name, constants)?;+ let pipeline = self.library.compute_pipeline_state(function_name, constants)?;- self.pipeline_cache- .borrow_mut()- .insert(cache_key.to_string(), pipeline.clone());+ self.pipeline_cache.borrow_mut().insert(cache_key.to_string(), pipeline.clone());Ok(pipeline)}@@ -208,22 +191,16 @@ impl Context for MTLContext {type Backend = Metal;fn new() -> Result<Rc<Self>, MTLError> {- let device: Retained<ProtocolObject<dyn MTLDevice>> =- <dyn metal::MTLDevice>::system_default().ok_or(- MTLError::Generic("cannot open system default device".into()),- )?;+ let device: Retained<ProtocolObject<dyn MTLDevice>> = <dyn metal::MTLDevice>::system_default()+ .ok_or(MTLError::Generic("cannot open system default device".into()))?;let command_queue = device.new_command_queue_with_max_command_buffer_count(1024).ok_or(MTLError::Generic("cannot create command queue".into()))?;- let library =- device.new_library_with_data(kernel::MTLB).map_err(|nserror| {- MTLError::Generic(format!(- "Failed to create Metal library: {}",- nserror- ))- })?;+ let library = device+ .new_library_with_data(kernel::MTLB)+ .map_err(|nserror| MTLError::Generic(format!("Failed to create Metal library: {}", nserror)))?;let architecture = DeviceArchitecture::from_device(&device);@@ -250,11 +227,7 @@ impl Context for MTLContext {.ok_or(MTLError::Generic("cannot create buffer".into()))}- fn create_command_buffer(- &self- ) -> Result<Retained<ProtocolObject<dyn MTLCommandBuffer>>, MTLError> {- self.command_queue- .command_buffer()- .ok_or(MTLError::Generic("cannot create command buffer".into()))+ fn create_command_buffer(&self) -> Result<Retained<ProtocolObject<dyn MTLCommandBuffer>>, MTLError> {+ self.command_queue.command_buffer().ok_or(MTLError::Generic("cannot create command buffer".into()))}}diff --git a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rsindex d984b3959..f7db9dc57 100644--- a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs@@ -1,6 +1,5 @@use crate::backends::metal::{- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, Metal,- ProtocolObject, Retained,+ MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, Metal, ProtocolObject, Retained,};use super::{@@ -46,9 +45,7 @@ impl EncodableBlock<Metal> for Activation {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,_parameters: &EncodingParameters<Metal>,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, _parameters);@@ -84,12 +81,6 @@ impl EncodableBlock<Metal> for Activation {},};- self.kernel.encode(- input_buffer,- output_buffer,- n as u32,- act_type,- encoder,- );+ self.kernel.encode(input_buffer, output_buffer, n as u32, act_type, encoder);}}diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rsindex 8b2479303..ced8da32a 100644--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs@@ -2,11 +2,10 @@use super::{EncodableBlock, Metal};use crate::backends::metal::{- KernelDataType, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,+ KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, ProtocolObject,+ Retained,kernel::attention::{- AttentionError, AttentionGemmArguments, AttentionKernel,- AttentionKernelVariant, AttentionSinglePassArguments,+ AttentionError, AttentionGemmArguments, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,AttentionTwoPassArguments, KVCacheUpdateArguments,},};@@ -67,9 +66,8 @@ impl EncodableBlock<Metal> for Attention {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &compute_encoder, parameters);compute_encoder.end_encoding();@@ -89,13 +87,7 @@ impl EncodableBlock<Metal> for Attention {compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,parameters: &EncodingParameters<Metal>,) {- let (- suffix_length,- num_heads,- head_dim,- num_groups,- max_sequence_length,- ) = {+ let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);let qkv_array = qkv_binding[0].borrow();let suffix_length = qkv_array.shape()[0];@@ -110,8 +102,7 @@ impl EncodableBlock<Metal> for Attention {let num_groups = keys_array.shape()[0];let max_sequence_length = if let Some(_kv) = state.cache_layers() {- let key_cache_binding =- state.arrays(&[ArrayId::Keys(self.layer_index)]);+ let key_cache_binding = state.arrays(&[ArrayId::Keys(self.layer_index)]);let key_cache_array = key_cache_binding[0].borrow();key_cache_array.shape()[1]} else {@@ -119,70 +110,46 @@ impl EncodableBlock<Metal> for Attention {suffix_length};- (- suffix_length,- num_heads,- head_dim,- num_groups,- max_sequence_length,- )+ (suffix_length, num_heads, head_dim, num_groups, max_sequence_length)};- let (segment_prefix_length, window_length) =- if let Some(cache_layers) = state.cache_layers() {- let cache = cache_layers.borrow();- let layer = cache.data[self.layer_index]- .as_transformer()- .expect("Attention kernel expects transformer layer state");- (- layer.projected_segment_prefix_length(- parameters.projection_step.unwrap_or(0),- ),- layer.window_length(),- )- } else {- // For classifiers without KV cache: no prefix, use configured sliding window- (0, self.sliding_window_size)- };+ let (segment_prefix_length, window_length) = if let Some(cache_layers) = state.cache_layers() {+ let cache = cache_layers.borrow();+ let layer = cache.data[self.layer_index]+ .as_transformer()+ .expect("Attention kernel expects transformer layer state");+ (layer.projected_segment_prefix_length(parameters.projection_step.unwrap_or(0)), layer.window_length())+ } else {+ // For classifiers without KV cache: no prefix, use configured sliding window+ (0, self.sliding_window_size)+ };let use_mask = window_length.is_some();let sequence_length = segment_prefix_length + suffix_length;let gqa_factor = num_heads / num_groups;- let scale =- self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());+ let scale = self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());let gemm_enabled = env_gemm_attention_enabled();- let use_gemm = gemm_enabled- && suffix_length > 8- && matches!(head_dim, 64 | 128 | 256);+ let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);if !gemm_enabled {static PRINT_ONCE: std::sync::Once = std::sync::Once::new();PRINT_ONCE.call_once(|| {- eprintln!(- "[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION"- );+ eprintln!("[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION");});}let variant = if use_gemm {AttentionKernelVariant::SinglePass} else {- self.kernel.choose_variant(- sequence_length,- head_dim,- self.is_causal,- use_mask,- )+ self.kernel.choose_variant(sequence_length, head_dim, self.is_causal, use_mask)};let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);let qkv_binding = state.arrays(&[ArrayId::QKV]);- let attention_bias_binding =- state.hashmaps(&[HashMapId::AttentionBias]);- let attention_output_binding =- state.arrays(&[ArrayId::AttentionOutput]);+ let attention_bias_binding = state.hashmaps(&[HashMapId::AttentionBias]);+ let attention_output_binding = state.arrays(&[ArrayId::AttentionOutput]);let mask_kv_seq_stride = 1;let mask_q_seq_stride = sequence_length as i32;@@ -202,10 +169,8 @@ impl EncodableBlock<Metal> for Attention {// Get KV cache buffers only if KV cache exists (LLM mode)let has_kv_cache = state.cache_layers().is_some();let (key_cache_buffer, value_cache_buffer) = if has_kv_cache {- let key_cache_binding =- state.arrays(&[ArrayId::Keys(self.layer_index)]);- let value_cache_binding =- state.arrays(&[ArrayId::Values(self.layer_index)]);+ let key_cache_binding = state.arrays(&[ArrayId::Keys(self.layer_index)]);+ let value_cache_binding = state.arrays(&[ArrayId::Values(self.layer_index)]);let key_cache_array = key_cache_binding[0].borrow_mut();let key_cache_buf = key_cache_array.buffer().clone();@@ -217,10 +182,8 @@ impl EncodableBlock<Metal> for Attention {} else {// For classifiers, we need a values buffer with [num_groups, suffix_length, head_dim] layout.// Use the KV cache update kernel to extract values from QKV into a dedicated extracted_values buffer.- let extracted_values_binding =- state.arrays(&[ArrayId::ExtractedValues]);- let extracted_values_array =- extracted_values_binding[0].borrow_mut();+ let extracted_values_binding = state.arrays(&[ArrayId::ExtractedValues]);+ let extracted_values_array = extracted_values_binding[0].borrow_mut();let extracted_values_buf = extracted_values_array.buffer().clone();// Reuse the KV cache update kernel to write values into extracted_values_buf.@@ -261,10 +224,7 @@ impl EncodableBlock<Metal> for Attention {array_ref.buffer().clone()}).unwrap_or_else(|| {- panic!(- "Attention bias buffer not found for window length {:?}",- window_length- );+ panic!("Attention bias buffer not found for window length {:?}", window_length);}),)} else {@@ -284,9 +244,7 @@ impl EncodableBlock<Metal> for Attention {let maxs_array = maxs_binding[0].borrow_mut();let maxs_buffer = maxs_array.buffer();- let sinks_buffer = sinks_binding- .as_ref()- .map(|binding| binding[0].borrow().buffer().clone());+ let sinks_buffer = sinks_binding.as_ref().map(|binding| binding[0].borrow().buffer().clone());// Only update KV cache for LLM mode (not for classifiers)if has_kv_cache {@@ -367,10 +325,7 @@ impl EncodableBlock<Metal> for Attention {is_causal: self.is_causal,},) {- eprintln!(- "Failed to encode single-pass attention: {:?}",- e- );+ eprintln!("Failed to encode single-pass attention: {:?}", e);}},AttentionKernelVariant::TwoPass => {diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rsindex a258bd31a..286c09a89 100644--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs@@ -3,22 +3,14 @@ use std::rc::Rc;#[cfg(not(feature = "tracing"))]use crate::backends::metal::MTLCommandEncoder;-use crate::backends::metal::{- MTLCommandBuffer, MTLComputeCommandEncoder, ProtocolObject, Retained,-};+use crate::backends::metal::{MTLCommandBuffer, MTLComputeCommandEncoder, ProtocolObject, Retained};use objc2::rc::autoreleasepool;-use super::{- Attention, EncodableBlock, Normalization, QKNorm, TensorAddSwap,- TensorCopy, transformer_layer,-};+use super::{Attention, EncodableBlock, Normalization, QKNorm, TensorAddSwap, TensorCopy, transformer_layer};use crate::forward_pass::state::{ArrayId, ForwardPassState};use crate::{DataType,- backends::metal::{- MTLContext, compilation_parameters::CompilationConfig,- kernel::KernelDataType,- },+ backends::metal::{MTLContext, compilation_parameters::CompilationConfig, kernel::KernelDataType},config::TransformerLayerConfig,encodable_block::EncodingParameters,parameters::ParameterTree,@@ -60,32 +52,18 @@ impl ClassifierLayer {) -> Self {autoreleasepool(|_| {let ctx = &*mtl_context; // Reference for functions expecting &MTLContext- let attention_config = layer_config- .mixer_config- .as_attention()- .expect("Classifier layers must use attention");- let intermediate_data_type: DataType = attention_config- .qkv_projection_config- .activation_precision()- .into();- let kernel_data_type: KernelDataType =- intermediate_data_type.into();-- let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>> =- Box::new(- TensorCopy::new(- ctx,- kernel_data_type,- vec![ArrayId::Main, ArrayId::Shortcut]- .into_boxed_slice(),- )+ let attention_config =+ layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");+ let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();+ let kernel_data_type: KernelDataType = intermediate_data_type.into();++ let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>> = Box::new(+ TensorCopy::new(ctx, kernel_data_type, vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice()).unwrap(),- );+ );let pre_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =- if let Some(norm_config) =- &layer_config.pre_attention_norm_config- {+ if let Some(norm_config) = &layer_config.pre_attention_norm_config {if layer_loader.subtree("pre_mixer_norm").is_ok() {Some(Box::new(Normalization::new(@@ -94,13 +72,9 @@ impl ClassifierLayer {norm_config.clone(),ArrayId::Main,ArrayId::Main,- &layer_loader- .subtree("pre_mixer_norm")- .unwrap(),+ &layer_loader.subtree("pre_mixer_norm").unwrap(),)- .expect(- "Failed to create pre-attention norm kernel",- ),+ .expect("Failed to create pre-attention norm kernel"),))} else {None@@ -113,11 +87,7 @@ impl ClassifierLayer {&attention_config.qkv_projection_config,attention_config.has_qkv_biases,model_dim,- [- num_heads * head_dim,- num_groups * head_dim,- num_groups * head_dim,- ],+ [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],ctx,&layer_loader.subtree("mixer.qkv_projection").unwrap(),ArrayId::Main,@@ -126,9 +96,7 @@ impl ClassifierLayer {.expect("Failed to create qkv projection");let qk_norm: Option<Box<dyn EncodableBlock<Metal>>> =- if attention_config.query_norm_config.is_some()- || attention_config.key_norm_config.is_some()- {+ if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {match QKNorm::new(ctx,intermediate_data_type,@@ -141,10 +109,7 @@ impl ClassifierLayer {head_dim,) {Ok(norm) => Some(Box::new(norm)),- Err(e) => panic!(- "Failed to create QK norm kernel for layer {}: {:?}",- layer_index, e- ),+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),}} else {None@@ -163,9 +128,7 @@ impl ClassifierLayer {.expect("Failed to create out projection");let post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =- if let Some(norm_config) =- &layer_config.post_attention_norm_config- {+ if let Some(norm_config) = &layer_config.post_attention_norm_config {Some(Box::new(Normalization::new(ctx,@@ -182,24 +145,14 @@ impl ClassifierLayer {};let mixer_residual_add: Box<dyn EncodableBlock<Metal>> = Box::new(- TensorAddSwap::new(- ctx,- kernel_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),- )- .unwrap(),+ TensorAddSwap::new(ctx, kernel_data_type, vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice())+ .unwrap(),);- let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<Metal>> =- Box::new(- TensorCopy::new(- ctx,- kernel_data_type,- vec![ArrayId::Main, ArrayId::Shortcut]- .into_boxed_slice(),- )+ let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<Metal>> = Box::new(+ TensorCopy::new(ctx, kernel_data_type, vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice()).unwrap(),- );+ );let pre_mlp_norm: Box<dyn EncodableBlock<Metal>> = Box::new(Normalization::new(@@ -253,12 +206,8 @@ impl ClassifierLayer {);let mlp_residual_add: Box<dyn EncodableBlock<Metal>> = Box::new(- TensorAddSwap::new(- ctx,- kernel_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),- )- .unwrap(),+ TensorAddSwap::new(ctx, kernel_data_type, vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice())+ .unwrap(),);Self {@@ -292,9 +241,8 @@ impl EncodableBlock<Metal> for ClassifierLayer {#[cfg(not(feature = "tracing"))]{if self.supports_shared_encoder() {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();return;@@ -302,27 +250,14 @@ impl EncodableBlock<Metal> for ClassifierLayer {}#[cfg(feature = "tracing")]- let layer_traces = state- .traces()- .borrow()- .layer_results- .get(self.layer_index)- .cloned();+ let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().inputs.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());}- self.copy_main_to_shortcut_mixer.encode(- state,- command_buffer,- parameters,- );+ self.copy_main_to_shortcut_mixer.encode(state, command_buffer, parameters);if let Some(ref pre_attn_norm) = self.pre_attention_norm {pre_attn_norm.encode(state, command_buffer, parameters);@@ -354,11 +289,7 @@ impl EncodableBlock<Metal> for ClassifierLayer {self.out_projection.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().attention.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());}if let Some(ref post_attn_norm) = self.post_attention_norm {@@ -376,59 +307,35 @@ impl EncodableBlock<Metal> for ClassifierLayer {self.mixer_residual_add.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().mlp_inputs.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());}- self.copy_main_to_shortcut_mlp.encode(- state,- command_buffer,- parameters,- );+ self.copy_main_to_shortcut_mlp.encode(state, command_buffer, parameters);self.pre_mlp_norm.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().pre_mlp_norm.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());}self.mlp.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().mlp.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());}if let Some(ref post_mlp_norm) = self.post_mlp_norm {post_mlp_norm.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().post_mlp_norm.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());}}self.mlp_residual_add.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().outputs.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());}let _ = parameters;@@ -443,34 +350,18 @@ impl EncodableBlock<Metal> for ClassifierLayer {#[cfg(not(feature = "tracing"))]{self.copy_main_to_shortcut_mixer.supports_shared_encoder()- && self- .pre_attention_norm- .as_ref()- .map(|b| b.supports_shared_encoder())- .unwrap_or(true)+ && self.pre_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)&& self.qkv_projection.supports_shared_encoder()- && self- .qk_norm- .as_ref()- .map(|b| b.supports_shared_encoder())- .unwrap_or(true)+ && self.qk_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)&& self.rope.supports_shared_encoder()&& self.attention.supports_shared_encoder()&& self.out_projection.supports_shared_encoder()- && self- .post_attention_norm- .as_ref()- .map(|b| b.supports_shared_encoder())- .unwrap_or(true)+ && self.post_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)&& self.mixer_residual_add.supports_shared_encoder()&& self.copy_main_to_shortcut_mlp.supports_shared_encoder()&& self.pre_mlp_norm.supports_shared_encoder()&& self.mlp.supports_shared_encoder()- && self- .post_mlp_norm- .as_ref()- .map(|b| b.supports_shared_encoder())- .unwrap_or(true)+ && self.post_mlp_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)&& self.mlp_residual_add.supports_shared_encoder()}}@@ -486,45 +377,35 @@ impl EncodableBlock<Metal> for ClassifierLayer {"encode_with_shared_encoder called on unsupported ClassifierLayer");- self.copy_main_to_shortcut_mixer- .encode_with_shared_encoder(state, encoder, parameters);+ self.copy_main_to_shortcut_mixer.encode_with_shared_encoder(state, encoder, parameters);if let Some(ref pre_attn_norm) = self.pre_attention_norm {- pre_attn_norm- .encode_with_shared_encoder(state, encoder, parameters);+ pre_attn_norm.encode_with_shared_encoder(state, encoder, parameters);}- self.qkv_projection- .encode_with_shared_encoder(state, encoder, parameters);+ self.qkv_projection.encode_with_shared_encoder(state, encoder, parameters);if let Some(ref qk_norm) = self.qk_norm {qk_norm.encode_with_shared_encoder(state, encoder, parameters);}self.rope.encode_with_shared_encoder(state, encoder, parameters);self.attention.encode_with_shared_encoder(state, encoder, parameters);- self.out_projection- .encode_with_shared_encoder(state, encoder, parameters);+ self.out_projection.encode_with_shared_encoder(state, encoder, parameters);if let Some(ref post_attn_norm) = self.post_attention_norm {- post_attn_norm- .encode_with_shared_encoder(state, encoder, parameters);+ post_attn_norm.encode_with_shared_encoder(state, encoder, parameters);}- self.mixer_residual_add- .encode_with_shared_encoder(state, encoder, parameters);+ self.mixer_residual_add.encode_with_shared_encoder(state, encoder, parameters);- self.copy_main_to_shortcut_mlp- .encode_with_shared_encoder(state, encoder, parameters);+ self.copy_main_to_shortcut_mlp.encode_with_shared_encoder(state, encoder, parameters);- self.pre_mlp_norm- .encode_with_shared_encoder(state, encoder, parameters);+ self.pre_mlp_norm.encode_with_shared_encoder(state, encoder, parameters);self.mlp.encode_with_shared_encoder(state, encoder, parameters);if let Some(ref post_mlp_norm) = self.post_mlp_norm {- post_mlp_norm- .encode_with_shared_encoder(state, encoder, parameters);+ post_mlp_norm.encode_with_shared_encoder(state, encoder, parameters);}- self.mlp_residual_add- .encode_with_shared_encoder(state, encoder, parameters);+ self.mlp_residual_add.encode_with_shared_encoder(state, encoder, parameters);}}diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rsindex d56b09207..19578d4d6 100644--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs@@ -7,8 +7,7 @@ use super::{EncodableBlock, LayerExecutables, RMSNorm, Rope};use crate::{DataType, DecoderConfig,backends::metal::{- KernelDataType, MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext,- ProtocolObject, Retained,+ KernelDataType, MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,compilation_parameters::CompilationConfig,encodable_block::transformer_layer::{embed_block, readout_block},},@@ -38,64 +37,43 @@ impl Decoder {root_weight_loader: &ParameterTree<MTLContext>,compilation_config: Rc<CompilationConfig>,) -> Self {- let decoder_weight_loader = root_weight_loader- .subtree("transformer")- .expect("transformer subtree not found");+ let decoder_weight_loader = root_weight_loader.subtree("transformer").expect("transformer subtree not found");- let embed =- embed_block(&decoder_config, &mtl_context, root_weight_loader);+ let embed = embed_block(&decoder_config, &mtl_context, root_weight_loader);- let readout =- readout_block(&decoder_config, &mtl_context, root_weight_loader);+ let readout = readout_block(&decoder_config, &mtl_context, root_weight_loader);let attention_data_type = Self::attention_data_type(&decoder_config);- let norm_reference_layer = decoder_config- .layer_configs- .as_ref()- .map(|configs| &configs[0])- .unwrap_or(&decoder_config.layer_config);- let norm_data_type: DataType = match &norm_reference_layer.mixer_config- {- MixerConfig::Attention(attention_config) => attention_config- .qkv_projection_config- .activation_precision()- .into(),- MixerConfig::Mamba(mamba_config) => {- mamba_config.in_projection_config.activation_precision().into()+ let norm_reference_layer =+ decoder_config.layer_configs.as_ref().map(|configs| &configs[0]).unwrap_or(&decoder_config.layer_config);+ let norm_data_type: DataType = match &norm_reference_layer.mixer_config {+ MixerConfig::Attention(attention_config) => {+ attention_config.qkv_projection_config.activation_precision().into()+ },+ MixerConfig::Mamba(mamba_config) => mamba_config.in_projection_config.activation_precision().into(),+ MixerConfig::ShortConv(short_conv_config) => {+ short_conv_config.in_projection_config.activation_precision().into()},- MixerConfig::ShortConv(short_conv_config) => short_conv_config- .in_projection_config- .activation_precision()- .into(),};let global_rope = if decoder_config.global_rope_config.is_some() {- attention_data_type.as_ref().map(|data_type| {- Self::create_rope_block(- &mtl_context,- (*data_type).into(),- RopeType::Global,- )- })+ attention_data_type+ .as_ref()+ .map(|data_type| Self::create_rope_block(&mtl_context, (*data_type).into(), RopeType::Global))} else {None};let local_rope = if decoder_config.local_rope_config.is_some() {- attention_data_type.as_ref().map(|data_type| {- Self::create_rope_block(- &mtl_context,- (*data_type).into(),- RopeType::Local,- )- })+ attention_data_type+ .as_ref()+ .map(|data_type| Self::create_rope_block(&mtl_context, (*data_type).into(), RopeType::Local))} else {None};let model_shape = ModelShape::from_decoder_config(&decoder_config);- let sliding_window_sizes =- model_shape.sliding_window_length_per_layer.clone();+ let sliding_window_sizes = model_shape.sliding_window_length_per_layer.clone();let layers = (0..decoder_config.num_layers).map(|layer_index| {@@ -107,13 +85,10 @@ impl Decoder {let layer_type = model_shape.layer_type(layer_index);let rope_for_layer = match layer_type {DecoderLayerType::Transformer => {- let mut rope_block = global_rope.clone().expect(- "Global rope missing for transformer layer",- );- if let (Some(_), Some(local_rope_block)) = (- sliding_window_sizes[layer_index],- local_rope.clone(),- ) {+ let mut rope_block = global_rope.clone().expect("Global rope missing for transformer layer");+ if let (Some(_), Some(local_rope_block)) =+ (sliding_window_sizes[layer_index], local_rope.clone())+ {rope_block = local_rope_block;}Some(rope_block)@@ -126,9 +101,7 @@ impl Decoder {} => None,};- let layer_loader = decoder_weight_loader- .subtree(&format!("layers.{}", layer_index))- .unwrap();+ let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();LayerExecutables::new(mtl_context.clone(),@@ -176,10 +149,8 @@ impl Decoder {kernel_data_type: KernelDataType,rope_type: RopeType,) -> Rc<Box<dyn EncodableBlock<Metal>>> {- let rotation: Box<dyn EncodableBlock<Metal>> = Box::new(- Rope::new(mtl_context, kernel_data_type, rope_type)- .expect("Failed to create Rope"),- );+ let rotation: Box<dyn EncodableBlock<Metal>> =+ Box::new(Rope::new(mtl_context, kernel_data_type, rope_type).expect("Failed to create Rope"));Rc::new(rotation)}@@ -190,12 +161,9 @@ impl Decoder {.as_ref().map(|configs| &configs[layer_index]).unwrap_or(&decoder_config.layer_config);- layer_config.attention_config().map(|attention_config| {- attention_config- .qkv_projection_config- .activation_precision()- .into()- })+ layer_config+ .attention_config()+ .map(|attention_config| attention_config.qkv_projection_config.activation_precision().into())})}}@@ -230,22 +198,14 @@ impl EncodableBlock<Metal> for Decoder {#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(- command_buffer,- ArrayId::Main,- traces.borrow().output_norm.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());}self.readout.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(- command_buffer,- ArrayId::Logits,- traces.borrow().logits.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Logits, traces.borrow().logits.clone());}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rsindex 1085df36a..c07daece3 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs@@ -7,9 +7,8 @@ use crate::{backends::{common::kernel::FullPrecisionEmbeddingLookupKernel,metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,- Retained, kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,+ ProtocolObject, Retained, kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,},},encodable_block::EncodingParameters,@@ -34,41 +33,31 @@ impl FullPrecisionEmbeddingLookup {input_scale: Option<f32>,parameter_tree: &ParameterTree<MTLContext>,) -> Result<Self, EmbeddingError> {- let kernel = FullPrecisionEmbeddingLookupMetalKernel::new(- mtl_context,- data_type.into(),- )?;+ let kernel = FullPrecisionEmbeddingLookupMetalKernel::new(mtl_context, data_type.into())?;let weights = match parameter_tree.leaf("weights") {Ok(weights) => weights,Err(_) => parameter_tree.leaf("input_weights").map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load weights: {:?}",- e- )))+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e)))})?,};if weights.shape() != [vocab_size, model_dim] {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Embedding lookup weights shape mismatch: got {:?}, \+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Embedding lookup weights shape mismatch: got {:?}, \expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim- ),- )));+ weights.shape(),+ vocab_size,+ model_dim+ ))));}if weights.data_type() != data_type {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Weights dtype mismatch: got {:?}, expected {:?}",- weights.data_type(),- data_type- ),- )));+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Weights dtype mismatch: got {:?}, expected {:?}",+ weights.data_type(),+ data_type+ ))));}let weights_buffer = weights.buffer().to_owned();@@ -90,9 +79,7 @@ impl EncodableBlock<Metal> for FullPrecisionEmbeddingLookup {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rsindex b951ac91f..d1fd4f083 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs@@ -1,8 +1,7 @@use std::cell::RefCell;use crate::backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,- ProtocolObject, Retained,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained,};use super::{@@ -35,46 +34,37 @@ impl FullPrecisionEmbeddingReadout {model_dim: usize,parameter_tree: &ParameterTree<MTLContext>,) -> Result<Self, EmbeddingError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(EmbeddingError::UnsupportedDataType(data_type));}let weights = match parameter_tree.leaf("weights") {Ok(weights) => weights,Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load weights: {:?}",- e- )))+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e)))})?,};if weights.shape() != [vocab_size, model_dim] {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim- ),- )));+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim+ ))));}if weights.data_type() != data_type {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Weights dtype mismatch: got {:?}, expected {:?}",- weights.data_type(),- data_type- ),- )));+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Weights dtype mismatch: got {:?}, expected {:?}",+ weights.data_type(),+ data_type+ ))));}let weights_buffer = weights.buffer().to_owned();- let mut kernel =- MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;+ let mut kernel = MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;kernel.precompile(_mtl_context).map_err(EmbeddingError::MetalError)?;Ok(Self {@@ -93,9 +83,7 @@ impl EncodableBlock<Metal> for FullPrecisionEmbeddingReadout {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rsindex a8bd084f6..d12da06a1 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs@@ -7,9 +7,8 @@ use crate::{backends::{common::{Context, kernel::QuantizedEmbeddingLookupKernel},metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,- Retained, kernel::dsl::QuantizedEmbeddingLookupMetalKernel,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,+ ProtocolObject, Retained, kernel::dsl::QuantizedEmbeddingLookupMetalKernel,},},config::QuantizationMode,@@ -96,110 +95,80 @@ impl QuantizedEmbeddingLookup {) -> Result<Self, EmbeddingError> {let packing_divisor = mode.packing_divisor();- let kernel = QuantizedEmbeddingLookupMetalKernel::new(- mtl_context,- data_type.into(),- )?;+ let kernel = QuantizedEmbeddingLookupMetalKernel::new(mtl_context, data_type.into())?;// Load weights [vocab_size, model_dim/packing_divisor] as storage_type- let weights = parameter_tree.leaf(weights_name).map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load weights: {:?}",- e- )))- })?;+ let weights = parameter_tree+ .leaf(weights_name)+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e))))?;if weights.data_type() != mode.storage_type() {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Expected packed weights of type {:?}, got {:?}",- mode.storage_type(),- weights.data_type()- ),- )));+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Expected packed weights of type {:?}, got {:?}",+ mode.storage_type(),+ weights.data_type()+ ))));}// Load scales [vocab_size, num_groups]- let scales = parameter_tree.leaf(scales_name).map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load scales: {:?}",- e- )))- })?;+ let scales = parameter_tree+ .leaf(scales_name)+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load scales: {:?}", e))))?;// Validate shapes and typeslet num_groups = (model_dim + group_size - 1) / group_size;if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim / packing_divisor- ),- )));+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim / packing_divisor+ ))));}if scales.shape() != [vocab_size, num_groups] {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",- scales.shape(),- vocab_size,- num_groups- ),- )));+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",+ scales.shape(),+ vocab_size,+ num_groups+ ))));}if scales.data_type() != data_type {- return Err(EmbeddingError::UnsupportedDataType(- scales.data_type(),- ));+ return Err(EmbeddingError::UnsupportedDataType(scales.data_type()));}// Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =- match parameter_tree.leaf(biases_name) {- Ok(deq_biases) => {- if deq_biases.shape() != [vocab_size, num_groups] {- return Err(EmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",- deq_biases.shape(),- vocab_size,- num_groups- )),- ));- }- if deq_biases.data_type() != data_type {- return Err(EmbeddingError::UnsupportedDataType(- deq_biases.data_type(),- ));- }- deq_biases.buffer().to_owned().into()- },- Err(_) => {- let elem_size: usize = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(EmbeddingError::UnsupportedDataType(- other,- ));- },- };- let size_bytes = vocab_size * num_groups * elem_size;- let buf = mtl_context- .create_buffer(size_bytes)- .expect("Failed to allocate buffer");- unsafe {- std::ptr::write_bytes(- metal::MTLBuffer::contents(&*buf).as_ptr(),- 0,- size_bytes as usize,- );- }- buf- },- };+ let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = match parameter_tree.leaf(biases_name) {+ Ok(deq_biases) => {+ if deq_biases.shape() != [vocab_size, num_groups] {+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",+ deq_biases.shape(),+ vocab_size,+ num_groups+ ))));+ }+ if deq_biases.data_type() != data_type {+ return Err(EmbeddingError::UnsupportedDataType(deq_biases.data_type()));+ }+ deq_biases.buffer().to_owned().into()+ },+ Err(_) => {+ let elem_size: usize = match data_type {+ DataType::F16 | DataType::BF16 => 2,+ DataType::F32 => 4,+ other => {+ return Err(EmbeddingError::UnsupportedDataType(other));+ },+ };+ let size_bytes = vocab_size * num_groups * elem_size;+ let buf = mtl_context.create_buffer(size_bytes).expect("Failed to allocate buffer");+ unsafe {+ std::ptr::write_bytes(metal::MTLBuffer::contents(&*buf).as_ptr(), 0, size_bytes as usize);+ }+ buf+ },+ };let weights_buffer = weights.buffer().to_owned().into();let scales_buffer = scales.buffer().to_owned().into();@@ -225,9 +194,7 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingLookup {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rsindex bde63ffbf..c312ed863 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs@@ -7,13 +7,9 @@ use crate::{backends::{common::Context,metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,- Retained,- kernel::quant_matmul::{- QuantizationType, QuantizedMatmulArguments,- QuantizedMatmulKernel,- },+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,+ ProtocolObject, Retained,+ kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},},},config::QuantizationMode,@@ -91,20 +87,14 @@ impl QuantizedEmbeddingReadout {parameter_tree: &ParameterTree<MTLContext>,) -> Result<Self, EmbeddingError> {// Load weights [vocab_size, model_dim/2] as U8- let weights = parameter_tree.leaf(weights_name).map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load weights: {:?}",- e- )))- })?;+ let weights = parameter_tree+ .leaf(weights_name)+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e))))?;// Load scales [vocab_size, num_groups]- let scales = parameter_tree.leaf(scales_name).map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load scales: {:?}",- e- )))- })?;+ let scales = parameter_tree+ .leaf(scales_name)+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load scales: {:?}", e))))?;// Validate shapeslet num_groups = (model_dim + group_size - 1) / group_size;@@ -114,77 +104,58 @@ impl QuantizedEmbeddingReadout {let weights_transposed = weights.shape()[0] == vocab_size;if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim / packing_divisor- ),- )));+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim / packing_divisor+ ))));}if scales.shape() != [vocab_size, num_groups] {- return Err(EmbeddingError::MetalError(MTLError::Generic(- format!(- "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",- scales.shape(),- vocab_size,- num_groups- ),- )));+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",+ scales.shape(),+ vocab_size,+ num_groups+ ))));}if scales.data_type() != data_type {- return Err(EmbeddingError::UnsupportedDataType(- scales.data_type(),- ));+ return Err(EmbeddingError::UnsupportedDataType(scales.data_type()));}// MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =- match parameter_tree.leaf(biases_name) {- Ok(deq_biases) => {- if deq_biases.shape() != [vocab_size, num_groups] {- return Err(EmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",- deq_biases.shape(),- vocab_size,- num_groups- )),- ));- }- if deq_biases.data_type() != data_type {- return Err(EmbeddingError::UnsupportedDataType(- deq_biases.data_type(),- ));- }- deq_biases.buffer().to_owned().into()- },- Err(_) => {- // Allocate zero-initialized biases buffer- let elem_size: usize = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(EmbeddingError::UnsupportedDataType(- other,- ));- },- };- let size_bytes = vocab_size * num_groups * elem_size;- let buf = mtl_context- .create_buffer(size_bytes)- .expect("Failed to allocate buffer");- unsafe {- std::ptr::write_bytes(- metal::MTLBuffer::contents(&*buf).as_ptr(),- 0,- size_bytes as usize,- );- }- buf- },- };+ let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = match parameter_tree.leaf(biases_name) {+ Ok(deq_biases) => {+ if deq_biases.shape() != [vocab_size, num_groups] {+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(+ "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",+ deq_biases.shape(),+ vocab_size,+ num_groups+ ))));+ }+ if deq_biases.data_type() != data_type {+ return Err(EmbeddingError::UnsupportedDataType(deq_biases.data_type()));+ }+ deq_biases.buffer().to_owned().into()+ },+ Err(_) => {+ // Allocate zero-initialized biases buffer+ let elem_size: usize = match data_type {+ DataType::F16 | DataType::BF16 => 2,+ DataType::F32 => 4,+ other => {+ return Err(EmbeddingError::UnsupportedDataType(other));+ },+ };+ let size_bytes = vocab_size * num_groups * elem_size;+ let buf = mtl_context.create_buffer(size_bytes).expect("Failed to allocate buffer");+ unsafe {+ std::ptr::write_bytes(metal::MTLBuffer::contents(&*buf).as_ptr(), 0, size_bytes as usize);+ }+ buf+ },+ };let weights_buffer = weights.buffer().to_owned().into();let scales_buffer = scales.buffer().to_owned().into();@@ -199,12 +170,7 @@ impl QuantizedEmbeddingReadout {QuantizationType::Mlx,weights_transposed,)- .map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to create kernel: {:?}",- e- )))- })?;+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to create kernel: {:?}", e))))?;Ok(Self {kernel,@@ -224,9 +190,7 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();@@ -273,8 +237,6 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {quantization_type: QuantizationType::Mlx,};- self.kernel- .encode(encoder, args)- .expect("Failed to encode quantized embedding readout kernel");+ self.kernel.encode(encoder, args).expect("Failed to encode quantized embedding readout kernel");}}diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rsindex 5f86335d5..34455bca8 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs@@ -5,24 +5,19 @@ use std::rc::Rc;#[cfg(not(feature = "tracing"))]use crate::backends::metal::MTLCommandEncoder;-use crate::backends::metal::{- MTLCommandBuffer, MTLComputeCommandEncoder, ProtocolObject, Retained,-};+use crate::backends::metal::{MTLCommandBuffer, MTLComputeCommandEncoder, ProtocolObject, Retained};use objc2::rc::autoreleasepool;use super::{super::{- Attention, EncodableBlock, MambaMixer, QKNorm, RMSNorm, ShortConvMixer,- TensorAddSwap, TensorCopy, transformer_layer,+ Attention, EncodableBlock, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap, TensorCopy,+ transformer_layer,},MixerExecutables,};use crate::{DataType, DecoderLayerConfig,- backends::metal::{- MTLContext, compilation_parameters::CompilationConfig,- kernel::KernelDataType,- },+ backends::metal::{MTLContext, compilation_parameters::CompilationConfig, kernel::KernelDataType},config::{DecoderLayerType, MixerConfig},encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState},@@ -61,33 +56,17 @@ impl LayerExecutables {) -> Self {autoreleasepool(|_| {let ctx = &*mtl_context; // Reference for functions expecting &MTLContext- let intermediate_data_type: DataType =- match &layer_config.mixer_config {- MixerConfig::Attention(attention) => attention- .qkv_projection_config- .activation_precision()- .into(),- MixerConfig::Mamba(mamba) => {- mamba.in_projection_config.activation_precision().into()- },- MixerConfig::ShortConv(short_conv) => short_conv- .in_projection_config- .activation_precision()- .into(),- };- let kernel_data_type: KernelDataType =- intermediate_data_type.into();+ let intermediate_data_type: DataType = match &layer_config.mixer_config {+ MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),+ MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),+ MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),+ };+ let kernel_data_type: KernelDataType = intermediate_data_type.into();- let copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>> =- Box::new(- TensorCopy::new(- ctx,- kernel_data_type,- vec![ArrayId::Main, ArrayId::Shortcut]- .into_boxed_slice(),- )+ let copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>> = Box::new(+ TensorCopy::new(ctx, kernel_data_type, vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice()).unwrap(),- );+ );let pre_attention_norm: Box<dyn EncodableBlock<Metal>> = Box::new(RMSNorm::new(@@ -103,32 +82,22 @@ impl LayerExecutables {let mixer = match &layer_config.mixer_config {MixerConfig::Attention(attention_config) => {- let rope_block = rope- .clone()- .expect("RoPE encoder missing for attention layer");+ let rope_block = rope.clone().expect("RoPE encoder missing for attention layer");let qkv_projection = transformer_layer::linear_block(&attention_config.qkv_projection_config,attention_config.has_qkv_biases,model_dim,- [- num_heads * head_dim,- num_groups * head_dim,- num_groups * head_dim,- ],+ [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],ctx,- &decoder_layer_loader- .subtree("mixer.qkv_projection")- .unwrap(),+ &decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),ArrayId::Main,ArrayId::QKV,).expect("Failed to create qkv projection");let qk_norm: Option<Box<dyn EncodableBlock<Metal>>> =- if attention_config.query_norm_config.is_some()- || attention_config.key_norm_config.is_some()- {+ if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {match QKNorm::new(ctx,intermediate_data_type,@@ -140,12 +109,8 @@ impl LayerExecutables {num_groups,head_dim,) {- Ok(qk_norm) => Some(Box::new(qk_norm)- as Box<dyn EncodableBlock<Metal>>),- Err(e) => panic!(- "Failed to create QK norm kernel for layer {}: {:?}",- layer_index, e- ),+ Ok(qk_norm) => Some(Box::new(qk_norm) as Box<dyn EncodableBlock<Metal>>),+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),}} else {None@@ -157,9 +122,7 @@ impl LayerExecutables {num_heads * head_dim,[model_dim],ctx,- &decoder_layer_loader- .subtree("mixer.out_projection")- .unwrap(),+ &decoder_layer_loader.subtree("mixer.out_projection").unwrap(),ArrayId::AttentionOutput,ArrayId::Main,)@@ -187,34 +150,32 @@ impl LayerExecutables {}},MixerConfig::Mamba(mamba_config) => {- let mixer: Box<dyn EncodableBlock<Metal>> =- Box::new(MambaMixer::new(- ctx,- layer_type.clone(),- mamba_config.clone(),- compilation_config.clone(),- layer_index,- model_dim,- num_heads,- head_dim,- num_groups,- decoder_layer_loader,- ));+ let mixer: Box<dyn EncodableBlock<Metal>> = Box::new(MambaMixer::new(+ ctx,+ layer_type.clone(),+ mamba_config.clone(),+ compilation_config.clone(),+ layer_index,+ model_dim,+ num_heads,+ head_dim,+ num_groups,+ decoder_layer_loader,+ ));MixerExecutables::StateSpace {mixer,}},MixerConfig::ShortConv(short_conv_config) => {- let mixer: Box<dyn EncodableBlock<Metal>> =- Box::new(ShortConvMixer::new(- ctx,- layer_type.clone(),- short_conv_config.clone(),- compilation_config.clone(),- layer_index,- model_dim,- decoder_layer_loader,- ));+ let mixer: Box<dyn EncodableBlock<Metal>> = Box::new(ShortConvMixer::new(+ ctx,+ layer_type.clone(),+ short_conv_config.clone(),+ compilation_config.clone(),+ layer_index,+ model_dim,+ decoder_layer_loader,+ ));MixerExecutables::ShortConv {mixer,}@@ -222,9 +183,7 @@ impl LayerExecutables {};let post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =- if let Some(norm_config) =- &layer_config.post_attention_norm_config- {+ if let Some(norm_config) = &layer_config.post_attention_norm_config {Some(Box::new(RMSNorm::new(ctx,@@ -232,9 +191,7 @@ impl LayerExecutables {norm_config.clone(),ArrayId::Main,ArrayId::Main,- &decoder_layer_loader- .subtree("post_mixer_norm")- .unwrap(),+ &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),).expect("Failed to create RMS norm kernel"),))@@ -242,16 +199,10 @@ impl LayerExecutables {None};- let main_shortcut_add_swap: Box<dyn EncodableBlock<Metal>> =- Box::new(- TensorAddSwap::new(- ctx,- kernel_data_type,- vec![ArrayId::Shortcut, ArrayId::Main]- .into_boxed_slice(),- )+ let main_shortcut_add_swap: Box<dyn EncodableBlock<Metal>> = Box::new(+ TensorAddSwap::new(ctx, kernel_data_type, vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice()).unwrap(),- );+ );let pre_mlp_norm: Box<dyn EncodableBlock<Metal>> = Box::new(RMSNorm::new(@@ -283,9 +234,7 @@ impl LayerExecutables {norm_config.clone(),ArrayId::Main,ArrayId::Main,- &decoder_layer_loader- .subtree("post_mlp_norm")- .unwrap(),+ &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),).expect("Failed to create RMS norm kernel"),))@@ -320,9 +269,8 @@ impl EncodableBlock<Metal> for LayerExecutables {#[cfg(not(feature = "tracing"))]{if self.supports_shared_encoder() {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();return;@@ -330,20 +278,11 @@ impl EncodableBlock<Metal> for LayerExecutables {}#[cfg(feature = "tracing")]- let layer_traces = state- .traces()- .borrow()- .layer_results- .get(self.layer_index)- .cloned();+ let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().inputs.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());}self.copy_main_to_shortcut.encode(state, command_buffer, parameters);@@ -352,11 +291,7 @@ impl EncodableBlock<Metal> for LayerExecutables {self.pre_attention_norm.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().pre_attention_norm.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());}match &self.mixer {@@ -376,11 +311,7 @@ impl EncodableBlock<Metal> for LayerExecutables {out_projection.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().attention.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());}},MixerExecutables::StateSpace {@@ -389,11 +320,7 @@ impl EncodableBlock<Metal> for LayerExecutables {mixer.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().attention.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());}},MixerExecutables::ShortConv {@@ -402,11 +329,7 @@ impl EncodableBlock<Metal> for LayerExecutables {mixer.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().attention.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());}},}@@ -429,42 +352,26 @@ impl EncodableBlock<Metal> for LayerExecutables {// main = input + attention_result#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().mlp_inputs.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());}self.pre_mlp_norm.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().pre_mlp_norm.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());}self.mlp.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().mlp.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());}if let Some(post_mlp_norm) = &self.post_mlp_norm {post_mlp_norm.encode(state, command_buffer, parameters);#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().post_mlp_norm.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());}}// main = mlp_result@@ -474,11 +381,7 @@ impl EncodableBlock<Metal> for LayerExecutables {// main = input + attention_result + mlp_result#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().outputs.clone(),- );+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());}let _ = parameters;@@ -501,10 +404,7 @@ impl EncodableBlock<Metal> for LayerExecutables {out_projection,} => {qkv_projection.supports_shared_encoder()- && qk_norm- .as_ref()- .map(|b| b.supports_shared_encoder())- .unwrap_or(true)+ && qk_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)&& rope.supports_shared_encoder()&& attention.supports_shared_encoder()&& out_projection.supports_shared_encoder()@@ -520,19 +420,11 @@ impl EncodableBlock<Metal> for LayerExecutables {self.copy_main_to_shortcut.supports_shared_encoder()&& self.pre_attention_norm.supports_shared_encoder()&& mixer_ok- && self- .post_attention_norm- .as_ref()- .map(|b| b.supports_shared_encoder())- .unwrap_or(true)+ && self.post_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)&& self.main_shortcut_add_swap.supports_shared_encoder()&& self.pre_mlp_norm.supports_shared_encoder()&& self.mlp.supports_shared_encoder()- && self- .post_mlp_norm- .as_ref()- .map(|b| b.supports_shared_encoder())- .unwrap_or(true)+ && self.post_mlp_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)}}@@ -547,10 +439,8 @@ impl EncodableBlock<Metal> for LayerExecutables {"encode_with_shared_encoder called on unsupported LayerExecutables");- self.copy_main_to_shortcut- .encode_with_shared_encoder(state, encoder, parameters);- self.pre_attention_norm- .encode_with_shared_encoder(state, encoder, parameters);+ self.copy_main_to_shortcut.encode_with_shared_encoder(state, encoder, parameters);+ self.pre_attention_norm.encode_with_shared_encoder(state, encoder, parameters);match &self.mixer {MixerExecutables::Attention {@@ -560,16 +450,13 @@ impl EncodableBlock<Metal> for LayerExecutables {attention,out_projection,} => {- qkv_projection- .encode_with_shared_encoder(state, encoder, parameters);+ qkv_projection.encode_with_shared_encoder(state, encoder, parameters);if let Some(norm) = qk_norm {norm.encode_with_shared_encoder(state, encoder, parameters);}rope.encode_with_shared_encoder(state, encoder, parameters);- attention- .encode_with_shared_encoder(state, encoder, parameters);- out_projection- .encode_with_shared_encoder(state, encoder, parameters);+ attention.encode_with_shared_encoder(state, encoder, parameters);+ out_projection.encode_with_shared_encoder(state, encoder, parameters);},MixerExecutables::StateSpace {mixer,@@ -584,23 +471,18 @@ impl EncodableBlock<Metal> for LayerExecutables {}if let Some(post_attention_norm) = &self.post_attention_norm {- post_attention_norm- .encode_with_shared_encoder(state, encoder, parameters);+ post_attention_norm.encode_with_shared_encoder(state, encoder, parameters);}- self.main_shortcut_add_swap- .encode_with_shared_encoder(state, encoder, parameters);+ self.main_shortcut_add_swap.encode_with_shared_encoder(state, encoder, parameters);- self.pre_mlp_norm- .encode_with_shared_encoder(state, encoder, parameters);+ self.pre_mlp_norm.encode_with_shared_encoder(state, encoder, parameters);self.mlp.encode_with_shared_encoder(state, encoder, parameters);if let Some(post_mlp_norm) = &self.post_mlp_norm {- post_mlp_norm- .encode_with_shared_encoder(state, encoder, parameters);+ post_mlp_norm.encode_with_shared_encoder(state, encoder, parameters);}- self.main_shortcut_add_swap- .encode_with_shared_encoder(state, encoder, parameters);+ self.main_shortcut_add_swap.encode_with_shared_encoder(state, encoder, parameters);}}diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rsindex 490544054..b5a9fe737 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs@@ -4,8 +4,7 @@ use std::cell::RefCell;use crate::{DataType,backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,Retained,encodable_block::EncodableBlock,kernel::matmul::{MatmulArguments, MatmulKernel},@@ -35,17 +34,16 @@ impl FullPrecisionLinear {input_array_id: ArrayId,output_array_id: ArrayId,) -> Result<Self, MTLError> {- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32)- {+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(MTLError::Generic(format!("Unsupported data type for full precision linear kernel: {:?}",precision)));}- let weights = parameter_tree.leaf("weights").map_err(|e| {- MTLError::Generic(format!("Failed to load weights: {:?}", e))- })?;+ let weights = parameter_tree+ .leaf("weights")+ .map_err(|e| MTLError::Generic(format!("Failed to load weights: {:?}", e)))?;let w_shape = weights.shape();if w_shape != [output_dim, input_dim] {@@ -63,8 +61,7 @@ impl FullPrecisionLinear {)));}- let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =- weights.buffer().to_owned().into();+ let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = weights.buffer().to_owned().into();let bias_buffer = match parameter_tree.leaf("biases") {Ok(biases) => {@@ -82,8 +79,7 @@ impl FullPrecisionLinear {precision)));}- let bias_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =- biases.buffer().to_owned().into();+ let bias_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = biases.buffer().to_owned().into();Some(bias_buffer)},Err(_) => None,@@ -120,9 +116,7 @@ impl EncodableBlock<Metal> for FullPrecisionLinear {let input_buffer = input_array_mut.buffer();let output_buffer = output_array_mut.buffer();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");let args = MatmulArguments {a: input_buffer,@@ -145,9 +139,7 @@ impl EncodableBlock<Metal> for FullPrecisionLinear {};let mut kernel = self.kernel.borrow_mut();- kernel- .encode(state.mtl_context(), &encoder, args)- .expect("Failed to encode matmul kernel");+ kernel.encode(state.mtl_context(), &encoder, args).expect("Failed to encode matmul kernel");encoder.end_encoding();@@ -197,8 +189,6 @@ impl EncodableBlock<Metal> for FullPrecisionLinear {};let mut kernel = self.kernel.borrow_mut();- kernel- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode matmul kernel");+ kernel.encode(state.mtl_context(), encoder, args).expect("Failed to encode matmul kernel");}}diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rsindex 58e96ca3f..8e85a4971 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs@@ -2,10 +2,7 @@ use crate::backends::metal::Metal;use crate::backends::{common::kernel::TensorAddBiasKernel,- metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, ProtocolObject, Retained,- },+ metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained},};use crate::{@@ -15,10 +12,7 @@ use crate::{encodable_block::EncodableBlock,kernel::{dsl::TensorAddBiasMetalKernel,- quant_matmul::{- QuantizationType, QuantizedMatmulArguments,- QuantizedMatmulKernel,- },+ quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},},},config::QuantizationConfig,@@ -53,19 +47,16 @@ impl QuantizedLinear {) -> Result<Self, MTLError> {let kernel_data_type: DataType = config.activation_precision.into();- if !matches!(- kernel_data_type,- DataType::F16 | DataType::BF16 | DataType::F32- ) {+ if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(MTLError::Generic(format!("Unsupported data type for quantized kernel: {:?}",kernel_data_type)));}- let weights = parameter_tree.leaf("weights").map_err(|e| {- MTLError::Generic(format!("Failed to load weights: {:?}", e))- })?;+ let weights = parameter_tree+ .leaf("weights")+ .map_err(|e| MTLError::Generic(format!("Failed to load weights: {:?}", e)))?;let packing_divisor = config.weight_quantization_mode.packing_divisor();let storage_type = config.weight_quantization_mode.storage_type();@@ -78,9 +69,8 @@ impl QuantizedLinear {)));}- let scales = parameter_tree.leaf("scales").map_err(|e| {- MTLError::Generic(format!("Failed to load scales: {:?}", e))- })?;+ let scales =+ parameter_tree.leaf("scales").map_err(|e| MTLError::Generic(format!("Failed to load scales: {:?}", e)))?;let k_g = (input_dim + config.group_size - 1) / config.group_size;let w_shape = weights.shape();@@ -117,23 +107,16 @@ impl QuantizedLinear {kernel_data_type)));}- let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =- scales.buffer().clone();- let biases_buf: Retained<ProtocolObject<dyn MTLBuffer>> =- deq_biases.buffer().clone();+ let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = scales.buffer().clone();+ let biases_buf: Retained<ProtocolObject<dyn MTLBuffer>> = deq_biases.buffer().clone();(QuantizationType::Mlx, biases_buf, scales_buffer)},Err(_) => {- let zero_points =- parameter_tree.leaf("zero_points").map_err(|e| {- MTLError::Generic(format!(- "Failed to load zero_points: {:?}",- e- ))- })?;+ let zero_points = parameter_tree+ .leaf("zero_points")+ .map_err(|e| MTLError::Generic(format!("Failed to load zero_points: {:?}", e)))?;let zp_shape = zero_points.shape();- let expected_zp_entries =- (k_g + packing_divisor - 1) / packing_divisor;+ let expected_zp_entries = (k_g + packing_divisor - 1) / packing_divisor;if !(w_shape == [output_dim, input_dim / packing_divisor]&& s_shape == [output_dim, k_g]&& zp_shape == [output_dim, expected_zp_entries])@@ -155,46 +138,38 @@ impl QuantizedLinear {storage_type)));}- let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =- scales.buffer().clone();- let zps_buf: Retained<ProtocolObject<dyn MTLBuffer>> =- zero_points.buffer().clone();+ let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = scales.buffer().clone();+ let zps_buf: Retained<ProtocolObject<dyn MTLBuffer>> = zero_points.buffer().clone();(QuantizationType::ZeroPoint, zps_buf, scales_buffer)},}};- let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =- weights.buffer().to_owned().into();+ let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = weights.buffer().to_owned().into();// Optional trainable bias support- let (bias_add_kernel, biases_buffer) =- match parameter_tree.leaf("biases") {- Ok(biases) => {- if biases.shape() != [output_dim] {- return Err(MTLError::Generic(format!(- "Bias shape mismatch: got {:?}, expected [{:?}]",- biases.shape(),- output_dim- )));- }- if biases.data_type() != kernel_data_type {- return Err(MTLError::Generic(format!(- "Bias dtype mismatch: got {:?}, expected {:?}",- biases.data_type(),- kernel_data_type- )));- }- let bias_add_kernel = Some(TensorAddBiasMetalKernel::new(- mtl_context,- kernel_data_type,- )?);- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =- biases.buffer().to_owned().into();- (bias_add_kernel, Some(biases_buffer))- },- Err(_) => (None, None),- };+ let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf("biases") {+ Ok(biases) => {+ if biases.shape() != [output_dim] {+ return Err(MTLError::Generic(format!(+ "Bias shape mismatch: got {:?}, expected [{:?}]",+ biases.shape(),+ output_dim+ )));+ }+ if biases.data_type() != kernel_data_type {+ return Err(MTLError::Generic(format!(+ "Bias dtype mismatch: got {:?}, expected {:?}",+ biases.data_type(),+ kernel_data_type+ )));+ }+ let bias_add_kernel = Some(TensorAddBiasMetalKernel::new(mtl_context, kernel_data_type)?);+ let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = biases.buffer().to_owned().into();+ (bias_add_kernel, Some(biases_buffer))+ },+ Err(_) => (None, None),+ };let g = config.group_size;let mode = config.weight_quantization_mode;@@ -243,9 +218,7 @@ impl EncodableBlock<Metal> for QuantizedLinear {let input_buffer = input_array_mut.buffer();let output_buffer = output_array_mut.buffer();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");let args = QuantizedMatmulArguments {a_buffer: input_buffer,@@ -260,15 +233,11 @@ impl EncodableBlock<Metal> for QuantizedLinear {quantization_type: self.quantization_type,};- self.kernel- .encode(&encoder, args)- .expect("Failed to encode quantized matmul kernel");+ self.kernel.encode(&encoder, args).expect("Failed to encode quantized matmul kernel");encoder.end_encoding();- if let (Some(bias_add), Some(bias_buf)) =- (&self.bias_add_kernel, &self.biases_buffer)- {+ if let (Some(bias_add), Some(bias_buf)) = (&self.bias_add_kernel, &self.biases_buffer) {let total_len = batch_size * self.output_dim;bias_add.encode_if(output_buffer,@@ -319,13 +288,9 @@ impl EncodableBlock<Metal> for QuantizedLinear {quantization_type: self.quantization_type,};- self.kernel- .encode(encoder, args)- .expect("Failed to encode quantized matmul kernel");+ self.kernel.encode(encoder, args).expect("Failed to encode quantized matmul kernel");- if let (Some(bias_add), Some(bias_buf)) =- (&self.bias_add_kernel, &self.biases_buffer)- {+ if let (Some(bias_add), Some(bias_buf)) = (&self.bias_add_kernel, &self.biases_buffer) {let total_len = batch_size * self.output_dim;bias_add.encode_if(output_buffer,diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rsindex f7c110a2c..32e438c09 100644--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs@@ -1,22 +1,18 @@//! Mamba2 SSM mixer encodable.use super::{EncodableBlock, Metal, transformer_layer};-use crate::backends::metal::kernel::ssm::ssd_prefill::{- SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode,-};+use crate::backends::metal::kernel::ssm::ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode};use crate::{DataType,backends::{common::kernel::{SSDUpdateKernel, SplitInProjKernel},metal::{- KernelDataType, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,- Retained,+ KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray,+ ProtocolObject, Retained,compilation_parameters::CompilationConfig,kernel::dsl::{SSDUpdateMetalKernel, SplitInProjMetalKernel},kernel::ssm::{- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,- conv1d_scan::Conv1dDecodeArguments,+ Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, conv1d_scan::Conv1dDecodeArguments,},},},@@ -59,32 +55,21 @@ impl MambaMixer {) -> Self {let _ = (num_heads, head_dim, num_groups);if !matches!(layer_type, DecoderLayerType::StateSpace { .. }) {- panic!(- "Layer {} marked as transformer but Mamba mixer config provided",- layer_index- );+ panic!("Layer {} marked as transformer but Mamba mixer config provided", layer_index);}let split_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);let conv_tree = resolve_subtree(&split_tree, &["conv", "conv1d"]);- let data_type: DataType =- mamba_config.in_projection_config.activation_precision().into();+ let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();let kernel_data_type: KernelDataType = data_type.into();let in_projection = transformer_layer::linear_block(&mamba_config.in_projection_config,mamba_config.has_in_biases,model_dim,- [- mamba_config.conv_dim(),- mamba_config.inner_dim(),- mamba_config.num_heads,- ],+ [mamba_config.conv_dim(), mamba_config.inner_dim(), mamba_config.num_heads],mtl_context,- &resolve_subtree(- decoder_layer_loader,- &["mixer.in_projection", "mixer.in_proj"],- ),+ &resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),ArrayId::Main,ArrayId::SsmInProj,)@@ -96,10 +81,7 @@ impl MambaMixer {mamba_config.inner_dim(),[model_dim],mtl_context,- &resolve_subtree(- decoder_layer_loader,- &["mixer.out_projection", "mixer.out_proj"],- ),+ &resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),ArrayId::AttentionOutput,ArrayId::Main,)@@ -112,23 +94,16 @@ impl MambaMixer {None};let gate_bias = split_tree.leaf("gate_bias").unwrap().clone();- let skip_connection_weight =- split_tree.leaf("skip_connection_weight").unwrap().clone();-- let split_inproj =- SplitInProjMetalKernel::new(mtl_context, kernel_data_type.into())- .expect("Failed to create split in-projection kernel");- let conv_scan = Conv1dScanKernel::new(- mtl_context,- kernel_data_type,- &mamba_config.activation,- )- .expect("Failed to create conv scan kernel");- let ssd_prefill = SSDPrefillKernels::new(mtl_context, kernel_data_type)- .expect("Failed to create SSD prefill kernel");- let ssd_update =- SSDUpdateMetalKernel::new(mtl_context, kernel_data_type.into())- .expect("Failed to create SSD decode kernel");+ let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap().clone();++ let split_inproj = SplitInProjMetalKernel::new(mtl_context, kernel_data_type.into())+ .expect("Failed to create split in-projection kernel");+ let conv_scan = Conv1dScanKernel::new(mtl_context, kernel_data_type, &mamba_config.activation)+ .expect("Failed to create conv scan kernel");+ let ssd_prefill =+ SSDPrefillKernels::new(mtl_context, kernel_data_type).expect("Failed to create SSD prefill kernel");+ let ssd_update = SSDUpdateMetalKernel::new(mtl_context, kernel_data_type.into())+ .expect("Failed to create SSD decode kernel");let prefill_mode = resolve_prefill_mode_from_env();Self {@@ -159,8 +134,7 @@ impl MambaMixer {return;}- self.in_projection- .encode_with_shared_encoder(state, encoder, parameters);+ self.in_projection.encode_with_shared_encoder(state, encoder, parameters);self.run_split_inproj(state, encoder, active_suffix_length);self.run_conv_scan(state, encoder, active_suffix_length);@@ -170,8 +144,7 @@ impl MambaMixer {self.run_prefill_ssm(state, encoder, active_suffix_length);}- self.out_projection- .encode_with_shared_encoder(state, encoder, parameters);+ self.out_projection.encode_with_shared_encoder(state, encoder, parameters);}fn run_split_inproj(@@ -281,9 +254,7 @@ impl MambaMixer {.expect("Failed to encode conv decode kernel");} else {let padded_buf = if state_stride > 0 {- let array = state- .conv_padded_buffer()- .expect("Missing conv padded buffer");+ let array = state.conv_padded_buffer().expect("Missing conv padded buffer");let borrow = array.borrow_mut();let buf = borrow.buffer().clone();drop(borrow);@@ -386,25 +357,12 @@ impl MambaMixer {state: &state_raw,y: &out_buf,suffix_len: suffix_length,- group_size: (self.config.num_heads / self.config.num_groups)- as i32,+ group_size: (self.config.num_heads / self.config.num_groups) as i32,state_size: self.config.state_dim as i32,- x_strides: [- self.config.num_heads * self.config.head_dim,- self.config.head_dim,- 1,- ],+ x_strides: [self.config.num_heads * self.config.head_dim, self.config.head_dim, 1],dt_strides: [self.config.num_heads, 1],- cb_strides: [- self.config.num_groups * self.config.state_dim,- self.config.state_dim,- 1,- ],- state_strides: [- self.config.head_dim * self.config.state_dim,- self.config.state_dim,- 1,- ],+ cb_strides: [self.config.num_groups * self.config.state_dim, self.config.state_dim, 1],+ state_strides: [self.config.head_dim * self.config.state_dim, self.config.state_dim, 1],channels: self.config.num_heads,head_dim: self.config.head_dim,},@@ -490,9 +448,7 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {match env::var("UZU_SSM_PREFILL_MODE") {Ok(value) => match value.trim().to_ascii_lowercase().as_str() {"seq" | "sequential" | "baseline" => SSDPrefillMode::Sequential,- "single" | "singlepass" | "single_pass" => {- SSDPrefillMode::SinglePass- },+ "single" | "singlepass" | "single_pass" => SSDPrefillMode::SinglePass,_ => SSDPrefillMode::SinglePass,},Err(_) => SSDPrefillMode::SinglePass,@@ -507,9 +463,8 @@ impl EncodableBlock<Metal> for MambaMixer {parameters: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_pipeline_with_encoder(state, &encoder, parameters);encoder.end_encoding();@@ -527,9 +482,7 @@ impl EncodableBlock<Metal> for MambaMixer {self.in_projection.encode(state, command_buffer, parameters);- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.run_split_inproj(state, &encoder, active_suffix_length);self.run_conv_scan(state, &encoder, active_suffix_length);if active_suffix_length == 1 {@@ -548,8 +501,7 @@ impl EncodableBlock<Metal> for MambaMixer {}fn supports_shared_encoder(&self) -> bool {- self.in_projection.supports_shared_encoder()- && self.out_projection.supports_shared_encoder()+ self.in_projection.supports_shared_encoder() && self.out_projection.supports_shared_encoder()}fn encode_with_shared_encoder(@@ -572,9 +524,6 @@ fn resolve_subtree<'tree>(}}- let missing =- candidates.first().copied().unwrap_or("<missing subtree name>");- loader.subtree(missing).unwrap_or_else(|_| {- panic!("Unable to resolve parameter subtree '{missing}'")- })+ let missing = candidates.first().copied().unwrap_or("<missing subtree name>");+ loader.subtree(missing).unwrap_or_else(|_| panic!("Unable to resolve parameter subtree '{missing}'"))}diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsindex 6eb7f499f..dc68466ca 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -3,8 +3,7 @@use std::{cell::RefCell, rc::Rc};use crate::backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,- ProtocolObject, Retained,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained,};use super::{EncodableBlock, Metal};@@ -15,10 +14,7 @@ use crate::{kernel::{mlp::{MlpActivationType, MlpGateActMulEncodable},mlp_fused::{MlpFusedArguments, MlpFusedKernel},- quant_matmul::{- MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments,- MlpFusedQmvKernel,- },+ quant_matmul::{MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments, MlpFusedQmvKernel},},},config::Activation,@@ -54,9 +50,8 @@ impl EncodableBlock<Metal> for MlpBlock {params: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, params);encoder.end_encoding();} else {@@ -65,17 +60,15 @@ impl EncodableBlock<Metal> for MlpBlock {// Gate act+mul (fused_up -> hidden){- let arrays =- state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);+ let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);let fused = arrays[0].borrow_mut();let hidden = arrays[1].borrow_mut();let m = fused.shape()[0] as i32;let fused_buf = fused.buffer();let hidden_buf = hidden.buffer();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.gate.encode(&encoder, fused_buf, hidden_buf, m).expect("Failed to encode MLP activation/mul kernel");@@ -112,9 +105,7 @@ impl EncodableBlock<Metal> for MlpBlock {let m = fused.shape()[0] as i32;let fused_buf = fused.buffer();let hidden_buf = hidden.buffer();- self.gate- .encode(encoder, fused_buf, hidden_buf, m)- .expect("Failed to encode MLP activation/mul kernel");+ self.gate.encode(encoder, fused_buf, hidden_buf, m).expect("Failed to encode MLP activation/mul kernel");drop(fused);drop(hidden);@@ -143,8 +134,7 @@ pub struct MlpFusedBlock {down: Box<dyn EncodableBlock<Metal>>,weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,scales_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- zero_points_or_biases_buffer:- Option<Retained<ProtocolObject<dyn MTLBuffer>>>,+ zero_points_or_biases_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,input_dim: usize,hidden_dim: usize,activation: MlpActivationType,@@ -201,27 +191,11 @@ impl MlpFusedBlock {input_array_id: ArrayId,hidden_array_id: ArrayId,) -> Result<Self, crate::backends::metal::MTLError> {- let qmv = MlpFusedQmvKernel::new(- &context,- data_type,- group_size,- mode,- quantization_type,- )- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!("{:?}", e))- })?;-- let qmm = MlpFusedQmmKernel::new(- &context,- data_type,- group_size,- mode,- quantization_type,- )- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!("{:?}", e))- })?;+ let qmv = MlpFusedQmvKernel::new(&context, data_type, group_size, mode, quantization_type)+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("{:?}", e)))?;++ let qmm = MlpFusedQmmKernel::new(&context, data_type, group_size, mode, quantization_type)+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("{:?}", e)))?;Ok(Self {context,@@ -267,18 +241,14 @@ impl MlpFusedBlock {batch_count: batch,activation: self.activation,};- kernel- .borrow_mut()- .encode(&self.context, encoder, &args)- .expect("Failed to encode MLP fused kernel");+ kernel.borrow_mut().encode(&self.context, encoder, &args).expect("Failed to encode MLP fused kernel");},MlpFusedUpKernel::Quantized {qmv,qmm,} => {let scales = self.scales_buffer.as_ref().unwrap();- let zp_or_biases =- self.zero_points_or_biases_buffer.as_ref().unwrap();+ let zp_or_biases = self.zero_points_or_biases_buffer.as_ref().unwrap();let is_decode = batch == 1;if is_decode {@@ -293,8 +263,7 @@ impl MlpFusedBlock {hidden_dim: self.hidden_dim as i32,batch_count: batch,};- qmv.encode(encoder, &args)- .expect("Failed to encode MLP fused QMV");+ qmv.encode(encoder, &args).expect("Failed to encode MLP fused QMV");} else {let args = MlpFusedQmmArguments {weights: &self.weights_buffer,@@ -307,8 +276,7 @@ impl MlpFusedBlock {input_dim: self.input_dim as i32,hidden_dim: self.hidden_dim as i32,};- qmm.encode(encoder, &args)- .expect("Failed to encode MLP fused QMM");+ qmm.encode(encoder, &args).expect("Failed to encode MLP fused QMM");}},}@@ -323,25 +291,22 @@ impl EncodableBlock<Metal> for MlpFusedBlock {params: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, params);encoder.end_encoding();} else {// Fused up + activation{- let arrays =- state.arrays(&[self.input_array_id, self.hidden_array_id]);+ let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);let input = arrays[0].borrow_mut();let hidden = arrays[1].borrow_mut();let batch = input.shape()[0] as i32;let input_buf = input.buffer();let hidden_buf = hidden.buffer();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_fused_up(&encoder, input_buf, 0, hidden_buf, batch);encoder.end_encoding();}diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex fc7339204..798c31d93 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -33,8 +33,7 @@ pub use attention::Attention;pub use classifier_layer::ClassifierLayer;pub use decoder::Decoder;pub use embedding::{- EmbeddingError, FullPrecisionEmbeddingLookup,- FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,+ EmbeddingError, FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,QuantizedEmbeddingReadout,};pub use layer::LayerExecutables;@@ -42,9 +41,7 @@ pub use linear::{FullPrecisionLinear, QuantizedLinear};pub(crate) use mamba_mixer::MambaMixer;pub use mlp::{MlpBlock, MlpFusedBlock, MlpFusedUpKernel};pub use moe_block::{MoeBlock, SharedMoeWeights};-pub use normalization::{- LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,-};+pub use normalization::{LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm};pub use pooling::Pooling;pub use prediction_head::ClassifierPredictionHead;pub use rope::Rope;diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rsindex 21053540e..111de03b4 100644--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs@@ -7,26 +7,19 @@ use super::{SharedMoeWeights,};use crate::{- Activation, DataType, LinearConfig, MixtureOfExpertsConfig,- RoutingFunctionConfig,+ Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,backends::{common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel},metal::{- KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,- MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext,- MetalArray, NSRange, ProtocolObject, Retained,+ KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,+ MTLComputeCommandEncoder, MTLContext, MetalArray, NSRange, ProtocolObject, Retained,kernel::{MoeGatherKernels,- dsl::{- MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,- },+ dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},moe::{- MoeBlockBasesArguments, MoeExpertsTwoPassArguments,- MoeExpertsTwoPassDecodeKernels,- MoeExpertsTwoPassPrefillKernel, MoeGatherArguments,- MoeRouterTopKArguments, MoeRouterTopKKernel,- MoeScatterArguments, MoeScatterKernels,- MoeScatterWithMapArguments,+ MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,+ MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeRouterTopKArguments, MoeRouterTopKKernel,+ MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments,},},},@@ -69,57 +62,36 @@ impl MoeBlock {hidden_dim: usize,parameter_tree: &ParameterTree<MTLContext>,) -> Result<Self, crate::backends::metal::MTLError> {- let activation_data_type: DataType = moe_config- .expert_config- .linear_config- .activation_precision()- .into();+ let activation_data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();let data_type: KernelDataType = activation_data_type.into();- let router_data_type: DataType =- moe_config.router_config.activation_precision().into();+ let router_data_type: DataType = moe_config.router_config.activation_precision().into();let router_kernel_data_type: KernelDataType = router_data_type.into();- let router_renorm = matches!(- moe_config.routing_function,- RoutingFunctionConfig::SoftmaxRouting- );+ let router_renorm = matches!(moe_config.routing_function, RoutingFunctionConfig::SoftmaxRouting);- let router_tree = parameter_tree.subtree("router").map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Router subtree error: {:?}",- e- ))- })?;+ let router_tree = parameter_tree+ .subtree("router")+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Router subtree error: {:?}", e)))?;let router = match &moe_config.router_config {LinearConfig::Quantized(_) => {- unimplemented!(- "Quantized router with fused router+topk not yet supported"- );+ unimplemented!("Quantized router with fused router+topk not yet supported");},LinearConfig::MLXQuantized(_) => {- unimplemented!(- "MLX quantized router with fused router+topk not yet supported"- );+ unimplemented!("MLX quantized router with fused router+topk not yet supported");},LinearConfig::FullPrecision {..} => {- let weights_arr = router_tree.leaf("weights").map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Router weights error: {:?}",- e- ))- })?;+ let weights_arr = router_tree+ .leaf("weights")+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Router weights error: {:?}", e)))?;let weights_buf = weights_arr.buffer();- let biases_arr = router_tree.leaf("biases").map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Router biases error: {:?}",- e- ))- })?;+ let biases_arr = router_tree+ .leaf("biases")+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Router biases error: {:?}", e)))?;let biases_buf = biases_arr.buffer();RouterBlock::Metal {@@ -134,129 +106,54 @@ impl MoeBlock {},};- let router_topk_kernel =- MoeRouterTopKKernel::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "RouterTopK fused kernel error: {:?}",- e- ))- })?;- let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context)- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Counts+offsets kernel error: {:?}",- e- ))+ let router_topk_kernel = MoeRouterTopKKernel::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!("RouterTopK fused kernel error: {:?}", e))})?;- let scatter_kernels = MoeScatterKernels::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Scatter kernels error: {:?}",- e- ))+ let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context)+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Counts+offsets kernel error: {:?}", e)))?;+ let scatter_kernels = MoeScatterKernels::new(context)+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Scatter kernels error: {:?}", e)))?;+ let gather_kernel = MoeGatherKernels::new(context)+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Gather kernel error: {:?}", e)))?;+ let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeKernels::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!("Experts two-pass decode kernel error: {:?}", e))})?;- let gather_kernel = MoeGatherKernels::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Gather kernel error: {:?}",- e- ))+ let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!("Experts two-pass prefill kernel error: {:?}", e))})?;- let experts_two_pass_decode_kernel =- MoeExpertsTwoPassDecodeKernels::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Experts two-pass decode kernel error: {:?}",- e- ))- })?;- let experts_two_pass_prefill_kernel =- MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Experts two-pass prefill kernel error: {:?}",- e- ))- })?;- let finalize_kernel =- MoeFinalizeMetalKernel::new(context, data_type.into()).map_err(- |e| {- crate::backends::metal::MTLError::Generic(format!(- "Finalize kernel error: {:?}",- e- ))- },- )?;+ let finalize_kernel = MoeFinalizeMetalKernel::new(context, data_type.into())+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Finalize kernel error: {:?}", e)))?;- let experts_tree = parameter_tree.subtree("experts").map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "experts subtree error: {:?}",- e- ))- })?;+ let experts_tree = parameter_tree+ .subtree("experts")+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("experts subtree error: {:?}", e)))?;let w13_arr = experts_tree.subtree("up_projection")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "up_projection error: {:?}",- e- ))- })?+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("up_projection error: {:?}", e)))?.leaf("weights")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "up weights error: {:?}",- e- ))- })?;+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("up weights error: {:?}", e)))?;let w13_buf = w13_arr.buffer().clone();let w2_arr = experts_tree.subtree("down_projection")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "down_projection error: {:?}",- e- ))- })?+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("down_projection error: {:?}", e)))?.leaf("weights")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "down weights error: {:?}",- e- ))- })?;+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("down weights error: {:?}", e)))?;let w2_buf = w2_arr.buffer().clone();let up_biases_arr = experts_tree.subtree("up_projection")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "up_projection biases error: {:?}",- e- ))- })?+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("up_projection biases error: {:?}", e)))?.leaf("biases")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "up biases error: {:?}",- e- ))- })?;+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("up biases error: {:?}", e)))?;let up_biases_buf = up_biases_arr.buffer().clone();let down_biases_arr = experts_tree.subtree("down_projection")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "down_projection biases error: {:?}",- e- ))- })?+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("down_projection biases error: {:?}", e)))?.leaf("biases")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "down biases error: {:?}",- e- ))- })?;+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("down biases error: {:?}", e)))?;let down_biases_buf = down_biases_arr.buffer().clone();let shared_weights = SharedMoeWeights {@@ -368,9 +265,7 @@ impl EncodableBlock<Metal> for MoeBlock {KernelDataType::Float32 => 4,};- let blit_encoder = root- .new_blit_command_encoder()- .expect("Failed to create blit command encoder");+ let blit_encoder = root.new_blit_command_encoder().expect("Failed to create blit command encoder");if suffix_length > 0 && k > 0 {let entries = suffix_length * k;@@ -394,19 +289,13 @@ impl EncodableBlock<Metal> for MoeBlock {}// Clear hidden buffer- let hidden_bytes =- (suffix_length * k * self.hidden_dim * dtype_size) as u64;+ let hidden_bytes = (suffix_length * k * self.hidden_dim * dtype_size) as u64;if hidden_bytes > 0 {- blit_encoder.fill_buffer_range_value(- &hidden_buf,- NSRange::new(0, hidden_bytes.try_into().unwrap()),- 0,- );+ blit_encoder.fill_buffer_range_value(&hidden_buf, NSRange::new(0, hidden_bytes.try_into().unwrap()), 0);}// Clear y_partial buffer- let y_partial_bytes =- (suffix_length * k * self.model_dim * dtype_size) as u64;+ let y_partial_bytes = (suffix_length * k * self.model_dim * dtype_size) as u64;if y_partial_bytes > 0 {blit_encoder.fill_buffer_range_value(&y_partial_buf,@@ -416,14 +305,9 @@ impl EncodableBlock<Metal> for MoeBlock {}// Clear x_perm buffer- let x_perm_bytes =- (suffix_length * k * self.model_dim * dtype_size) as u64;+ let x_perm_bytes = (suffix_length * k * self.model_dim * dtype_size) as u64;if x_perm_bytes > 0 {- blit_encoder.fill_buffer_range_value(- &x_perm_buf,- NSRange::new(0, x_perm_bytes.try_into().unwrap()),- 0,- );+ blit_encoder.fill_buffer_range_value(&x_perm_buf, NSRange::new(0, x_perm_bytes.try_into().unwrap()), 0);}}@@ -458,9 +342,8 @@ impl EncodableBlock<Metal> for MoeBlock {}{- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.counts_offsets_kernel.encode(&topk_ids_buf,&offsets_buf,@@ -529,15 +412,11 @@ impl EncodableBlock<Metal> for MoeBlock {},);- let gating_code = Self::gating_code_from_activation(- &self.moe_config.expert_config.activation,- );+ let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation);// Compute clipping values and alpha for expert kernels- let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]- .unwrap_or(f32::NEG_INFINITY);- let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]- .unwrap_or(f32::INFINITY);+ let gate_clip_min = self.moe_config.expert_config.gate_clipping[0].unwrap_or(f32::NEG_INFINITY);+ let gate_clip_max = self.moe_config.expert_config.gate_clipping[1].unwrap_or(f32::INFINITY);let up_clip_min = self.moe_config.expert_config.up_clipping[0];let up_clip_max = self.moe_config.expert_config.up_clipping[1];let silu_alpha = self.moe_config.expert_config.activation.alpha();@@ -617,9 +496,8 @@ impl EncodableBlock<Metal> for MoeBlock {}{- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.finalize_kernel.encode(&tok2row_buf,&topk_probs_buf,diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rsindex c06b3df89..a97905ad3 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs@@ -6,10 +6,8 @@ use crate::{backends::{common::kernel::LayerNormKernel,metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,- MTLResourceOptions, ProtocolObject, Retained,- kernel::dsl::LayerNormMetalKernel,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt,+ MTLError, MTLResourceOptions, ProtocolObject, Retained, kernel::dsl::LayerNormMetalKernel,},},config::{NormalizationConfig, UpcastMode},@@ -37,25 +35,19 @@ impl LayerNorm {) -> Result<Self, MTLError> {// Load scales from parameter treelet scales_param = parameter_tree.leaf("scales").map_err(|e| {- MTLError::Library(- crate::backends::metal::error::LibraryError::Custom(format!(- "Failed to load scales: {:?}",- e- )),- )+ MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(+ "Failed to load scales: {:?}",+ e+ )))})?;let scales_data = scales_param.as_bytes();let scales_buffer = context.device- .new_buffer_with_data(- scales_data,- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create scales buffer");- let accumulation_data_type: DataType =- config.accumulation_precision.into();+ let accumulation_data_type: DataType = config.accumulation_precision.into();let scale_data_type: DataType = config.scale_precision.into();let kernel = LayerNormMetalKernel::new(@@ -82,9 +74,8 @@ impl EncodableBlock<Metal> for LayerNorm {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &compute_encoder, parameters);compute_encoder.end_encoding();diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rsindex e7b65f5ea..10c27adbe 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs@@ -13,8 +13,7 @@ use crate::backends::metal::MTLError;use crate::{DataType,backends::metal::{- MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, ProtocolObject,- Retained, kernel::RMSNormError,+ MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained, kernel::RMSNormError,},config::NormalizationConfig,encodable_block::EncodingParameters,@@ -80,23 +79,15 @@ impl EncodableBlock<Metal> for Normalization {parameters: &EncodingParameters<Metal>,) {match self {- Normalization::LayerNorm(layer_norm) => {- layer_norm.encode(state, command_buffer, parameters)- },- Normalization::RMSNorm(rms_norm) => {- rms_norm.encode(state, command_buffer, parameters)- },+ Normalization::LayerNorm(layer_norm) => layer_norm.encode(state, command_buffer, parameters),+ Normalization::RMSNorm(rms_norm) => rms_norm.encode(state, command_buffer, parameters),}}fn supports_shared_encoder(&self) -> bool {match self {- Normalization::LayerNorm(layer_norm) => {- layer_norm.supports_shared_encoder()- },- Normalization::RMSNorm(rms_norm) => {- rms_norm.supports_shared_encoder()- },+ Normalization::LayerNorm(layer_norm) => layer_norm.supports_shared_encoder(),+ Normalization::RMSNorm(rms_norm) => rms_norm.supports_shared_encoder(),}}@@ -107,11 +98,8 @@ impl EncodableBlock<Metal> for Normalization {parameters: &EncodingParameters<Metal>,) {match self {- Normalization::LayerNorm(layer_norm) => layer_norm- .encode_with_shared_encoder(state, encoder, parameters),- Normalization::RMSNorm(rms_norm) => {- rms_norm.encode_with_shared_encoder(state, encoder, parameters)- },+ Normalization::LayerNorm(layer_norm) => layer_norm.encode_with_shared_encoder(state, encoder, parameters),+ Normalization::RMSNorm(rms_norm) => rms_norm.encode_with_shared_encoder(state, encoder, parameters),}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rsindex 2eedae5de..b429d68d0 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs@@ -4,13 +4,9 @@ use super::super::{EncodableBlock, Metal};use crate::{DataType,backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,MTLResourceOptions, ProtocolObject, Retained,- kernel::rms_norm::{- QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel,- RMSNormKernelType,- },+ kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel, RMSNormKernelType},},config::{NormalizationConfig, UpcastMode},encodable_block::EncodingParameters,@@ -50,34 +46,22 @@ impl QKNorm {// Setup query normalization if configuredif let Some(ref q_config) = query_config {- let scales_param =- parameter_tree.leaf("query_norm.scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(- crate::backends::metal::error::LibraryError::Custom(- format!("Failed to load query scales: {:?}", e),- ),- ))- })?;+ let scales_param = parameter_tree.leaf("query_norm.scales").map_err(|e| {+ RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(+ format!("Failed to load query scales: {:?}", e),+ )))+ })?;let scales_data = scales_param.as_bytes();- let scales_buffer = context.device.new_buffer_with_data(- scales_data,- MTLResourceOptions::STORAGE_MODE_SHARED,- );+ let scales_buffer =+ context.device.new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED);- let accumulation_data_type: DataType =- q_config.accumulation_precision.into();+ let accumulation_data_type: DataType = q_config.accumulation_precision.into();let scale_data_type: DataType = q_config.scale_precision.into();- let (input_type, scales_type, output_type) = match q_config- .upcast_mode- {- UpcastMode::OnlyNormalization => {- (intermediate_data_type, scale_data_type, scale_data_type)- },- UpcastMode::FullLayer => {- (intermediate_data_type, scale_data_type, scale_data_type)- },+ let (input_type, scales_type, output_type) = match q_config.upcast_mode {+ UpcastMode::OnlyNormalization => (intermediate_data_type, scale_data_type, scale_data_type),+ UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),};let kernel = RMSNormKernel::new_with_mode(@@ -96,34 +80,22 @@ impl QKNorm {// Setup key normalization if configuredif let Some(ref k_config) = key_config {- let scales_param =- parameter_tree.leaf("key_norm.scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(- crate::backends::metal::error::LibraryError::Custom(- format!("Failed to load key scales: {:?}", e),- ),- ))- })?;+ let scales_param = parameter_tree.leaf("key_norm.scales").map_err(|e| {+ RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(+ format!("Failed to load key scales: {:?}", e),+ )))+ })?;let scales_data = scales_param.as_bytes();- let scales_buffer = context.device.new_buffer_with_data(- scales_data,- MTLResourceOptions::STORAGE_MODE_SHARED,- );+ let scales_buffer =+ context.device.new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED);- let accumulation_data_type: DataType =- k_config.accumulation_precision.into();+ let accumulation_data_type: DataType = k_config.accumulation_precision.into();let scale_data_type: DataType = k_config.scale_precision.into();- let (input_type, scales_type, output_type) = match k_config- .upcast_mode- {- UpcastMode::OnlyNormalization => {- (intermediate_data_type, scale_data_type, scale_data_type)- },- UpcastMode::FullLayer => {- (intermediate_data_type, scale_data_type, scale_data_type)- },+ let (input_type, scales_type, output_type) = match k_config.upcast_mode {+ UpcastMode::OnlyNormalization => (intermediate_data_type, scale_data_type, scale_data_type),+ UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),};let kernel = RMSNormKernel::new_with_mode(@@ -162,9 +134,8 @@ impl EncodableBlock<Metal> for QKNorm {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &compute_encoder, parameters);compute_encoder.end_encoding();@@ -197,11 +168,7 @@ impl EncodableBlock<Metal> for QKNorm {let head_dim = self.head_dim as i32;// Process query normalization if configured- if let (- Some(query_kernel),- Some(query_scales_buffer),- Some(query_config),- ) =+ if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =(&self.query_kernel, &self.query_scales_buffer, &self.query_config){if let Err(e) = query_kernel.encode_qk_norm(@@ -220,10 +187,7 @@ impl EncodableBlock<Metal> for QKNorm {target: QKNormTarget::QueryHeads,},) {- eprintln!(- "Failed to encode query normalization kernel: {:?}",- e- );+ eprintln!("Failed to encode query normalization kernel: {:?}", e);}}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rsindex 4644e1f4a..fb54be5be 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs@@ -4,12 +4,9 @@ use super::super::{EncodableBlock, Metal};use crate::{DataType,backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,MTLResourceOptions, ProtocolObject, Retained,- kernel::rms_norm::{- RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType,- },+ kernel::rms_norm::{RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType},},config::{NormalizationConfig, UpcastMode},encodable_block::EncodingParameters,@@ -37,26 +34,20 @@ impl RMSNorm {) -> Result<Self, RMSNormError> {// Load scales from parameter treelet scales_param = parameter_tree.leaf("scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(- crate::backends::metal::error::LibraryError::Custom(format!(- "Failed to load scales: {:?}",- e- )),- ))+ RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(+ "Failed to load scales: {:?}",+ e+ ))))})?;// TODO: Don't create buffers dynamically, we need to use forward pass storage for thing like thislet scales_data = scales_param.as_bytes();let scales_buffer = context.device- .new_buffer_with_data(- scales_data,- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create scales buffer");- let accumulation_data_type: DataType =- config.accumulation_precision.into();+ let accumulation_data_type: DataType = config.accumulation_precision.into();let scale_data_type: DataType = config.scale_precision.into();let (input_type, scales_type, output_type) = match config.upcast_mode {@@ -105,9 +96,8 @@ impl EncodableBlock<Metal> for RMSNorm {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &compute_encoder, parameters);compute_encoder.end_encoding();@@ -149,8 +139,7 @@ impl EncodableBlock<Metal> for RMSNorm {} else {(0, state.active_suffix_length())};- let batch_len =- batch_len.min(suffix_length.saturating_sub(batch_start));+ let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));if batch_len == 0 {return;}diff --git a/crates/uzu/src/backends/metal/encodable_block/pooling.rs b/crates/uzu/src/backends/metal/encodable_block/pooling.rsindex 0fe17f7b2..62bd66c15 100644--- a/crates/uzu/src/backends/metal/encodable_block/pooling.rs+++ b/crates/uzu/src/backends/metal/encodable_block/pooling.rs@@ -7,9 +7,8 @@ use crate::{backends::{common::kernel::{PoolingClsKernel, PoolingMeanKernel},metal::{- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,- Retained,+ KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext,+ MTLError, ProtocolObject, Retained,kernel::dsl::{PoolingClsMetalKernel, PoolingMeanMetalKernel},},},@@ -34,12 +33,8 @@ impl PoolingKernel {encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,) {match self {- Self::Cls(k) => k.encode(- input, output, seq_len, hidden_dim, batch_size, encoder,- ),- Self::Mean(k) => k.encode(- input, output, seq_len, hidden_dim, batch_size, encoder,- ),+ Self::Cls(k) => k.encode(input, output, seq_len, hidden_dim, batch_size, encoder),+ Self::Mean(k) => k.encode(input, output, seq_len, hidden_dim, batch_size, encoder),}}}@@ -57,13 +52,8 @@ impl Pooling {model_dim: usize,) -> Result<Self, MTLError> {let pooling_kernel = match pooling_type {- PoolingType::Cls => PoolingKernel::Cls(PoolingClsMetalKernel::new(- context,- data_type.into(),- )?),- PoolingType::Mean => PoolingKernel::Mean(- PoolingMeanMetalKernel::new(context, data_type.into())?,- ),+ PoolingType::Cls => PoolingKernel::Cls(PoolingClsMetalKernel::new(context, data_type.into())?),+ PoolingType::Mean => PoolingKernel::Mean(PoolingMeanMetalKernel::new(context, data_type.into())?),};Ok(Self {pooling_kernel,@@ -79,17 +69,14 @@ impl EncodableBlock<Metal> for Pooling {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();#[cfg(feature = "tracing")]{let batch_size = 1;- let data_type =- { state.arrays(&[ArrayId::Main])[0].borrow().data_type() };+ let data_type = { state.arrays(&[ArrayId::Main])[0].borrow().data_type() };let arrays = state.arrays(&[ArrayId::ClassifierPooling]);let pooling_array = arrays[0].borrow();@@ -100,9 +87,7 @@ impl EncodableBlock<Metal> for Pooling {let trace_arr = traces_ref.output_pooling().borrow();let dst_buf = trace_arr.buffer();- let blit = command_buffer- .new_blit_command_encoder()- .expect("Failed to create blit command encoder");+ let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");blit.copy_buffer_to_buffer(output_buffer,0,diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rsindex efddba52a..a12e5d974 100644--- a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs+++ b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs@@ -3,10 +3,7 @@use super::{EncodableBlock, Metal};#[cfg(feature = "tracing")]use crate::backends::metal::MTLBlitCommandEncoder;-use crate::backends::metal::{- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,- ProtocolObject, Retained,-};+use crate::backends::metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained};use crate::encodable_block::EncodingParameters;#[cfg(feature = "tracing")]use crate::forward_pass::state::ArrayId;@@ -47,9 +44,8 @@ impl EncodableBlock<Metal> for ClassifierPredictionHead {parameters: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();} else {@@ -62,8 +58,7 @@ impl EncodableBlock<Metal> for ClassifierPredictionHead {#[cfg(feature = "tracing")]{let traces_rc = state.traces().clone();- let logits_arrays =- state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);+ let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);let logits_array_ref = logits_arrays[0].borrow();let linear_output_buffer = logits_array_ref.buffer();let data_type = logits_array_ref.data_type();@@ -73,19 +68,10 @@ impl EncodableBlock<Metal> for ClassifierPredictionHead {let trace_logits = traces_ref.logits.borrow();let dst_trace_buf = trace_logits.buffer();- let copy_size_bytes =- batch_size * self.num_labels * data_type.size_in_bytes();+ let copy_size_bytes = batch_size * self.num_labels * data_type.size_in_bytes();- let blit = command_buffer- .new_blit_command_encoder()- .expect("Failed to create blit command encoder");- blit.copy_buffer_to_buffer(- linear_output_buffer,- 0,- dst_trace_buf,- 0,- copy_size_bytes,- );+ let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");+ blit.copy_buffer_to_buffer(linear_output_buffer, 0, dst_trace_buf, 0, copy_size_bytes);blit.end_encoding();}diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rsindex d4671fb7a..7ff44133d 100644--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs@@ -1,9 +1,6 @@//! Rope (Rotary Position Embedding) encodable.-use crate::backends::metal::{- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,- ProtocolObject, Retained,-};+use crate::backends::metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained};use super::{EncodableBlock, Metal};use crate::backends::metal::{@@ -39,9 +36,8 @@ impl EncodableBlock<Metal> for Rope {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &compute_encoder, parameters);compute_encoder.end_encoding();@@ -75,8 +71,7 @@ impl EncodableBlock<Metal> for Rope {let keys_array = keys_binding[0].borrow();let num_groups = keys_array.shape()[0];- let cos_binding =- state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);+ let cos_binding = state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);let cos_array = cos_binding[0].borrow();let cos_shape = cos_array.shape();let rope_max_seq_len = cos_shape[0];@@ -96,12 +91,10 @@ impl EncodableBlock<Metal> for Rope {let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);let rotated_keys = rotated_keys_binding[0].borrow_mut();- let cos_buffer_binding =- state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);+ let cos_buffer_binding = state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);let rope_cosines = cos_buffer_binding[0].borrow_mut();- let sin_buffer_binding =- state.arrays(&[ArrayId::RopeSines(self.rope_type)]);+ let sin_buffer_binding = state.arrays(&[ArrayId::RopeSines(self.rope_type)]);let rope_sines = sin_buffer_binding[0].borrow_mut();let token_positions_offset = token_positions.offset();diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsindex 10e253d05..46feeaef7 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -6,13 +6,11 @@ use crate::{backends::{common::Context,metal::{- KernelDataType, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,- Retained,+ KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray,+ ProtocolObject, Retained,compilation_parameters::CompilationConfig,kernel::short_conv::{- ShortConvDecodeArguments, ShortConvKernel,- ShortConvPackArguments, ShortConvPrefillArguments,+ ShortConvDecodeArguments, ShortConvKernel, ShortConvPackArguments, ShortConvPrefillArguments,ShortConvTrieArguments,},},@@ -58,19 +56,13 @@ impl ShortConvMixer {decoder_layer_loader: &ParameterTree<MTLContext>,) -> Self {if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {- panic!(- "Layer {} marked as non-ShortConv but ShortConv config provided",- layer_index- );+ panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);}let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);- let data_type: DataType = short_conv_config- .in_projection_config- .activation_precision()- .into();+ let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();let kernel_data_type: KernelDataType = data_type.into();let in_projection = transformer_layer::linear_block(@@ -105,8 +97,7 @@ impl ShortConvMixer {};let short_conv_kernel =- ShortConvKernel::new(mtl_context, kernel_data_type)- .expect("Failed to create short conv kernel");+ ShortConvKernel::new(mtl_context, kernel_data_type).expect("Failed to create short conv kernel");Self {layer_index,@@ -133,9 +124,7 @@ impl ShortConvMixer {self.in_projection.encode(state, command_buffer, parameters);- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.run_conv(state, &encoder, active_suffix_length);encoder.end_encoding();@@ -158,13 +147,11 @@ impl ShortConvMixer {return;}- self.in_projection- .encode_with_shared_encoder(state, encoder, parameters);+ self.in_projection.encode_with_shared_encoder(state, encoder, parameters);self.run_conv(state, encoder, active_suffix_length);- self.out_projection- .encode_with_shared_encoder(state, encoder, parameters);+ self.out_projection.encode_with_shared_encoder(state, encoder, parameters);}fn clear_suffix_state_valid_range(@@ -175,9 +162,7 @@ impl ShortConvMixer {return;};let cache = cache_layers.borrow();- let layer = cache.data[self.layer_index]- .as_short_conv()- .expect("Expected ShortConv layer");+ let layer = cache.data[self.layer_index].as_short_conv().expect("Expected ShortConv layer");layer.clear_suffix_state_valid_range();}@@ -191,9 +176,7 @@ impl ShortConvMixer {return;};let cache = cache_layers.borrow();- let layer = cache.data[self.layer_index]- .as_short_conv()- .expect("Expected ShortConv layer");+ let layer = cache.data[self.layer_index].as_short_conv().expect("Expected ShortConv layer");layer.set_suffix_state_valid_range(start, len);}@@ -242,11 +225,8 @@ impl ShortConvMixer {compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {- let arrays = state.arrays(&[- ArrayId::SsmInProj,- ArrayId::ShortConvState(self.layer_index),- ArrayId::AttentionOutput,- ]);+ let arrays =+ state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);let in_proj = arrays[0].borrow_mut();let conv_state = arrays[1].borrow_mut();let out = arrays[2].borrow_mut();@@ -266,15 +246,11 @@ impl ShortConvMixer {let state_stride = kernel_size.saturating_sub(1);// Allocate temporary padded buffer- let data_type: DataType =- self.config.in_projection_config.activation_precision().into();+ let data_type: DataType = self.config.in_projection_config.activation_precision().into();let element_size = data_type.size_in_bytes();let padded_rows = state_stride + suffix_length;let padded_size = padded_rows * self.model_dim * element_size;- let padded_buf = state- .mtl_context()- .create_buffer(padded_size)- .expect("Failed to create padded buffer");+ let padded_buf = state.mtl_context().create_buffer(padded_size).expect("Failed to create padded buffer");self.short_conv_kernel.encode_pack(compute,@@ -341,23 +317,18 @@ impl ShortConvMixer {let suffix_state_buf = suffix_state.buffer().clone();let out_buf = out.buffer().clone();- let data_type: DataType =- self.config.in_projection_config.activation_precision().into();+ let data_type: DataType = self.config.in_projection_config.activation_precision().into();let elem_bytes = data_type.size_in_bytes();let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);let in_proj_stride = self.model_dim * 3;- let in_proj_offset =- in_proj.offset() + sampling_start * in_proj_stride * elem_bytes;- let out_offset =- out.offset() + sampling_start * self.model_dim * elem_bytes;- let suffix_state_offset = suffix_state.offset()- + sampling_start * self.model_dim * state_stride * elem_bytes;+ let in_proj_offset = in_proj.offset() + sampling_start * in_proj_stride * elem_bytes;+ let out_offset = out.offset() + sampling_start * self.model_dim * elem_bytes;+ let suffix_state_offset = suffix_state.offset() + sampling_start * self.model_dim * state_stride * elem_bytes;let base_state_offset = conv_state.offset();- let parents_offset =- parents.offset() + sampling_start * std::mem::size_of::<i32>();+ let parents_offset = parents.offset() + sampling_start * std::mem::size_of::<i32>();let conv_weight = self.conv_weight.clone();let weight_buf = conv_weight.buffer().clone();@@ -398,11 +369,8 @@ impl ShortConvMixer {compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {- let arrays = state.arrays(&[- ArrayId::SsmInProj,- ArrayId::ShortConvState(self.layer_index),- ArrayId::AttentionOutput,- ]);+ let arrays =+ state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);let in_proj = arrays[0].borrow_mut();let conv_state = arrays[1].borrow_mut();let out = arrays[2].borrow_mut();@@ -450,9 +418,8 @@ impl EncodableBlock<Metal> for ShortConvMixer {parameters: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_pipeline_with_encoder(state, &encoder, parameters);encoder.end_encoding();@@ -466,8 +433,7 @@ impl EncodableBlock<Metal> for ShortConvMixer {}fn supports_shared_encoder(&self) -> bool {- self.in_projection.supports_shared_encoder()- && self.out_projection.supports_shared_encoder()+ self.in_projection.supports_shared_encoder() && self.out_projection.supports_shared_encoder()}fn encode_with_shared_encoder(diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rsindex d6e29b2f2..82aa3c7ef 100644--- a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs@@ -2,16 +2,11 @@use crate::backends::{common::kernel::TensorAddSwapKernel,- metal::{- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,- ProtocolObject, Retained,- },+ metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained},};use super::{EncodableBlock, Metal};-use crate::backends::metal::{- MTLContext, MTLError, kernel::dsl::TensorAddSwapMetalKernel,-};+use crate::backends::metal::{MTLContext, MTLError, kernel::dsl::TensorAddSwapMetalKernel};use crate::encodable_block::EncodingParameters;use crate::forward_pass::state::{ArrayId, ForwardPassState};@@ -41,9 +36,7 @@ impl EncodableBlock<Metal> for TensorAddSwap {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();@@ -73,11 +66,6 @@ impl EncodableBlock<Metal> for TensorAddSwap {let skip_mtl_buffer = skip_array.buffer();let main_mtl_buffer = main_array.buffer();- self.kernel.encode(- skip_mtl_buffer,- main_mtl_buffer,- length as u32,- encoder,- );+ self.kernel.encode(skip_mtl_buffer, main_mtl_buffer, length as u32, encoder);}}diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rsindex d2663b814..d8c0627ae 100644--- a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs@@ -2,16 +2,11 @@use crate::backends::{common::kernel::TensorCopyKernel,- metal::{- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,- ProtocolObject, Retained,- },+ metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained},};use super::{EncodableBlock, Metal};-use crate::backends::metal::{- MTLContext, MTLError, kernel::dsl::TensorCopyMetalKernel,-};+use crate::backends::metal::{MTLContext, MTLError, kernel::dsl::TensorCopyMetalKernel};use crate::encodable_block::EncodingParameters;use crate::forward_pass::state::{ArrayId, ForwardPassState};@@ -41,9 +36,7 @@ impl EncodableBlock<Metal> for TensorCopy {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,parameters: &EncodingParameters<Metal>,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.encode_with_shared_encoder(state, &encoder, parameters);encoder.end_encoding();@@ -73,11 +66,6 @@ impl EncodableBlock<Metal> for TensorCopy {let source_mtl_buffer = source_array.buffer();let destination_mtl_buffer = destination_array.buffer();- self.kernel.encode(- source_mtl_buffer,- destination_mtl_buffer,- length as u32,- encoder,- );+ self.kernel.encode(source_mtl_buffer, destination_mtl_buffer, length as u32, encoder);}}diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 82d938c94..99d368e8f 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -2,10 +2,8 @@ use std::rc::Rc;use super::Metal;use super::{- EncodableBlock, FullPrecisionEmbeddingLookup,- FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock,- MlpFusedBlock, MoeBlock, QuantizedEmbeddingLookup,- QuantizedEmbeddingReadout, QuantizedLinear,+ EncodableBlock, FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock,+ MlpFusedBlock, MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,@@ -30,8 +28,7 @@ pub fn linear_block<const N: usize>() -> Result<Box<dyn EncodableBlock<Metal>>, MTLError> {let output_dimension_sum: usize = output_dimensions.iter().sum();match config {- LinearConfig::Quantized(quantization_config)- | LinearConfig::MLXQuantized(quantization_config) => {+ LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {let block = QuantizedLinear::new(context,quantization_config,@@ -59,9 +56,7 @@ pub fn linear_block<const N: usize>(},LinearConfig::QLoRA {..- } => Err(MTLError::Generic(- "QLoRA linear layer not supported".to_string(),- )),+ } => Err(MTLError::Generic("QLoRA linear layer not supported".to_string())),}}@@ -74,8 +69,7 @@ pub fn mlp_block(parameter_tree: &ParameterTree<MTLContext>,) -> Result<Box<dyn EncodableBlock<Metal>>, MTLError> {if let crate::config::MLPConfig::Dense(dense_config) = config {- let data_type: DataType =- dense_config.linear_config.activation_precision().into();+ let data_type: DataType = dense_config.linear_config.activation_precision().into();// Up projection (outputs 2*hidden_dimension for gate and up)let up_projection = linear_block(@@ -84,20 +78,14 @@ pub fn mlp_block(model_dimension,[2 * hidden_dimension],context,- ¶meter_tree- .subtree("up_projection")- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,+ ¶meter_tree.subtree("up_projection").map_err(|error| MTLError::Generic(format!("{:?}", error)))?,ArrayId::Main,ArrayId::MlpFusedUp,)?;// Gate activation + multiply- let gate_activation = MlpGateActMulEncodable::new(- context,- data_type,- dense_config.activation.clone(),- hidden_dimension,- )?;+ let gate_activation =+ MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)?;// Down projectionlet down_projection = linear_block(@@ -106,31 +94,17 @@ pub fn mlp_block(hidden_dimension,[model_dimension],context,- ¶meter_tree- .subtree("down_projection")- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,+ ¶meter_tree.subtree("down_projection").map_err(|error| MTLError::Generic(format!("{:?}", error)))?,ArrayId::MlpHidden,ArrayId::Main,)?;- return Ok(Box::new(MlpBlock::new(- up_projection,- gate_activation,- down_projection,- )));+ return Ok(Box::new(MlpBlock::new(up_projection, gate_activation, down_projection)));}- if let crate::config::MLPConfig::MixtureOfExperts(- mixture_of_experts_config,- ) = config- {- let mixture_of_experts_block = MoeBlock::new(- context,- mixture_of_experts_config,- model_dimension,- hidden_dimension,- parameter_tree,- )?;+ if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {+ let mixture_of_experts_block =+ MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;return Ok(Box::new(mixture_of_experts_block));}@@ -148,53 +122,35 @@ pub fn mlp_fused_block() -> Result<Box<dyn EncodableBlock<Metal>>, MTLError> {if let crate::config::MLPConfig::Dense(dense_config) = config {match &dense_config.linear_config {- LinearConfig::Quantized(quantization_config)- | LinearConfig::MLXQuantized(quantization_config) => {- let data_type: DataType =- dense_config.linear_config.activation_precision().into();+ LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {+ let data_type: DataType = dense_config.linear_config.activation_precision().into();// Load up_projection weights directly for fused kernel- let up_projection_tree =- parameter_tree.subtree("up_projection").map_err(- |error| MTLError::Generic(format!("{:?}", error)),- )?;-- let up_projection_weights =- up_projection_tree.leaf("weights").map_err(|error| {- MTLError::Generic(format!(- "Failed to load up weights: {:?}",- error- ))- })?;- let up_projection_weights_buffer =- up_projection_weights.buffer().into();-- let up_projection_scales =- up_projection_tree.leaf("scales").map_err(|error| {- MTLError::Generic(format!(- "Failed to load up scales: {:?}",- error- ))- })?;- let up_projection_scales_buffer =- up_projection_scales.buffer().into();+ let up_projection_tree = parameter_tree+ .subtree("up_projection")+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;++ let up_projection_weights = up_projection_tree+ .leaf("weights")+ .map_err(|error| MTLError::Generic(format!("Failed to load up weights: {:?}", error)))?;+ let up_projection_weights_buffer = up_projection_weights.buffer().into();++ let up_projection_scales = up_projection_tree+ .leaf("scales")+ .map_err(|error| MTLError::Generic(format!("Failed to load up scales: {:?}", error)))?;+ let up_projection_scales_buffer = up_projection_scales.buffer().into();// Load zero_points or biases depending on quantization type- let (- up_projection_zero_points_or_biases_buffer,- quantization_type,- ) = if let Ok(biases) = up_projection_tree.leaf("biases") {- (biases.buffer().into(), QuantizationType::Mlx)- } else if let Ok(zero_points) =- up_projection_tree.leaf("zero_points")- {- (zero_points.buffer().into(), QuantizationType::ZeroPoint)- } else {- return Err(MTLError::Generic(- "Missing zero_points or biases for quantized up_projection"- .to_string(),+ let (up_projection_zero_points_or_biases_buffer, quantization_type) =+ if let Ok(biases) = up_projection_tree.leaf("biases") {+ (biases.buffer().into(), QuantizationType::Mlx)+ } else if let Ok(zero_points) = up_projection_tree.leaf("zero_points") {+ (zero_points.buffer().into(), QuantizationType::ZeroPoint)+ } else {+ return Err(MTLError::Generic(+ "Missing zero_points or biases for quantized up_projection".to_string(),));- };+ };// Create down projection as separate linearlet down_projection = QuantizedLinear::new(@@ -202,9 +158,9 @@ pub fn mlp_fused_block(quantization_config,hidden_dimension,model_dimension,- ¶meter_tree.subtree("down_projection").map_err(- |error| MTLError::Generic(format!("{:?}", error)),- )?,+ ¶meter_tree+ .subtree("down_projection")+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,ArrayId::MlpHidden,ArrayId::Main,)?;@@ -233,20 +189,14 @@ pub fn mlp_fused_block(let data_type: DataType = (*precision).into();// Load up_projection weights directly for fused kernel- let up_projection_tree =- parameter_tree.subtree("up_projection").map_err(- |error| MTLError::Generic(format!("{:?}", error)),- )?;-- let up_projection_weights =- up_projection_tree.leaf("weights").map_err(|error| {- MTLError::Generic(format!(- "Failed to load up weights: {:?}",- error- ))- })?;- let up_projection_weights_buffer =- up_projection_weights.buffer().into();+ let up_projection_tree = parameter_tree+ .subtree("up_projection")+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;++ let up_projection_weights = up_projection_tree+ .leaf("weights")+ .map_err(|error| MTLError::Generic(format!("Failed to load up weights: {:?}", error)))?;+ let up_projection_weights_buffer = up_projection_weights.buffer().into();// Create down projection as separate linearlet down_projection = FullPrecisionLinear::new(@@ -254,9 +204,9 @@ pub fn mlp_fused_block(data_type,hidden_dimension,model_dimension,- ¶meter_tree.subtree("down_projection").map_err(- |error| MTLError::Generic(format!("{:?}", error)),- )?,+ ¶meter_tree+ .subtree("down_projection")+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,ArrayId::MlpHidden,ArrayId::Main,)?;@@ -277,24 +227,14 @@ pub fn mlp_fused_block(LinearConfig::QLoRA {..} => {- return Err(MTLError::Generic(- "QLoRA MLP not supported".to_string(),- ));+ return Err(MTLError::Generic("QLoRA MLP not supported".to_string()));},}}- if let crate::config::MLPConfig::MixtureOfExperts(- mixture_of_experts_config,- ) = config- {- let mixture_of_experts_block = MoeBlock::new(- &context,- mixture_of_experts_config,- model_dimension,- hidden_dimension,- parameter_tree,- )?;+ if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {+ let mixture_of_experts_block =+ MoeBlock::new(&context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;return Ok(Box::new(mixture_of_experts_block));}@@ -311,9 +251,7 @@ pub fn embed_block(common,precision,} => {- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = FullPrecisionEmbeddingLookup::new(context,@@ -331,9 +269,7 @@ pub fn embed_block(common,precision,} => {- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = FullPrecisionEmbeddingLookup::new(context,@@ -352,9 +288,7 @@ pub fn embed_block(activation_precision,..} => {- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = FullPrecisionEmbeddingLookup::new(context,@@ -375,12 +309,9 @@ pub fn embed_block(..} => {let data_type: DataType = (*activation_precision).into();- let input_scale =- config.embedding_config.common().input_scale.unwrap_or(1.0);+ let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = QuantizedEmbeddingLookup::new_untied_input(context,@@ -403,14 +334,11 @@ pub fn embed_block(} => {let data_type: DataType = (*activation_precision).into();- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");// For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dimlet group_size = config.model_dim;- let input_scale =- config.embedding_config.common().input_scale.unwrap_or(1.0);+ let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);let block = QuantizedEmbeddingLookup::new_tied(context,@@ -431,14 +359,10 @@ pub fn embed_block(embedding_quantization_mode,..} => {- let data_type: DataType =- config.output_norm_config.scale_precision.into();- let input_scale =- config.embedding_config.common().input_scale.unwrap_or(1.0);+ let data_type: DataType = config.output_norm_config.scale_precision.into();+ let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = QuantizedEmbeddingLookup::new_tied(context,@@ -467,9 +391,7 @@ pub fn readout_block(precision,..} => {- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = FullPrecisionEmbeddingReadout::new(context,@@ -486,9 +408,7 @@ pub fn readout_block(precision,..} => {- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = FullPrecisionEmbeddingReadout::new(context,@@ -508,9 +428,7 @@ pub fn readout_block(..} => {let data_type: DataType = (*activation_precision).into();- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = QuantizedEmbeddingReadout::new_untied_output(context,@@ -532,9 +450,7 @@ pub fn readout_block(} => {let data_type: DataType = (*activation_precision).into();- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");// For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dimlet group_size = config.model_dim;@@ -557,12 +473,9 @@ pub fn readout_block(embedding_quantization_mode,..} => {- let data_type: DataType =- config.output_norm_config.scale_precision.into();+ let data_type: DataType = config.output_norm_config.scale_precision.into();- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = QuantizedEmbeddingReadout::new_tied(context,@@ -582,12 +495,9 @@ pub fn readout_block(embedding_quantization_mode,..} => {- let data_type: DataType =- config.output_norm_config.scale_precision.into();+ let data_type: DataType = config.output_norm_config.scale_precision.into();- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");let block = QuantizedEmbeddingReadout::new_untied_output(context,diff --git a/crates/uzu/src/backends/metal/image.rs b/crates/uzu/src/backends/metal/image.rsindex 04e4b4afe..e11ce662d 100644--- a/crates/uzu/src/backends/metal/image.rs+++ b/crates/uzu/src/backends/metal/image.rs@@ -1,9 +1,8 @@use std::fmt;use crate::backends::metal::{- MTLDevice, MTLDeviceExt, MTLPixelFormat, MTLResourceExt, MTLStorageMode,- MTLTexture, MTLTextureDescriptor, MTLTextureUsage, ProtocolObject,- Retained,+ MTLDevice, MTLDeviceExt, MTLPixelFormat, MTLResourceExt, MTLStorageMode, MTLTexture, MTLTextureDescriptor,+ MTLTextureUsage, ProtocolObject, Retained,};#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -93,13 +92,7 @@ impl Image {height: u32,pixel_format: PixelFormat,) -> Self {- Self::new_with_usage(- device,- width,- height,- pixel_format,- TextureUsage::default(),- )+ Self::new_with_usage(device, width, height, pixel_format, TextureUsage::default())}pub fn new_with_usage(@@ -116,9 +109,7 @@ impl Image {descriptor.set_storage_mode(MTLStorageMode::Private);descriptor.set_usage(usage.into());- let texture = device- .new_texture_with_descriptor(&descriptor)- .expect("Failed to create texture");+ let texture = device.new_texture_with_descriptor(&descriptor).expect("Failed to create texture");texture.set_label(Some("Image"));Self {@@ -133,9 +124,7 @@ impl Image {device: &Retained<ProtocolObject<dyn MTLDevice>>,descriptor: &MTLTextureDescriptor,) -> Self {- let texture = device- .new_texture_with_descriptor(descriptor)- .expect("Failed to create texture");+ let texture = device.new_texture_with_descriptor(descriptor).expect("Failed to create texture");texture.set_label(Some("Image"));let width = descriptor.width() as u32;@@ -150,9 +139,7 @@ impl Image {}}- pub fn from_texture(- texture: Retained<ProtocolObject<dyn MTLTexture>>- ) -> Self {+ pub fn from_texture(texture: Retained<ProtocolObject<dyn MTLTexture>>) -> Self {let width = texture.width() as u32;let height = texture.height() as u32;let pixel_format = texture.pixel_format().into();@@ -204,10 +191,6 @@ impl fmt::Display for Image {&self,f: &mut fmt::Formatter<'_>,) -> fmt::Result {- write!(- f,- "Image({}x{}, {:?})",- self.width, self.height, self.pixel_format- )+ write!(f, "Image({}x{}, {:?})", self.width, self.height, self.pixel_format)}}diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rsindex 7d4cd65c1..979fd6471 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs@@ -4,8 +4,7 @@ use objc2::rc::Retained;use thiserror::Error;use crate::backends::metal::{- KernelDataType, MTLBuffer, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLDataType, MTLError,+ KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLDataType, MTLError,MTLFunctionConstantValues, MTLSize, ProtocolObject,};@@ -20,18 +19,10 @@ pub enum AttentionKernelVariant {type PipelineKey = (usize, bool, bool, bool); // (head_dim, has_sinks, is_causal, has_mask)pub struct AttentionKernelPipelines {- single_pass: HashMap<- PipelineKey,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,- two_pass_1: HashMap<- PipelineKey,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,- two_pass_2:- HashMap<usize, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- kv_cache_update:- Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,+ single_pass: HashMap<PipelineKey, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,+ two_pass_1: HashMap<PipelineKey, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,+ two_pass_2: HashMap<usize, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,+ kv_cache_update: Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,}pub struct AttentionKernel {@@ -55,16 +46,16 @@ pub struct AttentionSinglePassArguments<'a> {pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)pub gqa_factor: i32, // buffer(4)- pub sequence_length: i32, // buffer(5) - sequence_length- pub k_head_stride: i32, // buffer(6)- pub k_seq_stride: i32, // buffer(7)- pub v_head_stride: i32, // buffer(8)- pub v_seq_stride: i32, // buffer(9)- pub scale: f32, // buffer(10)+ pub sequence_length: i32, // buffer(5) - sequence_length+ pub k_head_stride: i32, // buffer(6)+ pub k_seq_stride: i32, // buffer(7)+ pub v_head_stride: i32, // buffer(8)+ pub v_seq_stride: i32, // buffer(9)+ pub scale: f32, // buffer(10)pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(11/12)- pub mask_kv_seq_stride: i32, // buffer(13)- pub mask_q_seq_stride: i32, // buffer(14)- pub mask_head_stride: i32, // buffer(15)+ pub mask_kv_seq_stride: i32, // buffer(13)+ pub mask_q_seq_stride: i32, // buffer(14)+ pub mask_head_stride: i32, // buffer(15)pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(16)pub num_heads: usize,pub suffix_length: usize,@@ -73,24 +64,24 @@ pub struct AttentionSinglePassArguments<'a> {}pub struct AttentionTwoPassArguments<'a> {- pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)- pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)- pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)+ pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)+ pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)+ pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)pub partials_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 1 output- pub sums_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) - pass 1 output- pub maxs_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) - pass 1 output- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 2 output- pub gqa_factor: i32, // buffer(6)- pub sequence_length: i32, // buffer(7) - sequence_length- pub k_head_stride: i32, // buffer(8)- pub k_seq_stride: i32, // buffer(9)- pub v_head_stride: i32, // buffer(10)- pub v_seq_stride: i32, // buffer(11)- pub scale: f32, // buffer(12)+ pub sums_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) - pass 1 output+ pub maxs_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) - pass 1 output+ pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 2 output+ pub gqa_factor: i32, // buffer(6)+ pub sequence_length: i32, // buffer(7) - sequence_length+ pub k_head_stride: i32, // buffer(8)+ pub k_seq_stride: i32, // buffer(9)+ pub v_head_stride: i32, // buffer(10)+ pub v_seq_stride: i32, // buffer(11)+ pub scale: f32, // buffer(12)pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(13/14)- pub mask_kv_seq_stride: i32, // buffer(15)- pub mask_q_seq_stride: i32, // buffer(16)- pub mask_head_stride: i32, // buffer(17)+ pub mask_kv_seq_stride: i32, // buffer(15)+ pub mask_q_seq_stride: i32, // buffer(16)+ pub mask_head_stride: i32, // buffer(17)pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(18)pub num_heads: usize,pub suffix_length: usize,@@ -140,36 +131,16 @@ fn make_function_constants(let bool_mask_value = false;let float_mask_value = has_mask_value;- function_constants.set_constant_value_type_at_index(- NonNull::from(&has_mask_value).cast(),- MTLDataType::Bool,- 20,- ); // has_mask+ function_constants.set_constant_value_type_at_index(NonNull::from(&has_mask_value).cast(), MTLDataType::Bool, 20); // has_maskfunction_constants.set_constant_value_type_at_index(NonNull::from(&query_transposed_value).cast(),MTLDataType::Bool,21,); // query_transposed- function_constants.set_constant_value_type_at_index(- NonNull::from(&bool_mask_value).cast(),- MTLDataType::Bool,- 23,- ); // bool_mask- function_constants.set_constant_value_type_at_index(- NonNull::from(&float_mask_value).cast(),- MTLDataType::Bool,- 24,- ); // float_mask- function_constants.set_constant_value_type_at_index(- NonNull::from(&is_causal_value).cast(),- MTLDataType::Bool,- 22,- ); // do_causal- function_constants.set_constant_value_type_at_index(- NonNull::from(&has_sinks_value).cast(),- MTLDataType::Bool,- 25,- ); // has_sinks+ function_constants.set_constant_value_type_at_index(NonNull::from(&bool_mask_value).cast(), MTLDataType::Bool, 23); // bool_mask+ function_constants.set_constant_value_type_at_index(NonNull::from(&float_mask_value).cast(), MTLDataType::Bool, 24); // float_mask+ function_constants.set_constant_value_type_at_index(NonNull::from(&is_causal_value).cast(), MTLDataType::Bool, 22); // do_causal+ function_constants.set_constant_value_type_at_index(NonNull::from(&has_sinks_value).cast(), MTLDataType::Bool, 25); // has_sinksfunction_constants}@@ -190,59 +161,28 @@ impl AttentionKernel {for &has_sinks_value in &[false, true] {for &is_causal_value in &[false, true] {for &has_mask_value in &[false, true] {- let function_constants = make_function_constants(- has_mask_value,- has_sinks_value,- is_causal_value,- );+ let function_constants = make_function_constants(has_mask_value, has_sinks_value, is_causal_value);for &head_dim in &supported_head_dims {if let Ok(pipeline) = context.compute_pipeline_state(- &format!(- "attention_single_pass_{}_{}",- data_suffix, head_dim- ),+ &format!("attention_single_pass_{}_{}", data_suffix, head_dim),Some(&function_constants),) {- single_pass.insert(- (- head_dim,- has_sinks_value,- is_causal_value,- has_mask_value,- ),- pipeline,- );+ single_pass.insert((head_dim, has_sinks_value, is_causal_value, has_mask_value), pipeline);}if let Ok(pipeline) = context.compute_pipeline_state(- &format!(- "attention_2pass_1_{}_{}",- data_suffix, head_dim- ),+ &format!("attention_2pass_1_{}_{}", data_suffix, head_dim),Some(&function_constants),) {- two_pass_1.insert(- (- head_dim,- has_sinks_value,- is_causal_value,- has_mask_value,- ),- pipeline,- );+ two_pass_1.insert((head_dim, has_sinks_value, is_causal_value, has_mask_value), pipeline);}if !two_pass_2.contains_key(&head_dim) {- if let Ok(pipeline) = context- .compute_pipeline_state(- &format!(- "attention_2pass_2_{}_{}",- data_suffix, head_dim- ),- Some(&function_constants),- )- {+ if let Ok(pipeline) = context.compute_pipeline_state(+ &format!("attention_2pass_2_{}_{}", data_suffix, head_dim),+ Some(&function_constants),+ ) {two_pass_2.insert(head_dim, pipeline);}}@@ -251,12 +191,7 @@ impl AttentionKernel {}}- let kv_cache_update = context- .compute_pipeline_state(- &format!("update_kv_cache_{}", data_suffix),- None,- )- .ok();+ let kv_cache_update = context.compute_pipeline_state(&format!("update_kv_cache_{}", data_suffix), None).ok();Ok(Self {data_type,@@ -275,9 +210,7 @@ impl AttentionKernel {is_causal: bool,has_mask: bool,) -> bool {- self.pipelines- .single_pass- .contains_key(&(head_dim, false, is_causal, has_mask))+ self.pipelines.single_pass.contains_key(&(head_dim, false, is_causal, has_mask))}pub fn supports_two_pass(@@ -286,9 +219,7 @@ impl AttentionKernel {is_causal: bool,has_mask: bool,) -> bool {- self.pipelines- .two_pass_1- .contains_key(&(head_dim, false, is_causal, has_mask))+ self.pipelines.two_pass_1.contains_key(&(head_dim, false, is_causal, has_mask))&& self.pipelines.two_pass_2.contains_key(&head_dim)}@@ -299,9 +230,7 @@ impl AttentionKernel {is_causal: bool,has_mask: bool,) -> AttentionKernelVariant {- if self.supports_two_pass(head_dim, is_causal, has_mask)- && sequence_length > 1024- {+ if self.supports_two_pass(head_dim, is_causal, has_mask) && sequence_length > 1024 {AttentionKernelVariant::TwoPass} else {AttentionKernelVariant::SinglePass@@ -321,82 +250,47 @@ impl AttentionKernel {.get(&(args.head_dim, has_sinks, args.is_causal, has_mask)).ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;- MTLComputeCommandEncoder::set_compute_pipeline_state(- compute_encoder,- pipeline,- );+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pipeline);- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.queries_buffer),- 0,- 0,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.keys_buffer),- 0,- 1,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.values_buffer),- 0,- 2,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.output_buffer),- 0,- 3,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);unsafe {MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.gqa_factor as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.gqa_factor as *const i32 as *mut _),size_of::<i32>(),4,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.sequence_length as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.sequence_length as *const i32 as *mut _),size_of::<i32>(),5,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.k_head_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.k_head_stride as *const i32 as *mut _),size_of::<i32>(),6,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.k_seq_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.k_seq_stride as *const i32 as *mut _),size_of::<i32>(),7,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.v_head_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.v_head_stride as *const i32 as *mut _),size_of::<i32>(),8,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.v_seq_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.v_seq_stride as *const i32 as *mut _),size_of::<i32>(),9,);@@ -409,34 +303,23 @@ impl AttentionKernel {}if let Some(mask_buffer) = args.mask_buffer {- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(mask_buffer),- 0,- 12,- ); // float_mask+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 12); // float_maskunsafe {MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.mask_kv_seq_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.mask_kv_seq_stride as *const i32 as *mut _),size_of::<i32>(),13,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.mask_q_seq_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.mask_q_seq_stride as *const i32 as *mut _),size_of::<i32>(),14,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.mask_head_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.mask_head_stride as *const i32 as *mut _),size_of::<i32>(),15,);@@ -444,12 +327,7 @@ impl AttentionKernel {}if let Some(sinks_buffer) = args.sinks_buffer {- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(sinks_buffer),- 0,- 16,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 16);}let threads_per_threadgroup = MTLSize {@@ -458,13 +336,9 @@ impl AttentionKernel {depth: 1,};- let threadgroups_per_grid =- MTLSize::new(args.num_heads, args.suffix_length, 1);+ let threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, 1);- compute_encoder.dispatch_threadgroups(- threadgroups_per_grid,- threads_per_threadgroup,- );+ compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);Ok(())}@@ -481,99 +355,55 @@ impl AttentionKernel {.get(&(args.head_dim, has_sinks, args.is_causal, has_mask)).ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;- let pass2_pipeline =- self.pipelines.two_pass_2.get(&args.head_dim).ok_or_else(|| {- AttentionError::UnsupportedHeadDim(args.head_dim)- })?;+ let pass2_pipeline = self+ .pipelines+ .two_pass_2+ .get(&args.head_dim)+ .ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;- MTLComputeCommandEncoder::set_compute_pipeline_state(- compute_encoder,- pass1_pipeline,- );+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pass1_pipeline);- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.queries_buffer),- 0,- 0,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.keys_buffer),- 0,- 1,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.values_buffer),- 0,- 2,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.partials_buffer),- 0,- 3,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.sums_buffer),- 0,- 4,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.maxs_buffer),- 0,- 5,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.partials_buffer), 0, 3);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.sums_buffer), 0, 4);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.maxs_buffer), 0, 5);unsafe {MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.gqa_factor as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.gqa_factor as *const i32 as *mut _),size_of::<i32>(),6,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.sequence_length as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.sequence_length as *const i32 as *mut _),size_of::<i32>(),7,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.k_head_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.k_head_stride as *const i32 as *mut _),size_of::<i32>(),8,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.k_seq_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.k_seq_stride as *const i32 as *mut _),size_of::<i32>(),9,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.v_head_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.v_head_stride as *const i32 as *mut _),size_of::<i32>(),10,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.v_seq_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.v_seq_stride as *const i32 as *mut _),size_of::<i32>(),11,);@@ -587,34 +417,23 @@ impl AttentionKernel {// Set mask buffer if presentif let Some(mask_buffer) = args.mask_buffer {- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(mask_buffer),- 0,- 14,- ); // float_mask+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 14); // float_maskunsafe {MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.mask_kv_seq_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.mask_kv_seq_stride as *const i32 as *mut _),size_of::<i32>(),15,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.mask_q_seq_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.mask_q_seq_stride as *const i32 as *mut _),size_of::<i32>(),16,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &args.mask_head_stride as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&args.mask_head_stride as *const i32 as *mut _),size_of::<i32>(),17,);@@ -622,12 +441,7 @@ impl AttentionKernel {}if let Some(sinks_buffer) = args.sinks_buffer {- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(sinks_buffer),- 0,- 18,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 18);}let total_blocks_count = 32u64;@@ -636,59 +450,25 @@ impl AttentionKernel {height: 1,depth: 1,};- let pass1_threadgroups_per_grid = MTLSize::new(- args.num_heads,- args.suffix_length,- total_blocks_count as usize,- );+ let pass1_threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, total_blocks_count as usize);- compute_encoder.dispatch_threadgroups(- pass1_threadgroups_per_grid,- pass1_threads_per_threadgroup,- );+ compute_encoder.dispatch_threadgroups(pass1_threadgroups_per_grid, pass1_threads_per_threadgroup);- MTLComputeCommandEncoder::set_compute_pipeline_state(- compute_encoder,- pass2_pipeline,- );+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pass2_pipeline);- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.partials_buffer),- 0,- 0,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.sums_buffer),- 0,- 1,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.maxs_buffer),- 0,- 2,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.output_buffer),- 0,- 3,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.partials_buffer), 0, 0);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.sums_buffer), 0, 1);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.maxs_buffer), 0, 2);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);let pass2_threads_per_threadgroup = MTLSize {width: 32 * 32,height: 1,depth: 1,};- let pass2_threadgroups_per_grid =- MTLSize::new(args.num_heads, args.suffix_length, 1);+ let pass2_threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, 1);- compute_encoder.dispatch_threadgroups(- pass2_threadgroups_per_grid,- pass2_threads_per_threadgroup,- );+ compute_encoder.dispatch_threadgroups(pass2_threadgroups_per_grid, pass2_threads_per_threadgroup);Ok(())}@@ -698,105 +478,63 @@ impl AttentionKernel {compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,args: KVCacheUpdateArguments,) -> Result<(), AttentionError> {- let pipeline =- self.pipelines.kv_cache_update.as_ref().ok_or_else(|| {- AttentionError::FunctionNotFound(- "KV cache update kernel".to_string(),- )- })?;-- MTLComputeCommandEncoder::set_compute_pipeline_state(- compute_encoder,- pipeline,- );+ let pipeline = self+ .pipelines+ .kv_cache_update+ .as_ref()+ .ok_or_else(|| AttentionError::FunctionNotFound("KV cache update kernel".to_string()))?;++ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pipeline);// Set buffers- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.rotated_keys_buffer),- 0,- 0,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.qkv_buffer),- 0,- 1,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.key_cache_buffer),- 0,- 2,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.value_cache_buffer),- 0,- 3,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.rotated_keys_buffer), 0, 0);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.qkv_buffer), 0, 1);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.key_cache_buffer), 0, 2);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.value_cache_buffer), 0, 3);// Set constantsunsafe {MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &(args.num_groups as i32) as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&(args.num_groups as i32) as *const i32 as *mut _),size_of::<i32>(),4,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &(args.num_heads as i32) as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&(args.num_heads as i32) as *const i32 as *mut _),size_of::<i32>(),5,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &(args.head_dim as i32) as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&(args.head_dim as i32) as *const i32 as *mut _),size_of::<i32>(),6,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &(args.suffix_length as i32) as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&(args.suffix_length as i32) as *const i32 as *mut _),size_of::<i32>(),7,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &(args.segment_prefix_length as i32) as *const i32- as *mut _,- ),+ NonNull::new_unchecked(&(args.segment_prefix_length as i32) as *const i32 as *mut _),size_of::<i32>(),8,);MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &(args.max_sequence_length as i32) as *const i32 as *mut _,- ),+ NonNull::new_unchecked(&(args.max_sequence_length as i32) as *const i32 as *mut _),size_of::<i32>(),9,);}- let threads_per_grid = MTLSize::new(- args.num_groups,- args.suffix_length,- args.head_dim as usize,- );+ let threads_per_grid = MTLSize::new(args.num_groups, args.suffix_length, args.head_dim as usize);- let threadgroup_depth =- std::cmp::min(args.head_dim.max(1), 64) as usize;+ let threadgroup_depth = std::cmp::min(args.head_dim.max(1), 64) as usize;MTLComputeCommandEncoder::dispatch_threads(compute_encoder,@@ -836,31 +574,11 @@ impl AttentionKernel {let has_sinks = args.sinks_buffer.is_some();let fcv = MTLFunctionConstantValues::new();- fcv.set_constant_value_type_at_index(- NonNull::from(&align_q).cast(),- MTLDataType::Bool,- 200,- );- fcv.set_constant_value_type_at_index(- NonNull::from(&align_k).cast(),- MTLDataType::Bool,- 201,- );- fcv.set_constant_value_type_at_index(- NonNull::from(&has_mask).cast(),- MTLDataType::Bool,- 300,- );- fcv.set_constant_value_type_at_index(- NonNull::from(&args.is_causal).cast(),- MTLDataType::Bool,- 301,- );- fcv.set_constant_value_type_at_index(- NonNull::from(&has_sinks).cast(),- MTLDataType::Bool,- 302,- );+ fcv.set_constant_value_type_at_index(NonNull::from(&align_q).cast(), MTLDataType::Bool, 200);+ fcv.set_constant_value_type_at_index(NonNull::from(&align_k).cast(), MTLDataType::Bool, 201);+ fcv.set_constant_value_type_at_index(NonNull::from(&has_mask).cast(), MTLDataType::Bool, 300);+ fcv.set_constant_value_type_at_index(NonNull::from(&args.is_causal).cast(), MTLDataType::Bool, 301);+ fcv.set_constant_value_type_at_index(NonNull::from(&has_sinks).cast(), MTLDataType::Bool, 302);// Kernel name matches gemm_attention.metal instantiations:// attention_gemm_{f16|bf16|f32}_{head_dim}_bk{bk}@@ -870,57 +588,24 @@ impl AttentionKernel {KernelDataType::Float32 => "f32",};- let function_name =- format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);+ let function_name = format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);let cache_key = format!("{}_aq{}_ak{}_m{}_c{}_s{}",- function_name,- align_q as u8,- align_k as u8,- has_mask as u8,- args.is_causal as u8,- has_sinks as u8+ function_name, align_q as u8, align_k as u8, has_mask as u8, args.is_causal as u8, has_sinks as u8);let pipeline = context- .compute_pipeline_state_cached(- &cache_key,- &function_name,- Some(&fcv),- )+ .compute_pipeline_state_cached(&cache_key, &function_name, Some(&fcv)).map_err(AttentionError::MetalError)?;- MTLComputeCommandEncoder::set_compute_pipeline_state(- compute_encoder,- &pipeline,- );+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, &pipeline);// Buffers- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.queries_buffer),- 0,- 0,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.keys_buffer),- 0,- 1,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.values_buffer),- 0,- 2,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(args.output_buffer),- 0,- 3,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);// Params (all strides in elements)let q_head_stride = (args.suffix_length * args.head_dim) as i64;@@ -971,43 +656,27 @@ impl AttentionKernel {unsafe {MTLComputeCommandEncoder::set_bytes(compute_encoder,- NonNull::new_unchecked(- &mask_params as *const AttnMaskParams as *mut _,- ),+ NonNull::new_unchecked(&mask_params as *const AttnMaskParams as *mut _),size_of::<AttnMaskParams>(),5,);}- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(mask_buffer),- 0,- 6,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 6);}if let Some(sinks_buffer) = args.sinks_buffer {- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(sinks_buffer),- 0,- 16,- );+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 16);}// Dispatch- let threadgroups_per_grid =- MTLSize::new(nq as usize, args.num_heads, 1);+ let threadgroups_per_grid = MTLSize::new(nq as usize, args.num_heads, 1);let threads_per_threadgroup = MTLSize {width: 32,height: WM as usize,depth: WN as usize,};- compute_encoder.dispatch_threadgroups(- threadgroups_per_grid,- threads_per_threadgroup,- );+ compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);Ok(())}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rsindex d05d3f17f..44f19c034 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs@@ -1,8 +1,6 @@mod matmul_arguments;mod transpose_configuration;-pub use crate::backends::common::gpu_types::{- GEMMAddMMParams, GEMMParams, GEMMSpiltKMlpFusedParams, GEMMSpiltKParams,-};+pub use crate::backends::common::gpu_types::{GEMMAddMMParams, GEMMParams, GEMMSpiltKMlpFusedParams, GEMMSpiltKParams};pub use matmul_arguments::MatmulArguments;pub(crate) use transpose_configuration::transpose_configuration;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rsindex 2e887ca30..91690f38a 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs@@ -43,21 +43,15 @@ pub(crate) fn choose_dispatch_descriptor(data_type: DataType,arguments: &MatmulArguments,) -> Result<MatmulDispatchDescriptor, MTLError> {- if let Some(descriptor) =- gemv::DispatchDescriptor::try_new(context, data_type, arguments)?- {+ if let Some(descriptor) = gemv::DispatchDescriptor::try_new(context, data_type, arguments)? {return Ok(MatmulDispatchDescriptor::Gemv(descriptor));}- if let Some(descriptor) =- split_k::DispatchDescriptor::try_new(context, data_type, arguments)?- {+ if let Some(descriptor) = split_k::DispatchDescriptor::try_new(context, data_type, arguments)? {return Ok(MatmulDispatchDescriptor::SplitK(descriptor));}- Ok(MatmulDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(- context, data_type, arguments,- )?))+ Ok(MatmulDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(context, data_type, arguments)?))}pub fn determine_kernel_variant(@@ -65,6 +59,5 @@ pub fn determine_kernel_variant(data_type: DataType,arguments: &MatmulArguments,) -> Result<MatmulKernelVariant, MTLError> {- choose_dispatch_descriptor(context, data_type, arguments)- .map(|d| d.variant())+ choose_dispatch_descriptor(context, data_type, arguments).map(|d| d.variant())}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rsindex 72208d5f0..47fb00a6c 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs@@ -1,14 +1,9 @@-use super::{- pipeline_configuration::PipelineConfiguration,- tile_configuration::TileConfiguration,-};+use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::TileConfiguration};use crate::{DataType,backends::metal::{DeviceClass, MTLContext, MTLError, MTLSize,- kernel::matmul::common::{- GEMMAddMMParams, GEMMParams, MatmulArguments,- },+ kernel::matmul::common::{GEMMAddMMParams, GEMMParams, MatmulArguments},},};@@ -27,11 +22,8 @@ impl DispatchDescriptor {data_type: DataType,arguments: &MatmulArguments,) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for GEMM: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for GEMM: {data_type:?}")));}let tile = select_tile(context, data_type, arguments);@@ -49,8 +41,7 @@ impl DispatchDescriptor {align_k: (k % tile.block_depth) == 0,has_batch: arguments.batch_count > 1,use_out_source: arguments.c.is_some(),- do_axpby: arguments.c.is_some()- && (arguments.alpha != 1.0 || arguments.beta != 0.0),+ do_axpby: arguments.c.is_some() && (arguments.alpha != 1.0 || arguments.beta != 0.0),};let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;@@ -61,15 +52,13 @@ impl DispatchDescriptor {let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;let tn_swizzled = tiles_n * tile_swizzle;- let elements_per_matrix_a =- (arguments.batch as i64) * (arguments.lda as i64);+ let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);let elements_per_matrix_b = if arguments.transpose_b {(arguments.output_dim as i64) * (arguments.ldb as i64)} else {(arguments.input_dim as i64) * (arguments.ldb as i64)};- let elements_per_matrix_d =- (arguments.batch as i64) * (arguments.ldd as i64);+ let elements_per_matrix_d = (arguments.batch as i64) * (arguments.ldd as i64);let params = GEMMParams {M: m,@@ -105,16 +94,8 @@ impl DispatchDescriptor {None};- let threads_per_threadgroup = MTLSize::new(- 32,- tile.warps_per_col as usize,- tile.warps_per_row as usize,- );- let threadgroups = MTLSize::new(- tn_swizzled as usize,- tm_swizzled as usize,- arguments.batch_count as usize,- );+ let threads_per_threadgroup = MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);+ let threadgroups = MTLSize::new(tn_swizzled as usize, tm_swizzled as usize, arguments.batch_count as usize);Ok(Self {pipeline_configuration,@@ -131,16 +112,14 @@ fn select_tile(data_type: DataType,arguments: &MatmulArguments,) -> TileConfiguration {- let overall_work_elements = (arguments.batch_count as i64)- * (arguments.batch as i64)- * (arguments.output_dim as i64);+ let overall_work_elements =+ (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);let is_float32 = matches!(data_type, DataType::F32);let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();if context.is_nax_available() && prefer_half_or_tf32 {let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);- let tile_rows =- (arguments.batch + base_tile.block_rows - 1) / base_tile.block_rows;+ let tile_rows = (arguments.batch + base_tile.block_rows - 1) / base_tile.block_rows;let swizzle_log2 = if tile_rows <= 3 {0} else {@@ -153,9 +132,7 @@ fn select_tile(}match context.architecture.device_class {- DeviceClass::Integrated- | DeviceClass::Phone- | DeviceClass::Unknown(_) => {+ DeviceClass::Integrated | DeviceClass::Phone | DeviceClass::Unknown(_) => {if prefer_half_or_tf32 {if !arguments.transpose_a && arguments.transpose_b {TileConfiguration::new(64, 32, 32, 2, 2, 0)@@ -171,9 +148,7 @@ fn select_tile(DeviceClass::Desktop => {if overall_work_elements >= (1_i64 << 20) {if prefer_half_or_tf32 {- if 2 * std::cmp::max(arguments.batch, arguments.output_dim)- > arguments.input_dim- {+ if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {TileConfiguration::new(64, 64, 16, 2, 2, 0)} else if !arguments.transpose_a && arguments.transpose_b {TileConfiguration::new(64, 32, 32, 2, 2, 0)diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex d115da827..8c739d928 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -1,14 +1,10 @@use std::{collections::HashMap, ptr::NonNull};-use super::{- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,- tile_configuration::TileConfiguration,-};+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration, tile_configuration::TileConfiguration};use crate::{DataType,backends::metal::{- ComputeEncoderSetValue, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError,+ ComputeEncoderSetValue, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,MTLFunctionConstantValues, ProtocolObject, Retained,kernel::matmul::common::{MatmulArguments, transpose_configuration},},@@ -16,19 +12,13 @@ use crate::{pub struct Kernel {data_type: DataType,- pipelines: HashMap<- PipelineConfiguration,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,+ pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,}impl Kernel {pub fn new(data_type: DataType) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for GEMM: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for GEMM: {data_type:?}")));}Ok(Self {data_type,@@ -41,36 +31,21 @@ impl Kernel {&mut self,context: &MTLContext,) -> Result<(), MTLError> {- let tiles_and_alignments: &[(- TileConfiguration,- &[(bool, bool, bool)],- )] = match self.data_type {+ let tiles_and_alignments: &[(TileConfiguration, &[(bool, bool, bool)])] = match self.data_type {DataType::BF16 => &[- (- TileConfiguration::new(64, 32, 32, 2, 2, 0),- &[(false, true, true), (true, true, true)],- ),+ (TileConfiguration::new(64, 32, 32, 2, 2, 0), &[(false, true, true), (true, true, true)]),(TileConfiguration::new(64, 64, 16, 2, 2, 0),- &[- (false, true, true),- (true, false, true),- (true, true, true),- ],- ),- (- TileConfiguration::new(64, 64, 16, 1, 2, 0),- &[(true, true, true)],+ &[(false, true, true), (true, false, true), (true, true, true)],),+ (TileConfiguration::new(64, 64, 16, 1, 2, 0), &[(true, true, true)]),],- DataType::F16 => &[(- TileConfiguration::new(64, 64, 16, 2, 2, 0),- &[(true, true, true), (false, true, true)],- )],- DataType::F32 => &[(- TileConfiguration::new(32, 64, 16, 1, 2, 0),- &[(false, true, true), (true, true, true)],- )],+ DataType::F16 => {+ &[(TileConfiguration::new(64, 64, 16, 2, 2, 0), &[(true, true, true), (false, true, true)])]+ },+ DataType::F32 => {+ &[(TileConfiguration::new(32, 64, 16, 1, 2, 0), &[(false, true, true), (true, true, true)])]+ },_ => return Ok(()),};@@ -108,10 +83,7 @@ impl Kernel {configuration: &PipelineConfiguration,) -> String {let type_name = self.type_name();- let transpose_suffix = transpose_configuration(- configuration.transpose_a,- configuration.transpose_b,- );+ let transpose_suffix = transpose_configuration(configuration.transpose_a, configuration.transpose_b);let prefix = if configuration.tile.is_nax() {"steel_gemm_nax"} else {@@ -135,8 +107,7 @@ impl Kernel {&mut self,context: &MTLContext,configuration: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if !self.pipelines.contains_key(configuration) {let kernel_name = self.kernel_name(configuration);let function_constants = MTLFunctionConstantValues::new();@@ -181,11 +152,8 @@ impl Kernel {configuration.use_out_source as u8,configuration.do_axpby as u8);- let pipeline_state = context.compute_pipeline_state_cached(- &cache_key,- &kernel_name,- Some(&function_constants),- )?;+ let pipeline_state =+ context.compute_pipeline_state_cached(&cache_key, &kernel_name, Some(&function_constants))?;self.pipelines.insert(configuration.clone(), pipeline_state);}Ok(self.pipelines.get(configuration).unwrap())@@ -198,10 +166,7 @@ impl Kernel {arguments: &MatmulArguments,descriptor: &DispatchDescriptor,) -> Result<bool, MTLError> {- let pipeline_state = self.get_or_compile_pipeline(- context,- &descriptor.pipeline_configuration,- )?;+ let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;encoder.set_compute_pipeline_state(pipeline_state);encoder.set_buffer(Some(arguments.a), arguments.a_offset as usize, 0);@@ -219,10 +184,7 @@ impl Kernel {encoder.set_value(addmm_params, 5);}- encoder.dispatch_threadgroups(- descriptor.threadgroups,- descriptor.threads_per_threadgroup,- );+ encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);Ok(false)}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rsindex 4df5febee..9cf36aa3f 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs@@ -1,13 +1,9 @@use std::sync::OnceLock;-use super::pipeline_configuration::{- PipelineConfiguration, select_configuration,-};+use super::pipeline_configuration::{PipelineConfiguration, select_configuration};use crate::{DataType,- backends::metal::{- MTLContext, MTLError, MTLSize, kernel::matmul::common::MatmulArguments,- },+ backends::metal::{MTLContext, MTLError, MTLSize, kernel::matmul::common::MatmulArguments},};const DEFAULT_GEMV_MAX_BATCH: i32 = 8;@@ -16,10 +12,7 @@ static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();fn max_gemv_batch_threshold() -> i32 {*GEMV_MAX_BATCH.get_or_init(|| {- std::env::var("UZU_GEMV_MAX_BATCH")- .ok()- .and_then(|s| s.parse().ok())- .unwrap_or(DEFAULT_GEMV_MAX_BATCH)+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)})}@@ -62,11 +55,8 @@ impl DispatchDescriptor {data_type: DataType,arguments: &MatmulArguments,) -> Result<Option<Self>, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported data type for GEMV: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported data type for GEMV: {data_type:?}")));}if arguments.transpose_a || !arguments.transpose_b {@@ -104,9 +94,7 @@ impl DispatchDescriptor {let (do_axpby, alpha, beta, bias_stride) = match axpby_source {AxpbySource::None => (false, 1.0f32, 0.0f32, 0),AxpbySource::Bias => (true, 1.0f32, 1.0f32, 1),- AxpbySource::C => {- (true, arguments.alpha, arguments.beta, arguments.ldd)- },+ AxpbySource::C => (true, arguments.alpha, arguments.beta, arguments.ldd),};let output_dimension = if matrix_is_rhs {@@ -116,10 +104,7 @@ impl DispatchDescriptor {};let mut batch_pack = 1;- if m == 4- && arguments.input_dim <= 2048- && (1536..=3072).contains(&output_dimension)- {+ if m == 4 && arguments.input_dim <= 2048 && (1536..=3072).contains(&output_dimension) {batch_pack = 2;} else if m <= 8 {batch_pack = if m >= 4 && m % 4 == 0 {@@ -157,8 +142,7 @@ impl DispatchDescriptor {1}];- let elements_per_matrix_a =- (arguments.batch as i64) * (arguments.lda as i64);+ let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);let elements_per_matrix_b = if arguments.transpose_b {(arguments.output_dim as i64) * (arguments.ldb as i64)} else {@@ -183,23 +167,17 @@ impl DispatchDescriptor {0}];- let output_elements_per_threadgroup =- pipeline_configuration.output_elements_per_threadgroup();+ let output_elements_per_threadgroup = pipeline_configuration.output_elements_per_threadgroup();let threadgroup_count_x =- ((output_dimension as u32 + output_elements_per_threadgroup - 1)- / output_elements_per_threadgroup) as u64;+ ((output_dimension as u32 + output_elements_per_threadgroup - 1) / output_elements_per_threadgroup) as u64;let threadgroup_count_z = batch_groups.max(1) as u64;let batch_rows = arguments.batch;let threadgroup_count_y = (batch_rows / batch_pack).max(1) as u64;- let threadgroups = MTLSize::new(- threadgroup_count_x as usize,- threadgroup_count_y as usize,- threadgroup_count_z as usize,- );- let threads_per_threadgroup =- pipeline_configuration.threads_per_threadgroup();+ let threadgroups =+ MTLSize::new(threadgroup_count_x as usize, threadgroup_count_y as usize, threadgroup_count_z as usize);+ let threads_per_threadgroup = pipeline_configuration.threads_per_threadgroup();let output_ld = arguments.ldd;let vector_ld = arguments.lda;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex ef92b6b37..ed55e89c1 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -7,9 +7,8 @@ use super::{use crate::{DataType,backends::metal::{- ComputeEncoderSetValue, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError, ProtocolObject,- Retained, kernel::matmul::common::MatmulArguments,+ ComputeEncoderSetValue, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,+ ProtocolObject, Retained, kernel::matmul::common::MatmulArguments,},};@@ -22,10 +21,7 @@ fn gemv_kernel_name(DataType::BF16 => "bfloat16",DataType::F32 => "float32",_ => {- return Err(MTLError::Generic(format!(- "Unsupported data type for GEMV: {:?}",- data_type- )));+ return Err(MTLError::Generic(format!("Unsupported data type for GEMV: {:?}", data_type)));},};@@ -58,20 +54,13 @@ fn gemv_kernel_name(pub struct Kernel {data_type: DataType,- pipelines: HashMap<- PipelineConfiguration,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,+ pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,}impl Kernel {pub fn new(data_type: DataType) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported data type for GEMV: {:?}",- data_type- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported data type for GEMV: {:?}", data_type)));}Ok(Self {data_type,@@ -115,12 +104,10 @@ impl Kernel {&mut self,context: &MTLContext,config: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if !self.pipelines.contains_key(config) {let kernel_name = gemv_kernel_name(self.data_type, config)?;- let pipeline =- context.compute_pipeline_state(&kernel_name, None)?;+ let pipeline = context.compute_pipeline_state(&kernel_name, None)?;self.pipelines.insert(*config, pipeline);}Ok(self.pipelines.get(config).unwrap())@@ -133,8 +120,7 @@ impl Kernel {arguments: &MatmulArguments,descriptor: &DispatchDescriptor,) -> Result<bool, MTLError> {- let pipeline =- self.get_pipeline(context, &descriptor.pipeline_configuration)?;+ let pipeline = self.get_pipeline(context, &descriptor.pipeline_configuration)?;encoder.set_compute_pipeline_state(pipeline);let (buf0, off0) = if descriptor.matrix_is_rhs {@@ -155,24 +141,18 @@ impl Kernel {match descriptor.axpby_source {AxpbySource::None => {return Err(MTLError::Generic(- "GEMV descriptor mismatch: do_axpby=true but axpby_source=None"- .to_owned(),+ "GEMV descriptor mismatch: do_axpby=true but axpby_source=None".to_owned(),));},AxpbySource::Bias => {- let bias = arguments.bias.ok_or_else(|| {- MTLError::Generic(- "GEMV descriptor requires bias buffer".to_owned(),- )- })?;+ let bias = arguments+ .bias+ .ok_or_else(|| MTLError::Generic("GEMV descriptor requires bias buffer".to_owned()))?;encoder.set_buffer(Some(bias), 0, 2);},AxpbySource::C => {- let c_buffer = arguments.c.ok_or_else(|| {- MTLError::Generic(- "GEMV descriptor requires C buffer".to_owned(),- )- })?;+ let c_buffer =+ arguments.c.ok_or_else(|| MTLError::Generic("GEMV descriptor requires C buffer".to_owned()))?;encoder.set_buffer(Some(c_buffer), 0, 2);},}@@ -195,10 +175,7 @@ impl Kernel {encoder.set_value(&descriptor.output_ld, 16);encoder.set_value(&descriptor.vector_ld, 17);- encoder.dispatch_threadgroups(- descriptor.threadgroups,- descriptor.threads_per_threadgroup,- );+ encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);Ok(descriptor.bias_is_fused())}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rsindex e21e979ec..96339e3dd 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs@@ -19,22 +19,14 @@ pub struct PipelineConfiguration {impl PipelineConfiguration {pub fn output_elements_per_threadgroup(&self) -> u32 {if self.transpose_matrix {- self.threadgroup_cols- * self.threads_per_simdgroup_col- * self.elements_per_thread_col+ self.threadgroup_cols * self.threads_per_simdgroup_col * self.elements_per_thread_col} else {- self.threadgroup_rows- * self.threads_per_simdgroup_row- * self.elements_per_thread_row+ self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row}}pub fn threads_per_threadgroup(&self) -> MTLSize {- MTLSize::new(- 32,- self.threadgroup_cols as usize,- self.threadgroup_rows as usize,- )+ MTLSize::new(32, self.threadgroup_cols as usize, self.threadgroup_rows as usize)}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex 4b6436e15..fccee162f 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -1,8 +1,6 @@use super::{common::MatmulArguments,- dispatch_descriptor::{- MatmulDispatchDescriptor, choose_dispatch_descriptor,- },+ dispatch_descriptor::{MatmulDispatchDescriptor, choose_dispatch_descriptor},gemm,gemv::GemvKernel,split_k::SplitKGemm,@@ -12,8 +10,8 @@ use crate::{backends::{common::kernel::TensorAddBiasKernel,metal::{- MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLError,- ProtocolObject, Retained, kernel::dsl::TensorAddBiasMetalKernel,+ MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject, Retained,+ kernel::dsl::TensorAddBiasMetalKernel,},},};@@ -28,11 +26,8 @@ pub struct MatmulKernel {impl MatmulKernel {pub fn new(data_type: DataType) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MatmulKernel: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for MatmulKernel: {data_type:?}")));}Ok(Self {@@ -60,9 +55,7 @@ impl MatmulKernel {Ok(())}- fn get_or_create_gemm(- &mut self- ) -> Result<&mut gemm::GemmKernel, MTLError> {+ fn get_or_create_gemm(&mut self) -> Result<&mut gemm::GemmKernel, MTLError> {if self.gemm.is_none() {self.gemm = Some(gemm::GemmKernel::new(self.data_type)?);}@@ -97,8 +90,7 @@ impl MatmulKernel {},MatmulDispatchDescriptor::SplitK(descriptor) => {let splitk = self.get_or_create_splitk()?;- splitk- .encode_descriptor(context, encoder, arguments, descriptor)+ splitk.encode_descriptor(context, encoder, arguments, descriptor)},MatmulDispatchDescriptor::Gemm(descriptor) => {let gemm = self.get_or_create_gemm()?;@@ -115,15 +107,9 @@ impl MatmulKernel {) -> Result<(), MTLError> {Self::apply_batch_collapse(&mut arguments);- let descriptor =- choose_dispatch_descriptor(context, self.data_type, &arguments)?;+ let descriptor = choose_dispatch_descriptor(context, self.data_type, &arguments)?;- let bias_fused = self.encode_dispatch_descriptor(- context,- encoder,- &arguments,- &descriptor,- )?;+ let bias_fused = self.encode_dispatch_descriptor(context, encoder, &arguments, &descriptor)?;if let Some(bias) = arguments.bias {if !bias_fused {@@ -150,18 +136,10 @@ impl MatmulKernel {}if self.bias_add.is_none() {- self.bias_add =- Some(TensorAddBiasMetalKernel::new(context, self.data_type)?);+ self.bias_add = Some(TensorAddBiasMetalKernel::new(context, self.data_type)?);}let bias_add = self.bias_add.as_ref().unwrap();- bias_add.encode(- arguments.d,- bias,- arguments.d,- n as u32,- total_len as u32,- encoder,- );+ bias_add.encode(arguments.d, bias, arguments.d, n as u32, total_len as u32, encoder);Ok(())}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rsindex bed064a57..cb32b7800 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs@@ -1,14 +1,9 @@-use super::{- pipeline_configuration::PipelineConfiguration,- tile_configuration::select_tile_configuration,-};+use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};use crate::{DataType,backends::metal::{MTLContext, MTLError, MTLSize,- kernel::matmul::common::{- GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments,- },+ kernel::matmul::common::{GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments},},};@@ -31,12 +26,8 @@ impl DispatchDescriptor {data_type: DataType,arguments: &MatmulArguments,) -> Result<Option<Self>, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for Split-K: {:?}",- data_type- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for Split-K: {:?}", data_type)));}if arguments.c.is_some() {@@ -72,11 +63,8 @@ impl DispatchDescriptor {let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;let output_elements_per_partition = m * n;- let accumulator_element_count = partition_count- * output_elements_per_partition- * arguments.batch_count;- let accumulator_bytes =- (accumulator_element_count as usize) * std::mem::size_of::<f32>();+ let accumulator_element_count = partition_count * output_elements_per_partition * arguments.batch_count;+ let accumulator_bytes = (accumulator_element_count as usize) * std::mem::size_of::<f32>();let params = SplitKGEMMParams {M: m,@@ -93,20 +81,12 @@ impl DispatchDescriptor {gemm_k_iterations_aligned: gemm_k_iterations,};- let partial_threads_per_threadgroup = MTLSize::new(- 32,- tile.warps_per_col as usize,- tile.warps_per_row as usize,- );- let partial_threadgroups = MTLSize::new(- tile_count_n as usize,- tile_count_m as usize,- partition_count as usize,- );+ let partial_threads_per_threadgroup =+ MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);+ let partial_threadgroups = MTLSize::new(tile_count_n as usize, tile_count_m as usize, partition_count as usize);let accum_total_threads = MTLSize::new(n as usize, m as usize, 1);- let accum_threads_per_threadgroup =- MTLSize::new(16.min(n as usize), 16.min(m as usize), 1);+ let accum_threads_per_threadgroup = MTLSize::new(16.min(n as usize), 16.min(m as usize), 1);Ok(Some(Self {pipeline_configuration,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rsindex e08b03516..ec5bed92e 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs@@ -1,43 +1,30 @@use std::collections::HashMap;-use super::{- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,-};+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};use crate::{DataType,backends::{common::Context,metal::{- ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError, ProtocolObject,- Retained,- kernel::matmul::common::{- MatmulArguments, transpose_configuration,- },+ ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,+ ProtocolObject, Retained,+ kernel::matmul::common::{MatmulArguments, transpose_configuration},},},};pub struct Kernel {data_type: DataType,- partial_pipelines: HashMap<- PipelineConfiguration,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,- accum_pipeline:- Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,+ partial_pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,+ accum_pipeline: Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,accumulator_buffer_bytes: usize,}impl Kernel {pub fn new(data_type: DataType) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for Split-K: {:?}",- data_type- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for Split-K: {:?}", data_type)));}Ok(Self {data_type,@@ -112,10 +99,7 @@ impl Kernel {DataType::F16 => Ok("float16"),DataType::BF16 => Ok("bfloat16"),DataType::F32 => Ok("float32"),- _ => Err(MTLError::Generic(format!(- "Unsupported dtype for Split-K: {:?}",- self.data_type- ))),+ _ => Err(MTLError::Generic(format!("Unsupported dtype for Split-K: {:?}", self.data_type))),}}@@ -129,9 +113,7 @@ impl Kernel {) -> Result<String, MTLError> {let in_name = self.steel_type_name()?;let out_name = self.splitk_partial_out_name();- let transpose_suffix =- transpose_configuration(config.transpose_a, config.transpose_b)- .as_str();+ let transpose_suffix = transpose_configuration(config.transpose_a, config.transpose_b).as_str();let mn_tag = if config.mn_aligned {"taligned"} else {@@ -159,19 +141,14 @@ impl Kernel {fn accum_kernel_name(&self) -> Result<String, MTLError> {let out_name = self.steel_type_name()?;- Ok(format!(- "steel_gemm_splitk_accum_{}_{}",- out_name,- self.splitk_partial_out_name()- ))+ Ok(format!("steel_gemm_splitk_accum_{}_{}", out_name, self.splitk_partial_out_name()))}fn get_partial_pipeline(&mut self,mtl: &MTLContext,config: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if !self.partial_pipelines.contains_key(config) {let name = self.partial_kernel_name(config)?;let ps = mtl.compute_pipeline_state(&name, None)?;@@ -183,8 +160,7 @@ impl Kernel {fn get_accum_pipeline(&mut self,mtl: &MTLContext,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if self.accum_pipeline.is_none() {let name = self.accum_kernel_name()?;let ps = mtl.compute_pipeline_state(&name, None)?;@@ -219,26 +195,17 @@ impl Kernel {descriptor: &DispatchDescriptor,) -> Result<bool, MTLError> {self.ensure_accumulator_buffer(context, descriptor.accumulator_bytes);- let accumulator_buffer = self- .accumulator_buffer- .as_ref()- .cloned()- .expect("Accumulator buffer must be initialized");+ let accumulator_buffer =+ self.accumulator_buffer.as_ref().cloned().expect("Accumulator buffer must be initialized");- let partial_pipeline_state = self.get_partial_pipeline(- context,- &descriptor.pipeline_configuration,- )?;+ let partial_pipeline_state = self.get_partial_pipeline(context, &descriptor.pipeline_configuration)?;encoder.set_compute_pipeline_state(partial_pipeline_state);encoder.set_buffer(Some(arguments.a), arguments.a_offset as usize, 0);encoder.set_buffer(Some(arguments.b), 0, 1);encoder.set_buffer(Some(&accumulator_buffer), 0, 2);encoder.set_value(&descriptor.params, 3);- encoder.dispatch_threadgroups(- descriptor.partial_threadgroups,- descriptor.partial_threads_per_threadgroup,- );+ encoder.dispatch_threadgroups(descriptor.partial_threadgroups, descriptor.partial_threads_per_threadgroup);let accum_pipeline_state = self.get_accum_pipeline(context)?;encoder.set_compute_pipeline_state(accum_pipeline_state);@@ -246,16 +213,12 @@ impl Kernel {encoder.set_buffer(Some(arguments.d), 0, 1);let partition_count = descriptor.partition_count;- let output_elements_per_partition =- descriptor.output_elements_per_partition;+ let output_elements_per_partition = descriptor.output_elements_per_partition;encoder.set_value(&partition_count, 2);encoder.set_value(&output_elements_per_partition, 3);encoder.set_value(&arguments.ldd, 4);- encoder.dispatch_threads(- descriptor.accum_total_threads,- descriptor.accum_threads_per_threadgroup,- );+ encoder.dispatch_threads(descriptor.accum_total_threads, descriptor.accum_threads_per_threadgroup);Ok(false)}@@ -265,15 +228,10 @@ impl Kernel {mtl: &MTLContext,required_bytes: usize,) {- if required_bytes <= self.accumulator_buffer_bytes- && self.accumulator_buffer.is_some()- {+ if required_bytes <= self.accumulator_buffer_bytes && self.accumulator_buffer.is_some() {return;}- self.accumulator_buffer = Some(- mtl.create_buffer(required_bytes)- .expect("Failed to create accumulator buffer"),- );+ self.accumulator_buffer = Some(mtl.create_buffer(required_bytes).expect("Failed to create accumulator buffer"));self.accumulator_buffer_bytes = required_bytes;}}diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rsindex ee493532d..bfd29a572 100644--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs@@ -8,9 +8,8 @@ use crate::{use objc2::rc::Retained;use crate::backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandEncoderExt,- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLSize,- ProtocolObject, error::MTLError, image::Image,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandEncoderExt, MTLComputeCommandEncoder,+ MTLComputePipelineState, MTLContext, MTLSize, ProtocolObject, error::MTLError, image::Image,metal_extensions::ComputeEncoderDispatch,};@@ -40,8 +39,7 @@ pub struct ScalePadNormalizeImage {impl ScalePadNormalizeImage {pub fn new(context: &MTLContext) -> Result<Self, MTLError> {let function_name = "scalePadNormalizeImage";- let pipeline_state =- context.compute_pipeline_state(&function_name, None)?;+ let pipeline_state = context.compute_pipeline_state(&function_name, None)?;Ok(Self {pipeline_state,})@@ -54,9 +52,8 @@ impl ScalePadNormalizeImage {image_params_ptr: *const std::ffi::c_void,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,) {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");compute_encoder.set_label(Some("ScalePadNormalizeImageEncoder"));compute_encoder.set_compute_pipeline_state(&self.pipeline_state);@@ -101,9 +98,7 @@ impl EncodableBlock<Metal> for ScalePadNormalizeImage {_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,_parameters: &EncodingParameters<Metal>,) {- unreachable!(- "ScalePadNormalizeImage does not support shared compute encoder"- );+ unreachable!("ScalePadNormalizeImage does not support shared compute encoder");}}@@ -115,8 +110,7 @@ pub struct ExtractImagePatches {impl ExtractImagePatches {pub fn new(context: &MTLContext) -> Result<Self, MTLError> {let function_name = "extractImagePatches";- let pipeline_state =- context.compute_pipeline_state(&function_name, None)?;+ let pipeline_state = context.compute_pipeline_state(&function_name, None)?;Ok(Self {pipeline_state,})@@ -129,14 +123,12 @@ impl ExtractImagePatches {patch_params_ptr: *const std::ffi::c_void,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,) {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");compute_encoder.set_label(Some("ExtractImagePatchesEncoder"));compute_encoder.set_compute_pipeline_state(&self.pipeline_state);- compute_encoder- .set_texture(Some(padded_normalized_image.texture_ref()), 0);+ compute_encoder.set_texture(Some(padded_normalized_image.texture_ref()), 0);compute_encoder.set_buffer(Some(output_buffer_mtl), 0, 0);unsafe {compute_encoder.set_bytes(diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rsindex a6eb8f90e..e0d2d9146 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs@@ -5,9 +5,8 @@ use crate::{backends::{common::kernel::MlpGateActMulKernel,metal::{- MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLDataType,- MTLError, MTLFunctionConstantValues, ProtocolObject, Retained,- kernel::dsl::MlpGateActMulMetalKernel,+ MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLDataType, MTLError, MTLFunctionConstantValues,+ ProtocolObject, Retained, kernel::dsl::MlpGateActMulMetalKernel,},},config::Activation,@@ -60,9 +59,7 @@ impl MlpFusedConfig {}/// Create function constants for MLP fused matmul- pub fn make_function_constants(- &self- ) -> Retained<MTLFunctionConstantValues> {+ pub fn make_function_constants(&self) -> Retained<MTLFunctionConstantValues> {let fcv = MTLFunctionConstantValues::new();let fused = true;fcv.set_constant_value_type_at_index(@@ -86,15 +83,10 @@ impl MlpFusedConfig {}/// Create function constants for non-fused (standard) matmul-pub fn make_non_fused_function_constants() -> Retained<MTLFunctionConstantValues>-{+pub fn make_non_fused_function_constants() -> Retained<MTLFunctionConstantValues> {let fcv = MTLFunctionConstantValues::new();let fused = false;- fcv.set_constant_value_type_at_index(- NonNull::from(&fused).cast(),- MTLDataType::Bool,- MLP_FUSED_FC_INDEX as usize,- );+ fcv.set_constant_value_type_at_index(NonNull::from(&fused).cast(), MTLDataType::Bool, MLP_FUSED_FC_INDEX as usize);fcv}@@ -135,14 +127,7 @@ impl MlpGateActMulEncodable {panic!("Identity activation is not supported for kernel")},};- self.kernel.encode(- fused_up,- hidden,- self.hidden_dim as i32,- m,- act_type,- encoder,- );+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, encoder);Ok(())}}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rsindex 3cd13a91d..da793f6e6 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs@@ -1,6 +1,4 @@-use crate::backends::metal::{- MTLBuffer, ProtocolObject, kernel::mlp::MlpActivationType,-};+use crate::backends::metal::{MTLBuffer, ProtocolObject, kernel::mlp::MlpActivationType};#[derive(Debug)]pub struct MlpFusedArguments<'a> {diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rsindex fca33bf46..9cbedf896 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs@@ -32,9 +32,7 @@ impl MlpFusedDispatchDescriptor {pub fn variant(&self) -> MlpFusedKernelVariant {match self {MlpFusedDispatchDescriptor::Gemv(_) => MlpFusedKernelVariant::Gemv,- MlpFusedDispatchDescriptor::SplitK(_) => {- MlpFusedKernelVariant::SplitK- },+ MlpFusedDispatchDescriptor::SplitK(_) => MlpFusedKernelVariant::SplitK,MlpFusedDispatchDescriptor::Gemm(_) => MlpFusedKernelVariant::Gemm,}}@@ -47,15 +45,11 @@ pub(crate) fn choose_dispatch_descriptor(arguments: &MlpFusedArguments,) -> Result<MlpFusedDispatchDescriptor, MTLError> {if weights_transposed {- if let Some(descriptor) =- gemv::DispatchDescriptor::try_new(context, data_type, arguments)?- {+ if let Some(descriptor) = gemv::DispatchDescriptor::try_new(context, data_type, arguments)? {return Ok(MlpFusedDispatchDescriptor::Gemv(descriptor));}- if let Some(descriptor) =- split_k::DispatchDescriptor::try_new(context, data_type, arguments)?- {+ if let Some(descriptor) = split_k::DispatchDescriptor::try_new(context, data_type, arguments)? {return Ok(MlpFusedDispatchDescriptor::SplitK(descriptor));}}@@ -74,11 +68,5 @@ pub fn determine_kernel_variant(weights_transposed: bool,arguments: &MlpFusedArguments,) -> Result<MlpFusedKernelVariant, MTLError> {- choose_dispatch_descriptor(- context,- data_type,- weights_transposed,- arguments,- )- .map(|d| d.variant())+ choose_dispatch_descriptor(context, data_type, weights_transposed, arguments).map(|d| d.variant())}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rsindex 139d8d89a..b8cc84c3e 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs@@ -1,14 +1,9 @@-use super::{- pipeline_configuration::PipelineConfiguration,- tile_configuration::select_tile_configuration,-};+use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};use crate::{DataType,backends::metal::{MTLContext, MTLError, MTLSize,- kernel::{- matmul::common::GEMMParams, mlp_fused::common::MlpFusedArguments,- },+ kernel::{matmul::common::GEMMParams, mlp_fused::common::MlpFusedArguments},},};@@ -28,18 +23,13 @@ impl DispatchDescriptor {weights_transposed: bool,arguments: &MlpFusedArguments,) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MLP fused GEMM: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMM: {data_type:?}")));}- let tile =- select_tile_configuration(arguments.batch, arguments.hidden_dim);+ let tile = select_tile_configuration(arguments.batch, arguments.hidden_dim);- let mn_aligned = arguments.batch % tile.block_rows == 0- && arguments.hidden_dim % tile.block_cols == 0;+ let mn_aligned = arguments.batch % tile.block_rows == 0 && arguments.hidden_dim % tile.block_cols == 0;let k_aligned = arguments.input_dim % tile.block_depth == 0;let pipeline_configuration = PipelineConfiguration {@@ -51,10 +41,8 @@ impl DispatchDescriptor {};let tiles_m = (arguments.batch + tile.block_rows - 1) / tile.block_rows;- let tiles_n =- (arguments.hidden_dim + tile.block_cols - 1) / tile.block_cols;- let gemm_k_iterations =- (arguments.input_dim + tile.block_depth - 1) / tile.block_depth;+ let tiles_n = (arguments.hidden_dim + tile.block_cols - 1) / tile.block_cols;+ let gemm_k_iterations = (arguments.input_dim + tile.block_depth - 1) / tile.block_depth;let params = GEMMParams {M: arguments.batch,diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rsindex 002bc9a42..2864e4073 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs@@ -1,25 +1,18 @@use std::{collections::HashMap, ptr::NonNull};-use super::{- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,-};+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};use crate::{DataType,backends::metal::{- ComputeEncoderSetValue, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError,- MTLFunctionConstantValues, ProtocolObject, Retained,- kernel::mlp_fused::common::MlpFusedArguments,+ ComputeEncoderSetValue, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,+ MTLFunctionConstantValues, ProtocolObject, Retained, kernel::mlp_fused::common::MlpFusedArguments,},};pub struct Kernel {data_type: DataType,weights_transposed: bool,- pipelines: HashMap<- PipelineConfiguration,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,+ pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,}impl Kernel {@@ -27,11 +20,8 @@ impl Kernel {data_type: DataType,weights_transposed: bool,) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MLP fused GEMM: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMM: {data_type:?}")));}Ok(Self {data_type,@@ -86,8 +76,7 @@ impl Kernel {&mut self,context: &MTLContext,configuration: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if !self.pipelines.contains_key(configuration) {let kernel_name = self.kernel_name(configuration);@@ -99,10 +88,7 @@ impl Kernel {52,);- let pipeline_state = context.compute_pipeline_state(- &kernel_name,- Some(&function_constants),- )?;+ let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;self.pipelines.insert(configuration.clone(), pipeline_state);}Ok(self.pipelines.get(configuration).unwrap())@@ -115,27 +101,17 @@ impl Kernel {arguments: &MlpFusedArguments,descriptor: &DispatchDescriptor,) -> Result<(), MTLError> {- let pipeline_state = self.get_or_compile_pipeline(- context,- &descriptor.pipeline_configuration,- )?;+ let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;encoder.set_compute_pipeline_state(pipeline_state);- encoder.set_buffer(- Some(arguments.input),- arguments.input_offset as usize,- 0,- );+ encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 0);encoder.set_buffer(Some(arguments.weights), 0, 1);encoder.set_buffer(Some(arguments.output), 0, 2);encoder.set_value(&descriptor.params, 3);encoder.set_value(&descriptor.hidden_dim, 10);- encoder.dispatch_threadgroups(- descriptor.threadgroups,- descriptor.threads_per_threadgroup,- );+ encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);Ok(())}}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rsindex f88969d08..b560342ce 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs@@ -1,12 +1,7 @@-use super::pipeline_configuration::{- PipelineConfiguration, select_configuration,-};+use super::pipeline_configuration::{PipelineConfiguration, select_configuration};use crate::{DataType,- backends::metal::{- MTLContext, MTLError, MTLSize,- kernel::mlp_fused::common::MlpFusedArguments,- },+ backends::metal::{MTLContext, MTLError, MTLSize, kernel::mlp_fused::common::MlpFusedArguments},};#[derive(Debug, Clone)]@@ -27,35 +22,23 @@ impl DispatchDescriptor {data_type: DataType,arguments: &MlpFusedArguments,) -> Result<Option<Self>, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MLP fused GEMV: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMV: {data_type:?}")));}if arguments.batch != 1 {return Ok(None);}- let pipeline_configuration =- select_configuration(arguments.hidden_dim, arguments.activation);+ let pipeline_configuration = select_configuration(arguments.hidden_dim, arguments.activation);- let output_elements_per_threadgroup =- pipeline_configuration.output_elements_per_threadgroup();- let threadgroup_count_x =- ((arguments.hidden_dim as u32 + output_elements_per_threadgroup- - 1)- / output_elements_per_threadgroup) as u64;+ let output_elements_per_threadgroup = pipeline_configuration.output_elements_per_threadgroup();+ let threadgroup_count_x = ((arguments.hidden_dim as u32 + output_elements_per_threadgroup - 1)+ / output_elements_per_threadgroup) as u64;let threadgroup_count_z = arguments.batch_count.max(1) as u64;- let threadgroups = MTLSize::new(- threadgroup_count_x as usize,- 1,- threadgroup_count_z as usize,- );- let threads_per_threadgroup =- pipeline_configuration.threads_per_threadgroup();+ let threadgroups = MTLSize::new(threadgroup_count_x as usize, 1, threadgroup_count_z as usize);+ let threads_per_threadgroup = pipeline_configuration.threads_per_threadgroup();Ok(Some(Self {pipeline_configuration,diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rsindex e113279eb..e065bd727 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs@@ -1,15 +1,11 @@use std::{collections::HashMap, ptr::NonNull};-use super::{- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,-};+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};use crate::{DataType,backends::metal::{- ComputeEncoderSetValue, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError,- MTLFunctionConstantValues, ProtocolObject, Retained,- kernel::mlp_fused::common::MlpFusedArguments,+ ComputeEncoderSetValue, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,+ MTLFunctionConstantValues, ProtocolObject, Retained, kernel::mlp_fused::common::MlpFusedArguments,},};@@ -22,9 +18,7 @@ fn kernel_name(DataType::BF16 => "bfloat16",DataType::F32 => "float32",_ => {- return Err(MTLError::Generic(format!(- "Unsupported data type for MLP fused GEMV: {data_type:?}"- )));+ return Err(MTLError::Generic(format!("Unsupported data type for MLP fused GEMV: {data_type:?}")));},};@@ -42,19 +36,13 @@ fn kernel_name(pub struct Kernel {data_type: DataType,- pipelines: HashMap<- PipelineConfiguration,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,+ pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,}impl Kernel {pub fn new(data_type: DataType) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MLP fused GEMV: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMV: {data_type:?}")));}Ok(Self {data_type,@@ -66,8 +54,7 @@ impl Kernel {&mut self,context: &MTLContext,configuration: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if !self.pipelines.contains_key(configuration) {let kernel_name = kernel_name(self.data_type, configuration)?;@@ -79,10 +66,7 @@ impl Kernel {52,);- let pipeline_state = context.compute_pipeline_state(- &kernel_name,- Some(&function_constants),- )?;+ let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;self.pipelines.insert(*configuration, pipeline_state);}Ok(self.pipelines.get(configuration).unwrap())@@ -95,18 +79,11 @@ impl Kernel {arguments: &MlpFusedArguments,descriptor: &DispatchDescriptor,) -> Result<(), MTLError> {- let pipeline_state = self.get_or_compile_pipeline(- context,- &descriptor.pipeline_configuration,- )?;+ let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;encoder.set_compute_pipeline_state(pipeline_state);encoder.set_buffer(Some(arguments.weights), 0, 0);- encoder.set_buffer(- Some(arguments.input),- arguments.input_offset as usize,- 1,- );+ encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 1);encoder.set_buffer(Some(arguments.output), 0, 3);encoder.set_value(&descriptor.input_dim, 4);@@ -115,10 +92,7 @@ impl Kernel {encoder.set_value(&descriptor.vector_batch_stride, 11);encoder.set_value(&descriptor.matrix_batch_stride, 12);- encoder.dispatch_threadgroups(- descriptor.threadgroups,- descriptor.threads_per_threadgroup,- );+ encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);Ok(())}}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rsindex 280e1cbf3..91130ed54 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs@@ -13,17 +13,11 @@ pub struct PipelineConfiguration {impl PipelineConfiguration {pub fn output_elements_per_threadgroup(&self) -> u32 {- self.threadgroup_rows- * self.threads_per_simdgroup_row- * self.elements_per_thread_row+ self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row}pub fn threads_per_threadgroup(&self) -> MTLSize {- MTLSize::new(- 32,- self.threadgroup_cols as usize,- self.threadgroup_rows as usize,- )+ MTLSize::new(32, self.threadgroup_cols as usize, self.threadgroup_rows as usize)}}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rsindex 0a951b099..013e3f5fd 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs@@ -2,9 +2,7 @@ use crate::backends::metal::{MTLComputeCommandEncoder, ProtocolObject};use super::{common::MlpFusedArguments,- dispatch_descriptor::{- MlpFusedDispatchDescriptor, choose_dispatch_descriptor,- },+ dispatch_descriptor::{MlpFusedDispatchDescriptor, choose_dispatch_descriptor},gemm::GemmKernel,gemv::GemvKernel,split_k::SplitKKernel,@@ -27,11 +25,8 @@ impl MlpFusedKernel {data_type: DataType,weights_transposed: bool,) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MlpFusedKernel: {data_type:?}"- )));+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!("Unsupported dtype for MlpFusedKernel: {data_type:?}")));}Ok(Self {@@ -45,8 +40,7 @@ impl MlpFusedKernel {fn get_or_create_gemm(&mut self) -> Result<&mut GemmKernel, MTLError> {if self.gemm.is_none() {- self.gemm =- Some(GemmKernel::new(self.data_type, self.weights_transposed)?);+ self.gemm = Some(GemmKernel::new(self.data_type, self.weights_transposed)?);}Ok(self.gemm.as_mut().unwrap())}@@ -79,8 +73,7 @@ impl MlpFusedKernel {},MlpFusedDispatchDescriptor::SplitK(descriptor) => {let splitk = self.get_or_create_splitk()?;- splitk- .encode_descriptor(context, encoder, arguments, descriptor)+ splitk.encode_descriptor(context, encoder, arguments, descriptor)},MlpFusedDispatchDescriptor::Gemm(descriptor) => {let gemm = self.get_or_create_gemm()?;@@ -95,18 +88,8 @@ impl MlpFusedKernel {encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,arguments: &MlpFusedArguments,) -> Result<(), MTLError> {- let descriptor = choose_dispatch_descriptor(- context,- self.data_type,- self.weights_transposed,- arguments,- )?;+ let descriptor = choose_dispatch_descriptor(context, self.data_type, self.weights_transposed, arguments)?;- self.encode_dispatch_descriptor(- context,- encoder,- arguments,- &descriptor,- )+ self.encode_dispatch_descriptor(context, encoder, arguments, &descriptor)}}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rsindex e668ca697..f503655f3 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs@@ -6,9 +6,7 @@ mod kernel;mod split_k;pub use common::MlpFusedArguments;-pub use dispatch_descriptor::{- MlpFusedKernelVariant, determine_kernel_variant,-};+pub use dispatch_descriptor::{MlpFusedKernelVariant, determine_kernel_variant};pub use gemm::GemmKernel;pub use gemv::GemvKernel;pub use kernel::MlpFusedKernel;diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rsindex a98eac2ea..574efb5b9 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs@@ -1,15 +1,9 @@-use super::{- pipeline_configuration::PipelineConfiguration,- tile_configuration::select_tile_configuration,-};+use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};use crate::{DataType,backends::metal::{MTLContext, MTLError, MTLSize,- kernel::{- matmul::common::GEMMSpiltKMlpFusedParams,- mlp_fused::common::MlpFusedArguments,- },+ kernel::{matmul::common::GEMMSpiltKMlpFusedParams, mlp_fused::common::MlpFusedArguments},},};@@ -37,11 +31,7 @@ impl DispatchDescriptor {return Ok(None);}- if !should_use_splitk(- arguments.batch,- arguments.hidden_dim,- arguments.input_dim,- ) {+ if !should_use_splitk(arguments.batch, arguments.hidden_dim, arguments.input_dim) {return Ok(None);}@@ -51,8 +41,7 @@ impl DispatchDescriptor {let tile = select_tile_configuration(batch, hidden_dim);let partition_count = compute_partition_count(input_dim);- let mn_aligned =- (batch % tile.tile_rows) == 0 && (hidden_dim % tile.tile_cols) == 0;+ let mn_aligned = (batch % tile.tile_rows) == 0 && (hidden_dim % tile.tile_cols) == 0;let k_aligned = (input_dim % tile.tile_depth) == 0;let pipeline_configuration = PipelineConfiguration {@@ -69,10 +58,8 @@ impl DispatchDescriptor {let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;let output_elements_per_partition = batch * hidden_dim;- let accumulator_element_count =- partition_count * output_elements_per_partition;- let accumulator_bytes =- (accumulator_element_count as usize) * std::mem::size_of::<f32>();+ let accumulator_element_count = partition_count * output_elements_per_partition;+ let accumulator_bytes = (accumulator_element_count as usize) * std::mem::size_of::<f32>();let params = GEMMSpiltKMlpFusedParams {M: batch,@@ -90,24 +77,12 @@ impl DispatchDescriptor {hidden_dim,};- let partial_threads_per_threadgroup = MTLSize::new(- 32,- tile.warps_per_col as usize,- tile.warps_per_row as usize,- );- let partial_threadgroups = MTLSize::new(- tile_count_n as usize,- tile_count_m as usize,- partition_count as usize,- );+ let partial_threads_per_threadgroup =+ MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);+ let partial_threadgroups = MTLSize::new(tile_count_n as usize, tile_count_m as usize, partition_count as usize);- let accum_total_threads =- MTLSize::new(hidden_dim as usize, batch as usize, 1);- let accum_threads_per_threadgroup = MTLSize::new(- 16.min(hidden_dim as usize),- 16.min(batch as usize),- 1,- );+ let accum_total_threads = MTLSize::new(hidden_dim as usize, batch as usize, 1);+ let accum_threads_per_threadgroup = MTLSize::new(16.min(hidden_dim as usize), 16.min(batch as usize), 1);Ok(Some(Self {pipeline_configuration,diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rsindex dced3cfb7..326a3c4da 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs@@ -1,33 +1,22 @@use std::{collections::HashMap, ptr::NonNull};-use super::{- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,-};+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};use crate::{DataType,backends::{common::Context,metal::{- ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError,+ ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,MTLFunctionConstantValues, ProtocolObject, Retained,- kernel::{- mlp::MlpActivationType, mlp_fused::common::MlpFusedArguments,- },+ kernel::{mlp::MlpActivationType, mlp_fused::common::MlpFusedArguments},},},};pub struct Kernel {data_type: DataType,- partial_pipelines: HashMap<- PipelineConfiguration,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,- accum_pipelines: HashMap<- MlpActivationType,- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- >,+ partial_pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,+ accum_pipelines: HashMap<MlpActivationType, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,up_accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,gate_accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,accumulator_buffer_bytes: usize,@@ -36,9 +25,7 @@ pub struct Kernel {impl Kernel {pub fn new(data_type: DataType) -> Result<Self, MTLError> {if !matches!(data_type, DataType::F16 | DataType::BF16) {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MLP fused Split-K: {data_type:?}"- )));+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused Split-K: {data_type:?}")));}Ok(Self {data_type,@@ -54,10 +41,7 @@ impl Kernel {match self.data_type {DataType::F16 => Ok("float16"),DataType::BF16 => Ok("bfloat16"),- _ => Err(MTLError::Generic(format!(- "Unsupported dtype for MLP Fused Split-K: {:?}",- self.data_type- ))),+ _ => Err(MTLError::Generic(format!("Unsupported dtype for MLP Fused Split-K: {:?}", self.data_type))),}}@@ -98,14 +82,11 @@ impl Kernel {&mut self,context: &MTLContext,configuration: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if !self.partial_pipelines.contains_key(configuration) {let kernel_name = self.partial_kernel_name(configuration)?;- let pipeline_state =- context.compute_pipeline_state(&kernel_name, None)?;- self.partial_pipelines- .insert(configuration.clone(), pipeline_state);+ let pipeline_state = context.compute_pipeline_state(&kernel_name, None)?;+ self.partial_pipelines.insert(configuration.clone(), pipeline_state);}Ok(self.partial_pipelines.get(configuration).unwrap())}@@ -114,8 +95,7 @@ impl Kernel {&mut self,context: &MTLContext,activation: MlpActivationType,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {if !self.accum_pipelines.contains_key(&activation) {let kernel_name = self.accum_kernel_name()?;let function_constants = MTLFunctionConstantValues::new();@@ -125,10 +105,7 @@ impl Kernel {metal::MTLDataType::UInt,52,);- let pipeline_state = context.compute_pipeline_state(- &kernel_name,- Some(&function_constants),- )?;+ let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;self.accum_pipelines.insert(activation, pipeline_state);}Ok(self.accum_pipelines.get(&activation).unwrap())@@ -146,8 +123,7 @@ impl Kernel {return;}self.up_accumulator_buffer = context.create_buffer(required_bytes).ok();- self.gate_accumulator_buffer =- context.create_buffer(required_bytes).ok();+ self.gate_accumulator_buffer = context.create_buffer(required_bytes).ok();self.accumulator_buffer_bytes = required_bytes;}@@ -160,10 +136,7 @@ impl Kernel {) -> Result<(), MTLError> {self.ensure_accumulator_buffers(context, descriptor.accumulator_bytes);self.get_partial_pipeline(context, &descriptor.pipeline_configuration)?;- self.get_accum_pipeline(- context,- descriptor.pipeline_configuration.activation,- )?;+ self.get_accum_pipeline(context, descriptor.pipeline_configuration.activation)?;let partial_pipeline_state = self.partial_pipelines@@ -173,29 +146,18 @@ impl Kernel {.accum_pipelines.get(&descriptor.pipeline_configuration.activation).expect("Accum pipeline must be initialized");- let up_accumulator_buffer = self- .up_accumulator_buffer- .as_ref()- .expect("Up accumulator buffer must be initialized");- let gate_accumulator_buffer = self- .gate_accumulator_buffer- .as_ref()- .expect("Gate accumulator buffer must be initialized");+ let up_accumulator_buffer =+ self.up_accumulator_buffer.as_ref().expect("Up accumulator buffer must be initialized");+ let gate_accumulator_buffer =+ self.gate_accumulator_buffer.as_ref().expect("Gate accumulator buffer must be initialized");encoder.set_compute_pipeline_state(partial_pipeline_state);- encoder.set_buffer(- Some(arguments.input),- arguments.input_offset as usize,- 0,- );+ encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 0);encoder.set_buffer(Some(arguments.weights), 0, 1);encoder.set_buffer(Some(up_accumulator_buffer), 0, 2);encoder.set_buffer(Some(gate_accumulator_buffer), 0, 3);encoder.set_value(&descriptor.params, 4);- encoder.dispatch_threadgroups(- descriptor.partial_threadgroups,- descriptor.partial_threads_per_threadgroup,- );+ encoder.dispatch_threadgroups(descriptor.partial_threadgroups, descriptor.partial_threads_per_threadgroup);encoder.set_compute_pipeline_state(accum_pipeline_state);encoder.set_buffer(Some(up_accumulator_buffer), 0, 0);@@ -205,10 +167,7 @@ impl Kernel {encoder.set_value(&descriptor.output_elements_per_partition, 4);encoder.set_value(&descriptor.ldd, 5);- encoder.dispatch_threads(- descriptor.accum_total_threads,- descriptor.accum_threads_per_threadgroup,- );+ encoder.dispatch_threads(descriptor.accum_total_threads, descriptor.accum_threads_per_threadgroup);Ok(())}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex a24d4084a..c9bd68bed 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -3,15 +3,14 @@ mod data_type;pub mod dsl {include!(concat!(env!("OUT_DIR"), "/dsl.rs"));}-pub(super) const MTLB: &[u8] =- include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));+pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));pub mod matmul;pub mod media_kernels;pub mod mlp;pub use mlp::{- MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX,- MlpActivationType, MlpFusedConfig, make_non_fused_function_constants,+ MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX, MlpActivationType, MlpFusedConfig,+ make_non_fused_function_constants,};pub mod mlp_fused;pub mod moe;@@ -23,41 +22,27 @@ pub mod ssm;pub mod token_copy;pub use attention::{- AttentionError, AttentionKernel, AttentionKernelVariant,- AttentionSinglePassArguments, AttentionTwoPassArguments,+ AttentionError, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments, AttentionTwoPassArguments,KVCacheUpdateArguments,};pub use data_type::KernelDataType;pub use matmul::{MatmulArguments, MatmulKernel};pub use moe::{- MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError,- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,- MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,- MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments,- MoeScatterError, MoeScatterKernels, MoeScatterWithMapArguments,-};-pub use rms_norm::{- QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormError,- RMSNormKernel, RMSNormKernelType,+ MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,+ MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,+ MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,+ MoeScatterWithMapArguments,};+pub use rms_norm::{QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType};pub use rope::{RopeError, RopeKernel, RopeKernelArguments};-pub use short_conv::{- ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError,- ShortConvPrefillArguments,-};+pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError, ShortConvPrefillArguments};pub use ssm::{- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,- SSDPrefillArguments, SSDPrefillMode, SSMKernelError,+ Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,};pub use token_copy::TokenCopyKernel;pub mod media {- pub use super::media_kernels::{- ExtractImagePatches, ScalePadNormalizeImage,- };+ pub use super::media_kernels::{ExtractImagePatches, ScalePadNormalizeImage};}-pub use media_kernels::{- ExtractImagePatches, ImageParameters, PatchParameters,- ScalePadNormalizeImage,-};+pub use media_kernels::{ExtractImagePatches, ImageParameters, PatchParameters, ScalePadNormalizeImage};diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rsindex b0293716f..1c0ff82a0 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs@@ -2,13 +2,12 @@ use std::{mem::size_of, ptr::NonNull};use super::{dtype_index, dtype_suffix};use crate::backends::metal::{- KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,- MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState,- MTLContext, MTLDataType, MTLError, MTLFunctionConstantValues, MTLSize,- NSRange, ProtocolObject, Retained,+ KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,+ MTLComputePipelineState, MTLContext, MTLDataType, MTLError, MTLFunctionConstantValues, MTLSize, NSRange,+ ProtocolObject, Retained,kernel::moe::tiles_map::{- MoeTileCountsArguments, MoeTileDispatchArguments,- MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,+ MoeTileScanArguments,},};@@ -29,15 +28,15 @@ pub enum MoeExpertsError {pub struct MoeExpertsArguments<'a> {pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - permuted inputpub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1] - expert segment offsets- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff, d_model] - transposed up projection weights- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model, d_ff] - transposed down projection weights- pub y_partial: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - output buffer- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff] - up projection biases- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model] - down projection biases- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]+ pub w13_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff, d_model] - transposed up projection weights+ pub w2_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model, d_ff] - transposed down projection weights+ pub y_partial: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - output buffer+ pub up_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff] - up projection biases+ pub down_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model] - down projection biases+ pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]pub tile_row_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [max_tiles * 3]- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [2]+ pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [max_tiles * 3]+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [2]pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]pub num_tiles_n: usize,pub t: usize,@@ -56,9 +55,8 @@ pub struct MoeExpertsArguments<'a> {pub struct MoeExpertsTwoPassPrefillKernel {tile_map: MoeTileMapKernels,- pass_a_indirect:- Vec<Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>>, // [gate][dtype]- pass_b_indirect: Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>, // [dtype]+ pass_a_indirect: Vec<Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>>, // [gate][dtype]+ pass_b_indirect: Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>, // [dtype]}#[derive(Debug)]@@ -93,44 +91,27 @@ pub struct MoeExpertsTwoPassArguments<'a> {impl MoeExpertsTwoPassPrefillKernel {pub fn new(ctx: &MTLContext) -> Result<Self, MoeExpertsError> {- let dtypes = [- KernelDataType::Float16,- KernelDataType::BFloat16,- KernelDataType::Float32,- ];+ let dtypes = [KernelDataType::Float16, KernelDataType::BFloat16, KernelDataType::Float32];let mut pass_a_indirect = vec![Vec::with_capacity(dtypes.len()); 4];for gate in 0u32..4u32 {for dtype in &dtypes {let dtype_suffix = dtype_suffix(*dtype);let fcv = MTLFunctionConstantValues::new();- fcv.set_constant_value_type_at_index(- NonNull::from(&gate).cast(),- MTLDataType::UInt,- 30,- );- let kernel_name = format!(- "moe_two_pass_prefill_pass_a_indirect_{}",- dtype_suffix- );+ fcv.set_constant_value_type_at_index(NonNull::from(&gate).cast(), MTLDataType::UInt, 30);+ let kernel_name = format!("moe_two_pass_prefill_pass_a_indirect_{}", dtype_suffix);let cache_key = format!("{}_gate_{}", kernel_name, gate);- pass_a_indirect[gate as usize].push(- ctx.compute_pipeline_state_cached(- &cache_key,- &kernel_name,- Some(&fcv),- )?,- );+ pass_a_indirect[gate as usize].push(ctx.compute_pipeline_state_cached(+ &cache_key,+ &kernel_name,+ Some(&fcv),+ )?);}}let mut pass_b_indirect = Vec::with_capacity(dtypes.len());for dtype in &dtypes {let dtype_suffix = dtype_suffix(*dtype);- let kernel_name = format!(- "moe_two_pass_prefill_pass_b_indirect_{}",- dtype_suffix- );- pass_b_indirect- .push(ctx.compute_pipeline_state(&kernel_name, None)?);+ let kernel_name = format!("moe_two_pass_prefill_pass_b_indirect_{}", dtype_suffix);+ pass_b_indirect.push(ctx.compute_pipeline_state(&kernel_name, None)?);}Ok(Self {tile_map: MoeTileMapKernels::new(ctx)?,@@ -154,14 +135,8 @@ impl MoeExpertsTwoPassPrefillKernel {KernelDataType::Float32 => 4,};let hidden_bytes = args.total_rows * args.d_ff * dtype_size;- let blit_encoder = command_buffer- .new_blit_command_encoder()- .expect("Failed to create blit command encoder");- blit_encoder.fill_buffer_range_value(- args.hidden_buffer,- NSRange::new(0, hidden_bytes),- 0,- );+ let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");+ blit_encoder.fill_buffer_range_value(args.hidden_buffer, NSRange::new(0, hidden_bytes), 0);blit_encoder.end_encoding();self.tile_map.encode_counts(command_buffer,@@ -214,9 +189,7 @@ impl MoeExpertsTwoPassPrefillKernel {},);let pass_a_pipeline = &self.pass_a_indirect[gate_idx][dtype_idx];- let encoder_a = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder_a = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");encoder_a.set_compute_pipeline_state(pass_a_pipeline);encoder_a.set_buffer(Some(args.x_perm_buffer), 0, 0);encoder_a.set_buffer(Some(args.expert_offsets), 0, 1);@@ -224,60 +197,20 @@ impl MoeExpertsTwoPassPrefillKernel {encoder_a.set_buffer(Some(args.up_biases), 0, 3);encoder_a.set_buffer(Some(args.hidden_buffer), 0, 4);unsafe {- encoder_a.set_bytes(- NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _),- size_of::<u32>(),- 5,- );- encoder_a.set_bytes(- NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _),- size_of::<u32>(),- 6,- );- encoder_a.set_bytes(- NonNull::new_unchecked(&e_u32 as *const _ as *mut _),- size_of::<u32>(),- 7,- );- encoder_a.set_bytes(- NonNull::new_unchecked(- &args.gate_clip_min as *const _ as *mut _,- ),- size_of::<f32>(),- 8,- );- encoder_a.set_bytes(- NonNull::new_unchecked(- &args.gate_clip_max as *const _ as *mut _,- ),- size_of::<f32>(),- 9,- );- encoder_a.set_bytes(- NonNull::new_unchecked(&args.up_clip_min as *const _ as *mut _),- size_of::<f32>(),- 10,- );- encoder_a.set_bytes(- NonNull::new_unchecked(&args.up_clip_max as *const _ as *mut _),- size_of::<f32>(),- 11,- );- encoder_a.set_bytes(- NonNull::new_unchecked(&args.silu_alpha as *const _ as *mut _),- size_of::<f32>(),- 12,- );+ encoder_a.set_bytes(NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _), size_of::<u32>(), 5);+ encoder_a.set_bytes(NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _), size_of::<u32>(), 6);+ encoder_a.set_bytes(NonNull::new_unchecked(&e_u32 as *const _ as *mut _), size_of::<u32>(), 7);+ encoder_a.set_bytes(NonNull::new_unchecked(&args.gate_clip_min as *const _ as *mut _), size_of::<f32>(), 8);+ encoder_a.set_bytes(NonNull::new_unchecked(&args.gate_clip_max as *const _ as *mut _), size_of::<f32>(), 9);+ encoder_a.set_bytes(NonNull::new_unchecked(&args.up_clip_min as *const _ as *mut _), size_of::<f32>(), 10);+ encoder_a.set_bytes(NonNull::new_unchecked(&args.up_clip_max as *const _ as *mut _), size_of::<f32>(), 11);+ encoder_a.set_bytes(NonNull::new_unchecked(&args.silu_alpha as *const _ as *mut _), size_of::<f32>(), 12);}encoder_a.set_buffer(Some(args.tile_map), 0, 13);// Match kernel config: WM=2, WN=2 => 4 SIMDgroups => 128 threadsconst SIMDGROUPS_PER_TG: u32 = 4;const THREADS_PER_TG: u32 = SIMDGROUPS_PER_TG * 32;- encoder_a.dispatch_threadgroups_indirect(- args.dispatch_args,- 0,- MTLSize::new(THREADS_PER_TG as usize, 1, 1),- );+ encoder_a.dispatch_threadgroups_indirect(args.dispatch_args, 0, MTLSize::new(THREADS_PER_TG as usize, 1, 1));encoder_a.end_encoding();self.tile_map.encode_dispatch_args(command_buffer,@@ -288,9 +221,7 @@ impl MoeExpertsTwoPassPrefillKernel {},);let pass_b_pipeline = &self.pass_b_indirect[dtype_idx];- let encoder_b = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder_b = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");encoder_b.set_compute_pipeline_state(pass_b_pipeline);encoder_b.set_buffer(Some(args.hidden_buffer), 0, 0);encoder_b.set_buffer(Some(args.expert_offsets), 0, 1);@@ -298,28 +229,12 @@ impl MoeExpertsTwoPassPrefillKernel {encoder_b.set_buffer(Some(args.down_biases), 0, 3);encoder_b.set_buffer(Some(args.output_buffer), 0, 4);unsafe {- encoder_b.set_bytes(- NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _),- size_of::<u32>(),- 5,- );- encoder_b.set_bytes(- NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _),- size_of::<u32>(),- 6,- );- encoder_b.set_bytes(- NonNull::new_unchecked(&e_u32 as *const _ as *mut _),- size_of::<u32>(),- 7,- );+ encoder_b.set_bytes(NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _), size_of::<u32>(), 5);+ encoder_b.set_bytes(NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _), size_of::<u32>(), 6);+ encoder_b.set_bytes(NonNull::new_unchecked(&e_u32 as *const _ as *mut _), size_of::<u32>(), 7);}encoder_b.set_buffer(Some(args.tile_map), 0, 8);- encoder_b.dispatch_threadgroups_indirect(- args.dispatch_args,- 0,- MTLSize::new(THREADS_PER_TG as usize, 1, 1),- );+ encoder_b.dispatch_threadgroups_indirect(args.dispatch_args, 0, MTLSize::new(THREADS_PER_TG as usize, 1, 1));encoder_b.end_encoding();Ok(())}diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_single.rsindex 14ed8876a..418fb4fd3 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs@@ -1,10 +1,6 @@-use crate::backends::common::kernel::{- MoeExpertsDecodeSinglePassAKernel as _,- MoeExpertsDecodeSinglePassBKernel as _,-};+use crate::backends::common::kernel::{MoeExpertsDecodeSinglePassAKernel as _, MoeExpertsDecodeSinglePassBKernel as _};use crate::backends::metal::kernel::dsl::{- MoeExpertsDecodeSinglePassAMetalKernel,- MoeExpertsDecodeSinglePassBMetalKernel,+ MoeExpertsDecodeSinglePassAMetalKernel, MoeExpertsDecodeSinglePassBMetalKernel,};use crate::backends::metal::{KernelDataType, MTLContext, MTLError};use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};@@ -12,11 +8,7 @@ use objc2::__framework_prelude::ProtocolObject;use objc2::Message;use objc2::rc::Retained;-static DTYPES: [KernelDataType; 3] = [- KernelDataType::Float16,- KernelDataType::BFloat16,- KernelDataType::Float32,-];+static DTYPES: [KernelDataType; 3] = [KernelDataType::Float16, KernelDataType::BFloat16, KernelDataType::Float32];/// Arguments for single-token MoE decode (T=1 optimized path)#[derive(Debug)]@@ -72,11 +64,7 @@ impl MoeExpertsSingleDecodeKernels {for gate in 0..4 {let mut kernels = vec![];for dtype in &DTYPES {- let kernel = MoeExpertsDecodeSinglePassAMetalKernel::new(- ctx,- (*dtype).into(),- gate,- )?;+ let kernel = MoeExpertsDecodeSinglePassAMetalKernel::new(ctx, (*dtype).into(), gate)?;kernels.push(kernel)}pass_a.push(kernels);@@ -84,10 +72,7 @@ impl MoeExpertsSingleDecodeKernels {let mut pass_b = Vec::with_capacity(DTYPES.len());for dtype in &DTYPES {- pass_b.push(MoeExpertsDecodeSinglePassBMetalKernel::new(- ctx,- (*dtype).into(),- )?);+ pass_b.push(MoeExpertsDecodeSinglePassBMetalKernel::new(ctx, (*dtype).into())?);}Ok(Self {@@ -106,15 +91,13 @@ impl MoeExpertsSingleDecodeKernels {}let gate_idx = args.gating_code.min(3) as usize;- let dtype_idx =- DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();+ let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();// Pass A: x @ W13[expert] -> hidden{let kernel = &self.pass_a[gate_idx][dtype_idx];- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");kernel.encode(&args.x.retain(),&args.topk_ids.retain(),@@ -137,9 +120,8 @@ impl MoeExpertsSingleDecodeKernels {// Pass B: 8 simdgroups (256 threads), outputs final y directly{let kernel = &self.pass_b[dtype_idx];- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");kernel.encode(&args.hidden.retain(),&args.topk_ids.retain(),diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rsindex 29cceb288..61d7b79df 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs@@ -7,22 +7,15 @@ use objc2::{use crate::{DataType,backends::{- common::kernel::{- MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel,- },+ common::kernel::{MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},metal::{MTLContext, MTLError,kernel::{MoeExpertsTwoPassArguments,- dsl::{- MoeExpertsDecodeDownFused2DMetalKernel,- MoeExpertsDecodePassAMetalKernel,- },+ dsl::{MoeExpertsDecodeDownFused2DMetalKernel, MoeExpertsDecodePassAMetalKernel},moe::{- MoePassARowMapArguments, MoePassATileBuildArguments,- MoePassATileCountsArguments, MoePassATileDispatchArguments,- MoePassATileKernels, MoePassATileScanArguments,- dtype_index,+ MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,+ MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments, dtype_index,},},},@@ -43,8 +36,7 @@ impl MoeExpertsTwoPassDecodeKernels {for gate in 0u32..4u32 {let mut kernels = vec![];for dtype in &DTYPES {- let kernel =- MoeExpertsDecodePassAMetalKernel::new(ctx, *dtype, gate)?;+ let kernel = MoeExpertsDecodePassAMetalKernel::new(ctx, *dtype, gate)?;kernels.push(kernel);}pass_a_indirect.push(kernels);@@ -52,11 +44,7 @@ impl MoeExpertsTwoPassDecodeKernels {let mut fused_down = Vec::with_capacity(DTYPES.len());for dtype in &DTYPES {- let kernel = MoeExpertsDecodeDownFused2DMetalKernel::new(- ctx,- *dtype,- DataType::F32,- )?;+ let kernel = MoeExpertsDecodeDownFused2DMetalKernel::new(ctx, *dtype, DataType::F32)?;fused_down.push(kernel);}@@ -129,9 +117,8 @@ impl MoeExpertsTwoPassDecodeKernels {// pass alet gate_idx = args.gating_code.min(3) as usize;let dtype_idx = dtype_index(args.data_type);- let pass_a_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let pass_a_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];pass_a_kernel.encode(&args.x_perm_buffer.retain(),@@ -155,9 +142,8 @@ impl MoeExpertsTwoPassDecodeKernels {// pass blet pass_b_kernel = &self.fused_down[dtype_idx];- let pass_b_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let pass_b_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");pass_b_kernel.encode(&args.hidden_buffer.retain(),&args.row_expert_map.retain(),diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.rs b/crates/uzu/src/backends/metal/kernel/moe/gather.rsindex 6196ec3e5..977e7d674 100644--- a/crates/uzu/src/backends/metal/kernel/moe/gather.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/gather.rs@@ -5,11 +5,9 @@ use crate::{backends::{common::kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},metal::{- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLContext, MTLError, ProtocolObject, Retained,- kernel::dsl::{- MoeGatherXPerm1DMetalKernel, MoeGatherXPerm2DMetalKernel,- },+ KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext, MTLError, ProtocolObject,+ Retained,+ kernel::dsl::{MoeGatherXPerm1DMetalKernel, MoeGatherXPerm2DMetalKernel},},},};@@ -46,9 +44,7 @@ impl MoeGatherKernels {dtype: KernelDataType,args: &MoeGatherArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");match dtype {KernelDataType::Float32 => self.f32.encode(&args.x_buffer.retain(),diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rsindex 9300ddfeb..5981a3a92 100644--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs@@ -12,24 +12,15 @@ pub mod tiles_pass_a;// Re-export public items from submodulespub use experts::{- MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,- MoeExpertsTwoPassPrefillKernel, MoeScatterError,-};-pub use experts_single::{- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels,+ MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeScatterError,};+pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels};pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeKernels;pub use gather::{MoeGatherArguments, MoeGatherKernels};-pub use router_topk::{- MoeRouterTopKArguments, MoeRouterTopKError, MoeRouterTopKKernel,-};-pub use scatter::{- MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels,- MoeScatterWithMapArguments,-};+pub use router_topk::{MoeRouterTopKArguments, MoeRouterTopKError, MoeRouterTopKKernel};+pub use scatter::{MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments};pub use tiles_pass_a::{- MoePassARowMapArguments, MoePassATileBuildArguments,- MoePassATileCountsArguments, MoePassATileDispatchArguments,+ MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,MoePassATileKernels, MoePassATileScanArguments,};diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs b/crates/uzu/src/backends/metal/kernel/moe/router_topk.rsindex ffdb5113c..8c30769fd 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs@@ -1,8 +1,6 @@use crate::backends::metal::{- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,- MTLSize, ProtocolObject, Retained,- metal_extensions::ComputeEncoderSetValue,+ KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState,+ MTLContext, MTLError, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,};const THREADS_PER_THREADGROUP: usize = 256;@@ -30,9 +28,9 @@ pub struct MoeRouterTopKKernel {#[derive(Debug)]pub struct MoeRouterTopKArguments<'a> {- pub input_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T, d_model]+ pub input_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T, d_model]pub weight_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model]- pub bias_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]+ pub bias_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]pub topk_ids_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T, K]pub topk_probs_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T, K]pub t: usize,@@ -44,12 +42,9 @@ pub struct MoeRouterTopKArguments<'a> {impl MoeRouterTopKKernel {pub fn new(mtl_context: &MTLContext) -> Result<Self, MoeRouterTopKError> {- let pipeline_f16 =- mtl_context.compute_pipeline_state("moe_router_topk_f16", None)?;- let pipeline_f32 =- mtl_context.compute_pipeline_state("moe_router_topk_f32", None)?;- let pipeline_bf16 =- mtl_context.compute_pipeline_state("moe_router_topk_bf16", None)?;+ let pipeline_f16 = mtl_context.compute_pipeline_state("moe_router_topk_f16", None)?;+ let pipeline_f32 = mtl_context.compute_pipeline_state("moe_router_topk_f32", None)?;+ let pipeline_bf16 = mtl_context.compute_pipeline_state("moe_router_topk_bf16", None)?;Ok(Self {pipeline_f16,pipeline_f32,@@ -83,19 +78,12 @@ impl MoeRouterTopKKernel {});}- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");match dtype {- KernelDataType::Float16 => {- compute_encoder.set_compute_pipeline_state(&self.pipeline_f16)- },- KernelDataType::Float32 => {- compute_encoder.set_compute_pipeline_state(&self.pipeline_f32)- },- KernelDataType::BFloat16 => {- compute_encoder.set_compute_pipeline_state(&self.pipeline_bf16)- },+ KernelDataType::Float16 => compute_encoder.set_compute_pipeline_state(&self.pipeline_f16),+ KernelDataType::Float32 => compute_encoder.set_compute_pipeline_state(&self.pipeline_f32),+ KernelDataType::BFloat16 => compute_encoder.set_compute_pipeline_state(&self.pipeline_bf16),}compute_encoder.set_buffer(Some(args.input_buffer), 0, 0);diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs b/crates/uzu/src/backends/metal/kernel/moe/scatter.rsindex 2c7509217..bc55f413a 100644--- a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter.rs@@ -1,7 +1,6 @@use crate::backends::metal::{- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLSize,- ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,+ KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState,+ MTLContext, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,};// ---- Scatter Buckets Kernels ----@@ -16,15 +15,11 @@ pub struct MoeScatterKernels {pipeline_bases: Retained<ProtocolObject<dyn MTLComputePipelineState>>,pipeline_scatter_f16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,pipeline_scatter_f32: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_scatter_bf16:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ pipeline_scatter_bf16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,// map variants- pipeline_scatter_map_f16:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_scatter_map_f32:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_scatter_map_bf16:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ pipeline_scatter_map_f16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ pipeline_scatter_map_f32: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ pipeline_scatter_map_bf16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,}#[derive(Debug)]@@ -61,20 +56,13 @@ pub struct MoeScatterWithMapArguments<'a> {impl MoeScatterKernels {pub fn new(mtl_context: &MTLContext) -> Result<Self, MoeScatterError> {- let pipeline_bases = mtl_context- .compute_pipeline_state("moe_block_bases_from_partials", None)?;- let pipeline_scatter_f16 = mtl_context- .compute_pipeline_state("moe_scatter_buckets_f16", None)?;- let pipeline_scatter_f32 = mtl_context- .compute_pipeline_state("moe_scatter_buckets_f32", None)?;- let pipeline_scatter_bf16 = mtl_context- .compute_pipeline_state("moe_scatter_buckets_bf16", None)?;- let pipeline_scatter_map_f16 = mtl_context- .compute_pipeline_state("moe_scatter_buckets_map_f16", None)?;- let pipeline_scatter_map_f32 = mtl_context- .compute_pipeline_state("moe_scatter_buckets_map_f32", None)?;- let pipeline_scatter_map_bf16 = mtl_context- .compute_pipeline_state("moe_scatter_buckets_map_bf16", None)?;+ let pipeline_bases = mtl_context.compute_pipeline_state("moe_block_bases_from_partials", None)?;+ let pipeline_scatter_f16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_f16", None)?;+ let pipeline_scatter_f32 = mtl_context.compute_pipeline_state("moe_scatter_buckets_f32", None)?;+ let pipeline_scatter_bf16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_bf16", None)?;+ let pipeline_scatter_map_f16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_f16", None)?;+ let pipeline_scatter_map_f32 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_f32", None)?;+ let pipeline_scatter_map_bf16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_bf16", None)?;Ok(Self {pipeline_bases,@@ -92,9 +80,8 @@ impl MoeScatterKernels {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,args: MoeBlockBasesArguments,) -> Result<(), MoeScatterError> {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");compute_encoder.set_compute_pipeline_state(&self.pipeline_bases);compute_encoder.set_buffer(Some(args.partials_buffer), 0, 0);compute_encoder.set_buffer(Some(args.block_bases_buffer), 0, 1);@@ -125,22 +112,18 @@ impl MoeScatterKernels {args: MoeScatterArguments,dtype: KernelDataType,) -> Result<(), MoeScatterError> {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");// Select pipeline based on dtypematch dtype {KernelDataType::Float16 => {- compute_encoder- .set_compute_pipeline_state(&self.pipeline_scatter_f16);+ compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f16);},KernelDataType::Float32 => {- compute_encoder- .set_compute_pipeline_state(&self.pipeline_scatter_f32);+ compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f32);},KernelDataType::BFloat16 => {- compute_encoder- .set_compute_pipeline_state(&self.pipeline_scatter_bf16);+ compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_bf16);},}compute_encoder.set_buffer(Some(args.topk_ids_buffer), 0, 0);@@ -176,9 +159,8 @@ impl MoeScatterKernels {args: MoeScatterWithMapArguments,dtype: KernelDataType,) -> Result<(), MoeScatterError> {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");let pipeline = match dtype {KernelDataType::Float16 => &self.pipeline_scatter_map_f16,KernelDataType::Float32 => &self.pipeline_scatter_map_f32,diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rsindex 22bbd7b18..ec5e4c7b7 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs@@ -1,13 +1,10 @@use crate::backends::{- common::kernel::{- MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel,- MoeWriteDispatchArgsKernel,- },+ common::kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},metal::{MTLContext, MTLError,kernel::dsl::{- MoeBuildTileMapMetalKernel, MoeTileCountsMetalKernel,- MoeTileScanMetalKernel, MoeWriteDispatchArgsMetalKernel,+ MoeBuildTileMapMetalKernel, MoeTileCountsMetalKernel, MoeTileScanMetalKernel,+ MoeWriteDispatchArgsMetalKernel,},},};@@ -37,15 +34,8 @@ impl MoeTileMapKernels {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,args: &MoeTileCountsArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");- self.counts.encode(- &args.offsets_buffer.retain(),- &args.tile_counts_buffer.retain(),- args.e as u32,- &encoder,- );+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");+ self.counts.encode(&args.offsets_buffer.retain(), &args.tile_counts_buffer.retain(), args.e as u32, &encoder);encoder.end_encoding();}@@ -54,9 +44,7 @@ impl MoeTileMapKernels {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,args: &MoeTileScanArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.scan.encode(&args.tile_counts_buffer.retain(),&args.tile_offsets_buffer.retain(),@@ -72,9 +60,7 @@ impl MoeTileMapKernels {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,args: &MoeTileMapBuildArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.build.encode(&args.expert_offsets.retain(),&args.tile_offsets.retain(),@@ -91,21 +77,14 @@ impl MoeTileMapKernels {command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,args: &MoeTileDispatchArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");- self.dispatch.encode(- &args.total_tiles.retain(),- &args.dispatch_args.retain(),- args.num_tiles_x,- &encoder,- );+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");+ self.dispatch.encode(&args.total_tiles.retain(), &args.dispatch_args.retain(), args.num_tiles_x, &encoder);encoder.end_encoding();}}pub struct MoeTileCountsArguments<'a> {- pub offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]+ pub offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]pub tile_counts_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]pub e: usize,}@@ -128,7 +107,7 @@ pub struct MoeTileMapBuildArguments<'a> {#[derive(Debug)]pub struct MoeTileDispatchArguments<'a> {- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]- pub num_tiles_x: u32, // x dimension for indirect dispatch+ pub num_tiles_x: u32, // x dimension for indirect dispatch}diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rsindex 8023d37d4..f9c6dd44a 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs@@ -2,17 +2,14 @@ use objc2::Message;use crate::backends::{common::kernel::{- MoePassABuildRowMapKernel, MoePassABuildTileMapKernel,- MoePassATileCountsKernel, MoePassATileScanKernel,+ MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel, MoePassATileScanKernel,MoePassAWriteDispatchArgsKernel,},metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext, MTLError,- ProtocolObject,+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext, MTLError, ProtocolObject,kernel::dsl::{- MoePassABuildRowMapMetalKernel, MoePassABuildTileMapMetalKernel,- MoePassATileCountsMetalKernel, MoePassATileScanMetalKernel,- MoePassAWriteDispatchArgsMetalKernel,+ MoePassABuildRowMapMetalKernel, MoePassABuildTileMapMetalKernel, MoePassATileCountsMetalKernel,+ MoePassATileScanMetalKernel, MoePassAWriteDispatchArgsMetalKernel,},},};@@ -27,9 +24,9 @@ pub struct MoePassATileCountsArguments<'a> {#[derive(Debug)]pub struct MoePassATileScanArguments<'a> {- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]+ pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]pub e: usize,}@@ -53,7 +50,7 @@ pub struct MoePassATileBuildArguments<'a> {#[derive(Debug)]pub struct MoePassATileDispatchArguments<'a> {- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]pub num_tiles_y: u32,}@@ -82,9 +79,7 @@ impl MoePassATileKernels {command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,args: &MoePassATileCountsArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.counts.encode(&args.expert_offsets.retain(),&args.tile_counts.retain(),@@ -100,9 +95,7 @@ impl MoePassATileKernels {command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,args: &MoePassATileScanArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.scan.encode(&args.tile_counts.retain(),&args.tile_offsets.retain(),@@ -118,9 +111,7 @@ impl MoePassATileKernels {command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,args: &MoePassARowMapArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.row_map.encode(&args.expert_offsets.retain(),&args.row_expert_map.retain(),@@ -136,9 +127,7 @@ impl MoePassATileKernels {command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,args: &MoePassATileBuildArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.build_map.encode(&args.expert_offsets.retain(),&args.tile_offsets.retain(),@@ -156,15 +145,8 @@ impl MoePassATileKernels {command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,args: &MoePassATileDispatchArguments,) {- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");- self.dispatch.encode(- &args.total_tiles.retain(),- &args.dispatch_args.retain(),- args.num_tiles_y,- &encoder,- );+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");+ self.dispatch.encode(&args.total_tiles.retain(), &args.dispatch_args.retain(), args.num_tiles_y, &encoder);encoder.end_encoding();}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsindex a34d37522..5ab10fa68 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -3,9 +3,8 @@ use std::{collections::HashMap, ptr::NonNull};use crate::{DataType,backends::metal::{- MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState,- MTLContext, MTLError, MTLFunctionConstantValues, MTLSize,- ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,+ MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues,+ MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,},config::QuantizationMode,};@@ -37,10 +36,7 @@ pub enum QuantizedMatmulError {}pub struct QuantizedMatmulKernel {- pipelines: HashMap<- KernelKind,- (Retained<ProtocolObject<dyn MTLComputePipelineState>>, u64, u64),- >,+ pipelines: HashMap<KernelKind, (Retained<ProtocolObject<dyn MTLComputePipelineState>>, u64, u64)>,output_dim: usize,weights_transposed: bool,}@@ -107,8 +103,7 @@ impl QuantizedMatmulKernel {quantization_type: QuantizationType,weights_transposed: bool,) -> Result<Self, QuantizedMatmulError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(QuantizedMatmulError::UnsupportedDataType(data_type));}@@ -122,41 +117,21 @@ impl QuantizedMatmulKernel {let mut pipelines = HashMap::new();- let kernel_name_mv = select_matrix_vector_kernel_name(- data_type,- group_size,- weights_transposed,- output_dim,- input_dim,- mode,- )?;+ let kernel_name_mv =+ select_matrix_vector_kernel_name(data_type, group_size, weights_transposed, output_dim, input_dim, mode)?;let cache_key_mv = format!("{}_mlx_{}", kernel_name_mv, use_mlx_quant);let pipeline_mv = mtl_context- .compute_pipeline_state_cached(- &cache_key_mv,- &kernel_name_mv,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key_mv, &kernel_name_mv, Some(&function_constants)).map_err(QuantizedMatmulError::MetalError)?;pipelines.insert(KernelKind::MatrixVector, (pipeline_mv, 32, 32));- let kernel_name_mm = select_qmm_kernel_name(- data_type,- group_size,- weights_transposed,- output_dim,- input_dim,- mode,- )?;+ let kernel_name_mm =+ select_qmm_kernel_name(data_type, group_size, weights_transposed, output_dim, input_dim, mode)?;let cache_key_mm = format!("{}_mlx_{}", kernel_name_mm, use_mlx_quant);let pipeline_mm = mtl_context- .compute_pipeline_state_cached(- &cache_key_mm,- &kernel_name_mm,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key_mm, &kernel_name_mm, Some(&function_constants)).map_err(QuantizedMatmulError::MetalError)?;let (bm, bn) = if kernel_name_mm.contains("_64x64") {@@ -196,12 +171,10 @@ impl QuantizedMatmulKernel {) -> Result<(), QuantizedMatmulError> {let variant = self.select_variant(args.batch as usize);let (pipeline, bm, bn) =- self.pipelines.get(&variant).ok_or_else(|| {- QuantizedMatmulError::InvalidDimensions {- m: args.batch as usize,- n: args.output_dim as usize,- k: args.input_dim as usize,- }+ self.pipelines.get(&variant).ok_or_else(|| QuantizedMatmulError::InvalidDimensions {+ m: args.batch as usize,+ n: args.output_dim as usize,+ k: args.input_dim as usize,})?;encoder.set_compute_pipeline_state(pipeline);@@ -230,28 +203,17 @@ impl QuantizedMatmulKernel {64};let n_tgp_y = (n + bn - 1) / bn;- let threadgroups =- MTLSize::new(m as usize, n_tgp_y as usize, 1);+ let threadgroups = MTLSize::new(m as usize, n_tgp_y as usize, 1);let threads_per_threadgroup = MTLSize::new(bk as usize, 2, 1);- encoder.dispatch_threadgroups(- threadgroups,- threads_per_threadgroup,- );+ encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);},KernelKind::MatrixMatrix => {let wm = 2;let wn = 2;- let threads_per_threadgroup =- MTLSize::new(32, wn as usize, wm as usize);- let threadgroups = MTLSize::new(- ((n as u64 + bn - 1) / bn) as usize,- ((m as u64 + bm - 1) / bm) as usize,- 1,- );- encoder.dispatch_threadgroups(- threadgroups,- threads_per_threadgroup,- );+ let threads_per_threadgroup = MTLSize::new(32, wn as usize, wm as usize);+ let threadgroups =+ MTLSize::new(((n as u64 + bn - 1) / bn) as usize, ((m as u64 + bm - 1) / bm) as usize, 1);+ encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);},}@@ -278,8 +240,7 @@ fn select_matrix_vector_kernel_name(input_dim: usize,mode: QuantizationMode,) -> Result<String, QuantizedMatmulError> {- let type_suffix = dtype_suffix(data_type)- .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;+ let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;let bits = match mode {QuantizationMode::UInt4 => 4,QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,@@ -304,8 +265,7 @@ fn select_qmm_kernel_name(input_dim: usize,mode: QuantizationMode,) -> Result<String, QuantizedMatmulError> {- let type_suffix = dtype_suffix(data_type)- .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;+ let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;let bits = match mode {QuantizationMode::UInt4 => 4,QuantizationMode::Int8 | QuantizationMode::UInt8 => 8,@@ -316,15 +276,11 @@ fn select_qmm_kernel_name(} else {""};- let mut kernel_name =- base_qmm_kernel_name(type_suffix, group_size, transpose_infix, bits)?;+ let mut kernel_name = base_qmm_kernel_name(type_suffix, group_size, transpose_infix, bits)?;if weights_transposed {if output_dim % 32 != 0 {kernel_name.push_str("_unaligned");- } else if type_suffix == "bf16"- && (group_size == 128 || group_size == 64)- && (bits == 4 || bits == 8)- {+ } else if type_suffix == "bf16" && (group_size == 128 || group_size == 64) && (bits == 4 || bits == 8) {kernel_name.push_str("_64x64");}} else if input_dim % 32 == 0 {@@ -376,8 +332,7 @@ impl MlpFusedQmvKernel {mode: QuantizationMode,quantization_type: QuantizationType,) -> Result<Self, QuantizedMatmulError> {- let type_suffix = dtype_suffix(data_type)- .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;+ let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;let bits = match mode {QuantizationMode::UInt4 => 4,@@ -388,8 +343,7 @@ impl MlpFusedQmvKernel {return Err(QuantizedMatmulError::UnsupportedBits(bits));}- let kernel_name =- format!("qmv_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);+ let kernel_name = format!("qmv_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);let function_constants = MTLFunctionConstantValues::new();let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);@@ -426,10 +380,8 @@ impl MlpFusedQmvKernel {// Dispatch: one threadgroup per 8 output rows (num_simdgroups * results_per_simdgroup)let rows_per_threadgroup = 8;- let n_tgp_y = ((args.hidden_dim + rows_per_threadgroup - 1)- / rows_per_threadgroup) as u64;- let threadgroups =- MTLSize::new(args.batch_count.max(1) as usize, n_tgp_y as usize, 1);+ let n_tgp_y = ((args.hidden_dim + rows_per_threadgroup - 1) / rows_per_threadgroup) as u64;+ let threadgroups = MTLSize::new(args.batch_count.max(1) as usize, n_tgp_y as usize, 1);let threads_per_threadgroup = MTLSize::new(32, 2, 1); // 2 simdgroups, 32 threads eachencoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);@@ -474,8 +426,7 @@ impl MlpFusedQmmKernel {mode: QuantizationMode,quantization_type: QuantizationType,) -> Result<Self, QuantizedMatmulError> {- let type_suffix = dtype_suffix(data_type)- .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;+ let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;let bits = match mode {QuantizationMode::UInt4 => 4,@@ -486,8 +437,7 @@ impl MlpFusedQmmKernel {return Err(QuantizedMatmulError::UnsupportedBits(bits));}- let kernel_name =- format!("qmm_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);+ let kernel_name = format!("qmm_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);let function_constants = MTLFunctionConstantValues::new();let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rsindex 19e9b5c8d..facef02b7 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs@@ -1,9 +1,8 @@use crate::{DataType,backends::metal::{- ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError, MTLSize, ProtocolObject,- Retained,+ ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,+ MTLSize, ProtocolObject, Retained,},};@@ -107,9 +106,7 @@ impl RMSNormKernel {full_layer,)?;- let pipeline = context- .compute_pipeline_state(&function_name, None)- .map_err(RMSNormError::MetalError)?;+ let pipeline = context.compute_pipeline_state(&function_name, None).map_err(RMSNormError::MetalError)?;Ok(Self {pipeline,@@ -144,18 +141,11 @@ impl RMSNormKernel {Ok(format!("{}_{}_{}_{}_{}{}",- base_name,- input_suffix,- scales_suffix,- output_suffix,- accum_suffix,- mode_suffix+ base_name, input_suffix, scales_suffix, output_suffix, accum_suffix, mode_suffix))}- fn data_type_to_suffix(- data_type: DataType- ) -> Result<&'static str, RMSNormError> {+ fn data_type_to_suffix(data_type: DataType) -> Result<&'static str, RMSNormError> {match data_type {DataType::F32 => Ok("f32"),DataType::F16 => Ok("f16"),@@ -169,9 +159,7 @@ impl RMSNormKernel {}}- fn accum_type_to_suffix(- data_type: DataType- ) -> Result<&'static str, RMSNormError> {+ fn accum_type_to_suffix(data_type: DataType) -> Result<&'static str, RMSNormError> {match data_type {DataType::F32 => Ok("f32"),DataType::F16 => Ok("f16"),@@ -196,17 +184,9 @@ impl RMSNormKernel {compute_encoder.set_compute_pipeline_state(&self.pipeline);// Set buffers- compute_encoder.set_buffer(- Some(args.input_buffer),- args.input_offset as usize,- 0,- );+ compute_encoder.set_buffer(Some(args.input_buffer), args.input_offset as usize, 0);compute_encoder.set_buffer(Some(args.scales_buffer), 0, 1);- compute_encoder.set_buffer(- Some(args.output_buffer),- args.output_offset as usize,- 2,- );+ compute_encoder.set_buffer(Some(args.output_buffer), args.output_offset as usize, 2);// Set parameterscompute_encoder.set_value(&args.batch_size, 3);@@ -226,10 +206,7 @@ impl RMSNormKernel {depth: 1,};- compute_encoder.dispatch_threadgroups(- threadgroups_per_grid,- threads_per_threadgroup,- );+ compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);Ok(())}@@ -264,12 +241,8 @@ impl RMSNormKernel {// We only ever normalize within [Q] and/or [K].let (head_offset, head_count): (u32, u32) = match args.target {QKNormTarget::QueryHeads => (0, args.num_q_heads as u32),- QKNormTarget::KeyHeads => {- (args.num_q_heads as u32, args.num_kv_heads as u32)- },- QKNormTarget::Both => {- (0, (args.num_q_heads + args.num_kv_heads) as u32)- },+ QKNormTarget::KeyHeads => (args.num_q_heads as u32, args.num_kv_heads as u32),+ QKNormTarget::Both => (0, (args.num_q_heads + args.num_kv_heads) as u32),};if args.batch_size <= 0 || args.head_dim <= 0 || head_count == 0 {@@ -284,11 +257,9 @@ impl RMSNormKernel {let simd_width = self.pipeline.thread_execution_width();let max_threads = self.pipeline.max_total_threads_per_threadgroup();let max_heads_per_threadgroup = (max_threads / simd_width).max(1);- let heads_per_threadgroup =- (head_count as usize).min(max_heads_per_threadgroup).max(1);+ let heads_per_threadgroup = (head_count as usize).min(max_heads_per_threadgroup).max(1);- let num_head_tiles =- (head_count as usize).div_ceil(heads_per_threadgroup);+ let num_head_tiles = (head_count as usize).div_ceil(heads_per_threadgroup);let threadgroups_per_grid = MTLSize {width: args.batch_size as usize,@@ -302,10 +273,7 @@ impl RMSNormKernel {depth: 1,};- compute_encoder.dispatch_threadgroups(- threadgroups_per_grid,- threads_per_threadgroup,- );+ compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);Ok(())}diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rsindex 2fadd633f..d11437399 100644--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs@@ -1,9 +1,8 @@use thiserror::Error;use crate::backends::metal::{- ComputeEncoderSetValue, KernelDataType, MTLBuffer,- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,- MTLSize, ProtocolObject, Retained,+ ComputeEncoderSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,+ MTLError, MTLSize, ProtocolObject, Retained,};pub struct RopeKernel {@@ -20,11 +19,11 @@ pub enum RopeError {#[allow(dead_code)]pub struct RopeKernelArguments<'a> {- pub qkv_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)+ pub qkv_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)pub cosines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)- pub sines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)+ pub sines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)pub token_positions_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)- pub token_positions_offset: usize, // byte offset into token_positions_buffer+ pub token_positions_offset: usize, // byte offset into token_positions_bufferpub rotated_queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4)pub rotated_keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5)pub head_dim: usize,@@ -39,18 +38,12 @@ impl RopeKernel {context: &MTLContext,data_type: KernelDataType,) -> Result<Self, RopeError> {- let function_name =- format!("applyRope_{}", data_type.function_name_suffix());+ let function_name = format!("applyRope_{}", data_type.function_name_suffix());- let pipeline_state = context- .compute_pipeline_state(&function_name, None)- .map_err(|e| {- eprintln!(- "Failed to create pipeline state for {}: {:?}",- function_name, e- );- RopeError::MetalError(e)- })?;+ let pipeline_state = context.compute_pipeline_state(&function_name, None).map_err(|e| {+ eprintln!("Failed to create pipeline state for {}: {:?}", function_name, e);+ RopeError::MetalError(e)+ })?;Ok(Self {pipeline_state,@@ -67,11 +60,7 @@ impl RopeKernel {compute_encoder.set_buffer(Some(args.qkv_buffer), 0, 0);compute_encoder.set_buffer(Some(args.cosines_buffer), 0, 1);compute_encoder.set_buffer(Some(args.sines_buffer), 0, 2);- compute_encoder.set_buffer(- Some(args.token_positions_buffer),- args.token_positions_offset as usize,- 3,- );+ compute_encoder.set_buffer(Some(args.token_positions_buffer), args.token_positions_offset as usize, 3);compute_encoder.set_buffer(Some(args.rotated_queries_buffer), 0, 4);compute_encoder.set_buffer(Some(args.rotated_keys_buffer), 0, 5);@@ -99,7 +88,6 @@ impl RopeKernel {depth: head_dim as usize,};- compute_encoder- .dispatch_threads(total_threads, threads_per_threadgroup);+ compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);}}diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rsindex 8f60fdfb6..b496e3523 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs@@ -1,9 +1,8 @@use std::ptr::NonNull;use crate::backends::metal::{- ComputeEncoderSetValue, KernelDataType, MTLBuffer,- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLDataType,- MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,+ ComputeEncoderSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,+ MTLDataType, MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,};#[derive(Debug, thiserror::Error)]@@ -16,32 +15,20 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {dt.function_name_suffix()}-fn make_function_constants(- has_bias: bool-) -> Retained<MTLFunctionConstantValues> {+fn make_function_constants(has_bias: bool) -> Retained<MTLFunctionConstantValues> {let function_constants = MTLFunctionConstantValues::new();- function_constants.set_constant_value_type_at_index(- NonNull::from(&has_bias).cast(),- MTLDataType::Bool,- 0,- );+ function_constants.set_constant_value_type_at_index(NonNull::from(&has_bias).cast(), MTLDataType::Bool, 0);function_constants}pub struct ShortConvKernel {pack_pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- prefill_pipeline_no_bias:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- prefill_pipeline_with_bias:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- decode_pipeline_no_bias:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- decode_pipeline_with_bias:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- trie_pipeline_no_bias:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- trie_pipeline_with_bias:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ prefill_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ prefill_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ decode_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ decode_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ trie_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ trie_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,}pub struct ShortConvPackArguments<'a> {@@ -112,61 +99,40 @@ impl ShortConvKernel {context: &MTLContext,data_type: KernelDataType,) -> Result<Self, ShortConvKernelError> {- let pack_name =- format!("short_conv_pack_kernel_{}", fn_suffix(data_type));- let prefill_name =- format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));- let decode_name =- format!("short_conv_decode_kernel_{}", fn_suffix(data_type));- let trie_name =- format!("short_conv_trie_kernel_{}", fn_suffix(data_type));-- let pack_pipeline = context- .compute_pipeline_state(&pack_name, None)- .map_err(ShortConvKernelError::MetalError)?;+ let pack_name = format!("short_conv_pack_kernel_{}", fn_suffix(data_type));+ let prefill_name = format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));+ let decode_name = format!("short_conv_decode_kernel_{}", fn_suffix(data_type));+ let trie_name = format!("short_conv_trie_kernel_{}", fn_suffix(data_type));++ let pack_pipeline =+ context.compute_pipeline_state(&pack_name, None).map_err(ShortConvKernelError::MetalError)?;let prefill_pipeline_no_bias = {let function_constants = make_function_constants(false);let cache_key = format!("{}_has_bias_0", prefill_name);context- .compute_pipeline_state_cached(- &cache_key,- &prefill_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &prefill_name, Some(&function_constants)).map_err(ShortConvKernelError::MetalError)?};let prefill_pipeline_with_bias = {let function_constants = make_function_constants(true);let cache_key = format!("{}_has_bias_1", prefill_name);context- .compute_pipeline_state_cached(- &cache_key,- &prefill_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &prefill_name, Some(&function_constants)).map_err(ShortConvKernelError::MetalError)?};let decode_pipeline_no_bias = {let function_constants = make_function_constants(false);let cache_key = format!("{}_has_bias_0", decode_name);context- .compute_pipeline_state_cached(- &cache_key,- &decode_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants)).map_err(ShortConvKernelError::MetalError)?};let decode_pipeline_with_bias = {let function_constants = make_function_constants(true);let cache_key = format!("{}_has_bias_1", decode_name);context- .compute_pipeline_state_cached(- &cache_key,- &decode_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants)).map_err(ShortConvKernelError::MetalError)?};@@ -174,22 +140,14 @@ impl ShortConvKernel {let function_constants = make_function_constants(false);let cache_key = format!("{}_has_bias_0", trie_name);context- .compute_pipeline_state_cached(- &cache_key,- &trie_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &trie_name, Some(&function_constants)).map_err(ShortConvKernelError::MetalError)?};let trie_pipeline_with_bias = {let function_constants = make_function_constants(true);let cache_key = format!("{}_has_bias_1", trie_name);context- .compute_pipeline_state_cached(- &cache_key,- &trie_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &trie_name, Some(&function_constants)).map_err(ShortConvKernelError::MetalError)?};@@ -356,18 +314,10 @@ impl ShortConvKernel {if has_bias {compute_encoder.set_buffer(args.b, 0, 2);}- compute_encoder.set_buffer(- Some(args.base_state),- args.base_state_offset,- 3,- );+ compute_encoder.set_buffer(Some(args.base_state), args.base_state_offset, 3);compute_encoder.set_buffer(Some(args.parents), args.parents_offset, 4);compute_encoder.set_buffer(Some(args.out), args.out_offset, 5);- compute_encoder.set_buffer(- Some(args.suffix_state),- args.suffix_state_offset,- 6,- );+ compute_encoder.set_buffer(Some(args.suffix_state), args.suffix_state_offset, 6);compute_encoder.set_value(&args.suffix_len, 7);compute_encoder.set_value(&args.kernel_size, 8);diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rsindex 36e1c486d..4d851a6bd 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs@@ -3,10 +3,8 @@ use std::ptr::NonNull;use super::{SSMKernelError, fn_suffix};use crate::{backends::metal::{- KernelDataType, MTLBuffer, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLDataType,- MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,- metal_extensions::ComputeEncoderSetValue,+ KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLDataType,+ MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,},config::Activation,};@@ -34,17 +32,9 @@ fn make_function_constants(let function_constants = MTLFunctionConstantValues::new();let activation_type = activation_to_int(activation);- function_constants.set_constant_value_type_at_index(- NonNull::from(&activation_type).cast(),- MTLDataType::Int,- 0,- );+ function_constants.set_constant_value_type_at_index(NonNull::from(&activation_type).cast(), MTLDataType::Int, 0);- function_constants.set_constant_value_type_at_index(- NonNull::from(&has_bias).cast(),- MTLDataType::Bool,- 1,- );+ function_constants.set_constant_value_type_at_index(NonNull::from(&has_bias).cast(), MTLDataType::Bool, 1);function_constants}@@ -53,19 +43,17 @@ pub struct Conv1dScanKernel {pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,pack_pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- decode_pipeline_no_bias:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,- decode_pipeline_with_bias:- Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ decode_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ decode_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,}pub struct Conv1dScanArguments<'a> {pub padded: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0) [prefix+suffix, channels]- pub w: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1) [channels, kernel]+ pub w: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1) [channels, kernel]pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(2) [channels]- pub x_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) [suffix, inner_dim]- pub b_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) [suffix, proj_dim]- pub c_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) [suffix, proj_dim]+ pub x_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) [suffix, inner_dim]+ pub b_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) [suffix, proj_dim]+ pub c_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) [suffix, proj_dim]pub state_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(6) [channels, kernel-1]pub suffix_len: usize,pub kernel_size: i32,@@ -111,58 +99,36 @@ impl Conv1dScanKernel {activation: &Activation,) -> Result<Self, SSMKernelError> {let fn_name = format!("conv1d_scan_kernel_{}", fn_suffix(data_type));- let pack_name =- format!("conv1d_pack_prefix_kernel_{}", fn_suffix(data_type));- let decode_name =- format!("conv1d_decode_kernel_{}", fn_suffix(data_type));+ let pack_name = format!("conv1d_pack_prefix_kernel_{}", fn_suffix(data_type));+ let decode_name = format!("conv1d_decode_kernel_{}", fn_suffix(data_type));let activation_id = activation_to_int(activation);let pipeline_no_bias = {let function_constants = make_function_constants(activation, false);let cache_key = format!("{}_act_{}_bias_0", fn_name, activation_id);context- .compute_pipeline_state_cached(- &cache_key,- &fn_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &fn_name, Some(&function_constants)).map_err(SSMKernelError::MetalError)?};let pipeline_with_bias = {let function_constants = make_function_constants(activation, true);let cache_key = format!("{}_act_{}_bias_1", fn_name, activation_id);context- .compute_pipeline_state_cached(- &cache_key,- &fn_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &fn_name, Some(&function_constants)).map_err(SSMKernelError::MetalError)?};- let pack_pipeline = context- .compute_pipeline_state(&pack_name, None)- .map_err(SSMKernelError::MetalError)?;+ let pack_pipeline = context.compute_pipeline_state(&pack_name, None).map_err(SSMKernelError::MetalError)?;let decode_pipeline_no_bias = {let function_constants = make_function_constants(activation, false);- let cache_key =- format!("{}_act_{}_bias_0", decode_name, activation_id);+ let cache_key = format!("{}_act_{}_bias_0", decode_name, activation_id);context- .compute_pipeline_state_cached(- &cache_key,- &decode_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants)).map_err(SSMKernelError::MetalError)?};let decode_pipeline_with_bias = {let function_constants = make_function_constants(activation, true);- let cache_key =- format!("{}_act_{}_bias_1", decode_name, activation_id);+ let cache_key = format!("{}_act_{}_bias_1", decode_name, activation_id);context- .compute_pipeline_state_cached(- &cache_key,- &decode_name,- Some(&function_constants),- )+ .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants)).map_err(SSMKernelError::MetalError)?};Ok(Self {@@ -204,8 +170,7 @@ impl Conv1dScanKernel {height: padded_rows,depth: 1,};- compute_encoder- .dispatch_threads(total_threads, threads_per_threadgroup);+ compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);Ok(())}@@ -256,8 +221,7 @@ impl Conv1dScanKernel {height: args.channels,depth: 1,};- compute_encoder- .dispatch_threads(total_threads, threads_per_threadgroup);+ compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);Ok(())}@@ -313,8 +277,7 @@ impl Conv1dScanKernel {height: channels,depth: 1,};- compute_encoder- .dispatch_threads(total_threads, threads_per_threadgroup);+ compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);Ok(())}}diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rsindex fe660542b..e21c5cad0 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs@@ -15,7 +15,5 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {pub mod conv1d_scan;pub mod ssd_prefill;-pub use conv1d_scan::{- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,-};+pub use conv1d_scan::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel};pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode};diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rsindex ec0e9dd57..15f0ded2a 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs@@ -1,15 +1,8 @@use crate::backends::{- common::kernel::{- SSDPrefill64Kernel as _, SSDPrefillKernel as _,- SSDPrefillSequentialKernel as _,- },+ common::kernel::{SSDPrefill64Kernel as _, SSDPrefillKernel as _, SSDPrefillSequentialKernel as _},metal::{- KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLContext,- MTLError, ProtocolObject,- kernel::dsl::{- SSDPrefill64MetalKernel, SSDPrefillMetalKernel,- SSDPrefillSequentialMetalKernel,- },+ KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,+ kernel::dsl::{SSDPrefill64MetalKernel, SSDPrefillMetalKernel, SSDPrefillSequentialMetalKernel},},};use objc2::Message;@@ -52,10 +45,8 @@ impl SSDPrefillKernels {data_type: KernelDataType,) -> Result<Self, MTLError> {let single = SSDPrefillMetalKernel::new(context, data_type.into())?;- let single_64 =- SSDPrefill64MetalKernel::new(context, data_type.into())?;- let sequential =- SSDPrefillSequentialMetalKernel::new(context, data_type.into())?;+ let single_64 = SSDPrefill64MetalKernel::new(context, data_type.into())?;+ let sequential = SSDPrefillSequentialMetalKernel::new(context, data_type.into())?;Ok(Self {single,single_64,@@ -83,26 +74,10 @@ impl SSDPrefillKernels {args.suffix_len as u32,args.group_size,args.state_size,- args.x_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.dt_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.cb_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.state_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),+ args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),args.channels as u32,args.head_dim as u32,compute_encoder,@@ -120,26 +95,10 @@ impl SSDPrefillKernels {args.suffix_len as u32,args.group_size,args.state_size,- args.x_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.dt_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.cb_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.state_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),+ args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),args.channels as u32,args.head_dim as u32,compute_encoder,@@ -158,26 +117,10 @@ impl SSDPrefillKernels {args.suffix_len as u32,args.group_size,args.state_size,- args.x_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.dt_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.cb_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),- args.state_strides- .iter()- .map(|x| *x as u32)- .collect::<Vec<_>>()- .as_slice(),+ args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),+ args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),args.channels as u32,args.head_dim as u32,compute_encoder,diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/token_copy/mod.rsindex d64232315..676c90d45 100644--- a/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs@@ -1,6 +1,6 @@use crate::backends::metal::{- MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,- MTLError, MTLSize, ProtocolObject, Retained,+ MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError, MTLSize, ProtocolObject,+ Retained,};/// Kernel for copying sampled tokens in async pipeline.@@ -14,10 +14,8 @@ pub struct TokenCopyKernel {impl TokenCopyKernel {pub fn new(context: &MTLContext) -> Result<Self, MTLError> {- let copy_to_token_ids =- context.compute_pipeline_state("copy_sampled_token", None)?;- let copy_to_results =- context.compute_pipeline_state("copy_token_to_results", None)?;+ let copy_to_token_ids = context.compute_pipeline_state("copy_sampled_token", None)?;+ let copy_to_results = context.compute_pipeline_state("copy_token_to_results", None)?;Ok(Self {copy_to_token_ids,diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rsindex 1b621d72d..c997bf438 100644--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs@@ -7,24 +7,22 @@ use std::{};use super::{- Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal,- compilation_parameters::CompilationConfig, kernel::TokenCopyKernel,+ Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal, compilation_parameters::CompilationConfig,+ kernel::TokenCopyKernel,};use crate::{DataType,backends::{common::{Context, kernel::MaskUpdateKernel},metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt,- MTLEvent, ProtocolObject, Retained,+ MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLEvent, ProtocolObject, Retained,kernel::dsl::MaskUpdateMetalKernel,},},config::{DecoderConfig, LanguageModelConfig, ModelMetadata},encodable_block::Sampling,forward_pass::{- cache_layers::CacheLayers, model_shape::ModelShape,- scratch_buffers::ScratchBuffers, state::SharedBuffers,+ cache_layers::CacheLayers, model_shape::ModelShape, scratch_buffers::ScratchBuffers, state::SharedBuffers,},language_model::rng::PRng,parameters::ParameterLoader,@@ -63,15 +61,12 @@ impl AsyncBuffers {max_tokens: usize,batch_size: usize,) -> Self {- let positions = context- .create_buffer(max_tokens * std::mem::size_of::<i32>())- .expect("Failed to create positions buffer");- let seeds = context- .create_buffer(max_tokens * std::mem::size_of::<u64>())- .expect("Failed to create seeds buffer");- let results = context- .create_buffer(batch_size * std::mem::size_of::<u32>())- .expect("Failed to create results buffer");+ let positions =+ context.create_buffer(max_tokens * std::mem::size_of::<i32>()).expect("Failed to create positions buffer");+ let seeds =+ context.create_buffer(max_tokens * std::mem::size_of::<u64>()).expect("Failed to create seeds buffer");+ let results =+ context.create_buffer(batch_size * std::mem::size_of::<u32>()).expect("Failed to create results buffer");let event = context.device.new_event().expect("Failed to create event");Self {@@ -161,77 +156,46 @@ impl LanguageModelGeneratorContext {model_path: &Path,decoding_config: &DecodingConfig,) -> Result<Self, Error> {- let context =- MTLContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;+ let context = MTLContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;- let command_buffer = context- .create_command_buffer()- .expect("Failed to create command buffer")- .to_owned();+ let command_buffer = context.create_command_buffer().expect("Failed to create command buffer").to_owned();let config_path = model_path.join("config.json");if !config_path.exists() {return Err(Error::UnableToLoadConfig);}- let config_file =- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;let model_metadata: ModelMetadata =- serde_json::from_reader(BufReader::new(config_file))- .map_err(|_| Error::UnableToLoadConfig)?;+ serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;// Extract language model config- let language_model_config = model_metadata- .model_config- .as_language_model()- .ok_or(Error::UnableToLoadConfig)?;-- let decoder_config = Rc::new(- language_model_config- .decoder_config()- .map_err(|_| Error::UnableToLoadConfig)?,- );+ let language_model_config = model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;++ let decoder_config = Rc::new(language_model_config.decoder_config().map_err(|_| Error::UnableToLoadConfig)?);let model_shape = ModelShape::from_decoder_config(&decoder_config);- let prefill_step_size =- decoding_config.prefill_step_size.resolve(language_model_config);+ let prefill_step_size = decoding_config.prefill_step_size.resolve(language_model_config);let generate_suffix_length = decoding_config.generate_suffix_length();- let max_prefix_length: usize =- decoding_config.context_length.resolve(language_model_config);- let max_suffix_length: usize =- std::cmp::max(prefill_step_size, generate_suffix_length);+ let max_prefix_length: usize = decoding_config.context_length.resolve(language_model_config);+ let max_suffix_length: usize = std::cmp::max(prefill_step_size, generate_suffix_length);let compilation_config = Rc::new(CompilationConfig::default());let weights_path = model_path.join("model.safetensors");if !weights_path.exists() {return Err(Error::UnableToLoadWeights);}- let weights_file = File::open(&weights_path)- .map_err(|_| Error::UnableToLoadWeights)?;- let loader = ParameterLoader::new(&weights_file, context.as_ref())- .map_err(|_| Error::UnableToLoadWeights)?;+ let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let root_loader_view = loader.tree();- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(- context.as_ref(),- &decoder_config,- &model_shape,- )));+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));shared_buffers.borrow_mut().update_data(&root_loader_view);- let scratch_buffers = ScratchBuffers::new(- context.as_ref(),- &decoder_config,- &model_shape,- max_prefix_length,- max_suffix_length,- );+ let scratch_buffers =+ ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_prefix_length, max_suffix_length);- let executables = Decoder::new(- context.clone(),- decoder_config.clone(),- &root_loader_view,- compilation_config.clone(),- );+ let executables =+ Decoder::new(context.clone(), decoder_config.clone(), &root_loader_view, compilation_config.clone());let cache_layers = Rc::new(RefCell::new(CacheLayers::new(context.as_ref(),@@ -240,28 +204,18 @@ impl LanguageModelGeneratorContext {max_suffix_length,)));- let intermediate_data_type: DataType =- decoder_config.output_norm_config.scale_precision.into();+ let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();let kernel_data_type: KernelDataType = intermediate_data_type.into();let kv_cache_update = Box::new(- KVCacheUpdate::new(- context.as_ref(),- intermediate_data_type,- max_prefix_length,- )- .map_err(|_| Error::UnableToCreateMetalContext)?,+ KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)+ .map_err(|_| Error::UnableToCreateMetalContext)?,);- let gpu_sampler = Sampling::<Metal>::new(- &context,- intermediate_data_type,- max_suffix_length,- decoder_config.vocab_size,- )- .map_err(|_| Error::UnableToCreateMetalContext)?;+ let gpu_sampler =+ Sampling::<Metal>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)+ .map_err(|_| Error::UnableToCreateMetalContext)?;- let token_copy = TokenCopyKernel::new(&context)- .map_err(|_| Error::UnableToCreateMetalContext)?;+ let token_copy = TokenCopyKernel::new(&context).map_err(|_| Error::UnableToCreateMetalContext)?;// Create mask update kernel if model has attention layerslet mask_update = if decoder_config.has_attention_layers() {@@ -273,10 +227,8 @@ impl LanguageModelGeneratorContext {None};- let async_batch_size =- decoding_config.async_batch_size.resolve(model_path);- let async_buffers =- AsyncBuffers::new(&context, max_prefix_length, async_batch_size);+ let async_batch_size = decoding_config.async_batch_size.resolve(model_path);+ let async_buffers = AsyncBuffers::new(&context, max_prefix_length, async_batch_size);let seed = PRng::new(decoding_config.sampling_seed.resolve());@@ -302,11 +254,7 @@ impl LanguageModelGeneratorContext {}pub fn reset_command_buffer(&mut self) {- self.command_buffer = self- .mtl_context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();+ self.command_buffer =+ self.mtl_context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();}}diff --git a/crates/uzu/src/backends/metal/media_utils.rs b/crates/uzu/src/backends/metal/media_utils.rsindex 6761abc4c..a29171d61 100644--- a/crates/uzu/src/backends/metal/media_utils.rs+++ b/crates/uzu/src/backends/metal/media_utils.rs@@ -3,14 +3,11 @@ use std::{cell::RefCell, mem, rc::Rc};use crate::{DataType,backends::metal::{- MTLCommandBuffer, MTLContext, MTLDevice, MTLDeviceExt,- MTLResourceOptions, MetalArray, ProtocolObject, Retained,+ MTLCommandBuffer, MTLContext, MTLDevice, MTLDeviceExt, MTLResourceOptions, MetalArray, ProtocolObject,+ Retained,error::MTLError,image::{Image, PixelFormat, TextureUsage},- kernel::media_kernels::{- ExtractImagePatches, ImageParameters, PatchParameters,- ScalePadNormalizeImage,- },+ kernel::media_kernels::{ExtractImagePatches, ImageParameters, PatchParameters, ScalePadNormalizeImage},},};@@ -43,10 +40,8 @@ fn calculate_preprocessing_dimensions(let scaled_width = (input_width as f32 * scale_factor).ceil() as u32;let scaled_height = (input_height as f32 * scale_factor).ceil() as u32;- let padded_width =- ((scaled_width + patch_size - 1) / patch_size) * patch_size;- let padded_height =- ((scaled_height + patch_size - 1) / patch_size) * patch_size;+ let padded_width = ((scaled_width + patch_size - 1) / patch_size) * patch_size;+ let padded_height = ((scaled_height + patch_size - 1) / patch_size) * patch_size;(padded_width, padded_height, scale_factor)}@@ -93,23 +88,19 @@ pub fn calculate_image_preprocessing_requirements(input_height: u32,params: &ImagePreprocessingParams,) -> ImagePreprocessingRequirements {- let (padded_width, padded_height, _scale_factor) =- calculate_preprocessing_dimensions(- input_width,- input_height,- params.min_pixels,- params.max_pixels,- params.patch_size,- );+ let (padded_width, padded_height, _scale_factor) = calculate_preprocessing_dimensions(+ input_width,+ input_height,+ params.min_pixels,+ params.max_pixels,+ params.patch_size,+ );let num_patches_x = padded_width / params.patch_size;let num_patches_y = padded_height / params.patch_size;let total_spatial_patches = num_patches_x * num_patches_y;- let output_buffer_element_count = total_spatial_patches- * params.temporal_slices- * params.num_channels- * params.patch_size- * params.patch_size;+ let output_buffer_element_count =+ total_spatial_patches * params.temporal_slices * params.num_channels * params.patch_size * params.patch_size;let intermediate_texture_usage = TextureUsage {shader_read: true,@@ -158,30 +149,12 @@ impl MetalImagePreprocessor {let padded_height = requirements.padded_height;let input_dimensions_data = [input_image.width(), input_image.height()];- let mut input_dimensions_tensor = data_to_mtl_tensor(- &self.device,- &input_dimensions_data,- vec![2],- DataType::F32,- )?;- let mut image_mean_tensor = data_to_mtl_tensor(- &self.device,- ¶ms.image_mean,- vec![3],- DataType::F32,- )?;- let mut image_std_tensor = data_to_mtl_tensor(- &self.device,- ¶ms.image_std,- vec![3],- DataType::F32,- )?;- let mut padding_value_rgb_tensor = data_to_mtl_tensor(- &self.device,- ¶ms.padding_value_rgb,- vec![1],- DataType::F32,- )?;+ let mut input_dimensions_tensor =+ data_to_mtl_tensor(&self.device, &input_dimensions_data, vec![2], DataType::F32)?;+ let mut image_mean_tensor = data_to_mtl_tensor(&self.device, ¶ms.image_mean, vec![3], DataType::F32)?;+ let mut image_std_tensor = data_to_mtl_tensor(&self.device, ¶ms.image_std, vec![3], DataType::F32)?;+ let mut padding_value_rgb_tensor =+ data_to_mtl_tensor(&self.device, ¶ms.padding_value_rgb, vec![1], DataType::F32)?;let _spn_tensors: Vec<&mut MetalArray> = vec![&mut input_dimensions_tensor,@@ -204,34 +177,15 @@ impl MetalImagePreprocessor {command_buffer,);- let mut final_output_buffer_array_ref =- output_patch_buffer_cell.borrow_mut();+ let mut final_output_buffer_array_ref = output_patch_buffer_cell.borrow_mut();let padded_dimensions_data = [padded_width, padded_height];- let mut padded_dimensions_tensor = data_to_mtl_tensor(- &self.device,- &padded_dimensions_data,- vec![2],- DataType::F32,- )?;- let mut patch_size_tensor = data_to_mtl_tensor(- &self.device,- ¶ms.patch_size,- vec![1],- DataType::F32,- )?;- let mut num_channels_tensor = data_to_mtl_tensor(- &self.device,- ¶ms.num_channels,- vec![1],- DataType::F32,- )?;- let mut temporal_slices_tensor = data_to_mtl_tensor(- &self.device,- ¶ms.temporal_slices,- vec![1],- DataType::F32,- )?;+ let mut padded_dimensions_tensor =+ data_to_mtl_tensor(&self.device, &padded_dimensions_data, vec![2], DataType::F32)?;+ let mut patch_size_tensor = data_to_mtl_tensor(&self.device, ¶ms.patch_size, vec![1], DataType::F32)?;+ let mut num_channels_tensor = data_to_mtl_tensor(&self.device, ¶ms.num_channels, vec![1], DataType::F32)?;+ let mut temporal_slices_tensor =+ data_to_mtl_tensor(&self.device, ¶ms.temporal_slices, vec![1], DataType::F32)?;let _eip_tensors: Vec<&mut MetalArray> = vec![&mut *final_output_buffer_array_ref,@@ -243,10 +197,7 @@ impl MetalImagePreprocessor {let _eip_images = [intermediate_texture];let patch_params = PatchParameters {- padded_dimensions: [- requirements.padded_width,- requirements.padded_height,- ],+ padded_dimensions: [requirements.padded_width, requirements.padded_height],patch_size: params.patch_size,num_channels: params.num_channels,temporal_slices: params.temporal_slices,@@ -269,11 +220,8 @@ fn data_to_mtl_tensor<T: Copy>(data_type: DataType,) -> Result<MetalArray, MTLError> {let size = mem::size_of::<T>();- let bytes = unsafe {- std::slice::from_raw_parts(data as *const T as *const u8, size)- };- let buffer = device- .new_buffer_with_data(bytes, MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let bytes = unsafe { std::slice::from_raw_parts(data as *const T as *const u8, size) };+ let buffer =+ device.new_buffer_with_data(bytes, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");unsafe { Ok(MetalArray::from_parts(buffer, 0, &shape, data_type)) }}diff --git a/crates/uzu/src/backends/metal/metal_extensions/command_buffer_timing.rs b/crates/uzu/src/backends/metal/metal_extensions/command_buffer_timing.rsindex d3e2df9b3..623408739 100644--- a/crates/uzu/src/backends/metal/metal_extensions/command_buffer_timing.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/command_buffer_timing.rs@@ -1,6 +1,4 @@-use crate::backends::metal::{- MTLCommandBuffer, MTLCommandBufferExt, ProtocolObject,-};+use crate::backends::metal::{MTLCommandBuffer, MTLCommandBufferExt, ProtocolObject};/// Extension trait providing convenience timing methods for command buffers.pub trait CommandBufferTimingExt {diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rsindex d68a37b57..c60bb24bd 100644--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs@@ -1,9 +1,6 @@use objc2::msg_send;-use crate::backends::metal::{- MTLBuffer, MTLCompareFunction, MTLComputeCommandEncoder, ProtocolObject,- Retained,-};+use crate::backends::metal::{MTLBuffer, MTLCompareFunction, MTLComputeCommandEncoder, ProtocolObject, Retained};/// Low-level, unsafe conditional control of Metal encoders./// This is internal; users should prefer the safe `ComputeEncoderConditional::condition`.@@ -34,9 +31,7 @@ pub trait ComputeEncoderConditional {ElseBlock: FnOnce();}-impl ComputeEncoderRawConditional- for ProtocolObject<dyn MTLComputeCommandEncoder>-{+impl ComputeEncoderRawConditional for ProtocolObject<dyn MTLComputeCommandEncoder> {fn encode_start_if(&self,predicate: &ProtocolObject<dyn MTLBuffer>,diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_dispatch.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_dispatch.rsindex 5b4fb5d6c..ce11d74b8 100644--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_dispatch.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_dispatch.rs@@ -3,8 +3,7 @@ use super::{device_extensions_features::{DeviceFeatures, Feature},};use crate::backends::metal::{- MTLCommandEncoderExt, MTLComputeCommandEncoder, MTLComputePipelineState,- MTLSize, ProtocolObject,+ MTLCommandEncoderExt, MTLComputeCommandEncoder, MTLComputePipelineState, MTLSize, ProtocolObject,};/// Extensions for MTLComputeCommandEncoder to simplify dispatch operations@@ -89,8 +88,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {size: usize,threadgroup_width: Option<usize>,) {- let tg_width = threadgroup_width- .unwrap_or(state.thread_execution_width() as usize);+ let tg_width = threadgroup_width.unwrap_or(state.thread_execution_width() as usize);let tg_size = MTLSize {width: tg_width,height: 1,@@ -114,8 +112,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {threadgroup_width: Option<usize>,) {let tg_size = MTLSize {- width: threadgroup_width- .unwrap_or(state.thread_execution_width() as usize),+ width: threadgroup_width.unwrap_or(state.thread_execution_width() as usize),height: 1,depth: 1,};@@ -152,8 +149,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {size: MTLSize,threadgroup_size: Option<MTLSize>,) {- let tg_size =- threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());+ let tg_size = threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());let count = MTLSize {width: (size.width + tg_size.width - 1) / tg_size.width,@@ -171,8 +167,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {size: MTLSize,threadgroup_size: Option<MTLSize>,) {- let tg_size =- threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());+ let tg_size = threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());self.set_compute_pipeline_state(state);self.dispatch_threads(size, tg_size);@@ -199,8 +194,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {size: MTLSize,threadgroup_size: Option<MTLSize>,) {- let tg_size =- threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());+ let tg_size = threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());let count = MTLSize {width: (size.width + tg_size.width - 1) / tg_size.width,@@ -218,8 +212,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {size: MTLSize,threadgroup_size: Option<MTLSize>,) {- let tg_size =- threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());+ let tg_size = threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());self.set_compute_pipeline_state(state);self.dispatch_threads(size, tg_size);diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_set_value.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_set_value.rsindex a80e4b30a..6010fbf9a 100644--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_set_value.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_set_value.rs@@ -25,8 +25,7 @@ impl ComputeEncoderSetValue for ProtocolObject<dyn MTLComputeCommandEncoder> {value: &T,index: usize,) {- let ptr = NonNull::new(value as *const T as *mut c_void)- .expect("value pointer should never be null");+ let ptr = NonNull::new(value as *const T as *mut c_void).expect("value pointer should never be null");self.set_bytes(ptr, size_of::<T>(), index);}@@ -35,8 +34,7 @@ impl ComputeEncoderSetValue for ProtocolObject<dyn MTLComputeCommandEncoder> {slice: &[T],index: usize,) {- let ptr = NonNull::new(slice.as_ptr() as *mut c_void)- .expect("slice pointer should never be null");+ let ptr = NonNull::new(slice.as_ptr() as *mut c_void).expect("slice pointer should never be null");self.set_bytes(ptr, size_of::<T>() * slice.len(), index);}}diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_pipeline_state_extensions_threads.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_pipeline_state_extensions_threads.rsindex e1a3d803a..8ab5944f7 100644--- a/crates/uzu/src/backends/metal/metal_extensions/compute_pipeline_state_extensions_threads.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_pipeline_state_extensions_threads.rs@@ -1,6 +1,4 @@-use crate::backends::metal::{- MTLComputePipelineState, MTLSize, ProtocolObject,-};+use crate::backends::metal::{MTLComputePipelineState, MTLSize, ProtocolObject};/// Extensions for ComputePipelineState to provide optimized threadgroup sizespub trait ComputePipelineStateThreads {@@ -29,9 +27,7 @@ pub trait ComputePipelineStateThreads {) -> MTLSize;}-impl ComputePipelineStateThreads- for ProtocolObject<dyn MTLComputePipelineState>-{+impl ComputePipelineStateThreads for ProtocolObject<dyn MTLComputePipelineState> {fn execution_width_threadgroup_size(&self) -> MTLSize {let w = self.thread_execution_width();diff --git a/crates/uzu/src/backends/metal/metal_extensions/device_extensions_features.rs b/crates/uzu/src/backends/metal/metal_extensions/device_extensions_features.rsindex 778d18a42..c732839a9 100644--- a/crates/uzu/src/backends/metal/metal_extensions/device_extensions_features.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/device_extensions_features.rs@@ -1,8 +1,7 @@use std::collections::HashSet;use crate::backends::metal::{- MTLDevice, MTLDeviceExt, MTLFeatureSet, MTLGPUFamily, MTLPixelFormat,- MTLReadWriteTextureTier, ProtocolObject,+ MTLDevice, MTLDeviceExt, MTLFeatureSet, MTLGPUFamily, MTLPixelFormat, MTLReadWriteTextureTier, ProtocolObject,};/// Enum representing various Metal features that may or may not be supported by a device.@@ -44,9 +43,7 @@ impl DeviceFeatures for ProtocolObject<dyn MTLDevice> {}#[cfg(target_os = "macos")]{- self.supports_feature_set(- MTLFeatureSet::macOS_GPUFamily1_v3,- )+ self.supports_feature_set(MTLFeatureSet::macOS_GPUFamily1_v3)}#[cfg(not(any(target_os = "ios", target_os = "macos")))]{@@ -55,46 +52,37 @@ impl DeviceFeatures for ProtocolObject<dyn MTLDevice> {},Feature::TileShaders => self.supports_family(MTLGPUFamily::Apple4),Feature::ReadWriteTextures(pixel_format) => {- let tier_one_supported_formats: HashSet<MTLPixelFormat> = [- MTLPixelFormat::R32Float,- MTLPixelFormat::R32Uint,- MTLPixelFormat::R32Sint,- ]- .into();+ let tier_one_supported_formats: HashSet<MTLPixelFormat> =+ [MTLPixelFormat::R32Float, MTLPixelFormat::R32Uint, MTLPixelFormat::R32Sint].into();- let tier_two_supported_formats: HashSet<MTLPixelFormat> =- tier_one_supported_formats- .union(- &[- MTLPixelFormat::RGBA32Float,- MTLPixelFormat::RGBA32Uint,- MTLPixelFormat::RGBA32Sint,- MTLPixelFormat::RGBA16Float,- MTLPixelFormat::RGBA16Uint,- MTLPixelFormat::RGBA16Sint,- MTLPixelFormat::RGBA8Unorm,- MTLPixelFormat::RGBA8Uint,- MTLPixelFormat::RGBA8Sint,- MTLPixelFormat::R16Float,- MTLPixelFormat::R16Uint,- MTLPixelFormat::R16Sint,- MTLPixelFormat::R8Unorm,- MTLPixelFormat::R8Uint,- MTLPixelFormat::R8Sint,- ]- .into_iter()- .collect(),- )- .cloned()- .collect();+ let tier_two_supported_formats: HashSet<MTLPixelFormat> = tier_one_supported_formats+ .union(+ &[+ MTLPixelFormat::RGBA32Float,+ MTLPixelFormat::RGBA32Uint,+ MTLPixelFormat::RGBA32Sint,+ MTLPixelFormat::RGBA16Float,+ MTLPixelFormat::RGBA16Uint,+ MTLPixelFormat::RGBA16Sint,+ MTLPixelFormat::RGBA8Unorm,+ MTLPixelFormat::RGBA8Uint,+ MTLPixelFormat::RGBA8Sint,+ MTLPixelFormat::R16Float,+ MTLPixelFormat::R16Uint,+ MTLPixelFormat::R16Sint,+ MTLPixelFormat::R8Unorm,+ MTLPixelFormat::R8Uint,+ MTLPixelFormat::R8Sint,+ ]+ .into_iter()+ .collect(),+ )+ .cloned()+ .collect();match self.read_write_texture_support() {- MTLReadWriteTextureTier::Tier1 => {- tier_one_supported_formats.contains(&pixel_format)- },- MTLReadWriteTextureTier::Tier2 => {- tier_two_supported_formats.contains(&pixel_format)- },+ MTLReadWriteTextureTier::Tier1 => tier_one_supported_formats.contains(&pixel_format),+ MTLReadWriteTextureTier::Tier2 => tier_two_supported_formats.contains(&pixel_format),MTLReadWriteTextureTier::None => false,}},diff --git a/crates/uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs b/crates/uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rsindex 3ddf8c424..8fac470da 100644--- a/crates/uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs@@ -2,8 +2,7 @@ use objc2::{rc::Retained, runtime::ProtocolObject};use objc2_foundation::NSError;use crate::backends::metal::{- MTLComputePipelineState, MTLDeviceExt, MTLFunctionConstantValues,- MTLLibrary, MTLLibraryExt,+ MTLComputePipelineState, MTLDeviceExt, MTLFunctionConstantValues, MTLLibrary, MTLLibraryExt,error::{LibraryError, MTLError},};@@ -23,16 +22,11 @@ impl LibraryPipelineExtensions for ProtocolObject<dyn MTLLibrary> {&self,function_name: &str,constants: Option<&MTLFunctionConstantValues>,- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {let function = match constants {Some(const_values) => {let mut error: *mut NSError = std::ptr::null_mut();- self.new_function_with_name_constant_values_error(- function_name,- const_values,- &mut error,- )+ self.new_function_with_name_constant_values_error(function_name, const_values, &mut error)},None => self.new_function_with_name(function_name),}@@ -40,14 +34,9 @@ impl LibraryPipelineExtensions for ProtocolObject<dyn MTLLibrary> {let device = self.device();- device.new_compute_pipeline_state_with_function(&function).map_err(- |e| {- MTLError::Library(LibraryError::Custom(format!(- "Pipeline state creation failed: {}",- e- )))- },- )+ device+ .new_compute_pipeline_state_with_function(&function)+ .map_err(|e| MTLError::Library(LibraryError::Custom(format!("Pipeline state creation failed: {}", e))))}}@@ -56,8 +45,7 @@ impl LibraryPipelineExtensions for Retained<ProtocolObject<dyn MTLLibrary>> {&self,function_name: &str,constants: Option<&MTLFunctionConstantValues>,- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>- {+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {(**self).compute_pipeline_state(function_name, constants)}}diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 6ca1d398b..33d97e6ca 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -19,17 +19,12 @@ pub use crate::backends::common::kernel::kv_cache_update::KVCacheUpdate;pub use array::MetalArray;pub use backend::Metal;pub use classifier_context::ClassifierContext;-pub use context::{- DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext,-};+pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext};pub use encodable_block::Decoder;pub use error::MTLError;pub use kernel::{KernelDataType, RopeKernel, dsl::MetalKernels};pub use language_model_generator_context::LanguageModelGeneratorContext;-pub use media_utils::{- ImagePreprocessingParams, ImagePreprocessingRequirements,- MetalImagePreprocessor,-};+pub use media_utils::{ImagePreprocessingParams, ImagePreprocessingRequirements, MetalImagePreprocessor};pub use metal::prelude::*;pub use metal_extensions::ComputeEncoderSetValue;pub use placement_analysis::PlacementAnalysis;diff --git a/crates/uzu/src/backends/metal/placement_analysis.rs b/crates/uzu/src/backends/metal/placement_analysis.rsindex 488ef21c0..f0c4f24f9 100644--- a/crates/uzu/src/backends/metal/placement_analysis.rs+++ b/crates/uzu/src/backends/metal/placement_analysis.rs@@ -71,22 +71,13 @@ impl PlacementAnalysis {let mut result = Self::default();// Extract layer counts- let re_layers = Regex::new(r"Layers Count: (\d+): ANE: (\d+) \(([\d\.e\-]+)%\), GPU: (\d+) \(([\d\.e\-]+)%\)").unwrap();+ let re_layers =+ Regex::new(r"Layers Count: (\d+): ANE: (\d+) \(([\d\.e\-]+)%\), GPU: (\d+) \(([\d\.e\-]+)%\)").unwrap();if let Some(captures) = re_layers.captures(log_output) {// println!("Found layer count information");- if let (- Some(total),- Some(ane),- Some(ane_pct),- Some(gpu),- Some(gpu_pct),- ) = (- captures.get(1),- captures.get(2),- captures.get(3),- captures.get(4),- captures.get(5),- ) {+ if let (Some(total), Some(ane), Some(ane_pct), Some(gpu), Some(gpu_pct)) =+ (captures.get(1), captures.get(2), captures.get(3), captures.get(4), captures.get(5))+ {result.total_layers = total.as_str().parse().unwrap_or(0);result.ane_layers = ane.as_str().parse().unwrap_or(0);result.gpu_layers = gpu.as_str().parse().unwrap_or(0);@@ -108,14 +99,9 @@ impl PlacementAnalysis {// println!("Found unplaced operations section");// The operations section begins after "Unplaced operations:"- if let Some(ops_section) =- log_output.split("Unplaced operations:").nth(1)- {+ if let Some(ops_section) = log_output.split("Unplaced operations:").nth(1) {// The operations section ends at "Couldn't find any ANERegionCall operation."- if let Some(ops_text) = ops_section- .split("Couldn't find any ANERegionCall operation")- .next()- {+ if let Some(ops_text) = ops_section.split("Couldn't find any ANERegionCall operation").next() {// Process operations sectionlet mut current_op_type = String::new();let mut _current_op_count = 0;@@ -126,20 +112,12 @@ impl PlacementAnalysis {// Process line by linefor line in ops_text.lines() {// Line like "mps.multiply (6):" starts a new operation type section- if let Some(caps) = Regex::new(r"(\w+\.\w+) \((\d+)\):")- .unwrap()- .captures(line)- {- if let (Some(op_type), Some(count)) =- (caps.get(1), caps.get(2))- {+ if let Some(caps) = Regex::new(r"(\w+\.\w+) \((\d+)\):").unwrap().captures(line) {+ if let (Some(op_type), Some(count)) = (caps.get(1), caps.get(2)) {current_op_type = op_type.as_str().to_string();- _current_op_count =- count.as_str().parse().unwrap_or(0);+ _current_op_count = count.as_str().parse().unwrap_or(0);// Insert immediately to handle logs that omit per-op lines- let trimmed = current_op_type- .trim_start_matches("mps.")- .to_string();+ let trimmed = current_op_type.trim_start_matches("mps.").to_string();unique_ops.insert(trimmed);// println!(// "Found operation type: {} ({})",@@ -149,9 +127,7 @@ impl PlacementAnalysis {// Lines starting with "%" contain actual operation instances} else if line.trim().starts_with('%') {// Store operation type **without** "mps." prefix so tests expect plain names- let trimmed = current_op_type- .trim_start_matches("mps.")- .to_string();+ let trimmed = current_op_type.trim_start_matches("mps.").to_string();unique_ops.insert(trimmed);}}@@ -170,9 +146,7 @@ impl PlacementAnalysis {// For ANE operations, we would need to extract them if present// Extract ANERegionCall occurrences so tests can validate presence- let ane_region_re =- Regex::new(r"ANERegionCall operations|ANERegionCall operation")- .unwrap();+ let ane_region_re = Regex::new(r"ANERegionCall operations|ANERegionCall operation").unwrap();if ane_region_re.is_match(log_output) {// We don't have detailed operation names β the tests only check for a placeholder string// so insert a canonical name once if any ANERegionCall string appears.@@ -225,9 +199,7 @@ mod tests {"#;let analysis = PlacementAnalysis::from_log_output(log_output);assert_eq!(analysis.ane_layers, 3);- assert!(- analysis.ane_operations.contains(&"ANE region call".to_string())- );+ assert!(analysis.ane_operations.contains(&"ANE region call".to_string()));}#[test]@@ -245,9 +217,7 @@ mod tests {assert_eq!(analysis.gpu_layers, 5);assert!(analysis.gpu_operations.contains(&"convolution".to_string()));assert_eq!(analysis.gpu_operations.len(), 1);- assert!(- analysis.ane_operations.contains(&"ANE region call".to_string())- );+ assert!(analysis.ane_operations.contains(&"ANE region call".to_string()));}#[test]diff --git a/crates/uzu/src/backends/metal/utils.rs b/crates/uzu/src/backends/metal/utils.rsindex 46306b3b0..59ba7a8b8 100644--- a/crates/uzu/src/backends/metal/utils.rs+++ b/crates/uzu/src/backends/metal/utils.rs@@ -59,17 +59,11 @@ impl StdoutCapture {self.stdout_thread = Some(std::thread::spawn(move || {let mut buf = [0u8; 4096];loop {- let bytes_read = libc::read(- read_fd,- buf.as_mut_ptr() as *mut libc::c_void,- buf.len(),- );+ let bytes_read = libc::read(read_fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len());if bytes_read <= 0 {break;}- if let Ok(s) =- std::str::from_utf8(&buf[0..bytes_read as usize])- {+ if let Ok(s) = std::str::from_utf8(&buf[0..bytes_read as usize]) {let mut b = buffer_clone.lock().unwrap();b.push_str(s);}@@ -92,17 +86,11 @@ impl StdoutCapture {self.stderr_thread = Some(std::thread::spawn(move || {let mut buf = [0u8; 4096];loop {- let bytes_read = libc::read(- read_fd,- buf.as_mut_ptr() as *mut libc::c_void,- buf.len(),- );+ let bytes_read = libc::read(read_fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len());if bytes_read <= 0 {break;}- if let Ok(s) =- std::str::from_utf8(&buf[0..bytes_read as usize])- {+ if let Ok(s) = std::str::from_utf8(&buf[0..bytes_read as usize]) {let mut b = buffer_clone.lock().unwrap();b.push_str(s);}diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 7aad798f7..a8db95eef 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -39,8 +39,7 @@ impl Classifier {let forward_start = Instant::now();#[cfg(feature = "tracing")]- let (logits, _traces) =- self.forward_pass_with_traces(&token_ids, &token_positions)?;+ let (logits, _traces) = self.forward_pass_with_traces(&token_ids, &token_positions)?;#[cfg(not(feature = "tracing"))]let logits = self.forward_pass(&token_ids, &token_positions)?;@@ -49,8 +48,7 @@ impl Classifier {let postprocessing_start = Instant::now();let probabilities = self.logits_to_probabilities(&logits)?;- let postprocessing_duration =- postprocessing_start.elapsed().as_secs_f64();+ let postprocessing_duration = postprocessing_start.elapsed().as_secs_f64();let (predicted_label, confidence) = probabilities.iter()@@ -108,16 +106,8 @@ impl Classifier {let encoding_params = EncodingParameters::new(false, true, false);- self.context.embed.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );- self.context.embedding_norm.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );+ self.context.embed.encode(&mut state, &self.context.command_buffer, &encoding_params);+ self.context.embedding_norm.encode(&mut state, &self.context.command_buffer, &encoding_params);#[cfg(feature = "tracing")]{let traces = state.traces().clone();@@ -129,17 +119,9 @@ impl Classifier {}for layer in self.context.layers.iter() {- layer.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );+ layer.encode(&mut state, &self.context.command_buffer, &encoding_params);}- self.context.output_norm.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );+ self.context.output_norm.encode(&mut state, &self.context.command_buffer, &encoding_params);#[cfg(feature = "tracing")]{let traces = state.traces().clone();@@ -150,17 +132,9 @@ impl Classifier {);}- self.context.pooling.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );+ self.context.pooling.encode(&mut state, &self.context.command_buffer, &encoding_params);- self.context.prediction_head.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );+ self.context.prediction_head.encode(&mut state, &self.context.command_buffer, &encoding_params);self.context.command_buffer.commit();self.context.command_buffer.wait_until_completed();@@ -195,38 +169,14 @@ impl Classifier {let encoding_params = EncodingParameters::new(false, true, false);- self.context.embed.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );- self.context.embedding_norm.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );+ self.context.embed.encode(&mut state, &self.context.command_buffer, &encoding_params);+ self.context.embedding_norm.encode(&mut state, &self.context.command_buffer, &encoding_params);for layer in self.context.layers.iter() {- layer.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );+ layer.encode(&mut state, &self.context.command_buffer, &encoding_params);}- self.context.output_norm.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );- self.context.pooling.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );- self.context.prediction_head.encode(- &mut state,- &self.context.command_buffer,- &encoding_params,- );+ self.context.output_norm.encode(&mut state, &self.context.command_buffer, &encoding_params);+ self.context.pooling.encode(&mut state, &self.context.command_buffer, &encoding_params);+ self.context.prediction_head.encode(&mut state, &self.context.command_buffer, &encoding_params);self.context.command_buffer.commit();self.context.command_buffer.wait_until_completed();@@ -241,8 +191,7 @@ impl Classifier {&self,state: &ForwardPassState<Metal>,) -> Result<Box<[f32]>, Error> {- let logits_arrays =- state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);+ let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);let logits_array = logits_arrays[0].borrow();let num_labels = self.context.model_config.model_config.num_labels;@@ -255,19 +204,11 @@ impl Classifier {},DataType::F16 => {let slice: &[half::f16] = bytemuck::cast_slice(buffer);- Ok(slice[..num_labels]- .iter()- .map(|&x| x.to_f32())- .collect::<Vec<_>>()- .into_boxed_slice())+ Ok(slice[..num_labels].iter().map(|&x| x.to_f32()).collect::<Vec<_>>().into_boxed_slice())},DataType::BF16 => {let slice: &[half::bf16] = bytemuck::cast_slice(buffer);- Ok(slice[..num_labels]- .iter()- .map(|&x| x.to_f32())- .collect::<Vec<_>>()- .into_boxed_slice())+ Ok(slice[..num_labels].iter().map(|&x| x.to_f32()).collect::<Vec<_>>().into_boxed_slice())},_ => Err(Error::UnableToDecodeText),}@@ -277,18 +218,14 @@ impl Classifier {&self,logits: &[f32],) -> Result<HashMap<String, f32>, Error> {- let output_labels =- &self.context.model_config.model_config.output_labels;+ let output_labels = &self.context.model_config.model_config.output_labels;let mut probabilities = HashMap::new();for (idx, &logit) in logits.iter().enumerate() {let prob = 1.0 / (1.0 + (-logit).exp());let label = if let Some(labels) = output_labels {- labels- .get(idx)- .map(|s| s.clone())- .unwrap_or_else(|| format!("class_{}", idx))+ labels.get(idx).map(|s| s.clone()).unwrap_or_else(|| format!("class_{}", idx))} else {format!("class_{}", idx)};diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rsindex 48d0f31b3..26d1020c5 100644--- a/crates/uzu/src/classifier/mod.rs+++ b/crates/uzu/src/classifier/mod.rs@@ -7,8 +7,6 @@ pub use result::{ClassificationOutput, ClassificationStats};// Re-export context from metal backendpub use crate::backends::metal::ClassifierContext;// Re-export encodables from metal backend for convenience-pub use crate::backends::metal::encodable_block::{- ClassifierLayer, ClassifierPredictionHead, Pooling,-};+pub use crate::backends::metal::encodable_block::{ClassifierLayer, ClassifierPredictionHead, Pooling};#[cfg(feature = "tracing")]pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rsindex ca1266c11..ff720dbab 100644--- a/crates/uzu/src/config/attention.rs+++ b/crates/uzu/src/config/attention.rs@@ -108,8 +108,7 @@ mod tests {has_out_biases: false,};- let deserialized_config: AttentionConfig =- from_str(config_str).unwrap();+ let deserialized_config: AttentionConfig = from_str(config_str).unwrap();assert_eq!(deserialized_config, ground_truth_config);}}diff --git a/crates/uzu/src/config/classifier/classifier_config.rs b/crates/uzu/src/config/classifier/classifier_config.rsindex e3e30e5b3..b6d2e4f06 100644--- a/crates/uzu/src/config/classifier/classifier_config.rs+++ b/crates/uzu/src/config/classifier/classifier_config.rs@@ -2,8 +2,8 @@ use serde::{Deserialize, Serialize};use super::{PoolingType, PredictionHeadConfig};use crate::{- DecoderConfig, DecoderLayerConfig, EmbeddingConfig, LinearConfig,- NormalizationConfig, TransformerConfig, config::ConfigError,+ DecoderConfig, DecoderLayerConfig, EmbeddingConfig, LinearConfig, NormalizationConfig, TransformerConfig,+ config::ConfigError,};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]@@ -37,24 +37,15 @@ impl ClassifierConfig {pub fn to_decoder_config(&self) -> Result<DecoderConfig, ConfigError> {// For classifier, we use the first layer config as the template// (most classifiers have homogeneous layers)- let first_layer = self- .transformer_config- .layer_configs- .first()- .ok_or(ConfigError::NoLayers)?;+ let first_layer = self.transformer_config.layer_configs.first().ok_or(ConfigError::NoLayers)?;let layer_config = DecoderLayerConfig {- pre_attention_norm_config: first_layer- .pre_attention_norm_config- .clone()- .unwrap_or_else(|| {- // If first layer has no pre-attention norm (like ModernBERT), use a default- self.transformer_config.output_norm_config.clone()- }),+ pre_attention_norm_config: first_layer.pre_attention_norm_config.clone().unwrap_or_else(|| {+ // If first layer has no pre-attention norm (like ModernBERT), use a default+ self.transformer_config.output_norm_config.clone()+ }),mixer_config: first_layer.mixer_config.clone(),- post_attention_norm_config: first_layer- .post_attention_norm_config- .clone(),+ post_attention_norm_config: first_layer.post_attention_norm_config.clone(),pre_mlp_norm_config: first_layer.pre_mlp_norm_config.clone(),mlp_config: first_layer.mlp_config.clone(),post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),@@ -62,12 +53,11 @@ impl ClassifierConfig {let first_mixer = &first_layer.mixer_config;- let num_heads =- self.num_heads.or(first_mixer.num_heads()).ok_or_else(|| {- ConfigError::MissingField("num_heads".to_string())- })?;- let num_groups =- self.num_groups.or(first_mixer.num_groups()).unwrap_or(num_heads);+ let num_heads = self+ .num_heads+ .or(first_mixer.num_heads())+ .ok_or_else(|| ConfigError::MissingField("num_heads".to_string()))?;+ let num_groups = self.num_groups.or(first_mixer.num_groups()).unwrap_or(num_heads);let head_dim = self.head_dim.or(first_mixer.head_dim())@@ -75,19 +65,10 @@ impl ClassifierConfig {Ok(DecoderConfig {embedding_config: self.embedding_config.clone(),- global_rope_config: self- .transformer_config- .global_rope_config- .clone(),- local_rope_config: self- .transformer_config- .local_rope_config- .clone(),+ global_rope_config: self.transformer_config.global_rope_config.clone(),+ local_rope_config: self.transformer_config.local_rope_config.clone(),layer_config,- output_norm_config: self- .transformer_config- .output_norm_config- .clone(),+ output_norm_config: self.transformer_config.output_norm_config.clone(),vocab_size: self.vocab_size,model_dim: self.model_dim,hidden_dim: self.transformer_config.hidden_dim,diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rsindex 7b23c74f5..b9ea318f2 100644--- a/crates/uzu/src/config/decoder.rs+++ b/crates/uzu/src/config/decoder.rs@@ -77,19 +77,14 @@ impl<'de> Deserialize<'de> for DecoderConfig {context_length,} = raw;- let layer_configs_boxed =- layer_configs.map(|layers| layers.into_boxed_slice());+ let layer_configs_boxed = layer_configs.map(|layers| layers.into_boxed_slice());let layer_config_value = if let Some(config) = layer_config {config} else if let Some(configs) = layer_configs_boxed.as_ref() {- configs.first().cloned().ok_or_else(|| {- de::Error::custom("layer_configs must not be empty")- })?+ configs.first().cloned().ok_or_else(|| de::Error::custom("layer_configs must not be empty"))?} else {- return Err(de::Error::custom(- "decoder config must include layer_config or layer_configs",- ));+ return Err(de::Error::custom("decoder config must include layer_config or layer_configs"));};let num_layers_value = if let Some(value) = num_layers {@@ -97,23 +92,18 @@ impl<'de> Deserialize<'de> for DecoderConfig {} else if let Some(configs) = layer_configs_boxed.as_ref() {configs.len()} else {- return Err(de::Error::custom(- "num_layers missing and layer_configs not provided",- ));+ return Err(de::Error::custom("num_layers missing and layer_configs not provided"));};- let (num_heads_value, num_groups_value, head_dim_value) =- match (num_heads, num_groups, head_dim) {- (Some(h), Some(g), Some(d)) => (h, g, d),- _ => derive_dims_from_layer(&layer_config_value).ok_or_else(- || {- de::Error::custom(- "num_heads/num_groups/head_dim missing and \+ let (num_heads_value, num_groups_value, head_dim_value) = match (num_heads, num_groups, head_dim) {+ (Some(h), Some(g), Some(d)) => (h, g, d),+ _ => derive_dims_from_layer(&layer_config_value).ok_or_else(|| {+ de::Error::custom(+ "num_heads/num_groups/head_dim missing and \cannot be derived from layer config",- )- },- )?,- };+ )+ })?,+ };let attention_scale_value = if attention_scale.is_some() {attention_scale@@ -121,35 +111,26 @@ impl<'de> Deserialize<'de> for DecoderConfig {layer_config_value.mixer_config.attention_scale()};- let sliding_window_sizes_boxed =- if let Some(sizes) = sliding_window_sizes {- Some(sizes.into_boxed_slice())- } else if let Some(configs) = layer_configs_boxed.as_ref() {- Some(- configs- .iter()- .map(|layer| layer.mixer_config.sliding_window_size())- .collect::<Vec<_>>()- .into_boxed_slice(),- )- } else {- None- };+ let sliding_window_sizes_boxed = if let Some(sizes) = sliding_window_sizes {+ Some(sizes.into_boxed_slice())+ } else if let Some(configs) = layer_configs_boxed.as_ref() {+ Some(+ configs+ .iter()+ .map(|layer| layer.mixer_config.sliding_window_size())+ .collect::<Vec<_>>()+ .into_boxed_slice(),+ )+ } else {+ None+ };- let explicit_layer_types =- layer_types.map(|types| types.into_boxed_slice());- let derived_layer_types =- if let Some(configs) = layer_configs_boxed.as_ref() {- Some(- configs- .iter()- .map(layer_type_from_config)- .collect::<Vec<_>>()- .into_boxed_slice(),- )- } else {- None- };+ let explicit_layer_types = layer_types.map(|types| types.into_boxed_slice());+ let derived_layer_types = if let Some(configs) = layer_configs_boxed.as_ref() {+ Some(configs.iter().map(layer_type_from_config).collect::<Vec<_>>().into_boxed_slice())+ } else {+ None+ };let layer_types_value = explicit_layer_types.or(derived_layer_types);Ok(Self {@@ -206,14 +187,8 @@ struct RawDecoderConfig {context_length: usize,}-fn derive_dims_from_layer(- layer: &DecoderLayerConfig-) -> Option<(usize, usize, usize)> {- Some((- layer.mixer_config.num_heads()?,- layer.mixer_config.num_groups()?,- layer.mixer_config.head_dim()?,- ))+fn derive_dims_from_layer(layer: &DecoderLayerConfig) -> Option<(usize, usize, usize)> {+ Some((layer.mixer_config.num_heads()?, layer.mixer_config.num_groups()?, layer.mixer_config.head_dim()?))}fn layer_type_from_config(layer: &DecoderLayerConfig) -> DecoderLayerType {@@ -241,12 +216,7 @@ impl DecoderConfig {self.layer_configs.as_ref().map(|configs| {- configs.iter().any(|config| {- matches!(- config.mixer_config,- crate::config::MixerConfig::Attention(_)- )- })+ configs.iter().any(|config| matches!(config.mixer_config, crate::config::MixerConfig::Attention(_)))}).unwrap_or(true)}@@ -420,9 +390,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -432,9 +400,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -458,9 +424,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,diff --git a/crates/uzu/src/config/decoder_layer.rs b/crates/uzu/src/config/decoder_layer.rsindex 41328a208..ccb9e2aca 100644--- a/crates/uzu/src/config/decoder_layer.rs+++ b/crates/uzu/src/config/decoder_layer.rs@@ -1,8 +1,8 @@use serde::{Deserialize, Serialize};use super::{- attention::AttentionConfig, mamba::Mamba2Config, mlp::MLPConfig,- normalization::NormalizationConfig, short_conv::ShortConvConfig,+ attention::AttentionConfig, mamba::Mamba2Config, mlp::MLPConfig, normalization::NormalizationConfig,+ short_conv::ShortConvConfig,};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]@@ -205,9 +205,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -217,9 +215,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -243,9 +239,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -264,8 +258,7 @@ mod tests {post_mlp_norm_config: None,};- let deserialized_config: DecoderLayerConfig =- from_str(config_str).unwrap();+ let deserialized_config: DecoderLayerConfig = from_str(config_str).unwrap();assert_eq!(deserialized_config, ground_truth_config);}}diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rsindex 490d76e37..92ce2c57c 100644--- a/crates/uzu/src/config/embedding.rs+++ b/crates/uzu/src/config/embedding.rs@@ -120,8 +120,7 @@ mod tests {activation_precision: ConfigDataType::BFloat16,};- let deserialized_config: EmbeddingConfig =- from_str(config_str).unwrap();+ let deserialized_config: EmbeddingConfig = from_str(config_str).unwrap();assert_eq!(deserialized_config, ground_truth_config);let semi_config_str = r#"@@ -173,8 +172,7 @@ mod tests {activation_precision: ConfigDataType::BFloat16,};- let deserialized: EmbeddingConfig =- from_str(mlx_quant_untied_str).unwrap();+ let deserialized: EmbeddingConfig = from_str(mlx_quant_untied_str).unwrap();assert_eq!(deserialized, mlx_quant_untied);}}diff --git a/crates/uzu/src/config/language_model.rs b/crates/uzu/src/config/language_model.rsindex 7cdd741d6..7ae4de0bd 100644--- a/crates/uzu/src/config/language_model.rs+++ b/crates/uzu/src/config/language_model.rs@@ -1,9 +1,8 @@use serde::{Deserialize, Serialize};use crate::{- DecoderConfig, DecoderLayerConfig, DecoderLayerType, EmbeddingConfig,- GenerationConfig, MessageProcessorConfig, MixerConfig, TransformerConfig,- config::ConfigError,+ DecoderConfig, DecoderLayerConfig, DecoderLayerType, EmbeddingConfig, GenerationConfig, MessageProcessorConfig,+ MixerConfig, TransformerConfig, config::ConfigError,};struct AttentionDims {@@ -27,8 +26,7 @@ impl InnerModelConfig {pub fn to_decoder_config(&self) -> Result<DecoderConfig, ConfigError> {let tf = &self.transformer_config;- let first_layer =- tf.layer_configs.first().ok_or(ConfigError::NoLayers)?;+ let first_layer = tf.layer_configs.first().ok_or(ConfigError::NoLayers)?;let layer_config = DecoderLayerConfig {pre_attention_norm_config: first_layer@@ -36,9 +34,7 @@ impl InnerModelConfig {.clone().unwrap_or_else(|| tf.output_norm_config.clone()),mixer_config: first_layer.mixer_config.clone(),- post_attention_norm_config: first_layer- .post_attention_norm_config- .clone(),+ post_attention_norm_config: first_layer.post_attention_norm_config.clone(),pre_mlp_norm_config: first_layer.pre_mlp_norm_config.clone(),mlp_config: first_layer.mlp_config.clone(),post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),@@ -70,9 +66,7 @@ impl InnerModelConfig {.clone().unwrap_or_else(|| tf.output_norm_config.clone()),mixer_config: layer.mixer_config.clone(),- post_attention_norm_config: layer- .post_attention_norm_config- .clone(),+ post_attention_norm_config: layer.post_attention_norm_config.clone(),pre_mlp_norm_config: layer.pre_mlp_norm_config.clone(),mlp_config: layer.mlp_config.clone(),post_mlp_norm_config: layer.post_mlp_norm_config.clone(),@@ -101,9 +95,7 @@ impl InnerModelConfig {})}- fn derive_attention_dims(- tf: &TransformerConfig- ) -> Result<AttentionDims, ConfigError> {+ fn derive_attention_dims(tf: &TransformerConfig) -> Result<AttentionDims, ConfigError> {if let (Some(num_heads), Some(head_dim)) = (tf.num_heads, tf.head_dim) {return Ok(AttentionDims {num_heads,@@ -113,17 +105,9 @@ impl InnerModelConfig {});}- if let Some(attn) = tf- .layer_configs- .iter()- .find_map(|layer| layer.mixer_config.as_attention())- {- let num_heads = attn.num_heads.ok_or_else(|| {- ConfigError::MissingField("num_heads".to_string())- })?;- let head_dim = attn.head_dim.ok_or_else(|| {- ConfigError::MissingField("head_dim".to_string())- })?;+ if let Some(attn) = tf.layer_configs.iter().find_map(|layer| layer.mixer_config.as_attention()) {+ let num_heads = attn.num_heads.ok_or_else(|| ConfigError::MissingField("num_heads".to_string()))?;+ let head_dim = attn.head_dim.ok_or_else(|| ConfigError::MissingField("head_dim".to_string()))?;return Ok(AttentionDims {num_heads,num_groups: attn.num_groups.unwrap_or(num_heads),@@ -132,11 +116,7 @@ impl InnerModelConfig {});}- if let Some(mamba) = tf- .layer_configs- .iter()- .find_map(|layer| layer.mixer_config.as_mamba())- {+ if let Some(mamba) = tf.layer_configs.iter().find_map(|layer| layer.mixer_config.as_mamba()) {return Ok(AttentionDims {num_heads: mamba.num_heads,num_groups: mamba.num_groups,diff --git a/crates/uzu/src/config/linear.rs b/crates/uzu/src/config/linear.rsindex 510eeb614..db8c21645 100644--- a/crates/uzu/src/config/linear.rs+++ b/crates/uzu/src/config/linear.rs@@ -36,12 +36,8 @@ impl LinearConfig {LinearConfig::FullPrecision {precision,} => *precision,- LinearConfig::Quantized(quantization) => {- quantization.activation_precision- },- LinearConfig::MLXQuantized(quantization) => {- quantization.activation_precision- },+ LinearConfig::Quantized(quantization) => quantization.activation_precision,+ LinearConfig::MLXQuantized(quantization) => quantization.activation_precision,LinearConfig::QLoRA {quantization,..diff --git a/crates/uzu/src/config/mlp.rs b/crates/uzu/src/config/mlp.rsindex cd1699f73..5ca9af14f 100644--- a/crates/uzu/src/config/mlp.rs+++ b/crates/uzu/src/config/mlp.rs@@ -137,37 +137,34 @@ mod tests {}"#;- let ground_truth_config =- MLPConfig::MixtureOfExperts(MixtureOfExpertsConfig {- mixture_size: 32,- num_experts_per_token: 4,- routing_function: RoutingFunctionConfig::SoftmaxRouting,- router_config: LinearConfig::FullPrecision {- precision: ConfigDataType::BFloat16,- },- router_has_biases: true,- expert_config: MoeExpertConfig {- linear_config: LinearConfig::QLoRA {- quantization: QuantizationConfig {- group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),- activation_precision: ConfigDataType::BFloat16,- },- lora_rank: 16,- lora_scale: 2.0,- },- activation: Activation::SiLU {- alpha: 1.0,+ let ground_truth_config = MLPConfig::MixtureOfExperts(MixtureOfExpertsConfig {+ mixture_size: 32,+ num_experts_per_token: 4,+ routing_function: RoutingFunctionConfig::SoftmaxRouting,+ router_config: LinearConfig::FullPrecision {+ precision: ConfigDataType::BFloat16,+ },+ router_has_biases: true,+ expert_config: MoeExpertConfig {+ linear_config: LinearConfig::QLoRA {+ quantization: QuantizationConfig {+ group_size: 32,+ weight_quantization_mode: QuantizationMode::UInt4,+ activation_quantization_mode: Some(QuantizationMode::Int8),+ activation_precision: ConfigDataType::BFloat16,},- has_up_biases: true,- has_down_biases: true,- gate_clipping: [None, Some(7.0)],- up_clipping: [-6.0, 8.0],+ lora_rank: 16,+ lora_scale: 2.0,},- });+ activation: Activation::SiLU {+ alpha: 1.0,+ },+ has_up_biases: true,+ has_down_biases: true,+ gate_clipping: [None, Some(7.0)],+ up_clipping: [-6.0, 8.0],+ },+ });let deserialized_config: MLPConfig = from_str(config_str).unwrap();assert_eq!(deserialized_config, ground_truth_config);diff --git a/crates/uzu/src/config/normalization/normalization_config.rs b/crates/uzu/src/config/normalization/normalization_config.rsindex 832ebb4a9..5d07d492c 100644--- a/crates/uzu/src/config/normalization/normalization_config.rs+++ b/crates/uzu/src/config/normalization/normalization_config.rs@@ -41,8 +41,7 @@ mod tests {subtract_mean: false,};- let deserialized_config: NormalizationConfig =- from_str(config_str).unwrap();+ let deserialized_config: NormalizationConfig = from_str(config_str).unwrap();assert_eq!(deserialized_config, ground_truth_config);}}diff --git a/crates/uzu/src/config/transformer_layer.rs b/crates/uzu/src/config/transformer_layer.rsindex d6809067d..791169dcb 100644--- a/crates/uzu/src/config/transformer_layer.rs+++ b/crates/uzu/src/config/transformer_layer.rs@@ -133,9 +133,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -145,9 +143,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -171,9 +167,7 @@ mod tests {quantization: QuantizationConfig {group_size: 32,weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(- QuantizationMode::Int8,- ),+ activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -192,8 +186,7 @@ mod tests {post_mlp_norm_config: None,};- let deserialized_config: TransformerLayerConfig =- from_str(config_str).unwrap();+ let deserialized_config: TransformerLayerConfig = from_str(config_str).unwrap();assert_eq!(deserialized_config, ground_truth_config);}}diff --git a/crates/uzu/src/data_type.rs b/crates/uzu/src/data_type.rsindex e2943aafa..a0abf72e0 100644--- a/crates/uzu/src/data_type.rs+++ b/crates/uzu/src/data_type.rs@@ -44,19 +44,9 @@ impl DataType {const fn dl_data_type_code(self) -> DLDataTypeCode {match self {DataType::BF16 => DLDataTypeCode::kDLBfloat,- DataType::F16 | DataType::F32 | DataType::F64 => {- DLDataTypeCode::kDLFloat- },- DataType::I4- | DataType::I8- | DataType::I16- | DataType::I32- | DataType::I64 => DLDataTypeCode::kDLInt,- DataType::U4- | DataType::U8- | DataType::U16- | DataType::U32- | DataType::U64 => DLDataTypeCode::kDLUInt,+ DataType::F16 | DataType::F32 | DataType::F64 => DLDataTypeCode::kDLFloat,+ DataType::I4 | DataType::I8 | DataType::I16 | DataType::I32 | DataType::I64 => DLDataTypeCode::kDLInt,+ DataType::U4 | DataType::U8 | DataType::U16 | DataType::U32 | DataType::U64 => DLDataTypeCode::kDLUInt,}}}diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 87e85ed6f..96b16788b 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -70,8 +70,6 @@ pub trait EncodableBlock<B: Backend> {command_buffer: &B::CommandBuffer,parameters: &EncodingParameters<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.encode_with_shared_encoder(state, encoder, parameters)- });+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, encoder, parameters));}}diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex 51c775ace..2fdf2d843 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -22,12 +22,7 @@ impl<B: Backend> Sampling<B> {max_batch_size: usize,max_vocab_size: usize,) -> Result<Self, SamplingError<B>> {- let kernel = SamplingKernel::new(- context,- data_type,- max_batch_size,- max_vocab_size,- )?;+ let kernel = SamplingKernel::new(context, data_type, max_batch_size, max_vocab_size)?;Ok(Self {kernel,})@@ -40,13 +35,8 @@ impl<B: Backend> Sampling<B> {max_vocab_size: usize,argmax_strategy: ArgmaxStrategy,) -> Result<Self, SamplingError<B>> {- let kernel = SamplingKernel::new_with_strategy(- context,- data_type,- max_batch_size,- max_vocab_size,- argmax_strategy,- )?;+ let kernel =+ SamplingKernel::new_with_strategy(context, data_type, max_batch_size, max_vocab_size, argmax_strategy)?;Ok(Self {kernel,})@@ -60,10 +50,7 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {encoder: &B::ComputeEncoder,_parameters: &EncodingParameters<B>,) {- assert!(- state.sampling_output().is_some(),- "Sampling output buffer must be pre-allocated"- );+ assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");let logits_binding = state.arrays(&[ArrayId::Logits]);let logits = logits_binding[0].borrow();@@ -82,19 +69,14 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {let output_buffer_ref = state.sampling_output().unwrap().borrow();let sampling_method = state.sampling_method().unwrap();- let seeds_offset =- seeds.offset() + sampling_start * std::mem::size_of::<u64>();+ let seeds_offset = seeds.offset() + sampling_start * std::mem::size_of::<u64>();- let (bitmask_buffer, bitmask_offset) =- state.token_bitmask().map_or((None, 0usize), |cell| {- let bitmask = cell.borrow();- let bitmask_row_len = bitmask.shape()[1];- let bitmask_offset = bitmask.offset()- + sampling_start- * bitmask_row_len- * std::mem::size_of::<u32>();- (Some(bitmask.buffer().clone()), bitmask_offset)- });+ let (bitmask_buffer, bitmask_offset) = state.token_bitmask().map_or((None, 0usize), |cell| {+ let bitmask = cell.borrow();+ let bitmask_row_len = bitmask.shape()[1];+ let bitmask_offset = bitmask.offset() + sampling_start * bitmask_row_len * std::mem::size_of::<u32>();+ (Some(bitmask.buffer().clone()), bitmask_offset)+ });if let Err(e) = self.kernel.encode_with_encoder(logits.buffer(),Some(seeds.buffer()),diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rsindex a95909ae8..87d8b82df 100644--- a/crates/uzu/src/forward_pass/cache_layers.rs+++ b/crates/uzu/src/forward_pass/cache_layers.rs@@ -8,10 +8,7 @@ use crate::{backends::common::{Backend, kernel::kv_cache_update::KVCacheUpdate},config::DecoderLayerType,forward_pass::{- kv_cache_layer::{- AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer,- KVCacheLayerState, KVSlice,- },+ kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState, KVSlice},model_shape::ModelShape,short_conv_layer::ShortConvLayer,ssm_layer::SSMLayer,@@ -34,14 +31,10 @@ pub enum CacheLayerSlice<B: Backend> {const ARRAY_TRANSFORMER_KEYS_LABEL: &str = "cache_layers_transformer_keys";const ARRAY_TRANSFORMER_VALUES_LABEL: &str = "cache_layers_transformer_values";-const ARRAY_STATE_SPACE_CONV_STATE_LABEL: &str =- "cache_layers_state_space_conv_state";-const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str =- "cache_layers_state_space_ssm_state";-const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str =- "cache_layers_short_conv_conv_state";-const ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL: &str =- "cache_layers_short_conv_suffix_state";+const ARRAY_STATE_SPACE_CONV_STATE_LABEL: &str = "cache_layers_state_space_conv_state";+const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str = "cache_layers_state_space_ssm_state";+const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str = "cache_layers_short_conv_conv_state";+const ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL: &str = "cache_layers_short_conv_suffix_state";impl<B: Backend> CacheLayer<B> {pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {@@ -106,9 +99,8 @@ impl<B: Backend> CacheLayers<B> {max_suffix_length: usize,) -> Self {let total_context_length = max_prefix_length.max(max_suffix_length);- let kv_shapes: Vec<[usize; 3]> = model_shape- .kv_cache_layer_shapes(max_prefix_length, max_suffix_length)- .collect();+ let kv_shapes: Vec<[usize; 3]> =+ model_shape.kv_cache_layer_shapes(max_prefix_length, max_suffix_length).collect();let data: Box<[CacheLayer<B>]> = model_shape.layer_types()@@ -117,11 +109,8 @@ impl<B: Backend> CacheLayers<B> {.map(|(layer_index, layer_type)| match layer_type {DecoderLayerType::Transformer => {let shape = kv_shapes[layer_index];- let window_length = model_shape- .sliding_window_length_per_layer[layer_index]- .filter(|&window_size| {- window_size < total_context_length- });+ let window_length = model_shape.sliding_window_length_per_layer[layer_index]+ .filter(|&window_size| window_size < total_context_length);let state = if let Some(w) = window_length {KVCacheLayerState::Windowed {@@ -140,16 +129,12 @@ impl<B: Backend> CacheLayers<B> {keys: RefCell::new(context.create_array(&shape,model_shape.kv_cache_data_type(),- &format!(- "{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),)),values: RefCell::new(context.create_array(&shape,model_shape.kv_cache_data_type(),- &format!(- "{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),)),prefix_token_positions: match &state {KVCacheLayerState::Full {@@ -158,9 +143,7 @@ impl<B: Backend> CacheLayers<B> {KVCacheLayerState::Windowed {window_length,..- } => (0..*window_length)- .map(|_| INVALID_POSITION)- .collect(),+ } => (0..*window_length).map(|_| INVALID_POSITION).collect(),},max_suffix_length,})@@ -181,47 +164,33 @@ impl<B: Backend> CacheLayers<B> {conv_state: RefCell::new(context.create_array(&conv_shape,dtype,- &format!(- "{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),)),ssm_state: RefCell::new(context.create_array(&ssm_shape,dtype,- &format!(- "{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),)),})},DecoderLayerType::ShortConv {kernel_size,} => {- let conv_shape = [- model_shape.model_dim(),- kernel_size.saturating_sub(1),- ];- let suffix_state_shape = [- max_suffix_length,- model_shape.model_dim(),- kernel_size.saturating_sub(1),- ];+ let conv_shape = [model_shape.model_dim(), kernel_size.saturating_sub(1)];+ let suffix_state_shape =+ [max_suffix_length, model_shape.model_dim(), kernel_size.saturating_sub(1)];let dtype = model_shape.activation_data_type();CacheLayer::ShortConv(ShortConvLayer {conv_state: RefCell::new(context.create_array(&conv_shape,dtype,- &format!(- "{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),)),suffix_state: RefCell::new(context.create_array(&suffix_state_shape,dtype,- &format!(- "{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),)),suffix_state_valid_start: Cell::new(0),suffix_state_valid_len: Cell::new(0),@@ -281,12 +250,7 @@ impl<B: Backend> CacheLayers<B> {for layer in self.data.iter() {if let CacheLayer::Transformer(layer) = layer {if let Some(array) = dst.get_mut(&layer.window_length()) {- layer.fill_attention_bias(- array,- suffix_token_positions,- suffix_length,- external_bias_fn,- );+ layer.fill_attention_bias(array, suffix_token_positions, suffix_length, external_bias_fn);}}}@@ -302,12 +266,7 @@ impl<B: Backend> CacheLayers<B> {for layer in self.data.iter() {if let CacheLayer::Transformer(layer) = layer {if let Some(cell) = dst.get(&layer.window_length()) {- layer.fill_attention_bias(- &mut cell.borrow_mut(),- suffix_token_positions,- suffix_length,- None,- );+ layer.fill_attention_bias(&mut cell.borrow_mut(), suffix_token_positions, suffix_length, None);}}}@@ -320,19 +279,12 @@ impl<B: Backend> CacheLayers<B> {command_buffer: &B::CommandBuffer,kv_cache_update: &KVCacheUpdate<B>,) {- let short_conv_commit_index =- accepted_suffix_indices.last().copied().unwrap_or(0);+ let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);for layer in self.data.iter_mut() {if let Some(layer) = layer.as_transformer_mut() {- layer.update_after_acceptance(- accepted_suffix_indices,- suffix_start,- command_buffer,- kv_cache_update,- );+ layer.update_after_acceptance(accepted_suffix_indices, suffix_start, command_buffer, kv_cache_update);} else if let Some(layer) = layer.as_short_conv() {- layer- .commit_from_suffix_state_if_valid(short_conv_commit_index);+ layer.commit_from_suffix_state_if_valid(short_conv_commit_index);}}}@@ -355,9 +307,7 @@ impl<B: Backend> CacheLayers<B> {self.data.iter().filter_map(|layer| match layer {- CacheLayer::Transformer(kv) => {- kv.attention_bias_update_after_acceptance(accepted_len)- },+ CacheLayer::Transformer(kv) => kv.attention_bias_update_after_acceptance(accepted_len),_ => None,}).collect()@@ -377,12 +327,8 @@ impl<B: Backend> CacheLayers<B> {};layers.push(CacheLayerSlice::Transformer(slice));},- CacheLayer::StateSpace(_) => {- layers.push(CacheLayerSlice::StateSpace)- },- CacheLayer::ShortConv(_) => {- layers.push(CacheLayerSlice::ShortConv)- },+ CacheLayer::StateSpace(_) => layers.push(CacheLayerSlice::StateSpace),+ CacheLayer::ShortConv(_) => layers.push(CacheLayerSlice::ShortConv),}}@@ -398,10 +344,7 @@ impl<B: Backend> CacheLayers<B> {) {for (layer, snapshot) in self.data.iter_mut().zip(slice.layers.iter()) {match (layer, snapshot) {- (- CacheLayer::Transformer(kv),- CacheLayerSlice::Transformer(s),- ) => {+ (CacheLayer::Transformer(kv), CacheLayerSlice::Transformer(s)) => {kv.apply_slice(&s, range.clone());},(CacheLayer::StateSpace(_), CacheLayerSlice::StateSpace) => {},@@ -437,16 +380,12 @@ impl<B: Backend> CacheLayers<B> {let mut new_keys = context.create_array(&new_shape,dtype,- &format!(- "{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),);let mut new_values = context.create_array(&new_shape,dtype,- &format!(- "{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),);if copy_rows > 0 {@@ -464,9 +403,7 @@ impl<B: Backend> CacheLayers<B> {state: layer.state.clone(),keys: RefCell::new(new_keys),values: RefCell::new(new_values),- prefix_token_positions: layer- .prefix_token_positions- .clone(),+ prefix_token_positions: layer.prefix_token_positions.clone(),max_suffix_length: layer.max_suffix_length,})},@@ -476,9 +413,7 @@ impl<B: Backend> CacheLayers<B> {let mut new_conv = context.create_array(&conv_shape,conv_dtype,- &format!(- "{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),);{let conv_src = layer.conv_state.borrow();@@ -490,9 +425,7 @@ impl<B: Backend> CacheLayers<B> {let mut new_ssm = context.create_array(&ssm_shape,ssm_dtype,- &format!(- "{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),);{let ssm_src = layer.ssm_state.borrow();@@ -510,24 +443,19 @@ impl<B: Backend> CacheLayers<B> {let mut new_conv = context.create_array(&conv_shape,conv_dtype,- &format!(- "{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),);{let conv_src = layer.conv_state.borrow();new_conv.copy_from_array(&conv_src);}- let suffix_shape =- layer.suffix_state.borrow().shape().to_vec();+ let suffix_shape = layer.suffix_state.borrow().shape().to_vec();let suffix_dtype = layer.suffix_state.borrow().data_type();let mut new_suffix = context.create_array(&suffix_shape,suffix_dtype,- &format!(- "{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"- ),+ &format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),);{new_suffix.as_bytes_mut().fill(0);diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rsindex 37348a84e..5851bc3ce 100644--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs@@ -109,22 +109,13 @@ impl<B: Backend> KVCacheLayer<B> {external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,) {let prefix_segment_length = self.prefix_segment_length();- fill_attention_bias(- dst,- suffix_length,- prefix_segment_length,- |row_index, column_index| {- if let Some(bias_fn) = external_bias_fn {- bias_fn(row_index, column_index)- } else {- self.bias_should_be_neg_inf(- row_index,- column_index,- suffix_token_positions,- )- }- },- );+ fill_attention_bias(dst, suffix_length, prefix_segment_length, |row_index, column_index| {+ if let Some(bias_fn) = external_bias_fn {+ bias_fn(row_index, column_index)+ } else {+ self.bias_should_be_neg_inf(row_index, column_index, suffix_token_positions)+ }+ });}pub fn bias_should_be_neg_inf(@@ -184,22 +175,14 @@ impl<B: Backend> KVCacheLayer<B> {}// Absolute positions of the *source* rows.- let source_indices: Vec<usize> = accepted_suffix_indices- .iter()- .map(|i| i + suffix_start.unwrap_or(*prefix_len))- .collect();+ let source_indices: Vec<usize> =+ accepted_suffix_indices.iter().map(|i| i + suffix_start.unwrap_or(*prefix_len)).collect();// Absolute positions of the *destination* rows.- let destination_indices: Vec<usize> = (*prefix_len- ..*prefix_len + accepted_suffix_indices.len())- .collect();+ let destination_indices: Vec<usize> =+ (*prefix_len..*prefix_len + accepted_suffix_indices.len()).collect();- self.scatter_if_required(- &source_indices,- &destination_indices,- command_buffer,- kv_cache_update,- );+ self.scatter_if_required(&source_indices, &destination_indices, command_buffer, kv_cache_update);},KVCacheLayerState::Windowed {@@ -207,31 +190,21 @@ impl<B: Backend> KVCacheLayer<B> {ring_length,window_length,} => {- let suffix_indices: Vec<usize> =- if accepted_suffix_indices.is_empty() {- vec![0]- } else {- accepted_suffix_indices.to_vec()- };+ let suffix_indices: Vec<usize> = if accepted_suffix_indices.is_empty() {+ vec![0]+ } else {+ accepted_suffix_indices.to_vec()+ };- let source_indices: Vec<usize> =- suffix_indices.iter().map(|i| i + *window_length).collect();+ let source_indices: Vec<usize> = suffix_indices.iter().map(|i| i + *window_length).collect();- let mut destination_indices =- Vec::with_capacity(suffix_indices.len());+ let mut destination_indices = Vec::with_capacity(suffix_indices.len());for i in 0..suffix_indices.len() {- destination_indices.push(- (*ring_length + *ring_offset + i) % *window_length,- );+ destination_indices.push((*ring_length + *ring_offset + i) % *window_length);}- self.scatter_if_required(- &source_indices,- &destination_indices,- command_buffer,- kv_cache_update,- );+ self.scatter_if_required(&source_indices, &destination_indices, command_buffer, kv_cache_update);},}}@@ -260,12 +233,7 @@ impl<B: Backend> KVCacheLayer<B> {value_shape: [v_shape[0], v_shape[1], v_shape[2]],};- let _ = kv_cache_update.encode(- &[layer_data],- source_indices,- destination_indices,- command_buffer,- );+ let _ = kv_cache_update.encode(&[layer_data], source_indices, destination_indices, command_buffer);}pub fn register_accepted_tokens(@@ -286,8 +254,7 @@ impl<B: Backend> KVCacheLayer<B> {} => {for &token_pos in token_positions {if *ring_length < *window_length {- let dst =- (*ring_offset + *ring_length) % *window_length;+ let dst = (*ring_offset + *ring_length) % *window_length;self.prefix_token_positions[dst] = token_pos;*ring_length += 1;@@ -319,17 +286,10 @@ impl<B: Backend> KVCacheLayer<B> {window_length,} => {let newest_slot = (ring_length > 0)- .then_some(- (ring_offset + ring_length + window_length - 1)- % window_length,- )+ .then_some((ring_offset + ring_length + window_length - 1) % window_length).unwrap_or(0);- let unmask_col = (ring_length > 0)- .then_some(newest_slot as i32)- .unwrap_or(-1);- let mask_col = (ring_length == window_length)- .then_some(ring_offset as i32)- .unwrap_or(-1);+ let unmask_col = (ring_length > 0).then_some(newest_slot as i32).unwrap_or(-1);+ let mask_col = (ring_length == window_length).then_some(ring_offset as i32).unwrap_or(-1);Some(AttentionBiasUpdate {key: Some(window_length),@@ -370,16 +330,8 @@ impl<B: Backend> KVCacheLayer<B> {let dtype = keys.data_type();let slice_shape = [num_groups, len, head_dim];- let mut slice_keys = context.create_array(- &slice_shape,- dtype,- "kv_cache_layer_slice_keys",- );- let mut slice_values = context.create_array(- &slice_shape,- dtype,- "kv_cache_layer_slice_values",- );+ let mut slice_keys = context.create_array(&slice_shape, dtype, "kv_cache_layer_slice_keys");+ let mut slice_values = context.create_array(&slice_shape, dtype, "kv_cache_layer_slice_values");let slots: Vec<usize> = (range.start..range.end).enumerate()@@ -393,10 +345,7 @@ impl<B: Backend> KVCacheLayer<B> {}).collect();- let positions: Vec<usize> = slots- .iter()- .map(|&s| self.prefix_token_positions[s])- .collect();+ let positions: Vec<usize> = slots.iter().map(|&s| self.prefix_token_positions[s]).collect();for (i, &slot) in slots.iter().enumerate() {slice_keys.copy_slice(&keys, 1, slot..slot + 1, i);@@ -440,8 +389,7 @@ impl<B: Backend> KVCacheLayer<B> {Some(r) => {let accepted = r.start;*prefix_len = base_prefix_len.saturating_add(accepted);- let keep_positions =- base_positions_len.saturating_add(accepted);+ let keep_positions = base_positions_len.saturating_add(accepted);self.prefix_token_positions.truncate(keep_positions);},},@@ -508,20 +456,9 @@ impl<B: Backend> KVCacheLayer<B> {for (i, &slot) in slots[r.clone()].iter().enumerate() {let src_i = r.start + i;- self.prefix_token_positions[slot] =- positions[src_i];- dst_keys.copy_slice(- keys,- 1,- src_i..src_i + 1,- slot,- );- dst_values.copy_slice(- values,- 1,- src_i..src_i + 1,- slot,- );+ self.prefix_token_positions[slot] = positions[src_i];+ dst_keys.copy_slice(keys, 1, src_i..src_i + 1, slot);+ dst_values.copy_slice(values, 1, src_i..src_i + 1, slot);}},}diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rsindex 32b32c0e5..12f0a3985 100644--- a/crates/uzu/src/forward_pass/model_shape.rs+++ b/crates/uzu/src/forward_pass/model_shape.rs@@ -29,19 +29,14 @@ pub struct ModelShape {impl ModelShape {pub fn from_decoder_config(decoder_config: &DecoderConfig) -> Self {- let activation_type: DataType =- match &decoder_config.layer_config.mlp_config {- crate::config::MLPConfig::Dense(d) => {- d.linear_config.activation_precision().into()- },- crate::config::MLPConfig::MixtureOfExperts(m) => {- m.expert_config.linear_config.activation_precision().into()- },- };+ let activation_type: DataType = match &decoder_config.layer_config.mlp_config {+ crate::config::MLPConfig::Dense(d) => d.linear_config.activation_precision().into(),+ crate::config::MLPConfig::MixtureOfExperts(m) => {+ m.expert_config.linear_config.activation_precision().into()+ },+ };let num_layers = decoder_config.num_layers;- let layer_types: Box<[DecoderLayerType]> = if let Some(layer_types) =- &decoder_config.layer_types- {+ let layer_types: Box<[DecoderLayerType]> = if let Some(layer_types) = &decoder_config.layer_types {assert_eq!(layer_types.len(),num_layers,@@ -75,8 +70,7 @@ impl ModelShape {max_mamba_head_dim = max_mamba_head_dim.max(*head_dim);max_mamba_conv_dim = max_mamba_conv_dim.max(*conv_dim);max_mamba_state_dim = max_mamba_state_dim.max(*state_dim);- max_mamba_kernel_size =- max_mamba_kernel_size.max(*kernel_size as usize);+ max_mamba_kernel_size = max_mamba_kernel_size.max(*kernel_size as usize);}}Self {@@ -368,9 +362,7 @@ impl ModelShape {}pub fn has_short_conv_layers(&self) -> bool {- self.layer_types.iter().any(|layer_type| {- matches!(layer_type, DecoderLayerType::ShortConv { .. })- })+ self.layer_types.iter().any(|layer_type| matches!(layer_type, DecoderLayerType::ShortConv { .. }))}pub fn max_mamba_conv_dim(&self) -> Option<usize> {@@ -409,9 +401,7 @@ impl ModelShape {suffix_length: usize,) -> Option<[usize; 2]> {match (self.max_mamba_conv_dim(), self.max_mamba_kernel_size()) {- (Some(conv_dim), Some(kernel_size)) if kernel_size > 0 => {- Some([suffix_length + kernel_size - 1, conv_dim])- },+ (Some(conv_dim), Some(kernel_size)) if kernel_size > 0 => Some([suffix_length + kernel_size - 1, conv_dim]),_ => None,}}@@ -474,9 +464,7 @@ impl ModelShape {suffix_length: usize,) -> Option<[usize; 3]> {if self.has_state_space_layers() {- self.max_mamba_state_dim().map(|state_dim| {- [suffix_length, self.max_mamba_groups, state_dim]- })+ self.max_mamba_state_dim().map(|state_dim| [suffix_length, self.max_mamba_groups, state_dim])} else {None}diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rsindex a203e0f31..e19a04dd9 100644--- a/crates/uzu/src/forward_pass/scratch_buffers.rs+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs@@ -46,8 +46,8 @@ pub struct ScratchBuffers<B: Backend> {// 2-pass attention intermediate bufferspub attention_partials: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count * head_dim]- pub attention_sums: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count]- pub attention_maxs: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count]+ pub attention_sums: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count]+ pub attention_maxs: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count]pub moe_topk_ids: Option<ArrayCell<B>>,pub moe_topk_probs: Option<ArrayCell<B>>,@@ -79,15 +79,10 @@ impl<B: Backend> ScratchBuffers<B> {max_suffix_len: usize,) -> Self {// Helper closure for allocation- let alloc =- |shape: &[usize], dtype: DataType, label: &str| -> ArrayCell<B> {- let array = context.create_array(- shape,- dtype,- &format!("scratch_buffers_{label}"),- );- RefCell::new(array)- };+ let alloc = |shape: &[usize], dtype: DataType, label: &str| -> ArrayCell<B> {+ let array = context.create_array(shape, dtype, &format!("scratch_buffers_{label}"));+ RefCell::new(array)+ };let act_ty = model_shape.activation_data_type();@@ -96,8 +91,7 @@ impl<B: Backend> ScratchBuffers<B> {MLPConfig::MixtureOfExperts(moe) => Some(moe),_ => None,};- let moe_max_routed =- moe.map(|moe| max_suffix_len * moe.num_experts_per_token);+ let moe_max_routed = moe.map(|moe| max_suffix_len * moe.num_experts_per_token);let moe_scatter_entries = moe.map(|moe| {let num_blocks = ((max_suffix_len + 255) / 256).max(1);let num_tiles = ((moe.mixture_size + 512 - 1) / 512).max(1);@@ -107,27 +101,11 @@ impl<B: Backend> ScratchBuffers<B> {Self {// 1-Dtoken_ids: alloc(&[max_suffix_len], DataType::U64, "token_ids"),- token_positions: alloc(- &[max_suffix_len],- DataType::I32,- "token_positions",- ),- token_parents: alloc(- &[max_suffix_len],- DataType::I32,- "token_parents",- ),- token_bitmask: alloc(- &model_shape.bitmask_shape(max_suffix_len),- DataType::U32,- "token_bitmask",- ),+ token_positions: alloc(&[max_suffix_len], DataType::I32, "token_positions"),+ token_parents: alloc(&[max_suffix_len], DataType::I32, "token_parents"),+ token_bitmask: alloc(&model_shape.bitmask_shape(max_suffix_len), DataType::U32, "token_bitmask"),token_seeds: alloc(&[max_suffix_len], DataType::U64, "token_seeds"),- sampling_output: alloc(- &[max_suffix_len],- DataType::U32,- "sampling_output",- ),+ sampling_output: alloc(&[max_suffix_len], DataType::U32, "sampling_output"),// 2-Dattention_window_size_to_bias: model_shape@@ -141,89 +119,36 @@ impl<B: Backend> ScratchBuffers<B> {Some(ws) => {format!("attention_bias_for_window_size_{ws}")},- None => {- "attention_bias_for_window_size_none".to_string()- },+ None => "attention_bias_for_window_size_none".to_string(),};- let shape =- [max_suffix_len, max_suffix_len + max_prefix_len];+ let shape = [max_suffix_len, max_suffix_len + max_prefix_len];(window_size, alloc(&shape, act_ty, &label))}).collect(),- logits: alloc(- &model_shape.logits_shape(max_suffix_len),- act_ty,- "logits",- ),- main: alloc(- &model_shape.main_shape(max_suffix_len),- act_ty,- "main",- ),- shortcut: alloc(- &model_shape.main_shape(max_suffix_len),- act_ty,- "shortcut",- ),+ logits: alloc(&model_shape.logits_shape(max_suffix_len), act_ty, "logits"),+ main: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "main"),+ shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "shortcut"),qkv: alloc(&model_shape.qkv_shape(max_suffix_len), act_ty, "qkv"),- attention_output: alloc(- &model_shape.attention_output_shape(max_suffix_len),- act_ty,- "attention_output",- ),- mlp_fused_up: alloc(- &model_shape.mlp_fused_up_shape(max_suffix_len),- act_ty,- "mlp_fused_up",- ),- mlp_hidden: alloc(- &model_shape.mlp_hidden_shape(max_suffix_len),- act_ty,- "mlp_hidden",- ),- ssm_inproj: model_shape- .ssm_inproj_shape(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "ssm_inproj")),- ssm_packed: model_shape- .ssm_packed_shape(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "ssm_packed")),+ attention_output: alloc(&model_shape.attention_output_shape(max_suffix_len), act_ty, "attention_output"),+ mlp_fused_up: alloc(&model_shape.mlp_fused_up_shape(max_suffix_len), act_ty, "mlp_fused_up"),+ mlp_hidden: alloc(&model_shape.mlp_hidden_shape(max_suffix_len), act_ty, "mlp_hidden"),+ ssm_inproj: model_shape.ssm_inproj_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_inproj")),+ ssm_packed: model_shape.ssm_packed_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_packed")),ssm_conv_padded: model_shape.ssm_conv_padded_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_conv_padded")),short_conv_padded: model_shape.short_conv_padded_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "short_conv_padded")),- ssm_x: model_shape- .ssm_x_shape(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "ssm_x")),- ssm_b: model_shape- .ssm_bc_shape(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "ssm_b")),- ssm_c: model_shape- .ssm_bc_shape(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "ssm_c")),- ssm_dt: model_shape- .ssm_dt_shape(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "ssm_dt")),- ssm_z: model_shape- .ssm_z_shape(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "ssm_z")),+ ssm_x: model_shape.ssm_x_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_x")),+ ssm_b: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_b")),+ ssm_c: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_c")),+ ssm_dt: model_shape.ssm_dt_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_dt")),+ ssm_z: model_shape.ssm_z_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_z")),// 3-D- rotated_queries: alloc(- &model_shape.rotated_queries_shape(max_suffix_len),- act_ty,- "rotated_queries",- ),- rotated_keys: alloc(- &model_shape.rotated_keys_shape(max_suffix_len),- act_ty,- "rotated_keys",- ),- extracted_values: alloc(- &model_shape.extracted_values_shape(max_suffix_len),- act_ty,- "extracted_values",- ),+ rotated_queries: alloc(&model_shape.rotated_queries_shape(max_suffix_len), act_ty, "rotated_queries"),+ rotated_keys: alloc(&model_shape.rotated_keys_shape(max_suffix_len), act_ty, "rotated_keys"),+ extracted_values: alloc(&model_shape.extracted_values_shape(max_suffix_len), act_ty, "extracted_values"),attention_partials: alloc(&model_shape.attention_partials_shape(max_suffix_len),@@ -235,130 +160,56 @@ impl<B: Backend> ScratchBuffers<B> {moe_topk_ids: moe.map(|moe| {alloc(- &model_shape.moe_topk_ids_shape(- max_suffix_len,- moe.num_experts_per_token,- ),+ &model_shape.moe_topk_ids_shape(max_suffix_len, moe.num_experts_per_token),DataType::U32,"moe_topk_ids",)}),moe_topk_probs: moe.map(|moe| {alloc(- &model_shape.moe_topk_probs_shape(- max_suffix_len,- moe.num_experts_per_token,- ),+ &model_shape.moe_topk_probs_shape(max_suffix_len, moe.num_experts_per_token),act_ty,"moe_topk_probs",)}),- moe_offsets: moe.map(|moe| {- alloc(- &model_shape.moe_offsets_shape(moe.mixture_size),- DataType::U32,- "moe_offsets",- )- }),- moe_sumk: moe.map(|_| {- alloc(&model_shape.moe_sumk_shape(), DataType::U32, "moe_sumk")- }),+ moe_offsets: moe+ .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.mixture_size), DataType::U32, "moe_offsets")),+ moe_sumk: moe.map(|_| alloc(&model_shape.moe_sumk_shape(), DataType::U32, "moe_sumk")),moe_bucketed_token_ids: moe_max_routed.map(|max_routed| {- alloc(- &model_shape.moe_bucketed_token_ids_shape(max_routed),- DataType::U32,- "moe_bucketed_token_ids",- )+ alloc(&model_shape.moe_bucketed_token_ids_shape(max_routed), DataType::U32, "moe_bucketed_token_ids")}),moe_bucketed_probs: moe_max_routed.map(|max_routed| {- alloc(- &model_shape.moe_bucketed_probs_shape(max_routed),- act_ty,- "moe_bucketed_probs",- )- }),- moe_x_perm: moe_max_routed.map(|max_routed| {- alloc(- &model_shape.moe_x_perm_shape(max_routed),- DataType::F16,- "moe_x_perm",- )+ alloc(&model_shape.moe_bucketed_probs_shape(max_routed), act_ty, "moe_bucketed_probs")}),+ moe_x_perm: moe_max_routed+ .map(|max_routed| alloc(&model_shape.moe_x_perm_shape(max_routed), DataType::F16, "moe_x_perm")),moe_tok2row: moe.map(|moe| {alloc(- &model_shape.moe_tok2row_shape(- max_suffix_len,- moe.num_experts_per_token,- ),+ &model_shape.moe_tok2row_shape(max_suffix_len, moe.num_experts_per_token),DataType::I32,"moe_tok2row",)}),- moe_y_partial: moe_max_routed.map(|max_routed| {- alloc(- &model_shape.moe_y_partial_shape(max_routed),- DataType::F16,- "moe_y_partial",- )- }),- moe_hidden: moe_max_routed.map(|max_routed| {- alloc(- &model_shape.moe_hidden_shape(max_routed),- DataType::F32,- "moe_hidden",- )- }),- moe_two_pass_row_expert_map: moe_max_routed.map(|max_routed| {- alloc(- &[max_routed],- DataType::U32,- "moe_two_pass_row_expert_map",- )- }),- moe_tile_counts: moe.map(|moe| {- alloc(- &model_shape.moe_counts_shape(moe.mixture_size),- DataType::U32,- "moe_tile_counts",- )- }),- moe_tile_offsets: moe.map(|moe| {- alloc(- &model_shape.moe_offsets_shape(moe.mixture_size),- DataType::U32,- "moe_tile_offsets",- )- }),- moe_tile_map: moe_max_routed.map(|max_routed| {- alloc(- &model_shape.moe_tile_map_shape(max_routed),- DataType::U32,- "moe_tile_map",- )- }),- moe_total_tiles: moe.map(|_| {- alloc(- &model_shape.moe_total_tiles_shape(),- DataType::U32,- "moe_total_tiles",- )- }),- moe_dispatch_args: moe.map(|_| {- alloc(- &model_shape.moe_dispatch_args_shape(),- DataType::U32,- "moe_dispatch_args",- )- }),- moe_scatter_partials: moe_scatter_entries.map(|entries| {- alloc(&[entries], DataType::U32, "moe_scatter_partials")- }),- moe_scatter_block_bases: moe_scatter_entries.map(|entries| {- alloc(&[entries], DataType::U32, "moe_scatter_block_bases")- }),- moe_block_alloc: moe_scatter_entries.map(|entries| {- alloc(&[entries], DataType::U32, "moe_block_alloc")- }),+ moe_y_partial: moe_max_routed+ .map(|max_routed| alloc(&model_shape.moe_y_partial_shape(max_routed), DataType::F16, "moe_y_partial")),+ moe_hidden: moe_max_routed+ .map(|max_routed| alloc(&model_shape.moe_hidden_shape(max_routed), DataType::F32, "moe_hidden")),+ moe_two_pass_row_expert_map: moe_max_routed+ .map(|max_routed| alloc(&[max_routed], DataType::U32, "moe_two_pass_row_expert_map")),+ moe_tile_counts: moe+ .map(|moe| alloc(&model_shape.moe_counts_shape(moe.mixture_size), DataType::U32, "moe_tile_counts")),+ moe_tile_offsets: moe+ .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.mixture_size), DataType::U32, "moe_tile_offsets")),+ moe_tile_map: moe_max_routed+ .map(|max_routed| alloc(&model_shape.moe_tile_map_shape(max_routed), DataType::U32, "moe_tile_map")),+ moe_total_tiles: moe.map(|_| alloc(&model_shape.moe_total_tiles_shape(), DataType::U32, "moe_total_tiles")),+ moe_dispatch_args: moe+ .map(|_| alloc(&model_shape.moe_dispatch_args_shape(), DataType::U32, "moe_dispatch_args")),+ moe_scatter_partials: moe_scatter_entries+ .map(|entries| alloc(&[entries], DataType::U32, "moe_scatter_partials")),+ moe_scatter_block_bases: moe_scatter_entries+ .map(|entries| alloc(&[entries], DataType::U32, "moe_scatter_block_bases")),+ moe_block_alloc: moe_scatter_entries.map(|entries| alloc(&[entries], DataType::U32, "moe_block_alloc")),}}}diff --git a/crates/uzu/src/forward_pass/short_conv_layer.rs b/crates/uzu/src/forward_pass/short_conv_layer.rsindex b01cfeea4..8cba95a33 100644--- a/crates/uzu/src/forward_pass/short_conv_layer.rs+++ b/crates/uzu/src/forward_pass/short_conv_layer.rs@@ -62,28 +62,17 @@ impl<B: Backend> ShortConvLayer<B> {let mut conv = self.conv_state.borrow_mut();let suffix = self.suffix_state.borrow();- assert_eq!(- conv.data_type(),- suffix.data_type(),- "ShortConv conv_state / suffix_state dtype mismatch"- );+ assert_eq!(conv.data_type(), suffix.data_type(), "ShortConv conv_state / suffix_state dtype mismatch");let [model_dim, state_stride] = {let shape = conv.shape();- assert_eq!(- shape.len(),- 2,- "ShortConv conv_state expected 2-D [model_dim, state_stride], got {:?}",- shape- );+ assert_eq!(shape.len(), 2, "ShortConv conv_state expected 2-D [model_dim, state_stride], got {:?}", shape);[shape[0], shape[1]]};let suffix_shape = suffix.shape();assert!(- suffix_shape.len() == 3- && suffix_shape[1] == model_dim- && suffix_shape[2] == state_stride,+ suffix_shape.len() == 3 && suffix_shape[1] == model_dim && suffix_shape[2] == state_stride,"ShortConv suffix_state expected 3-D [suffix_len, model_dim, state_stride], got {:?}",suffix_shape);@@ -95,8 +84,7 @@ impl<B: Backend> ShortConvLayer<B> {);let elem_bytes = conv.data_type().size_in_bytes();- let bytes_per_token =- model_dim.saturating_mul(state_stride).saturating_mul(elem_bytes);+ let bytes_per_token = model_dim.saturating_mul(state_stride).saturating_mul(elem_bytes);let src_start = commit_index.saturating_mul(bytes_per_token);let src_end = src_start.saturating_add(bytes_per_token);diff --git a/crates/uzu/src/forward_pass/state/array_id.rs b/crates/uzu/src/forward_pass/state/array_id.rsindex 4a07ef521..b0b6af703 100644--- a/crates/uzu/src/forward_pass/state/array_id.rs+++ b/crates/uzu/src/forward_pass/state/array_id.rs@@ -2,18 +2,7 @@ use serde::{Deserialize, Serialize};use super::RopeType;-#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- PartialOrd,- Ord,- Hash,- Serialize,- Deserialize,-)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]pub enum ArrayId {TokenIds,TokenPositions,diff --git a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/forward_pass/state/common_aux_buffers.rsindex 0323594ad..7c7b38beb 100644--- a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs+++ b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs@@ -29,37 +29,17 @@ impl<B: Backend> CommonAuxBuffers<B> {Self {suffix_length,main: scratch.main.view(&model_shape.main_shape(suffix_length)),- shortcut: scratch- .shortcut- .view(&model_shape.main_shape(suffix_length)),+ shortcut: scratch.shortcut.view(&model_shape.main_shape(suffix_length)),qkv: scratch.qkv.view(&model_shape.qkv_shape(suffix_length)),- attention_output: scratch- .attention_output- .view(&model_shape.attention_output_shape(suffix_length)),- mlp_fused_up: scratch- .mlp_fused_up- .view(&model_shape.mlp_fused_up_shape(suffix_length)),- mlp_hidden: scratch- .mlp_hidden- .view(&model_shape.mlp_hidden_shape(suffix_length)),- rotated_queries: scratch- .rotated_queries- .view(&model_shape.rotated_queries_shape(suffix_length)),- rotated_keys: scratch- .rotated_keys- .view(&model_shape.rotated_keys_shape(suffix_length)),- extracted_values: scratch- .extracted_values- .view(&model_shape.extracted_values_shape(suffix_length)),- attention_partials: scratch- .attention_partials- .view(&model_shape.attention_partials_shape(suffix_length)),- attention_sums: scratch- .attention_sums- .view(&model_shape.attention_sums_shape(suffix_length)),- attention_maxs: scratch- .attention_maxs- .view(&model_shape.attention_maxs_shape(suffix_length)),+ attention_output: scratch.attention_output.view(&model_shape.attention_output_shape(suffix_length)),+ mlp_fused_up: scratch.mlp_fused_up.view(&model_shape.mlp_fused_up_shape(suffix_length)),+ mlp_hidden: scratch.mlp_hidden.view(&model_shape.mlp_hidden_shape(suffix_length)),+ rotated_queries: scratch.rotated_queries.view(&model_shape.rotated_queries_shape(suffix_length)),+ rotated_keys: scratch.rotated_keys.view(&model_shape.rotated_keys_shape(suffix_length)),+ extracted_values: scratch.extracted_values.view(&model_shape.extracted_values_shape(suffix_length)),+ attention_partials: scratch.attention_partials.view(&model_shape.attention_partials_shape(suffix_length)),+ attention_sums: scratch.attention_sums.view(&model_shape.attention_sums_shape(suffix_length)),+ attention_maxs: scratch.attention_maxs.view(&model_shape.attention_maxs_shape(suffix_length)),}}}diff --git a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rsindex 27bfedd02..47cdde3b4 100644--- a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs+++ b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs@@ -49,14 +49,10 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {MLPConfig::MixtureOfExperts(moe) => Some(moe),_ => None,};- let max_routed =- moe.map(|moe| suffix_length * moe.num_experts_per_token);+ let max_routed = moe.map(|moe| suffix_length * moe.num_experts_per_token);let num_blocks = suffix_length.div_ceil(256).max(1);- let scatter_entries = moe.map(|moe| {- num_blocks * moe.mixture_size.div_ceil(512).max(1) * 512- });- let block_alloc_entries =- moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1));+ let scatter_entries = moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1) * 512);+ let block_alloc_entries = moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1));Self {ssm_inproj: scratch@@ -104,83 +100,46 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {.as_ref().zip(model_shape.ssm_z_shape(suffix_length)).map(|(buf, shape)| buf.view(&shape)),- moe_topk_ids: moe.zip(scratch.moe_topk_ids.as_ref()).map(- |(moe, buf)| {- buf.view(&model_shape.moe_topk_ids_shape(- suffix_length,- moe.num_experts_per_token,- ))- },- ),- moe_topk_probs: moe.zip(scratch.moe_topk_probs.as_ref()).map(- |(moe, buf)| {- buf.view(&model_shape.moe_topk_probs_shape(- suffix_length,- moe.num_experts_per_token,- ))- },- ),- moe_offsets: moe.zip(scratch.moe_offsets.as_ref()).map(- |(moe, buf)| {- buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))- },- ),- moe_sumk: moe- .and(scratch.moe_sumk.as_ref())- .map(|buf| buf.view(&model_shape.moe_sumk_shape())),+ moe_topk_ids: moe+ .zip(scratch.moe_topk_ids.as_ref())+ .map(|(moe, buf)| buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_experts_per_token))),+ moe_topk_probs: moe.zip(scratch.moe_topk_probs.as_ref()).map(|(moe, buf)| {+ buf.view(&model_shape.moe_topk_probs_shape(suffix_length, moe.num_experts_per_token))+ }),+ moe_offsets: moe+ .zip(scratch.moe_offsets.as_ref())+ .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))),+ moe_sumk: moe.and(scratch.moe_sumk.as_ref()).map(|buf| buf.view(&model_shape.moe_sumk_shape())),moe_bucketed_token_ids: max_routed.zip(scratch.moe_bucketed_token_ids.as_ref())- .map(|(max_routed, buf)| {- buf.view(- &model_shape.moe_bucketed_token_ids_shape(max_routed),- )- }),+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_bucketed_token_ids_shape(max_routed))),moe_bucketed_probs: max_routed.zip(scratch.moe_bucketed_probs.as_ref())- .map(|(max_routed, buf)| {- buf.view(&model_shape.moe_bucketed_probs_shape(max_routed))- }),- moe_x_perm: max_routed.zip(scratch.moe_x_perm.as_ref()).map(- |(max_routed, buf)| {- buf.view(&model_shape.moe_x_perm_shape(max_routed))- },- ),- moe_tok2row: moe.zip(scratch.moe_tok2row.as_ref()).map(- |(moe, buf)| {- buf.view(&model_shape.moe_tok2row_shape(- suffix_length,- moe.num_experts_per_token,- ))- },- ),- moe_y_partial: max_routed.zip(scratch.moe_y_partial.as_ref()).map(- |(max_routed, buf)| {- buf.view(&model_shape.moe_y_partial_shape(max_routed))- },- ),- moe_hidden: max_routed.zip(scratch.moe_hidden.as_ref()).map(- |(max_routed, buf)| {- buf.view(&model_shape.moe_hidden_shape(max_routed))- },- ),+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_bucketed_probs_shape(max_routed))),+ moe_x_perm: max_routed+ .zip(scratch.moe_x_perm.as_ref())+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_x_perm_shape(max_routed))),+ moe_tok2row: moe+ .zip(scratch.moe_tok2row.as_ref())+ .map(|(moe, buf)| buf.view(&model_shape.moe_tok2row_shape(suffix_length, moe.num_experts_per_token))),+ moe_y_partial: max_routed+ .zip(scratch.moe_y_partial.as_ref())+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_y_partial_shape(max_routed))),+ moe_hidden: max_routed+ .zip(scratch.moe_hidden.as_ref())+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_hidden_shape(max_routed))),moe_two_pass_row_expert_map: max_routed.zip(scratch.moe_two_pass_row_expert_map.as_ref()).map(|(max_routed, buf)| buf.view(&[max_routed])),- moe_tile_counts: moe.zip(scratch.moe_tile_counts.as_ref()).map(- |(moe, buf)| {- buf.view(&model_shape.moe_counts_shape(moe.mixture_size))- },- ),- moe_tile_offsets: moe.zip(scratch.moe_tile_offsets.as_ref()).map(- |(moe, buf)| {- buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))- },- ),- moe_tile_map: max_routed.zip(scratch.moe_tile_map.as_ref()).map(- |(max_routed, buf)| {- buf.view(&model_shape.moe_tile_map_shape(max_routed))- },- ),+ moe_tile_counts: moe+ .zip(scratch.moe_tile_counts.as_ref())+ .map(|(moe, buf)| buf.view(&model_shape.moe_counts_shape(moe.mixture_size))),+ moe_tile_offsets: moe+ .zip(scratch.moe_tile_offsets.as_ref())+ .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))),+ moe_tile_map: max_routed+ .zip(scratch.moe_tile_map.as_ref())+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_tile_map_shape(max_routed))),moe_total_tiles: moe.and(scratch.moe_total_tiles.as_ref()).map(|buf| buf.view(&model_shape.moe_total_tiles_shape())),diff --git a/crates/uzu/src/forward_pass/state/mod.rs b/crates/uzu/src/forward_pass/state/mod.rsindex 3aab46cde..efe1bbfc7 100644--- a/crates/uzu/src/forward_pass/state/mod.rs+++ b/crates/uzu/src/forward_pass/state/mod.rs@@ -18,7 +18,4 @@ pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;pub use rope_buffers::RopeBuffers;pub use shared_buffers::SharedBuffers;-pub use state::{- ClassifierModeState, ForwardPassMode, ForwardPassState,- LanguageModelGeneratorModeState,-};+pub use state::{ClassifierModeState, ForwardPassMode, ForwardPassState, LanguageModelGeneratorModeState};diff --git a/crates/uzu/src/forward_pass/state/rope_type.rs b/crates/uzu/src/forward_pass/state/rope_type.rsindex ea16a8877..374238c75 100644--- a/crates/uzu/src/forward_pass/state/rope_type.rs+++ b/crates/uzu/src/forward_pass/state/rope_type.rs@@ -1,17 +1,6 @@use serde::{Deserialize, Serialize};-#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- PartialOrd,- Ord,- Hash,- Serialize,- Deserialize,-)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]pub enum RopeType {Global,Local,diff --git a/crates/uzu/src/forward_pass/state/shared_buffers.rs b/crates/uzu/src/forward_pass/state/shared_buffers.rsindex fd7db21fb..bb1bb3bb3 100644--- a/crates/uzu/src/forward_pass/state/shared_buffers.rs+++ b/crates/uzu/src/forward_pass/state/shared_buffers.rs@@ -25,32 +25,22 @@ impl<B: Backend> SharedBuffers<B> {decoder_config: &DecoderConfig,model_shape: &ModelShape,) -> Self {- let global_rope = decoder_config- .global_rope_config- .is_some()- .then(|| RopeBuffers::new(context, model_shape));+ let global_rope = decoder_config.global_rope_config.is_some().then(|| RopeBuffers::new(context, model_shape));- let local_rope = decoder_config- .local_rope_config- .is_some()- .then(|| RopeBuffers::new(context, model_shape));+ let local_rope = decoder_config.local_rope_config.is_some().then(|| RopeBuffers::new(context, model_shape));- let attention_sinks = decoder_config- .layer_config- .attention_config()- .is_some_and(|c| c.has_sinks)- .then(|| {- let num_heads = decoder_config.num_heads;- (0..decoder_config.num_layers)- .map(|_| {- RefCell::new(context.create_array_uninitialized(- &[num_heads],- DataType::F32,- "shared_buffers_attention_sinks",- ))- })- .collect()- });+ let attention_sinks = decoder_config.layer_config.attention_config().is_some_and(|c| c.has_sinks).then(|| {+ let num_heads = decoder_config.num_heads;+ (0..decoder_config.num_layers)+ .map(|_| {+ RefCell::new(context.create_array_uninitialized(+ &[num_heads],+ DataType::F32,+ "shared_buffers_attention_sinks",+ ))+ })+ .collect()+ });Self {global_rope,@@ -63,9 +53,7 @@ impl<B: Backend> SharedBuffers<B> {&mut self,parameter_tree: &ParameterTree<B::Context>,) {- let transformer_tree = parameter_tree- .subtree("transformer")- .expect("transformer subtree not found");+ let transformer_tree = parameter_tree.subtree("transformer").expect("transformer subtree not found");if let Some(global_rope) = &mut self.global_rope {global_rope.update_data(&transformer_tree, "global_rope");@@ -76,9 +64,7 @@ impl<B: Backend> SharedBuffers<B> {if let Some(sinks_vec) = &mut self.attention_sinks {for (layer_idx, sink_cell) in sinks_vec.iter_mut().enumerate() {- let layer_tree = transformer_tree- .subtree(&format!("layers.{}", layer_idx))- .unwrap();+ let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_idx)).unwrap();let attn_tree = layer_tree.subtree("mixer").unwrap();let sinks_arr = attn_tree.leaf("sinks").unwrap();let mut dst = sink_cell.borrow_mut();@@ -91,25 +77,18 @@ impl<B: Backend> SharedBuffers<B> {},DataType::BF16 => {let src = sinks_arr.as_slice::<bf16>();- for (dst_val, src_val) in- dst_slice.iter_mut().zip(src.iter())- {+ for (dst_val, src_val) in dst_slice.iter_mut().zip(src.iter()) {*dst_val = f32::from(*src_val);}},DataType::F16 => {let src = sinks_arr.as_slice::<f16>();- for (dst_val, src_val) in- dst_slice.iter_mut().zip(src.iter())- {+ for (dst_val, src_val) in dst_slice.iter_mut().zip(src.iter()) {*dst_val = f32::from(*src_val);}},other => {- panic!(- "Unsupported attention sink data type: {:?}",- other- );+ panic!("Unsupported attention sink data type: {:?}", other);},}}diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex 19070e80c..9bd0fc6dc 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -11,10 +11,7 @@ use crate::{cache_layers::CacheLayers,model_shape::ModelShape,scratch_buffers::ScratchBuffers,- state::{- ArrayId, CommonAuxBuffers, HashMapId,- LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers,- },+ state::{ArrayId, CommonAuxBuffers, HashMapId, LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers},},session::parameter::SamplingMethod,utils::attention::fill_attention_bias,@@ -81,13 +78,9 @@ impl<B: Backend> ForwardPassState<B> {token_positions: &[usize],) -> ArrayCell<B> {let suffix_length = token_positions.len();- let token_positions_array =- scratch.token_positions.view(&[suffix_length]);- let token_positions_i32: Box<[i32]> =- token_positions.iter().map(|p| *p as i32).collect();- token_positions_array- .borrow_mut()- .copy_from_view(token_positions_i32.as_ref().into());+ let token_positions_array = scratch.token_positions.view(&[suffix_length]);+ let token_positions_i32: Box<[i32]> = token_positions.iter().map(|p| *p as i32).collect();+ token_positions_array.borrow_mut().copy_from_view(token_positions_i32.as_ref().into());token_positions_array}@@ -105,8 +98,7 @@ impl<B: Backend> ForwardPassState<B> {parents.fill(-1);if sampling_length > 0 {- let root_pos =- token_positions.get(sampling_start).copied().unwrap_or(0);+ let root_pos = token_positions.get(sampling_start).copied().unwrap_or(0);let mut stack: Vec<usize> = Vec::new();@@ -122,11 +114,7 @@ impl<B: Backend> ForwardPassState<B> {} else if let Some(&p) = stack.get(depth - 1) {p as i32} else {- debug_assert!(- false,- "invalid trie depth ordering: depth={depth}, stack_len={}",- stack.len()- );+ debug_assert!(false, "invalid trie depth ordering: depth={depth}, stack_len={}", stack.len());-1};parents[abs_idx] = parent_local;@@ -169,15 +157,8 @@ impl<B: Backend> ForwardPassState<B> {async_seeds: Option<(&B::NativeBuffer, usize)>,) -> Self {let suffix_length = token_ids.len();- assert_eq!(- suffix_length,- token_positions.len(),- "Tokens and positions must have same length"- );- assert!(- suffix_length <= cache_layers.borrow().max_suffix_length(),- "Suffix length exceeds KV cache capacity"- );+ assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");+ assert!(suffix_length <= cache_layers.borrow().max_suffix_length(), "Suffix length exceeds KV cache capacity");// Token IDs - optionally skip copy for async pathlet token_ids_cell = if skip_token_ids_copy {@@ -187,29 +168,23 @@ impl<B: Backend> ForwardPassState<B> {};// Token positions - use async buffer if provided- let token_positions_cell =- if let Some((async_buf, offset)) = async_positions {- let array = unsafe {- Array::from_parts(- async_buf.clone(),- offset * std::mem::size_of::<i32>(),- &[suffix_length],- DataType::I32,- )- };- RefCell::new(array)- } else {- Self::init_token_positions(scratch, token_positions)+ let token_positions_cell = if let Some((async_buf, offset)) = async_positions {+ let array = unsafe {+ Array::from_parts(+ async_buf.clone(),+ offset * std::mem::size_of::<i32>(),+ &[suffix_length],+ DataType::I32,+ )};+ RefCell::new(array)+ } else {+ Self::init_token_positions(scratch, token_positions)+ };// Trie parent indices (relative, within the sampling segment).// Only meaningful when sampling_length > 0.- let token_parents_cell = Self::init_token_parents(- scratch,- token_positions,- sampling_start,- sampling_length,- );+ let token_parents_cell = Self::init_token_parents(scratch, token_positions, sampling_start, sampling_length);// Token bitmasklet token_bitmask_cell = token_bitmask.map(|bitmask| {@@ -238,12 +213,10 @@ impl<B: Backend> ForwardPassState<B> {};// Logits- let logits_cell =- scratch.logits.view(&model_shape.logits_shape(suffix_length));+ let logits_cell = scratch.logits.view(&model_shape.logits_shape(suffix_length));// Sampling output- let sampling_output =- Some(scratch.sampling_output.view(&[suffix_length]));+ let sampling_output = Some(scratch.sampling_output.view(&[suffix_length]));// Attention bias (causal + sliding window)let attention_bias = Self::init_llm_attention_bias(@@ -256,40 +229,28 @@ impl<B: Backend> ForwardPassState<B> {);// Common aux buffers- let common_aux =- CommonAuxBuffers::new(scratch, model_shape, suffix_length);+ let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);// LLM-specific aux buffers- let llm_aux = Some(LanguageModelGeneratorAuxBuffers::new(- scratch,- decoder_config,- model_shape,- suffix_length,- ));+ let llm_aux = Some(LanguageModelGeneratorAuxBuffers::new(scratch, decoder_config, model_shape, suffix_length));// Traces#[cfg(feature = "tracing")]- let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(- context.as_ref(),- model_shape,- suffix_length,- )));-- let mode = ForwardPassMode::LanguageModelGenerator(- LanguageModelGeneratorModeState {- cache_layers,- token_seeds: token_seeds_cell,- logits: logits_cell,- sampling_output,- sampling_method: None,- #[cfg(feature = "tracing")]- traces,- active_suffix_length,- sampling_start,- sampling_length,- is_prefilling,- },- );+ let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(context.as_ref(), model_shape, suffix_length)));++ let mode = ForwardPassMode::LanguageModelGenerator(LanguageModelGeneratorModeState {+ cache_layers,+ token_seeds: token_seeds_cell,+ logits: logits_cell,+ sampling_output,+ sampling_method: None,+ #[cfg(feature = "tracing")]+ traces,+ active_suffix_length,+ sampling_start,+ sampling_length,+ is_prefilling,+ });Self {context,token_ids: token_ids_cell,@@ -317,10 +278,8 @@ impl<B: Backend> ForwardPassState<B> {.attention_window_size_to_bias.iter().map(|(window_size, buffer)| {- let prefix_length =- window_size.unwrap_or(cache_ref.max_prefix_length());- let attention_bias_shape =- [suffix_length, suffix_length + prefix_length];+ let prefix_length = window_size.unwrap_or(cache_ref.max_prefix_length());+ let attention_bias_shape = [suffix_length, suffix_length + prefix_length];let array = buffer.borrow().view(&attention_bias_shape);(*window_size, array)})@@ -339,10 +298,7 @@ impl<B: Backend> ForwardPassState<B> {);}- attention_bias_map- .into_iter()- .map(|(k, v)| (k, RefCell::new(v)))- .collect()+ attention_bias_map.into_iter().map(|(k, v)| (k, RefCell::new(v))).collect()}// ========================================================================@@ -364,31 +320,19 @@ impl<B: Backend> ForwardPassState<B> {assert_eq!(suffix_length, token_positions.len());let token_ids_cell = Self::init_token_ids(scratch, token_ids);- let token_positions_cell =- Self::init_token_positions(scratch, token_positions);- let token_parents_cell =- Self::init_token_parents(scratch, token_positions, 0, 0);+ let token_positions_cell = Self::init_token_positions(scratch, token_positions);+ let token_parents_cell = Self::init_token_parents(scratch, token_positions, 0, 0);// Attention bias (bidirectional or causal)- let attention_bias = Self::init_classifier_attention_bias(- scratch,- suffix_length,- bidirectional_attention,- );+ let attention_bias = Self::init_classifier_attention_bias(scratch, suffix_length, bidirectional_attention);// Common aux buffers- let common_aux =- CommonAuxBuffers::new(scratch, model_shape, suffix_length);+ let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);// Classifier-specific bufferslet model_dim = model_shape.main_shape(1)[1];- let classifier_state = Self::init_classifier_buffers(- context.as_ref(),- model_shape,- model_dim,- num_labels,- suffix_length,- );+ let classifier_state =+ Self::init_classifier_buffers(context.as_ref(), model_shape, model_dim, num_labels, suffix_length);let mode = ForwardPassMode::Classifier(classifier_state);@@ -425,37 +369,19 @@ impl<B: Backend> ForwardPassState<B> {if bidirectional_attention {if let Some(window_size) = window {let half_window = (window_size / 2) as isize;- fill_attention_bias(- bias_array,- suffix_length,- 0,- |row, col| {- let distance = (row as isize) - (col as isize);- distance.abs() > half_window- },- );+ fill_attention_bias(bias_array, suffix_length, 0, |row, col| {+ let distance = (row as isize) - (col as isize);+ distance.abs() > half_window+ });} else {- fill_attention_bias(- bias_array,- suffix_length,- 0,- |_row, _col| false,- );+ fill_attention_bias(bias_array, suffix_length, 0, |_row, _col| false);}} else {- fill_attention_bias(- bias_array,- suffix_length,- 0,- |row, col| row < col,- );+ fill_attention_bias(bias_array, suffix_length, 0, |row, col| row < col);}}- attention_bias_map- .into_iter()- .map(|(k, v)| (k, RefCell::new(v)))- .collect()+ attention_bias_map.into_iter().map(|(k, v)| (k, RefCell::new(v))).collect()}#[cfg(feature = "tracing")]@@ -471,17 +397,8 @@ impl<B: Backend> ForwardPassState<B> {let create_buffer = |size: usize| -> ArrayCell<B> {let buffer_size = size * data_type.size_in_bytes();- let buffer = context- .create_buffer(buffer_size)- .expect("Failed to create buffer");- RefCell::new(unsafe {- Array::from_parts(- buffer,- 0,- &[batch_size, size / batch_size],- data_type,- )- })+ let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");+ RefCell::new(unsafe { Array::from_parts(buffer, 0, &[batch_size, size / batch_size], data_type) })};ClassifierModeState {@@ -489,19 +406,9 @@ impl<B: Backend> ForwardPassState<B> {dense: create_buffer(batch_size * model_dim),norm: create_buffer(batch_size * model_dim),classifier_logits: {- let buffer_size =- batch_size * num_labels * data_type.size_in_bytes();- let buffer = context- .create_buffer(buffer_size)- .expect("Failed to create buffer");- RefCell::new(unsafe {- Array::from_parts(- buffer,- 0,- &[batch_size, num_labels],- data_type,- )- })+ let buffer_size = batch_size * num_labels * data_type.size_in_bytes();+ let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");+ RefCell::new(unsafe { Array::from_parts(buffer, 0, &[batch_size, num_labels], data_type) })},traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(context,@@ -526,12 +433,8 @@ impl<B: Backend> ForwardPassState<B> {let create_buffer = |dims: &[usize]| -> ArrayCell<B> {let size: usize = dims.iter().product();let buffer_size = size * data_type.size_in_bytes();- let buffer = context- .create_buffer(buffer_size)- .expect("Failed to create buffer");- RefCell::new(unsafe {- Array::from_parts(buffer, 0, dims, data_type)- })+ let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");+ RefCell::new(unsafe { Array::from_parts(buffer, 0, dims, data_type) })};ClassifierModeState {@@ -596,150 +499,88 @@ impl<B: Backend> ForwardPassState<B> {ArrayId::TokenIds => self.token_ids.clone(),ArrayId::TokenPositions => self.token_positions.clone(),ArrayId::TokenParents => self.token_parents.clone(),- ArrayId::TokenBitmask => {- self.token_bitmask.clone().expect("Token bitmask not available")- },+ ArrayId::TokenBitmask => self.token_bitmask.clone().expect("Token bitmask not available"),ArrayId::Main => self.common_aux.main.clone(),ArrayId::Shortcut => self.common_aux.shortcut.clone(),ArrayId::QKV => self.common_aux.qkv.clone(),- ArrayId::AttentionOutput => {- self.common_aux.attention_output.clone()- },+ ArrayId::AttentionOutput => self.common_aux.attention_output.clone(),ArrayId::MlpFusedUp => self.common_aux.mlp_fused_up.clone(),ArrayId::MlpHidden => self.common_aux.mlp_hidden.clone(),ArrayId::RotatedQueries => self.common_aux.rotated_queries.clone(),ArrayId::RotatedKeys => self.common_aux.rotated_keys.clone(),- ArrayId::ExtractedValues => {- self.common_aux.extracted_values.clone()- },- ArrayId::AttentionPartials => {- self.common_aux.attention_partials.clone()- },+ ArrayId::ExtractedValues => self.common_aux.extracted_values.clone(),+ ArrayId::AttentionPartials => self.common_aux.attention_partials.clone(),ArrayId::AttentionSums => self.common_aux.attention_sums.clone(),ArrayId::AttentionMaxs => self.common_aux.attention_maxs.clone(),// Shared buffer arrays- ArrayId::RopeCosines(_) | ArrayId::RopeSines(_) => self- .shared_buffer_array(id)- .expect("Shared buffer array should be available"),+ ArrayId::RopeCosines(_) | ArrayId::RopeSines(_) => {+ self.shared_buffer_array(id).expect("Shared buffer array should be available")+ },// LLM-specific arraysArrayId::Logits => self.llm_state().logits.clone(),ArrayId::TokenSeeds => self.llm_state().token_seeds.clone(),ArrayId::Keys(layer_index) => {let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index]- .as_transformer()- .expect("Expected transformer layer")- .keys- .clone()+ cache.data[layer_index].as_transformer().expect("Expected transformer layer").keys.clone()},ArrayId::Values(layer_index) => {let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index]- .as_transformer()- .expect("Expected transformer layer")- .values+ cache.data[layer_index].as_transformer().expect("Expected transformer layer").values.clone()+ },+ ArrayId::AttentionSinks(layer_index) => {+ self.shared_buffers.borrow().attention_sinks.as_ref().expect("Attention sinks not initialized")+ [layer_index].clone()},- ArrayId::AttentionSinks(layer_index) => self- .shared_buffers- .borrow()- .attention_sinks- .as_ref()- .expect("Attention sinks not initialized")[layer_index]- .clone(),// SSM arrays (LLM only)- ArrayId::SsmInProj => self- .llm_aux- .as_ref()- .and_then(|aux| aux.ssm_inproj.clone())- .expect("SSM inproj not initialized"),- ArrayId::SsmPacked(_) => self- .llm_aux- .as_ref()- .and_then(|aux| aux.ssm_packed.clone())- .expect("SSM packed not initialized"),+ ArrayId::SsmInProj => {+ self.llm_aux.as_ref().and_then(|aux| aux.ssm_inproj.clone()).expect("SSM inproj not initialized")+ },+ ArrayId::SsmPacked(_) => {+ self.llm_aux.as_ref().and_then(|aux| aux.ssm_packed.clone()).expect("SSM packed not initialized")+ },ArrayId::SsmConvState(layer_index) => {let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index]- .as_state_space()- .expect("Expected SSM layer")- .conv_state- .clone()+ cache.data[layer_index].as_state_space().expect("Expected SSM layer").conv_state.clone()},ArrayId::SsmState(layer_index) => {let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index]- .as_state_space()- .expect("Expected SSM layer")- .ssm_state- .clone()+ cache.data[layer_index].as_state_space().expect("Expected SSM layer").ssm_state.clone()},- ArrayId::SsmX(_) => self- .llm_aux- .as_ref()- .and_then(|aux| aux.ssm_x.clone())- .expect("SSM x not initialized"),- ArrayId::SsmB(_) => self- .llm_aux- .as_ref()- .and_then(|aux| aux.ssm_b.clone())- .expect("SSM b not initialized"),- ArrayId::SsmC(_) => self- .llm_aux- .as_ref()- .and_then(|aux| aux.ssm_c.clone())- .expect("SSM c not initialized"),- ArrayId::SsmDt(_) => self- .llm_aux- .as_ref()- .and_then(|aux| aux.ssm_dt.clone())- .expect("SSM dt not initialized"),- ArrayId::SsmZ(_) => self- .llm_aux- .as_ref()- .and_then(|aux| aux.ssm_z.clone())- .expect("SSM z not initialized"),+ ArrayId::SsmX(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_x.clone()).expect("SSM x not initialized"),+ ArrayId::SsmB(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_b.clone()).expect("SSM b not initialized"),+ ArrayId::SsmC(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_c.clone()).expect("SSM c not initialized"),+ ArrayId::SsmDt(_) => {+ self.llm_aux.as_ref().and_then(|aux| aux.ssm_dt.clone()).expect("SSM dt not initialized")+ },+ ArrayId::SsmZ(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_z.clone()).expect("SSM z not initialized"),ArrayId::ShortConvState(layer_index) => {let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index]- .as_short_conv()- .expect("Expected ShortConv layer")- .conv_state- .clone()+ cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").conv_state.clone()},ArrayId::ShortConvSuffixState(layer_index) => {let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index]- .as_short_conv()- .expect("Expected ShortConv layer")- .suffix_state- .clone()+ cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").suffix_state.clone()},// MoE arrays (LLM only)- ArrayId::MoeTopkIds => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_topk_ids.clone())- .expect("MoE topk_ids not initialized"),+ ArrayId::MoeTopkIds => {+ self.llm_aux.as_ref().and_then(|aux| aux.moe_topk_ids.clone()).expect("MoE topk_ids not initialized")+ },ArrayId::MoeTopkProbs => self.llm_aux.as_ref().and_then(|aux| aux.moe_topk_probs.clone()).expect("MoE topk_probs not initialized"),- ArrayId::MoeOffsets => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_offsets.clone())- .expect("MoE offsets not initialized"),- ArrayId::MoeSumK => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_sumk.clone())- .expect("MoE sumk not initialized"),+ ArrayId::MoeOffsets => {+ self.llm_aux.as_ref().and_then(|aux| aux.moe_offsets.clone()).expect("MoE offsets not initialized")+ },+ ArrayId::MoeSumK => {+ self.llm_aux.as_ref().and_then(|aux| aux.moe_sumk.clone()).expect("MoE sumk not initialized")+ },ArrayId::MoeBucketedTokenIds => self.llm_aux.as_ref()@@ -750,26 +591,18 @@ impl<B: Backend> ForwardPassState<B> {.as_ref().and_then(|aux| aux.moe_bucketed_probs.clone()).expect("MoE bucketed_probs not initialized"),- ArrayId::MoeXPerm => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_x_perm.clone())- .expect("MoE x_perm not initialized"),- ArrayId::MoeTok2Row => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_tok2row.clone())- .expect("MoE tok2row not initialized"),- ArrayId::MoeYPartial => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_y_partial.clone())- .expect("MoE y_partial not initialized"),- ArrayId::MoeHidden => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_hidden.clone())- .expect("MoE hidden not initialized"),+ ArrayId::MoeXPerm => {+ self.llm_aux.as_ref().and_then(|aux| aux.moe_x_perm.clone()).expect("MoE x_perm not initialized")+ },+ ArrayId::MoeTok2Row => {+ self.llm_aux.as_ref().and_then(|aux| aux.moe_tok2row.clone()).expect("MoE tok2row not initialized")+ },+ ArrayId::MoeYPartial => {+ self.llm_aux.as_ref().and_then(|aux| aux.moe_y_partial.clone()).expect("MoE y_partial not initialized")+ },+ ArrayId::MoeHidden => {+ self.llm_aux.as_ref().and_then(|aux| aux.moe_hidden.clone()).expect("MoE hidden not initialized")+ },ArrayId::MoeTwoPassRowExpertMap => self.llm_aux.as_ref()@@ -785,11 +618,9 @@ impl<B: Backend> ForwardPassState<B> {.as_ref().and_then(|aux| aux.moe_tile_offsets.clone()).expect("MoE tile_offsets not initialized"),- ArrayId::MoeTileMap => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_tile_map.clone())- .expect("MoE tile_map not initialized"),+ ArrayId::MoeTileMap => {+ self.llm_aux.as_ref().and_then(|aux| aux.moe_tile_map.clone()).expect("MoE tile_map not initialized")+ },ArrayId::MoeTotalTiles => self.llm_aux.as_ref()@@ -817,18 +648,10 @@ impl<B: Backend> ForwardPassState<B> {.expect("MoE block_alloc not initialized"),// Classifier-specific arrays- ArrayId::ClassifierPooling => {- self.classifier_state().pooling.clone()- },- ArrayId::ClassifierPredictionHeadDense => {- self.classifier_state().dense.clone()- },- ArrayId::ClassifierPredictionHeadNorm => {- self.classifier_state().norm.clone()- },- ArrayId::ClassifierPredictionHeadLogits => {- self.classifier_state().classifier_logits.clone()- },+ ArrayId::ClassifierPooling => self.classifier_state().pooling.clone(),+ ArrayId::ClassifierPredictionHeadDense => self.classifier_state().dense.clone(),+ ArrayId::ClassifierPredictionHeadNorm => self.classifier_state().norm.clone(),+ ArrayId::ClassifierPredictionHeadLogits => self.classifier_state().classifier_logits.clone(),}}@@ -839,32 +662,12 @@ impl<B: Backend> ForwardPassState<B> {let shared = self.shared_buffers.borrow();match id {ArrayId::RopeCosines(rope_type) => Some(match rope_type {- RopeType::Global => shared- .global_rope- .as_ref()- .expect("Global rope not initialized")- .cosines- .clone(),- RopeType::Local => shared- .local_rope- .as_ref()- .expect("Local rope not initialized")- .cosines- .clone(),+ RopeType::Global => shared.global_rope.as_ref().expect("Global rope not initialized").cosines.clone(),+ RopeType::Local => shared.local_rope.as_ref().expect("Local rope not initialized").cosines.clone(),}),ArrayId::RopeSines(rope_type) => Some(match rope_type {- RopeType::Global => shared- .global_rope- .as_ref()- .expect("Global rope not initialized")- .sines- .clone(),- RopeType::Local => shared- .local_rope- .as_ref()- .expect("Local rope not initialized")- .sines- .clone(),+ RopeType::Global => shared.global_rope.as_ref().expect("Global rope not initialized").sines.clone(),+ RopeType::Local => shared.local_rope.as_ref().expect("Local rope not initialized").sines.clone(),}),_ => None,}@@ -902,9 +705,7 @@ impl<B: Backend> ForwardPassState<B> {pub fn active_suffix_length(&self) -> usize {match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => {- state.active_suffix_length- },+ ForwardPassMode::LanguageModelGenerator(state) => state.active_suffix_length,ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,}}@@ -912,9 +713,7 @@ impl<B: Backend> ForwardPassState<B> {/// Start index (within the suffix batch) for which we need logits/sampling.pub fn sampling_start(&self) -> usize {match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => {- state.sampling_start- },+ ForwardPassMode::LanguageModelGenerator(state) => state.sampling_start,ForwardPassMode::Classifier(_) => 0,}}@@ -922,36 +721,28 @@ impl<B: Backend> ForwardPassState<B> {/// Number of batch items for which we need logits/sampling.pub fn sampling_length(&self) -> usize {match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => {- state.sampling_length- },+ ForwardPassMode::LanguageModelGenerator(state) => state.sampling_length,ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,}}pub fn is_prefilling(&self) -> bool {match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => {- state.is_prefilling- },+ ForwardPassMode::LanguageModelGenerator(state) => state.is_prefilling,ForwardPassMode::Classifier(_) => true,}}pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers<B>>>> {match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => {- Some(&state.cache_layers)- },+ ForwardPassMode::LanguageModelGenerator(state) => Some(&state.cache_layers),ForwardPassMode::Classifier(_) => None,}}pub fn sampling_output(&self) -> Option<&ArrayCell<B>> {match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => {- state.sampling_output.as_ref()- },+ ForwardPassMode::LanguageModelGenerator(state) => state.sampling_output.as_ref(),ForwardPassMode::Classifier(_) => None,}}@@ -964,22 +755,16 @@ impl<B: Backend> ForwardPassState<B> {}}- pub fn sampling_method_mut(- &mut self- ) -> Option<&mut Option<SamplingMethod>> {+ pub fn sampling_method_mut(&mut self) -> Option<&mut Option<SamplingMethod>> {match &mut self.mode {- ForwardPassMode::LanguageModelGenerator(state) => {- Some(&mut state.sampling_method)- },+ ForwardPassMode::LanguageModelGenerator(state) => Some(&mut state.sampling_method),ForwardPassMode::Classifier(_) => None,}}pub fn sampling_method(&self) -> Option<SamplingMethod> {match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => {- state.sampling_method- },+ ForwardPassMode::LanguageModelGenerator(state) => state.sampling_method,ForwardPassMode::Classifier(_) => None,}}@@ -989,9 +774,7 @@ impl<B: Backend> ForwardPassState<B> {source_array_id: ArrayId,destination_array: RefCell<Array<B>>,) {- destination_array- .borrow_mut()- .copy_from_array(&self.arrays(&[source_array_id])[0].borrow());+ destination_array.borrow_mut().copy_from_array(&self.arrays(&[source_array_id])[0].borrow());}pub fn encode_copy_array(diff --git a/crates/uzu/src/forward_pass/traces.rs b/crates/uzu/src/forward_pass/traces.rsindex 2e6a4fc34..683b30957 100644--- a/crates/uzu/src/forward_pass/traces.rs+++ b/crates/uzu/src/forward_pass/traces.rs@@ -22,13 +22,7 @@ fn create_layer_results<B: Backend>(suffix_length: usize,) -> Vec<Rc<RefCell<LayerActivationTrace<B>>>> {(0..model_shape.num_layers)- .map(|_| {- Rc::new(RefCell::new(LayerActivationTrace::new(- context,- model_shape,- suffix_length,- )))- })+ .map(|_| Rc::new(RefCell::new(LayerActivationTrace::new(context, model_shape, suffix_length)))).collect()}@@ -52,24 +46,13 @@ impl<B: Backend> LayerActivationTrace<B> {) -> Self {let main_shape = model_shape.main_shape(suffix_length);let activation_data_type = model_shape.activation_data_type();- let main = |label| {- create_trace_array(- context,- &main_shape,- activation_data_type,- label,- )- };+ let main = |label| create_trace_array(context, &main_shape, activation_data_type, label);Self {inputs: main("layer_activation_trace_inputs"),- pre_attention_norm: main(- "layer_activation_trace_pre_attention_norm",- ),+ pre_attention_norm: main("layer_activation_trace_pre_attention_norm"),attention: main("layer_activation_trace_attention"),- post_attention_norm: main(- "layer_activation_trace_post_attention_norm",- ),+ post_attention_norm: main("layer_activation_trace_post_attention_norm"),mlp_inputs: main("layer_activation_trace_mlp_inputs"),pre_mlp_norm: main("layer_activation_trace_pre_mlp_norm"),mlp: main("layer_activation_trace_mlp"),@@ -95,18 +78,12 @@ impl<B: Backend> ActivationTrace<B> {) -> Self {let activation_data_type = model_shape.activation_data_type();let main_shape = model_shape.main_shape(suffix_length);- let layer_results =- create_layer_results(context, model_shape, suffix_length);+ let layer_results = create_layer_results(context, model_shape, suffix_length);Self {embedding_norm: None,layer_results,- output_norm: create_trace_array(- context,- &main_shape,- activation_data_type,- "activation_trace_output_norm",- ),+ output_norm: create_trace_array(context, &main_shape, activation_data_type, "activation_trace_output_norm"),output_pooling: None,logits: create_trace_array(context,@@ -125,8 +102,7 @@ impl<B: Backend> ActivationTrace<B> {) -> Self {let activation_data_type = model_shape.activation_data_type();let main_shape = model_shape.main_shape(suffix_length);- let layer_results =- create_layer_results(context, model_shape, suffix_length);+ let layer_results = create_layer_results(context, model_shape, suffix_length);let model_dim = model_shape.main_shape(1)[1];Self {@@ -137,36 +113,22 @@ impl<B: Backend> ActivationTrace<B> {"activation_trace_embedding_norm",)),layer_results,- output_norm: create_trace_array(- context,- &main_shape,- activation_data_type,- "activation_trace_output_norm",- ),+ output_norm: create_trace_array(context, &main_shape, activation_data_type, "activation_trace_output_norm"),output_pooling: Some(create_trace_array(context,&[1, model_dim],activation_data_type,"activation_trace_output_pooling",)),- logits: create_trace_array(- context,- &[1, num_labels],- activation_data_type,- "activation_trace_logits",- ),+ logits: create_trace_array(context, &[1, num_labels], activation_data_type, "activation_trace_logits"),}}pub fn embedding_norm(&self) -> &ArrayCell<B> {- self.embedding_norm- .as_ref()- .expect("embedding_norm is only available for classifier traces")+ self.embedding_norm.as_ref().expect("embedding_norm is only available for classifier traces")}pub fn output_pooling(&self) -> &ArrayCell<B> {- self.output_pooling- .as_ref()- .expect("output_pooling is only available for classifier traces")+ self.output_pooling.as_ref().expect("output_pooling is only available for classifier traces")}}diff --git a/crates/uzu/src/language_model/gpu_capture.rs b/crates/uzu/src/language_model/gpu_capture.rsindex f4ff73708..29b013ddb 100644--- a/crates/uzu/src/language_model/gpu_capture.rs+++ b/crates/uzu/src/language_model/gpu_capture.rs@@ -6,10 +6,7 @@ use std::{use objc2_foundation::{NSString, NSURL};use crate::{- backends::metal::{- MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager,- MTLCommandQueueExt, MTLContext,- },+ backends::metal::{MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueueExt, MTLContext},utils::env_utils::MetalEnvVar,};@@ -22,10 +19,8 @@ pub struct GpuCaptureManager {impl GpuCaptureManager {pub fn new() -> Self {- let capture_prefill_enabled =- MetalEnvVar::CaptureFirstPrefill.is_enabled();- let capture_decode_enabled =- MetalEnvVar::CaptureFirstDecode.is_enabled();+ let capture_prefill_enabled = MetalEnvVar::CaptureFirstPrefill.is_enabled();+ let capture_decode_enabled = MetalEnvVar::CaptureFirstDecode.is_enabled();// Enable Metal capture layer BEFORE device creation if any capture is requestedif capture_prefill_enabled || capture_decode_enabled {@@ -46,18 +41,14 @@ impl GpuCaptureManager {&self,is_first_prefill: bool,) -> bool {- self.capture_prefill_enabled- && is_first_prefill- && !self.first_prefill_captured+ self.capture_prefill_enabled && is_first_prefill && !self.first_prefill_captured}pub fn should_capture_decode(&self,is_first_decode: bool,) -> bool {- self.capture_decode_enabled- && is_first_decode- && !self.first_decode_captured+ self.capture_decode_enabled && is_first_decode && !self.first_decode_captured}pub fn start_capture(@@ -65,10 +56,8 @@ impl GpuCaptureManager {mtl_context: &MTLContext,capture_type: &str,) -> Result<PathBuf, String> {- let timestamp = SystemTime::now()- .duration_since(UNIX_EPOCH)- .map_err(|e| format!("Time error: {}", e))?- .as_secs();+ let timestamp =+ SystemTime::now().duration_since(UNIX_EPOCH).map_err(|e| format!("Time error: {}", e))?.as_secs();let trace_path = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."))@@ -76,15 +65,12 @@ impl GpuCaptureManager {let capture_manager = MTLCaptureManager::shared_capture_manager();let capture_descriptor = MTLCaptureDescriptor::new();- capture_descriptor- .set_destination(MTLCaptureDestination::GPUTraceDocument);+ capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);// Set output URL using NSURLlet url_string = format!("file://{}", trace_path.display());unsafe {- if let Some(url) =- NSURL::URLWithString(&NSString::from_str(&url_string))- {+ if let Some(url) = NSURL::URLWithString(&NSString::from_str(&url_string)) {capture_descriptor.set_output_url(Some(&url));}}@@ -92,19 +78,14 @@ impl GpuCaptureManager {mtl_context.command_queue.set_label(Some("uzu_command_queue"));use objc2::rc::Retained;unsafe {- capture_descriptor.set_capture_object(Some(- &*(Retained::as_ptr(&mtl_context.command_queue).cast()),- ));+ capture_descriptor.set_capture_object(Some(&*(Retained::as_ptr(&mtl_context.command_queue).cast())));}capture_manager.start_capture_with_descriptor_error(&capture_descriptor).map_err(|e| format!("Failed to start GPU capture: {}", e))?;- println!(- "GPU capture started for first {}: {:?}",- capture_type, trace_path- );+ println!("GPU capture started for first {}: {:?}", capture_type, trace_path);Ok(trace_path)}diff --git a/crates/uzu/src/language_model/grammar.rs b/crates/uzu/src/language_model/grammar.rsindex c0dd86ca8..e53bd42ce 100644--- a/crates/uzu/src/language_model/grammar.rs+++ b/crates/uzu/src/language_model/grammar.rs@@ -1,9 +1,6 @@use std::iter::repeat_n;-use xgrammar::{- DLDataType, DLDevice, DLDeviceType, DLTensor, Grammar, GrammarCompiler,- GrammarMatcher, TokenizerInfo,-};+use xgrammar::{DLDataType, DLDevice, DLDeviceType, DLTensor, Grammar, GrammarCompiler, GrammarMatcher, TokenizerInfo};use crate::session::{config::GrammarConfig, types::Error};@@ -55,10 +52,8 @@ impl CompiledGrammarEngagementState {trigger_token_id: _,trigger_distance,} => {- let num_grammar_tokens =- usize::min(trigger_distance.unwrap_or(0), num_tokens);- *trigger_distance =- trigger_distance.and_then(|x| x.checked_sub(num_tokens));+ let num_grammar_tokens = usize::min(trigger_distance.unwrap_or(0), num_tokens);+ *trigger_distance = trigger_distance.and_then(|x| x.checked_sub(num_tokens));num_grammar_tokens},}@@ -97,18 +92,9 @@ impl CompiledGrammar {separators,strict_mode,} => {- let separators_ref =- separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));- Grammar::from_json_schema(- schema,- *any_whitespace,- *indent,- separators_ref,- *strict_mode,- None,- false,- )- .map_err(|error_message| Error::GrammarError(error_message))?+ let separators_ref = separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));+ Grammar::from_json_schema(schema, *any_whitespace, *indent, separators_ref, *strict_mode, None, false)+ .map_err(|error_message| Error::GrammarError(error_message))?},GrammarConfig::Regex {pattern,@@ -119,14 +105,12 @@ impl CompiledGrammar {};let mut compiler = GrammarCompiler::new(tokenizer_info, 8, true, -1).map_err(|error_message| Error::GrammarError(error_message))?;- let compiled = compiler- .compile_grammar(&grammar)- .map_err(|error_message| Error::GrammarError(error_message))?;+ let compiled =+ compiler.compile_grammar(&grammar).map_err(|error_message| Error::GrammarError(error_message))?;let matcher = GrammarMatcher::new(&compiled, None, true, -1).map_err(|error_message| Error::GrammarError(error_message))?;- let engagement_state = if let Some(trigger_token_id) = trigger_token_id- {+ let engagement_state = if let Some(trigger_token_id) = trigger_token_id {CompiledGrammarEngagementState::Triggered {trigger_token_id,trigger_distance: None,@@ -144,8 +128,7 @@ impl CompiledGrammar {pub fn next_bitmask(&mut self) -> Result<Option<Box<[u32]>>, Error> {if self.engagement_state.is_engaged() {- let mut cpu_mask = repeat_n(0, self.vocab_size.div_ceil(32))- .collect::<Box<[u32]>>();+ let mut cpu_mask = repeat_n(0, self.vocab_size.div_ceil(32)).collect::<Box<[u32]>>();let batch_mask_slice = &mut cpu_mask;let mut shape_i64 = [self.vocab_size.div_ceil(32) as i64];let mut bitmask_tensor = DLTensor {@@ -165,15 +148,9 @@ impl CompiledGrammar {byte_offset: 0,};- let success = self.matcher.fill_next_token_bitmask(- &mut bitmask_tensor,- 0,- false,- );+ let success = self.matcher.fill_next_token_bitmask(&mut bitmask_tensor, 0, false);if !success {- return Err(Error::GrammarError(- "Failed to fill next token bitmask".to_string(),- ));+ return Err(Error::GrammarError("Failed to fill next token bitmask".to_string()));}Ok(Some(cpu_mask))@@ -188,10 +165,7 @@ impl CompiledGrammar {) -> Result<(), Error> {if self.engagement_state.is_engaged() {if (token_id as usize) >= self.vocab_size {- return Err(Error::TokenOutOfGrammarRange(- token_id,- self.vocab_size,- ));+ return Err(Error::TokenOutOfGrammarRange(token_id, self.vocab_size));}if !self.matcher.accept_token(token_id as i32) {diff --git a/crates/uzu/src/language_model/gumbel.rs b/crates/uzu/src/language_model/gumbel.rsindex 73f371a9e..b2fcb18c6 100644--- a/crates/uzu/src/language_model/gumbel.rs+++ b/crates/uzu/src/language_model/gumbel.rs@@ -69,9 +69,7 @@ pub fn revidx(logit_idx: u32) -> (u32, u32) {let grain_idx = logit_idx_in_blockgrain / BLOCK_SIZE;let local_idx = logit_idx_in_blockgrain % BLOCK_SIZE;- let rng_offset = (global_idx * BLOCK_SIZE + local_idx)- * (GRAIN_SIZE + WORDS_PER_OFFSET - 1)- / WORDS_PER_OFFSET+ let rng_offset = (global_idx * BLOCK_SIZE + local_idx) * (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET+ grain_idx / WORDS_PER_OFFSET;let rng_word = grain_idx % WORDS_PER_OFFSET;diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex a0e76b611..08443e847 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -1,6 +1,4 @@-use std::{- collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant,-};+use std::{collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant};use itertools::{Either, Itertools, izip};@@ -16,8 +14,8 @@ use crate::{backends::{common::kernel::MaskUpdateKernel,metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,- MTLCommandBufferHandler, MTLCommandEncoder, MTLCommandQueue, Metal,+ MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler, MTLCommandEncoder,+ MTLCommandQueue, Metal,},},encodable_block::{EncodableBlock, EncodingParameters},@@ -51,10 +49,8 @@ impl LanguageModelGenerator {) -> Result<Self, Error> {let gpu_capture = GpuCaptureManager::new();- let context =- LanguageModelGeneratorContext::new(model_path, &decoding_config)?;- let prefill_step_size =- decoding_config.prefill_step_size.resolve(&context.model_config);+ let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;+ let prefill_step_size = decoding_config.prefill_step_size.resolve(&context.model_config);let generate_suffix_length = decoding_config.generate_suffix_length();let mut generator = Self {@@ -87,10 +83,7 @@ impl LanguageModelGenerator {let tokens_length = tokens.len();- let prefill_step_size = self- .decoding_config- .prefill_step_size- .resolve(&self.context.model_config);+ let prefill_step_size = self.decoding_config.prefill_step_size.resolve(&self.context.model_config);let prefill_steps = tokens_length.div_ceil(prefill_step_size);let prefill_size = prefill_steps * prefill_step_size;@@ -113,113 +106,73 @@ impl LanguageModelGenerator {let has_grammar = compiled_grammar.is_some();- let token_ids = tokens- .iter()- .copied()- .take(tokens_length - 1)- .chain(flat_trie.token_ids())- .chunks(prefill_step_size);+ let token_ids =+ tokens.iter().copied().take(tokens_length - 1).chain(flat_trie.token_ids()).chunks(prefill_step_size);- let token_positions = (prefix_offset- ..prefix_offset + tokens_length - 1)- .chain(flat_trie.token_positions().map(|trie_position| {- prefix_offset + tokens_length - 1 + trie_position- }))+ let token_positions = (prefix_offset..prefix_offset + tokens_length - 1)+ .chain(flat_trie.token_positions().map(|trie_position| prefix_offset + tokens_length - 1 + trie_position)).chunks(prefill_step_size);- let single_token_bitmask_size =- self.context.model_shape.bitmask_shape(1)[1];- let token_bitmasks = repeat_n(None, tokens_length - 1)- .chain(flat_trie.token_masks())- .chunks(prefill_step_size);+ let single_token_bitmask_size = self.context.model_shape.bitmask_shape(1)[1];+ let token_bitmasks = repeat_n(None, tokens_length - 1).chain(flat_trie.token_masks()).chunks(prefill_step_size);- let token_seeds = repeat_n(0, tokens_length - 1)- .chain(flat_trie.token_seeds())- .chunks(prefill_step_size);+ let token_seeds = repeat_n(0, tokens_length - 1).chain(flat_trie.token_seeds()).chunks(prefill_step_size);let mut last_state: Option<ForwardPassState<Metal>> = None;let mut run_times: Vec<f64> = Vec::new();// Process each prefill step and update the KV cache.- for (- step,- (- step_token_ids,- step_token_positions,- step_token_bitmasks,- step_token_seeds,- ),- ) in- izip!(&token_ids, &token_positions, &token_bitmasks, &token_seeds)- .enumerate()+ for (step, (step_token_ids, step_token_positions, step_token_bitmasks, step_token_seeds)) in+ izip!(&token_ids, &token_positions, &token_bitmasks, &token_seeds).enumerate(){let tokens_start_index = step * prefill_step_size;let tokens_end_index = tokens_start_index + prefill_step_size;let step_token_ids = step_token_ids.collect::<Box<[u64]>>();- let step_token_positions =- step_token_positions.collect::<Box<[usize]>>();+ let step_token_positions = step_token_positions.collect::<Box<[usize]>>();let step_token_seeds = step_token_seeds.collect::<Box<[u64]>>();let active_suffix_length = step_token_positions.len();let is_last_prefill_step = step == prefill_steps - 1;- let should_sample_after_step =- sample_suffix && is_last_prefill_step;+ let should_sample_after_step = sample_suffix && is_last_prefill_step;// If we sample on the last prefill step, we only need logits/sampling// for tokens that are beyond the prompt prefix (i.e. starting at the// suffix-root token, which is the last prompt token).- let (sampling_start, sampling_length) = if should_sample_after_step- {- let suffix_root_index_in_step =- (tokens_length - 1).saturating_sub(tokens_start_index);- let sampling_length = active_suffix_length- .saturating_sub(suffix_root_index_in_step);- debug_assert!(- sampling_length > 0,- "Expected at least one token to sample on the last prefill step"- );+ let (sampling_start, sampling_length) = if should_sample_after_step {+ let suffix_root_index_in_step = (tokens_length - 1).saturating_sub(tokens_start_index);+ let sampling_length = active_suffix_length.saturating_sub(suffix_root_index_in_step);+ debug_assert!(sampling_length > 0, "Expected at least one token to sample on the last prefill step");(suffix_root_index_in_step, sampling_length)} else {(0, 0)};- let step_token_bitmask: Option<Box<[u32]>> =- if has_grammar && sampling_length > 0 {- Some(- step_token_bitmasks- .map(|mask| match mask {- Some(mask) => Either::Left(- mask.iter()- .copied()- .take(single_token_bitmask_size)- .chain(repeat_n(- 0u32,- single_token_bitmask_size- .saturating_sub(mask.len()),- )),- ),- None => Either::Right(repeat_n(- u32::MAX,- single_token_bitmask_size,- )),- })- .flatten()- .collect::<Box<[u32]>>(),- )- } else {- // Drain the chunk iterator to keep the other chunked iterators aligned.- let _ = step_token_bitmasks.count();- None- };-- let should_capture =- self.gpu_capture.should_capture_prefill(step == 0);+ let step_token_bitmask: Option<Box<[u32]>> = if has_grammar && sampling_length > 0 {+ Some(+ step_token_bitmasks+ .map(|mask| match mask {+ Some(mask) => Either::Left(+ mask.iter()+ .copied()+ .take(single_token_bitmask_size)+ .chain(repeat_n(0u32, single_token_bitmask_size.saturating_sub(mask.len()))),+ ),+ None => Either::Right(repeat_n(u32::MAX, single_token_bitmask_size)),+ })+ .flatten()+ .collect::<Box<[u32]>>(),+ )+ } else {+ // Drain the chunk iterator to keep the other chunked iterators aligned.+ let _ = step_token_bitmasks.count();+ None+ };++ let should_capture = self.gpu_capture.should_capture_prefill(step == 0);if should_capture {- let _ = self- .gpu_capture- .start_capture(&self.context.mtl_context, "prefill");+ let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "prefill");}objc2::rc::autoreleasepool(|_pool| {@@ -251,36 +204,24 @@ impl LanguageModelGenerator {}// Register the accepted prompt tokens from this step.- let step_end_token_index =- std::cmp::min(tokens_end_index, tokens_length);- let tokens_processed_this_step =- step_end_token_index - tokens_start_index;+ let step_end_token_index = std::cmp::min(tokens_end_index, tokens_length);+ let tokens_processed_this_step = step_end_token_index - tokens_start_index;if tokens_processed_this_step > 0 {- let mut positions_for_step: Vec<usize> = (tokens_start_index- ..step_end_token_index)- .map(|idx| idx + prefix_offset)- .collect();+ let mut positions_for_step: Vec<usize> =+ (tokens_start_index..step_end_token_index).map(|idx| idx + prefix_offset).collect();if step == prefill_steps - 1 && sample_suffix {// Exclude the last token because it belongs to the suffix for sampling.positions_for_step.pop();}if !positions_for_step.is_empty() {- let accept_indices_for_step: Vec<usize> =- (0..positions_for_step.len()).collect();+ let accept_indices_for_step: Vec<usize> = (0..positions_for_step.len()).collect();if !accept_indices_for_step.is_empty() {- self.update_cache_layers(- &accept_indices_for_step,- None,- true,- );+ self.update_cache_layers(&accept_indices_for_step, None, true);}- self.context- .cache_layers- .borrow_mut()- .register_accepted_tokens(&positions_for_step);+ self.context.cache_layers.borrow_mut().register_accepted_tokens(&positions_for_step);if let Some(&last_idx) = positions_for_step.last() {self.registered_prefix_len = last_idx + 1;@@ -309,10 +250,7 @@ impl LanguageModelGenerator {flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());self.update_cache_layers(- &accepted_token_indices- .into_iter()- .map(|p| suffix_root_index + p)- .collect::<Box<[usize]>>(),+ &accepted_token_indices.into_iter().map(|p| suffix_root_index + p).collect::<Box<[usize]>>(),Some(last_suffix_start),false,);@@ -346,52 +284,36 @@ impl LanguageModelGenerator {let flat_trie = suffix_root.linearize();let active_suffix_length = flat_trie.len();- let token_ids = flat_trie- .token_ids()- .chain(repeat_n(0, suffix_length - active_suffix_length))- .collect::<Box<[u64]>>();-- let token_bitmask: Option<Box<[u32]>> =- compiled_grammar.is_some().then(|| {- let single_token_bitmask_size =- self.context.model_shape.bitmask_shape(1)[1];- flat_trie- .token_masks()- .chain(repeat_n(None, suffix_length - active_suffix_length))- .map(|mask| match mask {- Some(mask) => Either::Left(- mask.iter()- .copied()- .take(single_token_bitmask_size)- .chain(repeat_n(- 0u32,- single_token_bitmask_size- .saturating_sub(mask.len()),- )),- ),- None => Either::Right(repeat_n(- u32::MAX,- single_token_bitmask_size,- )),- })- .flatten()- .collect::<Box<[u32]>>()- });+ let token_ids =+ flat_trie.token_ids().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();++ let token_bitmask: Option<Box<[u32]>> = compiled_grammar.is_some().then(|| {+ let single_token_bitmask_size = self.context.model_shape.bitmask_shape(1)[1];+ flat_trie+ .token_masks()+ .chain(repeat_n(None, suffix_length - active_suffix_length))+ .map(|mask| match mask {+ Some(mask) => Either::Left(+ mask.iter()+ .copied()+ .take(single_token_bitmask_size)+ .chain(repeat_n(0u32, single_token_bitmask_size.saturating_sub(mask.len()))),+ ),+ None => Either::Right(repeat_n(u32::MAX, single_token_bitmask_size)),+ })+ .flatten()+ .collect::<Box<[u32]>>()+ });let start_position = self.tokens.len() - 1;let token_positions = flat_trie.token_positions().map(|trie_position| start_position + trie_position)- .chain(repeat_n(- INVALID_POSITION,- suffix_length - active_suffix_length,- ))+ .chain(repeat_n(INVALID_POSITION, suffix_length - active_suffix_length)).collect::<Box<[usize]>>();- let token_seeds = flat_trie- .token_seeds()- .chain(repeat_n(0, suffix_length - active_suffix_length))- .collect::<Box<[u64]>>();+ let token_seeds =+ flat_trie.token_seeds().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();let task = LanguageModelGeneratorRunTask {token_ids: &token_ids,@@ -405,13 +327,8 @@ impl LanguageModelGenerator {is_prefilling: false,};- let (mut state, run_time) = self.run_model(- task,- false,- self.allow_pre_encode(),- sampling_method,- self.should_fill_attention_bias(),- );+ let (mut state, run_time) =+ self.run_model(task, false, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias());let sampled_tokens = self.sample(&mut state)?;@@ -448,14 +365,8 @@ impl LanguageModelGenerator {&self.context.mtl_context,);- self.context- .async_buffers- .prepare_positions(prefill_count, tokens_to_generate);- self.context.async_buffers.prepare_seeds(- &self.context.seed,- prefill_count,- tokens_to_generate,- );+ self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);+ self.context.async_buffers.prepare_seeds(&self.context.seed, prefill_count, tokens_to_generate);self.context.async_buffers.reset_counter();}@@ -474,11 +385,7 @@ impl LanguageModelGenerator {whereF: FnOnce(u64) + Send + 'static,{- assert_eq!(- self.decoding_config.generate_suffix_length(),- 1,- "async_generate only supports suffix_length=1"- );+ assert_eq!(self.decoding_config.generate_suffix_length(), 1, "async_generate only supports suffix_length=1");// Extract values from async_buffers before mutable borrowlet current_counter = self.context.async_buffers.counter.get();@@ -487,8 +394,7 @@ impl LanguageModelGenerator {let slot = pass_idx % batch_size;let async_event = self.context.async_buffers.event.clone();let results_buffer = self.context.async_buffers.results.clone();- let async_positions_buffer =- self.context.async_buffers.positions.clone();+ let async_positions_buffer = self.context.async_buffers.positions.clone();let async_seeds_buffer = self.context.async_buffers.seeds.clone();let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;@@ -517,12 +423,9 @@ impl LanguageModelGenerator {let skip_token_ids_copy = pass_idx > 0;let is_first_decode = !is_continuation;- let should_capture =- self.gpu_capture.should_capture_decode(is_first_decode);+ let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);if should_capture {- let _ = self- .gpu_capture- .start_capture(&self.context.mtl_context, "decode");+ let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "decode");}let mut state = ForwardPassState::new_llm(@@ -555,8 +458,7 @@ impl LanguageModelGenerator {// Wait on previous pass if this is a continuationif is_continuation {- root_command_buffer- .encode_wait_for_event_value(&async_event, current_counter);+ root_command_buffer.encode_wait_for_event_value(&async_event, current_counter);}self.context.executables.encode(@@ -574,27 +476,14 @@ impl LanguageModelGenerator {// Copy sampled token: sampling_output β token_ids (for next pass)// and sampling_output β results[slot] (for callback)- let sampling_output = state- .sampling_output()- .expect("sampling_output must exist after sampling encode");+ let sampling_output = state.sampling_output().expect("sampling_output must exist after sampling encode");let sampling_output_buffer = sampling_output.borrow().buffer().clone();- let token_ids_buffer =- self.context.scratch_buffers.token_ids.borrow().buffer().clone();-- let encoder = root_command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");- self.context.token_copy.encode_to_token_ids(- &sampling_output_buffer,- &token_ids_buffer,- &encoder,- );- self.context.token_copy.encode_to_results(- &sampling_output_buffer,- &results_buffer,- slot,- &encoder,- );+ let token_ids_buffer = self.context.scratch_buffers.token_ids.borrow().buffer().clone();++ let encoder =+ root_command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");+ self.context.token_copy.encode_to_token_ids(&sampling_output_buffer, &token_ids_buffer, &encoder);+ self.context.token_copy.encode_to_results(&sampling_output_buffer, &results_buffer, slot, &encoder);encoder.end_encoding();// Scatter + register for all transformer layers@@ -606,35 +495,19 @@ impl LanguageModelGenerator {&cb,&self.context.kv_cache_update,);- self.context- .cache_layers- .borrow_mut()- .register_accepted_tokens(&[token_position]);+ self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);}- let encoder = root_command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");+ let encoder =+ root_command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");if let Some(mask_update) = &self.context.mask_update {- let updates: Vec<AttentionBiasUpdate> = self- .context- .cache_layers- .borrow()- .attention_bias_updates_after_acceptance(1);- for (window_size, mask_buffer) in- &self.context.scratch_buffers.attention_window_size_to_bias- {- if let Some(update) =- updates.iter().find(|u| &u.key == window_size)- {+ let updates: Vec<AttentionBiasUpdate> =+ self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);+ for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {+ if let Some(update) = updates.iter().find(|u| &u.key == window_size) {if update.unmask_col >= 0 || update.mask_col >= 0 {- mask_update.encode(- mask_buffer.borrow().buffer(),- update.unmask_col,- update.mask_col,- &encoder,- );+ mask_update.encode(mask_buffer.borrow().buffer(), update.unmask_col, update.mask_col, &encoder);}}}@@ -644,8 +517,7 @@ impl LanguageModelGenerator {// Signal event for next passlet next_counter = current_counter + 1;- root_command_buffer- .encode_signal_event_value(&async_event, next_counter);+ root_command_buffer.encode_signal_event_value(&async_event, next_counter);self.context.async_buffers.counter.set(next_counter);// Add completion handler@@ -654,8 +526,7 @@ impl LanguageModelGenerator {let handler = MTLCommandBufferHandler::new(move |_| {let token = {- let ptr =- results_buffer_clone.contents().as_ptr() as *const u32;+ let ptr = results_buffer_clone.contents().as_ptr() as *const u32;unsafe { *ptr.add(slot) as u64 }};if let Some(cb) = callback.lock().unwrap().take() {@@ -720,8 +591,7 @@ impl LanguageModelGenerator {is_prefilling: false,};- let (_, _) =- self.run_model(task, true, false, SamplingMethod::default(), false);+ let (_, _) = self.run_model(task, true, false, SamplingMethod::default(), false);}fn run_model(@@ -735,23 +605,16 @@ impl LanguageModelGenerator {objc2::rc::autoreleasepool(|_pool| {let run_start = Instant::now();- let mut state = task.create_state(- &mut self.context,- None,- should_fill_attention_bias,- );+ let mut state = task.create_state(&mut self.context, None, should_fill_attention_bias);if let Some(method) = state.sampling_method_mut() {*method = Some(sampling_method);}let is_first_decode = !warmup && task.token_ids.len() == 1;- let should_capture =- self.gpu_capture.should_capture_decode(is_first_decode);+ let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);if should_capture {- let _ = self- .gpu_capture- .start_capture(&self.context.mtl_context, "decode");+ let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "decode");}let encoded_task_key = task.encoded_task_key(self.tokens.len());@@ -790,19 +653,16 @@ impl LanguageModelGenerator {if allow_pre_encode {self.context.reset_command_buffer();- let next_task_key: String =- task.encoded_task_key(self.tokens.len() + 1);+ let next_task_key: String = task.encoded_task_key(self.tokens.len() + 1);let next_encoded_task = task.build_encoded_task(&self.context,&mut state,- &EncodingParameters::new(warmup, false, false)- .with_projection(1),+ &EncodingParameters::new(warmup, false, false).with_projection(1),next_task_key.clone(),);- self.encoded_tasks- .insert(next_task_key.clone(), next_encoded_task);+ self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);}root_command_buffer.wait_until_completed();@@ -819,7 +679,8 @@ impl LanguageModelGenerator {&mut self,state: &mut ForwardPassState<Metal>,) -> Result<Vec<u64>, Error> {- let sampling_output = state.sampling_output()+ let sampling_output = state+ .sampling_output().expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");let output_buffer = sampling_output.borrow();@@ -869,8 +730,7 @@ impl LanguageModelGenerator {fn allow_pre_encode(&self) -> bool {let metal_debug_active = MetalEnvVar::DeviceWrapperType.is_enabled();- let result =- self.decoding_config.allow_pre_encode && !metal_debug_active;+ let result = self.decoding_config.allow_pre_encode && !metal_debug_active;result}@@ -882,25 +742,18 @@ impl LanguageModelGenerator {let desired_prefix_len = self.tokens.len() - 1;if desired_prefix_len > self.registered_prefix_len {- let positions: Vec<usize> =- (self.registered_prefix_len..desired_prefix_len).collect();+ let positions: Vec<usize> = (self.registered_prefix_len..desired_prefix_len).collect();if !positions.is_empty() {- self.context- .cache_layers- .borrow_mut()- .register_accepted_tokens(&positions);+ self.context.cache_layers.borrow_mut().register_accepted_tokens(&positions);}self.registered_prefix_len = desired_prefix_len;}}fn should_fill_attention_bias(&self) -> bool {- let sliding_window_sizes =- self.context.model_shape.sliding_window_length_per_layer.clone();- let has_sliding_window =- sliding_window_sizes.iter().any(|size| size.is_some());- let has_speculative_suffix =- self.decoding_config.generate_suffix_length() > 1;+ let sliding_window_sizes = self.context.model_shape.sliding_window_length_per_layer.clone();+ let has_sliding_window = sliding_window_sizes.iter().any(|size| size.is_some());+ let has_speculative_suffix = self.decoding_config.generate_suffix_length() > 1;has_sliding_window || has_speculative_suffix}diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rsindex 966cfe721..ae0b098ed 100644--- a/crates/uzu/src/parameters/loader.rs+++ b/crates/uzu/src/parameters/loader.rs@@ -6,10 +6,7 @@ use std::{use thiserror::Error;-use super::safetensors_metadata::{- HashMetadata as STMetadata, HeaderLoadingError,- read_metadata as read_st_metadata,-};+use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingError, read_metadata as read_st_metadata};use crate::{DataType,array::{Array, ArrayContextExt},@@ -99,18 +96,11 @@ where&self,key: &str,) -> Result<Array<C::Backend>, ParameterLoaderError> {- let metadata_entry = self- .index- .get(key)- .ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;+ let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;let (offset, size) = (metadata_entry.offset, metadata_entry.size);let array_key = key.replace(".", "_");let array_label = format!("parameter_loader_{array_key}");- let mut array = self.context.create_array(- &metadata_entry.shape,- metadata_entry.data_type,- &array_label,- );+ let mut array = self.context.create_array(&metadata_entry.shape, metadata_entry.data_type, &array_label);let expected_size = array.size();if expected_size != size {return Err(ParameterLoaderError::SizeMismatch {@@ -131,10 +121,7 @@ whereshape: &mut Box<[usize]>,data_type: &mut DataType,) -> Result<(), ParameterLoaderError> {- let metadata_entry = self- .index- .get(key)- .ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;+ let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;self.file.read_exact_at(buf, metadata_entry.offset as u64)?;*shape = metadata_entry.shape.to_owned();*data_type = metadata_entry.data_type;@@ -170,9 +157,7 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {&self,name: &str,) -> String {- self.prefix- .as_ref()- .map_or_else(|| name.to_string(), |p| format!("{p}.{name}"))+ self.prefix.as_ref().map_or_else(|| name.to_string(), |p| format!("{p}.{name}"))}pub fn subtree(@@ -180,11 +165,7 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {name: &str,) -> Result<Self, ParameterLoaderError> {let new_prefix = self.join_prefix(name);- let num_suffixes = self- .loader- .keys()- .filter_map(|suffix| suffix.strip_prefix(&new_prefix))- .count();+ let num_suffixes = self.loader.keys().filter_map(|suffix| suffix.strip_prefix(&new_prefix)).count();if num_suffixes > 0 {Ok(Self {loader: self.loader,@@ -209,11 +190,6 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {shape: &mut Box<[usize]>,data_type: &mut DataType,) -> Result<(), ParameterLoaderError> {- self.loader.read_extract_at(- &self.join_prefix(name),- buf,- shape,- data_type,- )+ self.loader.read_extract_at(&self.join_prefix(name), buf, shape, data_type)}}diff --git a/crates/uzu/src/parameters/safetensors_metadata.rs b/crates/uzu/src/parameters/safetensors_metadata.rsindex e5f520b4a..edb0e04df 100644--- a/crates/uzu/src/parameters/safetensors_metadata.rs+++ b/crates/uzu/src/parameters/safetensors_metadata.rs@@ -11,9 +11,7 @@ use crate::DataType;pub enum HeaderLoadingError {#[error("The header is an invalid UTF-8 string and cannot be read.")]InvalidHeader,- #[error(- "The header does contain a valid string, but it is not valid JSON."- )]+ #[error("The header does contain a valid string, but it is not valid JSON.")]InvalidHeaderDeserialization,#[error("The header is large than 100Mo which is considered too large.")]HeaderTooLarge,@@ -41,9 +39,7 @@ pub struct TensorInfo {pub data_offsets: (usize, usize),}-#[derive(- Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Ord, PartialOrd,-)]+#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]#[non_exhaustive]pub enum Dtype {/// Boolan type@@ -116,29 +112,20 @@ impl From<DataType> for Dtype {const MAX_HEADER_SIZE: usize = 100_000_000;-pub fn read_metadata(- file: &File-) -> Result<(usize, HashMetadata), HeaderLoadingError> {+pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadingError> {let mut header_buffer = [0u8; size_of::<u64>()];- file.read_exact_at(&mut header_buffer, 0)- .map_err(|_| HeaderLoadingError::HeaderTooSmall)?;- let metadata_size: usize = u64::from_le_bytes(header_buffer)- .try_into()- .map_err(|_| HeaderLoadingError::HeaderTooLarge)?;+ file.read_exact_at(&mut header_buffer, 0).map_err(|_| HeaderLoadingError::HeaderTooSmall)?;+ let metadata_size: usize =+ u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;if metadata_size > MAX_HEADER_SIZE {return Err(HeaderLoadingError::InvalidHeaderLength);}- let stop = metadata_size- .checked_add(8)- .ok_or(HeaderLoadingError::InvalidHeaderLength)?;- let mut json_buffer: Box<[u8]> =- core::iter::repeat(0).take(stop - size_of::<u64>()).collect();- file.read_exact_at(&mut json_buffer, 8)- .map_err(|_| HeaderLoadingError::InvalidHeader)?;- let string = core::str::from_utf8(&json_buffer)- .map_err(|_| HeaderLoadingError::InvalidHeader)?;- let metadata: HashMetadata = serde_json::from_str(string)- .map_err(|_| HeaderLoadingError::InvalidHeaderDeserialization)?;+ let stop = metadata_size.checked_add(8).ok_or(HeaderLoadingError::InvalidHeaderLength)?;+ let mut json_buffer: Box<[u8]> = core::iter::repeat(0).take(stop - size_of::<u64>()).collect();+ file.read_exact_at(&mut json_buffer, 8).map_err(|_| HeaderLoadingError::InvalidHeader)?;+ let string = core::str::from_utf8(&json_buffer).map_err(|_| HeaderLoadingError::InvalidHeader)?;+ let metadata: HashMetadata =+ serde_json::from_str(string).map_err(|_| HeaderLoadingError::InvalidHeaderDeserialization)?;Ok((stop, metadata))}diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rsindex 405d00400..940c6a544 100644--- a/crates/uzu/src/prelude.rs+++ b/crates/uzu/src/prelude.rs@@ -9,13 +9,11 @@pub use crate::session::config::{DecodingConfig, RunConfig, SpeculatorConfig};// Session Parameterspub use crate::session::parameter::{- ContextLength, ContextMode, PrefillStepSize, SamplingMethod,- SamplingPolicy, SamplingSeed,+ ContextLength, ContextMode, PrefillStepSize, SamplingMethod, SamplingPolicy, SamplingSeed,};// Session Typespub use crate::session::types::{- Error, FinishReason, Input, Message, Output, ParsedText, Role, RunStats,- Stats, StepStats, Text, TotalStats,+ Error, FinishReason, Input, Message, Output, ParsedText, Role, RunStats, Stats, StepStats, Text, TotalStats,};// Speculatorspub use crate::speculators::speculator::Speculator;diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex f46663535..f16f730d0 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -28,15 +28,9 @@ use crate::{},session::{config::{DecodingConfig, RunConfig},- helpers::{- Context, InputProcessor, InputProcessorDefault, OutputParser,- is_directory_fits_ram,- },+ helpers::{Context, InputProcessor, InputProcessorDefault, OutputParser, is_directory_fits_ram},parameter::{ConfigResolvableValue, ContextMode},- types::{- Error, FinishReason, Input, Output, RunStats, Stats, StepStats,- TotalStats,- },+ types::{Error, FinishReason, Input, Output, RunStats, Stats, StepStats, TotalStats},},};@@ -81,12 +75,8 @@ impl ChatSession {if !config_path.exists() {return Err(Error::UnableToLoadConfig);}- let config_file =- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;- let model_metadata: ModelMetadata = serde_json::from_reader(- BufReader::new(config_file),- )- .map_err(|err| {+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;+ let model_metadata: ModelMetadata = serde_json::from_reader(BufReader::new(config_file)).map_err(|err| {eprintln!("Failed to parse config.json: {err}");Error::UnableToLoadConfig})?;@@ -97,12 +87,9 @@ impl ChatSession {.and_then(|lm| lm.decoder_config().ok()).and_then(|dc| dc.layer_configs).map(|layers| {- let has_non_attention = layers.iter().any(|layer| {- !matches!(layer.mixer_config, MixerConfig::Attention(_))- });- let has_mamba = layers.iter().any(|layer| {- matches!(layer.mixer_config, MixerConfig::Mamba(_))- });+ let has_non_attention =+ layers.iter().any(|layer| !matches!(layer.mixer_config, MixerConfig::Attention(_)));+ let has_mamba = layers.iter().any(|layer| matches!(layer.mixer_config, MixerConfig::Mamba(_)));(has_non_attention, has_mamba)}).unwrap_or((false, false));@@ -120,50 +107,29 @@ impl ChatSession {}}- if has_mamba_mixer- && decoding_config.speculator_config.number_of_speculated_tokens > 0- {+ if has_mamba_mixer && decoding_config.speculator_config.number_of_speculated_tokens > 0 {return Err(Error::UnsupportedSpeculatorConfigForModel);}- let language_model_config = model_metadata- .model_config- .as_language_model()- .ok_or(Error::UnableToLoadConfig)?;+ let language_model_config = model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;let tokenizer_path = model_path.join("tokenizer.json");if !tokenizer_path.exists() {return Err(Error::UnableToLoadTokenizer);}- let tokenizer = Tokenizer::from_file(&tokenizer_path)- .map_err(|_| Error::UnableToLoadTokenizer)?;+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;- let stop_token_ids: Vec<i32> = language_model_config- .generation_config- .stop_token_ids- .iter()- .map(|&x| x as i32)- .collect();- let tokenizer_info = TokenizerInfo::from_huggingface(- &tokenizer,- None,- Some(&stop_token_ids),- )- .map_err(|error_message| Error::GrammarError(error_message))?;-- let input_processor = InputProcessorDefault::new(- language_model_config.message_processor_config.clone(),- );+ let stop_token_ids: Vec<i32> =+ language_model_config.generation_config.stop_token_ids.iter().map(|&x| x as i32).collect();+ let tokenizer_info = TokenizerInfo::from_huggingface(&tokenizer, None, Some(&stop_token_ids))+ .map_err(|error_message| Error::GrammarError(error_message))?;- let output_parser = OutputParser::new(- language_model_config- .message_processor_config- .output_parser_regex- .clone(),- )?;+ let input_processor = InputProcessorDefault::new(language_model_config.message_processor_config.clone());++ let output_parser =+ OutputParser::new(language_model_config.message_processor_config.output_parser_regex.clone())?;- let llm = LanguageModelGenerator::new(&model_path, decoding_config)- .map_err(Error::from)?;+ let llm = LanguageModelGenerator::new(&model_path, decoding_config).map_err(Error::from)?;Ok(Self {model_path,@@ -186,20 +152,12 @@ impl ChatSession {whereF: Fn(Output) -> bool,{- let context_mode = self- .llm- .as_ref()- .ok_or(Error::LanguageModelGeneratorNotLoaded)?- .decoding_config- .context_mode- .clone();+ let context_mode =+ self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?.decoding_config.context_mode.clone();match &context_mode {ContextMode::None => {- let language_model_generator = self- .llm- .as_mut()- .ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;language_model_generator.reset_state();},ContextMode::Static {@@ -208,8 +166,7 @@ impl ChatSession {if self.static_context.is_none() {let mut prefill_config = config.clone();prefill_config.tokens_limit = 0;- let (_out, ctx) =- self.extend(input.clone(), None, prefill_config)?;+ let (_out, ctx) = self.extend(input.clone(), None, prefill_config)?;self.static_context = Some(ctx);}let tmp = self.static_context.take();@@ -228,10 +185,7 @@ impl ChatSession {| ContextMode::Static {..} => {- let language_model_generator = self- .llm- .as_mut()- .ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;language_model_generator.reset_state();},ContextMode::Dynamic => {},@@ -240,8 +194,7 @@ impl ChatSession {}pub fn reset(&mut self) -> Result<(), Error> {- let language_model_generator =- self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;language_model_generator.reset_state();Ok(())}@@ -255,11 +208,9 @@ impl ChatSession {config: RunConfig,) -> Result<(Output, Context), Error> {self.reconfigure_language_model_generator(context)?;- let output =- self.run_internal(input, config, None::<fn(Output) -> bool>)?;+ let output = self.run_internal(input, config, None::<fn(Output) -> bool>)?;let new_context = self.build_context_from_language_model_generator()?;- let language_model_generator =- self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;language_model_generator.reset_state();Ok((output, new_context))}@@ -274,11 +225,7 @@ impl ChatSession {F: Fn(Output) -> bool,{let run_start = Instant::now();- let text = self.input_processor.process(- &input,- config.enable_thinking,- config.tokens_limit > 0,- )?;+ let text = self.input_processor.process(&input, config.enable_thinking, config.tokens_limit > 0)?;let tokens: Vec<u64> = self.tokenizer.encode(text.as_str(), false)@@ -288,18 +235,11 @@ impl ChatSession {.map(|&id| id as u64).collect();- let language_model_config = self- .model_metadata- .model_config- .as_language_model()- .ok_or(Error::UnableToLoadConfig)?;-- let language_model_generator =- self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- let context_length = language_model_generator- .decoding_config- .context_length- .resolve(language_model_config);+ let language_model_config =+ self.model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;++ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let context_length = language_model_generator.decoding_config.context_length.resolve(language_model_config);if tokens.len() >= context_length {return Err(Error::ContextLengthExceeded);}@@ -307,26 +247,16 @@ impl ChatSession {let prefix_offset = language_model_generator.tokens.len();let prefix_len_before = prefix_offset;- let eos_tokens: Vec<u64> = language_model_config- .generation_config- .stop_token_ids- .iter()- .map(|&x| x as u64)- .collect();+ let eos_tokens: Vec<u64> =+ language_model_config.generation_config.stop_token_ids.iter().map(|&x| x as u64).collect();- let sampling_method =- config.sampling_policy.resolve(language_model_config);+ let sampling_method = config.sampling_policy.resolve(language_model_config);- let mut compiled_grammar: Option<CompiledGrammar> =- if let Some(ref grammar_config) = config.grammar_config {- Some(CompiledGrammar::from_config(- grammar_config,- None,- &self.tokenizer_info,- )?)- } else {- None- };+ let mut compiled_grammar: Option<CompiledGrammar> = if let Some(ref grammar_config) = config.grammar_config {+ Some(CompiledGrammar::from_config(grammar_config, None, &self.tokenizer_info)?)+ } else {+ None+ };let prefill_start = Instant::now();let sample_suffix = config.tokens_limit > 0;@@ -341,10 +271,8 @@ impl ChatSession {let prefill_duration = prefill_start.elapsed().as_secs_f64();language_model_generator.clear_cache();- let prefill_suffix_length = language_model_generator- .decoding_config- .prefill_step_size- .resolve(language_model_config);+ let prefill_suffix_length =+ language_model_generator.decoding_config.prefill_step_size.resolve(language_model_config);let run_context = RunContext {eos_tokens,@@ -358,19 +286,12 @@ impl ChatSession {run_start,};- let grammar_terminated = compiled_grammar- .as_ref()- .map(|g| g.is_terminated())- .unwrap_or(false);+ let grammar_terminated = compiled_grammar.as_ref().map(|g| g.is_terminated()).unwrap_or(false);let prefill_finish_reason = if grammar_terminated {Some(FinishReason::Stop)} else {- Self::check_finish_reason(- &run_context,- language_model_generator,- &prefill_tokens,- )+ Self::check_finish_reason(&run_context, language_model_generator, &prefill_tokens)};let prefill_output = Self::build_output(&self.tokenizer,@@ -392,21 +313,15 @@ impl ChatSession {if prefill_should_continue {return Ok(prefill_output);} else {- return Ok(prefill_output- .clone_with_finish_reason(Some(FinishReason::Cancelled)));+ return Ok(prefill_output.clone_with_finish_reason(Some(FinishReason::Cancelled)));}}let can_use_async =- language_model_generator.decoding_config.generate_suffix_length()- == 1- && compiled_grammar.is_none();+ language_model_generator.decoding_config.generate_suffix_length() == 1 && compiled_grammar.is_none();let generate_output = if can_use_async {- let batch_size = language_model_generator- .decoding_config- .async_batch_size- .resolve(&self.model_path);+ let batch_size = language_model_generator.decoding_config.async_batch_size.resolve(&self.model_path);Self::run_async_batch(&self.tokenizer,&self.output_parser,@@ -429,8 +344,7 @@ impl ChatSession {};language_model_generator.clear_cache();- Ok(generate_output- .clone_with_duration(run_start.elapsed().as_secs_f64()))+ Ok(generate_output.clone_with_duration(run_start.elapsed().as_secs_f64()))}fn run_sync_generate<F>(@@ -451,28 +365,19 @@ impl ChatSession {loop {let generate_start = Instant::now();- let generate_result = language_model_generator.generate(- compiled_grammar_mut.as_deref_mut(),- sampling_method,- )?;+ let generate_result =+ language_model_generator.generate(compiled_grammar_mut.as_deref_mut(), sampling_method)?;let generate_tokens = generate_result.tokens.clone();let generate_duration = generate_start.elapsed().as_secs_f64();generate_results.push(generate_result);generate_durations.push(generate_duration);- let grammar_terminated = compiled_grammar_mut- .as_ref()- .map(|g| g.is_terminated())- .unwrap_or(false);+ let grammar_terminated = compiled_grammar_mut.as_ref().map(|g| g.is_terminated()).unwrap_or(false);let generate_finish_reason = if grammar_terminated {Some(FinishReason::Stop)} else {- Self::check_finish_reason(- run_context,- language_model_generator,- &generate_tokens,- )+ Self::check_finish_reason(run_context, language_model_generator, &generate_tokens)};let generate_output = Self::build_output(tokenizer,@@ -494,9 +399,7 @@ impl ChatSession {if generate_should_continue {return Ok(generate_output);} else {- return Ok(generate_output.clone_with_finish_reason(Some(- FinishReason::Cancelled,- )));+ return Ok(generate_output.clone_with_finish_reason(Some(FinishReason::Cancelled)));}}}@@ -514,11 +417,8 @@ impl ChatSession {whereF: Fn(Output) -> bool,{- let remaining_by_limit = run_context- .tokens_limit- .saturating_sub(run_context.prefill_result.tokens.len());- let remaining_by_context =- run_context.context_length.saturating_sub(llm.tokens.len());+ let remaining_by_limit = run_context.tokens_limit.saturating_sub(run_context.prefill_result.tokens.len());+ let remaining_by_context = run_context.context_length.saturating_sub(llm.tokens.len());let tokens_to_generate = remaining_by_limit.min(remaining_by_context);llm.prepare_async(tokens_to_generate);@@ -527,47 +427,34 @@ impl ChatSession {let start_time = Instant::now();let last_nanos = Arc::new(AtomicU64::new(0));- let mut results: Vec<GenerateResult> =- Vec::with_capacity(tokens_to_generate);+ let mut results: Vec<GenerateResult> = Vec::with_capacity(tokens_to_generate);let mut durations: Vec<f64> = Vec::with_capacity(tokens_to_generate);let mut finish_reason = FinishReason::Length;let mut next_to_submit = 0;let mut in_flight = 0;while in_flight > 0 || next_to_submit < tokens_to_generate {- let batch_end =- std::cmp::min(next_to_submit + batch_size, tokens_to_generate);+ let batch_end = std::cmp::min(next_to_submit + batch_size, tokens_to_generate);let batch_submitted = batch_end - next_to_submit;let accepted_before_batch = results.len();- let cache_slice = llm- .context- .cache_layers- .borrow()- .slice(&llm.context.mtl_context, 0..batch_submitted);+ let cache_slice = llm.context.cache_layers.borrow().slice(&llm.context.mtl_context, 0..batch_submitted);for idx in next_to_submit..batch_end {let batch_sender = sender.clone();let last_callback_nanos = last_nanos.clone();let batch_start_time = start_time;llm.async_generate(idx, sampling_method, move |token| {- let now_nanos =- batch_start_time.elapsed().as_nanos() as u64;- let prev_nanos =- last_callback_nanos.swap(now_nanos, Ordering::SeqCst);- let _ = batch_sender.send((- idx,- token,- now_nanos.saturating_sub(prev_nanos),- ));+ let now_nanos = batch_start_time.elapsed().as_nanos() as u64;+ let prev_nanos = last_callback_nanos.swap(now_nanos, Ordering::SeqCst);+ let _ = batch_sender.send((idx, token, now_nanos.saturating_sub(prev_nanos)));})?;in_flight += 1;}next_to_submit = batch_end;for _ in 0..batch_submitted {- let (_, token, duration_nanos) =- receiver.recv().map_err(|_| Error::SamplingFailed)?;+ let (_, token, duration_nanos) = receiver.recv().map_err(|_| Error::SamplingFailed)?;in_flight -= 1;let duration = nanos_to_secs(duration_nanos);@@ -585,15 +472,8 @@ impl ChatSession {finish_reason = FinishReason::ContextLimitReached;true} else if let Some(progress_fn) = progress {- let output = Self::build_output(- tokenizer,- output_parser,- run_context,- llm,- &results,- &durations,- None,- )?;+ let output =+ Self::build_output(tokenizer, output_parser, run_context, llm, &results, &durations, None)?;if !progress_fn(output) {finish_reason = FinishReason::Cancelled;true@@ -610,13 +490,12 @@ impl ChatSession {in_flight -= 1;}if let Some(slice) = cache_slice {- let accepted_in_batch =- results.len().saturating_sub(accepted_before_batch);+ let accepted_in_batch = results.len().saturating_sub(accepted_before_batch);if accepted_in_batch < batch_submitted {- llm.context.cache_layers.borrow_mut().apply_slice(- &slice,- Some(accepted_in_batch..batch_submitted),- );+ llm.context+ .cache_layers+ .borrow_mut()+ .apply_slice(&slice, Some(accepted_in_batch..batch_submitted));}}return Self::build_output(@@ -632,70 +511,39 @@ impl ChatSession {}}- Self::build_output(- tokenizer,- output_parser,- run_context,- llm,- &results,- &durations,- Some(finish_reason),- )+ Self::build_output(tokenizer, output_parser, run_context, llm, &results, &durations, Some(finish_reason))}fn reconfigure_language_model_generator(&mut self,context: Option<&Context>,) -> Result<(), Error> {- let language_model_generator =- self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;language_model_generator.reset_state();if let Some(ctx) = context {- let mut llm_state =- language_model_generator.context.cache_layers.borrow_mut();- for (_layer_idx, (ctx_layer, gen_layer)) in ctx- .cache_layers- .data- .iter()- .zip(llm_state.data.iter_mut())- .enumerate()+ let mut llm_state = language_model_generator.context.cache_layers.borrow_mut();+ for (_layer_idx, (ctx_layer, gen_layer)) in+ ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate(){match (ctx_layer, gen_layer) {- (- CacheLayer::Transformer(src),- CacheLayer::Transformer(dst),- ) => {+ (CacheLayer::Transformer(src), CacheLayer::Transformer(dst)) => {let copy_rows = src.prefix_segment_length();if copy_rows > 0 {{let mut dst_keys = dst.keys.borrow_mut();let src_keys = src.keys.borrow();- dst_keys.copy_slice(- &src_keys,- 1,- 0..copy_rows,- 0,- );+ dst_keys.copy_slice(&src_keys, 1, 0..copy_rows, 0);}{let mut dst_values = dst.values.borrow_mut();let src_values = src.values.borrow();- dst_values.copy_slice(- &src_values,- 1,- 0..copy_rows,- 0,- );+ dst_values.copy_slice(&src_values, 1, 0..copy_rows, 0);}}dst.state = src.state.clone();- dst.prefix_token_positions =- src.prefix_token_positions.clone();+ dst.prefix_token_positions = src.prefix_token_positions.clone();},- (- CacheLayer::StateSpace(src),- CacheLayer::StateSpace(dst),- ) => {+ (CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {{let mut dst_conv = dst.conv_state.borrow_mut();let src_conv = src.conv_state.borrow();@@ -707,9 +555,7 @@ impl ChatSession {dst_ssm.copy_from_array(&src_ssm);}},- _ => panic!(- "Layer type mismatch when reconfiguring language model generator cache"- ),+ _ => panic!("Layer type mismatch when reconfiguring language model generator cache"),}}drop(llm_state);@@ -719,16 +565,10 @@ impl ChatSession {Ok(())}- fn build_context_from_language_model_generator(- &self- ) -> Result<Context, Error> {- let language_model_generator =- self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- let cache_layers = language_model_generator- .context- .cache_layers- .borrow()- .clone(&language_model_generator.context.mtl_context);+ fn build_context_from_language_model_generator(&self) -> Result<Context, Error> {+ let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let cache_layers =+ language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.mtl_context);let context = Context::new(language_model_generator.tokens.clone(),cache_layers,@@ -744,14 +584,10 @@ impl ChatSession {language_model_generator: &LanguageModelGenerator,new_tokens: &[u64],) -> Option<FinishReason> {- let start_idx =- run_context.prefix_len_before + run_context.input_tokens_len;- let total_new_tokens =- language_model_generator.tokens.len().saturating_sub(start_idx);- let has_eos =- new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));- let context_limit =- language_model_generator.tokens.len() >= run_context.context_length;+ let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;+ let total_new_tokens = language_model_generator.tokens.len().saturating_sub(start_idx);+ let has_eos = new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));+ let context_limit = language_model_generator.tokens.len() >= run_context.context_length;if has_eos {Some(FinishReason::Stop)@@ -774,9 +610,7 @@ impl ChatSession {.chain(generate_results.iter().flat_map(|r| r.tokens.iter())).map(|&v| v as u32).collect();- tokenizer- .decode(&generated_tokens, true)- .map_err(|_| Error::UnableToDecodeText)+ tokenizer.decode(&generated_tokens, true).map_err(|_| Error::UnableToDecodeText)}fn build_output(@@ -788,16 +622,10 @@ impl ChatSession {generate_durations: &[f64],finish_reason: Option<FinishReason>,) -> Result<Output, Error> {- let text = Self::decode_generated_tokens(- tokenizer,- &run_context.prefill_result.tokens,- generate_results,- )?;+ let text = Self::decode_generated_tokens(tokenizer, &run_context.prefill_result.tokens, generate_results)?;let parsed = output_parser.parse(text);- let start_idx =- run_context.prefix_len_before + run_context.input_tokens_len;- let output_tokens =- language_model_generator.tokens.len().saturating_sub(start_idx);+ let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;+ let output_tokens = language_model_generator.tokens.len().saturating_sub(start_idx);Ok(Output {text: parsed,@@ -807,9 +635,7 @@ impl ChatSession {run_context.prefill_suffix_length,generate_results.to_vec(),generate_durations.to_vec(),- language_model_generator- .decoding_config- .generate_suffix_length(),+ language_model_generator.decoding_config.generate_suffix_length(),run_context.run_start.elapsed().as_secs_f64(),run_context.input_tokens_len,output_tokens,@@ -834,13 +660,11 @@ impl ChatSession {let tokens_per_second = tokens_count as f64 / prefill_duration;let processed_tokens = tokens_count_input + tokens_count;- let processed_tokens_per_second =- processed_tokens as f64 / prefill_duration;+ let processed_tokens_per_second = processed_tokens as f64 / prefill_duration;let model_run_count = prefill_result.forwardpass_durations.len();let model_run_average_duration =- prefill_result.forwardpass_durations.iter().sum::<f64>()- / model_run_count as f64;+ prefill_result.forwardpass_durations.iter().sum::<f64>() / model_run_count as f64;StepStats {duration: prefill_duration,@@ -859,23 +683,16 @@ impl ChatSession {let generate_stats: Option<StepStats>;if generate_results.len() != 0 {let duration = generate_durations.iter().sum::<f64>();- let tokens_count = generate_results- .iter()- .flat_map(|result| result.tokens.clone())- .collect::<Vec<u64>>()- .len();+ let tokens_count =+ generate_results.iter().flat_map(|result| result.tokens.clone()).collect::<Vec<u64>>().len();let tokens_per_second: f64 = tokens_count as f64 / duration;let model_run_count = generate_results.len();- let model_run_average_duration = generate_results- .iter()- .map(|result| result.forwardpass_duration)- .sum::<f64>()- / model_run_count as f64;+ let model_run_average_duration =+ generate_results.iter().map(|result| result.forwardpass_duration).sum::<f64>() / model_run_count as f64;let run_count = generate_durations.len();- let run_average_duration =- generate_durations.iter().sum::<f64>() / run_count as f64;+ let run_average_duration = generate_durations.iter().sum::<f64>() / run_count as f64;generate_stats = Some(StepStats {duration,diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rsindex 8db0c4063..9ff58d6b9 100644--- a/crates/uzu/src/session/classification_session.rs+++ b/crates/uzu/src/session/classification_session.rs@@ -28,26 +28,20 @@ impl ClassificationSession {return Err(Error::ModelFolderNotFound);}- let config_file = std::fs::File::open(&config_path)- .map_err(|_| Error::UnableToLoadConfig)?;+ let config_file = std::fs::File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;let model_metadata: ModelMetadata =- serde_json::from_reader(std::io::BufReader::new(config_file))- .map_err(|_| Error::UnableToLoadConfig)?;+ serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;let tokenizer_path = model_path.join("tokenizer.json");if !tokenizer_path.exists() {return Err(Error::UnableToLoadTokenizer);}- let tokenizer = Tokenizer::from_file(&tokenizer_path)- .map_err(|_| Error::UnableToLoadTokenizer)?;+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;- let classifier_model_config = model_metadata- .model_config- .as_classifier()- .ok_or(Error::UnableToLoadConfig)?;- let input_processor = Box::new(InputProcessorDefault::new(- classifier_model_config.message_processor_config.clone(),- )) as Box<dyn InputProcessor>;+ let classifier_model_config = model_metadata.model_config.as_classifier().ok_or(Error::UnableToLoadConfig)?;+ let input_processor =+ Box::new(InputProcessorDefault::new(classifier_model_config.message_processor_config.clone()))+ as Box<dyn InputProcessor>;let classifier = Classifier::new(&model_path)?;Ok(Self {@@ -75,11 +69,9 @@ impl ClassificationSession {.collect();let token_positions: Vec<usize> = (0..tokens.len()).collect();- let preprocessing_duration =- preprocessing_start.elapsed().as_secs_f64();+ let preprocessing_duration = preprocessing_start.elapsed().as_secs_f64();- let mut output =- self.classifier.classify_tokens(tokens, token_positions)?;+ let mut output = self.classifier.classify_tokens(tokens, token_positions)?;output.stats.preprocessing_duration = preprocessing_duration;Ok(output)diff --git a/crates/uzu/src/session/config/decoding_config.rs b/crates/uzu/src/session/config/decoding_config.rsindex 4add97589..d2042126d 100644--- a/crates/uzu/src/session/config/decoding_config.rs+++ b/crates/uzu/src/session/config/decoding_config.rs@@ -1,9 +1,6 @@use crate::session::{config::SpeculatorConfig,- parameter::{- AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize,- SamplingSeed,- },+ parameter::{AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize, SamplingSeed},};#[derive(Clone)]diff --git a/crates/uzu/src/session/config/grammar_config.rs b/crates/uzu/src/session/config/grammar_config.rsindex 68f3fc91b..054dc7952 100644--- a/crates/uzu/src/session/config/grammar_config.rs+++ b/crates/uzu/src/session/config/grammar_config.rs@@ -63,8 +63,8 @@ impl GrammarConfig {T: JsonSchema,{let schema_root = schemars::schema_for!(T);- let schema_str = serde_json::to_string(&schema_root)- .map_err(|e| format!("Failed to serialize schema: {}", e))?;+ let schema_str =+ serde_json::to_string(&schema_root).map_err(|e| format!("Failed to serialize schema: {}", e))?;Ok(Self::JsonSchema {schema: schema_str,@@ -85,8 +85,8 @@ impl GrammarConfig {T: JsonSchema,{let schema_root = schemars::schema_for!(T);- let schema_str = serde_json::to_string(&schema_root)- .map_err(|e| format!("Failed to serialize schema: {}", e))?;+ let schema_str =+ serde_json::to_string(&schema_root).map_err(|e| format!("Failed to serialize schema: {}", e))?;Ok(Self::JsonSchema {schema: schema_str,diff --git a/crates/uzu/src/session/config/speculator_config.rs b/crates/uzu/src/session/config/speculator_config.rsindex cdcee7f6f..19e14c25c 100644--- a/crates/uzu/src/session/config/speculator_config.rs+++ b/crates/uzu/src/session/config/speculator_config.rs@@ -1,8 +1,6 @@use std::sync::Arc;-use crate::speculators::{- empty_speculator::EmptySpeculator, speculator::Speculator,-};+use crate::speculators::{empty_speculator::EmptySpeculator, speculator::Speculator};#[derive(Clone)]pub struct SpeculatorConfig {@@ -16,10 +14,8 @@ impl SpeculatorConfig {speculator: Arc<dyn Speculator>,) -> Self {let max_number_of_speculated_tokens: usize = 64;- let effective_number_of_speculated_tokens = std::cmp::min(- number_of_speculated_tokens,- max_number_of_speculated_tokens,- );+ let effective_number_of_speculated_tokens =+ std::cmp::min(number_of_speculated_tokens, max_number_of_speculated_tokens);Self {number_of_speculated_tokens: effective_number_of_speculated_tokens,speculator,diff --git a/crates/uzu/src/session/helpers/context.rs b/crates/uzu/src/session/helpers/context.rsindex c55e0d4bb..fa7c63627 100644--- a/crates/uzu/src/session/helpers/context.rs+++ b/crates/uzu/src/session/helpers/context.rs@@ -1,7 +1,4 @@-use crate::{- backends::metal::Metal, forward_pass::cache_layers::CacheLayers,- session::config::DecodingConfig,-};+use crate::{backends::metal::Metal, forward_pass::cache_layers::CacheLayers, session::config::DecodingConfig};pub struct Context {pub tokens: Vec<u64>,diff --git a/crates/uzu/src/session/helpers/input_processor.rs b/crates/uzu/src/session/helpers/input_processor.rsindex 8a3a27c94..d55887bff 100644--- a/crates/uzu/src/session/helpers/input_processor.rs+++ b/crates/uzu/src/session/helpers/input_processor.rs@@ -54,12 +54,8 @@ fn strftime_now(format_string: String) -> String {loop {let mut buffer = vec![0u8; buf_len];- let written = libc::strftime(- buffer.as_mut_ptr() as *mut libc::c_char,- buffer.len(),- c_format.as_ptr(),- &tm,- );+ let written =+ libc::strftime(buffer.as_mut_ptr() as *mut libc::c_char, buffer.len(), c_format.as_ptr(), &tm);if written > 0 {buffer.truncate(written as usize);@@ -84,17 +80,13 @@ impl InputProcessor for InputProcessorDefault {) -> Result<String, Error> {let messages = input.get_messages();for message in &messages {- if message.role != Role::Assistant- && message.reasoning_content.is_some()- {+ if message.role != Role::Assistant && message.reasoning_content.is_some() {return Err(Error::UnexpectedReasoningContent);}}- let messages = messages- .into_iter()- .map(|message| message.resolve(&self.message_processing_config))- .collect::<Vec<_>>();+ let messages =+ messages.into_iter().map(|message| message.resolve(&self.message_processing_config)).collect::<Vec<_>>();let template = self.message_processing_config.prompt_template.clone();let bos_token = self.message_processing_config.bos_token.clone();@@ -102,12 +94,8 @@ impl InputProcessor for InputProcessorDefault {let mut environment = Environment::new();environment.set_unknown_method_callback(unknown_method_callback);environment.add_function("strftime_now", strftime_now);- environment- .add_template(template_name, template.as_str())- .map_err(|_| Error::UnableToLoadPromptTemplate)?;- let template = environment- .get_template(template_name)- .map_err(|_| Error::UnableToLoadPromptTemplate)?;+ environment.add_template(template_name, template.as_str()).map_err(|_| Error::UnableToLoadPromptTemplate)?;+ let template = environment.get_template(template_name).map_err(|_| Error::UnableToLoadPromptTemplate)?;let result = template.render(context!(diff --git a/crates/uzu/src/session/helpers/output_parser.rs b/crates/uzu/src/session/helpers/output_parser.rsindex 42a9ab88d..c4dbeed0e 100644--- a/crates/uzu/src/session/helpers/output_parser.rs+++ b/crates/uzu/src/session/helpers/output_parser.rs@@ -31,12 +31,8 @@ impl OutputParser {let parsed_text = match &self.regex {Some(regex) => match regex.captures(&text) {Some(captures) => {- let chain_of_thought = captures- .name("chain_of_thought")- .map(|m| m.as_str().to_string());- let response = captures- .name("response")- .map(|m| m.as_str().to_string());+ let chain_of_thought = captures.name("chain_of_thought").map(|m| m.as_str().to_string());+ let response = captures.name("response").map(|m| m.as_str().to_string());ParsedText {chain_of_thought,response,diff --git a/crates/uzu/src/session/parameter/context_length.rs b/crates/uzu/src/session/parameter/context_length.rsindex 9abaca6e0..bdee47a44 100644--- a/crates/uzu/src/session/parameter/context_length.rs+++ b/crates/uzu/src/session/parameter/context_length.rs@@ -1,6 +1,4 @@-use crate::{- config::LanguageModelConfig, session::parameter::ConfigResolvableValue,-};+use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};#[derive(Debug, Clone, Copy)]pub enum ContextLength {diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rsindex fc81a7cc5..235f14076 100644--- a/crates/uzu/src/session/parameter/prefill_step_size.rs+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs@@ -1,10 +1,7 @@-use crate::{- config::LanguageModelConfig, session::parameter::ConfigResolvableValue,-};+use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};fn env_prefill_step_size_default_override() -> Option<usize> {- static OVERRIDE: std::sync::OnceLock<Option<usize>> =- std::sync::OnceLock::new();+ static OVERRIDE: std::sync::OnceLock<Option<usize>> = std::sync::OnceLock::new();*OVERRIDE.get_or_init(|| {let raw = std::env::var("UZU_PREFILL_STEP_SIZE").ok()?;let trimmed = raw.trim();@@ -34,8 +31,7 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {config: &LanguageModelConfig,) -> usize {let default_limit: usize = 4096;- let model_context_length =- config.model_config.transformer_config.context_length;+ let model_context_length = config.model_config.transformer_config.context_length;let minimal_sliding_window_size = config.decoder_config()@@ -49,11 +45,10 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {.min().unwrap_or(usize::MAX);- let maximal_value =- [model_context_length, minimal_sliding_window_size, default_limit]- .into_iter()- .min()- .unwrap_or(default_limit);+ let maximal_value = [model_context_length, minimal_sliding_window_size, default_limit]+ .into_iter()+ .min()+ .unwrap_or(default_limit);let proposed_value = match self {PrefillStepSize::Default => {diff --git a/crates/uzu/src/session/parameter/sampling_policy.rs b/crates/uzu/src/session/parameter/sampling_policy.rsindex 070d95ced..94d8d5b73 100644--- a/crates/uzu/src/session/parameter/sampling_policy.rs+++ b/crates/uzu/src/session/parameter/sampling_policy.rs@@ -1,6 +1,4 @@-use crate::{- config::LanguageModelConfig, session::parameter::ConfigResolvableValue,-};+use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};#[derive(Debug, Copy, Clone, PartialEq)]pub enum SamplingMethod {@@ -33,9 +31,7 @@ impl Default for SamplingPolicy {}}-impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod>- for SamplingPolicy-{+impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod> for SamplingPolicy {fn resolve(&self,config: &LanguageModelConfig,diff --git a/crates/uzu/src/session/types/message.rs b/crates/uzu/src/session/types/message.rsindex c49ab6154..f36961d3e 100644--- a/crates/uzu/src/session/types/message.rs+++ b/crates/uzu/src/session/types/message.rs@@ -43,9 +43,7 @@ impl Message {}}-impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>>- for Message-{+impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>> for Message {fn resolve(&self,config: &MessageProcessorConfig,@@ -56,10 +54,7 @@ impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>>Role::Assistant => config.assistant_role_name.clone(),};let content = self.content.clone();- let mut result = HashMap::from([- (String::from("role"), role),- (String::from("content"), content),- ]);+ let mut result = HashMap::from([(String::from("role"), role), (String::from("content"), content)]);if let Some(reasoning_content) = self.reasoning_content.clone() {result.insert(String::from("reasoning_content"), reasoning_content);}diff --git a/crates/uzu/src/tracer/mod.rs b/crates/uzu/src/tracer/mod.rsindex 43349c40d..7f377abaa 100644--- a/crates/uzu/src/tracer/mod.rs+++ b/crates/uzu/src/tracer/mod.rs@@ -7,6 +7,5 @@mod trace_validator;pub use trace_validator::{- ArrayTransform, TraceValidator, TracerValidationMetrics,- TracerValidationResult, TracerValidationResults,+ ArrayTransform, TraceValidator, TracerValidationMetrics, TracerValidationResult, TracerValidationResults,};diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex f358cae2f..22959739e 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -17,10 +17,7 @@ use num_traits::NumCast;use crate::{ArrayElement, DataType,- backends::metal::{- KVCacheUpdate, MTLCommandBuffer, MTLCommandQueue, MTLContext, Metal,- MetalArray,- },+ backends::metal::{KVCacheUpdate, MTLCommandBuffer, MTLCommandQueue, MTLContext, Metal, MetalArray},classifier::Classifier,config::ModelMetadata,encodable_block::{EncodableBlock, EncodingParameters, Sampling},@@ -37,10 +34,7 @@ use crate::{parameters::{ParameterLoader, ParameterTree, read_safetensors_metadata},session::{config::{DecodingConfig, SpeculatorConfig},- parameter::{- AsyncBatchSize, ConfigResolvableValue, ContextLength, ContextMode,- PrefillStepSize, SamplingSeed,- },+ parameter::{AsyncBatchSize, ConfigResolvableValue, ContextLength, ContextMode, PrefillStepSize, SamplingSeed},types::Error,},};@@ -141,8 +135,7 @@ impl TracerValidationResults {}pub fn is_valid(&self) -> bool {- self.number_of_tokens_violations()- <= self.number_of_allowed_tokens_violations()+ self.number_of_tokens_violations() <= self.number_of_allowed_tokens_violations()}}@@ -186,18 +179,15 @@ impl TraceValidator {return Err(Error::ModelFolderNotFound);}- let config_file =- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;let metadata: ModelMetadata =- serde_json::from_reader(std::io::BufReader::new(config_file))- .map_err(|_| Error::UnableToLoadConfig)?;+ serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;let context = if metadata.model_config.is_classifier() {let classifier = Classifier::new(model_path)?;ModelContext::Classifier(classifier)} else {- let prefill_step_size =- Self::determine_prefill_step_size(model_path);+ let prefill_step_size = Self::determine_prefill_step_size(model_path);let decoding_config = DecodingConfig::new(ContextMode::default(),ContextLength::default(),@@ -207,17 +197,9 @@ impl TraceValidator {AsyncBatchSize::default(),false,);- let mut llm_context = LanguageModelGeneratorContext::new(- model_path,- &decoding_config,- )?;- let desired_suffix_length =- prefill_step_size.max(decoding_config.generate_suffix_length());- Self::ensure_llm_context_capacity(- &decoding_config,- desired_suffix_length,- &mut llm_context,- );+ let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;+ let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());+ Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);ModelContext::LanguageModelGenerator(llm_context)};@@ -235,12 +217,8 @@ impl TraceValidator {}match &mut self.context {- ModelContext::LanguageModelGenerator(ctx) => {- Self::run_llm_validation(ctx, &traces_path)- },- ModelContext::Classifier(classifier) => {- Self::run_classifier_validation(classifier, &traces_path)- },+ ModelContext::LanguageModelGenerator(ctx) => Self::run_llm_validation(ctx, &traces_path),+ ModelContext::Classifier(classifier) => Self::run_classifier_validation(classifier, &traces_path),}}@@ -260,21 +238,13 @@ impl TraceValidator {ctx: &LanguageModelGeneratorContext,traces_path: &Path,) -> Result<TracerValidationResults, Error> {- let traces_file =- File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;+ let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;let traces_loader =- ParameterLoader::new(&traces_file, ctx.mtl_context.as_ref())- .map_err(|_| Error::UnableToLoadWeights)?;+ ParameterLoader::new(&traces_file, ctx.mtl_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let traces_view = traces_loader.tree();- let token_ids = Self::load_array_as_vec::<i32, u64>(- &traces_view,- "activation_trace.token_ids",- );- let token_positions = Self::load_array_as_vec::<i32, usize>(- &traces_view,- "activation_trace.token_positions",- );+ let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");+ let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");let token_seeds: Vec<u64> = vec![0; token_ids.len()];let mut state = ForwardPassState::new_llm(@@ -299,17 +269,9 @@ impl TraceValidator {None,);- let command_buffer = ctx- .mtl_context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer = ctx.mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");- ctx.executables.encode(- &mut state,- &command_buffer,- &EncodingParameters::new(false, false, false),- );+ ctx.executables.encode(&mut state, &command_buffer, &EncodingParameters::new(false, false, false));command_buffer.commit();command_buffer.wait_until_completed();@@ -317,53 +279,30 @@ impl TraceValidator {let data_type = ctx.model_shape.activation_data_type();// Common layer validation- let mut results =- Self::validate_layer_traces(&traces, &traces_view, data_type);+ let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);// LLM-specific: KV cache validationlet transformer_layers: Vec<usize> = {let cache = state.cache_layers().unwrap().borrow();- cache- .data- .iter()- .enumerate()- .filter_map(|(index, layer)| {- layer.as_transformer().map(|_| index)- })- .collect()+ cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_transformer().map(|_| index)).collect()};for index in transformer_layers {- let arrays =- state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);+ let arrays = state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);- if let Ok(expected) =- traces_view.leaf(&format!("updated_kv_cache.{}.keys", index))- {+ if let Ok(expected) = traces_view.leaf(&format!("updated_kv_cache.{}.keys", index)) {let keys = arrays[0].borrow();results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.keys", index),- metrics: Self::validate_array(- data_type,- &expected,- &keys,- Some(ArrayTransform::KVCacheSlice),- ),+ metrics: Self::validate_array(data_type, &expected, &keys, Some(ArrayTransform::KVCacheSlice)),});}- if let Ok(expected) =- traces_view.leaf(&format!("updated_kv_cache.{}.values", index))- {+ if let Ok(expected) = traces_view.leaf(&format!("updated_kv_cache.{}.values", index)) {let values = arrays[1].borrow();results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.values", index),- metrics: Self::validate_array(- data_type,- &expected,- &values,- Some(ArrayTransform::KVCacheSlice),- ),+ metrics: Self::validate_array(data_type, &expected, &values, Some(ArrayTransform::KVCacheSlice)),});}}@@ -371,30 +310,17 @@ impl TraceValidator {// LLM-specific: SSM state validationlet ssm_layers: Vec<usize> = {let cache = state.cache_layers().unwrap().borrow();- cache- .data- .iter()- .enumerate()- .filter_map(|(index, layer)| {- layer.as_state_space().map(|_| index)- })- .collect()+ cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_state_space().map(|_| index)).collect()};for index in ssm_layers {- let arrays = state.arrays(&[- ArrayId::SsmConvState(index),- ArrayId::SsmState(index),- ]);+ let arrays = state.arrays(&[ArrayId::SsmConvState(index), ArrayId::SsmState(index)]);let conv_state = arrays[0].borrow();let ssm_state = arrays[1].borrow();for path in [format!("updated_state.{}.conv_state", index),- format!(- "activation_trace.layer_results.{}.updated_state.conv_state",- index- ),+ format!("activation_trace.layer_results.{}.updated_state.conv_state", index),] {if let Ok(expected) = traces_view.leaf(&path) {results.push(TracerValidationResult {@@ -411,30 +337,21 @@ impl TraceValidator {for path in [format!("updated_state.{}.ssm_state", index),- format!(- "activation_trace.layer_results.{}.updated_state.ssm_state",- index- ),+ format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),] {if let Ok(expected) = traces_view.leaf(&path) {results.push(TracerValidationResult {name: path,- metrics: Self::validate_array(- data_type, &expected, &ssm_state, None,- ),+ metrics: Self::validate_array(data_type, &expected, &ssm_state, None),});}}}// LLM-specific: Token comparison- let tokens_violation_indices = if let Ok(expected_logits) =- traces_view.leaf("logits")- {- let expected_tokens =- Self::get_tokens_from_logits(&expected_logits);- let produced_tokens =- Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());+ let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf("logits") {+ let expected_tokens = Self::get_tokens_from_logits(&expected_logits);+ let produced_tokens = Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());expected_tokens.iter().zip(produced_tokens.iter())@@ -466,47 +383,34 @@ impl TraceValidator {classifier: &mut Classifier,traces_path: &Path,) -> Result<TracerValidationResults, Error> {- let traces_file =- File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;+ let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;let mtl_context = classifier.context.mtl_context.clone();let traces_loader =- ParameterLoader::new(&traces_file, mtl_context.as_ref())- .map_err(|_| Error::UnableToLoadWeights)?;+ ParameterLoader::new(&traces_file, mtl_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let traces_view = traces_loader.tree();- let has_token_ids =- traces_view.leaf("activation_trace.token_ids").is_ok();- let has_token_positions =- traces_view.leaf("activation_trace.token_positions").is_ok();+ let has_token_ids = traces_view.leaf("activation_trace.token_ids").is_ok();+ let has_token_positions = traces_view.leaf("activation_trace.token_positions").is_ok();if !has_token_ids || !has_token_positions {return Ok(Self::handle_missing_tokens(&traces_view));}- let token_ids = Self::load_array_as_vec::<i32, u64>(- &traces_view,- "activation_trace.token_ids",- );- let token_positions = Self::load_array_as_vec::<i32, usize>(- &traces_view,- "activation_trace.token_positions",- );+ let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");+ let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");let suffix_length = token_ids.len();- let (_logits, traces) = classifier- .forward_pass_with_traces(&token_ids, &token_positions)- .map_err(|_| Error::GenerateFailed)?;+ let (_logits, traces) =+ classifier.forward_pass_with_traces(&token_ids, &token_positions).map_err(|_| Error::GenerateFailed)?;let data_type = classifier.context.model_shape.activation_data_type();// Common layer validation- let mut results =- Self::validate_layer_traces(&traces, &traces_view, data_type);+ let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);// Classifier-specific: embedding_norm, output_pooling- let classifier_results =- Self::validate_classifier_traces(&traces, &traces_view, data_type);+ let classifier_results = Self::validate_classifier_traces(&traces, &traces_view, data_type);results.extend(classifier_results);Ok(TracerValidationResults {@@ -516,9 +420,7 @@ impl TraceValidator {})}- fn handle_missing_tokens(- traces_view: &ParameterTree<MTLContext>- ) -> TracerValidationResults {+ fn handle_missing_tokens(traces_view: &ParameterTree<MTLContext>) -> TracerValidationResults {if let Ok(expected_logits) = traces_view.leaf("logits") {let reference_shape = expected_logits.shape().to_vec();let metrics = TracerValidationMetrics {@@ -569,108 +471,62 @@ impl TraceValidator {) -> Vec<TracerValidationResult> {let mut results = Vec::new();- let validate = |path: &str,- array: &Ref<MetalArray>|- -> Option<TracerValidationResult> {+ let validate = |path: &str, array: &Ref<MetalArray>| -> Option<TracerValidationResult> {if traces_view.leaf(path).is_ok() {Some(TracerValidationResult {name: path.to_string(),- metrics: Self::validate_array_with_name(- data_type,- traces_view,- path,- array,- ),+ metrics: Self::validate_array_with_name(data_type, traces_view, path, array),})} else {None}};- for (index, layer_traces) in- traces.borrow().layer_results.iter().enumerate()- {+ for (index, layer_traces) in traces.borrow().layer_results.iter().enumerate() {let path = |suffix: &str| -> String {- format!(- "activation_trace.layer_results.{}.activation_trace.{}",- index, suffix- )+ format!("activation_trace.layer_results.{}.activation_trace.{}", index, suffix)};- if let Some(r) = validate(- &path("inputs"),- &layer_traces.borrow().inputs.borrow(),- ) {+ if let Some(r) = validate(&path("inputs"), &layer_traces.borrow().inputs.borrow()) {results.push(r);}- if let Some(r) = validate(- &path("pre_mixer_norm"),- &layer_traces.borrow().pre_attention_norm.borrow(),- ) {+ if let Some(r) = validate(&path("pre_mixer_norm"), &layer_traces.borrow().pre_attention_norm.borrow()) {results.push(r);}- if let Some(r) = validate(- &path("mixer"),- &layer_traces.borrow().attention.borrow(),- ) {+ if let Some(r) = validate(&path("mixer"), &layer_traces.borrow().attention.borrow()) {results.push(r);}- if let Some(r) = validate(- &path("post_mixer_norm"),- &layer_traces.borrow().post_attention_norm.borrow(),- ) {+ if let Some(r) = validate(&path("post_mixer_norm"), &layer_traces.borrow().post_attention_norm.borrow()) {results.push(r);}- if let Some(r) = validate(- &path("mlp_inputs"),- &layer_traces.borrow().mlp_inputs.borrow(),- ) {+ if let Some(r) = validate(&path("mlp_inputs"), &layer_traces.borrow().mlp_inputs.borrow()) {results.push(r);}- if let Some(r) = validate(- &path("pre_mlp_norm"),- &layer_traces.borrow().pre_mlp_norm.borrow(),- ) {+ if let Some(r) = validate(&path("pre_mlp_norm"), &layer_traces.borrow().pre_mlp_norm.borrow()) {results.push(r);}- if let Some(r) =- validate(&path("mlp"), &layer_traces.borrow().mlp.borrow())- {+ if let Some(r) = validate(&path("mlp"), &layer_traces.borrow().mlp.borrow()) {results.push(r);}- if let Some(r) = validate(- &path("post_mlp_norm"),- &layer_traces.borrow().post_mlp_norm.borrow(),- ) {+ if let Some(r) = validate(&path("post_mlp_norm"), &layer_traces.borrow().post_mlp_norm.borrow()) {results.push(r);}- let outputs_path =- format!("activation_trace.layer_results.{}.outputs", index);- if let Some(r) =- validate(&outputs_path, &layer_traces.borrow().outputs.borrow())- {+ let outputs_path = format!("activation_trace.layer_results.{}.outputs", index);+ if let Some(r) = validate(&outputs_path, &layer_traces.borrow().outputs.borrow()) {results.push(r);}}// Output norm (common to all models)- if let Some(r) = validate(- "activation_trace.output_norm",- &traces.borrow().output_norm.borrow(),- ) {+ if let Some(r) = validate("activation_trace.output_norm", &traces.borrow().output_norm.borrow()) {results.push(r);}// Logits (common to all models, but path may vary)- if let Some(r) = validate(- "activation_trace.logits",- &traces.borrow().logits.borrow(),- ) {+ if let Some(r) = validate("activation_trace.logits", &traces.borrow().logits.borrow()) {results.push(r);- } else if let Some(r) =- validate("logits", &traces.borrow().logits.borrow())- {+ } else if let Some(r) = validate("logits", &traces.borrow().logits.borrow()) {results.push(r);}@@ -728,21 +584,9 @@ impl TraceValidator {transform: Option<ArrayTransform>,) -> TracerValidationMetrics {match data_type {- DataType::F16 => Self::validate_array_of_type::<f16>(- expected_array,- produced_array,- transform,- ),- DataType::BF16 => Self::validate_array_of_type::<bf16>(- expected_array,- produced_array,- transform,- ),- DataType::F32 => Self::validate_array_of_type::<f32>(- expected_array,- produced_array,- transform,- ),+ DataType::F16 => Self::validate_array_of_type::<f16>(expected_array, produced_array, transform),+ DataType::BF16 => Self::validate_array_of_type::<bf16>(expected_array, produced_array, transform),+ DataType::F32 => Self::validate_array_of_type::<f32>(expected_array, produced_array, transform),_ => panic!("Unsupported data type: {:?}", data_type),}}@@ -774,12 +618,7 @@ impl TraceValidator {let sliced = permuted.slice(s![start.., .., ..]);let reshaped = sliced.into_owned()- .to_shape(IxDyn(&[- 1,- expected_tokens,- permuted.shape()[1],- permuted.shape()[2],- ]))+ .to_shape(IxDyn(&[1, expected_tokens, permuted.shape()[1], permuted.shape()[2]])).expect("Failed to reshape KV cache slice").to_owned();(expected_view.to_owned(), reshaped)@@ -814,18 +653,14 @@ impl TraceValidator {&& expected_shape.get(0) == Some(&1)&& expected_shape[1..] == produced_shape[..]{- expected_data = expected_data- .to_shape(IxDyn(&produced_shape))- .expect("Failed to reshape expected data")- .to_owned();+ expected_data =+ expected_data.to_shape(IxDyn(&produced_shape)).expect("Failed to reshape expected data").to_owned();} else if produced_shape.len() == expected_shape.len() + 1&& produced_shape.get(0) == Some(&1)&& produced_shape[1..] == expected_shape[..]{- produced_data = produced_data- .to_shape(IxDyn(&expected_shape))- .expect("Failed to reshape produced data")- .to_owned();+ produced_data =+ produced_data.to_shape(IxDyn(&expected_shape)).expect("Failed to reshape produced data").to_owned();}}@@ -837,22 +672,11 @@ impl TraceValidator {);}- let reference: Vec<f32> = expected_data- .iter()- .map(|value| NumCast::from(*value).unwrap_or(0.0))- .collect();-- let result: Vec<f32> = produced_data- .iter()- .map(|value| NumCast::from(*value).unwrap_or(0.0))- .collect();-- Self::compare_arrays(- &reference,- expected_data.shape().to_vec(),- &result,- produced_data.shape().to_vec(),- )+ let reference: Vec<f32> = expected_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();++ let result: Vec<f32> = produced_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();++ Self::compare_arrays(&reference, expected_data.shape().to_vec(), &result, produced_data.shape().to_vec())}fn compare_arrays(@@ -879,9 +703,7 @@ impl TraceValidator {let mut diff_max = 0.0f32;let mut result_nan = false;- for (i, (&exp, &prod)) in- reference.iter().zip(result.iter()).enumerate()- {+ for (i, (&exp, &prod)) in reference.iter().zip(result.iter()).enumerate() {if prod.is_nan() {result_nan = true;}@@ -922,8 +744,7 @@ impl TraceValidator {};let diff_avg = diff_sum / n;- let max_allowed_violations =- (fraction_of_allowed_violations * n).ceil() as usize;+ let max_allowed_violations = (fraction_of_allowed_violations * n).ceil() as usize;TracerValidationMetrics {atol,@@ -951,10 +772,7 @@ impl TraceValidator {// Utility Functions// ========================================================================- fn load_array_as_vec<- SourcePrecision: ArrayElement,- TargetPrecision: NumCast,- >(+ fn load_array_as_vec<SourcePrecision: ArrayElement, TargetPrecision: NumCast>(traces_view: &ParameterTree<MTLContext>,name: &str,) -> Vec<TargetPrecision> {@@ -966,20 +784,10 @@ impl TraceValidator {fn determine_prefill_step_size(model_path: &Path) -> usize {let traces_path = model_path.join("traces.safetensors");if let Ok(file) = File::open(&traces_path) {- if let Ok((_header_len, metadata)) =- read_safetensors_metadata(&file)- {- if let Some(tensor) =- metadata.tensors.get("activation_trace.token_ids")- {+ if let Ok((_header_len, metadata)) = read_safetensors_metadata(&file) {+ if let Some(tensor) = metadata.tensors.get("activation_trace.token_ids") {if let Some(&length) = tensor.shape.first() {- return tensor- .shape- .iter()- .copied()- .max()- .unwrap_or(length)- .max(1);+ return tensor.shape.iter().copied().max().unwrap_or(length).max(1);}}}@@ -992,8 +800,7 @@ impl TraceValidator {desired_suffix_length: usize,context: &mut LanguageModelGeneratorContext,) {- let resolved_prefix_length =- decoding_config.context_length.resolve(&context.model_config);+ let resolved_prefix_length = decoding_config.context_length.resolve(&context.model_config);let current_suffix_length = std::cmp::max(decoding_config.prefill_step_size.resolve(&context.model_config),decoding_config.generate_suffix_length(),@@ -1019,16 +826,11 @@ impl TraceValidator {desired_suffix_length,)));- let intermediate_dtype: DataType =- decoder_config.output_norm_config.scale_precision.into();+ let intermediate_dtype: DataType = decoder_config.output_norm_config.scale_precision.into();context.kv_cache_update = Box::new(- KVCacheUpdate::new(- context.mtl_context.as_ref(),- intermediate_dtype,- resolved_prefix_length,- )- .expect("Failed to create KV cache update kernel"),+ KVCacheUpdate::new(context.mtl_context.as_ref(), intermediate_dtype, resolved_prefix_length)+ .expect("Failed to create KV cache update kernel"),);context.gpu_sampler = Sampling::new(@@ -1043,22 +845,14 @@ impl TraceValidator {fn get_tokens_from_logits(logits: &MetalArray) -> Vec<u64> {let data_type = logits.data_type();match data_type {- DataType::F16 => {- Self::get_tokens_from_logits_of_type::<f16>(logits)- },- DataType::BF16 => {- Self::get_tokens_from_logits_of_type::<bf16>(logits)- },- DataType::F32 => {- Self::get_tokens_from_logits_of_type::<f32>(logits)- },+ DataType::F16 => Self::get_tokens_from_logits_of_type::<f16>(logits),+ DataType::BF16 => Self::get_tokens_from_logits_of_type::<bf16>(logits),+ DataType::F32 => Self::get_tokens_from_logits_of_type::<f32>(logits),_ => panic!("Unsupported data type: {:?}", data_type),}}- fn get_tokens_from_logits_of_type<Precision: ArrayElement>(- logits: &MetalArray- ) -> Vec<u64> {+ fn get_tokens_from_logits_of_type<Precision: ArrayElement>(logits: &MetalArray) -> Vec<u64> {let sampler = ArgmaxSampler {};sampler.sample(logits.as_view::<Precision>())}diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rsindex 4e827aa47..375b876e8 100644--- a/crates/uzu/src/trie.rs+++ b/crates/uzu/src/trie.rs@@ -1,9 +1,7 @@use thiserror::Error;use crate::{- language_model::{- grammar::CompiledGrammar, gumbel::speculator_sample, rng::PRng,- },+ language_model::{grammar::CompiledGrammar, gumbel::speculator_sample, rng::PRng},speculators::speculator::Speculator,};@@ -102,37 +100,22 @@ impl TrieNode {let mut length = 1;let mut height = 0;- let mask = compiled_grammar- .as_deref_mut()- .and_then(|g| g.next_bitmask().unwrap());- let mut root = Self::new(- *prefix.last().unwrap(),- mask,- seed.derive((prefix_length - 1) as u64),- );+ let mask = compiled_grammar.as_deref_mut().and_then(|g| g.next_bitmask().unwrap());+ let mut root = Self::new(*prefix.last().unwrap(), mask, seed.derive((prefix_length - 1) as u64));let mut cur_node = &mut root;let mut cur_node_width = 0;- let mut cur_node_speculator_weights =- speculator.speculate(&speculated_suffix);+ let mut cur_node_speculator_weights = speculator.speculate(&speculated_suffix);let mut next_node = None;while length < max_length {// Guuumbel speculator trick: both speculator and llm sample via gumbel max trick using the same noise for increased acceptance rate- if let Some(next_speculated_token) =- speculator_sample(cur_node.seed(), &cur_node_speculator_weights)- {+ if let Some(next_speculated_token) = speculator_sample(cur_node.seed(), &cur_node_speculator_weights) {// Add speculated token to the trie- let mask = if let Some(compiled_grammar) =- compiled_grammar.as_deref_mut()- {- if compiled_grammar- .accept_token(next_speculated_token)- .is_err()- {- cur_node_speculator_weights- .remove(&next_speculated_token);+ let mask = if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {+ if compiled_grammar.accept_token(next_speculated_token).is_err() {+ cur_node_speculator_weights.remove(&next_speculated_token);continue;}let next_bitmask = compiled_grammar.next_bitmask().unwrap();@@ -143,11 +126,7 @@ impl TrieNode {None};- let leaf_node = Self::new(- next_speculated_token,- mask,- seed.derive((prefix_length + height) as u64),- );+ let leaf_node = Self::new(next_speculated_token, mask, seed.derive((prefix_length + height) as u64));cur_node.add(leaf_node).unwrap();// If this is the first node we sampled (most likely to be selected after gumbel noise application) - set it as the next one@@ -166,8 +145,7 @@ impl TrieNode {} else if let Some(next_node_token) = next_node.take() {// Out of speculated tokens for this node, move onto the likeliest next nodespeculated_suffix.push(next_node_token);- if let Some(compiled_grammar) = compiled_grammar.as_deref_mut()- {+ if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {if compiled_grammar.accept_token(next_node_token).is_err() {break;}@@ -175,8 +153,7 @@ impl TrieNode {height += 1;cur_node = cur_node.get_mut(next_node_token).unwrap();cur_node_width = 0;- cur_node_speculator_weights =- speculator.speculate(&speculated_suffix);+ cur_node_speculator_weights = speculator.speculate(&speculated_suffix);continue;} else {// Dead end, exit@@ -276,10 +253,7 @@ impl<'a> FlatTrie<'a> {};if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {- assert!(- !compiled_grammar.is_terminated(),- "Grammar has terminated but llm continued generation"- );+ assert!(!compiled_grammar.is_terminated(), "Grammar has terminated but llm continued generation");}current_token = next_token;diff --git a/crates/uzu/src/utils/attention.rs b/crates/uzu/src/utils/attention.rsindex 67ea27ae3..a50c60a80 100644--- a/crates/uzu/src/utils/attention.rs+++ b/crates/uzu/src/utils/attention.rs@@ -17,12 +17,7 @@ pub fn fill_attention_bias(_ => panic!("Unsupported data type for attention bias fill"),};- fill_typed_for_dtype(- dst,- suffix_length,- prefix_segment_length,- should_be_neg_inf,- );+ fill_typed_for_dtype(dst, suffix_length, prefix_segment_length, should_be_neg_inf);}fn fill_typed<T: ArrayElement + Float>(diff --git a/crates/uzu/src/utils/device.rs b/crates/uzu/src/utils/device.rsindex 4d805505a..7e01c2fc0 100644--- a/crates/uzu/src/utils/device.rs+++ b/crates/uzu/src/utils/device.rs@@ -12,9 +12,7 @@ impl DeviceClass {pub fn detect() -> Self {use metal::MTLDeviceExt as _;let device: crate::backends::metal::Retained<- crate::backends::metal::ProtocolObject<- dyn crate::backends::metal::MTLDevice,- >,+ crate::backends::metal::ProtocolObject<dyn crate::backends::metal::MTLDevice>,> = match <dyn metal::MTLDevice>::system_default() {Some(dev) => dev,None => return DeviceClass::Unknown,diff --git a/crates/uzu/src/utils/model_size.rs b/crates/uzu/src/utils/model_size.rsindex 5e7a0eb89..98fab6bd3 100644--- a/crates/uzu/src/utils/model_size.rs+++ b/crates/uzu/src/utils/model_size.rs@@ -11,8 +11,7 @@ pub enum ModelSize {impl ModelSize {pub fn from_path(model_path: &Path) -> Self {let weights_path = model_path.join("model.safetensors");- let size_bytes =- std::fs::metadata(&weights_path).map(|m| m.len()).unwrap_or(0);+ let size_bytes = std::fs::metadata(&weights_path).map(|m| m.len()).unwrap_or(0);Self::from_bytes(size_bytes)}@@ -42,10 +41,7 @@ mod tests {.join(VERSION).join(MODEL_DIR_NAME);if !path.exists() {- panic!(- "Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.",- path- );+ panic!("Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.", path);}path}diff --git a/crates/uzu/src/utils/storage.rs b/crates/uzu/src/utils/storage.rsindex 4f39fa575..48c615766 100644--- a/crates/uzu/src/utils/storage.rs+++ b/crates/uzu/src/utils/storage.rs@@ -14,14 +14,10 @@ pub fn user_domain_path(dir: NSSearchPathDirectory) -> PathBuf {if let Some(path) = objc2::rc::autoreleasepool(|_| unsafe {let fm = NSFileManager::defaultManager();- let urls = fm.URLsForDirectory_inDomains(- dir,- NSSearchPathDomainMask::UserDomainMask,- );+ let urls = fm.URLsForDirectory_inDomains(dir, NSSearchPathDomainMask::UserDomainMask);urls.firstObject().map(|url| {- let ns_path: Retained<NSString> =- url.path().expect("URL should have a path");+ let ns_path: Retained<NSString> = url.path().expect("URL should have a path");PathBuf::from(ns_path.to_string())})}) {diff --git a/crates/uzu/tests/allocator_test/caching_allocator.rs b/crates/uzu/tests/allocator_test/caching_allocator.rsindex e24c049e5..e5852e80d 100644--- a/crates/uzu/tests/allocator_test/caching_allocator.rs+++ b/crates/uzu/tests/allocator_test/caching_allocator.rs@@ -27,10 +27,7 @@ impl AllocatorTrait for CachingAllocator {&self,size: usize,) -> Self::Buffer {- self.context- .allocator()- .alloc(BufferLifetime::Scratch, size)- .expect("Allocation failed")+ self.context.allocator().alloc(BufferLifetime::Scratch, size).expect("Allocation failed")}fn free(diff --git a/crates/uzu/tests/allocator_test/device_allocator.rs b/crates/uzu/tests/allocator_test/device_allocator.rsindex 905a429f8..0ee9a33ac 100644--- a/crates/uzu/tests/allocator_test/device_allocator.rs+++ b/crates/uzu/tests/allocator_test/device_allocator.rs@@ -13,8 +13,7 @@ pub struct DeviceAllocator {impl DeviceAllocator {pub fn new() -> Self {- let device =- <dyn MTLDevice>::system_default().expect("No Metal device");+ let device = <dyn MTLDevice>::system_default().expect("No Metal device");Self {device,@@ -31,10 +30,8 @@ impl AllocatorTrait for DeviceAllocator {&self,size: usize,) -> Self::Buffer {- let buffer = self- .device- .new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let buffer =+ self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let buf_size = buffer.length();let new_active = self.active_memory.get() + buf_size;diff --git a/crates/uzu/tests/allocator_test/heap_allocator.rs b/crates/uzu/tests/allocator_test/heap_allocator.rsindex 2793328eb..9376baaae 100644--- a/crates/uzu/tests/allocator_test/heap_allocator.rs+++ b/crates/uzu/tests/allocator_test/heap_allocator.rs@@ -1,9 +1,6 @@use std::cell::Cell;-use metal::{- MTLBuffer, MTLDevice, MTLDeviceExt, MTLHeap, MTLHeapDescriptor,- MTLResourceOptions, MTLStorageMode,-};+use metal::{MTLBuffer, MTLDevice, MTLDeviceExt, MTLHeap, MTLHeapDescriptor, MTLResourceOptions, MTLStorageMode};use objc2::{rc::Retained, runtime::ProtocolObject};use super::allocator_trait::AllocatorTrait;@@ -16,16 +13,13 @@ pub struct MTLHeapAllocator {impl MTLHeapAllocator {pub fn new(heap_size: usize) -> Self {- let device =- <dyn MTLDevice>::system_default().expect("No Metal device");+ let device = <dyn MTLDevice>::system_default().expect("No Metal device");let descriptor = MTLHeapDescriptor::new();descriptor.set_size(heap_size);descriptor.set_storage_mode(MTLStorageMode::Shared);- let heap = device- .new_heap_with_descriptor(&descriptor)- .expect("Failed to create heap");+ let heap = device.new_heap_with_descriptor(&descriptor).expect("Failed to create heap");Self {heap,diff --git a/crates/uzu/tests/allocator_test/inference_simulation.rs b/crates/uzu/tests/allocator_test/inference_simulation.rsindex cd92d610d..812108b1e 100644--- a/crates/uzu/tests/allocator_test/inference_simulation.rs+++ b/crates/uzu/tests/allocator_test/inference_simulation.rs@@ -1,9 +1,6 @@use std::time::Instant;-use super::{- allocation_stats::AllocationStats, allocator_trait::AllocatorTrait,- model_config::ModelConfig,-};+use super::{allocation_stats::AllocationStats, allocator_trait::AllocatorTrait, model_config::ModelConfig};const F16_SIZE: usize = 2;@@ -82,26 +79,16 @@ impl<'a, A: AllocatorTrait> InferenceSimulation<'a, A> {let qkv_size = batch_size * self.config.qkv_dim() * F16_SIZE;buffers.push(self.alloc_scratch(qkv_size));- let bias_size =- batch_size * (batch_size + self.current_prefix_len) * F16_SIZE;+ let bias_size = batch_size * (batch_size + self.current_prefix_len) * F16_SIZE;buffers.push(self.alloc_scratch(bias_size));- let rotated_q_size = self.config.num_heads- * batch_size- * self.config.head_dim- * F16_SIZE;+ let rotated_q_size = self.config.num_heads * batch_size * self.config.head_dim * F16_SIZE;buffers.push(self.alloc_scratch(rotated_q_size));- let rotated_k_size = self.config.num_groups- * batch_size- * self.config.head_dim- * F16_SIZE;+ let rotated_k_size = self.config.num_groups * batch_size * self.config.head_dim * F16_SIZE;buffers.push(self.alloc_scratch(rotated_k_size));- let attn_out_size = batch_size- * self.config.num_heads- * self.config.head_dim- * F16_SIZE;+ let attn_out_size = batch_size * self.config.num_heads * self.config.head_dim * F16_SIZE;buffers.push(self.alloc_scratch(attn_out_size));buffersdiff --git a/crates/uzu/tests/allocator_test/main.rs b/crates/uzu/tests/allocator_test/main.rsindex 8cc4c59e1..0d0741726 100644--- a/crates/uzu/tests/allocator_test/main.rs+++ b/crates/uzu/tests/allocator_test/main.rs@@ -13,9 +13,7 @@ use caching_allocator::CachingAllocator;use device_allocator::DeviceAllocator;use heap_allocator::MTLHeapAllocator;use model_config::ModelConfig;-use simulation_result::{- run_long_simulation, run_simulation, run_simulation_with_config,-};+use simulation_result::{run_long_simulation, run_simulation, run_simulation_with_config};use uzu::backends::common::Context;const KB: f64 = 1024.0;@@ -28,26 +26,17 @@ fn print_comparison(device_result: &simulation_result::SimulationResult,) {println!("\n=== {} ===", name);- println!(- "{:<20} {:>14} {:>14} {:>14}",- "Metric", "UZU Caching", "Heap", "Device"- );+ println!("{:<20} {:>14} {:>14} {:>14}", "Metric", "UZU Caching", "Heap", "Device");println!("{:-<62}", "");println!("{:<20} {:>14} {:>14} {:>14}",- "Allocations",- uzu_result.stats.total_allocs,- heap_result.stats.total_allocs,- device_result.stats.total_allocs+ "Allocations", uzu_result.stats.total_allocs, heap_result.stats.total_allocs, device_result.stats.total_allocs);println!("{:<20} {:>14} {:>14} {:>14}",- "Frees",- uzu_result.stats.total_frees,- heap_result.stats.total_frees,- device_result.stats.total_frees+ "Frees", uzu_result.stats.total_frees, heap_result.stats.total_frees, device_result.stats.total_frees);println!(@@ -93,15 +82,10 @@ fn print_comparison(println!();let device_time = device_result.stats.avg_alloc_time_us().max(0.001);- let uzu_speedup =- device_time / uzu_result.stats.avg_alloc_time_us().max(0.001);- let heap_speedup =- device_time / heap_result.stats.avg_alloc_time_us().max(0.001);+ let uzu_speedup = device_time / uzu_result.stats.avg_alloc_time_us().max(0.001);+ let heap_speedup = device_time / heap_result.stats.avg_alloc_time_us().max(0.001);- println!(- "Alloc speedup vs device: UZU {:.1}x, Heap {:.1}x",- uzu_speedup, heap_speedup- );+ println!("Alloc speedup vs device: UZU {:.1}x, Heap {:.1}x", uzu_speedup, heap_speedup);}fn print_memory_over_time(@@ -142,40 +126,17 @@ fn test_allocator_comparison() {println!("========================================");let uzu_allocator = CachingAllocator::new();- let uzu_result = run_simulation(- &uzu_allocator,- prompt_length,- generate_length,- prefill_step_size,- );+ let uzu_result = run_simulation(&uzu_allocator, prompt_length, generate_length, prefill_step_size);let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);- let heap_result = run_simulation(- &heap_allocator,- prompt_length,- generate_length,- prefill_step_size,- );+ let heap_result = run_simulation(&heap_allocator, prompt_length, generate_length, prefill_step_size);let device_allocator = DeviceAllocator::new();- let device_result = run_simulation(- &device_allocator,- prompt_length,- generate_length,- prefill_step_size,- );+ let device_result = run_simulation(&device_allocator, prompt_length, generate_length, prefill_step_size);- print_comparison(- "Short Conversation Results",- &uzu_result,- &heap_result,- &device_result,- );+ print_comparison("Short Conversation Results", &uzu_result, &heap_result, &device_result);- assert_eq!(- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,- "UZU allocator: allocs should equal frees"- );+ assert_eq!(uzu_result.stats.total_allocs, uzu_result.stats.total_frees, "UZU allocator: allocs should equal frees");assert_eq!(heap_result.stats.total_allocs, heap_result.stats.total_frees,"Heap allocator: allocs should equal frees"@@ -196,10 +157,7 @@ fn test_long_conversation_comparison() {println!("\n========================================");println!(" Long Conversation Test");- println!(- " {} conversations, ~{}p + ~{}g tokens each",- num_conversations, avg_prompt_length, avg_generate_length- );+ println!(" {} conversations, ~{}p + ~{}g tokens each", num_conversations, avg_prompt_length, avg_generate_length);println!("========================================");let gc_limit = 256 * 1024 * 1024;@@ -238,12 +196,7 @@ fn test_long_conversation_comparison() {snapshot_interval,);- print_comparison(- "Long Conversation Results",- &uzu_result,- &heap_result,- &device_result,- );+ print_comparison("Long Conversation Results", &uzu_result, &heap_result, &device_result);print_memory_over_time("UZU Caching", &uzu_result.snapshots, 15);print_memory_over_time("Heap", &heap_result.snapshots, 15);@@ -252,27 +205,18 @@ fn test_long_conversation_comparison() {println!("\n--- Memory Efficiency Summary ---");println!("UZU peak/requested ratio: {:.2}%",- (uzu_result.peak_memory_bytes as f64- / uzu_result.stats.total_bytes_requested as f64)- * 100.0+ (uzu_result.peak_memory_bytes as f64 / uzu_result.stats.total_bytes_requested as f64) * 100.0);println!("Heap peak/requested ratio: {:.2}%",- (heap_result.peak_memory_bytes as f64- / heap_result.stats.total_bytes_requested as f64)- * 100.0+ (heap_result.peak_memory_bytes as f64 / heap_result.stats.total_bytes_requested as f64) * 100.0);println!("Device peak/requested ratio: {:.2}%",- (device_result.peak_memory_bytes as f64- / device_result.stats.total_bytes_requested as f64)- * 100.0+ (device_result.peak_memory_bytes as f64 / device_result.stats.total_bytes_requested as f64) * 100.0);- assert_eq!(- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,- "UZU allocator: allocs should equal frees"- );+ assert_eq!(uzu_result.stats.total_allocs, uzu_result.stats.total_frees, "UZU allocator: allocs should equal frees");assert_eq!(device_result.stats.total_allocs, device_result.stats.total_frees,"Device allocator: allocs should equal frees"@@ -290,39 +234,21 @@ fn test_multi_model_comparison() {println!(" {}p + {}g tokens per model", prompt_length, generate_length);println!("========================================");- println!(- "\n{:<15} {:>12} {:>12} {:>12}",- "Model", "UZU (Β΅s)", "Heap (Β΅s)", "Device (Β΅s)"- );+ println!("\n{:<15} {:>12} {:>12} {:>12}", "Model", "UZU (Β΅s)", "Heap (Β΅s)", "Device (Β΅s)");println!("{:-<55}", "");for config in ModelConfig::all_models() {let uzu_allocator = CachingAllocator::new();- let uzu_result = run_simulation_with_config(- &uzu_allocator,- config,- prompt_length,- generate_length,- prefill_step_size,- );+ let uzu_result =+ run_simulation_with_config(&uzu_allocator, config, prompt_length, generate_length, prefill_step_size);let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);- let heap_result = run_simulation_with_config(- &heap_allocator,- config,- prompt_length,- generate_length,- prefill_step_size,- );+ let heap_result =+ run_simulation_with_config(&heap_allocator, config, prompt_length, generate_length, prefill_step_size);let device_allocator = DeviceAllocator::new();- let device_result = run_simulation_with_config(- &device_allocator,- config,- prompt_length,- generate_length,- prefill_step_size,- );+ let device_result =+ run_simulation_with_config(&device_allocator, config, prompt_length, generate_length, prefill_step_size);println!("{:<15} {:>12.2} {:>12.2} {:>12.2}",diff --git a/crates/uzu/tests/allocator_test/simulation_result.rs b/crates/uzu/tests/allocator_test/simulation_result.rsindex 822ae6e7e..d3f2a3184 100644--- a/crates/uzu/tests/allocator_test/simulation_result.rs+++ b/crates/uzu/tests/allocator_test/simulation_result.rs@@ -91,16 +91,13 @@ pub fn run_long_simulation_with_config<A: AllocatorTrait>(let mut all_snapshots = Vec::new();for conv_idx in 0..num_conversations {- let prompt_len =- avg_prompt_length + (conv_idx % 3) * (avg_prompt_length / 4);- let gen_len =- avg_generate_length + (conv_idx % 5) * (avg_generate_length / 8);+ let prompt_len = avg_prompt_length + (conv_idx % 3) * (avg_prompt_length / 4);+ let gen_len = avg_generate_length + (conv_idx % 5) * (avg_generate_length / 8);let prefill_snapshots = sim.run_prefill(prompt_len, prefill_step_size);all_snapshots.extend(prefill_snapshots);- let gen_snapshots =- sim.run_generation_with_sampling(gen_len, snapshot_interval);+ let gen_snapshots = sim.run_generation_with_sampling(gen_len, snapshot_interval);all_snapshots.extend(gen_snapshots);sim.current_prefix_len = 0;diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex 88ea91700..d8383a721 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -5,27 +5,24 @@ mod common;use std::mem::size_of;use bytemuck;-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,- MTLDeviceExt, MTLResourceOptions,-};+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use ndarray::{Array3, Array4, s};use uzu::backends::{common::Context,metal::{KernelDataType, MTLContext, ProtocolObject, Retained,kernel::attention::{- AttentionGemmArguments, AttentionKernel, AttentionKernelVariant,- AttentionSinglePassArguments, AttentionTwoPassArguments,+ AttentionGemmArguments, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,+ AttentionTwoPassArguments,},metal_extensions::CommandBufferTimingExt,},};fn reference_attention(- queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]- keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]- values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]+ queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]+ keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]+ values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]mask: Option<&Array3<f32>>, // [batch, seq_len, seq_len] or Nonesinks: Option<&[f32]>,scale: f32,@@ -44,8 +41,7 @@ fn reference_attention(for repeat in 0..n_repeats {let q_head = kv_head * n_repeats + repeat;- let q =- scaled_queries.slice(s![b, q_head, .., ..]).to_owned(); // [L, D]+ let q = scaled_queries.slice(s![b, q_head, .., ..]).to_owned(); // [L, D]let k = keys.slice(s![b, kv_head, .., ..]).to_owned(); // [L_kv, D]let v = values.slice(s![b, kv_head, .., ..]).to_owned(); // [L_kv, D]@@ -53,28 +49,17 @@ fn reference_attention(let mut scores = q.dot(&k.t());if let Some(mask_data) = mask {- let mask_slice = mask_data.slice(s![- b,- ..,- ..kv_seq_len.min(seq_len)- ]);- scores- .slice_mut(s![.., ..kv_seq_len.min(seq_len)])- .zip_mut_with(&mask_slice, |s, &m| *s += m);+ let mask_slice = mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);+ scores.slice_mut(s![.., ..kv_seq_len.min(seq_len)]).zip_mut_with(&mask_slice, |s, &m| *s += m);}// Softmax with optional sink logitfor i in 0..seq_len {let sink_logit = sinks.map(|s| s[q_head]);let row = scores.row(i);- let max_score = row.iter().fold(- sink_logit.unwrap_or(f32::NEG_INFINITY),- |acc, &x| acc.max(x),- );-- let mut sum_exp = sink_logit- .map(|sink| (sink - max_score).exp())- .unwrap_or(0.0);+ let max_score = row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |acc, &x| acc.max(x));++ let mut sum_exp = sink_logit.map(|sink| (sink - max_score).exp()).unwrap_or(0.0);for j in 0..kv_seq_len {scores[[i, j]] = (scores[[i, j]] - max_score).exp();sum_exp += scores[[i, j]];@@ -84,9 +69,7 @@ fn reference_attention(// Efficient matrix multiplication: output = scores @ Vlet head_output = scores.dot(&v);- output- .slice_mut(s![b, q_head, .., ..])- .assign(&head_output);+ output.slice_mut(s![b, q_head, .., ..]).assign(&head_output);}}} else {@@ -99,25 +82,17 @@ fn reference_attention(let mut scores = q.dot(&k.t());if let Some(mask_data) = mask {- let mask_slice =- mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);- scores- .slice_mut(s![.., ..kv_seq_len.min(seq_len)])- .zip_mut_with(&mask_slice, |s, &m| *s += m);+ let mask_slice = mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);+ scores.slice_mut(s![.., ..kv_seq_len.min(seq_len)]).zip_mut_with(&mask_slice, |s, &m| *s += m);}// Softmax with optional sink logitfor i in 0..seq_len {let sink_logit = sinks.map(|s| s[h]);let row = scores.row(i);- let max_score = row.iter().fold(- sink_logit.unwrap_or(f32::NEG_INFINITY),- |acc, &x| acc.max(x),- );-- let mut sum_exp = sink_logit- .map(|sink| (sink - max_score).exp())- .unwrap_or(0.0);+ let max_score = row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |acc, &x| acc.max(x));++ let mut sum_exp = sink_logit.map(|sink| (sink - max_score).exp()).unwrap_or(0.0);for j in 0..kv_seq_len {scores[[i, j]] = (scores[[i, j]] - max_score).exp();sum_exp += scores[[i, j]];@@ -147,29 +122,19 @@ fn create_test_data(let mut rng = StdRng::seed_from_u64(seed);- let queries = Array4::from_shape_fn(- (batch_size, num_heads, seq_len, head_dim),- |_| rng.random_range(-0.5..0.5),- );+ let queries = Array4::from_shape_fn((batch_size, num_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));- let keys = Array4::from_shape_fn(- (batch_size, num_kv_heads, seq_len, head_dim),- |_| rng.random_range(-0.5..0.5),- );+ let keys = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));- let values = Array4::from_shape_fn(- (batch_size, num_kv_heads, seq_len, head_dim),- |_| rng.random_range(-0.5..0.5),- );+ let values = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));- let mask =- Array3::from_shape_fn((batch_size, seq_len, seq_len), |(_, i, j)| {- if j <= i {- 0.0- } else {- -1e9- }- });+ let mask = Array3::from_shape_fn((batch_size, seq_len, seq_len), |(_, i, j)| {+ if j <= i {+ 0.0+ } else {+ -1e9+ }+ });(queries, keys, values, mask)}@@ -195,10 +160,7 @@ fn create_query_buffer(context.device- .new_buffer_with_data(- bytemuck::cast_slice(&query_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&query_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")}@@ -210,8 +172,7 @@ fn create_key_cache_buffer(let (_batch_size, num_kv_heads, seq_len, head_dim) = keys.dim();// Our kernel expects key cache layout: [num_kv_heads, max_seq_len, head_dim]- let mut key_cache_data =- vec![0.0f32; num_kv_heads * max_seq_len * head_dim];+ let mut key_cache_data = vec![0.0f32; num_kv_heads * max_seq_len * head_dim];for h in 0..num_kv_heads {for t in 0..seq_len {@@ -224,10 +185,7 @@ fn create_key_cache_buffer(context.device- .new_buffer_with_data(- bytemuck::cast_slice(&key_cache_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&key_cache_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")}@@ -239,8 +197,7 @@ fn create_value_cache_buffer(let (_batch_size, num_kv_heads, seq_len, head_dim) = values.dim();// Our kernel expects value cache layout: [num_kv_heads, max_seq_len, head_dim]- let mut value_cache_data =- vec![0.0f32; num_kv_heads * max_seq_len * head_dim];+ let mut value_cache_data = vec![0.0f32; num_kv_heads * max_seq_len * head_dim];for h in 0..num_kv_heads {for t in 0..seq_len {@@ -253,10 +210,7 @@ fn create_value_cache_buffer(context.device- .new_buffer_with_data(- bytemuck::cast_slice(&value_cache_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&value_cache_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")}@@ -280,10 +234,7 @@ fn create_mask_buffer(context.device- .new_buffer_with_data(- bytemuck::cast_slice(&mask_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&mask_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")}@@ -302,10 +253,7 @@ fn create_mask_2d_buffer(context.device- .new_buffer_with_data(- bytemuck::cast_slice(&mask_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&mask_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")}@@ -315,10 +263,7 @@ fn create_sinks_buffer() -> Retained<ProtocolObject<dyn MTLBuffer>> {context.device- .new_buffer_with_data(- bytemuck::cast_slice(&sinks),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&sinks), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")}@@ -329,8 +274,7 @@ fn convert_kernel_output(seq_len: usize,head_dim: usize,) -> Array4<f32> {- let mut kernel_output =- Array4::zeros((batch_size, num_heads, seq_len, head_dim));+ let mut kernel_output = Array4::zeros((batch_size, num_heads, seq_len, head_dim));for h in 0..num_heads {for t in 0..seq_len {for d in 0..head_dim {@@ -359,27 +303,18 @@ fn run_single_pass_attention(let query_buffer = create_query_buffer(queries, context);let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);- let value_cache_buffer =- create_value_cache_buffer(values, seq_len, context);+ let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));let output_buffer = context.device- .new_buffer(- num_heads * seq_len * head_dim * size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let args = AttentionSinglePassArguments {queries_buffer: &query_buffer,@@ -412,17 +347,9 @@ fn run_single_pass_attention(command_buffer.wait_until_completed();let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe {- std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)- };+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };- let kernel_output = convert_kernel_output(- output_slice,- batch_size,- num_heads,- seq_len,- head_dim,- );+ let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);Ok(kernel_output)}@@ -443,27 +370,18 @@ fn run_single_pass_attention_with_is_causal(let query_buffer = create_query_buffer(queries, context);let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);- let value_cache_buffer =- create_value_cache_buffer(values, seq_len, context);+ let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));let output_buffer = context.device- .new_buffer(- num_heads * seq_len * head_dim * size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let args = AttentionSinglePassArguments {queries_buffer: &query_buffer,@@ -496,17 +414,9 @@ fn run_single_pass_attention_with_is_causal(command_buffer.wait_until_completed();let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe {- std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)- };+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };- let kernel_output = convert_kernel_output(- output_slice,- batch_size,- num_heads,- seq_len,- head_dim,- );+ let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);Ok(kernel_output)}@@ -527,27 +437,18 @@ fn run_gemm_attention(let query_buffer = create_query_buffer(queries, context);let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);- let value_cache_buffer =- create_value_cache_buffer(values, seq_len, context);+ let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);let mask_buffer = mask.map(|m| create_mask_2d_buffer(m, context));let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));let output_buffer = context.device- .new_buffer(- num_heads * seq_len * head_dim * size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let args = AttentionGemmArguments {queries_buffer: &query_buffer,@@ -575,17 +476,9 @@ fn run_gemm_attention(command_buffer.wait_until_completed();let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe {- std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)- };+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };- let kernel_output = convert_kernel_output(- output_slice,- batch_size,- num_heads,- seq_len,- head_dim,- );+ let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);Ok(kernel_output)}@@ -596,11 +489,7 @@ fn compare_results(tolerance: f32,test_name: &str,) -> Result<(), String> {- let max_diff = kernel_output- .iter()- .zip(reference_output.iter())- .map(|(a, b)| (a - b).abs())- .fold(0.0f32, f32::max);+ let max_diff = kernel_output.iter().zip(reference_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);let relative_error = kernel_output.iter()@@ -618,29 +507,15 @@ fn compare_results(println!("Max relative error ({}): {}", test_name, relative_error);println!("Debug - Sample values ({}):", test_name);- println!(- "Kernel output [0,0,0,0:4]: {:?}",- &kernel_output.slice(s![0, 0, 0, 0..4]).to_vec()- );- println!(- "Reference output [0,0,0,0:4]: {:?}",- &reference_output.slice(s![0, 0, 0, 0..4]).to_vec()- );- println!(- "Kernel output [0,0,1,0:4]: {:?}",- &kernel_output.slice(s![0, 0, 1, 0..4]).to_vec()- );- println!(- "Reference output [0,0,1,0:4]: {:?}",- &reference_output.slice(s![0, 0, 1, 0..4]).to_vec()- );+ println!("Kernel output [0,0,0,0:4]: {:?}", &kernel_output.slice(s![0, 0, 0, 0..4]).to_vec());+ println!("Reference output [0,0,0,0:4]: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());+ println!("Kernel output [0,0,1,0:4]: {:?}", &kernel_output.slice(s![0, 0, 1, 0..4]).to_vec());+ println!("Reference output [0,0,1,0:4]: {:?}", &reference_output.slice(s![0, 0, 1, 0..4]).to_vec());let kernel_max = kernel_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));- let kernel_min =- kernel_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));+ let kernel_min = kernel_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));- let ref_min =- reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));+ let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));println!("Kernel output range: [{}, {}]", kernel_min, kernel_max);println!("Reference output range: [{}, {}]", ref_min, ref_max);@@ -668,27 +543,15 @@ fn test_single_pass_attention_basic() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 42,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);println!("Testing reference implementation without mask...");- let reference_output =- reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));- let ref_min =- reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));+ let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));println!("Reference output range (no mask): [{}, {}]", ref_min, ref_max);- println!(- "Reference sample values: {:?}",- &reference_output.slice(s![0, 0, 0, 0..4]).to_vec()- );+ println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {Ok(k) => k,@@ -705,19 +568,14 @@ fn test_single_pass_attention_basic() {head_dim,kernel.supports_single_pass(head_dim, is_causal, false));- println!(- "Supports two-pass for head_dim={}: {}",- head_dim,- kernel.supports_two_pass(head_dim, is_causal, false)- );+ println!("Supports two-pass for head_dim={}: {}", head_dim, kernel.supports_two_pass(head_dim, is_causal, false));if !kernel.supports_single_pass(head_dim, false, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}- let kernel_output = match run_single_pass_attention(- &kernel, &context, &queries, &keys, &values, None, None, scale,- ) {+ let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)+ {Ok(output) => output,Err(e) => {panic!("Failed to run single-pass attention: {:?}", e);@@ -725,12 +583,7 @@ fn test_single_pass_attention_basic() {};let tolerance = 1e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Single-pass attention",- ) {+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention") {panic!("{}", e);}}@@ -747,34 +600,18 @@ fn test_gemm_attention_basic() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 123,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);- let kernel = AttentionKernel::new(&context, KernelDataType::Float32)- .expect("Failed to create AttentionKernel");+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");- let reference_output =- reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);- let kernel_output = run_gemm_attention(- &kernel, &context, &queries, &keys, &values, None, None, scale,- /*is_causal=*/ false,- )- .expect("run gemm attention");+ let kernel_output =+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, None, scale, /*is_causal=*/ false)+ .expect("run gemm attention");let tolerance = 1e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Gemm attention",- ) {+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Gemm attention") {panic!("{}", e);}}@@ -791,34 +628,18 @@ fn test_gemm_attention_f32_head_dim_128() {let head_dim = 128;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 456,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);- let kernel = AttentionKernel::new(&context, KernelDataType::Float32)- .expect("Failed to create AttentionKernel");+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");- let reference_output =- reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);- let kernel_output = run_gemm_attention(- &kernel, &context, &queries, &keys, &values, None, None, scale,- /*is_causal=*/ false,- )- .expect("run gemm attention f32 head_dim=128");+ let kernel_output =+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, None, scale, /*is_causal=*/ false)+ .expect("run gemm attention f32 head_dim=128");let tolerance = 1e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Gemm attention f32 head_dim=128",- ) {+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Gemm attention f32 head_dim=128") {panic!("{}", e);}}@@ -836,21 +657,12 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = true;- let (queries, keys, values, mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 2026,- );+ let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);- let kernel = AttentionKernel::new(&context, KernelDataType::Float32)- .expect("Failed to create AttentionKernel");+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");// CPU reference (uses the same additive mask)- let reference_output =- reference_attention(&queries, &keys, &values, Some(&mask), None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);// vector attention path (single-pass)let vector_output = run_single_pass_attention_with_is_causal(@@ -867,46 +679,19 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {.expect("run vector attention");// matrix attention path (gemm)- let matrix_output = run_gemm_attention(- &kernel,- &context,- &queries,- &keys,- &values,- Some(&mask),- None,- scale,- is_causal,- )- .expect("run matrix attention");+ let matrix_output =+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale, is_causal)+ .expect("run matrix attention");// Compare both to CPUlet tol_cpu = 5e-2;- compare_results(- &vector_output,- &reference_output,- tol_cpu,- "vector single-pass attention vs CPU",- )- .unwrap();- compare_results(- &matrix_output,- &reference_output,- tol_cpu,- "matrix attention vs CPU",- )- .unwrap();+ compare_results(&vector_output, &reference_output, tol_cpu, "vector single-pass attention vs CPU").unwrap();+ compare_results(&matrix_output, &reference_output, tol_cpu, "matrix attention vs CPU").unwrap();- let max_diff_vector_matrix = vector_output- .iter()- .zip(matrix_output.iter())- .map(|(a, b)| (a - b).abs())- .fold(0.0f32, f32::max);+ let max_diff_vector_matrix =+ vector_output.iter().zip(matrix_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);let tol_vector_matrix = 5e-2;- println!(- "Max absolute difference (vector single-pass attention vs matrix attention): {}",- max_diff_vector_matrix- );+ println!("Max absolute difference (vector single-pass attention vs matrix attention): {}", max_diff_vector_matrix);assert!(max_diff_vector_matrix <= tol_vector_matrix,"vector single-pass attention and matrix attention differ too much: max_diff = {} (tol={})",@@ -927,14 +712,7 @@ fn test_single_pass_attention_with_mask() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 42,- );+ let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {Ok(k) => k,@@ -947,24 +725,15 @@ fn test_single_pass_attention_with_mask() {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}- let kernel_output = match run_single_pass_attention(- &kernel,- &context,- &queries,- &keys,- &values,- Some(&mask),- None,- scale,- ) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run single-pass attention with mask: {:?}", e);- },- };+ let kernel_output =+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale) {+ Ok(output) => output,+ Err(e) => {+ panic!("Failed to run single-pass attention with mask: {:?}", e);+ },+ };- let reference_output =- reference_attention(&queries, &keys, &values, Some(&mask), None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);println!("Mask values:");for i in 0..seq_len.min(4) {@@ -975,12 +744,7 @@ fn test_single_pass_attention_with_mask() {}let tolerance = 1e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Single-pass attention with mask",- ) {+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention with mask") {panic!("{}", e);}}@@ -997,18 +761,9 @@ fn test_single_pass_attention_with_sinks() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 444,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);- let sinks: Vec<f32> = (0..num_heads)- .map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25)- .collect();+ let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();println!("Using sinks: {:?}", sinks);let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {@@ -1022,38 +777,18 @@ fn test_single_pass_attention_with_sinks() {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}- let reference_output = reference_attention(- &queries,- &keys,- &values,- None,- Some(&sinks),- scale,- );+ let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);- let kernel_output = match run_single_pass_attention(- &kernel,- &context,- &queries,- &keys,- &values,- None,- Some(&sinks),- scale,- ) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run single-pass attention with sinks: {:?}", e);- },- };+ let kernel_output =+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, Some(&sinks), scale) {+ Ok(output) => output,+ Err(e) => {+ panic!("Failed to run single-pass attention with sinks: {:?}", e);+ },+ };let tolerance = 1e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Single-pass attention with sinks",- ) {+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention with sinks") {panic!("{}", e);}}@@ -1070,17 +805,9 @@ fn test_single_pass_attention_with_sinks_long_sequence() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 777,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);- let sinks: Vec<f32> =- (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();+ let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();println!("Long sequence sinks: {:?}", sinks);let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {@@ -1094,33 +821,15 @@ fn test_single_pass_attention_with_sinks_long_sequence() {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}- let reference_output = reference_attention(- &queries,- &keys,- &values,- None,- Some(&sinks),- scale,- );+ let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);- let kernel_output = match run_single_pass_attention(- &kernel,- &context,- &queries,- &keys,- &values,- None,- Some(&sinks),- scale,- ) {- Ok(output) => output,- Err(e) => {- panic!(- "Failed to run single-pass attention long sequence with sinks: {:?}",- e- );- },- };+ let kernel_output =+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, Some(&sinks), scale) {+ Ok(output) => output,+ Err(e) => {+ panic!("Failed to run single-pass attention long sequence with sinks: {:?}", e);+ },+ };let tolerance = 5e-2;if let Err(e) = compare_results(@@ -1145,14 +854,7 @@ fn test_single_pass_attention_gqa() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 42,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {Ok(k) => k,@@ -1169,25 +871,18 @@ fn test_single_pass_attention_gqa() {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}- let kernel_output = match run_single_pass_attention(- &kernel, &context, &queries, &keys, &values, None, None, scale,- ) {+ let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)+ {Ok(output) => output,Err(e) => {panic!("Failed to run single-pass attention GQA: {:?}", e);},};- let reference_output =- reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);let tolerance = 1e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Single-pass attention GQA",- ) {+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention GQA") {panic!("{}", e);}}@@ -1217,26 +912,17 @@ fn run_two_pass_attention(let partials_buffer = context.device- .new_buffer(- partials_size * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(partials_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let sums_buffer = context.device- .new_buffer(- sums_maxs_size * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let maxs_buffer = context.device- .new_buffer(- sums_maxs_size * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buffer = context@@ -1247,13 +933,8 @@ fn run_two_pass_attention().expect("Failed to create buffer");- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let args = AttentionTwoPassArguments {queries_buffer: &queries_buffer,@@ -1289,17 +970,9 @@ fn run_two_pass_attention(command_buffer.wait_until_completed();let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe {- std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)- };+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };- let kernel_output = convert_kernel_output(- output_slice,- batch_size,- num_heads,- seq_len,- head_dim,- );+ let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);Ok(kernel_output)}@@ -1330,27 +1003,14 @@ fn test_two_pass_attention() {let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);if !matches!(variant, AttentionKernelVariant::TwoPass) {- panic!(- "Two-pass not selected for seq_len={}. Got {:?}",- seq_len, variant- );+ panic!("Two-pass not selected for seq_len={}. Got {:?}", seq_len, variant);}- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 42,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let reference_output =- reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);- let kernel_output = match run_two_pass_attention(- &kernel, &context, &queries, &keys, &values, None, None, scale,- ) {+ let kernel_output = match run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale) {Ok(output) => output,Err(e) => {panic!("Failed to run two-pass attention: {:?}", e);@@ -1358,12 +1018,7 @@ fn test_two_pass_attention() {};let tolerance = 1e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Two-pass attention",- ) {+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention") {panic!("{}", e);}}@@ -1394,27 +1049,14 @@ fn test_two_pass_attention_gqa() {let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);if !matches!(variant, AttentionKernelVariant::TwoPass) {- panic!(- "Two-pass not selected for GQA seq_len={}. Got {:?}",- seq_len, variant- );+ panic!("Two-pass not selected for GQA seq_len={}. Got {:?}", seq_len, variant);}- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 42,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let reference_output =- reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);- let kernel_output = match run_two_pass_attention(- &kernel, &context, &queries, &keys, &values, None, None, scale,- ) {+ let kernel_output = match run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale) {Ok(output) => output,Err(e) => {panic!("Failed to run two-pass attention GQA: {:?}", e);@@ -1422,12 +1064,7 @@ fn test_two_pass_attention_gqa() {};let tolerance = 1e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Two-pass attention GQA",- ) {+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention GQA") {panic!("{}", e);}}@@ -1457,19 +1094,13 @@ fn perf_two_pass_attention() {let is_causal = false;if !kernel.supports_two_pass(head_dim, is_causal, false) {- println!(- "Skipping two-pass perf test: not supported for head_dim={}",- head_dim- );+ println!("Skipping two-pass perf test: not supported for head_dim={}", head_dim);return;}let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);if !matches!(variant, AttentionKernelVariant::TwoPass) {- println!(- "Skipping two-pass perf test: variant {:?} selected instead",- variant- );+ println!("Skipping two-pass perf test: variant {:?} selected instead", variant);return;}@@ -1478,48 +1109,30 @@ fn perf_two_pass_attention() {seq_len - suffix_length,suffix_length);- let (queries, keys, values, _mask) = create_test_data(- batch_size,- num_heads,- num_kv_heads,- seq_len,- head_dim,- 123,- );+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);// ---- Create buffers ----// For realistic inference, we only process queries for the suffix (new tokens)- let queries_suffix =- queries.slice(s![.., .., (seq_len - suffix_length).., ..]).to_owned();+ let queries_suffix = queries.slice(s![.., .., (seq_len - suffix_length).., ..]).to_owned();let queries_buffer = create_query_buffer(&queries_suffix, &context);let keys_buffer = create_key_cache_buffer(&keys, seq_len, &context);let values_buffer = create_value_cache_buffer(&values, seq_len, &context);let total_blocks_count = 32;- let partials_size =- num_heads * suffix_length * total_blocks_count * head_dim;+ let partials_size = num_heads * suffix_length * total_blocks_count * head_dim;let sums_maxs_size = num_heads * suffix_length * total_blocks_count;let partials_buffer = context.device- .new_buffer(- partials_size * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(partials_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let sums_buffer = context.device- .new_buffer(- sums_maxs_size * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let maxs_buffer = context.device- .new_buffer(- sums_maxs_size * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buffer = context.device@@ -1530,13 +1143,8 @@ fn perf_two_pass_attention() {.expect("Failed to create buffer");// ---- Launch and time ----- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let args = AttentionTwoPassArguments {queries_buffer: &queries_buffer,@@ -1603,12 +1211,7 @@ fn perf_two_pass_attention() {// ---- Sanity check ----let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe {- std::slice::from_raw_parts(- output_ptr,- num_heads * suffix_length * head_dim,- )- };+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * suffix_length * head_dim) };// Check for NaN/Inffor &val in output_slice.iter().take(100) {diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex 78cdbed95..b99e24e92 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -16,10 +16,7 @@ pub fn get_test_model_path() -> PathBuf {.join(VERSION).join(MODEL_DIR_NAME);if !path.exists() {- panic!(- "Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.",- path- );+ panic!("Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.", path);}path}@@ -58,11 +55,7 @@ impl Speculator for StaticSpeculator {&self,prefix: &[u64],) -> HashMap<u64, f32> {- self.responses- .get(prefix)- .cloned()- .or_else(|| self.default_response.clone())- .unwrap_or_default()+ self.responses.get(prefix).cloned().or_else(|| self.default_response.clone()).unwrap_or_default()}}diff --git a/crates/uzu/tests/context_mode_test.rs b/crates/uzu/tests/context_mode_test.rsindex 719fbd78b..2c4bbaebd 100644--- a/crates/uzu/tests/context_mode_test.rs+++ b/crates/uzu/tests/context_mode_test.rs@@ -5,8 +5,7 @@ use uzu::session::{Session,config::{DecodingConfig, RunConfig, SpeculatorConfig},parameter::{- AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize,- SamplingMethod, SamplingPolicy, SamplingSeed,+ AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize, SamplingMethod, SamplingPolicy, SamplingSeed,},types::{Input, Message, Output},};@@ -54,10 +53,8 @@ fn request(#[test]fn test_context_mode_none() {- let decoding_config =- build_decoding_config().with_context_mode(ContextMode::None);- let mut session =- Session::new(build_model_path(), decoding_config).unwrap();+ let decoding_config = build_decoding_config().with_context_mode(ContextMode::None);+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();let tokens_limit = 48;let input_text = "What is Alice's occupation?".to_string();@@ -77,98 +74,54 @@ fn test_context_mode_static() {let parameter_occupation = "Engineer".to_string();let parameter_city = "London".to_string();- let decoding_config =- build_decoding_config().with_context_mode(ContextMode::Static {- input: Input::Messages(vec![- Message::system(- "Always answer with one word or number, no punctuation"- .to_string(),- ),- Message::user(- format!(- "Name: {}\nAge: {}\nOccupation: {}\nCity: {}",- parameter_name,- parameter_age,- parameter_occupation,- parameter_city- )- .to_string(),- ),- ]),- });- let mut session =- Session::new(build_model_path(), decoding_config).unwrap();+ let decoding_config = build_decoding_config().with_context_mode(ContextMode::Static {+ input: Input::Messages(vec![+ Message::system("Always answer with one word or number, no punctuation".to_string()),+ Message::user(+ format!(+ "Name: {}\nAge: {}\nOccupation: {}\nCity: {}",+ parameter_name, parameter_age, parameter_occupation, parameter_city+ )+ .to_string(),+ ),+ ]),+ });+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();let tokens_limit = 48;- let response_occupation = request(- &mut session,- format!("What is {}'s occupation?", parameter_name),- tokens_limit,- );- let response_age = request(- &mut session,- format!("What is {}'s age?", parameter_name),- tokens_limit,- );- let response_city = request(- &mut session,- format!("What is {}'s city?", parameter_name),- tokens_limit,- );+ let response_occupation = request(&mut session, format!("What is {}'s occupation?", parameter_name), tokens_limit);+ let response_age = request(&mut session, format!("What is {}'s age?", parameter_name), tokens_limit);+ let response_city = request(&mut session, format!("What is {}'s city?", parameter_name), tokens_limit);println!("Occupation: {}", response_occupation);println!("Age: {}", response_age);println!("City: {}", response_city);- assert!(- response_occupation- .to_lowercase()- .contains(parameter_occupation.to_lowercase().as_str())- );- assert!(- response_age- .to_lowercase()- .contains(parameter_age.to_lowercase().as_str())- );- assert!(- response_city- .to_lowercase()- .contains(parameter_city.to_lowercase().as_str())- );+ assert!(response_occupation.to_lowercase().contains(parameter_occupation.to_lowercase().as_str()));+ assert!(response_age.to_lowercase().contains(parameter_age.to_lowercase().as_str()));+ assert!(response_city.to_lowercase().contains(parameter_city.to_lowercase().as_str()));}#[test]#[ignore = "Flaky test - depends on LLM output which varies even with fixed seed"]fn test_context_mode_dynamic() {- let decoding_config =- build_decoding_config().with_context_mode(ContextMode::Dynamic);- let mut session =- Session::new(build_model_path(), decoding_config).unwrap();-- let update = |session: &mut Session, input_text: String| {- request(session, input_text, 0)- };- let ask = |session: &mut Session, input_text: String| {- request(session, input_text, 48)- };+ let decoding_config = build_decoding_config().with_context_mode(ContextMode::Dynamic);+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();++ let update = |session: &mut Session, input_text: String| request(session, input_text, 0);+ let ask = |session: &mut Session, input_text: String| request(session, input_text, 48);update(&mut session, "Name: Dave. Occupation: Nurse. Age: 30.".to_string());let answer_1 = ask(&mut session, "What is Dave's occupation?".to_string());println!("Answer 1: {}", answer_1);assert!(answer_1.to_lowercase().contains("nurse"));- update(- &mut session,- "Update: Name: Dave. Occupation: Doctor. Age: 30.".to_string(),- );+ update(&mut session, "Update: Name: Dave. Occupation: Doctor. Age: 30.".to_string());let answer_2 = ask(&mut session, "What is Dave's occupation?".to_string());println!("Answer 2: {}", answer_2);assert!(answer_2.to_lowercase().contains("doctor"));- update(- &mut session,- "Update: Name: Dave. Occupation: Doctor. Age: 31.".to_string(),- );+ update(&mut session, "Update: Name: Dave. Occupation: Doctor. Age: 31.".to_string());let answer_3 = ask(&mut session, "How old is Dave?".to_string());println!("Answer 3: {}", answer_3);assert!(answer_3.to_lowercase().contains("31"));@@ -177,8 +130,7 @@ fn test_context_mode_dynamic() {&mut session,"Important update, Dave got promoted! Update: Name: Dave. Occupation: Director. Age: 32.".to_string(),);- let answer_4 =- ask(&mut session, "What is Dave's current occupation?".to_string());+ let answer_4 = ask(&mut session, "What is Dave's current occupation?".to_string());println!("Answer 4: {}", answer_4);assert!(answer_4.to_lowercase().contains("director"));@@ -194,15 +146,10 @@ fn test_context_mode_dynamic() {#[test]fn test_context_mode_dynamic_scenario() {- let decoding_config =- build_decoding_config().with_context_mode(ContextMode::Dynamic);- let mut session =- Session::new(build_model_path(), decoding_config).unwrap();-- let user_prompts = vec![- String::from("Tell about London"),- String::from("Compare with New York"),- ];+ let decoding_config = build_decoding_config().with_context_mode(ContextMode::Dynamic);+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();++ let user_prompts = vec![String::from("Tell about London"), String::from("Compare with New York")];println!("-------------------------");for user_prompt in user_prompts {diff --git a/crates/uzu/tests/grammar_json_schema_test.rs b/crates/uzu/tests/grammar_json_schema_test.rsindex 542a9207e..14d6cfcf7 100644--- a/crates/uzu/tests/grammar_json_schema_test.rs+++ b/crates/uzu/tests/grammar_json_schema_test.rs@@ -44,41 +44,29 @@ fn test_grammar(speculator_config: SpeculatorConfig) {let decoding_config = DecodingConfig::default().with_sampling_seed(uzu::prelude::SamplingSeed::Custom(42)).with_speculator_config(speculator_config);- let mut session = Session::new(model_dir, decoding_config)- .expect("Failed to create session");+ let mut session = Session::new(model_dir, decoding_config).expect("Failed to create session");- let input = Input::Text(- "Generate a detailed person profile with address and hobbies in JSON format.".to_string(),- );+ let input = Input::Text("Generate a detailed person profile with address and hobbies in JSON format.".to_string());- let grammar_config = GrammarConfig::from_json_schema_type::<Person>()- .expect("Failed to create grammar config");+ let grammar_config = GrammarConfig::from_json_schema_type::<Person>().expect("Failed to create grammar config");- let run_config = RunConfig::default()- .tokens_limit(1024)- .sampling_policy(SamplingPolicy::Default)- .grammar_config(grammar_config);+ let run_config =+ RunConfig::default().tokens_limit(1024).sampling_policy(SamplingPolicy::Default).grammar_config(grammar_config);- let output = session- .run(input, run_config, None::<fn(uzu::session::types::Output) -> bool>)- .expect("Failed to run session");+ let output =+ session.run(input, run_config, None::<fn(uzu::session::types::Output) -> bool>).expect("Failed to run session");let stats = output.stats;- let total_tokens = stats.total_stats.tokens_count_input- + stats.total_stats.tokens_count_output;+ let total_tokens = stats.total_stats.tokens_count_input + stats.total_stats.tokens_count_output;println!("\n=== Test Results ===");println!("Generated output length: {} chars", output.text.original.len());println!("Input tokens: {}, Output tokens: {}",- stats.total_stats.tokens_count_input,- stats.total_stats.tokens_count_output+ stats.total_stats.tokens_count_input, stats.total_stats.tokens_count_output);println!("Total time: {:.2}s", stats.total_stats.duration);- println!(- "Throughput: {:.2} tokens/s",- total_tokens as f64 / stats.total_stats.duration- );+ println!("Throughput: {:.2} tokens/s", total_tokens as f64 / stats.total_stats.duration);match serde_json::from_str::<Person>(&output.text.original) {Ok(person) => {@@ -86,10 +74,7 @@ fn test_grammar(speculator_config: SpeculatorConfig) {println!(" Name: {}", person.name);println!(" Age: {}", person.age);println!(" Email: {}", person.email);- println!(- " Address: {}, {}",- person.address.city, person.address.country- );+ println!(" Address: {}, {}", person.address.city, person.address.country);println!(" Hobbies: {:?}", person.hobbies);assert!(!person.name.is_empty());@@ -99,10 +84,7 @@ fn test_grammar(speculator_config: SpeculatorConfig) {assert!(!person.hobbies.is_empty());},Err(e) => {- panic!(- "β Generated invalid JSON: {}\nOutput: {}",- e, output.text.original- );+ panic!("β Generated invalid JSON: {}\nOutput: {}", e, output.text.original);},}}diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex c2ee30066..5490026fe 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -8,9 +8,7 @@ use uzu::{common::Context,metal::{KVCacheUpdate, MTLContext, Metal},},- forward_pass::kv_cache_layer::{- INVALID_POSITION, KVCacheLayer, KVCacheLayerState,- },+ forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},};#[derive(Debug)]@@ -49,16 +47,8 @@ fn make_test_layer(};let shape = [1, total_len.max(1), 1];- let keys = std::cell::RefCell::new(context.create_array(- &shape,- DataType::F32,- "kv_cache_keys",- ));- let values = std::cell::RefCell::new(context.create_array(- &shape,- DataType::F32,- "kv_cache_values",- ));+ let keys = std::cell::RefCell::new(context.create_array(&shape, DataType::F32, "kv_cache_keys"));+ let values = std::cell::RefCell::new(context.create_array(&shape, DataType::F32, "kv_cache_values"));let prefix_token_positions = match &state {KVCacheLayerState::Full {@@ -107,13 +97,12 @@ fn expected_after_update(initial: &[f32],) -> Vec<f32> {let mut expected = initial.to_vec();- let effective_indices: Vec<usize> = if accepted_indices.is_empty()- && matches!(state, KVCacheLayerState::Windowed { .. })- {- vec![0]- } else {- accepted_indices.to_vec()- };+ let effective_indices: Vec<usize> =+ if accepted_indices.is_empty() && matches!(state, KVCacheLayerState::Windowed { .. }) {+ vec![0]+ } else {+ accepted_indices.to_vec()+ };match state {KVCacheLayerState::Full {prefix_len,@@ -152,9 +141,7 @@ fn collect_mask((0..suffix_length).map(|row| {(0..prefix_segment_length + suffix_length)- .map(|col| {- layer.bias_should_be_neg_inf(row, col, suffix_positions)- })+ .map(|col| layer.bias_should_be_neg_inf(row, col, suffix_positions)).collect()}).collect()@@ -164,12 +151,8 @@ fn run_scenario(context: &MTLContext,scenario: &Scenario,) {- let mut layer = make_test_layer(- context,- scenario.state.clone(),- scenario.prefix_capacity,- scenario.suffix_capacity,- );+ let mut layer =+ make_test_layer(context, scenario.state.clone(), scenario.prefix_capacity, scenario.suffix_capacity);layer.prefix_token_positions = scenario.initial_prefix_positions.clone();@@ -177,16 +160,9 @@ fn run_scenario(let (initial_keys, initial_values) = fill_arrays(&mut layer);- let expected_keys = expected_after_update(- &state_before_update,- &scenario.accepted_suffix_indices,- &initial_keys,- );- let expected_values = expected_after_update(- &state_before_update,- &scenario.accepted_suffix_indices,- &initial_values,- );+ let expected_keys = expected_after_update(&state_before_update, &scenario.accepted_suffix_indices, &initial_keys);+ let expected_values =+ expected_after_update(&state_before_update, &scenario.accepted_suffix_indices, &initial_values);let total_sequence_length = match &layer.state {KVCacheLayerState::Full {@@ -199,29 +175,16 @@ fn run_scenario(};let mask = collect_mask(&layer, &scenario.suffix_token_positions);- assert_eq!(- mask, scenario.expected_mask,- "{}: bias mask mismatch",- scenario.name- );+ assert_eq!(mask, scenario.expected_mask, "{}: bias mask mismatch", scenario.name);- let kv_cache_update =- match KVCacheUpdate::new(context, DataType::F32, total_sequence_length)- {- Ok(update) => update,- Err(e) => {- panic!(- "Failed to create KV cache update for scenario {}: {:?}",- scenario.name, e- );- },- };+ let kv_cache_update = match KVCacheUpdate::new(context, DataType::F32, total_sequence_length) {+ Ok(update) => update,+ Err(e) => {+ panic!("Failed to create KV cache update for scenario {}: {:?}", scenario.name, e);+ },+ };- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();let root_command_buffer = command_buffer.clone();layer.update_after_acceptance(@@ -240,21 +203,13 @@ fn run_scenario(let keys_ref = layer.keys.borrow();keys_ref.as_slice::<f32>().to_vec()};- assert_eq!(- actual_keys, expected_keys,- "{}: key buffer mismatch",- scenario.name- );+ assert_eq!(actual_keys, expected_keys, "{}: key buffer mismatch", scenario.name);let actual_values = {let values_ref = layer.values.borrow();values_ref.as_slice::<f32>().to_vec()};- assert_eq!(- actual_values, expected_values,- "{}: value buffer mismatch",- scenario.name- );+ assert_eq!(actual_values, expected_values, "{}: value buffer mismatch", scenario.name);assert_eq!(layer.prefix_token_positions, scenario.expected_prefix_positions,@@ -267,11 +222,7 @@ fn run_scenario(prefix_len,} => {if let Some(expected_len) = scenario.expected_prefix_len {- assert_eq!(- *prefix_len, expected_len,- "{}: prefix length mismatch",- scenario.name- );+ assert_eq!(*prefix_len, expected_len, "{}: prefix length mismatch", scenario.name);}},KVCacheLayerState::Windowed {@@ -280,18 +231,10 @@ fn run_scenario(..} => {if let Some(expected_offset) = scenario.expected_ring_offset {- assert_eq!(- *ring_offset, expected_offset,- "{}: ring offset mismatch",- scenario.name- );+ assert_eq!(*ring_offset, expected_offset, "{}: ring offset mismatch", scenario.name);}if let Some(expected_length) = scenario.expected_ring_length {- assert_eq!(- *ring_length, expected_length,- "{}: ring length mismatch",- scenario.name- );+ assert_eq!(*ring_length, expected_length, "{}: ring length mismatch", scenario.name);}},}@@ -327,15 +270,9 @@ fn kv_cache_state_and_mask_scenarios() {suffix_start: None,expected_prefix_positions: vec![6, 7, 8, 9, 4, 5],expected_mask: vec![- vec![- false, true, false, false, false, false, false, true, true,- ],- vec![- false, true, true, false, false, false, false, false, true,- ],- vec![- false, true, true, true, false, false, false, false, false,- ],+ vec![false, true, false, false, false, false, false, true, true],+ vec![false, true, true, false, false, false, false, false, true],+ vec![false, true, true, true, false, false, false, false, false],],expected_ring_offset: Some(4),expected_ring_length: Some(6),@@ -351,14 +288,7 @@ fn kv_cache_state_and_mask_scenarios() {},prefix_capacity: 6,suffix_capacity: 3,- initial_prefix_positions: vec![- 0,- 1,- 2,- 3,- INVALID_POSITION,- INVALID_POSITION,- ],+ initial_prefix_positions: vec![0, 1, 2, 3, INVALID_POSITION, INVALID_POSITION],accepted_suffix_indices: vec![0, 1, 2],accepted_token_positions: vec![4, 5, 6],suffix_token_positions: vec![4, 5, 6],@@ -366,12 +296,8 @@ fn kv_cache_state_and_mask_scenarios() {expected_prefix_positions: vec![6, 1, 2, 3, 4, 5],expected_mask: vec![vec![false, false, false, false, true, true, false, true, true],- vec![- false, false, false, false, true, true, false, false, true,- ],- vec![- true, false, false, false, true, true, false, false, false,- ],+ vec![false, false, false, false, true, true, false, false, true],+ vec![true, false, false, false, true, true, false, false, false],],expected_ring_offset: Some(1),expected_ring_length: Some(6),@@ -393,9 +319,7 @@ fn kv_cache_state_and_mask_scenarios() {suffix_token_positions: vec![16],suffix_start: None,expected_prefix_positions: vec![11, 12, 13, 14, 15, 16],- expected_mask: vec![vec![- false, false, false, false, false, true, false,- ]],+ expected_mask: vec![vec![false, false, false, false, false, true, false]],expected_ring_offset: Some(0),expected_ring_length: Some(6),expected_prefix_len: None,@@ -473,16 +397,8 @@ fn kv_cache_slice_apply_contiguous_window() {let keys_after = layer.keys.borrow().as_slice::<f32>().to_vec();let values_after = layer.values.borrow().as_slice::<f32>().to_vec();- assert_eq!(- keys_after[0..4],- initial_keys[0..4],- "keys restored for contiguous slice"- );- assert_eq!(- values_after[0..4],- initial_values[0..4],- "values restored for contiguous slice"- );+ assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for contiguous slice");+ assert_eq!(values_after[0..4], initial_values[0..4], "values restored for contiguous slice");}#[test]@@ -519,24 +435,12 @@ fn kv_cache_slice_apply_wrap_window() {layer.apply_slice(&slice, None);- assert_eq!(- layer.prefix_token_positions,- vec![30, 31, 32, 33],- "positions restored for wrapped slice"- );+ assert_eq!(layer.prefix_token_positions, vec![30, 31, 32, 33], "positions restored for wrapped slice");let keys_after = layer.keys.borrow().as_slice::<f32>().to_vec();let values_after = layer.values.borrow().as_slice::<f32>().to_vec();- assert_eq!(- keys_after[0..4],- initial_keys[0..4],- "keys restored for wrapped slice"- );- assert_eq!(- values_after[0..4],- initial_values[0..4],- "values restored for wrapped slice"- );+ assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for wrapped slice");+ assert_eq!(values_after[0..4], initial_values[0..4], "values restored for wrapped slice");}#[test]@@ -568,11 +472,7 @@ fn kv_cache_slice_apply_full_restores_metadata() {layer.apply_slice(&slice, None);- assert_eq!(- layer.prefix_token_positions,- vec![1, 2, 3],- "full slice restores positions"- );+ assert_eq!(layer.prefix_token_positions, vec![1, 2, 3], "full slice restores positions");if let KVCacheLayerState::Full {prefix_len,} = &layer.statediff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rsindex 1af8f358c..82f22c04b 100644--- a/crates/uzu/tests/kv_cache_update_test.rs+++ b/crates/uzu/tests/kv_cache_update_test.rs@@ -1,9 +1,6 @@#![cfg(any(target_os = "macos", target_os = "ios"))]use bytemuck;-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt,- MTLResourceOptions,-};+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use ndarray::{Array, Array3, s};use uzu::backends::{common::{@@ -25,8 +22,7 @@ fn apply_swaps_3d<T: Clone>(let src = swap.source as usize;let dst = swap.destination as usize;let temp = array[(head, src, channel)].clone();- array[(head, src, channel)] =- array[(head, dst, channel)].clone();+ array[(head, src, channel)] = array[(head, dst, channel)].clone();array[(head, dst, channel)] = temp;}}@@ -50,17 +46,10 @@ fn test_random_pattern(context: &MTLContext) {println!("Testing with random pattern...");let max_sequence_length = 256usize;- let kv_cache_update = match KVCacheUpdate::new(- context,- uzu::DataType::F32,- max_sequence_length,- ) {+ let kv_cache_update = match KVCacheUpdate::new(context, uzu::DataType::F32, max_sequence_length) {Ok(update) => update,Err(e) => {- println!(- "Warning: Failed to create KV cache update: {:?}. Skipping test.",- e- );+ println!("Warning: Failed to create KV cache update: {:?}. Skipping test.", e);return;},};@@ -69,15 +58,13 @@ fn test_random_pattern(context: &MTLContext) {let seq_len = 15usize;let head_dim = 7usize;- let key_data = Array3::<f32>::from_shape_fn(- (num_heads, seq_len, head_dim),- |(h, t, c)| (h * 1_000_000 + t * 100 + c * 10) as f32,- );+ let key_data = Array3::<f32>::from_shape_fn((num_heads, seq_len, head_dim), |(h, t, c)| {+ (h * 1_000_000 + t * 100 + c * 10) as f32+ });- let value_data = Array3::<f32>::from_shape_fn(- (num_heads, seq_len, head_dim),- |(h, t, c)| (h * 1_000_000 + t * 100 + c * 10 + 1_000) as f32,- );+ let value_data = Array3::<f32>::from_shape_fn((num_heads, seq_len, head_dim), |(h, t, c)| {+ (h * 1_000_000 + t * 100 + c * 10 + 1_000) as f32+ });let source_indices = vec![0, 3, 6, 9, 12, 2, 5, 8, 11, 14];let destination_indices = vec![14, 11, 8, 5, 2, 12, 9, 6, 3, 0];@@ -112,23 +99,11 @@ fn test_random_pattern(context: &MTLContext) {value_shape: [num_heads, seq_len, head_dim],};- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- match kv_cache_update.encode(- &[kv_layer_data],- &source_indices,- &destination_indices,- &command_buffer,- ) {+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &command_buffer) {Ok(_) => {},Err(e) => {- println!(- "Warning: Failed to encode KV cache update: {:?}. Skipping test.",- e- );+ println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);return;},}@@ -141,22 +116,14 @@ fn test_random_pattern(context: &MTLContext) {let total_elems = num_heads * seq_len * head_dim;- let key_result_slice =- unsafe { std::slice::from_raw_parts(key_result_ptr, total_elems) };- let value_result_slice =- unsafe { std::slice::from_raw_parts(value_result_ptr, total_elems) };-- let key_result = Array::from_shape_vec(- (num_heads, seq_len, head_dim),- key_result_slice.to_vec(),- )- .expect("Failed to convert key result to ndarray");-- let value_result = Array::from_shape_vec(- (num_heads, seq_len, head_dim),- value_result_slice.to_vec(),- )- .expect("Failed to convert value result to ndarray");+ let key_result_slice = unsafe { std::slice::from_raw_parts(key_result_ptr, total_elems) };+ let value_result_slice = unsafe { std::slice::from_raw_parts(value_result_ptr, total_elems) };++ let key_result = Array::from_shape_vec((num_heads, seq_len, head_dim), key_result_slice.to_vec())+ .expect("Failed to convert key result to ndarray");++ let value_result = Array::from_shape_vec((num_heads, seq_len, head_dim), value_result_slice.to_vec())+ .expect("Failed to convert value result to ndarray");println!("Original keys head 0 rows 0,14:");println!("Row 0: {:?}", key_data.slice(s![0, 0, ..]));diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 25247df29..c5731e335 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -2,10 +2,7 @@use bytemuck;use half::bf16;-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,- MTLDeviceExt, MTLResourceOptions,-};+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use ndarray::Array2;use uzu::{DataType,@@ -29,24 +26,15 @@ fn run_metal_matmul() -> Vec<bf16> {let a_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(a_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(a_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let b_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(b_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(b_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let d_buf = ctx.device- .new_buffer(- m * n * core::mem::size_of::<bf16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(m * n * core::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let ldb = if transpose_b {@@ -57,14 +45,8 @@ fn run_metal_matmul(let mut kernel = MatmulKernel::new(DataType::BF16).expect("kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let enc = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");let encode_result = kernel.encode(ctx,&enc,@@ -129,17 +111,13 @@ fn generate_test_data(n: usize,transpose_b: bool,) -> (Vec<bf16>, Vec<bf16>) {- let a: Vec<bf16> = (0..(m * k))- .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01 - 0.06))- .collect();+ let a: Vec<bf16> = (0..(m * k)).map(|i| bf16::from_f32(((i % 13) as f32) * 0.01 - 0.06)).collect();let b_size = if transpose_b {n * k} else {k * n};- let b: Vec<bf16> = (0..b_size)- .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.15))- .collect();+ let b: Vec<bf16> = (0..b_size).map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.15)).collect();(a, b)}@@ -154,8 +132,7 @@ struct TestCase {fn test_cases() -> Vec<TestCase> {let base_tolerance = 0.01;- let batch_sizes: [usize; 12] =- [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];+ let batch_sizes: [usize; 12] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];let model_shapes: [(usize, usize); 51] = [// Qwen2.5-Coder-0.5B@@ -221,10 +198,8 @@ fn test_cases() -> Vec<TestCase> {(17408, 5120),];- let mut cases: Vec<(usize, usize, usize, bool)> = model_shapes- .iter()- .flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n, true)))- .collect();+ let mut cases: Vec<(usize, usize, usize, bool)> =+ model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n, true))).collect();// Non-transposed B (for attention AV matmul)for &(k, n) in &[(1024, 1024), (2048, 2048), (4096, 4096)] {@@ -267,8 +242,7 @@ fn compare_results(let mut max_diff_idx = 0;let mut diff_count = 0;- for (i, (&m_val, &r_val)) in metal.iter().zip(reference.iter()).enumerate()- {+ for (i, (&m_val, &r_val)) in metal.iter().zip(reference.iter()).enumerate() {let mf = m_val.to_f32();let rf = r_val.to_f32();let diff = (mf - rf).abs();@@ -301,25 +275,9 @@ fn matmul_correctness_comprehensive() {let mut failed = Vec::new();for case in &cases {- let (a, b) =- generate_test_data(case.m, case.k, case.n, case.transpose_b);- let metal_result = run_metal_matmul(- &ctx,- &a,- &b,- case.m,- case.k,- case.n,- case.transpose_b,- );- let reference = run_ndarray_matmul(- &a,- &b,- case.m,- case.k,- case.n,- case.transpose_b,- );+ let (a, b) = generate_test_data(case.m, case.k, case.n, case.transpose_b);+ let metal_result = run_metal_matmul(&ctx, &a, &b, case.m, case.k, case.n, case.transpose_b);+ let reference = run_ndarray_matmul(&a, &b, case.m, case.k, case.n, case.transpose_b);let trans_str = if case.transpose_b {"T"@@ -330,23 +288,13 @@ fn matmul_correctness_comprehensive() {match compare_results(&metal_result, &reference, case.tolerance) {Ok(()) => {passed += 1;- eprintln!(- "β m={} k={} n={} B={}",- case.m, case.k, case.n, trans_str- );+ eprintln!("β m={} k={} n={} B={}", case.m, case.k, case.n, trans_str);},Err((max_diff, idx, count)) => {failed.push((case, max_diff, idx, count));eprintln!("β m={} k={} n={} B={} max_diff={:.6} at idx {} ({} exceed tol {})",- case.m,- case.k,- case.n,- trans_str,- max_diff,- idx,- count,- case.tolerance+ case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance);},}@@ -364,14 +312,7 @@ fn matmul_correctness_comprehensive() {};eprintln!(" m={} k={} n={} B={}: max_diff={:.6} at idx {}, {} exceed tol {}",- case.m,- case.k,- case.n,- trans_str,- max_diff,- idx,- count,- case.tolerance+ case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance);}panic!("{} tests failed", failed.len());diff --git a/crates/uzu/tests/mlp_fused_correctness_test.rs b/crates/uzu/tests/mlp_fused_correctness_test.rsindex 88575a151..7d407a348 100644--- a/crates/uzu/tests/mlp_fused_correctness_test.rs+++ b/crates/uzu/tests/mlp_fused_correctness_test.rs@@ -3,10 +3,7 @@use bytemuck;use half::f16;-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,- MTLDeviceExt, MTLResourceOptions,-};+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use objc2::rc::autoreleasepool;use uzu::{DataType,@@ -81,36 +78,21 @@ fn run_fused_gemv() -> Vec<f16> {let input_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(input),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(input), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let weights_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(weights),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(weights), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buf = ctx.device- .new_buffer(- hidden_dim * core::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(hidden_dim * core::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let mut kernel = MlpFusedKernel::new(DataType::F16, true).expect("kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let enc = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");let args = MlpFusedArguments {input: &input_buf,@@ -150,36 +132,21 @@ fn run_fused_gemm() -> Vec<f16> {let input_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(input),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(input), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let weights_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(weights),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(weights), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buf = ctx.device- .new_buffer(- m * hidden_dim * core::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(m * hidden_dim * core::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let mut kernel = MlpFusedKernel::new(DataType::F16, true).expect("kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let enc = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");let args = MlpFusedArguments {input: &input_buf,@@ -252,47 +219,25 @@ fn test_mlp_fused_gemv_silu_small() {let m = 1;// Generate random-ish test data- let input: Vec<f32> =- (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();- let weights_up: Vec<f32> = (0..hidden_dim * k)- .map(|i| (i as f32 * 0.017).cos() * 0.3)- .collect();- let weights_gate: Vec<f32> = (0..hidden_dim * k)- .map(|i| (i as f32 * 0.023).sin() * 0.3)- .collect();+ let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();+ let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();+ let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();// Create concatenated weights [up; gate] row-major- let mut weights_concat: Vec<f32> =- Vec::with_capacity(2 * hidden_dim * k);+ let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);weights_concat.extend(&weights_up);weights_concat.extend(&weights_gate);// CPU reference- let expected = mlp_fused_reference(- &input,- &weights_up,- &weights_gate,- m,- k,- hidden_dim,- MlpActivationType::SiLU,- );+ let expected =+ mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);// Convert to f16- let input_f16: Vec<f16> =- input.iter().map(|&x| f16::from_f32(x)).collect();- let weights_f16: Vec<f16> =- weights_concat.iter().map(|&x| f16::from_f32(x)).collect();+ let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();+ let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();// Run fused kernel- let actual = run_fused_gemv(- &ctx,- &input_f16,- &weights_f16,- k,- hidden_dim,- MlpActivationType::SiLU,- );+ let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::SiLU);let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);println!("GEMV SiLU small: max_diff = {}", max_diff);@@ -315,43 +260,21 @@ fn test_mlp_fused_gemv_gelu_small() {let hidden_dim = 32;let m = 1;- let input: Vec<f32> =- (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();- let weights_up: Vec<f32> = (0..hidden_dim * k)- .map(|i| (i as f32 * 0.017).cos() * 0.3)- .collect();- let weights_gate: Vec<f32> = (0..hidden_dim * k)- .map(|i| (i as f32 * 0.023).sin() * 0.3)- .collect();-- let mut weights_concat: Vec<f32> =- Vec::with_capacity(2 * hidden_dim * k);+ let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();+ let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();+ let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();++ let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);weights_concat.extend(&weights_up);weights_concat.extend(&weights_gate);- let expected = mlp_fused_reference(- &input,- &weights_up,- &weights_gate,- m,- k,- hidden_dim,- MlpActivationType::Gelu,- );-- let input_f16: Vec<f16> =- input.iter().map(|&x| f16::from_f32(x)).collect();- let weights_f16: Vec<f16> =- weights_concat.iter().map(|&x| f16::from_f32(x)).collect();-- let actual = run_fused_gemv(- &ctx,- &input_f16,- &weights_f16,- k,- hidden_dim,- MlpActivationType::Gelu,- );+ let expected =+ mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::Gelu);++ let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();+ let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();++ let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::Gelu);let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);println!("GEMV GELU small: max_diff = {}", max_diff);@@ -374,44 +297,21 @@ fn test_mlp_fused_gemm_silu_medium() {let k = 64;let hidden_dim = 64;- let input: Vec<f32> =- (0..m * k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();- let weights_up: Vec<f32> = (0..hidden_dim * k)- .map(|i| (i as f32 * 0.017).cos() * 0.3)- .collect();- let weights_gate: Vec<f32> = (0..hidden_dim * k)- .map(|i| (i as f32 * 0.023).sin() * 0.3)- .collect();-- let mut weights_concat: Vec<f32> =- Vec::with_capacity(2 * hidden_dim * k);+ let input: Vec<f32> = (0..m * k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();+ let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();+ let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();++ let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);weights_concat.extend(&weights_up);weights_concat.extend(&weights_gate);- let expected = mlp_fused_reference(- &input,- &weights_up,- &weights_gate,- m,- k,- hidden_dim,- MlpActivationType::SiLU,- );-- let input_f16: Vec<f16> =- input.iter().map(|&x| f16::from_f32(x)).collect();- let weights_f16: Vec<f16> =- weights_concat.iter().map(|&x| f16::from_f32(x)).collect();-- let actual = run_fused_gemm(- &ctx,- &input_f16,- &weights_f16,- m,- k,- hidden_dim,- MlpActivationType::SiLU,- );+ let expected =+ mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);++ let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();+ let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();++ let actual = run_fused_gemm(&ctx, &input_f16, &weights_f16, m, k, hidden_dim, MlpActivationType::SiLU);let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.05);println!("GEMM SiLU medium: max_diff = {}", max_diff);@@ -435,43 +335,21 @@ fn test_mlp_fused_gemv_realistic_dimensions() {let hidden_dim = 256; // hidden dim (half of up+gate)let m = 1;- let input: Vec<f32> =- (0..k).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();- let weights_up: Vec<f32> = (0..hidden_dim * k)- .map(|i| (i as f32 * 0.001).cos() * 0.1)- .collect();- let weights_gate: Vec<f32> = (0..hidden_dim * k)- .map(|i| (i as f32 * 0.002).sin() * 0.1)- .collect();-- let mut weights_concat: Vec<f32> =- Vec::with_capacity(2 * hidden_dim * k);+ let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();+ let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.001).cos() * 0.1).collect();+ let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.002).sin() * 0.1).collect();++ let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);weights_concat.extend(&weights_up);weights_concat.extend(&weights_gate);- let expected = mlp_fused_reference(- &input,- &weights_up,- &weights_gate,- m,- k,- hidden_dim,- MlpActivationType::SiLU,- );-- let input_f16: Vec<f16> =- input.iter().map(|&x| f16::from_f32(x)).collect();- let weights_f16: Vec<f16> =- weights_concat.iter().map(|&x| f16::from_f32(x)).collect();-- let actual = run_fused_gemv(- &ctx,- &input_f16,- &weights_f16,- k,- hidden_dim,- MlpActivationType::SiLU,- );+ let expected =+ mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);++ let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();+ let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();++ let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::SiLU);let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.02);println!("GEMV realistic: max_diff = {}", max_diff);diff --git a/crates/uzu/tests/mlp_fused_performance_test.rs b/crates/uzu/tests/mlp_fused_performance_test.rsindex fc3542408..2333f1747 100644--- a/crates/uzu/tests/mlp_fused_performance_test.rs+++ b/crates/uzu/tests/mlp_fused_performance_test.rs@@ -4,10 +4,7 @@use std::time::{Duration, Instant};use half::f16;-use metal::{- MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt,- MTLResourceOptions,-};+use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use objc2::rc::autoreleasepool;use uzu::{DataType,@@ -16,14 +13,10 @@ use uzu::{metal::{MTLContext,kernel::{- matmul::{- MatmulArguments, MatmulKernel,- determine_kernel_variant as determine_matmul_variant,- },+ matmul::{MatmulArguments, MatmulKernel, determine_kernel_variant as determine_matmul_variant},mlp::MlpActivationType,mlp_fused::{- MlpFusedArguments, MlpFusedKernel,- determine_kernel_variant as determine_mlp_fused_variant,+ MlpFusedArguments, MlpFusedKernel, determine_kernel_variant as determine_mlp_fused_variant,},},},@@ -66,10 +59,7 @@ fn benchmark_gemv_fused() -> BenchmarkResult {let input_buffer = context.device- .new_buffer(- input_dimension * std::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(input_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let weights_buffer = context.device@@ -80,21 +70,14 @@ fn benchmark_gemv_fused(.expect("Failed to create buffer");let output_buffer = context.device- .new_buffer(- hidden_dimension * std::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let fused_up_buffer = context.device- .new_buffer(- 2 * hidden_dimension * std::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(2 * hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut fused_kernel =- MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");+ let mut fused_kernel = MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");let mut matmul_kernel = MatmulKernel::new(DataType::F16).expect("matmul");let fused_args = MlpFusedArguments {@@ -114,14 +97,9 @@ fn benchmark_gemv_fused(// Warmup fusedfor _ in 0..warmup_count {- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");fused_kernel.encode(context, &encoder, &fused_args).unwrap();encoder.end_encoding();command_buffer.commit();@@ -131,14 +109,9 @@ fn benchmark_gemv_fused(// Benchmark fusedlet fused_start = Instant::now();for _ in 0..iteration_count {- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");fused_kernel.encode(context, &encoder, &fused_args).unwrap();encoder.end_encoding();command_buffer.commit();@@ -167,25 +140,16 @@ fn benchmark_gemv_fused(};let fused_kernel_name =- determine_mlp_fused_variant(context, DataType::F16, true, &fused_args)- .map(|v| v.as_str())- .unwrap_or("?");+ determine_mlp_fused_variant(context, DataType::F16, true, &fused_args).map(|v| v.as_str()).unwrap_or("?");let unfused_kernel_name =- determine_matmul_variant(context, DataType::F16, &matmul_args)- .map(|v| v.as_str())- .unwrap_or("?");+ determine_matmul_variant(context, DataType::F16, &matmul_args).map(|v| v.as_str()).unwrap_or("?");// Warmup unfusedfor _ in 0..warmup_count {- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();encoder.end_encoding();command_buffer.commit();@@ -195,14 +159,9 @@ fn benchmark_gemv_fused(// Benchmark unfusedlet unfused_start = Instant::now();for _ in 0..iteration_count {- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();encoder.end_encoding();command_buffer.commit();@@ -210,12 +169,7 @@ fn benchmark_gemv_fused(}let unfused_duration = unfused_start.elapsed() / iteration_count as u32;- BenchmarkResult::new(- fused_duration,- unfused_duration,- fused_kernel_name,- unfused_kernel_name,- )+ BenchmarkResult::new(fused_duration, unfused_duration, fused_kernel_name, unfused_kernel_name)}fn benchmark_gemm_fused(@@ -228,10 +182,7 @@ fn benchmark_gemm_fused() -> BenchmarkResult {let input_buffer = context.device- .new_buffer(- batch_size * input_dimension * std::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(batch_size * input_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let weights_buffer = context.device@@ -242,10 +193,7 @@ fn benchmark_gemm_fused(.expect("Failed to create buffer");let output_buffer = context.device- .new_buffer(- batch_size * hidden_dimension * std::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(batch_size * hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let fused_up_buffer = context.device@@ -255,8 +203,7 @@ fn benchmark_gemm_fused().expect("Failed to create buffer");- let mut fused_kernel =- MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");+ let mut fused_kernel = MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");let mut matmul_kernel = MatmulKernel::new(DataType::F16).expect("matmul");let fused_args = MlpFusedArguments {@@ -276,14 +223,9 @@ fn benchmark_gemm_fused(// Warmup fusedfor _ in 0..warmup_count {- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");fused_kernel.encode(context, &encoder, &fused_args).unwrap();encoder.end_encoding();command_buffer.commit();@@ -292,14 +234,9 @@ fn benchmark_gemm_fused(let fused_start = Instant::now();for _ in 0..iteration_count {- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");fused_kernel.encode(context, &encoder, &fused_args).unwrap();encoder.end_encoding();command_buffer.commit();@@ -328,25 +265,16 @@ fn benchmark_gemm_fused(};let fused_kernel_name =- determine_mlp_fused_variant(context, DataType::F16, true, &fused_args)- .map(|v| v.as_str())- .unwrap_or("?");+ determine_mlp_fused_variant(context, DataType::F16, true, &fused_args).map(|v| v.as_str()).unwrap_or("?");let unfused_kernel_name =- determine_matmul_variant(context, DataType::F16, &matmul_args)- .map(|v| v.as_str())- .unwrap_or("?");+ determine_matmul_variant(context, DataType::F16, &matmul_args).map(|v| v.as_str()).unwrap_or("?");// Warmup unfusedfor _ in 0..warmup_count {- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();encoder.end_encoding();command_buffer.commit();@@ -355,14 +283,9 @@ fn benchmark_gemm_fused(let unfused_start = Instant::now();for _ in 0..iteration_count {- let command_buffer = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer")- .to_owned();- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();encoder.end_encoding();command_buffer.commit();@@ -370,12 +293,7 @@ fn benchmark_gemm_fused(}let unfused_duration = unfused_start.elapsed() / iteration_count as u32;- BenchmarkResult::new(- fused_duration,- unfused_duration,- fused_kernel_name,- unfused_kernel_name,- )+ BenchmarkResult::new(fused_duration, unfused_duration, fused_kernel_name, unfused_kernel_name)}/// MLP shapes from real model configs@@ -425,27 +343,14 @@ fn mlp_fused_performance_benchmark() {println!("{:>10} {:>10} {:>10} {:>10} {:>12} {:>10} {:>12} {:>10}",- "batch",- "input",- "hidden",- "Fused",- "Fused (Β΅s)",- "Unfused",- "Unfused (Β΅s)",- "Speedup"+ "batch", "input", "hidden", "Fused", "Fused (Β΅s)", "Unfused", "Unfused (Β΅s)", "Speedup");println!("{}", "-".repeat(98));for &batch_size in BATCH_SIZES {for &(input_dimension, hidden_dimension) in MLP_SHAPES {let result = if batch_size == 1 {- benchmark_gemv_fused(- &context,- input_dimension,- hidden_dimension,- iteration_count,- warmup_count,- )+ benchmark_gemv_fused(&context, input_dimension, hidden_dimension, iteration_count, warmup_count)} else {benchmark_gemm_fused(&context,diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rsindex 4e8a96a6a..7a9b9ef00 100644--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs@@ -56,18 +56,13 @@ fn gen_topk_ids_from_logits(// Generate random input and router weightslet d_model = 64; // arbitrary model dimension for test- let input_f32: Vec<f32> =- (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();- let weight_f32: Vec<f32> =- (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();- let bias_f32: Vec<f32> =- (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();+ let input_f32: Vec<f32> = (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();+ let weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();+ let bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();// Convert to bf16- let input: Vec<bf16> =- input_f32.iter().map(|&x| bf16::from_f32(x)).collect();- let weight: Vec<bf16> =- weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();+ let input: Vec<bf16> = input_f32.iter().map(|&x| bf16::from_f32(x)).collect();+ let weight: Vec<bf16> = weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();let bias: Vec<bf16> = bias_f32.iter().map(|&x| bf16::from_f32(x)).collect();let input_buf = alloc_buffer_with_data(ctx, &input);@@ -77,12 +72,8 @@ fn gen_topk_ids_from_logits(let topk_probs_buf = alloc_buffer::<bf16>(ctx, t * k);// Use fused router+topk kernel- let router_topk =- MoeRouterTopKKernel::new(ctx).expect("router_topk kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let router_topk = MoeRouterTopKKernel::new(ctx).expect("router_topk kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let args = MoeRouterTopKArguments {input_buffer: &input_buf,weight_buffer: &weight_buf,@@ -95,9 +86,7 @@ fn gen_topk_ids_from_logits(k,renorm: true,};- router_topk- .encode(&cb, KernelDataType::BFloat16, args)- .expect("encode router_topk");+ router_topk.encode(&cb, KernelDataType::BFloat16, args).expect("encode router_topk");cb.commit();cb.wait_until_completed();@@ -117,8 +106,7 @@ fn test_counts_offsets_fused_parity_random() {if k > e {continue;}- let (topk_ids, topk_ids_buf) =- gen_topk_ids_from_logits(&ctx, t, e, k);+ let (topk_ids, topk_ids_buf) = gen_topk_ids_from_logits(&ctx, t, e, k);// CPU referencelet counts_cpu = cpu_bucket_counts(&topk_ids, t, k, e);@@ -130,12 +118,8 @@ fn test_counts_offsets_fused_parity_random() {let num_tiles = ((e + 511) / 512).max(1);let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);- let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx)- .expect("fused kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");kernel.encode(@@ -155,25 +139,13 @@ fn test_counts_offsets_fused_parity_random() {// Verify offsetslet offsets_ptr = offsets_buf.contents().as_ptr() as *const u32;- let offsets_gpu =- unsafe { std::slice::from_raw_parts(offsets_ptr, e + 1) };- assert_eq!(- offsets_gpu,- &offsets_cpu[..],- "offsets mismatch T={} E={} K={}",- t,- e,- k- );+ let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_ptr, e + 1) };+ assert_eq!(offsets_gpu, &offsets_cpu[..], "offsets mismatch T={} E={} K={}", t, e, k);// Verify sumlet sum_ptr = sum_k_buf.contents().as_ptr() as *const u32;let sum_gpu = unsafe { *sum_ptr };- assert_eq!(- sum_gpu, sum_cpu,- "sum mismatch T={} E={} K={}",- t, e, k- );+ assert_eq!(sum_gpu, sum_cpu, "sum mismatch T={} E={} K={}", t, e, k);}}}@@ -197,33 +169,15 @@ fn test_counts_offsets_fused_edge_cases() {let num_tiles = ((e + 511) / 512).max(1);let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);- let kernel =- MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");- kernel.encode(- &topk_ids_buf,- &offsets_buf,- &sum_k_buf,- &partials_buf,- t as u32,- e as u32,- k as u32,- &encoder,- );+ kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);encoder.end_encoding();cb.commit();cb.wait_until_completed();- let offsets_gpu = unsafe {- std::slice::from_raw_parts(- offsets_buf.contents().as_ptr() as *const u32,- e + 1,- )- };+ let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };let mut expected_offsets = vec![0u32; e + 1];expected_offsets[4..].fill((t * k) as u32);@@ -238,32 +192,14 @@ fn test_counts_offsets_fused_edge_cases() {let num_tiles = ((e + 511) / 512).max(1);let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);- let kernel =- MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");- kernel.encode(- &topk_ids_buf,- &offsets_buf,- &sum_k_buf,- &partials_buf,- t as u32,- e as u32,- k as u32,- &encoder,- );+ kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);encoder.end_encoding();cb.commit();cb.wait_until_completed();- let offsets_gpu = unsafe {- std::slice::from_raw_parts(- offsets_buf.contents().as_ptr() as *const u32,- e + 1,- )- };+ let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };assert!(offsets_gpu.iter().all(|&v| v == 0));}diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex 46937eb8c..cc5df1800 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -6,13 +6,11 @@ use uzu::backends::{metal::{MTLContext,kernel::{- KernelDataType, MoeBlockBasesArguments, MoeScatterKernels,- MoeScatterWithMapArguments,+ KernelDataType, MoeBlockBasesArguments, MoeScatterKernels, MoeScatterWithMapArguments,dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},moe::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel,- MoeGatherArguments, MoeGatherKernels, MoeRouterTopKArguments,- MoeRouterTopKKernel,+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,+ MoeRouterTopKArguments, MoeRouterTopKKernel,},},},@@ -46,10 +44,10 @@ fn moe_cpu_reference(x: &[bf16],router_weight: &[f32], // [E, d_model] - kept as F32 for router (computed before BF16 conversion)router_bias: &[f32], // [E]- w13: &[bf16], // source layout [E, d_model, 2*d_ff] (GPU transposes to [E, 2*d_ff, d_model])- w2: &[bf16], // [E, d_ff, d_model]- up_biases: &[bf16], // [E, 2*d_ff]- down_biases: &[bf16], // [E, d_model]+ w13: &[bf16], // source layout [E, d_model, 2*d_ff] (GPU transposes to [E, 2*d_ff, d_model])+ w2: &[bf16], // [E, d_ff, d_model]+ up_biases: &[bf16], // [E, 2*d_ff]+ down_biases: &[bf16], // [E, d_model]t: usize,e: usize,k: usize,@@ -78,23 +76,15 @@ fn moe_cpu_reference(}// TopK selection- let mut indices_and_logits: Vec<(usize, f32)> =- token_logits.iter().enumerate().map(|(i, &v)| (i, v)).collect();+ let mut indices_and_logits: Vec<(usize, f32)> = token_logits.iter().enumerate().map(|(i, &v)| (i, v)).collect();indices_and_logits.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());- let top_k_indices: Vec<usize> =- indices_and_logits[..k].iter().map(|&(i, _)| i).collect();- let top_k_logits: Vec<f32> =- indices_and_logits[..k].iter().map(|&(_, v)| v).collect();+ let top_k_indices: Vec<usize> = indices_and_logits[..k].iter().map(|&(i, _)| i).collect();+ let top_k_logits: Vec<f32> = indices_and_logits[..k].iter().map(|&(_, v)| v).collect();// Softmax normalization (matching Python: jax.nn.softmax)- let max_logit =- top_k_logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);- let exp_sum: f32 =- top_k_logits.iter().map(|&v| (v - max_logit).exp()).sum();- let weights: Vec<f32> = top_k_logits- .iter()- .map(|&v| (v - max_logit).exp() / exp_sum)- .collect();+ let max_logit = top_k_logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);+ let exp_sum: f32 = top_k_logits.iter().map(|&v| (v - max_logit).exp()).sum();+ let weights: Vec<f32> = top_k_logits.iter().map(|&v| (v - max_logit).exp() / exp_sum).collect();// Process each selected expert (matching Python: vmap(run_expert))for (expert_idx, &weight) in top_k_indices.iter().zip(weights.iter()) {@@ -113,8 +103,7 @@ fn moe_cpu_reference(for ff_idx in 0..d_ff {let mut up_sum = f32::from(up_biases[bias_up_offset + ff_idx]);- let mut gate_sum =- f32::from(up_biases[bias_up_offset + d_ff + ff_idx]);+ let mut gate_sum = f32::from(up_biases[bias_up_offset + d_ff + ff_idx]);for input_idx in 0..d_model {let w_base = w13_offset + input_idx * 2 * d_ff;@@ -154,15 +143,13 @@ fn moe_cpu_reference(for out_idx in 0..d_model {let mut fc2_sum = 0.0f32;for ff_idx in 0..d_ff {- let w2_val =- f32::from(w2[w2_offset + ff_idx * d_model + out_idx]);+ let w2_val = f32::from(w2[w2_offset + ff_idx * d_model + out_idx]);fc2_sum += hidden[ff_idx] * w2_val;}// Two-pass: keep in F32 (no intermediate quantization)// Add down bias in F32- let down_bias_val =- f32::from(down_biases[bias_down_offset + out_idx]);+ let down_bias_val = f32::from(down_biases[bias_down_offset + out_idx]);let with_bias = fc2_sum + down_bias_val;// Only quantize to BF16 once at the end (matches two-pass final write)@@ -266,43 +253,26 @@ fn run_moe_parity_test_internal(eprintln!("\n[{}] T={}, E={}, K={}, d_model={}, d_ff={}, alpha={}, gate_clip={:?}, up_clip={:?}, mode={}",- test_name,- t,- e,- k,- d_model,- d_ff,- silu_alpha,- gate_clip,- up_clip,- prefill_mode+ test_name, t, e, k, d_model, d_ff, silu_alpha, gate_clip, up_clip, prefill_mode);// Random BF16 inputs- let x: Vec<bf16> = (0..t * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();// Generate random router weights and biases for CPU reference- let router_weight_f32: Vec<f32> =- (0..e * d_model).map(|_| rng.random_range(-0.5..0.5)).collect();- let router_bias_f32: Vec<f32> =- (0..e).map(|_| rng.random_range(-0.1..0.1)).collect();+ let router_weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-0.5..0.5)).collect();+ let router_bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.1..0.1)).collect();// Convert router weights/bias to BF16 for GPU- let router_weight_bf16: Vec<bf16> =- router_weight_f32.iter().map(|&v| bf16::from_f32(v)).collect();- let router_bias_bf16: Vec<bf16> =- router_bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();+ let router_weight_bf16: Vec<bf16> = router_weight_f32.iter().map(|&v| bf16::from_f32(v)).collect();+ let router_bias_bf16: Vec<bf16> = router_bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();// Generate random expert weights and biaseslet w13_len = e * d_model * 2 * d_ff;let w2_len = e * d_ff * d_model;// Generate W13 in original layout [E, d_model, 2*d_ff] for CPU reference- let w13_cpu: Vec<bf16> = (0..w13_len)- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))- .collect();+ let w13_cpu: Vec<bf16> = (0..w13_len).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();// Transpose to GPU layout [E, 2*d_ff, d_model]let mut w13_gpu = vec![bf16::from_f32(0.0); w13_len];@@ -319,9 +289,7 @@ fn run_moe_parity_test_internal(}// Generate W2 in original layout [E, d_ff, d_model] for CPU reference- let w2_cpu: Vec<bf16> = (0..w2_len)- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))- .collect();+ let w2_cpu: Vec<bf16> = (0..w2_len).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();// Transpose W2 to GPU layout [E, d_model, d_ff]let mut w2_gpu = vec![bf16::from_f32(0.0); w2_len];@@ -331,17 +299,12 @@ fn run_moe_parity_test_internal(for dm in 0..d_model {// src: [E, d_ff, d_model] -> index: expert_offset + ff * d_model + dm// dst: [E, d_model, d_ff] -> index: expert_offset + dm * d_ff + ff- w2_gpu[expert_offset + dm * d_ff + ff] =- w2_cpu[expert_offset + ff * d_model + dm];+ w2_gpu[expert_offset + dm * d_ff + ff] = w2_cpu[expert_offset + ff * d_model + dm];}}}- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))- .collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();// Create Metal bufferslet x_buf = alloc_buffer_with_data(&ctx, &x);@@ -381,10 +344,7 @@ fn run_moe_parity_test_internal(// Encode ALL kernels in one command buffereprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");// Router + TopK (fused kernel)let router_topk = MoeRouterTopKKernel::new(&ctx).expect("router+topk");@@ -407,19 +367,9 @@ fn run_moe_parity_test_internal().expect("router+topk encode");- let fused_kernel =- MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let fused_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");let encoder = cb.new_compute_command_encoder().expect("encoder");- fused_kernel.encode(- &topk_ids_buf,- &offsets_buf,- &sumk_buf,- &partials_buf,- t as u32,- e as u32,- k as u32,- &encoder,- );+ fused_kernel.encode(&topk_ids_buf, &offsets_buf, &sumk_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);encoder.end_encoding();let scatter = MoeScatterKernels::new(&ctx).expect("scatter");@@ -517,9 +467,7 @@ fn run_moe_parity_test_internal().expect("experts encode");- let finalize =- MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into())- .expect("finalize");+ let finalize = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize");let encoder = cb.new_compute_command_encoder().expect("encoder");finalize.encode(&tok2row_buf,@@ -539,14 +487,8 @@ fn run_moe_parity_test_internal(eprintln!("[E2E] GPU execution completed");// Read GPU output- let y_out_bf16 = unsafe {- std::slice::from_raw_parts(- y_out_buf.contents().as_ptr() as *const bf16,- t * d_model,- )- };- let y_out_gpu: Vec<f32> =- y_out_bf16.iter().map(|&v| f32::from(v)).collect();+ let y_out_bf16 = unsafe { std::slice::from_raw_parts(y_out_buf.contents().as_ptr() as *const bf16, t * d_model) };+ let y_out_gpu: Vec<f32> = y_out_bf16.iter().map(|&v| f32::from(v)).collect();// Validate GPU output is finitelet nan_count = y_out_gpu.iter().filter(|v| v.is_nan()).count();@@ -570,85 +512,46 @@ fn run_moe_parity_test_internal();// Probe tile bookkeeping- let total_tiles_cpu = unsafe {- std::slice::from_raw_parts(- total_tiles_buf.contents().as_ptr() as *const u32,- 2,- )- };- let dispatch_args_cpu = unsafe {- std::slice::from_raw_parts(- dispatch_args_buf.contents().as_ptr() as *const u32,- 3,- )- };- let tile_offsets_cpu = unsafe {- std::slice::from_raw_parts(- tile_offsets_buf.contents().as_ptr() as *const u32,- (e + 1).min(8),- )- };- let sumk_val =- unsafe { *(sumk_buf.contents().as_ptr() as *const u32) } as usize;+ let total_tiles_cpu =+ unsafe { std::slice::from_raw_parts(total_tiles_buf.contents().as_ptr() as *const u32, 2) };+ let dispatch_args_cpu =+ unsafe { std::slice::from_raw_parts(dispatch_args_buf.contents().as_ptr() as *const u32, 3) };+ let tile_offsets_cpu =+ unsafe { std::slice::from_raw_parts(tile_offsets_buf.contents().as_ptr() as *const u32, (e + 1).min(8)) };+ let sumk_val = unsafe { *(sumk_buf.contents().as_ptr() as *const u32) } as usize;eprintln!("[E2E] Tile bookkeeping:");eprintln!("[E2E] total_tiles={}, dispatch_args=({}, {}, {})",- total_tiles_cpu[0],- dispatch_args_cpu[0],- dispatch_args_cpu[1],- dispatch_args_cpu[2]- );- eprintln!(- "[E2E] tile_offsets[0..{}]={:?}",- (e + 1).min(8),- &tile_offsets_cpu+ total_tiles_cpu[0], dispatch_args_cpu[0], dispatch_args_cpu[1], dispatch_args_cpu[2]);+ eprintln!("[E2E] tile_offsets[0..{}]={:?}", (e + 1).min(8), &tile_offsets_cpu);eprintln!("[E2E] sumk={}, num_tiles_k={}", sumk_val, num_tiles_k);// For multi-token tests with large d_ff, verify gather output (x_perm)if t > 1 && d_ff >= 256 {let x_perm_cpu = unsafe {- std::slice::from_raw_parts(- x_perm_buf.contents().as_ptr() as *const bf16,- sumk_val * d_model,- )+ std::slice::from_raw_parts(x_perm_buf.contents().as_ptr() as *const bf16, sumk_val * d_model)};eprintln!("[E2E] x_perm diagnostics (sumk={}):", sumk_val);- eprintln!(- "[E2E] Row 0 [0:8]: {:?}",- &x_perm_cpu[0..8]- .iter()- .map(|&v| f32::from(v))- .collect::<Vec<_>>()- );+ eprintln!("[E2E] Row 0 [0:8]: {:?}", &x_perm_cpu[0..8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());if sumk_val > 1 {eprintln!("[E2E] Row 1 [{}:{}]: {:?}",d_model,d_model + 8,- &x_perm_cpu[d_model..d_model + 8]- .iter()- .map(|&v| f32::from(v))- .collect::<Vec<_>>()+ &x_perm_cpu[d_model..d_model + 8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());}// Check tile_map for first few tileslet tile_map_cpu = unsafe {- std::slice::from_raw_parts(- tile_map_buf.contents().as_ptr() as *const u32,- 12.min(max_tiles * 3),- )+ std::slice::from_raw_parts(tile_map_buf.contents().as_ptr() as *const u32, 12.min(max_tiles * 3))};eprintln!("[E2E] tile_map (first 4 tiles): {:?}", &tile_map_cpu);// CRITICAL: Check tok2row mapping for multi-token tests- let tok2row_cpu = unsafe {- std::slice::from_raw_parts(- tok2row_buf.contents().as_ptr() as *const i32,- t * k,- )- };+ let tok2row_cpu =+ unsafe { std::slice::from_raw_parts(tok2row_buf.contents().as_ptr() as *const i32, t * k) };eprintln!("[E2E] tok2row[0..{}]: {:?}", t * k, &tok2row_cpu);eprintln!("[E2E] Expected: token 0βrow {}, token 1βrow {}",@@ -662,32 +565,17 @@ fn run_moe_parity_test_internal(// Check y_partial at specific indices where finalize reads for token 1let y_partial_full = unsafe {- std::slice::from_raw_parts(- y_partial_buf.contents().as_ptr() as *const bf16,- sumk_val * d_model,- )+ std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sumk_val * d_model)};eprintln!("[E2E] y_partial spot check:");- eprintln!(- "[E2E] y_partial[48] (row 0, col 48) = {:.6}",- f32::from(y_partial_full[48])- );+ eprintln!("[E2E] y_partial[48] (row 0, col 48) = {:.6}", f32::from(y_partial_full[48]));if sumk_val > 1 {// Check positions where mismatches occur (odd indices near tile boundaries)- eprintln!(- "[E2E] Row 1 positions (even=working, odd=corrupted?):"- );- for &pos in &[- 48, 56, 57, 58, 59, 60, 61, 62, 63, 64, 120, 121, 122, 123,- 124, 125, 126, 127, 128,- ] {+ eprintln!("[E2E] Row 1 positions (even=working, odd=corrupted?):");+ for &pos in &[48, 56, 57, 58, 59, 60, 61, 62, 63, 64, 120, 121, 122, 123, 124, 125, 126, 127, 128] {let idx = 1 * d_model + pos;if idx < y_partial_full.len() {- eprintln!(- "[E2E] y_partial[row1, col {}] = {:.3}",- pos,- f32::from(y_partial_full[idx])- );+ eprintln!("[E2E] y_partial[row1, col {}] = {:.3}", pos, f32::from(y_partial_full[idx]));}}}@@ -738,12 +626,11 @@ fn run_moe_parity_test_internal(let threshold_abs = 1e-3;if rel_diff > threshold_rel && abs_diff > threshold_abs {- let print_limit =- if test_name.contains("BoundarySweep_D1024_FF256_T2") {- usize::MAX // Print ALL for debugging- } else {- 10- };+ let print_limit = if test_name.contains("BoundarySweep_D1024_FF256_T2") {+ usize::MAX // Print ALL for debugging+ } else {+ 10+ };if mismatches < print_limit {eprintln!("[{}] Mismatch idx {}: GPU={:.6}, CPU={:.6}, abs={:.6}, rel={:.6}",@@ -763,41 +650,21 @@ fn run_moe_parity_test_internal(// Debug: print some sample values from intermediate bufferseprintln!("[E2E] === DEBUG: Intermediate values ===");- let topk_ids_gpu = unsafe {- std::slice::from_raw_parts(- topk_ids_buf.contents().as_ptr() as *const i32,- t * k,- )- };- let topk_probs_gpu = unsafe {- std::slice::from_raw_parts(- topk_probs_buf.contents().as_ptr() as *const bf16,- t * k,- )- };+ let topk_ids_gpu = unsafe { std::slice::from_raw_parts(topk_ids_buf.contents().as_ptr() as *const i32, t * k) };+ let topk_probs_gpu =+ unsafe { std::slice::from_raw_parts(topk_probs_buf.contents().as_ptr() as *const bf16, t * k) };eprintln!("[E2E] TopK IDs: {:?}", topk_ids_gpu);- eprintln!(- "[E2E] TopK Probs: {:?}",- topk_probs_gpu.iter().map(|&v| f32::from(v)).collect::<Vec<_>>()- );+ eprintln!("[E2E] TopK Probs: {:?}", topk_probs_gpu.iter().map(|&v| f32::from(v)).collect::<Vec<_>>());- let y_partial_gpu = unsafe {- std::slice::from_raw_parts(- y_partial_buf.contents().as_ptr() as *const bf16,- max_sumk * d_model,- )- };- let sumk_actual =- unsafe { *(sumk_buf.contents().as_ptr() as *const u32) } as usize;+ let y_partial_gpu =+ unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, max_sumk * d_model) };+ let sumk_actual = unsafe { *(sumk_buf.contents().as_ptr() as *const u32) } as usize;eprintln!("[E2E] sumk={}", sumk_actual);let sample_size = 16.min(sumk_actual * d_model);eprintln!("[E2E] y_partial sample (first {}): {:?}",sample_size,- &y_partial_gpu[..sample_size]- .iter()- .map(|&v| f32::from(v))- .collect::<Vec<_>>()+ &y_partial_gpu[..sample_size].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());// For multi-row debugging: print both rows@@ -806,52 +673,29 @@ fn run_moe_parity_test_internal("[E2E] y_partial row 1 [{}-{}]: {:?}",d_model,d_model + 16,- &y_partial_gpu[d_model..d_model + 16]- .iter()- .map(|&v| f32::from(v))- .collect::<Vec<_>>()+ &y_partial_gpu[d_model..d_model + 16].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());}let out_sample_size = 16.min(t * d_model);- eprintln!(- "[E2E] y_out sample (token 0): {:?}",- &y_out_gpu[..out_sample_size]- );+ eprintln!("[E2E] y_out sample (token 0): {:?}", &y_out_gpu[..out_sample_size]);if t > 1 && d_model >= 512 {eprintln!("[E2E] y_out sample (token 1, first 32): {:?}",- &y_out_gpu[d_model..d_model + 32]- .iter()- .map(|&v| f32::from(v))- .collect::<Vec<_>>()+ &y_out_gpu[d_model..d_model + 32].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());}- eprintln!(- "[E2E] CPU ref sample (token 0): {:?}",- &y_cpu[..out_sample_size]- );+ eprintln!("[E2E] CPU ref sample (token 0): {:?}", &y_cpu[..out_sample_size]);if t > 1 && d_model >= 512 {- eprintln!(- "[E2E] CPU ref sample (token 1): {:?}",- &y_cpu[d_model..d_model + 16]- );+ eprintln!("[E2E] CPU ref sample (token 1): {:?}", &y_cpu[d_model..d_model + 16]);// Compare token 1 outputs element-wise (more positions to find pattern)let gpu_t1 = &y_out_gpu[d_model..];let cpu_t1 = &y_cpu[d_model..];eprintln!("[E2E] Token 1 detailed comparison:");- for i in- [0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 17, 18, 48, 64, 121, 128].iter()- {+ for i in [0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 17, 18, 48, 64, 121, 128].iter() {if *i < d_model {let diff = (f32::from(gpu_t1[*i]) - cpu_t1[*i]).abs();- eprintln!(- "[E2E] [{}]: GPU={:.6}, CPU={:.6}, diff={:.6}",- i,- f32::from(gpu_t1[*i]),- cpu_t1[*i],- diff- );+ eprintln!("[E2E] [{}]: GPU={:.6}, CPU={:.6}, diff={:.6}", i, f32::from(gpu_t1[*i]), cpu_t1[*i], diff);}}}@@ -865,10 +709,7 @@ fn run_moe_parity_test_internal(let mean_abs_threshold = 0.1f32.max(multi_expert_drift);if mean_abs_error >= mean_abs_threshold {- panic!(- "[{}] Mean absolute error {:.6} exceeds threshold {:.6}",- test_name, mean_abs_error, mean_abs_threshold- );+ panic!("[{}] Mean absolute error {:.6} exceeds threshold {:.6}", test_name, mean_abs_error, mean_abs_threshold);}// Warn on high max_rel but don't fail (can be inflated by near-zero values)diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rsindex f154d8195..1a684516f 100644--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs@@ -6,9 +6,8 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::backends::metal::{KernelDataType, MTLContext,kernel::moe::{- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels,- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,- MoeExpertsTwoPassPrefillKernel,+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,+ MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel,},};@@ -48,20 +47,14 @@ fn run_decode_case(let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);let sum_k = t * k;- eprintln!(- "\n[decode/two-pass] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}",- name, t, d_model, d_ff, e, k, sum_k- );+ eprintln!("\n[decode/two-pass] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}", name, t, d_model, d_ff, e, k, sum_k);- let x_perm: Vec<bf16> = (0..sum_k * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();let offsets = build_offsets(e, sum_k);// Generate W13 in original layout [E, d_model, 2*d_ff]- let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();+ let w13_original: Vec<bf16> =+ (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();// Transpose to GPU layout [E, 2*d_ff, d_model]let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];@@ -76,18 +69,11 @@ fn run_decode_case(}}}- let w2: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();-- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx)- .expect("experts decode kernel");+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();++ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx).expect("experts decode kernel");let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);@@ -143,10 +129,7 @@ fn run_decode_case(};for _ in 0..warmup {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb, &make_two_pass_args());cb.commit();cb.wait_until_completed();@@ -155,10 +138,7 @@ fn run_decode_case(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb, &make_two_pass_args());cb.commit();cb.wait_until_completed();@@ -170,14 +150,10 @@ fn run_decode_case(let median = times[times.len() / 2];let min = times[0];let max = times[times.len() - 1];- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()- / times.len() as f64;+ let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;let std = var.sqrt();- eprintln!(- " mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms",- mean, median, min, max, std- );+ eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);eprintln!(" β Latency: {:.1} Β΅s/token", (mean / t as f64) * 1000.0);}@@ -186,15 +162,11 @@ fn run_decode_case(fn test_two_pass_decode_speed() {let ctx = create_ctx();- let cases = vec![- ("T1_E16_D1024_F4096", 1, 1024, 4096, 16, 4, 3, 20),- ("T1_E8_D2048_F8192", 1, 2048, 8192, 8, 4, 3, 20),- ];+ let cases =+ vec![("T1_E16_D1024_F4096", 1, 1024, 4096, 16, 4, 3, 20), ("T1_E8_D2048_F8192", 1, 2048, 8192, 8, 4, 3, 20)];for (name, t, d_model, d_ff, e, k, warmup, iters) in &cases {- run_decode_case(- &ctx, name, *t, *d_model, *d_ff, *e, *k, *warmup, *iters,- );+ run_decode_case(&ctx, name, *t, *d_model, *d_ff, *e, *k, *warmup, *iters);}}@@ -214,19 +186,13 @@ fn run_two_pass_prefill_case(let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);let sum_k = t * k;- eprintln!(- "\n[2-pass prefill] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}",- name, t, d_model, d_ff, e, k, sum_k- );+ eprintln!("\n[2-pass prefill] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}", name, t, d_model, d_ff, e, k, sum_k);- let x_perm: Vec<bf16> = (0..sum_k * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();let offsets = build_offsets(e, sum_k);- let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();+ let w13_original: Vec<bf16> =+ (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];for expert in 0..e {@@ -240,18 +206,11 @@ fn run_two_pass_prefill_case(}}}- let w2: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();-- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(ctx)- .expect("experts prefill kernel");+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();++ let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(ctx).expect("experts prefill kernel");let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);@@ -303,13 +262,8 @@ fn run_two_pass_prefill_case(};for _ in 0..warmup {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- experts_kernel- .encode(&cb, make_two_pass_args())- .expect("two-pass prefill encode");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ experts_kernel.encode(&cb, make_two_pass_args()).expect("two-pass prefill encode");cb.commit();cb.wait_until_completed();}@@ -317,13 +271,8 @@ fn run_two_pass_prefill_case(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- experts_kernel- .encode(&cb, make_two_pass_args())- .expect("two-pass prefill encode");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ experts_kernel.encode(&cb, make_two_pass_args()).expect("two-pass prefill encode");cb.commit();cb.wait_until_completed();times.push(start.elapsed().as_secs_f64() * 1000.0);@@ -334,18 +283,11 @@ fn run_two_pass_prefill_case(let median = times[times.len() / 2];let min = times[0];let max = times[times.len() - 1];- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()- / times.len() as f64;+ let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;let std = var.sqrt();- eprintln!(- " mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms",- mean, median, min, max, std- );- eprintln!(- " β Throughput: {:.1} Β΅s/token (mean / sum_k)",- (mean / sum_k as f64) * 1000.0- );+ eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);+ eprintln!(" β Throughput: {:.1} Β΅s/token (mean / sum_k)", (mean / sum_k as f64) * 1000.0);}#[test]@@ -360,9 +302,7 @@ fn test_two_pass_prefill_speed() {];for (name, t, d_model, d_ff, e, k, warmup, iters) in cases {- run_two_pass_prefill_case(- &ctx, name, t, d_model, d_ff, e, k, warmup, iters,- );+ run_two_pass_prefill_case(&ctx, name, t, d_model, d_ff, e, k, warmup, iters);}}@@ -378,39 +318,25 @@ fn run_fused_single_token_case() {let mut rng = StdRng::seed_from_u64(0xF053ED);- eprintln!(- "\n[fused single-token] {} => D={}, FF={}, E={}, K={}",- name, d_model, d_ff, e, k- );+ eprintln!("\n[fused single-token] {} => D={}, FF={}, E={}, K={}", name, d_model, d_ff, e, k);- let x: Vec<bf16> = (0..d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();let topk_probs: Vec<bf16> = {- let raw: Vec<f32> =- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();let sum: f32 = raw.iter().sum();raw.iter().map(|p| bf16::from_f32(p / sum)).collect()};- let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let w2_all: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();-- let fused_kernel =- MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");+ let w13_all: Vec<bf16> =+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();++ let fused_kernel = MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");let x_buf = alloc_buffer_with_data(ctx, &x);let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);@@ -445,10 +371,7 @@ fn run_fused_single_token_case(};for _ in 0..warmup {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(&cb, make_args());cb.commit();cb.wait_until_completed();@@ -457,10 +380,7 @@ fn run_fused_single_token_case(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(&cb, make_args());cb.commit();cb.wait_until_completed();@@ -472,14 +392,10 @@ fn run_fused_single_token_case(let median = times[times.len() / 2];let min = times[0];let max = times[times.len() - 1];- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()- / times.len() as f64;+ let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;let std = var.sqrt();- eprintln!(- " mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms",- mean, median, min, max, std- );+ eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);eprintln!(" β Latency: {:.1} Β΅s/token", mean * 1000.0);}@@ -495,9 +411,7 @@ fn test_fused_single_token_speed() {];for (name, d_model, d_ff, e, k, warmup, iters) in cases {- run_fused_single_token_case(- &ctx, name, d_model, d_ff, e, k, warmup, iters,- );+ run_fused_single_token_case(&ctx, name, d_model, d_ff, e, k, warmup, iters);}}@@ -522,19 +436,13 @@ fn test_single_token_indirect_vs_fused() {eprintln!("[{}] D={}, FF={}, E={}, K={}", name, d_model, d_ff, e, k);// Run indirect decode (T=1)- let indirect_mean = run_indirect_decode_timed(- &ctx, 1, d_model, d_ff, e, k, warmup, iters,- );+ let indirect_mean = run_indirect_decode_timed(&ctx, 1, d_model, d_ff, e, k, warmup, iters);// Run fused decode- let fused_mean =- run_fused_decode_timed(&ctx, d_model, d_ff, e, k, warmup, iters);+ let fused_mean = run_fused_decode_timed(&ctx, d_model, d_ff, e, k, warmup, iters);let speedup = indirect_mean / fused_mean;- eprintln!(- " Indirect: {:.3}ms, Fused: {:.3}ms, Speedup: {:.2}x\n",- indirect_mean, fused_mean, speedup- );+ eprintln!(" Indirect: {:.3}ms, Fused: {:.3}ms, Speedup: {:.2}x\n", indirect_mean, fused_mean, speedup);}}@@ -553,14 +461,11 @@ fn run_indirect_decode_timed(let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);let sum_k = t * k;- let x_perm: Vec<bf16> = (0..sum_k * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();let offsets = build_offsets(e, sum_k);- let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();+ let w13_original: Vec<bf16> =+ (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];for expert in 0..e {@@ -574,18 +479,11 @@ fn run_indirect_decode_timed(}}}- let w2: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();-- let experts_kernel =- MoeExpertsTwoPassDecodeKernels::new(ctx).expect("decode kernel");+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();++ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx).expect("decode kernel");let x_perm_buf = alloc_buffer_with_data(ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(ctx, &offsets);@@ -636,10 +534,7 @@ fn run_indirect_decode_timed(};for _ in 0..warmup {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb, &make_args());cb.commit();cb.wait_until_completed();@@ -648,10 +543,7 @@ fn run_indirect_decode_timed(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb, &make_args());cb.commit();cb.wait_until_completed();@@ -672,34 +564,23 @@ fn run_fused_decode_timed() -> f64 {let mut rng = StdRng::seed_from_u64(0xF053ED);- let x: Vec<bf16> = (0..d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();let topk_probs: Vec<bf16> = {- let raw: Vec<f32> =- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();let sum: f32 = raw.iter().sum();raw.iter().map(|p| bf16::from_f32(p / sum)).collect()};- let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let w2_all: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();-- let fused_kernel =- MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");+ let w13_all: Vec<bf16> =+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();++ let fused_kernel = MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");let x_buf = alloc_buffer_with_data(ctx, &x);let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);@@ -734,10 +615,7 @@ fn run_fused_decode_timed(};for _ in 0..warmup {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(&cb, make_args());cb.commit();cb.wait_until_completed();@@ -746,10 +624,7 @@ fn run_fused_decode_timed(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(&cb, make_args());cb.commit();cb.wait_until_completed();diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rsindex 3461f7c23..a7a0e3892 100644--- a/crates/uzu/tests/moe/moe_experts_test.rs+++ b/crates/uzu/tests/moe/moe_experts_test.rs@@ -14,17 +14,14 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::backends::metal::{KernelDataType,kernel::moe::{- MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,- MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel,+ MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,+ MoeExpertsTwoPassPrefillKernel,},};#[path = "moe_test_utils.rs"]mod test_utils;-use test_utils::{- alloc_buffer, alloc_buffer_with_data, assert_bf16_close, cpu_tile_counts,- cpu_tile_scan, create_ctx,-};+use test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, cpu_tile_counts, cpu_tile_scan, create_ctx};use uzu::backends::metal::kernel::moe::MoeExpertsSingleDecodeArguments;/// Test data for MoE experts@@ -47,35 +44,23 @@ impl MoeTestData {d_ff: usize,e: usize,) -> Self {- let x: Vec<bf16> = (0..t * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let topk_ids: Vec<i32> = (0..t * k)- .map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32)- .collect();+ let topk_ids: Vec<i32> = (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();let topk_probs: Vec<bf16> = (0..t).flat_map(|_| {- let raw: Vec<f32> =- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();let sum: f32 = raw.iter().sum();raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()}).collect();- let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let w2: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();+ let w13: Vec<bf16> =+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();Self {x,@@ -262,13 +247,10 @@ fn cpu_moe_reference(}} else {// SwiGLU or GEGLU - need gate projection too- let mut acc_gate =- f32::from(up_biases[bias_base + d_ff + h]);+ let mut acc_gate = f32::from(up_biases[bias_base + d_ff + h]);for d in 0..d_model {let x_val = f32::from(x[x_offset + d]);- let w_val = f32::from(- w13_all[w13_base + (d_ff + h) * d_model + d],- );+ let w_val = f32::from(w13_all[w13_base + (d_ff + h) * d_model + d]);acc_gate += x_val * w_val;}@@ -334,23 +316,16 @@ fn test_two_pass_decode_correctness() {let gating_code = 2u32; // SwiGLUlet silu_alpha = 1.0f32;- eprintln!(- "[2-pass decode] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}",- t, k, sum_k, d_model, d_ff, e- );+ eprintln!("[2-pass decode] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}", t, k, sum_k, d_model, d_ff, e);// Generate input x [T, d_model]- let x: Vec<bf16> = (0..t * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();// Generate routing: topk_ids [T*K], topk_probs [T*K]- let topk_ids: Vec<i32> =- (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();+ let topk_ids: Vec<i32> = (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();let topk_probs: Vec<bf16> = (0..t).flat_map(|_| {- let raw: Vec<f32> =- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();let sum: f32 = raw.iter().sum();raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()})@@ -388,18 +363,10 @@ fn test_two_pass_decode_correctness() {}// Generate weights- let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let w2: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();+ let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();// CPU reference - end-to-endlet y_expected = cpu_moe_reference(@@ -445,12 +412,8 @@ fn test_two_pass_decode_correctness() {let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);// Execute 2-pass decode kernel- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx)- .expect("MoeExpertsTwoPassDecodeKernel::new");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");const K_TILE: usize = 64;let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;@@ -491,12 +454,8 @@ fn test_two_pass_decode_correctness() {cb.wait_until_completed();// Read GPU partial output and do CPU finalize (weighted sum)- let y_partial_gpu = unsafe {- std::slice::from_raw_parts(- y_partial_buf.contents().as_ptr() as *const bf16,- sum_k * d_model,- )- };+ let y_partial_gpu =+ unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };// Finalize: y[t] = Ξ£_k prob[t,k] * y_partial[t*k + k_idx]let mut y_gpu = vec![bf16::from_f32(0.0); t * d_model];@@ -517,9 +476,7 @@ fn test_two_pass_decode_correctness() {let mut max_abs_error = 0.0f32;let mut max_idx = 0;- for (i, (&gpu_val, &cpu_val)) in- y_gpu.iter().zip(y_expected.iter()).enumerate()- {+ for (i, (&gpu_val, &cpu_val)) in y_gpu.iter().zip(y_expected.iter()).enumerate() {let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();if abs_error > max_abs_error {max_abs_error = abs_error;@@ -555,10 +512,7 @@ fn test_two_pass_decode_multi_token() {let gating_code = 2u32;let silu_alpha = 1.0f32;- eprintln!(- "[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}",- t, k, d_model, d_ff, e- );+ eprintln!("[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);@@ -586,8 +540,7 @@ fn test_two_pass_decode_multi_token() {// GPU bufferslet x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);- let row_expert_map_buf =- alloc_buffer_with_data(&ctx, &scatter.row_expert_map);+ let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);@@ -603,12 +556,8 @@ fn test_two_pass_decode_multi_token() {let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- let experts_kernel =- MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb,&MoeExpertsTwoPassArguments {@@ -643,20 +592,9 @@ fn test_two_pass_decode_multi_token() {cb.commit();cb.wait_until_completed();- let y_partial_gpu = unsafe {- std::slice::from_raw_parts(- y_partial_buf.contents().as_ptr() as *const bf16,- sum_k * d_model,- )- };- let y_gpu = gather_and_finalize(- y_partial_gpu,- &data.topk_probs,- &scatter.perm_idx,- t,- k,- d_model,- );+ let y_partial_gpu =+ unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };+ let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass decode multi-token");eprintln!("[2-pass decode multi-token] β PASSED");@@ -676,10 +614,7 @@ fn test_two_pass_prefill_correctness() {let gating_code = 2u32;let silu_alpha = 1.0f32;- eprintln!(- "[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}",- t, k, d_model, d_ff, e- );+ eprintln!("[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);@@ -707,8 +642,7 @@ fn test_two_pass_prefill_correctness() {// GPU bufferslet x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);- let row_expert_map_buf =- alloc_buffer_with_data(&ctx, &scatter.row_expert_map);+ let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);@@ -724,12 +658,8 @@ fn test_two_pass_prefill_correctness() {let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- let experts_kernel =- MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb,@@ -766,20 +696,9 @@ fn test_two_pass_prefill_correctness() {cb.commit();cb.wait_until_completed();- let y_partial_gpu = unsafe {- std::slice::from_raw_parts(- y_partial_buf.contents().as_ptr() as *const bf16,- sum_k * d_model,- )- };- let y_gpu = gather_and_finalize(- y_partial_gpu,- &data.topk_probs,- &scatter.perm_idx,- t,- k,- d_model,- );+ let y_partial_gpu =+ unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };+ let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass prefill");eprintln!("[2-pass prefill] β PASSED");@@ -818,37 +737,24 @@ fn test_fused_single_token_decode() {let e = 8;let k = 2;- eprintln!(- "[fused single-token] d_model={}, d_ff={}, E={}, K={}",- d_model, d_ff, e, k- );+ eprintln!("[fused single-token] d_model={}, d_ff={}, E={}, K={}", d_model, d_ff, e, k);// Generate test data- let x: Vec<bf16> = (0..d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();let topk_probs: Vec<bf16> = {- let raw: Vec<f32> =- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();let sum: f32 = raw.iter().sum();raw.iter().map(|p| bf16::from_f32(p / sum)).collect()};- let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let w2_all: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();+ let w13_all: Vec<bf16> =+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();// CPU reference (gating_code=2 for SwiGLU)let gating_code = 2u32;@@ -885,12 +791,8 @@ fn test_fused_single_token_decode() {let y_buf = alloc_buffer::<bf16>(&ctx, d_model);// Run fused decode kernel- let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx)- .expect("MoeExpertsSingleDecodeKernel::new");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(&cb,@@ -921,21 +823,14 @@ fn test_fused_single_token_decode() {cb.wait_until_completed();// Read GPU output- let y_gpu = unsafe {- std::slice::from_raw_parts(- y_buf.contents().as_ptr() as *const bf16,- d_model,- )- };+ let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };// Compute error metricslet mut max_abs_error = 0.0f32;let mut sum_abs_error = 0.0f32;let mut max_idx = 0;- for (i, (&gpu_val, &cpu_val)) in- y_gpu.iter().zip(y_expected.iter()).enumerate()- {+ for (i, (&gpu_val, &cpu_val)) in y_gpu.iter().zip(y_expected.iter()).enumerate() {let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();sum_abs_error += abs_error;if abs_error > max_abs_error {@@ -955,12 +850,7 @@ fn test_fused_single_token_decode() {);let tolerance = 0.02;- assert_bf16_close(- y_gpu,- &y_expected,- tolerance,- "fused single-token output",- );+ assert_bf16_close(y_gpu, &y_expected, tolerance, "fused single-token output");eprintln!("[fused single-token] PASSED (tolerance={:.4})", tolerance);}@@ -975,36 +865,23 @@ fn test_fused_single_token_k4() {let e = 8;let k = 4;- eprintln!(- "[fused single-token K=4] d_model={}, d_ff={}, E={}, K={}",- d_model, d_ff, e, k- );+ eprintln!("[fused single-token K=4] d_model={}, d_ff={}, E={}, K={}", d_model, d_ff, e, k);- let x: Vec<bf16> = (0..d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();let topk_ids: Vec<i32> = (0..k).map(|i| ((i * 2) % e) as i32).collect();let topk_probs: Vec<bf16> = {- let raw: Vec<f32> =- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();let sum: f32 = raw.iter().sum();raw.iter().map(|p| bf16::from_f32(p / sum)).collect()};- let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let w2_all: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))- .collect();+ let w13_all: Vec<bf16> =+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let gating_code = 2u32;let silu_alpha = 1.0f32;@@ -1038,12 +915,8 @@ fn test_fused_single_token_k4() {let hidden_buf = alloc_buffer::<f32>(&ctx, k * d_ff);let y_buf = alloc_buffer::<bf16>(&ctx, d_model);- let fused_kernel =- MoeExpertsSingleDecodeKernels::new(&ctx).expect("fused kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx).expect("fused kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(&cb,MoeExpertsSingleDecodeArguments {@@ -1071,21 +944,14 @@ fn test_fused_single_token_k4() {cb.commit();cb.wait_until_completed();- let y_gpu = unsafe {- std::slice::from_raw_parts(- y_buf.contents().as_ptr() as *const bf16,- d_model,- )- };+ let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };// Compute error metricslet mut max_abs_error = 0.0f32;let mut sum_abs_error = 0.0f32;let mut max_idx = 0;- for (i, (&gpu_val, &cpu_val)) in- y_gpu.iter().zip(y_expected.iter()).enumerate()- {+ for (i, (&gpu_val, &cpu_val)) in y_gpu.iter().zip(y_expected.iter()).enumerate() {let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();sum_abs_error += abs_error;if abs_error > max_abs_error {@@ -1105,12 +971,7 @@ fn test_fused_single_token_k4() {);let tolerance = 0.02;- assert_bf16_close(- y_gpu,- &y_expected,- tolerance,- "fused single-token K=4 output",- );+ assert_bf16_close(y_gpu, &y_expected, tolerance, "fused single-token K=4 output");eprintln!("[fused single-token K=4] PASSED (tolerance={:.4})", tolerance);}diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rsindex e6656dd96..4b4efdf7d 100644--- a/crates/uzu/tests/moe/moe_finalize_test.rs+++ b/crates/uzu/tests/moe/moe_finalize_test.rs@@ -5,13 +5,9 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};use half::bf16;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::backends::common::kernel::MoeFinalizeKernel;-use uzu::backends::metal::{- KernelDataType, kernel::dsl::MoeFinalizeMetalKernel,-};+use uzu::backends::metal::{KernelDataType, kernel::dsl::MoeFinalizeMetalKernel};-use super::test_utils::{- alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx,-};+use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};fn cpu_finalize(tok2row: &[i32],@@ -30,8 +26,7 @@ fn cpu_finalize(let row = tok2row[idx];if row >= 0 {let rowu = row as usize;- acc += f32::from(probs[idx])- * f32::from(y_partial[rowu * d_model + f]);+ acc += f32::from(probs[idx]) * f32::from(y_partial[rowu * d_model + f]);}}y[ti * d_model + f] = bf16::from_f32(acc);@@ -54,10 +49,7 @@ fn test_finalize_correctness() {];for (t, k, d_model, sum_k) in shapes {- eprintln!(- "[FinalizeTest] T={}, K={}, d_model={}, sum_k={}",- t, k, d_model, sum_k- );+ eprintln!("[FinalizeTest] T={}, K={}, d_model={}, sum_k={}", t, k, d_model, sum_k);// Generate random tok2row mapping: maps (token, k_idx) β row in y_partial// Some entries can be -1 (no expert selected)@@ -77,14 +69,10 @@ fn test_finalize_correctness() {}// Generate random probabilities (should sum to 1 per token, but not critical for unit test)- let probs: Vec<bf16> = (0..t * k)- .map(|_| bf16::from_f32(rng.random_range(0.0..1.0)))- .collect();+ let probs: Vec<bf16> = (0..t * k).map(|_| bf16::from_f32(rng.random_range(0.0..1.0))).collect();// Generate random y_partial (expert outputs)- let y_partial: Vec<bf16> = (0..sum_k * d_model)- .map(|_| bf16::from_f32(rng.random_range(-2.0..2.0)))- .collect();+ let y_partial: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-2.0..2.0))).collect();// CPU referencelet y_cpu = cpu_finalize(&tok2row, &probs, &y_partial, t, d_model, k);@@ -96,13 +84,8 @@ fn test_finalize_correctness() {let y_out_buf = alloc_buffer::<bf16>(&ctx, t * d_model);// Execute finalize kernel- let finalize =- MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into())- .expect("finalize kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let finalize = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");finalize.encode(&tok2row_buf,@@ -119,12 +102,7 @@ fn test_finalize_correctness() {cb.wait_until_completed();// Compare- let y_gpu = unsafe {- std::slice::from_raw_parts(- y_out_buf.contents().as_ptr() as *const bf16,- t * d_model,- )- };+ let y_gpu = unsafe { std::slice::from_raw_parts(y_out_buf.contents().as_ptr() as *const bf16, t * d_model) };// BF16 has limited precision, especially for weighted sumsassert_bf16_close(y_gpu, &y_cpu, 1e-2, "finalize output");diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rsindex 19fad640f..144b22f8d 100644--- a/crates/uzu/tests/moe/moe_gather_test.rs+++ b/crates/uzu/tests/moe/moe_gather_test.rs@@ -9,9 +9,7 @@ use uzu::backends::metal::kernel::{moe::{MoeGatherArguments, MoeGatherKernels},};-use super::test_utils::{- alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx,-};+use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};/// CPU reference for gather operation: x_perm[i] = x[bucketed_ids[i]]///@@ -37,8 +35,7 @@ pub fn cpu_gather(if token_id >= 0 && (token_id as usize) < t {let src_offset = (token_id as usize) * d_model;let dst_offset = row * d_model;- x_perm[dst_offset..dst_offset + d_model]- .copy_from_slice(&x[src_offset..src_offset + d_model]);+ x_perm[dst_offset..dst_offset + d_model].copy_from_slice(&x[src_offset..src_offset + d_model]);}}x_perm@@ -61,13 +58,10 @@ fn test_gather_correctness() {eprintln!("[GatherTest] T={}, sum_k={}, d_model={}", t, sum_k, d_model);// Random input- let x: Vec<bf16> = (0..t * d_model)- .map(|_| bf16::from_f32(rng.random_range(-2.0..2.0)))- .collect();+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-2.0..2.0))).collect();// Random bucketed_ids with valid token indices- let bucketed_ids: Vec<i32> =- (0..sum_k).map(|_| rng.random_range(0..t as i32)).collect();+ let bucketed_ids: Vec<i32> = (0..sum_k).map(|_| rng.random_range(0..t as i32)).collect();let x_cpu = cpu_gather(&x, &bucketed_ids, t, d_model, sum_k);@@ -82,10 +76,7 @@ fn test_gather_correctness() {// Execute gather kernel using kernel structlet gather = MoeGatherKernels::new(&ctx).expect("MoeGatherKernel::new");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");gather.encode(&cb,KernelDataType::BFloat16,@@ -103,12 +94,8 @@ fn test_gather_correctness() {cb.wait_until_completed();// Compare- let x_gpu = unsafe {- std::slice::from_raw_parts(- x_perm_buf.contents().as_ptr() as *const bf16,- sum_k * d_model,- )- };+ let x_gpu =+ unsafe { std::slice::from_raw_parts(x_perm_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };assert_bf16_close(x_gpu, &x_cpu, 1e-6, "gather output");diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex 6af1e2978..6766e7eda 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -10,10 +10,8 @@ use uzu::backends::{kernel::{dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},moe::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,- MoeGatherArguments, MoeGatherKernels, MoeRouterTopKArguments,- MoeRouterTopKKernel, MoeScatterKernels,- MoeScatterWithMapArguments,+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels, MoeGatherArguments, MoeGatherKernels,+ MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterKernels, MoeScatterWithMapArguments,},},},@@ -43,8 +41,7 @@ impl PerfResult {let min = sorted[0];let max = sorted[sorted.len() - 1];- let variance = sorted.iter().map(|t| (t - mean).powi(2)).sum::<f64>()- / sorted.len() as f64;+ let variance = sorted.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / sorted.len() as f64;let std_dev = variance.sqrt();Self {@@ -60,12 +57,7 @@ impl PerfResult {fn print(&self) {eprintln!(" {:<30} mean={:8.3}ms median={:8.3}ms min={:8.3}ms max={:8.3}ms std={:6.3}ms",- self.name,- self.mean_ms,- self.median_ms,- self.min_ms,- self.max_ms,- self.std_dev_ms+ self.name, self.mean_ms, self.median_ms, self.min_ms, self.max_ms, self.std_dev_ms);}}@@ -103,30 +95,18 @@ fn test_moe_e2e_decode_perf() {let ctx = create_ctx();let mut rng = StdRng::seed_from_u64(0xA0075);- let configs = vec![- ("Small", 1, 1024, 256, 8, 2),- ("Medium", 1, 2048, 1024, 16, 2),- ("Production", 1, 4096, 14336, 16, 2),- ];+ let configs =+ vec![("Small", 1, 1024, 256, 8, 2), ("Medium", 1, 2048, 1024, 16, 2), ("Production", 1, 4096, 14336, 16, 2)];eprintln!("\n=== End-to-End MoE Performance (DECODE, T=1) ===");for (config_name, t, d_model, d_ff, e, k) in configs {- eprintln!(- "\n{} T={}, D={}, H={}, E={}, K={}",- config_name, t, d_model, d_ff, e, k- );+ eprintln!("\n{} T={}, D={}, H={}, E={}, K={}", config_name, t, d_model, d_ff, e, k);// Generate data- let x: Vec<bf16> = (0..t * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();- let router_w: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))- .collect();- let router_b: Vec<bf16> = (0..e)- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))- .collect();+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();+ let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();+ let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();// Buffers (simplified - just key buffers for timing)let x_buf = alloc_buffer_with_data(&ctx, &x);@@ -135,15 +115,11 @@ fn test_moe_e2e_decode_perf() {let topk_ids_buf = alloc_buffer::<i32>(&ctx, t * k);let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);- let router_topk =- MoeRouterTopKKernel::new(&ctx).expect("router+topk fused kernel");+ let router_topk = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused kernel");// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");router_topk.encode(&cb,@@ -188,21 +164,12 @@ fn test_moe_e2e_prefill_perf() {eprintln!("\n=== End-to-End MoE Performance (PREFILL, T>1) ===");for (config_name, t, d_model, d_ff, e, k) in configs {- eprintln!(- "\n{} T={}, D={}, H={}, E={}, K={}",- config_name, t, d_model, d_ff, e, k- );+ eprintln!("\n{} T={}, D={}, H={}, E={}, K={}", config_name, t, d_model, d_ff, e, k);// Generate data- let x: Vec<bf16> = (0..t * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();- let router_w: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))- .collect();- let router_b: Vec<bf16> = (0..e)- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))- .collect();+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();+ let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();+ let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();// Buffers (simplified - just key buffers for timing)let x_buf = alloc_buffer_with_data(&ctx, &x);@@ -211,15 +178,11 @@ fn test_moe_e2e_prefill_perf() {let topk_ids_buf = alloc_buffer::<i32>(&ctx, t * k);let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);- let router_topk =- MoeRouterTopKKernel::new(&ctx).expect("router+topk fused kernel");+ let router_topk = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused kernel");// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");router_topk.encode(&cb,@@ -260,21 +223,14 @@ fn test_moe_pipeline_breakdown_decode() {let mut rng = StdRng::seed_from_u64(0xDECADE);eprintln!("\n=== MoE Pipeline Breakdown (DECODE, T=1) ===");- eprintln!(- "Measures ALL MoE kernels: RouterβTopKβCountsβOffsetsβScatterβGatherβExpertsβFinalize\n"- );+ eprintln!("Measures ALL MoE kernels: RouterβTopKβCountsβOffsetsβScatterβGatherβExpertsβFinalize\n");let (t, d_model, d_ff, e, k) = (1, 4096, 14336, 16, 2);// Allocate buffers- let x: Vec<bf16> = (0..t * d_model)- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))- .collect();- let router_w: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))- .collect();- let router_b: Vec<bf16> =- (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();+ let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();+ let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();let x_buf = alloc_buffer_with_data(&ctx, &x);let router_w_buf = alloc_buffer_with_data(&ctx, &router_w);@@ -294,9 +250,8 @@ fn test_moe_pipeline_breakdown_decode() {// Expert weights buffers// Generate W13 in original layout [E, d_model, 2*d_ff]- let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))- .collect();+ let w13_original: Vec<bf16> =+ (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();// Transpose to GPU layout [E, 2*d_ff, d_model]let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];@@ -312,15 +267,9 @@ fn test_moe_pipeline_breakdown_decode() {}}- let w2: Vec<bf16> = (0..e * d_ff * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))- .collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))- .collect();- let down_biases: Vec<bf16> = (0..e * d_model)- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))- .collect();+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();let w13_buf = alloc_buffer_with_data(&ctx, &w13);let w2_buf = alloc_buffer_with_data(&ctx, &w2);@@ -348,74 +297,58 @@ fn test_moe_pipeline_breakdown_decode() {let bucketed_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);// Create kernel structs (use production-validated encoding logic)- let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx)- .expect("counts+offsets fused");+ let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("counts+offsets fused");let scatter_kernel = MoeScatterKernels::new(&ctx).expect("scatter");let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx)- .expect("experts two-pass decode");- let finalize_kernel =- MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into())- .expect("finalize");- let router_topk_fused_kernel =- MoeRouterTopKKernel::new(&ctx).expect("router+topk fused");+ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("experts two-pass decode");+ let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize");+ let router_topk_fused_kernel = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused");// Testing: Router + TopK + Counts+Offsets (FUSED)- let router_topk_fused_perf =- time_kernel("Router+TopK (FUSED)", 2, 5, || {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- router_topk_fused_kernel- .encode(- &cb,- KernelDataType::BFloat16,- MoeRouterTopKArguments {- input_buffer: &x_buf,- weight_buffer: &router_w_buf,- bias_buffer: &router_b_buf,- topk_ids_buffer: &topk_ids_buf,- topk_probs_buffer: &topk_probs_buf,- t,- d_model,- e,- k,- renorm: true,- },- )- .expect("router_topk_fused");- cb.commit();- cb.wait_until_completed();- });+ let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ router_topk_fused_kernel+ .encode(+ &cb,+ KernelDataType::BFloat16,+ MoeRouterTopKArguments {+ input_buffer: &x_buf,+ weight_buffer: &router_w_buf,+ bias_buffer: &router_b_buf,+ topk_ids_buffer: &topk_ids_buf,+ topk_probs_buffer: &topk_probs_buf,+ t,+ d_model,+ e,+ k,+ renorm: true,+ },+ )+ .expect("router_topk_fused");+ cb.commit();+ cb.wait_until_completed();+ });- let counts_offsets_perf =- time_kernel("Counts+Offsets (FUSED)", 2, 5, || {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");- counts_offsets_kernel.encode(- &topk_ids_buf,- &offsets_buf,- &sumk_buf,- &partials_buf,- t as u32,- e as u32,- k as u32,- &encoder,- );- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();- });+ let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let encoder = cb.new_compute_command_encoder().expect("encoder");+ counts_offsets_kernel.encode(+ &topk_ids_buf,+ &offsets_buf,+ &sumk_buf,+ &partials_buf,+ t as u32,+ e as u32,+ k as u32,+ &encoder,+ );+ encoder.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ });let scatter_perf = time_kernel("Scatter", 2, 5, || {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");scatter_kernel.encode_block_bases(&cb,@@ -433,21 +366,20 @@ fn test_moe_pipeline_breakdown_decode() {.encode_scatter_with_map(&cb,MoeScatterWithMapArguments {- base:- uzu::backends::metal::kernel::moe::MoeScatterArguments {- topk_ids_buffer: &topk_ids_buf,- topk_probs_buffer: &topk_probs_buf,- offsets_buffer: &offsets_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- out_ids_buffer: &bucketed_ids_buf,- out_probs_buffer: &bucketed_probs_buf,- t,- e,- k,- num_blocks,- num_tiles,- },+ base: uzu::backends::metal::kernel::moe::MoeScatterArguments {+ topk_ids_buffer: &topk_ids_buf,+ topk_probs_buffer: &topk_probs_buf,+ offsets_buffer: &offsets_buf,+ block_bases_buffer: &block_bases_buf,+ block_alloc_buffer: &block_alloc_buf,+ out_ids_buffer: &bucketed_ids_buf,+ out_probs_buffer: &bucketed_probs_buf,+ t,+ e,+ k,+ num_blocks,+ num_tiles,+ },tok2row_buffer: &tok2row_buf,},KernelDataType::BFloat16,@@ -458,10 +390,7 @@ fn test_moe_pipeline_breakdown_decode() {});let gather_perf = time_kernel("Gather", 2, 5, || {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");gather_kernel.encode(&cb,KernelDataType::BFloat16,@@ -480,10 +409,7 @@ fn test_moe_pipeline_breakdown_decode() {});let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb,&MoeExpertsTwoPassArguments {@@ -520,10 +446,7 @@ fn test_moe_pipeline_breakdown_decode() {});let finalize_perf = time_kernel("Finalize", 2, 5, || {- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");finalize_kernel.encode(&tok2row_buf,@@ -557,9 +480,7 @@ fn test_moe_pipeline_breakdown_decode() {+ finalize_perf.mean_ms)* 1000.0;- eprintln!(- "\n βββ Per-Kernel Latency (Production D=4096, H=14336, E=16, K=2, T=1) βββ"- );+ eprintln!("\n βββ Per-Kernel Latency (Production D=4096, H=14336, E=16, K=2, T=1) βββ");eprintln!(" Router+TopK (FUSED): {:8.1} us ({:5.1}%)",router_topk_fused_perf.mean_ms * 1000.0,@@ -592,10 +513,6 @@ fn test_moe_pipeline_breakdown_decode() {);eprintln!(" βββββββββββββββββββββββββββββββββββββββββββ");eprintln!(" TOTAL: {:8.1} us (100.0%)", total_us);- eprintln!(- "\n Note: Times include Metal CB overhead (~10-50ms per kernel)."- );- eprintln!(- " Real GPU compute is much faster, but relative % shows bottleneck."- );+ eprintln!("\n Note: Times include Metal CB overhead (~10-50ms per kernel).");+ eprintln!(" Real GPU compute is much faster, but relative % shows bottleneck.");}diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rsindex d3f0717f7..18e9435f4 100644--- a/crates/uzu/tests/moe/moe_router_topk_test.rs+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs@@ -81,9 +81,7 @@ pub fn cpu_topk_select_f32(let v = row[expert];let mut insert_pos = None;for j in (0..k).rev() {- if v > best_vals[j]- || (v == best_vals[j] && (expert as i32) < best_ids[j])- {+ if v > best_vals[j] || (v == best_vals[j] && (expert as i32) < best_ids[j]) {insert_pos = Some(j);}}@@ -101,8 +99,7 @@ pub fn cpu_topk_select_f32(ids[base + kk] = best_ids[kk];}if renorm {- let max_v =- best_vals.iter().copied().fold(f32::NEG_INFINITY, f32::max);+ let max_v = best_vals.iter().copied().fold(f32::NEG_INFINITY, f32::max);let mut exps = vec![0.0f32; k];let mut sum = 0.0f32;for kk in 0..k {@@ -138,25 +135,18 @@ fn run_router_topk_once(renorm: bool,) {let mut rng = StdRng::seed_from_u64(1234);- let input_f32: Vec<f32> =- (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();- let weight_f32: Vec<f32> =- (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();- let bias_f32: Vec<f32> =- (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();+ let input_f32: Vec<f32> = (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();+ let weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();+ let bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();// Convert to bf16- let input: Vec<bf16> =- input_f32.iter().map(|&x| bf16::from_f32(x)).collect();- let weight: Vec<bf16> =- weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();+ let input: Vec<bf16> = input_f32.iter().map(|&x| bf16::from_f32(x)).collect();+ let weight: Vec<bf16> = weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();let bias: Vec<bf16> = bias_f32.iter().map(|&x| bf16::from_f32(x)).collect();// Compute CPU reference using bf16 inputs (with f32 accumulation)- let logits_ref =- cpu_router_logits_bf16(&input, &weight, &bias, t, e, d_model);- let (ids_ref, probs_ref) =- cpu_topk_select_f32(&logits_ref, t, e, k, renorm);+ let logits_ref = cpu_router_logits_bf16(&input, &weight, &bias, t, e, d_model);+ let (ids_ref, probs_ref) = cpu_topk_select_f32(&logits_ref, t, e, k, renorm);let input_buf = alloc_buffer_with_data(ctx, &input);let weight_buf = alloc_buffer_with_data(ctx, &weight);@@ -165,10 +155,7 @@ fn run_router_topk_once(// For BFloat16 kernel, probs buffer must be bf16, not f32let probs_buf = alloc_buffer::<bf16>(ctx, t * k);- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let args = MoeRouterTopKArguments {input_buffer: &input_buf,weight_buffer: &weight_buf,@@ -181,21 +168,16 @@ fn run_router_topk_once(k,renorm,};- kernel- .encode(&cb, KernelDataType::BFloat16, args)- .expect("encode fused router+topk");+ kernel.encode(&cb, KernelDataType::BFloat16, args).expect("encode fused router+topk");cb.commit();cb.wait_until_completed();let ids_ptr = ids_buf.contents().as_ptr() as *const i32;let probs_ptr = probs_buf.contents().as_ptr() as *const bf16;- let ids_gpu =- unsafe { std::slice::from_raw_parts(ids_ptr, t * k) }.to_vec();- let probs_bf16_gpu =- unsafe { std::slice::from_raw_parts(probs_ptr, t * k) }.to_vec();+ let ids_gpu = unsafe { std::slice::from_raw_parts(ids_ptr, t * k) }.to_vec();+ let probs_bf16_gpu = unsafe { std::slice::from_raw_parts(probs_ptr, t * k) }.to_vec();// Convert bf16 to f32 for comparison- let probs_gpu: Vec<f32> =- probs_bf16_gpu.iter().map(|&h| f32::from(h)).collect();+ let probs_gpu: Vec<f32> = probs_bf16_gpu.iter().map(|&h| f32::from(h)).collect();assert_eq!(ids_gpu, ids_ref,@@ -239,13 +221,8 @@ fn test_router_topk_fused_matches_reference() {let ctx = create_ctx();let kernel = MoeRouterTopKKernel::new(&ctx).expect("kernel");- let configs = [- (1usize, 64usize, 32usize, 4usize),- (2, 128, 64, 8),- (4, 256, 128, 16),- (8, 256, 256, 32),- (1, 512, 512, 64),- ];+ let configs =+ [(1usize, 64usize, 32usize, 4usize), (2, 128, 64, 8), (4, 256, 128, 16), (8, 256, 256, 32), (1, 512, 512, 64)];let renorm_options = [false, true];for &(t, d_model, e, k) in &configs {diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rsindex a964fc5a3..8cfe18237 100644--- a/crates/uzu/tests/moe/moe_scatter_test.rs+++ b/crates/uzu/tests/moe/moe_scatter_test.rs@@ -8,8 +8,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::backends::common::kernel::MoeCountsOffsetsFusedKernel;use uzu::backends::metal::kernel::dsl::MoeCountsOffsetsFusedMetalKernel;use uzu::backends::metal::kernel::{- KernelDataType, MoeBlockBasesArguments, MoeScatterArguments,- MoeScatterKernels,+ KernelDataType, MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels,moe::{MoeRouterTopKArguments, MoeRouterTopKKernel},};@@ -61,19 +60,13 @@ fn test_scatter_buckets_parity() {// Generate random input and router weights for fused kernellet d_model = 64;- let input_f32: Vec<f32> =- (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();- let weight_f32: Vec<f32> =- (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();- let bias_f32: Vec<f32> =- (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();-- let input: Vec<bf16> =- input_f32.iter().map(|&x| bf16::from_f32(x)).collect();- let weight: Vec<bf16> =- weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();- let bias: Vec<bf16> =- bias_f32.iter().map(|&x| bf16::from_f32(x)).collect();+ let input_f32: Vec<f32> = (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();+ let weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();+ let bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();++ let input: Vec<bf16> = input_f32.iter().map(|&x| bf16::from_f32(x)).collect();+ let weight: Vec<bf16> = weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();+ let bias: Vec<bf16> = bias_f32.iter().map(|&x| bf16::from_f32(x)).collect();let input_buf = alloc_buffer_with_data(&ctx, &input);let weight_buf = alloc_buffer_with_data(&ctx, &weight);@@ -83,10 +76,7 @@ fn test_scatter_buckets_parity() {// Use fused router+topk kernellet router_topk = MoeRouterTopKKernel::new(&ctx).expect("router_topk");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");router_topk.encode(&cb,@@ -109,32 +99,18 @@ fn test_scatter_buckets_parity() {cb.wait_until_completed();// Read back CPU probs for reference compare- let probs_bf16 = unsafe {- std::slice::from_raw_parts(- topk_probs_buf.contents().as_ptr() as *const bf16,- t * k,- )- };- let topk_probs_cpu: Vec<f32> =- probs_bf16.iter().map(|&h| f32::from(h)).collect();- let topk_ids_cpu = unsafe {- std::slice::from_raw_parts(- topk_ids_buf.contents().as_ptr() as *const i32,- t * k,- )- };+ let probs_bf16 =+ unsafe { std::slice::from_raw_parts(topk_probs_buf.contents().as_ptr() as *const bf16, t * k) };+ let topk_probs_cpu: Vec<f32> = probs_bf16.iter().map(|&h| f32::from(h)).collect();+ let topk_ids_cpu = unsafe { std::slice::from_raw_parts(topk_ids_buf.contents().as_ptr() as *const i32, t * k) };let offsets_buf = alloc_buffer::<u32>(&ctx, e + 1);let sumk_buf = alloc_buffer::<u32>(&ctx, 1);let num_tiles = ((e + 511) / 512).max(1);let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);- let fused_kernel =- MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let fused_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");fused_kernel.encode(&topk_ids_buf,@@ -157,10 +133,7 @@ fn test_scatter_buckets_parity() {let block_bases_buf = alloc_buffer::<u32>(&ctx, entries);let scatter = MoeScatterKernels::new(&ctx).expect("scatter kernels");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);scatter.encode_block_bases(@@ -178,19 +151,11 @@ fn test_scatter_buckets_parity() {cb.commit();cb.wait_until_completed();- let sumk = unsafe {- std::slice::from_raw_parts(- sumk_buf.contents().as_ptr() as *const u32,- 1,- )- }[0] as usize;+ let sumk = unsafe { std::slice::from_raw_parts(sumk_buf.contents().as_ptr() as *const u32, 1) }[0] as usize;let out_ids_buf = alloc_buffer::<i32>(&ctx, sumk);let out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");scatter.encode_scatter(&cb,@@ -214,30 +179,13 @@ fn test_scatter_buckets_parity() {cb.commit();cb.wait_until_completed();- let out_ids = unsafe {- std::slice::from_raw_parts(- out_ids_buf.contents().as_ptr() as *const i32,- sumk,- )- };- let out_probs_h = unsafe {- std::slice::from_raw_parts(- out_probs_buf.contents().as_ptr() as *const bf16,- sumk,- )- };- let _out_probs: Vec<f32> =- out_probs_h.iter().map(|&h| f32::from(h)).collect();+ let out_ids = unsafe { std::slice::from_raw_parts(out_ids_buf.contents().as_ptr() as *const i32, sumk) };+ let out_probs_h = unsafe { std::slice::from_raw_parts(out_probs_buf.contents().as_ptr() as *const bf16, sumk) };+ let _out_probs: Vec<f32> = out_probs_h.iter().map(|&h| f32::from(h)).collect();// CPU reference- let (cpu_ids, _cpu_probs, offsets_cpu) =- cpu_expert_buckets(topk_ids_cpu, &topk_probs_cpu, t, e, k);- let offsets_gpu = unsafe {- std::slice::from_raw_parts(- offsets_buf.contents().as_ptr() as *const u32,- e + 1,- )- };+ let (cpu_ids, _cpu_probs, offsets_cpu) = cpu_expert_buckets(topk_ids_cpu, &topk_probs_cpu, t, e, k);+ let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };assert_eq!(offsets_gpu, &offsets_cpu[..]);// Compare per-expert multisets of idsdiff --git a/crates/uzu/tests/moe/moe_test_utils.rs b/crates/uzu/tests/moe/moe_test_utils.rsindex f33b21708..b9cf882e8 100644--- a/crates/uzu/tests/moe/moe_test_utils.rs+++ b/crates/uzu/tests/moe/moe_test_utils.rs@@ -30,10 +30,7 @@ pub fn alloc_buffer_with_data<T: bytemuck::NoUninit>(ctx.create_buffer(1).expect("Failed to create empty buffer")} else {ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")}}diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rsindex 41a6d92a2..13afb70a1 100644--- a/crates/uzu/tests/moe/moe_tiles_test.rs+++ b/crates/uzu/tests/moe/moe_tiles_test.rs@@ -2,14 +2,9 @@use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu::backends::metal::kernel::moe::tiles_map::{- MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments,-};+use uzu::backends::metal::kernel::moe::tiles_map::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments};-use super::test_utils::{- alloc_buffer, alloc_buffer_with_data, cpu_tile_counts, cpu_tile_scan,- create_ctx,-};+use super::test_utils::{alloc_buffer, alloc_buffer_with_data, cpu_tile_counts, cpu_tile_scan, create_ctx};#[test]fn test_tile_counts_correctness() {@@ -36,12 +31,8 @@ fn test_tile_counts_correctness() {let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);// Execute kernel using kernel struct- let tile_kernel =- MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(&cb,&MoeTileCountsArguments {@@ -54,19 +45,10 @@ fn test_tile_counts_correctness() {cb.wait_until_completed();// Compare- let tile_counts_gpu = unsafe {- std::slice::from_raw_parts(- tile_counts_buf.contents().as_ptr() as *const u32,- e,- )- };-- assert_eq!(- tile_counts_gpu,- &tile_counts_cpu[..],- "Tile counts mismatch for E={}",- e- );+ let tile_counts_gpu =+ unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };++ assert_eq!(tile_counts_gpu, &tile_counts_cpu[..], "Tile counts mismatch for E={}", e);eprintln!("[TileCountsTest] β PASSED");}@@ -84,8 +66,7 @@ fn test_tile_scan_correctness() {eprintln!("[TileScanTest] E={}", e);// Generate random tile counts- let tile_counts: Vec<u32> =- (0..e).map(|_| rng.random_range(0..20)).collect();+ let tile_counts: Vec<u32> = (0..e).map(|_| rng.random_range(0..20)).collect();let (tile_offsets_cpu, total_tiles_cpu) = cpu_tile_scan(&tile_counts);@@ -95,12 +76,8 @@ fn test_tile_scan_correctness() {let total_tiles_buf = alloc_buffer::<u32>(&ctx, 1);// Execute kernel using kernel struct- let tile_kernel =- MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_scan(&cb,&MoeTileScanArguments {@@ -114,26 +91,12 @@ fn test_tile_scan_correctness() {cb.wait_until_completed();// Compare- let tile_offsets_gpu = unsafe {- std::slice::from_raw_parts(- tile_offsets_buf.contents().as_ptr() as *const u32,- e + 1,- )- };- let total_tiles_gpu =- unsafe { *(total_tiles_buf.contents().as_ptr() as *const u32) };-- assert_eq!(- tile_offsets_gpu,- &tile_offsets_cpu[..],- "Tile offsets mismatch for E={}",- e- );- assert_eq!(- total_tiles_gpu, total_tiles_cpu,- "Total tiles mismatch for E={}",- e- );+ let tile_offsets_gpu =+ unsafe { std::slice::from_raw_parts(tile_offsets_buf.contents().as_ptr() as *const u32, e + 1) };+ let total_tiles_gpu = unsafe { *(total_tiles_buf.contents().as_ptr() as *const u32) };++ assert_eq!(tile_offsets_gpu, &tile_offsets_cpu[..], "Tile offsets mismatch for E={}", e);+ assert_eq!(total_tiles_gpu, total_tiles_cpu, "Total tiles mismatch for E={}", e);eprintln!("[TileScanTest] β PASSED");}@@ -153,12 +116,8 @@ fn test_tile_edge_cases() {let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);- let tile_kernel =- MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(&cb,&MoeTileCountsArguments {@@ -170,12 +129,8 @@ fn test_tile_edge_cases() {cb.commit();cb.wait_until_completed();- let tile_counts_gpu = unsafe {- std::slice::from_raw_parts(- tile_counts_buf.contents().as_ptr() as *const u32,- e,- )- };+ let tile_counts_gpu =+ unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };assert_eq!(tile_counts_gpu, &tile_counts_cpu[..]);assert!(tile_counts_gpu.iter().all(|&c| c == 0));@@ -192,12 +147,8 @@ fn test_tile_edge_cases() {let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);- let tile_kernel =- MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");- let cb = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(&cb,&MoeTileCountsArguments {@@ -209,12 +160,8 @@ fn test_tile_edge_cases() {cb.commit();cb.wait_until_completed();- let tile_counts_gpu = unsafe {- std::slice::from_raw_parts(- tile_counts_buf.contents().as_ptr() as *const u32,- e,- )- };+ let tile_counts_gpu =+ unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };assert_eq!(tile_counts_gpu, &tile_counts_cpu[..]);// With BM=16: seg_lens=[10, 5, 15, 20] -> tile_counts=[1, 1, 1, 2]diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rsindex 6fb1b2b48..f3d877a36 100644--- a/crates/uzu/tests/parameters_loader_test.rs+++ b/crates/uzu/tests/parameters_loader_test.rs@@ -12,13 +12,10 @@ use uzu::{fn test_parameter_loader_basic() {let weights_path = crate::common::get_test_weights_path();let context = MTLContext::new().expect("Failed to create MTLContext");- let file = std::fs::File::open(&weights_path)- .expect("Weights file not found; run download script");+ let file = std::fs::File::open(&weights_path).expect("Weights file not found; run download script");- let loader =- ParameterLoader::new(&file, context.as_ref()).expect("create loader");- let embeddings =- loader.get("embedding.weights").expect("weights embeddings");+ let loader = ParameterLoader::new(&file, context.as_ref()).expect("create loader");+ let embeddings = loader.get("embedding.weights").expect("weights embeddings");let view = embeddings.as_view::<bf16>();assert!(is_close!(view[[5usize, 3usize]], bf16::from_f32(-0.01819)));diff --git a/crates/uzu/tests/person_schema_metal_bitmask_test.rs b/crates/uzu/tests/person_schema_metal_bitmask_test.rsindex 4b045991c..e3454fe5b 100644--- a/crates/uzu/tests/person_schema_metal_bitmask_test.rs+++ b/crates/uzu/tests/person_schema_metal_bitmask_test.rs@@ -7,10 +7,7 @@ use schemars::JsonSchema;use serde::{Deserialize, Serialize};use tokenizers::Tokenizer;use uzu::{DataType, backends::metal::MetalArray};-use xgrammar::{- DLDevice, DLDeviceType, DLTensor, Grammar, GrammarCompiler, GrammarMatcher,- TokenizerInfo,-};+use xgrammar::{DLDevice, DLDeviceType, DLTensor, Grammar, GrammarCompiler, GrammarMatcher, TokenizerInfo};#[derive(Debug, Serialize, Deserialize, JsonSchema)]struct Person {@@ -24,10 +21,7 @@ fn person_schema_metal_bitmask() {let crate_version = env!("CARGO_PKG_VERSION");let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));let repo_root = manifest_dir.parent().unwrap().parent().unwrap();- let model_dir = repo_root- .join("models")- .join(crate_version)- .join("Llama-3.2-1B-Instruct");+ let model_dir = repo_root.join("models").join(crate_version).join("Llama-3.2-1B-Instruct");let tokenizer_path = model_dir.join("tokenizer.json");if !tokenizer_path.exists() {@@ -39,26 +33,13 @@ fn person_schema_metal_bitmask() {}let schema_root = schemars::schema_for!(Person);- let schema_str =- serde_json::to_string(&schema_root).expect("schema to string");+ let schema_str = serde_json::to_string(&schema_root).expect("schema to string");- let tokenizer =- Tokenizer::from_file(&tokenizer_path).expect("load tokenizer.json");- let tokenizer_info =- TokenizerInfo::from_huggingface(&tokenizer, None, None).unwrap();+ let tokenizer = Tokenizer::from_file(&tokenizer_path).expect("load tokenizer.json");+ let tokenizer_info = TokenizerInfo::from_huggingface(&tokenizer, None, None).unwrap();- let grammar = Grammar::from_json_schema(- &schema_str,- true,- Some(2),- Some((",", ":")),- true,- None,- false,- )- .unwrap();- let mut compiler =- GrammarCompiler::new(&tokenizer_info, 8, true, -1).unwrap();+ let grammar = Grammar::from_json_schema(&schema_str, true, Some(2), Some((",", ":")), true, None, false).unwrap();+ let mut compiler = GrammarCompiler::new(&tokenizer_info, 8, true, -1).unwrap();let compiled = compiler.compile_grammar(&grammar).unwrap();let mut matcher = GrammarMatcher::new(&compiled, None, true, -1).unwrap();@@ -80,11 +61,8 @@ fn person_schema_metal_bitmask() {// xgrammar expects an int32 mask tensor of shape [buffer_size]let elems = batch * buffer_size;let bytes = elems * core::mem::size_of::<i32>();- let buffer = device- .new_buffer(bytes, MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let metal_bitmask =- unsafe { MetalArray::from_parts(buffer, 0, &shape, DataType::I32) };+ let buffer = device.new_buffer(bytes, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");+ let metal_bitmask = unsafe { MetalArray::from_parts(buffer, 0, &shape, DataType::I32) };let mut shape_i64 = [buffer_size as i64];let mut bitmask_tensor = DLTensor {diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rsindex 8ea1e3b1c..dab71fd17 100644--- a/crates/uzu/tests/quant_matmul_test.rs+++ b/crates/uzu/tests/quant_matmul_test.rs@@ -2,20 +2,14 @@ use std::time::Instant;use bytemuck;use half::{bf16, f16};-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,- MTLDeviceExt, MTLResourceOptions,-};+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use uzu::{DataType,backends::{common::Context,metal::{MTLContext, ProtocolObject, Retained,- kernel::quant_matmul::{- QuantizationType, QuantizedMatmulArguments,- QuantizedMatmulKernel,- },+ kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},},},config::QuantizationMode,@@ -141,8 +135,7 @@ fn get_zp_value(((byte >> 4) & 0x0F) as f32}} else {- zero_points.get(row_idx * stride + group_idx).copied().unwrap_or(0)- as f32+ zero_points.get(row_idx * stride + group_idx).copied().unwrap_or(0) as f32}}@@ -191,19 +184,12 @@ fn cpu_reference(let val_a = a[i * input_dim + l];let scale = scales[l * num_groups + group_idx];- let bias =- if quantization_type == QuantizationType::ZeroPoint {- let zp_val_qvm = get_zp_value(- zero_points,- zero_points_stride,- l,- group_idx,- bits,- );- -scale * zp_val_qvm- } else {- biases[l * num_groups + group_idx]- };+ let bias = if quantization_type == QuantizationType::ZeroPoint {+ let zp_val_qvm = get_zp_value(zero_points, zero_points_stride, l, group_idx, bits);+ -scale * zp_val_qvm+ } else {+ biases[l * num_groups + group_idx]+ };acc += val_a * (scale * val_q + bias);}} else {@@ -211,13 +197,7 @@ fn cpu_reference(let scale = scales[j * num_groups + g];let bias = match quantization_type {QuantizationType::ZeroPoint => {- let zp = get_zp_value(- zero_points,- zero_points_stride,- j,- g,- bits,- );+ let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);-scale * zp},QuantizationType::Mlx => biases[j * num_groups + g],@@ -315,13 +295,9 @@ fn generate_test_quant_params(if bits == 4 {let byte_index = k * zero_points_stride + (g >> 1);if (g & 1) == 0 {- zero_points[byte_index] = (zero_points[byte_index]- & 0xF0)- | (zp_val_u8 & 0x0F);+ zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val_u8 & 0x0F);} else {- zero_points[byte_index] = (zero_points[byte_index]- & 0x0F)- | ((zp_val_u8 & 0x0F) << 4);+ zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val_u8 & 0x0F) << 4);}} else {zero_points[k * zero_points_stride + g] = zp_val_u8;@@ -338,8 +314,7 @@ fn generate_test_quant_params(} else {zp_val_u8 as f32};- biases[k * num_groups + g] =- quantize_value(-s * zp_val, data_type);+ biases[k * num_groups + g] = quantize_value(-s * zp_val, data_type);}}} else {@@ -359,13 +334,9 @@ fn generate_test_quant_params(if bits == 4 {let byte_index = j * zero_points_stride + (g >> 1);if (g & 1) == 0 {- zero_points[byte_index] = (zero_points[byte_index]- & 0xF0)- | (zp_val & 0x0F);+ zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val & 0x0F);} else {- zero_points[byte_index] = (zero_points[byte_index]- & 0x0F)- | ((zp_val & 0x0F) << 4);+ zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);}} else {let byte_index = j * zero_points_stride + g;@@ -386,8 +357,7 @@ fn generate_test_quant_params(for g in 0..num_groups {let base_val = g * 3;let bias_val = (base_val % 19) as f32 * 0.125;- biases[k * num_groups + g] =- quantize_value(bias_val, data_type);+ biases[k * num_groups + g] = quantize_value(bias_val, data_type);}}} else {@@ -399,8 +369,7 @@ fn generate_test_quant_params(g * 3};let bias_val = (base_val % 19) as f32 * 0.125;- biases[j * num_groups + g] =- quantize_value(bias_val, data_type);+ biases[j * num_groups + g] = quantize_value(bias_val, data_type);}}}@@ -422,31 +391,20 @@ fn buffer_from_f32_slice() -> Retained<ProtocolObject<dyn MTLBuffer>> {match dtype {DataType::F16 => {- let data: Vec<f16> =- values.iter().map(|&v| f16::from_f32(v)).collect();+ let data: Vec<f16> = values.iter().map(|&v| f16::from_f32(v)).collect();ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")},DataType::BF16 => {- let data: Vec<bf16> =- values.iter().map(|&v| bf16::from_f32(v)).collect();+ let data: Vec<bf16> = values.iter().map(|&v| bf16::from_f32(v)).collect();ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer")},DataType::F32 => ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(values),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(values), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer"),other => {panic!("Unsupported dtype for buffer_from_f32_slice: {:?}", other)@@ -468,8 +426,7 @@ fn execute_quantized_matmul(data_type: DataType,bits: usize,) -> ExecutionResult {- let weights_quant =- create_test_weights(output_dim, input_dim, weights_transposed, bits);+ let weights_quant = create_test_weights(output_dim, input_dim, weights_transposed, bits);let weights_packed = if bits == 4 {pack_u4_weights(&weights_quant)} else {@@ -503,29 +460,19 @@ fn execute_quantized_matmul(let w_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&weights_packed),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&weights_packed), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let s_buf = buffer_from_f32_slice(ctx, data_type, ¶ms.scales);let b_buf = match quantization_type {QuantizationType::ZeroPoint => ctx.device- .new_buffer_with_data(- ¶ms.zero_points,- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(¶ms.zero_points, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer"),- QuantizationType::Mlx => {- buffer_from_f32_slice(ctx, data_type, ¶ms.biases)- },+ QuantizationType::Mlx => buffer_from_f32_slice(ctx, data_type, ¶ms.biases),};let x_buf = buffer_from_f32_slice(ctx, data_type, &x_f32);- let y_buf = ctx- .create_buffer(batch * output_dim * data_type.size_in_bytes())- .expect("Failed to create buffer");+ let y_buf = ctx.create_buffer(batch * output_dim * data_type.size_in_bytes()).expect("Failed to create buffer");let kernel = QuantizedMatmulKernel::new(&ctx,@@ -557,13 +504,8 @@ fn execute_quantized_matmul(output_dim: output_dim as i32,quantization_type,};- let cb_ref = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let encoder = cb_ref- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&encoder, args).unwrap();encoder.end_encoding();cb_ref.commit();@@ -585,13 +527,8 @@ fn execute_quantized_matmul(output_dim: output_dim as i32,quantization_type,};- let cb_ref = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let encoder = cb_ref- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&encoder, args).unwrap();encoder.end_encoding();cb_ref.commit();@@ -620,23 +557,17 @@ fn execute_quantized_matmul(let y_out_f32: Vec<f32> = match data_type {DataType::F16 => {let y_ptr = y_buf.contents().as_ptr() as *const f16;- let y_out = unsafe {- std::slice::from_raw_parts(y_ptr, batch * output_dim)- };+ let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };y_out.iter().map(|&v| v.to_f32()).collect()},DataType::BF16 => {let y_ptr = y_buf.contents().as_ptr() as *const bf16;- let y_out = unsafe {- std::slice::from_raw_parts(y_ptr, batch * output_dim)- };+ let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };y_out.iter().map(|&v| v.to_f32()).collect()},DataType::F32 => {let y_ptr = y_buf.contents().as_ptr() as *const f32;- let y_out = unsafe {- std::slice::from_raw_parts(y_ptr, batch * output_dim)- };+ let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };y_out.to_vec()},other => panic!("Unsupported dtype for validation: {:?}", other),@@ -644,17 +575,12 @@ fn execute_quantized_matmul(let mut debug_prints = 0;- for (i, (&exp, &got)) in- y_expected.iter().zip(y_out_f32.iter()).enumerate()- {+ for (i, (&exp, &got)) in y_expected.iter().zip(y_out_f32.iter()).enumerate() {let diff = (exp - got).abs();if check_tolerance(exp, got) {if debug_prints < 16 {- println!(- "\n detail idx {} diff {} exp {} got {}",- i, diff, exp, got- );+ println!("\n detail idx {} diff {} exp {} got {}", i, diff, exp, got);}debug_prints += 1;}@@ -676,16 +602,9 @@ fn execute_quantized_matmul(.last().map(|(i, _)| i).unwrap_or(0);- println!(- "\nTotal errors: {} out of {} outputs",- debug_prints,- batch * output_dim- );+ println!("\nTotal errors: {} out of {} outputs", debug_prints, batch * output_dim);println!("Error range: indices {}-{}", first_error, last_error);- println!(- "Dims batch={} output_dim={} input_dim={}",- batch, output_dim, input_dim- );+ println!("Dims batch={} output_dim={} input_dim={}", batch, output_dim, input_dim);panic!("Validation failed with {} mismatches", debug_prints);}@@ -708,8 +627,7 @@ struct TestConfig {const QMV_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];const QVM_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];-const QMM_DIMS: &[(usize, usize, usize)] =- &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];+const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];fn run_kernel_test(ctx: &MTLContext,@@ -778,9 +696,7 @@ fn test_quant_gmv() {for config in &configs {for &(output_dim, input_dim) in QMV_DIMS {- run_kernel_test(- &ctx, 1, output_dim, input_dim, true, config, true, 1,- );+ run_kernel_test(&ctx, 1, output_dim, input_dim, true, config, true, 1);}}}@@ -824,9 +740,7 @@ fn test_quant_qvm() {for config in &configs {for &(output_dim, input_dim) in QVM_DIMS {- run_kernel_test(- &ctx, 1, output_dim, input_dim, false, config, true, 1,- );+ run_kernel_test(&ctx, 1, output_dim, input_dim, false, config, true, 1);}}}@@ -870,9 +784,7 @@ fn test_quant_gmm() {for config in &configs {for &(batch, output_dim, input_dim) in QMM_DIMS {- run_kernel_test(- &ctx, batch, output_dim, input_dim, false, config, true, 1,- );+ run_kernel_test(&ctx, batch, output_dim, input_dim, false, config, true, 1);}}}@@ -916,9 +828,7 @@ fn test_quant_gmm_transposed() {for config in &configs {for &(batch, output_dim, input_dim) in QMM_DIMS {- run_kernel_test(- &ctx, batch, output_dim, input_dim, true, config, true, 1,- );+ run_kernel_test(&ctx, batch, output_dim, input_dim, true, config, true, 1);}}}@@ -983,9 +893,7 @@ fn test_quant_matmul_perf() {for config in &configs {for &(batch, output_dim, input_dim) in &shapes {- let result = run_kernel_test(- &ctx, batch, output_dim, input_dim, false, config, false, 20,- );+ let result = run_kernel_test(&ctx, batch, output_dim, input_dim, false, config, false, 20);let avg = result.elapsed / 20.0 * 1000.0; // msprintln!(diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rsindex c4225ad4b..40c9e8dd3 100644--- a/crates/uzu/tests/rms_norm_kernel_test.rs+++ b/crates/uzu/tests/rms_norm_kernel_test.rs@@ -3,29 +3,21 @@use bytemuck;use half::{bf16, f16};-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,- MTLDeviceExt, MTLResourceOptions,-};+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use uzu::{DataType,backends::{common::Context,metal::{MTLContext,- kernel::rms_norm::{- QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormKernel,- RMSNormKernelType,- },+ kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormKernel, RMSNormKernelType},metal_extensions::CommandBufferTimingExt,},},};// Helper trait to unify different float types for testing-trait TestFloat:- Copy + Clone + std::fmt::Debug + PartialEq + bytemuck::NoUninit-{+trait TestFloat: Copy + Clone + std::fmt::Debug + PartialEq + bytemuck::NoUninit {fn from_f32(val: f32) -> Self;fn to_f32(self) -> f32;fn size_of() -> usize;@@ -92,47 +84,26 @@ fn compute_expected_rms_norm(DataType::F16 => {// Convert inputs to their actual types, then to F16 for accumulationlet input_f16: Vec<f16> = match input_type {- DataType::F32 => {- input_data_f32.iter().map(|&x| f16::from_f32(x)).collect()- },- DataType::F16 => input_data_f32- .iter()- .map(|&x| f16::from_f32(f16::from_f32(x).to_f32()))- .collect(),- DataType::BF16 => input_data_f32- .iter()- .map(|&x| f16::from_f32(bf16::from_f32(x).to_f32()))- .collect(),+ DataType::F32 => input_data_f32.iter().map(|&x| f16::from_f32(x)).collect(),+ DataType::F16 => input_data_f32.iter().map(|&x| f16::from_f32(f16::from_f32(x).to_f32())).collect(),+ DataType::BF16 => input_data_f32.iter().map(|&x| f16::from_f32(bf16::from_f32(x).to_f32())).collect(),_ => panic!("Unsupported input type: {:?}", input_type),};let scale_f16: Vec<f16> = match scale_type {- DataType::F32 => {- scale_data_f32.iter().map(|&x| f16::from_f32(x)).collect()- },- DataType::F16 => scale_data_f32- .iter()- .map(|&x| f16::from_f32(f16::from_f32(x).to_f32()))- .collect(),- DataType::BF16 => scale_data_f32- .iter()- .map(|&x| f16::from_f32(bf16::from_f32(x).to_f32()))- .collect(),+ DataType::F32 => scale_data_f32.iter().map(|&x| f16::from_f32(x)).collect(),+ DataType::F16 => scale_data_f32.iter().map(|&x| f16::from_f32(f16::from_f32(x).to_f32())).collect(),+ DataType::BF16 => scale_data_f32.iter().map(|&x| f16::from_f32(bf16::from_f32(x).to_f32())).collect(),_ => panic!("Unsupported scale type: {:?}", scale_type),};let epsilon_f16 = f16::from_f32(epsilon);let scale_offset_f16 = f16::from_f32(scale_offset);// Compute mean square in F16 (matches Metal kernel logic)- let mean_square_f16 = input_f16- .iter()- .map(|&x| x * x)- .fold(f16::ZERO, |acc, x| acc + x)+ let mean_square_f16 = input_f16.iter().map(|&x| x * x).fold(f16::ZERO, |acc, x| acc + x)/ f16::from_f32(input_f16.len() as f32);// Use rsqrt like the Metal kernel (inverse square root)- let rms_norm_f16 = f16::from_f32(- (mean_square_f16 + epsilon_f16).to_f32().sqrt().recip(),- );+ let rms_norm_f16 = f16::from_f32((mean_square_f16 + epsilon_f16).to_f32().sqrt().recip());// Apply normalization and scaling based on mode (matches Metal kernel exactly)if full_layer_mode {@@ -164,47 +135,26 @@ fn compute_expected_rms_norm(DataType::BF16 => {// Convert inputs to their actual types, then to BF16 for accumulationlet input_bf16: Vec<bf16> = match input_type {- DataType::F32 => {- input_data_f32.iter().map(|&x| bf16::from_f32(x)).collect()- },- DataType::F16 => input_data_f32- .iter()- .map(|&x| bf16::from_f32(f16::from_f32(x).to_f32()))- .collect(),- DataType::BF16 => input_data_f32- .iter()- .map(|&x| bf16::from_f32(bf16::from_f32(x).to_f32()))- .collect(),+ DataType::F32 => input_data_f32.iter().map(|&x| bf16::from_f32(x)).collect(),+ DataType::F16 => input_data_f32.iter().map(|&x| bf16::from_f32(f16::from_f32(x).to_f32())).collect(),+ DataType::BF16 => input_data_f32.iter().map(|&x| bf16::from_f32(bf16::from_f32(x).to_f32())).collect(),_ => panic!("Unsupported input type: {:?}", input_type),};let scale_bf16: Vec<bf16> = match scale_type {- DataType::F32 => {- scale_data_f32.iter().map(|&x| bf16::from_f32(x)).collect()- },- DataType::F16 => scale_data_f32- .iter()- .map(|&x| bf16::from_f32(f16::from_f32(x).to_f32()))- .collect(),- DataType::BF16 => scale_data_f32- .iter()- .map(|&x| bf16::from_f32(bf16::from_f32(x).to_f32()))- .collect(),+ DataType::F32 => scale_data_f32.iter().map(|&x| bf16::from_f32(x)).collect(),+ DataType::F16 => scale_data_f32.iter().map(|&x| bf16::from_f32(f16::from_f32(x).to_f32())).collect(),+ DataType::BF16 => scale_data_f32.iter().map(|&x| bf16::from_f32(bf16::from_f32(x).to_f32())).collect(),_ => panic!("Unsupported scale type: {:?}", scale_type),};let epsilon_bf16 = bf16::from_f32(epsilon);let scale_offset_bf16 = bf16::from_f32(scale_offset);// Compute mean square in BF16 (matches Metal kernel logic)- let mean_square_bf16 = input_bf16- .iter()- .map(|&x| x * x)- .fold(bf16::ZERO, |acc, x| acc + x)+ let mean_square_bf16 = input_bf16.iter().map(|&x| x * x).fold(bf16::ZERO, |acc, x| acc + x)/ bf16::from_f32(input_bf16.len() as f32);// Use rsqrt like the Metal kernel (inverse square root)- let rms_norm_bf16 = bf16::from_f32(- (mean_square_bf16 + epsilon_bf16).to_f32().sqrt().recip(),- );+ let rms_norm_bf16 = bf16::from_f32((mean_square_bf16 + epsilon_bf16).to_f32().sqrt().recip());// Apply normalization and scaling based on mode (matches Metal kernel exactly)if full_layer_mode {@@ -237,33 +187,19 @@ fn compute_expected_rms_norm(// Convert inputs to their actual types, then to F32 for accumulationlet input_f32: Vec<f32> = match input_type {DataType::F32 => input_data_f32.to_vec(),- DataType::F16 => input_data_f32- .iter()- .map(|&x| f16::from_f32(x).to_f32())- .collect(),- DataType::BF16 => input_data_f32- .iter()- .map(|&x| bf16::from_f32(x).to_f32())- .collect(),+ DataType::F16 => input_data_f32.iter().map(|&x| f16::from_f32(x).to_f32()).collect(),+ DataType::BF16 => input_data_f32.iter().map(|&x| bf16::from_f32(x).to_f32()).collect(),_ => panic!("Unsupported input type: {:?}", input_type),};let scale_f32: Vec<f32> = match scale_type {DataType::F32 => scale_data_f32.to_vec(),- DataType::F16 => scale_data_f32- .iter()- .map(|&x| f16::from_f32(x).to_f32())- .collect(),- DataType::BF16 => scale_data_f32- .iter()- .map(|&x| bf16::from_f32(x).to_f32())- .collect(),+ DataType::F16 => scale_data_f32.iter().map(|&x| f16::from_f32(x).to_f32()).collect(),+ DataType::BF16 => scale_data_f32.iter().map(|&x| bf16::from_f32(x).to_f32()).collect(),_ => panic!("Unsupported scale type: {:?}", scale_type),};// Compute mean square in F32 (matches Metal kernel logic)- let mean_square: f32 =- input_f32.iter().map(|&x| x * x).sum::<f32>()- / (input_f32.len() as f32);+ let mean_square: f32 = input_f32.iter().map(|&x| x * x).sum::<f32>() / (input_f32.len() as f32);// Use rsqrt like the Metal kernel (inverse square root)let rms_norm = (mean_square + epsilon).sqrt().recip();@@ -327,17 +263,14 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(.map(|i| {// Create a sine wave pattern with values roughly in [-2, 2] range// This gives us reasonable values that work well with F16 precision- 2.0 * f32::sin(i as f32 * 0.01)- + 0.1 * (i as f32 / model_dim as f32)+ 2.0 * f32::sin(i as f32 * 0.01) + 0.1 * (i as f32 / model_dim as f32)}).collect();let scale_data_f32: Vec<f32> = vec![1.0; model_dim as usize]; // Unit scales// Convert to target types- let input_data: Vec<InputT> =- input_data_f32.iter().map(|&x| InputT::from_f32(x)).collect();- let scale_data: Vec<ScaleT> =- scale_data_f32.iter().map(|&x| ScaleT::from_f32(x)).collect();+ let input_data: Vec<InputT> = input_data_f32.iter().map(|&x| InputT::from_f32(x)).collect();+ let scale_data: Vec<ScaleT> = scale_data_f32.iter().map(|&x| ScaleT::from_f32(x)).collect();// Calculate expected outputlet expected_output_f32 = compute_expected_rms_norm(@@ -354,26 +287,17 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(// Create Metal bufferslet input_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&input_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&input_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let scale_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&scale_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buffer = mtl_context.device- .new_buffer(- model_dim as usize * OutputT::size_of(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(model_dim as usize * OutputT::size_of(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");// Create RMS norm kernel@@ -388,14 +312,9 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(.expect("Failed to create RMS norm kernel");// Create command buffer and encode- let command_buffer_ref = mtl_context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let _ = kernel.encode(&compute_encoder,@@ -418,10 +337,8 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(// Read resultslet output_ptr = output_buffer.contents().as_ptr() as *const OutputT;- let output_data =- unsafe { std::slice::from_raw_parts(output_ptr, model_dim as usize) };- let output_data_f32: Vec<f32> =- output_data.iter().map(|&x| x.to_f32()).collect();+ let output_data = unsafe { std::slice::from_raw_parts(output_ptr, model_dim as usize) };+ let output_data_f32: Vec<f32> = output_data.iter().map(|&x| x.to_f32()).collect();println!("Types: Input={:?}, Scale={:?}, Output={:?}, Accum={:?}",@@ -432,11 +349,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(println!("Output (first 8): {:?}", &output_data_f32[..8]);println!("RMS value: {}",- f32::sqrt(- input_data_f32.iter().map(|&x| x * x).sum::<f32>()- / (model_dim as f32)- + epsilon- )+ f32::sqrt(input_data_f32.iter().map(|&x| x * x).sum::<f32>() / (model_dim as f32) + epsilon));// Verify results with tolerance for different precisions@@ -450,9 +363,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(1e-5};- for (i, (&expected, &actual)) in- expected_output_f32.iter().zip(output_data_f32.iter()).enumerate()- {+ for (i, (&expected, &actual)) in expected_output_f32.iter().zip(output_data_f32.iter()).enumerate() {let diff = (expected - actual).abs();assert!(diff < tolerance,@@ -494,33 +405,22 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(let input_data_f32: Vec<f32> = vec![1e-4, 2e-4, 3e-4, 4e-4]; // Scaled up for F16 precisionlet scale_data_f32: Vec<f32> = vec![2.0, 0.5, 1.5, 0.8]; // Non-unit scales- let input_data: Vec<InputT> =- input_data_f32.iter().map(|&x| InputT::from_f32(x)).collect();- let scale_data: Vec<ScaleT> =- scale_data_f32.iter().map(|&x| ScaleT::from_f32(x)).collect();+ let input_data: Vec<InputT> = input_data_f32.iter().map(|&x| InputT::from_f32(x)).collect();+ let scale_data: Vec<ScaleT> = scale_data_f32.iter().map(|&x| ScaleT::from_f32(x)).collect();let input_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&input_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&input_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let scale_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&scale_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buffer = mtl_context.device- .new_buffer(- 4 * OutputT::size_of(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(4 * OutputT::size_of(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let kernel = RMSNormKernel::new(@@ -533,14 +433,9 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>().expect("Failed to create RMS norm kernel");- let command_buffer_ref = mtl_context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let _ = kernel.encode(&compute_encoder,@@ -567,13 +462,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(// Check that all values are finite (no NaN or infinity)for (i, &value) in output_data.iter().enumerate() {let f32_value = value.to_f32();- assert!(- f32_value.is_finite(),- "Output at index {} is not finite: {} (type {:?})",- i,- f32_value,- output_type- );+ assert!(f32_value.is_finite(), "Output at index {} is not finite: {} (type {:?})", i, f32_value, output_type);}println!(@@ -585,237 +474,112 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(// All F32 accumulation tests#[test]fn test_rms_norm_f32_f32_f32_f32() {- test_rms_norm_basic_typed::<f32, f32, f32>(- DataType::F32,- DataType::F32,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_f32_f16_f32_f32() {- test_rms_norm_basic_typed::<f32, f16, f32>(- DataType::F32,- DataType::F16,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<f32, f16, f32>(DataType::F32, DataType::F16, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_f32_f32_f16_f32() {- test_rms_norm_basic_typed::<f32, f32, f16>(- DataType::F32,- DataType::F32,- DataType::F16,- DataType::F32,- );+ test_rms_norm_basic_typed::<f32, f32, f16>(DataType::F32, DataType::F32, DataType::F16, DataType::F32);}#[test]fn test_rms_norm_f32_f16_f16_f32() {- test_rms_norm_basic_typed::<f32, f16, f16>(- DataType::F32,- DataType::F16,- DataType::F16,- DataType::F32,- );+ test_rms_norm_basic_typed::<f32, f16, f16>(DataType::F32, DataType::F16, DataType::F16, DataType::F32);}#[test]fn test_rms_norm_f16_f32_f32_f32() {- test_rms_norm_basic_typed::<f16, f32, f32>(- DataType::F16,- DataType::F32,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<f16, f32, f32>(DataType::F16, DataType::F32, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_f16_f16_f32_f32() {- test_rms_norm_basic_typed::<f16, f16, f32>(- DataType::F16,- DataType::F16,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<f16, f16, f32>(DataType::F16, DataType::F16, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_f16_f32_f16_f32() {- test_rms_norm_basic_typed::<f16, f32, f16>(- DataType::F16,- DataType::F32,- DataType::F16,- DataType::F32,- );+ test_rms_norm_basic_typed::<f16, f32, f16>(DataType::F16, DataType::F32, DataType::F16, DataType::F32);}#[test]fn test_rms_norm_f16_f16_f16_f32() {- test_rms_norm_basic_typed::<f16, f16, f16>(- DataType::F16,- DataType::F16,- DataType::F16,- DataType::F32,- );+ test_rms_norm_basic_typed::<f16, f16, f16>(DataType::F16, DataType::F16, DataType::F16, DataType::F32);}// F16 accumulation tests#[test]fn test_rms_norm_f32_f32_f32_f16() {- test_rms_norm_basic_typed::<f32, f32, f32>(- DataType::F32,- DataType::F32,- DataType::F32,- DataType::F16,- );+ test_rms_norm_basic_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F16);}#[test]fn test_rms_norm_f32_f16_f32_f16() {- test_rms_norm_basic_typed::<f32, f16, f32>(- DataType::F32,- DataType::F16,- DataType::F32,- DataType::F16,- );+ test_rms_norm_basic_typed::<f32, f16, f32>(DataType::F32, DataType::F16, DataType::F32, DataType::F16);}#[test]fn test_rms_norm_f32_f32_f16_f16() {- test_rms_norm_basic_typed::<f32, f32, f16>(- DataType::F32,- DataType::F32,- DataType::F16,- DataType::F16,- );+ test_rms_norm_basic_typed::<f32, f32, f16>(DataType::F32, DataType::F32, DataType::F16, DataType::F16);}#[test]fn test_rms_norm_f32_f16_f16_f16() {- test_rms_norm_basic_typed::<f32, f16, f16>(- DataType::F32,- DataType::F16,- DataType::F16,- DataType::F16,- );+ test_rms_norm_basic_typed::<f32, f16, f16>(DataType::F32, DataType::F16, DataType::F16, DataType::F16);}#[test]fn test_rms_norm_f16_f32_f32_f16() {- test_rms_norm_basic_typed::<f16, f32, f32>(- DataType::F16,- DataType::F32,- DataType::F32,- DataType::F16,- );+ test_rms_norm_basic_typed::<f16, f32, f32>(DataType::F16, DataType::F32, DataType::F32, DataType::F16);}#[test]fn test_rms_norm_f16_f16_f32_f16() {- test_rms_norm_basic_typed::<f16, f16, f32>(- DataType::F16,- DataType::F16,- DataType::F32,- DataType::F16,- );+ test_rms_norm_basic_typed::<f16, f16, f32>(DataType::F16, DataType::F16, DataType::F32, DataType::F16);}#[test]fn test_rms_norm_f16_f32_f16_f16() {- test_rms_norm_basic_typed::<f16, f32, f16>(- DataType::F16,- DataType::F32,- DataType::F16,- DataType::F16,- );+ test_rms_norm_basic_typed::<f16, f32, f16>(DataType::F16, DataType::F32, DataType::F16, DataType::F16);}#[test]fn test_rms_norm_f16_f16_f16_f16() {- test_rms_norm_basic_typed::<f16, f16, f16>(- DataType::F16,- DataType::F16,- DataType::F16,- DataType::F16,- );+ test_rms_norm_basic_typed::<f16, f16, f16>(DataType::F16, DataType::F16, DataType::F16, DataType::F16);}// BFloat16 tests (using f32 storage)#[test]fn test_rms_norm_bf16_bf16_f32_f32() {- test_rms_norm_basic_typed::<bf16, bf16, f32>(- DataType::BF16,- DataType::BF16,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<bf16, bf16, f32>(DataType::BF16, DataType::BF16, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_bf16_f32_f32_f32() {- test_rms_norm_basic_typed::<bf16, f32, f32>(- DataType::BF16,- DataType::F32,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<bf16, f32, f32>(DataType::BF16, DataType::F32, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_bf16_f16_f32_f32() {- test_rms_norm_basic_typed::<bf16, f16, f32>(- DataType::BF16,- DataType::F16,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<bf16, f16, f32>(DataType::BF16, DataType::F16, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_f32_bf16_f32_f32() {- test_rms_norm_basic_typed::<f32, bf16, f32>(- DataType::F32,- DataType::BF16,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<f32, bf16, f32>(DataType::F32, DataType::BF16, DataType::F32, DataType::F32);}// Edge case tests for critical combinations#[test]fn test_rms_norm_edge_f32_f32_f32_f32() {- test_rms_norm_edge_cases_typed::<f32, f32, f32>(- DataType::F32,- DataType::F32,- DataType::F32,- DataType::F32,- );+ test_rms_norm_edge_cases_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_edge_f16_f16_f16_f16() {- test_rms_norm_edge_cases_typed::<f16, f16, f16>(- DataType::F16,- DataType::F16,- DataType::F16,- DataType::F16,- );+ test_rms_norm_edge_cases_typed::<f16, f16, f16>(DataType::F16, DataType::F16, DataType::F16, DataType::F16);}#[test]fn test_rms_norm_edge_f32_f16_f16_f32() {- test_rms_norm_edge_cases_typed::<f32, f16, f16>(- DataType::F32,- DataType::F16,- DataType::F16,- DataType::F32,- );+ test_rms_norm_edge_cases_typed::<f32, f16, f16>(DataType::F32, DataType::F16, DataType::F16, DataType::F32);}// Legacy test wrappers for compatibility#[test]fn test_rms_norm_kernel_basic() {- test_rms_norm_basic_typed::<f32, f32, f32>(- DataType::F32,- DataType::F32,- DataType::F32,- DataType::F32,- );+ test_rms_norm_basic_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F32);}#[test]fn test_rms_norm_edge_cases() {- test_rms_norm_edge_cases_typed::<f32, f32, f32>(- DataType::F32,- DataType::F32,- DataType::F32,- DataType::F32,- );+ test_rms_norm_edge_cases_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F32);}fn perf_rms_norm_with_size(@@ -863,18 +627,12 @@ fn perf_rms_norm_with_size(// ---- Create Metal buffers ----let input_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&input_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&input_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let scale_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&scale_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buffer = mtl_context@@ -886,14 +644,9 @@ fn perf_rms_norm_with_size(.expect("Failed to create buffer");// ---- Launch and time ----- let command_buffer_ref = mtl_context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let _ = kernel.encode(&compute_encoder,@@ -938,39 +691,23 @@ fn perf_rms_norm_with_size(// ---- Sanity check results ----let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_data = unsafe {- std::slice::from_raw_parts(- output_ptr,- (batch_size * model_dim) as usize,- )- };+ let output_data = unsafe { std::slice::from_raw_parts(output_ptr, (batch_size * model_dim) as usize) };// Sample check for large outputslet sample_size = std::cmp::min(1000, output_data.len());- for i in (0..output_data.len())- .step_by(std::cmp::max(1, output_data.len() / sample_size))- {- assert!(- output_data[i].is_finite(),- "Output at index {} is not finite: {}",- i,- output_data[i]- );+ for i in (0..output_data.len()).step_by(std::cmp::max(1, output_data.len() / sample_size)) {+ assert!(output_data[i].is_finite(), "Output at index {} is not finite: {}", i, output_data[i]);}// Check that normalization is working- let mean_abs =- output_data.iter().take(sample_size).map(|x| x.abs()).sum::<f32>()- / sample_size as f32;+ let mean_abs = output_data.iter().take(sample_size).map(|x| x.abs()).sum::<f32>() / sample_size as f32;assert!(mean_abs > 0.01 && mean_abs < 100.0,"Mean absolute value {} seems unreasonable - normalization may not be working",mean_abs);- println!(- "β RMS norm performance test passed with reasonable output values"- );+ println!("β RMS norm performance test passed with reasonable output values");}#[test]@@ -1024,8 +761,7 @@ fn qk_norm_test() {let k_offset = (num_q_heads * head_dim) as usize;for head in 0..num_kv_heads {for dim in 0..head_dim {- let idx =- batch_offset + k_offset + (head * head_dim + dim) as usize;+ let idx = batch_offset + k_offset + (head * head_dim + dim) as usize;qkv_data[idx] = 2.0 + (head as f32) * 0.1 + (dim as f32) * 0.01;}}@@ -1034,8 +770,7 @@ fn qk_norm_test() {let v_offset = ((num_q_heads + num_kv_heads) * head_dim) as usize;for head in 0..num_kv_heads {for dim in 0..head_dim {- let idx =- batch_offset + v_offset + (head * head_dim + dim) as usize;+ let idx = batch_offset + v_offset + (head * head_dim + dim) as usize;qkv_data[idx] = 3.0 + (head as f32) * 0.1 + (dim as f32) * 0.01;}}@@ -1046,26 +781,17 @@ fn qk_norm_test() {// Create bufferslet qkv_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&qkv_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&qkv_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let q_scales_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&scale_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let k_scales_buffer = mtl_context.device- .new_buffer_with_data(- bytemuck::cast_slice(&scale_data),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");// Create QK norm kernels@@ -1093,13 +819,8 @@ fn qk_norm_test() {// Test Q head normalization{- let command_buffer = mtl_context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let _ = q_kernel.encode_qk_norm(&compute_encoder,@@ -1108,7 +829,7 @@ fn qk_norm_test() {scales_buffer: &q_scales_buffer,qkv_output_buffer: &qkv_buffer,batch_size,- num_q_heads, // Now correctly passes actual head count+ num_q_heads, // Now correctly passes actual head countnum_kv_heads, // Now correctly passes actual head counthead_dim,epsilon,@@ -1123,12 +844,8 @@ fn qk_norm_test() {}// Verify Q heads were normalized (should have different values now)- let output_data = unsafe {- std::slice::from_raw_parts(- qkv_buffer.contents().as_ptr() as *const f32,- qkv_data.len(),- )- };+ let output_data =+ unsafe { std::slice::from_raw_parts(qkv_buffer.contents().as_ptr() as *const f32, qkv_data.len()) };// Check that Q heads were processed (values should be different from original)for head in 0..num_q_heads as usize {@@ -1164,13 +881,8 @@ fn qk_norm_test() {// Test K head normalization{- let command_buffer = mtl_context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let _ = k_kernel.encode_qk_norm(&compute_encoder,@@ -1179,7 +891,7 @@ fn qk_norm_test() {scales_buffer: &k_scales_buffer,qkv_output_buffer: &qkv_buffer,batch_size,- num_q_heads, // Now correctly passes actual head count+ num_q_heads, // Now correctly passes actual head countnum_kv_heads, // Now correctly passes actual head counthead_dim,epsilon,@@ -1194,12 +906,8 @@ fn qk_norm_test() {}// Verify K heads were normalized- let final_data = unsafe {- std::slice::from_raw_parts(- qkv_buffer.contents().as_ptr() as *const f32,- qkv_data.len(),- )- };+ let final_data =+ unsafe { std::slice::from_raw_parts(qkv_buffer.contents().as_ptr() as *const f32, qkv_data.len()) };// Check that K heads were processedfor head in 0..num_kv_heads as usize {diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex 2341f5594..14a27695b 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -1,9 +1,6 @@mod common;use bytemuck;-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,- MTLDeviceExt, MTLResourceOptions,-};+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use rand::{RngExt, seq::SliceRandom};// for Vec::shuffleuse uzu::{@@ -12,16 +9,14 @@ use uzu::{common::{Context,kernel::{- GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel,- TopPKernel,+ GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel, TopPKernel,sampling::{ArgmaxStrategy, SamplingKernel},},},metal::{KernelDataType, MTLContext, Metal,kernel::dsl::{- GumbelMetalKernel, MinPMetalKernel, TemperatureMetalKernel,- TopKMetalKernel, TopPMetalKernel,+ GumbelMetalKernel, MinPMetalKernel, TemperatureMetalKernel, TopKMetalKernel, TopPMetalKernel,},metal_extensions::CommandBufferTimingExt,},@@ -38,13 +33,9 @@ fn cpu_reference_top_p(top_p: f32,) -> Vec<f32> {// 1. softβmax (unnormalised)- let max_logit =- row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);- let mut softmax: Vec<(usize, f32)> = row_logits- .iter()- .enumerate()- .map(|(idx, &z)| (idx, (z - max_logit).exp()))- .collect();+ let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);+ let mut softmax: Vec<(usize, f32)> =+ row_logits.iter().enumerate().map(|(idx, &z)| (idx, (z - max_logit).exp())).collect();let total_mass: f32 = softmax.iter().map(|(_, p)| p).sum();// 2. sort descending@@ -74,8 +65,7 @@ fn cpu_reference_min_p(row_logits: &[f32],min_p: f32,) -> Vec<f32> {- let max_logit =- row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);+ let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);let threshold = max_logit + min_p.ln();row_logits@@ -102,14 +92,8 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {let batch_size = 2;let vocab_size = 4;- let kernel = SamplingKernel::<Metal>::new_with_strategy(- &context,- DataType::F32,- batch_size,- vocab_size,- strategy,- )- .expect("Failed to create argmax kernel");+ let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, batch_size, vocab_size, strategy)+ .expect("Failed to create argmax kernel");// Create test data: batch_size=2, vocab_size=4// First batch: [1.0, 3.0, 2.0, 0.5] -> should select index 1@@ -121,30 +105,19 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {let logits_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&test_logits),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buffer = context.device- .new_buffer(- batch_size * std::mem::size_of::<u32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();// Run sampling- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(&logits_buffer,@@ -168,19 +141,10 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {let result_ptr = output_buffer.contents().as_ptr() as *const u32;let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };- assert_eq!(- results[0], 1,- "First batch should select token 1 (highest logit: 3.0)"- );- assert_eq!(- results[1], 2,- "Second batch should select token 2 (highest logit: 2.5)"- );-- println!(- "β Argmax sampling test passed with {:?} strategy - selected tokens: {:?}",- strategy, results- );+ assert_eq!(results[0], 1, "First batch should select token 1 (highest logit: 3.0)");+ assert_eq!(results[1], 2, "Second batch should select token 2 (highest logit: 2.5)");++ println!("β Argmax sampling test passed with {:?} strategy - selected tokens: {:?}", strategy, results);}#[test]@@ -220,19 +184,13 @@ fn test_topp_sampling_from_prob_exact_match(let p = (k as f32) * 0.1;// Create kernel- let kernel = SamplingKernel::<Metal>::new(- &context,- DataType::F32,- batch_size,- vocab_size,- )- .expect("Failed to create sampling kernel");+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)+ .expect("Failed to create sampling kernel");// Build probability tablelet low_prob = (1.0 - p) / (vocab_size - k) as f32;let mut probs = vec![low_prob; batch_size * vocab_size];- let mut high_prob_token_sets: Vec<HashSet<usize>> =- Vec::with_capacity(batch_size);+ let mut high_prob_token_sets: Vec<HashSet<usize>> = Vec::with_capacity(batch_size);// Use std::sample equivalent (Fisher-Yates shuffle + take first k)let mut rng = StdRng::seed_from_u64(42); // Deterministic for reproducibility@@ -241,8 +199,7 @@ fn test_topp_sampling_from_prob_exact_match(for b in 0..batch_size {// Equivalent to std::sample - shuffle and take first kall_token_ids.shuffle(&mut rng);- let high_prob_tokens: HashSet<usize> =- all_token_ids[..k].iter().cloned().collect();+ let high_prob_tokens: HashSet<usize> = all_token_ids[..k].iter().cloned().collect();// Set selected tokens to high probabilityfor &token_id in &high_prob_tokens {@@ -266,10 +223,7 @@ fn test_topp_sampling_from_prob_exact_match(// GPU bufferslet output_buf = context.device- .new_buffer(- batch_size * std::mem::size_of::<u32>(),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(batch_size * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");// Counter for samples@@ -281,30 +235,18 @@ fn test_topp_sampling_from_prob_exact_match(// Create fresh logits buffer since kernel mutates in-placelet logits_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let seeds: Vec<u64> =- vec![TEST_SAMPLING_SEED + draw as u64; batch_size];+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];let seeds_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let cb_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(&logits_buf,@@ -329,14 +271,10 @@ fn test_topp_sampling_from_prob_exact_match(cb_ref.wait_until_completed();let ptr = output_buf.contents().as_ptr() as *const u32;- let sampled_ids =- unsafe { std::slice::from_raw_parts(ptr, batch_size) };+ let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };for (i, &sampled_id) in sampled_ids.iter().enumerate() {- assert!(- (sampled_id as usize) < vocab_size,- "Sampled token out of range"- );+ assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");counter[i * vocab_size + sampled_id as usize] += 1;}}@@ -355,10 +293,7 @@ fn test_topp_sampling_from_prob_exact_match(}}- println!(- "batch_size: {}, p: {:.1}, vocab_size: {}, accuracy test passed.",- batch_size, p, vocab_size- );+ println!("batch_size: {}, p: {:.1}, vocab_size: {}, accuracy test passed.", batch_size, p, vocab_size);}#[test]@@ -393,8 +328,7 @@ fn test_topp_sampling_statistical_large() {// ===== 3. Build kernel =====let kernel =- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB)- .expect("Failed to create sampling kernel");+ SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");// ===== 4. Generate reproducible random logits =====let mut rng = StdRng::seed_from_u64(42);@@ -415,10 +349,7 @@ fn test_topp_sampling_statistical_large() {// ===== 6. Allocate output & counters =====let output_buf = context.device- .new_buffer(- BATCH * std::mem::size_of::<u32>(),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let mut counters = vec![0u32; BATCH * VOCAB];@@ -427,29 +358,18 @@ fn test_topp_sampling_statistical_large() {// Create fresh logits buffer since kernel mutates in-placelet logits_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];let seeds_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let cb_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -490,25 +410,15 @@ fn test_topp_sampling_statistical_large() {if expected < 1e-12 {continue;} // ignore tiny- let observed =- (counters[b * VOCAB + j] as f32) / (NUM_DRAWS as f32);+ let observed = (counters[b * VOCAB + j] as f32) / (NUM_DRAWS as f32);if observed > 0.0 {kl += observed * (observed.ln() - expected.ln());}}- assert!(- kl < TOLERANCE_KL,- "Row {} KL {:.4} exceeded tolerance {:.3}",- b,- kl,- TOLERANCE_KL- );+ assert!(kl < TOLERANCE_KL, "Row {} KL {:.4} exceeded tolerance {:.3}", b, kl, TOLERANCE_KL);}- println!(- "β Large statistical Topβp test passed (KL < {:.3})",- TOLERANCE_KL- );+ println!("β Large statistical Topβp test passed (KL < {:.3})", TOLERANCE_KL);}#[test]@@ -533,8 +443,7 @@ fn perf_topp_128k_vocab() {// ---- Kernel ----let kernel =- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB)- .expect("Failed to create sampling kernel");+ SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");// ---- Build random logits ----let mut rng = StdRng::seed_from_u64(123);@@ -545,38 +454,24 @@ fn perf_topp_128k_vocab() {let logits_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];let seeds_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buf = context.device- .new_buffer(- BATCH * std::mem::size_of::<u32>(),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");// ---- Launch once and time ----- let cb_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -651,14 +546,8 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {const VOCAB: usize = 128000; // 128K// ---- Kernel ----- let kernel = SamplingKernel::<Metal>::new_with_strategy(- &context,- DataType::F32,- BATCH,- VOCAB,- strategy,- )- .expect("Failed to create Argmax kernel");+ let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, BATCH, VOCAB, strategy)+ .expect("Failed to create Argmax kernel");// ---- Build random logits ----let mut rng = StdRng::seed_from_u64(123);@@ -669,38 +558,24 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {let logits_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];let seeds_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buf = context.device- .new_buffer(- BATCH * std::mem::size_of::<u32>(),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");// ---- Launch once and time ----- let cb_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -756,8 +631,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {let selected_logit = row_logits[selected_token];// Find the actual maximum logit- let max_logit =- row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);+ let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);assert!((selected_logit - max_logit).abs() < 1e-5,@@ -785,13 +659,8 @@ fn test_categorical_sampling() {let batch_size = 2;let vocab_size = 4;- let kernel = SamplingKernel::<Metal>::new(- &context,- DataType::F32,- batch_size,- vocab_size,- )- .expect("Failed to create sampling kernel");+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)+ .expect("Failed to create sampling kernel");// Create test data with different probability distributions// First batch: [1.0, 2.0, 1.5, 0.5] -> softmax: [0.134, 0.366, 0.201, 0.082]@@ -803,10 +672,7 @@ fn test_categorical_sampling() {let output_buffer = context.device- .new_buffer(- batch_size * std::mem::size_of::<u32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");// Run sampling multiple times to check distribution@@ -817,30 +683,18 @@ fn test_categorical_sampling() {// Create fresh logits buffer since kernel mutates in-placelet logits_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&test_logits),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let seeds_vec =- vec![TEST_SAMPLING_SEED + sample_idx as u64; batch_size];+ let seeds_vec = vec![TEST_SAMPLING_SEED + sample_idx as u64; batch_size];let seeds_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds_vec),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -867,8 +721,7 @@ fn test_categorical_sampling() {command_buffer_ref.wait_until_completed();let result_ptr = output_buffer.contents().as_ptr() as *const u32;- let results =- unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };+ let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };for (batch_idx, &token) in results.iter().enumerate() {assert!(@@ -883,8 +736,7 @@ fn test_categorical_sampling() {// Check that all batches produced valid outputsfor batch_idx in 0..batch_size {- let batch_counts =- &counts[batch_idx * vocab_size..(batch_idx + 1) * vocab_size];+ let batch_counts = &counts[batch_idx * vocab_size..(batch_idx + 1) * vocab_size];let total_samples: usize = batch_counts.iter().sum();assert_eq!(total_samples, num_samples,@@ -909,10 +761,7 @@ fn test_categorical_sampling() {}}- println!(- "β Categorical sampling test passed - sampled {} times per batch",- num_samples- );+ println!("β Categorical sampling test passed - sampled {} times per batch", num_samples);println!(" Batch 0 counts: {:?}", &counts[0..vocab_size]);println!(" Batch 1 counts: {:?}", &counts[vocab_size..2 * vocab_size]);}@@ -935,8 +784,7 @@ fn test_categorical_sampling_statistical() {const TOLERANCE: f32 = 0.05; // 5% tolerancelet kernel =- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB)- .expect("Failed to create sampling kernel");+ SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");// Generate random logitslet mut rng = StdRng::seed_from_u64(42);@@ -949,8 +797,7 @@ fn test_categorical_sampling_statistical() {let mut expected_probs = vec![0.0f32; BATCH * VOCAB];for batch_idx in 0..BATCH {let batch_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];- let max_logit =- batch_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);+ let max_logit = batch_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);let mut sum_exp = 0.0f32;for j in 0..VOCAB {@@ -967,10 +814,7 @@ fn test_categorical_sampling_statistical() {let output_buffer = context.device- .new_buffer(- BATCH * std::mem::size_of::<u32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(BATCH * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let mut counts = vec![0; BATCH * VOCAB];@@ -980,29 +824,18 @@ fn test_categorical_sampling_statistical() {// Create fresh logits buffer since kernel mutates in-placelet logits_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let seeds_vec = vec![TEST_SAMPLING_SEED + sample_idx as u64; BATCH];let seeds_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds_vec),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -1039,8 +872,7 @@ fn test_categorical_sampling_statistical() {// Check statistical accuracyfor batch_idx in 0..BATCH {for token_idx in 0..VOCAB {- let observed_freq = counts[batch_idx * VOCAB + token_idx] as f32- / NUM_SAMPLES as f32;+ let observed_freq = counts[batch_idx * VOCAB + token_idx] as f32 / NUM_SAMPLES as f32;let expected_freq = expected_probs[batch_idx * VOCAB + token_idx];if expected_freq > 0.01 {@@ -1060,10 +892,7 @@ fn test_categorical_sampling_statistical() {}}- println!(- "β Categorical statistical test passed (tolerance: {:.1}%)",- TOLERANCE * 100.0- );+ println!("β Categorical statistical test passed (tolerance: {:.1}%)", TOLERANCE * 100.0);}#[test]@@ -1084,8 +913,7 @@ fn perf_categorical_128k_vocab() {const VOCAB: usize = 128000; // 128Klet kernel =- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB)- .expect("Failed to create sampling kernel");+ SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");// Build random logitslet mut rng = StdRng::seed_from_u64(123);@@ -1096,38 +924,24 @@ fn perf_categorical_128k_vocab() {let logits_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];let seeds_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let output_buf = context.device- .new_buffer(- BATCH * std::mem::size_of::<u32>(),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");// Launch and time- let cb_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -1198,59 +1012,35 @@ fn test_temperature_gpu_cpu_match() {const RTOL: f32 = 1e-6;const ATOL: f32 = 1e-6;- let kernel =- TemperatureMetalKernel::new(&context, KernelDataType::Float32.into())- .expect("Failed to create temperature kernel");+ let kernel = TemperatureMetalKernel::new(&context, KernelDataType::Float32.into())+ .expect("Failed to create temperature kernel");- let logits: Vec<f32> = (0..BATCH * VOCAB)- .map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01)- .collect();+ let logits: Vec<f32> = (0..BATCH * VOCAB).map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01).collect();let logits_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let processed_buffer = context.device- .new_buffer(- logits.len() * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");- kernel.encode(- &logits_buffer,- &processed_buffer,- BATCH as u32,- VOCAB as u32,- TEMPERATURE,- &compute_encoder,- );+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();command_buffer_ref.wait_until_completed();let gpu_ptr = processed_buffer.contents().as_ptr() as *const f32;- let gpu_results =- unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };+ let gpu_results = unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };- for (idx, (&logit, &processed)) in- logits.iter().zip(gpu_results.iter()).enumerate()- {+ for (idx, (&logit, &processed)) in logits.iter().zip(gpu_results.iter()).enumerate() {let expected = logit / TEMPERATURE;let abs_diff = (expected - processed).abs();let tolerance = ATOL + RTOL * expected.abs();@@ -1265,10 +1055,7 @@ fn test_temperature_gpu_cpu_match() {);}- println!(- "β Temperature processor gpu cpu match (temp={}, rtol={}, atol={})",- TEMPERATURE, RTOL, ATOL- );+ println!("β Temperature processor gpu cpu match (temp={}, rtol={}, atol={})", TEMPERATURE, RTOL, ATOL);}#[test]@@ -1287,8 +1074,7 @@ fn test_topk_gpu_cpu_match() {const VOCAB: usize = 1024;const TOPK: u32 = 16;- let kernel = TopKMetalKernel::new(&context, KernelDataType::Float32.into())- .expect("Failed to create topk kernel");+ let kernel = TopKMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create topk kernel");let mut rng = StdRng::seed_from_u64(42);let mut logits = vec![0.0f32; BATCH * VOCAB];@@ -1298,58 +1084,37 @@ fn test_topk_gpu_cpu_match() {let logits_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let processed_buffer = context.device- .new_buffer(- logits.len() * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");- kernel.encode(- &logits_buffer,- &processed_buffer,- BATCH as u32,- VOCAB as u32,- TOPK,- &compute_encoder,- );+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();command_buffer_ref.wait_until_completed();let results_ptr = processed_buffer.contents().as_ptr() as *const f32;- let all_results =- unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };+ let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };for batch_idx in 0..BATCH {let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];- let mut cpu_sorted_logits: Vec<(usize, f32)> =- cpu_logits.iter().copied().enumerate().collect();+ let mut cpu_sorted_logits: Vec<(usize, f32)> = cpu_logits.iter().copied().enumerate().collect();cpu_sorted_logits.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));let mut cpu_processed_logits: Vec<f32> = vec![f32::NEG_INFINITY; VOCAB];for (idx, val) in cpu_sorted_logits.into_iter().take(TOPK as usize) {cpu_processed_logits[idx] = val;}- let gpu_processed_logits =- &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];+ let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];assert_eq!(cpu_processed_logits, gpu_processed_logits);}@@ -1373,8 +1138,7 @@ fn test_topp_gpu_cpu_match() {const VOCAB: usize = 1024;const TOPP: f32 = 0.9;- let kernel = TopPMetalKernel::new(&context, KernelDataType::Float32.into())- .expect("Failed to create topp kernel");+ let kernel = TopPMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create topp kernel");let mut rng = StdRng::seed_from_u64(42);let mut logits = vec![0.0f32; BATCH * VOCAB];@@ -1384,45 +1148,26 @@ fn test_topp_gpu_cpu_match() {let logits_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let processed_buffer = context.device- .new_buffer(- logits.len() * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");- kernel.encode(- &logits_buffer,- &processed_buffer,- BATCH as u32,- VOCAB as u32,- TOPP,- &compute_encoder,- );+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();command_buffer_ref.wait_until_completed();let results_ptr = processed_buffer.contents().as_ptr() as *const f32;- let all_results =- unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };+ let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };for batch_idx in 0..BATCH {let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];@@ -1438,8 +1183,7 @@ fn test_topp_gpu_cpu_match() {}).collect::<Vec<f32>>();- let gpu_processed_logits =- &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];+ let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];assert_eq!(&cpu_processed_logits, gpu_processed_logits);}@@ -1463,8 +1207,7 @@ fn test_minp_gpu_cpu_match() {const VOCAB: usize = 1024;const MINP: f32 = 0.1;- let kernel = MinPMetalKernel::new(&context, KernelDataType::Float32.into())- .expect("Failed to create minp kernel");+ let kernel = MinPMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create minp kernel");let mut rng = StdRng::seed_from_u64(42);let mut logits = vec![0.0f32; BATCH * VOCAB];@@ -1474,52 +1217,32 @@ fn test_minp_gpu_cpu_match() {let logits_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let processed_buffer = context.device- .new_buffer(- logits.len() * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");- kernel.encode(- &logits_buffer,- &processed_buffer,- BATCH as u32,- VOCAB as u32,- MINP,- &compute_encoder,- );+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, MINP, &compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();command_buffer_ref.wait_until_completed();let results_ptr = processed_buffer.contents().as_ptr() as *const f32;- let all_results =- unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };+ let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };for batch_idx in 0..BATCH {let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];let cpu_processed_logits = cpu_reference_min_p(cpu_logits, MINP);- let gpu_processed_logits =- &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];+ let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];assert_eq!(&cpu_processed_logits, gpu_processed_logits);}@@ -1544,13 +1267,8 @@ fn test_minp_sampling_exact_match(},};- let kernel = SamplingKernel::<Metal>::new(- &context,- DataType::F32,- batch_size,- vocab_size,- )- .expect("Failed to create sampling kernel");+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)+ .expect("Failed to create sampling kernel");// Build logits where some tokens have high probability and others have low// For min_p filtering, tokens with probability < min_p * max_prob are masked@@ -1560,16 +1278,14 @@ fn test_minp_sampling_exact_match(let low_logit = -50.0f32; // exp(-50) β 0let mut logits = vec![low_logit; batch_size * vocab_size];- let mut high_prob_token_sets: Vec<HashSet<usize>> =- Vec::with_capacity(batch_size);+ let mut high_prob_token_sets: Vec<HashSet<usize>> = Vec::with_capacity(batch_size);let mut rng = StdRng::seed_from_u64(42);let mut all_token_ids: Vec<usize> = (0..vocab_size).collect();for b in 0..batch_size {all_token_ids.shuffle(&mut rng);- let high_prob_tokens: HashSet<usize> =- all_token_ids[..k].iter().cloned().collect();+ let high_prob_tokens: HashSet<usize> = all_token_ids[..k].iter().cloned().collect();for &token_id in &high_prob_tokens {logits[b * vocab_size + token_id] = high_logit;@@ -1579,10 +1295,7 @@ fn test_minp_sampling_exact_match(let output_buf = context.device- .new_buffer(- batch_size * std::mem::size_of::<u32>(),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(batch_size * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let num_samples = 1000;@@ -1592,30 +1305,18 @@ fn test_minp_sampling_exact_match(// Create fresh logits buffer since kernel mutates in-placelet logits_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let seeds: Vec<u64> =- vec![TEST_SAMPLING_SEED + draw as u64; batch_size];+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];let seeds_buf = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds),- metal::MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let cb_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(&logits_buf,@@ -1640,14 +1341,10 @@ fn test_minp_sampling_exact_match(cb_ref.wait_until_completed();let ptr = output_buf.contents().as_ptr() as *const u32;- let sampled_ids =- unsafe { std::slice::from_raw_parts(ptr, batch_size) };+ let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };for (i, &sampled_id) in sampled_ids.iter().enumerate() {- assert!(- (sampled_id as usize) < vocab_size,- "Sampled token out of range"- );+ assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");counter[i * vocab_size + sampled_id as usize] += 1;}}@@ -1666,10 +1363,7 @@ fn test_minp_sampling_exact_match(}}- println!(- "batch_size: {}, min_p: {:.2}, vocab_size: {}, accuracy test passed.",- batch_size, min_p, vocab_size- );+ println!("batch_size: {}, min_p: {:.2}, vocab_size: {}, accuracy test passed.", batch_size, min_p, vocab_size);}#[test]@@ -1698,68 +1392,43 @@ fn test_gumbel_gpu_cpu_match() {const ATOL: f32 = 1e-6;let kernel =- GumbelMetalKernel::new(&context, KernelDataType::Float32.into())- .expect("Failed to create gumbel kernel");+ GumbelMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create gumbel kernel");let logits = vec![0.0f32; BATCH * VOCAB];let seeds: Vec<u64> = (0_u64..BATCH as u64).collect();let logits_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&logits),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let seeds_buffer = context.device- .new_buffer_with_data(- bytemuck::cast_slice(&seeds),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let gumbel_logits_buffer = context.device- .new_buffer(- BATCH * VOCAB * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(BATCH * VOCAB * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");- kernel.encode(- &logits_buffer,- &seeds_buffer,- &gumbel_logits_buffer,- BATCH as u32,- VOCAB as u32,- &compute_encoder,- );+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &seeds_buffer, &gumbel_logits_buffer, BATCH as u32, VOCAB as u32, &compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();command_buffer_ref.wait_until_completed();let result_ptr = gumbel_logits_buffer.contents().as_ptr() as *const f32;- let all_results =- unsafe { std::slice::from_raw_parts(result_ptr, BATCH * VOCAB) };+ let all_results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH * VOCAB) };for (batch_idx, batch_seed) in seeds.iter().copied().enumerate() {let results = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];- for (logit_idx, gpu_logit_value) in results.iter().copied().enumerate()- {- let cpu_logit_value =- gumbel_float(batch_seed, revidx(logit_idx as u32));+ for (logit_idx, gpu_logit_value) in results.iter().copied().enumerate() {+ let cpu_logit_value = gumbel_float(batch_seed, revidx(logit_idx as u32));let abs_diff = (cpu_logit_value - gpu_logit_value).abs();let tolerance = ATOL + RTOL * cpu_logit_value.abs();assert!(@@ -1769,9 +1438,5 @@ fn test_gumbel_gpu_cpu_match() {}}- println!(- "β Gumbel cpu gpu match test passed (rtol: {:.1}%, atol: {:.1e})",- RTOL * 100.0,- ATOL- );+ println!("β Gumbel cpu gpu match test passed (rtol: {:.1}%, atol: {:.1e})", RTOL * 100.0, ATOL);}diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rsindex 5659e7364..f6d117228 100644--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs@@ -4,8 +4,7 @@ use std::mem::size_of;use bytemuck;use metal::{- MTLBlitCommandEncoder, MTLCommandBuffer, MTLCommandEncoder,- MTLCommandQueue, MTLDeviceExt, MTLResourceOptions,+ MTLBlitCommandEncoder, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions,};use uzu::backends::{common::Context,@@ -19,8 +18,7 @@ use uzu::backends::{};use uzu::config::Activation;-const STORAGE_MODE: MTLResourceOptions =- MTLResourceOptions::STORAGE_MODE_SHARED;+const STORAGE_MODE: MTLResourceOptions = MTLResourceOptions::STORAGE_MODE_SHARED;fn silu_scalar(x: f32) -> f32 {let y = 1.0 / (1.0 + (-x).exp());@@ -40,11 +38,7 @@ fn write_buffer(data: &[f32],) {unsafe {- std::ptr::copy_nonoverlapping(- data.as_ptr(),- buf.contents().as_ptr() as *mut f32,- data.len(),- );+ std::ptr::copy_nonoverlapping(data.as_ptr(), buf.contents().as_ptr() as *mut f32, data.len());}}@@ -54,22 +48,14 @@ fn read_buffer() -> Vec<f32> {let mut out = vec![0.0f32; len];unsafe {- std::ptr::copy_nonoverlapping(- buf.contents().as_ptr() as *const f32,- out.as_mut_ptr(),- len,- );+ std::ptr::copy_nonoverlapping(buf.contents().as_ptr() as *const f32, out.as_mut_ptr(), len);}out}fn zero_buffer(buf: &ProtocolObject<dyn MTLBuffer>) {unsafe {- std::ptr::write_bytes(- buf.contents().as_ptr(),- 0,- buf.length() as usize,- );+ std::ptr::write_bytes(buf.contents().as_ptr(), 0, buf.length() as usize);}}@@ -101,8 +87,7 @@ fn ssd_prefill_cpu_reference(for dh in 0..head_dim {let state_base = h * state_strides[0] + dh * state_strides[1];for token in 0..suffix_len {- let x_idx =- token * x_strides[0] + h * x_strides[1] + dh * x_strides[2];+ let x_idx = token * x_strides[0] + h * x_strides[1] + dh * x_strides[2];let dt_idx = token * dt_strides[0] + h * dt_strides[1];let cb_base = token * cb_strides[0] + group_idx * cb_strides[1];@@ -119,8 +104,7 @@ fn ssd_prefill_cpu_reference(let cb_idx = cb_base + s * cb_strides[2];let b_coeff = b_data[cb_idx];let c_coeff = c_data[cb_idx];- let new_state = decay_val * state[state_idx]- + dt_scaled_input * b_coeff;+ let new_state = decay_val * state[state_idx] + dt_scaled_input * b_coeff;state[state_idx] = new_state;acc += new_state * c_coeff;}@@ -171,20 +155,13 @@ impl SSDPrefillFixture {let total_cb = suffix_len * group_count * state_dim;let total_state = num_heads * head_dim * state_dim;- let x_data: Vec<f32> =- (0..total_x).map(|i| ((i % 17) as f32) * 0.01 - 0.05).collect();- let dt_data: Vec<f32> =- (0..total_dt).map(|i| ((i % 13) as f32) * 0.2 - 1.5).collect();- let b_data: Vec<f32> =- (0..total_cb).map(|i| ((i % 11) as f32) * 0.02 - 0.05).collect();- let c_data: Vec<f32> =- (0..total_cb).map(|i| ((i % 19) as f32) * 0.01 - 0.02).collect();- let d_data: Vec<f32> =- (0..num_heads).map(|i| ((i % 3) as f32) * 0.05 - 0.05).collect();- let z_data: Vec<f32> =- (0..total_x).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();- let state_init: Vec<f32> =- (0..total_state).map(|i| ((i % 29) as f32) * 0.03 - 0.4).collect();+ let x_data: Vec<f32> = (0..total_x).map(|i| ((i % 17) as f32) * 0.01 - 0.05).collect();+ let dt_data: Vec<f32> = (0..total_dt).map(|i| ((i % 13) as f32) * 0.2 - 1.5).collect();+ let b_data: Vec<f32> = (0..total_cb).map(|i| ((i % 11) as f32) * 0.02 - 0.05).collect();+ let c_data: Vec<f32> = (0..total_cb).map(|i| ((i % 19) as f32) * 0.01 - 0.02).collect();+ let d_data: Vec<f32> = (0..num_heads).map(|i| ((i % 3) as f32) * 0.05 - 0.05).collect();+ let z_data: Vec<f32> = (0..total_x).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();+ let state_init: Vec<f32> = (0..total_state).map(|i| ((i % 29) as f32) * 0.03 - 0.4).collect();let x_strides = [num_heads * head_dim, head_dim, 1usize];let dt_strides = [num_heads, 1usize];@@ -224,47 +201,25 @@ fn run_prefill_kernel_mode() -> (Vec<f32>, Vec<f32>, Option<(Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>)>) {let device = &ctx.device;let x_buf = device- .new_buffer_with_data(- bytemuck::cast_slice(&fixture.x_data),- STORAGE_MODE,- )+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.x_data), STORAGE_MODE).expect("Failed to create buffer");let dt_buf = device- .new_buffer_with_data(- bytemuck::cast_slice(&fixture.dt_data),- STORAGE_MODE,- )+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.dt_data), STORAGE_MODE).expect("Failed to create buffer");let b_buf = device- .new_buffer_with_data(- bytemuck::cast_slice(&fixture.b_data),- STORAGE_MODE,- )+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.b_data), STORAGE_MODE).expect("Failed to create buffer");let c_buf = device- .new_buffer_with_data(- bytemuck::cast_slice(&fixture.c_data),- STORAGE_MODE,- )+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.c_data), STORAGE_MODE).expect("Failed to create buffer");let d_buf = device- .new_buffer_with_data(- bytemuck::cast_slice(&fixture.d_data),- STORAGE_MODE,- )+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.d_data), STORAGE_MODE).expect("Failed to create buffer");let z_buf = device- .new_buffer_with_data(- bytemuck::cast_slice(&fixture.z_data),- STORAGE_MODE,- )- .expect("Failed to create buffer");- let state_buf = device- .new_buffer(fixture.total_state * 4, STORAGE_MODE)- .expect("Failed to create buffer");- let y_buf = device- .new_buffer(fixture.total_x * 4, STORAGE_MODE)+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.z_data), STORAGE_MODE).expect("Failed to create buffer");+ let state_buf = device.new_buffer(fixture.total_state * 4, STORAGE_MODE).expect("Failed to create buffer");+ let y_buf = device.new_buffer(fixture.total_x * 4, STORAGE_MODE).expect("Failed to create buffer");write_buffer(&state_buf, &fixture.state_init);zero_buffer(&y_buf);@@ -289,13 +244,8 @@ fn run_prefill_kernel_mode(head_dim: fixture.head_dim,};- let command_buffer = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&encoder, args, mode);encoder.end_encoding();command_buffer.commit();@@ -326,31 +276,19 @@ fn run_conv_scan_once(let total_state = channels * tap_count;let y_buf = if alias_io {- device- .new_buffer_with_data(bytemuck::cast_slice(x_data), STORAGE_MODE)- .expect("Failed to create buffer")+ device.new_buffer_with_data(bytemuck::cast_slice(x_data), STORAGE_MODE).expect("Failed to create buffer")} else {- let buf = device- .new_buffer(total_x * size_of::<f32>(), STORAGE_MODE)- .expect("Failed to create buffer");+ let buf = device.new_buffer(total_x * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");zero_buffer(&buf);buf};- let w_buf = device- .new_buffer_with_data(bytemuck::cast_slice(w_data), STORAGE_MODE)- .expect("Failed to create buffer");- let b_buf = device- .new_buffer_with_data(bytemuck::cast_slice(b_data), STORAGE_MODE)- .expect("Failed to create buffer");- let state_buf = device- .new_buffer(total_state * size_of::<f32>(), STORAGE_MODE)- .expect("Failed to create buffer");+ let w_buf =+ device.new_buffer_with_data(bytemuck::cast_slice(w_data), STORAGE_MODE).expect("Failed to create buffer");+ let b_buf =+ device.new_buffer_with_data(bytemuck::cast_slice(b_data), STORAGE_MODE).expect("Failed to create buffer");+ let state_buf = device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");let scratch_buf = if use_scratch && tap_count > 0 {- Some(- device- .new_buffer(total_state * size_of::<f32>(), STORAGE_MODE)- .expect("Failed to create buffer"),- )+ Some(device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer"))} else {None};@@ -361,9 +299,8 @@ fn run_conv_scan_once(}let padded_len = tap_count + suffix_len;- let padded_buf = device- .new_buffer(padded_len * channels * size_of::<f32>(), STORAGE_MODE)- .expect("Failed to create buffer");+ let padded_buf =+ device.new_buffer(padded_len * channels * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");{let mut host = vec![0.0f32; padded_len * channels];for tap in 0..tap_count {@@ -373,8 +310,7 @@ fn run_conv_scan_once(}for token in 0..suffix_len {for ch in 0..channels {- host[(tap_count + token) * channels + ch] =- x_data[token * channels + ch];+ host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];}}write_buffer(&padded_buf, &host);@@ -397,22 +333,15 @@ fn run_conv_scan_once(proj_dim: 0,};- let command_buffer = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");- let encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&encoder, args).unwrap();encoder.end_encoding();if let Some(ref scratch) = scratch_buf {let bytes = channels * tap_count * size_of::<f32>();if bytes > 0 {- let blit = command_buffer- .new_blit_command_encoder()- .expect("Failed to create blit encoder");+ let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit encoder");blit.copy_buffer_to_buffer(scratch, 0, &state_buf, 0, bytes);blit.end_encoding();}@@ -434,10 +363,8 @@ fn assert_deterministic_for_mode(mode: SSDPrefillMode) {let kernel = SSDPrefillKernels::new(&ctx, KernelDataType::Float32).unwrap();let fixture = SSDPrefillFixture::new();- let (y_a, state_a, _) =- run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);- let (y_b, state_b, _) =- run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);+ let (y_a, state_a, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);+ let (y_b, state_b, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);assert_eq!(y_a, y_b, "Prefill outputs differ in {:?} mode", mode);assert_eq!(state_a, state_b, "Prefill states differ in {:?} mode", mode);@@ -470,8 +397,7 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {fixture.state_strides,);- let (y_gpu, state_gpu, _) =- run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);+ let (y_gpu, state_gpu, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);let tolerance = 5e-5f32;let mut max_y_diff = 0.0f32;@@ -536,9 +462,7 @@ fn conv1d_scan_is_deterministic() {return;};let activation = Activation::Identity;- let kernel =- Conv1dScanKernel::new(&ctx, KernelDataType::Float32, &activation)- .unwrap();+ let kernel = Conv1dScanKernel::new(&ctx, KernelDataType::Float32, &activation).unwrap();let suffix_len = 192usize;let channels = 8usize;@@ -548,15 +472,10 @@ fn conv1d_scan_is_deterministic() {let total_x = suffix_len * channels;let total_state = channels * tap_count;- let x_data: Vec<f32> =- (0..total_x).map(|i| ((i % 31) as f32) * 0.02 - 0.3).collect();- let w_data: Vec<f32> = (0..(channels * kernel_size as usize))- .map(|i| ((i % 17) as f32) * 0.01 - 0.04)- .collect();- let b_data: Vec<f32> =- (0..channels).map(|i| ((i % 5) as f32) * 0.03 - 0.07).collect();- let state_init: Vec<f32> =- (0..total_state).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();+ let x_data: Vec<f32> = (0..total_x).map(|i| ((i % 31) as f32) * 0.02 - 0.3).collect();+ let w_data: Vec<f32> = (0..(channels * kernel_size as usize)).map(|i| ((i % 17) as f32) * 0.01 - 0.04).collect();+ let b_data: Vec<f32> = (0..channels).map(|i| ((i % 5) as f32) * 0.03 - 0.07).collect();+ let state_init: Vec<f32> = (0..total_state).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();let use_scratch = tap_count > 0 && suffix_len > 1;@@ -590,13 +509,7 @@ fn conv1d_scan_is_deterministic() {alias_io,);- assert_eq!(- first.0, second.0,- "Conv outputs differ (alias_io={alias_io})"- );- assert_eq!(- first.1, second.1,- "Conv states differ (alias_io={alias_io})"- );+ assert_eq!(first.0, second.0, "Conv outputs differ (alias_io={alias_io})");+ assert_eq!(first.1, second.1, "Conv states differ (alias_io={alias_io})");}}diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rsindex 0aa225d4c..f5e27100f 100644--- a/crates/uzu/tests/ssm_update_test.rs+++ b/crates/uzu/tests/ssm_update_test.rs@@ -2,10 +2,7 @@use bytemuck;use half::bf16;-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,- MTLDeviceExt, MTLResourceOptions,-};+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};use uzu::backends::{common::Context,common::kernel::SSDUpdateKernel,@@ -50,8 +47,7 @@ fn ssm_update_ref_bf16(let state_offset = idx * n;let mut acc = 0.0f32;for i in 0..n {- let new_s = state[state_offset + i].to_f32()- * (a[i].to_f32() * delta).exp()+ let new_s = state[state_offset + i].to_f32() * (a[i].to_f32() * delta).exp()+ b[cb_offset + i].to_f32() * delta * this_x;next_state[state_offset + i] = bf16::from_f32(new_s);acc += new_s * c[cb_offset + i].to_f32();@@ -107,8 +103,7 @@ fn ssd_update_no_z_ref_bf16(let mut acc = 0.0f32;for i in 0..n {let s_old = state[state_idx0 + i].to_f32();- let new_s = s_old * this_decay- + b[cb_idx0 + i].to_f32() * dt_scaled_input;+ let new_s = s_old * this_decay + b[cb_idx0 + i].to_f32() * dt_scaled_input;next_state[state_idx0 + i] = bf16::from_f32(new_s);acc += new_s * c[cb_idx0 + i].to_f32();}@@ -162,8 +157,7 @@ fn ssd_update_ref_bf16(let mut acc = 0.0f32;for i in 0..n {let s_old = state[state_idx0 + i].to_f32();- let new_s = s_old * this_decay- + b[cb_idx0 + i].to_f32() * dt_scaled_input;+ let new_s = s_old * this_decay + b[cb_idx0 + i].to_f32() * dt_scaled_input;next_state[state_idx0 + i] = bf16::from_f32(new_s);acc += new_s * c[cb_idx0 + i].to_f32();}@@ -188,29 +182,15 @@ fn ssd_update_with_z_bf16() {let g = 2usize;let n = 8usize;- let x: Vec<bf16> = (0..bsz * h * dh)- .map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.2))- .collect();- let z: Vec<bf16> = (0..bsz * h * dh)- .map(|i| bf16::from_f32(((i % 5) as f32) * 0.1 - 0.1))- .collect();- let dt: Vec<bf16> = (0..bsz * h)- .map(|i| bf16::from_f32(((i % 5) as f32) * 0.3 - 1.0))- .collect();- let b: Vec<bf16> = (0..bsz * g * n)- .map(|i| bf16::from_f32(((i % 11) as f32) * 0.02 - 0.05))- .collect();- let c: Vec<bf16> = (0..bsz * g * n)- .map(|i| bf16::from_f32(((i % 13) as f32) * 0.015))- .collect();- let d: Vec<bf16> =- (0..h).map(|i| bf16::from_f32(((i % 3) as f32) * 0.05)).collect();- let state: Vec<bf16> = (0..bsz * h * dh * n)- .map(|i| bf16::from_f32(((i % 23) as f32) * 0.01 - 0.05))- .collect();+ let x: Vec<bf16> = (0..bsz * h * dh).map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.2)).collect();+ let z: Vec<bf16> = (0..bsz * h * dh).map(|i| bf16::from_f32(((i % 5) as f32) * 0.1 - 0.1)).collect();+ let dt: Vec<bf16> = (0..bsz * h).map(|i| bf16::from_f32(((i % 5) as f32) * 0.3 - 1.0)).collect();+ let b: Vec<bf16> = (0..bsz * g * n).map(|i| bf16::from_f32(((i % 11) as f32) * 0.02 - 0.05)).collect();+ let c: Vec<bf16> = (0..bsz * g * n).map(|i| bf16::from_f32(((i % 13) as f32) * 0.015)).collect();+ let d: Vec<bf16> = (0..h).map(|i| bf16::from_f32(((i % 3) as f32) * 0.05)).collect();+ let state: Vec<bf16> = (0..bsz * h * dh * n).map(|i| bf16::from_f32(((i % 23) as f32) * 0.01 - 0.05)).collect();- let (y_exp, ns_exp) =- ssd_update_ref_bf16(&x, &dt, &b, &c, &d, &z, &state, bsz, h, dh, g, n);+ let (y_exp, ns_exp) = ssd_update_ref_bf16(&x, &dt, &b, &c, &d, &z, &state, bsz, h, dh, g, n);let y_exp_f32: Vec<f32> = y_exp.iter().map(|&v| v.to_f32()).collect();let ns_exp_f32: Vec<f32> = ns_exp.iter().map(|&v| v.to_f32()).collect();@@ -221,79 +201,45 @@ fn ssd_update_with_z_bf16() {let x_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&x),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&x), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let dt_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&dt),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&dt), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let b_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&b),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&b), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let c_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&c),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&c), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let d_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&d),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&d), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let z_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&z),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&z), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let state_buf = ctx.device- .new_buffer_with_data(- bytemuck::cast_slice(&state),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer_with_data(bytemuck::cast_slice(&state), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let y_buf = ctx.device- .new_buffer(- bsz * h * dh * std::mem::size_of::<bf16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(bsz * h * dh * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");let ns_buf = ctx.device- .new_buffer(- bsz * h * dh * n * std::mem::size_of::<bf16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )+ .new_buffer(bsz * h * dh * n * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let kernel =- SSDUpdateMetalKernel::new(&ctx, KernelDataType::BFloat16.into())- .unwrap();- let cb_ref = ctx- .command_queue- .command_buffer()- .expect("Failed to create command buffer");+ let kernel = SSDUpdateMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).unwrap();+ let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let enc = cb- .new_compute_command_encoder()- .expect("Failed to create compute encoder");+ let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&x_buf,&dt_buf,@@ -323,8 +269,7 @@ fn ssd_update_with_z_bf16() {let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h * dh) };let y_out_f32: Vec<f32> = y_out.iter().map(|&v| v.to_f32()).collect();let ns_ptr = ns_buf.contents().as_ptr() as *const bf16;- let ns_out =- unsafe { std::slice::from_raw_parts(ns_ptr, bsz * h * dh * n) };+ let ns_out = unsafe { std::slice::from_raw_parts(ns_ptr, bsz * h * dh * n) };let ns_out_f32: Vec<f32> = ns_out.iter().map(|&v| v.to_f32()).collect();let tol = 2e-2;for (i, (a, b)) in y_out_f32.iter().zip(y_exp_f32.iter()).enumerate() {diff --git a/crates/uzu/tests/text_session_test.rs b/crates/uzu/tests/text_session_test.rsindex 42706dcc8..22be26ab6 100644--- a/crates/uzu/tests/text_session_test.rs+++ b/crates/uzu/tests/text_session_test.rs@@ -29,17 +29,13 @@ fn test_text_session_base() {#[test]fn test_text_session_scenario() {let system_prompt = String::from("You are a helpful assistant.");- let user_prompts = vec![- String::from("Tell about London"),- String::from("Compare with New York"),- ];+ let user_prompts = vec![String::from("Tell about London"), String::from("Compare with New York")];run_scenario(Some(system_prompt), user_prompts);}#[test]fn test_text_session_stability() {- let mut session =- Session::new(build_model_path(), build_decoding_config()).unwrap();+ let mut session = Session::new(build_model_path(), build_decoding_config()).unwrap();println!("Index | TTFT, s | Prompt t/s | Generate t/s");for index in 0..10 {let input = Input::Text(build_default_text());@@ -67,8 +63,7 @@ fn run(decoding_config: DecodingConfig,tokens_limit: u64,) {- let mut session =- Session::new(build_model_path(), decoding_config).unwrap();+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();let input = Input::Text(text);let output = session.run(@@ -83,15 +78,9 @@ fn run(let empty_response = String::from("None");println!("-------------------------");- println!(- "{}",- output.text.parsed.chain_of_thought.unwrap_or(empty_response.clone())- );+ println!("{}", output.text.parsed.chain_of_thought.unwrap_or(empty_response.clone()));println!("-------------------------");- println!(- "{}",- output.text.parsed.response.unwrap_or(empty_response.clone())- );+ println!("{}", output.text.parsed.response.unwrap_or(empty_response.clone()));println!("-------------------------");println!("{:#?}", output.stats);println!("-------------------------");@@ -103,8 +92,7 @@ fn run_scenario(system_prompt: Option<String>,user_prompts: Vec<String>,) {- let mut session =- Session::new(build_model_path(), build_decoding_config()).unwrap();+ let mut session = Session::new(build_model_path(), build_decoding_config()).unwrap();let mut messages: Vec<Message> = vec![];if let Some(system_prompt) = system_prompt {diff --git a/crates/uzu/tests/tracer_test.rs b/crates/uzu/tests/tracer_test.rsindex ef3aeff20..a8928a7f2 100644--- a/crates/uzu/tests/tracer_test.rs+++ b/crates/uzu/tests/tracer_test.rs@@ -15,10 +15,7 @@ fn test_tracer() {// Ensure traces file present, otherwise skiplet traces_path = crate::common::get_traces_path();if !traces_path.exists() {- println!(- "Skipping tracer test: traces file missing at {:?}",- traces_path- );+ println!("Skipping tracer test: traces file missing at {:?}", traces_path);return;}@@ -58,27 +55,17 @@ fn test_tracer() {);println!("{}: {}", result.name, text);if !valid {- println!(- "{}",- colored_text(result.metrics.message().as_str(), false)- );+ println!("{}", colored_text(result.metrics.message().as_str(), false));}}println!("-------------------------");println!("number_of_tokens_violations: {}",colored_text(- format!(- "{} / {}",- results.number_of_tokens_violations(),- results.number_of_allowed_tokens_violations()- )- .as_str(),+ format!("{} / {}", results.number_of_tokens_violations(), results.number_of_allowed_tokens_violations())+ .as_str(),results.is_valid(),));- println!(- "tokens_violation_indices: {:?}",- results.tokens_violation_indices- );+ println!("tokens_violation_indices: {:?}", results.tokens_violation_indices);}diff --git a/crates/uzu/tests/trie_test.rs b/crates/uzu/tests/trie_test.rsindex c5fcb2cf5..751135df8 100644--- a/crates/uzu/tests/trie_test.rs+++ b/crates/uzu/tests/trie_test.rs@@ -22,10 +22,7 @@ fn verify_sprout(assert_eq!(flat_trie.index(&TrieNode::new(0, None, 0)), None);assert_eq!(flat_trie.token_ids().collect::<Vec<u64>>(), vec![0]);assert_eq!(flat_trie.token_positions().collect::<Vec<usize>>(), vec![0]);- assert_eq!(- flat_trie.token_seeds().collect::<Vec<u64>>(),- vec![expected_seed]- );+ assert_eq!(flat_trie.token_seeds().collect::<Vec<u64>>(), vec![expected_seed]);}#[test]@@ -108,10 +105,7 @@ fn test_trie_from_speculator_stick() {let mut hs = HashMap::new();for i in 1..10 {- hs.insert(- (0..i).collect(),- vec![(i, 0.988), (888, 0.001), (999, 0.001)].into_iter().collect(),- );+ hs.insert((0..i).collect(), vec![(i, 0.988), (888, 0.001), (999, 0.001)].into_iter().collect());}let speculator = StaticSpeculator::new(hs);@@ -180,12 +174,7 @@ fn test_trie_from_speculator_bush() {let rng = PRng::new(0);let mut hs = HashMap::new();- hs.insert(- vec![0],- vec![(1, 0.33), (2, 0.33), (3, 0.33), (888, 0.005), (999, 0.005)]- .into_iter()- .collect(),- );+ hs.insert(vec![0], vec![(1, 0.33), (2, 0.33), (3, 0.33), (888, 0.005), (999, 0.005)].into_iter().collect());let speculator = StaticSpeculator::new(hs);let trie_root = TrieNode::from_speculator(diff --git a/rustfmt.toml b/rustfmt.tomlindex 9b128290a..29d67d950 100644--- a/rustfmt.toml+++ b/rustfmt.toml@@ -6,4 +6,4 @@ imports_granularity = "Crate"fn_params_layout = "Vertical"match_block_trailing_comma = trueuse_small_heuristics = "Off"-max_width = 80+max_width = 120