Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

rustfmt-max-width (#164)

Aleksandr Golokoz7bc54256a447
diff --git a/crates/benchmarks/src/runner/helpers/memory_usage.rs b/crates/benchmarks/src/runner/helpers/memory_usage.rs
index 751f9e9e2..e93ebf88c 100644
--- a/crates/benchmarks/src/runner/helpers/memory_usage.rs
+++ b/crates/benchmarks/src/runner/helpers/memory_usage.rs
@@ -22,15 +22,9 @@ pub fn get_memory_usage() -> Option<u64> {
suspend_count: 0,
};
- let mut count: mach_msg_type_number_t =
- (mem::size_of::<task_basic_info>() / mem::size_of::<u32>()) as u32;
+ let mut count: mach_msg_type_number_t = (mem::size_of::<task_basic_info>() / mem::size_of::<u32>()) as u32;
- let result = task_info(
- task,
- TASK_BASIC_INFO,
- &mut info as *mut _ as *mut i32,
- &mut count,
- );
+ let result = task_info(task, TASK_BASIC_INFO, &mut info as *mut _ as *mut i32, &mut count);
if result == KERN_SUCCESS {
Some(info.resident_size as u64)
diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rs
index 9d4232674..a6b0f382d 100644
--- a/crates/benchmarks/src/runner/runner.rs
+++ b/crates/benchmarks/src/runner/runner.rs
@@ -9,9 +9,7 @@ use uzu::{
session::{
ChatSession,
config::{DecodingConfig, RunConfig},
- parameter::{
- ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy,
- },
+ parameter::{ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy},
types::{Input, Output},
},
};
@@ -40,14 +38,9 @@ impl Runner {
}
}
- fn minimal_context_length(
- &self
- ) -> Result<usize, Box<dyn std::error::Error>> {
+ fn minimal_context_length(&self) -> Result<usize, Box<dyn std::error::Error>> {
let input = Input::Messages(self.task.messages.clone());
- let mut session = ChatSession::new(
- PathBuf::from(self.model_path.clone()),
- DecodingConfig::default(),
- )?;
+ let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()), DecodingConfig::default())?;
let run_config = RunConfig::default().tokens_limit(1);
let output = session.run(
input.clone(),
@@ -56,8 +49,7 @@ impl Runner {
return true;
}),
)?;
- return Ok(output.stats.total_stats.tokens_count_input as usize
- + self.task.tokens_limit as usize);
+ return Ok(output.stats.total_stats.tokens_count_input as usize + self.task.tokens_limit as usize);
}
pub fn run<F>(
@@ -69,29 +61,18 @@ impl Runner {
{
let context_length = self.minimal_context_length()?;
- let mut decoding_config = DecodingConfig::default()
- .with_context_length(ContextLength::Custom(context_length));
+ let mut decoding_config = DecodingConfig::default().with_context_length(ContextLength::Custom(context_length));
if let Some(prefill_step_size) = self.prefill_step_size {
- decoding_config = decoding_config.with_prefill_step_size(
- PrefillStepSize::Custom(prefill_step_size),
- );
+ decoding_config = decoding_config.with_prefill_step_size(PrefillStepSize::Custom(prefill_step_size));
}
- let mut session = ChatSession::new(
- PathBuf::from(self.model_path.clone()),
- decoding_config,
- )?;
+ let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()), decoding_config)?;
- let precision =
- session.model_metadata.model_config.as_language_model().map(
- |config| {
- config
- .model_config
- .transformer_config
- .output_norm_config
- .scale_precision
- },
- );
+ let precision = session
+ .model_metadata
+ .model_config
+ .as_language_model()
+ .map(|config| config.model_config.transformer_config.output_norm_config.scale_precision);
let device = self.get_device_info();
@@ -99,16 +80,13 @@ impl Runner {
let mut results: Vec<TaskResult> = Vec::new();
for run_index in 0..self.task.number_of_runs {
- let timestamp =
- SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
+ let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
- let mut run_config =
- RunConfig::default().tokens_limit(self.task.tokens_limit);
+ let mut run_config = RunConfig::default().tokens_limit(self.task.tokens_limit);
if self.task.greedy {
- run_config =
- run_config.sampling_policy(SamplingPolicy::Custom {
- value: SamplingMethod::Greedy,
- });
+ run_config = run_config.sampling_policy(SamplingPolicy::Custom {
+ value: SamplingMethod::Greedy,
+ });
}
let output = session.run(
input.clone(),
@@ -128,27 +106,16 @@ impl Runner {
precision,
memory_used,
tokens_count_input: output.stats.total_stats.tokens_count_input,
- tokens_count_output: output
- .stats
- .total_stats
- .tokens_count_output,
+ tokens_count_output: output.stats.total_stats.tokens_count_output,
time_to_first_token: output.stats.prefill_stats.duration,
- prompt_tokens_per_second: output
- .stats
- .prefill_stats
- .processed_tokens_per_second,
- generate_tokens_per_second: output
- .stats
- .generate_stats
- .map(|stats| stats.tokens_per_second),
+ prompt_tokens_per_second: output.stats.prefill_stats.processed_tokens_per_second,
+ generate_tokens_per_second: output.stats.generate_stats.map(|stats| stats.tokens_per_second),
text: output.text.original,
};
results.push(result);
if let Some(progress) = progress.as_mut() {
- progress(
- (run_index + 1) as f64 / self.task.number_of_runs as f64,
- );
+ progress((run_index + 1) as f64 / self.task.number_of_runs as f64);
}
}
@@ -162,8 +129,7 @@ impl Runner {
system_info.refresh_all();
let os_name = System::long_os_version();
- let cpu_name =
- system_info.cpus().first().map(|cpu| cpu.brand().to_string());
+ let cpu_name = system_info.cpus().first().map(|cpu| cpu.brand().to_string());
let memory_total = system_info.total_memory();
let device = Device {
diff --git a/crates/cli/src/handlers/bench.rs b/crates/cli/src/handlers/bench.rs
index 4c420383e..f46a7b707 100644
--- a/crates/cli/src/handlers/bench.rs
+++ b/crates/cli/src/handlers/bench.rs
@@ -1,10 +1,7 @@
use std::fs;
use benchmarks::{runner::Runner, types::Task};
-use comfy_table::{
- CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS,
- presets::UTF8_FULL,
-};
+use comfy_table::{CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL};
use indicatif::ProgressBar;
pub fn handle_bench(
@@ -27,8 +24,7 @@ pub fn handle_bench(
let results_data = serde_json::to_string_pretty(&results)?;
fs::write(output_path, results_data)?;
- let tokens_count_input =
- results.first().expect("No results").tokens_count_input;
+ let tokens_count_input = results.first().expect("No results").tokens_count_input;
println!("Input tokens count: {}", tokens_count_input);
let memory_used_list = results
@@ -36,25 +32,16 @@ pub fn handle_bench(
.filter_map(|result| result.memory_used)
.map(|memory_used| (memory_used as f64) / 1024.0 / 1024.0 / 1024.0)
.collect::<Vec<f64>>();
- let time_to_first_token_list = results
- .iter()
- .map(|result| result.time_to_first_token)
- .collect::<Vec<f64>>();
- let prompt_tokens_per_second_list = results
- .iter()
- .map(|result| result.prompt_tokens_per_second)
- .collect::<Vec<f64>>();
- let generate_tokens_per_second_list = results
- .iter()
- .filter_map(|result| result.generate_tokens_per_second)
- .collect::<Vec<f64>>();
+ let time_to_first_token_list = results.iter().map(|result| result.time_to_first_token).collect::<Vec<f64>>();
+ let prompt_tokens_per_second_list =
+ results.iter().map(|result| result.prompt_tokens_per_second).collect::<Vec<f64>>();
+ let generate_tokens_per_second_list =
+ results.iter().filter_map(|result| result.generate_tokens_per_second).collect::<Vec<f64>>();
let memory_used_metric = calculate_metric(memory_used_list);
let time_to_first_token_metric = calculate_metric(time_to_first_token_list);
- let prompt_tokens_per_second_metric =
- calculate_metric(prompt_tokens_per_second_list);
- let generate_tokens_per_second_metric =
- calculate_metric(generate_tokens_per_second_list);
+ let prompt_tokens_per_second_metric = calculate_metric(prompt_tokens_per_second_list);
+ let generate_tokens_per_second_metric = calculate_metric(generate_tokens_per_second_list);
let mut table = Table::new();
table
@@ -65,10 +52,7 @@ pub fn handle_bench(
.add_row(vec!["Used memory, GB", memory_used_metric.as_str()])
.add_row(vec!["TTFT, s", time_to_first_token_metric.as_str()])
.add_row(vec!["Prompt, t/s", prompt_tokens_per_second_metric.as_str()])
- .add_row(vec![
- "Generate, t/s",
- generate_tokens_per_second_metric.as_str(),
- ]);
+ .add_row(vec!["Generate, t/s", generate_tokens_per_second_metric.as_str()]);
let column = table.column_mut(1).expect("Column not found");
column.set_cell_alignment(CellAlignment::Right);
println!("{table}");
diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rs
index 651a8d4c5..feee0c1d5 100644
--- a/crates/cli/src/handlers/run.rs
+++ b/crates/cli/src/handlers/run.rs
@@ -16,18 +16,14 @@ use crate::server::load_session;
fn format_output(output: Output) -> String {
let stats = &output.stats;
- let tokens_per_second = if let Some(generate_stats) = &stats.generate_stats
- {
+ let tokens_per_second = if let Some(generate_stats) = &stats.generate_stats {
generate_stats.tokens_per_second
} else {
stats.prefill_stats.tokens_per_second
};
let style_stats = Style::new().bold();
- let stats_info = style_stats.apply_to(format!(
- "{:.3}s, {:.3}t/s",
- stats.total_stats.duration, tokens_per_second,
- ));
+ let stats_info = style_stats.apply_to(format!("{:.3}s, {:.3}t/s", stats.total_stats.duration, tokens_per_second,));
let result = format!("{}\n\n{}", output.text.original, stats_info,);
result
@@ -76,11 +72,7 @@ pub fn handle_run(
let progress_bar_message_limit: usize = 1024;
let progress_bar = ProgressBar::new_spinner();
progress_bar.enable_steady_tick(std::time::Duration::from_millis(100));
- progress_bar.set_style(
- ProgressStyle::default_spinner()
- .template("{spinner:.green} {msg}")
- .unwrap(),
- );
+ progress_bar.set_style(ProgressStyle::default_spinner().template("{spinner:.green} {msg}").unwrap());
let progress_bar_for_progress = progress_bar.clone();
let is_model_running_for_progress = is_model_running.clone();
@@ -103,12 +95,7 @@ pub fn handle_run(
let session_output = match session.run(
Input::Text(input.to_string()),
- RunConfig::new(
- tokens_limit as u64,
- true,
- SamplingPolicy::Default,
- None,
- ),
+ RunConfig::new(tokens_limit as u64, true, SamplingPolicy::Default, None),
Some(session_progress),
) {
Ok(output) => output,
@@ -121,9 +108,7 @@ pub fn handle_run(
};
let result = format_output(session_output);
- progress_bar.set_style(
- ProgressStyle::default_spinner().template("{msg}").unwrap(),
- );
+ progress_bar.set_style(ProgressStyle::default_spinner().template("{msg}").unwrap());
progress_bar.finish_and_clear();
println!("{}", result);
is_model_running.store(false, Ordering::SeqCst);
diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rs
index 6aea93b25..3972aaa5a 100644
--- a/crates/cli/src/server/chat_completions.rs
+++ b/crates/cli/src/server/chat_completions.rs
@@ -6,10 +6,7 @@ use uuid::Uuid;
use uzu::session::{
config::RunConfig,
parameter::SamplingPolicy,
- types::{
- FinishReason, Input, Message, Output, Role, RunStats, Stats, StepStats,
- TotalStats,
- },
+ types::{FinishReason, Input, Message, Output, Role, RunStats, Stats, StepStats, TotalStats},
};
use crate::server::SessionState;
@@ -53,10 +50,7 @@ pub fn handle_chat_completions(
println!("πŸ“¨ [{}] Incoming chat completion request:", id);
println!(" Messages: {} message(s)", request.messages.len());
- println!(
- " Max tokens: {}",
- request.max_completion_tokens.unwrap_or(2048)
- );
+ println!(" Max tokens: {}", request.max_completion_tokens.unwrap_or(2048));
for (i, msg) in request.messages.iter().enumerate() {
let content_preview = if msg.content.len() > 10000 {
@@ -79,58 +73,53 @@ pub fn handle_chat_completions(
})
.collect();
let input = Input::Messages(messages);
- let run_config =
- RunConfig::new(tokens_limit, true, SamplingPolicy::Default, None);
+ let run_config = RunConfig::new(tokens_limit, true, SamplingPolicy::Default, None);
let start_time = std::time::Instant::now();
let mut session = state.session_wrapper.lock();
- let output =
- match session.run(input, run_config, None::<fn(Output) -> bool>) {
- Ok(output) => output,
- Err(e) => {
- eprintln!("❌ [{}] Session run error: {}", id, e);
- // Return error response
- let error_response = ChatCompletionResponse {
- id: id.clone(),
- object: "chat.completion".to_string(),
- created: SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .unwrap()
- .as_secs() as i64,
- model: model_name.clone(),
- choices: vec![ChatCompletionChoice {
- index: 0,
- message: ChatMessage {
- content: format!("Error: {}", e),
- role: Role::Assistant,
- },
- finish_reason: "error".to_string(),
- }],
- stats: Stats {
- prefill_stats: StepStats {
- duration: 0.0,
- suffix_length: 0,
- tokens_count: 0,
- tokens_per_second: 0.0,
- processed_tokens_per_second: 0.0,
- model_run: RunStats {
- count: 0,
- average_duration: 0.0,
- },
- run: None,
- },
- generate_stats: None,
- total_stats: TotalStats {
- duration: 0.0,
- tokens_count_input: 0,
- tokens_count_output: 0,
+ let output = match session.run(input, run_config, None::<fn(Output) -> bool>) {
+ Ok(output) => output,
+ Err(e) => {
+ eprintln!("❌ [{}] Session run error: {}", id, e);
+ // Return error response
+ let error_response = ChatCompletionResponse {
+ id: id.clone(),
+ object: "chat.completion".to_string(),
+ created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64,
+ model: model_name.clone(),
+ choices: vec![ChatCompletionChoice {
+ index: 0,
+ message: ChatMessage {
+ content: format!("Error: {}", e),
+ role: Role::Assistant,
+ },
+ finish_reason: "error".to_string(),
+ }],
+ stats: Stats {
+ prefill_stats: StepStats {
+ duration: 0.0,
+ suffix_length: 0,
+ tokens_count: 0,
+ tokens_per_second: 0.0,
+ processed_tokens_per_second: 0.0,
+ model_run: RunStats {
+ count: 0,
+ average_duration: 0.0,
},
+ run: None,
},
- };
- return Json(error_response);
- },
- };
+ generate_stats: None,
+ total_stats: TotalStats {
+ duration: 0.0,
+ tokens_count_input: 0,
+ tokens_count_output: 0,
+ },
+ },
+ };
+ return Json(error_response);
+ },
+ };
let processing_time = start_time.elapsed();
@@ -149,14 +138,7 @@ pub fn handle_chat_completions(
println!(" Response length: {} chars", text.original.len());
println!(" Finish reason: {:?}", finish_reason_val);
println!(" Processing time: {:.3}s", processing_time.as_secs_f64());
- println!(
- " Tokens/second: {:.1}",
- stats
- .generate_stats
- .as_ref()
- .map(|s| s.tokens_per_second)
- .unwrap_or(0.0)
- );
+ println!(" Tokens/second: {:.1}", stats.generate_stats.as_ref().map(|s| s.tokens_per_second).unwrap_or(0.0));
print!(" Response preview: ");
let text = text.original.clone();
@@ -171,8 +153,7 @@ pub fn handle_chat_completions(
let response = ChatCompletionResponse {
id: id.clone(),
object: "chat.completion".to_string(),
- created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()
- as i64,
+ created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64,
model: model_name,
choices: vec![ChatCompletionChoice {
index: 0,
diff --git a/crates/cli/src/server/main.rs b/crates/cli/src/server/main.rs
index 3a2bc8ba0..6a94ccfdb 100644
--- a/crates/cli/src/server/main.rs
+++ b/crates/cli/src/server/main.rs
@@ -3,9 +3,7 @@ use std::path::PathBuf;
use log::LevelFilter;
use rocket::{Config, config::LogLevel, log::private as log, routes};
-use crate::server::{
- SessionState, SessionWrapper, handle_chat_completions, load_session,
-};
+use crate::server::{SessionState, SessionWrapper, handle_chat_completions, load_session};
struct SilentLogger;
static SILENT_LOGGER: SilentLogger = SilentLogger;
@@ -30,8 +28,7 @@ pub async fn run_server(
prefill_step_size: Option<usize>,
) {
// Install the silent logger **before** Rocket initializes its own logger.
- let _ = log::set_logger(&SILENT_LOGGER)
- .map(|_| log::set_max_level(LevelFilter::Off));
+ let _ = log::set_logger(&SILENT_LOGGER).map(|_| log::set_max_level(LevelFilter::Off));
let config = Config {
workers: 1,
@@ -39,22 +36,12 @@ pub async fn run_server(
..Config::default()
};
- let model_name = PathBuf::from(model_path.clone())
- .file_name()
- .unwrap()
- .to_str()
- .unwrap()
- .to_string();
+ let model_name = PathBuf::from(model_path.clone()).file_name().unwrap().to_str().unwrap().to_string();
println!("πŸš€ Starting server with model: {}", model_name);
println!("πŸ“‚ Model path: {}", model_path);
- println!(
- "🌐 Server will be available at: http://localhost:{}",
- config.port
- );
- println!(
- "πŸ“ Endpoints:\n POST /chat/completions - Chat completions API\n"
- );
+ println!("🌐 Server will be available at: http://localhost:{}", config.port);
+ println!("πŸ“ Endpoints:\n POST /chat/completions - Chat completions API\n");
let session = load_session(model_path, prefill_step_size, None);
let state = SessionState {
@@ -62,9 +49,5 @@ pub async fn run_server(
session_wrapper: SessionWrapper::new(session),
};
- let _ = rocket::custom(config)
- .manage(state)
- .mount("/", routes![handle_chat_completions])
- .launch()
- .await;
+ let _ = rocket::custom(config).manage(state).mount("/", routes![handle_chat_completions]).launch().await;
}
diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rs
index 53717e3bc..117abcae2 100644
--- a/crates/cli/src/server/state.rs
+++ b/crates/cli/src/server/state.rs
@@ -36,19 +36,11 @@ pub fn load_session(
let style_bold = Style::new().bold();
let model_path_buf = PathBuf::from(model_path);
- let model_name = style_bold
- .apply_to(
- model_path_buf.file_name().unwrap().to_str().unwrap().to_string(),
- )
- .to_string();
+ let model_name = style_bold.apply_to(model_path_buf.file_name().unwrap().to_str().unwrap().to_string()).to_string();
let progress_bar = ProgressBar::new_spinner();
progress_bar.enable_steady_tick(std::time::Duration::from_millis(100));
- progress_bar.set_style(
- ProgressStyle::default_spinner()
- .template("{spinner:.green} Loading: {msg}")
- .unwrap(),
- );
+ progress_bar.set_style(ProgressStyle::default_spinner().template("{spinner:.green} Loading: {msg}").unwrap());
progress_bar.set_message(model_name.clone());
let prefill_step_size_config: PrefillStepSize;
@@ -58,18 +50,14 @@ pub fn load_session(
prefill_step_size_config = PrefillStepSize::Default;
}
- let decoding_config = DecodingConfig::default()
- .with_prefill_step_size(prefill_step_size_config)
- .with_sampling_seed(match seed {
+ let decoding_config =
+ DecodingConfig::default().with_prefill_step_size(prefill_step_size_config).with_sampling_seed(match seed {
Some(seed) => SamplingSeed::Custom(seed),
None => SamplingSeed::Default,
});
- let session = Session::new(model_path_buf, decoding_config)
- .expect("Failed to create session");
+ let session = Session::new(model_path_buf, decoding_config).expect("Failed to create session");
- progress_bar.set_style(
- ProgressStyle::default_spinner().template("Loaded: {msg}").unwrap(),
- );
+ progress_bar.set_style(ProgressStyle::default_spinner().template("Loaded: {msg}").unwrap());
progress_bar.finish_with_message(model_name.clone());
session
diff --git a/crates/uzu/build/common/caching.rs b/crates/uzu/build/common/caching.rs
index 84e1b498f..e992ae939 100644
--- a/crates/uzu/build/common/caching.rs
+++ b/crates/uzu/build/common/caching.rs
@@ -8,11 +8,8 @@ pub fn build_system_hash() -> anyhow::Result<&'static blake3::Hash> {
Ok(bsh)
} else {
let bsh = blake3::hash(
- &fs::read(
- &std::env::current_exe()
- .context("cannot get build system executable")?,
- )
- .context("cannot read build system executable")?,
+ &fs::read(&std::env::current_exe().context("cannot get build system executable")?)
+ .context("cannot read build system executable")?,
);
Ok(BUILD_SYSTEM_HASH.get_or_init(|| bsh))
diff --git a/crates/uzu/build/common/codegen.rs b/crates/uzu/build/common/codegen.rs
index da55239a6..5b8a97cd2 100644
--- a/crates/uzu/build/common/codegen.rs
+++ b/crates/uzu/build/common/codegen.rs
@@ -10,15 +10,11 @@ pub fn write_tokens(
let tokens = tokens.into();
let file = file.as_ref();
- let parsed = syn::parse2(tokens.into())
- .context("cannot parse generated bindings")?;
+ let parsed = syn::parse2(tokens.into()).context("cannot parse generated bindings")?;
fs::write(&file, prettyplease::unparse(&parsed))
.with_context(|| format!("cannot write file {}", file.display()))?;
- std::process::Command::new("rustfmt")
- .arg(&file)
- .status()
- .context("rustfmt failed")?;
+ std::process::Command::new("rustfmt").arg(&file).status().context("rustfmt failed")?;
Ok(())
}
diff --git a/crates/uzu/build/common/compiler.rs b/crates/uzu/build/common/compiler.rs
index 109c81b17..9ffc5b0a7 100644
--- a/crates/uzu/build/common/compiler.rs
+++ b/crates/uzu/build/common/compiler.rs
@@ -6,7 +6,5 @@ use super::kernel::Kernel;
#[async_trait]
pub trait Compiler {
- async fn build(
- &self
- ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>;
+ async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>;
}
diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rs
index a1291fc4f..29a89a5bb 100644
--- a/crates/uzu/build/common/traitgen.rs
+++ b/crates/uzu/build/common/traitgen.rs
@@ -56,8 +56,7 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
})
.collect::<(Vec<_>, Vec<_>)>();
- let encode_generics =
- encode_generics.into_iter().flatten().collect::<Vec<_>>();
+ let encode_generics = encode_generics.into_iter().flatten().collect::<Vec<_>>();
let kernel_trait = quote! {
pub trait #trait_name: Sized {
@@ -70,17 +69,13 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
}
};
- let kernel_type =
- quote! { type #trait_name: #trait_name<Backend = Self::Backend>; };
+ let kernel_type = quote! { type #trait_name: #trait_name<Backend = Self::Backend>; };
(kernel_trait, kernel_type)
}
-pub fn traitgen_all(
- backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>
-) -> anyhow::Result<()> {
- let out_dir =
- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
+pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>) -> anyhow::Result<()> {
+ let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
let mut kernels: HashMap<Box<[Box<str>]>, Box<[Kernel]>> = HashMap::new();
@@ -99,9 +94,7 @@ pub fn traitgen_all(
let mut kernel_traits = Vec::new();
let mut kernel_types = Vec::new();
- for (_file_path, file_kernels) in
- kernels.into_iter().sorted_by_key(|(p, _k)| p.join("::"))
- {
+ for (_file_path, file_kernels) in kernels.into_iter().sorted_by_key(|(p, _k)| p.join("::")) {
for (tr, ty) in file_kernels.iter().map(traitgen) {
kernel_traits.push(tr);
kernel_types.push(ty);
@@ -137,8 +130,7 @@ pub fn traitgen_all(
}
};
- write_tokens(kernel_traits, out_dir.join("traits.rs"))
- .context("cannot write kernel traits")?;
+ write_tokens(kernel_traits, out_dir.join("traits.rs")).context("cannot write kernel traits")?;
Ok(())
}
diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rs
index 19baafcdc..080e2b45a 100644
--- a/crates/uzu/build/gpu_types/mod.rs
+++ b/crates/uzu/build/gpu_types/mod.rs
@@ -2,10 +2,7 @@ use std::{collections::HashMap, env, fs, path::PathBuf};
use anyhow::Context;
use async_trait::async_trait;
-use syn::{
- Expr, ExprLit, Fields, Item, ItemStruct, Lit, Meta, Type, TypeArray,
- TypePath,
-};
+use syn::{Expr, ExprLit, Fields, Item, ItemStruct, Lit, Meta, Type, TypeArray, TypePath};
use tokio::task::spawn_blocking;
use crate::common::{compiler::Compiler, kernel::Kernel};
@@ -20,22 +17,13 @@ pub struct GpuTypesCompiler {
impl GpuTypesCompiler {
pub fn new() -> anyhow::Result<Self> {
- let crate_dir = PathBuf::from(
- env::var("CARGO_MANIFEST_DIR")
- .context("missing CARGO_MANIFEST_DIR")?,
- );
+ let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?);
let generated_dir = crate_dir.join("src/backends/metal/generated");
- fs::create_dir_all(&generated_dir).with_context(|| {
- format!("cannot create {}", generated_dir.display())
- })?;
+ fs::create_dir_all(&generated_dir).with_context(|| format!("cannot create {}", generated_dir.display()))?;
- let build_dir =
- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?)
- .join("gpu_types");
- fs::create_dir_all(&build_dir).with_context(|| {
- format!("cannot create {}", build_dir.display())
- })?;
+ let build_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?).join("gpu_types");
+ fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;
Ok(Self {
crate_dir,
@@ -49,25 +37,20 @@ impl GpuTypesCompiler {
}
fn generate(&self) -> anyhow::Result<()> {
- let gpu_types_dir =
- self.crate_dir.join("src/backends/common/gpu_types");
+ let gpu_types_dir = self.crate_dir.join("src/backends/common/gpu_types");
let source_files: Vec<PathBuf> = fs::read_dir(&gpu_types_dir)
- .with_context(|| {
- format!("cannot read {}", gpu_types_dir.display())
- })?
+ .with_context(|| format!("cannot read {}", gpu_types_dir.display()))?
.filter_map(|entry| entry.ok())
.map(|entry| entry.path())
.filter(|path| {
- path.extension().map(|ext| ext == "rs").unwrap_or(false)
- && path.file_stem() != Some("mod".as_ref())
+ path.extension().map(|ext| ext == "rs").unwrap_or(false) && path.file_stem() != Some("mod".as_ref())
})
.collect();
let mut hasher = blake3::Hasher::new();
for path in &source_files {
- let content = fs::read(path)
- .with_context(|| format!("cannot read {}", path.display()))?;
+ let content = fs::read(path).with_context(|| format!("cannot read {}", path.display()))?;
hasher.update(&content);
}
@@ -87,9 +70,8 @@ impl GpuTypesCompiler {
self.generate_header_for_file(src_path)?;
}
- fs::write(&cached_hash_path, &source_hash).with_context(|| {
- format!("cannot write {}", cached_hash_path.display())
- })?;
+ fs::write(&cached_hash_path, &source_hash)
+ .with_context(|| format!("cannot write {}", cached_hash_path.display()))?;
debug_log!("gpu_types generation done");
@@ -100,24 +82,18 @@ impl GpuTypesCompiler {
&self,
src_path: &PathBuf,
) -> anyhow::Result<()> {
- let module_name = src_path
- .file_stem()
- .and_then(|s| s.to_str())
- .context("invalid source file name")?;
+ let module_name = src_path.file_stem().and_then(|s| s.to_str()).context("invalid source file name")?;
- let source = fs::read_to_string(src_path)
- .with_context(|| format!("cannot read {}", src_path.display()))?;
+ let source = fs::read_to_string(src_path).with_context(|| format!("cannot read {}", src_path.display()))?;
- let syntax = syn::parse_file(&source)
- .with_context(|| format!("cannot parse {}", src_path.display()))?;
+ let syntax = syn::parse_file(&source).with_context(|| format!("cannot parse {}", src_path.display()))?;
let mut repr_c_structs: Vec<ItemStruct> = Vec::new();
for item in syntax.items {
if let Item::Struct(item_struct) = item {
let has_repr_c = item_struct.attrs.iter().any(|attribute| {
if attribute.path().is_ident("repr") {
- if let Ok(ident) = attribute.parse_args::<syn::Ident>()
- {
+ if let Ok(ident) = attribute.parse_args::<syn::Ident>() {
return ident == "C";
}
}
@@ -156,18 +132,12 @@ impl GpuTypesCompiler {
if let Fields::Named(named_fields) = &rust_struct.fields {
for field in &named_fields.named {
- let field_name = field
- .ident
- .as_ref()
- .map(|ident| ident.to_string())
- .unwrap_or_default();
+ let field_name = field.ident.as_ref().map(|ident| ident.to_string()).unwrap_or_default();
let c_type = rust_type_to_c(&field.ty);
for attribute in &field.attrs {
if attribute.path().is_ident("doc") {
- if let Meta::NameValue(meta_name_value) =
- &attribute.meta
- {
+ if let Meta::NameValue(meta_name_value) = &attribute.meta {
if let Expr::Lit(ExprLit {
lit: Lit::Str(literal_string),
..
@@ -177,8 +147,7 @@ impl GpuTypesCompiler {
let doc_comment = doc_comment.trim();
if !doc_comment.is_empty() {
c_struct_definitions.push_str(" /**");
- c_struct_definitions
- .push_str(doc_comment);
+ c_struct_definitions.push_str(doc_comment);
c_struct_definitions.push_str(" */\n");
}
}
@@ -186,15 +155,10 @@ impl GpuTypesCompiler {
}
}
- if let Some((base_type, array_size)) =
- c_type.split_once('[')
- {
- c_struct_definitions.push_str(&format!(
- " {base_type} {field_name}[{array_size};\n"
- ));
+ if let Some((base_type, array_size)) = c_type.split_once('[') {
+ c_struct_definitions.push_str(&format!(" {base_type} {field_name}[{array_size};\n"));
} else {
- c_struct_definitions
- .push_str(&format!(" {c_type} {field_name};\n"));
+ c_struct_definitions.push_str(&format!(" {c_type} {field_name};\n"));
}
}
}
@@ -244,9 +208,7 @@ namespace {module_name} {{
);
let output_path = self.generated_dir.join(format!("{module_name}.h"));
- fs::write(&output_path, &header_content).with_context(|| {
- format!("cannot write {}", output_path.display())
- })?;
+ fs::write(&output_path, &header_content).with_context(|| format!("cannot write {}", output_path.display()))?;
debug_log!(" -> {}", output_path.display());
@@ -303,9 +265,7 @@ fn rust_type_to_c(rust_type: &Type) -> String {
#[async_trait]
impl Compiler for GpuTypesCompiler {
- async fn build(
- &self
- ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
+ async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
let crate_dir = self.crate_dir.clone();
let generated_dir = self.generated_dir.clone();
let build_dir = self.build_dir.clone();
diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rs
index 8eab62ce7..bc25ca1bd 100644
--- a/crates/uzu/build/main.rs
+++ b/crates/uzu/build/main.rs
@@ -16,42 +16,31 @@ async fn main() -> anyhow::Result<()> {
println!("cargo::rerun-if-changed=.");
if envs::build_always() {
- println!(
- "cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun"
- );
+ println!("cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun");
}
debug_log!("build script started");
if envs::build_clean() {
- let out_dir =
- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
+ let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
if out_dir.exists() {
- fs::remove_dir_all(&out_dir).with_context(|| {
- format!("cannot clean {}", out_dir.display())
- })?;
- fs::create_dir_all(&out_dir).with_context(|| {
- format!("cannot recreate {}", out_dir.display())
- })?;
+ fs::remove_dir_all(&out_dir).with_context(|| format!("cannot clean {}", out_dir.display()))?;
+ fs::create_dir_all(&out_dir).with_context(|| format!("cannot recreate {}", out_dir.display()))?;
}
debug_log!("cleaned caches");
}
let gpu_types_compiler = gpu_types::GpuTypesCompiler::new()?;
- let generated_header_dir =
- gpu_types_compiler.generated_header_dir().clone();
+ let generated_header_dir = gpu_types_compiler.generated_header_dir().clone();
gpu_types_compiler.build().await?;
debug_log!("gpu_types build done");
let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();
#[cfg(feature = "metal")]
- compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(
- generated_header_dir,
- )?));
+ compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));
- let backends_kernels =
- try_join_all(compilers.iter().map(|c| c.build())).await?;
+ let backends_kernels = try_join_all(compilers.iter().map(|c| c.build())).await?;
debug_log!("backend build end");
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rs
index 84d2f09fa..f61fc77d5 100644
--- a/crates/uzu/build/metal/ast.rs
+++ b/crates/uzu/build/metal/ast.rs
@@ -1,10 +1,7 @@
use anyhow::{Context, bail};
use serde::{Deserialize, Serialize};
-use crate::common::kernel::{
- Kernel, KernelArgument, KernelArgumentType, KernelParameter,
- KernelParameterType,
-};
+use crate::common::kernel::{Kernel, KernelArgument, KernelArgumentType, KernelParameter, KernelParameterType};
pub type MetalAstNode = clang_ast::Node<MetalAstKind>;
@@ -43,9 +40,7 @@ pub enum MetalAstKind {
Other,
}
-fn annotation_from_ast_node(
- annotation_node: MetalAstNode
-) -> anyhow::Result<Box<[Box<str>]>> {
+fn annotation_from_ast_node(annotation_node: MetalAstNode) -> anyhow::Result<Box<[Box<str>]>> {
if !matches!(annotation_node.kind, MetalAstKind::AnnotateAttr) {
bail!(
"unexpected kind of root node: MetalAstKind::AnnotateAttr expected, but {:?} found",
@@ -62,26 +57,17 @@ fn annotation_from_ast_node(
};
if constant_expr.inner.len() != 1 {
- bail!(
- "ConstantExpr must have exactly one child, found {}",
- constant_expr.inner.len()
- );
+ bail!("ConstantExpr must have exactly one child, found {}", constant_expr.inner.len());
}
let mut implicit_cast_expr = constant_expr.inner.pop().unwrap();
let MetalAstKind::ImplicitCastExpr = implicit_cast_expr.kind else {
- bail!(
- "expected ImplicitCastExpr, found {:?}",
- implicit_cast_expr.kind
- );
+ bail!("expected ImplicitCastExpr, found {:?}", implicit_cast_expr.kind);
};
if implicit_cast_expr.inner.len() != 1 {
- bail!(
- "ImplicitCastExpr must have exactly one child, found {}",
- implicit_cast_expr.inner.len()
- );
+ bail!("ImplicitCastExpr must have exactly one child, found {}", implicit_cast_expr.inner.len());
}
let string_literal = implicit_cast_expr.inner.pop().unwrap();
@@ -90,15 +76,11 @@ fn annotation_from_ast_node(
value,
} = string_literal.kind
else {
- bail!(
- "expected StringLiteral, found {:?}",
- string_literal.kind
- );
+ bail!("expected StringLiteral, found {:?}", string_literal.kind);
};
// NOTE: string literal includes "" (and is probably not parsed?), using json parse here for now
- Ok(serde_json::from_str(&value)
- .context("failed to parse string literal")?)
+ Ok(serde_json::from_str(&value).context("failed to parse string literal")?)
})
.collect()
}
@@ -145,9 +127,9 @@ impl MetalArgument {
["uint"] | ["uint32_t"] | ["unsigned", "int"] => Ok("u32".into()),
["int"] | ["int32_t"] => Ok("i32".into()),
["float"] => Ok("f32".into()),
- [vpath] if vpath.starts_with("uzu::") => Ok(vpath
- .replacen("uzu::", "crate::backends::common::gpu_types::", 1)
- .into()),
+ [vpath] if vpath.starts_with("uzu::") => {
+ Ok(vpath.replacen("uzu::", "crate::backends::common::gpu_types::", 1).into())
+ },
_ => bail!("unknown scalar type: {c_type}"),
}
}
@@ -159,11 +141,10 @@ impl MetalArgument {
let MetalAstKind::ParmVarDecl {
name,
range,
- ty:
- ParmVarDeclType {
- qual_type,
- desugared_qual_type,
- },
+ ty: ParmVarDeclType {
+ qual_type,
+ desugared_qual_type,
+ },
} = argument_node.kind
else {
bail!("argument isn't ParmVarDecl: {:?}", argument_node.kind);
@@ -177,27 +158,16 @@ impl MetalArgument {
bail!("more than one annotation on argument ast node");
}
- let annotation =
- if let Some(annotation_node) = argument_node.inner.first() {
- Some(annotation_from_ast_node(annotation_node.clone())?)
- } else {
- None
- };
+ let annotation = if let Some(annotation_node) = argument_node.inner.first() {
+ Some(annotation_from_ast_node(annotation_node.clone())?)
+ } else {
+ None
+ };
- let start_offset = range
- .begin
- .spelling_loc
- .context("no start location in source range")?
- .offset;
- let end_offset = range
- .end
- .spelling_loc
- .context("no end location in source range")?
- .offset;
+ let start_offset = range.begin.spelling_loc.context("no start location in source range")?.offset;
+ let end_offset = range.end.spelling_loc.context("no end location in source range")?.offset;
let source =
- str::from_utf8(&source.as_bytes()[start_offset..=end_offset])
- .context("source range is not utf-8")?
- .into();
+ str::from_utf8(&source.as_bytes()[start_offset..=end_offset]).context("source range is not utf-8")?.into();
Ok(Self {
name,
@@ -219,58 +189,36 @@ impl MetalArgument {
match &*annotation_key {
"dsl.specialize" => {
if !annotation.is_empty() {
- bail!(
- "dsl.specialize takes no arguments, got {}",
- annotation.len()
- );
+ bail!("dsl.specialize takes no arguments, got {}", annotation.len());
}
let rust_type = Self::scalar_type_to_rust(&self.c_type)?;
Ok(MetalArgumentType::Specialize(rust_type.into()))
},
"dsl.axis" => {
if annotation.len() != 2 {
- bail!(
- "dsl.axis requires 2 arguments, got {}",
- annotation.len()
- );
+ bail!("dsl.axis requires 2 arguments, got {}", annotation.len());
}
- Ok(MetalArgumentType::Axis(
- annotation.remove(0),
- annotation.remove(0),
- ))
+ Ok(MetalArgumentType::Axis(annotation.remove(0), annotation.remove(0)))
},
"dsl.groups" => {
if annotation.len() != 1 {
- bail!(
- "dsl.groups requires 1 argument, got {}",
- annotation.len()
- );
+ bail!("dsl.groups requires 1 argument, got {}", annotation.len());
}
let dim = annotation.remove(0);
match dim.as_ref() {
- "INDIRECT" => Ok(MetalArgumentType::Groups(
- MetalGroupsType::Indirect,
- )),
- _ => Ok(MetalArgumentType::Groups(
- MetalGroupsType::Direct(dim),
- )),
+ "INDIRECT" => Ok(MetalArgumentType::Groups(MetalGroupsType::Indirect)),
+ _ => Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(dim))),
}
},
"dsl.threads" => {
if annotation.len() != 1 {
- bail!(
- "dsl.threads requires 1 argument, got {}",
- annotation.len()
- );
+ bail!("dsl.threads requires 1 argument, got {}", annotation.len());
}
Ok(MetalArgumentType::Threads(annotation.remove(0)))
},
_ => bail!("unknown annotation: {annotation_key}"),
}
- } else if self.c_type.contains("device")
- && self.c_type.contains('*')
- && !self.c_type.contains('&')
- {
+ } else if self.c_type.contains("device") && self.c_type.contains('*') && !self.c_type.contains('&') {
Ok(MetalArgumentType::Buffer)
} else if let ["const", "constant", c_type_scalar, "&"] =
self.c_type.split_whitespace().collect::<Vec<_>>().as_slice()
@@ -286,23 +234,12 @@ impl MetalArgument {
Self::scalar_type_to_rust(c_type_scalar)?.into(),
MetalConstantType::Array,
)))
- } else if self.c_type.contains("threadgroup")
- && self.c_type.contains('*')
- {
- let lbracket = self
- .source
- .rfind('[')
- .context("threadgroup missing size bracket")?
- + 1;
- let rbracket = self
- .source
- .rfind(']')
- .context("threadgroup missing size bracket")?;
+ } else if self.c_type.contains("threadgroup") && self.c_type.contains('*') {
+ let lbracket = self.source.rfind('[').context("threadgroup missing size bracket")? + 1;
+ let rbracket = self.source.rfind(']').context("threadgroup missing size bracket")?;
let size_expr = &self.source[lbracket..rbracket];
Ok(MetalArgumentType::Shared(Some(size_expr.into())))
- } else if self.c_type.contains("threadgroup")
- && self.c_type.contains('&')
- {
+ } else if self.c_type.contains("threadgroup") && self.c_type.contains('&') {
Ok(MetalArgumentType::Shared(None))
} else {
bail!("cannot parse c type: {}", self.c_type);
@@ -344,39 +281,27 @@ pub struct MetalKernelInfo {
impl MetalKernelInfo {
pub fn has_axis(&self) -> bool {
- self.arguments.iter().any(|a| {
- matches!(a.argument_type(), Ok(MetalArgumentType::Axis(..)))
- })
+ self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Axis(..))))
}
pub fn has_groups(&self) -> bool {
- self.arguments.iter().any(|a| {
- matches!(a.argument_type(), Ok(MetalArgumentType::Groups(_)))
- })
+ self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Groups(_))))
}
pub fn has_groups_direct(&self) -> bool {
- self.arguments.iter().any(|a| {
- matches!(
- a.argument_type(),
- Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(_)))
- )
- })
+ self.arguments
+ .iter()
+ .any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(_)))))
}
pub fn has_groups_indirect(&self) -> bool {
- self.arguments.iter().any(|a| {
- matches!(
- a.argument_type(),
- Ok(MetalArgumentType::Groups(MetalGroupsType::Indirect))
- )
- })
+ self.arguments
+ .iter()
+ .any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Groups(MetalGroupsType::Indirect))))
}
pub fn has_threads(&self) -> bool {
- self.arguments.iter().any(|a| {
- matches!(a.argument_type(), Ok(MetalArgumentType::Threads(_)))
- })
+ self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Threads(_))))
}
pub fn to_kernel(&self) -> Kernel {
@@ -400,26 +325,18 @@ impl MetalKernelInfo {
name: a.name.clone(),
ty: KernelArgumentType::Buffer,
}),
- Ok(MetalArgumentType::Groups(
- MetalGroupsType::Indirect,
- )) if !indirect_flag => {
+ Ok(MetalArgumentType::Groups(MetalGroupsType::Indirect)) if !indirect_flag => {
indirect_flag = true;
Some(KernelArgument {
name: "__dsl_indirect_dispatch_buffer".into(),
ty: KernelArgumentType::Buffer,
})
},
- Ok(MetalArgumentType::Constant((
- ty,
- MetalConstantType::Scalar,
- ))) => Some(KernelArgument {
+ Ok(MetalArgumentType::Constant((ty, MetalConstantType::Scalar))) => Some(KernelArgument {
name: a.name.clone(),
ty: KernelArgumentType::Scalar(ty),
}),
- Ok(MetalArgumentType::Constant((
- ty,
- MetalConstantType::Array,
- ))) => Some(KernelArgument {
+ Ok(MetalArgumentType::Constant((ty, MetalConstantType::Array))) => Some(KernelArgument {
name: a.name.clone(),
ty: KernelArgumentType::Constant(ty),
}),
@@ -435,48 +352,41 @@ impl MetalKernelInfo {
node: MetalAstNode,
source: &str,
) -> anyhow::Result<Option<Self>> {
- let (is_template, template_parameters, node) =
- if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {
- let mut template_parameters = Vec::new();
- let mut function_node = None;
-
- for child in node.inner {
- match child.kind {
- MetalAstKind::TemplateTypeParmDecl {
- name,
- } => {
- let name = name
- .context("template parameter missing name")?;
- template_parameters.push(name);
- },
- MetalAstKind::FunctionDecl {
- name: _,
- } => {
- function_node = Some(child);
- },
- _ => (),
- }
+ let (is_template, template_parameters, node) = if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {
+ let mut template_parameters = Vec::new();
+ let mut function_node = None;
+
+ for child in node.inner {
+ match child.kind {
+ MetalAstKind::TemplateTypeParmDecl {
+ name,
+ } => {
+ let name = name.context("template parameter missing name")?;
+ template_parameters.push(name);
+ },
+ MetalAstKind::FunctionDecl {
+ name: _,
+ } => {
+ function_node = Some(child);
+ },
+ _ => (),
}
+ }
- let node = function_node.context(
- "unexpected kind of root node: template without function",
- )?;
+ let node = function_node.context("unexpected kind of root node: template without function")?;
- (true, template_parameters, node)
- } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {
- (false, Vec::new(), node)
- } else {
- return Ok(None);
- };
+ (true, template_parameters, node)
+ } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {
+ (false, Vec::new(), node)
+ } else {
+ return Ok(None);
+ };
let MetalAstKind::FunctionDecl {
name,
} = node.kind
else {
- bail!(
- "unexpected kind of root node: function expected, but {:?} found",
- node.kind
- );
+ bail!("unexpected kind of root node: function expected, but {:?} found", node.kind);
};
let mut arg_nodes = Vec::new();
@@ -489,9 +399,7 @@ impl MetalKernelInfo {
range: _,
ty: _,
} => arg_nodes.push(node),
- MetalAstKind::AnnotateAttr => {
- annotations.push(annotation_from_ast_node(node)?)
- },
+ MetalAstKind::AnnotateAttr => annotations.push(annotation_from_ast_node(node)?),
_ => (),
}
}
@@ -520,10 +428,7 @@ impl MetalKernelInfo {
bail!("malformed dsl.variants annotation");
};
- let variants = variant_values
- .split(',')
- .map(|v| v.trim().into())
- .collect::<Box<[Box<str>]>>();
+ let variants = variant_values.split(',').map(|v| v.trim().into()).collect::<Box<[Box<str>]>>();
Ok(MetalTypeParameter {
name: typename.clone(),
@@ -537,20 +442,10 @@ impl MetalKernelInfo {
}
if is_template {
- let template_names = template_parameters
- .iter()
- .map(|name| name.as_ref())
- .collect::<Vec<_>>();
- let variant_names = variants
- .iter()
- .map(|variant| variant.name.as_ref())
- .collect::<Vec<_>>();
+ let template_names = template_parameters.iter().map(|name| name.as_ref()).collect::<Vec<_>>();
+ let variant_names = variants.iter().map(|variant| variant.name.as_ref()).collect::<Vec<_>>();
if template_names != variant_names {
- bail!(
- "template parameters {:?} do not match dsl.variants order {:?}",
- template_names,
- variant_names
- );
+ bail!("template parameters {:?} do not match dsl.variants order {:?}", template_names, variant_names);
}
}
@@ -574,25 +469,24 @@ impl MetalKernelInfo {
}
pub fn validate_raw_kernel(node: &MetalAstNode) -> anyhow::Result<()> {
- let (node, is_template) =
- if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {
- let inner_fn = node.inner.iter().find(|c| {
- matches!(
- c.kind,
- MetalAstKind::FunctionDecl {
- name: _
- }
- )
- });
- match inner_fn {
- Some(n) => (n, true),
- None => return Ok(()),
- }
- } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {
- (node, false)
- } else {
- return Ok(());
- };
+ let (node, is_template) = if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {
+ let inner_fn = node.inner.iter().find(|c| {
+ matches!(
+ c.kind,
+ MetalAstKind::FunctionDecl {
+ name: _
+ }
+ )
+ });
+ match inner_fn {
+ Some(n) => (n, true),
+ None => return Ok(()),
+ }
+ } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {
+ (node, false)
+ } else {
+ return Ok(());
+ };
let MetalAstKind::FunctionDecl {
name,
@@ -608,15 +502,10 @@ pub fn validate_raw_kernel(node: &MetalAstNode) -> anyhow::Result<()> {
for child in &node.inner {
match &child.kind {
MetalAstKind::MetalKernelAttr => has_metal_kernel_attr = true,
- MetalAstKind::MetalMaxTotalThreadsPerThreadGroupAttr => {
- has_max_threads_attr = true
- },
+ MetalAstKind::MetalMaxTotalThreadsPerThreadGroupAttr => has_max_threads_attr = true,
MetalAstKind::AnnotateAttr => {
- if let Ok(annotation) = annotation_from_ast_node(child.clone())
- {
- if annotation.first().map(|s| s.as_ref())
- == Some("dsl.kernel")
- {
+ if let Ok(annotation) = annotation_from_ast_node(child.clone()) {
+ if annotation.first().map(|s| s.as_ref()) == Some("dsl.kernel") {
has_dsl_kernel = true;
}
}
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index 9b556c8da..2f9e34446 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -22,33 +22,18 @@ pub fn bindgen(
let struct_name = format_ident!("{kernel_name}MetalKernel");
let parse_expr = |expr: &Box<str>| -> anyhow::Result<TokenStream> {
- syn::parse_str(expr.as_ref()).with_context(|| {
- format!(
- "cannot parse rust expression `{}` in kernel `{}`",
- expr, kernel_name
- )
- })
+ syn::parse_str(expr.as_ref())
+ .with_context(|| format!("cannot parse rust expression `{}` in kernel `{}`", expr, kernel_name))
};
- let (variants_extra_arguments, variants_kernel_format) = if let Some(
- variants,
- ) =
- &kernel.variants
- {
- let variant_names = variants
- .iter()
- .map(|type_parameter| {
- format_ident!("{}", type_parameter.name.as_ref())
- })
- .collect::<Vec<_>>();
+ let (variants_extra_arguments, variants_kernel_format) = if let Some(variants) = &kernel.variants {
+ let variant_names =
+ variants.iter().map(|type_parameter| format_ident!("{}", type_parameter.name.as_ref())).collect::<Vec<_>>();
let kernel_format = repeat_n("{}", variant_names.len() + 1).join("_");
(
- variant_names
- .iter()
- .map(|name| quote! { #[allow(non_snake_case)] #name: crate::DataType })
- .collect(),
+ variant_names.iter().map(|name| quote! { #[allow(non_snake_case)] #name: crate::DataType }).collect(),
quote! { &format!(#kernel_format, #kernel_name #(, KernelDataType::from(#variant_names).function_name_suffix())*) },
)
} else {
@@ -56,15 +41,10 @@ pub fn bindgen(
};
let base_index = specialize_indices.get(&kernel.name).copied();
- let (specialize_args, specialize_setup): (
- Vec<TokenStream>,
- Vec<TokenStream>,
- ) = kernel
+ let (specialize_args, specialize_setup): (Vec<TokenStream>, Vec<TokenStream>) = kernel
.arguments
.iter()
- .filter(|a| {
- matches!(a.argument_type(), Ok(MetalArgumentType::Specialize(_)))
- })
+ .filter(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Specialize(_))))
.enumerate()
.map(|(i, a)| {
let arg_name = format_ident!("{}", a.name.as_ref());
@@ -166,8 +146,7 @@ pub fn bindgen(
})
.multiunzip();
- let encode_generics =
- encode_generics.into_iter().flatten().collect::<Vec<_>>();
+ let encode_generics = encode_generics.into_iter().flatten().collect::<Vec<_>>();
let (dispatch, elements) = if kernel.has_axis() {
if kernel.has_groups() || kernel.has_threads() {
@@ -178,10 +157,9 @@ pub fn bindgen(
.arguments
.iter()
.filter_map(|a| match a.argument_type() {
- Ok(MetalArgumentType::Axis(
- threads_rexprs,
- threads_per_group_rexprs,
- )) => Some((threads_rexprs, threads_per_group_rexprs)),
+ Ok(MetalArgumentType::Axis(threads_rexprs, threads_per_group_rexprs)) => {
+ Some((threads_rexprs, threads_per_group_rexprs))
+ },
_ => None,
})
.map(|(threads_rexprs, threads_per_group_rexprs)| {
@@ -192,8 +170,7 @@ pub fn bindgen(
.collect::<anyhow::Result<Vec<(TokenStream, TokenStream)>>>()?;
axis.extend(repeat_n((quote! {1}, quote! {1}), 3 - axis.len()));
- let (threads, threads_per_group): (Vec<TokenStream>, Vec<TokenStream>) =
- axis.into_iter().unzip();
+ let (threads, threads_per_group): (Vec<TokenStream>, Vec<TokenStream>) = axis.into_iter().unzip();
(
quote! {
@@ -209,9 +186,7 @@ pub fn bindgen(
.arguments
.iter()
.filter_map(|a| {
- if let Ok(MetalArgumentType::Threads(rexprs)) =
- a.argument_type()
- {
+ if let Ok(MetalArgumentType::Threads(rexprs)) = a.argument_type() {
Some(parse_expr(&rexprs))
} else {
None
@@ -241,10 +216,7 @@ pub fn bindgen(
.arguments
.iter()
.filter_map(|a| {
- if let Ok(MetalArgumentType::Groups(
- MetalGroupsType::Direct(rexprs),
- )) = a.argument_type()
- {
+ if let Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(rexprs))) = a.argument_type() {
Some(parse_expr(&rexprs))
} else {
None
diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rs
index cf232c231..45ec2ff0a 100644
--- a/crates/uzu/build/metal/compiler.rs
+++ b/crates/uzu/build/metal/compiler.rs
@@ -19,10 +19,7 @@ use super::{
wrapper::{SpecializeBaseIndices, wrappers},
};
use crate::{
- common::{
- caching, codegen::write_tokens, compiler::Compiler, envs,
- kernel::Kernel,
- },
+ common::{caching, codegen::write_tokens, compiler::Compiler, envs, kernel::Kernel},
debug_log,
};
@@ -55,10 +52,7 @@ impl ObjectInfo {
}
fn is_nax_source(path: &Path) -> bool {
- path.file_name()
- .and_then(|s| s.to_str())
- .map(|s| s.contains("_nax"))
- .unwrap_or(false)
+ path.file_name().and_then(|s| s.to_str()).map(|s| s.contains("_nax")).unwrap_or(false)
}
async fn hash_dependencies(
@@ -66,41 +60,28 @@ async fn hash_dependencies(
) -> anyhow::Result<HashMap<Box<str>, [u8; blake3::OUT_LEN]>> {
let futures: Vec<_> = dependencies
.map(|path| async move {
- let mut file = tokio::fs::File::open(path.as_ref())
- .await
- .with_context(|| format!("cannot open file {}", path))?;
+ let mut file =
+ tokio::fs::File::open(path.as_ref()).await.with_context(|| format!("cannot open file {}", path))?;
let mut contents = Vec::new();
- file.read_to_end(&mut contents)
- .await
- .with_context(|| format!("cannot read file {}", path))?;
- let hash = spawn_blocking(move || blake3::hash(&contents))
- .await
- .context("hash task panicked")?;
+ file.read_to_end(&mut contents).await.with_context(|| format!("cannot read file {}", path))?;
+ let hash = spawn_blocking(move || blake3::hash(&contents)).await.context("hash task panicked")?;
Ok::<_, anyhow::Error>((path, hash.into()))
})
.collect();
try_join_all(futures).await.map(|v| v.into_iter().collect())
}
-fn objects_hash<'a>(
- objects: impl IntoIterator<Item = &'a ObjectInfo>
-) -> anyhow::Result<blake3::Hash> {
+fn objects_hash<'a>(objects: impl IntoIterator<Item = &'a ObjectInfo>) -> anyhow::Result<blake3::Hash> {
let mut hasher = blake3::Hasher::new();
- hasher.update(
- caching::build_system_hash()
- .context("cannot get build system hash")?
- .as_bytes(),
- );
- let mut paths: Vec<_> =
- objects.into_iter().map(|o| &o.object_path).collect();
+ hasher.update(caching::build_system_hash().context("cannot get build system hash")?.as_bytes());
+ let mut paths: Vec<_> = objects.into_iter().map(|o| &o.object_path).collect();
paths.sort();
for path in paths {
let path_bytes = path.to_string_lossy();
let path_bytes = path_bytes.as_bytes();
hasher.update(&(path_bytes.len() as u32).to_le_bytes());
hasher.update(path_bytes);
- let contents = fs::read(path)
- .with_context(|| format!("cannot read {}", path.display()))?;
+ let contents = fs::read(path).with_context(|| format!("cannot read {}", path.display()))?;
hasher.update(&(contents.len() as u32).to_le_bytes());
hasher.update(&contents);
}
@@ -117,22 +98,15 @@ pub struct MetalCompiler {
impl MetalCompiler {
pub fn new_with_include_dir(include_dir: PathBuf) -> anyhow::Result<Self> {
- let src_dir = PathBuf::from(
- env::var("CARGO_MANIFEST_DIR")
- .context("missing CARGO_MANIFEST_DIR")?,
- )
- .join("src/backends/metal/kernel");
+ let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?)
+ .join("src/backends/metal/kernel");
- let out_dir =
- PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
+ let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
let build_dir = out_dir.join("metal");
- fs::create_dir_all(&build_dir).with_context(|| {
- format!("cannot create {}", build_dir.display())
- })?;
+ fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;
let toolchain =
- MetalToolchain::from_env_with_include_dir(Some(include_dir))
- .context("cannot create toolchain")?;
+ MetalToolchain::from_env_with_include_dir(Some(include_dir)).context("cannot create toolchain")?;
Ok(Self {
src_dir,
@@ -146,55 +120,35 @@ impl MetalCompiler {
&self,
source_path: PathBuf,
) -> anyhow::Result<ObjectInfo> {
- let buildsystem_hash = caching::build_system_hash()
- .context("cannot get build system cache")?
- .as_bytes()
- .clone();
+ let buildsystem_hash =
+ caching::build_system_hash().context("cannot get build system cache")?.as_bytes().clone();
- let source_path_pretty =
- source_path.file_name().unwrap().to_str().unwrap();
+ let source_path_pretty = source_path.file_name().unwrap().to_str().unwrap();
debug_log!("compile start: {source_path_pretty}");
let source_path_display = source_path.display().to_string();
- let source_path_hash =
- blake3::hash(source_path.to_string_lossy().as_bytes());
- let source_file_name = source_path
- .with_extension("")
- .file_name()
- .context("source path has no file name")?
- .to_os_string();
+ let source_path_hash = blake3::hash(source_path.to_string_lossy().as_bytes());
+ let source_file_name =
+ source_path.with_extension("").file_name().context("source path has no file name")?.to_os_string();
- let source_path_rel = source_path
- .strip_prefix(&self.src_dir)
- .context("source is not in src_dir")?;
+ let source_path_rel = source_path.strip_prefix(&self.src_dir).context("source is not in src_dir")?;
let build_dir = self.build_dir.join(source_path_hash.to_string());
- let objectinfo_path =
- build_dir.join(&source_file_name).with_extension("objectinfo");
+ let objectinfo_path = build_dir.join(&source_file_name).with_extension("objectinfo");
if build_dir.exists() {
if let Ok(contents) = tokio::fs::read(&objectinfo_path).await
- && let Ok(cached) =
- serde_json::from_slice::<ObjectInfo>(&contents)
+ && let Ok(cached) = serde_json::from_slice::<ObjectInfo>(&contents)
&& buildsystem_hash == cached.buildsystem_hash
- && let Ok(dependency_hashes) =
- hash_dependencies(cached.dependency_hashes.keys().cloned())
- .await
+ && let Ok(dependency_hashes) = hash_dependencies(cached.dependency_hashes.keys().cloned()).await
&& dependency_hashes == cached.dependency_hashes
{
if envs::build_debug() {
- let kernel_list = cached
- .kernels
- .iter()
- .map(|k| k.name.as_ref())
- .collect::<Vec<_>>()
- .join(", ");
+ let kernel_list = cached.kernels.iter().map(|k| k.name.as_ref()).collect::<Vec<_>>().join(", ");
if kernel_list.is_empty() {
debug_log!("compile cached: {source_path_pretty}");
} else {
- debug_log!(
- "compile cached: {source_path_pretty} (kernels: [{kernel_list}])"
- );
+ debug_log!("compile cached: {source_path_pretty} (kernels: [{kernel_list}])");
}
}
return Ok(cached);
@@ -202,35 +156,31 @@ impl MetalCompiler {
fs::remove_dir_all(&build_dir).ok();
}
- fs::create_dir_all(&build_dir).with_context(|| {
- format!("cannot create build directory {}", build_dir.display())
- })?;
+ fs::create_dir_all(&build_dir)
+ .with_context(|| format!("cannot create build directory {}", build_dir.display()))?;
- let (metal_kernel_infos, dependencies) =
- self.toolchain.analyze(&source_path).await.with_context(|| {
- format!("cannot analyze {}", source_path_display)
- })?;
+ let (metal_kernel_infos, dependencies) = self
+ .toolchain
+ .analyze(&source_path)
+ .await
+ .with_context(|| format!("cannot analyze {}", source_path_display))?;
let kernel_infos: Vec<MetalKernelInfo> = metal_kernel_infos.collect();
- let (wrapper_strs, specialize_indices) = wrappers(&kernel_infos)
- .context("cannot generate kernel wrappers")?;
+ let (wrapper_strs, specialize_indices) = wrappers(&kernel_infos).context("cannot generate kernel wrappers")?;
let mut footer = String::new();
for wrapper in wrapper_strs.iter() {
footer.push_str(wrapper);
}
- let object_path =
- build_dir.join(&source_file_name).with_extension("air");
+ let object_path = build_dir.join(&source_file_name).with_extension("air");
let compile_output = self
.toolchain
.compile(&source_path, &footer, &object_path)
.await
- .with_context(|| {
- format!("cannot compile {}", source_path_display)
- })?;
+ .with_context(|| format!("cannot compile {}", source_path_display))?;
if let Some(warnings) = &compile_output {
for line in warnings.lines() {
@@ -238,9 +188,7 @@ impl MetalCompiler {
}
}
- let dependency_hashes = hash_dependencies(dependencies)
- .await
- .context("cannot hash dependencies")?;
+ let dependency_hashes = hash_dependencies(dependencies).await.context("cannot hash dependencies")?;
let object_info = ObjectInfo {
src_rel_path: source_path_rel.into(),
@@ -253,27 +201,16 @@ impl MetalCompiler {
fs::write(
&objectinfo_path,
- serde_json::to_string_pretty(&object_info)
- .context("failed to serialize object info")?
- .as_bytes(),
+ serde_json::to_string_pretty(&object_info).context("failed to serialize object info")?.as_bytes(),
)
- .with_context(|| {
- format!("cannot write object info {}", objectinfo_path.display())
- })?;
+ .with_context(|| format!("cannot write object info {}", objectinfo_path.display()))?;
if envs::build_debug() {
- let kernel_list = object_info
- .kernels
- .iter()
- .map(|k| k.name.as_ref())
- .collect::<Vec<_>>()
- .join(", ");
+ let kernel_list = object_info.kernels.iter().map(|k| k.name.as_ref()).collect::<Vec<_>>().join(", ");
if kernel_list.is_empty() {
debug_log!("compile end: {source_path_pretty}");
} else {
- debug_log!(
- "compile end: {source_path_pretty} (kernels: [{kernel_list}])"
- );
+ debug_log!("compile end: {source_path_pretty} (kernels: [{kernel_list}])");
}
}
@@ -310,9 +247,8 @@ impl MetalCompiler {
}
}
- fs::write(&hash_path, hash.to_string()).with_context(|| {
- format!("cannot write hash file {}", hash_path.display())
- })?;
+ fs::write(&hash_path, hash.to_string())
+ .with_context(|| format!("cannot write hash file {}", hash_path.display()))?;
debug_log!("linking end");
@@ -341,12 +277,10 @@ impl MetalCompiler {
.into_iter()
.flat_map(|o| o.kernels.iter().map(|k| (k, &o.specialize_indices)))
.map(|(k, specialize_indices)| {
- super::bindgen::bindgen(k, specialize_indices).with_context(
- || format!("cannot generate bindings for {}", k.name),
- )
+ super::bindgen::bindgen(k, specialize_indices)
+ .with_context(|| format!("cannot generate bindings for {}", k.name))
})
- .collect::<anyhow::Result<(Vec<TokenStream>, Vec<TokenStream>)>>(
- )?;
+ .collect::<anyhow::Result<(Vec<TokenStream>, Vec<TokenStream>)>>()?;
let tokens = quote! {
use crate::backends::metal::{
@@ -376,9 +310,8 @@ impl MetalCompiler {
write_tokens(tokens, &out_path).context("cannot write bindings")?;
- fs::write(&hash_path, hash.to_string()).with_context(|| {
- format!("cannot write hash file {}", hash_path.display())
- })?;
+ fs::write(&hash_path, hash.to_string())
+ .with_context(|| format!("cannot write hash file {}", hash_path.display()))?;
debug_log!("bindgen end");
@@ -388,9 +321,7 @@ impl MetalCompiler {
#[async_trait]
impl Compiler for MetalCompiler {
- async fn build(
- &self
- ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
+ async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
let nax_enabled = cfg!(feature = "metal-nax");
debug_log!(
"metal nax {}",
@@ -404,18 +335,12 @@ impl Compiler for MetalCompiler {
let metal_sources: Vec<PathBuf> = WalkDir::new(&self.src_dir)
.into_iter()
.filter_map(|e| e.ok())
- .filter(|e| {
- e.file_type().is_file()
- && e.path().extension().and_then(|s| s.to_str())
- == Some("metal")
- })
+ .filter(|e| e.file_type().is_file() && e.path().extension().and_then(|s| s.to_str()) == Some("metal"))
.map(|e| e.into_path())
.filter(|p| nax_enabled || !is_nax_source(p))
.collect();
- let num_concurrent_compiles =
- std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4)
- * 2;
+ let num_concurrent_compiles = std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4) * 2;
let objects: Vec<ObjectInfo> = stream::iter(metal_sources)
.map(|p| self.compile(p))
diff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rs
index 0c3d540cb..7c4743f10 100644
--- a/crates/uzu/build/metal/toolchain.rs
+++ b/crates/uzu/build/metal/toolchain.rs
@@ -10,9 +10,7 @@ use anyhow::{Context, bail};
use tempfile::NamedTempFile;
use tokio::{io::AsyncWriteExt, process::Command};
-use super::ast::{
- MetalAstKind, MetalAstNode, MetalKernelInfo, validate_raw_kernel,
-};
+use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo, validate_raw_kernel};
#[derive(Debug)]
pub enum MetalSdk {
@@ -31,9 +29,7 @@ impl MetalSdk {
if target_os == "ios" {
if target.contains("macabi") || target_env == "macabi" {
Ok(Self::MacCatalyst)
- } else if target.contains("ios")
- && (target.contains("86_64") || target_env == "sim")
- {
+ } else if target.contains("ios") && (target.contains("86_64") || target_env == "sim") {
Ok(Self::IPhoneSimulator)
} else {
Ok(Self::IPhoneOS)
@@ -41,18 +37,14 @@ impl MetalSdk {
} else if target_os == "macos" {
Ok(Self::MacOSX)
} else {
- bail!(
- "cannot find matching metal sdk for ({target:?}, {target_os:?}, {target_env:?})"
- );
+ bail!("cannot find matching metal sdk for ({target:?}, {target_os:?}, {target_env:?})");
}
}
pub fn from_env() -> anyhow::Result<Self> {
let target = env::var("TARGET").context("missing TARGET")?;
- let target_os = env::var("CARGO_CFG_TARGET_OS")
- .context("missing CARGO_CFG_TARGET_OS")?;
- let target_env = env::var("CARGO_CFG_TARGET_ENV")
- .context("missing CARGO_CFG_TARGET_ENV")?;
+ let target_os = env::var("CARGO_CFG_TARGET_OS").context("missing CARGO_CFG_TARGET_OS")?;
+ let target_env = env::var("CARGO_CFG_TARGET_ENV").context("missing CARGO_CFG_TARGET_ENV")?;
let sdk = Self::from_parts(&target, &target_os, &target_env)?;
@@ -118,21 +110,16 @@ pub struct MetalToolchain {
}
impl MetalToolchain {
- pub fn from_env_with_include_dir(
- include_dir: Option<PathBuf>
- ) -> anyhow::Result<Self> {
+ pub fn from_env_with_include_dir(include_dir: Option<PathBuf>) -> anyhow::Result<Self> {
let sdk = MetalSdk::from_env().context("cannot get sdk")?;
let std = MetalStd::default();
- let opt_flags = match env::var("OPT_LEVEL")
- .context("missing OPT_LEVEL")?
- .as_str()
- {
+ let opt_flags = match env::var("OPT_LEVEL").context("missing OPT_LEVEL")?.as_str() {
"0" => {
[
OsString::from("-O1"), // matmul kernels compiled with -O0 are broken and require a reboot to unfreeze the os
OsString::from("-gline-tables-only"), // debug with line tables only
- OsString::from("-frecord-sources"), // include source code
+ OsString::from("-frecord-sources"), // include source code
]
.into()
},
@@ -141,11 +128,7 @@ impl MetalToolchain {
};
let extra_options = Box::new([
- OsString::from(format!(
- "-m{}-version-min={}",
- sdk.os(),
- std.min_os()
- )),
+ OsString::from(format!("-m{}-version-min={}", sdk.os(), std.min_os())),
// NOTE: Previous build.rs script didn't forward metal compiler warnings to cargo warnings, the new one does.
// This is temporary to avoid warning spam without modifying unrelated things in the same pull request as introducing the dsl and converting kernels
OsString::from("-Wno-sign-compare"),
@@ -182,14 +165,10 @@ impl MetalToolchain {
pub async fn analyze(
&self,
path: impl AsRef<Path>,
- ) -> anyhow::Result<(
- impl Iterator<Item = MetalKernelInfo>,
- impl Iterator<Item = Box<str>>,
- )> {
+ ) -> anyhow::Result<(impl Iterator<Item = MetalKernelInfo>, impl Iterator<Item = Box<str>>)> {
let path = path.as_ref();
- let depfile_path =
- NamedTempFile::new().context("cannot create temporary file")?;
+ let depfile_path = NamedTempFile::new().context("cannot create temporary file")?;
let mut cmd = self.xcrun();
cmd.arg("metal")
@@ -208,19 +187,14 @@ impl MetalToolchain {
.stdout(Stdio::piped())
.stderr(Stdio::piped());
- let analyze_output =
- cmd.output().await.context("cannot execute metal analyzer")?;
+ let analyze_output = cmd.output().await.context("cannot execute metal analyzer")?;
if !analyze_output.status.success() {
- bail!(
- "metal analyzer failed: {}",
- String::from_utf8_lossy(&analyze_output.stderr)
- );
+ bail!("metal analyzer failed: {}", String::from_utf8_lossy(&analyze_output.stderr));
}
let ast_root =
- serde_json::from_slice::<MetalAstNode>(&analyze_output.stdout)
- .context("cannot deserialize ast dump")?;
+ serde_json::from_slice::<MetalAstNode>(&analyze_output.stdout).context("cannot deserialize ast dump")?;
if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {
bail!(
@@ -229,30 +203,21 @@ impl MetalToolchain {
);
}
- let source_contents =
- fs::read_to_string(path).context("cannot read source file")?;
+ let source_contents = fs::read_to_string(path).context("cannot read source file")?;
for node in &ast_root.inner {
- validate_raw_kernel(node)
- .context("validation of legacy (non-dsl) kernel failed")?;
+ validate_raw_kernel(node).context("validation of legacy (non-dsl) kernel failed")?;
}
let kernel_infos = ast_root
.inner
.into_iter()
- .filter_map(|node| {
- MetalKernelInfo::from_ast_node_and_source(
- node,
- &source_contents,
- )
- .transpose()
- })
+ .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())
.collect::<anyhow::Result<Vec<_>>>()
.context("cannot parse kernel infos from AST")?
.into_iter();
- let depfile_contents = fs::read_to_string(depfile_path.path())
- .context("cannot read depfile")?;
+ let depfile_contents = fs::read_to_string(depfile_path.path()).context("cannot read depfile")?;
let dependencies = depfile::parse(&depfile_contents)
.map_err(|e| anyhow::anyhow!("cannot parse depfile: {e}"))?
@@ -289,8 +254,7 @@ impl MetalToolchain {
.stdin(Stdio::piped())
.stderr(Stdio::piped());
- let mut compile_child =
- cmd.spawn().context("cannot execute metal compiler")?;
+ let mut compile_child = cmd.spawn().context("cannot execute metal compiler")?;
compile_child
.stdin
@@ -300,14 +264,9 @@ impl MetalToolchain {
.await
.context("cannot write to metal compiler stdin")?;
- let compile_output = compile_child
- .wait_with_output()
- .await
- .context("cannot wait on metal compiler")?;
+ let compile_output = compile_child.wait_with_output().await.context("cannot wait on metal compiler")?;
- let stderr = String::from_utf8_lossy(&compile_output.stderr)
- .into_owned()
- .into_boxed_str();
+ let stderr = String::from_utf8_lossy(&compile_output.stderr).into_owned().into_boxed_str();
if !compile_output.status.success() {
bail!("metal compiler failed: {stderr}");
@@ -330,18 +289,14 @@ impl MetalToolchain {
let link_output = self
.xcrun()
.arg("metallib")
- .args(
- objects.into_iter().map(|p| p.as_ref().as_os_str().to_owned()),
- )
+ .args(objects.into_iter().map(|p| p.as_ref().as_os_str().to_owned()))
.arg("-o")
.arg(output.as_ref())
.output()
.await
.context("cannot execute metal linker")?;
- let stderr = String::from_utf8_lossy(&link_output.stderr)
- .into_owned()
- .into_boxed_str();
+ let stderr = String::from_utf8_lossy(&link_output.stderr).into_owned().into_boxed_str();
if !link_output.status.success() {
bail!("metal linker failed: {stderr}");
diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rs
index 6348506a1..9e0825467 100644
--- a/crates/uzu/build/metal/wrapper.rs
+++ b/crates/uzu/build/metal/wrapper.rs
@@ -7,9 +7,7 @@ use super::ast::{MetalArgumentType, MetalKernelInfo};
pub type SpecializeBaseIndices = HashMap<Box<str>, usize>;
-pub fn wrappers(
- kernels: &[MetalKernelInfo]
-) -> anyhow::Result<(Box<[Box<str>]>, SpecializeBaseIndices)> {
+pub fn wrappers(kernels: &[MetalKernelInfo]) -> anyhow::Result<(Box<[Box<str>]>, SpecializeBaseIndices)> {
let mut all_wrappers = Vec::new();
let mut base_indices = SpecializeBaseIndices::new();
let mut next_index = 0usize;
@@ -18,12 +16,7 @@ pub fn wrappers(
let specialize_count = kernel
.arguments
.iter()
- .filter(|a| {
- matches!(
- a.argument_type(),
- Ok(MetalArgumentType::Specialize(_))
- )
- })
+ .filter(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Specialize(_))))
.count();
if specialize_count > 0 {
@@ -31,8 +24,7 @@ pub fn wrappers(
next_index += specialize_count;
}
- let kernel_wrappers =
- kernel_wrappers(kernel, base_indices.get(&kernel.name))?;
+ let kernel_wrappers = kernel_wrappers(kernel, base_indices.get(&kernel.name))?;
all_wrappers.extend(kernel_wrappers.into_vec());
}
@@ -54,10 +46,7 @@ fn kernel_wrappers(
.map(|(i, a)| {
let c_type = a.c_type.trim_start_matches("const ");
let idx = base + i;
- format!(
- "constant {c_type} __dsl_specialize_{}_{} [[function_constant({idx})]];\n",
- kernel.name, a.name
- )
+ format!("constant {c_type} __dsl_specialize_{}_{} [[function_constant({idx})]];\n", kernel.name, a.name)
})
.collect::<String>()
} else {
@@ -82,52 +71,37 @@ fn kernel_wrappers(
} else {
vec![None]
} {
- let (wrapper_name, underlying_name) =
- if let Some(type_variant) = &type_variant {
- (
- format!(
- "{}_{}",
- kernel.name,
- type_variant.iter().map(|(_k, v)| v).join("_")
- ),
- format!(
- "{}<{}>",
- kernel.name,
- type_variant.iter().map(|(_k, v)| v).join(", ")
- ),
- )
- } else {
- (kernel.name.to_string(), kernel.name.to_string())
- };
+ let (wrapper_name, underlying_name) = if let Some(type_variant) = &type_variant {
+ (
+ format!("{}_{}", kernel.name, type_variant.iter().map(|(_k, v)| v).join("_")),
+ format!("{}<{}>", kernel.name, type_variant.iter().map(|(_k, v)| v).join(", ")),
+ )
+ } else {
+ (kernel.name.to_string(), kernel.name.to_string())
+ };
let max_total_threads_per_threadgroup = kernel
.arguments
.iter()
.filter_map(|a| match a.argument_type() {
- Ok(MetalArgumentType::Axis(_, l))
- | Ok(MetalArgumentType::Threads(l)) => Some(format!("({l})")),
+ Ok(MetalArgumentType::Axis(_, l)) | Ok(MetalArgumentType::Threads(l)) => Some(format!("({l})")),
_ => None,
})
.collect::<Vec<_>>();
- let max_total_threads_per_threadgroup =
- if !max_total_threads_per_threadgroup.is_empty() {
- max_total_threads_per_threadgroup.join(" * ")
- } else {
- "1".to_string()
- };
+ let max_total_threads_per_threadgroup = if !max_total_threads_per_threadgroup.is_empty() {
+ max_total_threads_per_threadgroup.join(" * ")
+ } else {
+ "1".to_string()
+ };
- let wrapper_argument_replace = type_variant
- .as_ref()
- .map(|tv| tv.iter().cloned().collect::<HashMap<_, _>>());
+ let wrapper_argument_replace = type_variant.as_ref().map(|tv| tv.iter().cloned().collect::<HashMap<_, _>>());
let apply_replace = |s: &str| {
s.split_whitespace()
.map(|token| {
- if let Some(wrapper_argument_replace) =
- &wrapper_argument_replace
- && let Some(replacement) =
- wrapper_argument_replace.get(token)
+ if let Some(wrapper_argument_replace) = &wrapper_argument_replace
+ && let Some(replacement) = wrapper_argument_replace.get(token)
{
replacement
} else {
@@ -141,22 +115,12 @@ fn kernel_wrappers(
let mut wrapper_arguments = kernel
.arguments
.iter()
- .filter(|a| {
- matches!(
- a.argument_type(),
- Ok(MetalArgumentType::Buffer)
- | Ok(MetalArgumentType::Constant(_))
- )
- })
+ .filter(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Buffer) | Ok(MetalArgumentType::Constant(_))))
.enumerate()
.map(|(i, a)| match a.argument_type() {
- Ok(MetalArgumentType::Buffer)
- | Ok(MetalArgumentType::Constant(_)) => format!(
- "{} {} [[buffer({})]]",
- apply_replace(&a.c_type),
- a.name,
- i
- ),
+ Ok(MetalArgumentType::Buffer) | Ok(MetalArgumentType::Constant(_)) => {
+ format!("{} {} [[buffer({})]]", apply_replace(&a.c_type), a.name, i)
+ },
_ => unreachable!(),
})
.collect::<Vec<_>>();
@@ -166,41 +130,28 @@ fn kernel_wrappers(
bail!("mixing groups/threads and axis is not supported");
}
- wrapper_arguments.push(
- "uint3 __dsl_axis_idx [[thread_position_in_grid]]".into(),
- );
+ wrapper_arguments.push("uint3 __dsl_axis_idx [[thread_position_in_grid]]".into());
}
if kernel.has_groups() {
- wrapper_arguments.push(
- "uint3 __dsl_group_idx [[threadgroup_position_in_grid]]".into(),
- );
+ wrapper_arguments.push("uint3 __dsl_group_idx [[threadgroup_position_in_grid]]".into());
}
if kernel.has_threads() {
- wrapper_arguments.push(
- "uint3 __dsl_thread_idx [[thread_position_in_threadgroup]]"
- .into(),
- );
+ wrapper_arguments.push("uint3 __dsl_thread_idx [[thread_position_in_threadgroup]]".into());
}
let wrapper_arguments = wrapper_arguments.join(", ");
- let shared_definitions =
- kernel.arguments.iter().filter_map(|a| match a.argument_type() {
- Ok(MetalArgumentType::Shared(Some(len))) => Some(format!(
- "{} {}[{}]",
- apply_replace(&a.c_type.replace('*', "")),
- a.name,
- len.as_ref(),
- )),
- Ok(MetalArgumentType::Shared(None)) => Some(format!(
- "{} {}",
- apply_replace(&a.c_type.replace('&', "")),
- a.name
- )),
- _ => None,
- });
+ let shared_definitions = kernel.arguments.iter().filter_map(|a| match a.argument_type() {
+ Ok(MetalArgumentType::Shared(Some(len))) => {
+ Some(format!("{} {}[{}]", apply_replace(&a.c_type.replace('*', "")), a.name, len.as_ref(),))
+ },
+ Ok(MetalArgumentType::Shared(None)) => {
+ Some(format!("{} {}", apply_replace(&a.c_type.replace('&', "")), a.name))
+ },
+ _ => None,
+ });
let underlying_arguments = {
let mut group_axis_letters = ["x", "y", "z"].iter();
@@ -210,43 +161,30 @@ fn kernel_wrappers(
.arguments
.iter()
.map(|a| match a.argument_type().unwrap() {
- MetalArgumentType::Buffer
- | MetalArgumentType::Constant(_)
- | MetalArgumentType::Shared(_) => a.name.to_string(),
+ MetalArgumentType::Buffer | MetalArgumentType::Constant(_) | MetalArgumentType::Shared(_) => {
+ a.name.to_string()
+ },
MetalArgumentType::Specialize(_) => {
format!("__dsl_specialize_{}_{}", kernel.name, a.name)
},
MetalArgumentType::Axis(..) => {
- format!(
- "__dsl_axis_idx.{}",
- group_axis_letters.next().unwrap()
- )
+ format!("__dsl_axis_idx.{}", group_axis_letters.next().unwrap())
},
MetalArgumentType::Groups(_) => {
- format!(
- "__dsl_group_idx.{}",
- group_axis_letters.next().unwrap()
- )
+ format!("__dsl_group_idx.{}", group_axis_letters.next().unwrap())
},
MetalArgumentType::Threads(_) => {
- format!(
- "__dsl_thread_idx.{}",
- thread_axis_letters.next().unwrap()
- )
+ format!("__dsl_thread_idx.{}", thread_axis_letters.next().unwrap())
},
})
.collect::<Vec<_>>()
.join(", ")
};
- let underlying_call =
- format!("{underlying_name}({underlying_arguments})");
+ let underlying_call = format!("{underlying_name}({underlying_arguments})");
- let wrapper_body = shared_definitions
- .chain(once(underlying_call))
- .map(|l| format!(" {l};\n"))
- .collect::<Vec<_>>()
- .join("");
+ let wrapper_body =
+ shared_definitions.chain(once(underlying_call)).map(|l| format!(" {l};\n")).collect::<Vec<_>>().join("");
kernel_wrappers.push(
format!(
diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rs
index 07524801f..67ac64b41 100644
--- a/crates/uzu/src/array.rs
+++ b/crates/uzu/src/array.rs
@@ -61,14 +61,7 @@ impl<B: Backend> Array<B> {
&self,
shape: &[usize],
) -> Self {
- unsafe {
- Self::from_parts(
- self.buffer.clone(),
- self.offset,
- shape,
- self.data_type,
- )
- }
+ unsafe { Self::from_parts(self.buffer.clone(), self.offset, shape, self.data_type) }
}
// Getters
@@ -112,21 +105,11 @@ impl<B: Backend> Array<B> {
}
pub fn as_bytes(&self) -> &[u8] {
- unsafe {
- std::slice::from_raw_parts(
- self.cpu_ptr().as_ptr() as *const u8,
- self.size(),
- )
- }
+ unsafe { std::slice::from_raw_parts(self.cpu_ptr().as_ptr() as *const u8, self.size()) }
}
pub fn as_bytes_mut(&mut self) -> &mut [u8] {
- unsafe {
- std::slice::from_raw_parts_mut(
- self.cpu_ptr().as_ptr() as *mut u8,
- self.size(),
- )
- }
+ unsafe { std::slice::from_raw_parts_mut(self.cpu_ptr().as_ptr() as *mut u8, self.size()) }
}
pub fn as_slice<T: ArrayElement>(&self) -> &[T] {
@@ -140,8 +123,7 @@ impl<B: Backend> Array<B> {
}
pub fn as_view<T: ArrayElement>(&self) -> ArrayView<'_, T, IxDyn> {
- ArrayView::from_shape(IxDyn(self.shape()), self.as_slice::<T>())
- .expect("Failed to create array view")
+ ArrayView::from_shape(IxDyn(self.shape()), self.as_slice::<T>()).expect("Failed to create array view")
}
pub fn copy_from_array<C: Backend>(
@@ -168,17 +150,12 @@ impl<B: Backend> Array<B> {
assert_eq!(a, b, "Shapes must match on all non-sliced axes");
}
}
- assert_eq!(
- self.data_type(),
- source.data_type(),
- "Arrays must have the same data type"
- );
+ assert_eq!(self.data_type(), source.data_type(), "Arrays must have the same data type");
let elem_size = self.data_type().size_in_bytes();
// The number of contiguous elements to copy for each slice operation
- let block_size_elems =
- self.shape().iter().skip(axis + 1).product::<usize>();
+ let block_size_elems = self.shape().iter().skip(axis + 1).product::<usize>();
let block_size_bytes = block_size_elems * elem_size;
// The total number of blocks to copy
@@ -199,8 +176,7 @@ impl<B: Backend> Array<B> {
let src_block_start = i * src_stride_bytes;
let dst_block_start = i * dst_stride_bytes;
- let src_start =
- src_block_start + src_range.start * block_size_bytes;
+ let src_start = src_block_start + src_range.start * block_size_bytes;
let dst_start = dst_block_start + dst_offset * block_size_bytes;
let src_slice = &src_buf[src_start..src_start + copy_bytes];
@@ -267,8 +243,7 @@ pub trait ArrayContextExt {
data_type: DataType,
label: &str,
) -> Array<Self::Backend> {
- let mut array =
- self.create_array_uninitialized(shape, data_type, label);
+ let mut array = self.create_array_uninitialized(shape, data_type, label);
array.as_bytes_mut().fill(0);
array
}
@@ -285,9 +260,7 @@ impl<C: Context> ArrayContextExt for C {
) -> Array<Self::Backend> {
let buffer_size_bytes = size_for_shape(shape, data_type);
- let buffer = self
- .create_buffer(buffer_size_bytes)
- .expect("Failed to create buffer");
+ let buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");
buffer.set_label(Some(label));
unsafe { Array::from_parts(buffer, 0, shape, data_type) }
diff --git a/crates/uzu/src/backends/common/allocator/allocator.rs b/crates/uzu/src/backends/common/allocator/allocator.rs
index 57e1ad196..66bf849b1 100644
--- a/crates/uzu/src/backends/common/allocator/allocator.rs
+++ b/crates/uzu/src/backends/common/allocator/allocator.rs
@@ -7,9 +7,7 @@ use super::{
AllocError,
scratch_pool::{ScratchPool, align_size},
};
-use crate::backends::common::{
- Backend, Buffer, BufferLifetime, Context, NativeBuffer,
-};
+use crate::backends::common::{Backend, Buffer, BufferLifetime, Context, NativeBuffer};
pub struct Allocator<B: Backend> {
context: Weak<B::Context>,
@@ -36,11 +34,9 @@ impl<B: Backend> Allocator<B> {
&self,
size: usize,
) -> Result<B::NativeBuffer, AllocError> {
- let context = self.context.upgrade().ok_or_else(|| {
- AllocError::AllocationFailed {
- size,
- reason: "context has been dropped".to_string(),
- }
+ let context = self.context.upgrade().ok_or_else(|| AllocError::AllocationFailed {
+ size,
+ reason: "context has been dropped".to_string(),
})?;
context.create_buffer(size).map_err(|e| AllocError::AllocationFailed {
size,
@@ -141,9 +137,7 @@ impl<B: Backend> Allocator<B> {
self.track_allocation(aligned_size);
(buffer, false)
},
- BufferLifetime::Scratch => {
- (self.alloc_scratch_internal(aligned_size)?, true)
- },
+ BufferLifetime::Scratch => (self.alloc_scratch_internal(aligned_size)?, true),
};
Ok(self.wrap_buffer(native, is_scratch))
diff --git a/crates/uzu/src/backends/common/allocator/error.rs b/crates/uzu/src/backends/common/allocator/error.rs
index f6c4c949c..bb109b172 100644
--- a/crates/uzu/src/backends/common/allocator/error.rs
+++ b/crates/uzu/src/backends/common/allocator/error.rs
@@ -22,11 +22,7 @@ impl fmt::Display for AllocError {
requested,
available,
} => {
- write!(
- f,
- "Out of memory: requested {} bytes, {} available",
- requested, available
- )
+ write!(f, "Out of memory: requested {} bytes, {} available", requested, available)
},
AllocError::AllocationFailed {
size,
diff --git a/crates/uzu/src/backends/common/allocator/scratch_pool.rs b/crates/uzu/src/backends/common/allocator/scratch_pool.rs
index 1c62f71f9..963deb3d2 100644
--- a/crates/uzu/src/backends/common/allocator/scratch_pool.rs
+++ b/crates/uzu/src/backends/common/allocator/scratch_pool.rs
@@ -157,11 +157,10 @@ impl<B> ScratchPool<B> {
}
}
- let (key, slot, bucket_idx) =
- match (found_key, found_slot, found_bucket_idx) {
- (Some(k), Some(s), Some(i)) => (k, s, i),
- _ => return None,
- };
+ let (key, slot, bucket_idx) = match (found_key, found_slot, found_bucket_idx) {
+ (Some(k), Some(s), Some(i)) => (k, s, i),
+ _ => return None,
+ };
self.remove_slot_from_bucket(key, slot, bucket_idx);
self.remove_from_lru(slot);
@@ -248,8 +247,7 @@ impl<B> ScratchPool<B> {
self.size_index.clear();
self.lru_head = None;
self.lru_tail = None;
- self.total_allocated =
- self.total_allocated.saturating_sub(self.pool_size);
+ self.total_allocated = self.total_allocated.saturating_sub(self.pool_size);
self.pool_size = 0;
count
diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rs
index cb674cda9..240289230 100644
--- a/crates/uzu/src/backends/common/context.rs
+++ b/crates/uzu/src/backends/common/context.rs
@@ -12,15 +12,9 @@ pub trait Context: Sized {
fn create_buffer(
&self,
size: usize,
- ) -> Result<
- <Self::Backend as Backend>::NativeBuffer,
- <Self::Backend as Backend>::Error,
- >;
+ ) -> Result<<Self::Backend as Backend>::NativeBuffer, <Self::Backend as Backend>::Error>;
fn create_command_buffer(
&self
- ) -> Result<
- <Self::Backend as Backend>::CommandBuffer,
- <Self::Backend as Backend>::Error,
- >;
+ ) -> Result<<Self::Backend as Backend>::CommandBuffer, <Self::Backend as Backend>::Error>;
}
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
index ab6af972f..50cc11aff 100644
--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -2,10 +2,7 @@ use thiserror::Error;
use crate::{
DataType,
- backends::common::{
- Backend, CommandBuffer, Kernels, gpu_types::Swap,
- kernel::KVCacheUpdateKernel,
- },
+ backends::common::{Backend, CommandBuffer, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},
};
pub struct KVLayerData<B: Backend> {
@@ -44,10 +41,8 @@ impl<B: Backend> KVCacheUpdate<B> {
data_type: DataType,
max_sequence_length: usize,
) -> Result<Self, KVCacheUpdateError<B>> {
- let kernel = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(
- context, data_type,
- )
- .map_err(KVCacheUpdateError::BackendError)?;
+ let kernel = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(context, data_type)
+ .map_err(KVCacheUpdateError::BackendError)?;
Ok(Self {
kernel,
@@ -63,12 +58,7 @@ impl<B: Backend> KVCacheUpdate<B> {
command_buffer: &B::CommandBuffer,
) -> Result<(), KVCacheUpdateError<B>> {
command_buffer.with_compute_encoder(|encoder| {
- self.encode_with_encoder(
- in_place_data,
- source_indices,
- destination_indices,
- encoder,
- )
+ self.encode_with_encoder(in_place_data, source_indices, destination_indices, encoder)
})
}
@@ -92,18 +82,14 @@ impl<B: Backend> KVCacheUpdate<B> {
// For small swap counts, use set_bytes to avoid shared buffer race in async mode.
// Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.
let use_inline_bytes = swaps.len() <= 32;
- assert!(
- use_inline_bytes,
- "non-inline is not supported yet (and is broken anyways due to a data race)"
- );
+ assert!(use_inline_bytes, "non-inline is not supported yet (and is broken anyways due to a data race)");
for layer_data in in_place_data {
if layer_data.key_shape != layer_data.value_shape {
return Err(KVCacheUpdateError::ShapeMismatch);
}
- let [num_heads, max_sequence_length, head_dim] =
- layer_data.key_shape;
+ let [num_heads, max_sequence_length, head_dim] = layer_data.key_shape;
self.kernel.encode(
&layer_data.key_buffer,
diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rs
index 82bef77ba..189b49013 100644
--- a/crates/uzu/src/backends/common/kernel/sampling.rs
+++ b/crates/uzu/src/backends/common/kernel/sampling.rs
@@ -7,9 +7,8 @@ use crate::{
backends::common::{
Context,
kernel::{
- ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel,
- BitmaskKernel, GumbelKernel, MinPKernel, TemperatureKernel,
- TopKKernel, TopPKernel,
+ ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel, BitmaskKernel, GumbelKernel, MinPKernel,
+ TemperatureKernel, TopKKernel, TopPKernel,
},
},
session::parameter::SamplingMethod,
@@ -83,13 +82,7 @@ impl<B: Backend> SamplingKernel<B> {
max_batch_size: usize,
max_vocab_size: usize,
) -> Result<Self, SamplingError<B>> {
- Self::new_with_strategy(
- context,
- data_type,
- max_batch_size,
- max_vocab_size,
- ArgmaxStrategy::TwoPass,
- )
+ Self::new_with_strategy(context, data_type, max_batch_size, max_vocab_size, ArgmaxStrategy::TwoPass)
}
pub fn new_with_strategy(
@@ -100,56 +93,39 @@ impl<B: Backend> SamplingKernel<B> {
argmax_strategy: ArgmaxStrategy,
) -> Result<Self, SamplingError<B>> {
let bitmask =
- <B::Kernels as Kernels>::BitmaskKernel::new(context, data_type)
- .map_err(SamplingError::BackendError)?;
+ <B::Kernels as Kernels>::BitmaskKernel::new(context, data_type).map_err(SamplingError::BackendError)?;
let temperature =
- <B::Kernels as Kernels>::TemperatureKernel::new(context, data_type)
- .map_err(SamplingError::BackendError)?;
- let topk = <B::Kernels as Kernels>::TopKKernel::new(context, data_type)
- .map_err(SamplingError::BackendError)?;
- let topp = <B::Kernels as Kernels>::TopPKernel::new(context, data_type)
- .map_err(SamplingError::BackendError)?;
- let minp = <B::Kernels as Kernels>::MinPKernel::new(context, data_type)
- .map_err(SamplingError::BackendError)?;
+ <B::Kernels as Kernels>::TemperatureKernel::new(context, data_type).map_err(SamplingError::BackendError)?;
+ let topk = <B::Kernels as Kernels>::TopKKernel::new(context, data_type).map_err(SamplingError::BackendError)?;
+ let topp = <B::Kernels as Kernels>::TopPKernel::new(context, data_type).map_err(SamplingError::BackendError)?;
+ let minp = <B::Kernels as Kernels>::MinPKernel::new(context, data_type).map_err(SamplingError::BackendError)?;
let gumbel =
- <B::Kernels as Kernels>::GumbelKernel::new(context, data_type)
- .map_err(SamplingError::BackendError)?;
+ <B::Kernels as Kernels>::GumbelKernel::new(context, data_type).map_err(SamplingError::BackendError)?;
let argmax_implementation = match argmax_strategy {
ArgmaxStrategy::SinglePass => {
- let kernel = <B::Kernels as Kernels>::ArgmaxSingleKernel::new(
- context, data_type,
- )
- .map_err(SamplingError::BackendError)?;
+ let kernel = <B::Kernels as Kernels>::ArgmaxSingleKernel::new(context, data_type)
+ .map_err(SamplingError::BackendError)?;
ArgmaxImplementation::SinglePass {
kernel,
}
},
ArgmaxStrategy::TwoPass => {
- let main_kernel =
- <B::Kernels as Kernels>::ArgmaxMainKernel::new(
- context, data_type,
- )
+ let main_kernel = <B::Kernels as Kernels>::ArgmaxMainKernel::new(context, data_type)
.map_err(SamplingError::BackendError)?;
let final_kernel =
- <B::Kernels as Kernels>::ArgmaxFinalKernel::new(context)
- .map_err(SamplingError::BackendError)?;
+ <B::Kernels as Kernels>::ArgmaxFinalKernel::new(context).map_err(SamplingError::BackendError)?;
let block_size = 1024;
let grain_size = 4;
let elements_per_group = block_size * grain_size;
- let max_vocab_groups_per_batch =
- (max_vocab_size + elements_per_group - 1)
- / elements_per_group;
- let max_partial_results =
- max_batch_size * max_vocab_groups_per_batch;
+ let max_vocab_groups_per_batch = (max_vocab_size + elements_per_group - 1) / elements_per_group;
+ let max_partial_results = max_batch_size * max_vocab_groups_per_batch;
let partial_results_buffer = context
- .create_buffer(
- max_partial_results * size_of::<ArgmaxPair>(),
- )
+ .create_buffer(max_partial_results * size_of::<ArgmaxPair>())
.expect("Failed to create partial results buffer");
ArgmaxImplementation::TwoPass {
@@ -187,16 +163,10 @@ impl<B: Backend> SamplingKernel<B> {
compute_encoder: &B::ComputeEncoder,
) -> Result<(), SamplingError<B>> {
if batch_size > self.max_batch_size {
- return Err(SamplingError::BatchSizeExceeded(
- batch_size,
- self.max_batch_size,
- ));
+ return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));
}
if vocab_size > self.max_vocab_size {
- return Err(SamplingError::VocabSizeExceeded(
- vocab_size,
- self.max_vocab_size,
- ));
+ return Err(SamplingError::VocabSizeExceeded(vocab_size, self.max_vocab_size));
}
if let Some(bitmask_buffer) = bitmask_buffer {
@@ -263,10 +233,7 @@ impl<B: Backend> SamplingKernel<B> {
self.gumbel.encode(
logits_buffer,
- (
- seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,
- seeds_offset,
- ),
+ (seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?, seeds_offset),
logits_buffer,
batch_size as u32,
vocab_size as u32,
diff --git a/crates/uzu/src/backends/cpu/argmax.rs b/crates/uzu/src/backends/cpu/argmax.rs
index 61cf89d7c..9e6212780 100644
--- a/crates/uzu/src/backends/cpu/argmax.rs
+++ b/crates/uzu/src/backends/cpu/argmax.rs
@@ -24,8 +24,7 @@ pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {
for chunk in 0..4 {
let base = i + chunk * 8;
for j in 0..8 {
- let val: f32 = NumCast::from(*ptr.add(base + j))
- .unwrap_or(f32::NEG_INFINITY);
+ let val: f32 = NumCast::from(*ptr.add(base + j)).unwrap_or(f32::NEG_INFINITY);
if val > max_val || (val == max_val && base + j < max_idx) {
max_val = val;
max_idx = base + j;
@@ -38,8 +37,7 @@ pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {
// Handle remaining 16x chunks
while i + 16 <= len {
for j in 0..16 {
- let val: f32 =
- NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);
+ let val: f32 = NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);
if val > max_val || (val == max_val && i + j < max_idx) {
max_val = val;
max_idx = i + j;
@@ -51,8 +49,7 @@ pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {
// Handle remaining 8x chunks
while i + 8 <= len {
for j in 0..8 {
- let val: f32 =
- NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);
+ let val: f32 = NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);
if val > max_val || (val == max_val && i + j < max_idx) {
max_val = val;
max_idx = i + j;
@@ -63,8 +60,7 @@ pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {
// Handle remaining elements
while i < len {
- let val: f32 =
- NumCast::from(*ptr.add(i)).unwrap_or(f32::NEG_INFINITY);
+ let val: f32 = NumCast::from(*ptr.add(i)).unwrap_or(f32::NEG_INFINITY);
if val > max_val || (val == max_val && i < max_idx) {
max_val = val;
max_idx = i;
@@ -80,20 +76,14 @@ pub fn simple_argmax<T: ArrayElement>(input: &[T]) -> usize {
input
.iter()
.enumerate()
- .fold(
- (0, f32::NEG_INFINITY),
- |(best_index, best_value), (index, &value)| {
- let value_f32 =
- NumCast::from(value).unwrap_or(f32::NEG_INFINITY);
- if value_f32 > best_value
- || (value_f32 == best_value && index < best_index)
- {
- (index, value_f32)
- } else {
- (best_index, best_value)
- }
- },
- )
+ .fold((0, f32::NEG_INFINITY), |(best_index, best_value), (index, &value)| {
+ let value_f32 = NumCast::from(value).unwrap_or(f32::NEG_INFINITY);
+ if value_f32 > best_value || (value_f32 == best_value && index < best_index) {
+ (index, value_f32)
+ } else {
+ (best_index, best_value)
+ }
+ })
.0
}
@@ -150,8 +140,7 @@ pub unsafe fn neon_argmax_f32(input: &[f32]) -> usize {
for j in 1..4 {
if max_vals_array[j] > global_max
- || (max_vals_array[j] == global_max
- && (max_indices_array[j] as usize) < global_max_idx)
+ || (max_vals_array[j] == global_max && (max_indices_array[j] as usize) < global_max_idx)
{
global_max = max_vals_array[j];
global_max_idx = max_indices_array[j] as usize;
@@ -182,10 +171,7 @@ pub fn neon_optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {
// For f32, use direct NEON implementation
if std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>() {
unsafe {
- let f32_slice = std::slice::from_raw_parts(
- input.as_ptr() as *const f32,
- input.len(),
- );
+ let f32_slice = std::slice::from_raw_parts(input.as_ptr() as *const f32, input.len());
return neon_argmax_f32(f32_slice);
}
}
@@ -225,11 +211,7 @@ mod tests {
// Compare NEON result to expected
let neon_result = neon_optimized_argmax(&data);
- assert_eq!(
- expected, neon_result,
- "64k array test failed: expected {}, got {}",
- expected, neon_result
- );
+ assert_eq!(expected, neon_result, "64k array test failed: expected {}, got {}", expected, neon_result);
assert_eq!(expected, 42_000, "Expected max should be at index 42_000");
}
@@ -280,34 +262,15 @@ mod tests {
if batch_idx == 0 {
println!("128k argmax performance comparison:");
- println!(
- "NEON: {:.3} ms",
- neon_duration.as_secs_f64() * 1000.0
- );
- println!(
- "Simple: {:.3} ms",
- simple_duration.as_secs_f64() * 1000.0
- );
- println!(
- "Optimized: {:.3} ms",
- optimized_duration.as_secs_f64() * 1000.0
- );
-
- if simple_duration.as_nanos() > 0
- && neon_duration.as_nanos() > 0
- {
- let speedup_simple = simple_duration.as_secs_f64()
- / neon_duration.as_secs_f64();
- let speedup_optimized = optimized_duration.as_secs_f64()
- / neon_duration.as_secs_f64();
- println!(
- "NEON vs Simple speedup: {:.2}x",
- speedup_simple
- );
- println!(
- "NEON vs Optimized speedup: {:.2}x",
- speedup_optimized
- );
+ println!("NEON: {:.3} ms", neon_duration.as_secs_f64() * 1000.0);
+ println!("Simple: {:.3} ms", simple_duration.as_secs_f64() * 1000.0);
+ println!("Optimized: {:.3} ms", optimized_duration.as_secs_f64() * 1000.0);
+
+ if simple_duration.as_nanos() > 0 && neon_duration.as_nanos() > 0 {
+ let speedup_simple = simple_duration.as_secs_f64() / neon_duration.as_secs_f64();
+ let speedup_optimized = optimized_duration.as_secs_f64() / neon_duration.as_secs_f64();
+ println!("NEON vs Simple speedup: {:.2}x", speedup_simple);
+ println!("NEON vs Optimized speedup: {:.2}x", speedup_optimized);
}
}
diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rs
index 763324f34..559046842 100644
--- a/crates/uzu/src/backends/metal/backend.rs
+++ b/crates/uzu/src/backends/metal/backend.rs
@@ -1,7 +1,4 @@
-use metal::{
- MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,
- MTLComputeCommandEncoder,
-};
+use metal::{MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{MTLContext, MTLError, MetalKernels};
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
index f1ccb6e94..6bc5a4d0a 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -2,22 +2,17 @@ use crate::backends::metal::Metal;
use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
use super::{
- KernelDataType, MTLCommandBuffer, MTLCommandQueue, MTLContext,
- ProtocolObject, Retained,
+ KernelDataType, MTLCommandBuffer, MTLCommandQueue, MTLContext, ProtocolObject, Retained,
compilation_parameters::CompilationConfig,
encodable_block::{
- Activation, ClassifierLayer, ClassifierPredictionHead, Normalization,
- Pooling, Rope,
+ Activation, ClassifierLayer, ClassifierPredictionHead, Normalization, Pooling, Rope,
transformer_layer::{embed_block, linear_block},
},
kernel::dsl::SigmoidMetalKernel,
};
use crate::{
DataType,
- backends::{
- common::Context, common::kernel::SigmoidKernel,
- metal::error::ClassifierError,
- },
+ backends::{common::Context, common::kernel::SigmoidKernel, metal::error::ClassifierError},
config::{ClassifierModelConfig, ModelMetadata},
encodable_block::EncodableBlock,
forward_pass::{
@@ -54,34 +49,22 @@ pub struct ClassifierContext {
impl ClassifierContext {
pub fn new(model_path: &Path) -> Result<Self, Error> {
- let context =
- MTLContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;
+ let context = MTLContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;
- let command_buffer = context
- .create_command_buffer()
- .map_err(|_| Error::UnableToCreateMetalContext)?;
+ let command_buffer = context.create_command_buffer().map_err(|_| Error::UnableToCreateMetalContext)?;
let config_path = model_path.join("config.json");
if !config_path.exists() {
return Err(Error::UnableToLoadConfig);
}
- let config_file =
- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
let model_metadata: ModelMetadata =
- serde_json::from_reader(BufReader::new(config_file))
- .map_err(|_| Error::UnableToLoadConfig)?;
+ serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
- let classifier_model_config = model_metadata
- .model_config
- .as_classifier()
- .ok_or(Error::UnableToLoadConfig)?;
-
- let decoder_config = Rc::new(
- classifier_model_config
- .model_config
- .to_decoder_config()
- .map_err(|_| Error::UnableToLoadConfig)?,
- );
+ let classifier_model_config = model_metadata.model_config.as_classifier().ok_or(Error::UnableToLoadConfig)?;
+
+ let decoder_config =
+ Rc::new(classifier_model_config.model_config.to_decoder_config().map_err(|_| Error::UnableToLoadConfig)?);
let model_shape = ModelShape::from_decoder_config(&decoder_config);
let compilation_config = Rc::new(CompilationConfig::default());
@@ -89,23 +72,14 @@ impl ClassifierContext {
if !weights_path.exists() {
return Err(Error::UnableToLoadWeights);
}
- let weights_file = File::open(&weights_path)
- .map_err(|_| Error::UnableToLoadWeights)?;
- let loader = ParameterLoader::new(&weights_file, context.as_ref())
- .map_err(|_| Error::UnableToLoadWeights)?;
+ let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let root_loader_view = loader.tree();
- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(
- context.as_ref(),
- &decoder_config,
- &model_shape,
- )));
- let transformer_tree =
- root_loader_view.subtree("transformer").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound(
- "transformer".to_string(),
- ))
- })?;
+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));
+ let transformer_tree = root_loader_view
+ .subtree("transformer")
+ .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("transformer".to_string())))?;
{
let mut shared_bufs = shared_buffers.borrow_mut();
@@ -128,17 +102,13 @@ impl ClassifierContext {
let embed = embed_block(&decoder_config, &context, &root_loader_view);
- let global_rope =
- Self::create_rope_block(&context, data_type, RopeType::Global)
- .map_err(Error::Classifier)?;
+ let global_rope = Self::create_rope_block(&context, data_type, RopeType::Global).map_err(Error::Classifier)?;
let local_rope = classifier_model_config
.model_config
.transformer_config
.local_rope_config
.as_ref()
- .map(|_| {
- Self::create_rope_block(&context, data_type, RopeType::Local)
- })
+ .map(|_| Self::create_rope_block(&context, data_type, RopeType::Local))
.transpose()
.map_err(Error::Classifier)?;
@@ -150,9 +120,7 @@ impl ClassifierContext {
.enumerate()
.map(|(layer_index, layer_config)| {
let mut rope = global_rope.clone();
- let attn = layer_config
- .attention_config()
- .ok_or(ClassifierError::NonAttentionMixer)?;
+ let attn = layer_config.attention_config().ok_or(ClassifierError::NonAttentionMixer)?;
if attn.sliding_window_size.is_some() {
if let Some(local_rope_block) = local_rope.clone() {
@@ -161,32 +129,18 @@ impl ClassifierContext {
}
let num_heads = attn.num_heads.ok_or_else(|| {
- ClassifierError::MissingConfigField(format!(
- "num_heads in layer {}",
- layer_index
- ))
- })?;
- let head_dim = attn.head_dim.ok_or_else(|| {
- ClassifierError::MissingConfigField(format!(
- "head_dim in layer {}",
- layer_index
- ))
+ ClassifierError::MissingConfigField(format!("num_heads in layer {}", layer_index))
})?;
+ let head_dim = attn
+ .head_dim
+ .ok_or_else(|| ClassifierError::MissingConfigField(format!("head_dim in layer {}", layer_index)))?;
let num_groups = attn.num_groups.ok_or_else(|| {
- ClassifierError::MissingConfigField(format!(
- "num_groups in layer {}",
- layer_index
- ))
+ ClassifierError::MissingConfigField(format!("num_groups in layer {}", layer_index))
})?;
let layer_tree = transformer_tree
.subtree(&format!("layers.{}", layer_index))
- .map_err(|_| {
- ClassifierError::WeightSubtreeNotFound(format!(
- "layers.{}",
- layer_index
- ))
- })?;
+ .map_err(|_| ClassifierError::WeightSubtreeNotFound(format!("layers.{}", layer_index)))?;
Ok(ClassifierLayer::new(
context.clone(),
@@ -194,10 +148,7 @@ impl ClassifierContext {
compilation_config.clone(),
layer_index,
classifier_model_config.model_config.model_dim,
- classifier_model_config
- .model_config
- .transformer_config
- .hidden_dim,
+ classifier_model_config.model_config.transformer_config.hidden_dim,
num_heads,
head_dim,
num_groups,
@@ -210,47 +161,26 @@ impl ClassifierContext {
.map_err(Error::Classifier)?
.into_boxed_slice();
- let output_norm_tree =
- transformer_tree.subtree("output_norm").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound(
- "output_norm".to_string(),
- ))
- })?;
+ let output_norm_tree = transformer_tree
+ .subtree("output_norm")
+ .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("output_norm".to_string())))?;
let output_norm = Normalization::new(
&context,
data_type,
- classifier_model_config
- .model_config
- .transformer_config
- .output_norm_config
- .clone(),
+ classifier_model_config.model_config.transformer_config.output_norm_config.clone(),
ArrayId::Main,
ArrayId::Main,
&output_norm_tree,
)
- .map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(format!(
- "output norm: {:?}",
- e
- )))
- })?;
+ .map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("output norm: {:?}", e))))?;
- let context_length =
- classifier_model_config.model_config.context_length;
- let scratch_buffers = ScratchBuffers::new(
- context.as_ref(),
- &decoder_config,
- &model_shape,
- context_length,
- context_length,
- );
+ let context_length = classifier_model_config.model_config.context_length;
+ let scratch_buffers =
+ ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, context_length, context_length);
- let embedding_norm_tree =
- root_loader_view.subtree("embedding_norm").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound(
- "embedding_norm".to_string(),
- ))
- })?;
+ let embedding_norm_tree = root_loader_view
+ .subtree("embedding_norm")
+ .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("embedding_norm".to_string())))?;
let embedding_norm = Normalization::new(
&context,
data_type,
@@ -259,33 +189,20 @@ impl ClassifierContext {
ArrayId::Main,
&embedding_norm_tree,
)
- .map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(format!(
- "embedding norm: {:?}",
- e
- )))
- })?;
+ .map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("embedding norm: {:?}", e))))?;
let model_dim = classifier_model_config.model_config.model_dim;
let num_labels = classifier_model_config.model_config.num_labels;
- let prediction_head_config =
- &classifier_model_config.model_config.prediction_head_config;
- let prediction_head_tree =
- root_loader_view.subtree("prediction_head").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound(
- "prediction_head".to_string(),
- ))
- })?;
+ let prediction_head_config = &classifier_model_config.model_config.prediction_head_config;
+ let prediction_head_tree = root_loader_view
+ .subtree("prediction_head")
+ .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head".to_string())))?;
- let prediction_head_data_type: DataType =
- prediction_head_config.dense_config.activation_precision().into();
+ let prediction_head_data_type: DataType = prediction_head_config.dense_config.activation_precision().into();
- let prediction_head_dense_tree =
- prediction_head_tree.subtree("dense").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound(
- "prediction_head.dense".to_string(),
- ))
- })?;
+ let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))
+ })?;
let prediction_head_dense = linear_block::<1>(
&prediction_head_config.dense_config,
prediction_head_config.use_dense_bias,
@@ -306,18 +223,13 @@ impl ClassifierContext {
ArrayId::ClassifierPredictionHeadDense,
)
.map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(
- format!("prediction head activation: {:?}", e),
- ))
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))
})?,
);
- let prediction_head_norm_tree =
- prediction_head_tree.subtree("norm").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound(
- "prediction_head.norm".to_string(),
- ))
- })?;
+ let prediction_head_norm_tree = prediction_head_tree.subtree("norm").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.norm".to_string()))
+ })?;
let prediction_head_norm = Normalization::new(
&context,
prediction_head_data_type,
@@ -327,18 +239,12 @@ impl ClassifierContext {
&prediction_head_norm_tree,
)
.map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(format!(
- "prediction head norm: {:?}",
- e
- )))
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head norm: {:?}", e)))
})?;
- let prediction_head_readout_tree =
- prediction_head_tree.subtree("readout").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound(
- "prediction_head.readout".to_string(),
- ))
- })?;
+ let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))
+ })?;
let prediction_head_final_linear = linear_block::<1>(
&prediction_head_config.readout_config,
true,
@@ -350,19 +256,13 @@ impl ClassifierContext {
ArrayId::ClassifierPredictionHeadLogits,
)
.map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(format!(
- "prediction head readout: {:?}",
- e
- )))
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head readout: {:?}", e)))
})?;
- let sigmoid_kernel =
- SigmoidMetalKernel::new(&context, data_type.into()).map_err(
- |e| {
- eprintln!("Failed to create sigmoid kernel: {:?}", e);
- Error::UnableToCreateMetalContext
- },
- )?;
+ let sigmoid_kernel = SigmoidMetalKernel::new(&context, data_type.into()).map_err(|e| {
+ eprintln!("Failed to create sigmoid kernel: {:?}", e);
+ Error::UnableToCreateMetalContext
+ })?;
let pooling = Box::new(
Pooling::new(
@@ -379,9 +279,7 @@ impl ClassifierContext {
let prediction_head = Box::new(ClassifierPredictionHead::new(
prediction_head_dense.map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(
- format!("prediction head dense: {:?}", e),
- ))
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))
})?,
prediction_head_activation,
Box::new(prediction_head_norm),
@@ -416,24 +314,14 @@ impl ClassifierContext {
let kernel_data_type: KernelDataType = data_type.into();
let rotation: Box<dyn EncodableBlock<Metal>> = Box::new(
- Rope::new(mtl_context, kernel_data_type, rope_type).map_err(
- |e| {
- ClassifierError::KernelCreationFailed(format!(
- "RoPE: {:?}",
- e
- ))
- },
- )?,
+ Rope::new(mtl_context, kernel_data_type, rope_type)
+ .map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?,
);
Ok(Rc::new(rotation))
}
pub fn reset_command_buffer(&mut self) {
- self.command_buffer = self
- .mtl_context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
+ self.command_buffer =
+ self.mtl_context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
}
}
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index 5741b68a9..3f03a900b 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -10,13 +10,9 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
fn with_compute_encoder<T>(
&self,
- callback: impl FnOnce(
- &<Self::Backend as crate::backends::common::Backend>::ComputeEncoder,
- ) -> T,
+ callback: impl FnOnce(&<Self::Backend as crate::backends::common::Backend>::ComputeEncoder) -> T,
) -> T {
- let encoder = self
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = self.new_compute_command_encoder().expect("Failed to create compute command encoder");
let ret = callback(&encoder);
@@ -27,13 +23,9 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
fn with_copy_encoder<T>(
&self,
- callback: impl FnOnce(
- &<Self::Backend as crate::backends::common::Backend>::CopyEncoder,
- ) -> T,
+ callback: impl FnOnce(&<Self::Backend as crate::backends::common::Backend>::CopyEncoder) -> T,
) -> T {
- let encoder = self
- .new_blit_command_encoder()
- .expect("Failed to create blit command encoder");
+ let encoder = self.new_blit_command_encoder().expect("Failed to create blit command encoder");
let ret = callback(&encoder);
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index be16f6d3b..40169a5af 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -3,14 +3,12 @@ use std::{cell::RefCell, collections::HashMap, env, rc::Rc};
use metal::{MTLBuffer, MTLCommandBuffer};
use objc2::{rc::Retained, runtime::ProtocolObject};
-use super::{
- Metal, error::MTLError, kernel, metal_extensions::LibraryPipelineExtensions,
-};
+use super::{Metal, error::MTLError, kernel, metal_extensions::LibraryPipelineExtensions};
use crate::backends::{
common::{Allocator, Context},
metal::{
- MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt,
- MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions,
+ MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLFunctionConstantValues, MTLLibrary,
+ MTLResourceOptions,
},
};
@@ -70,8 +68,7 @@ impl DeviceArchitecture {
// Extract generation from architecture name
// Format is typically like "Apple M3 Pro" or the GPU name "applegpu_g15d"
- let (generation, device_class) =
- Self::parse_architecture_info(&arch_string);
+ let (generation, device_class) = Self::parse_architecture_info(&arch_string);
Self {
generation,
@@ -100,10 +97,7 @@ impl DeviceArchitecture {
DeviceGeneration::Unknown(0)
};
- let device_class = if arch.contains("Max")
- || arch.contains("Ultra")
- || arch.contains("Pro")
- {
+ let device_class = if arch.contains("Max") || arch.contains("Ultra") || arch.contains("Pro") {
DeviceClass::Desktop
} else if arch.contains("iPhone") || arch.contains("iPad") {
DeviceClass::Phone
@@ -133,9 +127,7 @@ pub struct MTLContext {
pub command_queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,
pub architecture: DeviceArchitecture,
pub library: Retained<ProtocolObject<dyn MTLLibrary>>,
- pipeline_cache: RefCell<
- HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- >,
+ pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
allocator: Allocator<Metal>,
}
@@ -162,22 +154,17 @@ impl MTLContext {
/// TF32 toggle via UZU_TF32 environment variable.
pub fn tf32_enabled(&self) -> bool {
- env::var("UZU_TF32")
- .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
- .unwrap_or(false)
+ env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)
}
pub fn compute_pipeline_state(
&self,
function_name: &str,
constants: Option<&MTLFunctionConstantValues>,
- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
// Only cache pipelines without constants
if constants.is_some() {
- return self
- .library
- .compute_pipeline_state(function_name, constants);
+ return self.library.compute_pipeline_state(function_name, constants);
}
self.compute_pipeline_state_cached(function_name, function_name, None)
}
@@ -187,18 +174,14 @@ impl MTLContext {
cache_key: &str,
function_name: &str,
constants: Option<&MTLFunctionConstantValues>,
- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if let Some(pipeline) = self.pipeline_cache.borrow().get(cache_key) {
return Ok(pipeline.clone());
}
- let pipeline =
- self.library.compute_pipeline_state(function_name, constants)?;
+ let pipeline = self.library.compute_pipeline_state(function_name, constants)?;
- self.pipeline_cache
- .borrow_mut()
- .insert(cache_key.to_string(), pipeline.clone());
+ self.pipeline_cache.borrow_mut().insert(cache_key.to_string(), pipeline.clone());
Ok(pipeline)
}
@@ -208,22 +191,16 @@ impl Context for MTLContext {
type Backend = Metal;
fn new() -> Result<Rc<Self>, MTLError> {
- let device: Retained<ProtocolObject<dyn MTLDevice>> =
- <dyn metal::MTLDevice>::system_default().ok_or(
- MTLError::Generic("cannot open system default device".into()),
- )?;
+ let device: Retained<ProtocolObject<dyn MTLDevice>> = <dyn metal::MTLDevice>::system_default()
+ .ok_or(MTLError::Generic("cannot open system default device".into()))?;
let command_queue = device
.new_command_queue_with_max_command_buffer_count(1024)
.ok_or(MTLError::Generic("cannot create command queue".into()))?;
- let library =
- device.new_library_with_data(kernel::MTLB).map_err(|nserror| {
- MTLError::Generic(format!(
- "Failed to create Metal library: {}",
- nserror
- ))
- })?;
+ let library = device
+ .new_library_with_data(kernel::MTLB)
+ .map_err(|nserror| MTLError::Generic(format!("Failed to create Metal library: {}", nserror)))?;
let architecture = DeviceArchitecture::from_device(&device);
@@ -250,11 +227,7 @@ impl Context for MTLContext {
.ok_or(MTLError::Generic("cannot create buffer".into()))
}
- fn create_command_buffer(
- &self
- ) -> Result<Retained<ProtocolObject<dyn MTLCommandBuffer>>, MTLError> {
- self.command_queue
- .command_buffer()
- .ok_or(MTLError::Generic("cannot create command buffer".into()))
+ fn create_command_buffer(&self) -> Result<Retained<ProtocolObject<dyn MTLCommandBuffer>>, MTLError> {
+ self.command_queue.command_buffer().ok_or(MTLError::Generic("cannot create command buffer".into()))
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs
index d984b3959..f7db9dc57 100644
--- a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs
@@ -1,6 +1,5 @@
use crate::backends::metal::{
- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, Metal,
- ProtocolObject, Retained,
+ MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, Metal, ProtocolObject, Retained,
};
use super::{
@@ -46,9 +45,7 @@ impl EncodableBlock<Metal> for Activation {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
_parameters: &EncodingParameters<Metal>,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, _parameters);
@@ -84,12 +81,6 @@ impl EncodableBlock<Metal> for Activation {
},
};
- self.kernel.encode(
- input_buffer,
- output_buffer,
- n as u32,
- act_type,
- encoder,
- );
+ self.kernel.encode(input_buffer, output_buffer, n as u32, act_type, encoder);
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rs
index 8b2479303..ced8da32a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs
@@ -2,11 +2,10 @@
use super::{EncodableBlock, Metal};
use crate::backends::metal::{
- KernelDataType, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,
+ KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, ProtocolObject,
+ Retained,
kernel::attention::{
- AttentionError, AttentionGemmArguments, AttentionKernel,
- AttentionKernelVariant, AttentionSinglePassArguments,
+ AttentionError, AttentionGemmArguments, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,
AttentionTwoPassArguments, KVCacheUpdateArguments,
},
};
@@ -67,9 +66,8 @@ impl EncodableBlock<Metal> for Attention {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &compute_encoder, parameters);
compute_encoder.end_encoding();
@@ -89,13 +87,7 @@ impl EncodableBlock<Metal> for Attention {
compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
parameters: &EncodingParameters<Metal>,
) {
- let (
- suffix_length,
- num_heads,
- head_dim,
- num_groups,
- max_sequence_length,
- ) = {
+ let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
let qkv_array = qkv_binding[0].borrow();
let suffix_length = qkv_array.shape()[0];
@@ -110,8 +102,7 @@ impl EncodableBlock<Metal> for Attention {
let num_groups = keys_array.shape()[0];
let max_sequence_length = if let Some(_kv) = state.cache_layers() {
- let key_cache_binding =
- state.arrays(&[ArrayId::Keys(self.layer_index)]);
+ let key_cache_binding = state.arrays(&[ArrayId::Keys(self.layer_index)]);
let key_cache_array = key_cache_binding[0].borrow();
key_cache_array.shape()[1]
} else {
@@ -119,70 +110,46 @@ impl EncodableBlock<Metal> for Attention {
suffix_length
};
- (
- suffix_length,
- num_heads,
- head_dim,
- num_groups,
- max_sequence_length,
- )
+ (suffix_length, num_heads, head_dim, num_groups, max_sequence_length)
};
- let (segment_prefix_length, window_length) =
- if let Some(cache_layers) = state.cache_layers() {
- let cache = cache_layers.borrow();
- let layer = cache.data[self.layer_index]
- .as_transformer()
- .expect("Attention kernel expects transformer layer state");
- (
- layer.projected_segment_prefix_length(
- parameters.projection_step.unwrap_or(0),
- ),
- layer.window_length(),
- )
- } else {
- // For classifiers without KV cache: no prefix, use configured sliding window
- (0, self.sliding_window_size)
- };
+ let (segment_prefix_length, window_length) = if let Some(cache_layers) = state.cache_layers() {
+ let cache = cache_layers.borrow();
+ let layer = cache.data[self.layer_index]
+ .as_transformer()
+ .expect("Attention kernel expects transformer layer state");
+ (layer.projected_segment_prefix_length(parameters.projection_step.unwrap_or(0)), layer.window_length())
+ } else {
+ // For classifiers without KV cache: no prefix, use configured sliding window
+ (0, self.sliding_window_size)
+ };
let use_mask = window_length.is_some();
let sequence_length = segment_prefix_length + suffix_length;
let gqa_factor = num_heads / num_groups;
- let scale =
- self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());
+ let scale = self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());
let gemm_enabled = env_gemm_attention_enabled();
- let use_gemm = gemm_enabled
- && suffix_length > 8
- && matches!(head_dim, 64 | 128 | 256);
+ let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);
if !gemm_enabled {
static PRINT_ONCE: std::sync::Once = std::sync::Once::new();
PRINT_ONCE.call_once(|| {
- eprintln!(
- "[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION"
- );
+ eprintln!("[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION");
});
}
let variant = if use_gemm {
AttentionKernelVariant::SinglePass
} else {
- self.kernel.choose_variant(
- sequence_length,
- head_dim,
- self.is_causal,
- use_mask,
- )
+ self.kernel.choose_variant(sequence_length, head_dim, self.is_causal, use_mask)
};
let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
let qkv_binding = state.arrays(&[ArrayId::QKV]);
- let attention_bias_binding =
- state.hashmaps(&[HashMapId::AttentionBias]);
- let attention_output_binding =
- state.arrays(&[ArrayId::AttentionOutput]);
+ let attention_bias_binding = state.hashmaps(&[HashMapId::AttentionBias]);
+ let attention_output_binding = state.arrays(&[ArrayId::AttentionOutput]);
let mask_kv_seq_stride = 1;
let mask_q_seq_stride = sequence_length as i32;
@@ -202,10 +169,8 @@ impl EncodableBlock<Metal> for Attention {
// Get KV cache buffers only if KV cache exists (LLM mode)
let has_kv_cache = state.cache_layers().is_some();
let (key_cache_buffer, value_cache_buffer) = if has_kv_cache {
- let key_cache_binding =
- state.arrays(&[ArrayId::Keys(self.layer_index)]);
- let value_cache_binding =
- state.arrays(&[ArrayId::Values(self.layer_index)]);
+ let key_cache_binding = state.arrays(&[ArrayId::Keys(self.layer_index)]);
+ let value_cache_binding = state.arrays(&[ArrayId::Values(self.layer_index)]);
let key_cache_array = key_cache_binding[0].borrow_mut();
let key_cache_buf = key_cache_array.buffer().clone();
@@ -217,10 +182,8 @@ impl EncodableBlock<Metal> for Attention {
} else {
// For classifiers, we need a values buffer with [num_groups, suffix_length, head_dim] layout.
// Use the KV cache update kernel to extract values from QKV into a dedicated extracted_values buffer.
- let extracted_values_binding =
- state.arrays(&[ArrayId::ExtractedValues]);
- let extracted_values_array =
- extracted_values_binding[0].borrow_mut();
+ let extracted_values_binding = state.arrays(&[ArrayId::ExtractedValues]);
+ let extracted_values_array = extracted_values_binding[0].borrow_mut();
let extracted_values_buf = extracted_values_array.buffer().clone();
// Reuse the KV cache update kernel to write values into extracted_values_buf.
@@ -261,10 +224,7 @@ impl EncodableBlock<Metal> for Attention {
array_ref.buffer().clone()
})
.unwrap_or_else(|| {
- panic!(
- "Attention bias buffer not found for window length {:?}",
- window_length
- );
+ panic!("Attention bias buffer not found for window length {:?}", window_length);
}),
)
} else {
@@ -284,9 +244,7 @@ impl EncodableBlock<Metal> for Attention {
let maxs_array = maxs_binding[0].borrow_mut();
let maxs_buffer = maxs_array.buffer();
- let sinks_buffer = sinks_binding
- .as_ref()
- .map(|binding| binding[0].borrow().buffer().clone());
+ let sinks_buffer = sinks_binding.as_ref().map(|binding| binding[0].borrow().buffer().clone());
// Only update KV cache for LLM mode (not for classifiers)
if has_kv_cache {
@@ -367,10 +325,7 @@ impl EncodableBlock<Metal> for Attention {
is_causal: self.is_causal,
},
) {
- eprintln!(
- "Failed to encode single-pass attention: {:?}",
- e
- );
+ eprintln!("Failed to encode single-pass attention: {:?}", e);
}
},
AttentionKernelVariant::TwoPass => {
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
index a258bd31a..286c09a89 100644
--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
@@ -3,22 +3,14 @@ use std::rc::Rc;
#[cfg(not(feature = "tracing"))]
use crate::backends::metal::MTLCommandEncoder;
-use crate::backends::metal::{
- MTLCommandBuffer, MTLComputeCommandEncoder, ProtocolObject, Retained,
-};
+use crate::backends::metal::{MTLCommandBuffer, MTLComputeCommandEncoder, ProtocolObject, Retained};
use objc2::rc::autoreleasepool;
-use super::{
- Attention, EncodableBlock, Normalization, QKNorm, TensorAddSwap,
- TensorCopy, transformer_layer,
-};
+use super::{Attention, EncodableBlock, Normalization, QKNorm, TensorAddSwap, TensorCopy, transformer_layer};
use crate::forward_pass::state::{ArrayId, ForwardPassState};
use crate::{
DataType,
- backends::metal::{
- MTLContext, compilation_parameters::CompilationConfig,
- kernel::KernelDataType,
- },
+ backends::metal::{MTLContext, compilation_parameters::CompilationConfig, kernel::KernelDataType},
config::TransformerLayerConfig,
encodable_block::EncodingParameters,
parameters::ParameterTree,
@@ -60,32 +52,18 @@ impl ClassifierLayer {
) -> Self {
autoreleasepool(|_| {
let ctx = &*mtl_context; // Reference for functions expecting &MTLContext
- let attention_config = layer_config
- .mixer_config
- .as_attention()
- .expect("Classifier layers must use attention");
- let intermediate_data_type: DataType = attention_config
- .qkv_projection_config
- .activation_precision()
- .into();
- let kernel_data_type: KernelDataType =
- intermediate_data_type.into();
-
- let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>> =
- Box::new(
- TensorCopy::new(
- ctx,
- kernel_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut]
- .into_boxed_slice(),
- )
+ let attention_config =
+ layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
+ let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
+ let kernel_data_type: KernelDataType = intermediate_data_type.into();
+
+ let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>> = Box::new(
+ TensorCopy::new(ctx, kernel_data_type, vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice())
.unwrap(),
- );
+ );
let pre_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =
- if let Some(norm_config) =
- &layer_config.pre_attention_norm_config
- {
+ if let Some(norm_config) = &layer_config.pre_attention_norm_config {
if layer_loader.subtree("pre_mixer_norm").is_ok() {
Some(Box::new(
Normalization::new(
@@ -94,13 +72,9 @@ impl ClassifierLayer {
norm_config.clone(),
ArrayId::Main,
ArrayId::Main,
- &layer_loader
- .subtree("pre_mixer_norm")
- .unwrap(),
+ &layer_loader.subtree("pre_mixer_norm").unwrap(),
)
- .expect(
- "Failed to create pre-attention norm kernel",
- ),
+ .expect("Failed to create pre-attention norm kernel"),
))
} else {
None
@@ -113,11 +87,7 @@ impl ClassifierLayer {
&attention_config.qkv_projection_config,
attention_config.has_qkv_biases,
model_dim,
- [
- num_heads * head_dim,
- num_groups * head_dim,
- num_groups * head_dim,
- ],
+ [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
ctx,
&layer_loader.subtree("mixer.qkv_projection").unwrap(),
ArrayId::Main,
@@ -126,9 +96,7 @@ impl ClassifierLayer {
.expect("Failed to create qkv projection");
let qk_norm: Option<Box<dyn EncodableBlock<Metal>>> =
- if attention_config.query_norm_config.is_some()
- || attention_config.key_norm_config.is_some()
- {
+ if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
match QKNorm::new(
ctx,
intermediate_data_type,
@@ -141,10 +109,7 @@ impl ClassifierLayer {
head_dim,
) {
Ok(norm) => Some(Box::new(norm)),
- Err(e) => panic!(
- "Failed to create QK norm kernel for layer {}: {:?}",
- layer_index, e
- ),
+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
}
} else {
None
@@ -163,9 +128,7 @@ impl ClassifierLayer {
.expect("Failed to create out projection");
let post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =
- if let Some(norm_config) =
- &layer_config.post_attention_norm_config
- {
+ if let Some(norm_config) = &layer_config.post_attention_norm_config {
Some(Box::new(
Normalization::new(
ctx,
@@ -182,24 +145,14 @@ impl ClassifierLayer {
};
let mixer_residual_add: Box<dyn EncodableBlock<Metal>> = Box::new(
- TensorAddSwap::new(
- ctx,
- kernel_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
- )
- .unwrap(),
+ TensorAddSwap::new(ctx, kernel_data_type, vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice())
+ .unwrap(),
);
- let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<Metal>> =
- Box::new(
- TensorCopy::new(
- ctx,
- kernel_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut]
- .into_boxed_slice(),
- )
+ let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<Metal>> = Box::new(
+ TensorCopy::new(ctx, kernel_data_type, vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice())
.unwrap(),
- );
+ );
let pre_mlp_norm: Box<dyn EncodableBlock<Metal>> = Box::new(
Normalization::new(
@@ -253,12 +206,8 @@ impl ClassifierLayer {
);
let mlp_residual_add: Box<dyn EncodableBlock<Metal>> = Box::new(
- TensorAddSwap::new(
- ctx,
- kernel_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
- )
- .unwrap(),
+ TensorAddSwap::new(ctx, kernel_data_type, vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice())
+ .unwrap(),
);
Self {
@@ -292,9 +241,8 @@ impl EncodableBlock<Metal> for ClassifierLayer {
#[cfg(not(feature = "tracing"))]
{
if self.supports_shared_encoder() {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
return;
@@ -302,27 +250,14 @@ impl EncodableBlock<Metal> for ClassifierLayer {
}
#[cfg(feature = "tracing")]
- let layer_traces = state
- .traces()
- .borrow()
- .layer_results
- .get(self.layer_index)
- .cloned();
+ let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().inputs.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());
}
- self.copy_main_to_shortcut_mixer.encode(
- state,
- command_buffer,
- parameters,
- );
+ self.copy_main_to_shortcut_mixer.encode(state, command_buffer, parameters);
if let Some(ref pre_attn_norm) = self.pre_attention_norm {
pre_attn_norm.encode(state, command_buffer, parameters);
@@ -354,11 +289,7 @@ impl EncodableBlock<Metal> for ClassifierLayer {
self.out_projection.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().attention.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
}
if let Some(ref post_attn_norm) = self.post_attention_norm {
@@ -376,59 +307,35 @@ impl EncodableBlock<Metal> for ClassifierLayer {
self.mixer_residual_add.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().mlp_inputs.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
}
- self.copy_main_to_shortcut_mlp.encode(
- state,
- command_buffer,
- parameters,
- );
+ self.copy_main_to_shortcut_mlp.encode(state, command_buffer, parameters);
self.pre_mlp_norm.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().pre_mlp_norm.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
}
self.mlp.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().mlp.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());
}
if let Some(ref post_mlp_norm) = self.post_mlp_norm {
post_mlp_norm.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().post_mlp_norm.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
}
}
self.mlp_residual_add.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().outputs.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());
}
let _ = parameters;
@@ -443,34 +350,18 @@ impl EncodableBlock<Metal> for ClassifierLayer {
#[cfg(not(feature = "tracing"))]
{
self.copy_main_to_shortcut_mixer.supports_shared_encoder()
- && self
- .pre_attention_norm
- .as_ref()
- .map(|b| b.supports_shared_encoder())
- .unwrap_or(true)
+ && self.pre_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
&& self.qkv_projection.supports_shared_encoder()
- && self
- .qk_norm
- .as_ref()
- .map(|b| b.supports_shared_encoder())
- .unwrap_or(true)
+ && self.qk_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
&& self.rope.supports_shared_encoder()
&& self.attention.supports_shared_encoder()
&& self.out_projection.supports_shared_encoder()
- && self
- .post_attention_norm
- .as_ref()
- .map(|b| b.supports_shared_encoder())
- .unwrap_or(true)
+ && self.post_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
&& self.mixer_residual_add.supports_shared_encoder()
&& self.copy_main_to_shortcut_mlp.supports_shared_encoder()
&& self.pre_mlp_norm.supports_shared_encoder()
&& self.mlp.supports_shared_encoder()
- && self
- .post_mlp_norm
- .as_ref()
- .map(|b| b.supports_shared_encoder())
- .unwrap_or(true)
+ && self.post_mlp_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
&& self.mlp_residual_add.supports_shared_encoder()
}
}
@@ -486,45 +377,35 @@ impl EncodableBlock<Metal> for ClassifierLayer {
"encode_with_shared_encoder called on unsupported ClassifierLayer"
);
- self.copy_main_to_shortcut_mixer
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.copy_main_to_shortcut_mixer.encode_with_shared_encoder(state, encoder, parameters);
if let Some(ref pre_attn_norm) = self.pre_attention_norm {
- pre_attn_norm
- .encode_with_shared_encoder(state, encoder, parameters);
+ pre_attn_norm.encode_with_shared_encoder(state, encoder, parameters);
}
- self.qkv_projection
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.qkv_projection.encode_with_shared_encoder(state, encoder, parameters);
if let Some(ref qk_norm) = self.qk_norm {
qk_norm.encode_with_shared_encoder(state, encoder, parameters);
}
self.rope.encode_with_shared_encoder(state, encoder, parameters);
self.attention.encode_with_shared_encoder(state, encoder, parameters);
- self.out_projection
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.out_projection.encode_with_shared_encoder(state, encoder, parameters);
if let Some(ref post_attn_norm) = self.post_attention_norm {
- post_attn_norm
- .encode_with_shared_encoder(state, encoder, parameters);
+ post_attn_norm.encode_with_shared_encoder(state, encoder, parameters);
}
- self.mixer_residual_add
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.mixer_residual_add.encode_with_shared_encoder(state, encoder, parameters);
- self.copy_main_to_shortcut_mlp
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.copy_main_to_shortcut_mlp.encode_with_shared_encoder(state, encoder, parameters);
- self.pre_mlp_norm
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.pre_mlp_norm.encode_with_shared_encoder(state, encoder, parameters);
self.mlp.encode_with_shared_encoder(state, encoder, parameters);
if let Some(ref post_mlp_norm) = self.post_mlp_norm {
- post_mlp_norm
- .encode_with_shared_encoder(state, encoder, parameters);
+ post_mlp_norm.encode_with_shared_encoder(state, encoder, parameters);
}
- self.mlp_residual_add
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.mlp_residual_add.encode_with_shared_encoder(state, encoder, parameters);
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
index d56b09207..19578d4d6 100644
--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
@@ -7,8 +7,7 @@ use super::{EncodableBlock, LayerExecutables, RMSNorm, Rope};
use crate::{
DataType, DecoderConfig,
backends::metal::{
- KernelDataType, MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext,
- ProtocolObject, Retained,
+ KernelDataType, MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,
compilation_parameters::CompilationConfig,
encodable_block::transformer_layer::{embed_block, readout_block},
},
@@ -38,64 +37,43 @@ impl Decoder {
root_weight_loader: &ParameterTree<MTLContext>,
compilation_config: Rc<CompilationConfig>,
) -> Self {
- let decoder_weight_loader = root_weight_loader
- .subtree("transformer")
- .expect("transformer subtree not found");
+ let decoder_weight_loader = root_weight_loader.subtree("transformer").expect("transformer subtree not found");
- let embed =
- embed_block(&decoder_config, &mtl_context, root_weight_loader);
+ let embed = embed_block(&decoder_config, &mtl_context, root_weight_loader);
- let readout =
- readout_block(&decoder_config, &mtl_context, root_weight_loader);
+ let readout = readout_block(&decoder_config, &mtl_context, root_weight_loader);
let attention_data_type = Self::attention_data_type(&decoder_config);
- let norm_reference_layer = decoder_config
- .layer_configs
- .as_ref()
- .map(|configs| &configs[0])
- .unwrap_or(&decoder_config.layer_config);
- let norm_data_type: DataType = match &norm_reference_layer.mixer_config
- {
- MixerConfig::Attention(attention_config) => attention_config
- .qkv_projection_config
- .activation_precision()
- .into(),
- MixerConfig::Mamba(mamba_config) => {
- mamba_config.in_projection_config.activation_precision().into()
+ let norm_reference_layer =
+ decoder_config.layer_configs.as_ref().map(|configs| &configs[0]).unwrap_or(&decoder_config.layer_config);
+ let norm_data_type: DataType = match &norm_reference_layer.mixer_config {
+ MixerConfig::Attention(attention_config) => {
+ attention_config.qkv_projection_config.activation_precision().into()
+ },
+ MixerConfig::Mamba(mamba_config) => mamba_config.in_projection_config.activation_precision().into(),
+ MixerConfig::ShortConv(short_conv_config) => {
+ short_conv_config.in_projection_config.activation_precision().into()
},
- MixerConfig::ShortConv(short_conv_config) => short_conv_config
- .in_projection_config
- .activation_precision()
- .into(),
};
let global_rope = if decoder_config.global_rope_config.is_some() {
- attention_data_type.as_ref().map(|data_type| {
- Self::create_rope_block(
- &mtl_context,
- (*data_type).into(),
- RopeType::Global,
- )
- })
+ attention_data_type
+ .as_ref()
+ .map(|data_type| Self::create_rope_block(&mtl_context, (*data_type).into(), RopeType::Global))
} else {
None
};
let local_rope = if decoder_config.local_rope_config.is_some() {
- attention_data_type.as_ref().map(|data_type| {
- Self::create_rope_block(
- &mtl_context,
- (*data_type).into(),
- RopeType::Local,
- )
- })
+ attention_data_type
+ .as_ref()
+ .map(|data_type| Self::create_rope_block(&mtl_context, (*data_type).into(), RopeType::Local))
} else {
None
};
let model_shape = ModelShape::from_decoder_config(&decoder_config);
- let sliding_window_sizes =
- model_shape.sliding_window_length_per_layer.clone();
+ let sliding_window_sizes = model_shape.sliding_window_length_per_layer.clone();
let layers = (0..decoder_config.num_layers)
.map(|layer_index| {
@@ -107,13 +85,10 @@ impl Decoder {
let layer_type = model_shape.layer_type(layer_index);
let rope_for_layer = match layer_type {
DecoderLayerType::Transformer => {
- let mut rope_block = global_rope.clone().expect(
- "Global rope missing for transformer layer",
- );
- if let (Some(_), Some(local_rope_block)) = (
- sliding_window_sizes[layer_index],
- local_rope.clone(),
- ) {
+ let mut rope_block = global_rope.clone().expect("Global rope missing for transformer layer");
+ if let (Some(_), Some(local_rope_block)) =
+ (sliding_window_sizes[layer_index], local_rope.clone())
+ {
rope_block = local_rope_block;
}
Some(rope_block)
@@ -126,9 +101,7 @@ impl Decoder {
} => None,
};
- let layer_loader = decoder_weight_loader
- .subtree(&format!("layers.{}", layer_index))
- .unwrap();
+ let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();
LayerExecutables::new(
mtl_context.clone(),
@@ -176,10 +149,8 @@ impl Decoder {
kernel_data_type: KernelDataType,
rope_type: RopeType,
) -> Rc<Box<dyn EncodableBlock<Metal>>> {
- let rotation: Box<dyn EncodableBlock<Metal>> = Box::new(
- Rope::new(mtl_context, kernel_data_type, rope_type)
- .expect("Failed to create Rope"),
- );
+ let rotation: Box<dyn EncodableBlock<Metal>> =
+ Box::new(Rope::new(mtl_context, kernel_data_type, rope_type).expect("Failed to create Rope"));
Rc::new(rotation)
}
@@ -190,12 +161,9 @@ impl Decoder {
.as_ref()
.map(|configs| &configs[layer_index])
.unwrap_or(&decoder_config.layer_config);
- layer_config.attention_config().map(|attention_config| {
- attention_config
- .qkv_projection_config
- .activation_precision()
- .into()
- })
+ layer_config
+ .attention_config()
+ .map(|attention_config| attention_config.qkv_projection_config.activation_precision().into())
})
}
}
@@ -230,22 +198,14 @@ impl EncodableBlock<Metal> for Decoder {
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- traces.borrow().output_norm.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());
}
self.readout.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(
- command_buffer,
- ArrayId::Logits,
- traces.borrow().logits.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Logits, traces.borrow().logits.clone());
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
index 1085df36a..c07daece3 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
@@ -7,9 +7,8 @@ use crate::{
backends::{
common::kernel::FullPrecisionEmbeddingLookupKernel,
metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
- Retained, kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,
+ ProtocolObject, Retained, kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,
},
},
encodable_block::EncodingParameters,
@@ -34,41 +33,31 @@ impl FullPrecisionEmbeddingLookup {
input_scale: Option<f32>,
parameter_tree: &ParameterTree<MTLContext>,
) -> Result<Self, EmbeddingError> {
- let kernel = FullPrecisionEmbeddingLookupMetalKernel::new(
- mtl_context,
- data_type.into(),
- )?;
+ let kernel = FullPrecisionEmbeddingLookupMetalKernel::new(mtl_context, data_type.into())?;
let weights = match parameter_tree.leaf("weights") {
Ok(weights) => weights,
Err(_) => parameter_tree.leaf("input_weights").map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load weights: {:?}",
- e
- )))
+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e)))
})?,
};
if weights.shape() != [vocab_size, model_dim] {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Embedding lookup weights shape mismatch: got {:?}, \
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Embedding lookup weights shape mismatch: got {:?}, \
expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim
- ),
- )));
+ weights.shape(),
+ vocab_size,
+ model_dim
+ ))));
}
if weights.data_type() != data_type {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Weights dtype mismatch: got {:?}, expected {:?}",
- weights.data_type(),
- data_type
- ),
- )));
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Weights dtype mismatch: got {:?}, expected {:?}",
+ weights.data_type(),
+ data_type
+ ))));
}
let weights_buffer = weights.buffer().to_owned();
@@ -90,9 +79,7 @@ impl EncodableBlock<Metal> for FullPrecisionEmbeddingLookup {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
index b951ac91f..d1fd4f083 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
@@ -1,8 +1,7 @@
use std::cell::RefCell;
use crate::backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
- ProtocolObject, Retained,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained,
};
use super::{
@@ -35,46 +34,37 @@ impl FullPrecisionEmbeddingReadout {
model_dim: usize,
parameter_tree: &ParameterTree<MTLContext>,
) -> Result<Self, EmbeddingError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(EmbeddingError::UnsupportedDataType(data_type));
}
let weights = match parameter_tree.leaf("weights") {
Ok(weights) => weights,
Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load weights: {:?}",
- e
- )))
+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e)))
})?,
};
if weights.shape() != [vocab_size, model_dim] {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim
- ),
- )));
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim
+ ))));
}
if weights.data_type() != data_type {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Weights dtype mismatch: got {:?}, expected {:?}",
- weights.data_type(),
- data_type
- ),
- )));
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Weights dtype mismatch: got {:?}, expected {:?}",
+ weights.data_type(),
+ data_type
+ ))));
}
let weights_buffer = weights.buffer().to_owned();
- let mut kernel =
- MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;
+ let mut kernel = MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;
kernel.precompile(_mtl_context).map_err(EmbeddingError::MetalError)?;
Ok(Self {
@@ -93,9 +83,7 @@ impl EncodableBlock<Metal> for FullPrecisionEmbeddingReadout {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
index a8bd084f6..d12da06a1 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
@@ -7,9 +7,8 @@ use crate::{
backends::{
common::{Context, kernel::QuantizedEmbeddingLookupKernel},
metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
- Retained, kernel::dsl::QuantizedEmbeddingLookupMetalKernel,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,
+ ProtocolObject, Retained, kernel::dsl::QuantizedEmbeddingLookupMetalKernel,
},
},
config::QuantizationMode,
@@ -96,110 +95,80 @@ impl QuantizedEmbeddingLookup {
) -> Result<Self, EmbeddingError> {
let packing_divisor = mode.packing_divisor();
- let kernel = QuantizedEmbeddingLookupMetalKernel::new(
- mtl_context,
- data_type.into(),
- )?;
+ let kernel = QuantizedEmbeddingLookupMetalKernel::new(mtl_context, data_type.into())?;
// Load weights [vocab_size, model_dim/packing_divisor] as storage_type
- let weights = parameter_tree.leaf(weights_name).map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load weights: {:?}",
- e
- )))
- })?;
+ let weights = parameter_tree
+ .leaf(weights_name)
+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e))))?;
if weights.data_type() != mode.storage_type() {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Expected packed weights of type {:?}, got {:?}",
- mode.storage_type(),
- weights.data_type()
- ),
- )));
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Expected packed weights of type {:?}, got {:?}",
+ mode.storage_type(),
+ weights.data_type()
+ ))));
}
// Load scales [vocab_size, num_groups]
- let scales = parameter_tree.leaf(scales_name).map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load scales: {:?}",
- e
- )))
- })?;
+ let scales = parameter_tree
+ .leaf(scales_name)
+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load scales: {:?}", e))))?;
// Validate shapes and types
let num_groups = (model_dim + group_size - 1) / group_size;
if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim / packing_divisor
- ),
- )));
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim / packing_divisor
+ ))));
}
if scales.shape() != [vocab_size, num_groups] {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",
- scales.shape(),
- vocab_size,
- num_groups
- ),
- )));
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",
+ scales.shape(),
+ vocab_size,
+ num_groups
+ ))));
}
if scales.data_type() != data_type {
- return Err(EmbeddingError::UnsupportedDataType(
- scales.data_type(),
- ));
+ return Err(EmbeddingError::UnsupportedDataType(scales.data_type()));
}
// Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")
- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =
- match parameter_tree.leaf(biases_name) {
- Ok(deq_biases) => {
- if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(EmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",
- deq_biases.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if deq_biases.data_type() != data_type {
- return Err(EmbeddingError::UnsupportedDataType(
- deq_biases.data_type(),
- ));
- }
- deq_biases.buffer().to_owned().into()
- },
- Err(_) => {
- let elem_size: usize = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(EmbeddingError::UnsupportedDataType(
- other,
- ));
- },
- };
- let size_bytes = vocab_size * num_groups * elem_size;
- let buf = mtl_context
- .create_buffer(size_bytes)
- .expect("Failed to allocate buffer");
- unsafe {
- std::ptr::write_bytes(
- metal::MTLBuffer::contents(&*buf).as_ptr(),
- 0,
- size_bytes as usize,
- );
- }
- buf
- },
- };
+ let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = match parameter_tree.leaf(biases_name) {
+ Ok(deq_biases) => {
+ if deq_biases.shape() != [vocab_size, num_groups] {
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",
+ deq_biases.shape(),
+ vocab_size,
+ num_groups
+ ))));
+ }
+ if deq_biases.data_type() != data_type {
+ return Err(EmbeddingError::UnsupportedDataType(deq_biases.data_type()));
+ }
+ deq_biases.buffer().to_owned().into()
+ },
+ Err(_) => {
+ let elem_size: usize = match data_type {
+ DataType::F16 | DataType::BF16 => 2,
+ DataType::F32 => 4,
+ other => {
+ return Err(EmbeddingError::UnsupportedDataType(other));
+ },
+ };
+ let size_bytes = vocab_size * num_groups * elem_size;
+ let buf = mtl_context.create_buffer(size_bytes).expect("Failed to allocate buffer");
+ unsafe {
+ std::ptr::write_bytes(metal::MTLBuffer::contents(&*buf).as_ptr(), 0, size_bytes as usize);
+ }
+ buf
+ },
+ };
let weights_buffer = weights.buffer().to_owned().into();
let scales_buffer = scales.buffer().to_owned().into();
@@ -225,9 +194,7 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingLookup {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
index bde63ffbf..c312ed863 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
@@ -7,13 +7,9 @@ use crate::{
backends::{
common::Context,
metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
- Retained,
- kernel::quant_matmul::{
- QuantizationType, QuantizedMatmulArguments,
- QuantizedMatmulKernel,
- },
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,
+ ProtocolObject, Retained,
+ kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},
},
},
config::QuantizationMode,
@@ -91,20 +87,14 @@ impl QuantizedEmbeddingReadout {
parameter_tree: &ParameterTree<MTLContext>,
) -> Result<Self, EmbeddingError> {
// Load weights [vocab_size, model_dim/2] as U8
- let weights = parameter_tree.leaf(weights_name).map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load weights: {:?}",
- e
- )))
- })?;
+ let weights = parameter_tree
+ .leaf(weights_name)
+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e))))?;
// Load scales [vocab_size, num_groups]
- let scales = parameter_tree.leaf(scales_name).map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load scales: {:?}",
- e
- )))
- })?;
+ let scales = parameter_tree
+ .leaf(scales_name)
+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load scales: {:?}", e))))?;
// Validate shapes
let num_groups = (model_dim + group_size - 1) / group_size;
@@ -114,77 +104,58 @@ impl QuantizedEmbeddingReadout {
let weights_transposed = weights.shape()[0] == vocab_size;
if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim / packing_divisor
- ),
- )));
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim / packing_divisor
+ ))));
}
if scales.shape() != [vocab_size, num_groups] {
- return Err(EmbeddingError::MetalError(MTLError::Generic(
- format!(
- "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",
- scales.shape(),
- vocab_size,
- num_groups
- ),
- )));
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",
+ scales.shape(),
+ vocab_size,
+ num_groups
+ ))));
}
if scales.data_type() != data_type {
- return Err(EmbeddingError::UnsupportedDataType(
- scales.data_type(),
- ));
+ return Err(EmbeddingError::UnsupportedDataType(scales.data_type()));
}
// MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]
- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =
- match parameter_tree.leaf(biases_name) {
- Ok(deq_biases) => {
- if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(EmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",
- deq_biases.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if deq_biases.data_type() != data_type {
- return Err(EmbeddingError::UnsupportedDataType(
- deq_biases.data_type(),
- ));
- }
- deq_biases.buffer().to_owned().into()
- },
- Err(_) => {
- // Allocate zero-initialized biases buffer
- let elem_size: usize = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(EmbeddingError::UnsupportedDataType(
- other,
- ));
- },
- };
- let size_bytes = vocab_size * num_groups * elem_size;
- let buf = mtl_context
- .create_buffer(size_bytes)
- .expect("Failed to allocate buffer");
- unsafe {
- std::ptr::write_bytes(
- metal::MTLBuffer::contents(&*buf).as_ptr(),
- 0,
- size_bytes as usize,
- );
- }
- buf
- },
- };
+ let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = match parameter_tree.leaf(biases_name) {
+ Ok(deq_biases) => {
+ if deq_biases.shape() != [vocab_size, num_groups] {
+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
+ "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",
+ deq_biases.shape(),
+ vocab_size,
+ num_groups
+ ))));
+ }
+ if deq_biases.data_type() != data_type {
+ return Err(EmbeddingError::UnsupportedDataType(deq_biases.data_type()));
+ }
+ deq_biases.buffer().to_owned().into()
+ },
+ Err(_) => {
+ // Allocate zero-initialized biases buffer
+ let elem_size: usize = match data_type {
+ DataType::F16 | DataType::BF16 => 2,
+ DataType::F32 => 4,
+ other => {
+ return Err(EmbeddingError::UnsupportedDataType(other));
+ },
+ };
+ let size_bytes = vocab_size * num_groups * elem_size;
+ let buf = mtl_context.create_buffer(size_bytes).expect("Failed to allocate buffer");
+ unsafe {
+ std::ptr::write_bytes(metal::MTLBuffer::contents(&*buf).as_ptr(), 0, size_bytes as usize);
+ }
+ buf
+ },
+ };
let weights_buffer = weights.buffer().to_owned().into();
let scales_buffer = scales.buffer().to_owned().into();
@@ -199,12 +170,7 @@ impl QuantizedEmbeddingReadout {
QuantizationType::Mlx,
weights_transposed,
)
- .map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to create kernel: {:?}",
- e
- )))
- })?;
+ .map_err(|e| EmbeddingError::MetalError(MTLError::Generic(format!("Failed to create kernel: {:?}", e))))?;
Ok(Self {
kernel,
@@ -224,9 +190,7 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
@@ -273,8 +237,6 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {
quantization_type: QuantizationType::Mlx,
};
- self.kernel
- .encode(encoder, args)
- .expect("Failed to encode quantized embedding readout kernel");
+ self.kernel.encode(encoder, args).expect("Failed to encode quantized embedding readout kernel");
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
index 5f86335d5..34455bca8 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -5,24 +5,19 @@ use std::rc::Rc;
#[cfg(not(feature = "tracing"))]
use crate::backends::metal::MTLCommandEncoder;
-use crate::backends::metal::{
- MTLCommandBuffer, MTLComputeCommandEncoder, ProtocolObject, Retained,
-};
+use crate::backends::metal::{MTLCommandBuffer, MTLComputeCommandEncoder, ProtocolObject, Retained};
use objc2::rc::autoreleasepool;
use super::{
super::{
- Attention, EncodableBlock, MambaMixer, QKNorm, RMSNorm, ShortConvMixer,
- TensorAddSwap, TensorCopy, transformer_layer,
+ Attention, EncodableBlock, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap, TensorCopy,
+ transformer_layer,
},
MixerExecutables,
};
use crate::{
DataType, DecoderLayerConfig,
- backends::metal::{
- MTLContext, compilation_parameters::CompilationConfig,
- kernel::KernelDataType,
- },
+ backends::metal::{MTLContext, compilation_parameters::CompilationConfig, kernel::KernelDataType},
config::{DecoderLayerType, MixerConfig},
encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState},
@@ -61,33 +56,17 @@ impl LayerExecutables {
) -> Self {
autoreleasepool(|_| {
let ctx = &*mtl_context; // Reference for functions expecting &MTLContext
- let intermediate_data_type: DataType =
- match &layer_config.mixer_config {
- MixerConfig::Attention(attention) => attention
- .qkv_projection_config
- .activation_precision()
- .into(),
- MixerConfig::Mamba(mamba) => {
- mamba.in_projection_config.activation_precision().into()
- },
- MixerConfig::ShortConv(short_conv) => short_conv
- .in_projection_config
- .activation_precision()
- .into(),
- };
- let kernel_data_type: KernelDataType =
- intermediate_data_type.into();
+ let intermediate_data_type: DataType = match &layer_config.mixer_config {
+ MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),
+ MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),
+ MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
+ };
+ let kernel_data_type: KernelDataType = intermediate_data_type.into();
- let copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>> =
- Box::new(
- TensorCopy::new(
- ctx,
- kernel_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut]
- .into_boxed_slice(),
- )
+ let copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>> = Box::new(
+ TensorCopy::new(ctx, kernel_data_type, vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice())
.unwrap(),
- );
+ );
let pre_attention_norm: Box<dyn EncodableBlock<Metal>> = Box::new(
RMSNorm::new(
@@ -103,32 +82,22 @@ impl LayerExecutables {
let mixer = match &layer_config.mixer_config {
MixerConfig::Attention(attention_config) => {
- let rope_block = rope
- .clone()
- .expect("RoPE encoder missing for attention layer");
+ let rope_block = rope.clone().expect("RoPE encoder missing for attention layer");
let qkv_projection = transformer_layer::linear_block(
&attention_config.qkv_projection_config,
attention_config.has_qkv_biases,
model_dim,
- [
- num_heads * head_dim,
- num_groups * head_dim,
- num_groups * head_dim,
- ],
+ [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
ctx,
- &decoder_layer_loader
- .subtree("mixer.qkv_projection")
- .unwrap(),
+ &decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),
ArrayId::Main,
ArrayId::QKV,
)
.expect("Failed to create qkv projection");
let qk_norm: Option<Box<dyn EncodableBlock<Metal>>> =
- if attention_config.query_norm_config.is_some()
- || attention_config.key_norm_config.is_some()
- {
+ if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
match QKNorm::new(
ctx,
intermediate_data_type,
@@ -140,12 +109,8 @@ impl LayerExecutables {
num_groups,
head_dim,
) {
- Ok(qk_norm) => Some(Box::new(qk_norm)
- as Box<dyn EncodableBlock<Metal>>),
- Err(e) => panic!(
- "Failed to create QK norm kernel for layer {}: {:?}",
- layer_index, e
- ),
+ Ok(qk_norm) => Some(Box::new(qk_norm) as Box<dyn EncodableBlock<Metal>>),
+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
}
} else {
None
@@ -157,9 +122,7 @@ impl LayerExecutables {
num_heads * head_dim,
[model_dim],
ctx,
- &decoder_layer_loader
- .subtree("mixer.out_projection")
- .unwrap(),
+ &decoder_layer_loader.subtree("mixer.out_projection").unwrap(),
ArrayId::AttentionOutput,
ArrayId::Main,
)
@@ -187,34 +150,32 @@ impl LayerExecutables {
}
},
MixerConfig::Mamba(mamba_config) => {
- let mixer: Box<dyn EncodableBlock<Metal>> =
- Box::new(MambaMixer::new(
- ctx,
- layer_type.clone(),
- mamba_config.clone(),
- compilation_config.clone(),
- layer_index,
- model_dim,
- num_heads,
- head_dim,
- num_groups,
- decoder_layer_loader,
- ));
+ let mixer: Box<dyn EncodableBlock<Metal>> = Box::new(MambaMixer::new(
+ ctx,
+ layer_type.clone(),
+ mamba_config.clone(),
+ compilation_config.clone(),
+ layer_index,
+ model_dim,
+ num_heads,
+ head_dim,
+ num_groups,
+ decoder_layer_loader,
+ ));
MixerExecutables::StateSpace {
mixer,
}
},
MixerConfig::ShortConv(short_conv_config) => {
- let mixer: Box<dyn EncodableBlock<Metal>> =
- Box::new(ShortConvMixer::new(
- ctx,
- layer_type.clone(),
- short_conv_config.clone(),
- compilation_config.clone(),
- layer_index,
- model_dim,
- decoder_layer_loader,
- ));
+ let mixer: Box<dyn EncodableBlock<Metal>> = Box::new(ShortConvMixer::new(
+ ctx,
+ layer_type.clone(),
+ short_conv_config.clone(),
+ compilation_config.clone(),
+ layer_index,
+ model_dim,
+ decoder_layer_loader,
+ ));
MixerExecutables::ShortConv {
mixer,
}
@@ -222,9 +183,7 @@ impl LayerExecutables {
};
let post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =
- if let Some(norm_config) =
- &layer_config.post_attention_norm_config
- {
+ if let Some(norm_config) = &layer_config.post_attention_norm_config {
Some(Box::new(
RMSNorm::new(
ctx,
@@ -232,9 +191,7 @@ impl LayerExecutables {
norm_config.clone(),
ArrayId::Main,
ArrayId::Main,
- &decoder_layer_loader
- .subtree("post_mixer_norm")
- .unwrap(),
+ &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
)
.expect("Failed to create RMS norm kernel"),
))
@@ -242,16 +199,10 @@ impl LayerExecutables {
None
};
- let main_shortcut_add_swap: Box<dyn EncodableBlock<Metal>> =
- Box::new(
- TensorAddSwap::new(
- ctx,
- kernel_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main]
- .into_boxed_slice(),
- )
+ let main_shortcut_add_swap: Box<dyn EncodableBlock<Metal>> = Box::new(
+ TensorAddSwap::new(ctx, kernel_data_type, vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice())
.unwrap(),
- );
+ );
let pre_mlp_norm: Box<dyn EncodableBlock<Metal>> = Box::new(
RMSNorm::new(
@@ -283,9 +234,7 @@ impl LayerExecutables {
norm_config.clone(),
ArrayId::Main,
ArrayId::Main,
- &decoder_layer_loader
- .subtree("post_mlp_norm")
- .unwrap(),
+ &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
)
.expect("Failed to create RMS norm kernel"),
))
@@ -320,9 +269,8 @@ impl EncodableBlock<Metal> for LayerExecutables {
#[cfg(not(feature = "tracing"))]
{
if self.supports_shared_encoder() {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
return;
@@ -330,20 +278,11 @@ impl EncodableBlock<Metal> for LayerExecutables {
}
#[cfg(feature = "tracing")]
- let layer_traces = state
- .traces()
- .borrow()
- .layer_results
- .get(self.layer_index)
- .cloned();
+ let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().inputs.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());
}
self.copy_main_to_shortcut.encode(state, command_buffer, parameters);
@@ -352,11 +291,7 @@ impl EncodableBlock<Metal> for LayerExecutables {
self.pre_attention_norm.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().pre_attention_norm.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
}
match &self.mixer {
@@ -376,11 +311,7 @@ impl EncodableBlock<Metal> for LayerExecutables {
out_projection.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().attention.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
}
},
MixerExecutables::StateSpace {
@@ -389,11 +320,7 @@ impl EncodableBlock<Metal> for LayerExecutables {
mixer.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().attention.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
}
},
MixerExecutables::ShortConv {
@@ -402,11 +329,7 @@ impl EncodableBlock<Metal> for LayerExecutables {
mixer.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().attention.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
}
},
}
@@ -429,42 +352,26 @@ impl EncodableBlock<Metal> for LayerExecutables {
// main = input + attention_result
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().mlp_inputs.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
}
self.pre_mlp_norm.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().pre_mlp_norm.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
}
self.mlp.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().mlp.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());
}
if let Some(post_mlp_norm) = &self.post_mlp_norm {
post_mlp_norm.encode(state, command_buffer, parameters);
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().post_mlp_norm.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
}
}
// main = mlp_result
@@ -474,11 +381,7 @@ impl EncodableBlock<Metal> for LayerExecutables {
// main = input + attention_result + mlp_result
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().outputs.clone(),
- );
+ state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());
}
let _ = parameters;
@@ -501,10 +404,7 @@ impl EncodableBlock<Metal> for LayerExecutables {
out_projection,
} => {
qkv_projection.supports_shared_encoder()
- && qk_norm
- .as_ref()
- .map(|b| b.supports_shared_encoder())
- .unwrap_or(true)
+ && qk_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
&& rope.supports_shared_encoder()
&& attention.supports_shared_encoder()
&& out_projection.supports_shared_encoder()
@@ -520,19 +420,11 @@ impl EncodableBlock<Metal> for LayerExecutables {
self.copy_main_to_shortcut.supports_shared_encoder()
&& self.pre_attention_norm.supports_shared_encoder()
&& mixer_ok
- && self
- .post_attention_norm
- .as_ref()
- .map(|b| b.supports_shared_encoder())
- .unwrap_or(true)
+ && self.post_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
&& self.main_shortcut_add_swap.supports_shared_encoder()
&& self.pre_mlp_norm.supports_shared_encoder()
&& self.mlp.supports_shared_encoder()
- && self
- .post_mlp_norm
- .as_ref()
- .map(|b| b.supports_shared_encoder())
- .unwrap_or(true)
+ && self.post_mlp_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
}
}
@@ -547,10 +439,8 @@ impl EncodableBlock<Metal> for LayerExecutables {
"encode_with_shared_encoder called on unsupported LayerExecutables"
);
- self.copy_main_to_shortcut
- .encode_with_shared_encoder(state, encoder, parameters);
- self.pre_attention_norm
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.copy_main_to_shortcut.encode_with_shared_encoder(state, encoder, parameters);
+ self.pre_attention_norm.encode_with_shared_encoder(state, encoder, parameters);
match &self.mixer {
MixerExecutables::Attention {
@@ -560,16 +450,13 @@ impl EncodableBlock<Metal> for LayerExecutables {
attention,
out_projection,
} => {
- qkv_projection
- .encode_with_shared_encoder(state, encoder, parameters);
+ qkv_projection.encode_with_shared_encoder(state, encoder, parameters);
if let Some(norm) = qk_norm {
norm.encode_with_shared_encoder(state, encoder, parameters);
}
rope.encode_with_shared_encoder(state, encoder, parameters);
- attention
- .encode_with_shared_encoder(state, encoder, parameters);
- out_projection
- .encode_with_shared_encoder(state, encoder, parameters);
+ attention.encode_with_shared_encoder(state, encoder, parameters);
+ out_projection.encode_with_shared_encoder(state, encoder, parameters);
},
MixerExecutables::StateSpace {
mixer,
@@ -584,23 +471,18 @@ impl EncodableBlock<Metal> for LayerExecutables {
}
if let Some(post_attention_norm) = &self.post_attention_norm {
- post_attention_norm
- .encode_with_shared_encoder(state, encoder, parameters);
+ post_attention_norm.encode_with_shared_encoder(state, encoder, parameters);
}
- self.main_shortcut_add_swap
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.main_shortcut_add_swap.encode_with_shared_encoder(state, encoder, parameters);
- self.pre_mlp_norm
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.pre_mlp_norm.encode_with_shared_encoder(state, encoder, parameters);
self.mlp.encode_with_shared_encoder(state, encoder, parameters);
if let Some(post_mlp_norm) = &self.post_mlp_norm {
- post_mlp_norm
- .encode_with_shared_encoder(state, encoder, parameters);
+ post_mlp_norm.encode_with_shared_encoder(state, encoder, parameters);
}
- self.main_shortcut_add_swap
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.main_shortcut_add_swap.encode_with_shared_encoder(state, encoder, parameters);
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
index 490544054..b5a9fe737 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
@@ -4,8 +4,7 @@ use std::cell::RefCell;
use crate::{
DataType,
backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
Retained,
encodable_block::EncodableBlock,
kernel::matmul::{MatmulArguments, MatmulKernel},
@@ -35,17 +34,16 @@ impl FullPrecisionLinear {
input_array_id: ArrayId,
output_array_id: ArrayId,
) -> Result<Self, MTLError> {
- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32)
- {
+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(MTLError::Generic(format!(
"Unsupported data type for full precision linear kernel: {:?}",
precision
)));
}
- let weights = parameter_tree.leaf("weights").map_err(|e| {
- MTLError::Generic(format!("Failed to load weights: {:?}", e))
- })?;
+ let weights = parameter_tree
+ .leaf("weights")
+ .map_err(|e| MTLError::Generic(format!("Failed to load weights: {:?}", e)))?;
let w_shape = weights.shape();
if w_shape != [output_dim, input_dim] {
@@ -63,8 +61,7 @@ impl FullPrecisionLinear {
)));
}
- let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =
- weights.buffer().to_owned().into();
+ let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = weights.buffer().to_owned().into();
let bias_buffer = match parameter_tree.leaf("biases") {
Ok(biases) => {
@@ -82,8 +79,7 @@ impl FullPrecisionLinear {
precision
)));
}
- let bias_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =
- biases.buffer().to_owned().into();
+ let bias_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = biases.buffer().to_owned().into();
Some(bias_buffer)
},
Err(_) => None,
@@ -120,9 +116,7 @@ impl EncodableBlock<Metal> for FullPrecisionLinear {
let input_buffer = input_array_mut.buffer();
let output_buffer = output_array_mut.buffer();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
let args = MatmulArguments {
a: input_buffer,
@@ -145,9 +139,7 @@ impl EncodableBlock<Metal> for FullPrecisionLinear {
};
let mut kernel = self.kernel.borrow_mut();
- kernel
- .encode(state.mtl_context(), &encoder, args)
- .expect("Failed to encode matmul kernel");
+ kernel.encode(state.mtl_context(), &encoder, args).expect("Failed to encode matmul kernel");
encoder.end_encoding();
@@ -197,8 +189,6 @@ impl EncodableBlock<Metal> for FullPrecisionLinear {
};
let mut kernel = self.kernel.borrow_mut();
- kernel
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode matmul kernel");
+ kernel.encode(state.mtl_context(), encoder, args).expect("Failed to encode matmul kernel");
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
index 58e96ca3f..8e85a4971 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
@@ -2,10 +2,7 @@ use crate::backends::metal::Metal;
use crate::backends::{
common::kernel::TensorAddBiasKernel,
- metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, ProtocolObject, Retained,
- },
+ metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained},
};
use crate::{
@@ -15,10 +12,7 @@ use crate::{
encodable_block::EncodableBlock,
kernel::{
dsl::TensorAddBiasMetalKernel,
- quant_matmul::{
- QuantizationType, QuantizedMatmulArguments,
- QuantizedMatmulKernel,
- },
+ quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},
},
},
config::QuantizationConfig,
@@ -53,19 +47,16 @@ impl QuantizedLinear {
) -> Result<Self, MTLError> {
let kernel_data_type: DataType = config.activation_precision.into();
- if !matches!(
- kernel_data_type,
- DataType::F16 | DataType::BF16 | DataType::F32
- ) {
+ if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(MTLError::Generic(format!(
"Unsupported data type for quantized kernel: {:?}",
kernel_data_type
)));
}
- let weights = parameter_tree.leaf("weights").map_err(|e| {
- MTLError::Generic(format!("Failed to load weights: {:?}", e))
- })?;
+ let weights = parameter_tree
+ .leaf("weights")
+ .map_err(|e| MTLError::Generic(format!("Failed to load weights: {:?}", e)))?;
let packing_divisor = config.weight_quantization_mode.packing_divisor();
let storage_type = config.weight_quantization_mode.storage_type();
@@ -78,9 +69,8 @@ impl QuantizedLinear {
)));
}
- let scales = parameter_tree.leaf("scales").map_err(|e| {
- MTLError::Generic(format!("Failed to load scales: {:?}", e))
- })?;
+ let scales =
+ parameter_tree.leaf("scales").map_err(|e| MTLError::Generic(format!("Failed to load scales: {:?}", e)))?;
let k_g = (input_dim + config.group_size - 1) / config.group_size;
let w_shape = weights.shape();
@@ -117,23 +107,16 @@ impl QuantizedLinear {
kernel_data_type
)));
}
- let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =
- scales.buffer().clone();
- let biases_buf: Retained<ProtocolObject<dyn MTLBuffer>> =
- deq_biases.buffer().clone();
+ let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = scales.buffer().clone();
+ let biases_buf: Retained<ProtocolObject<dyn MTLBuffer>> = deq_biases.buffer().clone();
(QuantizationType::Mlx, biases_buf, scales_buffer)
},
Err(_) => {
- let zero_points =
- parameter_tree.leaf("zero_points").map_err(|e| {
- MTLError::Generic(format!(
- "Failed to load zero_points: {:?}",
- e
- ))
- })?;
+ let zero_points = parameter_tree
+ .leaf("zero_points")
+ .map_err(|e| MTLError::Generic(format!("Failed to load zero_points: {:?}", e)))?;
let zp_shape = zero_points.shape();
- let expected_zp_entries =
- (k_g + packing_divisor - 1) / packing_divisor;
+ let expected_zp_entries = (k_g + packing_divisor - 1) / packing_divisor;
if !(w_shape == [output_dim, input_dim / packing_divisor]
&& s_shape == [output_dim, k_g]
&& zp_shape == [output_dim, expected_zp_entries])
@@ -155,46 +138,38 @@ impl QuantizedLinear {
storage_type
)));
}
- let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =
- scales.buffer().clone();
- let zps_buf: Retained<ProtocolObject<dyn MTLBuffer>> =
- zero_points.buffer().clone();
+ let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = scales.buffer().clone();
+ let zps_buf: Retained<ProtocolObject<dyn MTLBuffer>> = zero_points.buffer().clone();
(QuantizationType::ZeroPoint, zps_buf, scales_buffer)
},
}
};
- let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =
- weights.buffer().to_owned().into();
+ let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = weights.buffer().to_owned().into();
// Optional trainable bias support
- let (bias_add_kernel, biases_buffer) =
- match parameter_tree.leaf("biases") {
- Ok(biases) => {
- if biases.shape() != [output_dim] {
- return Err(MTLError::Generic(format!(
- "Bias shape mismatch: got {:?}, expected [{:?}]",
- biases.shape(),
- output_dim
- )));
- }
- if biases.data_type() != kernel_data_type {
- return Err(MTLError::Generic(format!(
- "Bias dtype mismatch: got {:?}, expected {:?}",
- biases.data_type(),
- kernel_data_type
- )));
- }
- let bias_add_kernel = Some(TensorAddBiasMetalKernel::new(
- mtl_context,
- kernel_data_type,
- )?);
- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> =
- biases.buffer().to_owned().into();
- (bias_add_kernel, Some(biases_buffer))
- },
- Err(_) => (None, None),
- };
+ let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf("biases") {
+ Ok(biases) => {
+ if biases.shape() != [output_dim] {
+ return Err(MTLError::Generic(format!(
+ "Bias shape mismatch: got {:?}, expected [{:?}]",
+ biases.shape(),
+ output_dim
+ )));
+ }
+ if biases.data_type() != kernel_data_type {
+ return Err(MTLError::Generic(format!(
+ "Bias dtype mismatch: got {:?}, expected {:?}",
+ biases.data_type(),
+ kernel_data_type
+ )));
+ }
+ let bias_add_kernel = Some(TensorAddBiasMetalKernel::new(mtl_context, kernel_data_type)?);
+ let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = biases.buffer().to_owned().into();
+ (bias_add_kernel, Some(biases_buffer))
+ },
+ Err(_) => (None, None),
+ };
let g = config.group_size;
let mode = config.weight_quantization_mode;
@@ -243,9 +218,7 @@ impl EncodableBlock<Metal> for QuantizedLinear {
let input_buffer = input_array_mut.buffer();
let output_buffer = output_array_mut.buffer();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
let args = QuantizedMatmulArguments {
a_buffer: input_buffer,
@@ -260,15 +233,11 @@ impl EncodableBlock<Metal> for QuantizedLinear {
quantization_type: self.quantization_type,
};
- self.kernel
- .encode(&encoder, args)
- .expect("Failed to encode quantized matmul kernel");
+ self.kernel.encode(&encoder, args).expect("Failed to encode quantized matmul kernel");
encoder.end_encoding();
- if let (Some(bias_add), Some(bias_buf)) =
- (&self.bias_add_kernel, &self.biases_buffer)
- {
+ if let (Some(bias_add), Some(bias_buf)) = (&self.bias_add_kernel, &self.biases_buffer) {
let total_len = batch_size * self.output_dim;
bias_add.encode_if(
output_buffer,
@@ -319,13 +288,9 @@ impl EncodableBlock<Metal> for QuantizedLinear {
quantization_type: self.quantization_type,
};
- self.kernel
- .encode(encoder, args)
- .expect("Failed to encode quantized matmul kernel");
+ self.kernel.encode(encoder, args).expect("Failed to encode quantized matmul kernel");
- if let (Some(bias_add), Some(bias_buf)) =
- (&self.bias_add_kernel, &self.biases_buffer)
- {
+ if let (Some(bias_add), Some(bias_buf)) = (&self.bias_add_kernel, &self.biases_buffer) {
let total_len = batch_size * self.output_dim;
bias_add.encode_if(
output_buffer,
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
index f7c110a2c..32e438c09 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
@@ -1,22 +1,18 @@
//! Mamba2 SSM mixer encodable.
use super::{EncodableBlock, Metal, transformer_layer};
-use crate::backends::metal::kernel::ssm::ssd_prefill::{
- SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode,
-};
+use crate::backends::metal::kernel::ssm::ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode};
use crate::{
DataType,
backends::{
common::kernel::{SSDUpdateKernel, SplitInProjKernel},
metal::{
- KernelDataType, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,
- Retained,
+ KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray,
+ ProtocolObject, Retained,
compilation_parameters::CompilationConfig,
kernel::dsl::{SSDUpdateMetalKernel, SplitInProjMetalKernel},
kernel::ssm::{
- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
- conv1d_scan::Conv1dDecodeArguments,
+ Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, conv1d_scan::Conv1dDecodeArguments,
},
},
},
@@ -59,32 +55,21 @@ impl MambaMixer {
) -> Self {
let _ = (num_heads, head_dim, num_groups);
if !matches!(layer_type, DecoderLayerType::StateSpace { .. }) {
- panic!(
- "Layer {} marked as transformer but Mamba mixer config provided",
- layer_index
- );
+ panic!("Layer {} marked as transformer but Mamba mixer config provided", layer_index);
}
let split_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);
let conv_tree = resolve_subtree(&split_tree, &["conv", "conv1d"]);
- let data_type: DataType =
- mamba_config.in_projection_config.activation_precision().into();
+ let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();
let kernel_data_type: KernelDataType = data_type.into();
let in_projection = transformer_layer::linear_block(
&mamba_config.in_projection_config,
mamba_config.has_in_biases,
model_dim,
- [
- mamba_config.conv_dim(),
- mamba_config.inner_dim(),
- mamba_config.num_heads,
- ],
+ [mamba_config.conv_dim(), mamba_config.inner_dim(), mamba_config.num_heads],
mtl_context,
- &resolve_subtree(
- decoder_layer_loader,
- &["mixer.in_projection", "mixer.in_proj"],
- ),
+ &resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),
ArrayId::Main,
ArrayId::SsmInProj,
)
@@ -96,10 +81,7 @@ impl MambaMixer {
mamba_config.inner_dim(),
[model_dim],
mtl_context,
- &resolve_subtree(
- decoder_layer_loader,
- &["mixer.out_projection", "mixer.out_proj"],
- ),
+ &resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),
ArrayId::AttentionOutput,
ArrayId::Main,
)
@@ -112,23 +94,16 @@ impl MambaMixer {
None
};
let gate_bias = split_tree.leaf("gate_bias").unwrap().clone();
- let skip_connection_weight =
- split_tree.leaf("skip_connection_weight").unwrap().clone();
-
- let split_inproj =
- SplitInProjMetalKernel::new(mtl_context, kernel_data_type.into())
- .expect("Failed to create split in-projection kernel");
- let conv_scan = Conv1dScanKernel::new(
- mtl_context,
- kernel_data_type,
- &mamba_config.activation,
- )
- .expect("Failed to create conv scan kernel");
- let ssd_prefill = SSDPrefillKernels::new(mtl_context, kernel_data_type)
- .expect("Failed to create SSD prefill kernel");
- let ssd_update =
- SSDUpdateMetalKernel::new(mtl_context, kernel_data_type.into())
- .expect("Failed to create SSD decode kernel");
+ let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap().clone();
+
+ let split_inproj = SplitInProjMetalKernel::new(mtl_context, kernel_data_type.into())
+ .expect("Failed to create split in-projection kernel");
+ let conv_scan = Conv1dScanKernel::new(mtl_context, kernel_data_type, &mamba_config.activation)
+ .expect("Failed to create conv scan kernel");
+ let ssd_prefill =
+ SSDPrefillKernels::new(mtl_context, kernel_data_type).expect("Failed to create SSD prefill kernel");
+ let ssd_update = SSDUpdateMetalKernel::new(mtl_context, kernel_data_type.into())
+ .expect("Failed to create SSD decode kernel");
let prefill_mode = resolve_prefill_mode_from_env();
Self {
@@ -159,8 +134,7 @@ impl MambaMixer {
return;
}
- self.in_projection
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.in_projection.encode_with_shared_encoder(state, encoder, parameters);
self.run_split_inproj(state, encoder, active_suffix_length);
self.run_conv_scan(state, encoder, active_suffix_length);
@@ -170,8 +144,7 @@ impl MambaMixer {
self.run_prefill_ssm(state, encoder, active_suffix_length);
}
- self.out_projection
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.out_projection.encode_with_shared_encoder(state, encoder, parameters);
}
fn run_split_inproj(
@@ -281,9 +254,7 @@ impl MambaMixer {
.expect("Failed to encode conv decode kernel");
} else {
let padded_buf = if state_stride > 0 {
- let array = state
- .conv_padded_buffer()
- .expect("Missing conv padded buffer");
+ let array = state.conv_padded_buffer().expect("Missing conv padded buffer");
let borrow = array.borrow_mut();
let buf = borrow.buffer().clone();
drop(borrow);
@@ -386,25 +357,12 @@ impl MambaMixer {
state: &state_raw,
y: &out_buf,
suffix_len: suffix_length,
- group_size: (self.config.num_heads / self.config.num_groups)
- as i32,
+ group_size: (self.config.num_heads / self.config.num_groups) as i32,
state_size: self.config.state_dim as i32,
- x_strides: [
- self.config.num_heads * self.config.head_dim,
- self.config.head_dim,
- 1,
- ],
+ x_strides: [self.config.num_heads * self.config.head_dim, self.config.head_dim, 1],
dt_strides: [self.config.num_heads, 1],
- cb_strides: [
- self.config.num_groups * self.config.state_dim,
- self.config.state_dim,
- 1,
- ],
- state_strides: [
- self.config.head_dim * self.config.state_dim,
- self.config.state_dim,
- 1,
- ],
+ cb_strides: [self.config.num_groups * self.config.state_dim, self.config.state_dim, 1],
+ state_strides: [self.config.head_dim * self.config.state_dim, self.config.state_dim, 1],
channels: self.config.num_heads,
head_dim: self.config.head_dim,
},
@@ -490,9 +448,7 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
match env::var("UZU_SSM_PREFILL_MODE") {
Ok(value) => match value.trim().to_ascii_lowercase().as_str() {
"seq" | "sequential" | "baseline" => SSDPrefillMode::Sequential,
- "single" | "singlepass" | "single_pass" => {
- SSDPrefillMode::SinglePass
- },
+ "single" | "singlepass" | "single_pass" => SSDPrefillMode::SinglePass,
_ => SSDPrefillMode::SinglePass,
},
Err(_) => SSDPrefillMode::SinglePass,
@@ -507,9 +463,8 @@ impl EncodableBlock<Metal> for MambaMixer {
parameters: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_pipeline_with_encoder(state, &encoder, parameters);
encoder.end_encoding();
@@ -527,9 +482,7 @@ impl EncodableBlock<Metal> for MambaMixer {
self.in_projection.encode(state, command_buffer, parameters);
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.run_split_inproj(state, &encoder, active_suffix_length);
self.run_conv_scan(state, &encoder, active_suffix_length);
if active_suffix_length == 1 {
@@ -548,8 +501,7 @@ impl EncodableBlock<Metal> for MambaMixer {
}
fn supports_shared_encoder(&self) -> bool {
- self.in_projection.supports_shared_encoder()
- && self.out_projection.supports_shared_encoder()
+ self.in_projection.supports_shared_encoder() && self.out_projection.supports_shared_encoder()
}
fn encode_with_shared_encoder(
@@ -572,9 +524,6 @@ fn resolve_subtree<'tree>(
}
}
- let missing =
- candidates.first().copied().unwrap_or("<missing subtree name>");
- loader.subtree(missing).unwrap_or_else(|_| {
- panic!("Unable to resolve parameter subtree '{missing}'")
- })
+ let missing = candidates.first().copied().unwrap_or("<missing subtree name>");
+ loader.subtree(missing).unwrap_or_else(|_| panic!("Unable to resolve parameter subtree '{missing}'"))
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
index 6eb7f499f..dc68466ca 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
@@ -3,8 +3,7 @@
use std::{cell::RefCell, rc::Rc};
use crate::backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
- ProtocolObject, Retained,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained,
};
use super::{EncodableBlock, Metal};
@@ -15,10 +14,7 @@ use crate::{
kernel::{
mlp::{MlpActivationType, MlpGateActMulEncodable},
mlp_fused::{MlpFusedArguments, MlpFusedKernel},
- quant_matmul::{
- MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments,
- MlpFusedQmvKernel,
- },
+ quant_matmul::{MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments, MlpFusedQmvKernel},
},
},
config::Activation,
@@ -54,9 +50,8 @@ impl EncodableBlock<Metal> for MlpBlock {
params: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, params);
encoder.end_encoding();
} else {
@@ -65,17 +60,15 @@ impl EncodableBlock<Metal> for MlpBlock {
// Gate act+mul (fused_up -> hidden)
{
- let arrays =
- state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
+ let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
let fused = arrays[0].borrow_mut();
let hidden = arrays[1].borrow_mut();
let m = fused.shape()[0] as i32;
let fused_buf = fused.buffer();
let hidden_buf = hidden.buffer();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.gate
.encode(&encoder, fused_buf, hidden_buf, m)
.expect("Failed to encode MLP activation/mul kernel");
@@ -112,9 +105,7 @@ impl EncodableBlock<Metal> for MlpBlock {
let m = fused.shape()[0] as i32;
let fused_buf = fused.buffer();
let hidden_buf = hidden.buffer();
- self.gate
- .encode(encoder, fused_buf, hidden_buf, m)
- .expect("Failed to encode MLP activation/mul kernel");
+ self.gate.encode(encoder, fused_buf, hidden_buf, m).expect("Failed to encode MLP activation/mul kernel");
drop(fused);
drop(hidden);
@@ -143,8 +134,7 @@ pub struct MlpFusedBlock {
down: Box<dyn EncodableBlock<Metal>>,
weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
scales_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- zero_points_or_biases_buffer:
- Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
+ zero_points_or_biases_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
input_dim: usize,
hidden_dim: usize,
activation: MlpActivationType,
@@ -201,27 +191,11 @@ impl MlpFusedBlock {
input_array_id: ArrayId,
hidden_array_id: ArrayId,
) -> Result<Self, crate::backends::metal::MTLError> {
- let qmv = MlpFusedQmvKernel::new(
- &context,
- data_type,
- group_size,
- mode,
- quantization_type,
- )
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!("{:?}", e))
- })?;
-
- let qmm = MlpFusedQmmKernel::new(
- &context,
- data_type,
- group_size,
- mode,
- quantization_type,
- )
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!("{:?}", e))
- })?;
+ let qmv = MlpFusedQmvKernel::new(&context, data_type, group_size, mode, quantization_type)
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("{:?}", e)))?;
+
+ let qmm = MlpFusedQmmKernel::new(&context, data_type, group_size, mode, quantization_type)
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("{:?}", e)))?;
Ok(Self {
context,
@@ -267,18 +241,14 @@ impl MlpFusedBlock {
batch_count: batch,
activation: self.activation,
};
- kernel
- .borrow_mut()
- .encode(&self.context, encoder, &args)
- .expect("Failed to encode MLP fused kernel");
+ kernel.borrow_mut().encode(&self.context, encoder, &args).expect("Failed to encode MLP fused kernel");
},
MlpFusedUpKernel::Quantized {
qmv,
qmm,
} => {
let scales = self.scales_buffer.as_ref().unwrap();
- let zp_or_biases =
- self.zero_points_or_biases_buffer.as_ref().unwrap();
+ let zp_or_biases = self.zero_points_or_biases_buffer.as_ref().unwrap();
let is_decode = batch == 1;
if is_decode {
@@ -293,8 +263,7 @@ impl MlpFusedBlock {
hidden_dim: self.hidden_dim as i32,
batch_count: batch,
};
- qmv.encode(encoder, &args)
- .expect("Failed to encode MLP fused QMV");
+ qmv.encode(encoder, &args).expect("Failed to encode MLP fused QMV");
} else {
let args = MlpFusedQmmArguments {
weights: &self.weights_buffer,
@@ -307,8 +276,7 @@ impl MlpFusedBlock {
input_dim: self.input_dim as i32,
hidden_dim: self.hidden_dim as i32,
};
- qmm.encode(encoder, &args)
- .expect("Failed to encode MLP fused QMM");
+ qmm.encode(encoder, &args).expect("Failed to encode MLP fused QMM");
}
},
}
@@ -323,25 +291,22 @@ impl EncodableBlock<Metal> for MlpFusedBlock {
params: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, params);
encoder.end_encoding();
} else {
// Fused up + activation
{
- let arrays =
- state.arrays(&[self.input_array_id, self.hidden_array_id]);
+ let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);
let input = arrays[0].borrow_mut();
let hidden = arrays[1].borrow_mut();
let batch = input.shape()[0] as i32;
let input_buf = input.buffer();
let hidden_buf = hidden.buffer();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_fused_up(&encoder, input_buf, 0, hidden_buf, batch);
encoder.end_encoding();
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index fc7339204..798c31d93 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -33,8 +33,7 @@ pub use attention::Attention;
pub use classifier_layer::ClassifierLayer;
pub use decoder::Decoder;
pub use embedding::{
- EmbeddingError, FullPrecisionEmbeddingLookup,
- FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,
+ EmbeddingError, FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,
QuantizedEmbeddingReadout,
};
pub use layer::LayerExecutables;
@@ -42,9 +41,7 @@ pub use linear::{FullPrecisionLinear, QuantizedLinear};
pub(crate) use mamba_mixer::MambaMixer;
pub use mlp::{MlpBlock, MlpFusedBlock, MlpFusedUpKernel};
pub use moe_block::{MoeBlock, SharedMoeWeights};
-pub use normalization::{
- LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,
-};
+pub use normalization::{LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm};
pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
pub use rope::Rope;
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
index 21053540e..111de03b4 100644
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
@@ -7,26 +7,19 @@ use super::{
SharedMoeWeights,
};
use crate::{
- Activation, DataType, LinearConfig, MixtureOfExpertsConfig,
- RoutingFunctionConfig,
+ Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,
backends::{
common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel},
metal::{
- KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,
- MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext,
- MetalArray, NSRange, ProtocolObject, Retained,
+ KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
+ MTLComputeCommandEncoder, MTLContext, MetalArray, NSRange, ProtocolObject, Retained,
kernel::{
MoeGatherKernels,
- dsl::{
- MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
- },
+ dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
moe::{
- MoeBlockBasesArguments, MoeExpertsTwoPassArguments,
- MoeExpertsTwoPassDecodeKernels,
- MoeExpertsTwoPassPrefillKernel, MoeGatherArguments,
- MoeRouterTopKArguments, MoeRouterTopKKernel,
- MoeScatterArguments, MoeScatterKernels,
- MoeScatterWithMapArguments,
+ MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,
+ MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeRouterTopKArguments, MoeRouterTopKKernel,
+ MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments,
},
},
},
@@ -69,57 +62,36 @@ impl MoeBlock {
hidden_dim: usize,
parameter_tree: &ParameterTree<MTLContext>,
) -> Result<Self, crate::backends::metal::MTLError> {
- let activation_data_type: DataType = moe_config
- .expert_config
- .linear_config
- .activation_precision()
- .into();
+ let activation_data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();
let data_type: KernelDataType = activation_data_type.into();
- let router_data_type: DataType =
- moe_config.router_config.activation_precision().into();
+ let router_data_type: DataType = moe_config.router_config.activation_precision().into();
let router_kernel_data_type: KernelDataType = router_data_type.into();
- let router_renorm = matches!(
- moe_config.routing_function,
- RoutingFunctionConfig::SoftmaxRouting
- );
+ let router_renorm = matches!(moe_config.routing_function, RoutingFunctionConfig::SoftmaxRouting);
- let router_tree = parameter_tree.subtree("router").map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Router subtree error: {:?}",
- e
- ))
- })?;
+ let router_tree = parameter_tree
+ .subtree("router")
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Router subtree error: {:?}", e)))?;
let router = match &moe_config.router_config {
LinearConfig::Quantized(_) => {
- unimplemented!(
- "Quantized router with fused router+topk not yet supported"
- );
+ unimplemented!("Quantized router with fused router+topk not yet supported");
},
LinearConfig::MLXQuantized(_) => {
- unimplemented!(
- "MLX quantized router with fused router+topk not yet supported"
- );
+ unimplemented!("MLX quantized router with fused router+topk not yet supported");
},
LinearConfig::FullPrecision {
..
} => {
- let weights_arr = router_tree.leaf("weights").map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Router weights error: {:?}",
- e
- ))
- })?;
+ let weights_arr = router_tree
+ .leaf("weights")
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Router weights error: {:?}", e)))?;
let weights_buf = weights_arr.buffer();
- let biases_arr = router_tree.leaf("biases").map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Router biases error: {:?}",
- e
- ))
- })?;
+ let biases_arr = router_tree
+ .leaf("biases")
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Router biases error: {:?}", e)))?;
let biases_buf = biases_arr.buffer();
RouterBlock::Metal {
@@ -134,129 +106,54 @@ impl MoeBlock {
},
};
- let router_topk_kernel =
- MoeRouterTopKKernel::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "RouterTopK fused kernel error: {:?}",
- e
- ))
- })?;
- let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context)
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Counts+offsets kernel error: {:?}",
- e
- ))
+ let router_topk_kernel = MoeRouterTopKKernel::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!("RouterTopK fused kernel error: {:?}", e))
})?;
- let scatter_kernels = MoeScatterKernels::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Scatter kernels error: {:?}",
- e
- ))
+ let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context)
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Counts+offsets kernel error: {:?}", e)))?;
+ let scatter_kernels = MoeScatterKernels::new(context)
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Scatter kernels error: {:?}", e)))?;
+ let gather_kernel = MoeGatherKernels::new(context)
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Gather kernel error: {:?}", e)))?;
+ let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeKernels::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!("Experts two-pass decode kernel error: {:?}", e))
})?;
- let gather_kernel = MoeGatherKernels::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Gather kernel error: {:?}",
- e
- ))
+ let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!("Experts two-pass prefill kernel error: {:?}", e))
})?;
- let experts_two_pass_decode_kernel =
- MoeExpertsTwoPassDecodeKernels::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Experts two-pass decode kernel error: {:?}",
- e
- ))
- })?;
- let experts_two_pass_prefill_kernel =
- MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Experts two-pass prefill kernel error: {:?}",
- e
- ))
- })?;
- let finalize_kernel =
- MoeFinalizeMetalKernel::new(context, data_type.into()).map_err(
- |e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Finalize kernel error: {:?}",
- e
- ))
- },
- )?;
+ let finalize_kernel = MoeFinalizeMetalKernel::new(context, data_type.into())
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("Finalize kernel error: {:?}", e)))?;
- let experts_tree = parameter_tree.subtree("experts").map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "experts subtree error: {:?}",
- e
- ))
- })?;
+ let experts_tree = parameter_tree
+ .subtree("experts")
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("experts subtree error: {:?}", e)))?;
let w13_arr = experts_tree
.subtree("up_projection")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "up_projection error: {:?}",
- e
- ))
- })?
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("up_projection error: {:?}", e)))?
.leaf("weights")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "up weights error: {:?}",
- e
- ))
- })?;
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("up weights error: {:?}", e)))?;
let w13_buf = w13_arr.buffer().clone();
let w2_arr = experts_tree
.subtree("down_projection")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "down_projection error: {:?}",
- e
- ))
- })?
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("down_projection error: {:?}", e)))?
.leaf("weights")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "down weights error: {:?}",
- e
- ))
- })?;
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("down weights error: {:?}", e)))?;
let w2_buf = w2_arr.buffer().clone();
let up_biases_arr = experts_tree
.subtree("up_projection")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "up_projection biases error: {:?}",
- e
- ))
- })?
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("up_projection biases error: {:?}", e)))?
.leaf("biases")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "up biases error: {:?}",
- e
- ))
- })?;
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("up biases error: {:?}", e)))?;
let up_biases_buf = up_biases_arr.buffer().clone();
let down_biases_arr = experts_tree
.subtree("down_projection")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "down_projection biases error: {:?}",
- e
- ))
- })?
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("down_projection biases error: {:?}", e)))?
.leaf("biases")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "down biases error: {:?}",
- e
- ))
- })?;
+ .map_err(|e| crate::backends::metal::MTLError::Generic(format!("down biases error: {:?}", e)))?;
let down_biases_buf = down_biases_arr.buffer().clone();
let shared_weights = SharedMoeWeights {
@@ -368,9 +265,7 @@ impl EncodableBlock<Metal> for MoeBlock {
KernelDataType::Float32 => 4,
};
- let blit_encoder = root
- .new_blit_command_encoder()
- .expect("Failed to create blit command encoder");
+ let blit_encoder = root.new_blit_command_encoder().expect("Failed to create blit command encoder");
if suffix_length > 0 && k > 0 {
let entries = suffix_length * k;
@@ -394,19 +289,13 @@ impl EncodableBlock<Metal> for MoeBlock {
}
// Clear hidden buffer
- let hidden_bytes =
- (suffix_length * k * self.hidden_dim * dtype_size) as u64;
+ let hidden_bytes = (suffix_length * k * self.hidden_dim * dtype_size) as u64;
if hidden_bytes > 0 {
- blit_encoder.fill_buffer_range_value(
- &hidden_buf,
- NSRange::new(0, hidden_bytes.try_into().unwrap()),
- 0,
- );
+ blit_encoder.fill_buffer_range_value(&hidden_buf, NSRange::new(0, hidden_bytes.try_into().unwrap()), 0);
}
// Clear y_partial buffer
- let y_partial_bytes =
- (suffix_length * k * self.model_dim * dtype_size) as u64;
+ let y_partial_bytes = (suffix_length * k * self.model_dim * dtype_size) as u64;
if y_partial_bytes > 0 {
blit_encoder.fill_buffer_range_value(
&y_partial_buf,
@@ -416,14 +305,9 @@ impl EncodableBlock<Metal> for MoeBlock {
}
// Clear x_perm buffer
- let x_perm_bytes =
- (suffix_length * k * self.model_dim * dtype_size) as u64;
+ let x_perm_bytes = (suffix_length * k * self.model_dim * dtype_size) as u64;
if x_perm_bytes > 0 {
- blit_encoder.fill_buffer_range_value(
- &x_perm_buf,
- NSRange::new(0, x_perm_bytes.try_into().unwrap()),
- 0,
- );
+ blit_encoder.fill_buffer_range_value(&x_perm_buf, NSRange::new(0, x_perm_bytes.try_into().unwrap()), 0);
}
}
@@ -458,9 +342,8 @@ impl EncodableBlock<Metal> for MoeBlock {
}
{
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.counts_offsets_kernel.encode(
&topk_ids_buf,
&offsets_buf,
@@ -529,15 +412,11 @@ impl EncodableBlock<Metal> for MoeBlock {
},
);
- let gating_code = Self::gating_code_from_activation(
- &self.moe_config.expert_config.activation,
- );
+ let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation);
// Compute clipping values and alpha for expert kernels
- let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]
- .unwrap_or(f32::NEG_INFINITY);
- let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]
- .unwrap_or(f32::INFINITY);
+ let gate_clip_min = self.moe_config.expert_config.gate_clipping[0].unwrap_or(f32::NEG_INFINITY);
+ let gate_clip_max = self.moe_config.expert_config.gate_clipping[1].unwrap_or(f32::INFINITY);
let up_clip_min = self.moe_config.expert_config.up_clipping[0];
let up_clip_max = self.moe_config.expert_config.up_clipping[1];
let silu_alpha = self.moe_config.expert_config.activation.alpha();
@@ -617,9 +496,8 @@ impl EncodableBlock<Metal> for MoeBlock {
}
{
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.finalize_kernel.encode(
&tok2row_buf,
&topk_probs_buf,
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
index c06b3df89..a97905ad3 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
@@ -6,10 +6,8 @@ use crate::{
backends::{
common::kernel::LayerNormKernel,
metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
- MTLResourceOptions, ProtocolObject, Retained,
- kernel::dsl::LayerNormMetalKernel,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt,
+ MTLError, MTLResourceOptions, ProtocolObject, Retained, kernel::dsl::LayerNormMetalKernel,
},
},
config::{NormalizationConfig, UpcastMode},
@@ -37,25 +35,19 @@ impl LayerNorm {
) -> Result<Self, MTLError> {
// Load scales from parameter tree
let scales_param = parameter_tree.leaf("scales").map_err(|e| {
- MTLError::Library(
- crate::backends::metal::error::LibraryError::Custom(format!(
- "Failed to load scales: {:?}",
- e
- )),
- )
+ MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(
+ "Failed to load scales: {:?}",
+ e
+ )))
})?;
let scales_data = scales_param.as_bytes();
let scales_buffer = context
.device
- .new_buffer_with_data(
- scales_data,
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create scales buffer");
- let accumulation_data_type: DataType =
- config.accumulation_precision.into();
+ let accumulation_data_type: DataType = config.accumulation_precision.into();
let scale_data_type: DataType = config.scale_precision.into();
let kernel = LayerNormMetalKernel::new(
@@ -82,9 +74,8 @@ impl EncodableBlock<Metal> for LayerNorm {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &compute_encoder, parameters);
compute_encoder.end_encoding();
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
index e7b65f5ea..10c27adbe 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
@@ -13,8 +13,7 @@ use crate::backends::metal::MTLError;
use crate::{
DataType,
backends::metal::{
- MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, ProtocolObject,
- Retained, kernel::RMSNormError,
+ MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained, kernel::RMSNormError,
},
config::NormalizationConfig,
encodable_block::EncodingParameters,
@@ -80,23 +79,15 @@ impl EncodableBlock<Metal> for Normalization {
parameters: &EncodingParameters<Metal>,
) {
match self {
- Normalization::LayerNorm(layer_norm) => {
- layer_norm.encode(state, command_buffer, parameters)
- },
- Normalization::RMSNorm(rms_norm) => {
- rms_norm.encode(state, command_buffer, parameters)
- },
+ Normalization::LayerNorm(layer_norm) => layer_norm.encode(state, command_buffer, parameters),
+ Normalization::RMSNorm(rms_norm) => rms_norm.encode(state, command_buffer, parameters),
}
}
fn supports_shared_encoder(&self) -> bool {
match self {
- Normalization::LayerNorm(layer_norm) => {
- layer_norm.supports_shared_encoder()
- },
- Normalization::RMSNorm(rms_norm) => {
- rms_norm.supports_shared_encoder()
- },
+ Normalization::LayerNorm(layer_norm) => layer_norm.supports_shared_encoder(),
+ Normalization::RMSNorm(rms_norm) => rms_norm.supports_shared_encoder(),
}
}
@@ -107,11 +98,8 @@ impl EncodableBlock<Metal> for Normalization {
parameters: &EncodingParameters<Metal>,
) {
match self {
- Normalization::LayerNorm(layer_norm) => layer_norm
- .encode_with_shared_encoder(state, encoder, parameters),
- Normalization::RMSNorm(rms_norm) => {
- rms_norm.encode_with_shared_encoder(state, encoder, parameters)
- },
+ Normalization::LayerNorm(layer_norm) => layer_norm.encode_with_shared_encoder(state, encoder, parameters),
+ Normalization::RMSNorm(rms_norm) => rms_norm.encode_with_shared_encoder(state, encoder, parameters),
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
index 2eedae5de..b429d68d0 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
@@ -4,13 +4,9 @@ use super::super::{EncodableBlock, Metal};
use crate::{
DataType,
backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
MTLResourceOptions, ProtocolObject, Retained,
- kernel::rms_norm::{
- QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel,
- RMSNormKernelType,
- },
+ kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel, RMSNormKernelType},
},
config::{NormalizationConfig, UpcastMode},
encodable_block::EncodingParameters,
@@ -50,34 +46,22 @@ impl QKNorm {
// Setup query normalization if configured
if let Some(ref q_config) = query_config {
- let scales_param =
- parameter_tree.leaf("query_norm.scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(
- crate::backends::metal::error::LibraryError::Custom(
- format!("Failed to load query scales: {:?}", e),
- ),
- ))
- })?;
+ let scales_param = parameter_tree.leaf("query_norm.scales").map_err(|e| {
+ RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(
+ format!("Failed to load query scales: {:?}", e),
+ )))
+ })?;
let scales_data = scales_param.as_bytes();
- let scales_buffer = context.device.new_buffer_with_data(
- scales_data,
- MTLResourceOptions::STORAGE_MODE_SHARED,
- );
+ let scales_buffer =
+ context.device.new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED);
- let accumulation_data_type: DataType =
- q_config.accumulation_precision.into();
+ let accumulation_data_type: DataType = q_config.accumulation_precision.into();
let scale_data_type: DataType = q_config.scale_precision.into();
- let (input_type, scales_type, output_type) = match q_config
- .upcast_mode
- {
- UpcastMode::OnlyNormalization => {
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
- UpcastMode::FullLayer => {
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
+ let (input_type, scales_type, output_type) = match q_config.upcast_mode {
+ UpcastMode::OnlyNormalization => (intermediate_data_type, scale_data_type, scale_data_type),
+ UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),
};
let kernel = RMSNormKernel::new_with_mode(
@@ -96,34 +80,22 @@ impl QKNorm {
// Setup key normalization if configured
if let Some(ref k_config) = key_config {
- let scales_param =
- parameter_tree.leaf("key_norm.scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(
- crate::backends::metal::error::LibraryError::Custom(
- format!("Failed to load key scales: {:?}", e),
- ),
- ))
- })?;
+ let scales_param = parameter_tree.leaf("key_norm.scales").map_err(|e| {
+ RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(
+ format!("Failed to load key scales: {:?}", e),
+ )))
+ })?;
let scales_data = scales_param.as_bytes();
- let scales_buffer = context.device.new_buffer_with_data(
- scales_data,
- MTLResourceOptions::STORAGE_MODE_SHARED,
- );
+ let scales_buffer =
+ context.device.new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED);
- let accumulation_data_type: DataType =
- k_config.accumulation_precision.into();
+ let accumulation_data_type: DataType = k_config.accumulation_precision.into();
let scale_data_type: DataType = k_config.scale_precision.into();
- let (input_type, scales_type, output_type) = match k_config
- .upcast_mode
- {
- UpcastMode::OnlyNormalization => {
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
- UpcastMode::FullLayer => {
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
+ let (input_type, scales_type, output_type) = match k_config.upcast_mode {
+ UpcastMode::OnlyNormalization => (intermediate_data_type, scale_data_type, scale_data_type),
+ UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),
};
let kernel = RMSNormKernel::new_with_mode(
@@ -162,9 +134,8 @@ impl EncodableBlock<Metal> for QKNorm {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &compute_encoder, parameters);
compute_encoder.end_encoding();
@@ -197,11 +168,7 @@ impl EncodableBlock<Metal> for QKNorm {
let head_dim = self.head_dim as i32;
// Process query normalization if configured
- if let (
- Some(query_kernel),
- Some(query_scales_buffer),
- Some(query_config),
- ) =
+ if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =
(&self.query_kernel, &self.query_scales_buffer, &self.query_config)
{
if let Err(e) = query_kernel.encode_qk_norm(
@@ -220,10 +187,7 @@ impl EncodableBlock<Metal> for QKNorm {
target: QKNormTarget::QueryHeads,
},
) {
- eprintln!(
- "Failed to encode query normalization kernel: {:?}",
- e
- );
+ eprintln!("Failed to encode query normalization kernel: {:?}", e);
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
index 4644e1f4a..fb54be5be 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
@@ -4,12 +4,9 @@ use super::super::{EncodableBlock, Metal};
use crate::{
DataType,
backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
MTLResourceOptions, ProtocolObject, Retained,
- kernel::rms_norm::{
- RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType,
- },
+ kernel::rms_norm::{RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType},
},
config::{NormalizationConfig, UpcastMode},
encodable_block::EncodingParameters,
@@ -37,26 +34,20 @@ impl RMSNorm {
) -> Result<Self, RMSNormError> {
// Load scales from parameter tree
let scales_param = parameter_tree.leaf("scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(
- crate::backends::metal::error::LibraryError::Custom(format!(
- "Failed to load scales: {:?}",
- e
- )),
- ))
+ RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(
+ "Failed to load scales: {:?}",
+ e
+ ))))
})?;
// TODO: Don't create buffers dynamically, we need to use forward pass storage for thing like this
let scales_data = scales_param.as_bytes();
let scales_buffer = context
.device
- .new_buffer_with_data(
- scales_data,
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create scales buffer");
- let accumulation_data_type: DataType =
- config.accumulation_precision.into();
+ let accumulation_data_type: DataType = config.accumulation_precision.into();
let scale_data_type: DataType = config.scale_precision.into();
let (input_type, scales_type, output_type) = match config.upcast_mode {
@@ -105,9 +96,8 @@ impl EncodableBlock<Metal> for RMSNorm {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &compute_encoder, parameters);
compute_encoder.end_encoding();
@@ -149,8 +139,7 @@ impl EncodableBlock<Metal> for RMSNorm {
} else {
(0, state.active_suffix_length())
};
- let batch_len =
- batch_len.min(suffix_length.saturating_sub(batch_start));
+ let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));
if batch_len == 0 {
return;
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/pooling.rs b/crates/uzu/src/backends/metal/encodable_block/pooling.rs
index 0fe17f7b2..62bd66c15 100644
--- a/crates/uzu/src/backends/metal/encodable_block/pooling.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/pooling.rs
@@ -7,9 +7,8 @@ use crate::{
backends::{
common::kernel::{PoolingClsKernel, PoolingMeanKernel},
metal::{
- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
- Retained,
+ KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext,
+ MTLError, ProtocolObject, Retained,
kernel::dsl::{PoolingClsMetalKernel, PoolingMeanMetalKernel},
},
},
@@ -34,12 +33,8 @@ impl PoolingKernel {
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
) {
match self {
- Self::Cls(k) => k.encode(
- input, output, seq_len, hidden_dim, batch_size, encoder,
- ),
- Self::Mean(k) => k.encode(
- input, output, seq_len, hidden_dim, batch_size, encoder,
- ),
+ Self::Cls(k) => k.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
+ Self::Mean(k) => k.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
}
}
}
@@ -57,13 +52,8 @@ impl Pooling {
model_dim: usize,
) -> Result<Self, MTLError> {
let pooling_kernel = match pooling_type {
- PoolingType::Cls => PoolingKernel::Cls(PoolingClsMetalKernel::new(
- context,
- data_type.into(),
- )?),
- PoolingType::Mean => PoolingKernel::Mean(
- PoolingMeanMetalKernel::new(context, data_type.into())?,
- ),
+ PoolingType::Cls => PoolingKernel::Cls(PoolingClsMetalKernel::new(context, data_type.into())?),
+ PoolingType::Mean => PoolingKernel::Mean(PoolingMeanMetalKernel::new(context, data_type.into())?),
};
Ok(Self {
pooling_kernel,
@@ -79,17 +69,14 @@ impl EncodableBlock<Metal> for Pooling {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
#[cfg(feature = "tracing")]
{
let batch_size = 1;
- let data_type =
- { state.arrays(&[ArrayId::Main])[0].borrow().data_type() };
+ let data_type = { state.arrays(&[ArrayId::Main])[0].borrow().data_type() };
let arrays = state.arrays(&[ArrayId::ClassifierPooling]);
let pooling_array = arrays[0].borrow();
@@ -100,9 +87,7 @@ impl EncodableBlock<Metal> for Pooling {
let trace_arr = traces_ref.output_pooling().borrow();
let dst_buf = trace_arr.buffer();
- let blit = command_buffer
- .new_blit_command_encoder()
- .expect("Failed to create blit command encoder");
+ let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");
blit.copy_buffer_to_buffer(
output_buffer,
0,
diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
index efddba52a..a12e5d974 100644
--- a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
@@ -3,10 +3,7 @@
use super::{EncodableBlock, Metal};
#[cfg(feature = "tracing")]
use crate::backends::metal::MTLBlitCommandEncoder;
-use crate::backends::metal::{
- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
- ProtocolObject, Retained,
-};
+use crate::backends::metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained};
use crate::encodable_block::EncodingParameters;
#[cfg(feature = "tracing")]
use crate::forward_pass::state::ArrayId;
@@ -47,9 +44,8 @@ impl EncodableBlock<Metal> for ClassifierPredictionHead {
parameters: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
} else {
@@ -62,8 +58,7 @@ impl EncodableBlock<Metal> for ClassifierPredictionHead {
#[cfg(feature = "tracing")]
{
let traces_rc = state.traces().clone();
- let logits_arrays =
- state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
+ let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
let logits_array_ref = logits_arrays[0].borrow();
let linear_output_buffer = logits_array_ref.buffer();
let data_type = logits_array_ref.data_type();
@@ -73,19 +68,10 @@ impl EncodableBlock<Metal> for ClassifierPredictionHead {
let trace_logits = traces_ref.logits.borrow();
let dst_trace_buf = trace_logits.buffer();
- let copy_size_bytes =
- batch_size * self.num_labels * data_type.size_in_bytes();
+ let copy_size_bytes = batch_size * self.num_labels * data_type.size_in_bytes();
- let blit = command_buffer
- .new_blit_command_encoder()
- .expect("Failed to create blit command encoder");
- blit.copy_buffer_to_buffer(
- linear_output_buffer,
- 0,
- dst_trace_buf,
- 0,
- copy_size_bytes,
- );
+ let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");
+ blit.copy_buffer_to_buffer(linear_output_buffer, 0, dst_trace_buf, 0, copy_size_bytes);
blit.end_encoding();
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rs
index d4671fb7a..7ff44133d 100644
--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs
@@ -1,9 +1,6 @@
//! Rope (Rotary Position Embedding) encodable.
-use crate::backends::metal::{
- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
- ProtocolObject, Retained,
-};
+use crate::backends::metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained};
use super::{EncodableBlock, Metal};
use crate::backends::metal::{
@@ -39,9 +36,8 @@ impl EncodableBlock<Metal> for Rope {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &compute_encoder, parameters);
compute_encoder.end_encoding();
@@ -75,8 +71,7 @@ impl EncodableBlock<Metal> for Rope {
let keys_array = keys_binding[0].borrow();
let num_groups = keys_array.shape()[0];
- let cos_binding =
- state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
+ let cos_binding = state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
let cos_array = cos_binding[0].borrow();
let cos_shape = cos_array.shape();
let rope_max_seq_len = cos_shape[0];
@@ -96,12 +91,10 @@ impl EncodableBlock<Metal> for Rope {
let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
let rotated_keys = rotated_keys_binding[0].borrow_mut();
- let cos_buffer_binding =
- state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
+ let cos_buffer_binding = state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
let rope_cosines = cos_buffer_binding[0].borrow_mut();
- let sin_buffer_binding =
- state.arrays(&[ArrayId::RopeSines(self.rope_type)]);
+ let sin_buffer_binding = state.arrays(&[ArrayId::RopeSines(self.rope_type)]);
let rope_sines = sin_buffer_binding[0].borrow_mut();
let token_positions_offset = token_positions.offset();
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index 10e253d05..46feeaef7 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -6,13 +6,11 @@ use crate::{
backends::{
common::Context,
metal::{
- KernelDataType, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,
- Retained,
+ KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray,
+ ProtocolObject, Retained,
compilation_parameters::CompilationConfig,
kernel::short_conv::{
- ShortConvDecodeArguments, ShortConvKernel,
- ShortConvPackArguments, ShortConvPrefillArguments,
+ ShortConvDecodeArguments, ShortConvKernel, ShortConvPackArguments, ShortConvPrefillArguments,
ShortConvTrieArguments,
},
},
@@ -58,19 +56,13 @@ impl ShortConvMixer {
decoder_layer_loader: &ParameterTree<MTLContext>,
) -> Self {
if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {
- panic!(
- "Layer {} marked as non-ShortConv but ShortConv config provided",
- layer_index
- );
+ panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);
}
let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);
let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);
- let data_type: DataType = short_conv_config
- .in_projection_config
- .activation_precision()
- .into();
+ let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();
let kernel_data_type: KernelDataType = data_type.into();
let in_projection = transformer_layer::linear_block(
@@ -105,8 +97,7 @@ impl ShortConvMixer {
};
let short_conv_kernel =
- ShortConvKernel::new(mtl_context, kernel_data_type)
- .expect("Failed to create short conv kernel");
+ ShortConvKernel::new(mtl_context, kernel_data_type).expect("Failed to create short conv kernel");
Self {
layer_index,
@@ -133,9 +124,7 @@ impl ShortConvMixer {
self.in_projection.encode(state, command_buffer, parameters);
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.run_conv(state, &encoder, active_suffix_length);
encoder.end_encoding();
@@ -158,13 +147,11 @@ impl ShortConvMixer {
return;
}
- self.in_projection
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.in_projection.encode_with_shared_encoder(state, encoder, parameters);
self.run_conv(state, encoder, active_suffix_length);
- self.out_projection
- .encode_with_shared_encoder(state, encoder, parameters);
+ self.out_projection.encode_with_shared_encoder(state, encoder, parameters);
}
fn clear_suffix_state_valid_range(
@@ -175,9 +162,7 @@ impl ShortConvMixer {
return;
};
let cache = cache_layers.borrow();
- let layer = cache.data[self.layer_index]
- .as_short_conv()
- .expect("Expected ShortConv layer");
+ let layer = cache.data[self.layer_index].as_short_conv().expect("Expected ShortConv layer");
layer.clear_suffix_state_valid_range();
}
@@ -191,9 +176,7 @@ impl ShortConvMixer {
return;
};
let cache = cache_layers.borrow();
- let layer = cache.data[self.layer_index]
- .as_short_conv()
- .expect("Expected ShortConv layer");
+ let layer = cache.data[self.layer_index].as_short_conv().expect("Expected ShortConv layer");
layer.set_suffix_state_valid_range(start, len);
}
@@ -242,11 +225,8 @@ impl ShortConvMixer {
compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
- let arrays = state.arrays(&[
- ArrayId::SsmInProj,
- ArrayId::ShortConvState(self.layer_index),
- ArrayId::AttentionOutput,
- ]);
+ let arrays =
+ state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);
let in_proj = arrays[0].borrow_mut();
let conv_state = arrays[1].borrow_mut();
let out = arrays[2].borrow_mut();
@@ -266,15 +246,11 @@ impl ShortConvMixer {
let state_stride = kernel_size.saturating_sub(1);
// Allocate temporary padded buffer
- let data_type: DataType =
- self.config.in_projection_config.activation_precision().into();
+ let data_type: DataType = self.config.in_projection_config.activation_precision().into();
let element_size = data_type.size_in_bytes();
let padded_rows = state_stride + suffix_length;
let padded_size = padded_rows * self.model_dim * element_size;
- let padded_buf = state
- .mtl_context()
- .create_buffer(padded_size)
- .expect("Failed to create padded buffer");
+ let padded_buf = state.mtl_context().create_buffer(padded_size).expect("Failed to create padded buffer");
self.short_conv_kernel
.encode_pack(
compute,
@@ -341,23 +317,18 @@ impl ShortConvMixer {
let suffix_state_buf = suffix_state.buffer().clone();
let out_buf = out.buffer().clone();
- let data_type: DataType =
- self.config.in_projection_config.activation_precision().into();
+ let data_type: DataType = self.config.in_projection_config.activation_precision().into();
let elem_bytes = data_type.size_in_bytes();
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
let in_proj_stride = self.model_dim * 3;
- let in_proj_offset =
- in_proj.offset() + sampling_start * in_proj_stride * elem_bytes;
- let out_offset =
- out.offset() + sampling_start * self.model_dim * elem_bytes;
- let suffix_state_offset = suffix_state.offset()
- + sampling_start * self.model_dim * state_stride * elem_bytes;
+ let in_proj_offset = in_proj.offset() + sampling_start * in_proj_stride * elem_bytes;
+ let out_offset = out.offset() + sampling_start * self.model_dim * elem_bytes;
+ let suffix_state_offset = suffix_state.offset() + sampling_start * self.model_dim * state_stride * elem_bytes;
let base_state_offset = conv_state.offset();
- let parents_offset =
- parents.offset() + sampling_start * std::mem::size_of::<i32>();
+ let parents_offset = parents.offset() + sampling_start * std::mem::size_of::<i32>();
let conv_weight = self.conv_weight.clone();
let weight_buf = conv_weight.buffer().clone();
@@ -398,11 +369,8 @@ impl ShortConvMixer {
compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
- let arrays = state.arrays(&[
- ArrayId::SsmInProj,
- ArrayId::ShortConvState(self.layer_index),
- ArrayId::AttentionOutput,
- ]);
+ let arrays =
+ state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);
let in_proj = arrays[0].borrow_mut();
let conv_state = arrays[1].borrow_mut();
let out = arrays[2].borrow_mut();
@@ -450,9 +418,8 @@ impl EncodableBlock<Metal> for ShortConvMixer {
parameters: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_pipeline_with_encoder(state, &encoder, parameters);
encoder.end_encoding();
@@ -466,8 +433,7 @@ impl EncodableBlock<Metal> for ShortConvMixer {
}
fn supports_shared_encoder(&self) -> bool {
- self.in_projection.supports_shared_encoder()
- && self.out_projection.supports_shared_encoder()
+ self.in_projection.supports_shared_encoder() && self.out_projection.supports_shared_encoder()
}
fn encode_with_shared_encoder(
diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
index d6e29b2f2..82aa3c7ef 100644
--- a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
@@ -2,16 +2,11 @@
use crate::backends::{
common::kernel::TensorAddSwapKernel,
- metal::{
- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
- ProtocolObject, Retained,
- },
+ metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained},
};
use super::{EncodableBlock, Metal};
-use crate::backends::metal::{
- MTLContext, MTLError, kernel::dsl::TensorAddSwapMetalKernel,
-};
+use crate::backends::metal::{MTLContext, MTLError, kernel::dsl::TensorAddSwapMetalKernel};
use crate::encodable_block::EncodingParameters;
use crate::forward_pass::state::{ArrayId, ForwardPassState};
@@ -41,9 +36,7 @@ impl EncodableBlock<Metal> for TensorAddSwap {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
@@ -73,11 +66,6 @@ impl EncodableBlock<Metal> for TensorAddSwap {
let skip_mtl_buffer = skip_array.buffer();
let main_mtl_buffer = main_array.buffer();
- self.kernel.encode(
- skip_mtl_buffer,
- main_mtl_buffer,
- length as u32,
- encoder,
- );
+ self.kernel.encode(skip_mtl_buffer, main_mtl_buffer, length as u32, encoder);
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
index d2663b814..d8c0627ae 100644
--- a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
@@ -2,16 +2,11 @@
use crate::backends::{
common::kernel::TensorCopyKernel,
- metal::{
- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
- ProtocolObject, Retained,
- },
+ metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained},
};
use super::{EncodableBlock, Metal};
-use crate::backends::metal::{
- MTLContext, MTLError, kernel::dsl::TensorCopyMetalKernel,
-};
+use crate::backends::metal::{MTLContext, MTLError, kernel::dsl::TensorCopyMetalKernel};
use crate::encodable_block::EncodingParameters;
use crate::forward_pass::state::{ArrayId, ForwardPassState};
@@ -41,9 +36,7 @@ impl EncodableBlock<Metal> for TensorCopy {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
parameters: &EncodingParameters<Metal>,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.encode_with_shared_encoder(state, &encoder, parameters);
encoder.end_encoding();
@@ -73,11 +66,6 @@ impl EncodableBlock<Metal> for TensorCopy {
let source_mtl_buffer = source_array.buffer();
let destination_mtl_buffer = destination_array.buffer();
- self.kernel.encode(
- source_mtl_buffer,
- destination_mtl_buffer,
- length as u32,
- encoder,
- );
+ self.kernel.encode(source_mtl_buffer, destination_mtl_buffer, length as u32, encoder);
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 82d938c94..99d368e8f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -2,10 +2,8 @@ use std::rc::Rc;
use super::Metal;
use super::{
- EncodableBlock, FullPrecisionEmbeddingLookup,
- FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock,
- MlpFusedBlock, MoeBlock, QuantizedEmbeddingLookup,
- QuantizedEmbeddingReadout, QuantizedLinear,
+ EncodableBlock, FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock,
+ MlpFusedBlock, MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
@@ -30,8 +28,7 @@ pub fn linear_block<const N: usize>(
) -> Result<Box<dyn EncodableBlock<Metal>>, MTLError> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
match config {
- LinearConfig::Quantized(quantization_config)
- | LinearConfig::MLXQuantized(quantization_config) => {
+ LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {
let block = QuantizedLinear::new(
context,
quantization_config,
@@ -59,9 +56,7 @@ pub fn linear_block<const N: usize>(
},
LinearConfig::QLoRA {
..
- } => Err(MTLError::Generic(
- "QLoRA linear layer not supported".to_string(),
- )),
+ } => Err(MTLError::Generic("QLoRA linear layer not supported".to_string())),
}
}
@@ -74,8 +69,7 @@ pub fn mlp_block(
parameter_tree: &ParameterTree<MTLContext>,
) -> Result<Box<dyn EncodableBlock<Metal>>, MTLError> {
if let crate::config::MLPConfig::Dense(dense_config) = config {
- let data_type: DataType =
- dense_config.linear_config.activation_precision().into();
+ let data_type: DataType = dense_config.linear_config.activation_precision().into();
// Up projection (outputs 2*hidden_dimension for gate and up)
let up_projection = linear_block(
@@ -84,20 +78,14 @@ pub fn mlp_block(
model_dimension,
[2 * hidden_dimension],
context,
- &parameter_tree
- .subtree("up_projection")
- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
+ &parameter_tree.subtree("up_projection").map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
ArrayId::Main,
ArrayId::MlpFusedUp,
)?;
// Gate activation + multiply
- let gate_activation = MlpGateActMulEncodable::new(
- context,
- data_type,
- dense_config.activation.clone(),
- hidden_dimension,
- )?;
+ let gate_activation =
+ MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)?;
// Down projection
let down_projection = linear_block(
@@ -106,31 +94,17 @@ pub fn mlp_block(
hidden_dimension,
[model_dimension],
context,
- &parameter_tree
- .subtree("down_projection")
- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
+ &parameter_tree.subtree("down_projection").map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
ArrayId::MlpHidden,
ArrayId::Main,
)?;
- return Ok(Box::new(MlpBlock::new(
- up_projection,
- gate_activation,
- down_projection,
- )));
+ return Ok(Box::new(MlpBlock::new(up_projection, gate_activation, down_projection)));
}
- if let crate::config::MLPConfig::MixtureOfExperts(
- mixture_of_experts_config,
- ) = config
- {
- let mixture_of_experts_block = MoeBlock::new(
- context,
- mixture_of_experts_config,
- model_dimension,
- hidden_dimension,
- parameter_tree,
- )?;
+ if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
+ let mixture_of_experts_block =
+ MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;
return Ok(Box::new(mixture_of_experts_block));
}
@@ -148,53 +122,35 @@ pub fn mlp_fused_block(
) -> Result<Box<dyn EncodableBlock<Metal>>, MTLError> {
if let crate::config::MLPConfig::Dense(dense_config) = config {
match &dense_config.linear_config {
- LinearConfig::Quantized(quantization_config)
- | LinearConfig::MLXQuantized(quantization_config) => {
- let data_type: DataType =
- dense_config.linear_config.activation_precision().into();
+ LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {
+ let data_type: DataType = dense_config.linear_config.activation_precision().into();
// Load up_projection weights directly for fused kernel
- let up_projection_tree =
- parameter_tree.subtree("up_projection").map_err(
- |error| MTLError::Generic(format!("{:?}", error)),
- )?;
-
- let up_projection_weights =
- up_projection_tree.leaf("weights").map_err(|error| {
- MTLError::Generic(format!(
- "Failed to load up weights: {:?}",
- error
- ))
- })?;
- let up_projection_weights_buffer =
- up_projection_weights.buffer().into();
-
- let up_projection_scales =
- up_projection_tree.leaf("scales").map_err(|error| {
- MTLError::Generic(format!(
- "Failed to load up scales: {:?}",
- error
- ))
- })?;
- let up_projection_scales_buffer =
- up_projection_scales.buffer().into();
+ let up_projection_tree = parameter_tree
+ .subtree("up_projection")
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
+
+ let up_projection_weights = up_projection_tree
+ .leaf("weights")
+ .map_err(|error| MTLError::Generic(format!("Failed to load up weights: {:?}", error)))?;
+ let up_projection_weights_buffer = up_projection_weights.buffer().into();
+
+ let up_projection_scales = up_projection_tree
+ .leaf("scales")
+ .map_err(|error| MTLError::Generic(format!("Failed to load up scales: {:?}", error)))?;
+ let up_projection_scales_buffer = up_projection_scales.buffer().into();
// Load zero_points or biases depending on quantization type
- let (
- up_projection_zero_points_or_biases_buffer,
- quantization_type,
- ) = if let Ok(biases) = up_projection_tree.leaf("biases") {
- (biases.buffer().into(), QuantizationType::Mlx)
- } else if let Ok(zero_points) =
- up_projection_tree.leaf("zero_points")
- {
- (zero_points.buffer().into(), QuantizationType::ZeroPoint)
- } else {
- return Err(MTLError::Generic(
- "Missing zero_points or biases for quantized up_projection"
- .to_string(),
+ let (up_projection_zero_points_or_biases_buffer, quantization_type) =
+ if let Ok(biases) = up_projection_tree.leaf("biases") {
+ (biases.buffer().into(), QuantizationType::Mlx)
+ } else if let Ok(zero_points) = up_projection_tree.leaf("zero_points") {
+ (zero_points.buffer().into(), QuantizationType::ZeroPoint)
+ } else {
+ return Err(MTLError::Generic(
+ "Missing zero_points or biases for quantized up_projection".to_string(),
));
- };
+ };
// Create down projection as separate linear
let down_projection = QuantizedLinear::new(
@@ -202,9 +158,9 @@ pub fn mlp_fused_block(
quantization_config,
hidden_dimension,
model_dimension,
- &parameter_tree.subtree("down_projection").map_err(
- |error| MTLError::Generic(format!("{:?}", error)),
- )?,
+ &parameter_tree
+ .subtree("down_projection")
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
ArrayId::MlpHidden,
ArrayId::Main,
)?;
@@ -233,20 +189,14 @@ pub fn mlp_fused_block(
let data_type: DataType = (*precision).into();
// Load up_projection weights directly for fused kernel
- let up_projection_tree =
- parameter_tree.subtree("up_projection").map_err(
- |error| MTLError::Generic(format!("{:?}", error)),
- )?;
-
- let up_projection_weights =
- up_projection_tree.leaf("weights").map_err(|error| {
- MTLError::Generic(format!(
- "Failed to load up weights: {:?}",
- error
- ))
- })?;
- let up_projection_weights_buffer =
- up_projection_weights.buffer().into();
+ let up_projection_tree = parameter_tree
+ .subtree("up_projection")
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
+
+ let up_projection_weights = up_projection_tree
+ .leaf("weights")
+ .map_err(|error| MTLError::Generic(format!("Failed to load up weights: {:?}", error)))?;
+ let up_projection_weights_buffer = up_projection_weights.buffer().into();
// Create down projection as separate linear
let down_projection = FullPrecisionLinear::new(
@@ -254,9 +204,9 @@ pub fn mlp_fused_block(
data_type,
hidden_dimension,
model_dimension,
- &parameter_tree.subtree("down_projection").map_err(
- |error| MTLError::Generic(format!("{:?}", error)),
- )?,
+ &parameter_tree
+ .subtree("down_projection")
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
ArrayId::MlpHidden,
ArrayId::Main,
)?;
@@ -277,24 +227,14 @@ pub fn mlp_fused_block(
LinearConfig::QLoRA {
..
} => {
- return Err(MTLError::Generic(
- "QLoRA MLP not supported".to_string(),
- ));
+ return Err(MTLError::Generic("QLoRA MLP not supported".to_string()));
},
}
}
- if let crate::config::MLPConfig::MixtureOfExperts(
- mixture_of_experts_config,
- ) = config
- {
- let mixture_of_experts_block = MoeBlock::new(
- &context,
- mixture_of_experts_config,
- model_dimension,
- hidden_dimension,
- parameter_tree,
- )?;
+ if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
+ let mixture_of_experts_block =
+ MoeBlock::new(&context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;
return Ok(Box::new(mixture_of_experts_block));
}
@@ -311,9 +251,7 @@ pub fn embed_block(
common,
precision,
} => {
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = FullPrecisionEmbeddingLookup::new(
context,
@@ -331,9 +269,7 @@ pub fn embed_block(
common,
precision,
} => {
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = FullPrecisionEmbeddingLookup::new(
context,
@@ -352,9 +288,7 @@ pub fn embed_block(
activation_precision,
..
} => {
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = FullPrecisionEmbeddingLookup::new(
context,
@@ -375,12 +309,9 @@ pub fn embed_block(
..
} => {
let data_type: DataType = (*activation_precision).into();
- let input_scale =
- config.embedding_config.common().input_scale.unwrap_or(1.0);
+ let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = QuantizedEmbeddingLookup::new_untied_input(
context,
@@ -403,14 +334,11 @@ pub fn embed_block(
} => {
let data_type: DataType = (*activation_precision).into();
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
// For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim
let group_size = config.model_dim;
- let input_scale =
- config.embedding_config.common().input_scale.unwrap_or(1.0);
+ let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);
let block = QuantizedEmbeddingLookup::new_tied(
context,
@@ -431,14 +359,10 @@ pub fn embed_block(
embedding_quantization_mode,
..
} => {
- let data_type: DataType =
- config.output_norm_config.scale_precision.into();
- let input_scale =
- config.embedding_config.common().input_scale.unwrap_or(1.0);
+ let data_type: DataType = config.output_norm_config.scale_precision.into();
+ let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = QuantizedEmbeddingLookup::new_tied(
context,
@@ -467,9 +391,7 @@ pub fn readout_block(
precision,
..
} => {
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = FullPrecisionEmbeddingReadout::new(
context,
@@ -486,9 +408,7 @@ pub fn readout_block(
precision,
..
} => {
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = FullPrecisionEmbeddingReadout::new(
context,
@@ -508,9 +428,7 @@ pub fn readout_block(
..
} => {
let data_type: DataType = (*activation_precision).into();
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = QuantizedEmbeddingReadout::new_untied_output(
context,
@@ -532,9 +450,7 @@ pub fn readout_block(
} => {
let data_type: DataType = (*activation_precision).into();
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
// For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim
let group_size = config.model_dim;
@@ -557,12 +473,9 @@ pub fn readout_block(
embedding_quantization_mode,
..
} => {
- let data_type: DataType =
- config.output_norm_config.scale_precision.into();
+ let data_type: DataType = config.output_norm_config.scale_precision.into();
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = QuantizedEmbeddingReadout::new_tied(
context,
@@ -582,12 +495,9 @@ pub fn readout_block(
embedding_quantization_mode,
..
} => {
- let data_type: DataType =
- config.output_norm_config.scale_precision.into();
+ let data_type: DataType = config.output_norm_config.scale_precision.into();
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
+ let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
let block = QuantizedEmbeddingReadout::new_untied_output(
context,
diff --git a/crates/uzu/src/backends/metal/image.rs b/crates/uzu/src/backends/metal/image.rs
index 04e4b4afe..e11ce662d 100644
--- a/crates/uzu/src/backends/metal/image.rs
+++ b/crates/uzu/src/backends/metal/image.rs
@@ -1,9 +1,8 @@
use std::fmt;
use crate::backends::metal::{
- MTLDevice, MTLDeviceExt, MTLPixelFormat, MTLResourceExt, MTLStorageMode,
- MTLTexture, MTLTextureDescriptor, MTLTextureUsage, ProtocolObject,
- Retained,
+ MTLDevice, MTLDeviceExt, MTLPixelFormat, MTLResourceExt, MTLStorageMode, MTLTexture, MTLTextureDescriptor,
+ MTLTextureUsage, ProtocolObject, Retained,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -93,13 +92,7 @@ impl Image {
height: u32,
pixel_format: PixelFormat,
) -> Self {
- Self::new_with_usage(
- device,
- width,
- height,
- pixel_format,
- TextureUsage::default(),
- )
+ Self::new_with_usage(device, width, height, pixel_format, TextureUsage::default())
}
pub fn new_with_usage(
@@ -116,9 +109,7 @@ impl Image {
descriptor.set_storage_mode(MTLStorageMode::Private);
descriptor.set_usage(usage.into());
- let texture = device
- .new_texture_with_descriptor(&descriptor)
- .expect("Failed to create texture");
+ let texture = device.new_texture_with_descriptor(&descriptor).expect("Failed to create texture");
texture.set_label(Some("Image"));
Self {
@@ -133,9 +124,7 @@ impl Image {
device: &Retained<ProtocolObject<dyn MTLDevice>>,
descriptor: &MTLTextureDescriptor,
) -> Self {
- let texture = device
- .new_texture_with_descriptor(descriptor)
- .expect("Failed to create texture");
+ let texture = device.new_texture_with_descriptor(descriptor).expect("Failed to create texture");
texture.set_label(Some("Image"));
let width = descriptor.width() as u32;
@@ -150,9 +139,7 @@ impl Image {
}
}
- pub fn from_texture(
- texture: Retained<ProtocolObject<dyn MTLTexture>>
- ) -> Self {
+ pub fn from_texture(texture: Retained<ProtocolObject<dyn MTLTexture>>) -> Self {
let width = texture.width() as u32;
let height = texture.height() as u32;
let pixel_format = texture.pixel_format().into();
@@ -204,10 +191,6 @@ impl fmt::Display for Image {
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
- write!(
- f,
- "Image({}x{}, {:?})",
- self.width, self.height, self.pixel_format
- )
+ write!(f, "Image({}x{}, {:?})", self.width, self.height, self.pixel_format)
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index 7d4cd65c1..979fd6471 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -4,8 +4,7 @@ use objc2::rc::Retained;
use thiserror::Error;
use crate::backends::metal::{
- KernelDataType, MTLBuffer, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLDataType, MTLError,
+ KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLDataType, MTLError,
MTLFunctionConstantValues, MTLSize, ProtocolObject,
};
@@ -20,18 +19,10 @@ pub enum AttentionKernelVariant {
type PipelineKey = (usize, bool, bool, bool); // (head_dim, has_sinks, is_causal, has_mask)
pub struct AttentionKernelPipelines {
- single_pass: HashMap<
- PipelineKey,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
- two_pass_1: HashMap<
- PipelineKey,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
- two_pass_2:
- HashMap<usize, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- kv_cache_update:
- Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
+ single_pass: HashMap<PipelineKey, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
+ two_pass_1: HashMap<PipelineKey, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
+ two_pass_2: HashMap<usize, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
+ kv_cache_update: Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
}
pub struct AttentionKernel {
@@ -55,16 +46,16 @@ pub struct AttentionSinglePassArguments<'a> {
pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)
pub gqa_factor: i32, // buffer(4)
- pub sequence_length: i32, // buffer(5) - sequence_length
- pub k_head_stride: i32, // buffer(6)
- pub k_seq_stride: i32, // buffer(7)
- pub v_head_stride: i32, // buffer(8)
- pub v_seq_stride: i32, // buffer(9)
- pub scale: f32, // buffer(10)
+ pub sequence_length: i32, // buffer(5) - sequence_length
+ pub k_head_stride: i32, // buffer(6)
+ pub k_seq_stride: i32, // buffer(7)
+ pub v_head_stride: i32, // buffer(8)
+ pub v_seq_stride: i32, // buffer(9)
+ pub scale: f32, // buffer(10)
pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(11/12)
- pub mask_kv_seq_stride: i32, // buffer(13)
- pub mask_q_seq_stride: i32, // buffer(14)
- pub mask_head_stride: i32, // buffer(15)
+ pub mask_kv_seq_stride: i32, // buffer(13)
+ pub mask_q_seq_stride: i32, // buffer(14)
+ pub mask_head_stride: i32, // buffer(15)
pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(16)
pub num_heads: usize,
pub suffix_length: usize,
@@ -73,24 +64,24 @@ pub struct AttentionSinglePassArguments<'a> {
}
pub struct AttentionTwoPassArguments<'a> {
- pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
- pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)
- pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
+ pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
+ pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)
+ pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
pub partials_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 1 output
- pub sums_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) - pass 1 output
- pub maxs_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) - pass 1 output
- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 2 output
- pub gqa_factor: i32, // buffer(6)
- pub sequence_length: i32, // buffer(7) - sequence_length
- pub k_head_stride: i32, // buffer(8)
- pub k_seq_stride: i32, // buffer(9)
- pub v_head_stride: i32, // buffer(10)
- pub v_seq_stride: i32, // buffer(11)
- pub scale: f32, // buffer(12)
+ pub sums_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) - pass 1 output
+ pub maxs_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) - pass 1 output
+ pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 2 output
+ pub gqa_factor: i32, // buffer(6)
+ pub sequence_length: i32, // buffer(7) - sequence_length
+ pub k_head_stride: i32, // buffer(8)
+ pub k_seq_stride: i32, // buffer(9)
+ pub v_head_stride: i32, // buffer(10)
+ pub v_seq_stride: i32, // buffer(11)
+ pub scale: f32, // buffer(12)
pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(13/14)
- pub mask_kv_seq_stride: i32, // buffer(15)
- pub mask_q_seq_stride: i32, // buffer(16)
- pub mask_head_stride: i32, // buffer(17)
+ pub mask_kv_seq_stride: i32, // buffer(15)
+ pub mask_q_seq_stride: i32, // buffer(16)
+ pub mask_head_stride: i32, // buffer(17)
pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(18)
pub num_heads: usize,
pub suffix_length: usize,
@@ -140,36 +131,16 @@ fn make_function_constants(
let bool_mask_value = false;
let float_mask_value = has_mask_value;
- function_constants.set_constant_value_type_at_index(
- NonNull::from(&has_mask_value).cast(),
- MTLDataType::Bool,
- 20,
- ); // has_mask
+ function_constants.set_constant_value_type_at_index(NonNull::from(&has_mask_value).cast(), MTLDataType::Bool, 20); // has_mask
function_constants.set_constant_value_type_at_index(
NonNull::from(&query_transposed_value).cast(),
MTLDataType::Bool,
21,
); // query_transposed
- function_constants.set_constant_value_type_at_index(
- NonNull::from(&bool_mask_value).cast(),
- MTLDataType::Bool,
- 23,
- ); // bool_mask
- function_constants.set_constant_value_type_at_index(
- NonNull::from(&float_mask_value).cast(),
- MTLDataType::Bool,
- 24,
- ); // float_mask
- function_constants.set_constant_value_type_at_index(
- NonNull::from(&is_causal_value).cast(),
- MTLDataType::Bool,
- 22,
- ); // do_causal
- function_constants.set_constant_value_type_at_index(
- NonNull::from(&has_sinks_value).cast(),
- MTLDataType::Bool,
- 25,
- ); // has_sinks
+ function_constants.set_constant_value_type_at_index(NonNull::from(&bool_mask_value).cast(), MTLDataType::Bool, 23); // bool_mask
+ function_constants.set_constant_value_type_at_index(NonNull::from(&float_mask_value).cast(), MTLDataType::Bool, 24); // float_mask
+ function_constants.set_constant_value_type_at_index(NonNull::from(&is_causal_value).cast(), MTLDataType::Bool, 22); // do_causal
+ function_constants.set_constant_value_type_at_index(NonNull::from(&has_sinks_value).cast(), MTLDataType::Bool, 25); // has_sinks
function_constants
}
@@ -190,59 +161,28 @@ impl AttentionKernel {
for &has_sinks_value in &[false, true] {
for &is_causal_value in &[false, true] {
for &has_mask_value in &[false, true] {
- let function_constants = make_function_constants(
- has_mask_value,
- has_sinks_value,
- is_causal_value,
- );
+ let function_constants = make_function_constants(has_mask_value, has_sinks_value, is_causal_value);
for &head_dim in &supported_head_dims {
if let Ok(pipeline) = context.compute_pipeline_state(
- &format!(
- "attention_single_pass_{}_{}",
- data_suffix, head_dim
- ),
+ &format!("attention_single_pass_{}_{}", data_suffix, head_dim),
Some(&function_constants),
) {
- single_pass.insert(
- (
- head_dim,
- has_sinks_value,
- is_causal_value,
- has_mask_value,
- ),
- pipeline,
- );
+ single_pass.insert((head_dim, has_sinks_value, is_causal_value, has_mask_value), pipeline);
}
if let Ok(pipeline) = context.compute_pipeline_state(
- &format!(
- "attention_2pass_1_{}_{}",
- data_suffix, head_dim
- ),
+ &format!("attention_2pass_1_{}_{}", data_suffix, head_dim),
Some(&function_constants),
) {
- two_pass_1.insert(
- (
- head_dim,
- has_sinks_value,
- is_causal_value,
- has_mask_value,
- ),
- pipeline,
- );
+ two_pass_1.insert((head_dim, has_sinks_value, is_causal_value, has_mask_value), pipeline);
}
if !two_pass_2.contains_key(&head_dim) {
- if let Ok(pipeline) = context
- .compute_pipeline_state(
- &format!(
- "attention_2pass_2_{}_{}",
- data_suffix, head_dim
- ),
- Some(&function_constants),
- )
- {
+ if let Ok(pipeline) = context.compute_pipeline_state(
+ &format!("attention_2pass_2_{}_{}", data_suffix, head_dim),
+ Some(&function_constants),
+ ) {
two_pass_2.insert(head_dim, pipeline);
}
}
@@ -251,12 +191,7 @@ impl AttentionKernel {
}
}
- let kv_cache_update = context
- .compute_pipeline_state(
- &format!("update_kv_cache_{}", data_suffix),
- None,
- )
- .ok();
+ let kv_cache_update = context.compute_pipeline_state(&format!("update_kv_cache_{}", data_suffix), None).ok();
Ok(Self {
data_type,
@@ -275,9 +210,7 @@ impl AttentionKernel {
is_causal: bool,
has_mask: bool,
) -> bool {
- self.pipelines
- .single_pass
- .contains_key(&(head_dim, false, is_causal, has_mask))
+ self.pipelines.single_pass.contains_key(&(head_dim, false, is_causal, has_mask))
}
pub fn supports_two_pass(
@@ -286,9 +219,7 @@ impl AttentionKernel {
is_causal: bool,
has_mask: bool,
) -> bool {
- self.pipelines
- .two_pass_1
- .contains_key(&(head_dim, false, is_causal, has_mask))
+ self.pipelines.two_pass_1.contains_key(&(head_dim, false, is_causal, has_mask))
&& self.pipelines.two_pass_2.contains_key(&head_dim)
}
@@ -299,9 +230,7 @@ impl AttentionKernel {
is_causal: bool,
has_mask: bool,
) -> AttentionKernelVariant {
- if self.supports_two_pass(head_dim, is_causal, has_mask)
- && sequence_length > 1024
- {
+ if self.supports_two_pass(head_dim, is_causal, has_mask) && sequence_length > 1024 {
AttentionKernelVariant::TwoPass
} else {
AttentionKernelVariant::SinglePass
@@ -321,82 +250,47 @@ impl AttentionKernel {
.get(&(args.head_dim, has_sinks, args.is_causal, has_mask))
.ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
- MTLComputeCommandEncoder::set_compute_pipeline_state(
- compute_encoder,
- pipeline,
- );
+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pipeline);
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.queries_buffer),
- 0,
- 0,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.keys_buffer),
- 0,
- 1,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.values_buffer),
- 0,
- 2,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.output_buffer),
- 0,
- 3,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);
unsafe {
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.gqa_factor as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.gqa_factor as *const i32 as *mut _),
size_of::<i32>(),
4,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.sequence_length as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.sequence_length as *const i32 as *mut _),
size_of::<i32>(),
5,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.k_head_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.k_head_stride as *const i32 as *mut _),
size_of::<i32>(),
6,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.k_seq_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.k_seq_stride as *const i32 as *mut _),
size_of::<i32>(),
7,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.v_head_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.v_head_stride as *const i32 as *mut _),
size_of::<i32>(),
8,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.v_seq_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.v_seq_stride as *const i32 as *mut _),
size_of::<i32>(),
9,
);
@@ -409,34 +303,23 @@ impl AttentionKernel {
}
if let Some(mask_buffer) = args.mask_buffer {
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(mask_buffer),
- 0,
- 12,
- ); // float_mask
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 12); // float_mask
unsafe {
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.mask_kv_seq_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.mask_kv_seq_stride as *const i32 as *mut _),
size_of::<i32>(),
13,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.mask_q_seq_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.mask_q_seq_stride as *const i32 as *mut _),
size_of::<i32>(),
14,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.mask_head_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.mask_head_stride as *const i32 as *mut _),
size_of::<i32>(),
15,
);
@@ -444,12 +327,7 @@ impl AttentionKernel {
}
if let Some(sinks_buffer) = args.sinks_buffer {
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(sinks_buffer),
- 0,
- 16,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 16);
}
let threads_per_threadgroup = MTLSize {
@@ -458,13 +336,9 @@ impl AttentionKernel {
depth: 1,
};
- let threadgroups_per_grid =
- MTLSize::new(args.num_heads, args.suffix_length, 1);
+ let threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, 1);
- compute_encoder.dispatch_threadgroups(
- threadgroups_per_grid,
- threads_per_threadgroup,
- );
+ compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);
Ok(())
}
@@ -481,99 +355,55 @@ impl AttentionKernel {
.get(&(args.head_dim, has_sinks, args.is_causal, has_mask))
.ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
- let pass2_pipeline =
- self.pipelines.two_pass_2.get(&args.head_dim).ok_or_else(|| {
- AttentionError::UnsupportedHeadDim(args.head_dim)
- })?;
+ let pass2_pipeline = self
+ .pipelines
+ .two_pass_2
+ .get(&args.head_dim)
+ .ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
- MTLComputeCommandEncoder::set_compute_pipeline_state(
- compute_encoder,
- pass1_pipeline,
- );
+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pass1_pipeline);
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.queries_buffer),
- 0,
- 0,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.keys_buffer),
- 0,
- 1,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.values_buffer),
- 0,
- 2,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.partials_buffer),
- 0,
- 3,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.sums_buffer),
- 0,
- 4,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.maxs_buffer),
- 0,
- 5,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.partials_buffer), 0, 3);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.sums_buffer), 0, 4);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.maxs_buffer), 0, 5);
unsafe {
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.gqa_factor as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.gqa_factor as *const i32 as *mut _),
size_of::<i32>(),
6,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.sequence_length as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.sequence_length as *const i32 as *mut _),
size_of::<i32>(),
7,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.k_head_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.k_head_stride as *const i32 as *mut _),
size_of::<i32>(),
8,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.k_seq_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.k_seq_stride as *const i32 as *mut _),
size_of::<i32>(),
9,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.v_head_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.v_head_stride as *const i32 as *mut _),
size_of::<i32>(),
10,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.v_seq_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.v_seq_stride as *const i32 as *mut _),
size_of::<i32>(),
11,
);
@@ -587,34 +417,23 @@ impl AttentionKernel {
// Set mask buffer if present
if let Some(mask_buffer) = args.mask_buffer {
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(mask_buffer),
- 0,
- 14,
- ); // float_mask
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 14); // float_mask
unsafe {
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.mask_kv_seq_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.mask_kv_seq_stride as *const i32 as *mut _),
size_of::<i32>(),
15,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.mask_q_seq_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.mask_q_seq_stride as *const i32 as *mut _),
size_of::<i32>(),
16,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &args.mask_head_stride as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&args.mask_head_stride as *const i32 as *mut _),
size_of::<i32>(),
17,
);
@@ -622,12 +441,7 @@ impl AttentionKernel {
}
if let Some(sinks_buffer) = args.sinks_buffer {
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(sinks_buffer),
- 0,
- 18,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 18);
}
let total_blocks_count = 32u64;
@@ -636,59 +450,25 @@ impl AttentionKernel {
height: 1,
depth: 1,
};
- let pass1_threadgroups_per_grid = MTLSize::new(
- args.num_heads,
- args.suffix_length,
- total_blocks_count as usize,
- );
+ let pass1_threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, total_blocks_count as usize);
- compute_encoder.dispatch_threadgroups(
- pass1_threadgroups_per_grid,
- pass1_threads_per_threadgroup,
- );
+ compute_encoder.dispatch_threadgroups(pass1_threadgroups_per_grid, pass1_threads_per_threadgroup);
- MTLComputeCommandEncoder::set_compute_pipeline_state(
- compute_encoder,
- pass2_pipeline,
- );
+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pass2_pipeline);
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.partials_buffer),
- 0,
- 0,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.sums_buffer),
- 0,
- 1,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.maxs_buffer),
- 0,
- 2,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.output_buffer),
- 0,
- 3,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.partials_buffer), 0, 0);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.sums_buffer), 0, 1);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.maxs_buffer), 0, 2);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);
let pass2_threads_per_threadgroup = MTLSize {
width: 32 * 32,
height: 1,
depth: 1,
};
- let pass2_threadgroups_per_grid =
- MTLSize::new(args.num_heads, args.suffix_length, 1);
+ let pass2_threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, 1);
- compute_encoder.dispatch_threadgroups(
- pass2_threadgroups_per_grid,
- pass2_threads_per_threadgroup,
- );
+ compute_encoder.dispatch_threadgroups(pass2_threadgroups_per_grid, pass2_threads_per_threadgroup);
Ok(())
}
@@ -698,105 +478,63 @@ impl AttentionKernel {
compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
args: KVCacheUpdateArguments,
) -> Result<(), AttentionError> {
- let pipeline =
- self.pipelines.kv_cache_update.as_ref().ok_or_else(|| {
- AttentionError::FunctionNotFound(
- "KV cache update kernel".to_string(),
- )
- })?;
-
- MTLComputeCommandEncoder::set_compute_pipeline_state(
- compute_encoder,
- pipeline,
- );
+ let pipeline = self
+ .pipelines
+ .kv_cache_update
+ .as_ref()
+ .ok_or_else(|| AttentionError::FunctionNotFound("KV cache update kernel".to_string()))?;
+
+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pipeline);
// Set buffers
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.rotated_keys_buffer),
- 0,
- 0,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.qkv_buffer),
- 0,
- 1,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.key_cache_buffer),
- 0,
- 2,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.value_cache_buffer),
- 0,
- 3,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.rotated_keys_buffer), 0, 0);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.qkv_buffer), 0, 1);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.key_cache_buffer), 0, 2);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.value_cache_buffer), 0, 3);
// Set constants
unsafe {
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &(args.num_groups as i32) as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&(args.num_groups as i32) as *const i32 as *mut _),
size_of::<i32>(),
4,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &(args.num_heads as i32) as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&(args.num_heads as i32) as *const i32 as *mut _),
size_of::<i32>(),
5,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &(args.head_dim as i32) as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&(args.head_dim as i32) as *const i32 as *mut _),
size_of::<i32>(),
6,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &(args.suffix_length as i32) as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&(args.suffix_length as i32) as *const i32 as *mut _),
size_of::<i32>(),
7,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &(args.segment_prefix_length as i32) as *const i32
- as *mut _,
- ),
+ NonNull::new_unchecked(&(args.segment_prefix_length as i32) as *const i32 as *mut _),
size_of::<i32>(),
8,
);
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &(args.max_sequence_length as i32) as *const i32 as *mut _,
- ),
+ NonNull::new_unchecked(&(args.max_sequence_length as i32) as *const i32 as *mut _),
size_of::<i32>(),
9,
);
}
- let threads_per_grid = MTLSize::new(
- args.num_groups,
- args.suffix_length,
- args.head_dim as usize,
- );
+ let threads_per_grid = MTLSize::new(args.num_groups, args.suffix_length, args.head_dim as usize);
- let threadgroup_depth =
- std::cmp::min(args.head_dim.max(1), 64) as usize;
+ let threadgroup_depth = std::cmp::min(args.head_dim.max(1), 64) as usize;
MTLComputeCommandEncoder::dispatch_threads(
compute_encoder,
@@ -836,31 +574,11 @@ impl AttentionKernel {
let has_sinks = args.sinks_buffer.is_some();
let fcv = MTLFunctionConstantValues::new();
- fcv.set_constant_value_type_at_index(
- NonNull::from(&align_q).cast(),
- MTLDataType::Bool,
- 200,
- );
- fcv.set_constant_value_type_at_index(
- NonNull::from(&align_k).cast(),
- MTLDataType::Bool,
- 201,
- );
- fcv.set_constant_value_type_at_index(
- NonNull::from(&has_mask).cast(),
- MTLDataType::Bool,
- 300,
- );
- fcv.set_constant_value_type_at_index(
- NonNull::from(&args.is_causal).cast(),
- MTLDataType::Bool,
- 301,
- );
- fcv.set_constant_value_type_at_index(
- NonNull::from(&has_sinks).cast(),
- MTLDataType::Bool,
- 302,
- );
+ fcv.set_constant_value_type_at_index(NonNull::from(&align_q).cast(), MTLDataType::Bool, 200);
+ fcv.set_constant_value_type_at_index(NonNull::from(&align_k).cast(), MTLDataType::Bool, 201);
+ fcv.set_constant_value_type_at_index(NonNull::from(&has_mask).cast(), MTLDataType::Bool, 300);
+ fcv.set_constant_value_type_at_index(NonNull::from(&args.is_causal).cast(), MTLDataType::Bool, 301);
+ fcv.set_constant_value_type_at_index(NonNull::from(&has_sinks).cast(), MTLDataType::Bool, 302);
// Kernel name matches gemm_attention.metal instantiations:
// attention_gemm_{f16|bf16|f32}_{head_dim}_bk{bk}
@@ -870,57 +588,24 @@ impl AttentionKernel {
KernelDataType::Float32 => "f32",
};
- let function_name =
- format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);
+ let function_name = format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);
let cache_key = format!(
"{}_aq{}_ak{}_m{}_c{}_s{}",
- function_name,
- align_q as u8,
- align_k as u8,
- has_mask as u8,
- args.is_causal as u8,
- has_sinks as u8
+ function_name, align_q as u8, align_k as u8, has_mask as u8, args.is_causal as u8, has_sinks as u8
);
let pipeline = context
- .compute_pipeline_state_cached(
- &cache_key,
- &function_name,
- Some(&fcv),
- )
+ .compute_pipeline_state_cached(&cache_key, &function_name, Some(&fcv))
.map_err(AttentionError::MetalError)?;
- MTLComputeCommandEncoder::set_compute_pipeline_state(
- compute_encoder,
- &pipeline,
- );
+ MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, &pipeline);
// Buffers
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.queries_buffer),
- 0,
- 0,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.keys_buffer),
- 0,
- 1,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.values_buffer),
- 0,
- 2,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(args.output_buffer),
- 0,
- 3,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);
// Params (all strides in elements)
let q_head_stride = (args.suffix_length * args.head_dim) as i64;
@@ -971,43 +656,27 @@ impl AttentionKernel {
unsafe {
MTLComputeCommandEncoder::set_bytes(
compute_encoder,
- NonNull::new_unchecked(
- &mask_params as *const AttnMaskParams as *mut _,
- ),
+ NonNull::new_unchecked(&mask_params as *const AttnMaskParams as *mut _),
size_of::<AttnMaskParams>(),
5,
);
}
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(mask_buffer),
- 0,
- 6,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 6);
}
if let Some(sinks_buffer) = args.sinks_buffer {
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(sinks_buffer),
- 0,
- 16,
- );
+ MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 16);
}
// Dispatch
- let threadgroups_per_grid =
- MTLSize::new(nq as usize, args.num_heads, 1);
+ let threadgroups_per_grid = MTLSize::new(nq as usize, args.num_heads, 1);
let threads_per_threadgroup = MTLSize {
width: 32,
height: WM as usize,
depth: WN as usize,
};
- compute_encoder.dispatch_threadgroups(
- threadgroups_per_grid,
- threads_per_threadgroup,
- );
+ compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);
Ok(())
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
index d05d3f17f..44f19c034 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
@@ -1,8 +1,6 @@
mod matmul_arguments;
mod transpose_configuration;
-pub use crate::backends::common::gpu_types::{
- GEMMAddMMParams, GEMMParams, GEMMSpiltKMlpFusedParams, GEMMSpiltKParams,
-};
+pub use crate::backends::common::gpu_types::{GEMMAddMMParams, GEMMParams, GEMMSpiltKMlpFusedParams, GEMMSpiltKParams};
pub use matmul_arguments::MatmulArguments;
pub(crate) use transpose_configuration::transpose_configuration;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
index 2e887ca30..91690f38a 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
@@ -43,21 +43,15 @@ pub(crate) fn choose_dispatch_descriptor(
data_type: DataType,
arguments: &MatmulArguments,
) -> Result<MatmulDispatchDescriptor, MTLError> {
- if let Some(descriptor) =
- gemv::DispatchDescriptor::try_new(context, data_type, arguments)?
- {
+ if let Some(descriptor) = gemv::DispatchDescriptor::try_new(context, data_type, arguments)? {
return Ok(MatmulDispatchDescriptor::Gemv(descriptor));
}
- if let Some(descriptor) =
- split_k::DispatchDescriptor::try_new(context, data_type, arguments)?
- {
+ if let Some(descriptor) = split_k::DispatchDescriptor::try_new(context, data_type, arguments)? {
return Ok(MatmulDispatchDescriptor::SplitK(descriptor));
}
- Ok(MatmulDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(
- context, data_type, arguments,
- )?))
+ Ok(MatmulDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(context, data_type, arguments)?))
}
pub fn determine_kernel_variant(
@@ -65,6 +59,5 @@ pub fn determine_kernel_variant(
data_type: DataType,
arguments: &MatmulArguments,
) -> Result<MatmulKernelVariant, MTLError> {
- choose_dispatch_descriptor(context, data_type, arguments)
- .map(|d| d.variant())
+ choose_dispatch_descriptor(context, data_type, arguments).map(|d| d.variant())
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
index 72208d5f0..47fb00a6c 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
@@ -1,14 +1,9 @@
-use super::{
- pipeline_configuration::PipelineConfiguration,
- tile_configuration::TileConfiguration,
-};
+use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::TileConfiguration};
use crate::{
DataType,
backends::metal::{
DeviceClass, MTLContext, MTLError, MTLSize,
- kernel::matmul::common::{
- GEMMAddMMParams, GEMMParams, MatmulArguments,
- },
+ kernel::matmul::common::{GEMMAddMMParams, GEMMParams, MatmulArguments},
},
};
@@ -27,11 +22,8 @@ impl DispatchDescriptor {
data_type: DataType,
arguments: &MatmulArguments,
) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for GEMM: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for GEMM: {data_type:?}")));
}
let tile = select_tile(context, data_type, arguments);
@@ -49,8 +41,7 @@ impl DispatchDescriptor {
align_k: (k % tile.block_depth) == 0,
has_batch: arguments.batch_count > 1,
use_out_source: arguments.c.is_some(),
- do_axpby: arguments.c.is_some()
- && (arguments.alpha != 1.0 || arguments.beta != 0.0),
+ do_axpby: arguments.c.is_some() && (arguments.alpha != 1.0 || arguments.beta != 0.0),
};
let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;
@@ -61,15 +52,13 @@ impl DispatchDescriptor {
let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
let tn_swizzled = tiles_n * tile_swizzle;
- let elements_per_matrix_a =
- (arguments.batch as i64) * (arguments.lda as i64);
+ let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);
let elements_per_matrix_b = if arguments.transpose_b {
(arguments.output_dim as i64) * (arguments.ldb as i64)
} else {
(arguments.input_dim as i64) * (arguments.ldb as i64)
};
- let elements_per_matrix_d =
- (arguments.batch as i64) * (arguments.ldd as i64);
+ let elements_per_matrix_d = (arguments.batch as i64) * (arguments.ldd as i64);
let params = GEMMParams {
M: m,
@@ -105,16 +94,8 @@ impl DispatchDescriptor {
None
};
- let threads_per_threadgroup = MTLSize::new(
- 32,
- tile.warps_per_col as usize,
- tile.warps_per_row as usize,
- );
- let threadgroups = MTLSize::new(
- tn_swizzled as usize,
- tm_swizzled as usize,
- arguments.batch_count as usize,
- );
+ let threads_per_threadgroup = MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);
+ let threadgroups = MTLSize::new(tn_swizzled as usize, tm_swizzled as usize, arguments.batch_count as usize);
Ok(Self {
pipeline_configuration,
@@ -131,16 +112,14 @@ fn select_tile(
data_type: DataType,
arguments: &MatmulArguments,
) -> TileConfiguration {
- let overall_work_elements = (arguments.batch_count as i64)
- * (arguments.batch as i64)
- * (arguments.output_dim as i64);
+ let overall_work_elements =
+ (arguments.batch_count as i64) * (arguments.batch as i64) * (arguments.output_dim as i64);
let is_float32 = matches!(data_type, DataType::F32);
let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();
if context.is_nax_available() && prefer_half_or_tf32 {
let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);
- let tile_rows =
- (arguments.batch + base_tile.block_rows - 1) / base_tile.block_rows;
+ let tile_rows = (arguments.batch + base_tile.block_rows - 1) / base_tile.block_rows;
let swizzle_log2 = if tile_rows <= 3 {
0
} else {
@@ -153,9 +132,7 @@ fn select_tile(
}
match context.architecture.device_class {
- DeviceClass::Integrated
- | DeviceClass::Phone
- | DeviceClass::Unknown(_) => {
+ DeviceClass::Integrated | DeviceClass::Phone | DeviceClass::Unknown(_) => {
if prefer_half_or_tf32 {
if !arguments.transpose_a && arguments.transpose_b {
TileConfiguration::new(64, 32, 32, 2, 2, 0)
@@ -171,9 +148,7 @@ fn select_tile(
DeviceClass::Desktop => {
if overall_work_elements >= (1_i64 << 20) {
if prefer_half_or_tf32 {
- if 2 * std::cmp::max(arguments.batch, arguments.output_dim)
- > arguments.input_dim
- {
+ if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {
TileConfiguration::new(64, 64, 16, 2, 2, 0)
} else if !arguments.transpose_a && arguments.transpose_b {
TileConfiguration::new(64, 32, 32, 2, 2, 0)
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index d115da827..8c739d928 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -1,14 +1,10 @@
use std::{collections::HashMap, ptr::NonNull};
-use super::{
- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,
- tile_configuration::TileConfiguration,
-};
+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration, tile_configuration::TileConfiguration};
use crate::{
DataType,
backends::metal::{
- ComputeEncoderSetValue, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError,
+ ComputeEncoderSetValue, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
MTLFunctionConstantValues, ProtocolObject, Retained,
kernel::matmul::common::{MatmulArguments, transpose_configuration},
},
@@ -16,19 +12,13 @@ use crate::{
pub struct Kernel {
data_type: DataType,
- pipelines: HashMap<
- PipelineConfiguration,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
+ pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
}
impl Kernel {
pub fn new(data_type: DataType) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for GEMM: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for GEMM: {data_type:?}")));
}
Ok(Self {
data_type,
@@ -41,36 +31,21 @@ impl Kernel {
&mut self,
context: &MTLContext,
) -> Result<(), MTLError> {
- let tiles_and_alignments: &[(
- TileConfiguration,
- &[(bool, bool, bool)],
- )] = match self.data_type {
+ let tiles_and_alignments: &[(TileConfiguration, &[(bool, bool, bool)])] = match self.data_type {
DataType::BF16 => &[
- (
- TileConfiguration::new(64, 32, 32, 2, 2, 0),
- &[(false, true, true), (true, true, true)],
- ),
+ (TileConfiguration::new(64, 32, 32, 2, 2, 0), &[(false, true, true), (true, true, true)]),
(
TileConfiguration::new(64, 64, 16, 2, 2, 0),
- &[
- (false, true, true),
- (true, false, true),
- (true, true, true),
- ],
- ),
- (
- TileConfiguration::new(64, 64, 16, 1, 2, 0),
- &[(true, true, true)],
+ &[(false, true, true), (true, false, true), (true, true, true)],
),
+ (TileConfiguration::new(64, 64, 16, 1, 2, 0), &[(true, true, true)]),
],
- DataType::F16 => &[(
- TileConfiguration::new(64, 64, 16, 2, 2, 0),
- &[(true, true, true), (false, true, true)],
- )],
- DataType::F32 => &[(
- TileConfiguration::new(32, 64, 16, 1, 2, 0),
- &[(false, true, true), (true, true, true)],
- )],
+ DataType::F16 => {
+ &[(TileConfiguration::new(64, 64, 16, 2, 2, 0), &[(true, true, true), (false, true, true)])]
+ },
+ DataType::F32 => {
+ &[(TileConfiguration::new(32, 64, 16, 1, 2, 0), &[(false, true, true), (true, true, true)])]
+ },
_ => return Ok(()),
};
@@ -108,10 +83,7 @@ impl Kernel {
configuration: &PipelineConfiguration,
) -> String {
let type_name = self.type_name();
- let transpose_suffix = transpose_configuration(
- configuration.transpose_a,
- configuration.transpose_b,
- );
+ let transpose_suffix = transpose_configuration(configuration.transpose_a, configuration.transpose_b);
let prefix = if configuration.tile.is_nax() {
"steel_gemm_nax"
} else {
@@ -135,8 +107,7 @@ impl Kernel {
&mut self,
context: &MTLContext,
configuration: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if !self.pipelines.contains_key(configuration) {
let kernel_name = self.kernel_name(configuration);
let function_constants = MTLFunctionConstantValues::new();
@@ -181,11 +152,8 @@ impl Kernel {
configuration.use_out_source as u8,
configuration.do_axpby as u8
);
- let pipeline_state = context.compute_pipeline_state_cached(
- &cache_key,
- &kernel_name,
- Some(&function_constants),
- )?;
+ let pipeline_state =
+ context.compute_pipeline_state_cached(&cache_key, &kernel_name, Some(&function_constants))?;
self.pipelines.insert(configuration.clone(), pipeline_state);
}
Ok(self.pipelines.get(configuration).unwrap())
@@ -198,10 +166,7 @@ impl Kernel {
arguments: &MatmulArguments,
descriptor: &DispatchDescriptor,
) -> Result<bool, MTLError> {
- let pipeline_state = self.get_or_compile_pipeline(
- context,
- &descriptor.pipeline_configuration,
- )?;
+ let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;
encoder.set_compute_pipeline_state(pipeline_state);
encoder.set_buffer(Some(arguments.a), arguments.a_offset as usize, 0);
@@ -219,10 +184,7 @@ impl Kernel {
encoder.set_value(addmm_params, 5);
}
- encoder.dispatch_threadgroups(
- descriptor.threadgroups,
- descriptor.threads_per_threadgroup,
- );
+ encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);
Ok(false)
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs
index 4df5febee..9cf36aa3f 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs
@@ -1,13 +1,9 @@
use std::sync::OnceLock;
-use super::pipeline_configuration::{
- PipelineConfiguration, select_configuration,
-};
+use super::pipeline_configuration::{PipelineConfiguration, select_configuration};
use crate::{
DataType,
- backends::metal::{
- MTLContext, MTLError, MTLSize, kernel::matmul::common::MatmulArguments,
- },
+ backends::metal::{MTLContext, MTLError, MTLSize, kernel::matmul::common::MatmulArguments},
};
const DEFAULT_GEMV_MAX_BATCH: i32 = 8;
@@ -16,10 +12,7 @@ static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();
fn max_gemv_batch_threshold() -> i32 {
*GEMV_MAX_BATCH.get_or_init(|| {
- std::env::var("UZU_GEMV_MAX_BATCH")
- .ok()
- .and_then(|s| s.parse().ok())
- .unwrap_or(DEFAULT_GEMV_MAX_BATCH)
+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
})
}
@@ -62,11 +55,8 @@ impl DispatchDescriptor {
data_type: DataType,
arguments: &MatmulArguments,
) -> Result<Option<Self>, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for GEMV: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported data type for GEMV: {data_type:?}")));
}
if arguments.transpose_a || !arguments.transpose_b {
@@ -104,9 +94,7 @@ impl DispatchDescriptor {
let (do_axpby, alpha, beta, bias_stride) = match axpby_source {
AxpbySource::None => (false, 1.0f32, 0.0f32, 0),
AxpbySource::Bias => (true, 1.0f32, 1.0f32, 1),
- AxpbySource::C => {
- (true, arguments.alpha, arguments.beta, arguments.ldd)
- },
+ AxpbySource::C => (true, arguments.alpha, arguments.beta, arguments.ldd),
};
let output_dimension = if matrix_is_rhs {
@@ -116,10 +104,7 @@ impl DispatchDescriptor {
};
let mut batch_pack = 1;
- if m == 4
- && arguments.input_dim <= 2048
- && (1536..=3072).contains(&output_dimension)
- {
+ if m == 4 && arguments.input_dim <= 2048 && (1536..=3072).contains(&output_dimension) {
batch_pack = 2;
} else if m <= 8 {
batch_pack = if m >= 4 && m % 4 == 0 {
@@ -157,8 +142,7 @@ impl DispatchDescriptor {
1
}];
- let elements_per_matrix_a =
- (arguments.batch as i64) * (arguments.lda as i64);
+ let elements_per_matrix_a = (arguments.batch as i64) * (arguments.lda as i64);
let elements_per_matrix_b = if arguments.transpose_b {
(arguments.output_dim as i64) * (arguments.ldb as i64)
} else {
@@ -183,23 +167,17 @@ impl DispatchDescriptor {
0
}];
- let output_elements_per_threadgroup =
- pipeline_configuration.output_elements_per_threadgroup();
+ let output_elements_per_threadgroup = pipeline_configuration.output_elements_per_threadgroup();
let threadgroup_count_x =
- ((output_dimension as u32 + output_elements_per_threadgroup - 1)
- / output_elements_per_threadgroup) as u64;
+ ((output_dimension as u32 + output_elements_per_threadgroup - 1) / output_elements_per_threadgroup) as u64;
let threadgroup_count_z = batch_groups.max(1) as u64;
let batch_rows = arguments.batch;
let threadgroup_count_y = (batch_rows / batch_pack).max(1) as u64;
- let threadgroups = MTLSize::new(
- threadgroup_count_x as usize,
- threadgroup_count_y as usize,
- threadgroup_count_z as usize,
- );
- let threads_per_threadgroup =
- pipeline_configuration.threads_per_threadgroup();
+ let threadgroups =
+ MTLSize::new(threadgroup_count_x as usize, threadgroup_count_y as usize, threadgroup_count_z as usize);
+ let threads_per_threadgroup = pipeline_configuration.threads_per_threadgroup();
let output_ld = arguments.ldd;
let vector_ld = arguments.lda;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index ef92b6b37..ed55e89c1 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -7,9 +7,8 @@ use super::{
use crate::{
DataType,
backends::metal::{
- ComputeEncoderSetValue, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError, ProtocolObject,
- Retained, kernel::matmul::common::MatmulArguments,
+ ComputeEncoderSetValue, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
+ ProtocolObject, Retained, kernel::matmul::common::MatmulArguments,
},
};
@@ -22,10 +21,7 @@ fn gemv_kernel_name(
DataType::BF16 => "bfloat16",
DataType::F32 => "float32",
_ => {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for GEMV: {:?}",
- data_type
- )));
+ return Err(MTLError::Generic(format!("Unsupported data type for GEMV: {:?}", data_type)));
},
};
@@ -58,20 +54,13 @@ fn gemv_kernel_name(
pub struct Kernel {
data_type: DataType,
- pipelines: HashMap<
- PipelineConfiguration,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
+ pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
}
impl Kernel {
pub fn new(data_type: DataType) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for GEMV: {:?}",
- data_type
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported data type for GEMV: {:?}", data_type)));
}
Ok(Self {
data_type,
@@ -115,12 +104,10 @@ impl Kernel {
&mut self,
context: &MTLContext,
config: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if !self.pipelines.contains_key(config) {
let kernel_name = gemv_kernel_name(self.data_type, config)?;
- let pipeline =
- context.compute_pipeline_state(&kernel_name, None)?;
+ let pipeline = context.compute_pipeline_state(&kernel_name, None)?;
self.pipelines.insert(*config, pipeline);
}
Ok(self.pipelines.get(config).unwrap())
@@ -133,8 +120,7 @@ impl Kernel {
arguments: &MatmulArguments,
descriptor: &DispatchDescriptor,
) -> Result<bool, MTLError> {
- let pipeline =
- self.get_pipeline(context, &descriptor.pipeline_configuration)?;
+ let pipeline = self.get_pipeline(context, &descriptor.pipeline_configuration)?;
encoder.set_compute_pipeline_state(pipeline);
let (buf0, off0) = if descriptor.matrix_is_rhs {
@@ -155,24 +141,18 @@ impl Kernel {
match descriptor.axpby_source {
AxpbySource::None => {
return Err(MTLError::Generic(
- "GEMV descriptor mismatch: do_axpby=true but axpby_source=None"
- .to_owned(),
+ "GEMV descriptor mismatch: do_axpby=true but axpby_source=None".to_owned(),
));
},
AxpbySource::Bias => {
- let bias = arguments.bias.ok_or_else(|| {
- MTLError::Generic(
- "GEMV descriptor requires bias buffer".to_owned(),
- )
- })?;
+ let bias = arguments
+ .bias
+ .ok_or_else(|| MTLError::Generic("GEMV descriptor requires bias buffer".to_owned()))?;
encoder.set_buffer(Some(bias), 0, 2);
},
AxpbySource::C => {
- let c_buffer = arguments.c.ok_or_else(|| {
- MTLError::Generic(
- "GEMV descriptor requires C buffer".to_owned(),
- )
- })?;
+ let c_buffer =
+ arguments.c.ok_or_else(|| MTLError::Generic("GEMV descriptor requires C buffer".to_owned()))?;
encoder.set_buffer(Some(c_buffer), 0, 2);
},
}
@@ -195,10 +175,7 @@ impl Kernel {
encoder.set_value(&descriptor.output_ld, 16);
encoder.set_value(&descriptor.vector_ld, 17);
- encoder.dispatch_threadgroups(
- descriptor.threadgroups,
- descriptor.threads_per_threadgroup,
- );
+ encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);
Ok(descriptor.bias_is_fused())
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs
index e21e979ec..96339e3dd 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs
@@ -19,22 +19,14 @@ pub struct PipelineConfiguration {
impl PipelineConfiguration {
pub fn output_elements_per_threadgroup(&self) -> u32 {
if self.transpose_matrix {
- self.threadgroup_cols
- * self.threads_per_simdgroup_col
- * self.elements_per_thread_col
+ self.threadgroup_cols * self.threads_per_simdgroup_col * self.elements_per_thread_col
} else {
- self.threadgroup_rows
- * self.threads_per_simdgroup_row
- * self.elements_per_thread_row
+ self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row
}
}
pub fn threads_per_threadgroup(&self) -> MTLSize {
- MTLSize::new(
- 32,
- self.threadgroup_cols as usize,
- self.threadgroup_rows as usize,
- )
+ MTLSize::new(32, self.threadgroup_cols as usize, self.threadgroup_rows as usize)
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 4b6436e15..fccee162f 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -1,8 +1,6 @@
use super::{
common::MatmulArguments,
- dispatch_descriptor::{
- MatmulDispatchDescriptor, choose_dispatch_descriptor,
- },
+ dispatch_descriptor::{MatmulDispatchDescriptor, choose_dispatch_descriptor},
gemm,
gemv::GemvKernel,
split_k::SplitKGemm,
@@ -12,8 +10,8 @@ use crate::{
backends::{
common::kernel::TensorAddBiasKernel,
metal::{
- MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLError,
- ProtocolObject, Retained, kernel::dsl::TensorAddBiasMetalKernel,
+ MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject, Retained,
+ kernel::dsl::TensorAddBiasMetalKernel,
},
},
};
@@ -28,11 +26,8 @@ pub struct MatmulKernel {
impl MatmulKernel {
pub fn new(data_type: DataType) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MatmulKernel: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for MatmulKernel: {data_type:?}")));
}
Ok(Self {
@@ -60,9 +55,7 @@ impl MatmulKernel {
Ok(())
}
- fn get_or_create_gemm(
- &mut self
- ) -> Result<&mut gemm::GemmKernel, MTLError> {
+ fn get_or_create_gemm(&mut self) -> Result<&mut gemm::GemmKernel, MTLError> {
if self.gemm.is_none() {
self.gemm = Some(gemm::GemmKernel::new(self.data_type)?);
}
@@ -97,8 +90,7 @@ impl MatmulKernel {
},
MatmulDispatchDescriptor::SplitK(descriptor) => {
let splitk = self.get_or_create_splitk()?;
- splitk
- .encode_descriptor(context, encoder, arguments, descriptor)
+ splitk.encode_descriptor(context, encoder, arguments, descriptor)
},
MatmulDispatchDescriptor::Gemm(descriptor) => {
let gemm = self.get_or_create_gemm()?;
@@ -115,15 +107,9 @@ impl MatmulKernel {
) -> Result<(), MTLError> {
Self::apply_batch_collapse(&mut arguments);
- let descriptor =
- choose_dispatch_descriptor(context, self.data_type, &arguments)?;
+ let descriptor = choose_dispatch_descriptor(context, self.data_type, &arguments)?;
- let bias_fused = self.encode_dispatch_descriptor(
- context,
- encoder,
- &arguments,
- &descriptor,
- )?;
+ let bias_fused = self.encode_dispatch_descriptor(context, encoder, &arguments, &descriptor)?;
if let Some(bias) = arguments.bias {
if !bias_fused {
@@ -150,18 +136,10 @@ impl MatmulKernel {
}
if self.bias_add.is_none() {
- self.bias_add =
- Some(TensorAddBiasMetalKernel::new(context, self.data_type)?);
+ self.bias_add = Some(TensorAddBiasMetalKernel::new(context, self.data_type)?);
}
let bias_add = self.bias_add.as_ref().unwrap();
- bias_add.encode(
- arguments.d,
- bias,
- arguments.d,
- n as u32,
- total_len as u32,
- encoder,
- );
+ bias_add.encode(arguments.d, bias, arguments.d, n as u32, total_len as u32, encoder);
Ok(())
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs
index bed064a57..cb32b7800 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs
@@ -1,14 +1,9 @@
-use super::{
- pipeline_configuration::PipelineConfiguration,
- tile_configuration::select_tile_configuration,
-};
+use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};
use crate::{
DataType,
backends::metal::{
MTLContext, MTLError, MTLSize,
- kernel::matmul::common::{
- GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments,
- },
+ kernel::matmul::common::{GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments},
},
};
@@ -31,12 +26,8 @@ impl DispatchDescriptor {
data_type: DataType,
arguments: &MatmulArguments,
) -> Result<Option<Self>, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for Split-K: {:?}",
- data_type
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for Split-K: {:?}", data_type)));
}
if arguments.c.is_some() {
@@ -72,11 +63,8 @@ impl DispatchDescriptor {
let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;
let output_elements_per_partition = m * n;
- let accumulator_element_count = partition_count
- * output_elements_per_partition
- * arguments.batch_count;
- let accumulator_bytes =
- (accumulator_element_count as usize) * std::mem::size_of::<f32>();
+ let accumulator_element_count = partition_count * output_elements_per_partition * arguments.batch_count;
+ let accumulator_bytes = (accumulator_element_count as usize) * std::mem::size_of::<f32>();
let params = SplitKGEMMParams {
M: m,
@@ -93,20 +81,12 @@ impl DispatchDescriptor {
gemm_k_iterations_aligned: gemm_k_iterations,
};
- let partial_threads_per_threadgroup = MTLSize::new(
- 32,
- tile.warps_per_col as usize,
- tile.warps_per_row as usize,
- );
- let partial_threadgroups = MTLSize::new(
- tile_count_n as usize,
- tile_count_m as usize,
- partition_count as usize,
- );
+ let partial_threads_per_threadgroup =
+ MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);
+ let partial_threadgroups = MTLSize::new(tile_count_n as usize, tile_count_m as usize, partition_count as usize);
let accum_total_threads = MTLSize::new(n as usize, m as usize, 1);
- let accum_threads_per_threadgroup =
- MTLSize::new(16.min(n as usize), 16.min(m as usize), 1);
+ let accum_threads_per_threadgroup = MTLSize::new(16.min(n as usize), 16.min(m as usize), 1);
Ok(Some(Self {
pipeline_configuration,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
index e08b03516..ec5bed92e 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
@@ -1,43 +1,30 @@
use std::collections::HashMap;
-use super::{
- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,
-};
+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};
use crate::{
DataType,
backends::{
common::Context,
metal::{
- ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError, ProtocolObject,
- Retained,
- kernel::matmul::common::{
- MatmulArguments, transpose_configuration,
- },
+ ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
+ ProtocolObject, Retained,
+ kernel::matmul::common::{MatmulArguments, transpose_configuration},
},
},
};
pub struct Kernel {
data_type: DataType,
- partial_pipelines: HashMap<
- PipelineConfiguration,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
- accum_pipeline:
- Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
+ partial_pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
+ accum_pipeline: Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
accumulator_buffer_bytes: usize,
}
impl Kernel {
pub fn new(data_type: DataType) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for Split-K: {:?}",
- data_type
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for Split-K: {:?}", data_type)));
}
Ok(Self {
data_type,
@@ -112,10 +99,7 @@ impl Kernel {
DataType::F16 => Ok("float16"),
DataType::BF16 => Ok("bfloat16"),
DataType::F32 => Ok("float32"),
- _ => Err(MTLError::Generic(format!(
- "Unsupported dtype for Split-K: {:?}",
- self.data_type
- ))),
+ _ => Err(MTLError::Generic(format!("Unsupported dtype for Split-K: {:?}", self.data_type))),
}
}
@@ -129,9 +113,7 @@ impl Kernel {
) -> Result<String, MTLError> {
let in_name = self.steel_type_name()?;
let out_name = self.splitk_partial_out_name();
- let transpose_suffix =
- transpose_configuration(config.transpose_a, config.transpose_b)
- .as_str();
+ let transpose_suffix = transpose_configuration(config.transpose_a, config.transpose_b).as_str();
let mn_tag = if config.mn_aligned {
"taligned"
} else {
@@ -159,19 +141,14 @@ impl Kernel {
fn accum_kernel_name(&self) -> Result<String, MTLError> {
let out_name = self.steel_type_name()?;
- Ok(format!(
- "steel_gemm_splitk_accum_{}_{}",
- out_name,
- self.splitk_partial_out_name()
- ))
+ Ok(format!("steel_gemm_splitk_accum_{}_{}", out_name, self.splitk_partial_out_name()))
}
fn get_partial_pipeline(
&mut self,
mtl: &MTLContext,
config: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if !self.partial_pipelines.contains_key(config) {
let name = self.partial_kernel_name(config)?;
let ps = mtl.compute_pipeline_state(&name, None)?;
@@ -183,8 +160,7 @@ impl Kernel {
fn get_accum_pipeline(
&mut self,
mtl: &MTLContext,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if self.accum_pipeline.is_none() {
let name = self.accum_kernel_name()?;
let ps = mtl.compute_pipeline_state(&name, None)?;
@@ -219,26 +195,17 @@ impl Kernel {
descriptor: &DispatchDescriptor,
) -> Result<bool, MTLError> {
self.ensure_accumulator_buffer(context, descriptor.accumulator_bytes);
- let accumulator_buffer = self
- .accumulator_buffer
- .as_ref()
- .cloned()
- .expect("Accumulator buffer must be initialized");
+ let accumulator_buffer =
+ self.accumulator_buffer.as_ref().cloned().expect("Accumulator buffer must be initialized");
- let partial_pipeline_state = self.get_partial_pipeline(
- context,
- &descriptor.pipeline_configuration,
- )?;
+ let partial_pipeline_state = self.get_partial_pipeline(context, &descriptor.pipeline_configuration)?;
encoder.set_compute_pipeline_state(partial_pipeline_state);
encoder.set_buffer(Some(arguments.a), arguments.a_offset as usize, 0);
encoder.set_buffer(Some(arguments.b), 0, 1);
encoder.set_buffer(Some(&accumulator_buffer), 0, 2);
encoder.set_value(&descriptor.params, 3);
- encoder.dispatch_threadgroups(
- descriptor.partial_threadgroups,
- descriptor.partial_threads_per_threadgroup,
- );
+ encoder.dispatch_threadgroups(descriptor.partial_threadgroups, descriptor.partial_threads_per_threadgroup);
let accum_pipeline_state = self.get_accum_pipeline(context)?;
encoder.set_compute_pipeline_state(accum_pipeline_state);
@@ -246,16 +213,12 @@ impl Kernel {
encoder.set_buffer(Some(arguments.d), 0, 1);
let partition_count = descriptor.partition_count;
- let output_elements_per_partition =
- descriptor.output_elements_per_partition;
+ let output_elements_per_partition = descriptor.output_elements_per_partition;
encoder.set_value(&partition_count, 2);
encoder.set_value(&output_elements_per_partition, 3);
encoder.set_value(&arguments.ldd, 4);
- encoder.dispatch_threads(
- descriptor.accum_total_threads,
- descriptor.accum_threads_per_threadgroup,
- );
+ encoder.dispatch_threads(descriptor.accum_total_threads, descriptor.accum_threads_per_threadgroup);
Ok(false)
}
@@ -265,15 +228,10 @@ impl Kernel {
mtl: &MTLContext,
required_bytes: usize,
) {
- if required_bytes <= self.accumulator_buffer_bytes
- && self.accumulator_buffer.is_some()
- {
+ if required_bytes <= self.accumulator_buffer_bytes && self.accumulator_buffer.is_some() {
return;
}
- self.accumulator_buffer = Some(
- mtl.create_buffer(required_bytes)
- .expect("Failed to create accumulator buffer"),
- );
+ self.accumulator_buffer = Some(mtl.create_buffer(required_bytes).expect("Failed to create accumulator buffer"));
self.accumulator_buffer_bytes = required_bytes;
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
index ee493532d..bfd29a572 100644
--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
@@ -8,9 +8,8 @@ use crate::{
use objc2::rc::Retained;
use crate::backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandEncoderExt,
- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLSize,
- ProtocolObject, error::MTLError, image::Image,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandEncoderExt, MTLComputeCommandEncoder,
+ MTLComputePipelineState, MTLContext, MTLSize, ProtocolObject, error::MTLError, image::Image,
metal_extensions::ComputeEncoderDispatch,
};
@@ -40,8 +39,7 @@ pub struct ScalePadNormalizeImage {
impl ScalePadNormalizeImage {
pub fn new(context: &MTLContext) -> Result<Self, MTLError> {
let function_name = "scalePadNormalizeImage";
- let pipeline_state =
- context.compute_pipeline_state(&function_name, None)?;
+ let pipeline_state = context.compute_pipeline_state(&function_name, None)?;
Ok(Self {
pipeline_state,
})
@@ -54,9 +52,8 @@ impl ScalePadNormalizeImage {
image_params_ptr: *const std::ffi::c_void,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
) {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
compute_encoder.set_label(Some("ScalePadNormalizeImageEncoder"));
compute_encoder.set_compute_pipeline_state(&self.pipeline_state);
@@ -101,9 +98,7 @@ impl EncodableBlock<Metal> for ScalePadNormalizeImage {
_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
_parameters: &EncodingParameters<Metal>,
) {
- unreachable!(
- "ScalePadNormalizeImage does not support shared compute encoder"
- );
+ unreachable!("ScalePadNormalizeImage does not support shared compute encoder");
}
}
@@ -115,8 +110,7 @@ pub struct ExtractImagePatches {
impl ExtractImagePatches {
pub fn new(context: &MTLContext) -> Result<Self, MTLError> {
let function_name = "extractImagePatches";
- let pipeline_state =
- context.compute_pipeline_state(&function_name, None)?;
+ let pipeline_state = context.compute_pipeline_state(&function_name, None)?;
Ok(Self {
pipeline_state,
})
@@ -129,14 +123,12 @@ impl ExtractImagePatches {
patch_params_ptr: *const std::ffi::c_void,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
) {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
compute_encoder.set_label(Some("ExtractImagePatchesEncoder"));
compute_encoder.set_compute_pipeline_state(&self.pipeline_state);
- compute_encoder
- .set_texture(Some(padded_normalized_image.texture_ref()), 0);
+ compute_encoder.set_texture(Some(padded_normalized_image.texture_ref()), 0);
compute_encoder.set_buffer(Some(output_buffer_mtl), 0, 0);
unsafe {
compute_encoder.set_bytes(
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
index a6eb8f90e..e0d2d9146 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
@@ -5,9 +5,8 @@ use crate::{
backends::{
common::kernel::MlpGateActMulKernel,
metal::{
- MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLDataType,
- MTLError, MTLFunctionConstantValues, ProtocolObject, Retained,
- kernel::dsl::MlpGateActMulMetalKernel,
+ MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLDataType, MTLError, MTLFunctionConstantValues,
+ ProtocolObject, Retained, kernel::dsl::MlpGateActMulMetalKernel,
},
},
config::Activation,
@@ -60,9 +59,7 @@ impl MlpFusedConfig {
}
/// Create function constants for MLP fused matmul
- pub fn make_function_constants(
- &self
- ) -> Retained<MTLFunctionConstantValues> {
+ pub fn make_function_constants(&self) -> Retained<MTLFunctionConstantValues> {
let fcv = MTLFunctionConstantValues::new();
let fused = true;
fcv.set_constant_value_type_at_index(
@@ -86,15 +83,10 @@ impl MlpFusedConfig {
}
/// Create function constants for non-fused (standard) matmul
-pub fn make_non_fused_function_constants() -> Retained<MTLFunctionConstantValues>
-{
+pub fn make_non_fused_function_constants() -> Retained<MTLFunctionConstantValues> {
let fcv = MTLFunctionConstantValues::new();
let fused = false;
- fcv.set_constant_value_type_at_index(
- NonNull::from(&fused).cast(),
- MTLDataType::Bool,
- MLP_FUSED_FC_INDEX as usize,
- );
+ fcv.set_constant_value_type_at_index(NonNull::from(&fused).cast(), MTLDataType::Bool, MLP_FUSED_FC_INDEX as usize);
fcv
}
@@ -135,14 +127,7 @@ impl MlpGateActMulEncodable {
panic!("Identity activation is not supported for kernel")
},
};
- self.kernel.encode(
- fused_up,
- hidden,
- self.hidden_dim as i32,
- m,
- act_type,
- encoder,
- );
+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, encoder);
Ok(())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs
index 3cd13a91d..da793f6e6 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs
@@ -1,6 +1,4 @@
-use crate::backends::metal::{
- MTLBuffer, ProtocolObject, kernel::mlp::MlpActivationType,
-};
+use crate::backends::metal::{MTLBuffer, ProtocolObject, kernel::mlp::MlpActivationType};
#[derive(Debug)]
pub struct MlpFusedArguments<'a> {
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs
index fca33bf46..9cbedf896 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs
@@ -32,9 +32,7 @@ impl MlpFusedDispatchDescriptor {
pub fn variant(&self) -> MlpFusedKernelVariant {
match self {
MlpFusedDispatchDescriptor::Gemv(_) => MlpFusedKernelVariant::Gemv,
- MlpFusedDispatchDescriptor::SplitK(_) => {
- MlpFusedKernelVariant::SplitK
- },
+ MlpFusedDispatchDescriptor::SplitK(_) => MlpFusedKernelVariant::SplitK,
MlpFusedDispatchDescriptor::Gemm(_) => MlpFusedKernelVariant::Gemm,
}
}
@@ -47,15 +45,11 @@ pub(crate) fn choose_dispatch_descriptor(
arguments: &MlpFusedArguments,
) -> Result<MlpFusedDispatchDescriptor, MTLError> {
if weights_transposed {
- if let Some(descriptor) =
- gemv::DispatchDescriptor::try_new(context, data_type, arguments)?
- {
+ if let Some(descriptor) = gemv::DispatchDescriptor::try_new(context, data_type, arguments)? {
return Ok(MlpFusedDispatchDescriptor::Gemv(descriptor));
}
- if let Some(descriptor) =
- split_k::DispatchDescriptor::try_new(context, data_type, arguments)?
- {
+ if let Some(descriptor) = split_k::DispatchDescriptor::try_new(context, data_type, arguments)? {
return Ok(MlpFusedDispatchDescriptor::SplitK(descriptor));
}
}
@@ -74,11 +68,5 @@ pub fn determine_kernel_variant(
weights_transposed: bool,
arguments: &MlpFusedArguments,
) -> Result<MlpFusedKernelVariant, MTLError> {
- choose_dispatch_descriptor(
- context,
- data_type,
- weights_transposed,
- arguments,
- )
- .map(|d| d.variant())
+ choose_dispatch_descriptor(context, data_type, weights_transposed, arguments).map(|d| d.variant())
}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs
index 139d8d89a..b8cc84c3e 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs
@@ -1,14 +1,9 @@
-use super::{
- pipeline_configuration::PipelineConfiguration,
- tile_configuration::select_tile_configuration,
-};
+use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};
use crate::{
DataType,
backends::metal::{
MTLContext, MTLError, MTLSize,
- kernel::{
- matmul::common::GEMMParams, mlp_fused::common::MlpFusedArguments,
- },
+ kernel::{matmul::common::GEMMParams, mlp_fused::common::MlpFusedArguments},
},
};
@@ -28,18 +23,13 @@ impl DispatchDescriptor {
weights_transposed: bool,
arguments: &MlpFusedArguments,
) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MLP fused GEMM: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMM: {data_type:?}")));
}
- let tile =
- select_tile_configuration(arguments.batch, arguments.hidden_dim);
+ let tile = select_tile_configuration(arguments.batch, arguments.hidden_dim);
- let mn_aligned = arguments.batch % tile.block_rows == 0
- && arguments.hidden_dim % tile.block_cols == 0;
+ let mn_aligned = arguments.batch % tile.block_rows == 0 && arguments.hidden_dim % tile.block_cols == 0;
let k_aligned = arguments.input_dim % tile.block_depth == 0;
let pipeline_configuration = PipelineConfiguration {
@@ -51,10 +41,8 @@ impl DispatchDescriptor {
};
let tiles_m = (arguments.batch + tile.block_rows - 1) / tile.block_rows;
- let tiles_n =
- (arguments.hidden_dim + tile.block_cols - 1) / tile.block_cols;
- let gemm_k_iterations =
- (arguments.input_dim + tile.block_depth - 1) / tile.block_depth;
+ let tiles_n = (arguments.hidden_dim + tile.block_cols - 1) / tile.block_cols;
+ let gemm_k_iterations = (arguments.input_dim + tile.block_depth - 1) / tile.block_depth;
let params = GEMMParams {
M: arguments.batch,
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs
index 002bc9a42..2864e4073 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs
@@ -1,25 +1,18 @@
use std::{collections::HashMap, ptr::NonNull};
-use super::{
- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,
-};
+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};
use crate::{
DataType,
backends::metal::{
- ComputeEncoderSetValue, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError,
- MTLFunctionConstantValues, ProtocolObject, Retained,
- kernel::mlp_fused::common::MlpFusedArguments,
+ ComputeEncoderSetValue, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
+ MTLFunctionConstantValues, ProtocolObject, Retained, kernel::mlp_fused::common::MlpFusedArguments,
},
};
pub struct Kernel {
data_type: DataType,
weights_transposed: bool,
- pipelines: HashMap<
- PipelineConfiguration,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
+ pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
}
impl Kernel {
@@ -27,11 +20,8 @@ impl Kernel {
data_type: DataType,
weights_transposed: bool,
) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MLP fused GEMM: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMM: {data_type:?}")));
}
Ok(Self {
data_type,
@@ -86,8 +76,7 @@ impl Kernel {
&mut self,
context: &MTLContext,
configuration: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if !self.pipelines.contains_key(configuration) {
let kernel_name = self.kernel_name(configuration);
@@ -99,10 +88,7 @@ impl Kernel {
52,
);
- let pipeline_state = context.compute_pipeline_state(
- &kernel_name,
- Some(&function_constants),
- )?;
+ let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;
self.pipelines.insert(configuration.clone(), pipeline_state);
}
Ok(self.pipelines.get(configuration).unwrap())
@@ -115,27 +101,17 @@ impl Kernel {
arguments: &MlpFusedArguments,
descriptor: &DispatchDescriptor,
) -> Result<(), MTLError> {
- let pipeline_state = self.get_or_compile_pipeline(
- context,
- &descriptor.pipeline_configuration,
- )?;
+ let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;
encoder.set_compute_pipeline_state(pipeline_state);
- encoder.set_buffer(
- Some(arguments.input),
- arguments.input_offset as usize,
- 0,
- );
+ encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 0);
encoder.set_buffer(Some(arguments.weights), 0, 1);
encoder.set_buffer(Some(arguments.output), 0, 2);
encoder.set_value(&descriptor.params, 3);
encoder.set_value(&descriptor.hidden_dim, 10);
- encoder.dispatch_threadgroups(
- descriptor.threadgroups,
- descriptor.threads_per_threadgroup,
- );
+ encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);
Ok(())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs
index f88969d08..b560342ce 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs
@@ -1,12 +1,7 @@
-use super::pipeline_configuration::{
- PipelineConfiguration, select_configuration,
-};
+use super::pipeline_configuration::{PipelineConfiguration, select_configuration};
use crate::{
DataType,
- backends::metal::{
- MTLContext, MTLError, MTLSize,
- kernel::mlp_fused::common::MlpFusedArguments,
- },
+ backends::metal::{MTLContext, MTLError, MTLSize, kernel::mlp_fused::common::MlpFusedArguments},
};
#[derive(Debug, Clone)]
@@ -27,35 +22,23 @@ impl DispatchDescriptor {
data_type: DataType,
arguments: &MlpFusedArguments,
) -> Result<Option<Self>, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MLP fused GEMV: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMV: {data_type:?}")));
}
if arguments.batch != 1 {
return Ok(None);
}
- let pipeline_configuration =
- select_configuration(arguments.hidden_dim, arguments.activation);
+ let pipeline_configuration = select_configuration(arguments.hidden_dim, arguments.activation);
- let output_elements_per_threadgroup =
- pipeline_configuration.output_elements_per_threadgroup();
- let threadgroup_count_x =
- ((arguments.hidden_dim as u32 + output_elements_per_threadgroup
- - 1)
- / output_elements_per_threadgroup) as u64;
+ let output_elements_per_threadgroup = pipeline_configuration.output_elements_per_threadgroup();
+ let threadgroup_count_x = ((arguments.hidden_dim as u32 + output_elements_per_threadgroup - 1)
+ / output_elements_per_threadgroup) as u64;
let threadgroup_count_z = arguments.batch_count.max(1) as u64;
- let threadgroups = MTLSize::new(
- threadgroup_count_x as usize,
- 1,
- threadgroup_count_z as usize,
- );
- let threads_per_threadgroup =
- pipeline_configuration.threads_per_threadgroup();
+ let threadgroups = MTLSize::new(threadgroup_count_x as usize, 1, threadgroup_count_z as usize);
+ let threads_per_threadgroup = pipeline_configuration.threads_per_threadgroup();
Ok(Some(Self {
pipeline_configuration,
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs
index e113279eb..e065bd727 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs
@@ -1,15 +1,11 @@
use std::{collections::HashMap, ptr::NonNull};
-use super::{
- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,
-};
+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};
use crate::{
DataType,
backends::metal::{
- ComputeEncoderSetValue, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError,
- MTLFunctionConstantValues, ProtocolObject, Retained,
- kernel::mlp_fused::common::MlpFusedArguments,
+ ComputeEncoderSetValue, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
+ MTLFunctionConstantValues, ProtocolObject, Retained, kernel::mlp_fused::common::MlpFusedArguments,
},
};
@@ -22,9 +18,7 @@ fn kernel_name(
DataType::BF16 => "bfloat16",
DataType::F32 => "float32",
_ => {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for MLP fused GEMV: {data_type:?}"
- )));
+ return Err(MTLError::Generic(format!("Unsupported data type for MLP fused GEMV: {data_type:?}")));
},
};
@@ -42,19 +36,13 @@ fn kernel_name(
pub struct Kernel {
data_type: DataType,
- pipelines: HashMap<
- PipelineConfiguration,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
+ pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
}
impl Kernel {
pub fn new(data_type: DataType) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MLP fused GEMV: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMV: {data_type:?}")));
}
Ok(Self {
data_type,
@@ -66,8 +54,7 @@ impl Kernel {
&mut self,
context: &MTLContext,
configuration: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if !self.pipelines.contains_key(configuration) {
let kernel_name = kernel_name(self.data_type, configuration)?;
@@ -79,10 +66,7 @@ impl Kernel {
52,
);
- let pipeline_state = context.compute_pipeline_state(
- &kernel_name,
- Some(&function_constants),
- )?;
+ let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;
self.pipelines.insert(*configuration, pipeline_state);
}
Ok(self.pipelines.get(configuration).unwrap())
@@ -95,18 +79,11 @@ impl Kernel {
arguments: &MlpFusedArguments,
descriptor: &DispatchDescriptor,
) -> Result<(), MTLError> {
- let pipeline_state = self.get_or_compile_pipeline(
- context,
- &descriptor.pipeline_configuration,
- )?;
+ let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;
encoder.set_compute_pipeline_state(pipeline_state);
encoder.set_buffer(Some(arguments.weights), 0, 0);
- encoder.set_buffer(
- Some(arguments.input),
- arguments.input_offset as usize,
- 1,
- );
+ encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 1);
encoder.set_buffer(Some(arguments.output), 0, 3);
encoder.set_value(&descriptor.input_dim, 4);
@@ -115,10 +92,7 @@ impl Kernel {
encoder.set_value(&descriptor.vector_batch_stride, 11);
encoder.set_value(&descriptor.matrix_batch_stride, 12);
- encoder.dispatch_threadgroups(
- descriptor.threadgroups,
- descriptor.threads_per_threadgroup,
- );
+ encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);
Ok(())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs
index 280e1cbf3..91130ed54 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs
@@ -13,17 +13,11 @@ pub struct PipelineConfiguration {
impl PipelineConfiguration {
pub fn output_elements_per_threadgroup(&self) -> u32 {
- self.threadgroup_rows
- * self.threads_per_simdgroup_row
- * self.elements_per_thread_row
+ self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row
}
pub fn threads_per_threadgroup(&self) -> MTLSize {
- MTLSize::new(
- 32,
- self.threadgroup_cols as usize,
- self.threadgroup_rows as usize,
- )
+ MTLSize::new(32, self.threadgroup_cols as usize, self.threadgroup_rows as usize)
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs
index 0a951b099..013e3f5fd 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs
@@ -2,9 +2,7 @@ use crate::backends::metal::{MTLComputeCommandEncoder, ProtocolObject};
use super::{
common::MlpFusedArguments,
- dispatch_descriptor::{
- MlpFusedDispatchDescriptor, choose_dispatch_descriptor,
- },
+ dispatch_descriptor::{MlpFusedDispatchDescriptor, choose_dispatch_descriptor},
gemm::GemmKernel,
gemv::GemvKernel,
split_k::SplitKKernel,
@@ -27,11 +25,8 @@ impl MlpFusedKernel {
data_type: DataType,
weights_transposed: bool,
) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MlpFusedKernel: {data_type:?}"
- )));
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!("Unsupported dtype for MlpFusedKernel: {data_type:?}")));
}
Ok(Self {
@@ -45,8 +40,7 @@ impl MlpFusedKernel {
fn get_or_create_gemm(&mut self) -> Result<&mut GemmKernel, MTLError> {
if self.gemm.is_none() {
- self.gemm =
- Some(GemmKernel::new(self.data_type, self.weights_transposed)?);
+ self.gemm = Some(GemmKernel::new(self.data_type, self.weights_transposed)?);
}
Ok(self.gemm.as_mut().unwrap())
}
@@ -79,8 +73,7 @@ impl MlpFusedKernel {
},
MlpFusedDispatchDescriptor::SplitK(descriptor) => {
let splitk = self.get_or_create_splitk()?;
- splitk
- .encode_descriptor(context, encoder, arguments, descriptor)
+ splitk.encode_descriptor(context, encoder, arguments, descriptor)
},
MlpFusedDispatchDescriptor::Gemm(descriptor) => {
let gemm = self.get_or_create_gemm()?;
@@ -95,18 +88,8 @@ impl MlpFusedKernel {
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
arguments: &MlpFusedArguments,
) -> Result<(), MTLError> {
- let descriptor = choose_dispatch_descriptor(
- context,
- self.data_type,
- self.weights_transposed,
- arguments,
- )?;
+ let descriptor = choose_dispatch_descriptor(context, self.data_type, self.weights_transposed, arguments)?;
- self.encode_dispatch_descriptor(
- context,
- encoder,
- arguments,
- &descriptor,
- )
+ self.encode_dispatch_descriptor(context, encoder, arguments, &descriptor)
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs
index e668ca697..f503655f3 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs
@@ -6,9 +6,7 @@ mod kernel;
mod split_k;
pub use common::MlpFusedArguments;
-pub use dispatch_descriptor::{
- MlpFusedKernelVariant, determine_kernel_variant,
-};
+pub use dispatch_descriptor::{MlpFusedKernelVariant, determine_kernel_variant};
pub use gemm::GemmKernel;
pub use gemv::GemvKernel;
pub use kernel::MlpFusedKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs
index a98eac2ea..574efb5b9 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs
@@ -1,15 +1,9 @@
-use super::{
- pipeline_configuration::PipelineConfiguration,
- tile_configuration::select_tile_configuration,
-};
+use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};
use crate::{
DataType,
backends::metal::{
MTLContext, MTLError, MTLSize,
- kernel::{
- matmul::common::GEMMSpiltKMlpFusedParams,
- mlp_fused::common::MlpFusedArguments,
- },
+ kernel::{matmul::common::GEMMSpiltKMlpFusedParams, mlp_fused::common::MlpFusedArguments},
},
};
@@ -37,11 +31,7 @@ impl DispatchDescriptor {
return Ok(None);
}
- if !should_use_splitk(
- arguments.batch,
- arguments.hidden_dim,
- arguments.input_dim,
- ) {
+ if !should_use_splitk(arguments.batch, arguments.hidden_dim, arguments.input_dim) {
return Ok(None);
}
@@ -51,8 +41,7 @@ impl DispatchDescriptor {
let tile = select_tile_configuration(batch, hidden_dim);
let partition_count = compute_partition_count(input_dim);
- let mn_aligned =
- (batch % tile.tile_rows) == 0 && (hidden_dim % tile.tile_cols) == 0;
+ let mn_aligned = (batch % tile.tile_rows) == 0 && (hidden_dim % tile.tile_cols) == 0;
let k_aligned = (input_dim % tile.tile_depth) == 0;
let pipeline_configuration = PipelineConfiguration {
@@ -69,10 +58,8 @@ impl DispatchDescriptor {
let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;
let output_elements_per_partition = batch * hidden_dim;
- let accumulator_element_count =
- partition_count * output_elements_per_partition;
- let accumulator_bytes =
- (accumulator_element_count as usize) * std::mem::size_of::<f32>();
+ let accumulator_element_count = partition_count * output_elements_per_partition;
+ let accumulator_bytes = (accumulator_element_count as usize) * std::mem::size_of::<f32>();
let params = GEMMSpiltKMlpFusedParams {
M: batch,
@@ -90,24 +77,12 @@ impl DispatchDescriptor {
hidden_dim,
};
- let partial_threads_per_threadgroup = MTLSize::new(
- 32,
- tile.warps_per_col as usize,
- tile.warps_per_row as usize,
- );
- let partial_threadgroups = MTLSize::new(
- tile_count_n as usize,
- tile_count_m as usize,
- partition_count as usize,
- );
+ let partial_threads_per_threadgroup =
+ MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);
+ let partial_threadgroups = MTLSize::new(tile_count_n as usize, tile_count_m as usize, partition_count as usize);
- let accum_total_threads =
- MTLSize::new(hidden_dim as usize, batch as usize, 1);
- let accum_threads_per_threadgroup = MTLSize::new(
- 16.min(hidden_dim as usize),
- 16.min(batch as usize),
- 1,
- );
+ let accum_total_threads = MTLSize::new(hidden_dim as usize, batch as usize, 1);
+ let accum_threads_per_threadgroup = MTLSize::new(16.min(hidden_dim as usize), 16.min(batch as usize), 1);
Ok(Some(Self {
pipeline_configuration,
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs
index dced3cfb7..326a3c4da 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs
@@ -1,33 +1,22 @@
use std::{collections::HashMap, ptr::NonNull};
-use super::{
- DispatchDescriptor, pipeline_configuration::PipelineConfiguration,
-};
+use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};
use crate::{
DataType,
backends::{
common::Context,
metal::{
- ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError,
+ ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
MTLFunctionConstantValues, ProtocolObject, Retained,
- kernel::{
- mlp::MlpActivationType, mlp_fused::common::MlpFusedArguments,
- },
+ kernel::{mlp::MlpActivationType, mlp_fused::common::MlpFusedArguments},
},
},
};
pub struct Kernel {
data_type: DataType,
- partial_pipelines: HashMap<
- PipelineConfiguration,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
- accum_pipelines: HashMap<
- MlpActivationType,
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- >,
+ partial_pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
+ accum_pipelines: HashMap<MlpActivationType, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
up_accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
gate_accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
accumulator_buffer_bytes: usize,
@@ -36,9 +25,7 @@ pub struct Kernel {
impl Kernel {
pub fn new(data_type: DataType) -> Result<Self, MTLError> {
if !matches!(data_type, DataType::F16 | DataType::BF16) {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MLP fused Split-K: {data_type:?}"
- )));
+ return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused Split-K: {data_type:?}")));
}
Ok(Self {
data_type,
@@ -54,10 +41,7 @@ impl Kernel {
match self.data_type {
DataType::F16 => Ok("float16"),
DataType::BF16 => Ok("bfloat16"),
- _ => Err(MTLError::Generic(format!(
- "Unsupported dtype for MLP Fused Split-K: {:?}",
- self.data_type
- ))),
+ _ => Err(MTLError::Generic(format!("Unsupported dtype for MLP Fused Split-K: {:?}", self.data_type))),
}
}
@@ -98,14 +82,11 @@ impl Kernel {
&mut self,
context: &MTLContext,
configuration: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if !self.partial_pipelines.contains_key(configuration) {
let kernel_name = self.partial_kernel_name(configuration)?;
- let pipeline_state =
- context.compute_pipeline_state(&kernel_name, None)?;
- self.partial_pipelines
- .insert(configuration.clone(), pipeline_state);
+ let pipeline_state = context.compute_pipeline_state(&kernel_name, None)?;
+ self.partial_pipelines.insert(configuration.clone(), pipeline_state);
}
Ok(self.partial_pipelines.get(configuration).unwrap())
}
@@ -114,8 +95,7 @@ impl Kernel {
&mut self,
context: &MTLContext,
activation: MlpActivationType,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
if !self.accum_pipelines.contains_key(&activation) {
let kernel_name = self.accum_kernel_name()?;
let function_constants = MTLFunctionConstantValues::new();
@@ -125,10 +105,7 @@ impl Kernel {
metal::MTLDataType::UInt,
52,
);
- let pipeline_state = context.compute_pipeline_state(
- &kernel_name,
- Some(&function_constants),
- )?;
+ let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;
self.accum_pipelines.insert(activation, pipeline_state);
}
Ok(self.accum_pipelines.get(&activation).unwrap())
@@ -146,8 +123,7 @@ impl Kernel {
return;
}
self.up_accumulator_buffer = context.create_buffer(required_bytes).ok();
- self.gate_accumulator_buffer =
- context.create_buffer(required_bytes).ok();
+ self.gate_accumulator_buffer = context.create_buffer(required_bytes).ok();
self.accumulator_buffer_bytes = required_bytes;
}
@@ -160,10 +136,7 @@ impl Kernel {
) -> Result<(), MTLError> {
self.ensure_accumulator_buffers(context, descriptor.accumulator_bytes);
self.get_partial_pipeline(context, &descriptor.pipeline_configuration)?;
- self.get_accum_pipeline(
- context,
- descriptor.pipeline_configuration.activation,
- )?;
+ self.get_accum_pipeline(context, descriptor.pipeline_configuration.activation)?;
let partial_pipeline_state = self
.partial_pipelines
@@ -173,29 +146,18 @@ impl Kernel {
.accum_pipelines
.get(&descriptor.pipeline_configuration.activation)
.expect("Accum pipeline must be initialized");
- let up_accumulator_buffer = self
- .up_accumulator_buffer
- .as_ref()
- .expect("Up accumulator buffer must be initialized");
- let gate_accumulator_buffer = self
- .gate_accumulator_buffer
- .as_ref()
- .expect("Gate accumulator buffer must be initialized");
+ let up_accumulator_buffer =
+ self.up_accumulator_buffer.as_ref().expect("Up accumulator buffer must be initialized");
+ let gate_accumulator_buffer =
+ self.gate_accumulator_buffer.as_ref().expect("Gate accumulator buffer must be initialized");
encoder.set_compute_pipeline_state(partial_pipeline_state);
- encoder.set_buffer(
- Some(arguments.input),
- arguments.input_offset as usize,
- 0,
- );
+ encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 0);
encoder.set_buffer(Some(arguments.weights), 0, 1);
encoder.set_buffer(Some(up_accumulator_buffer), 0, 2);
encoder.set_buffer(Some(gate_accumulator_buffer), 0, 3);
encoder.set_value(&descriptor.params, 4);
- encoder.dispatch_threadgroups(
- descriptor.partial_threadgroups,
- descriptor.partial_threads_per_threadgroup,
- );
+ encoder.dispatch_threadgroups(descriptor.partial_threadgroups, descriptor.partial_threads_per_threadgroup);
encoder.set_compute_pipeline_state(accum_pipeline_state);
encoder.set_buffer(Some(up_accumulator_buffer), 0, 0);
@@ -205,10 +167,7 @@ impl Kernel {
encoder.set_value(&descriptor.output_elements_per_partition, 4);
encoder.set_value(&descriptor.ldd, 5);
- encoder.dispatch_threads(
- descriptor.accum_total_threads,
- descriptor.accum_threads_per_threadgroup,
- );
+ encoder.dispatch_threads(descriptor.accum_total_threads, descriptor.accum_threads_per_threadgroup);
Ok(())
}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index a24d4084a..c9bd68bed 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -3,15 +3,14 @@ mod data_type;
pub mod dsl {
include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
}
-pub(super) const MTLB: &[u8] =
- include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
+pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
pub mod matmul;
pub mod media_kernels;
pub mod mlp;
pub use mlp::{
- MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX,
- MlpActivationType, MlpFusedConfig, make_non_fused_function_constants,
+ MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX, MlpActivationType, MlpFusedConfig,
+ make_non_fused_function_constants,
};
pub mod mlp_fused;
pub mod moe;
@@ -23,41 +22,27 @@ pub mod ssm;
pub mod token_copy;
pub use attention::{
- AttentionError, AttentionKernel, AttentionKernelVariant,
- AttentionSinglePassArguments, AttentionTwoPassArguments,
+ AttentionError, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments, AttentionTwoPassArguments,
KVCacheUpdateArguments,
};
pub use data_type::KernelDataType;
pub use matmul::{MatmulArguments, MatmulKernel};
pub use moe::{
- MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError,
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,
- MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,
- MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments,
- MoeScatterError, MoeScatterKernels, MoeScatterWithMapArguments,
-};
-pub use rms_norm::{
- QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormError,
- RMSNormKernel, RMSNormKernelType,
+ MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,
+ MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,
+ MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,
+ MoeScatterWithMapArguments,
};
+pub use rms_norm::{QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType};
pub use rope::{RopeError, RopeKernel, RopeKernelArguments};
-pub use short_conv::{
- ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError,
- ShortConvPrefillArguments,
-};
+pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError, ShortConvPrefillArguments};
pub use ssm::{
- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
- SSDPrefillArguments, SSDPrefillMode, SSMKernelError,
+ Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,
};
pub use token_copy::TokenCopyKernel;
pub mod media {
- pub use super::media_kernels::{
- ExtractImagePatches, ScalePadNormalizeImage,
- };
+ pub use super::media_kernels::{ExtractImagePatches, ScalePadNormalizeImage};
}
-pub use media_kernels::{
- ExtractImagePatches, ImageParameters, PatchParameters,
- ScalePadNormalizeImage,
-};
+pub use media_kernels::{ExtractImagePatches, ImageParameters, PatchParameters, ScalePadNormalizeImage};
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
index b0293716f..1c0ff82a0 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
@@ -2,13 +2,12 @@ use std::{mem::size_of, ptr::NonNull};
use super::{dtype_index, dtype_suffix};
use crate::backends::metal::{
- KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,
- MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLContext, MTLDataType, MTLError, MTLFunctionConstantValues, MTLSize,
- NSRange, ProtocolObject, Retained,
+ KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
+ MTLComputePipelineState, MTLContext, MTLDataType, MTLError, MTLFunctionConstantValues, MTLSize, NSRange,
+ ProtocolObject, Retained,
kernel::moe::tiles_map::{
- MoeTileCountsArguments, MoeTileDispatchArguments,
- MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,
+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,
+ MoeTileScanArguments,
},
};
@@ -29,15 +28,15 @@ pub enum MoeExpertsError {
pub struct MoeExpertsArguments<'a> {
pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - permuted input
pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1] - expert segment offsets
- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff, d_model] - transposed up projection weights
- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model, d_ff] - transposed down projection weights
- pub y_partial: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - output buffer
- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff] - up projection biases
- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model] - down projection biases
- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]
+ pub w13_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff, d_model] - transposed up projection weights
+ pub w2_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model, d_ff] - transposed down projection weights
+ pub y_partial: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - output buffer
+ pub up_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff] - up projection biases
+ pub down_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model] - down projection biases
+ pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]
pub tile_row_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [max_tiles * 3]
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [2]
+ pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [max_tiles * 3]
+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [2]
pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]
pub num_tiles_n: usize,
pub t: usize,
@@ -56,9 +55,8 @@ pub struct MoeExpertsArguments<'a> {
pub struct MoeExpertsTwoPassPrefillKernel {
tile_map: MoeTileMapKernels,
- pass_a_indirect:
- Vec<Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>>, // [gate][dtype]
- pass_b_indirect: Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>, // [dtype]
+ pass_a_indirect: Vec<Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>>, // [gate][dtype]
+ pass_b_indirect: Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>, // [dtype]
}
#[derive(Debug)]
@@ -93,44 +91,27 @@ pub struct MoeExpertsTwoPassArguments<'a> {
impl MoeExpertsTwoPassPrefillKernel {
pub fn new(ctx: &MTLContext) -> Result<Self, MoeExpertsError> {
- let dtypes = [
- KernelDataType::Float16,
- KernelDataType::BFloat16,
- KernelDataType::Float32,
- ];
+ let dtypes = [KernelDataType::Float16, KernelDataType::BFloat16, KernelDataType::Float32];
let mut pass_a_indirect = vec![Vec::with_capacity(dtypes.len()); 4];
for gate in 0u32..4u32 {
for dtype in &dtypes {
let dtype_suffix = dtype_suffix(*dtype);
let fcv = MTLFunctionConstantValues::new();
- fcv.set_constant_value_type_at_index(
- NonNull::from(&gate).cast(),
- MTLDataType::UInt,
- 30,
- );
- let kernel_name = format!(
- "moe_two_pass_prefill_pass_a_indirect_{}",
- dtype_suffix
- );
+ fcv.set_constant_value_type_at_index(NonNull::from(&gate).cast(), MTLDataType::UInt, 30);
+ let kernel_name = format!("moe_two_pass_prefill_pass_a_indirect_{}", dtype_suffix);
let cache_key = format!("{}_gate_{}", kernel_name, gate);
- pass_a_indirect[gate as usize].push(
- ctx.compute_pipeline_state_cached(
- &cache_key,
- &kernel_name,
- Some(&fcv),
- )?,
- );
+ pass_a_indirect[gate as usize].push(ctx.compute_pipeline_state_cached(
+ &cache_key,
+ &kernel_name,
+ Some(&fcv),
+ )?);
}
}
let mut pass_b_indirect = Vec::with_capacity(dtypes.len());
for dtype in &dtypes {
let dtype_suffix = dtype_suffix(*dtype);
- let kernel_name = format!(
- "moe_two_pass_prefill_pass_b_indirect_{}",
- dtype_suffix
- );
- pass_b_indirect
- .push(ctx.compute_pipeline_state(&kernel_name, None)?);
+ let kernel_name = format!("moe_two_pass_prefill_pass_b_indirect_{}", dtype_suffix);
+ pass_b_indirect.push(ctx.compute_pipeline_state(&kernel_name, None)?);
}
Ok(Self {
tile_map: MoeTileMapKernels::new(ctx)?,
@@ -154,14 +135,8 @@ impl MoeExpertsTwoPassPrefillKernel {
KernelDataType::Float32 => 4,
};
let hidden_bytes = args.total_rows * args.d_ff * dtype_size;
- let blit_encoder = command_buffer
- .new_blit_command_encoder()
- .expect("Failed to create blit command encoder");
- blit_encoder.fill_buffer_range_value(
- args.hidden_buffer,
- NSRange::new(0, hidden_bytes),
- 0,
- );
+ let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");
+ blit_encoder.fill_buffer_range_value(args.hidden_buffer, NSRange::new(0, hidden_bytes), 0);
blit_encoder.end_encoding();
self.tile_map.encode_counts(
command_buffer,
@@ -214,9 +189,7 @@ impl MoeExpertsTwoPassPrefillKernel {
},
);
let pass_a_pipeline = &self.pass_a_indirect[gate_idx][dtype_idx];
- let encoder_a = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder_a = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
encoder_a.set_compute_pipeline_state(pass_a_pipeline);
encoder_a.set_buffer(Some(args.x_perm_buffer), 0, 0);
encoder_a.set_buffer(Some(args.expert_offsets), 0, 1);
@@ -224,60 +197,20 @@ impl MoeExpertsTwoPassPrefillKernel {
encoder_a.set_buffer(Some(args.up_biases), 0, 3);
encoder_a.set_buffer(Some(args.hidden_buffer), 0, 4);
unsafe {
- encoder_a.set_bytes(
- NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _),
- size_of::<u32>(),
- 5,
- );
- encoder_a.set_bytes(
- NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _),
- size_of::<u32>(),
- 6,
- );
- encoder_a.set_bytes(
- NonNull::new_unchecked(&e_u32 as *const _ as *mut _),
- size_of::<u32>(),
- 7,
- );
- encoder_a.set_bytes(
- NonNull::new_unchecked(
- &args.gate_clip_min as *const _ as *mut _,
- ),
- size_of::<f32>(),
- 8,
- );
- encoder_a.set_bytes(
- NonNull::new_unchecked(
- &args.gate_clip_max as *const _ as *mut _,
- ),
- size_of::<f32>(),
- 9,
- );
- encoder_a.set_bytes(
- NonNull::new_unchecked(&args.up_clip_min as *const _ as *mut _),
- size_of::<f32>(),
- 10,
- );
- encoder_a.set_bytes(
- NonNull::new_unchecked(&args.up_clip_max as *const _ as *mut _),
- size_of::<f32>(),
- 11,
- );
- encoder_a.set_bytes(
- NonNull::new_unchecked(&args.silu_alpha as *const _ as *mut _),
- size_of::<f32>(),
- 12,
- );
+ encoder_a.set_bytes(NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _), size_of::<u32>(), 5);
+ encoder_a.set_bytes(NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _), size_of::<u32>(), 6);
+ encoder_a.set_bytes(NonNull::new_unchecked(&e_u32 as *const _ as *mut _), size_of::<u32>(), 7);
+ encoder_a.set_bytes(NonNull::new_unchecked(&args.gate_clip_min as *const _ as *mut _), size_of::<f32>(), 8);
+ encoder_a.set_bytes(NonNull::new_unchecked(&args.gate_clip_max as *const _ as *mut _), size_of::<f32>(), 9);
+ encoder_a.set_bytes(NonNull::new_unchecked(&args.up_clip_min as *const _ as *mut _), size_of::<f32>(), 10);
+ encoder_a.set_bytes(NonNull::new_unchecked(&args.up_clip_max as *const _ as *mut _), size_of::<f32>(), 11);
+ encoder_a.set_bytes(NonNull::new_unchecked(&args.silu_alpha as *const _ as *mut _), size_of::<f32>(), 12);
}
encoder_a.set_buffer(Some(args.tile_map), 0, 13);
// Match kernel config: WM=2, WN=2 => 4 SIMDgroups => 128 threads
const SIMDGROUPS_PER_TG: u32 = 4;
const THREADS_PER_TG: u32 = SIMDGROUPS_PER_TG * 32;
- encoder_a.dispatch_threadgroups_indirect(
- args.dispatch_args,
- 0,
- MTLSize::new(THREADS_PER_TG as usize, 1, 1),
- );
+ encoder_a.dispatch_threadgroups_indirect(args.dispatch_args, 0, MTLSize::new(THREADS_PER_TG as usize, 1, 1));
encoder_a.end_encoding();
self.tile_map.encode_dispatch_args(
command_buffer,
@@ -288,9 +221,7 @@ impl MoeExpertsTwoPassPrefillKernel {
},
);
let pass_b_pipeline = &self.pass_b_indirect[dtype_idx];
- let encoder_b = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder_b = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
encoder_b.set_compute_pipeline_state(pass_b_pipeline);
encoder_b.set_buffer(Some(args.hidden_buffer), 0, 0);
encoder_b.set_buffer(Some(args.expert_offsets), 0, 1);
@@ -298,28 +229,12 @@ impl MoeExpertsTwoPassPrefillKernel {
encoder_b.set_buffer(Some(args.down_biases), 0, 3);
encoder_b.set_buffer(Some(args.output_buffer), 0, 4);
unsafe {
- encoder_b.set_bytes(
- NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _),
- size_of::<u32>(),
- 5,
- );
- encoder_b.set_bytes(
- NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _),
- size_of::<u32>(),
- 6,
- );
- encoder_b.set_bytes(
- NonNull::new_unchecked(&e_u32 as *const _ as *mut _),
- size_of::<u32>(),
- 7,
- );
+ encoder_b.set_bytes(NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _), size_of::<u32>(), 5);
+ encoder_b.set_bytes(NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _), size_of::<u32>(), 6);
+ encoder_b.set_bytes(NonNull::new_unchecked(&e_u32 as *const _ as *mut _), size_of::<u32>(), 7);
}
encoder_b.set_buffer(Some(args.tile_map), 0, 8);
- encoder_b.dispatch_threadgroups_indirect(
- args.dispatch_args,
- 0,
- MTLSize::new(THREADS_PER_TG as usize, 1, 1),
- );
+ encoder_b.dispatch_threadgroups_indirect(args.dispatch_args, 0, MTLSize::new(THREADS_PER_TG as usize, 1, 1));
encoder_b.end_encoding();
Ok(())
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs
index 14ed8876a..418fb4fd3 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs
@@ -1,10 +1,6 @@
-use crate::backends::common::kernel::{
- MoeExpertsDecodeSinglePassAKernel as _,
- MoeExpertsDecodeSinglePassBKernel as _,
-};
+use crate::backends::common::kernel::{MoeExpertsDecodeSinglePassAKernel as _, MoeExpertsDecodeSinglePassBKernel as _};
use crate::backends::metal::kernel::dsl::{
- MoeExpertsDecodeSinglePassAMetalKernel,
- MoeExpertsDecodeSinglePassBMetalKernel,
+ MoeExpertsDecodeSinglePassAMetalKernel, MoeExpertsDecodeSinglePassBMetalKernel,
};
use crate::backends::metal::{KernelDataType, MTLContext, MTLError};
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};
@@ -12,11 +8,7 @@ use objc2::__framework_prelude::ProtocolObject;
use objc2::Message;
use objc2::rc::Retained;
-static DTYPES: [KernelDataType; 3] = [
- KernelDataType::Float16,
- KernelDataType::BFloat16,
- KernelDataType::Float32,
-];
+static DTYPES: [KernelDataType; 3] = [KernelDataType::Float16, KernelDataType::BFloat16, KernelDataType::Float32];
/// Arguments for single-token MoE decode (T=1 optimized path)
#[derive(Debug)]
@@ -72,11 +64,7 @@ impl MoeExpertsSingleDecodeKernels {
for gate in 0..4 {
let mut kernels = vec![];
for dtype in &DTYPES {
- let kernel = MoeExpertsDecodeSinglePassAMetalKernel::new(
- ctx,
- (*dtype).into(),
- gate,
- )?;
+ let kernel = MoeExpertsDecodeSinglePassAMetalKernel::new(ctx, (*dtype).into(), gate)?;
kernels.push(kernel)
}
pass_a.push(kernels);
@@ -84,10 +72,7 @@ impl MoeExpertsSingleDecodeKernels {
let mut pass_b = Vec::with_capacity(DTYPES.len());
for dtype in &DTYPES {
- pass_b.push(MoeExpertsDecodeSinglePassBMetalKernel::new(
- ctx,
- (*dtype).into(),
- )?);
+ pass_b.push(MoeExpertsDecodeSinglePassBMetalKernel::new(ctx, (*dtype).into())?);
}
Ok(Self {
@@ -106,15 +91,13 @@ impl MoeExpertsSingleDecodeKernels {
}
let gate_idx = args.gating_code.min(3) as usize;
- let dtype_idx =
- DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
+ let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
// Pass A: x @ W13[expert] -> hidden
{
let kernel = &self.pass_a[gate_idx][dtype_idx];
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
kernel.encode(
&args.x.retain(),
&args.topk_ids.retain(),
@@ -137,9 +120,8 @@ impl MoeExpertsSingleDecodeKernels {
// Pass B: 8 simdgroups (256 threads), outputs final y directly
{
let kernel = &self.pass_b[dtype_idx];
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
kernel.encode(
&args.hidden.retain(),
&args.topk_ids.retain(),
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs
index 29cceb288..61d7b79df 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs
@@ -7,22 +7,15 @@ use objc2::{
use crate::{
DataType,
backends::{
- common::kernel::{
- MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel,
- },
+ common::kernel::{MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},
metal::{
MTLContext, MTLError,
kernel::{
MoeExpertsTwoPassArguments,
- dsl::{
- MoeExpertsDecodeDownFused2DMetalKernel,
- MoeExpertsDecodePassAMetalKernel,
- },
+ dsl::{MoeExpertsDecodeDownFused2DMetalKernel, MoeExpertsDecodePassAMetalKernel},
moe::{
- MoePassARowMapArguments, MoePassATileBuildArguments,
- MoePassATileCountsArguments, MoePassATileDispatchArguments,
- MoePassATileKernels, MoePassATileScanArguments,
- dtype_index,
+ MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,
+ MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments, dtype_index,
},
},
},
@@ -43,8 +36,7 @@ impl MoeExpertsTwoPassDecodeKernels {
for gate in 0u32..4u32 {
let mut kernels = vec![];
for dtype in &DTYPES {
- let kernel =
- MoeExpertsDecodePassAMetalKernel::new(ctx, *dtype, gate)?;
+ let kernel = MoeExpertsDecodePassAMetalKernel::new(ctx, *dtype, gate)?;
kernels.push(kernel);
}
pass_a_indirect.push(kernels);
@@ -52,11 +44,7 @@ impl MoeExpertsTwoPassDecodeKernels {
let mut fused_down = Vec::with_capacity(DTYPES.len());
for dtype in &DTYPES {
- let kernel = MoeExpertsDecodeDownFused2DMetalKernel::new(
- ctx,
- *dtype,
- DataType::F32,
- )?;
+ let kernel = MoeExpertsDecodeDownFused2DMetalKernel::new(ctx, *dtype, DataType::F32)?;
fused_down.push(kernel);
}
@@ -129,9 +117,8 @@ impl MoeExpertsTwoPassDecodeKernels {
// pass a
let gate_idx = args.gating_code.min(3) as usize;
let dtype_idx = dtype_index(args.data_type);
- let pass_a_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let pass_a_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];
pass_a_kernel.encode(
&args.x_perm_buffer.retain(),
@@ -155,9 +142,8 @@ impl MoeExpertsTwoPassDecodeKernels {
// pass b
let pass_b_kernel = &self.fused_down[dtype_idx];
- let pass_b_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let pass_b_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
pass_b_kernel.encode(
&args.hidden_buffer.retain(),
&args.row_expert_map.retain(),
diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.rs b/crates/uzu/src/backends/metal/kernel/moe/gather.rs
index 6196ec3e5..977e7d674 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/gather.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/gather.rs
@@ -5,11 +5,9 @@ use crate::{
backends::{
common::kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
metal::{
- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLContext, MTLError, ProtocolObject, Retained,
- kernel::dsl::{
- MoeGatherXPerm1DMetalKernel, MoeGatherXPerm2DMetalKernel,
- },
+ KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext, MTLError, ProtocolObject,
+ Retained,
+ kernel::dsl::{MoeGatherXPerm1DMetalKernel, MoeGatherXPerm2DMetalKernel},
},
},
};
@@ -46,9 +44,7 @@ impl MoeGatherKernels {
dtype: KernelDataType,
args: &MoeGatherArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
match dtype {
KernelDataType::Float32 => self.f32.encode(
&args.x_buffer.retain(),
diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
index 9300ddfeb..5981a3a92 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
@@ -12,24 +12,15 @@ pub mod tiles_pass_a;
// Re-export public items from submodules
pub use experts::{
- MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,
- MoeExpertsTwoPassPrefillKernel, MoeScatterError,
-};
-pub use experts_single::{
- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels,
+ MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeScatterError,
};
+pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels};
pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeKernels;
pub use gather::{MoeGatherArguments, MoeGatherKernels};
-pub use router_topk::{
- MoeRouterTopKArguments, MoeRouterTopKError, MoeRouterTopKKernel,
-};
-pub use scatter::{
- MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels,
- MoeScatterWithMapArguments,
-};
+pub use router_topk::{MoeRouterTopKArguments, MoeRouterTopKError, MoeRouterTopKKernel};
+pub use scatter::{MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments};
pub use tiles_pass_a::{
- MoePassARowMapArguments, MoePassATileBuildArguments,
- MoePassATileCountsArguments, MoePassATileDispatchArguments,
+ MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,
MoePassATileKernels, MoePassATileScanArguments,
};
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs b/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs
index ffdb5113c..8c30769fd 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs
@@ -1,8 +1,6 @@
use crate::backends::metal::{
- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
- MTLSize, ProtocolObject, Retained,
- metal_extensions::ComputeEncoderSetValue,
+ KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState,
+ MTLContext, MTLError, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
};
const THREADS_PER_THREADGROUP: usize = 256;
@@ -30,9 +28,9 @@ pub struct MoeRouterTopKKernel {
#[derive(Debug)]
pub struct MoeRouterTopKArguments<'a> {
- pub input_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T, d_model]
+ pub input_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T, d_model]
pub weight_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model]
- pub bias_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]
+ pub bias_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]
pub topk_ids_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T, K]
pub topk_probs_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T, K]
pub t: usize,
@@ -44,12 +42,9 @@ pub struct MoeRouterTopKArguments<'a> {
impl MoeRouterTopKKernel {
pub fn new(mtl_context: &MTLContext) -> Result<Self, MoeRouterTopKError> {
- let pipeline_f16 =
- mtl_context.compute_pipeline_state("moe_router_topk_f16", None)?;
- let pipeline_f32 =
- mtl_context.compute_pipeline_state("moe_router_topk_f32", None)?;
- let pipeline_bf16 =
- mtl_context.compute_pipeline_state("moe_router_topk_bf16", None)?;
+ let pipeline_f16 = mtl_context.compute_pipeline_state("moe_router_topk_f16", None)?;
+ let pipeline_f32 = mtl_context.compute_pipeline_state("moe_router_topk_f32", None)?;
+ let pipeline_bf16 = mtl_context.compute_pipeline_state("moe_router_topk_bf16", None)?;
Ok(Self {
pipeline_f16,
pipeline_f32,
@@ -83,19 +78,12 @@ impl MoeRouterTopKKernel {
});
}
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
match dtype {
- KernelDataType::Float16 => {
- compute_encoder.set_compute_pipeline_state(&self.pipeline_f16)
- },
- KernelDataType::Float32 => {
- compute_encoder.set_compute_pipeline_state(&self.pipeline_f32)
- },
- KernelDataType::BFloat16 => {
- compute_encoder.set_compute_pipeline_state(&self.pipeline_bf16)
- },
+ KernelDataType::Float16 => compute_encoder.set_compute_pipeline_state(&self.pipeline_f16),
+ KernelDataType::Float32 => compute_encoder.set_compute_pipeline_state(&self.pipeline_f32),
+ KernelDataType::BFloat16 => compute_encoder.set_compute_pipeline_state(&self.pipeline_bf16),
}
compute_encoder.set_buffer(Some(args.input_buffer), 0, 0);
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs b/crates/uzu/src/backends/metal/kernel/moe/scatter.rs
index 2c7509217..bc55f413a 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter.rs
@@ -1,7 +1,6 @@
use crate::backends::metal::{
- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLSize,
- ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
+ KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState,
+ MTLContext, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
};
// ---- Scatter Buckets Kernels ----
@@ -16,15 +15,11 @@ pub struct MoeScatterKernels {
pipeline_bases: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
pipeline_scatter_f16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
pipeline_scatter_f32: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_scatter_bf16:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ pipeline_scatter_bf16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
// map variants
- pipeline_scatter_map_f16:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_scatter_map_f32:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_scatter_map_bf16:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ pipeline_scatter_map_f16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ pipeline_scatter_map_f32: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ pipeline_scatter_map_bf16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
}
#[derive(Debug)]
@@ -61,20 +56,13 @@ pub struct MoeScatterWithMapArguments<'a> {
impl MoeScatterKernels {
pub fn new(mtl_context: &MTLContext) -> Result<Self, MoeScatterError> {
- let pipeline_bases = mtl_context
- .compute_pipeline_state("moe_block_bases_from_partials", None)?;
- let pipeline_scatter_f16 = mtl_context
- .compute_pipeline_state("moe_scatter_buckets_f16", None)?;
- let pipeline_scatter_f32 = mtl_context
- .compute_pipeline_state("moe_scatter_buckets_f32", None)?;
- let pipeline_scatter_bf16 = mtl_context
- .compute_pipeline_state("moe_scatter_buckets_bf16", None)?;
- let pipeline_scatter_map_f16 = mtl_context
- .compute_pipeline_state("moe_scatter_buckets_map_f16", None)?;
- let pipeline_scatter_map_f32 = mtl_context
- .compute_pipeline_state("moe_scatter_buckets_map_f32", None)?;
- let pipeline_scatter_map_bf16 = mtl_context
- .compute_pipeline_state("moe_scatter_buckets_map_bf16", None)?;
+ let pipeline_bases = mtl_context.compute_pipeline_state("moe_block_bases_from_partials", None)?;
+ let pipeline_scatter_f16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_f16", None)?;
+ let pipeline_scatter_f32 = mtl_context.compute_pipeline_state("moe_scatter_buckets_f32", None)?;
+ let pipeline_scatter_bf16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_bf16", None)?;
+ let pipeline_scatter_map_f16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_f16", None)?;
+ let pipeline_scatter_map_f32 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_f32", None)?;
+ let pipeline_scatter_map_bf16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_bf16", None)?;
Ok(Self {
pipeline_bases,
@@ -92,9 +80,8 @@ impl MoeScatterKernels {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
args: MoeBlockBasesArguments,
) -> Result<(), MoeScatterError> {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
compute_encoder.set_compute_pipeline_state(&self.pipeline_bases);
compute_encoder.set_buffer(Some(args.partials_buffer), 0, 0);
compute_encoder.set_buffer(Some(args.block_bases_buffer), 0, 1);
@@ -125,22 +112,18 @@ impl MoeScatterKernels {
args: MoeScatterArguments,
dtype: KernelDataType,
) -> Result<(), MoeScatterError> {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
// Select pipeline based on dtype
match dtype {
KernelDataType::Float16 => {
- compute_encoder
- .set_compute_pipeline_state(&self.pipeline_scatter_f16);
+ compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f16);
},
KernelDataType::Float32 => {
- compute_encoder
- .set_compute_pipeline_state(&self.pipeline_scatter_f32);
+ compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f32);
},
KernelDataType::BFloat16 => {
- compute_encoder
- .set_compute_pipeline_state(&self.pipeline_scatter_bf16);
+ compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_bf16);
},
}
compute_encoder.set_buffer(Some(args.topk_ids_buffer), 0, 0);
@@ -176,9 +159,8 @@ impl MoeScatterKernels {
args: MoeScatterWithMapArguments,
dtype: KernelDataType,
) -> Result<(), MoeScatterError> {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
let pipeline = match dtype {
KernelDataType::Float16 => &self.pipeline_scatter_map_f16,
KernelDataType::Float32 => &self.pipeline_scatter_map_f32,
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs
index 22bbd7b18..ec5e4c7b7 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs
@@ -1,13 +1,10 @@
use crate::backends::{
- common::kernel::{
- MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel,
- MoeWriteDispatchArgsKernel,
- },
+ common::kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},
metal::{
MTLContext, MTLError,
kernel::dsl::{
- MoeBuildTileMapMetalKernel, MoeTileCountsMetalKernel,
- MoeTileScanMetalKernel, MoeWriteDispatchArgsMetalKernel,
+ MoeBuildTileMapMetalKernel, MoeTileCountsMetalKernel, MoeTileScanMetalKernel,
+ MoeWriteDispatchArgsMetalKernel,
},
},
};
@@ -37,15 +34,8 @@ impl MoeTileMapKernels {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
args: &MoeTileCountsArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
- self.counts.encode(
- &args.offsets_buffer.retain(),
- &args.tile_counts_buffer.retain(),
- args.e as u32,
- &encoder,
- );
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
+ self.counts.encode(&args.offsets_buffer.retain(), &args.tile_counts_buffer.retain(), args.e as u32, &encoder);
encoder.end_encoding();
}
@@ -54,9 +44,7 @@ impl MoeTileMapKernels {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
args: &MoeTileScanArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.scan.encode(
&args.tile_counts_buffer.retain(),
&args.tile_offsets_buffer.retain(),
@@ -72,9 +60,7 @@ impl MoeTileMapKernels {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
args: &MoeTileMapBuildArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.build.encode(
&args.expert_offsets.retain(),
&args.tile_offsets.retain(),
@@ -91,21 +77,14 @@ impl MoeTileMapKernels {
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
args: &MoeTileDispatchArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
- self.dispatch.encode(
- &args.total_tiles.retain(),
- &args.dispatch_args.retain(),
- args.num_tiles_x,
- &encoder,
- );
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
+ self.dispatch.encode(&args.total_tiles.retain(), &args.dispatch_args.retain(), args.num_tiles_x, &encoder);
encoder.end_encoding();
}
}
pub struct MoeTileCountsArguments<'a> {
- pub offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
+ pub offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
pub tile_counts_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]
pub e: usize,
}
@@ -128,7 +107,7 @@ pub struct MoeTileMapBuildArguments<'a> {
#[derive(Debug)]
pub struct MoeTileDispatchArguments<'a> {
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]
- pub num_tiles_x: u32, // x dimension for indirect dispatch
+ pub num_tiles_x: u32, // x dimension for indirect dispatch
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs
index 8023d37d4..f9c6dd44a 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs
@@ -2,17 +2,14 @@ use objc2::Message;
use crate::backends::{
common::kernel::{
- MoePassABuildRowMapKernel, MoePassABuildTileMapKernel,
- MoePassATileCountsKernel, MoePassATileScanKernel,
+ MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel, MoePassATileScanKernel,
MoePassAWriteDispatchArgsKernel,
},
metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext, MTLError,
- ProtocolObject,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext, MTLError, ProtocolObject,
kernel::dsl::{
- MoePassABuildRowMapMetalKernel, MoePassABuildTileMapMetalKernel,
- MoePassATileCountsMetalKernel, MoePassATileScanMetalKernel,
- MoePassAWriteDispatchArgsMetalKernel,
+ MoePassABuildRowMapMetalKernel, MoePassABuildTileMapMetalKernel, MoePassATileCountsMetalKernel,
+ MoePassATileScanMetalKernel, MoePassAWriteDispatchArgsMetalKernel,
},
},
};
@@ -27,9 +24,9 @@ pub struct MoePassATileCountsArguments<'a> {
#[derive(Debug)]
pub struct MoePassATileScanArguments<'a> {
- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]
+ pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]
pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
pub e: usize,
}
@@ -53,7 +50,7 @@ pub struct MoePassATileBuildArguments<'a> {
#[derive(Debug)]
pub struct MoePassATileDispatchArguments<'a> {
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]
pub num_tiles_y: u32,
}
@@ -82,9 +79,7 @@ impl MoePassATileKernels {
command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
args: &MoePassATileCountsArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.counts.encode(
&args.expert_offsets.retain(),
&args.tile_counts.retain(),
@@ -100,9 +95,7 @@ impl MoePassATileKernels {
command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
args: &MoePassATileScanArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.scan.encode(
&args.tile_counts.retain(),
&args.tile_offsets.retain(),
@@ -118,9 +111,7 @@ impl MoePassATileKernels {
command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
args: &MoePassARowMapArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.row_map.encode(
&args.expert_offsets.retain(),
&args.row_expert_map.retain(),
@@ -136,9 +127,7 @@ impl MoePassATileKernels {
command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
args: &MoePassATileBuildArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.build_map.encode(
&args.expert_offsets.retain(),
&args.tile_offsets.retain(),
@@ -156,15 +145,8 @@ impl MoePassATileKernels {
command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
args: &MoePassATileDispatchArguments,
) {
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
- self.dispatch.encode(
- &args.total_tiles.retain(),
- &args.dispatch_args.retain(),
- args.num_tiles_y,
- &encoder,
- );
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
+ self.dispatch.encode(&args.total_tiles.retain(), &args.dispatch_args.retain(), args.num_tiles_y, &encoder);
encoder.end_encoding();
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index a34d37522..5ab10fa68 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -3,9 +3,8 @@ use std::{collections::HashMap, ptr::NonNull};
use crate::{
DataType,
backends::metal::{
- MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLContext, MTLError, MTLFunctionConstantValues, MTLSize,
- ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
+ MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues,
+ MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
},
config::QuantizationMode,
};
@@ -37,10 +36,7 @@ pub enum QuantizedMatmulError {
}
pub struct QuantizedMatmulKernel {
- pipelines: HashMap<
- KernelKind,
- (Retained<ProtocolObject<dyn MTLComputePipelineState>>, u64, u64),
- >,
+ pipelines: HashMap<KernelKind, (Retained<ProtocolObject<dyn MTLComputePipelineState>>, u64, u64)>,
output_dim: usize,
weights_transposed: bool,
}
@@ -107,8 +103,7 @@ impl QuantizedMatmulKernel {
quantization_type: QuantizationType,
weights_transposed: bool,
) -> Result<Self, QuantizedMatmulError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(QuantizedMatmulError::UnsupportedDataType(data_type));
}
@@ -122,41 +117,21 @@ impl QuantizedMatmulKernel {
let mut pipelines = HashMap::new();
- let kernel_name_mv = select_matrix_vector_kernel_name(
- data_type,
- group_size,
- weights_transposed,
- output_dim,
- input_dim,
- mode,
- )?;
+ let kernel_name_mv =
+ select_matrix_vector_kernel_name(data_type, group_size, weights_transposed, output_dim, input_dim, mode)?;
let cache_key_mv = format!("{}_mlx_{}", kernel_name_mv, use_mlx_quant);
let pipeline_mv = mtl_context
- .compute_pipeline_state_cached(
- &cache_key_mv,
- &kernel_name_mv,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key_mv, &kernel_name_mv, Some(&function_constants))
.map_err(QuantizedMatmulError::MetalError)?;
pipelines.insert(KernelKind::MatrixVector, (pipeline_mv, 32, 32));
- let kernel_name_mm = select_qmm_kernel_name(
- data_type,
- group_size,
- weights_transposed,
- output_dim,
- input_dim,
- mode,
- )?;
+ let kernel_name_mm =
+ select_qmm_kernel_name(data_type, group_size, weights_transposed, output_dim, input_dim, mode)?;
let cache_key_mm = format!("{}_mlx_{}", kernel_name_mm, use_mlx_quant);
let pipeline_mm = mtl_context
- .compute_pipeline_state_cached(
- &cache_key_mm,
- &kernel_name_mm,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key_mm, &kernel_name_mm, Some(&function_constants))
.map_err(QuantizedMatmulError::MetalError)?;
let (bm, bn) = if kernel_name_mm.contains("_64x64") {
@@ -196,12 +171,10 @@ impl QuantizedMatmulKernel {
) -> Result<(), QuantizedMatmulError> {
let variant = self.select_variant(args.batch as usize);
let (pipeline, bm, bn) =
- self.pipelines.get(&variant).ok_or_else(|| {
- QuantizedMatmulError::InvalidDimensions {
- m: args.batch as usize,
- n: args.output_dim as usize,
- k: args.input_dim as usize,
- }
+ self.pipelines.get(&variant).ok_or_else(|| QuantizedMatmulError::InvalidDimensions {
+ m: args.batch as usize,
+ n: args.output_dim as usize,
+ k: args.input_dim as usize,
})?;
encoder.set_compute_pipeline_state(pipeline);
@@ -230,28 +203,17 @@ impl QuantizedMatmulKernel {
64
};
let n_tgp_y = (n + bn - 1) / bn;
- let threadgroups =
- MTLSize::new(m as usize, n_tgp_y as usize, 1);
+ let threadgroups = MTLSize::new(m as usize, n_tgp_y as usize, 1);
let threads_per_threadgroup = MTLSize::new(bk as usize, 2, 1);
- encoder.dispatch_threadgroups(
- threadgroups,
- threads_per_threadgroup,
- );
+ encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);
},
KernelKind::MatrixMatrix => {
let wm = 2;
let wn = 2;
- let threads_per_threadgroup =
- MTLSize::new(32, wn as usize, wm as usize);
- let threadgroups = MTLSize::new(
- ((n as u64 + bn - 1) / bn) as usize,
- ((m as u64 + bm - 1) / bm) as usize,
- 1,
- );
- encoder.dispatch_threadgroups(
- threadgroups,
- threads_per_threadgroup,
- );
+ let threads_per_threadgroup = MTLSize::new(32, wn as usize, wm as usize);
+ let threadgroups =
+ MTLSize::new(((n as u64 + bn - 1) / bn) as usize, ((m as u64 + bm - 1) / bm) as usize, 1);
+ encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);
},
}
@@ -278,8 +240,7 @@ fn select_matrix_vector_kernel_name(
input_dim: usize,
mode: QuantizationMode,
) -> Result<String, QuantizedMatmulError> {
- let type_suffix = dtype_suffix(data_type)
- .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
+ let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
let bits = match mode {
QuantizationMode::UInt4 => 4,
QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,
@@ -304,8 +265,7 @@ fn select_qmm_kernel_name(
input_dim: usize,
mode: QuantizationMode,
) -> Result<String, QuantizedMatmulError> {
- let type_suffix = dtype_suffix(data_type)
- .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
+ let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
let bits = match mode {
QuantizationMode::UInt4 => 4,
QuantizationMode::Int8 | QuantizationMode::UInt8 => 8,
@@ -316,15 +276,11 @@ fn select_qmm_kernel_name(
} else {
""
};
- let mut kernel_name =
- base_qmm_kernel_name(type_suffix, group_size, transpose_infix, bits)?;
+ let mut kernel_name = base_qmm_kernel_name(type_suffix, group_size, transpose_infix, bits)?;
if weights_transposed {
if output_dim % 32 != 0 {
kernel_name.push_str("_unaligned");
- } else if type_suffix == "bf16"
- && (group_size == 128 || group_size == 64)
- && (bits == 4 || bits == 8)
- {
+ } else if type_suffix == "bf16" && (group_size == 128 || group_size == 64) && (bits == 4 || bits == 8) {
kernel_name.push_str("_64x64");
}
} else if input_dim % 32 == 0 {
@@ -376,8 +332,7 @@ impl MlpFusedQmvKernel {
mode: QuantizationMode,
quantization_type: QuantizationType,
) -> Result<Self, QuantizedMatmulError> {
- let type_suffix = dtype_suffix(data_type)
- .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
+ let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
let bits = match mode {
QuantizationMode::UInt4 => 4,
@@ -388,8 +343,7 @@ impl MlpFusedQmvKernel {
return Err(QuantizedMatmulError::UnsupportedBits(bits));
}
- let kernel_name =
- format!("qmv_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);
+ let kernel_name = format!("qmv_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);
let function_constants = MTLFunctionConstantValues::new();
let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);
@@ -426,10 +380,8 @@ impl MlpFusedQmvKernel {
// Dispatch: one threadgroup per 8 output rows (num_simdgroups * results_per_simdgroup)
let rows_per_threadgroup = 8;
- let n_tgp_y = ((args.hidden_dim + rows_per_threadgroup - 1)
- / rows_per_threadgroup) as u64;
- let threadgroups =
- MTLSize::new(args.batch_count.max(1) as usize, n_tgp_y as usize, 1);
+ let n_tgp_y = ((args.hidden_dim + rows_per_threadgroup - 1) / rows_per_threadgroup) as u64;
+ let threadgroups = MTLSize::new(args.batch_count.max(1) as usize, n_tgp_y as usize, 1);
let threads_per_threadgroup = MTLSize::new(32, 2, 1); // 2 simdgroups, 32 threads each
encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);
@@ -474,8 +426,7 @@ impl MlpFusedQmmKernel {
mode: QuantizationMode,
quantization_type: QuantizationType,
) -> Result<Self, QuantizedMatmulError> {
- let type_suffix = dtype_suffix(data_type)
- .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
+ let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
let bits = match mode {
QuantizationMode::UInt4 => 4,
@@ -486,8 +437,7 @@ impl MlpFusedQmmKernel {
return Err(QuantizedMatmulError::UnsupportedBits(bits));
}
- let kernel_name =
- format!("qmm_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);
+ let kernel_name = format!("qmm_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);
let function_constants = MTLFunctionConstantValues::new();
let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
index 19e9b5c8d..facef02b7 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
@@ -1,9 +1,8 @@
use crate::{
DataType,
backends::metal::{
- ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError, MTLSize, ProtocolObject,
- Retained,
+ ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
+ MTLSize, ProtocolObject, Retained,
},
};
@@ -107,9 +106,7 @@ impl RMSNormKernel {
full_layer,
)?;
- let pipeline = context
- .compute_pipeline_state(&function_name, None)
- .map_err(RMSNormError::MetalError)?;
+ let pipeline = context.compute_pipeline_state(&function_name, None).map_err(RMSNormError::MetalError)?;
Ok(Self {
pipeline,
@@ -144,18 +141,11 @@ impl RMSNormKernel {
Ok(format!(
"{}_{}_{}_{}_{}{}",
- base_name,
- input_suffix,
- scales_suffix,
- output_suffix,
- accum_suffix,
- mode_suffix
+ base_name, input_suffix, scales_suffix, output_suffix, accum_suffix, mode_suffix
))
}
- fn data_type_to_suffix(
- data_type: DataType
- ) -> Result<&'static str, RMSNormError> {
+ fn data_type_to_suffix(data_type: DataType) -> Result<&'static str, RMSNormError> {
match data_type {
DataType::F32 => Ok("f32"),
DataType::F16 => Ok("f16"),
@@ -169,9 +159,7 @@ impl RMSNormKernel {
}
}
- fn accum_type_to_suffix(
- data_type: DataType
- ) -> Result<&'static str, RMSNormError> {
+ fn accum_type_to_suffix(data_type: DataType) -> Result<&'static str, RMSNormError> {
match data_type {
DataType::F32 => Ok("f32"),
DataType::F16 => Ok("f16"),
@@ -196,17 +184,9 @@ impl RMSNormKernel {
compute_encoder.set_compute_pipeline_state(&self.pipeline);
// Set buffers
- compute_encoder.set_buffer(
- Some(args.input_buffer),
- args.input_offset as usize,
- 0,
- );
+ compute_encoder.set_buffer(Some(args.input_buffer), args.input_offset as usize, 0);
compute_encoder.set_buffer(Some(args.scales_buffer), 0, 1);
- compute_encoder.set_buffer(
- Some(args.output_buffer),
- args.output_offset as usize,
- 2,
- );
+ compute_encoder.set_buffer(Some(args.output_buffer), args.output_offset as usize, 2);
// Set parameters
compute_encoder.set_value(&args.batch_size, 3);
@@ -226,10 +206,7 @@ impl RMSNormKernel {
depth: 1,
};
- compute_encoder.dispatch_threadgroups(
- threadgroups_per_grid,
- threads_per_threadgroup,
- );
+ compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);
Ok(())
}
@@ -264,12 +241,8 @@ impl RMSNormKernel {
// We only ever normalize within [Q] and/or [K].
let (head_offset, head_count): (u32, u32) = match args.target {
QKNormTarget::QueryHeads => (0, args.num_q_heads as u32),
- QKNormTarget::KeyHeads => {
- (args.num_q_heads as u32, args.num_kv_heads as u32)
- },
- QKNormTarget::Both => {
- (0, (args.num_q_heads + args.num_kv_heads) as u32)
- },
+ QKNormTarget::KeyHeads => (args.num_q_heads as u32, args.num_kv_heads as u32),
+ QKNormTarget::Both => (0, (args.num_q_heads + args.num_kv_heads) as u32),
};
if args.batch_size <= 0 || args.head_dim <= 0 || head_count == 0 {
@@ -284,11 +257,9 @@ impl RMSNormKernel {
let simd_width = self.pipeline.thread_execution_width();
let max_threads = self.pipeline.max_total_threads_per_threadgroup();
let max_heads_per_threadgroup = (max_threads / simd_width).max(1);
- let heads_per_threadgroup =
- (head_count as usize).min(max_heads_per_threadgroup).max(1);
+ let heads_per_threadgroup = (head_count as usize).min(max_heads_per_threadgroup).max(1);
- let num_head_tiles =
- (head_count as usize).div_ceil(heads_per_threadgroup);
+ let num_head_tiles = (head_count as usize).div_ceil(heads_per_threadgroup);
let threadgroups_per_grid = MTLSize {
width: args.batch_size as usize,
@@ -302,10 +273,7 @@ impl RMSNormKernel {
depth: 1,
};
- compute_encoder.dispatch_threadgroups(
- threadgroups_per_grid,
- threads_per_threadgroup,
- );
+ compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);
Ok(())
}
diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
index 2fadd633f..d11437399 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
@@ -1,9 +1,8 @@
use thiserror::Error;
use crate::backends::metal::{
- ComputeEncoderSetValue, KernelDataType, MTLBuffer,
- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
- MTLSize, ProtocolObject, Retained,
+ ComputeEncoderSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
+ MTLError, MTLSize, ProtocolObject, Retained,
};
pub struct RopeKernel {
@@ -20,11 +19,11 @@ pub enum RopeError {
#[allow(dead_code)]
pub struct RopeKernelArguments<'a> {
- pub qkv_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
+ pub qkv_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
pub cosines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)
- pub sines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
+ pub sines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
pub token_positions_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)
- pub token_positions_offset: usize, // byte offset into token_positions_buffer
+ pub token_positions_offset: usize, // byte offset into token_positions_buffer
pub rotated_queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4)
pub rotated_keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5)
pub head_dim: usize,
@@ -39,18 +38,12 @@ impl RopeKernel {
context: &MTLContext,
data_type: KernelDataType,
) -> Result<Self, RopeError> {
- let function_name =
- format!("applyRope_{}", data_type.function_name_suffix());
+ let function_name = format!("applyRope_{}", data_type.function_name_suffix());
- let pipeline_state = context
- .compute_pipeline_state(&function_name, None)
- .map_err(|e| {
- eprintln!(
- "Failed to create pipeline state for {}: {:?}",
- function_name, e
- );
- RopeError::MetalError(e)
- })?;
+ let pipeline_state = context.compute_pipeline_state(&function_name, None).map_err(|e| {
+ eprintln!("Failed to create pipeline state for {}: {:?}", function_name, e);
+ RopeError::MetalError(e)
+ })?;
Ok(Self {
pipeline_state,
@@ -67,11 +60,7 @@ impl RopeKernel {
compute_encoder.set_buffer(Some(args.qkv_buffer), 0, 0);
compute_encoder.set_buffer(Some(args.cosines_buffer), 0, 1);
compute_encoder.set_buffer(Some(args.sines_buffer), 0, 2);
- compute_encoder.set_buffer(
- Some(args.token_positions_buffer),
- args.token_positions_offset as usize,
- 3,
- );
+ compute_encoder.set_buffer(Some(args.token_positions_buffer), args.token_positions_offset as usize, 3);
compute_encoder.set_buffer(Some(args.rotated_queries_buffer), 0, 4);
compute_encoder.set_buffer(Some(args.rotated_keys_buffer), 0, 5);
@@ -99,7 +88,6 @@ impl RopeKernel {
depth: head_dim as usize,
};
- compute_encoder
- .dispatch_threads(total_threads, threads_per_threadgroup);
+ compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
index 8f60fdfb6..b496e3523 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
@@ -1,9 +1,8 @@
use std::ptr::NonNull;
use crate::backends::metal::{
- ComputeEncoderSetValue, KernelDataType, MTLBuffer,
- MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLDataType,
- MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
+ ComputeEncoderSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
+ MTLDataType, MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
};
#[derive(Debug, thiserror::Error)]
@@ -16,32 +15,20 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {
dt.function_name_suffix()
}
-fn make_function_constants(
- has_bias: bool
-) -> Retained<MTLFunctionConstantValues> {
+fn make_function_constants(has_bias: bool) -> Retained<MTLFunctionConstantValues> {
let function_constants = MTLFunctionConstantValues::new();
- function_constants.set_constant_value_type_at_index(
- NonNull::from(&has_bias).cast(),
- MTLDataType::Bool,
- 0,
- );
+ function_constants.set_constant_value_type_at_index(NonNull::from(&has_bias).cast(), MTLDataType::Bool, 0);
function_constants
}
pub struct ShortConvKernel {
pack_pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- prefill_pipeline_no_bias:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- prefill_pipeline_with_bias:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- decode_pipeline_no_bias:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- decode_pipeline_with_bias:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- trie_pipeline_no_bias:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- trie_pipeline_with_bias:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ prefill_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ prefill_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ decode_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ decode_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ trie_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ trie_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
}
pub struct ShortConvPackArguments<'a> {
@@ -112,61 +99,40 @@ impl ShortConvKernel {
context: &MTLContext,
data_type: KernelDataType,
) -> Result<Self, ShortConvKernelError> {
- let pack_name =
- format!("short_conv_pack_kernel_{}", fn_suffix(data_type));
- let prefill_name =
- format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));
- let decode_name =
- format!("short_conv_decode_kernel_{}", fn_suffix(data_type));
- let trie_name =
- format!("short_conv_trie_kernel_{}", fn_suffix(data_type));
-
- let pack_pipeline = context
- .compute_pipeline_state(&pack_name, None)
- .map_err(ShortConvKernelError::MetalError)?;
+ let pack_name = format!("short_conv_pack_kernel_{}", fn_suffix(data_type));
+ let prefill_name = format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));
+ let decode_name = format!("short_conv_decode_kernel_{}", fn_suffix(data_type));
+ let trie_name = format!("short_conv_trie_kernel_{}", fn_suffix(data_type));
+
+ let pack_pipeline =
+ context.compute_pipeline_state(&pack_name, None).map_err(ShortConvKernelError::MetalError)?;
let prefill_pipeline_no_bias = {
let function_constants = make_function_constants(false);
let cache_key = format!("{}_has_bias_0", prefill_name);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &prefill_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &prefill_name, Some(&function_constants))
.map_err(ShortConvKernelError::MetalError)?
};
let prefill_pipeline_with_bias = {
let function_constants = make_function_constants(true);
let cache_key = format!("{}_has_bias_1", prefill_name);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &prefill_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &prefill_name, Some(&function_constants))
.map_err(ShortConvKernelError::MetalError)?
};
let decode_pipeline_no_bias = {
let function_constants = make_function_constants(false);
let cache_key = format!("{}_has_bias_0", decode_name);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &decode_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))
.map_err(ShortConvKernelError::MetalError)?
};
let decode_pipeline_with_bias = {
let function_constants = make_function_constants(true);
let cache_key = format!("{}_has_bias_1", decode_name);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &decode_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))
.map_err(ShortConvKernelError::MetalError)?
};
@@ -174,22 +140,14 @@ impl ShortConvKernel {
let function_constants = make_function_constants(false);
let cache_key = format!("{}_has_bias_0", trie_name);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &trie_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &trie_name, Some(&function_constants))
.map_err(ShortConvKernelError::MetalError)?
};
let trie_pipeline_with_bias = {
let function_constants = make_function_constants(true);
let cache_key = format!("{}_has_bias_1", trie_name);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &trie_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &trie_name, Some(&function_constants))
.map_err(ShortConvKernelError::MetalError)?
};
@@ -356,18 +314,10 @@ impl ShortConvKernel {
if has_bias {
compute_encoder.set_buffer(args.b, 0, 2);
}
- compute_encoder.set_buffer(
- Some(args.base_state),
- args.base_state_offset,
- 3,
- );
+ compute_encoder.set_buffer(Some(args.base_state), args.base_state_offset, 3);
compute_encoder.set_buffer(Some(args.parents), args.parents_offset, 4);
compute_encoder.set_buffer(Some(args.out), args.out_offset, 5);
- compute_encoder.set_buffer(
- Some(args.suffix_state),
- args.suffix_state_offset,
- 6,
- );
+ compute_encoder.set_buffer(Some(args.suffix_state), args.suffix_state_offset, 6);
compute_encoder.set_value(&args.suffix_len, 7);
compute_encoder.set_value(&args.kernel_size, 8);
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
index 36e1c486d..4d851a6bd 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
@@ -3,10 +3,8 @@ use std::ptr::NonNull;
use super::{SSMKernelError, fn_suffix};
use crate::{
backends::metal::{
- KernelDataType, MTLBuffer, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLDataType,
- MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
- metal_extensions::ComputeEncoderSetValue,
+ KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLDataType,
+ MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
},
config::Activation,
};
@@ -34,17 +32,9 @@ fn make_function_constants(
let function_constants = MTLFunctionConstantValues::new();
let activation_type = activation_to_int(activation);
- function_constants.set_constant_value_type_at_index(
- NonNull::from(&activation_type).cast(),
- MTLDataType::Int,
- 0,
- );
+ function_constants.set_constant_value_type_at_index(NonNull::from(&activation_type).cast(), MTLDataType::Int, 0);
- function_constants.set_constant_value_type_at_index(
- NonNull::from(&has_bias).cast(),
- MTLDataType::Bool,
- 1,
- );
+ function_constants.set_constant_value_type_at_index(NonNull::from(&has_bias).cast(), MTLDataType::Bool, 1);
function_constants
}
@@ -53,19 +43,17 @@ pub struct Conv1dScanKernel {
pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
pack_pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- decode_pipeline_no_bias:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- decode_pipeline_with_bias:
- Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ decode_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ decode_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
}
pub struct Conv1dScanArguments<'a> {
pub padded: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0) [prefix+suffix, channels]
- pub w: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1) [channels, kernel]
+ pub w: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1) [channels, kernel]
pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(2) [channels]
- pub x_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) [suffix, inner_dim]
- pub b_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) [suffix, proj_dim]
- pub c_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) [suffix, proj_dim]
+ pub x_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) [suffix, inner_dim]
+ pub b_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) [suffix, proj_dim]
+ pub c_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) [suffix, proj_dim]
pub state_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(6) [channels, kernel-1]
pub suffix_len: usize,
pub kernel_size: i32,
@@ -111,58 +99,36 @@ impl Conv1dScanKernel {
activation: &Activation,
) -> Result<Self, SSMKernelError> {
let fn_name = format!("conv1d_scan_kernel_{}", fn_suffix(data_type));
- let pack_name =
- format!("conv1d_pack_prefix_kernel_{}", fn_suffix(data_type));
- let decode_name =
- format!("conv1d_decode_kernel_{}", fn_suffix(data_type));
+ let pack_name = format!("conv1d_pack_prefix_kernel_{}", fn_suffix(data_type));
+ let decode_name = format!("conv1d_decode_kernel_{}", fn_suffix(data_type));
let activation_id = activation_to_int(activation);
let pipeline_no_bias = {
let function_constants = make_function_constants(activation, false);
let cache_key = format!("{}_act_{}_bias_0", fn_name, activation_id);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &fn_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &fn_name, Some(&function_constants))
.map_err(SSMKernelError::MetalError)?
};
let pipeline_with_bias = {
let function_constants = make_function_constants(activation, true);
let cache_key = format!("{}_act_{}_bias_1", fn_name, activation_id);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &fn_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &fn_name, Some(&function_constants))
.map_err(SSMKernelError::MetalError)?
};
- let pack_pipeline = context
- .compute_pipeline_state(&pack_name, None)
- .map_err(SSMKernelError::MetalError)?;
+ let pack_pipeline = context.compute_pipeline_state(&pack_name, None).map_err(SSMKernelError::MetalError)?;
let decode_pipeline_no_bias = {
let function_constants = make_function_constants(activation, false);
- let cache_key =
- format!("{}_act_{}_bias_0", decode_name, activation_id);
+ let cache_key = format!("{}_act_{}_bias_0", decode_name, activation_id);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &decode_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))
.map_err(SSMKernelError::MetalError)?
};
let decode_pipeline_with_bias = {
let function_constants = make_function_constants(activation, true);
- let cache_key =
- format!("{}_act_{}_bias_1", decode_name, activation_id);
+ let cache_key = format!("{}_act_{}_bias_1", decode_name, activation_id);
context
- .compute_pipeline_state_cached(
- &cache_key,
- &decode_name,
- Some(&function_constants),
- )
+ .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))
.map_err(SSMKernelError::MetalError)?
};
Ok(Self {
@@ -204,8 +170,7 @@ impl Conv1dScanKernel {
height: padded_rows,
depth: 1,
};
- compute_encoder
- .dispatch_threads(total_threads, threads_per_threadgroup);
+ compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);
Ok(())
}
@@ -256,8 +221,7 @@ impl Conv1dScanKernel {
height: args.channels,
depth: 1,
};
- compute_encoder
- .dispatch_threads(total_threads, threads_per_threadgroup);
+ compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);
Ok(())
}
@@ -313,8 +277,7 @@ impl Conv1dScanKernel {
height: channels,
depth: 1,
};
- compute_encoder
- .dispatch_threads(total_threads, threads_per_threadgroup);
+ compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);
Ok(())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
index fe660542b..e21c5cad0 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
@@ -15,7 +15,5 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {
pub mod conv1d_scan;
pub mod ssd_prefill;
-pub use conv1d_scan::{
- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
-};
+pub use conv1d_scan::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel};
pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode};
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs
index ec0e9dd57..15f0ded2a 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs
@@ -1,15 +1,8 @@
use crate::backends::{
- common::kernel::{
- SSDPrefill64Kernel as _, SSDPrefillKernel as _,
- SSDPrefillSequentialKernel as _,
- },
+ common::kernel::{SSDPrefill64Kernel as _, SSDPrefillKernel as _, SSDPrefillSequentialKernel as _},
metal::{
- KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLContext,
- MTLError, ProtocolObject,
- kernel::dsl::{
- SSDPrefill64MetalKernel, SSDPrefillMetalKernel,
- SSDPrefillSequentialMetalKernel,
- },
+ KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
+ kernel::dsl::{SSDPrefill64MetalKernel, SSDPrefillMetalKernel, SSDPrefillSequentialMetalKernel},
},
};
use objc2::Message;
@@ -52,10 +45,8 @@ impl SSDPrefillKernels {
data_type: KernelDataType,
) -> Result<Self, MTLError> {
let single = SSDPrefillMetalKernel::new(context, data_type.into())?;
- let single_64 =
- SSDPrefill64MetalKernel::new(context, data_type.into())?;
- let sequential =
- SSDPrefillSequentialMetalKernel::new(context, data_type.into())?;
+ let single_64 = SSDPrefill64MetalKernel::new(context, data_type.into())?;
+ let sequential = SSDPrefillSequentialMetalKernel::new(context, data_type.into())?;
Ok(Self {
single,
single_64,
@@ -83,26 +74,10 @@ impl SSDPrefillKernels {
args.suffix_len as u32,
args.group_size,
args.state_size,
- args.x_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.dt_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.cb_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.state_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
+ args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
args.channels as u32,
args.head_dim as u32,
compute_encoder,
@@ -120,26 +95,10 @@ impl SSDPrefillKernels {
args.suffix_len as u32,
args.group_size,
args.state_size,
- args.x_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.dt_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.cb_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.state_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
+ args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
args.channels as u32,
args.head_dim as u32,
compute_encoder,
@@ -158,26 +117,10 @@ impl SSDPrefillKernels {
args.suffix_len as u32,
args.group_size,
args.state_size,
- args.x_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.dt_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.cb_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
- args.state_strides
- .iter()
- .map(|x| *x as u32)
- .collect::<Vec<_>>()
- .as_slice(),
+ args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
+ args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
args.channels as u32,
args.head_dim as u32,
compute_encoder,
diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs
index d64232315..676c90d45 100644
--- a/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs
@@ -1,6 +1,6 @@
use crate::backends::metal::{
- MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
- MTLError, MTLSize, ProtocolObject, Retained,
+ MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError, MTLSize, ProtocolObject,
+ Retained,
};
/// Kernel for copying sampled tokens in async pipeline.
@@ -14,10 +14,8 @@ pub struct TokenCopyKernel {
impl TokenCopyKernel {
pub fn new(context: &MTLContext) -> Result<Self, MTLError> {
- let copy_to_token_ids =
- context.compute_pipeline_state("copy_sampled_token", None)?;
- let copy_to_results =
- context.compute_pipeline_state("copy_token_to_results", None)?;
+ let copy_to_token_ids = context.compute_pipeline_state("copy_sampled_token", None)?;
+ let copy_to_results = context.compute_pipeline_state("copy_token_to_results", None)?;
Ok(Self {
copy_to_token_ids,
diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rs
index 1b621d72d..c997bf438 100644
--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs
+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs
@@ -7,24 +7,22 @@ use std::{
};
use super::{
- Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal,
- compilation_parameters::CompilationConfig, kernel::TokenCopyKernel,
+ Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal, compilation_parameters::CompilationConfig,
+ kernel::TokenCopyKernel,
};
use crate::{
DataType,
backends::{
common::{Context, kernel::MaskUpdateKernel},
metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt,
- MTLEvent, ProtocolObject, Retained,
+ MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLEvent, ProtocolObject, Retained,
kernel::dsl::MaskUpdateMetalKernel,
},
},
config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
encodable_block::Sampling,
forward_pass::{
- cache_layers::CacheLayers, model_shape::ModelShape,
- scratch_buffers::ScratchBuffers, state::SharedBuffers,
+ cache_layers::CacheLayers, model_shape::ModelShape, scratch_buffers::ScratchBuffers, state::SharedBuffers,
},
language_model::rng::PRng,
parameters::ParameterLoader,
@@ -63,15 +61,12 @@ impl AsyncBuffers {
max_tokens: usize,
batch_size: usize,
) -> Self {
- let positions = context
- .create_buffer(max_tokens * std::mem::size_of::<i32>())
- .expect("Failed to create positions buffer");
- let seeds = context
- .create_buffer(max_tokens * std::mem::size_of::<u64>())
- .expect("Failed to create seeds buffer");
- let results = context
- .create_buffer(batch_size * std::mem::size_of::<u32>())
- .expect("Failed to create results buffer");
+ let positions =
+ context.create_buffer(max_tokens * std::mem::size_of::<i32>()).expect("Failed to create positions buffer");
+ let seeds =
+ context.create_buffer(max_tokens * std::mem::size_of::<u64>()).expect("Failed to create seeds buffer");
+ let results =
+ context.create_buffer(batch_size * std::mem::size_of::<u32>()).expect("Failed to create results buffer");
let event = context.device.new_event().expect("Failed to create event");
Self {
@@ -161,77 +156,46 @@ impl LanguageModelGeneratorContext {
model_path: &Path,
decoding_config: &DecodingConfig,
) -> Result<Self, Error> {
- let context =
- MTLContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;
+ let context = MTLContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;
- let command_buffer = context
- .create_command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
+ let command_buffer = context.create_command_buffer().expect("Failed to create command buffer").to_owned();
let config_path = model_path.join("config.json");
if !config_path.exists() {
return Err(Error::UnableToLoadConfig);
}
- let config_file =
- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
let model_metadata: ModelMetadata =
- serde_json::from_reader(BufReader::new(config_file))
- .map_err(|_| Error::UnableToLoadConfig)?;
+ serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
// Extract language model config
- let language_model_config = model_metadata
- .model_config
- .as_language_model()
- .ok_or(Error::UnableToLoadConfig)?;
-
- let decoder_config = Rc::new(
- language_model_config
- .decoder_config()
- .map_err(|_| Error::UnableToLoadConfig)?,
- );
+ let language_model_config = model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;
+
+ let decoder_config = Rc::new(language_model_config.decoder_config().map_err(|_| Error::UnableToLoadConfig)?);
let model_shape = ModelShape::from_decoder_config(&decoder_config);
- let prefill_step_size =
- decoding_config.prefill_step_size.resolve(language_model_config);
+ let prefill_step_size = decoding_config.prefill_step_size.resolve(language_model_config);
let generate_suffix_length = decoding_config.generate_suffix_length();
- let max_prefix_length: usize =
- decoding_config.context_length.resolve(language_model_config);
- let max_suffix_length: usize =
- std::cmp::max(prefill_step_size, generate_suffix_length);
+ let max_prefix_length: usize = decoding_config.context_length.resolve(language_model_config);
+ let max_suffix_length: usize = std::cmp::max(prefill_step_size, generate_suffix_length);
let compilation_config = Rc::new(CompilationConfig::default());
let weights_path = model_path.join("model.safetensors");
if !weights_path.exists() {
return Err(Error::UnableToLoadWeights);
}
- let weights_file = File::open(&weights_path)
- .map_err(|_| Error::UnableToLoadWeights)?;
- let loader = ParameterLoader::new(&weights_file, context.as_ref())
- .map_err(|_| Error::UnableToLoadWeights)?;
+ let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let root_loader_view = loader.tree();
- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(
- context.as_ref(),
- &decoder_config,
- &model_shape,
- )));
+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));
shared_buffers.borrow_mut().update_data(&root_loader_view);
- let scratch_buffers = ScratchBuffers::new(
- context.as_ref(),
- &decoder_config,
- &model_shape,
- max_prefix_length,
- max_suffix_length,
- );
+ let scratch_buffers =
+ ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_prefix_length, max_suffix_length);
- let executables = Decoder::new(
- context.clone(),
- decoder_config.clone(),
- &root_loader_view,
- compilation_config.clone(),
- );
+ let executables =
+ Decoder::new(context.clone(), decoder_config.clone(), &root_loader_view, compilation_config.clone());
let cache_layers = Rc::new(RefCell::new(CacheLayers::new(
context.as_ref(),
@@ -240,28 +204,18 @@ impl LanguageModelGeneratorContext {
max_suffix_length,
)));
- let intermediate_data_type: DataType =
- decoder_config.output_norm_config.scale_precision.into();
+ let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();
let kernel_data_type: KernelDataType = intermediate_data_type.into();
let kv_cache_update = Box::new(
- KVCacheUpdate::new(
- context.as_ref(),
- intermediate_data_type,
- max_prefix_length,
- )
- .map_err(|_| Error::UnableToCreateMetalContext)?,
+ KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)
+ .map_err(|_| Error::UnableToCreateMetalContext)?,
);
- let gpu_sampler = Sampling::<Metal>::new(
- &context,
- intermediate_data_type,
- max_suffix_length,
- decoder_config.vocab_size,
- )
- .map_err(|_| Error::UnableToCreateMetalContext)?;
+ let gpu_sampler =
+ Sampling::<Metal>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)
+ .map_err(|_| Error::UnableToCreateMetalContext)?;
- let token_copy = TokenCopyKernel::new(&context)
- .map_err(|_| Error::UnableToCreateMetalContext)?;
+ let token_copy = TokenCopyKernel::new(&context).map_err(|_| Error::UnableToCreateMetalContext)?;
// Create mask update kernel if model has attention layers
let mask_update = if decoder_config.has_attention_layers() {
@@ -273,10 +227,8 @@ impl LanguageModelGeneratorContext {
None
};
- let async_batch_size =
- decoding_config.async_batch_size.resolve(model_path);
- let async_buffers =
- AsyncBuffers::new(&context, max_prefix_length, async_batch_size);
+ let async_batch_size = decoding_config.async_batch_size.resolve(model_path);
+ let async_buffers = AsyncBuffers::new(&context, max_prefix_length, async_batch_size);
let seed = PRng::new(decoding_config.sampling_seed.resolve());
@@ -302,11 +254,7 @@ impl LanguageModelGeneratorContext {
}
pub fn reset_command_buffer(&mut self) {
- self.command_buffer = self
- .mtl_context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
+ self.command_buffer =
+ self.mtl_context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
}
}
diff --git a/crates/uzu/src/backends/metal/media_utils.rs b/crates/uzu/src/backends/metal/media_utils.rs
index 6761abc4c..a29171d61 100644
--- a/crates/uzu/src/backends/metal/media_utils.rs
+++ b/crates/uzu/src/backends/metal/media_utils.rs
@@ -3,14 +3,11 @@ use std::{cell::RefCell, mem, rc::Rc};
use crate::{
DataType,
backends::metal::{
- MTLCommandBuffer, MTLContext, MTLDevice, MTLDeviceExt,
- MTLResourceOptions, MetalArray, ProtocolObject, Retained,
+ MTLCommandBuffer, MTLContext, MTLDevice, MTLDeviceExt, MTLResourceOptions, MetalArray, ProtocolObject,
+ Retained,
error::MTLError,
image::{Image, PixelFormat, TextureUsage},
- kernel::media_kernels::{
- ExtractImagePatches, ImageParameters, PatchParameters,
- ScalePadNormalizeImage,
- },
+ kernel::media_kernels::{ExtractImagePatches, ImageParameters, PatchParameters, ScalePadNormalizeImage},
},
};
@@ -43,10 +40,8 @@ fn calculate_preprocessing_dimensions(
let scaled_width = (input_width as f32 * scale_factor).ceil() as u32;
let scaled_height = (input_height as f32 * scale_factor).ceil() as u32;
- let padded_width =
- ((scaled_width + patch_size - 1) / patch_size) * patch_size;
- let padded_height =
- ((scaled_height + patch_size - 1) / patch_size) * patch_size;
+ let padded_width = ((scaled_width + patch_size - 1) / patch_size) * patch_size;
+ let padded_height = ((scaled_height + patch_size - 1) / patch_size) * patch_size;
(padded_width, padded_height, scale_factor)
}
@@ -93,23 +88,19 @@ pub fn calculate_image_preprocessing_requirements(
input_height: u32,
params: &ImagePreprocessingParams,
) -> ImagePreprocessingRequirements {
- let (padded_width, padded_height, _scale_factor) =
- calculate_preprocessing_dimensions(
- input_width,
- input_height,
- params.min_pixels,
- params.max_pixels,
- params.patch_size,
- );
+ let (padded_width, padded_height, _scale_factor) = calculate_preprocessing_dimensions(
+ input_width,
+ input_height,
+ params.min_pixels,
+ params.max_pixels,
+ params.patch_size,
+ );
let num_patches_x = padded_width / params.patch_size;
let num_patches_y = padded_height / params.patch_size;
let total_spatial_patches = num_patches_x * num_patches_y;
- let output_buffer_element_count = total_spatial_patches
- * params.temporal_slices
- * params.num_channels
- * params.patch_size
- * params.patch_size;
+ let output_buffer_element_count =
+ total_spatial_patches * params.temporal_slices * params.num_channels * params.patch_size * params.patch_size;
let intermediate_texture_usage = TextureUsage {
shader_read: true,
@@ -158,30 +149,12 @@ impl MetalImagePreprocessor {
let padded_height = requirements.padded_height;
let input_dimensions_data = [input_image.width(), input_image.height()];
- let mut input_dimensions_tensor = data_to_mtl_tensor(
- &self.device,
- &input_dimensions_data,
- vec![2],
- DataType::F32,
- )?;
- let mut image_mean_tensor = data_to_mtl_tensor(
- &self.device,
- &params.image_mean,
- vec![3],
- DataType::F32,
- )?;
- let mut image_std_tensor = data_to_mtl_tensor(
- &self.device,
- &params.image_std,
- vec![3],
- DataType::F32,
- )?;
- let mut padding_value_rgb_tensor = data_to_mtl_tensor(
- &self.device,
- &params.padding_value_rgb,
- vec![1],
- DataType::F32,
- )?;
+ let mut input_dimensions_tensor =
+ data_to_mtl_tensor(&self.device, &input_dimensions_data, vec![2], DataType::F32)?;
+ let mut image_mean_tensor = data_to_mtl_tensor(&self.device, &params.image_mean, vec![3], DataType::F32)?;
+ let mut image_std_tensor = data_to_mtl_tensor(&self.device, &params.image_std, vec![3], DataType::F32)?;
+ let mut padding_value_rgb_tensor =
+ data_to_mtl_tensor(&self.device, &params.padding_value_rgb, vec![1], DataType::F32)?;
let _spn_tensors: Vec<&mut MetalArray> = vec![
&mut input_dimensions_tensor,
@@ -204,34 +177,15 @@ impl MetalImagePreprocessor {
command_buffer,
);
- let mut final_output_buffer_array_ref =
- output_patch_buffer_cell.borrow_mut();
+ let mut final_output_buffer_array_ref = output_patch_buffer_cell.borrow_mut();
let padded_dimensions_data = [padded_width, padded_height];
- let mut padded_dimensions_tensor = data_to_mtl_tensor(
- &self.device,
- &padded_dimensions_data,
- vec![2],
- DataType::F32,
- )?;
- let mut patch_size_tensor = data_to_mtl_tensor(
- &self.device,
- &params.patch_size,
- vec![1],
- DataType::F32,
- )?;
- let mut num_channels_tensor = data_to_mtl_tensor(
- &self.device,
- &params.num_channels,
- vec![1],
- DataType::F32,
- )?;
- let mut temporal_slices_tensor = data_to_mtl_tensor(
- &self.device,
- &params.temporal_slices,
- vec![1],
- DataType::F32,
- )?;
+ let mut padded_dimensions_tensor =
+ data_to_mtl_tensor(&self.device, &padded_dimensions_data, vec![2], DataType::F32)?;
+ let mut patch_size_tensor = data_to_mtl_tensor(&self.device, &params.patch_size, vec![1], DataType::F32)?;
+ let mut num_channels_tensor = data_to_mtl_tensor(&self.device, &params.num_channels, vec![1], DataType::F32)?;
+ let mut temporal_slices_tensor =
+ data_to_mtl_tensor(&self.device, &params.temporal_slices, vec![1], DataType::F32)?;
let _eip_tensors: Vec<&mut MetalArray> = vec![
&mut *final_output_buffer_array_ref,
@@ -243,10 +197,7 @@ impl MetalImagePreprocessor {
let _eip_images = [intermediate_texture];
let patch_params = PatchParameters {
- padded_dimensions: [
- requirements.padded_width,
- requirements.padded_height,
- ],
+ padded_dimensions: [requirements.padded_width, requirements.padded_height],
patch_size: params.patch_size,
num_channels: params.num_channels,
temporal_slices: params.temporal_slices,
@@ -269,11 +220,8 @@ fn data_to_mtl_tensor<T: Copy>(
data_type: DataType,
) -> Result<MetalArray, MTLError> {
let size = mem::size_of::<T>();
- let bytes = unsafe {
- std::slice::from_raw_parts(data as *const T as *const u8, size)
- };
- let buffer = device
- .new_buffer_with_data(bytes, MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let bytes = unsafe { std::slice::from_raw_parts(data as *const T as *const u8, size) };
+ let buffer =
+ device.new_buffer_with_data(bytes, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");
unsafe { Ok(MetalArray::from_parts(buffer, 0, &shape, data_type)) }
}
diff --git a/crates/uzu/src/backends/metal/metal_extensions/command_buffer_timing.rs b/crates/uzu/src/backends/metal/metal_extensions/command_buffer_timing.rs
index d3e2df9b3..623408739 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/command_buffer_timing.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/command_buffer_timing.rs
@@ -1,6 +1,4 @@
-use crate::backends::metal::{
- MTLCommandBuffer, MTLCommandBufferExt, ProtocolObject,
-};
+use crate::backends::metal::{MTLCommandBuffer, MTLCommandBufferExt, ProtocolObject};
/// Extension trait providing convenience timing methods for command buffers.
pub trait CommandBufferTimingExt {
diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
index d68a37b57..c60bb24bd 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
@@ -1,9 +1,6 @@
use objc2::msg_send;
-use crate::backends::metal::{
- MTLBuffer, MTLCompareFunction, MTLComputeCommandEncoder, ProtocolObject,
- Retained,
-};
+use crate::backends::metal::{MTLBuffer, MTLCompareFunction, MTLComputeCommandEncoder, ProtocolObject, Retained};
/// Low-level, unsafe conditional control of Metal encoders.
/// This is internal; users should prefer the safe `ComputeEncoderConditional::condition`.
@@ -34,9 +31,7 @@ pub trait ComputeEncoderConditional {
ElseBlock: FnOnce();
}
-impl ComputeEncoderRawConditional
- for ProtocolObject<dyn MTLComputeCommandEncoder>
-{
+impl ComputeEncoderRawConditional for ProtocolObject<dyn MTLComputeCommandEncoder> {
fn encode_start_if(
&self,
predicate: &ProtocolObject<dyn MTLBuffer>,
diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_dispatch.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_dispatch.rs
index 5b4fb5d6c..ce11d74b8 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_dispatch.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_dispatch.rs
@@ -3,8 +3,7 @@ use super::{
device_extensions_features::{DeviceFeatures, Feature},
};
use crate::backends::metal::{
- MTLCommandEncoderExt, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLSize, ProtocolObject,
+ MTLCommandEncoderExt, MTLComputeCommandEncoder, MTLComputePipelineState, MTLSize, ProtocolObject,
};
/// Extensions for MTLComputeCommandEncoder to simplify dispatch operations
@@ -89,8 +88,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {
size: usize,
threadgroup_width: Option<usize>,
) {
- let tg_width = threadgroup_width
- .unwrap_or(state.thread_execution_width() as usize);
+ let tg_width = threadgroup_width.unwrap_or(state.thread_execution_width() as usize);
let tg_size = MTLSize {
width: tg_width,
height: 1,
@@ -114,8 +112,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {
threadgroup_width: Option<usize>,
) {
let tg_size = MTLSize {
- width: threadgroup_width
- .unwrap_or(state.thread_execution_width() as usize),
+ width: threadgroup_width.unwrap_or(state.thread_execution_width() as usize),
height: 1,
depth: 1,
};
@@ -152,8 +149,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {
size: MTLSize,
threadgroup_size: Option<MTLSize>,
) {
- let tg_size =
- threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());
+ let tg_size = threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());
let count = MTLSize {
width: (size.width + tg_size.width - 1) / tg_size.width,
@@ -171,8 +167,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {
size: MTLSize,
threadgroup_size: Option<MTLSize>,
) {
- let tg_size =
- threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());
+ let tg_size = threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());
self.set_compute_pipeline_state(state);
self.dispatch_threads(size, tg_size);
@@ -199,8 +194,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {
size: MTLSize,
threadgroup_size: Option<MTLSize>,
) {
- let tg_size =
- threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());
+ let tg_size = threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());
let count = MTLSize {
width: (size.width + tg_size.width - 1) / tg_size.width,
@@ -218,8 +212,7 @@ impl ComputeEncoderDispatch for ProtocolObject<dyn MTLComputeCommandEncoder> {
size: MTLSize,
threadgroup_size: Option<MTLSize>,
) {
- let tg_size =
- threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());
+ let tg_size = threadgroup_size.unwrap_or_else(|| state.max_2d_threadgroup_size());
self.set_compute_pipeline_state(state);
self.dispatch_threads(size, tg_size);
diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_set_value.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_set_value.rs
index a80e4b30a..6010fbf9a 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_set_value.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_set_value.rs
@@ -25,8 +25,7 @@ impl ComputeEncoderSetValue for ProtocolObject<dyn MTLComputeCommandEncoder> {
value: &T,
index: usize,
) {
- let ptr = NonNull::new(value as *const T as *mut c_void)
- .expect("value pointer should never be null");
+ let ptr = NonNull::new(value as *const T as *mut c_void).expect("value pointer should never be null");
self.set_bytes(ptr, size_of::<T>(), index);
}
@@ -35,8 +34,7 @@ impl ComputeEncoderSetValue for ProtocolObject<dyn MTLComputeCommandEncoder> {
slice: &[T],
index: usize,
) {
- let ptr = NonNull::new(slice.as_ptr() as *mut c_void)
- .expect("slice pointer should never be null");
+ let ptr = NonNull::new(slice.as_ptr() as *mut c_void).expect("slice pointer should never be null");
self.set_bytes(ptr, size_of::<T>() * slice.len(), index);
}
}
diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_pipeline_state_extensions_threads.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_pipeline_state_extensions_threads.rs
index e1a3d803a..8ab5944f7 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/compute_pipeline_state_extensions_threads.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_pipeline_state_extensions_threads.rs
@@ -1,6 +1,4 @@
-use crate::backends::metal::{
- MTLComputePipelineState, MTLSize, ProtocolObject,
-};
+use crate::backends::metal::{MTLComputePipelineState, MTLSize, ProtocolObject};
/// Extensions for ComputePipelineState to provide optimized threadgroup sizes
pub trait ComputePipelineStateThreads {
@@ -29,9 +27,7 @@ pub trait ComputePipelineStateThreads {
) -> MTLSize;
}
-impl ComputePipelineStateThreads
- for ProtocolObject<dyn MTLComputePipelineState>
-{
+impl ComputePipelineStateThreads for ProtocolObject<dyn MTLComputePipelineState> {
fn execution_width_threadgroup_size(&self) -> MTLSize {
let w = self.thread_execution_width();
diff --git a/crates/uzu/src/backends/metal/metal_extensions/device_extensions_features.rs b/crates/uzu/src/backends/metal/metal_extensions/device_extensions_features.rs
index 778d18a42..c732839a9 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/device_extensions_features.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/device_extensions_features.rs
@@ -1,8 +1,7 @@
use std::collections::HashSet;
use crate::backends::metal::{
- MTLDevice, MTLDeviceExt, MTLFeatureSet, MTLGPUFamily, MTLPixelFormat,
- MTLReadWriteTextureTier, ProtocolObject,
+ MTLDevice, MTLDeviceExt, MTLFeatureSet, MTLGPUFamily, MTLPixelFormat, MTLReadWriteTextureTier, ProtocolObject,
};
/// Enum representing various Metal features that may or may not be supported by a device.
@@ -44,9 +43,7 @@ impl DeviceFeatures for ProtocolObject<dyn MTLDevice> {
}
#[cfg(target_os = "macos")]
{
- self.supports_feature_set(
- MTLFeatureSet::macOS_GPUFamily1_v3,
- )
+ self.supports_feature_set(MTLFeatureSet::macOS_GPUFamily1_v3)
}
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
@@ -55,46 +52,37 @@ impl DeviceFeatures for ProtocolObject<dyn MTLDevice> {
},
Feature::TileShaders => self.supports_family(MTLGPUFamily::Apple4),
Feature::ReadWriteTextures(pixel_format) => {
- let tier_one_supported_formats: HashSet<MTLPixelFormat> = [
- MTLPixelFormat::R32Float,
- MTLPixelFormat::R32Uint,
- MTLPixelFormat::R32Sint,
- ]
- .into();
+ let tier_one_supported_formats: HashSet<MTLPixelFormat> =
+ [MTLPixelFormat::R32Float, MTLPixelFormat::R32Uint, MTLPixelFormat::R32Sint].into();
- let tier_two_supported_formats: HashSet<MTLPixelFormat> =
- tier_one_supported_formats
- .union(
- &[
- MTLPixelFormat::RGBA32Float,
- MTLPixelFormat::RGBA32Uint,
- MTLPixelFormat::RGBA32Sint,
- MTLPixelFormat::RGBA16Float,
- MTLPixelFormat::RGBA16Uint,
- MTLPixelFormat::RGBA16Sint,
- MTLPixelFormat::RGBA8Unorm,
- MTLPixelFormat::RGBA8Uint,
- MTLPixelFormat::RGBA8Sint,
- MTLPixelFormat::R16Float,
- MTLPixelFormat::R16Uint,
- MTLPixelFormat::R16Sint,
- MTLPixelFormat::R8Unorm,
- MTLPixelFormat::R8Uint,
- MTLPixelFormat::R8Sint,
- ]
- .into_iter()
- .collect(),
- )
- .cloned()
- .collect();
+ let tier_two_supported_formats: HashSet<MTLPixelFormat> = tier_one_supported_formats
+ .union(
+ &[
+ MTLPixelFormat::RGBA32Float,
+ MTLPixelFormat::RGBA32Uint,
+ MTLPixelFormat::RGBA32Sint,
+ MTLPixelFormat::RGBA16Float,
+ MTLPixelFormat::RGBA16Uint,
+ MTLPixelFormat::RGBA16Sint,
+ MTLPixelFormat::RGBA8Unorm,
+ MTLPixelFormat::RGBA8Uint,
+ MTLPixelFormat::RGBA8Sint,
+ MTLPixelFormat::R16Float,
+ MTLPixelFormat::R16Uint,
+ MTLPixelFormat::R16Sint,
+ MTLPixelFormat::R8Unorm,
+ MTLPixelFormat::R8Uint,
+ MTLPixelFormat::R8Sint,
+ ]
+ .into_iter()
+ .collect(),
+ )
+ .cloned()
+ .collect();
match self.read_write_texture_support() {
- MTLReadWriteTextureTier::Tier1 => {
- tier_one_supported_formats.contains(&pixel_format)
- },
- MTLReadWriteTextureTier::Tier2 => {
- tier_two_supported_formats.contains(&pixel_format)
- },
+ MTLReadWriteTextureTier::Tier1 => tier_one_supported_formats.contains(&pixel_format),
+ MTLReadWriteTextureTier::Tier2 => tier_two_supported_formats.contains(&pixel_format),
MTLReadWriteTextureTier::None => false,
}
},
diff --git a/crates/uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs b/crates/uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs
index 3ddf8c424..8fac470da 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs
@@ -2,8 +2,7 @@ use objc2::{rc::Retained, runtime::ProtocolObject};
use objc2_foundation::NSError;
use crate::backends::metal::{
- MTLComputePipelineState, MTLDeviceExt, MTLFunctionConstantValues,
- MTLLibrary, MTLLibraryExt,
+ MTLComputePipelineState, MTLDeviceExt, MTLFunctionConstantValues, MTLLibrary, MTLLibraryExt,
error::{LibraryError, MTLError},
};
@@ -23,16 +22,11 @@ impl LibraryPipelineExtensions for ProtocolObject<dyn MTLLibrary> {
&self,
function_name: &str,
constants: Option<&MTLFunctionConstantValues>,
- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
let function = match constants {
Some(const_values) => {
let mut error: *mut NSError = std::ptr::null_mut();
- self.new_function_with_name_constant_values_error(
- function_name,
- const_values,
- &mut error,
- )
+ self.new_function_with_name_constant_values_error(function_name, const_values, &mut error)
},
None => self.new_function_with_name(function_name),
}
@@ -40,14 +34,9 @@ impl LibraryPipelineExtensions for ProtocolObject<dyn MTLLibrary> {
let device = self.device();
- device.new_compute_pipeline_state_with_function(&function).map_err(
- |e| {
- MTLError::Library(LibraryError::Custom(format!(
- "Pipeline state creation failed: {}",
- e
- )))
- },
- )
+ device
+ .new_compute_pipeline_state_with_function(&function)
+ .map_err(|e| MTLError::Library(LibraryError::Custom(format!("Pipeline state creation failed: {}", e))))
}
}
@@ -56,8 +45,7 @@ impl LibraryPipelineExtensions for Retained<ProtocolObject<dyn MTLLibrary>> {
&self,
function_name: &str,
constants: Option<&MTLFunctionConstantValues>,
- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError>
- {
+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
(**self).compute_pipeline_state(function_name, constants)
}
}
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 6ca1d398b..33d97e6ca 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -19,17 +19,12 @@ pub use crate::backends::common::kernel::kv_cache_update::KVCacheUpdate;
pub use array::MetalArray;
pub use backend::Metal;
pub use classifier_context::ClassifierContext;
-pub use context::{
- DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext,
-};
+pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext};
pub use encodable_block::Decoder;
pub use error::MTLError;
pub use kernel::{KernelDataType, RopeKernel, dsl::MetalKernels};
pub use language_model_generator_context::LanguageModelGeneratorContext;
-pub use media_utils::{
- ImagePreprocessingParams, ImagePreprocessingRequirements,
- MetalImagePreprocessor,
-};
+pub use media_utils::{ImagePreprocessingParams, ImagePreprocessingRequirements, MetalImagePreprocessor};
pub use metal::prelude::*;
pub use metal_extensions::ComputeEncoderSetValue;
pub use placement_analysis::PlacementAnalysis;
diff --git a/crates/uzu/src/backends/metal/placement_analysis.rs b/crates/uzu/src/backends/metal/placement_analysis.rs
index 488ef21c0..f0c4f24f9 100644
--- a/crates/uzu/src/backends/metal/placement_analysis.rs
+++ b/crates/uzu/src/backends/metal/placement_analysis.rs
@@ -71,22 +71,13 @@ impl PlacementAnalysis {
let mut result = Self::default();
// Extract layer counts
- let re_layers = Regex::new(r"Layers Count: (\d+): ANE: (\d+) \(([\d\.e\-]+)%\), GPU: (\d+) \(([\d\.e\-]+)%\)").unwrap();
+ let re_layers =
+ Regex::new(r"Layers Count: (\d+): ANE: (\d+) \(([\d\.e\-]+)%\), GPU: (\d+) \(([\d\.e\-]+)%\)").unwrap();
if let Some(captures) = re_layers.captures(log_output) {
// println!("Found layer count information");
- if let (
- Some(total),
- Some(ane),
- Some(ane_pct),
- Some(gpu),
- Some(gpu_pct),
- ) = (
- captures.get(1),
- captures.get(2),
- captures.get(3),
- captures.get(4),
- captures.get(5),
- ) {
+ if let (Some(total), Some(ane), Some(ane_pct), Some(gpu), Some(gpu_pct)) =
+ (captures.get(1), captures.get(2), captures.get(3), captures.get(4), captures.get(5))
+ {
result.total_layers = total.as_str().parse().unwrap_or(0);
result.ane_layers = ane.as_str().parse().unwrap_or(0);
result.gpu_layers = gpu.as_str().parse().unwrap_or(0);
@@ -108,14 +99,9 @@ impl PlacementAnalysis {
// println!("Found unplaced operations section");
// The operations section begins after "Unplaced operations:"
- if let Some(ops_section) =
- log_output.split("Unplaced operations:").nth(1)
- {
+ if let Some(ops_section) = log_output.split("Unplaced operations:").nth(1) {
// The operations section ends at "Couldn't find any ANERegionCall operation."
- if let Some(ops_text) = ops_section
- .split("Couldn't find any ANERegionCall operation")
- .next()
- {
+ if let Some(ops_text) = ops_section.split("Couldn't find any ANERegionCall operation").next() {
// Process operations section
let mut current_op_type = String::new();
let mut _current_op_count = 0;
@@ -126,20 +112,12 @@ impl PlacementAnalysis {
// Process line by line
for line in ops_text.lines() {
// Line like "mps.multiply (6):" starts a new operation type section
- if let Some(caps) = Regex::new(r"(\w+\.\w+) \((\d+)\):")
- .unwrap()
- .captures(line)
- {
- if let (Some(op_type), Some(count)) =
- (caps.get(1), caps.get(2))
- {
+ if let Some(caps) = Regex::new(r"(\w+\.\w+) \((\d+)\):").unwrap().captures(line) {
+ if let (Some(op_type), Some(count)) = (caps.get(1), caps.get(2)) {
current_op_type = op_type.as_str().to_string();
- _current_op_count =
- count.as_str().parse().unwrap_or(0);
+ _current_op_count = count.as_str().parse().unwrap_or(0);
// Insert immediately to handle logs that omit per-op lines
- let trimmed = current_op_type
- .trim_start_matches("mps.")
- .to_string();
+ let trimmed = current_op_type.trim_start_matches("mps.").to_string();
unique_ops.insert(trimmed);
// println!(
// "Found operation type: {} ({})",
@@ -149,9 +127,7 @@ impl PlacementAnalysis {
// Lines starting with "%" contain actual operation instances
} else if line.trim().starts_with('%') {
// Store operation type **without** "mps." prefix so tests expect plain names
- let trimmed = current_op_type
- .trim_start_matches("mps.")
- .to_string();
+ let trimmed = current_op_type.trim_start_matches("mps.").to_string();
unique_ops.insert(trimmed);
}
}
@@ -170,9 +146,7 @@ impl PlacementAnalysis {
// For ANE operations, we would need to extract them if present
// Extract ANERegionCall occurrences so tests can validate presence
- let ane_region_re =
- Regex::new(r"ANERegionCall operations|ANERegionCall operation")
- .unwrap();
+ let ane_region_re = Regex::new(r"ANERegionCall operations|ANERegionCall operation").unwrap();
if ane_region_re.is_match(log_output) {
// We don't have detailed operation names – the tests only check for a placeholder string
// so insert a canonical name once if any ANERegionCall string appears.
@@ -225,9 +199,7 @@ mod tests {
"#;
let analysis = PlacementAnalysis::from_log_output(log_output);
assert_eq!(analysis.ane_layers, 3);
- assert!(
- analysis.ane_operations.contains(&"ANE region call".to_string())
- );
+ assert!(analysis.ane_operations.contains(&"ANE region call".to_string()));
}
#[test]
@@ -245,9 +217,7 @@ mod tests {
assert_eq!(analysis.gpu_layers, 5);
assert!(analysis.gpu_operations.contains(&"convolution".to_string()));
assert_eq!(analysis.gpu_operations.len(), 1);
- assert!(
- analysis.ane_operations.contains(&"ANE region call".to_string())
- );
+ assert!(analysis.ane_operations.contains(&"ANE region call".to_string()));
}
#[test]
diff --git a/crates/uzu/src/backends/metal/utils.rs b/crates/uzu/src/backends/metal/utils.rs
index 46306b3b0..59ba7a8b8 100644
--- a/crates/uzu/src/backends/metal/utils.rs
+++ b/crates/uzu/src/backends/metal/utils.rs
@@ -59,17 +59,11 @@ impl StdoutCapture {
self.stdout_thread = Some(std::thread::spawn(move || {
let mut buf = [0u8; 4096];
loop {
- let bytes_read = libc::read(
- read_fd,
- buf.as_mut_ptr() as *mut libc::c_void,
- buf.len(),
- );
+ let bytes_read = libc::read(read_fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len());
if bytes_read <= 0 {
break;
}
- if let Ok(s) =
- std::str::from_utf8(&buf[0..bytes_read as usize])
- {
+ if let Ok(s) = std::str::from_utf8(&buf[0..bytes_read as usize]) {
let mut b = buffer_clone.lock().unwrap();
b.push_str(s);
}
@@ -92,17 +86,11 @@ impl StdoutCapture {
self.stderr_thread = Some(std::thread::spawn(move || {
let mut buf = [0u8; 4096];
loop {
- let bytes_read = libc::read(
- read_fd,
- buf.as_mut_ptr() as *mut libc::c_void,
- buf.len(),
- );
+ let bytes_read = libc::read(read_fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len());
if bytes_read <= 0 {
break;
}
- if let Ok(s) =
- std::str::from_utf8(&buf[0..bytes_read as usize])
- {
+ if let Ok(s) = std::str::from_utf8(&buf[0..bytes_read as usize]) {
let mut b = buffer_clone.lock().unwrap();
b.push_str(s);
}
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 7aad798f7..a8db95eef 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -39,8 +39,7 @@ impl Classifier {
let forward_start = Instant::now();
#[cfg(feature = "tracing")]
- let (logits, _traces) =
- self.forward_pass_with_traces(&token_ids, &token_positions)?;
+ let (logits, _traces) = self.forward_pass_with_traces(&token_ids, &token_positions)?;
#[cfg(not(feature = "tracing"))]
let logits = self.forward_pass(&token_ids, &token_positions)?;
@@ -49,8 +48,7 @@ impl Classifier {
let postprocessing_start = Instant::now();
let probabilities = self.logits_to_probabilities(&logits)?;
- let postprocessing_duration =
- postprocessing_start.elapsed().as_secs_f64();
+ let postprocessing_duration = postprocessing_start.elapsed().as_secs_f64();
let (predicted_label, confidence) = probabilities
.iter()
@@ -108,16 +106,8 @@ impl Classifier {
let encoding_params = EncodingParameters::new(false, true, false);
- self.context.embed.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
- self.context.embedding_norm.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
+ self.context.embed.encode(&mut state, &self.context.command_buffer, &encoding_params);
+ self.context.embedding_norm.encode(&mut state, &self.context.command_buffer, &encoding_params);
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
@@ -129,17 +119,9 @@ impl Classifier {
}
for layer in self.context.layers.iter() {
- layer.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
+ layer.encode(&mut state, &self.context.command_buffer, &encoding_params);
}
- self.context.output_norm.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
+ self.context.output_norm.encode(&mut state, &self.context.command_buffer, &encoding_params);
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
@@ -150,17 +132,9 @@ impl Classifier {
);
}
- self.context.pooling.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
+ self.context.pooling.encode(&mut state, &self.context.command_buffer, &encoding_params);
- self.context.prediction_head.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
+ self.context.prediction_head.encode(&mut state, &self.context.command_buffer, &encoding_params);
self.context.command_buffer.commit();
self.context.command_buffer.wait_until_completed();
@@ -195,38 +169,14 @@ impl Classifier {
let encoding_params = EncodingParameters::new(false, true, false);
- self.context.embed.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
- self.context.embedding_norm.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
+ self.context.embed.encode(&mut state, &self.context.command_buffer, &encoding_params);
+ self.context.embedding_norm.encode(&mut state, &self.context.command_buffer, &encoding_params);
for layer in self.context.layers.iter() {
- layer.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
+ layer.encode(&mut state, &self.context.command_buffer, &encoding_params);
}
- self.context.output_norm.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
- self.context.pooling.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
- self.context.prediction_head.encode(
- &mut state,
- &self.context.command_buffer,
- &encoding_params,
- );
+ self.context.output_norm.encode(&mut state, &self.context.command_buffer, &encoding_params);
+ self.context.pooling.encode(&mut state, &self.context.command_buffer, &encoding_params);
+ self.context.prediction_head.encode(&mut state, &self.context.command_buffer, &encoding_params);
self.context.command_buffer.commit();
self.context.command_buffer.wait_until_completed();
@@ -241,8 +191,7 @@ impl Classifier {
&self,
state: &ForwardPassState<Metal>,
) -> Result<Box<[f32]>, Error> {
- let logits_arrays =
- state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
+ let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
let logits_array = logits_arrays[0].borrow();
let num_labels = self.context.model_config.model_config.num_labels;
@@ -255,19 +204,11 @@ impl Classifier {
},
DataType::F16 => {
let slice: &[half::f16] = bytemuck::cast_slice(buffer);
- Ok(slice[..num_labels]
- .iter()
- .map(|&x| x.to_f32())
- .collect::<Vec<_>>()
- .into_boxed_slice())
+ Ok(slice[..num_labels].iter().map(|&x| x.to_f32()).collect::<Vec<_>>().into_boxed_slice())
},
DataType::BF16 => {
let slice: &[half::bf16] = bytemuck::cast_slice(buffer);
- Ok(slice[..num_labels]
- .iter()
- .map(|&x| x.to_f32())
- .collect::<Vec<_>>()
- .into_boxed_slice())
+ Ok(slice[..num_labels].iter().map(|&x| x.to_f32()).collect::<Vec<_>>().into_boxed_slice())
},
_ => Err(Error::UnableToDecodeText),
}
@@ -277,18 +218,14 @@ impl Classifier {
&self,
logits: &[f32],
) -> Result<HashMap<String, f32>, Error> {
- let output_labels =
- &self.context.model_config.model_config.output_labels;
+ let output_labels = &self.context.model_config.model_config.output_labels;
let mut probabilities = HashMap::new();
for (idx, &logit) in logits.iter().enumerate() {
let prob = 1.0 / (1.0 + (-logit).exp());
let label = if let Some(labels) = output_labels {
- labels
- .get(idx)
- .map(|s| s.clone())
- .unwrap_or_else(|| format!("class_{}", idx))
+ labels.get(idx).map(|s| s.clone()).unwrap_or_else(|| format!("class_{}", idx))
} else {
format!("class_{}", idx)
};
diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rs
index 48d0f31b3..26d1020c5 100644
--- a/crates/uzu/src/classifier/mod.rs
+++ b/crates/uzu/src/classifier/mod.rs
@@ -7,8 +7,6 @@ pub use result::{ClassificationOutput, ClassificationStats};
// Re-export context from metal backend
pub use crate::backends::metal::ClassifierContext;
// Re-export encodables from metal backend for convenience
-pub use crate::backends::metal::encodable_block::{
- ClassifierLayer, ClassifierPredictionHead, Pooling,
-};
+pub use crate::backends::metal::encodable_block::{ClassifierLayer, ClassifierPredictionHead, Pooling};
#[cfg(feature = "tracing")]
pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};
diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rs
index ca1266c11..ff720dbab 100644
--- a/crates/uzu/src/config/attention.rs
+++ b/crates/uzu/src/config/attention.rs
@@ -108,8 +108,7 @@ mod tests {
has_out_biases: false,
};
- let deserialized_config: AttentionConfig =
- from_str(config_str).unwrap();
+ let deserialized_config: AttentionConfig = from_str(config_str).unwrap();
assert_eq!(deserialized_config, ground_truth_config);
}
}
diff --git a/crates/uzu/src/config/classifier/classifier_config.rs b/crates/uzu/src/config/classifier/classifier_config.rs
index e3e30e5b3..b6d2e4f06 100644
--- a/crates/uzu/src/config/classifier/classifier_config.rs
+++ b/crates/uzu/src/config/classifier/classifier_config.rs
@@ -2,8 +2,8 @@ use serde::{Deserialize, Serialize};
use super::{PoolingType, PredictionHeadConfig};
use crate::{
- DecoderConfig, DecoderLayerConfig, EmbeddingConfig, LinearConfig,
- NormalizationConfig, TransformerConfig, config::ConfigError,
+ DecoderConfig, DecoderLayerConfig, EmbeddingConfig, LinearConfig, NormalizationConfig, TransformerConfig,
+ config::ConfigError,
};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
@@ -37,24 +37,15 @@ impl ClassifierConfig {
pub fn to_decoder_config(&self) -> Result<DecoderConfig, ConfigError> {
// For classifier, we use the first layer config as the template
// (most classifiers have homogeneous layers)
- let first_layer = self
- .transformer_config
- .layer_configs
- .first()
- .ok_or(ConfigError::NoLayers)?;
+ let first_layer = self.transformer_config.layer_configs.first().ok_or(ConfigError::NoLayers)?;
let layer_config = DecoderLayerConfig {
- pre_attention_norm_config: first_layer
- .pre_attention_norm_config
- .clone()
- .unwrap_or_else(|| {
- // If first layer has no pre-attention norm (like ModernBERT), use a default
- self.transformer_config.output_norm_config.clone()
- }),
+ pre_attention_norm_config: first_layer.pre_attention_norm_config.clone().unwrap_or_else(|| {
+ // If first layer has no pre-attention norm (like ModernBERT), use a default
+ self.transformer_config.output_norm_config.clone()
+ }),
mixer_config: first_layer.mixer_config.clone(),
- post_attention_norm_config: first_layer
- .post_attention_norm_config
- .clone(),
+ post_attention_norm_config: first_layer.post_attention_norm_config.clone(),
pre_mlp_norm_config: first_layer.pre_mlp_norm_config.clone(),
mlp_config: first_layer.mlp_config.clone(),
post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),
@@ -62,12 +53,11 @@ impl ClassifierConfig {
let first_mixer = &first_layer.mixer_config;
- let num_heads =
- self.num_heads.or(first_mixer.num_heads()).ok_or_else(|| {
- ConfigError::MissingField("num_heads".to_string())
- })?;
- let num_groups =
- self.num_groups.or(first_mixer.num_groups()).unwrap_or(num_heads);
+ let num_heads = self
+ .num_heads
+ .or(first_mixer.num_heads())
+ .ok_or_else(|| ConfigError::MissingField("num_heads".to_string()))?;
+ let num_groups = self.num_groups.or(first_mixer.num_groups()).unwrap_or(num_heads);
let head_dim = self
.head_dim
.or(first_mixer.head_dim())
@@ -75,19 +65,10 @@ impl ClassifierConfig {
Ok(DecoderConfig {
embedding_config: self.embedding_config.clone(),
- global_rope_config: self
- .transformer_config
- .global_rope_config
- .clone(),
- local_rope_config: self
- .transformer_config
- .local_rope_config
- .clone(),
+ global_rope_config: self.transformer_config.global_rope_config.clone(),
+ local_rope_config: self.transformer_config.local_rope_config.clone(),
layer_config,
- output_norm_config: self
- .transformer_config
- .output_norm_config
- .clone(),
+ output_norm_config: self.transformer_config.output_norm_config.clone(),
vocab_size: self.vocab_size,
model_dim: self.model_dim,
hidden_dim: self.transformer_config.hidden_dim,
diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rs
index 7b23c74f5..b9ea318f2 100644
--- a/crates/uzu/src/config/decoder.rs
+++ b/crates/uzu/src/config/decoder.rs
@@ -77,19 +77,14 @@ impl<'de> Deserialize<'de> for DecoderConfig {
context_length,
} = raw;
- let layer_configs_boxed =
- layer_configs.map(|layers| layers.into_boxed_slice());
+ let layer_configs_boxed = layer_configs.map(|layers| layers.into_boxed_slice());
let layer_config_value = if let Some(config) = layer_config {
config
} else if let Some(configs) = layer_configs_boxed.as_ref() {
- configs.first().cloned().ok_or_else(|| {
- de::Error::custom("layer_configs must not be empty")
- })?
+ configs.first().cloned().ok_or_else(|| de::Error::custom("layer_configs must not be empty"))?
} else {
- return Err(de::Error::custom(
- "decoder config must include layer_config or layer_configs",
- ));
+ return Err(de::Error::custom("decoder config must include layer_config or layer_configs"));
};
let num_layers_value = if let Some(value) = num_layers {
@@ -97,23 +92,18 @@ impl<'de> Deserialize<'de> for DecoderConfig {
} else if let Some(configs) = layer_configs_boxed.as_ref() {
configs.len()
} else {
- return Err(de::Error::custom(
- "num_layers missing and layer_configs not provided",
- ));
+ return Err(de::Error::custom("num_layers missing and layer_configs not provided"));
};
- let (num_heads_value, num_groups_value, head_dim_value) =
- match (num_heads, num_groups, head_dim) {
- (Some(h), Some(g), Some(d)) => (h, g, d),
- _ => derive_dims_from_layer(&layer_config_value).ok_or_else(
- || {
- de::Error::custom(
- "num_heads/num_groups/head_dim missing and \
+ let (num_heads_value, num_groups_value, head_dim_value) = match (num_heads, num_groups, head_dim) {
+ (Some(h), Some(g), Some(d)) => (h, g, d),
+ _ => derive_dims_from_layer(&layer_config_value).ok_or_else(|| {
+ de::Error::custom(
+ "num_heads/num_groups/head_dim missing and \
cannot be derived from layer config",
- )
- },
- )?,
- };
+ )
+ })?,
+ };
let attention_scale_value = if attention_scale.is_some() {
attention_scale
@@ -121,35 +111,26 @@ impl<'de> Deserialize<'de> for DecoderConfig {
layer_config_value.mixer_config.attention_scale()
};
- let sliding_window_sizes_boxed =
- if let Some(sizes) = sliding_window_sizes {
- Some(sizes.into_boxed_slice())
- } else if let Some(configs) = layer_configs_boxed.as_ref() {
- Some(
- configs
- .iter()
- .map(|layer| layer.mixer_config.sliding_window_size())
- .collect::<Vec<_>>()
- .into_boxed_slice(),
- )
- } else {
- None
- };
+ let sliding_window_sizes_boxed = if let Some(sizes) = sliding_window_sizes {
+ Some(sizes.into_boxed_slice())
+ } else if let Some(configs) = layer_configs_boxed.as_ref() {
+ Some(
+ configs
+ .iter()
+ .map(|layer| layer.mixer_config.sliding_window_size())
+ .collect::<Vec<_>>()
+ .into_boxed_slice(),
+ )
+ } else {
+ None
+ };
- let explicit_layer_types =
- layer_types.map(|types| types.into_boxed_slice());
- let derived_layer_types =
- if let Some(configs) = layer_configs_boxed.as_ref() {
- Some(
- configs
- .iter()
- .map(layer_type_from_config)
- .collect::<Vec<_>>()
- .into_boxed_slice(),
- )
- } else {
- None
- };
+ let explicit_layer_types = layer_types.map(|types| types.into_boxed_slice());
+ let derived_layer_types = if let Some(configs) = layer_configs_boxed.as_ref() {
+ Some(configs.iter().map(layer_type_from_config).collect::<Vec<_>>().into_boxed_slice())
+ } else {
+ None
+ };
let layer_types_value = explicit_layer_types.or(derived_layer_types);
Ok(Self {
@@ -206,14 +187,8 @@ struct RawDecoderConfig {
context_length: usize,
}
-fn derive_dims_from_layer(
- layer: &DecoderLayerConfig
-) -> Option<(usize, usize, usize)> {
- Some((
- layer.mixer_config.num_heads()?,
- layer.mixer_config.num_groups()?,
- layer.mixer_config.head_dim()?,
- ))
+fn derive_dims_from_layer(layer: &DecoderLayerConfig) -> Option<(usize, usize, usize)> {
+ Some((layer.mixer_config.num_heads()?, layer.mixer_config.num_groups()?, layer.mixer_config.head_dim()?))
}
fn layer_type_from_config(layer: &DecoderLayerConfig) -> DecoderLayerType {
@@ -241,12 +216,7 @@ impl DecoderConfig {
self.layer_configs
.as_ref()
.map(|configs| {
- configs.iter().any(|config| {
- matches!(
- config.mixer_config,
- crate::config::MixerConfig::Attention(_)
- )
- })
+ configs.iter().any(|config| matches!(config.mixer_config, crate::config::MixerConfig::Attention(_)))
})
.unwrap_or(true)
}
@@ -420,9 +390,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -432,9 +400,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -458,9 +424,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
diff --git a/crates/uzu/src/config/decoder_layer.rs b/crates/uzu/src/config/decoder_layer.rs
index 41328a208..ccb9e2aca 100644
--- a/crates/uzu/src/config/decoder_layer.rs
+++ b/crates/uzu/src/config/decoder_layer.rs
@@ -1,8 +1,8 @@
use serde::{Deserialize, Serialize};
use super::{
- attention::AttentionConfig, mamba::Mamba2Config, mlp::MLPConfig,
- normalization::NormalizationConfig, short_conv::ShortConvConfig,
+ attention::AttentionConfig, mamba::Mamba2Config, mlp::MLPConfig, normalization::NormalizationConfig,
+ short_conv::ShortConvConfig,
};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
@@ -205,9 +205,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -217,9 +215,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -243,9 +239,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -264,8 +258,7 @@ mod tests {
post_mlp_norm_config: None,
};
- let deserialized_config: DecoderLayerConfig =
- from_str(config_str).unwrap();
+ let deserialized_config: DecoderLayerConfig = from_str(config_str).unwrap();
assert_eq!(deserialized_config, ground_truth_config);
}
}
diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rs
index 490d76e37..92ce2c57c 100644
--- a/crates/uzu/src/config/embedding.rs
+++ b/crates/uzu/src/config/embedding.rs
@@ -120,8 +120,7 @@ mod tests {
activation_precision: ConfigDataType::BFloat16,
};
- let deserialized_config: EmbeddingConfig =
- from_str(config_str).unwrap();
+ let deserialized_config: EmbeddingConfig = from_str(config_str).unwrap();
assert_eq!(deserialized_config, ground_truth_config);
let semi_config_str = r#"
@@ -173,8 +172,7 @@ mod tests {
activation_precision: ConfigDataType::BFloat16,
};
- let deserialized: EmbeddingConfig =
- from_str(mlx_quant_untied_str).unwrap();
+ let deserialized: EmbeddingConfig = from_str(mlx_quant_untied_str).unwrap();
assert_eq!(deserialized, mlx_quant_untied);
}
}
diff --git a/crates/uzu/src/config/language_model.rs b/crates/uzu/src/config/language_model.rs
index 7cdd741d6..7ae4de0bd 100644
--- a/crates/uzu/src/config/language_model.rs
+++ b/crates/uzu/src/config/language_model.rs
@@ -1,9 +1,8 @@
use serde::{Deserialize, Serialize};
use crate::{
- DecoderConfig, DecoderLayerConfig, DecoderLayerType, EmbeddingConfig,
- GenerationConfig, MessageProcessorConfig, MixerConfig, TransformerConfig,
- config::ConfigError,
+ DecoderConfig, DecoderLayerConfig, DecoderLayerType, EmbeddingConfig, GenerationConfig, MessageProcessorConfig,
+ MixerConfig, TransformerConfig, config::ConfigError,
};
struct AttentionDims {
@@ -27,8 +26,7 @@ impl InnerModelConfig {
pub fn to_decoder_config(&self) -> Result<DecoderConfig, ConfigError> {
let tf = &self.transformer_config;
- let first_layer =
- tf.layer_configs.first().ok_or(ConfigError::NoLayers)?;
+ let first_layer = tf.layer_configs.first().ok_or(ConfigError::NoLayers)?;
let layer_config = DecoderLayerConfig {
pre_attention_norm_config: first_layer
@@ -36,9 +34,7 @@ impl InnerModelConfig {
.clone()
.unwrap_or_else(|| tf.output_norm_config.clone()),
mixer_config: first_layer.mixer_config.clone(),
- post_attention_norm_config: first_layer
- .post_attention_norm_config
- .clone(),
+ post_attention_norm_config: first_layer.post_attention_norm_config.clone(),
pre_mlp_norm_config: first_layer.pre_mlp_norm_config.clone(),
mlp_config: first_layer.mlp_config.clone(),
post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),
@@ -70,9 +66,7 @@ impl InnerModelConfig {
.clone()
.unwrap_or_else(|| tf.output_norm_config.clone()),
mixer_config: layer.mixer_config.clone(),
- post_attention_norm_config: layer
- .post_attention_norm_config
- .clone(),
+ post_attention_norm_config: layer.post_attention_norm_config.clone(),
pre_mlp_norm_config: layer.pre_mlp_norm_config.clone(),
mlp_config: layer.mlp_config.clone(),
post_mlp_norm_config: layer.post_mlp_norm_config.clone(),
@@ -101,9 +95,7 @@ impl InnerModelConfig {
})
}
- fn derive_attention_dims(
- tf: &TransformerConfig
- ) -> Result<AttentionDims, ConfigError> {
+ fn derive_attention_dims(tf: &TransformerConfig) -> Result<AttentionDims, ConfigError> {
if let (Some(num_heads), Some(head_dim)) = (tf.num_heads, tf.head_dim) {
return Ok(AttentionDims {
num_heads,
@@ -113,17 +105,9 @@ impl InnerModelConfig {
});
}
- if let Some(attn) = tf
- .layer_configs
- .iter()
- .find_map(|layer| layer.mixer_config.as_attention())
- {
- let num_heads = attn.num_heads.ok_or_else(|| {
- ConfigError::MissingField("num_heads".to_string())
- })?;
- let head_dim = attn.head_dim.ok_or_else(|| {
- ConfigError::MissingField("head_dim".to_string())
- })?;
+ if let Some(attn) = tf.layer_configs.iter().find_map(|layer| layer.mixer_config.as_attention()) {
+ let num_heads = attn.num_heads.ok_or_else(|| ConfigError::MissingField("num_heads".to_string()))?;
+ let head_dim = attn.head_dim.ok_or_else(|| ConfigError::MissingField("head_dim".to_string()))?;
return Ok(AttentionDims {
num_heads,
num_groups: attn.num_groups.unwrap_or(num_heads),
@@ -132,11 +116,7 @@ impl InnerModelConfig {
});
}
- if let Some(mamba) = tf
- .layer_configs
- .iter()
- .find_map(|layer| layer.mixer_config.as_mamba())
- {
+ if let Some(mamba) = tf.layer_configs.iter().find_map(|layer| layer.mixer_config.as_mamba()) {
return Ok(AttentionDims {
num_heads: mamba.num_heads,
num_groups: mamba.num_groups,
diff --git a/crates/uzu/src/config/linear.rs b/crates/uzu/src/config/linear.rs
index 510eeb614..db8c21645 100644
--- a/crates/uzu/src/config/linear.rs
+++ b/crates/uzu/src/config/linear.rs
@@ -36,12 +36,8 @@ impl LinearConfig {
LinearConfig::FullPrecision {
precision,
} => *precision,
- LinearConfig::Quantized(quantization) => {
- quantization.activation_precision
- },
- LinearConfig::MLXQuantized(quantization) => {
- quantization.activation_precision
- },
+ LinearConfig::Quantized(quantization) => quantization.activation_precision,
+ LinearConfig::MLXQuantized(quantization) => quantization.activation_precision,
LinearConfig::QLoRA {
quantization,
..
diff --git a/crates/uzu/src/config/mlp.rs b/crates/uzu/src/config/mlp.rs
index cd1699f73..5ca9af14f 100644
--- a/crates/uzu/src/config/mlp.rs
+++ b/crates/uzu/src/config/mlp.rs
@@ -137,37 +137,34 @@ mod tests {
}
"#;
- let ground_truth_config =
- MLPConfig::MixtureOfExperts(MixtureOfExpertsConfig {
- mixture_size: 32,
- num_experts_per_token: 4,
- routing_function: RoutingFunctionConfig::SoftmaxRouting,
- router_config: LinearConfig::FullPrecision {
- precision: ConfigDataType::BFloat16,
- },
- router_has_biases: true,
- expert_config: MoeExpertConfig {
- linear_config: LinearConfig::QLoRA {
- quantization: QuantizationConfig {
- group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
- activation_precision: ConfigDataType::BFloat16,
- },
- lora_rank: 16,
- lora_scale: 2.0,
- },
- activation: Activation::SiLU {
- alpha: 1.0,
+ let ground_truth_config = MLPConfig::MixtureOfExperts(MixtureOfExpertsConfig {
+ mixture_size: 32,
+ num_experts_per_token: 4,
+ routing_function: RoutingFunctionConfig::SoftmaxRouting,
+ router_config: LinearConfig::FullPrecision {
+ precision: ConfigDataType::BFloat16,
+ },
+ router_has_biases: true,
+ expert_config: MoeExpertConfig {
+ linear_config: LinearConfig::QLoRA {
+ quantization: QuantizationConfig {
+ group_size: 32,
+ weight_quantization_mode: QuantizationMode::UInt4,
+ activation_quantization_mode: Some(QuantizationMode::Int8),
+ activation_precision: ConfigDataType::BFloat16,
},
- has_up_biases: true,
- has_down_biases: true,
- gate_clipping: [None, Some(7.0)],
- up_clipping: [-6.0, 8.0],
+ lora_rank: 16,
+ lora_scale: 2.0,
},
- });
+ activation: Activation::SiLU {
+ alpha: 1.0,
+ },
+ has_up_biases: true,
+ has_down_biases: true,
+ gate_clipping: [None, Some(7.0)],
+ up_clipping: [-6.0, 8.0],
+ },
+ });
let deserialized_config: MLPConfig = from_str(config_str).unwrap();
assert_eq!(deserialized_config, ground_truth_config);
diff --git a/crates/uzu/src/config/normalization/normalization_config.rs b/crates/uzu/src/config/normalization/normalization_config.rs
index 832ebb4a9..5d07d492c 100644
--- a/crates/uzu/src/config/normalization/normalization_config.rs
+++ b/crates/uzu/src/config/normalization/normalization_config.rs
@@ -41,8 +41,7 @@ mod tests {
subtract_mean: false,
};
- let deserialized_config: NormalizationConfig =
- from_str(config_str).unwrap();
+ let deserialized_config: NormalizationConfig = from_str(config_str).unwrap();
assert_eq!(deserialized_config, ground_truth_config);
}
}
diff --git a/crates/uzu/src/config/transformer_layer.rs b/crates/uzu/src/config/transformer_layer.rs
index d6809067d..791169dcb 100644
--- a/crates/uzu/src/config/transformer_layer.rs
+++ b/crates/uzu/src/config/transformer_layer.rs
@@ -133,9 +133,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -145,9 +143,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -171,9 +167,7 @@ mod tests {
quantization: QuantizationConfig {
group_size: 32,
weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(
- QuantizationMode::Int8,
- ),
+ activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -192,8 +186,7 @@ mod tests {
post_mlp_norm_config: None,
};
- let deserialized_config: TransformerLayerConfig =
- from_str(config_str).unwrap();
+ let deserialized_config: TransformerLayerConfig = from_str(config_str).unwrap();
assert_eq!(deserialized_config, ground_truth_config);
}
}
diff --git a/crates/uzu/src/data_type.rs b/crates/uzu/src/data_type.rs
index e2943aafa..a0abf72e0 100644
--- a/crates/uzu/src/data_type.rs
+++ b/crates/uzu/src/data_type.rs
@@ -44,19 +44,9 @@ impl DataType {
const fn dl_data_type_code(self) -> DLDataTypeCode {
match self {
DataType::BF16 => DLDataTypeCode::kDLBfloat,
- DataType::F16 | DataType::F32 | DataType::F64 => {
- DLDataTypeCode::kDLFloat
- },
- DataType::I4
- | DataType::I8
- | DataType::I16
- | DataType::I32
- | DataType::I64 => DLDataTypeCode::kDLInt,
- DataType::U4
- | DataType::U8
- | DataType::U16
- | DataType::U32
- | DataType::U64 => DLDataTypeCode::kDLUInt,
+ DataType::F16 | DataType::F32 | DataType::F64 => DLDataTypeCode::kDLFloat,
+ DataType::I4 | DataType::I8 | DataType::I16 | DataType::I32 | DataType::I64 => DLDataTypeCode::kDLInt,
+ DataType::U4 | DataType::U8 | DataType::U16 | DataType::U32 | DataType::U64 => DLDataTypeCode::kDLUInt,
}
}
}
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 87e85ed6f..96b16788b 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -70,8 +70,6 @@ pub trait EncodableBlock<B: Backend> {
command_buffer: &B::CommandBuffer,
parameters: &EncodingParameters<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.encode_with_shared_encoder(state, encoder, parameters)
- });
+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, encoder, parameters));
}
}
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index 51c775ace..2fdf2d843 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -22,12 +22,7 @@ impl<B: Backend> Sampling<B> {
max_batch_size: usize,
max_vocab_size: usize,
) -> Result<Self, SamplingError<B>> {
- let kernel = SamplingKernel::new(
- context,
- data_type,
- max_batch_size,
- max_vocab_size,
- )?;
+ let kernel = SamplingKernel::new(context, data_type, max_batch_size, max_vocab_size)?;
Ok(Self {
kernel,
})
@@ -40,13 +35,8 @@ impl<B: Backend> Sampling<B> {
max_vocab_size: usize,
argmax_strategy: ArgmaxStrategy,
) -> Result<Self, SamplingError<B>> {
- let kernel = SamplingKernel::new_with_strategy(
- context,
- data_type,
- max_batch_size,
- max_vocab_size,
- argmax_strategy,
- )?;
+ let kernel =
+ SamplingKernel::new_with_strategy(context, data_type, max_batch_size, max_vocab_size, argmax_strategy)?;
Ok(Self {
kernel,
})
@@ -60,10 +50,7 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {
encoder: &B::ComputeEncoder,
_parameters: &EncodingParameters<B>,
) {
- assert!(
- state.sampling_output().is_some(),
- "Sampling output buffer must be pre-allocated"
- );
+ assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");
let logits_binding = state.arrays(&[ArrayId::Logits]);
let logits = logits_binding[0].borrow();
@@ -82,19 +69,14 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {
let output_buffer_ref = state.sampling_output().unwrap().borrow();
let sampling_method = state.sampling_method().unwrap();
- let seeds_offset =
- seeds.offset() + sampling_start * std::mem::size_of::<u64>();
+ let seeds_offset = seeds.offset() + sampling_start * std::mem::size_of::<u64>();
- let (bitmask_buffer, bitmask_offset) =
- state.token_bitmask().map_or((None, 0usize), |cell| {
- let bitmask = cell.borrow();
- let bitmask_row_len = bitmask.shape()[1];
- let bitmask_offset = bitmask.offset()
- + sampling_start
- * bitmask_row_len
- * std::mem::size_of::<u32>();
- (Some(bitmask.buffer().clone()), bitmask_offset)
- });
+ let (bitmask_buffer, bitmask_offset) = state.token_bitmask().map_or((None, 0usize), |cell| {
+ let bitmask = cell.borrow();
+ let bitmask_row_len = bitmask.shape()[1];
+ let bitmask_offset = bitmask.offset() + sampling_start * bitmask_row_len * std::mem::size_of::<u32>();
+ (Some(bitmask.buffer().clone()), bitmask_offset)
+ });
if let Err(e) = self.kernel.encode_with_encoder(
logits.buffer(),
Some(seeds.buffer()),
diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rs
index a95909ae8..87d8b82df 100644
--- a/crates/uzu/src/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/forward_pass/cache_layers.rs
@@ -8,10 +8,7 @@ use crate::{
backends::common::{Backend, kernel::kv_cache_update::KVCacheUpdate},
config::DecoderLayerType,
forward_pass::{
- kv_cache_layer::{
- AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer,
- KVCacheLayerState, KVSlice,
- },
+ kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState, KVSlice},
model_shape::ModelShape,
short_conv_layer::ShortConvLayer,
ssm_layer::SSMLayer,
@@ -34,14 +31,10 @@ pub enum CacheLayerSlice<B: Backend> {
const ARRAY_TRANSFORMER_KEYS_LABEL: &str = "cache_layers_transformer_keys";
const ARRAY_TRANSFORMER_VALUES_LABEL: &str = "cache_layers_transformer_values";
-const ARRAY_STATE_SPACE_CONV_STATE_LABEL: &str =
- "cache_layers_state_space_conv_state";
-const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str =
- "cache_layers_state_space_ssm_state";
-const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str =
- "cache_layers_short_conv_conv_state";
-const ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL: &str =
- "cache_layers_short_conv_suffix_state";
+const ARRAY_STATE_SPACE_CONV_STATE_LABEL: &str = "cache_layers_state_space_conv_state";
+const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str = "cache_layers_state_space_ssm_state";
+const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str = "cache_layers_short_conv_conv_state";
+const ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL: &str = "cache_layers_short_conv_suffix_state";
impl<B: Backend> CacheLayer<B> {
pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {
@@ -106,9 +99,8 @@ impl<B: Backend> CacheLayers<B> {
max_suffix_length: usize,
) -> Self {
let total_context_length = max_prefix_length.max(max_suffix_length);
- let kv_shapes: Vec<[usize; 3]> = model_shape
- .kv_cache_layer_shapes(max_prefix_length, max_suffix_length)
- .collect();
+ let kv_shapes: Vec<[usize; 3]> =
+ model_shape.kv_cache_layer_shapes(max_prefix_length, max_suffix_length).collect();
let data: Box<[CacheLayer<B>]> = model_shape
.layer_types()
@@ -117,11 +109,8 @@ impl<B: Backend> CacheLayers<B> {
.map(|(layer_index, layer_type)| match layer_type {
DecoderLayerType::Transformer => {
let shape = kv_shapes[layer_index];
- let window_length = model_shape
- .sliding_window_length_per_layer[layer_index]
- .filter(|&window_size| {
- window_size < total_context_length
- });
+ let window_length = model_shape.sliding_window_length_per_layer[layer_index]
+ .filter(|&window_size| window_size < total_context_length);
let state = if let Some(w) = window_length {
KVCacheLayerState::Windowed {
@@ -140,16 +129,12 @@ impl<B: Backend> CacheLayers<B> {
keys: RefCell::new(context.create_array(
&shape,
model_shape.kv_cache_data_type(),
- &format!(
- "{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),
)),
values: RefCell::new(context.create_array(
&shape,
model_shape.kv_cache_data_type(),
- &format!(
- "{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),
)),
prefix_token_positions: match &state {
KVCacheLayerState::Full {
@@ -158,9 +143,7 @@ impl<B: Backend> CacheLayers<B> {
KVCacheLayerState::Windowed {
window_length,
..
- } => (0..*window_length)
- .map(|_| INVALID_POSITION)
- .collect(),
+ } => (0..*window_length).map(|_| INVALID_POSITION).collect(),
},
max_suffix_length,
})
@@ -181,47 +164,33 @@ impl<B: Backend> CacheLayers<B> {
conv_state: RefCell::new(context.create_array(
&conv_shape,
dtype,
- &format!(
- "{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),
)),
ssm_state: RefCell::new(context.create_array(
&ssm_shape,
dtype,
- &format!(
- "{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),
)),
})
},
DecoderLayerType::ShortConv {
kernel_size,
} => {
- let conv_shape = [
- model_shape.model_dim(),
- kernel_size.saturating_sub(1),
- ];
- let suffix_state_shape = [
- max_suffix_length,
- model_shape.model_dim(),
- kernel_size.saturating_sub(1),
- ];
+ let conv_shape = [model_shape.model_dim(), kernel_size.saturating_sub(1)];
+ let suffix_state_shape =
+ [max_suffix_length, model_shape.model_dim(), kernel_size.saturating_sub(1)];
let dtype = model_shape.activation_data_type();
CacheLayer::ShortConv(ShortConvLayer {
conv_state: RefCell::new(context.create_array(
&conv_shape,
dtype,
- &format!(
- "{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),
)),
suffix_state: RefCell::new(context.create_array(
&suffix_state_shape,
dtype,
- &format!(
- "{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),
)),
suffix_state_valid_start: Cell::new(0),
suffix_state_valid_len: Cell::new(0),
@@ -281,12 +250,7 @@ impl<B: Backend> CacheLayers<B> {
for layer in self.data.iter() {
if let CacheLayer::Transformer(layer) = layer {
if let Some(array) = dst.get_mut(&layer.window_length()) {
- layer.fill_attention_bias(
- array,
- suffix_token_positions,
- suffix_length,
- external_bias_fn,
- );
+ layer.fill_attention_bias(array, suffix_token_positions, suffix_length, external_bias_fn);
}
}
}
@@ -302,12 +266,7 @@ impl<B: Backend> CacheLayers<B> {
for layer in self.data.iter() {
if let CacheLayer::Transformer(layer) = layer {
if let Some(cell) = dst.get(&layer.window_length()) {
- layer.fill_attention_bias(
- &mut cell.borrow_mut(),
- suffix_token_positions,
- suffix_length,
- None,
- );
+ layer.fill_attention_bias(&mut cell.borrow_mut(), suffix_token_positions, suffix_length, None);
}
}
}
@@ -320,19 +279,12 @@ impl<B: Backend> CacheLayers<B> {
command_buffer: &B::CommandBuffer,
kv_cache_update: &KVCacheUpdate<B>,
) {
- let short_conv_commit_index =
- accepted_suffix_indices.last().copied().unwrap_or(0);
+ let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);
for layer in self.data.iter_mut() {
if let Some(layer) = layer.as_transformer_mut() {
- layer.update_after_acceptance(
- accepted_suffix_indices,
- suffix_start,
- command_buffer,
- kv_cache_update,
- );
+ layer.update_after_acceptance(accepted_suffix_indices, suffix_start, command_buffer, kv_cache_update);
} else if let Some(layer) = layer.as_short_conv() {
- layer
- .commit_from_suffix_state_if_valid(short_conv_commit_index);
+ layer.commit_from_suffix_state_if_valid(short_conv_commit_index);
}
}
}
@@ -355,9 +307,7 @@ impl<B: Backend> CacheLayers<B> {
self.data
.iter()
.filter_map(|layer| match layer {
- CacheLayer::Transformer(kv) => {
- kv.attention_bias_update_after_acceptance(accepted_len)
- },
+ CacheLayer::Transformer(kv) => kv.attention_bias_update_after_acceptance(accepted_len),
_ => None,
})
.collect()
@@ -377,12 +327,8 @@ impl<B: Backend> CacheLayers<B> {
};
layers.push(CacheLayerSlice::Transformer(slice));
},
- CacheLayer::StateSpace(_) => {
- layers.push(CacheLayerSlice::StateSpace)
- },
- CacheLayer::ShortConv(_) => {
- layers.push(CacheLayerSlice::ShortConv)
- },
+ CacheLayer::StateSpace(_) => layers.push(CacheLayerSlice::StateSpace),
+ CacheLayer::ShortConv(_) => layers.push(CacheLayerSlice::ShortConv),
}
}
@@ -398,10 +344,7 @@ impl<B: Backend> CacheLayers<B> {
) {
for (layer, snapshot) in self.data.iter_mut().zip(slice.layers.iter()) {
match (layer, snapshot) {
- (
- CacheLayer::Transformer(kv),
- CacheLayerSlice::Transformer(s),
- ) => {
+ (CacheLayer::Transformer(kv), CacheLayerSlice::Transformer(s)) => {
kv.apply_slice(&s, range.clone());
},
(CacheLayer::StateSpace(_), CacheLayerSlice::StateSpace) => {},
@@ -437,16 +380,12 @@ impl<B: Backend> CacheLayers<B> {
let mut new_keys = context.create_array(
&new_shape,
dtype,
- &format!(
- "{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),
);
let mut new_values = context.create_array(
&new_shape,
dtype,
- &format!(
- "{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),
);
if copy_rows > 0 {
@@ -464,9 +403,7 @@ impl<B: Backend> CacheLayers<B> {
state: layer.state.clone(),
keys: RefCell::new(new_keys),
values: RefCell::new(new_values),
- prefix_token_positions: layer
- .prefix_token_positions
- .clone(),
+ prefix_token_positions: layer.prefix_token_positions.clone(),
max_suffix_length: layer.max_suffix_length,
})
},
@@ -476,9 +413,7 @@ impl<B: Backend> CacheLayers<B> {
let mut new_conv = context.create_array(
&conv_shape,
conv_dtype,
- &format!(
- "{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),
);
{
let conv_src = layer.conv_state.borrow();
@@ -490,9 +425,7 @@ impl<B: Backend> CacheLayers<B> {
let mut new_ssm = context.create_array(
&ssm_shape,
ssm_dtype,
- &format!(
- "{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),
);
{
let ssm_src = layer.ssm_state.borrow();
@@ -510,24 +443,19 @@ impl<B: Backend> CacheLayers<B> {
let mut new_conv = context.create_array(
&conv_shape,
conv_dtype,
- &format!(
- "{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),
);
{
let conv_src = layer.conv_state.borrow();
new_conv.copy_from_array(&conv_src);
}
- let suffix_shape =
- layer.suffix_state.borrow().shape().to_vec();
+ let suffix_shape = layer.suffix_state.borrow().shape().to_vec();
let suffix_dtype = layer.suffix_state.borrow().data_type();
let mut new_suffix = context.create_array(
&suffix_shape,
suffix_dtype,
- &format!(
- "{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"
- ),
+ &format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),
);
{
new_suffix.as_bytes_mut().fill(0);
diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rs
index 37348a84e..5851bc3ce 100644
--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs
@@ -109,22 +109,13 @@ impl<B: Backend> KVCacheLayer<B> {
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
) {
let prefix_segment_length = self.prefix_segment_length();
- fill_attention_bias(
- dst,
- suffix_length,
- prefix_segment_length,
- |row_index, column_index| {
- if let Some(bias_fn) = external_bias_fn {
- bias_fn(row_index, column_index)
- } else {
- self.bias_should_be_neg_inf(
- row_index,
- column_index,
- suffix_token_positions,
- )
- }
- },
- );
+ fill_attention_bias(dst, suffix_length, prefix_segment_length, |row_index, column_index| {
+ if let Some(bias_fn) = external_bias_fn {
+ bias_fn(row_index, column_index)
+ } else {
+ self.bias_should_be_neg_inf(row_index, column_index, suffix_token_positions)
+ }
+ });
}
pub fn bias_should_be_neg_inf(
@@ -184,22 +175,14 @@ impl<B: Backend> KVCacheLayer<B> {
}
// Absolute positions of the *source* rows.
- let source_indices: Vec<usize> = accepted_suffix_indices
- .iter()
- .map(|i| i + suffix_start.unwrap_or(*prefix_len))
- .collect();
+ let source_indices: Vec<usize> =
+ accepted_suffix_indices.iter().map(|i| i + suffix_start.unwrap_or(*prefix_len)).collect();
// Absolute positions of the *destination* rows.
- let destination_indices: Vec<usize> = (*prefix_len
- ..*prefix_len + accepted_suffix_indices.len())
- .collect();
+ let destination_indices: Vec<usize> =
+ (*prefix_len..*prefix_len + accepted_suffix_indices.len()).collect();
- self.scatter_if_required(
- &source_indices,
- &destination_indices,
- command_buffer,
- kv_cache_update,
- );
+ self.scatter_if_required(&source_indices, &destination_indices, command_buffer, kv_cache_update);
},
KVCacheLayerState::Windowed {
@@ -207,31 +190,21 @@ impl<B: Backend> KVCacheLayer<B> {
ring_length,
window_length,
} => {
- let suffix_indices: Vec<usize> =
- if accepted_suffix_indices.is_empty() {
- vec![0]
- } else {
- accepted_suffix_indices.to_vec()
- };
+ let suffix_indices: Vec<usize> = if accepted_suffix_indices.is_empty() {
+ vec![0]
+ } else {
+ accepted_suffix_indices.to_vec()
+ };
- let source_indices: Vec<usize> =
- suffix_indices.iter().map(|i| i + *window_length).collect();
+ let source_indices: Vec<usize> = suffix_indices.iter().map(|i| i + *window_length).collect();
- let mut destination_indices =
- Vec::with_capacity(suffix_indices.len());
+ let mut destination_indices = Vec::with_capacity(suffix_indices.len());
for i in 0..suffix_indices.len() {
- destination_indices.push(
- (*ring_length + *ring_offset + i) % *window_length,
- );
+ destination_indices.push((*ring_length + *ring_offset + i) % *window_length);
}
- self.scatter_if_required(
- &source_indices,
- &destination_indices,
- command_buffer,
- kv_cache_update,
- );
+ self.scatter_if_required(&source_indices, &destination_indices, command_buffer, kv_cache_update);
},
}
}
@@ -260,12 +233,7 @@ impl<B: Backend> KVCacheLayer<B> {
value_shape: [v_shape[0], v_shape[1], v_shape[2]],
};
- let _ = kv_cache_update.encode(
- &[layer_data],
- source_indices,
- destination_indices,
- command_buffer,
- );
+ let _ = kv_cache_update.encode(&[layer_data], source_indices, destination_indices, command_buffer);
}
pub fn register_accepted_tokens(
@@ -286,8 +254,7 @@ impl<B: Backend> KVCacheLayer<B> {
} => {
for &token_pos in token_positions {
if *ring_length < *window_length {
- let dst =
- (*ring_offset + *ring_length) % *window_length;
+ let dst = (*ring_offset + *ring_length) % *window_length;
self.prefix_token_positions[dst] = token_pos;
*ring_length += 1;
@@ -319,17 +286,10 @@ impl<B: Backend> KVCacheLayer<B> {
window_length,
} => {
let newest_slot = (ring_length > 0)
- .then_some(
- (ring_offset + ring_length + window_length - 1)
- % window_length,
- )
+ .then_some((ring_offset + ring_length + window_length - 1) % window_length)
.unwrap_or(0);
- let unmask_col = (ring_length > 0)
- .then_some(newest_slot as i32)
- .unwrap_or(-1);
- let mask_col = (ring_length == window_length)
- .then_some(ring_offset as i32)
- .unwrap_or(-1);
+ let unmask_col = (ring_length > 0).then_some(newest_slot as i32).unwrap_or(-1);
+ let mask_col = (ring_length == window_length).then_some(ring_offset as i32).unwrap_or(-1);
Some(AttentionBiasUpdate {
key: Some(window_length),
@@ -370,16 +330,8 @@ impl<B: Backend> KVCacheLayer<B> {
let dtype = keys.data_type();
let slice_shape = [num_groups, len, head_dim];
- let mut slice_keys = context.create_array(
- &slice_shape,
- dtype,
- "kv_cache_layer_slice_keys",
- );
- let mut slice_values = context.create_array(
- &slice_shape,
- dtype,
- "kv_cache_layer_slice_values",
- );
+ let mut slice_keys = context.create_array(&slice_shape, dtype, "kv_cache_layer_slice_keys");
+ let mut slice_values = context.create_array(&slice_shape, dtype, "kv_cache_layer_slice_values");
let slots: Vec<usize> = (range.start..range.end)
.enumerate()
@@ -393,10 +345,7 @@ impl<B: Backend> KVCacheLayer<B> {
})
.collect();
- let positions: Vec<usize> = slots
- .iter()
- .map(|&s| self.prefix_token_positions[s])
- .collect();
+ let positions: Vec<usize> = slots.iter().map(|&s| self.prefix_token_positions[s]).collect();
for (i, &slot) in slots.iter().enumerate() {
slice_keys.copy_slice(&keys, 1, slot..slot + 1, i);
@@ -440,8 +389,7 @@ impl<B: Backend> KVCacheLayer<B> {
Some(r) => {
let accepted = r.start;
*prefix_len = base_prefix_len.saturating_add(accepted);
- let keep_positions =
- base_positions_len.saturating_add(accepted);
+ let keep_positions = base_positions_len.saturating_add(accepted);
self.prefix_token_positions.truncate(keep_positions);
},
},
@@ -508,20 +456,9 @@ impl<B: Backend> KVCacheLayer<B> {
for (i, &slot) in slots[r.clone()].iter().enumerate() {
let src_i = r.start + i;
- self.prefix_token_positions[slot] =
- positions[src_i];
- dst_keys.copy_slice(
- keys,
- 1,
- src_i..src_i + 1,
- slot,
- );
- dst_values.copy_slice(
- values,
- 1,
- src_i..src_i + 1,
- slot,
- );
+ self.prefix_token_positions[slot] = positions[src_i];
+ dst_keys.copy_slice(keys, 1, src_i..src_i + 1, slot);
+ dst_values.copy_slice(values, 1, src_i..src_i + 1, slot);
}
},
}
diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rs
index 32b32c0e5..12f0a3985 100644
--- a/crates/uzu/src/forward_pass/model_shape.rs
+++ b/crates/uzu/src/forward_pass/model_shape.rs
@@ -29,19 +29,14 @@ pub struct ModelShape {
impl ModelShape {
pub fn from_decoder_config(decoder_config: &DecoderConfig) -> Self {
- let activation_type: DataType =
- match &decoder_config.layer_config.mlp_config {
- crate::config::MLPConfig::Dense(d) => {
- d.linear_config.activation_precision().into()
- },
- crate::config::MLPConfig::MixtureOfExperts(m) => {
- m.expert_config.linear_config.activation_precision().into()
- },
- };
+ let activation_type: DataType = match &decoder_config.layer_config.mlp_config {
+ crate::config::MLPConfig::Dense(d) => d.linear_config.activation_precision().into(),
+ crate::config::MLPConfig::MixtureOfExperts(m) => {
+ m.expert_config.linear_config.activation_precision().into()
+ },
+ };
let num_layers = decoder_config.num_layers;
- let layer_types: Box<[DecoderLayerType]> = if let Some(layer_types) =
- &decoder_config.layer_types
- {
+ let layer_types: Box<[DecoderLayerType]> = if let Some(layer_types) = &decoder_config.layer_types {
assert_eq!(
layer_types.len(),
num_layers,
@@ -75,8 +70,7 @@ impl ModelShape {
max_mamba_head_dim = max_mamba_head_dim.max(*head_dim);
max_mamba_conv_dim = max_mamba_conv_dim.max(*conv_dim);
max_mamba_state_dim = max_mamba_state_dim.max(*state_dim);
- max_mamba_kernel_size =
- max_mamba_kernel_size.max(*kernel_size as usize);
+ max_mamba_kernel_size = max_mamba_kernel_size.max(*kernel_size as usize);
}
}
Self {
@@ -368,9 +362,7 @@ impl ModelShape {
}
pub fn has_short_conv_layers(&self) -> bool {
- self.layer_types.iter().any(|layer_type| {
- matches!(layer_type, DecoderLayerType::ShortConv { .. })
- })
+ self.layer_types.iter().any(|layer_type| matches!(layer_type, DecoderLayerType::ShortConv { .. }))
}
pub fn max_mamba_conv_dim(&self) -> Option<usize> {
@@ -409,9 +401,7 @@ impl ModelShape {
suffix_length: usize,
) -> Option<[usize; 2]> {
match (self.max_mamba_conv_dim(), self.max_mamba_kernel_size()) {
- (Some(conv_dim), Some(kernel_size)) if kernel_size > 0 => {
- Some([suffix_length + kernel_size - 1, conv_dim])
- },
+ (Some(conv_dim), Some(kernel_size)) if kernel_size > 0 => Some([suffix_length + kernel_size - 1, conv_dim]),
_ => None,
}
}
@@ -474,9 +464,7 @@ impl ModelShape {
suffix_length: usize,
) -> Option<[usize; 3]> {
if self.has_state_space_layers() {
- self.max_mamba_state_dim().map(|state_dim| {
- [suffix_length, self.max_mamba_groups, state_dim]
- })
+ self.max_mamba_state_dim().map(|state_dim| [suffix_length, self.max_mamba_groups, state_dim])
} else {
None
}
diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rs
index a203e0f31..e19a04dd9 100644
--- a/crates/uzu/src/forward_pass/scratch_buffers.rs
+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs
@@ -46,8 +46,8 @@ pub struct ScratchBuffers<B: Backend> {
// 2-pass attention intermediate buffers
pub attention_partials: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count * head_dim]
- pub attention_sums: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count]
- pub attention_maxs: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count]
+ pub attention_sums: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count]
+ pub attention_maxs: ArrayCell<B>, // [num_heads * max_suffix_len * total_blocks_count]
pub moe_topk_ids: Option<ArrayCell<B>>,
pub moe_topk_probs: Option<ArrayCell<B>>,
@@ -79,15 +79,10 @@ impl<B: Backend> ScratchBuffers<B> {
max_suffix_len: usize,
) -> Self {
// Helper closure for allocation
- let alloc =
- |shape: &[usize], dtype: DataType, label: &str| -> ArrayCell<B> {
- let array = context.create_array(
- shape,
- dtype,
- &format!("scratch_buffers_{label}"),
- );
- RefCell::new(array)
- };
+ let alloc = |shape: &[usize], dtype: DataType, label: &str| -> ArrayCell<B> {
+ let array = context.create_array(shape, dtype, &format!("scratch_buffers_{label}"));
+ RefCell::new(array)
+ };
let act_ty = model_shape.activation_data_type();
@@ -96,8 +91,7 @@ impl<B: Backend> ScratchBuffers<B> {
MLPConfig::MixtureOfExperts(moe) => Some(moe),
_ => None,
};
- let moe_max_routed =
- moe.map(|moe| max_suffix_len * moe.num_experts_per_token);
+ let moe_max_routed = moe.map(|moe| max_suffix_len * moe.num_experts_per_token);
let moe_scatter_entries = moe.map(|moe| {
let num_blocks = ((max_suffix_len + 255) / 256).max(1);
let num_tiles = ((moe.mixture_size + 512 - 1) / 512).max(1);
@@ -107,27 +101,11 @@ impl<B: Backend> ScratchBuffers<B> {
Self {
// 1-D
token_ids: alloc(&[max_suffix_len], DataType::U64, "token_ids"),
- token_positions: alloc(
- &[max_suffix_len],
- DataType::I32,
- "token_positions",
- ),
- token_parents: alloc(
- &[max_suffix_len],
- DataType::I32,
- "token_parents",
- ),
- token_bitmask: alloc(
- &model_shape.bitmask_shape(max_suffix_len),
- DataType::U32,
- "token_bitmask",
- ),
+ token_positions: alloc(&[max_suffix_len], DataType::I32, "token_positions"),
+ token_parents: alloc(&[max_suffix_len], DataType::I32, "token_parents"),
+ token_bitmask: alloc(&model_shape.bitmask_shape(max_suffix_len), DataType::U32, "token_bitmask"),
token_seeds: alloc(&[max_suffix_len], DataType::U64, "token_seeds"),
- sampling_output: alloc(
- &[max_suffix_len],
- DataType::U32,
- "sampling_output",
- ),
+ sampling_output: alloc(&[max_suffix_len], DataType::U32, "sampling_output"),
// 2-D
attention_window_size_to_bias: model_shape
@@ -141,89 +119,36 @@ impl<B: Backend> ScratchBuffers<B> {
Some(ws) => {
format!("attention_bias_for_window_size_{ws}")
},
- None => {
- "attention_bias_for_window_size_none".to_string()
- },
+ None => "attention_bias_for_window_size_none".to_string(),
};
- let shape =
- [max_suffix_len, max_suffix_len + max_prefix_len];
+ let shape = [max_suffix_len, max_suffix_len + max_prefix_len];
(window_size, alloc(&shape, act_ty, &label))
})
.collect(),
- logits: alloc(
- &model_shape.logits_shape(max_suffix_len),
- act_ty,
- "logits",
- ),
- main: alloc(
- &model_shape.main_shape(max_suffix_len),
- act_ty,
- "main",
- ),
- shortcut: alloc(
- &model_shape.main_shape(max_suffix_len),
- act_ty,
- "shortcut",
- ),
+ logits: alloc(&model_shape.logits_shape(max_suffix_len), act_ty, "logits"),
+ main: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "main"),
+ shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "shortcut"),
qkv: alloc(&model_shape.qkv_shape(max_suffix_len), act_ty, "qkv"),
- attention_output: alloc(
- &model_shape.attention_output_shape(max_suffix_len),
- act_ty,
- "attention_output",
- ),
- mlp_fused_up: alloc(
- &model_shape.mlp_fused_up_shape(max_suffix_len),
- act_ty,
- "mlp_fused_up",
- ),
- mlp_hidden: alloc(
- &model_shape.mlp_hidden_shape(max_suffix_len),
- act_ty,
- "mlp_hidden",
- ),
- ssm_inproj: model_shape
- .ssm_inproj_shape(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "ssm_inproj")),
- ssm_packed: model_shape
- .ssm_packed_shape(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "ssm_packed")),
+ attention_output: alloc(&model_shape.attention_output_shape(max_suffix_len), act_ty, "attention_output"),
+ mlp_fused_up: alloc(&model_shape.mlp_fused_up_shape(max_suffix_len), act_ty, "mlp_fused_up"),
+ mlp_hidden: alloc(&model_shape.mlp_hidden_shape(max_suffix_len), act_ty, "mlp_hidden"),
+ ssm_inproj: model_shape.ssm_inproj_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_inproj")),
+ ssm_packed: model_shape.ssm_packed_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_packed")),
ssm_conv_padded: model_shape
.ssm_conv_padded_shape(max_suffix_len)
.map(|shape| alloc(&shape, act_ty, "ssm_conv_padded")),
short_conv_padded: model_shape
.short_conv_padded_shape(max_suffix_len)
.map(|shape| alloc(&shape, act_ty, "short_conv_padded")),
- ssm_x: model_shape
- .ssm_x_shape(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "ssm_x")),
- ssm_b: model_shape
- .ssm_bc_shape(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "ssm_b")),
- ssm_c: model_shape
- .ssm_bc_shape(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "ssm_c")),
- ssm_dt: model_shape
- .ssm_dt_shape(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "ssm_dt")),
- ssm_z: model_shape
- .ssm_z_shape(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "ssm_z")),
+ ssm_x: model_shape.ssm_x_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_x")),
+ ssm_b: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_b")),
+ ssm_c: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_c")),
+ ssm_dt: model_shape.ssm_dt_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_dt")),
+ ssm_z: model_shape.ssm_z_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_z")),
// 3-D
- rotated_queries: alloc(
- &model_shape.rotated_queries_shape(max_suffix_len),
- act_ty,
- "rotated_queries",
- ),
- rotated_keys: alloc(
- &model_shape.rotated_keys_shape(max_suffix_len),
- act_ty,
- "rotated_keys",
- ),
- extracted_values: alloc(
- &model_shape.extracted_values_shape(max_suffix_len),
- act_ty,
- "extracted_values",
- ),
+ rotated_queries: alloc(&model_shape.rotated_queries_shape(max_suffix_len), act_ty, "rotated_queries"),
+ rotated_keys: alloc(&model_shape.rotated_keys_shape(max_suffix_len), act_ty, "rotated_keys"),
+ extracted_values: alloc(&model_shape.extracted_values_shape(max_suffix_len), act_ty, "extracted_values"),
attention_partials: alloc(
&model_shape.attention_partials_shape(max_suffix_len),
@@ -235,130 +160,56 @@ impl<B: Backend> ScratchBuffers<B> {
moe_topk_ids: moe.map(|moe| {
alloc(
- &model_shape.moe_topk_ids_shape(
- max_suffix_len,
- moe.num_experts_per_token,
- ),
+ &model_shape.moe_topk_ids_shape(max_suffix_len, moe.num_experts_per_token),
DataType::U32,
"moe_topk_ids",
)
}),
moe_topk_probs: moe.map(|moe| {
alloc(
- &model_shape.moe_topk_probs_shape(
- max_suffix_len,
- moe.num_experts_per_token,
- ),
+ &model_shape.moe_topk_probs_shape(max_suffix_len, moe.num_experts_per_token),
act_ty,
"moe_topk_probs",
)
}),
- moe_offsets: moe.map(|moe| {
- alloc(
- &model_shape.moe_offsets_shape(moe.mixture_size),
- DataType::U32,
- "moe_offsets",
- )
- }),
- moe_sumk: moe.map(|_| {
- alloc(&model_shape.moe_sumk_shape(), DataType::U32, "moe_sumk")
- }),
+ moe_offsets: moe
+ .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.mixture_size), DataType::U32, "moe_offsets")),
+ moe_sumk: moe.map(|_| alloc(&model_shape.moe_sumk_shape(), DataType::U32, "moe_sumk")),
moe_bucketed_token_ids: moe_max_routed.map(|max_routed| {
- alloc(
- &model_shape.moe_bucketed_token_ids_shape(max_routed),
- DataType::U32,
- "moe_bucketed_token_ids",
- )
+ alloc(&model_shape.moe_bucketed_token_ids_shape(max_routed), DataType::U32, "moe_bucketed_token_ids")
}),
moe_bucketed_probs: moe_max_routed.map(|max_routed| {
- alloc(
- &model_shape.moe_bucketed_probs_shape(max_routed),
- act_ty,
- "moe_bucketed_probs",
- )
- }),
- moe_x_perm: moe_max_routed.map(|max_routed| {
- alloc(
- &model_shape.moe_x_perm_shape(max_routed),
- DataType::F16,
- "moe_x_perm",
- )
+ alloc(&model_shape.moe_bucketed_probs_shape(max_routed), act_ty, "moe_bucketed_probs")
}),
+ moe_x_perm: moe_max_routed
+ .map(|max_routed| alloc(&model_shape.moe_x_perm_shape(max_routed), DataType::F16, "moe_x_perm")),
moe_tok2row: moe.map(|moe| {
alloc(
- &model_shape.moe_tok2row_shape(
- max_suffix_len,
- moe.num_experts_per_token,
- ),
+ &model_shape.moe_tok2row_shape(max_suffix_len, moe.num_experts_per_token),
DataType::I32,
"moe_tok2row",
)
}),
- moe_y_partial: moe_max_routed.map(|max_routed| {
- alloc(
- &model_shape.moe_y_partial_shape(max_routed),
- DataType::F16,
- "moe_y_partial",
- )
- }),
- moe_hidden: moe_max_routed.map(|max_routed| {
- alloc(
- &model_shape.moe_hidden_shape(max_routed),
- DataType::F32,
- "moe_hidden",
- )
- }),
- moe_two_pass_row_expert_map: moe_max_routed.map(|max_routed| {
- alloc(
- &[max_routed],
- DataType::U32,
- "moe_two_pass_row_expert_map",
- )
- }),
- moe_tile_counts: moe.map(|moe| {
- alloc(
- &model_shape.moe_counts_shape(moe.mixture_size),
- DataType::U32,
- "moe_tile_counts",
- )
- }),
- moe_tile_offsets: moe.map(|moe| {
- alloc(
- &model_shape.moe_offsets_shape(moe.mixture_size),
- DataType::U32,
- "moe_tile_offsets",
- )
- }),
- moe_tile_map: moe_max_routed.map(|max_routed| {
- alloc(
- &model_shape.moe_tile_map_shape(max_routed),
- DataType::U32,
- "moe_tile_map",
- )
- }),
- moe_total_tiles: moe.map(|_| {
- alloc(
- &model_shape.moe_total_tiles_shape(),
- DataType::U32,
- "moe_total_tiles",
- )
- }),
- moe_dispatch_args: moe.map(|_| {
- alloc(
- &model_shape.moe_dispatch_args_shape(),
- DataType::U32,
- "moe_dispatch_args",
- )
- }),
- moe_scatter_partials: moe_scatter_entries.map(|entries| {
- alloc(&[entries], DataType::U32, "moe_scatter_partials")
- }),
- moe_scatter_block_bases: moe_scatter_entries.map(|entries| {
- alloc(&[entries], DataType::U32, "moe_scatter_block_bases")
- }),
- moe_block_alloc: moe_scatter_entries.map(|entries| {
- alloc(&[entries], DataType::U32, "moe_block_alloc")
- }),
+ moe_y_partial: moe_max_routed
+ .map(|max_routed| alloc(&model_shape.moe_y_partial_shape(max_routed), DataType::F16, "moe_y_partial")),
+ moe_hidden: moe_max_routed
+ .map(|max_routed| alloc(&model_shape.moe_hidden_shape(max_routed), DataType::F32, "moe_hidden")),
+ moe_two_pass_row_expert_map: moe_max_routed
+ .map(|max_routed| alloc(&[max_routed], DataType::U32, "moe_two_pass_row_expert_map")),
+ moe_tile_counts: moe
+ .map(|moe| alloc(&model_shape.moe_counts_shape(moe.mixture_size), DataType::U32, "moe_tile_counts")),
+ moe_tile_offsets: moe
+ .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.mixture_size), DataType::U32, "moe_tile_offsets")),
+ moe_tile_map: moe_max_routed
+ .map(|max_routed| alloc(&model_shape.moe_tile_map_shape(max_routed), DataType::U32, "moe_tile_map")),
+ moe_total_tiles: moe.map(|_| alloc(&model_shape.moe_total_tiles_shape(), DataType::U32, "moe_total_tiles")),
+ moe_dispatch_args: moe
+ .map(|_| alloc(&model_shape.moe_dispatch_args_shape(), DataType::U32, "moe_dispatch_args")),
+ moe_scatter_partials: moe_scatter_entries
+ .map(|entries| alloc(&[entries], DataType::U32, "moe_scatter_partials")),
+ moe_scatter_block_bases: moe_scatter_entries
+ .map(|entries| alloc(&[entries], DataType::U32, "moe_scatter_block_bases")),
+ moe_block_alloc: moe_scatter_entries.map(|entries| alloc(&[entries], DataType::U32, "moe_block_alloc")),
}
}
}
diff --git a/crates/uzu/src/forward_pass/short_conv_layer.rs b/crates/uzu/src/forward_pass/short_conv_layer.rs
index b01cfeea4..8cba95a33 100644
--- a/crates/uzu/src/forward_pass/short_conv_layer.rs
+++ b/crates/uzu/src/forward_pass/short_conv_layer.rs
@@ -62,28 +62,17 @@ impl<B: Backend> ShortConvLayer<B> {
let mut conv = self.conv_state.borrow_mut();
let suffix = self.suffix_state.borrow();
- assert_eq!(
- conv.data_type(),
- suffix.data_type(),
- "ShortConv conv_state / suffix_state dtype mismatch"
- );
+ assert_eq!(conv.data_type(), suffix.data_type(), "ShortConv conv_state / suffix_state dtype mismatch");
let [model_dim, state_stride] = {
let shape = conv.shape();
- assert_eq!(
- shape.len(),
- 2,
- "ShortConv conv_state expected 2-D [model_dim, state_stride], got {:?}",
- shape
- );
+ assert_eq!(shape.len(), 2, "ShortConv conv_state expected 2-D [model_dim, state_stride], got {:?}", shape);
[shape[0], shape[1]]
};
let suffix_shape = suffix.shape();
assert!(
- suffix_shape.len() == 3
- && suffix_shape[1] == model_dim
- && suffix_shape[2] == state_stride,
+ suffix_shape.len() == 3 && suffix_shape[1] == model_dim && suffix_shape[2] == state_stride,
"ShortConv suffix_state expected 3-D [suffix_len, model_dim, state_stride], got {:?}",
suffix_shape
);
@@ -95,8 +84,7 @@ impl<B: Backend> ShortConvLayer<B> {
);
let elem_bytes = conv.data_type().size_in_bytes();
- let bytes_per_token =
- model_dim.saturating_mul(state_stride).saturating_mul(elem_bytes);
+ let bytes_per_token = model_dim.saturating_mul(state_stride).saturating_mul(elem_bytes);
let src_start = commit_index.saturating_mul(bytes_per_token);
let src_end = src_start.saturating_add(bytes_per_token);
diff --git a/crates/uzu/src/forward_pass/state/array_id.rs b/crates/uzu/src/forward_pass/state/array_id.rs
index 4a07ef521..b0b6af703 100644
--- a/crates/uzu/src/forward_pass/state/array_id.rs
+++ b/crates/uzu/src/forward_pass/state/array_id.rs
@@ -2,18 +2,7 @@ use serde::{Deserialize, Serialize};
use super::RopeType;
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- PartialOrd,
- Ord,
- Hash,
- Serialize,
- Deserialize,
-)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum ArrayId {
TokenIds,
TokenPositions,
diff --git a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
index 0323594ad..7c7b38beb 100644
--- a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
@@ -29,37 +29,17 @@ impl<B: Backend> CommonAuxBuffers<B> {
Self {
suffix_length,
main: scratch.main.view(&model_shape.main_shape(suffix_length)),
- shortcut: scratch
- .shortcut
- .view(&model_shape.main_shape(suffix_length)),
+ shortcut: scratch.shortcut.view(&model_shape.main_shape(suffix_length)),
qkv: scratch.qkv.view(&model_shape.qkv_shape(suffix_length)),
- attention_output: scratch
- .attention_output
- .view(&model_shape.attention_output_shape(suffix_length)),
- mlp_fused_up: scratch
- .mlp_fused_up
- .view(&model_shape.mlp_fused_up_shape(suffix_length)),
- mlp_hidden: scratch
- .mlp_hidden
- .view(&model_shape.mlp_hidden_shape(suffix_length)),
- rotated_queries: scratch
- .rotated_queries
- .view(&model_shape.rotated_queries_shape(suffix_length)),
- rotated_keys: scratch
- .rotated_keys
- .view(&model_shape.rotated_keys_shape(suffix_length)),
- extracted_values: scratch
- .extracted_values
- .view(&model_shape.extracted_values_shape(suffix_length)),
- attention_partials: scratch
- .attention_partials
- .view(&model_shape.attention_partials_shape(suffix_length)),
- attention_sums: scratch
- .attention_sums
- .view(&model_shape.attention_sums_shape(suffix_length)),
- attention_maxs: scratch
- .attention_maxs
- .view(&model_shape.attention_maxs_shape(suffix_length)),
+ attention_output: scratch.attention_output.view(&model_shape.attention_output_shape(suffix_length)),
+ mlp_fused_up: scratch.mlp_fused_up.view(&model_shape.mlp_fused_up_shape(suffix_length)),
+ mlp_hidden: scratch.mlp_hidden.view(&model_shape.mlp_hidden_shape(suffix_length)),
+ rotated_queries: scratch.rotated_queries.view(&model_shape.rotated_queries_shape(suffix_length)),
+ rotated_keys: scratch.rotated_keys.view(&model_shape.rotated_keys_shape(suffix_length)),
+ extracted_values: scratch.extracted_values.view(&model_shape.extracted_values_shape(suffix_length)),
+ attention_partials: scratch.attention_partials.view(&model_shape.attention_partials_shape(suffix_length)),
+ attention_sums: scratch.attention_sums.view(&model_shape.attention_sums_shape(suffix_length)),
+ attention_maxs: scratch.attention_maxs.view(&model_shape.attention_maxs_shape(suffix_length)),
}
}
}
diff --git a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
index 27bfedd02..47cdde3b4 100644
--- a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
@@ -49,14 +49,10 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {
MLPConfig::MixtureOfExperts(moe) => Some(moe),
_ => None,
};
- let max_routed =
- moe.map(|moe| suffix_length * moe.num_experts_per_token);
+ let max_routed = moe.map(|moe| suffix_length * moe.num_experts_per_token);
let num_blocks = suffix_length.div_ceil(256).max(1);
- let scatter_entries = moe.map(|moe| {
- num_blocks * moe.mixture_size.div_ceil(512).max(1) * 512
- });
- let block_alloc_entries =
- moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1));
+ let scatter_entries = moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1) * 512);
+ let block_alloc_entries = moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1));
Self {
ssm_inproj: scratch
@@ -104,83 +100,46 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {
.as_ref()
.zip(model_shape.ssm_z_shape(suffix_length))
.map(|(buf, shape)| buf.view(&shape)),
- moe_topk_ids: moe.zip(scratch.moe_topk_ids.as_ref()).map(
- |(moe, buf)| {
- buf.view(&model_shape.moe_topk_ids_shape(
- suffix_length,
- moe.num_experts_per_token,
- ))
- },
- ),
- moe_topk_probs: moe.zip(scratch.moe_topk_probs.as_ref()).map(
- |(moe, buf)| {
- buf.view(&model_shape.moe_topk_probs_shape(
- suffix_length,
- moe.num_experts_per_token,
- ))
- },
- ),
- moe_offsets: moe.zip(scratch.moe_offsets.as_ref()).map(
- |(moe, buf)| {
- buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))
- },
- ),
- moe_sumk: moe
- .and(scratch.moe_sumk.as_ref())
- .map(|buf| buf.view(&model_shape.moe_sumk_shape())),
+ moe_topk_ids: moe
+ .zip(scratch.moe_topk_ids.as_ref())
+ .map(|(moe, buf)| buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_experts_per_token))),
+ moe_topk_probs: moe.zip(scratch.moe_topk_probs.as_ref()).map(|(moe, buf)| {
+ buf.view(&model_shape.moe_topk_probs_shape(suffix_length, moe.num_experts_per_token))
+ }),
+ moe_offsets: moe
+ .zip(scratch.moe_offsets.as_ref())
+ .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))),
+ moe_sumk: moe.and(scratch.moe_sumk.as_ref()).map(|buf| buf.view(&model_shape.moe_sumk_shape())),
moe_bucketed_token_ids: max_routed
.zip(scratch.moe_bucketed_token_ids.as_ref())
- .map(|(max_routed, buf)| {
- buf.view(
- &model_shape.moe_bucketed_token_ids_shape(max_routed),
- )
- }),
+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_bucketed_token_ids_shape(max_routed))),
moe_bucketed_probs: max_routed
.zip(scratch.moe_bucketed_probs.as_ref())
- .map(|(max_routed, buf)| {
- buf.view(&model_shape.moe_bucketed_probs_shape(max_routed))
- }),
- moe_x_perm: max_routed.zip(scratch.moe_x_perm.as_ref()).map(
- |(max_routed, buf)| {
- buf.view(&model_shape.moe_x_perm_shape(max_routed))
- },
- ),
- moe_tok2row: moe.zip(scratch.moe_tok2row.as_ref()).map(
- |(moe, buf)| {
- buf.view(&model_shape.moe_tok2row_shape(
- suffix_length,
- moe.num_experts_per_token,
- ))
- },
- ),
- moe_y_partial: max_routed.zip(scratch.moe_y_partial.as_ref()).map(
- |(max_routed, buf)| {
- buf.view(&model_shape.moe_y_partial_shape(max_routed))
- },
- ),
- moe_hidden: max_routed.zip(scratch.moe_hidden.as_ref()).map(
- |(max_routed, buf)| {
- buf.view(&model_shape.moe_hidden_shape(max_routed))
- },
- ),
+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_bucketed_probs_shape(max_routed))),
+ moe_x_perm: max_routed
+ .zip(scratch.moe_x_perm.as_ref())
+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_x_perm_shape(max_routed))),
+ moe_tok2row: moe
+ .zip(scratch.moe_tok2row.as_ref())
+ .map(|(moe, buf)| buf.view(&model_shape.moe_tok2row_shape(suffix_length, moe.num_experts_per_token))),
+ moe_y_partial: max_routed
+ .zip(scratch.moe_y_partial.as_ref())
+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_y_partial_shape(max_routed))),
+ moe_hidden: max_routed
+ .zip(scratch.moe_hidden.as_ref())
+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_hidden_shape(max_routed))),
moe_two_pass_row_expert_map: max_routed
.zip(scratch.moe_two_pass_row_expert_map.as_ref())
.map(|(max_routed, buf)| buf.view(&[max_routed])),
- moe_tile_counts: moe.zip(scratch.moe_tile_counts.as_ref()).map(
- |(moe, buf)| {
- buf.view(&model_shape.moe_counts_shape(moe.mixture_size))
- },
- ),
- moe_tile_offsets: moe.zip(scratch.moe_tile_offsets.as_ref()).map(
- |(moe, buf)| {
- buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))
- },
- ),
- moe_tile_map: max_routed.zip(scratch.moe_tile_map.as_ref()).map(
- |(max_routed, buf)| {
- buf.view(&model_shape.moe_tile_map_shape(max_routed))
- },
- ),
+ moe_tile_counts: moe
+ .zip(scratch.moe_tile_counts.as_ref())
+ .map(|(moe, buf)| buf.view(&model_shape.moe_counts_shape(moe.mixture_size))),
+ moe_tile_offsets: moe
+ .zip(scratch.moe_tile_offsets.as_ref())
+ .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))),
+ moe_tile_map: max_routed
+ .zip(scratch.moe_tile_map.as_ref())
+ .map(|(max_routed, buf)| buf.view(&model_shape.moe_tile_map_shape(max_routed))),
moe_total_tiles: moe
.and(scratch.moe_total_tiles.as_ref())
.map(|buf| buf.view(&model_shape.moe_total_tiles_shape())),
diff --git a/crates/uzu/src/forward_pass/state/mod.rs b/crates/uzu/src/forward_pass/state/mod.rs
index 3aab46cde..efe1bbfc7 100644
--- a/crates/uzu/src/forward_pass/state/mod.rs
+++ b/crates/uzu/src/forward_pass/state/mod.rs
@@ -18,7 +18,4 @@ pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;
pub use rope_buffers::RopeBuffers;
pub use shared_buffers::SharedBuffers;
-pub use state::{
- ClassifierModeState, ForwardPassMode, ForwardPassState,
- LanguageModelGeneratorModeState,
-};
+pub use state::{ClassifierModeState, ForwardPassMode, ForwardPassState, LanguageModelGeneratorModeState};
diff --git a/crates/uzu/src/forward_pass/state/rope_type.rs b/crates/uzu/src/forward_pass/state/rope_type.rs
index ea16a8877..374238c75 100644
--- a/crates/uzu/src/forward_pass/state/rope_type.rs
+++ b/crates/uzu/src/forward_pass/state/rope_type.rs
@@ -1,17 +1,6 @@
use serde::{Deserialize, Serialize};
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- PartialOrd,
- Ord,
- Hash,
- Serialize,
- Deserialize,
-)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum RopeType {
Global,
Local,
diff --git a/crates/uzu/src/forward_pass/state/shared_buffers.rs b/crates/uzu/src/forward_pass/state/shared_buffers.rs
index fd7db21fb..bb1bb3bb3 100644
--- a/crates/uzu/src/forward_pass/state/shared_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/shared_buffers.rs
@@ -25,32 +25,22 @@ impl<B: Backend> SharedBuffers<B> {
decoder_config: &DecoderConfig,
model_shape: &ModelShape,
) -> Self {
- let global_rope = decoder_config
- .global_rope_config
- .is_some()
- .then(|| RopeBuffers::new(context, model_shape));
+ let global_rope = decoder_config.global_rope_config.is_some().then(|| RopeBuffers::new(context, model_shape));
- let local_rope = decoder_config
- .local_rope_config
- .is_some()
- .then(|| RopeBuffers::new(context, model_shape));
+ let local_rope = decoder_config.local_rope_config.is_some().then(|| RopeBuffers::new(context, model_shape));
- let attention_sinks = decoder_config
- .layer_config
- .attention_config()
- .is_some_and(|c| c.has_sinks)
- .then(|| {
- let num_heads = decoder_config.num_heads;
- (0..decoder_config.num_layers)
- .map(|_| {
- RefCell::new(context.create_array_uninitialized(
- &[num_heads],
- DataType::F32,
- "shared_buffers_attention_sinks",
- ))
- })
- .collect()
- });
+ let attention_sinks = decoder_config.layer_config.attention_config().is_some_and(|c| c.has_sinks).then(|| {
+ let num_heads = decoder_config.num_heads;
+ (0..decoder_config.num_layers)
+ .map(|_| {
+ RefCell::new(context.create_array_uninitialized(
+ &[num_heads],
+ DataType::F32,
+ "shared_buffers_attention_sinks",
+ ))
+ })
+ .collect()
+ });
Self {
global_rope,
@@ -63,9 +53,7 @@ impl<B: Backend> SharedBuffers<B> {
&mut self,
parameter_tree: &ParameterTree<B::Context>,
) {
- let transformer_tree = parameter_tree
- .subtree("transformer")
- .expect("transformer subtree not found");
+ let transformer_tree = parameter_tree.subtree("transformer").expect("transformer subtree not found");
if let Some(global_rope) = &mut self.global_rope {
global_rope.update_data(&transformer_tree, "global_rope");
@@ -76,9 +64,7 @@ impl<B: Backend> SharedBuffers<B> {
if let Some(sinks_vec) = &mut self.attention_sinks {
for (layer_idx, sink_cell) in sinks_vec.iter_mut().enumerate() {
- let layer_tree = transformer_tree
- .subtree(&format!("layers.{}", layer_idx))
- .unwrap();
+ let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_idx)).unwrap();
let attn_tree = layer_tree.subtree("mixer").unwrap();
let sinks_arr = attn_tree.leaf("sinks").unwrap();
let mut dst = sink_cell.borrow_mut();
@@ -91,25 +77,18 @@ impl<B: Backend> SharedBuffers<B> {
},
DataType::BF16 => {
let src = sinks_arr.as_slice::<bf16>();
- for (dst_val, src_val) in
- dst_slice.iter_mut().zip(src.iter())
- {
+ for (dst_val, src_val) in dst_slice.iter_mut().zip(src.iter()) {
*dst_val = f32::from(*src_val);
}
},
DataType::F16 => {
let src = sinks_arr.as_slice::<f16>();
- for (dst_val, src_val) in
- dst_slice.iter_mut().zip(src.iter())
- {
+ for (dst_val, src_val) in dst_slice.iter_mut().zip(src.iter()) {
*dst_val = f32::from(*src_val);
}
},
other => {
- panic!(
- "Unsupported attention sink data type: {:?}",
- other
- );
+ panic!("Unsupported attention sink data type: {:?}", other);
},
}
}
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index 19070e80c..9bd0fc6dc 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -11,10 +11,7 @@ use crate::{
cache_layers::CacheLayers,
model_shape::ModelShape,
scratch_buffers::ScratchBuffers,
- state::{
- ArrayId, CommonAuxBuffers, HashMapId,
- LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers,
- },
+ state::{ArrayId, CommonAuxBuffers, HashMapId, LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers},
},
session::parameter::SamplingMethod,
utils::attention::fill_attention_bias,
@@ -81,13 +78,9 @@ impl<B: Backend> ForwardPassState<B> {
token_positions: &[usize],
) -> ArrayCell<B> {
let suffix_length = token_positions.len();
- let token_positions_array =
- scratch.token_positions.view(&[suffix_length]);
- let token_positions_i32: Box<[i32]> =
- token_positions.iter().map(|p| *p as i32).collect();
- token_positions_array
- .borrow_mut()
- .copy_from_view(token_positions_i32.as_ref().into());
+ let token_positions_array = scratch.token_positions.view(&[suffix_length]);
+ let token_positions_i32: Box<[i32]> = token_positions.iter().map(|p| *p as i32).collect();
+ token_positions_array.borrow_mut().copy_from_view(token_positions_i32.as_ref().into());
token_positions_array
}
@@ -105,8 +98,7 @@ impl<B: Backend> ForwardPassState<B> {
parents.fill(-1);
if sampling_length > 0 {
- let root_pos =
- token_positions.get(sampling_start).copied().unwrap_or(0);
+ let root_pos = token_positions.get(sampling_start).copied().unwrap_or(0);
let mut stack: Vec<usize> = Vec::new();
@@ -122,11 +114,7 @@ impl<B: Backend> ForwardPassState<B> {
} else if let Some(&p) = stack.get(depth - 1) {
p as i32
} else {
- debug_assert!(
- false,
- "invalid trie depth ordering: depth={depth}, stack_len={}",
- stack.len()
- );
+ debug_assert!(false, "invalid trie depth ordering: depth={depth}, stack_len={}", stack.len());
-1
};
parents[abs_idx] = parent_local;
@@ -169,15 +157,8 @@ impl<B: Backend> ForwardPassState<B> {
async_seeds: Option<(&B::NativeBuffer, usize)>,
) -> Self {
let suffix_length = token_ids.len();
- assert_eq!(
- suffix_length,
- token_positions.len(),
- "Tokens and positions must have same length"
- );
- assert!(
- suffix_length <= cache_layers.borrow().max_suffix_length(),
- "Suffix length exceeds KV cache capacity"
- );
+ assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");
+ assert!(suffix_length <= cache_layers.borrow().max_suffix_length(), "Suffix length exceeds KV cache capacity");
// Token IDs - optionally skip copy for async path
let token_ids_cell = if skip_token_ids_copy {
@@ -187,29 +168,23 @@ impl<B: Backend> ForwardPassState<B> {
};
// Token positions - use async buffer if provided
- let token_positions_cell =
- if let Some((async_buf, offset)) = async_positions {
- let array = unsafe {
- Array::from_parts(
- async_buf.clone(),
- offset * std::mem::size_of::<i32>(),
- &[suffix_length],
- DataType::I32,
- )
- };
- RefCell::new(array)
- } else {
- Self::init_token_positions(scratch, token_positions)
+ let token_positions_cell = if let Some((async_buf, offset)) = async_positions {
+ let array = unsafe {
+ Array::from_parts(
+ async_buf.clone(),
+ offset * std::mem::size_of::<i32>(),
+ &[suffix_length],
+ DataType::I32,
+ )
};
+ RefCell::new(array)
+ } else {
+ Self::init_token_positions(scratch, token_positions)
+ };
// Trie parent indices (relative, within the sampling segment).
// Only meaningful when sampling_length > 0.
- let token_parents_cell = Self::init_token_parents(
- scratch,
- token_positions,
- sampling_start,
- sampling_length,
- );
+ let token_parents_cell = Self::init_token_parents(scratch, token_positions, sampling_start, sampling_length);
// Token bitmask
let token_bitmask_cell = token_bitmask.map(|bitmask| {
@@ -238,12 +213,10 @@ impl<B: Backend> ForwardPassState<B> {
};
// Logits
- let logits_cell =
- scratch.logits.view(&model_shape.logits_shape(suffix_length));
+ let logits_cell = scratch.logits.view(&model_shape.logits_shape(suffix_length));
// Sampling output
- let sampling_output =
- Some(scratch.sampling_output.view(&[suffix_length]));
+ let sampling_output = Some(scratch.sampling_output.view(&[suffix_length]));
// Attention bias (causal + sliding window)
let attention_bias = Self::init_llm_attention_bias(
@@ -256,40 +229,28 @@ impl<B: Backend> ForwardPassState<B> {
);
// Common aux buffers
- let common_aux =
- CommonAuxBuffers::new(scratch, model_shape, suffix_length);
+ let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);
// LLM-specific aux buffers
- let llm_aux = Some(LanguageModelGeneratorAuxBuffers::new(
- scratch,
- decoder_config,
- model_shape,
- suffix_length,
- ));
+ let llm_aux = Some(LanguageModelGeneratorAuxBuffers::new(scratch, decoder_config, model_shape, suffix_length));
// Traces
#[cfg(feature = "tracing")]
- let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(
- context.as_ref(),
- model_shape,
- suffix_length,
- )));
-
- let mode = ForwardPassMode::LanguageModelGenerator(
- LanguageModelGeneratorModeState {
- cache_layers,
- token_seeds: token_seeds_cell,
- logits: logits_cell,
- sampling_output,
- sampling_method: None,
- #[cfg(feature = "tracing")]
- traces,
- active_suffix_length,
- sampling_start,
- sampling_length,
- is_prefilling,
- },
- );
+ let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(context.as_ref(), model_shape, suffix_length)));
+
+ let mode = ForwardPassMode::LanguageModelGenerator(LanguageModelGeneratorModeState {
+ cache_layers,
+ token_seeds: token_seeds_cell,
+ logits: logits_cell,
+ sampling_output,
+ sampling_method: None,
+ #[cfg(feature = "tracing")]
+ traces,
+ active_suffix_length,
+ sampling_start,
+ sampling_length,
+ is_prefilling,
+ });
Self {
context,
token_ids: token_ids_cell,
@@ -317,10 +278,8 @@ impl<B: Backend> ForwardPassState<B> {
.attention_window_size_to_bias
.iter()
.map(|(window_size, buffer)| {
- let prefix_length =
- window_size.unwrap_or(cache_ref.max_prefix_length());
- let attention_bias_shape =
- [suffix_length, suffix_length + prefix_length];
+ let prefix_length = window_size.unwrap_or(cache_ref.max_prefix_length());
+ let attention_bias_shape = [suffix_length, suffix_length + prefix_length];
let array = buffer.borrow().view(&attention_bias_shape);
(*window_size, array)
})
@@ -339,10 +298,7 @@ impl<B: Backend> ForwardPassState<B> {
);
}
- attention_bias_map
- .into_iter()
- .map(|(k, v)| (k, RefCell::new(v)))
- .collect()
+ attention_bias_map.into_iter().map(|(k, v)| (k, RefCell::new(v))).collect()
}
// ========================================================================
@@ -364,31 +320,19 @@ impl<B: Backend> ForwardPassState<B> {
assert_eq!(suffix_length, token_positions.len());
let token_ids_cell = Self::init_token_ids(scratch, token_ids);
- let token_positions_cell =
- Self::init_token_positions(scratch, token_positions);
- let token_parents_cell =
- Self::init_token_parents(scratch, token_positions, 0, 0);
+ let token_positions_cell = Self::init_token_positions(scratch, token_positions);
+ let token_parents_cell = Self::init_token_parents(scratch, token_positions, 0, 0);
// Attention bias (bidirectional or causal)
- let attention_bias = Self::init_classifier_attention_bias(
- scratch,
- suffix_length,
- bidirectional_attention,
- );
+ let attention_bias = Self::init_classifier_attention_bias(scratch, suffix_length, bidirectional_attention);
// Common aux buffers
- let common_aux =
- CommonAuxBuffers::new(scratch, model_shape, suffix_length);
+ let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);
// Classifier-specific buffers
let model_dim = model_shape.main_shape(1)[1];
- let classifier_state = Self::init_classifier_buffers(
- context.as_ref(),
- model_shape,
- model_dim,
- num_labels,
- suffix_length,
- );
+ let classifier_state =
+ Self::init_classifier_buffers(context.as_ref(), model_shape, model_dim, num_labels, suffix_length);
let mode = ForwardPassMode::Classifier(classifier_state);
@@ -425,37 +369,19 @@ impl<B: Backend> ForwardPassState<B> {
if bidirectional_attention {
if let Some(window_size) = window {
let half_window = (window_size / 2) as isize;
- fill_attention_bias(
- bias_array,
- suffix_length,
- 0,
- |row, col| {
- let distance = (row as isize) - (col as isize);
- distance.abs() > half_window
- },
- );
+ fill_attention_bias(bias_array, suffix_length, 0, |row, col| {
+ let distance = (row as isize) - (col as isize);
+ distance.abs() > half_window
+ });
} else {
- fill_attention_bias(
- bias_array,
- suffix_length,
- 0,
- |_row, _col| false,
- );
+ fill_attention_bias(bias_array, suffix_length, 0, |_row, _col| false);
}
} else {
- fill_attention_bias(
- bias_array,
- suffix_length,
- 0,
- |row, col| row < col,
- );
+ fill_attention_bias(bias_array, suffix_length, 0, |row, col| row < col);
}
}
- attention_bias_map
- .into_iter()
- .map(|(k, v)| (k, RefCell::new(v)))
- .collect()
+ attention_bias_map.into_iter().map(|(k, v)| (k, RefCell::new(v))).collect()
}
#[cfg(feature = "tracing")]
@@ -471,17 +397,8 @@ impl<B: Backend> ForwardPassState<B> {
let create_buffer = |size: usize| -> ArrayCell<B> {
let buffer_size = size * data_type.size_in_bytes();
- let buffer = context
- .create_buffer(buffer_size)
- .expect("Failed to create buffer");
- RefCell::new(unsafe {
- Array::from_parts(
- buffer,
- 0,
- &[batch_size, size / batch_size],
- data_type,
- )
- })
+ let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
+ RefCell::new(unsafe { Array::from_parts(buffer, 0, &[batch_size, size / batch_size], data_type) })
};
ClassifierModeState {
@@ -489,19 +406,9 @@ impl<B: Backend> ForwardPassState<B> {
dense: create_buffer(batch_size * model_dim),
norm: create_buffer(batch_size * model_dim),
classifier_logits: {
- let buffer_size =
- batch_size * num_labels * data_type.size_in_bytes();
- let buffer = context
- .create_buffer(buffer_size)
- .expect("Failed to create buffer");
- RefCell::new(unsafe {
- Array::from_parts(
- buffer,
- 0,
- &[batch_size, num_labels],
- data_type,
- )
- })
+ let buffer_size = batch_size * num_labels * data_type.size_in_bytes();
+ let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
+ RefCell::new(unsafe { Array::from_parts(buffer, 0, &[batch_size, num_labels], data_type) })
},
traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(
context,
@@ -526,12 +433,8 @@ impl<B: Backend> ForwardPassState<B> {
let create_buffer = |dims: &[usize]| -> ArrayCell<B> {
let size: usize = dims.iter().product();
let buffer_size = size * data_type.size_in_bytes();
- let buffer = context
- .create_buffer(buffer_size)
- .expect("Failed to create buffer");
- RefCell::new(unsafe {
- Array::from_parts(buffer, 0, dims, data_type)
- })
+ let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
+ RefCell::new(unsafe { Array::from_parts(buffer, 0, dims, data_type) })
};
ClassifierModeState {
@@ -596,150 +499,88 @@ impl<B: Backend> ForwardPassState<B> {
ArrayId::TokenIds => self.token_ids.clone(),
ArrayId::TokenPositions => self.token_positions.clone(),
ArrayId::TokenParents => self.token_parents.clone(),
- ArrayId::TokenBitmask => {
- self.token_bitmask.clone().expect("Token bitmask not available")
- },
+ ArrayId::TokenBitmask => self.token_bitmask.clone().expect("Token bitmask not available"),
ArrayId::Main => self.common_aux.main.clone(),
ArrayId::Shortcut => self.common_aux.shortcut.clone(),
ArrayId::QKV => self.common_aux.qkv.clone(),
- ArrayId::AttentionOutput => {
- self.common_aux.attention_output.clone()
- },
+ ArrayId::AttentionOutput => self.common_aux.attention_output.clone(),
ArrayId::MlpFusedUp => self.common_aux.mlp_fused_up.clone(),
ArrayId::MlpHidden => self.common_aux.mlp_hidden.clone(),
ArrayId::RotatedQueries => self.common_aux.rotated_queries.clone(),
ArrayId::RotatedKeys => self.common_aux.rotated_keys.clone(),
- ArrayId::ExtractedValues => {
- self.common_aux.extracted_values.clone()
- },
- ArrayId::AttentionPartials => {
- self.common_aux.attention_partials.clone()
- },
+ ArrayId::ExtractedValues => self.common_aux.extracted_values.clone(),
+ ArrayId::AttentionPartials => self.common_aux.attention_partials.clone(),
ArrayId::AttentionSums => self.common_aux.attention_sums.clone(),
ArrayId::AttentionMaxs => self.common_aux.attention_maxs.clone(),
// Shared buffer arrays
- ArrayId::RopeCosines(_) | ArrayId::RopeSines(_) => self
- .shared_buffer_array(id)
- .expect("Shared buffer array should be available"),
+ ArrayId::RopeCosines(_) | ArrayId::RopeSines(_) => {
+ self.shared_buffer_array(id).expect("Shared buffer array should be available")
+ },
// LLM-specific arrays
ArrayId::Logits => self.llm_state().logits.clone(),
ArrayId::TokenSeeds => self.llm_state().token_seeds.clone(),
ArrayId::Keys(layer_index) => {
let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index]
- .as_transformer()
- .expect("Expected transformer layer")
- .keys
- .clone()
+ cache.data[layer_index].as_transformer().expect("Expected transformer layer").keys.clone()
},
ArrayId::Values(layer_index) => {
let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index]
- .as_transformer()
- .expect("Expected transformer layer")
- .values
+ cache.data[layer_index].as_transformer().expect("Expected transformer layer").values.clone()
+ },
+ ArrayId::AttentionSinks(layer_index) => {
+ self.shared_buffers.borrow().attention_sinks.as_ref().expect("Attention sinks not initialized")
+ [layer_index]
.clone()
},
- ArrayId::AttentionSinks(layer_index) => self
- .shared_buffers
- .borrow()
- .attention_sinks
- .as_ref()
- .expect("Attention sinks not initialized")[layer_index]
- .clone(),
// SSM arrays (LLM only)
- ArrayId::SsmInProj => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.ssm_inproj.clone())
- .expect("SSM inproj not initialized"),
- ArrayId::SsmPacked(_) => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.ssm_packed.clone())
- .expect("SSM packed not initialized"),
+ ArrayId::SsmInProj => {
+ self.llm_aux.as_ref().and_then(|aux| aux.ssm_inproj.clone()).expect("SSM inproj not initialized")
+ },
+ ArrayId::SsmPacked(_) => {
+ self.llm_aux.as_ref().and_then(|aux| aux.ssm_packed.clone()).expect("SSM packed not initialized")
+ },
ArrayId::SsmConvState(layer_index) => {
let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index]
- .as_state_space()
- .expect("Expected SSM layer")
- .conv_state
- .clone()
+ cache.data[layer_index].as_state_space().expect("Expected SSM layer").conv_state.clone()
},
ArrayId::SsmState(layer_index) => {
let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index]
- .as_state_space()
- .expect("Expected SSM layer")
- .ssm_state
- .clone()
+ cache.data[layer_index].as_state_space().expect("Expected SSM layer").ssm_state.clone()
},
- ArrayId::SsmX(_) => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.ssm_x.clone())
- .expect("SSM x not initialized"),
- ArrayId::SsmB(_) => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.ssm_b.clone())
- .expect("SSM b not initialized"),
- ArrayId::SsmC(_) => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.ssm_c.clone())
- .expect("SSM c not initialized"),
- ArrayId::SsmDt(_) => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.ssm_dt.clone())
- .expect("SSM dt not initialized"),
- ArrayId::SsmZ(_) => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.ssm_z.clone())
- .expect("SSM z not initialized"),
+ ArrayId::SsmX(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_x.clone()).expect("SSM x not initialized"),
+ ArrayId::SsmB(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_b.clone()).expect("SSM b not initialized"),
+ ArrayId::SsmC(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_c.clone()).expect("SSM c not initialized"),
+ ArrayId::SsmDt(_) => {
+ self.llm_aux.as_ref().and_then(|aux| aux.ssm_dt.clone()).expect("SSM dt not initialized")
+ },
+ ArrayId::SsmZ(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_z.clone()).expect("SSM z not initialized"),
ArrayId::ShortConvState(layer_index) => {
let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index]
- .as_short_conv()
- .expect("Expected ShortConv layer")
- .conv_state
- .clone()
+ cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").conv_state.clone()
},
ArrayId::ShortConvSuffixState(layer_index) => {
let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index]
- .as_short_conv()
- .expect("Expected ShortConv layer")
- .suffix_state
- .clone()
+ cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").suffix_state.clone()
},
// MoE arrays (LLM only)
- ArrayId::MoeTopkIds => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_topk_ids.clone())
- .expect("MoE topk_ids not initialized"),
+ ArrayId::MoeTopkIds => {
+ self.llm_aux.as_ref().and_then(|aux| aux.moe_topk_ids.clone()).expect("MoE topk_ids not initialized")
+ },
ArrayId::MoeTopkProbs => self
.llm_aux
.as_ref()
.and_then(|aux| aux.moe_topk_probs.clone())
.expect("MoE topk_probs not initialized"),
- ArrayId::MoeOffsets => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_offsets.clone())
- .expect("MoE offsets not initialized"),
- ArrayId::MoeSumK => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_sumk.clone())
- .expect("MoE sumk not initialized"),
+ ArrayId::MoeOffsets => {
+ self.llm_aux.as_ref().and_then(|aux| aux.moe_offsets.clone()).expect("MoE offsets not initialized")
+ },
+ ArrayId::MoeSumK => {
+ self.llm_aux.as_ref().and_then(|aux| aux.moe_sumk.clone()).expect("MoE sumk not initialized")
+ },
ArrayId::MoeBucketedTokenIds => self
.llm_aux
.as_ref()
@@ -750,26 +591,18 @@ impl<B: Backend> ForwardPassState<B> {
.as_ref()
.and_then(|aux| aux.moe_bucketed_probs.clone())
.expect("MoE bucketed_probs not initialized"),
- ArrayId::MoeXPerm => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_x_perm.clone())
- .expect("MoE x_perm not initialized"),
- ArrayId::MoeTok2Row => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_tok2row.clone())
- .expect("MoE tok2row not initialized"),
- ArrayId::MoeYPartial => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_y_partial.clone())
- .expect("MoE y_partial not initialized"),
- ArrayId::MoeHidden => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_hidden.clone())
- .expect("MoE hidden not initialized"),
+ ArrayId::MoeXPerm => {
+ self.llm_aux.as_ref().and_then(|aux| aux.moe_x_perm.clone()).expect("MoE x_perm not initialized")
+ },
+ ArrayId::MoeTok2Row => {
+ self.llm_aux.as_ref().and_then(|aux| aux.moe_tok2row.clone()).expect("MoE tok2row not initialized")
+ },
+ ArrayId::MoeYPartial => {
+ self.llm_aux.as_ref().and_then(|aux| aux.moe_y_partial.clone()).expect("MoE y_partial not initialized")
+ },
+ ArrayId::MoeHidden => {
+ self.llm_aux.as_ref().and_then(|aux| aux.moe_hidden.clone()).expect("MoE hidden not initialized")
+ },
ArrayId::MoeTwoPassRowExpertMap => self
.llm_aux
.as_ref()
@@ -785,11 +618,9 @@ impl<B: Backend> ForwardPassState<B> {
.as_ref()
.and_then(|aux| aux.moe_tile_offsets.clone())
.expect("MoE tile_offsets not initialized"),
- ArrayId::MoeTileMap => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_tile_map.clone())
- .expect("MoE tile_map not initialized"),
+ ArrayId::MoeTileMap => {
+ self.llm_aux.as_ref().and_then(|aux| aux.moe_tile_map.clone()).expect("MoE tile_map not initialized")
+ },
ArrayId::MoeTotalTiles => self
.llm_aux
.as_ref()
@@ -817,18 +648,10 @@ impl<B: Backend> ForwardPassState<B> {
.expect("MoE block_alloc not initialized"),
// Classifier-specific arrays
- ArrayId::ClassifierPooling => {
- self.classifier_state().pooling.clone()
- },
- ArrayId::ClassifierPredictionHeadDense => {
- self.classifier_state().dense.clone()
- },
- ArrayId::ClassifierPredictionHeadNorm => {
- self.classifier_state().norm.clone()
- },
- ArrayId::ClassifierPredictionHeadLogits => {
- self.classifier_state().classifier_logits.clone()
- },
+ ArrayId::ClassifierPooling => self.classifier_state().pooling.clone(),
+ ArrayId::ClassifierPredictionHeadDense => self.classifier_state().dense.clone(),
+ ArrayId::ClassifierPredictionHeadNorm => self.classifier_state().norm.clone(),
+ ArrayId::ClassifierPredictionHeadLogits => self.classifier_state().classifier_logits.clone(),
}
}
@@ -839,32 +662,12 @@ impl<B: Backend> ForwardPassState<B> {
let shared = self.shared_buffers.borrow();
match id {
ArrayId::RopeCosines(rope_type) => Some(match rope_type {
- RopeType::Global => shared
- .global_rope
- .as_ref()
- .expect("Global rope not initialized")
- .cosines
- .clone(),
- RopeType::Local => shared
- .local_rope
- .as_ref()
- .expect("Local rope not initialized")
- .cosines
- .clone(),
+ RopeType::Global => shared.global_rope.as_ref().expect("Global rope not initialized").cosines.clone(),
+ RopeType::Local => shared.local_rope.as_ref().expect("Local rope not initialized").cosines.clone(),
}),
ArrayId::RopeSines(rope_type) => Some(match rope_type {
- RopeType::Global => shared
- .global_rope
- .as_ref()
- .expect("Global rope not initialized")
- .sines
- .clone(),
- RopeType::Local => shared
- .local_rope
- .as_ref()
- .expect("Local rope not initialized")
- .sines
- .clone(),
+ RopeType::Global => shared.global_rope.as_ref().expect("Global rope not initialized").sines.clone(),
+ RopeType::Local => shared.local_rope.as_ref().expect("Local rope not initialized").sines.clone(),
}),
_ => None,
}
@@ -902,9 +705,7 @@ impl<B: Backend> ForwardPassState<B> {
pub fn active_suffix_length(&self) -> usize {
match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => {
- state.active_suffix_length
- },
+ ForwardPassMode::LanguageModelGenerator(state) => state.active_suffix_length,
ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,
}
}
@@ -912,9 +713,7 @@ impl<B: Backend> ForwardPassState<B> {
/// Start index (within the suffix batch) for which we need logits/sampling.
pub fn sampling_start(&self) -> usize {
match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => {
- state.sampling_start
- },
+ ForwardPassMode::LanguageModelGenerator(state) => state.sampling_start,
ForwardPassMode::Classifier(_) => 0,
}
}
@@ -922,36 +721,28 @@ impl<B: Backend> ForwardPassState<B> {
/// Number of batch items for which we need logits/sampling.
pub fn sampling_length(&self) -> usize {
match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => {
- state.sampling_length
- },
+ ForwardPassMode::LanguageModelGenerator(state) => state.sampling_length,
ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,
}
}
pub fn is_prefilling(&self) -> bool {
match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => {
- state.is_prefilling
- },
+ ForwardPassMode::LanguageModelGenerator(state) => state.is_prefilling,
ForwardPassMode::Classifier(_) => true,
}
}
pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers<B>>>> {
match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => {
- Some(&state.cache_layers)
- },
+ ForwardPassMode::LanguageModelGenerator(state) => Some(&state.cache_layers),
ForwardPassMode::Classifier(_) => None,
}
}
pub fn sampling_output(&self) -> Option<&ArrayCell<B>> {
match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => {
- state.sampling_output.as_ref()
- },
+ ForwardPassMode::LanguageModelGenerator(state) => state.sampling_output.as_ref(),
ForwardPassMode::Classifier(_) => None,
}
}
@@ -964,22 +755,16 @@ impl<B: Backend> ForwardPassState<B> {
}
}
- pub fn sampling_method_mut(
- &mut self
- ) -> Option<&mut Option<SamplingMethod>> {
+ pub fn sampling_method_mut(&mut self) -> Option<&mut Option<SamplingMethod>> {
match &mut self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => {
- Some(&mut state.sampling_method)
- },
+ ForwardPassMode::LanguageModelGenerator(state) => Some(&mut state.sampling_method),
ForwardPassMode::Classifier(_) => None,
}
}
pub fn sampling_method(&self) -> Option<SamplingMethod> {
match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => {
- state.sampling_method
- },
+ ForwardPassMode::LanguageModelGenerator(state) => state.sampling_method,
ForwardPassMode::Classifier(_) => None,
}
}
@@ -989,9 +774,7 @@ impl<B: Backend> ForwardPassState<B> {
source_array_id: ArrayId,
destination_array: RefCell<Array<B>>,
) {
- destination_array
- .borrow_mut()
- .copy_from_array(&self.arrays(&[source_array_id])[0].borrow());
+ destination_array.borrow_mut().copy_from_array(&self.arrays(&[source_array_id])[0].borrow());
}
pub fn encode_copy_array(
diff --git a/crates/uzu/src/forward_pass/traces.rs b/crates/uzu/src/forward_pass/traces.rs
index 2e6a4fc34..683b30957 100644
--- a/crates/uzu/src/forward_pass/traces.rs
+++ b/crates/uzu/src/forward_pass/traces.rs
@@ -22,13 +22,7 @@ fn create_layer_results<B: Backend>(
suffix_length: usize,
) -> Vec<Rc<RefCell<LayerActivationTrace<B>>>> {
(0..model_shape.num_layers)
- .map(|_| {
- Rc::new(RefCell::new(LayerActivationTrace::new(
- context,
- model_shape,
- suffix_length,
- )))
- })
+ .map(|_| Rc::new(RefCell::new(LayerActivationTrace::new(context, model_shape, suffix_length))))
.collect()
}
@@ -52,24 +46,13 @@ impl<B: Backend> LayerActivationTrace<B> {
) -> Self {
let main_shape = model_shape.main_shape(suffix_length);
let activation_data_type = model_shape.activation_data_type();
- let main = |label| {
- create_trace_array(
- context,
- &main_shape,
- activation_data_type,
- label,
- )
- };
+ let main = |label| create_trace_array(context, &main_shape, activation_data_type, label);
Self {
inputs: main("layer_activation_trace_inputs"),
- pre_attention_norm: main(
- "layer_activation_trace_pre_attention_norm",
- ),
+ pre_attention_norm: main("layer_activation_trace_pre_attention_norm"),
attention: main("layer_activation_trace_attention"),
- post_attention_norm: main(
- "layer_activation_trace_post_attention_norm",
- ),
+ post_attention_norm: main("layer_activation_trace_post_attention_norm"),
mlp_inputs: main("layer_activation_trace_mlp_inputs"),
pre_mlp_norm: main("layer_activation_trace_pre_mlp_norm"),
mlp: main("layer_activation_trace_mlp"),
@@ -95,18 +78,12 @@ impl<B: Backend> ActivationTrace<B> {
) -> Self {
let activation_data_type = model_shape.activation_data_type();
let main_shape = model_shape.main_shape(suffix_length);
- let layer_results =
- create_layer_results(context, model_shape, suffix_length);
+ let layer_results = create_layer_results(context, model_shape, suffix_length);
Self {
embedding_norm: None,
layer_results,
- output_norm: create_trace_array(
- context,
- &main_shape,
- activation_data_type,
- "activation_trace_output_norm",
- ),
+ output_norm: create_trace_array(context, &main_shape, activation_data_type, "activation_trace_output_norm"),
output_pooling: None,
logits: create_trace_array(
context,
@@ -125,8 +102,7 @@ impl<B: Backend> ActivationTrace<B> {
) -> Self {
let activation_data_type = model_shape.activation_data_type();
let main_shape = model_shape.main_shape(suffix_length);
- let layer_results =
- create_layer_results(context, model_shape, suffix_length);
+ let layer_results = create_layer_results(context, model_shape, suffix_length);
let model_dim = model_shape.main_shape(1)[1];
Self {
@@ -137,36 +113,22 @@ impl<B: Backend> ActivationTrace<B> {
"activation_trace_embedding_norm",
)),
layer_results,
- output_norm: create_trace_array(
- context,
- &main_shape,
- activation_data_type,
- "activation_trace_output_norm",
- ),
+ output_norm: create_trace_array(context, &main_shape, activation_data_type, "activation_trace_output_norm"),
output_pooling: Some(create_trace_array(
context,
&[1, model_dim],
activation_data_type,
"activation_trace_output_pooling",
)),
- logits: create_trace_array(
- context,
- &[1, num_labels],
- activation_data_type,
- "activation_trace_logits",
- ),
+ logits: create_trace_array(context, &[1, num_labels], activation_data_type, "activation_trace_logits"),
}
}
pub fn embedding_norm(&self) -> &ArrayCell<B> {
- self.embedding_norm
- .as_ref()
- .expect("embedding_norm is only available for classifier traces")
+ self.embedding_norm.as_ref().expect("embedding_norm is only available for classifier traces")
}
pub fn output_pooling(&self) -> &ArrayCell<B> {
- self.output_pooling
- .as_ref()
- .expect("output_pooling is only available for classifier traces")
+ self.output_pooling.as_ref().expect("output_pooling is only available for classifier traces")
}
}
diff --git a/crates/uzu/src/language_model/gpu_capture.rs b/crates/uzu/src/language_model/gpu_capture.rs
index f4ff73708..29b013ddb 100644
--- a/crates/uzu/src/language_model/gpu_capture.rs
+++ b/crates/uzu/src/language_model/gpu_capture.rs
@@ -6,10 +6,7 @@ use std::{
use objc2_foundation::{NSString, NSURL};
use crate::{
- backends::metal::{
- MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager,
- MTLCommandQueueExt, MTLContext,
- },
+ backends::metal::{MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueueExt, MTLContext},
utils::env_utils::MetalEnvVar,
};
@@ -22,10 +19,8 @@ pub struct GpuCaptureManager {
impl GpuCaptureManager {
pub fn new() -> Self {
- let capture_prefill_enabled =
- MetalEnvVar::CaptureFirstPrefill.is_enabled();
- let capture_decode_enabled =
- MetalEnvVar::CaptureFirstDecode.is_enabled();
+ let capture_prefill_enabled = MetalEnvVar::CaptureFirstPrefill.is_enabled();
+ let capture_decode_enabled = MetalEnvVar::CaptureFirstDecode.is_enabled();
// Enable Metal capture layer BEFORE device creation if any capture is requested
if capture_prefill_enabled || capture_decode_enabled {
@@ -46,18 +41,14 @@ impl GpuCaptureManager {
&self,
is_first_prefill: bool,
) -> bool {
- self.capture_prefill_enabled
- && is_first_prefill
- && !self.first_prefill_captured
+ self.capture_prefill_enabled && is_first_prefill && !self.first_prefill_captured
}
pub fn should_capture_decode(
&self,
is_first_decode: bool,
) -> bool {
- self.capture_decode_enabled
- && is_first_decode
- && !self.first_decode_captured
+ self.capture_decode_enabled && is_first_decode && !self.first_decode_captured
}
pub fn start_capture(
@@ -65,10 +56,8 @@ impl GpuCaptureManager {
mtl_context: &MTLContext,
capture_type: &str,
) -> Result<PathBuf, String> {
- let timestamp = SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .map_err(|e| format!("Time error: {}", e))?
- .as_secs();
+ let timestamp =
+ SystemTime::now().duration_since(UNIX_EPOCH).map_err(|e| format!("Time error: {}", e))?.as_secs();
let trace_path = std::env::current_dir()
.unwrap_or_else(|_| PathBuf::from("."))
@@ -76,15 +65,12 @@ impl GpuCaptureManager {
let capture_manager = MTLCaptureManager::shared_capture_manager();
let capture_descriptor = MTLCaptureDescriptor::new();
- capture_descriptor
- .set_destination(MTLCaptureDestination::GPUTraceDocument);
+ capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);
// Set output URL using NSURL
let url_string = format!("file://{}", trace_path.display());
unsafe {
- if let Some(url) =
- NSURL::URLWithString(&NSString::from_str(&url_string))
- {
+ if let Some(url) = NSURL::URLWithString(&NSString::from_str(&url_string)) {
capture_descriptor.set_output_url(Some(&url));
}
}
@@ -92,19 +78,14 @@ impl GpuCaptureManager {
mtl_context.command_queue.set_label(Some("uzu_command_queue"));
use objc2::rc::Retained;
unsafe {
- capture_descriptor.set_capture_object(Some(
- &*(Retained::as_ptr(&mtl_context.command_queue).cast()),
- ));
+ capture_descriptor.set_capture_object(Some(&*(Retained::as_ptr(&mtl_context.command_queue).cast())));
}
capture_manager
.start_capture_with_descriptor_error(&capture_descriptor)
.map_err(|e| format!("Failed to start GPU capture: {}", e))?;
- println!(
- "GPU capture started for first {}: {:?}",
- capture_type, trace_path
- );
+ println!("GPU capture started for first {}: {:?}", capture_type, trace_path);
Ok(trace_path)
}
diff --git a/crates/uzu/src/language_model/grammar.rs b/crates/uzu/src/language_model/grammar.rs
index c0dd86ca8..e53bd42ce 100644
--- a/crates/uzu/src/language_model/grammar.rs
+++ b/crates/uzu/src/language_model/grammar.rs
@@ -1,9 +1,6 @@
use std::iter::repeat_n;
-use xgrammar::{
- DLDataType, DLDevice, DLDeviceType, DLTensor, Grammar, GrammarCompiler,
- GrammarMatcher, TokenizerInfo,
-};
+use xgrammar::{DLDataType, DLDevice, DLDeviceType, DLTensor, Grammar, GrammarCompiler, GrammarMatcher, TokenizerInfo};
use crate::session::{config::GrammarConfig, types::Error};
@@ -55,10 +52,8 @@ impl CompiledGrammarEngagementState {
trigger_token_id: _,
trigger_distance,
} => {
- let num_grammar_tokens =
- usize::min(trigger_distance.unwrap_or(0), num_tokens);
- *trigger_distance =
- trigger_distance.and_then(|x| x.checked_sub(num_tokens));
+ let num_grammar_tokens = usize::min(trigger_distance.unwrap_or(0), num_tokens);
+ *trigger_distance = trigger_distance.and_then(|x| x.checked_sub(num_tokens));
num_grammar_tokens
},
}
@@ -97,18 +92,9 @@ impl CompiledGrammar {
separators,
strict_mode,
} => {
- let separators_ref =
- separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));
- Grammar::from_json_schema(
- schema,
- *any_whitespace,
- *indent,
- separators_ref,
- *strict_mode,
- None,
- false,
- )
- .map_err(|error_message| Error::GrammarError(error_message))?
+ let separators_ref = separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));
+ Grammar::from_json_schema(schema, *any_whitespace, *indent, separators_ref, *strict_mode, None, false)
+ .map_err(|error_message| Error::GrammarError(error_message))?
},
GrammarConfig::Regex {
pattern,
@@ -119,14 +105,12 @@ impl CompiledGrammar {
};
let mut compiler = GrammarCompiler::new(tokenizer_info, 8, true, -1)
.map_err(|error_message| Error::GrammarError(error_message))?;
- let compiled = compiler
- .compile_grammar(&grammar)
- .map_err(|error_message| Error::GrammarError(error_message))?;
+ let compiled =
+ compiler.compile_grammar(&grammar).map_err(|error_message| Error::GrammarError(error_message))?;
let matcher = GrammarMatcher::new(&compiled, None, true, -1)
.map_err(|error_message| Error::GrammarError(error_message))?;
- let engagement_state = if let Some(trigger_token_id) = trigger_token_id
- {
+ let engagement_state = if let Some(trigger_token_id) = trigger_token_id {
CompiledGrammarEngagementState::Triggered {
trigger_token_id,
trigger_distance: None,
@@ -144,8 +128,7 @@ impl CompiledGrammar {
pub fn next_bitmask(&mut self) -> Result<Option<Box<[u32]>>, Error> {
if self.engagement_state.is_engaged() {
- let mut cpu_mask = repeat_n(0, self.vocab_size.div_ceil(32))
- .collect::<Box<[u32]>>();
+ let mut cpu_mask = repeat_n(0, self.vocab_size.div_ceil(32)).collect::<Box<[u32]>>();
let batch_mask_slice = &mut cpu_mask;
let mut shape_i64 = [self.vocab_size.div_ceil(32) as i64];
let mut bitmask_tensor = DLTensor {
@@ -165,15 +148,9 @@ impl CompiledGrammar {
byte_offset: 0,
};
- let success = self.matcher.fill_next_token_bitmask(
- &mut bitmask_tensor,
- 0,
- false,
- );
+ let success = self.matcher.fill_next_token_bitmask(&mut bitmask_tensor, 0, false);
if !success {
- return Err(Error::GrammarError(
- "Failed to fill next token bitmask".to_string(),
- ));
+ return Err(Error::GrammarError("Failed to fill next token bitmask".to_string()));
}
Ok(Some(cpu_mask))
@@ -188,10 +165,7 @@ impl CompiledGrammar {
) -> Result<(), Error> {
if self.engagement_state.is_engaged() {
if (token_id as usize) >= self.vocab_size {
- return Err(Error::TokenOutOfGrammarRange(
- token_id,
- self.vocab_size,
- ));
+ return Err(Error::TokenOutOfGrammarRange(token_id, self.vocab_size));
}
if !self.matcher.accept_token(token_id as i32) {
diff --git a/crates/uzu/src/language_model/gumbel.rs b/crates/uzu/src/language_model/gumbel.rs
index 73f371a9e..b2fcb18c6 100644
--- a/crates/uzu/src/language_model/gumbel.rs
+++ b/crates/uzu/src/language_model/gumbel.rs
@@ -69,9 +69,7 @@ pub fn revidx(logit_idx: u32) -> (u32, u32) {
let grain_idx = logit_idx_in_blockgrain / BLOCK_SIZE;
let local_idx = logit_idx_in_blockgrain % BLOCK_SIZE;
- let rng_offset = (global_idx * BLOCK_SIZE + local_idx)
- * (GRAIN_SIZE + WORDS_PER_OFFSET - 1)
- / WORDS_PER_OFFSET
+ let rng_offset = (global_idx * BLOCK_SIZE + local_idx) * (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET
+ grain_idx / WORDS_PER_OFFSET;
let rng_word = grain_idx % WORDS_PER_OFFSET;
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index a0e76b611..08443e847 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -1,6 +1,4 @@
-use std::{
- collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant,
-};
+use std::{collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant};
use itertools::{Either, Itertools, izip};
@@ -16,8 +14,8 @@ use crate::{
backends::{
common::kernel::MaskUpdateKernel,
metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,
- MTLCommandBufferHandler, MTLCommandEncoder, MTLCommandQueue, Metal,
+ MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler, MTLCommandEncoder,
+ MTLCommandQueue, Metal,
},
},
encodable_block::{EncodableBlock, EncodingParameters},
@@ -51,10 +49,8 @@ impl LanguageModelGenerator {
) -> Result<Self, Error> {
let gpu_capture = GpuCaptureManager::new();
- let context =
- LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
- let prefill_step_size =
- decoding_config.prefill_step_size.resolve(&context.model_config);
+ let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
+ let prefill_step_size = decoding_config.prefill_step_size.resolve(&context.model_config);
let generate_suffix_length = decoding_config.generate_suffix_length();
let mut generator = Self {
@@ -87,10 +83,7 @@ impl LanguageModelGenerator {
let tokens_length = tokens.len();
- let prefill_step_size = self
- .decoding_config
- .prefill_step_size
- .resolve(&self.context.model_config);
+ let prefill_step_size = self.decoding_config.prefill_step_size.resolve(&self.context.model_config);
let prefill_steps = tokens_length.div_ceil(prefill_step_size);
let prefill_size = prefill_steps * prefill_step_size;
@@ -113,113 +106,73 @@ impl LanguageModelGenerator {
let has_grammar = compiled_grammar.is_some();
- let token_ids = tokens
- .iter()
- .copied()
- .take(tokens_length - 1)
- .chain(flat_trie.token_ids())
- .chunks(prefill_step_size);
+ let token_ids =
+ tokens.iter().copied().take(tokens_length - 1).chain(flat_trie.token_ids()).chunks(prefill_step_size);
- let token_positions = (prefix_offset
- ..prefix_offset + tokens_length - 1)
- .chain(flat_trie.token_positions().map(|trie_position| {
- prefix_offset + tokens_length - 1 + trie_position
- }))
+ let token_positions = (prefix_offset..prefix_offset + tokens_length - 1)
+ .chain(flat_trie.token_positions().map(|trie_position| prefix_offset + tokens_length - 1 + trie_position))
.chunks(prefill_step_size);
- let single_token_bitmask_size =
- self.context.model_shape.bitmask_shape(1)[1];
- let token_bitmasks = repeat_n(None, tokens_length - 1)
- .chain(flat_trie.token_masks())
- .chunks(prefill_step_size);
+ let single_token_bitmask_size = self.context.model_shape.bitmask_shape(1)[1];
+ let token_bitmasks = repeat_n(None, tokens_length - 1).chain(flat_trie.token_masks()).chunks(prefill_step_size);
- let token_seeds = repeat_n(0, tokens_length - 1)
- .chain(flat_trie.token_seeds())
- .chunks(prefill_step_size);
+ let token_seeds = repeat_n(0, tokens_length - 1).chain(flat_trie.token_seeds()).chunks(prefill_step_size);
let mut last_state: Option<ForwardPassState<Metal>> = None;
let mut run_times: Vec<f64> = Vec::new();
// Process each prefill step and update the KV cache.
- for (
- step,
- (
- step_token_ids,
- step_token_positions,
- step_token_bitmasks,
- step_token_seeds,
- ),
- ) in
- izip!(&token_ids, &token_positions, &token_bitmasks, &token_seeds)
- .enumerate()
+ for (step, (step_token_ids, step_token_positions, step_token_bitmasks, step_token_seeds)) in
+ izip!(&token_ids, &token_positions, &token_bitmasks, &token_seeds).enumerate()
{
let tokens_start_index = step * prefill_step_size;
let tokens_end_index = tokens_start_index + prefill_step_size;
let step_token_ids = step_token_ids.collect::<Box<[u64]>>();
- let step_token_positions =
- step_token_positions.collect::<Box<[usize]>>();
+ let step_token_positions = step_token_positions.collect::<Box<[usize]>>();
let step_token_seeds = step_token_seeds.collect::<Box<[u64]>>();
let active_suffix_length = step_token_positions.len();
let is_last_prefill_step = step == prefill_steps - 1;
- let should_sample_after_step =
- sample_suffix && is_last_prefill_step;
+ let should_sample_after_step = sample_suffix && is_last_prefill_step;
// If we sample on the last prefill step, we only need logits/sampling
// for tokens that are beyond the prompt prefix (i.e. starting at the
// suffix-root token, which is the last prompt token).
- let (sampling_start, sampling_length) = if should_sample_after_step
- {
- let suffix_root_index_in_step =
- (tokens_length - 1).saturating_sub(tokens_start_index);
- let sampling_length = active_suffix_length
- .saturating_sub(suffix_root_index_in_step);
- debug_assert!(
- sampling_length > 0,
- "Expected at least one token to sample on the last prefill step"
- );
+ let (sampling_start, sampling_length) = if should_sample_after_step {
+ let suffix_root_index_in_step = (tokens_length - 1).saturating_sub(tokens_start_index);
+ let sampling_length = active_suffix_length.saturating_sub(suffix_root_index_in_step);
+ debug_assert!(sampling_length > 0, "Expected at least one token to sample on the last prefill step");
(suffix_root_index_in_step, sampling_length)
} else {
(0, 0)
};
- let step_token_bitmask: Option<Box<[u32]>> =
- if has_grammar && sampling_length > 0 {
- Some(
- step_token_bitmasks
- .map(|mask| match mask {
- Some(mask) => Either::Left(
- mask.iter()
- .copied()
- .take(single_token_bitmask_size)
- .chain(repeat_n(
- 0u32,
- single_token_bitmask_size
- .saturating_sub(mask.len()),
- )),
- ),
- None => Either::Right(repeat_n(
- u32::MAX,
- single_token_bitmask_size,
- )),
- })
- .flatten()
- .collect::<Box<[u32]>>(),
- )
- } else {
- // Drain the chunk iterator to keep the other chunked iterators aligned.
- let _ = step_token_bitmasks.count();
- None
- };
-
- let should_capture =
- self.gpu_capture.should_capture_prefill(step == 0);
+ let step_token_bitmask: Option<Box<[u32]>> = if has_grammar && sampling_length > 0 {
+ Some(
+ step_token_bitmasks
+ .map(|mask| match mask {
+ Some(mask) => Either::Left(
+ mask.iter()
+ .copied()
+ .take(single_token_bitmask_size)
+ .chain(repeat_n(0u32, single_token_bitmask_size.saturating_sub(mask.len()))),
+ ),
+ None => Either::Right(repeat_n(u32::MAX, single_token_bitmask_size)),
+ })
+ .flatten()
+ .collect::<Box<[u32]>>(),
+ )
+ } else {
+ // Drain the chunk iterator to keep the other chunked iterators aligned.
+ let _ = step_token_bitmasks.count();
+ None
+ };
+
+ let should_capture = self.gpu_capture.should_capture_prefill(step == 0);
if should_capture {
- let _ = self
- .gpu_capture
- .start_capture(&self.context.mtl_context, "prefill");
+ let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "prefill");
}
objc2::rc::autoreleasepool(|_pool| {
@@ -251,36 +204,24 @@ impl LanguageModelGenerator {
}
// Register the accepted prompt tokens from this step.
- let step_end_token_index =
- std::cmp::min(tokens_end_index, tokens_length);
- let tokens_processed_this_step =
- step_end_token_index - tokens_start_index;
+ let step_end_token_index = std::cmp::min(tokens_end_index, tokens_length);
+ let tokens_processed_this_step = step_end_token_index - tokens_start_index;
if tokens_processed_this_step > 0 {
- let mut positions_for_step: Vec<usize> = (tokens_start_index
- ..step_end_token_index)
- .map(|idx| idx + prefix_offset)
- .collect();
+ let mut positions_for_step: Vec<usize> =
+ (tokens_start_index..step_end_token_index).map(|idx| idx + prefix_offset).collect();
if step == prefill_steps - 1 && sample_suffix {
// Exclude the last token because it belongs to the suffix for sampling.
positions_for_step.pop();
}
if !positions_for_step.is_empty() {
- let accept_indices_for_step: Vec<usize> =
- (0..positions_for_step.len()).collect();
+ let accept_indices_for_step: Vec<usize> = (0..positions_for_step.len()).collect();
if !accept_indices_for_step.is_empty() {
- self.update_cache_layers(
- &accept_indices_for_step,
- None,
- true,
- );
+ self.update_cache_layers(&accept_indices_for_step, None, true);
}
- self.context
- .cache_layers
- .borrow_mut()
- .register_accepted_tokens(&positions_for_step);
+ self.context.cache_layers.borrow_mut().register_accepted_tokens(&positions_for_step);
if let Some(&last_idx) = positions_for_step.last() {
self.registered_prefix_len = last_idx + 1;
@@ -309,10 +250,7 @@ impl LanguageModelGenerator {
flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());
self.update_cache_layers(
- &accepted_token_indices
- .into_iter()
- .map(|p| suffix_root_index + p)
- .collect::<Box<[usize]>>(),
+ &accepted_token_indices.into_iter().map(|p| suffix_root_index + p).collect::<Box<[usize]>>(),
Some(last_suffix_start),
false,
);
@@ -346,52 +284,36 @@ impl LanguageModelGenerator {
let flat_trie = suffix_root.linearize();
let active_suffix_length = flat_trie.len();
- let token_ids = flat_trie
- .token_ids()
- .chain(repeat_n(0, suffix_length - active_suffix_length))
- .collect::<Box<[u64]>>();
-
- let token_bitmask: Option<Box<[u32]>> =
- compiled_grammar.is_some().then(|| {
- let single_token_bitmask_size =
- self.context.model_shape.bitmask_shape(1)[1];
- flat_trie
- .token_masks()
- .chain(repeat_n(None, suffix_length - active_suffix_length))
- .map(|mask| match mask {
- Some(mask) => Either::Left(
- mask.iter()
- .copied()
- .take(single_token_bitmask_size)
- .chain(repeat_n(
- 0u32,
- single_token_bitmask_size
- .saturating_sub(mask.len()),
- )),
- ),
- None => Either::Right(repeat_n(
- u32::MAX,
- single_token_bitmask_size,
- )),
- })
- .flatten()
- .collect::<Box<[u32]>>()
- });
+ let token_ids =
+ flat_trie.token_ids().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();
+
+ let token_bitmask: Option<Box<[u32]>> = compiled_grammar.is_some().then(|| {
+ let single_token_bitmask_size = self.context.model_shape.bitmask_shape(1)[1];
+ flat_trie
+ .token_masks()
+ .chain(repeat_n(None, suffix_length - active_suffix_length))
+ .map(|mask| match mask {
+ Some(mask) => Either::Left(
+ mask.iter()
+ .copied()
+ .take(single_token_bitmask_size)
+ .chain(repeat_n(0u32, single_token_bitmask_size.saturating_sub(mask.len()))),
+ ),
+ None => Either::Right(repeat_n(u32::MAX, single_token_bitmask_size)),
+ })
+ .flatten()
+ .collect::<Box<[u32]>>()
+ });
let start_position = self.tokens.len() - 1;
let token_positions = flat_trie
.token_positions()
.map(|trie_position| start_position + trie_position)
- .chain(repeat_n(
- INVALID_POSITION,
- suffix_length - active_suffix_length,
- ))
+ .chain(repeat_n(INVALID_POSITION, suffix_length - active_suffix_length))
.collect::<Box<[usize]>>();
- let token_seeds = flat_trie
- .token_seeds()
- .chain(repeat_n(0, suffix_length - active_suffix_length))
- .collect::<Box<[u64]>>();
+ let token_seeds =
+ flat_trie.token_seeds().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();
let task = LanguageModelGeneratorRunTask {
token_ids: &token_ids,
@@ -405,13 +327,8 @@ impl LanguageModelGenerator {
is_prefilling: false,
};
- let (mut state, run_time) = self.run_model(
- task,
- false,
- self.allow_pre_encode(),
- sampling_method,
- self.should_fill_attention_bias(),
- );
+ let (mut state, run_time) =
+ self.run_model(task, false, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias());
let sampled_tokens = self.sample(&mut state)?;
@@ -448,14 +365,8 @@ impl LanguageModelGenerator {
&self.context.mtl_context,
);
- self.context
- .async_buffers
- .prepare_positions(prefill_count, tokens_to_generate);
- self.context.async_buffers.prepare_seeds(
- &self.context.seed,
- prefill_count,
- tokens_to_generate,
- );
+ self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);
+ self.context.async_buffers.prepare_seeds(&self.context.seed, prefill_count, tokens_to_generate);
self.context.async_buffers.reset_counter();
}
@@ -474,11 +385,7 @@ impl LanguageModelGenerator {
where
F: FnOnce(u64) + Send + 'static,
{
- assert_eq!(
- self.decoding_config.generate_suffix_length(),
- 1,
- "async_generate only supports suffix_length=1"
- );
+ assert_eq!(self.decoding_config.generate_suffix_length(), 1, "async_generate only supports suffix_length=1");
// Extract values from async_buffers before mutable borrow
let current_counter = self.context.async_buffers.counter.get();
@@ -487,8 +394,7 @@ impl LanguageModelGenerator {
let slot = pass_idx % batch_size;
let async_event = self.context.async_buffers.event.clone();
let results_buffer = self.context.async_buffers.results.clone();
- let async_positions_buffer =
- self.context.async_buffers.positions.clone();
+ let async_positions_buffer = self.context.async_buffers.positions.clone();
let async_seeds_buffer = self.context.async_buffers.seeds.clone();
let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
@@ -517,12 +423,9 @@ impl LanguageModelGenerator {
let skip_token_ids_copy = pass_idx > 0;
let is_first_decode = !is_continuation;
- let should_capture =
- self.gpu_capture.should_capture_decode(is_first_decode);
+ let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
if should_capture {
- let _ = self
- .gpu_capture
- .start_capture(&self.context.mtl_context, "decode");
+ let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "decode");
}
let mut state = ForwardPassState::new_llm(
@@ -555,8 +458,7 @@ impl LanguageModelGenerator {
// Wait on previous pass if this is a continuation
if is_continuation {
- root_command_buffer
- .encode_wait_for_event_value(&async_event, current_counter);
+ root_command_buffer.encode_wait_for_event_value(&async_event, current_counter);
}
self.context.executables.encode(
@@ -574,27 +476,14 @@ impl LanguageModelGenerator {
// Copy sampled token: sampling_output β†’ token_ids (for next pass)
// and sampling_output β†’ results[slot] (for callback)
- let sampling_output = state
- .sampling_output()
- .expect("sampling_output must exist after sampling encode");
+ let sampling_output = state.sampling_output().expect("sampling_output must exist after sampling encode");
let sampling_output_buffer = sampling_output.borrow().buffer().clone();
- let token_ids_buffer =
- self.context.scratch_buffers.token_ids.borrow().buffer().clone();
-
- let encoder = root_command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
- self.context.token_copy.encode_to_token_ids(
- &sampling_output_buffer,
- &token_ids_buffer,
- &encoder,
- );
- self.context.token_copy.encode_to_results(
- &sampling_output_buffer,
- &results_buffer,
- slot,
- &encoder,
- );
+ let token_ids_buffer = self.context.scratch_buffers.token_ids.borrow().buffer().clone();
+
+ let encoder =
+ root_command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
+ self.context.token_copy.encode_to_token_ids(&sampling_output_buffer, &token_ids_buffer, &encoder);
+ self.context.token_copy.encode_to_results(&sampling_output_buffer, &results_buffer, slot, &encoder);
encoder.end_encoding();
// Scatter + register for all transformer layers
@@ -606,35 +495,19 @@ impl LanguageModelGenerator {
&cb,
&self.context.kv_cache_update,
);
- self.context
- .cache_layers
- .borrow_mut()
- .register_accepted_tokens(&[token_position]);
+ self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
}
- let encoder = root_command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
+ let encoder =
+ root_command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
if let Some(mask_update) = &self.context.mask_update {
- let updates: Vec<AttentionBiasUpdate> = self
- .context
- .cache_layers
- .borrow()
- .attention_bias_updates_after_acceptance(1);
- for (window_size, mask_buffer) in
- &self.context.scratch_buffers.attention_window_size_to_bias
- {
- if let Some(update) =
- updates.iter().find(|u| &u.key == window_size)
- {
+ let updates: Vec<AttentionBiasUpdate> =
+ self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
+ for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {
+ if let Some(update) = updates.iter().find(|u| &u.key == window_size) {
if update.unmask_col >= 0 || update.mask_col >= 0 {
- mask_update.encode(
- mask_buffer.borrow().buffer(),
- update.unmask_col,
- update.mask_col,
- &encoder,
- );
+ mask_update.encode(mask_buffer.borrow().buffer(), update.unmask_col, update.mask_col, &encoder);
}
}
}
@@ -644,8 +517,7 @@ impl LanguageModelGenerator {
// Signal event for next pass
let next_counter = current_counter + 1;
- root_command_buffer
- .encode_signal_event_value(&async_event, next_counter);
+ root_command_buffer.encode_signal_event_value(&async_event, next_counter);
self.context.async_buffers.counter.set(next_counter);
// Add completion handler
@@ -654,8 +526,7 @@ impl LanguageModelGenerator {
let handler = MTLCommandBufferHandler::new(move |_| {
let token = {
- let ptr =
- results_buffer_clone.contents().as_ptr() as *const u32;
+ let ptr = results_buffer_clone.contents().as_ptr() as *const u32;
unsafe { *ptr.add(slot) as u64 }
};
if let Some(cb) = callback.lock().unwrap().take() {
@@ -720,8 +591,7 @@ impl LanguageModelGenerator {
is_prefilling: false,
};
- let (_, _) =
- self.run_model(task, true, false, SamplingMethod::default(), false);
+ let (_, _) = self.run_model(task, true, false, SamplingMethod::default(), false);
}
fn run_model(
@@ -735,23 +605,16 @@ impl LanguageModelGenerator {
objc2::rc::autoreleasepool(|_pool| {
let run_start = Instant::now();
- let mut state = task.create_state(
- &mut self.context,
- None,
- should_fill_attention_bias,
- );
+ let mut state = task.create_state(&mut self.context, None, should_fill_attention_bias);
if let Some(method) = state.sampling_method_mut() {
*method = Some(sampling_method);
}
let is_first_decode = !warmup && task.token_ids.len() == 1;
- let should_capture =
- self.gpu_capture.should_capture_decode(is_first_decode);
+ let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
if should_capture {
- let _ = self
- .gpu_capture
- .start_capture(&self.context.mtl_context, "decode");
+ let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "decode");
}
let encoded_task_key = task.encoded_task_key(self.tokens.len());
@@ -790,19 +653,16 @@ impl LanguageModelGenerator {
if allow_pre_encode {
self.context.reset_command_buffer();
- let next_task_key: String =
- task.encoded_task_key(self.tokens.len() + 1);
+ let next_task_key: String = task.encoded_task_key(self.tokens.len() + 1);
let next_encoded_task = task.build_encoded_task(
&self.context,
&mut state,
- &EncodingParameters::new(warmup, false, false)
- .with_projection(1),
+ &EncodingParameters::new(warmup, false, false).with_projection(1),
next_task_key.clone(),
);
- self.encoded_tasks
- .insert(next_task_key.clone(), next_encoded_task);
+ self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);
}
root_command_buffer.wait_until_completed();
@@ -819,7 +679,8 @@ impl LanguageModelGenerator {
&mut self,
state: &mut ForwardPassState<Metal>,
) -> Result<Vec<u64>, Error> {
- let sampling_output = state.sampling_output()
+ let sampling_output = state
+ .sampling_output()
.expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");
let output_buffer = sampling_output.borrow();
@@ -869,8 +730,7 @@ impl LanguageModelGenerator {
fn allow_pre_encode(&self) -> bool {
let metal_debug_active = MetalEnvVar::DeviceWrapperType.is_enabled();
- let result =
- self.decoding_config.allow_pre_encode && !metal_debug_active;
+ let result = self.decoding_config.allow_pre_encode && !metal_debug_active;
result
}
@@ -882,25 +742,18 @@ impl LanguageModelGenerator {
let desired_prefix_len = self.tokens.len() - 1;
if desired_prefix_len > self.registered_prefix_len {
- let positions: Vec<usize> =
- (self.registered_prefix_len..desired_prefix_len).collect();
+ let positions: Vec<usize> = (self.registered_prefix_len..desired_prefix_len).collect();
if !positions.is_empty() {
- self.context
- .cache_layers
- .borrow_mut()
- .register_accepted_tokens(&positions);
+ self.context.cache_layers.borrow_mut().register_accepted_tokens(&positions);
}
self.registered_prefix_len = desired_prefix_len;
}
}
fn should_fill_attention_bias(&self) -> bool {
- let sliding_window_sizes =
- self.context.model_shape.sliding_window_length_per_layer.clone();
- let has_sliding_window =
- sliding_window_sizes.iter().any(|size| size.is_some());
- let has_speculative_suffix =
- self.decoding_config.generate_suffix_length() > 1;
+ let sliding_window_sizes = self.context.model_shape.sliding_window_length_per_layer.clone();
+ let has_sliding_window = sliding_window_sizes.iter().any(|size| size.is_some());
+ let has_speculative_suffix = self.decoding_config.generate_suffix_length() > 1;
has_sliding_window || has_speculative_suffix
}
diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rs
index 966cfe721..ae0b098ed 100644
--- a/crates/uzu/src/parameters/loader.rs
+++ b/crates/uzu/src/parameters/loader.rs
@@ -6,10 +6,7 @@ use std::{
use thiserror::Error;
-use super::safetensors_metadata::{
- HashMetadata as STMetadata, HeaderLoadingError,
- read_metadata as read_st_metadata,
-};
+use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingError, read_metadata as read_st_metadata};
use crate::{
DataType,
array::{Array, ArrayContextExt},
@@ -99,18 +96,11 @@ where
&self,
key: &str,
) -> Result<Array<C::Backend>, ParameterLoaderError> {
- let metadata_entry = self
- .index
- .get(key)
- .ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
+ let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
let (offset, size) = (metadata_entry.offset, metadata_entry.size);
let array_key = key.replace(".", "_");
let array_label = format!("parameter_loader_{array_key}");
- let mut array = self.context.create_array(
- &metadata_entry.shape,
- metadata_entry.data_type,
- &array_label,
- );
+ let mut array = self.context.create_array(&metadata_entry.shape, metadata_entry.data_type, &array_label);
let expected_size = array.size();
if expected_size != size {
return Err(ParameterLoaderError::SizeMismatch {
@@ -131,10 +121,7 @@ where
shape: &mut Box<[usize]>,
data_type: &mut DataType,
) -> Result<(), ParameterLoaderError> {
- let metadata_entry = self
- .index
- .get(key)
- .ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
+ let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
self.file.read_exact_at(buf, metadata_entry.offset as u64)?;
*shape = metadata_entry.shape.to_owned();
*data_type = metadata_entry.data_type;
@@ -170,9 +157,7 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
&self,
name: &str,
) -> String {
- self.prefix
- .as_ref()
- .map_or_else(|| name.to_string(), |p| format!("{p}.{name}"))
+ self.prefix.as_ref().map_or_else(|| name.to_string(), |p| format!("{p}.{name}"))
}
pub fn subtree(
@@ -180,11 +165,7 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
name: &str,
) -> Result<Self, ParameterLoaderError> {
let new_prefix = self.join_prefix(name);
- let num_suffixes = self
- .loader
- .keys()
- .filter_map(|suffix| suffix.strip_prefix(&new_prefix))
- .count();
+ let num_suffixes = self.loader.keys().filter_map(|suffix| suffix.strip_prefix(&new_prefix)).count();
if num_suffixes > 0 {
Ok(Self {
loader: self.loader,
@@ -209,11 +190,6 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
shape: &mut Box<[usize]>,
data_type: &mut DataType,
) -> Result<(), ParameterLoaderError> {
- self.loader.read_extract_at(
- &self.join_prefix(name),
- buf,
- shape,
- data_type,
- )
+ self.loader.read_extract_at(&self.join_prefix(name), buf, shape, data_type)
}
}
diff --git a/crates/uzu/src/parameters/safetensors_metadata.rs b/crates/uzu/src/parameters/safetensors_metadata.rs
index e5f520b4a..edb0e04df 100644
--- a/crates/uzu/src/parameters/safetensors_metadata.rs
+++ b/crates/uzu/src/parameters/safetensors_metadata.rs
@@ -11,9 +11,7 @@ use crate::DataType;
pub enum HeaderLoadingError {
#[error("The header is an invalid UTF-8 string and cannot be read.")]
InvalidHeader,
- #[error(
- "The header does contain a valid string, but it is not valid JSON."
- )]
+ #[error("The header does contain a valid string, but it is not valid JSON.")]
InvalidHeaderDeserialization,
#[error("The header is large than 100Mo which is considered too large.")]
HeaderTooLarge,
@@ -41,9 +39,7 @@ pub struct TensorInfo {
pub data_offsets: (usize, usize),
}
-#[derive(
- Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Ord, PartialOrd,
-)]
+#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
#[non_exhaustive]
pub enum Dtype {
/// Boolan type
@@ -116,29 +112,20 @@ impl From<DataType> for Dtype {
const MAX_HEADER_SIZE: usize = 100_000_000;
-pub fn read_metadata(
- file: &File
-) -> Result<(usize, HashMetadata), HeaderLoadingError> {
+pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadingError> {
let mut header_buffer = [0u8; size_of::<u64>()];
- file.read_exact_at(&mut header_buffer, 0)
- .map_err(|_| HeaderLoadingError::HeaderTooSmall)?;
- let metadata_size: usize = u64::from_le_bytes(header_buffer)
- .try_into()
- .map_err(|_| HeaderLoadingError::HeaderTooLarge)?;
+ file.read_exact_at(&mut header_buffer, 0).map_err(|_| HeaderLoadingError::HeaderTooSmall)?;
+ let metadata_size: usize =
+ u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;
if metadata_size > MAX_HEADER_SIZE {
return Err(HeaderLoadingError::InvalidHeaderLength);
}
- let stop = metadata_size
- .checked_add(8)
- .ok_or(HeaderLoadingError::InvalidHeaderLength)?;
- let mut json_buffer: Box<[u8]> =
- core::iter::repeat(0).take(stop - size_of::<u64>()).collect();
- file.read_exact_at(&mut json_buffer, 8)
- .map_err(|_| HeaderLoadingError::InvalidHeader)?;
- let string = core::str::from_utf8(&json_buffer)
- .map_err(|_| HeaderLoadingError::InvalidHeader)?;
- let metadata: HashMetadata = serde_json::from_str(string)
- .map_err(|_| HeaderLoadingError::InvalidHeaderDeserialization)?;
+ let stop = metadata_size.checked_add(8).ok_or(HeaderLoadingError::InvalidHeaderLength)?;
+ let mut json_buffer: Box<[u8]> = core::iter::repeat(0).take(stop - size_of::<u64>()).collect();
+ file.read_exact_at(&mut json_buffer, 8).map_err(|_| HeaderLoadingError::InvalidHeader)?;
+ let string = core::str::from_utf8(&json_buffer).map_err(|_| HeaderLoadingError::InvalidHeader)?;
+ let metadata: HashMetadata =
+ serde_json::from_str(string).map_err(|_| HeaderLoadingError::InvalidHeaderDeserialization)?;
Ok((stop, metadata))
}
diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rs
index 405d00400..940c6a544 100644
--- a/crates/uzu/src/prelude.rs
+++ b/crates/uzu/src/prelude.rs
@@ -9,13 +9,11 @@
pub use crate::session::config::{DecodingConfig, RunConfig, SpeculatorConfig};
// Session Parameters
pub use crate::session::parameter::{
- ContextLength, ContextMode, PrefillStepSize, SamplingMethod,
- SamplingPolicy, SamplingSeed,
+ ContextLength, ContextMode, PrefillStepSize, SamplingMethod, SamplingPolicy, SamplingSeed,
};
// Session Types
pub use crate::session::types::{
- Error, FinishReason, Input, Message, Output, ParsedText, Role, RunStats,
- Stats, StepStats, Text, TotalStats,
+ Error, FinishReason, Input, Message, Output, ParsedText, Role, RunStats, Stats, StepStats, Text, TotalStats,
};
// Speculators
pub use crate::speculators::speculator::Speculator;
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index f46663535..f16f730d0 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -28,15 +28,9 @@ use crate::{
},
session::{
config::{DecodingConfig, RunConfig},
- helpers::{
- Context, InputProcessor, InputProcessorDefault, OutputParser,
- is_directory_fits_ram,
- },
+ helpers::{Context, InputProcessor, InputProcessorDefault, OutputParser, is_directory_fits_ram},
parameter::{ConfigResolvableValue, ContextMode},
- types::{
- Error, FinishReason, Input, Output, RunStats, Stats, StepStats,
- TotalStats,
- },
+ types::{Error, FinishReason, Input, Output, RunStats, Stats, StepStats, TotalStats},
},
};
@@ -81,12 +75,8 @@ impl ChatSession {
if !config_path.exists() {
return Err(Error::UnableToLoadConfig);
}
- let config_file =
- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
- let model_metadata: ModelMetadata = serde_json::from_reader(
- BufReader::new(config_file),
- )
- .map_err(|err| {
+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
+ let model_metadata: ModelMetadata = serde_json::from_reader(BufReader::new(config_file)).map_err(|err| {
eprintln!("Failed to parse config.json: {err}");
Error::UnableToLoadConfig
})?;
@@ -97,12 +87,9 @@ impl ChatSession {
.and_then(|lm| lm.decoder_config().ok())
.and_then(|dc| dc.layer_configs)
.map(|layers| {
- let has_non_attention = layers.iter().any(|layer| {
- !matches!(layer.mixer_config, MixerConfig::Attention(_))
- });
- let has_mamba = layers.iter().any(|layer| {
- matches!(layer.mixer_config, MixerConfig::Mamba(_))
- });
+ let has_non_attention =
+ layers.iter().any(|layer| !matches!(layer.mixer_config, MixerConfig::Attention(_)));
+ let has_mamba = layers.iter().any(|layer| matches!(layer.mixer_config, MixerConfig::Mamba(_)));
(has_non_attention, has_mamba)
})
.unwrap_or((false, false));
@@ -120,50 +107,29 @@ impl ChatSession {
}
}
- if has_mamba_mixer
- && decoding_config.speculator_config.number_of_speculated_tokens > 0
- {
+ if has_mamba_mixer && decoding_config.speculator_config.number_of_speculated_tokens > 0 {
return Err(Error::UnsupportedSpeculatorConfigForModel);
}
- let language_model_config = model_metadata
- .model_config
- .as_language_model()
- .ok_or(Error::UnableToLoadConfig)?;
+ let language_model_config = model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;
let tokenizer_path = model_path.join("tokenizer.json");
if !tokenizer_path.exists() {
return Err(Error::UnableToLoadTokenizer);
}
- let tokenizer = Tokenizer::from_file(&tokenizer_path)
- .map_err(|_| Error::UnableToLoadTokenizer)?;
+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;
- let stop_token_ids: Vec<i32> = language_model_config
- .generation_config
- .stop_token_ids
- .iter()
- .map(|&x| x as i32)
- .collect();
- let tokenizer_info = TokenizerInfo::from_huggingface(
- &tokenizer,
- None,
- Some(&stop_token_ids),
- )
- .map_err(|error_message| Error::GrammarError(error_message))?;
-
- let input_processor = InputProcessorDefault::new(
- language_model_config.message_processor_config.clone(),
- );
+ let stop_token_ids: Vec<i32> =
+ language_model_config.generation_config.stop_token_ids.iter().map(|&x| x as i32).collect();
+ let tokenizer_info = TokenizerInfo::from_huggingface(&tokenizer, None, Some(&stop_token_ids))
+ .map_err(|error_message| Error::GrammarError(error_message))?;
- let output_parser = OutputParser::new(
- language_model_config
- .message_processor_config
- .output_parser_regex
- .clone(),
- )?;
+ let input_processor = InputProcessorDefault::new(language_model_config.message_processor_config.clone());
+
+ let output_parser =
+ OutputParser::new(language_model_config.message_processor_config.output_parser_regex.clone())?;
- let llm = LanguageModelGenerator::new(&model_path, decoding_config)
- .map_err(Error::from)?;
+ let llm = LanguageModelGenerator::new(&model_path, decoding_config).map_err(Error::from)?;
Ok(Self {
model_path,
@@ -186,20 +152,12 @@ impl ChatSession {
where
F: Fn(Output) -> bool,
{
- let context_mode = self
- .llm
- .as_ref()
- .ok_or(Error::LanguageModelGeneratorNotLoaded)?
- .decoding_config
- .context_mode
- .clone();
+ let context_mode =
+ self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?.decoding_config.context_mode.clone();
match &context_mode {
ContextMode::None => {
- let language_model_generator = self
- .llm
- .as_mut()
- .ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
language_model_generator.reset_state();
},
ContextMode::Static {
@@ -208,8 +166,7 @@ impl ChatSession {
if self.static_context.is_none() {
let mut prefill_config = config.clone();
prefill_config.tokens_limit = 0;
- let (_out, ctx) =
- self.extend(input.clone(), None, prefill_config)?;
+ let (_out, ctx) = self.extend(input.clone(), None, prefill_config)?;
self.static_context = Some(ctx);
}
let tmp = self.static_context.take();
@@ -228,10 +185,7 @@ impl ChatSession {
| ContextMode::Static {
..
} => {
- let language_model_generator = self
- .llm
- .as_mut()
- .ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
language_model_generator.reset_state();
},
ContextMode::Dynamic => {},
@@ -240,8 +194,7 @@ impl ChatSession {
}
pub fn reset(&mut self) -> Result<(), Error> {
- let language_model_generator =
- self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
language_model_generator.reset_state();
Ok(())
}
@@ -255,11 +208,9 @@ impl ChatSession {
config: RunConfig,
) -> Result<(Output, Context), Error> {
self.reconfigure_language_model_generator(context)?;
- let output =
- self.run_internal(input, config, None::<fn(Output) -> bool>)?;
+ let output = self.run_internal(input, config, None::<fn(Output) -> bool>)?;
let new_context = self.build_context_from_language_model_generator()?;
- let language_model_generator =
- self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
language_model_generator.reset_state();
Ok((output, new_context))
}
@@ -274,11 +225,7 @@ impl ChatSession {
F: Fn(Output) -> bool,
{
let run_start = Instant::now();
- let text = self.input_processor.process(
- &input,
- config.enable_thinking,
- config.tokens_limit > 0,
- )?;
+ let text = self.input_processor.process(&input, config.enable_thinking, config.tokens_limit > 0)?;
let tokens: Vec<u64> = self
.tokenizer
.encode(text.as_str(), false)
@@ -288,18 +235,11 @@ impl ChatSession {
.map(|&id| id as u64)
.collect();
- let language_model_config = self
- .model_metadata
- .model_config
- .as_language_model()
- .ok_or(Error::UnableToLoadConfig)?;
-
- let language_model_generator =
- self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- let context_length = language_model_generator
- .decoding_config
- .context_length
- .resolve(language_model_config);
+ let language_model_config =
+ self.model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;
+
+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let context_length = language_model_generator.decoding_config.context_length.resolve(language_model_config);
if tokens.len() >= context_length {
return Err(Error::ContextLengthExceeded);
}
@@ -307,26 +247,16 @@ impl ChatSession {
let prefix_offset = language_model_generator.tokens.len();
let prefix_len_before = prefix_offset;
- let eos_tokens: Vec<u64> = language_model_config
- .generation_config
- .stop_token_ids
- .iter()
- .map(|&x| x as u64)
- .collect();
+ let eos_tokens: Vec<u64> =
+ language_model_config.generation_config.stop_token_ids.iter().map(|&x| x as u64).collect();
- let sampling_method =
- config.sampling_policy.resolve(language_model_config);
+ let sampling_method = config.sampling_policy.resolve(language_model_config);
- let mut compiled_grammar: Option<CompiledGrammar> =
- if let Some(ref grammar_config) = config.grammar_config {
- Some(CompiledGrammar::from_config(
- grammar_config,
- None,
- &self.tokenizer_info,
- )?)
- } else {
- None
- };
+ let mut compiled_grammar: Option<CompiledGrammar> = if let Some(ref grammar_config) = config.grammar_config {
+ Some(CompiledGrammar::from_config(grammar_config, None, &self.tokenizer_info)?)
+ } else {
+ None
+ };
let prefill_start = Instant::now();
let sample_suffix = config.tokens_limit > 0;
@@ -341,10 +271,8 @@ impl ChatSession {
let prefill_duration = prefill_start.elapsed().as_secs_f64();
language_model_generator.clear_cache();
- let prefill_suffix_length = language_model_generator
- .decoding_config
- .prefill_step_size
- .resolve(language_model_config);
+ let prefill_suffix_length =
+ language_model_generator.decoding_config.prefill_step_size.resolve(language_model_config);
let run_context = RunContext {
eos_tokens,
@@ -358,19 +286,12 @@ impl ChatSession {
run_start,
};
- let grammar_terminated = compiled_grammar
- .as_ref()
- .map(|g| g.is_terminated())
- .unwrap_or(false);
+ let grammar_terminated = compiled_grammar.as_ref().map(|g| g.is_terminated()).unwrap_or(false);
let prefill_finish_reason = if grammar_terminated {
Some(FinishReason::Stop)
} else {
- Self::check_finish_reason(
- &run_context,
- language_model_generator,
- &prefill_tokens,
- )
+ Self::check_finish_reason(&run_context, language_model_generator, &prefill_tokens)
};
let prefill_output = Self::build_output(
&self.tokenizer,
@@ -392,21 +313,15 @@ impl ChatSession {
if prefill_should_continue {
return Ok(prefill_output);
} else {
- return Ok(prefill_output
- .clone_with_finish_reason(Some(FinishReason::Cancelled)));
+ return Ok(prefill_output.clone_with_finish_reason(Some(FinishReason::Cancelled)));
}
}
let can_use_async =
- language_model_generator.decoding_config.generate_suffix_length()
- == 1
- && compiled_grammar.is_none();
+ language_model_generator.decoding_config.generate_suffix_length() == 1 && compiled_grammar.is_none();
let generate_output = if can_use_async {
- let batch_size = language_model_generator
- .decoding_config
- .async_batch_size
- .resolve(&self.model_path);
+ let batch_size = language_model_generator.decoding_config.async_batch_size.resolve(&self.model_path);
Self::run_async_batch(
&self.tokenizer,
&self.output_parser,
@@ -429,8 +344,7 @@ impl ChatSession {
};
language_model_generator.clear_cache();
- Ok(generate_output
- .clone_with_duration(run_start.elapsed().as_secs_f64()))
+ Ok(generate_output.clone_with_duration(run_start.elapsed().as_secs_f64()))
}
fn run_sync_generate<F>(
@@ -451,28 +365,19 @@ impl ChatSession {
loop {
let generate_start = Instant::now();
- let generate_result = language_model_generator.generate(
- compiled_grammar_mut.as_deref_mut(),
- sampling_method,
- )?;
+ let generate_result =
+ language_model_generator.generate(compiled_grammar_mut.as_deref_mut(), sampling_method)?;
let generate_tokens = generate_result.tokens.clone();
let generate_duration = generate_start.elapsed().as_secs_f64();
generate_results.push(generate_result);
generate_durations.push(generate_duration);
- let grammar_terminated = compiled_grammar_mut
- .as_ref()
- .map(|g| g.is_terminated())
- .unwrap_or(false);
+ let grammar_terminated = compiled_grammar_mut.as_ref().map(|g| g.is_terminated()).unwrap_or(false);
let generate_finish_reason = if grammar_terminated {
Some(FinishReason::Stop)
} else {
- Self::check_finish_reason(
- run_context,
- language_model_generator,
- &generate_tokens,
- )
+ Self::check_finish_reason(run_context, language_model_generator, &generate_tokens)
};
let generate_output = Self::build_output(
tokenizer,
@@ -494,9 +399,7 @@ impl ChatSession {
if generate_should_continue {
return Ok(generate_output);
} else {
- return Ok(generate_output.clone_with_finish_reason(Some(
- FinishReason::Cancelled,
- )));
+ return Ok(generate_output.clone_with_finish_reason(Some(FinishReason::Cancelled)));
}
}
}
@@ -514,11 +417,8 @@ impl ChatSession {
where
F: Fn(Output) -> bool,
{
- let remaining_by_limit = run_context
- .tokens_limit
- .saturating_sub(run_context.prefill_result.tokens.len());
- let remaining_by_context =
- run_context.context_length.saturating_sub(llm.tokens.len());
+ let remaining_by_limit = run_context.tokens_limit.saturating_sub(run_context.prefill_result.tokens.len());
+ let remaining_by_context = run_context.context_length.saturating_sub(llm.tokens.len());
let tokens_to_generate = remaining_by_limit.min(remaining_by_context);
llm.prepare_async(tokens_to_generate);
@@ -527,47 +427,34 @@ impl ChatSession {
let start_time = Instant::now();
let last_nanos = Arc::new(AtomicU64::new(0));
- let mut results: Vec<GenerateResult> =
- Vec::with_capacity(tokens_to_generate);
+ let mut results: Vec<GenerateResult> = Vec::with_capacity(tokens_to_generate);
let mut durations: Vec<f64> = Vec::with_capacity(tokens_to_generate);
let mut finish_reason = FinishReason::Length;
let mut next_to_submit = 0;
let mut in_flight = 0;
while in_flight > 0 || next_to_submit < tokens_to_generate {
- let batch_end =
- std::cmp::min(next_to_submit + batch_size, tokens_to_generate);
+ let batch_end = std::cmp::min(next_to_submit + batch_size, tokens_to_generate);
let batch_submitted = batch_end - next_to_submit;
let accepted_before_batch = results.len();
- let cache_slice = llm
- .context
- .cache_layers
- .borrow()
- .slice(&llm.context.mtl_context, 0..batch_submitted);
+ let cache_slice = llm.context.cache_layers.borrow().slice(&llm.context.mtl_context, 0..batch_submitted);
for idx in next_to_submit..batch_end {
let batch_sender = sender.clone();
let last_callback_nanos = last_nanos.clone();
let batch_start_time = start_time;
llm.async_generate(idx, sampling_method, move |token| {
- let now_nanos =
- batch_start_time.elapsed().as_nanos() as u64;
- let prev_nanos =
- last_callback_nanos.swap(now_nanos, Ordering::SeqCst);
- let _ = batch_sender.send((
- idx,
- token,
- now_nanos.saturating_sub(prev_nanos),
- ));
+ let now_nanos = batch_start_time.elapsed().as_nanos() as u64;
+ let prev_nanos = last_callback_nanos.swap(now_nanos, Ordering::SeqCst);
+ let _ = batch_sender.send((idx, token, now_nanos.saturating_sub(prev_nanos)));
})?;
in_flight += 1;
}
next_to_submit = batch_end;
for _ in 0..batch_submitted {
- let (_, token, duration_nanos) =
- receiver.recv().map_err(|_| Error::SamplingFailed)?;
+ let (_, token, duration_nanos) = receiver.recv().map_err(|_| Error::SamplingFailed)?;
in_flight -= 1;
let duration = nanos_to_secs(duration_nanos);
@@ -585,15 +472,8 @@ impl ChatSession {
finish_reason = FinishReason::ContextLimitReached;
true
} else if let Some(progress_fn) = progress {
- let output = Self::build_output(
- tokenizer,
- output_parser,
- run_context,
- llm,
- &results,
- &durations,
- None,
- )?;
+ let output =
+ Self::build_output(tokenizer, output_parser, run_context, llm, &results, &durations, None)?;
if !progress_fn(output) {
finish_reason = FinishReason::Cancelled;
true
@@ -610,13 +490,12 @@ impl ChatSession {
in_flight -= 1;
}
if let Some(slice) = cache_slice {
- let accepted_in_batch =
- results.len().saturating_sub(accepted_before_batch);
+ let accepted_in_batch = results.len().saturating_sub(accepted_before_batch);
if accepted_in_batch < batch_submitted {
- llm.context.cache_layers.borrow_mut().apply_slice(
- &slice,
- Some(accepted_in_batch..batch_submitted),
- );
+ llm.context
+ .cache_layers
+ .borrow_mut()
+ .apply_slice(&slice, Some(accepted_in_batch..batch_submitted));
}
}
return Self::build_output(
@@ -632,70 +511,39 @@ impl ChatSession {
}
}
- Self::build_output(
- tokenizer,
- output_parser,
- run_context,
- llm,
- &results,
- &durations,
- Some(finish_reason),
- )
+ Self::build_output(tokenizer, output_parser, run_context, llm, &results, &durations, Some(finish_reason))
}
fn reconfigure_language_model_generator(
&mut self,
context: Option<&Context>,
) -> Result<(), Error> {
- let language_model_generator =
- self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
language_model_generator.reset_state();
if let Some(ctx) = context {
- let mut llm_state =
- language_model_generator.context.cache_layers.borrow_mut();
- for (_layer_idx, (ctx_layer, gen_layer)) in ctx
- .cache_layers
- .data
- .iter()
- .zip(llm_state.data.iter_mut())
- .enumerate()
+ let mut llm_state = language_model_generator.context.cache_layers.borrow_mut();
+ for (_layer_idx, (ctx_layer, gen_layer)) in
+ ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()
{
match (ctx_layer, gen_layer) {
- (
- CacheLayer::Transformer(src),
- CacheLayer::Transformer(dst),
- ) => {
+ (CacheLayer::Transformer(src), CacheLayer::Transformer(dst)) => {
let copy_rows = src.prefix_segment_length();
if copy_rows > 0 {
{
let mut dst_keys = dst.keys.borrow_mut();
let src_keys = src.keys.borrow();
- dst_keys.copy_slice(
- &src_keys,
- 1,
- 0..copy_rows,
- 0,
- );
+ dst_keys.copy_slice(&src_keys, 1, 0..copy_rows, 0);
}
{
let mut dst_values = dst.values.borrow_mut();
let src_values = src.values.borrow();
- dst_values.copy_slice(
- &src_values,
- 1,
- 0..copy_rows,
- 0,
- );
+ dst_values.copy_slice(&src_values, 1, 0..copy_rows, 0);
}
}
dst.state = src.state.clone();
- dst.prefix_token_positions =
- src.prefix_token_positions.clone();
+ dst.prefix_token_positions = src.prefix_token_positions.clone();
},
- (
- CacheLayer::StateSpace(src),
- CacheLayer::StateSpace(dst),
- ) => {
+ (CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {
{
let mut dst_conv = dst.conv_state.borrow_mut();
let src_conv = src.conv_state.borrow();
@@ -707,9 +555,7 @@ impl ChatSession {
dst_ssm.copy_from_array(&src_ssm);
}
},
- _ => panic!(
- "Layer type mismatch when reconfiguring language model generator cache"
- ),
+ _ => panic!("Layer type mismatch when reconfiguring language model generator cache"),
}
}
drop(llm_state);
@@ -719,16 +565,10 @@ impl ChatSession {
Ok(())
}
- fn build_context_from_language_model_generator(
- &self
- ) -> Result<Context, Error> {
- let language_model_generator =
- self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- let cache_layers = language_model_generator
- .context
- .cache_layers
- .borrow()
- .clone(&language_model_generator.context.mtl_context);
+ fn build_context_from_language_model_generator(&self) -> Result<Context, Error> {
+ let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let cache_layers =
+ language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.mtl_context);
let context = Context::new(
language_model_generator.tokens.clone(),
cache_layers,
@@ -744,14 +584,10 @@ impl ChatSession {
language_model_generator: &LanguageModelGenerator,
new_tokens: &[u64],
) -> Option<FinishReason> {
- let start_idx =
- run_context.prefix_len_before + run_context.input_tokens_len;
- let total_new_tokens =
- language_model_generator.tokens.len().saturating_sub(start_idx);
- let has_eos =
- new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));
- let context_limit =
- language_model_generator.tokens.len() >= run_context.context_length;
+ let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;
+ let total_new_tokens = language_model_generator.tokens.len().saturating_sub(start_idx);
+ let has_eos = new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));
+ let context_limit = language_model_generator.tokens.len() >= run_context.context_length;
if has_eos {
Some(FinishReason::Stop)
@@ -774,9 +610,7 @@ impl ChatSession {
.chain(generate_results.iter().flat_map(|r| r.tokens.iter()))
.map(|&v| v as u32)
.collect();
- tokenizer
- .decode(&generated_tokens, true)
- .map_err(|_| Error::UnableToDecodeText)
+ tokenizer.decode(&generated_tokens, true).map_err(|_| Error::UnableToDecodeText)
}
fn build_output(
@@ -788,16 +622,10 @@ impl ChatSession {
generate_durations: &[f64],
finish_reason: Option<FinishReason>,
) -> Result<Output, Error> {
- let text = Self::decode_generated_tokens(
- tokenizer,
- &run_context.prefill_result.tokens,
- generate_results,
- )?;
+ let text = Self::decode_generated_tokens(tokenizer, &run_context.prefill_result.tokens, generate_results)?;
let parsed = output_parser.parse(text);
- let start_idx =
- run_context.prefix_len_before + run_context.input_tokens_len;
- let output_tokens =
- language_model_generator.tokens.len().saturating_sub(start_idx);
+ let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;
+ let output_tokens = language_model_generator.tokens.len().saturating_sub(start_idx);
Ok(Output {
text: parsed,
@@ -807,9 +635,7 @@ impl ChatSession {
run_context.prefill_suffix_length,
generate_results.to_vec(),
generate_durations.to_vec(),
- language_model_generator
- .decoding_config
- .generate_suffix_length(),
+ language_model_generator.decoding_config.generate_suffix_length(),
run_context.run_start.elapsed().as_secs_f64(),
run_context.input_tokens_len,
output_tokens,
@@ -834,13 +660,11 @@ impl ChatSession {
let tokens_per_second = tokens_count as f64 / prefill_duration;
let processed_tokens = tokens_count_input + tokens_count;
- let processed_tokens_per_second =
- processed_tokens as f64 / prefill_duration;
+ let processed_tokens_per_second = processed_tokens as f64 / prefill_duration;
let model_run_count = prefill_result.forwardpass_durations.len();
let model_run_average_duration =
- prefill_result.forwardpass_durations.iter().sum::<f64>()
- / model_run_count as f64;
+ prefill_result.forwardpass_durations.iter().sum::<f64>() / model_run_count as f64;
StepStats {
duration: prefill_duration,
@@ -859,23 +683,16 @@ impl ChatSession {
let generate_stats: Option<StepStats>;
if generate_results.len() != 0 {
let duration = generate_durations.iter().sum::<f64>();
- let tokens_count = generate_results
- .iter()
- .flat_map(|result| result.tokens.clone())
- .collect::<Vec<u64>>()
- .len();
+ let tokens_count =
+ generate_results.iter().flat_map(|result| result.tokens.clone()).collect::<Vec<u64>>().len();
let tokens_per_second: f64 = tokens_count as f64 / duration;
let model_run_count = generate_results.len();
- let model_run_average_duration = generate_results
- .iter()
- .map(|result| result.forwardpass_duration)
- .sum::<f64>()
- / model_run_count as f64;
+ let model_run_average_duration =
+ generate_results.iter().map(|result| result.forwardpass_duration).sum::<f64>() / model_run_count as f64;
let run_count = generate_durations.len();
- let run_average_duration =
- generate_durations.iter().sum::<f64>() / run_count as f64;
+ let run_average_duration = generate_durations.iter().sum::<f64>() / run_count as f64;
generate_stats = Some(StepStats {
duration,
diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rs
index 8db0c4063..9ff58d6b9 100644
--- a/crates/uzu/src/session/classification_session.rs
+++ b/crates/uzu/src/session/classification_session.rs
@@ -28,26 +28,20 @@ impl ClassificationSession {
return Err(Error::ModelFolderNotFound);
}
- let config_file = std::fs::File::open(&config_path)
- .map_err(|_| Error::UnableToLoadConfig)?;
+ let config_file = std::fs::File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
let model_metadata: ModelMetadata =
- serde_json::from_reader(std::io::BufReader::new(config_file))
- .map_err(|_| Error::UnableToLoadConfig)?;
+ serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
let tokenizer_path = model_path.join("tokenizer.json");
if !tokenizer_path.exists() {
return Err(Error::UnableToLoadTokenizer);
}
- let tokenizer = Tokenizer::from_file(&tokenizer_path)
- .map_err(|_| Error::UnableToLoadTokenizer)?;
+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;
- let classifier_model_config = model_metadata
- .model_config
- .as_classifier()
- .ok_or(Error::UnableToLoadConfig)?;
- let input_processor = Box::new(InputProcessorDefault::new(
- classifier_model_config.message_processor_config.clone(),
- )) as Box<dyn InputProcessor>;
+ let classifier_model_config = model_metadata.model_config.as_classifier().ok_or(Error::UnableToLoadConfig)?;
+ let input_processor =
+ Box::new(InputProcessorDefault::new(classifier_model_config.message_processor_config.clone()))
+ as Box<dyn InputProcessor>;
let classifier = Classifier::new(&model_path)?;
Ok(Self {
@@ -75,11 +69,9 @@ impl ClassificationSession {
.collect();
let token_positions: Vec<usize> = (0..tokens.len()).collect();
- let preprocessing_duration =
- preprocessing_start.elapsed().as_secs_f64();
+ let preprocessing_duration = preprocessing_start.elapsed().as_secs_f64();
- let mut output =
- self.classifier.classify_tokens(tokens, token_positions)?;
+ let mut output = self.classifier.classify_tokens(tokens, token_positions)?;
output.stats.preprocessing_duration = preprocessing_duration;
Ok(output)
diff --git a/crates/uzu/src/session/config/decoding_config.rs b/crates/uzu/src/session/config/decoding_config.rs
index 4add97589..d2042126d 100644
--- a/crates/uzu/src/session/config/decoding_config.rs
+++ b/crates/uzu/src/session/config/decoding_config.rs
@@ -1,9 +1,6 @@
use crate::session::{
config::SpeculatorConfig,
- parameter::{
- AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize,
- SamplingSeed,
- },
+ parameter::{AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize, SamplingSeed},
};
#[derive(Clone)]
diff --git a/crates/uzu/src/session/config/grammar_config.rs b/crates/uzu/src/session/config/grammar_config.rs
index 68f3fc91b..054dc7952 100644
--- a/crates/uzu/src/session/config/grammar_config.rs
+++ b/crates/uzu/src/session/config/grammar_config.rs
@@ -63,8 +63,8 @@ impl GrammarConfig {
T: JsonSchema,
{
let schema_root = schemars::schema_for!(T);
- let schema_str = serde_json::to_string(&schema_root)
- .map_err(|e| format!("Failed to serialize schema: {}", e))?;
+ let schema_str =
+ serde_json::to_string(&schema_root).map_err(|e| format!("Failed to serialize schema: {}", e))?;
Ok(Self::JsonSchema {
schema: schema_str,
@@ -85,8 +85,8 @@ impl GrammarConfig {
T: JsonSchema,
{
let schema_root = schemars::schema_for!(T);
- let schema_str = serde_json::to_string(&schema_root)
- .map_err(|e| format!("Failed to serialize schema: {}", e))?;
+ let schema_str =
+ serde_json::to_string(&schema_root).map_err(|e| format!("Failed to serialize schema: {}", e))?;
Ok(Self::JsonSchema {
schema: schema_str,
diff --git a/crates/uzu/src/session/config/speculator_config.rs b/crates/uzu/src/session/config/speculator_config.rs
index cdcee7f6f..19e14c25c 100644
--- a/crates/uzu/src/session/config/speculator_config.rs
+++ b/crates/uzu/src/session/config/speculator_config.rs
@@ -1,8 +1,6 @@
use std::sync::Arc;
-use crate::speculators::{
- empty_speculator::EmptySpeculator, speculator::Speculator,
-};
+use crate::speculators::{empty_speculator::EmptySpeculator, speculator::Speculator};
#[derive(Clone)]
pub struct SpeculatorConfig {
@@ -16,10 +14,8 @@ impl SpeculatorConfig {
speculator: Arc<dyn Speculator>,
) -> Self {
let max_number_of_speculated_tokens: usize = 64;
- let effective_number_of_speculated_tokens = std::cmp::min(
- number_of_speculated_tokens,
- max_number_of_speculated_tokens,
- );
+ let effective_number_of_speculated_tokens =
+ std::cmp::min(number_of_speculated_tokens, max_number_of_speculated_tokens);
Self {
number_of_speculated_tokens: effective_number_of_speculated_tokens,
speculator,
diff --git a/crates/uzu/src/session/helpers/context.rs b/crates/uzu/src/session/helpers/context.rs
index c55e0d4bb..fa7c63627 100644
--- a/crates/uzu/src/session/helpers/context.rs
+++ b/crates/uzu/src/session/helpers/context.rs
@@ -1,7 +1,4 @@
-use crate::{
- backends::metal::Metal, forward_pass::cache_layers::CacheLayers,
- session::config::DecodingConfig,
-};
+use crate::{backends::metal::Metal, forward_pass::cache_layers::CacheLayers, session::config::DecodingConfig};
pub struct Context {
pub tokens: Vec<u64>,
diff --git a/crates/uzu/src/session/helpers/input_processor.rs b/crates/uzu/src/session/helpers/input_processor.rs
index 8a3a27c94..d55887bff 100644
--- a/crates/uzu/src/session/helpers/input_processor.rs
+++ b/crates/uzu/src/session/helpers/input_processor.rs
@@ -54,12 +54,8 @@ fn strftime_now(format_string: String) -> String {
loop {
let mut buffer = vec![0u8; buf_len];
- let written = libc::strftime(
- buffer.as_mut_ptr() as *mut libc::c_char,
- buffer.len(),
- c_format.as_ptr(),
- &tm,
- );
+ let written =
+ libc::strftime(buffer.as_mut_ptr() as *mut libc::c_char, buffer.len(), c_format.as_ptr(), &tm);
if written > 0 {
buffer.truncate(written as usize);
@@ -84,17 +80,13 @@ impl InputProcessor for InputProcessorDefault {
) -> Result<String, Error> {
let messages = input.get_messages();
for message in &messages {
- if message.role != Role::Assistant
- && message.reasoning_content.is_some()
- {
+ if message.role != Role::Assistant && message.reasoning_content.is_some() {
return Err(Error::UnexpectedReasoningContent);
}
}
- let messages = messages
- .into_iter()
- .map(|message| message.resolve(&self.message_processing_config))
- .collect::<Vec<_>>();
+ let messages =
+ messages.into_iter().map(|message| message.resolve(&self.message_processing_config)).collect::<Vec<_>>();
let template = self.message_processing_config.prompt_template.clone();
let bos_token = self.message_processing_config.bos_token.clone();
@@ -102,12 +94,8 @@ impl InputProcessor for InputProcessorDefault {
let mut environment = Environment::new();
environment.set_unknown_method_callback(unknown_method_callback);
environment.add_function("strftime_now", strftime_now);
- environment
- .add_template(template_name, template.as_str())
- .map_err(|_| Error::UnableToLoadPromptTemplate)?;
- let template = environment
- .get_template(template_name)
- .map_err(|_| Error::UnableToLoadPromptTemplate)?;
+ environment.add_template(template_name, template.as_str()).map_err(|_| Error::UnableToLoadPromptTemplate)?;
+ let template = environment.get_template(template_name).map_err(|_| Error::UnableToLoadPromptTemplate)?;
let result = template
.render(context!(
diff --git a/crates/uzu/src/session/helpers/output_parser.rs b/crates/uzu/src/session/helpers/output_parser.rs
index 42a9ab88d..c4dbeed0e 100644
--- a/crates/uzu/src/session/helpers/output_parser.rs
+++ b/crates/uzu/src/session/helpers/output_parser.rs
@@ -31,12 +31,8 @@ impl OutputParser {
let parsed_text = match &self.regex {
Some(regex) => match regex.captures(&text) {
Some(captures) => {
- let chain_of_thought = captures
- .name("chain_of_thought")
- .map(|m| m.as_str().to_string());
- let response = captures
- .name("response")
- .map(|m| m.as_str().to_string());
+ let chain_of_thought = captures.name("chain_of_thought").map(|m| m.as_str().to_string());
+ let response = captures.name("response").map(|m| m.as_str().to_string());
ParsedText {
chain_of_thought,
response,
diff --git a/crates/uzu/src/session/parameter/context_length.rs b/crates/uzu/src/session/parameter/context_length.rs
index 9abaca6e0..bdee47a44 100644
--- a/crates/uzu/src/session/parameter/context_length.rs
+++ b/crates/uzu/src/session/parameter/context_length.rs
@@ -1,6 +1,4 @@
-use crate::{
- config::LanguageModelConfig, session::parameter::ConfigResolvableValue,
-};
+use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};
#[derive(Debug, Clone, Copy)]
pub enum ContextLength {
diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rs
index fc81a7cc5..235f14076 100644
--- a/crates/uzu/src/session/parameter/prefill_step_size.rs
+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs
@@ -1,10 +1,7 @@
-use crate::{
- config::LanguageModelConfig, session::parameter::ConfigResolvableValue,
-};
+use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};
fn env_prefill_step_size_default_override() -> Option<usize> {
- static OVERRIDE: std::sync::OnceLock<Option<usize>> =
- std::sync::OnceLock::new();
+ static OVERRIDE: std::sync::OnceLock<Option<usize>> = std::sync::OnceLock::new();
*OVERRIDE.get_or_init(|| {
let raw = std::env::var("UZU_PREFILL_STEP_SIZE").ok()?;
let trimmed = raw.trim();
@@ -34,8 +31,7 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
config: &LanguageModelConfig,
) -> usize {
let default_limit: usize = 4096;
- let model_context_length =
- config.model_config.transformer_config.context_length;
+ let model_context_length = config.model_config.transformer_config.context_length;
let minimal_sliding_window_size = config
.decoder_config()
@@ -49,11 +45,10 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
.min()
.unwrap_or(usize::MAX);
- let maximal_value =
- [model_context_length, minimal_sliding_window_size, default_limit]
- .into_iter()
- .min()
- .unwrap_or(default_limit);
+ let maximal_value = [model_context_length, minimal_sliding_window_size, default_limit]
+ .into_iter()
+ .min()
+ .unwrap_or(default_limit);
let proposed_value = match self {
PrefillStepSize::Default => {
diff --git a/crates/uzu/src/session/parameter/sampling_policy.rs b/crates/uzu/src/session/parameter/sampling_policy.rs
index 070d95ced..94d8d5b73 100644
--- a/crates/uzu/src/session/parameter/sampling_policy.rs
+++ b/crates/uzu/src/session/parameter/sampling_policy.rs
@@ -1,6 +1,4 @@
-use crate::{
- config::LanguageModelConfig, session::parameter::ConfigResolvableValue,
-};
+use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum SamplingMethod {
@@ -33,9 +31,7 @@ impl Default for SamplingPolicy {
}
}
-impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod>
- for SamplingPolicy
-{
+impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod> for SamplingPolicy {
fn resolve(
&self,
config: &LanguageModelConfig,
diff --git a/crates/uzu/src/session/types/message.rs b/crates/uzu/src/session/types/message.rs
index c49ab6154..f36961d3e 100644
--- a/crates/uzu/src/session/types/message.rs
+++ b/crates/uzu/src/session/types/message.rs
@@ -43,9 +43,7 @@ impl Message {
}
}
-impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>>
- for Message
-{
+impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>> for Message {
fn resolve(
&self,
config: &MessageProcessorConfig,
@@ -56,10 +54,7 @@ impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>>
Role::Assistant => config.assistant_role_name.clone(),
};
let content = self.content.clone();
- let mut result = HashMap::from([
- (String::from("role"), role),
- (String::from("content"), content),
- ]);
+ let mut result = HashMap::from([(String::from("role"), role), (String::from("content"), content)]);
if let Some(reasoning_content) = self.reasoning_content.clone() {
result.insert(String::from("reasoning_content"), reasoning_content);
}
diff --git a/crates/uzu/src/tracer/mod.rs b/crates/uzu/src/tracer/mod.rs
index 43349c40d..7f377abaa 100644
--- a/crates/uzu/src/tracer/mod.rs
+++ b/crates/uzu/src/tracer/mod.rs
@@ -7,6 +7,5 @@
mod trace_validator;
pub use trace_validator::{
- ArrayTransform, TraceValidator, TracerValidationMetrics,
- TracerValidationResult, TracerValidationResults,
+ ArrayTransform, TraceValidator, TracerValidationMetrics, TracerValidationResult, TracerValidationResults,
};
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index f358cae2f..22959739e 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -17,10 +17,7 @@ use num_traits::NumCast;
use crate::{
ArrayElement, DataType,
- backends::metal::{
- KVCacheUpdate, MTLCommandBuffer, MTLCommandQueue, MTLContext, Metal,
- MetalArray,
- },
+ backends::metal::{KVCacheUpdate, MTLCommandBuffer, MTLCommandQueue, MTLContext, Metal, MetalArray},
classifier::Classifier,
config::ModelMetadata,
encodable_block::{EncodableBlock, EncodingParameters, Sampling},
@@ -37,10 +34,7 @@ use crate::{
parameters::{ParameterLoader, ParameterTree, read_safetensors_metadata},
session::{
config::{DecodingConfig, SpeculatorConfig},
- parameter::{
- AsyncBatchSize, ConfigResolvableValue, ContextLength, ContextMode,
- PrefillStepSize, SamplingSeed,
- },
+ parameter::{AsyncBatchSize, ConfigResolvableValue, ContextLength, ContextMode, PrefillStepSize, SamplingSeed},
types::Error,
},
};
@@ -141,8 +135,7 @@ impl TracerValidationResults {
}
pub fn is_valid(&self) -> bool {
- self.number_of_tokens_violations()
- <= self.number_of_allowed_tokens_violations()
+ self.number_of_tokens_violations() <= self.number_of_allowed_tokens_violations()
}
}
@@ -186,18 +179,15 @@ impl TraceValidator {
return Err(Error::ModelFolderNotFound);
}
- let config_file =
- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
let metadata: ModelMetadata =
- serde_json::from_reader(std::io::BufReader::new(config_file))
- .map_err(|_| Error::UnableToLoadConfig)?;
+ serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
let context = if metadata.model_config.is_classifier() {
let classifier = Classifier::new(model_path)?;
ModelContext::Classifier(classifier)
} else {
- let prefill_step_size =
- Self::determine_prefill_step_size(model_path);
+ let prefill_step_size = Self::determine_prefill_step_size(model_path);
let decoding_config = DecodingConfig::new(
ContextMode::default(),
ContextLength::default(),
@@ -207,17 +197,9 @@ impl TraceValidator {
AsyncBatchSize::default(),
false,
);
- let mut llm_context = LanguageModelGeneratorContext::new(
- model_path,
- &decoding_config,
- )?;
- let desired_suffix_length =
- prefill_step_size.max(decoding_config.generate_suffix_length());
- Self::ensure_llm_context_capacity(
- &decoding_config,
- desired_suffix_length,
- &mut llm_context,
- );
+ let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
+ let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());
+ Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);
ModelContext::LanguageModelGenerator(llm_context)
};
@@ -235,12 +217,8 @@ impl TraceValidator {
}
match &mut self.context {
- ModelContext::LanguageModelGenerator(ctx) => {
- Self::run_llm_validation(ctx, &traces_path)
- },
- ModelContext::Classifier(classifier) => {
- Self::run_classifier_validation(classifier, &traces_path)
- },
+ ModelContext::LanguageModelGenerator(ctx) => Self::run_llm_validation(ctx, &traces_path),
+ ModelContext::Classifier(classifier) => Self::run_classifier_validation(classifier, &traces_path),
}
}
@@ -260,21 +238,13 @@ impl TraceValidator {
ctx: &LanguageModelGeneratorContext,
traces_path: &Path,
) -> Result<TracerValidationResults, Error> {
- let traces_file =
- File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
let traces_loader =
- ParameterLoader::new(&traces_file, ctx.mtl_context.as_ref())
- .map_err(|_| Error::UnableToLoadWeights)?;
+ ParameterLoader::new(&traces_file, ctx.mtl_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let traces_view = traces_loader.tree();
- let token_ids = Self::load_array_as_vec::<i32, u64>(
- &traces_view,
- "activation_trace.token_ids",
- );
- let token_positions = Self::load_array_as_vec::<i32, usize>(
- &traces_view,
- "activation_trace.token_positions",
- );
+ let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");
+ let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");
let token_seeds: Vec<u64> = vec![0; token_ids.len()];
let mut state = ForwardPassState::new_llm(
@@ -299,17 +269,9 @@ impl TraceValidator {
None,
);
- let command_buffer = ctx
- .mtl_context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer = ctx.mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
- ctx.executables.encode(
- &mut state,
- &command_buffer,
- &EncodingParameters::new(false, false, false),
- );
+ ctx.executables.encode(&mut state, &command_buffer, &EncodingParameters::new(false, false, false));
command_buffer.commit();
command_buffer.wait_until_completed();
@@ -317,53 +279,30 @@ impl TraceValidator {
let data_type = ctx.model_shape.activation_data_type();
// Common layer validation
- let mut results =
- Self::validate_layer_traces(&traces, &traces_view, data_type);
+ let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);
// LLM-specific: KV cache validation
let transformer_layers: Vec<usize> = {
let cache = state.cache_layers().unwrap().borrow();
- cache
- .data
- .iter()
- .enumerate()
- .filter_map(|(index, layer)| {
- layer.as_transformer().map(|_| index)
- })
- .collect()
+ cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_transformer().map(|_| index)).collect()
};
for index in transformer_layers {
- let arrays =
- state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);
+ let arrays = state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);
- if let Ok(expected) =
- traces_view.leaf(&format!("updated_kv_cache.{}.keys", index))
- {
+ if let Ok(expected) = traces_view.leaf(&format!("updated_kv_cache.{}.keys", index)) {
let keys = arrays[0].borrow();
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.keys", index),
- metrics: Self::validate_array(
- data_type,
- &expected,
- &keys,
- Some(ArrayTransform::KVCacheSlice),
- ),
+ metrics: Self::validate_array(data_type, &expected, &keys, Some(ArrayTransform::KVCacheSlice)),
});
}
- if let Ok(expected) =
- traces_view.leaf(&format!("updated_kv_cache.{}.values", index))
- {
+ if let Ok(expected) = traces_view.leaf(&format!("updated_kv_cache.{}.values", index)) {
let values = arrays[1].borrow();
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.values", index),
- metrics: Self::validate_array(
- data_type,
- &expected,
- &values,
- Some(ArrayTransform::KVCacheSlice),
- ),
+ metrics: Self::validate_array(data_type, &expected, &values, Some(ArrayTransform::KVCacheSlice)),
});
}
}
@@ -371,30 +310,17 @@ impl TraceValidator {
// LLM-specific: SSM state validation
let ssm_layers: Vec<usize> = {
let cache = state.cache_layers().unwrap().borrow();
- cache
- .data
- .iter()
- .enumerate()
- .filter_map(|(index, layer)| {
- layer.as_state_space().map(|_| index)
- })
- .collect()
+ cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_state_space().map(|_| index)).collect()
};
for index in ssm_layers {
- let arrays = state.arrays(&[
- ArrayId::SsmConvState(index),
- ArrayId::SsmState(index),
- ]);
+ let arrays = state.arrays(&[ArrayId::SsmConvState(index), ArrayId::SsmState(index)]);
let conv_state = arrays[0].borrow();
let ssm_state = arrays[1].borrow();
for path in [
format!("updated_state.{}.conv_state", index),
- format!(
- "activation_trace.layer_results.{}.updated_state.conv_state",
- index
- ),
+ format!("activation_trace.layer_results.{}.updated_state.conv_state", index),
] {
if let Ok(expected) = traces_view.leaf(&path) {
results.push(TracerValidationResult {
@@ -411,30 +337,21 @@ impl TraceValidator {
for path in [
format!("updated_state.{}.ssm_state", index),
- format!(
- "activation_trace.layer_results.{}.updated_state.ssm_state",
- index
- ),
+ format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),
] {
if let Ok(expected) = traces_view.leaf(&path) {
results.push(TracerValidationResult {
name: path,
- metrics: Self::validate_array(
- data_type, &expected, &ssm_state, None,
- ),
+ metrics: Self::validate_array(data_type, &expected, &ssm_state, None),
});
}
}
}
// LLM-specific: Token comparison
- let tokens_violation_indices = if let Ok(expected_logits) =
- traces_view.leaf("logits")
- {
- let expected_tokens =
- Self::get_tokens_from_logits(&expected_logits);
- let produced_tokens =
- Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());
+ let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf("logits") {
+ let expected_tokens = Self::get_tokens_from_logits(&expected_logits);
+ let produced_tokens = Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());
expected_tokens
.iter()
.zip(produced_tokens.iter())
@@ -466,47 +383,34 @@ impl TraceValidator {
classifier: &mut Classifier,
traces_path: &Path,
) -> Result<TracerValidationResults, Error> {
- let traces_file =
- File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
let mtl_context = classifier.context.mtl_context.clone();
let traces_loader =
- ParameterLoader::new(&traces_file, mtl_context.as_ref())
- .map_err(|_| Error::UnableToLoadWeights)?;
+ ParameterLoader::new(&traces_file, mtl_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let traces_view = traces_loader.tree();
- let has_token_ids =
- traces_view.leaf("activation_trace.token_ids").is_ok();
- let has_token_positions =
- traces_view.leaf("activation_trace.token_positions").is_ok();
+ let has_token_ids = traces_view.leaf("activation_trace.token_ids").is_ok();
+ let has_token_positions = traces_view.leaf("activation_trace.token_positions").is_ok();
if !has_token_ids || !has_token_positions {
return Ok(Self::handle_missing_tokens(&traces_view));
}
- let token_ids = Self::load_array_as_vec::<i32, u64>(
- &traces_view,
- "activation_trace.token_ids",
- );
- let token_positions = Self::load_array_as_vec::<i32, usize>(
- &traces_view,
- "activation_trace.token_positions",
- );
+ let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");
+ let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");
let suffix_length = token_ids.len();
- let (_logits, traces) = classifier
- .forward_pass_with_traces(&token_ids, &token_positions)
- .map_err(|_| Error::GenerateFailed)?;
+ let (_logits, traces) =
+ classifier.forward_pass_with_traces(&token_ids, &token_positions).map_err(|_| Error::GenerateFailed)?;
let data_type = classifier.context.model_shape.activation_data_type();
// Common layer validation
- let mut results =
- Self::validate_layer_traces(&traces, &traces_view, data_type);
+ let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);
// Classifier-specific: embedding_norm, output_pooling
- let classifier_results =
- Self::validate_classifier_traces(&traces, &traces_view, data_type);
+ let classifier_results = Self::validate_classifier_traces(&traces, &traces_view, data_type);
results.extend(classifier_results);
Ok(TracerValidationResults {
@@ -516,9 +420,7 @@ impl TraceValidator {
})
}
- fn handle_missing_tokens(
- traces_view: &ParameterTree<MTLContext>
- ) -> TracerValidationResults {
+ fn handle_missing_tokens(traces_view: &ParameterTree<MTLContext>) -> TracerValidationResults {
if let Ok(expected_logits) = traces_view.leaf("logits") {
let reference_shape = expected_logits.shape().to_vec();
let metrics = TracerValidationMetrics {
@@ -569,108 +471,62 @@ impl TraceValidator {
) -> Vec<TracerValidationResult> {
let mut results = Vec::new();
- let validate = |path: &str,
- array: &Ref<MetalArray>|
- -> Option<TracerValidationResult> {
+ let validate = |path: &str, array: &Ref<MetalArray>| -> Option<TracerValidationResult> {
if traces_view.leaf(path).is_ok() {
Some(TracerValidationResult {
name: path.to_string(),
- metrics: Self::validate_array_with_name(
- data_type,
- traces_view,
- path,
- array,
- ),
+ metrics: Self::validate_array_with_name(data_type, traces_view, path, array),
})
} else {
None
}
};
- for (index, layer_traces) in
- traces.borrow().layer_results.iter().enumerate()
- {
+ for (index, layer_traces) in traces.borrow().layer_results.iter().enumerate() {
let path = |suffix: &str| -> String {
- format!(
- "activation_trace.layer_results.{}.activation_trace.{}",
- index, suffix
- )
+ format!("activation_trace.layer_results.{}.activation_trace.{}", index, suffix)
};
- if let Some(r) = validate(
- &path("inputs"),
- &layer_traces.borrow().inputs.borrow(),
- ) {
+ if let Some(r) = validate(&path("inputs"), &layer_traces.borrow().inputs.borrow()) {
results.push(r);
}
- if let Some(r) = validate(
- &path("pre_mixer_norm"),
- &layer_traces.borrow().pre_attention_norm.borrow(),
- ) {
+ if let Some(r) = validate(&path("pre_mixer_norm"), &layer_traces.borrow().pre_attention_norm.borrow()) {
results.push(r);
}
- if let Some(r) = validate(
- &path("mixer"),
- &layer_traces.borrow().attention.borrow(),
- ) {
+ if let Some(r) = validate(&path("mixer"), &layer_traces.borrow().attention.borrow()) {
results.push(r);
}
- if let Some(r) = validate(
- &path("post_mixer_norm"),
- &layer_traces.borrow().post_attention_norm.borrow(),
- ) {
+ if let Some(r) = validate(&path("post_mixer_norm"), &layer_traces.borrow().post_attention_norm.borrow()) {
results.push(r);
}
- if let Some(r) = validate(
- &path("mlp_inputs"),
- &layer_traces.borrow().mlp_inputs.borrow(),
- ) {
+ if let Some(r) = validate(&path("mlp_inputs"), &layer_traces.borrow().mlp_inputs.borrow()) {
results.push(r);
}
- if let Some(r) = validate(
- &path("pre_mlp_norm"),
- &layer_traces.borrow().pre_mlp_norm.borrow(),
- ) {
+ if let Some(r) = validate(&path("pre_mlp_norm"), &layer_traces.borrow().pre_mlp_norm.borrow()) {
results.push(r);
}
- if let Some(r) =
- validate(&path("mlp"), &layer_traces.borrow().mlp.borrow())
- {
+ if let Some(r) = validate(&path("mlp"), &layer_traces.borrow().mlp.borrow()) {
results.push(r);
}
- if let Some(r) = validate(
- &path("post_mlp_norm"),
- &layer_traces.borrow().post_mlp_norm.borrow(),
- ) {
+ if let Some(r) = validate(&path("post_mlp_norm"), &layer_traces.borrow().post_mlp_norm.borrow()) {
results.push(r);
}
- let outputs_path =
- format!("activation_trace.layer_results.{}.outputs", index);
- if let Some(r) =
- validate(&outputs_path, &layer_traces.borrow().outputs.borrow())
- {
+ let outputs_path = format!("activation_trace.layer_results.{}.outputs", index);
+ if let Some(r) = validate(&outputs_path, &layer_traces.borrow().outputs.borrow()) {
results.push(r);
}
}
// Output norm (common to all models)
- if let Some(r) = validate(
- "activation_trace.output_norm",
- &traces.borrow().output_norm.borrow(),
- ) {
+ if let Some(r) = validate("activation_trace.output_norm", &traces.borrow().output_norm.borrow()) {
results.push(r);
}
// Logits (common to all models, but path may vary)
- if let Some(r) = validate(
- "activation_trace.logits",
- &traces.borrow().logits.borrow(),
- ) {
+ if let Some(r) = validate("activation_trace.logits", &traces.borrow().logits.borrow()) {
results.push(r);
- } else if let Some(r) =
- validate("logits", &traces.borrow().logits.borrow())
- {
+ } else if let Some(r) = validate("logits", &traces.borrow().logits.borrow()) {
results.push(r);
}
@@ -728,21 +584,9 @@ impl TraceValidator {
transform: Option<ArrayTransform>,
) -> TracerValidationMetrics {
match data_type {
- DataType::F16 => Self::validate_array_of_type::<f16>(
- expected_array,
- produced_array,
- transform,
- ),
- DataType::BF16 => Self::validate_array_of_type::<bf16>(
- expected_array,
- produced_array,
- transform,
- ),
- DataType::F32 => Self::validate_array_of_type::<f32>(
- expected_array,
- produced_array,
- transform,
- ),
+ DataType::F16 => Self::validate_array_of_type::<f16>(expected_array, produced_array, transform),
+ DataType::BF16 => Self::validate_array_of_type::<bf16>(expected_array, produced_array, transform),
+ DataType::F32 => Self::validate_array_of_type::<f32>(expected_array, produced_array, transform),
_ => panic!("Unsupported data type: {:?}", data_type),
}
}
@@ -774,12 +618,7 @@ impl TraceValidator {
let sliced = permuted.slice(s![start.., .., ..]);
let reshaped = sliced
.into_owned()
- .to_shape(IxDyn(&[
- 1,
- expected_tokens,
- permuted.shape()[1],
- permuted.shape()[2],
- ]))
+ .to_shape(IxDyn(&[1, expected_tokens, permuted.shape()[1], permuted.shape()[2]]))
.expect("Failed to reshape KV cache slice")
.to_owned();
(expected_view.to_owned(), reshaped)
@@ -814,18 +653,14 @@ impl TraceValidator {
&& expected_shape.get(0) == Some(&1)
&& expected_shape[1..] == produced_shape[..]
{
- expected_data = expected_data
- .to_shape(IxDyn(&produced_shape))
- .expect("Failed to reshape expected data")
- .to_owned();
+ expected_data =
+ expected_data.to_shape(IxDyn(&produced_shape)).expect("Failed to reshape expected data").to_owned();
} else if produced_shape.len() == expected_shape.len() + 1
&& produced_shape.get(0) == Some(&1)
&& produced_shape[1..] == expected_shape[..]
{
- produced_data = produced_data
- .to_shape(IxDyn(&expected_shape))
- .expect("Failed to reshape produced data")
- .to_owned();
+ produced_data =
+ produced_data.to_shape(IxDyn(&expected_shape)).expect("Failed to reshape produced data").to_owned();
}
}
@@ -837,22 +672,11 @@ impl TraceValidator {
);
}
- let reference: Vec<f32> = expected_data
- .iter()
- .map(|value| NumCast::from(*value).unwrap_or(0.0))
- .collect();
-
- let result: Vec<f32> = produced_data
- .iter()
- .map(|value| NumCast::from(*value).unwrap_or(0.0))
- .collect();
-
- Self::compare_arrays(
- &reference,
- expected_data.shape().to_vec(),
- &result,
- produced_data.shape().to_vec(),
- )
+ let reference: Vec<f32> = expected_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();
+
+ let result: Vec<f32> = produced_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();
+
+ Self::compare_arrays(&reference, expected_data.shape().to_vec(), &result, produced_data.shape().to_vec())
}
fn compare_arrays(
@@ -879,9 +703,7 @@ impl TraceValidator {
let mut diff_max = 0.0f32;
let mut result_nan = false;
- for (i, (&exp, &prod)) in
- reference.iter().zip(result.iter()).enumerate()
- {
+ for (i, (&exp, &prod)) in reference.iter().zip(result.iter()).enumerate() {
if prod.is_nan() {
result_nan = true;
}
@@ -922,8 +744,7 @@ impl TraceValidator {
};
let diff_avg = diff_sum / n;
- let max_allowed_violations =
- (fraction_of_allowed_violations * n).ceil() as usize;
+ let max_allowed_violations = (fraction_of_allowed_violations * n).ceil() as usize;
TracerValidationMetrics {
atol,
@@ -951,10 +772,7 @@ impl TraceValidator {
// Utility Functions
// ========================================================================
- fn load_array_as_vec<
- SourcePrecision: ArrayElement,
- TargetPrecision: NumCast,
- >(
+ fn load_array_as_vec<SourcePrecision: ArrayElement, TargetPrecision: NumCast>(
traces_view: &ParameterTree<MTLContext>,
name: &str,
) -> Vec<TargetPrecision> {
@@ -966,20 +784,10 @@ impl TraceValidator {
fn determine_prefill_step_size(model_path: &Path) -> usize {
let traces_path = model_path.join("traces.safetensors");
if let Ok(file) = File::open(&traces_path) {
- if let Ok((_header_len, metadata)) =
- read_safetensors_metadata(&file)
- {
- if let Some(tensor) =
- metadata.tensors.get("activation_trace.token_ids")
- {
+ if let Ok((_header_len, metadata)) = read_safetensors_metadata(&file) {
+ if let Some(tensor) = metadata.tensors.get("activation_trace.token_ids") {
if let Some(&length) = tensor.shape.first() {
- return tensor
- .shape
- .iter()
- .copied()
- .max()
- .unwrap_or(length)
- .max(1);
+ return tensor.shape.iter().copied().max().unwrap_or(length).max(1);
}
}
}
@@ -992,8 +800,7 @@ impl TraceValidator {
desired_suffix_length: usize,
context: &mut LanguageModelGeneratorContext,
) {
- let resolved_prefix_length =
- decoding_config.context_length.resolve(&context.model_config);
+ let resolved_prefix_length = decoding_config.context_length.resolve(&context.model_config);
let current_suffix_length = std::cmp::max(
decoding_config.prefill_step_size.resolve(&context.model_config),
decoding_config.generate_suffix_length(),
@@ -1019,16 +826,11 @@ impl TraceValidator {
desired_suffix_length,
)));
- let intermediate_dtype: DataType =
- decoder_config.output_norm_config.scale_precision.into();
+ let intermediate_dtype: DataType = decoder_config.output_norm_config.scale_precision.into();
context.kv_cache_update = Box::new(
- KVCacheUpdate::new(
- context.mtl_context.as_ref(),
- intermediate_dtype,
- resolved_prefix_length,
- )
- .expect("Failed to create KV cache update kernel"),
+ KVCacheUpdate::new(context.mtl_context.as_ref(), intermediate_dtype, resolved_prefix_length)
+ .expect("Failed to create KV cache update kernel"),
);
context.gpu_sampler = Sampling::new(
@@ -1043,22 +845,14 @@ impl TraceValidator {
fn get_tokens_from_logits(logits: &MetalArray) -> Vec<u64> {
let data_type = logits.data_type();
match data_type {
- DataType::F16 => {
- Self::get_tokens_from_logits_of_type::<f16>(logits)
- },
- DataType::BF16 => {
- Self::get_tokens_from_logits_of_type::<bf16>(logits)
- },
- DataType::F32 => {
- Self::get_tokens_from_logits_of_type::<f32>(logits)
- },
+ DataType::F16 => Self::get_tokens_from_logits_of_type::<f16>(logits),
+ DataType::BF16 => Self::get_tokens_from_logits_of_type::<bf16>(logits),
+ DataType::F32 => Self::get_tokens_from_logits_of_type::<f32>(logits),
_ => panic!("Unsupported data type: {:?}", data_type),
}
}
- fn get_tokens_from_logits_of_type<Precision: ArrayElement>(
- logits: &MetalArray
- ) -> Vec<u64> {
+ fn get_tokens_from_logits_of_type<Precision: ArrayElement>(logits: &MetalArray) -> Vec<u64> {
let sampler = ArgmaxSampler {};
sampler.sample(logits.as_view::<Precision>())
}
diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rs
index 4e827aa47..375b876e8 100644
--- a/crates/uzu/src/trie.rs
+++ b/crates/uzu/src/trie.rs
@@ -1,9 +1,7 @@
use thiserror::Error;
use crate::{
- language_model::{
- grammar::CompiledGrammar, gumbel::speculator_sample, rng::PRng,
- },
+ language_model::{grammar::CompiledGrammar, gumbel::speculator_sample, rng::PRng},
speculators::speculator::Speculator,
};
@@ -102,37 +100,22 @@ impl TrieNode {
let mut length = 1;
let mut height = 0;
- let mask = compiled_grammar
- .as_deref_mut()
- .and_then(|g| g.next_bitmask().unwrap());
- let mut root = Self::new(
- *prefix.last().unwrap(),
- mask,
- seed.derive((prefix_length - 1) as u64),
- );
+ let mask = compiled_grammar.as_deref_mut().and_then(|g| g.next_bitmask().unwrap());
+ let mut root = Self::new(*prefix.last().unwrap(), mask, seed.derive((prefix_length - 1) as u64));
let mut cur_node = &mut root;
let mut cur_node_width = 0;
- let mut cur_node_speculator_weights =
- speculator.speculate(&speculated_suffix);
+ let mut cur_node_speculator_weights = speculator.speculate(&speculated_suffix);
let mut next_node = None;
while length < max_length {
// Guuumbel speculator trick: both speculator and llm sample via gumbel max trick using the same noise for increased acceptance rate
- if let Some(next_speculated_token) =
- speculator_sample(cur_node.seed(), &cur_node_speculator_weights)
- {
+ if let Some(next_speculated_token) = speculator_sample(cur_node.seed(), &cur_node_speculator_weights) {
// Add speculated token to the trie
- let mask = if let Some(compiled_grammar) =
- compiled_grammar.as_deref_mut()
- {
- if compiled_grammar
- .accept_token(next_speculated_token)
- .is_err()
- {
- cur_node_speculator_weights
- .remove(&next_speculated_token);
+ let mask = if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {
+ if compiled_grammar.accept_token(next_speculated_token).is_err() {
+ cur_node_speculator_weights.remove(&next_speculated_token);
continue;
}
let next_bitmask = compiled_grammar.next_bitmask().unwrap();
@@ -143,11 +126,7 @@ impl TrieNode {
None
};
- let leaf_node = Self::new(
- next_speculated_token,
- mask,
- seed.derive((prefix_length + height) as u64),
- );
+ let leaf_node = Self::new(next_speculated_token, mask, seed.derive((prefix_length + height) as u64));
cur_node.add(leaf_node).unwrap();
// If this is the first node we sampled (most likely to be selected after gumbel noise application) - set it as the next one
@@ -166,8 +145,7 @@ impl TrieNode {
} else if let Some(next_node_token) = next_node.take() {
// Out of speculated tokens for this node, move onto the likeliest next node
speculated_suffix.push(next_node_token);
- if let Some(compiled_grammar) = compiled_grammar.as_deref_mut()
- {
+ if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {
if compiled_grammar.accept_token(next_node_token).is_err() {
break;
}
@@ -175,8 +153,7 @@ impl TrieNode {
height += 1;
cur_node = cur_node.get_mut(next_node_token).unwrap();
cur_node_width = 0;
- cur_node_speculator_weights =
- speculator.speculate(&speculated_suffix);
+ cur_node_speculator_weights = speculator.speculate(&speculated_suffix);
continue;
} else {
// Dead end, exit
@@ -276,10 +253,7 @@ impl<'a> FlatTrie<'a> {
};
if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {
- assert!(
- !compiled_grammar.is_terminated(),
- "Grammar has terminated but llm continued generation"
- );
+ assert!(!compiled_grammar.is_terminated(), "Grammar has terminated but llm continued generation");
}
current_token = next_token;
diff --git a/crates/uzu/src/utils/attention.rs b/crates/uzu/src/utils/attention.rs
index 67ea27ae3..a50c60a80 100644
--- a/crates/uzu/src/utils/attention.rs
+++ b/crates/uzu/src/utils/attention.rs
@@ -17,12 +17,7 @@ pub fn fill_attention_bias(
_ => panic!("Unsupported data type for attention bias fill"),
};
- fill_typed_for_dtype(
- dst,
- suffix_length,
- prefix_segment_length,
- should_be_neg_inf,
- );
+ fill_typed_for_dtype(dst, suffix_length, prefix_segment_length, should_be_neg_inf);
}
fn fill_typed<T: ArrayElement + Float>(
diff --git a/crates/uzu/src/utils/device.rs b/crates/uzu/src/utils/device.rs
index 4d805505a..7e01c2fc0 100644
--- a/crates/uzu/src/utils/device.rs
+++ b/crates/uzu/src/utils/device.rs
@@ -12,9 +12,7 @@ impl DeviceClass {
pub fn detect() -> Self {
use metal::MTLDeviceExt as _;
let device: crate::backends::metal::Retained<
- crate::backends::metal::ProtocolObject<
- dyn crate::backends::metal::MTLDevice,
- >,
+ crate::backends::metal::ProtocolObject<dyn crate::backends::metal::MTLDevice>,
> = match <dyn metal::MTLDevice>::system_default() {
Some(dev) => dev,
None => return DeviceClass::Unknown,
diff --git a/crates/uzu/src/utils/model_size.rs b/crates/uzu/src/utils/model_size.rs
index 5e7a0eb89..98fab6bd3 100644
--- a/crates/uzu/src/utils/model_size.rs
+++ b/crates/uzu/src/utils/model_size.rs
@@ -11,8 +11,7 @@ pub enum ModelSize {
impl ModelSize {
pub fn from_path(model_path: &Path) -> Self {
let weights_path = model_path.join("model.safetensors");
- let size_bytes =
- std::fs::metadata(&weights_path).map(|m| m.len()).unwrap_or(0);
+ let size_bytes = std::fs::metadata(&weights_path).map(|m| m.len()).unwrap_or(0);
Self::from_bytes(size_bytes)
}
@@ -42,10 +41,7 @@ mod tests {
.join(VERSION)
.join(MODEL_DIR_NAME);
if !path.exists() {
- panic!(
- "Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.",
- path
- );
+ panic!("Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.", path);
}
path
}
diff --git a/crates/uzu/src/utils/storage.rs b/crates/uzu/src/utils/storage.rs
index 4f39fa575..48c615766 100644
--- a/crates/uzu/src/utils/storage.rs
+++ b/crates/uzu/src/utils/storage.rs
@@ -14,14 +14,10 @@ pub fn user_domain_path(dir: NSSearchPathDirectory) -> PathBuf {
if let Some(path) = objc2::rc::autoreleasepool(|_| unsafe {
let fm = NSFileManager::defaultManager();
- let urls = fm.URLsForDirectory_inDomains(
- dir,
- NSSearchPathDomainMask::UserDomainMask,
- );
+ let urls = fm.URLsForDirectory_inDomains(dir, NSSearchPathDomainMask::UserDomainMask);
urls.firstObject().map(|url| {
- let ns_path: Retained<NSString> =
- url.path().expect("URL should have a path");
+ let ns_path: Retained<NSString> = url.path().expect("URL should have a path");
PathBuf::from(ns_path.to_string())
})
}) {
diff --git a/crates/uzu/tests/allocator_test/caching_allocator.rs b/crates/uzu/tests/allocator_test/caching_allocator.rs
index e24c049e5..e5852e80d 100644
--- a/crates/uzu/tests/allocator_test/caching_allocator.rs
+++ b/crates/uzu/tests/allocator_test/caching_allocator.rs
@@ -27,10 +27,7 @@ impl AllocatorTrait for CachingAllocator {
&self,
size: usize,
) -> Self::Buffer {
- self.context
- .allocator()
- .alloc(BufferLifetime::Scratch, size)
- .expect("Allocation failed")
+ self.context.allocator().alloc(BufferLifetime::Scratch, size).expect("Allocation failed")
}
fn free(
diff --git a/crates/uzu/tests/allocator_test/device_allocator.rs b/crates/uzu/tests/allocator_test/device_allocator.rs
index 905a429f8..0ee9a33ac 100644
--- a/crates/uzu/tests/allocator_test/device_allocator.rs
+++ b/crates/uzu/tests/allocator_test/device_allocator.rs
@@ -13,8 +13,7 @@ pub struct DeviceAllocator {
impl DeviceAllocator {
pub fn new() -> Self {
- let device =
- <dyn MTLDevice>::system_default().expect("No Metal device");
+ let device = <dyn MTLDevice>::system_default().expect("No Metal device");
Self {
device,
@@ -31,10 +30,8 @@ impl AllocatorTrait for DeviceAllocator {
&self,
size: usize,
) -> Self::Buffer {
- let buffer = self
- .device
- .new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let buffer =
+ self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");
let buf_size = buffer.length();
let new_active = self.active_memory.get() + buf_size;
diff --git a/crates/uzu/tests/allocator_test/heap_allocator.rs b/crates/uzu/tests/allocator_test/heap_allocator.rs
index 2793328eb..9376baaae 100644
--- a/crates/uzu/tests/allocator_test/heap_allocator.rs
+++ b/crates/uzu/tests/allocator_test/heap_allocator.rs
@@ -1,9 +1,6 @@
use std::cell::Cell;
-use metal::{
- MTLBuffer, MTLDevice, MTLDeviceExt, MTLHeap, MTLHeapDescriptor,
- MTLResourceOptions, MTLStorageMode,
-};
+use metal::{MTLBuffer, MTLDevice, MTLDeviceExt, MTLHeap, MTLHeapDescriptor, MTLResourceOptions, MTLStorageMode};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::allocator_trait::AllocatorTrait;
@@ -16,16 +13,13 @@ pub struct MTLHeapAllocator {
impl MTLHeapAllocator {
pub fn new(heap_size: usize) -> Self {
- let device =
- <dyn MTLDevice>::system_default().expect("No Metal device");
+ let device = <dyn MTLDevice>::system_default().expect("No Metal device");
let descriptor = MTLHeapDescriptor::new();
descriptor.set_size(heap_size);
descriptor.set_storage_mode(MTLStorageMode::Shared);
- let heap = device
- .new_heap_with_descriptor(&descriptor)
- .expect("Failed to create heap");
+ let heap = device.new_heap_with_descriptor(&descriptor).expect("Failed to create heap");
Self {
heap,
diff --git a/crates/uzu/tests/allocator_test/inference_simulation.rs b/crates/uzu/tests/allocator_test/inference_simulation.rs
index cd92d610d..812108b1e 100644
--- a/crates/uzu/tests/allocator_test/inference_simulation.rs
+++ b/crates/uzu/tests/allocator_test/inference_simulation.rs
@@ -1,9 +1,6 @@
use std::time::Instant;
-use super::{
- allocation_stats::AllocationStats, allocator_trait::AllocatorTrait,
- model_config::ModelConfig,
-};
+use super::{allocation_stats::AllocationStats, allocator_trait::AllocatorTrait, model_config::ModelConfig};
const F16_SIZE: usize = 2;
@@ -82,26 +79,16 @@ impl<'a, A: AllocatorTrait> InferenceSimulation<'a, A> {
let qkv_size = batch_size * self.config.qkv_dim() * F16_SIZE;
buffers.push(self.alloc_scratch(qkv_size));
- let bias_size =
- batch_size * (batch_size + self.current_prefix_len) * F16_SIZE;
+ let bias_size = batch_size * (batch_size + self.current_prefix_len) * F16_SIZE;
buffers.push(self.alloc_scratch(bias_size));
- let rotated_q_size = self.config.num_heads
- * batch_size
- * self.config.head_dim
- * F16_SIZE;
+ let rotated_q_size = self.config.num_heads * batch_size * self.config.head_dim * F16_SIZE;
buffers.push(self.alloc_scratch(rotated_q_size));
- let rotated_k_size = self.config.num_groups
- * batch_size
- * self.config.head_dim
- * F16_SIZE;
+ let rotated_k_size = self.config.num_groups * batch_size * self.config.head_dim * F16_SIZE;
buffers.push(self.alloc_scratch(rotated_k_size));
- let attn_out_size = batch_size
- * self.config.num_heads
- * self.config.head_dim
- * F16_SIZE;
+ let attn_out_size = batch_size * self.config.num_heads * self.config.head_dim * F16_SIZE;
buffers.push(self.alloc_scratch(attn_out_size));
buffers
diff --git a/crates/uzu/tests/allocator_test/main.rs b/crates/uzu/tests/allocator_test/main.rs
index 8cc4c59e1..0d0741726 100644
--- a/crates/uzu/tests/allocator_test/main.rs
+++ b/crates/uzu/tests/allocator_test/main.rs
@@ -13,9 +13,7 @@ use caching_allocator::CachingAllocator;
use device_allocator::DeviceAllocator;
use heap_allocator::MTLHeapAllocator;
use model_config::ModelConfig;
-use simulation_result::{
- run_long_simulation, run_simulation, run_simulation_with_config,
-};
+use simulation_result::{run_long_simulation, run_simulation, run_simulation_with_config};
use uzu::backends::common::Context;
const KB: f64 = 1024.0;
@@ -28,26 +26,17 @@ fn print_comparison(
device_result: &simulation_result::SimulationResult,
) {
println!("\n=== {} ===", name);
- println!(
- "{:<20} {:>14} {:>14} {:>14}",
- "Metric", "UZU Caching", "Heap", "Device"
- );
+ println!("{:<20} {:>14} {:>14} {:>14}", "Metric", "UZU Caching", "Heap", "Device");
println!("{:-<62}", "");
println!(
"{:<20} {:>14} {:>14} {:>14}",
- "Allocations",
- uzu_result.stats.total_allocs,
- heap_result.stats.total_allocs,
- device_result.stats.total_allocs
+ "Allocations", uzu_result.stats.total_allocs, heap_result.stats.total_allocs, device_result.stats.total_allocs
);
println!(
"{:<20} {:>14} {:>14} {:>14}",
- "Frees",
- uzu_result.stats.total_frees,
- heap_result.stats.total_frees,
- device_result.stats.total_frees
+ "Frees", uzu_result.stats.total_frees, heap_result.stats.total_frees, device_result.stats.total_frees
);
println!(
@@ -93,15 +82,10 @@ fn print_comparison(
println!();
let device_time = device_result.stats.avg_alloc_time_us().max(0.001);
- let uzu_speedup =
- device_time / uzu_result.stats.avg_alloc_time_us().max(0.001);
- let heap_speedup =
- device_time / heap_result.stats.avg_alloc_time_us().max(0.001);
+ let uzu_speedup = device_time / uzu_result.stats.avg_alloc_time_us().max(0.001);
+ let heap_speedup = device_time / heap_result.stats.avg_alloc_time_us().max(0.001);
- println!(
- "Alloc speedup vs device: UZU {:.1}x, Heap {:.1}x",
- uzu_speedup, heap_speedup
- );
+ println!("Alloc speedup vs device: UZU {:.1}x, Heap {:.1}x", uzu_speedup, heap_speedup);
}
fn print_memory_over_time(
@@ -142,40 +126,17 @@ fn test_allocator_comparison() {
println!("========================================");
let uzu_allocator = CachingAllocator::new();
- let uzu_result = run_simulation(
- &uzu_allocator,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
+ let uzu_result = run_simulation(&uzu_allocator, prompt_length, generate_length, prefill_step_size);
let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);
- let heap_result = run_simulation(
- &heap_allocator,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
+ let heap_result = run_simulation(&heap_allocator, prompt_length, generate_length, prefill_step_size);
let device_allocator = DeviceAllocator::new();
- let device_result = run_simulation(
- &device_allocator,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
+ let device_result = run_simulation(&device_allocator, prompt_length, generate_length, prefill_step_size);
- print_comparison(
- "Short Conversation Results",
- &uzu_result,
- &heap_result,
- &device_result,
- );
+ print_comparison("Short Conversation Results", &uzu_result, &heap_result, &device_result);
- assert_eq!(
- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,
- "UZU allocator: allocs should equal frees"
- );
+ assert_eq!(uzu_result.stats.total_allocs, uzu_result.stats.total_frees, "UZU allocator: allocs should equal frees");
assert_eq!(
heap_result.stats.total_allocs, heap_result.stats.total_frees,
"Heap allocator: allocs should equal frees"
@@ -196,10 +157,7 @@ fn test_long_conversation_comparison() {
println!("\n========================================");
println!(" Long Conversation Test");
- println!(
- " {} conversations, ~{}p + ~{}g tokens each",
- num_conversations, avg_prompt_length, avg_generate_length
- );
+ println!(" {} conversations, ~{}p + ~{}g tokens each", num_conversations, avg_prompt_length, avg_generate_length);
println!("========================================");
let gc_limit = 256 * 1024 * 1024;
@@ -238,12 +196,7 @@ fn test_long_conversation_comparison() {
snapshot_interval,
);
- print_comparison(
- "Long Conversation Results",
- &uzu_result,
- &heap_result,
- &device_result,
- );
+ print_comparison("Long Conversation Results", &uzu_result, &heap_result, &device_result);
print_memory_over_time("UZU Caching", &uzu_result.snapshots, 15);
print_memory_over_time("Heap", &heap_result.snapshots, 15);
@@ -252,27 +205,18 @@ fn test_long_conversation_comparison() {
println!("\n--- Memory Efficiency Summary ---");
println!(
"UZU peak/requested ratio: {:.2}%",
- (uzu_result.peak_memory_bytes as f64
- / uzu_result.stats.total_bytes_requested as f64)
- * 100.0
+ (uzu_result.peak_memory_bytes as f64 / uzu_result.stats.total_bytes_requested as f64) * 100.0
);
println!(
"Heap peak/requested ratio: {:.2}%",
- (heap_result.peak_memory_bytes as f64
- / heap_result.stats.total_bytes_requested as f64)
- * 100.0
+ (heap_result.peak_memory_bytes as f64 / heap_result.stats.total_bytes_requested as f64) * 100.0
);
println!(
"Device peak/requested ratio: {:.2}%",
- (device_result.peak_memory_bytes as f64
- / device_result.stats.total_bytes_requested as f64)
- * 100.0
+ (device_result.peak_memory_bytes as f64 / device_result.stats.total_bytes_requested as f64) * 100.0
);
- assert_eq!(
- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,
- "UZU allocator: allocs should equal frees"
- );
+ assert_eq!(uzu_result.stats.total_allocs, uzu_result.stats.total_frees, "UZU allocator: allocs should equal frees");
assert_eq!(
device_result.stats.total_allocs, device_result.stats.total_frees,
"Device allocator: allocs should equal frees"
@@ -290,39 +234,21 @@ fn test_multi_model_comparison() {
println!(" {}p + {}g tokens per model", prompt_length, generate_length);
println!("========================================");
- println!(
- "\n{:<15} {:>12} {:>12} {:>12}",
- "Model", "UZU (Β΅s)", "Heap (Β΅s)", "Device (Β΅s)"
- );
+ println!("\n{:<15} {:>12} {:>12} {:>12}", "Model", "UZU (Β΅s)", "Heap (Β΅s)", "Device (Β΅s)");
println!("{:-<55}", "");
for config in ModelConfig::all_models() {
let uzu_allocator = CachingAllocator::new();
- let uzu_result = run_simulation_with_config(
- &uzu_allocator,
- config,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
+ let uzu_result =
+ run_simulation_with_config(&uzu_allocator, config, prompt_length, generate_length, prefill_step_size);
let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);
- let heap_result = run_simulation_with_config(
- &heap_allocator,
- config,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
+ let heap_result =
+ run_simulation_with_config(&heap_allocator, config, prompt_length, generate_length, prefill_step_size);
let device_allocator = DeviceAllocator::new();
- let device_result = run_simulation_with_config(
- &device_allocator,
- config,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
+ let device_result =
+ run_simulation_with_config(&device_allocator, config, prompt_length, generate_length, prefill_step_size);
println!(
"{:<15} {:>12.2} {:>12.2} {:>12.2}",
diff --git a/crates/uzu/tests/allocator_test/simulation_result.rs b/crates/uzu/tests/allocator_test/simulation_result.rs
index 822ae6e7e..d3f2a3184 100644
--- a/crates/uzu/tests/allocator_test/simulation_result.rs
+++ b/crates/uzu/tests/allocator_test/simulation_result.rs
@@ -91,16 +91,13 @@ pub fn run_long_simulation_with_config<A: AllocatorTrait>(
let mut all_snapshots = Vec::new();
for conv_idx in 0..num_conversations {
- let prompt_len =
- avg_prompt_length + (conv_idx % 3) * (avg_prompt_length / 4);
- let gen_len =
- avg_generate_length + (conv_idx % 5) * (avg_generate_length / 8);
+ let prompt_len = avg_prompt_length + (conv_idx % 3) * (avg_prompt_length / 4);
+ let gen_len = avg_generate_length + (conv_idx % 5) * (avg_generate_length / 8);
let prefill_snapshots = sim.run_prefill(prompt_len, prefill_step_size);
all_snapshots.extend(prefill_snapshots);
- let gen_snapshots =
- sim.run_generation_with_sampling(gen_len, snapshot_interval);
+ let gen_snapshots = sim.run_generation_with_sampling(gen_len, snapshot_interval);
all_snapshots.extend(gen_snapshots);
sim.current_prefix_len = 0;
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index 88ea91700..d8383a721 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -5,27 +5,24 @@ mod common;
use std::mem::size_of;
use bytemuck;
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,
- MTLDeviceExt, MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use ndarray::{Array3, Array4, s};
use uzu::backends::{
common::Context,
metal::{
KernelDataType, MTLContext, ProtocolObject, Retained,
kernel::attention::{
- AttentionGemmArguments, AttentionKernel, AttentionKernelVariant,
- AttentionSinglePassArguments, AttentionTwoPassArguments,
+ AttentionGemmArguments, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,
+ AttentionTwoPassArguments,
},
metal_extensions::CommandBufferTimingExt,
},
};
fn reference_attention(
- queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]
- keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
- values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
+ queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]
+ keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
+ values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
mask: Option<&Array3<f32>>, // [batch, seq_len, seq_len] or None
sinks: Option<&[f32]>,
scale: f32,
@@ -44,8 +41,7 @@ fn reference_attention(
for repeat in 0..n_repeats {
let q_head = kv_head * n_repeats + repeat;
- let q =
- scaled_queries.slice(s![b, q_head, .., ..]).to_owned(); // [L, D]
+ let q = scaled_queries.slice(s![b, q_head, .., ..]).to_owned(); // [L, D]
let k = keys.slice(s![b, kv_head, .., ..]).to_owned(); // [L_kv, D]
let v = values.slice(s![b, kv_head, .., ..]).to_owned(); // [L_kv, D]
@@ -53,28 +49,17 @@ fn reference_attention(
let mut scores = q.dot(&k.t());
if let Some(mask_data) = mask {
- let mask_slice = mask_data.slice(s![
- b,
- ..,
- ..kv_seq_len.min(seq_len)
- ]);
- scores
- .slice_mut(s![.., ..kv_seq_len.min(seq_len)])
- .zip_mut_with(&mask_slice, |s, &m| *s += m);
+ let mask_slice = mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);
+ scores.slice_mut(s![.., ..kv_seq_len.min(seq_len)]).zip_mut_with(&mask_slice, |s, &m| *s += m);
}
// Softmax with optional sink logit
for i in 0..seq_len {
let sink_logit = sinks.map(|s| s[q_head]);
let row = scores.row(i);
- let max_score = row.iter().fold(
- sink_logit.unwrap_or(f32::NEG_INFINITY),
- |acc, &x| acc.max(x),
- );
-
- let mut sum_exp = sink_logit
- .map(|sink| (sink - max_score).exp())
- .unwrap_or(0.0);
+ let max_score = row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |acc, &x| acc.max(x));
+
+ let mut sum_exp = sink_logit.map(|sink| (sink - max_score).exp()).unwrap_or(0.0);
for j in 0..kv_seq_len {
scores[[i, j]] = (scores[[i, j]] - max_score).exp();
sum_exp += scores[[i, j]];
@@ -84,9 +69,7 @@ fn reference_attention(
// Efficient matrix multiplication: output = scores @ V
let head_output = scores.dot(&v);
- output
- .slice_mut(s![b, q_head, .., ..])
- .assign(&head_output);
+ output.slice_mut(s![b, q_head, .., ..]).assign(&head_output);
}
}
} else {
@@ -99,25 +82,17 @@ fn reference_attention(
let mut scores = q.dot(&k.t());
if let Some(mask_data) = mask {
- let mask_slice =
- mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);
- scores
- .slice_mut(s![.., ..kv_seq_len.min(seq_len)])
- .zip_mut_with(&mask_slice, |s, &m| *s += m);
+ let mask_slice = mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);
+ scores.slice_mut(s![.., ..kv_seq_len.min(seq_len)]).zip_mut_with(&mask_slice, |s, &m| *s += m);
}
// Softmax with optional sink logit
for i in 0..seq_len {
let sink_logit = sinks.map(|s| s[h]);
let row = scores.row(i);
- let max_score = row.iter().fold(
- sink_logit.unwrap_or(f32::NEG_INFINITY),
- |acc, &x| acc.max(x),
- );
-
- let mut sum_exp = sink_logit
- .map(|sink| (sink - max_score).exp())
- .unwrap_or(0.0);
+ let max_score = row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |acc, &x| acc.max(x));
+
+ let mut sum_exp = sink_logit.map(|sink| (sink - max_score).exp()).unwrap_or(0.0);
for j in 0..kv_seq_len {
scores[[i, j]] = (scores[[i, j]] - max_score).exp();
sum_exp += scores[[i, j]];
@@ -147,29 +122,19 @@ fn create_test_data(
let mut rng = StdRng::seed_from_u64(seed);
- let queries = Array4::from_shape_fn(
- (batch_size, num_heads, seq_len, head_dim),
- |_| rng.random_range(-0.5..0.5),
- );
+ let queries = Array4::from_shape_fn((batch_size, num_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));
- let keys = Array4::from_shape_fn(
- (batch_size, num_kv_heads, seq_len, head_dim),
- |_| rng.random_range(-0.5..0.5),
- );
+ let keys = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));
- let values = Array4::from_shape_fn(
- (batch_size, num_kv_heads, seq_len, head_dim),
- |_| rng.random_range(-0.5..0.5),
- );
+ let values = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));
- let mask =
- Array3::from_shape_fn((batch_size, seq_len, seq_len), |(_, i, j)| {
- if j <= i {
- 0.0
- } else {
- -1e9
- }
- });
+ let mask = Array3::from_shape_fn((batch_size, seq_len, seq_len), |(_, i, j)| {
+ if j <= i {
+ 0.0
+ } else {
+ -1e9
+ }
+ });
(queries, keys, values, mask)
}
@@ -195,10 +160,7 @@ fn create_query_buffer(
context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&query_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&query_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
}
@@ -210,8 +172,7 @@ fn create_key_cache_buffer(
let (_batch_size, num_kv_heads, seq_len, head_dim) = keys.dim();
// Our kernel expects key cache layout: [num_kv_heads, max_seq_len, head_dim]
- let mut key_cache_data =
- vec![0.0f32; num_kv_heads * max_seq_len * head_dim];
+ let mut key_cache_data = vec![0.0f32; num_kv_heads * max_seq_len * head_dim];
for h in 0..num_kv_heads {
for t in 0..seq_len {
@@ -224,10 +185,7 @@ fn create_key_cache_buffer(
context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&key_cache_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&key_cache_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
}
@@ -239,8 +197,7 @@ fn create_value_cache_buffer(
let (_batch_size, num_kv_heads, seq_len, head_dim) = values.dim();
// Our kernel expects value cache layout: [num_kv_heads, max_seq_len, head_dim]
- let mut value_cache_data =
- vec![0.0f32; num_kv_heads * max_seq_len * head_dim];
+ let mut value_cache_data = vec![0.0f32; num_kv_heads * max_seq_len * head_dim];
for h in 0..num_kv_heads {
for t in 0..seq_len {
@@ -253,10 +210,7 @@ fn create_value_cache_buffer(
context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&value_cache_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&value_cache_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
}
@@ -280,10 +234,7 @@ fn create_mask_buffer(
context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&mask_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&mask_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
}
@@ -302,10 +253,7 @@ fn create_mask_2d_buffer(
context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&mask_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&mask_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
}
@@ -315,10 +263,7 @@ fn create_sinks_buffer(
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&sinks),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&sinks), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
}
@@ -329,8 +274,7 @@ fn convert_kernel_output(
seq_len: usize,
head_dim: usize,
) -> Array4<f32> {
- let mut kernel_output =
- Array4::zeros((batch_size, num_heads, seq_len, head_dim));
+ let mut kernel_output = Array4::zeros((batch_size, num_heads, seq_len, head_dim));
for h in 0..num_heads {
for t in 0..seq_len {
for d in 0..head_dim {
@@ -359,27 +303,18 @@ fn run_single_pass_attention(
let query_buffer = create_query_buffer(queries, context);
let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
- let value_cache_buffer =
- create_value_cache_buffer(values, seq_len, context);
+ let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));
let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
let output_buffer = context
.device
- .new_buffer(
- num_heads * seq_len * head_dim * size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let args = AttentionSinglePassArguments {
queries_buffer: &query_buffer,
@@ -412,17 +347,9 @@ fn run_single_pass_attention(
command_buffer.wait_until_completed();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe {
- std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)
- };
+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
- let kernel_output = convert_kernel_output(
- output_slice,
- batch_size,
- num_heads,
- seq_len,
- head_dim,
- );
+ let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);
Ok(kernel_output)
}
@@ -443,27 +370,18 @@ fn run_single_pass_attention_with_is_causal(
let query_buffer = create_query_buffer(queries, context);
let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
- let value_cache_buffer =
- create_value_cache_buffer(values, seq_len, context);
+ let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));
let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
let output_buffer = context
.device
- .new_buffer(
- num_heads * seq_len * head_dim * size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let args = AttentionSinglePassArguments {
queries_buffer: &query_buffer,
@@ -496,17 +414,9 @@ fn run_single_pass_attention_with_is_causal(
command_buffer.wait_until_completed();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe {
- std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)
- };
+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
- let kernel_output = convert_kernel_output(
- output_slice,
- batch_size,
- num_heads,
- seq_len,
- head_dim,
- );
+ let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);
Ok(kernel_output)
}
@@ -527,27 +437,18 @@ fn run_gemm_attention(
let query_buffer = create_query_buffer(queries, context);
let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
- let value_cache_buffer =
- create_value_cache_buffer(values, seq_len, context);
+ let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
let mask_buffer = mask.map(|m| create_mask_2d_buffer(m, context));
let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
let output_buffer = context
.device
- .new_buffer(
- num_heads * seq_len * head_dim * size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let args = AttentionGemmArguments {
queries_buffer: &query_buffer,
@@ -575,17 +476,9 @@ fn run_gemm_attention(
command_buffer.wait_until_completed();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe {
- std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)
- };
+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
- let kernel_output = convert_kernel_output(
- output_slice,
- batch_size,
- num_heads,
- seq_len,
- head_dim,
- );
+ let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);
Ok(kernel_output)
}
@@ -596,11 +489,7 @@ fn compare_results(
tolerance: f32,
test_name: &str,
) -> Result<(), String> {
- let max_diff = kernel_output
- .iter()
- .zip(reference_output.iter())
- .map(|(a, b)| (a - b).abs())
- .fold(0.0f32, f32::max);
+ let max_diff = kernel_output.iter().zip(reference_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);
let relative_error = kernel_output
.iter()
@@ -618,29 +507,15 @@ fn compare_results(
println!("Max relative error ({}): {}", test_name, relative_error);
println!("Debug - Sample values ({}):", test_name);
- println!(
- "Kernel output [0,0,0,0:4]: {:?}",
- &kernel_output.slice(s![0, 0, 0, 0..4]).to_vec()
- );
- println!(
- "Reference output [0,0,0,0:4]: {:?}",
- &reference_output.slice(s![0, 0, 0, 0..4]).to_vec()
- );
- println!(
- "Kernel output [0,0,1,0:4]: {:?}",
- &kernel_output.slice(s![0, 0, 1, 0..4]).to_vec()
- );
- println!(
- "Reference output [0,0,1,0:4]: {:?}",
- &reference_output.slice(s![0, 0, 1, 0..4]).to_vec()
- );
+ println!("Kernel output [0,0,0,0:4]: {:?}", &kernel_output.slice(s![0, 0, 0, 0..4]).to_vec());
+ println!("Reference output [0,0,0,0:4]: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());
+ println!("Kernel output [0,0,1,0:4]: {:?}", &kernel_output.slice(s![0, 0, 1, 0..4]).to_vec());
+ println!("Reference output [0,0,1,0:4]: {:?}", &reference_output.slice(s![0, 0, 1, 0..4]).to_vec());
let kernel_max = kernel_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));
- let kernel_min =
- kernel_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
+ let kernel_min = kernel_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));
- let ref_min =
- reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
+ let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
println!("Kernel output range: [{}, {}]", kernel_min, kernel_max);
println!("Reference output range: [{}, {}]", ref_min, ref_max);
@@ -668,27 +543,15 @@ fn test_single_pass_attention_basic() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 42,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
println!("Testing reference implementation without mask...");
- let reference_output =
- reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));
- let ref_min =
- reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
+ let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
println!("Reference output range (no mask): [{}, {}]", ref_min, ref_max);
- println!(
- "Reference sample values: {:?}",
- &reference_output.slice(s![0, 0, 0, 0..4]).to_vec()
- );
+ println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());
let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
Ok(k) => k,
@@ -705,19 +568,14 @@ fn test_single_pass_attention_basic() {
head_dim,
kernel.supports_single_pass(head_dim, is_causal, false)
);
- println!(
- "Supports two-pass for head_dim={}: {}",
- head_dim,
- kernel.supports_two_pass(head_dim, is_causal, false)
- );
+ println!("Supports two-pass for head_dim={}: {}", head_dim, kernel.supports_two_pass(head_dim, is_causal, false));
if !kernel.supports_single_pass(head_dim, false, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
- let kernel_output = match run_single_pass_attention(
- &kernel, &context, &queries, &keys, &values, None, None, scale,
- ) {
+ let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)
+ {
Ok(output) => output,
Err(e) => {
panic!("Failed to run single-pass attention: {:?}", e);
@@ -725,12 +583,7 @@ fn test_single_pass_attention_basic() {
};
let tolerance = 1e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Single-pass attention",
- ) {
+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention") {
panic!("{}", e);
}
}
@@ -747,34 +600,18 @@ fn test_gemm_attention_basic() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 123,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
- let kernel = AttentionKernel::new(&context, KernelDataType::Float32)
- .expect("Failed to create AttentionKernel");
+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");
- let reference_output =
- reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
- let kernel_output = run_gemm_attention(
- &kernel, &context, &queries, &keys, &values, None, None, scale,
- /*is_causal=*/ false,
- )
- .expect("run gemm attention");
+ let kernel_output =
+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, None, scale, /*is_causal=*/ false)
+ .expect("run gemm attention");
let tolerance = 1e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Gemm attention",
- ) {
+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Gemm attention") {
panic!("{}", e);
}
}
@@ -791,34 +628,18 @@ fn test_gemm_attention_f32_head_dim_128() {
let head_dim = 128;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 456,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);
- let kernel = AttentionKernel::new(&context, KernelDataType::Float32)
- .expect("Failed to create AttentionKernel");
+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");
- let reference_output =
- reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
- let kernel_output = run_gemm_attention(
- &kernel, &context, &queries, &keys, &values, None, None, scale,
- /*is_causal=*/ false,
- )
- .expect("run gemm attention f32 head_dim=128");
+ let kernel_output =
+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, None, scale, /*is_causal=*/ false)
+ .expect("run gemm attention f32 head_dim=128");
let tolerance = 1e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Gemm attention f32 head_dim=128",
- ) {
+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Gemm attention f32 head_dim=128") {
panic!("{}", e);
}
}
@@ -836,21 +657,12 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = true;
- let (queries, keys, values, mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 2026,
- );
+ let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);
- let kernel = AttentionKernel::new(&context, KernelDataType::Float32)
- .expect("Failed to create AttentionKernel");
+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");
// CPU reference (uses the same additive mask)
- let reference_output =
- reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
// vector attention path (single-pass)
let vector_output = run_single_pass_attention_with_is_causal(
@@ -867,46 +679,19 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
.expect("run vector attention");
// matrix attention path (gemm)
- let matrix_output = run_gemm_attention(
- &kernel,
- &context,
- &queries,
- &keys,
- &values,
- Some(&mask),
- None,
- scale,
- is_causal,
- )
- .expect("run matrix attention");
+ let matrix_output =
+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale, is_causal)
+ .expect("run matrix attention");
// Compare both to CPU
let tol_cpu = 5e-2;
- compare_results(
- &vector_output,
- &reference_output,
- tol_cpu,
- "vector single-pass attention vs CPU",
- )
- .unwrap();
- compare_results(
- &matrix_output,
- &reference_output,
- tol_cpu,
- "matrix attention vs CPU",
- )
- .unwrap();
+ compare_results(&vector_output, &reference_output, tol_cpu, "vector single-pass attention vs CPU").unwrap();
+ compare_results(&matrix_output, &reference_output, tol_cpu, "matrix attention vs CPU").unwrap();
- let max_diff_vector_matrix = vector_output
- .iter()
- .zip(matrix_output.iter())
- .map(|(a, b)| (a - b).abs())
- .fold(0.0f32, f32::max);
+ let max_diff_vector_matrix =
+ vector_output.iter().zip(matrix_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);
let tol_vector_matrix = 5e-2;
- println!(
- "Max absolute difference (vector single-pass attention vs matrix attention): {}",
- max_diff_vector_matrix
- );
+ println!("Max absolute difference (vector single-pass attention vs matrix attention): {}", max_diff_vector_matrix);
assert!(
max_diff_vector_matrix <= tol_vector_matrix,
"vector single-pass attention and matrix attention differ too much: max_diff = {} (tol={})",
@@ -927,14 +712,7 @@ fn test_single_pass_attention_with_mask() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 42,
- );
+ let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
Ok(k) => k,
@@ -947,24 +725,15 @@ fn test_single_pass_attention_with_mask() {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
- let kernel_output = match run_single_pass_attention(
- &kernel,
- &context,
- &queries,
- &keys,
- &values,
- Some(&mask),
- None,
- scale,
- ) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run single-pass attention with mask: {:?}", e);
- },
- };
+ let kernel_output =
+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale) {
+ Ok(output) => output,
+ Err(e) => {
+ panic!("Failed to run single-pass attention with mask: {:?}", e);
+ },
+ };
- let reference_output =
- reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
println!("Mask values:");
for i in 0..seq_len.min(4) {
@@ -975,12 +744,7 @@ fn test_single_pass_attention_with_mask() {
}
let tolerance = 1e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Single-pass attention with mask",
- ) {
+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention with mask") {
panic!("{}", e);
}
}
@@ -997,18 +761,9 @@ fn test_single_pass_attention_with_sinks() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 444,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);
- let sinks: Vec<f32> = (0..num_heads)
- .map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25)
- .collect();
+ let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();
println!("Using sinks: {:?}", sinks);
let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
@@ -1022,38 +777,18 @@ fn test_single_pass_attention_with_sinks() {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
- let reference_output = reference_attention(
- &queries,
- &keys,
- &values,
- None,
- Some(&sinks),
- scale,
- );
+ let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);
- let kernel_output = match run_single_pass_attention(
- &kernel,
- &context,
- &queries,
- &keys,
- &values,
- None,
- Some(&sinks),
- scale,
- ) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run single-pass attention with sinks: {:?}", e);
- },
- };
+ let kernel_output =
+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, Some(&sinks), scale) {
+ Ok(output) => output,
+ Err(e) => {
+ panic!("Failed to run single-pass attention with sinks: {:?}", e);
+ },
+ };
let tolerance = 1e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Single-pass attention with sinks",
- ) {
+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention with sinks") {
panic!("{}", e);
}
}
@@ -1070,17 +805,9 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 777,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);
- let sinks: Vec<f32> =
- (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
+ let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
println!("Long sequence sinks: {:?}", sinks);
let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
@@ -1094,33 +821,15 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
- let reference_output = reference_attention(
- &queries,
- &keys,
- &values,
- None,
- Some(&sinks),
- scale,
- );
+ let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);
- let kernel_output = match run_single_pass_attention(
- &kernel,
- &context,
- &queries,
- &keys,
- &values,
- None,
- Some(&sinks),
- scale,
- ) {
- Ok(output) => output,
- Err(e) => {
- panic!(
- "Failed to run single-pass attention long sequence with sinks: {:?}",
- e
- );
- },
- };
+ let kernel_output =
+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, Some(&sinks), scale) {
+ Ok(output) => output,
+ Err(e) => {
+ panic!("Failed to run single-pass attention long sequence with sinks: {:?}", e);
+ },
+ };
let tolerance = 5e-2;
if let Err(e) = compare_results(
@@ -1145,14 +854,7 @@ fn test_single_pass_attention_gqa() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 42,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
Ok(k) => k,
@@ -1169,25 +871,18 @@ fn test_single_pass_attention_gqa() {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
- let kernel_output = match run_single_pass_attention(
- &kernel, &context, &queries, &keys, &values, None, None, scale,
- ) {
+ let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)
+ {
Ok(output) => output,
Err(e) => {
panic!("Failed to run single-pass attention GQA: {:?}", e);
},
};
- let reference_output =
- reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
let tolerance = 1e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Single-pass attention GQA",
- ) {
+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention GQA") {
panic!("{}", e);
}
}
@@ -1217,26 +912,17 @@ fn run_two_pass_attention(
let partials_buffer = context
.device
- .new_buffer(
- partials_size * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(partials_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let sums_buffer = context
.device
- .new_buffer(
- sums_maxs_size * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let maxs_buffer = context
.device
- .new_buffer(
- sums_maxs_size * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buffer = context
@@ -1247,13 +933,8 @@ fn run_two_pass_attention(
)
.expect("Failed to create buffer");
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let args = AttentionTwoPassArguments {
queries_buffer: &queries_buffer,
@@ -1289,17 +970,9 @@ fn run_two_pass_attention(
command_buffer.wait_until_completed();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe {
- std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)
- };
+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
- let kernel_output = convert_kernel_output(
- output_slice,
- batch_size,
- num_heads,
- seq_len,
- head_dim,
- );
+ let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);
Ok(kernel_output)
}
@@ -1330,27 +1003,14 @@ fn test_two_pass_attention() {
let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
- panic!(
- "Two-pass not selected for seq_len={}. Got {:?}",
- seq_len, variant
- );
+ panic!("Two-pass not selected for seq_len={}. Got {:?}", seq_len, variant);
}
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 42,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let reference_output =
- reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
- let kernel_output = match run_two_pass_attention(
- &kernel, &context, &queries, &keys, &values, None, None, scale,
- ) {
+ let kernel_output = match run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale) {
Ok(output) => output,
Err(e) => {
panic!("Failed to run two-pass attention: {:?}", e);
@@ -1358,12 +1018,7 @@ fn test_two_pass_attention() {
};
let tolerance = 1e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Two-pass attention",
- ) {
+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention") {
panic!("{}", e);
}
}
@@ -1394,27 +1049,14 @@ fn test_two_pass_attention_gqa() {
let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
- panic!(
- "Two-pass not selected for GQA seq_len={}. Got {:?}",
- seq_len, variant
- );
+ panic!("Two-pass not selected for GQA seq_len={}. Got {:?}", seq_len, variant);
}
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 42,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let reference_output =
- reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
- let kernel_output = match run_two_pass_attention(
- &kernel, &context, &queries, &keys, &values, None, None, scale,
- ) {
+ let kernel_output = match run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale) {
Ok(output) => output,
Err(e) => {
panic!("Failed to run two-pass attention GQA: {:?}", e);
@@ -1422,12 +1064,7 @@ fn test_two_pass_attention_gqa() {
};
let tolerance = 1e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Two-pass attention GQA",
- ) {
+ if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention GQA") {
panic!("{}", e);
}
}
@@ -1457,19 +1094,13 @@ fn perf_two_pass_attention() {
let is_causal = false;
if !kernel.supports_two_pass(head_dim, is_causal, false) {
- println!(
- "Skipping two-pass perf test: not supported for head_dim={}",
- head_dim
- );
+ println!("Skipping two-pass perf test: not supported for head_dim={}", head_dim);
return;
}
let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
- println!(
- "Skipping two-pass perf test: variant {:?} selected instead",
- variant
- );
+ println!("Skipping two-pass perf test: variant {:?} selected instead", variant);
return;
}
@@ -1478,48 +1109,30 @@ fn perf_two_pass_attention() {
seq_len - suffix_length,
suffix_length
);
- let (queries, keys, values, _mask) = create_test_data(
- batch_size,
- num_heads,
- num_kv_heads,
- seq_len,
- head_dim,
- 123,
- );
+ let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
// ---- Create buffers ----
// For realistic inference, we only process queries for the suffix (new tokens)
- let queries_suffix =
- queries.slice(s![.., .., (seq_len - suffix_length).., ..]).to_owned();
+ let queries_suffix = queries.slice(s![.., .., (seq_len - suffix_length).., ..]).to_owned();
let queries_buffer = create_query_buffer(&queries_suffix, &context);
let keys_buffer = create_key_cache_buffer(&keys, seq_len, &context);
let values_buffer = create_value_cache_buffer(&values, seq_len, &context);
let total_blocks_count = 32;
- let partials_size =
- num_heads * suffix_length * total_blocks_count * head_dim;
+ let partials_size = num_heads * suffix_length * total_blocks_count * head_dim;
let sums_maxs_size = num_heads * suffix_length * total_blocks_count;
let partials_buffer = context
.device
- .new_buffer(
- partials_size * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(partials_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let sums_buffer = context
.device
- .new_buffer(
- sums_maxs_size * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let maxs_buffer = context
.device
- .new_buffer(
- sums_maxs_size * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buffer = context
.device
@@ -1530,13 +1143,8 @@ fn perf_two_pass_attention() {
.expect("Failed to create buffer");
// ---- Launch and time ----
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let args = AttentionTwoPassArguments {
queries_buffer: &queries_buffer,
@@ -1603,12 +1211,7 @@ fn perf_two_pass_attention() {
// ---- Sanity check ----
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe {
- std::slice::from_raw_parts(
- output_ptr,
- num_heads * suffix_length * head_dim,
- )
- };
+ let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * suffix_length * head_dim) };
// Check for NaN/Inf
for &val in output_slice.iter().take(100) {
diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rs
index 78cdbed95..b99e24e92 100644
--- a/crates/uzu/tests/common/mod.rs
+++ b/crates/uzu/tests/common/mod.rs
@@ -16,10 +16,7 @@ pub fn get_test_model_path() -> PathBuf {
.join(VERSION)
.join(MODEL_DIR_NAME);
if !path.exists() {
- panic!(
- "Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.",
- path
- );
+ panic!("Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.", path);
}
path
}
@@ -58,11 +55,7 @@ impl Speculator for StaticSpeculator {
&self,
prefix: &[u64],
) -> HashMap<u64, f32> {
- self.responses
- .get(prefix)
- .cloned()
- .or_else(|| self.default_response.clone())
- .unwrap_or_default()
+ self.responses.get(prefix).cloned().or_else(|| self.default_response.clone()).unwrap_or_default()
}
}
diff --git a/crates/uzu/tests/context_mode_test.rs b/crates/uzu/tests/context_mode_test.rs
index 719fbd78b..2c4bbaebd 100644
--- a/crates/uzu/tests/context_mode_test.rs
+++ b/crates/uzu/tests/context_mode_test.rs
@@ -5,8 +5,7 @@ use uzu::session::{
Session,
config::{DecodingConfig, RunConfig, SpeculatorConfig},
parameter::{
- AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize,
- SamplingMethod, SamplingPolicy, SamplingSeed,
+ AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize, SamplingMethod, SamplingPolicy, SamplingSeed,
},
types::{Input, Message, Output},
};
@@ -54,10 +53,8 @@ fn request(
#[test]
fn test_context_mode_none() {
- let decoding_config =
- build_decoding_config().with_context_mode(ContextMode::None);
- let mut session =
- Session::new(build_model_path(), decoding_config).unwrap();
+ let decoding_config = build_decoding_config().with_context_mode(ContextMode::None);
+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();
let tokens_limit = 48;
let input_text = "What is Alice's occupation?".to_string();
@@ -77,98 +74,54 @@ fn test_context_mode_static() {
let parameter_occupation = "Engineer".to_string();
let parameter_city = "London".to_string();
- let decoding_config =
- build_decoding_config().with_context_mode(ContextMode::Static {
- input: Input::Messages(vec![
- Message::system(
- "Always answer with one word or number, no punctuation"
- .to_string(),
- ),
- Message::user(
- format!(
- "Name: {}\nAge: {}\nOccupation: {}\nCity: {}",
- parameter_name,
- parameter_age,
- parameter_occupation,
- parameter_city
- )
- .to_string(),
- ),
- ]),
- });
- let mut session =
- Session::new(build_model_path(), decoding_config).unwrap();
+ let decoding_config = build_decoding_config().with_context_mode(ContextMode::Static {
+ input: Input::Messages(vec![
+ Message::system("Always answer with one word or number, no punctuation".to_string()),
+ Message::user(
+ format!(
+ "Name: {}\nAge: {}\nOccupation: {}\nCity: {}",
+ parameter_name, parameter_age, parameter_occupation, parameter_city
+ )
+ .to_string(),
+ ),
+ ]),
+ });
+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();
let tokens_limit = 48;
- let response_occupation = request(
- &mut session,
- format!("What is {}'s occupation?", parameter_name),
- tokens_limit,
- );
- let response_age = request(
- &mut session,
- format!("What is {}'s age?", parameter_name),
- tokens_limit,
- );
- let response_city = request(
- &mut session,
- format!("What is {}'s city?", parameter_name),
- tokens_limit,
- );
+ let response_occupation = request(&mut session, format!("What is {}'s occupation?", parameter_name), tokens_limit);
+ let response_age = request(&mut session, format!("What is {}'s age?", parameter_name), tokens_limit);
+ let response_city = request(&mut session, format!("What is {}'s city?", parameter_name), tokens_limit);
println!("Occupation: {}", response_occupation);
println!("Age: {}", response_age);
println!("City: {}", response_city);
- assert!(
- response_occupation
- .to_lowercase()
- .contains(parameter_occupation.to_lowercase().as_str())
- );
- assert!(
- response_age
- .to_lowercase()
- .contains(parameter_age.to_lowercase().as_str())
- );
- assert!(
- response_city
- .to_lowercase()
- .contains(parameter_city.to_lowercase().as_str())
- );
+ assert!(response_occupation.to_lowercase().contains(parameter_occupation.to_lowercase().as_str()));
+ assert!(response_age.to_lowercase().contains(parameter_age.to_lowercase().as_str()));
+ assert!(response_city.to_lowercase().contains(parameter_city.to_lowercase().as_str()));
}
#[test]
#[ignore = "Flaky test - depends on LLM output which varies even with fixed seed"]
fn test_context_mode_dynamic() {
- let decoding_config =
- build_decoding_config().with_context_mode(ContextMode::Dynamic);
- let mut session =
- Session::new(build_model_path(), decoding_config).unwrap();
-
- let update = |session: &mut Session, input_text: String| {
- request(session, input_text, 0)
- };
- let ask = |session: &mut Session, input_text: String| {
- request(session, input_text, 48)
- };
+ let decoding_config = build_decoding_config().with_context_mode(ContextMode::Dynamic);
+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();
+
+ let update = |session: &mut Session, input_text: String| request(session, input_text, 0);
+ let ask = |session: &mut Session, input_text: String| request(session, input_text, 48);
update(&mut session, "Name: Dave. Occupation: Nurse. Age: 30.".to_string());
let answer_1 = ask(&mut session, "What is Dave's occupation?".to_string());
println!("Answer 1: {}", answer_1);
assert!(answer_1.to_lowercase().contains("nurse"));
- update(
- &mut session,
- "Update: Name: Dave. Occupation: Doctor. Age: 30.".to_string(),
- );
+ update(&mut session, "Update: Name: Dave. Occupation: Doctor. Age: 30.".to_string());
let answer_2 = ask(&mut session, "What is Dave's occupation?".to_string());
println!("Answer 2: {}", answer_2);
assert!(answer_2.to_lowercase().contains("doctor"));
- update(
- &mut session,
- "Update: Name: Dave. Occupation: Doctor. Age: 31.".to_string(),
- );
+ update(&mut session, "Update: Name: Dave. Occupation: Doctor. Age: 31.".to_string());
let answer_3 = ask(&mut session, "How old is Dave?".to_string());
println!("Answer 3: {}", answer_3);
assert!(answer_3.to_lowercase().contains("31"));
@@ -177,8 +130,7 @@ fn test_context_mode_dynamic() {
&mut session,
"Important update, Dave got promoted! Update: Name: Dave. Occupation: Director. Age: 32.".to_string(),
);
- let answer_4 =
- ask(&mut session, "What is Dave's current occupation?".to_string());
+ let answer_4 = ask(&mut session, "What is Dave's current occupation?".to_string());
println!("Answer 4: {}", answer_4);
assert!(answer_4.to_lowercase().contains("director"));
@@ -194,15 +146,10 @@ fn test_context_mode_dynamic() {
#[test]
fn test_context_mode_dynamic_scenario() {
- let decoding_config =
- build_decoding_config().with_context_mode(ContextMode::Dynamic);
- let mut session =
- Session::new(build_model_path(), decoding_config).unwrap();
-
- let user_prompts = vec![
- String::from("Tell about London"),
- String::from("Compare with New York"),
- ];
+ let decoding_config = build_decoding_config().with_context_mode(ContextMode::Dynamic);
+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();
+
+ let user_prompts = vec![String::from("Tell about London"), String::from("Compare with New York")];
println!("-------------------------");
for user_prompt in user_prompts {
diff --git a/crates/uzu/tests/grammar_json_schema_test.rs b/crates/uzu/tests/grammar_json_schema_test.rs
index 542a9207e..14d6cfcf7 100644
--- a/crates/uzu/tests/grammar_json_schema_test.rs
+++ b/crates/uzu/tests/grammar_json_schema_test.rs
@@ -44,41 +44,29 @@ fn test_grammar(speculator_config: SpeculatorConfig) {
let decoding_config = DecodingConfig::default()
.with_sampling_seed(uzu::prelude::SamplingSeed::Custom(42))
.with_speculator_config(speculator_config);
- let mut session = Session::new(model_dir, decoding_config)
- .expect("Failed to create session");
+ let mut session = Session::new(model_dir, decoding_config).expect("Failed to create session");
- let input = Input::Text(
- "Generate a detailed person profile with address and hobbies in JSON format.".to_string(),
- );
+ let input = Input::Text("Generate a detailed person profile with address and hobbies in JSON format.".to_string());
- let grammar_config = GrammarConfig::from_json_schema_type::<Person>()
- .expect("Failed to create grammar config");
+ let grammar_config = GrammarConfig::from_json_schema_type::<Person>().expect("Failed to create grammar config");
- let run_config = RunConfig::default()
- .tokens_limit(1024)
- .sampling_policy(SamplingPolicy::Default)
- .grammar_config(grammar_config);
+ let run_config =
+ RunConfig::default().tokens_limit(1024).sampling_policy(SamplingPolicy::Default).grammar_config(grammar_config);
- let output = session
- .run(input, run_config, None::<fn(uzu::session::types::Output) -> bool>)
- .expect("Failed to run session");
+ let output =
+ session.run(input, run_config, None::<fn(uzu::session::types::Output) -> bool>).expect("Failed to run session");
let stats = output.stats;
- let total_tokens = stats.total_stats.tokens_count_input
- + stats.total_stats.tokens_count_output;
+ let total_tokens = stats.total_stats.tokens_count_input + stats.total_stats.tokens_count_output;
println!("\n=== Test Results ===");
println!("Generated output length: {} chars", output.text.original.len());
println!(
"Input tokens: {}, Output tokens: {}",
- stats.total_stats.tokens_count_input,
- stats.total_stats.tokens_count_output
+ stats.total_stats.tokens_count_input, stats.total_stats.tokens_count_output
);
println!("Total time: {:.2}s", stats.total_stats.duration);
- println!(
- "Throughput: {:.2} tokens/s",
- total_tokens as f64 / stats.total_stats.duration
- );
+ println!("Throughput: {:.2} tokens/s", total_tokens as f64 / stats.total_stats.duration);
match serde_json::from_str::<Person>(&output.text.original) {
Ok(person) => {
@@ -86,10 +74,7 @@ fn test_grammar(speculator_config: SpeculatorConfig) {
println!(" Name: {}", person.name);
println!(" Age: {}", person.age);
println!(" Email: {}", person.email);
- println!(
- " Address: {}, {}",
- person.address.city, person.address.country
- );
+ println!(" Address: {}, {}", person.address.city, person.address.country);
println!(" Hobbies: {:?}", person.hobbies);
assert!(!person.name.is_empty());
@@ -99,10 +84,7 @@ fn test_grammar(speculator_config: SpeculatorConfig) {
assert!(!person.hobbies.is_empty());
},
Err(e) => {
- panic!(
- "βœ— Generated invalid JSON: {}\nOutput: {}",
- e, output.text.original
- );
+ panic!("βœ— Generated invalid JSON: {}\nOutput: {}", e, output.text.original);
},
}
}
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index c2ee30066..5490026fe 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -8,9 +8,7 @@ use uzu::{
common::Context,
metal::{KVCacheUpdate, MTLContext, Metal},
},
- forward_pass::kv_cache_layer::{
- INVALID_POSITION, KVCacheLayer, KVCacheLayerState,
- },
+ forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},
};
#[derive(Debug)]
@@ -49,16 +47,8 @@ fn make_test_layer(
};
let shape = [1, total_len.max(1), 1];
- let keys = std::cell::RefCell::new(context.create_array(
- &shape,
- DataType::F32,
- "kv_cache_keys",
- ));
- let values = std::cell::RefCell::new(context.create_array(
- &shape,
- DataType::F32,
- "kv_cache_values",
- ));
+ let keys = std::cell::RefCell::new(context.create_array(&shape, DataType::F32, "kv_cache_keys"));
+ let values = std::cell::RefCell::new(context.create_array(&shape, DataType::F32, "kv_cache_values"));
let prefix_token_positions = match &state {
KVCacheLayerState::Full {
@@ -107,13 +97,12 @@ fn expected_after_update(
initial: &[f32],
) -> Vec<f32> {
let mut expected = initial.to_vec();
- let effective_indices: Vec<usize> = if accepted_indices.is_empty()
- && matches!(state, KVCacheLayerState::Windowed { .. })
- {
- vec![0]
- } else {
- accepted_indices.to_vec()
- };
+ let effective_indices: Vec<usize> =
+ if accepted_indices.is_empty() && matches!(state, KVCacheLayerState::Windowed { .. }) {
+ vec![0]
+ } else {
+ accepted_indices.to_vec()
+ };
match state {
KVCacheLayerState::Full {
prefix_len,
@@ -152,9 +141,7 @@ fn collect_mask(
(0..suffix_length)
.map(|row| {
(0..prefix_segment_length + suffix_length)
- .map(|col| {
- layer.bias_should_be_neg_inf(row, col, suffix_positions)
- })
+ .map(|col| layer.bias_should_be_neg_inf(row, col, suffix_positions))
.collect()
})
.collect()
@@ -164,12 +151,8 @@ fn run_scenario(
context: &MTLContext,
scenario: &Scenario,
) {
- let mut layer = make_test_layer(
- context,
- scenario.state.clone(),
- scenario.prefix_capacity,
- scenario.suffix_capacity,
- );
+ let mut layer =
+ make_test_layer(context, scenario.state.clone(), scenario.prefix_capacity, scenario.suffix_capacity);
layer.prefix_token_positions = scenario.initial_prefix_positions.clone();
@@ -177,16 +160,9 @@ fn run_scenario(
let (initial_keys, initial_values) = fill_arrays(&mut layer);
- let expected_keys = expected_after_update(
- &state_before_update,
- &scenario.accepted_suffix_indices,
- &initial_keys,
- );
- let expected_values = expected_after_update(
- &state_before_update,
- &scenario.accepted_suffix_indices,
- &initial_values,
- );
+ let expected_keys = expected_after_update(&state_before_update, &scenario.accepted_suffix_indices, &initial_keys);
+ let expected_values =
+ expected_after_update(&state_before_update, &scenario.accepted_suffix_indices, &initial_values);
let total_sequence_length = match &layer.state {
KVCacheLayerState::Full {
@@ -199,29 +175,16 @@ fn run_scenario(
};
let mask = collect_mask(&layer, &scenario.suffix_token_positions);
- assert_eq!(
- mask, scenario.expected_mask,
- "{}: bias mask mismatch",
- scenario.name
- );
+ assert_eq!(mask, scenario.expected_mask, "{}: bias mask mismatch", scenario.name);
- let kv_cache_update =
- match KVCacheUpdate::new(context, DataType::F32, total_sequence_length)
- {
- Ok(update) => update,
- Err(e) => {
- panic!(
- "Failed to create KV cache update for scenario {}: {:?}",
- scenario.name, e
- );
- },
- };
+ let kv_cache_update = match KVCacheUpdate::new(context, DataType::F32, total_sequence_length) {
+ Ok(update) => update,
+ Err(e) => {
+ panic!("Failed to create KV cache update for scenario {}: {:?}", scenario.name, e);
+ },
+ };
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
let root_command_buffer = command_buffer.clone();
layer.update_after_acceptance(
@@ -240,21 +203,13 @@ fn run_scenario(
let keys_ref = layer.keys.borrow();
keys_ref.as_slice::<f32>().to_vec()
};
- assert_eq!(
- actual_keys, expected_keys,
- "{}: key buffer mismatch",
- scenario.name
- );
+ assert_eq!(actual_keys, expected_keys, "{}: key buffer mismatch", scenario.name);
let actual_values = {
let values_ref = layer.values.borrow();
values_ref.as_slice::<f32>().to_vec()
};
- assert_eq!(
- actual_values, expected_values,
- "{}: value buffer mismatch",
- scenario.name
- );
+ assert_eq!(actual_values, expected_values, "{}: value buffer mismatch", scenario.name);
assert_eq!(
layer.prefix_token_positions, scenario.expected_prefix_positions,
@@ -267,11 +222,7 @@ fn run_scenario(
prefix_len,
} => {
if let Some(expected_len) = scenario.expected_prefix_len {
- assert_eq!(
- *prefix_len, expected_len,
- "{}: prefix length mismatch",
- scenario.name
- );
+ assert_eq!(*prefix_len, expected_len, "{}: prefix length mismatch", scenario.name);
}
},
KVCacheLayerState::Windowed {
@@ -280,18 +231,10 @@ fn run_scenario(
..
} => {
if let Some(expected_offset) = scenario.expected_ring_offset {
- assert_eq!(
- *ring_offset, expected_offset,
- "{}: ring offset mismatch",
- scenario.name
- );
+ assert_eq!(*ring_offset, expected_offset, "{}: ring offset mismatch", scenario.name);
}
if let Some(expected_length) = scenario.expected_ring_length {
- assert_eq!(
- *ring_length, expected_length,
- "{}: ring length mismatch",
- scenario.name
- );
+ assert_eq!(*ring_length, expected_length, "{}: ring length mismatch", scenario.name);
}
},
}
@@ -327,15 +270,9 @@ fn kv_cache_state_and_mask_scenarios() {
suffix_start: None,
expected_prefix_positions: vec![6, 7, 8, 9, 4, 5],
expected_mask: vec![
- vec![
- false, true, false, false, false, false, false, true, true,
- ],
- vec![
- false, true, true, false, false, false, false, false, true,
- ],
- vec![
- false, true, true, true, false, false, false, false, false,
- ],
+ vec![false, true, false, false, false, false, false, true, true],
+ vec![false, true, true, false, false, false, false, false, true],
+ vec![false, true, true, true, false, false, false, false, false],
],
expected_ring_offset: Some(4),
expected_ring_length: Some(6),
@@ -351,14 +288,7 @@ fn kv_cache_state_and_mask_scenarios() {
},
prefix_capacity: 6,
suffix_capacity: 3,
- initial_prefix_positions: vec![
- 0,
- 1,
- 2,
- 3,
- INVALID_POSITION,
- INVALID_POSITION,
- ],
+ initial_prefix_positions: vec![0, 1, 2, 3, INVALID_POSITION, INVALID_POSITION],
accepted_suffix_indices: vec![0, 1, 2],
accepted_token_positions: vec![4, 5, 6],
suffix_token_positions: vec![4, 5, 6],
@@ -366,12 +296,8 @@ fn kv_cache_state_and_mask_scenarios() {
expected_prefix_positions: vec![6, 1, 2, 3, 4, 5],
expected_mask: vec![
vec![false, false, false, false, true, true, false, true, true],
- vec![
- false, false, false, false, true, true, false, false, true,
- ],
- vec![
- true, false, false, false, true, true, false, false, false,
- ],
+ vec![false, false, false, false, true, true, false, false, true],
+ vec![true, false, false, false, true, true, false, false, false],
],
expected_ring_offset: Some(1),
expected_ring_length: Some(6),
@@ -393,9 +319,7 @@ fn kv_cache_state_and_mask_scenarios() {
suffix_token_positions: vec![16],
suffix_start: None,
expected_prefix_positions: vec![11, 12, 13, 14, 15, 16],
- expected_mask: vec![vec![
- false, false, false, false, false, true, false,
- ]],
+ expected_mask: vec![vec![false, false, false, false, false, true, false]],
expected_ring_offset: Some(0),
expected_ring_length: Some(6),
expected_prefix_len: None,
@@ -473,16 +397,8 @@ fn kv_cache_slice_apply_contiguous_window() {
let keys_after = layer.keys.borrow().as_slice::<f32>().to_vec();
let values_after = layer.values.borrow().as_slice::<f32>().to_vec();
- assert_eq!(
- keys_after[0..4],
- initial_keys[0..4],
- "keys restored for contiguous slice"
- );
- assert_eq!(
- values_after[0..4],
- initial_values[0..4],
- "values restored for contiguous slice"
- );
+ assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for contiguous slice");
+ assert_eq!(values_after[0..4], initial_values[0..4], "values restored for contiguous slice");
}
#[test]
@@ -519,24 +435,12 @@ fn kv_cache_slice_apply_wrap_window() {
layer.apply_slice(&slice, None);
- assert_eq!(
- layer.prefix_token_positions,
- vec![30, 31, 32, 33],
- "positions restored for wrapped slice"
- );
+ assert_eq!(layer.prefix_token_positions, vec![30, 31, 32, 33], "positions restored for wrapped slice");
let keys_after = layer.keys.borrow().as_slice::<f32>().to_vec();
let values_after = layer.values.borrow().as_slice::<f32>().to_vec();
- assert_eq!(
- keys_after[0..4],
- initial_keys[0..4],
- "keys restored for wrapped slice"
- );
- assert_eq!(
- values_after[0..4],
- initial_values[0..4],
- "values restored for wrapped slice"
- );
+ assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for wrapped slice");
+ assert_eq!(values_after[0..4], initial_values[0..4], "values restored for wrapped slice");
}
#[test]
@@ -568,11 +472,7 @@ fn kv_cache_slice_apply_full_restores_metadata() {
layer.apply_slice(&slice, None);
- assert_eq!(
- layer.prefix_token_positions,
- vec![1, 2, 3],
- "full slice restores positions"
- );
+ assert_eq!(layer.prefix_token_positions, vec![1, 2, 3], "full slice restores positions");
if let KVCacheLayerState::Full {
prefix_len,
} = &layer.state
diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rs
index 1af8f358c..82f22c04b 100644
--- a/crates/uzu/tests/kv_cache_update_test.rs
+++ b/crates/uzu/tests/kv_cache_update_test.rs
@@ -1,9 +1,6 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
use bytemuck;
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt,
- MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use ndarray::{Array, Array3, s};
use uzu::backends::{
common::{
@@ -25,8 +22,7 @@ fn apply_swaps_3d<T: Clone>(
let src = swap.source as usize;
let dst = swap.destination as usize;
let temp = array[(head, src, channel)].clone();
- array[(head, src, channel)] =
- array[(head, dst, channel)].clone();
+ array[(head, src, channel)] = array[(head, dst, channel)].clone();
array[(head, dst, channel)] = temp;
}
}
@@ -50,17 +46,10 @@ fn test_random_pattern(context: &MTLContext) {
println!("Testing with random pattern...");
let max_sequence_length = 256usize;
- let kv_cache_update = match KVCacheUpdate::new(
- context,
- uzu::DataType::F32,
- max_sequence_length,
- ) {
+ let kv_cache_update = match KVCacheUpdate::new(context, uzu::DataType::F32, max_sequence_length) {
Ok(update) => update,
Err(e) => {
- println!(
- "Warning: Failed to create KV cache update: {:?}. Skipping test.",
- e
- );
+ println!("Warning: Failed to create KV cache update: {:?}. Skipping test.", e);
return;
},
};
@@ -69,15 +58,13 @@ fn test_random_pattern(context: &MTLContext) {
let seq_len = 15usize;
let head_dim = 7usize;
- let key_data = Array3::<f32>::from_shape_fn(
- (num_heads, seq_len, head_dim),
- |(h, t, c)| (h * 1_000_000 + t * 100 + c * 10) as f32,
- );
+ let key_data = Array3::<f32>::from_shape_fn((num_heads, seq_len, head_dim), |(h, t, c)| {
+ (h * 1_000_000 + t * 100 + c * 10) as f32
+ });
- let value_data = Array3::<f32>::from_shape_fn(
- (num_heads, seq_len, head_dim),
- |(h, t, c)| (h * 1_000_000 + t * 100 + c * 10 + 1_000) as f32,
- );
+ let value_data = Array3::<f32>::from_shape_fn((num_heads, seq_len, head_dim), |(h, t, c)| {
+ (h * 1_000_000 + t * 100 + c * 10 + 1_000) as f32
+ });
let source_indices = vec![0, 3, 6, 9, 12, 2, 5, 8, 11, 14];
let destination_indices = vec![14, 11, 8, 5, 2, 12, 9, 6, 3, 0];
@@ -112,23 +99,11 @@ fn test_random_pattern(context: &MTLContext) {
value_shape: [num_heads, seq_len, head_dim],
};
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- match kv_cache_update.encode(
- &[kv_layer_data],
- &source_indices,
- &destination_indices,
- &command_buffer,
- ) {
+ let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &command_buffer) {
Ok(_) => {},
Err(e) => {
- println!(
- "Warning: Failed to encode KV cache update: {:?}. Skipping test.",
- e
- );
+ println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);
return;
},
}
@@ -141,22 +116,14 @@ fn test_random_pattern(context: &MTLContext) {
let total_elems = num_heads * seq_len * head_dim;
- let key_result_slice =
- unsafe { std::slice::from_raw_parts(key_result_ptr, total_elems) };
- let value_result_slice =
- unsafe { std::slice::from_raw_parts(value_result_ptr, total_elems) };
-
- let key_result = Array::from_shape_vec(
- (num_heads, seq_len, head_dim),
- key_result_slice.to_vec(),
- )
- .expect("Failed to convert key result to ndarray");
-
- let value_result = Array::from_shape_vec(
- (num_heads, seq_len, head_dim),
- value_result_slice.to_vec(),
- )
- .expect("Failed to convert value result to ndarray");
+ let key_result_slice = unsafe { std::slice::from_raw_parts(key_result_ptr, total_elems) };
+ let value_result_slice = unsafe { std::slice::from_raw_parts(value_result_ptr, total_elems) };
+
+ let key_result = Array::from_shape_vec((num_heads, seq_len, head_dim), key_result_slice.to_vec())
+ .expect("Failed to convert key result to ndarray");
+
+ let value_result = Array::from_shape_vec((num_heads, seq_len, head_dim), value_result_slice.to_vec())
+ .expect("Failed to convert value result to ndarray");
println!("Original keys head 0 rows 0,14:");
println!("Row 0: {:?}", key_data.slice(s![0, 0, ..]));
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 25247df29..c5731e335 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -2,10 +2,7 @@
use bytemuck;
use half::bf16;
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,
- MTLDeviceExt, MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use ndarray::Array2;
use uzu::{
DataType,
@@ -29,24 +26,15 @@ fn run_metal_matmul(
) -> Vec<bf16> {
let a_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(a_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(a_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let b_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(b_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(b_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let d_buf = ctx
.device
- .new_buffer(
- m * n * core::mem::size_of::<bf16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(m * n * core::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let ldb = if transpose_b {
@@ -57,14 +45,8 @@ fn run_metal_matmul(
let mut kernel = MatmulKernel::new(DataType::BF16).expect("kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let enc = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
let encode_result = kernel.encode(
ctx,
&enc,
@@ -129,17 +111,13 @@ fn generate_test_data(
n: usize,
transpose_b: bool,
) -> (Vec<bf16>, Vec<bf16>) {
- let a: Vec<bf16> = (0..(m * k))
- .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01 - 0.06))
- .collect();
+ let a: Vec<bf16> = (0..(m * k)).map(|i| bf16::from_f32(((i % 13) as f32) * 0.01 - 0.06)).collect();
let b_size = if transpose_b {
n * k
} else {
k * n
};
- let b: Vec<bf16> = (0..b_size)
- .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.15))
- .collect();
+ let b: Vec<bf16> = (0..b_size).map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.15)).collect();
(a, b)
}
@@ -154,8 +132,7 @@ struct TestCase {
fn test_cases() -> Vec<TestCase> {
let base_tolerance = 0.01;
- let batch_sizes: [usize; 12] =
- [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];
+ let batch_sizes: [usize; 12] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048];
let model_shapes: [(usize, usize); 51] = [
// Qwen2.5-Coder-0.5B
@@ -221,10 +198,8 @@ fn test_cases() -> Vec<TestCase> {
(17408, 5120),
];
- let mut cases: Vec<(usize, usize, usize, bool)> = model_shapes
- .iter()
- .flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n, true)))
- .collect();
+ let mut cases: Vec<(usize, usize, usize, bool)> =
+ model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n, true))).collect();
// Non-transposed B (for attention AV matmul)
for &(k, n) in &[(1024, 1024), (2048, 2048), (4096, 4096)] {
@@ -267,8 +242,7 @@ fn compare_results(
let mut max_diff_idx = 0;
let mut diff_count = 0;
- for (i, (&m_val, &r_val)) in metal.iter().zip(reference.iter()).enumerate()
- {
+ for (i, (&m_val, &r_val)) in metal.iter().zip(reference.iter()).enumerate() {
let mf = m_val.to_f32();
let rf = r_val.to_f32();
let diff = (mf - rf).abs();
@@ -301,25 +275,9 @@ fn matmul_correctness_comprehensive() {
let mut failed = Vec::new();
for case in &cases {
- let (a, b) =
- generate_test_data(case.m, case.k, case.n, case.transpose_b);
- let metal_result = run_metal_matmul(
- &ctx,
- &a,
- &b,
- case.m,
- case.k,
- case.n,
- case.transpose_b,
- );
- let reference = run_ndarray_matmul(
- &a,
- &b,
- case.m,
- case.k,
- case.n,
- case.transpose_b,
- );
+ let (a, b) = generate_test_data(case.m, case.k, case.n, case.transpose_b);
+ let metal_result = run_metal_matmul(&ctx, &a, &b, case.m, case.k, case.n, case.transpose_b);
+ let reference = run_ndarray_matmul(&a, &b, case.m, case.k, case.n, case.transpose_b);
let trans_str = if case.transpose_b {
"T"
@@ -330,23 +288,13 @@ fn matmul_correctness_comprehensive() {
match compare_results(&metal_result, &reference, case.tolerance) {
Ok(()) => {
passed += 1;
- eprintln!(
- "βœ“ m={} k={} n={} B={}",
- case.m, case.k, case.n, trans_str
- );
+ eprintln!("βœ“ m={} k={} n={} B={}", case.m, case.k, case.n, trans_str);
},
Err((max_diff, idx, count)) => {
failed.push((case, max_diff, idx, count));
eprintln!(
"βœ— m={} k={} n={} B={} max_diff={:.6} at idx {} ({} exceed tol {})",
- case.m,
- case.k,
- case.n,
- trans_str,
- max_diff,
- idx,
- count,
- case.tolerance
+ case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance
);
},
}
@@ -364,14 +312,7 @@ fn matmul_correctness_comprehensive() {
};
eprintln!(
" m={} k={} n={} B={}: max_diff={:.6} at idx {}, {} exceed tol {}",
- case.m,
- case.k,
- case.n,
- trans_str,
- max_diff,
- idx,
- count,
- case.tolerance
+ case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance
);
}
panic!("{} tests failed", failed.len());
diff --git a/crates/uzu/tests/mlp_fused_correctness_test.rs b/crates/uzu/tests/mlp_fused_correctness_test.rs
index 88575a151..7d407a348 100644
--- a/crates/uzu/tests/mlp_fused_correctness_test.rs
+++ b/crates/uzu/tests/mlp_fused_correctness_test.rs
@@ -3,10 +3,7 @@
use bytemuck;
use half::f16;
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,
- MTLDeviceExt, MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use objc2::rc::autoreleasepool;
use uzu::{
DataType,
@@ -81,36 +78,21 @@ fn run_fused_gemv(
) -> Vec<f16> {
let input_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(input),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(input), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let weights_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(weights),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(weights), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buf = ctx
.device
- .new_buffer(
- hidden_dim * core::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(hidden_dim * core::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let mut kernel = MlpFusedKernel::new(DataType::F16, true).expect("kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let enc = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
let args = MlpFusedArguments {
input: &input_buf,
@@ -150,36 +132,21 @@ fn run_fused_gemm(
) -> Vec<f16> {
let input_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(input),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(input), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let weights_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(weights),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(weights), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buf = ctx
.device
- .new_buffer(
- m * hidden_dim * core::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(m * hidden_dim * core::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let mut kernel = MlpFusedKernel::new(DataType::F16, true).expect("kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let enc = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
let args = MlpFusedArguments {
input: &input_buf,
@@ -252,47 +219,25 @@ fn test_mlp_fused_gemv_silu_small() {
let m = 1;
// Generate random-ish test data
- let input: Vec<f32> =
- (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
- let weights_up: Vec<f32> = (0..hidden_dim * k)
- .map(|i| (i as f32 * 0.017).cos() * 0.3)
- .collect();
- let weights_gate: Vec<f32> = (0..hidden_dim * k)
- .map(|i| (i as f32 * 0.023).sin() * 0.3)
- .collect();
+ let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
+ let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();
+ let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();
// Create concatenated weights [up; gate] row-major
- let mut weights_concat: Vec<f32> =
- Vec::with_capacity(2 * hidden_dim * k);
+ let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);
weights_concat.extend(&weights_up);
weights_concat.extend(&weights_gate);
// CPU reference
- let expected = mlp_fused_reference(
- &input,
- &weights_up,
- &weights_gate,
- m,
- k,
- hidden_dim,
- MlpActivationType::SiLU,
- );
+ let expected =
+ mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);
// Convert to f16
- let input_f16: Vec<f16> =
- input.iter().map(|&x| f16::from_f32(x)).collect();
- let weights_f16: Vec<f16> =
- weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
+ let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();
+ let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
// Run fused kernel
- let actual = run_fused_gemv(
- &ctx,
- &input_f16,
- &weights_f16,
- k,
- hidden_dim,
- MlpActivationType::SiLU,
- );
+ let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::SiLU);
let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);
println!("GEMV SiLU small: max_diff = {}", max_diff);
@@ -315,43 +260,21 @@ fn test_mlp_fused_gemv_gelu_small() {
let hidden_dim = 32;
let m = 1;
- let input: Vec<f32> =
- (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
- let weights_up: Vec<f32> = (0..hidden_dim * k)
- .map(|i| (i as f32 * 0.017).cos() * 0.3)
- .collect();
- let weights_gate: Vec<f32> = (0..hidden_dim * k)
- .map(|i| (i as f32 * 0.023).sin() * 0.3)
- .collect();
-
- let mut weights_concat: Vec<f32> =
- Vec::with_capacity(2 * hidden_dim * k);
+ let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
+ let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();
+ let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();
+
+ let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);
weights_concat.extend(&weights_up);
weights_concat.extend(&weights_gate);
- let expected = mlp_fused_reference(
- &input,
- &weights_up,
- &weights_gate,
- m,
- k,
- hidden_dim,
- MlpActivationType::Gelu,
- );
-
- let input_f16: Vec<f16> =
- input.iter().map(|&x| f16::from_f32(x)).collect();
- let weights_f16: Vec<f16> =
- weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
-
- let actual = run_fused_gemv(
- &ctx,
- &input_f16,
- &weights_f16,
- k,
- hidden_dim,
- MlpActivationType::Gelu,
- );
+ let expected =
+ mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::Gelu);
+
+ let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();
+ let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
+
+ let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::Gelu);
let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);
println!("GEMV GELU small: max_diff = {}", max_diff);
@@ -374,44 +297,21 @@ fn test_mlp_fused_gemm_silu_medium() {
let k = 64;
let hidden_dim = 64;
- let input: Vec<f32> =
- (0..m * k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
- let weights_up: Vec<f32> = (0..hidden_dim * k)
- .map(|i| (i as f32 * 0.017).cos() * 0.3)
- .collect();
- let weights_gate: Vec<f32> = (0..hidden_dim * k)
- .map(|i| (i as f32 * 0.023).sin() * 0.3)
- .collect();
-
- let mut weights_concat: Vec<f32> =
- Vec::with_capacity(2 * hidden_dim * k);
+ let input: Vec<f32> = (0..m * k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
+ let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();
+ let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();
+
+ let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);
weights_concat.extend(&weights_up);
weights_concat.extend(&weights_gate);
- let expected = mlp_fused_reference(
- &input,
- &weights_up,
- &weights_gate,
- m,
- k,
- hidden_dim,
- MlpActivationType::SiLU,
- );
-
- let input_f16: Vec<f16> =
- input.iter().map(|&x| f16::from_f32(x)).collect();
- let weights_f16: Vec<f16> =
- weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
-
- let actual = run_fused_gemm(
- &ctx,
- &input_f16,
- &weights_f16,
- m,
- k,
- hidden_dim,
- MlpActivationType::SiLU,
- );
+ let expected =
+ mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);
+
+ let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();
+ let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
+
+ let actual = run_fused_gemm(&ctx, &input_f16, &weights_f16, m, k, hidden_dim, MlpActivationType::SiLU);
let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.05);
println!("GEMM SiLU medium: max_diff = {}", max_diff);
@@ -435,43 +335,21 @@ fn test_mlp_fused_gemv_realistic_dimensions() {
let hidden_dim = 256; // hidden dim (half of up+gate)
let m = 1;
- let input: Vec<f32> =
- (0..k).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();
- let weights_up: Vec<f32> = (0..hidden_dim * k)
- .map(|i| (i as f32 * 0.001).cos() * 0.1)
- .collect();
- let weights_gate: Vec<f32> = (0..hidden_dim * k)
- .map(|i| (i as f32 * 0.002).sin() * 0.1)
- .collect();
-
- let mut weights_concat: Vec<f32> =
- Vec::with_capacity(2 * hidden_dim * k);
+ let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();
+ let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.001).cos() * 0.1).collect();
+ let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.002).sin() * 0.1).collect();
+
+ let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);
weights_concat.extend(&weights_up);
weights_concat.extend(&weights_gate);
- let expected = mlp_fused_reference(
- &input,
- &weights_up,
- &weights_gate,
- m,
- k,
- hidden_dim,
- MlpActivationType::SiLU,
- );
-
- let input_f16: Vec<f16> =
- input.iter().map(|&x| f16::from_f32(x)).collect();
- let weights_f16: Vec<f16> =
- weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
-
- let actual = run_fused_gemv(
- &ctx,
- &input_f16,
- &weights_f16,
- k,
- hidden_dim,
- MlpActivationType::SiLU,
- );
+ let expected =
+ mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);
+
+ let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();
+ let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
+
+ let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::SiLU);
let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.02);
println!("GEMV realistic: max_diff = {}", max_diff);
diff --git a/crates/uzu/tests/mlp_fused_performance_test.rs b/crates/uzu/tests/mlp_fused_performance_test.rs
index fc3542408..2333f1747 100644
--- a/crates/uzu/tests/mlp_fused_performance_test.rs
+++ b/crates/uzu/tests/mlp_fused_performance_test.rs
@@ -4,10 +4,7 @@
use std::time::{Duration, Instant};
use half::f16;
-use metal::{
- MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt,
- MTLResourceOptions,
-};
+use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use objc2::rc::autoreleasepool;
use uzu::{
DataType,
@@ -16,14 +13,10 @@ use uzu::{
metal::{
MTLContext,
kernel::{
- matmul::{
- MatmulArguments, MatmulKernel,
- determine_kernel_variant as determine_matmul_variant,
- },
+ matmul::{MatmulArguments, MatmulKernel, determine_kernel_variant as determine_matmul_variant},
mlp::MlpActivationType,
mlp_fused::{
- MlpFusedArguments, MlpFusedKernel,
- determine_kernel_variant as determine_mlp_fused_variant,
+ MlpFusedArguments, MlpFusedKernel, determine_kernel_variant as determine_mlp_fused_variant,
},
},
},
@@ -66,10 +59,7 @@ fn benchmark_gemv_fused(
) -> BenchmarkResult {
let input_buffer = context
.device
- .new_buffer(
- input_dimension * std::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(input_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let weights_buffer = context
.device
@@ -80,21 +70,14 @@ fn benchmark_gemv_fused(
.expect("Failed to create buffer");
let output_buffer = context
.device
- .new_buffer(
- hidden_dimension * std::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let fused_up_buffer = context
.device
- .new_buffer(
- 2 * hidden_dimension * std::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(2 * hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut fused_kernel =
- MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");
+ let mut fused_kernel = MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");
let mut matmul_kernel = MatmulKernel::new(DataType::F16).expect("matmul");
let fused_args = MlpFusedArguments {
@@ -114,14 +97,9 @@ fn benchmark_gemv_fused(
// Warmup fused
for _ in 0..warmup_count {
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
fused_kernel.encode(context, &encoder, &fused_args).unwrap();
encoder.end_encoding();
command_buffer.commit();
@@ -131,14 +109,9 @@ fn benchmark_gemv_fused(
// Benchmark fused
let fused_start = Instant::now();
for _ in 0..iteration_count {
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
fused_kernel.encode(context, &encoder, &fused_args).unwrap();
encoder.end_encoding();
command_buffer.commit();
@@ -167,25 +140,16 @@ fn benchmark_gemv_fused(
};
let fused_kernel_name =
- determine_mlp_fused_variant(context, DataType::F16, true, &fused_args)
- .map(|v| v.as_str())
- .unwrap_or("?");
+ determine_mlp_fused_variant(context, DataType::F16, true, &fused_args).map(|v| v.as_str()).unwrap_or("?");
let unfused_kernel_name =
- determine_matmul_variant(context, DataType::F16, &matmul_args)
- .map(|v| v.as_str())
- .unwrap_or("?");
+ determine_matmul_variant(context, DataType::F16, &matmul_args).map(|v| v.as_str()).unwrap_or("?");
// Warmup unfused
for _ in 0..warmup_count {
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();
encoder.end_encoding();
command_buffer.commit();
@@ -195,14 +159,9 @@ fn benchmark_gemv_fused(
// Benchmark unfused
let unfused_start = Instant::now();
for _ in 0..iteration_count {
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();
encoder.end_encoding();
command_buffer.commit();
@@ -210,12 +169,7 @@ fn benchmark_gemv_fused(
}
let unfused_duration = unfused_start.elapsed() / iteration_count as u32;
- BenchmarkResult::new(
- fused_duration,
- unfused_duration,
- fused_kernel_name,
- unfused_kernel_name,
- )
+ BenchmarkResult::new(fused_duration, unfused_duration, fused_kernel_name, unfused_kernel_name)
}
fn benchmark_gemm_fused(
@@ -228,10 +182,7 @@ fn benchmark_gemm_fused(
) -> BenchmarkResult {
let input_buffer = context
.device
- .new_buffer(
- batch_size * input_dimension * std::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(batch_size * input_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let weights_buffer = context
.device
@@ -242,10 +193,7 @@ fn benchmark_gemm_fused(
.expect("Failed to create buffer");
let output_buffer = context
.device
- .new_buffer(
- batch_size * hidden_dimension * std::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(batch_size * hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let fused_up_buffer = context
.device
@@ -255,8 +203,7 @@ fn benchmark_gemm_fused(
)
.expect("Failed to create buffer");
- let mut fused_kernel =
- MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");
+ let mut fused_kernel = MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");
let mut matmul_kernel = MatmulKernel::new(DataType::F16).expect("matmul");
let fused_args = MlpFusedArguments {
@@ -276,14 +223,9 @@ fn benchmark_gemm_fused(
// Warmup fused
for _ in 0..warmup_count {
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
fused_kernel.encode(context, &encoder, &fused_args).unwrap();
encoder.end_encoding();
command_buffer.commit();
@@ -292,14 +234,9 @@ fn benchmark_gemm_fused(
let fused_start = Instant::now();
for _ in 0..iteration_count {
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
fused_kernel.encode(context, &encoder, &fused_args).unwrap();
encoder.end_encoding();
command_buffer.commit();
@@ -328,25 +265,16 @@ fn benchmark_gemm_fused(
};
let fused_kernel_name =
- determine_mlp_fused_variant(context, DataType::F16, true, &fused_args)
- .map(|v| v.as_str())
- .unwrap_or("?");
+ determine_mlp_fused_variant(context, DataType::F16, true, &fused_args).map(|v| v.as_str()).unwrap_or("?");
let unfused_kernel_name =
- determine_matmul_variant(context, DataType::F16, &matmul_args)
- .map(|v| v.as_str())
- .unwrap_or("?");
+ determine_matmul_variant(context, DataType::F16, &matmul_args).map(|v| v.as_str()).unwrap_or("?");
// Warmup unfused
for _ in 0..warmup_count {
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();
encoder.end_encoding();
command_buffer.commit();
@@ -355,14 +283,9 @@ fn benchmark_gemm_fused(
let unfused_start = Instant::now();
for _ in 0..iteration_count {
- let command_buffer = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer")
- .to_owned();
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();
encoder.end_encoding();
command_buffer.commit();
@@ -370,12 +293,7 @@ fn benchmark_gemm_fused(
}
let unfused_duration = unfused_start.elapsed() / iteration_count as u32;
- BenchmarkResult::new(
- fused_duration,
- unfused_duration,
- fused_kernel_name,
- unfused_kernel_name,
- )
+ BenchmarkResult::new(fused_duration, unfused_duration, fused_kernel_name, unfused_kernel_name)
}
/// MLP shapes from real model configs
@@ -425,27 +343,14 @@ fn mlp_fused_performance_benchmark() {
println!(
"{:>10} {:>10} {:>10} {:>10} {:>12} {:>10} {:>12} {:>10}",
- "batch",
- "input",
- "hidden",
- "Fused",
- "Fused (Β΅s)",
- "Unfused",
- "Unfused (Β΅s)",
- "Speedup"
+ "batch", "input", "hidden", "Fused", "Fused (Β΅s)", "Unfused", "Unfused (Β΅s)", "Speedup"
);
println!("{}", "-".repeat(98));
for &batch_size in BATCH_SIZES {
for &(input_dimension, hidden_dimension) in MLP_SHAPES {
let result = if batch_size == 1 {
- benchmark_gemv_fused(
- &context,
- input_dimension,
- hidden_dimension,
- iteration_count,
- warmup_count,
- )
+ benchmark_gemv_fused(&context, input_dimension, hidden_dimension, iteration_count, warmup_count)
} else {
benchmark_gemm_fused(
&context,
diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
index 4e8a96a6a..7a9b9ef00 100644
--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs
+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
@@ -56,18 +56,13 @@ fn gen_topk_ids_from_logits(
// Generate random input and router weights
let d_model = 64; // arbitrary model dimension for test
- let input_f32: Vec<f32> =
- (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
- let weight_f32: Vec<f32> =
- (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
- let bias_f32: Vec<f32> =
- (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();
+ let input_f32: Vec<f32> = (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
+ let weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
+ let bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();
// Convert to bf16
- let input: Vec<bf16> =
- input_f32.iter().map(|&x| bf16::from_f32(x)).collect();
- let weight: Vec<bf16> =
- weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();
+ let input: Vec<bf16> = input_f32.iter().map(|&x| bf16::from_f32(x)).collect();
+ let weight: Vec<bf16> = weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();
let bias: Vec<bf16> = bias_f32.iter().map(|&x| bf16::from_f32(x)).collect();
let input_buf = alloc_buffer_with_data(ctx, &input);
@@ -77,12 +72,8 @@ fn gen_topk_ids_from_logits(
let topk_probs_buf = alloc_buffer::<bf16>(ctx, t * k);
// Use fused router+topk kernel
- let router_topk =
- MoeRouterTopKKernel::new(ctx).expect("router_topk kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let router_topk = MoeRouterTopKKernel::new(ctx).expect("router_topk kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let args = MoeRouterTopKArguments {
input_buffer: &input_buf,
weight_buffer: &weight_buf,
@@ -95,9 +86,7 @@ fn gen_topk_ids_from_logits(
k,
renorm: true,
};
- router_topk
- .encode(&cb, KernelDataType::BFloat16, args)
- .expect("encode router_topk");
+ router_topk.encode(&cb, KernelDataType::BFloat16, args).expect("encode router_topk");
cb.commit();
cb.wait_until_completed();
@@ -117,8 +106,7 @@ fn test_counts_offsets_fused_parity_random() {
if k > e {
continue;
}
- let (topk_ids, topk_ids_buf) =
- gen_topk_ids_from_logits(&ctx, t, e, k);
+ let (topk_ids, topk_ids_buf) = gen_topk_ids_from_logits(&ctx, t, e, k);
// CPU reference
let counts_cpu = cpu_bucket_counts(&topk_ids, t, k, e);
@@ -130,12 +118,8 @@ fn test_counts_offsets_fused_parity_random() {
let num_tiles = ((e + 511) / 512).max(1);
let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);
- let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx)
- .expect("fused kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
kernel.encode(
@@ -155,25 +139,13 @@ fn test_counts_offsets_fused_parity_random() {
// Verify offsets
let offsets_ptr = offsets_buf.contents().as_ptr() as *const u32;
- let offsets_gpu =
- unsafe { std::slice::from_raw_parts(offsets_ptr, e + 1) };
- assert_eq!(
- offsets_gpu,
- &offsets_cpu[..],
- "offsets mismatch T={} E={} K={}",
- t,
- e,
- k
- );
+ let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_ptr, e + 1) };
+ assert_eq!(offsets_gpu, &offsets_cpu[..], "offsets mismatch T={} E={} K={}", t, e, k);
// Verify sum
let sum_ptr = sum_k_buf.contents().as_ptr() as *const u32;
let sum_gpu = unsafe { *sum_ptr };
- assert_eq!(
- sum_gpu, sum_cpu,
- "sum mismatch T={} E={} K={}",
- t, e, k
- );
+ assert_eq!(sum_gpu, sum_cpu, "sum mismatch T={} E={} K={}", t, e, k);
}
}
}
@@ -197,33 +169,15 @@ fn test_counts_offsets_fused_edge_cases() {
let num_tiles = ((e + 511) / 512).max(1);
let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);
- let kernel =
- MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
- kernel.encode(
- &topk_ids_buf,
- &offsets_buf,
- &sum_k_buf,
- &partials_buf,
- t as u32,
- e as u32,
- k as u32,
- &encoder,
- );
+ kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
encoder.end_encoding();
cb.commit();
cb.wait_until_completed();
- let offsets_gpu = unsafe {
- std::slice::from_raw_parts(
- offsets_buf.contents().as_ptr() as *const u32,
- e + 1,
- )
- };
+ let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };
let mut expected_offsets = vec![0u32; e + 1];
expected_offsets[4..].fill((t * k) as u32);
@@ -238,32 +192,14 @@ fn test_counts_offsets_fused_edge_cases() {
let num_tiles = ((e + 511) / 512).max(1);
let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);
- let kernel =
- MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
- kernel.encode(
- &topk_ids_buf,
- &offsets_buf,
- &sum_k_buf,
- &partials_buf,
- t as u32,
- e as u32,
- k as u32,
- &encoder,
- );
+ kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
encoder.end_encoding();
cb.commit();
cb.wait_until_completed();
- let offsets_gpu = unsafe {
- std::slice::from_raw_parts(
- offsets_buf.contents().as_ptr() as *const u32,
- e + 1,
- )
- };
+ let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };
assert!(offsets_gpu.iter().all(|&v| v == 0));
}
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index 46937eb8c..cc5df1800 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -6,13 +6,11 @@ use uzu::backends::{
metal::{
MTLContext,
kernel::{
- KernelDataType, MoeBlockBasesArguments, MoeScatterKernels,
- MoeScatterWithMapArguments,
+ KernelDataType, MoeBlockBasesArguments, MoeScatterKernels, MoeScatterWithMapArguments,
dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel,
- MoeGatherArguments, MoeGatherKernels, MoeRouterTopKArguments,
- MoeRouterTopKKernel,
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,
+ MoeRouterTopKArguments, MoeRouterTopKKernel,
},
},
},
@@ -46,10 +44,10 @@ fn moe_cpu_reference(
x: &[bf16],
router_weight: &[f32], // [E, d_model] - kept as F32 for router (computed before BF16 conversion)
router_bias: &[f32], // [E]
- w13: &[bf16], // source layout [E, d_model, 2*d_ff] (GPU transposes to [E, 2*d_ff, d_model])
- w2: &[bf16], // [E, d_ff, d_model]
- up_biases: &[bf16], // [E, 2*d_ff]
- down_biases: &[bf16], // [E, d_model]
+ w13: &[bf16], // source layout [E, d_model, 2*d_ff] (GPU transposes to [E, 2*d_ff, d_model])
+ w2: &[bf16], // [E, d_ff, d_model]
+ up_biases: &[bf16], // [E, 2*d_ff]
+ down_biases: &[bf16], // [E, d_model]
t: usize,
e: usize,
k: usize,
@@ -78,23 +76,15 @@ fn moe_cpu_reference(
}
// TopK selection
- let mut indices_and_logits: Vec<(usize, f32)> =
- token_logits.iter().enumerate().map(|(i, &v)| (i, v)).collect();
+ let mut indices_and_logits: Vec<(usize, f32)> = token_logits.iter().enumerate().map(|(i, &v)| (i, v)).collect();
indices_and_logits.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
- let top_k_indices: Vec<usize> =
- indices_and_logits[..k].iter().map(|&(i, _)| i).collect();
- let top_k_logits: Vec<f32> =
- indices_and_logits[..k].iter().map(|&(_, v)| v).collect();
+ let top_k_indices: Vec<usize> = indices_and_logits[..k].iter().map(|&(i, _)| i).collect();
+ let top_k_logits: Vec<f32> = indices_and_logits[..k].iter().map(|&(_, v)| v).collect();
// Softmax normalization (matching Python: jax.nn.softmax)
- let max_logit =
- top_k_logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
- let exp_sum: f32 =
- top_k_logits.iter().map(|&v| (v - max_logit).exp()).sum();
- let weights: Vec<f32> = top_k_logits
- .iter()
- .map(|&v| (v - max_logit).exp() / exp_sum)
- .collect();
+ let max_logit = top_k_logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
+ let exp_sum: f32 = top_k_logits.iter().map(|&v| (v - max_logit).exp()).sum();
+ let weights: Vec<f32> = top_k_logits.iter().map(|&v| (v - max_logit).exp() / exp_sum).collect();
// Process each selected expert (matching Python: vmap(run_expert))
for (expert_idx, &weight) in top_k_indices.iter().zip(weights.iter()) {
@@ -113,8 +103,7 @@ fn moe_cpu_reference(
for ff_idx in 0..d_ff {
let mut up_sum = f32::from(up_biases[bias_up_offset + ff_idx]);
- let mut gate_sum =
- f32::from(up_biases[bias_up_offset + d_ff + ff_idx]);
+ let mut gate_sum = f32::from(up_biases[bias_up_offset + d_ff + ff_idx]);
for input_idx in 0..d_model {
let w_base = w13_offset + input_idx * 2 * d_ff;
@@ -154,15 +143,13 @@ fn moe_cpu_reference(
for out_idx in 0..d_model {
let mut fc2_sum = 0.0f32;
for ff_idx in 0..d_ff {
- let w2_val =
- f32::from(w2[w2_offset + ff_idx * d_model + out_idx]);
+ let w2_val = f32::from(w2[w2_offset + ff_idx * d_model + out_idx]);
fc2_sum += hidden[ff_idx] * w2_val;
}
// Two-pass: keep in F32 (no intermediate quantization)
// Add down bias in F32
- let down_bias_val =
- f32::from(down_biases[bias_down_offset + out_idx]);
+ let down_bias_val = f32::from(down_biases[bias_down_offset + out_idx]);
let with_bias = fc2_sum + down_bias_val;
// Only quantize to BF16 once at the end (matches two-pass final write)
@@ -266,43 +253,26 @@ fn run_moe_parity_test_internal(
eprintln!(
"\n[{}] T={}, E={}, K={}, d_model={}, d_ff={}, alpha={}, gate_clip={:?}, up_clip={:?}, mode={}",
- test_name,
- t,
- e,
- k,
- d_model,
- d_ff,
- silu_alpha,
- gate_clip,
- up_clip,
- prefill_mode
+ test_name, t, e, k, d_model, d_ff, silu_alpha, gate_clip, up_clip, prefill_mode
);
// Random BF16 inputs
- let x: Vec<bf16> = (0..t * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
// Generate random router weights and biases for CPU reference
- let router_weight_f32: Vec<f32> =
- (0..e * d_model).map(|_| rng.random_range(-0.5..0.5)).collect();
- let router_bias_f32: Vec<f32> =
- (0..e).map(|_| rng.random_range(-0.1..0.1)).collect();
+ let router_weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-0.5..0.5)).collect();
+ let router_bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.1..0.1)).collect();
// Convert router weights/bias to BF16 for GPU
- let router_weight_bf16: Vec<bf16> =
- router_weight_f32.iter().map(|&v| bf16::from_f32(v)).collect();
- let router_bias_bf16: Vec<bf16> =
- router_bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+ let router_weight_bf16: Vec<bf16> = router_weight_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+ let router_bias_bf16: Vec<bf16> = router_bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
// Generate random expert weights and biases
let w13_len = e * d_model * 2 * d_ff;
let w2_len = e * d_ff * d_model;
// Generate W13 in original layout [E, d_model, 2*d_ff] for CPU reference
- let w13_cpu: Vec<bf16> = (0..w13_len)
- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))
- .collect();
+ let w13_cpu: Vec<bf16> = (0..w13_len).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
// Transpose to GPU layout [E, 2*d_ff, d_model]
let mut w13_gpu = vec![bf16::from_f32(0.0); w13_len];
@@ -319,9 +289,7 @@ fn run_moe_parity_test_internal(
}
// Generate W2 in original layout [E, d_ff, d_model] for CPU reference
- let w2_cpu: Vec<bf16> = (0..w2_len)
- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))
- .collect();
+ let w2_cpu: Vec<bf16> = (0..w2_len).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
// Transpose W2 to GPU layout [E, d_model, d_ff]
let mut w2_gpu = vec![bf16::from_f32(0.0); w2_len];
@@ -331,17 +299,12 @@ fn run_moe_parity_test_internal(
for dm in 0..d_model {
// src: [E, d_ff, d_model] -> index: expert_offset + ff * d_model + dm
// dst: [E, d_model, d_ff] -> index: expert_offset + dm * d_ff + ff
- w2_gpu[expert_offset + dm * d_ff + ff] =
- w2_cpu[expert_offset + ff * d_model + dm];
+ w2_gpu[expert_offset + dm * d_ff + ff] = w2_cpu[expert_offset + ff * d_model + dm];
}
}
}
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))
- .collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
// Create Metal buffers
let x_buf = alloc_buffer_with_data(&ctx, &x);
@@ -381,10 +344,7 @@ fn run_moe_parity_test_internal(
// Encode ALL kernels in one command buffer
eprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
// Router + TopK (fused kernel)
let router_topk = MoeRouterTopKKernel::new(&ctx).expect("router+topk");
@@ -407,19 +367,9 @@ fn run_moe_parity_test_internal(
)
.expect("router+topk encode");
- let fused_kernel =
- MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let fused_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
let encoder = cb.new_compute_command_encoder().expect("encoder");
- fused_kernel.encode(
- &topk_ids_buf,
- &offsets_buf,
- &sumk_buf,
- &partials_buf,
- t as u32,
- e as u32,
- k as u32,
- &encoder,
- );
+ fused_kernel.encode(&topk_ids_buf, &offsets_buf, &sumk_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
encoder.end_encoding();
let scatter = MoeScatterKernels::new(&ctx).expect("scatter");
@@ -517,9 +467,7 @@ fn run_moe_parity_test_internal(
)
.expect("experts encode");
- let finalize =
- MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into())
- .expect("finalize");
+ let finalize = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize");
let encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
&tok2row_buf,
@@ -539,14 +487,8 @@ fn run_moe_parity_test_internal(
eprintln!("[E2E] GPU execution completed");
// Read GPU output
- let y_out_bf16 = unsafe {
- std::slice::from_raw_parts(
- y_out_buf.contents().as_ptr() as *const bf16,
- t * d_model,
- )
- };
- let y_out_gpu: Vec<f32> =
- y_out_bf16.iter().map(|&v| f32::from(v)).collect();
+ let y_out_bf16 = unsafe { std::slice::from_raw_parts(y_out_buf.contents().as_ptr() as *const bf16, t * d_model) };
+ let y_out_gpu: Vec<f32> = y_out_bf16.iter().map(|&v| f32::from(v)).collect();
// Validate GPU output is finite
let nan_count = y_out_gpu.iter().filter(|v| v.is_nan()).count();
@@ -570,85 +512,46 @@ fn run_moe_parity_test_internal(
);
// Probe tile bookkeeping
- let total_tiles_cpu = unsafe {
- std::slice::from_raw_parts(
- total_tiles_buf.contents().as_ptr() as *const u32,
- 2,
- )
- };
- let dispatch_args_cpu = unsafe {
- std::slice::from_raw_parts(
- dispatch_args_buf.contents().as_ptr() as *const u32,
- 3,
- )
- };
- let tile_offsets_cpu = unsafe {
- std::slice::from_raw_parts(
- tile_offsets_buf.contents().as_ptr() as *const u32,
- (e + 1).min(8),
- )
- };
- let sumk_val =
- unsafe { *(sumk_buf.contents().as_ptr() as *const u32) } as usize;
+ let total_tiles_cpu =
+ unsafe { std::slice::from_raw_parts(total_tiles_buf.contents().as_ptr() as *const u32, 2) };
+ let dispatch_args_cpu =
+ unsafe { std::slice::from_raw_parts(dispatch_args_buf.contents().as_ptr() as *const u32, 3) };
+ let tile_offsets_cpu =
+ unsafe { std::slice::from_raw_parts(tile_offsets_buf.contents().as_ptr() as *const u32, (e + 1).min(8)) };
+ let sumk_val = unsafe { *(sumk_buf.contents().as_ptr() as *const u32) } as usize;
eprintln!("[E2E] Tile bookkeeping:");
eprintln!(
"[E2E] total_tiles={}, dispatch_args=({}, {}, {})",
- total_tiles_cpu[0],
- dispatch_args_cpu[0],
- dispatch_args_cpu[1],
- dispatch_args_cpu[2]
- );
- eprintln!(
- "[E2E] tile_offsets[0..{}]={:?}",
- (e + 1).min(8),
- &tile_offsets_cpu
+ total_tiles_cpu[0], dispatch_args_cpu[0], dispatch_args_cpu[1], dispatch_args_cpu[2]
);
+ eprintln!("[E2E] tile_offsets[0..{}]={:?}", (e + 1).min(8), &tile_offsets_cpu);
eprintln!("[E2E] sumk={}, num_tiles_k={}", sumk_val, num_tiles_k);
// For multi-token tests with large d_ff, verify gather output (x_perm)
if t > 1 && d_ff >= 256 {
let x_perm_cpu = unsafe {
- std::slice::from_raw_parts(
- x_perm_buf.contents().as_ptr() as *const bf16,
- sumk_val * d_model,
- )
+ std::slice::from_raw_parts(x_perm_buf.contents().as_ptr() as *const bf16, sumk_val * d_model)
};
eprintln!("[E2E] x_perm diagnostics (sumk={}):", sumk_val);
- eprintln!(
- "[E2E] Row 0 [0:8]: {:?}",
- &x_perm_cpu[0..8]
- .iter()
- .map(|&v| f32::from(v))
- .collect::<Vec<_>>()
- );
+ eprintln!("[E2E] Row 0 [0:8]: {:?}", &x_perm_cpu[0..8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());
if sumk_val > 1 {
eprintln!(
"[E2E] Row 1 [{}:{}]: {:?}",
d_model,
d_model + 8,
- &x_perm_cpu[d_model..d_model + 8]
- .iter()
- .map(|&v| f32::from(v))
- .collect::<Vec<_>>()
+ &x_perm_cpu[d_model..d_model + 8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()
);
}
// Check tile_map for first few tiles
let tile_map_cpu = unsafe {
- std::slice::from_raw_parts(
- tile_map_buf.contents().as_ptr() as *const u32,
- 12.min(max_tiles * 3),
- )
+ std::slice::from_raw_parts(tile_map_buf.contents().as_ptr() as *const u32, 12.min(max_tiles * 3))
};
eprintln!("[E2E] tile_map (first 4 tiles): {:?}", &tile_map_cpu);
// CRITICAL: Check tok2row mapping for multi-token tests
- let tok2row_cpu = unsafe {
- std::slice::from_raw_parts(
- tok2row_buf.contents().as_ptr() as *const i32,
- t * k,
- )
- };
+ let tok2row_cpu =
+ unsafe { std::slice::from_raw_parts(tok2row_buf.contents().as_ptr() as *const i32, t * k) };
eprintln!("[E2E] tok2row[0..{}]: {:?}", t * k, &tok2row_cpu);
eprintln!(
"[E2E] Expected: token 0β†’row {}, token 1β†’row {}",
@@ -662,32 +565,17 @@ fn run_moe_parity_test_internal(
// Check y_partial at specific indices where finalize reads for token 1
let y_partial_full = unsafe {
- std::slice::from_raw_parts(
- y_partial_buf.contents().as_ptr() as *const bf16,
- sumk_val * d_model,
- )
+ std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sumk_val * d_model)
};
eprintln!("[E2E] y_partial spot check:");
- eprintln!(
- "[E2E] y_partial[48] (row 0, col 48) = {:.6}",
- f32::from(y_partial_full[48])
- );
+ eprintln!("[E2E] y_partial[48] (row 0, col 48) = {:.6}", f32::from(y_partial_full[48]));
if sumk_val > 1 {
// Check positions where mismatches occur (odd indices near tile boundaries)
- eprintln!(
- "[E2E] Row 1 positions (even=working, odd=corrupted?):"
- );
- for &pos in &[
- 48, 56, 57, 58, 59, 60, 61, 62, 63, 64, 120, 121, 122, 123,
- 124, 125, 126, 127, 128,
- ] {
+ eprintln!("[E2E] Row 1 positions (even=working, odd=corrupted?):");
+ for &pos in &[48, 56, 57, 58, 59, 60, 61, 62, 63, 64, 120, 121, 122, 123, 124, 125, 126, 127, 128] {
let idx = 1 * d_model + pos;
if idx < y_partial_full.len() {
- eprintln!(
- "[E2E] y_partial[row1, col {}] = {:.3}",
- pos,
- f32::from(y_partial_full[idx])
- );
+ eprintln!("[E2E] y_partial[row1, col {}] = {:.3}", pos, f32::from(y_partial_full[idx]));
}
}
}
@@ -738,12 +626,11 @@ fn run_moe_parity_test_internal(
let threshold_abs = 1e-3;
if rel_diff > threshold_rel && abs_diff > threshold_abs {
- let print_limit =
- if test_name.contains("BoundarySweep_D1024_FF256_T2") {
- usize::MAX // Print ALL for debugging
- } else {
- 10
- };
+ let print_limit = if test_name.contains("BoundarySweep_D1024_FF256_T2") {
+ usize::MAX // Print ALL for debugging
+ } else {
+ 10
+ };
if mismatches < print_limit {
eprintln!(
"[{}] Mismatch idx {}: GPU={:.6}, CPU={:.6}, abs={:.6}, rel={:.6}",
@@ -763,41 +650,21 @@ fn run_moe_parity_test_internal(
// Debug: print some sample values from intermediate buffers
eprintln!("[E2E] === DEBUG: Intermediate values ===");
- let topk_ids_gpu = unsafe {
- std::slice::from_raw_parts(
- topk_ids_buf.contents().as_ptr() as *const i32,
- t * k,
- )
- };
- let topk_probs_gpu = unsafe {
- std::slice::from_raw_parts(
- topk_probs_buf.contents().as_ptr() as *const bf16,
- t * k,
- )
- };
+ let topk_ids_gpu = unsafe { std::slice::from_raw_parts(topk_ids_buf.contents().as_ptr() as *const i32, t * k) };
+ let topk_probs_gpu =
+ unsafe { std::slice::from_raw_parts(topk_probs_buf.contents().as_ptr() as *const bf16, t * k) };
eprintln!("[E2E] TopK IDs: {:?}", topk_ids_gpu);
- eprintln!(
- "[E2E] TopK Probs: {:?}",
- topk_probs_gpu.iter().map(|&v| f32::from(v)).collect::<Vec<_>>()
- );
+ eprintln!("[E2E] TopK Probs: {:?}", topk_probs_gpu.iter().map(|&v| f32::from(v)).collect::<Vec<_>>());
- let y_partial_gpu = unsafe {
- std::slice::from_raw_parts(
- y_partial_buf.contents().as_ptr() as *const bf16,
- max_sumk * d_model,
- )
- };
- let sumk_actual =
- unsafe { *(sumk_buf.contents().as_ptr() as *const u32) } as usize;
+ let y_partial_gpu =
+ unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, max_sumk * d_model) };
+ let sumk_actual = unsafe { *(sumk_buf.contents().as_ptr() as *const u32) } as usize;
eprintln!("[E2E] sumk={}", sumk_actual);
let sample_size = 16.min(sumk_actual * d_model);
eprintln!(
"[E2E] y_partial sample (first {}): {:?}",
sample_size,
- &y_partial_gpu[..sample_size]
- .iter()
- .map(|&v| f32::from(v))
- .collect::<Vec<_>>()
+ &y_partial_gpu[..sample_size].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()
);
// For multi-row debugging: print both rows
@@ -806,52 +673,29 @@ fn run_moe_parity_test_internal(
"[E2E] y_partial row 1 [{}-{}]: {:?}",
d_model,
d_model + 16,
- &y_partial_gpu[d_model..d_model + 16]
- .iter()
- .map(|&v| f32::from(v))
- .collect::<Vec<_>>()
+ &y_partial_gpu[d_model..d_model + 16].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()
);
}
let out_sample_size = 16.min(t * d_model);
- eprintln!(
- "[E2E] y_out sample (token 0): {:?}",
- &y_out_gpu[..out_sample_size]
- );
+ eprintln!("[E2E] y_out sample (token 0): {:?}", &y_out_gpu[..out_sample_size]);
if t > 1 && d_model >= 512 {
eprintln!(
"[E2E] y_out sample (token 1, first 32): {:?}",
- &y_out_gpu[d_model..d_model + 32]
- .iter()
- .map(|&v| f32::from(v))
- .collect::<Vec<_>>()
+ &y_out_gpu[d_model..d_model + 32].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()
);
}
- eprintln!(
- "[E2E] CPU ref sample (token 0): {:?}",
- &y_cpu[..out_sample_size]
- );
+ eprintln!("[E2E] CPU ref sample (token 0): {:?}", &y_cpu[..out_sample_size]);
if t > 1 && d_model >= 512 {
- eprintln!(
- "[E2E] CPU ref sample (token 1): {:?}",
- &y_cpu[d_model..d_model + 16]
- );
+ eprintln!("[E2E] CPU ref sample (token 1): {:?}", &y_cpu[d_model..d_model + 16]);
// Compare token 1 outputs element-wise (more positions to find pattern)
let gpu_t1 = &y_out_gpu[d_model..];
let cpu_t1 = &y_cpu[d_model..];
eprintln!("[E2E] Token 1 detailed comparison:");
- for i in
- [0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 17, 18, 48, 64, 121, 128].iter()
- {
+ for i in [0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 17, 18, 48, 64, 121, 128].iter() {
if *i < d_model {
let diff = (f32::from(gpu_t1[*i]) - cpu_t1[*i]).abs();
- eprintln!(
- "[E2E] [{}]: GPU={:.6}, CPU={:.6}, diff={:.6}",
- i,
- f32::from(gpu_t1[*i]),
- cpu_t1[*i],
- diff
- );
+ eprintln!("[E2E] [{}]: GPU={:.6}, CPU={:.6}, diff={:.6}", i, f32::from(gpu_t1[*i]), cpu_t1[*i], diff);
}
}
}
@@ -865,10 +709,7 @@ fn run_moe_parity_test_internal(
let mean_abs_threshold = 0.1f32.max(multi_expert_drift);
if mean_abs_error >= mean_abs_threshold {
- panic!(
- "[{}] Mean absolute error {:.6} exceeds threshold {:.6}",
- test_name, mean_abs_error, mean_abs_threshold
- );
+ panic!("[{}] Mean absolute error {:.6} exceeds threshold {:.6}", test_name, mean_abs_error, mean_abs_threshold);
}
// Warn on high max_rel but don't fail (can be inflated by near-zero values)
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index f154d8195..1a684516f 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -6,9 +6,8 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::backends::metal::{
KernelDataType, MTLContext,
kernel::moe::{
- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels,
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,
- MoeExpertsTwoPassPrefillKernel,
+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
+ MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel,
},
};
@@ -48,20 +47,14 @@ fn run_decode_case(
let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
let sum_k = t * k;
- eprintln!(
- "\n[decode/two-pass] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}",
- name, t, d_model, d_ff, e, k, sum_k
- );
+ eprintln!("\n[decode/two-pass] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}", name, t, d_model, d_ff, e, k, sum_k);
- let x_perm: Vec<bf16> = (0..sum_k * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
let offsets = build_offsets(e, sum_k);
// Generate W13 in original layout [E, d_model, 2*d_ff]
- let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
+ let w13_original: Vec<bf16> =
+ (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
// Transpose to GPU layout [E, 2*d_ff, d_model]
let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];
@@ -76,18 +69,11 @@ fn run_decode_case(
}
}
}
- let w2: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
-
- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx)
- .expect("experts decode kernel");
+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+
+ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx).expect("experts decode kernel");
let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
@@ -143,10 +129,7 @@ fn run_decode_case(
};
for _ in 0..warmup {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(&cb, &make_two_pass_args());
cb.commit();
cb.wait_until_completed();
@@ -155,10 +138,7 @@ fn run_decode_case(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(&cb, &make_two_pass_args());
cb.commit();
cb.wait_until_completed();
@@ -170,14 +150,10 @@ fn run_decode_case(
let median = times[times.len() / 2];
let min = times[0];
let max = times[times.len() - 1];
- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()
- / times.len() as f64;
+ let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;
let std = var.sqrt();
- eprintln!(
- " mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms",
- mean, median, min, max, std
- );
+ eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);
eprintln!(" β†’ Latency: {:.1} Β΅s/token", (mean / t as f64) * 1000.0);
}
@@ -186,15 +162,11 @@ fn run_decode_case(
fn test_two_pass_decode_speed() {
let ctx = create_ctx();
- let cases = vec![
- ("T1_E16_D1024_F4096", 1, 1024, 4096, 16, 4, 3, 20),
- ("T1_E8_D2048_F8192", 1, 2048, 8192, 8, 4, 3, 20),
- ];
+ let cases =
+ vec![("T1_E16_D1024_F4096", 1, 1024, 4096, 16, 4, 3, 20), ("T1_E8_D2048_F8192", 1, 2048, 8192, 8, 4, 3, 20)];
for (name, t, d_model, d_ff, e, k, warmup, iters) in &cases {
- run_decode_case(
- &ctx, name, *t, *d_model, *d_ff, *e, *k, *warmup, *iters,
- );
+ run_decode_case(&ctx, name, *t, *d_model, *d_ff, *e, *k, *warmup, *iters);
}
}
@@ -214,19 +186,13 @@ fn run_two_pass_prefill_case(
let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
let sum_k = t * k;
- eprintln!(
- "\n[2-pass prefill] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}",
- name, t, d_model, d_ff, e, k, sum_k
- );
+ eprintln!("\n[2-pass prefill] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}", name, t, d_model, d_ff, e, k, sum_k);
- let x_perm: Vec<bf16> = (0..sum_k * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
let offsets = build_offsets(e, sum_k);
- let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
+ let w13_original: Vec<bf16> =
+ (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];
for expert in 0..e {
@@ -240,18 +206,11 @@ fn run_two_pass_prefill_case(
}
}
}
- let w2: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
-
- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(ctx)
- .expect("experts prefill kernel");
+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+
+ let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(ctx).expect("experts prefill kernel");
let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
@@ -303,13 +262,8 @@ fn run_two_pass_prefill_case(
};
for _ in 0..warmup {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- experts_kernel
- .encode(&cb, make_two_pass_args())
- .expect("two-pass prefill encode");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ experts_kernel.encode(&cb, make_two_pass_args()).expect("two-pass prefill encode");
cb.commit();
cb.wait_until_completed();
}
@@ -317,13 +271,8 @@ fn run_two_pass_prefill_case(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- experts_kernel
- .encode(&cb, make_two_pass_args())
- .expect("two-pass prefill encode");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ experts_kernel.encode(&cb, make_two_pass_args()).expect("two-pass prefill encode");
cb.commit();
cb.wait_until_completed();
times.push(start.elapsed().as_secs_f64() * 1000.0);
@@ -334,18 +283,11 @@ fn run_two_pass_prefill_case(
let median = times[times.len() / 2];
let min = times[0];
let max = times[times.len() - 1];
- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()
- / times.len() as f64;
+ let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;
let std = var.sqrt();
- eprintln!(
- " mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms",
- mean, median, min, max, std
- );
- eprintln!(
- " β†’ Throughput: {:.1} Β΅s/token (mean / sum_k)",
- (mean / sum_k as f64) * 1000.0
- );
+ eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);
+ eprintln!(" β†’ Throughput: {:.1} Β΅s/token (mean / sum_k)", (mean / sum_k as f64) * 1000.0);
}
#[test]
@@ -360,9 +302,7 @@ fn test_two_pass_prefill_speed() {
];
for (name, t, d_model, d_ff, e, k, warmup, iters) in cases {
- run_two_pass_prefill_case(
- &ctx, name, t, d_model, d_ff, e, k, warmup, iters,
- );
+ run_two_pass_prefill_case(&ctx, name, t, d_model, d_ff, e, k, warmup, iters);
}
}
@@ -378,39 +318,25 @@ fn run_fused_single_token_case(
) {
let mut rng = StdRng::seed_from_u64(0xF053ED);
- eprintln!(
- "\n[fused single-token] {} => D={}, FF={}, E={}, K={}",
- name, d_model, d_ff, e, k
- );
+ eprintln!("\n[fused single-token] {} => D={}, FF={}, E={}, K={}", name, d_model, d_ff, e, k);
- let x: Vec<bf16> = (0..d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();
let topk_probs: Vec<bf16> = {
- let raw: Vec<f32> =
- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
let sum: f32 = raw.iter().sum();
raw.iter().map(|p| bf16::from_f32(p / sum)).collect()
};
- let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let w2_all: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
-
- let fused_kernel =
- MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");
+ let w13_all: Vec<bf16> =
+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+
+ let fused_kernel = MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");
let x_buf = alloc_buffer_with_data(ctx, &x);
let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);
@@ -445,10 +371,7 @@ fn run_fused_single_token_case(
};
for _ in 0..warmup {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(&cb, make_args());
cb.commit();
cb.wait_until_completed();
@@ -457,10 +380,7 @@ fn run_fused_single_token_case(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(&cb, make_args());
cb.commit();
cb.wait_until_completed();
@@ -472,14 +392,10 @@ fn run_fused_single_token_case(
let median = times[times.len() / 2];
let min = times[0];
let max = times[times.len() - 1];
- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()
- / times.len() as f64;
+ let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;
let std = var.sqrt();
- eprintln!(
- " mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms",
- mean, median, min, max, std
- );
+ eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);
eprintln!(" β†’ Latency: {:.1} Β΅s/token", mean * 1000.0);
}
@@ -495,9 +411,7 @@ fn test_fused_single_token_speed() {
];
for (name, d_model, d_ff, e, k, warmup, iters) in cases {
- run_fused_single_token_case(
- &ctx, name, d_model, d_ff, e, k, warmup, iters,
- );
+ run_fused_single_token_case(&ctx, name, d_model, d_ff, e, k, warmup, iters);
}
}
@@ -522,19 +436,13 @@ fn test_single_token_indirect_vs_fused() {
eprintln!("[{}] D={}, FF={}, E={}, K={}", name, d_model, d_ff, e, k);
// Run indirect decode (T=1)
- let indirect_mean = run_indirect_decode_timed(
- &ctx, 1, d_model, d_ff, e, k, warmup, iters,
- );
+ let indirect_mean = run_indirect_decode_timed(&ctx, 1, d_model, d_ff, e, k, warmup, iters);
// Run fused decode
- let fused_mean =
- run_fused_decode_timed(&ctx, d_model, d_ff, e, k, warmup, iters);
+ let fused_mean = run_fused_decode_timed(&ctx, d_model, d_ff, e, k, warmup, iters);
let speedup = indirect_mean / fused_mean;
- eprintln!(
- " Indirect: {:.3}ms, Fused: {:.3}ms, Speedup: {:.2}x\n",
- indirect_mean, fused_mean, speedup
- );
+ eprintln!(" Indirect: {:.3}ms, Fused: {:.3}ms, Speedup: {:.2}x\n", indirect_mean, fused_mean, speedup);
}
}
@@ -553,14 +461,11 @@ fn run_indirect_decode_timed(
let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
let sum_k = t * k;
- let x_perm: Vec<bf16> = (0..sum_k * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
let offsets = build_offsets(e, sum_k);
- let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
+ let w13_original: Vec<bf16> =
+ (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];
for expert in 0..e {
@@ -574,18 +479,11 @@ fn run_indirect_decode_timed(
}
}
}
- let w2: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
-
- let experts_kernel =
- MoeExpertsTwoPassDecodeKernels::new(ctx).expect("decode kernel");
+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+
+ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx).expect("decode kernel");
let x_perm_buf = alloc_buffer_with_data(ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(ctx, &offsets);
@@ -636,10 +534,7 @@ fn run_indirect_decode_timed(
};
for _ in 0..warmup {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(&cb, &make_args());
cb.commit();
cb.wait_until_completed();
@@ -648,10 +543,7 @@ fn run_indirect_decode_timed(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(&cb, &make_args());
cb.commit();
cb.wait_until_completed();
@@ -672,34 +564,23 @@ fn run_fused_decode_timed(
) -> f64 {
let mut rng = StdRng::seed_from_u64(0xF053ED);
- let x: Vec<bf16> = (0..d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();
let topk_probs: Vec<bf16> = {
- let raw: Vec<f32> =
- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
let sum: f32 = raw.iter().sum();
raw.iter().map(|p| bf16::from_f32(p / sum)).collect()
};
- let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let w2_all: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
-
- let fused_kernel =
- MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");
+ let w13_all: Vec<bf16> =
+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+
+ let fused_kernel = MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");
let x_buf = alloc_buffer_with_data(ctx, &x);
let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);
@@ -734,10 +615,7 @@ fn run_fused_decode_timed(
};
for _ in 0..warmup {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(&cb, make_args());
cb.commit();
cb.wait_until_completed();
@@ -746,10 +624,7 @@ fn run_fused_decode_timed(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(&cb, make_args());
cb.commit();
cb.wait_until_completed();
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index 3461f7c23..a7a0e3892 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -14,17 +14,14 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::backends::metal::{
KernelDataType,
kernel::moe::{
- MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
- MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel,
+ MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,
+ MoeExpertsTwoPassPrefillKernel,
},
};
#[path = "moe_test_utils.rs"]
mod test_utils;
-use test_utils::{
- alloc_buffer, alloc_buffer_with_data, assert_bf16_close, cpu_tile_counts,
- cpu_tile_scan, create_ctx,
-};
+use test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, cpu_tile_counts, cpu_tile_scan, create_ctx};
use uzu::backends::metal::kernel::moe::MoeExpertsSingleDecodeArguments;
/// Test data for MoE experts
@@ -47,35 +44,23 @@ impl MoeTestData {
d_ff: usize,
e: usize,
) -> Self {
- let x: Vec<bf16> = (0..t * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
- let topk_ids: Vec<i32> = (0..t * k)
- .map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32)
- .collect();
+ let topk_ids: Vec<i32> = (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();
let topk_probs: Vec<bf16> = (0..t)
.flat_map(|_| {
- let raw: Vec<f32> =
- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
let sum: f32 = raw.iter().sum();
raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()
})
.collect();
- let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let w2: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
+ let w13: Vec<bf16> =
+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
Self {
x,
@@ -262,13 +247,10 @@ fn cpu_moe_reference(
}
} else {
// SwiGLU or GEGLU - need gate projection too
- let mut acc_gate =
- f32::from(up_biases[bias_base + d_ff + h]);
+ let mut acc_gate = f32::from(up_biases[bias_base + d_ff + h]);
for d in 0..d_model {
let x_val = f32::from(x[x_offset + d]);
- let w_val = f32::from(
- w13_all[w13_base + (d_ff + h) * d_model + d],
- );
+ let w_val = f32::from(w13_all[w13_base + (d_ff + h) * d_model + d]);
acc_gate += x_val * w_val;
}
@@ -334,23 +316,16 @@ fn test_two_pass_decode_correctness() {
let gating_code = 2u32; // SwiGLU
let silu_alpha = 1.0f32;
- eprintln!(
- "[2-pass decode] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}",
- t, k, sum_k, d_model, d_ff, e
- );
+ eprintln!("[2-pass decode] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}", t, k, sum_k, d_model, d_ff, e);
// Generate input x [T, d_model]
- let x: Vec<bf16> = (0..t * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
// Generate routing: topk_ids [T*K], topk_probs [T*K]
- let topk_ids: Vec<i32> =
- (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();
+ let topk_ids: Vec<i32> = (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();
let topk_probs: Vec<bf16> = (0..t)
.flat_map(|_| {
- let raw: Vec<f32> =
- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
let sum: f32 = raw.iter().sum();
raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()
})
@@ -388,18 +363,10 @@ fn test_two_pass_decode_correctness() {
}
// Generate weights
- let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let w2: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
+ let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
// CPU reference - end-to-end
let y_expected = cpu_moe_reference(
@@ -445,12 +412,8 @@ fn test_two_pass_decode_correctness() {
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
// Execute 2-pass decode kernel
- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx)
- .expect("MoeExpertsTwoPassDecodeKernel::new");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
const K_TILE: usize = 64;
let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;
@@ -491,12 +454,8 @@ fn test_two_pass_decode_correctness() {
cb.wait_until_completed();
// Read GPU partial output and do CPU finalize (weighted sum)
- let y_partial_gpu = unsafe {
- std::slice::from_raw_parts(
- y_partial_buf.contents().as_ptr() as *const bf16,
- sum_k * d_model,
- )
- };
+ let y_partial_gpu =
+ unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };
// Finalize: y[t] = Ξ£_k prob[t,k] * y_partial[t*k + k_idx]
let mut y_gpu = vec![bf16::from_f32(0.0); t * d_model];
@@ -517,9 +476,7 @@ fn test_two_pass_decode_correctness() {
let mut max_abs_error = 0.0f32;
let mut max_idx = 0;
- for (i, (&gpu_val, &cpu_val)) in
- y_gpu.iter().zip(y_expected.iter()).enumerate()
- {
+ for (i, (&gpu_val, &cpu_val)) in y_gpu.iter().zip(y_expected.iter()).enumerate() {
let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();
if abs_error > max_abs_error {
max_abs_error = abs_error;
@@ -555,10 +512,7 @@ fn test_two_pass_decode_multi_token() {
let gating_code = 2u32;
let silu_alpha = 1.0f32;
- eprintln!(
- "[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}",
- t, k, d_model, d_ff, e
- );
+ eprintln!("[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);
let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);
let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);
@@ -586,8 +540,7 @@ fn test_two_pass_decode_multi_token() {
// GPU buffers
let x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);
- let row_expert_map_buf =
- alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
+ let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);
let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);
let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);
@@ -603,12 +556,8 @@ fn test_two_pass_decode_multi_token() {
let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- let experts_kernel =
- MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(
&cb,
&MoeExpertsTwoPassArguments {
@@ -643,20 +592,9 @@ fn test_two_pass_decode_multi_token() {
cb.commit();
cb.wait_until_completed();
- let y_partial_gpu = unsafe {
- std::slice::from_raw_parts(
- y_partial_buf.contents().as_ptr() as *const bf16,
- sum_k * d_model,
- )
- };
- let y_gpu = gather_and_finalize(
- y_partial_gpu,
- &data.topk_probs,
- &scatter.perm_idx,
- t,
- k,
- d_model,
- );
+ let y_partial_gpu =
+ unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };
+ let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass decode multi-token");
eprintln!("[2-pass decode multi-token] βœ“ PASSED");
@@ -676,10 +614,7 @@ fn test_two_pass_prefill_correctness() {
let gating_code = 2u32;
let silu_alpha = 1.0f32;
- eprintln!(
- "[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}",
- t, k, d_model, d_ff, e
- );
+ eprintln!("[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);
let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);
let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);
@@ -707,8 +642,7 @@ fn test_two_pass_prefill_correctness() {
// GPU buffers
let x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);
- let row_expert_map_buf =
- alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
+ let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);
let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);
let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);
@@ -724,12 +658,8 @@ fn test_two_pass_prefill_correctness() {
let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- let experts_kernel =
- MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel
.encode(
&cb,
@@ -766,20 +696,9 @@ fn test_two_pass_prefill_correctness() {
cb.commit();
cb.wait_until_completed();
- let y_partial_gpu = unsafe {
- std::slice::from_raw_parts(
- y_partial_buf.contents().as_ptr() as *const bf16,
- sum_k * d_model,
- )
- };
- let y_gpu = gather_and_finalize(
- y_partial_gpu,
- &data.topk_probs,
- &scatter.perm_idx,
- t,
- k,
- d_model,
- );
+ let y_partial_gpu =
+ unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };
+ let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass prefill");
eprintln!("[2-pass prefill] βœ“ PASSED");
@@ -818,37 +737,24 @@ fn test_fused_single_token_decode() {
let e = 8;
let k = 2;
- eprintln!(
- "[fused single-token] d_model={}, d_ff={}, E={}, K={}",
- d_model, d_ff, e, k
- );
+ eprintln!("[fused single-token] d_model={}, d_ff={}, E={}, K={}", d_model, d_ff, e, k);
// Generate test data
- let x: Vec<bf16> = (0..d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();
let topk_probs: Vec<bf16> = {
- let raw: Vec<f32> =
- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
let sum: f32 = raw.iter().sum();
raw.iter().map(|p| bf16::from_f32(p / sum)).collect()
};
- let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let w2_all: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
+ let w13_all: Vec<bf16> =
+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
// CPU reference (gating_code=2 for SwiGLU)
let gating_code = 2u32;
@@ -885,12 +791,8 @@ fn test_fused_single_token_decode() {
let y_buf = alloc_buffer::<bf16>(&ctx, d_model);
// Run fused decode kernel
- let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx)
- .expect("MoeExpertsSingleDecodeKernel::new");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(
&cb,
@@ -921,21 +823,14 @@ fn test_fused_single_token_decode() {
cb.wait_until_completed();
// Read GPU output
- let y_gpu = unsafe {
- std::slice::from_raw_parts(
- y_buf.contents().as_ptr() as *const bf16,
- d_model,
- )
- };
+ let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };
// Compute error metrics
let mut max_abs_error = 0.0f32;
let mut sum_abs_error = 0.0f32;
let mut max_idx = 0;
- for (i, (&gpu_val, &cpu_val)) in
- y_gpu.iter().zip(y_expected.iter()).enumerate()
- {
+ for (i, (&gpu_val, &cpu_val)) in y_gpu.iter().zip(y_expected.iter()).enumerate() {
let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();
sum_abs_error += abs_error;
if abs_error > max_abs_error {
@@ -955,12 +850,7 @@ fn test_fused_single_token_decode() {
);
let tolerance = 0.02;
- assert_bf16_close(
- y_gpu,
- &y_expected,
- tolerance,
- "fused single-token output",
- );
+ assert_bf16_close(y_gpu, &y_expected, tolerance, "fused single-token output");
eprintln!("[fused single-token] PASSED (tolerance={:.4})", tolerance);
}
@@ -975,36 +865,23 @@ fn test_fused_single_token_k4() {
let e = 8;
let k = 4;
- eprintln!(
- "[fused single-token K=4] d_model={}, d_ff={}, E={}, K={}",
- d_model, d_ff, e, k
- );
+ eprintln!("[fused single-token K=4] d_model={}, d_ff={}, E={}, K={}", d_model, d_ff, e, k);
- let x: Vec<bf16> = (0..d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
let topk_ids: Vec<i32> = (0..k).map(|i| ((i * 2) % e) as i32).collect();
let topk_probs: Vec<bf16> = {
- let raw: Vec<f32> =
- (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
let sum: f32 = raw.iter().sum();
raw.iter().map(|p| bf16::from_f32(p / sum)).collect()
};
- let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let w2_all: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
- .collect();
+ let w13_all: Vec<bf16> =
+ (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let gating_code = 2u32;
let silu_alpha = 1.0f32;
@@ -1038,12 +915,8 @@ fn test_fused_single_token_k4() {
let hidden_buf = alloc_buffer::<f32>(&ctx, k * d_ff);
let y_buf = alloc_buffer::<bf16>(&ctx, d_model);
- let fused_kernel =
- MoeExpertsSingleDecodeKernels::new(&ctx).expect("fused kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx).expect("fused kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(
&cb,
MoeExpertsSingleDecodeArguments {
@@ -1071,21 +944,14 @@ fn test_fused_single_token_k4() {
cb.commit();
cb.wait_until_completed();
- let y_gpu = unsafe {
- std::slice::from_raw_parts(
- y_buf.contents().as_ptr() as *const bf16,
- d_model,
- )
- };
+ let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };
// Compute error metrics
let mut max_abs_error = 0.0f32;
let mut sum_abs_error = 0.0f32;
let mut max_idx = 0;
- for (i, (&gpu_val, &cpu_val)) in
- y_gpu.iter().zip(y_expected.iter()).enumerate()
- {
+ for (i, (&gpu_val, &cpu_val)) in y_gpu.iter().zip(y_expected.iter()).enumerate() {
let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();
sum_abs_error += abs_error;
if abs_error > max_abs_error {
@@ -1105,12 +971,7 @@ fn test_fused_single_token_k4() {
);
let tolerance = 0.02;
- assert_bf16_close(
- y_gpu,
- &y_expected,
- tolerance,
- "fused single-token K=4 output",
- );
+ assert_bf16_close(y_gpu, &y_expected, tolerance, "fused single-token K=4 output");
eprintln!("[fused single-token K=4] PASSED (tolerance={:.4})", tolerance);
}
diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rs
index e6656dd96..4b4efdf7d 100644
--- a/crates/uzu/tests/moe/moe_finalize_test.rs
+++ b/crates/uzu/tests/moe/moe_finalize_test.rs
@@ -5,13 +5,9 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use half::bf16;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::backends::common::kernel::MoeFinalizeKernel;
-use uzu::backends::metal::{
- KernelDataType, kernel::dsl::MoeFinalizeMetalKernel,
-};
+use uzu::backends::metal::{KernelDataType, kernel::dsl::MoeFinalizeMetalKernel};
-use super::test_utils::{
- alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx,
-};
+use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};
fn cpu_finalize(
tok2row: &[i32],
@@ -30,8 +26,7 @@ fn cpu_finalize(
let row = tok2row[idx];
if row >= 0 {
let rowu = row as usize;
- acc += f32::from(probs[idx])
- * f32::from(y_partial[rowu * d_model + f]);
+ acc += f32::from(probs[idx]) * f32::from(y_partial[rowu * d_model + f]);
}
}
y[ti * d_model + f] = bf16::from_f32(acc);
@@ -54,10 +49,7 @@ fn test_finalize_correctness() {
];
for (t, k, d_model, sum_k) in shapes {
- eprintln!(
- "[FinalizeTest] T={}, K={}, d_model={}, sum_k={}",
- t, k, d_model, sum_k
- );
+ eprintln!("[FinalizeTest] T={}, K={}, d_model={}, sum_k={}", t, k, d_model, sum_k);
// Generate random tok2row mapping: maps (token, k_idx) β†’ row in y_partial
// Some entries can be -1 (no expert selected)
@@ -77,14 +69,10 @@ fn test_finalize_correctness() {
}
// Generate random probabilities (should sum to 1 per token, but not critical for unit test)
- let probs: Vec<bf16> = (0..t * k)
- .map(|_| bf16::from_f32(rng.random_range(0.0..1.0)))
- .collect();
+ let probs: Vec<bf16> = (0..t * k).map(|_| bf16::from_f32(rng.random_range(0.0..1.0))).collect();
// Generate random y_partial (expert outputs)
- let y_partial: Vec<bf16> = (0..sum_k * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-2.0..2.0)))
- .collect();
+ let y_partial: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-2.0..2.0))).collect();
// CPU reference
let y_cpu = cpu_finalize(&tok2row, &probs, &y_partial, t, d_model, k);
@@ -96,13 +84,8 @@ fn test_finalize_correctness() {
let y_out_buf = alloc_buffer::<bf16>(&ctx, t * d_model);
// Execute finalize kernel
- let finalize =
- MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into())
- .expect("finalize kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let finalize = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
&tok2row_buf,
@@ -119,12 +102,7 @@ fn test_finalize_correctness() {
cb.wait_until_completed();
// Compare
- let y_gpu = unsafe {
- std::slice::from_raw_parts(
- y_out_buf.contents().as_ptr() as *const bf16,
- t * d_model,
- )
- };
+ let y_gpu = unsafe { std::slice::from_raw_parts(y_out_buf.contents().as_ptr() as *const bf16, t * d_model) };
// BF16 has limited precision, especially for weighted sums
assert_bf16_close(y_gpu, &y_cpu, 1e-2, "finalize output");
diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rs
index 19fad640f..144b22f8d 100644
--- a/crates/uzu/tests/moe/moe_gather_test.rs
+++ b/crates/uzu/tests/moe/moe_gather_test.rs
@@ -9,9 +9,7 @@ use uzu::backends::metal::kernel::{
moe::{MoeGatherArguments, MoeGatherKernels},
};
-use super::test_utils::{
- alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx,
-};
+use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};
/// CPU reference for gather operation: x_perm[i] = x[bucketed_ids[i]]
///
@@ -37,8 +35,7 @@ pub fn cpu_gather(
if token_id >= 0 && (token_id as usize) < t {
let src_offset = (token_id as usize) * d_model;
let dst_offset = row * d_model;
- x_perm[dst_offset..dst_offset + d_model]
- .copy_from_slice(&x[src_offset..src_offset + d_model]);
+ x_perm[dst_offset..dst_offset + d_model].copy_from_slice(&x[src_offset..src_offset + d_model]);
}
}
x_perm
@@ -61,13 +58,10 @@ fn test_gather_correctness() {
eprintln!("[GatherTest] T={}, sum_k={}, d_model={}", t, sum_k, d_model);
// Random input
- let x: Vec<bf16> = (0..t * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-2.0..2.0)))
- .collect();
+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-2.0..2.0))).collect();
// Random bucketed_ids with valid token indices
- let bucketed_ids: Vec<i32> =
- (0..sum_k).map(|_| rng.random_range(0..t as i32)).collect();
+ let bucketed_ids: Vec<i32> = (0..sum_k).map(|_| rng.random_range(0..t as i32)).collect();
let x_cpu = cpu_gather(&x, &bucketed_ids, t, d_model, sum_k);
@@ -82,10 +76,7 @@ fn test_gather_correctness() {
// Execute gather kernel using kernel struct
let gather = MoeGatherKernels::new(&ctx).expect("MoeGatherKernel::new");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
gather.encode(
&cb,
KernelDataType::BFloat16,
@@ -103,12 +94,8 @@ fn test_gather_correctness() {
cb.wait_until_completed();
// Compare
- let x_gpu = unsafe {
- std::slice::from_raw_parts(
- x_perm_buf.contents().as_ptr() as *const bf16,
- sum_k * d_model,
- )
- };
+ let x_gpu =
+ unsafe { std::slice::from_raw_parts(x_perm_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };
assert_bf16_close(x_gpu, &x_cpu, 1e-6, "gather output");
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index 6af1e2978..6766e7eda 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -10,10 +10,8 @@ use uzu::backends::{
kernel::{
dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,
- MoeGatherArguments, MoeGatherKernels, MoeRouterTopKArguments,
- MoeRouterTopKKernel, MoeScatterKernels,
- MoeScatterWithMapArguments,
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels, MoeGatherArguments, MoeGatherKernels,
+ MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterKernels, MoeScatterWithMapArguments,
},
},
},
@@ -43,8 +41,7 @@ impl PerfResult {
let min = sorted[0];
let max = sorted[sorted.len() - 1];
- let variance = sorted.iter().map(|t| (t - mean).powi(2)).sum::<f64>()
- / sorted.len() as f64;
+ let variance = sorted.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / sorted.len() as f64;
let std_dev = variance.sqrt();
Self {
@@ -60,12 +57,7 @@ impl PerfResult {
fn print(&self) {
eprintln!(
" {:<30} mean={:8.3}ms median={:8.3}ms min={:8.3}ms max={:8.3}ms std={:6.3}ms",
- self.name,
- self.mean_ms,
- self.median_ms,
- self.min_ms,
- self.max_ms,
- self.std_dev_ms
+ self.name, self.mean_ms, self.median_ms, self.min_ms, self.max_ms, self.std_dev_ms
);
}
}
@@ -103,30 +95,18 @@ fn test_moe_e2e_decode_perf() {
let ctx = create_ctx();
let mut rng = StdRng::seed_from_u64(0xA0075);
- let configs = vec![
- ("Small", 1, 1024, 256, 8, 2),
- ("Medium", 1, 2048, 1024, 16, 2),
- ("Production", 1, 4096, 14336, 16, 2),
- ];
+ let configs =
+ vec![("Small", 1, 1024, 256, 8, 2), ("Medium", 1, 2048, 1024, 16, 2), ("Production", 1, 4096, 14336, 16, 2)];
eprintln!("\n=== End-to-End MoE Performance (DECODE, T=1) ===");
for (config_name, t, d_model, d_ff, e, k) in configs {
- eprintln!(
- "\n{} T={}, D={}, H={}, E={}, K={}",
- config_name, t, d_model, d_ff, e, k
- );
+ eprintln!("\n{} T={}, D={}, H={}, E={}, K={}", config_name, t, d_model, d_ff, e, k);
// Generate data
- let x: Vec<bf16> = (0..t * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
- let router_w: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))
- .collect();
- let router_b: Vec<bf16> = (0..e)
- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))
- .collect();
+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
+ let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
+ let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
// Buffers (simplified - just key buffers for timing)
let x_buf = alloc_buffer_with_data(&ctx, &x);
@@ -135,15 +115,11 @@ fn test_moe_e2e_decode_perf() {
let topk_ids_buf = alloc_buffer::<i32>(&ctx, t * k);
let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);
- let router_topk =
- MoeRouterTopKKernel::new(&ctx).expect("router+topk fused kernel");
+ let router_topk = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused kernel");
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
router_topk
.encode(
&cb,
@@ -188,21 +164,12 @@ fn test_moe_e2e_prefill_perf() {
eprintln!("\n=== End-to-End MoE Performance (PREFILL, T>1) ===");
for (config_name, t, d_model, d_ff, e, k) in configs {
- eprintln!(
- "\n{} T={}, D={}, H={}, E={}, K={}",
- config_name, t, d_model, d_ff, e, k
- );
+ eprintln!("\n{} T={}, D={}, H={}, E={}, K={}", config_name, t, d_model, d_ff, e, k);
// Generate data
- let x: Vec<bf16> = (0..t * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
- let router_w: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))
- .collect();
- let router_b: Vec<bf16> = (0..e)
- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))
- .collect();
+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
+ let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
+ let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
// Buffers (simplified - just key buffers for timing)
let x_buf = alloc_buffer_with_data(&ctx, &x);
@@ -211,15 +178,11 @@ fn test_moe_e2e_prefill_perf() {
let topk_ids_buf = alloc_buffer::<i32>(&ctx, t * k);
let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);
- let router_topk =
- MoeRouterTopKKernel::new(&ctx).expect("router+topk fused kernel");
+ let router_topk = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused kernel");
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
router_topk
.encode(
&cb,
@@ -260,21 +223,14 @@ fn test_moe_pipeline_breakdown_decode() {
let mut rng = StdRng::seed_from_u64(0xDECADE);
eprintln!("\n=== MoE Pipeline Breakdown (DECODE, T=1) ===");
- eprintln!(
- "Measures ALL MoE kernels: Router→TopK→Counts→Offsets→Scatter→Gather→Experts→Finalize\n"
- );
+ eprintln!("Measures ALL MoE kernels: Router→TopK→Counts→Offsets→Scatter→Gather→Experts→Finalize\n");
let (t, d_model, d_ff, e, k) = (1, 4096, 14336, 16, 2);
// Allocate buffers
- let x: Vec<bf16> = (0..t * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
- .collect();
- let router_w: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))
- .collect();
- let router_b: Vec<bf16> =
- (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
+ let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
+ let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
let x_buf = alloc_buffer_with_data(&ctx, &x);
let router_w_buf = alloc_buffer_with_data(&ctx, &router_w);
@@ -294,9 +250,8 @@ fn test_moe_pipeline_breakdown_decode() {
// Expert weights buffers
// Generate W13 in original layout [E, d_model, 2*d_ff]
- let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))
- .collect();
+ let w13_original: Vec<bf16> =
+ (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
// Transpose to GPU layout [E, 2*d_ff, d_model]
let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];
@@ -312,15 +267,9 @@ fn test_moe_pipeline_breakdown_decode() {
}
}
- let w2: Vec<bf16> = (0..e * d_ff * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.5..0.5)))
- .collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))
- .collect();
- let down_biases: Vec<bf16> = (0..e * d_model)
- .map(|_| bf16::from_f32(rng.random_range(-0.1..0.1)))
- .collect();
+ let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
+ let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
let w13_buf = alloc_buffer_with_data(&ctx, &w13);
let w2_buf = alloc_buffer_with_data(&ctx, &w2);
@@ -348,74 +297,58 @@ fn test_moe_pipeline_breakdown_decode() {
let bucketed_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);
// Create kernel structs (use production-validated encoding logic)
- let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx)
- .expect("counts+offsets fused");
+ let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("counts+offsets fused");
let scatter_kernel = MoeScatterKernels::new(&ctx).expect("scatter");
let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");
- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx)
- .expect("experts two-pass decode");
- let finalize_kernel =
- MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into())
- .expect("finalize");
- let router_topk_fused_kernel =
- MoeRouterTopKKernel::new(&ctx).expect("router+topk fused");
+ let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("experts two-pass decode");
+ let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize");
+ let router_topk_fused_kernel = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused");
// Testing: Router + TopK + Counts+Offsets (FUSED)
- let router_topk_fused_perf =
- time_kernel("Router+TopK (FUSED)", 2, 5, || {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- router_topk_fused_kernel
- .encode(
- &cb,
- KernelDataType::BFloat16,
- MoeRouterTopKArguments {
- input_buffer: &x_buf,
- weight_buffer: &router_w_buf,
- bias_buffer: &router_b_buf,
- topk_ids_buffer: &topk_ids_buf,
- topk_probs_buffer: &topk_probs_buf,
- t,
- d_model,
- e,
- k,
- renorm: true,
- },
- )
- .expect("router_topk_fused");
- cb.commit();
- cb.wait_until_completed();
- });
+ let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ router_topk_fused_kernel
+ .encode(
+ &cb,
+ KernelDataType::BFloat16,
+ MoeRouterTopKArguments {
+ input_buffer: &x_buf,
+ weight_buffer: &router_w_buf,
+ bias_buffer: &router_b_buf,
+ topk_ids_buffer: &topk_ids_buf,
+ topk_probs_buffer: &topk_probs_buf,
+ t,
+ d_model,
+ e,
+ k,
+ renorm: true,
+ },
+ )
+ .expect("router_topk_fused");
+ cb.commit();
+ cb.wait_until_completed();
+ });
- let counts_offsets_perf =
- time_kernel("Counts+Offsets (FUSED)", 2, 5, || {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
- counts_offsets_kernel.encode(
- &topk_ids_buf,
- &offsets_buf,
- &sumk_buf,
- &partials_buf,
- t as u32,
- e as u32,
- k as u32,
- &encoder,
- );
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
- });
+ let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let encoder = cb.new_compute_command_encoder().expect("encoder");
+ counts_offsets_kernel.encode(
+ &topk_ids_buf,
+ &offsets_buf,
+ &sumk_buf,
+ &partials_buf,
+ t as u32,
+ e as u32,
+ k as u32,
+ &encoder,
+ );
+ encoder.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ });
let scatter_perf = time_kernel("Scatter", 2, 5, || {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
scatter_kernel
.encode_block_bases(
&cb,
@@ -433,21 +366,20 @@ fn test_moe_pipeline_breakdown_decode() {
.encode_scatter_with_map(
&cb,
MoeScatterWithMapArguments {
- base:
- uzu::backends::metal::kernel::moe::MoeScatterArguments {
- topk_ids_buffer: &topk_ids_buf,
- topk_probs_buffer: &topk_probs_buf,
- offsets_buffer: &offsets_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- out_ids_buffer: &bucketed_ids_buf,
- out_probs_buffer: &bucketed_probs_buf,
- t,
- e,
- k,
- num_blocks,
- num_tiles,
- },
+ base: uzu::backends::metal::kernel::moe::MoeScatterArguments {
+ topk_ids_buffer: &topk_ids_buf,
+ topk_probs_buffer: &topk_probs_buf,
+ offsets_buffer: &offsets_buf,
+ block_bases_buffer: &block_bases_buf,
+ block_alloc_buffer: &block_alloc_buf,
+ out_ids_buffer: &bucketed_ids_buf,
+ out_probs_buffer: &bucketed_probs_buf,
+ t,
+ e,
+ k,
+ num_blocks,
+ num_tiles,
+ },
tok2row_buffer: &tok2row_buf,
},
KernelDataType::BFloat16,
@@ -458,10 +390,7 @@ fn test_moe_pipeline_breakdown_decode() {
});
let gather_perf = time_kernel("Gather", 2, 5, || {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
gather_kernel.encode(
&cb,
KernelDataType::BFloat16,
@@ -480,10 +409,7 @@ fn test_moe_pipeline_breakdown_decode() {
});
let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(
&cb,
&MoeExpertsTwoPassArguments {
@@ -520,10 +446,7 @@ fn test_moe_pipeline_breakdown_decode() {
});
let finalize_perf = time_kernel("Finalize", 2, 5, || {
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
finalize_kernel.encode(
&tok2row_buf,
@@ -557,9 +480,7 @@ fn test_moe_pipeline_breakdown_decode() {
+ finalize_perf.mean_ms)
* 1000.0;
- eprintln!(
- "\n ═══ Per-Kernel Latency (Production D=4096, H=14336, E=16, K=2, T=1) ═══"
- );
+ eprintln!("\n ═══ Per-Kernel Latency (Production D=4096, H=14336, E=16, K=2, T=1) ═══");
eprintln!(
" Router+TopK (FUSED): {:8.1} us ({:5.1}%)",
router_topk_fused_perf.mean_ms * 1000.0,
@@ -592,10 +513,6 @@ fn test_moe_pipeline_breakdown_decode() {
);
eprintln!(" ═══════════════════════════════════════════");
eprintln!(" TOTAL: {:8.1} us (100.0%)", total_us);
- eprintln!(
- "\n Note: Times include Metal CB overhead (~10-50ms per kernel)."
- );
- eprintln!(
- " Real GPU compute is much faster, but relative % shows bottleneck."
- );
+ eprintln!("\n Note: Times include Metal CB overhead (~10-50ms per kernel).");
+ eprintln!(" Real GPU compute is much faster, but relative % shows bottleneck.");
}
diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rs
index d3f0717f7..18e9435f4 100644
--- a/crates/uzu/tests/moe/moe_router_topk_test.rs
+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs
@@ -81,9 +81,7 @@ pub fn cpu_topk_select_f32(
let v = row[expert];
let mut insert_pos = None;
for j in (0..k).rev() {
- if v > best_vals[j]
- || (v == best_vals[j] && (expert as i32) < best_ids[j])
- {
+ if v > best_vals[j] || (v == best_vals[j] && (expert as i32) < best_ids[j]) {
insert_pos = Some(j);
}
}
@@ -101,8 +99,7 @@ pub fn cpu_topk_select_f32(
ids[base + kk] = best_ids[kk];
}
if renorm {
- let max_v =
- best_vals.iter().copied().fold(f32::NEG_INFINITY, f32::max);
+ let max_v = best_vals.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let mut exps = vec![0.0f32; k];
let mut sum = 0.0f32;
for kk in 0..k {
@@ -138,25 +135,18 @@ fn run_router_topk_once(
renorm: bool,
) {
let mut rng = StdRng::seed_from_u64(1234);
- let input_f32: Vec<f32> =
- (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
- let weight_f32: Vec<f32> =
- (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
- let bias_f32: Vec<f32> =
- (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();
+ let input_f32: Vec<f32> = (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
+ let weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
+ let bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();
// Convert to bf16
- let input: Vec<bf16> =
- input_f32.iter().map(|&x| bf16::from_f32(x)).collect();
- let weight: Vec<bf16> =
- weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();
+ let input: Vec<bf16> = input_f32.iter().map(|&x| bf16::from_f32(x)).collect();
+ let weight: Vec<bf16> = weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();
let bias: Vec<bf16> = bias_f32.iter().map(|&x| bf16::from_f32(x)).collect();
// Compute CPU reference using bf16 inputs (with f32 accumulation)
- let logits_ref =
- cpu_router_logits_bf16(&input, &weight, &bias, t, e, d_model);
- let (ids_ref, probs_ref) =
- cpu_topk_select_f32(&logits_ref, t, e, k, renorm);
+ let logits_ref = cpu_router_logits_bf16(&input, &weight, &bias, t, e, d_model);
+ let (ids_ref, probs_ref) = cpu_topk_select_f32(&logits_ref, t, e, k, renorm);
let input_buf = alloc_buffer_with_data(ctx, &input);
let weight_buf = alloc_buffer_with_data(ctx, &weight);
@@ -165,10 +155,7 @@ fn run_router_topk_once(
// For BFloat16 kernel, probs buffer must be bf16, not f32
let probs_buf = alloc_buffer::<bf16>(ctx, t * k);
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let args = MoeRouterTopKArguments {
input_buffer: &input_buf,
weight_buffer: &weight_buf,
@@ -181,21 +168,16 @@ fn run_router_topk_once(
k,
renorm,
};
- kernel
- .encode(&cb, KernelDataType::BFloat16, args)
- .expect("encode fused router+topk");
+ kernel.encode(&cb, KernelDataType::BFloat16, args).expect("encode fused router+topk");
cb.commit();
cb.wait_until_completed();
let ids_ptr = ids_buf.contents().as_ptr() as *const i32;
let probs_ptr = probs_buf.contents().as_ptr() as *const bf16;
- let ids_gpu =
- unsafe { std::slice::from_raw_parts(ids_ptr, t * k) }.to_vec();
- let probs_bf16_gpu =
- unsafe { std::slice::from_raw_parts(probs_ptr, t * k) }.to_vec();
+ let ids_gpu = unsafe { std::slice::from_raw_parts(ids_ptr, t * k) }.to_vec();
+ let probs_bf16_gpu = unsafe { std::slice::from_raw_parts(probs_ptr, t * k) }.to_vec();
// Convert bf16 to f32 for comparison
- let probs_gpu: Vec<f32> =
- probs_bf16_gpu.iter().map(|&h| f32::from(h)).collect();
+ let probs_gpu: Vec<f32> = probs_bf16_gpu.iter().map(|&h| f32::from(h)).collect();
assert_eq!(
ids_gpu, ids_ref,
@@ -239,13 +221,8 @@ fn test_router_topk_fused_matches_reference() {
let ctx = create_ctx();
let kernel = MoeRouterTopKKernel::new(&ctx).expect("kernel");
- let configs = [
- (1usize, 64usize, 32usize, 4usize),
- (2, 128, 64, 8),
- (4, 256, 128, 16),
- (8, 256, 256, 32),
- (1, 512, 512, 64),
- ];
+ let configs =
+ [(1usize, 64usize, 32usize, 4usize), (2, 128, 64, 8), (4, 256, 128, 16), (8, 256, 256, 32), (1, 512, 512, 64)];
let renorm_options = [false, true];
for &(t, d_model, e, k) in &configs {
diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rs
index a964fc5a3..8cfe18237 100644
--- a/crates/uzu/tests/moe/moe_scatter_test.rs
+++ b/crates/uzu/tests/moe/moe_scatter_test.rs
@@ -8,8 +8,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::backends::common::kernel::MoeCountsOffsetsFusedKernel;
use uzu::backends::metal::kernel::dsl::MoeCountsOffsetsFusedMetalKernel;
use uzu::backends::metal::kernel::{
- KernelDataType, MoeBlockBasesArguments, MoeScatterArguments,
- MoeScatterKernels,
+ KernelDataType, MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels,
moe::{MoeRouterTopKArguments, MoeRouterTopKKernel},
};
@@ -61,19 +60,13 @@ fn test_scatter_buckets_parity() {
// Generate random input and router weights for fused kernel
let d_model = 64;
- let input_f32: Vec<f32> =
- (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
- let weight_f32: Vec<f32> =
- (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
- let bias_f32: Vec<f32> =
- (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();
-
- let input: Vec<bf16> =
- input_f32.iter().map(|&x| bf16::from_f32(x)).collect();
- let weight: Vec<bf16> =
- weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();
- let bias: Vec<bf16> =
- bias_f32.iter().map(|&x| bf16::from_f32(x)).collect();
+ let input_f32: Vec<f32> = (0..t * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
+ let weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-1.0..1.0)).collect();
+ let bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.5..0.5)).collect();
+
+ let input: Vec<bf16> = input_f32.iter().map(|&x| bf16::from_f32(x)).collect();
+ let weight: Vec<bf16> = weight_f32.iter().map(|&x| bf16::from_f32(x)).collect();
+ let bias: Vec<bf16> = bias_f32.iter().map(|&x| bf16::from_f32(x)).collect();
let input_buf = alloc_buffer_with_data(&ctx, &input);
let weight_buf = alloc_buffer_with_data(&ctx, &weight);
@@ -83,10 +76,7 @@ fn test_scatter_buckets_parity() {
// Use fused router+topk kernel
let router_topk = MoeRouterTopKKernel::new(&ctx).expect("router_topk");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
router_topk
.encode(
&cb,
@@ -109,32 +99,18 @@ fn test_scatter_buckets_parity() {
cb.wait_until_completed();
// Read back CPU probs for reference compare
- let probs_bf16 = unsafe {
- std::slice::from_raw_parts(
- topk_probs_buf.contents().as_ptr() as *const bf16,
- t * k,
- )
- };
- let topk_probs_cpu: Vec<f32> =
- probs_bf16.iter().map(|&h| f32::from(h)).collect();
- let topk_ids_cpu = unsafe {
- std::slice::from_raw_parts(
- topk_ids_buf.contents().as_ptr() as *const i32,
- t * k,
- )
- };
+ let probs_bf16 =
+ unsafe { std::slice::from_raw_parts(topk_probs_buf.contents().as_ptr() as *const bf16, t * k) };
+ let topk_probs_cpu: Vec<f32> = probs_bf16.iter().map(|&h| f32::from(h)).collect();
+ let topk_ids_cpu = unsafe { std::slice::from_raw_parts(topk_ids_buf.contents().as_ptr() as *const i32, t * k) };
let offsets_buf = alloc_buffer::<u32>(&ctx, e + 1);
let sumk_buf = alloc_buffer::<u32>(&ctx, 1);
let num_tiles = ((e + 511) / 512).max(1);
let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);
- let fused_kernel =
- MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let fused_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
fused_kernel.encode(
&topk_ids_buf,
@@ -157,10 +133,7 @@ fn test_scatter_buckets_parity() {
let block_bases_buf = alloc_buffer::<u32>(&ctx, entries);
let scatter = MoeScatterKernels::new(&ctx).expect("scatter kernels");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);
scatter
.encode_block_bases(
@@ -178,19 +151,11 @@ fn test_scatter_buckets_parity() {
cb.commit();
cb.wait_until_completed();
- let sumk = unsafe {
- std::slice::from_raw_parts(
- sumk_buf.contents().as_ptr() as *const u32,
- 1,
- )
- }[0] as usize;
+ let sumk = unsafe { std::slice::from_raw_parts(sumk_buf.contents().as_ptr() as *const u32, 1) }[0] as usize;
let out_ids_buf = alloc_buffer::<i32>(&ctx, sumk);
let out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
scatter
.encode_scatter(
&cb,
@@ -214,30 +179,13 @@ fn test_scatter_buckets_parity() {
cb.commit();
cb.wait_until_completed();
- let out_ids = unsafe {
- std::slice::from_raw_parts(
- out_ids_buf.contents().as_ptr() as *const i32,
- sumk,
- )
- };
- let out_probs_h = unsafe {
- std::slice::from_raw_parts(
- out_probs_buf.contents().as_ptr() as *const bf16,
- sumk,
- )
- };
- let _out_probs: Vec<f32> =
- out_probs_h.iter().map(|&h| f32::from(h)).collect();
+ let out_ids = unsafe { std::slice::from_raw_parts(out_ids_buf.contents().as_ptr() as *const i32, sumk) };
+ let out_probs_h = unsafe { std::slice::from_raw_parts(out_probs_buf.contents().as_ptr() as *const bf16, sumk) };
+ let _out_probs: Vec<f32> = out_probs_h.iter().map(|&h| f32::from(h)).collect();
// CPU reference
- let (cpu_ids, _cpu_probs, offsets_cpu) =
- cpu_expert_buckets(topk_ids_cpu, &topk_probs_cpu, t, e, k);
- let offsets_gpu = unsafe {
- std::slice::from_raw_parts(
- offsets_buf.contents().as_ptr() as *const u32,
- e + 1,
- )
- };
+ let (cpu_ids, _cpu_probs, offsets_cpu) = cpu_expert_buckets(topk_ids_cpu, &topk_probs_cpu, t, e, k);
+ let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };
assert_eq!(offsets_gpu, &offsets_cpu[..]);
// Compare per-expert multisets of ids
diff --git a/crates/uzu/tests/moe/moe_test_utils.rs b/crates/uzu/tests/moe/moe_test_utils.rs
index f33b21708..b9cf882e8 100644
--- a/crates/uzu/tests/moe/moe_test_utils.rs
+++ b/crates/uzu/tests/moe/moe_test_utils.rs
@@ -30,10 +30,7 @@ pub fn alloc_buffer_with_data<T: bytemuck::NoUninit>(
ctx.create_buffer(1).expect("Failed to create empty buffer")
} else {
ctx.device
- .new_buffer_with_data(
- bytemuck::cast_slice(data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
}
}
diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rs
index 41a6d92a2..13afb70a1 100644
--- a/crates/uzu/tests/moe/moe_tiles_test.rs
+++ b/crates/uzu/tests/moe/moe_tiles_test.rs
@@ -2,14 +2,9 @@
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::metal::kernel::moe::tiles_map::{
- MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments,
-};
+use uzu::backends::metal::kernel::moe::tiles_map::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments};
-use super::test_utils::{
- alloc_buffer, alloc_buffer_with_data, cpu_tile_counts, cpu_tile_scan,
- create_ctx,
-};
+use super::test_utils::{alloc_buffer, alloc_buffer_with_data, cpu_tile_counts, cpu_tile_scan, create_ctx};
#[test]
fn test_tile_counts_correctness() {
@@ -36,12 +31,8 @@ fn test_tile_counts_correctness() {
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
// Execute kernel using kernel struct
- let tile_kernel =
- MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
&cb,
&MoeTileCountsArguments {
@@ -54,19 +45,10 @@ fn test_tile_counts_correctness() {
cb.wait_until_completed();
// Compare
- let tile_counts_gpu = unsafe {
- std::slice::from_raw_parts(
- tile_counts_buf.contents().as_ptr() as *const u32,
- e,
- )
- };
-
- assert_eq!(
- tile_counts_gpu,
- &tile_counts_cpu[..],
- "Tile counts mismatch for E={}",
- e
- );
+ let tile_counts_gpu =
+ unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };
+
+ assert_eq!(tile_counts_gpu, &tile_counts_cpu[..], "Tile counts mismatch for E={}", e);
eprintln!("[TileCountsTest] βœ“ PASSED");
}
@@ -84,8 +66,7 @@ fn test_tile_scan_correctness() {
eprintln!("[TileScanTest] E={}", e);
// Generate random tile counts
- let tile_counts: Vec<u32> =
- (0..e).map(|_| rng.random_range(0..20)).collect();
+ let tile_counts: Vec<u32> = (0..e).map(|_| rng.random_range(0..20)).collect();
let (tile_offsets_cpu, total_tiles_cpu) = cpu_tile_scan(&tile_counts);
@@ -95,12 +76,8 @@ fn test_tile_scan_correctness() {
let total_tiles_buf = alloc_buffer::<u32>(&ctx, 1);
// Execute kernel using kernel struct
- let tile_kernel =
- MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_scan(
&cb,
&MoeTileScanArguments {
@@ -114,26 +91,12 @@ fn test_tile_scan_correctness() {
cb.wait_until_completed();
// Compare
- let tile_offsets_gpu = unsafe {
- std::slice::from_raw_parts(
- tile_offsets_buf.contents().as_ptr() as *const u32,
- e + 1,
- )
- };
- let total_tiles_gpu =
- unsafe { *(total_tiles_buf.contents().as_ptr() as *const u32) };
-
- assert_eq!(
- tile_offsets_gpu,
- &tile_offsets_cpu[..],
- "Tile offsets mismatch for E={}",
- e
- );
- assert_eq!(
- total_tiles_gpu, total_tiles_cpu,
- "Total tiles mismatch for E={}",
- e
- );
+ let tile_offsets_gpu =
+ unsafe { std::slice::from_raw_parts(tile_offsets_buf.contents().as_ptr() as *const u32, e + 1) };
+ let total_tiles_gpu = unsafe { *(total_tiles_buf.contents().as_ptr() as *const u32) };
+
+ assert_eq!(tile_offsets_gpu, &tile_offsets_cpu[..], "Tile offsets mismatch for E={}", e);
+ assert_eq!(total_tiles_gpu, total_tiles_cpu, "Total tiles mismatch for E={}", e);
eprintln!("[TileScanTest] βœ“ PASSED");
}
@@ -153,12 +116,8 @@ fn test_tile_edge_cases() {
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
- let tile_kernel =
- MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
&cb,
&MoeTileCountsArguments {
@@ -170,12 +129,8 @@ fn test_tile_edge_cases() {
cb.commit();
cb.wait_until_completed();
- let tile_counts_gpu = unsafe {
- std::slice::from_raw_parts(
- tile_counts_buf.contents().as_ptr() as *const u32,
- e,
- )
- };
+ let tile_counts_gpu =
+ unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };
assert_eq!(tile_counts_gpu, &tile_counts_cpu[..]);
assert!(tile_counts_gpu.iter().all(|&c| c == 0));
@@ -192,12 +147,8 @@ fn test_tile_edge_cases() {
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
- let tile_kernel =
- MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
- let cb = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
+ let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
&cb,
&MoeTileCountsArguments {
@@ -209,12 +160,8 @@ fn test_tile_edge_cases() {
cb.commit();
cb.wait_until_completed();
- let tile_counts_gpu = unsafe {
- std::slice::from_raw_parts(
- tile_counts_buf.contents().as_ptr() as *const u32,
- e,
- )
- };
+ let tile_counts_gpu =
+ unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };
assert_eq!(tile_counts_gpu, &tile_counts_cpu[..]);
// With BM=16: seg_lens=[10, 5, 15, 20] -> tile_counts=[1, 1, 1, 2]
diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rs
index 6fb1b2b48..f3d877a36 100644
--- a/crates/uzu/tests/parameters_loader_test.rs
+++ b/crates/uzu/tests/parameters_loader_test.rs
@@ -12,13 +12,10 @@ use uzu::{
fn test_parameter_loader_basic() {
let weights_path = crate::common::get_test_weights_path();
let context = MTLContext::new().expect("Failed to create MTLContext");
- let file = std::fs::File::open(&weights_path)
- .expect("Weights file not found; run download script");
+ let file = std::fs::File::open(&weights_path).expect("Weights file not found; run download script");
- let loader =
- ParameterLoader::new(&file, context.as_ref()).expect("create loader");
- let embeddings =
- loader.get("embedding.weights").expect("weights embeddings");
+ let loader = ParameterLoader::new(&file, context.as_ref()).expect("create loader");
+ let embeddings = loader.get("embedding.weights").expect("weights embeddings");
let view = embeddings.as_view::<bf16>();
assert!(is_close!(view[[5usize, 3usize]], bf16::from_f32(-0.01819)));
diff --git a/crates/uzu/tests/person_schema_metal_bitmask_test.rs b/crates/uzu/tests/person_schema_metal_bitmask_test.rs
index 4b045991c..e3454fe5b 100644
--- a/crates/uzu/tests/person_schema_metal_bitmask_test.rs
+++ b/crates/uzu/tests/person_schema_metal_bitmask_test.rs
@@ -7,10 +7,7 @@ use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use tokenizers::Tokenizer;
use uzu::{DataType, backends::metal::MetalArray};
-use xgrammar::{
- DLDevice, DLDeviceType, DLTensor, Grammar, GrammarCompiler, GrammarMatcher,
- TokenizerInfo,
-};
+use xgrammar::{DLDevice, DLDeviceType, DLTensor, Grammar, GrammarCompiler, GrammarMatcher, TokenizerInfo};
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
struct Person {
@@ -24,10 +21,7 @@ fn person_schema_metal_bitmask() {
let crate_version = env!("CARGO_PKG_VERSION");
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let repo_root = manifest_dir.parent().unwrap().parent().unwrap();
- let model_dir = repo_root
- .join("models")
- .join(crate_version)
- .join("Llama-3.2-1B-Instruct");
+ let model_dir = repo_root.join("models").join(crate_version).join("Llama-3.2-1B-Instruct");
let tokenizer_path = model_dir.join("tokenizer.json");
if !tokenizer_path.exists() {
@@ -39,26 +33,13 @@ fn person_schema_metal_bitmask() {
}
let schema_root = schemars::schema_for!(Person);
- let schema_str =
- serde_json::to_string(&schema_root).expect("schema to string");
+ let schema_str = serde_json::to_string(&schema_root).expect("schema to string");
- let tokenizer =
- Tokenizer::from_file(&tokenizer_path).expect("load tokenizer.json");
- let tokenizer_info =
- TokenizerInfo::from_huggingface(&tokenizer, None, None).unwrap();
+ let tokenizer = Tokenizer::from_file(&tokenizer_path).expect("load tokenizer.json");
+ let tokenizer_info = TokenizerInfo::from_huggingface(&tokenizer, None, None).unwrap();
- let grammar = Grammar::from_json_schema(
- &schema_str,
- true,
- Some(2),
- Some((",", ":")),
- true,
- None,
- false,
- )
- .unwrap();
- let mut compiler =
- GrammarCompiler::new(&tokenizer_info, 8, true, -1).unwrap();
+ let grammar = Grammar::from_json_schema(&schema_str, true, Some(2), Some((",", ":")), true, None, false).unwrap();
+ let mut compiler = GrammarCompiler::new(&tokenizer_info, 8, true, -1).unwrap();
let compiled = compiler.compile_grammar(&grammar).unwrap();
let mut matcher = GrammarMatcher::new(&compiled, None, true, -1).unwrap();
@@ -80,11 +61,8 @@ fn person_schema_metal_bitmask() {
// xgrammar expects an int32 mask tensor of shape [buffer_size]
let elems = batch * buffer_size;
let bytes = elems * core::mem::size_of::<i32>();
- let buffer = device
- .new_buffer(bytes, MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let metal_bitmask =
- unsafe { MetalArray::from_parts(buffer, 0, &shape, DataType::I32) };
+ let buffer = device.new_buffer(bytes, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");
+ let metal_bitmask = unsafe { MetalArray::from_parts(buffer, 0, &shape, DataType::I32) };
let mut shape_i64 = [buffer_size as i64];
let mut bitmask_tensor = DLTensor {
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
index 8ea1e3b1c..dab71fd17 100644
--- a/crates/uzu/tests/quant_matmul_test.rs
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -2,20 +2,14 @@ use std::time::Instant;
use bytemuck;
use half::{bf16, f16};
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,
- MTLDeviceExt, MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use uzu::{
DataType,
backends::{
common::Context,
metal::{
MTLContext, ProtocolObject, Retained,
- kernel::quant_matmul::{
- QuantizationType, QuantizedMatmulArguments,
- QuantizedMatmulKernel,
- },
+ kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},
},
},
config::QuantizationMode,
@@ -141,8 +135,7 @@ fn get_zp_value(
((byte >> 4) & 0x0F) as f32
}
} else {
- zero_points.get(row_idx * stride + group_idx).copied().unwrap_or(0)
- as f32
+ zero_points.get(row_idx * stride + group_idx).copied().unwrap_or(0) as f32
}
}
@@ -191,19 +184,12 @@ fn cpu_reference(
let val_a = a[i * input_dim + l];
let scale = scales[l * num_groups + group_idx];
- let bias =
- if quantization_type == QuantizationType::ZeroPoint {
- let zp_val_qvm = get_zp_value(
- zero_points,
- zero_points_stride,
- l,
- group_idx,
- bits,
- );
- -scale * zp_val_qvm
- } else {
- biases[l * num_groups + group_idx]
- };
+ let bias = if quantization_type == QuantizationType::ZeroPoint {
+ let zp_val_qvm = get_zp_value(zero_points, zero_points_stride, l, group_idx, bits);
+ -scale * zp_val_qvm
+ } else {
+ biases[l * num_groups + group_idx]
+ };
acc += val_a * (scale * val_q + bias);
}
} else {
@@ -211,13 +197,7 @@ fn cpu_reference(
let scale = scales[j * num_groups + g];
let bias = match quantization_type {
QuantizationType::ZeroPoint => {
- let zp = get_zp_value(
- zero_points,
- zero_points_stride,
- j,
- g,
- bits,
- );
+ let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);
-scale * zp
},
QuantizationType::Mlx => biases[j * num_groups + g],
@@ -315,13 +295,9 @@ fn generate_test_quant_params(
if bits == 4 {
let byte_index = k * zero_points_stride + (g >> 1);
if (g & 1) == 0 {
- zero_points[byte_index] = (zero_points[byte_index]
- & 0xF0)
- | (zp_val_u8 & 0x0F);
+ zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val_u8 & 0x0F);
} else {
- zero_points[byte_index] = (zero_points[byte_index]
- & 0x0F)
- | ((zp_val_u8 & 0x0F) << 4);
+ zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val_u8 & 0x0F) << 4);
}
} else {
zero_points[k * zero_points_stride + g] = zp_val_u8;
@@ -338,8 +314,7 @@ fn generate_test_quant_params(
} else {
zp_val_u8 as f32
};
- biases[k * num_groups + g] =
- quantize_value(-s * zp_val, data_type);
+ biases[k * num_groups + g] = quantize_value(-s * zp_val, data_type);
}
}
} else {
@@ -359,13 +334,9 @@ fn generate_test_quant_params(
if bits == 4 {
let byte_index = j * zero_points_stride + (g >> 1);
if (g & 1) == 0 {
- zero_points[byte_index] = (zero_points[byte_index]
- & 0xF0)
- | (zp_val & 0x0F);
+ zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val & 0x0F);
} else {
- zero_points[byte_index] = (zero_points[byte_index]
- & 0x0F)
- | ((zp_val & 0x0F) << 4);
+ zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);
}
} else {
let byte_index = j * zero_points_stride + g;
@@ -386,8 +357,7 @@ fn generate_test_quant_params(
for g in 0..num_groups {
let base_val = g * 3;
let bias_val = (base_val % 19) as f32 * 0.125;
- biases[k * num_groups + g] =
- quantize_value(bias_val, data_type);
+ biases[k * num_groups + g] = quantize_value(bias_val, data_type);
}
}
} else {
@@ -399,8 +369,7 @@ fn generate_test_quant_params(
g * 3
};
let bias_val = (base_val % 19) as f32 * 0.125;
- biases[j * num_groups + g] =
- quantize_value(bias_val, data_type);
+ biases[j * num_groups + g] = quantize_value(bias_val, data_type);
}
}
}
@@ -422,31 +391,20 @@ fn buffer_from_f32_slice(
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
match dtype {
DataType::F16 => {
- let data: Vec<f16> =
- values.iter().map(|&v| f16::from_f32(v)).collect();
+ let data: Vec<f16> = values.iter().map(|&v| f16::from_f32(v)).collect();
ctx.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
},
DataType::BF16 => {
- let data: Vec<bf16> =
- values.iter().map(|&v| bf16::from_f32(v)).collect();
+ let data: Vec<bf16> = values.iter().map(|&v| bf16::from_f32(v)).collect();
ctx.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer")
},
DataType::F32 => ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(values),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(values), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer"),
other => {
panic!("Unsupported dtype for buffer_from_f32_slice: {:?}", other)
@@ -468,8 +426,7 @@ fn execute_quantized_matmul(
data_type: DataType,
bits: usize,
) -> ExecutionResult {
- let weights_quant =
- create_test_weights(output_dim, input_dim, weights_transposed, bits);
+ let weights_quant = create_test_weights(output_dim, input_dim, weights_transposed, bits);
let weights_packed = if bits == 4 {
pack_u4_weights(&weights_quant)
} else {
@@ -503,29 +460,19 @@ fn execute_quantized_matmul(
let w_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&weights_packed),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&weights_packed), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let s_buf = buffer_from_f32_slice(ctx, data_type, &params.scales);
let b_buf = match quantization_type {
QuantizationType::ZeroPoint => ctx
.device
- .new_buffer_with_data(
- &params.zero_points,
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(&params.zero_points, MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer"),
- QuantizationType::Mlx => {
- buffer_from_f32_slice(ctx, data_type, &params.biases)
- },
+ QuantizationType::Mlx => buffer_from_f32_slice(ctx, data_type, &params.biases),
};
let x_buf = buffer_from_f32_slice(ctx, data_type, &x_f32);
- let y_buf = ctx
- .create_buffer(batch * output_dim * data_type.size_in_bytes())
- .expect("Failed to create buffer");
+ let y_buf = ctx.create_buffer(batch * output_dim * data_type.size_in_bytes()).expect("Failed to create buffer");
let kernel = QuantizedMatmulKernel::new(
&ctx,
@@ -557,13 +504,8 @@ fn execute_quantized_matmul(
output_dim: output_dim as i32,
quantization_type,
};
- let cb_ref = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let encoder = cb_ref
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(&encoder, args).unwrap();
encoder.end_encoding();
cb_ref.commit();
@@ -585,13 +527,8 @@ fn execute_quantized_matmul(
output_dim: output_dim as i32,
quantization_type,
};
- let cb_ref = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let encoder = cb_ref
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(&encoder, args).unwrap();
encoder.end_encoding();
cb_ref.commit();
@@ -620,23 +557,17 @@ fn execute_quantized_matmul(
let y_out_f32: Vec<f32> = match data_type {
DataType::F16 => {
let y_ptr = y_buf.contents().as_ptr() as *const f16;
- let y_out = unsafe {
- std::slice::from_raw_parts(y_ptr, batch * output_dim)
- };
+ let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
y_out.iter().map(|&v| v.to_f32()).collect()
},
DataType::BF16 => {
let y_ptr = y_buf.contents().as_ptr() as *const bf16;
- let y_out = unsafe {
- std::slice::from_raw_parts(y_ptr, batch * output_dim)
- };
+ let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
y_out.iter().map(|&v| v.to_f32()).collect()
},
DataType::F32 => {
let y_ptr = y_buf.contents().as_ptr() as *const f32;
- let y_out = unsafe {
- std::slice::from_raw_parts(y_ptr, batch * output_dim)
- };
+ let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
y_out.to_vec()
},
other => panic!("Unsupported dtype for validation: {:?}", other),
@@ -644,17 +575,12 @@ fn execute_quantized_matmul(
let mut debug_prints = 0;
- for (i, (&exp, &got)) in
- y_expected.iter().zip(y_out_f32.iter()).enumerate()
- {
+ for (i, (&exp, &got)) in y_expected.iter().zip(y_out_f32.iter()).enumerate() {
let diff = (exp - got).abs();
if check_tolerance(exp, got) {
if debug_prints < 16 {
- println!(
- "\n detail idx {} diff {} exp {} got {}",
- i, diff, exp, got
- );
+ println!("\n detail idx {} diff {} exp {} got {}", i, diff, exp, got);
}
debug_prints += 1;
}
@@ -676,16 +602,9 @@ fn execute_quantized_matmul(
.last()
.map(|(i, _)| i)
.unwrap_or(0);
- println!(
- "\nTotal errors: {} out of {} outputs",
- debug_prints,
- batch * output_dim
- );
+ println!("\nTotal errors: {} out of {} outputs", debug_prints, batch * output_dim);
println!("Error range: indices {}-{}", first_error, last_error);
- println!(
- "Dims batch={} output_dim={} input_dim={}",
- batch, output_dim, input_dim
- );
+ println!("Dims batch={} output_dim={} input_dim={}", batch, output_dim, input_dim);
panic!("Validation failed with {} mismatches", debug_prints);
}
@@ -708,8 +627,7 @@ struct TestConfig {
const QMV_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];
const QVM_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];
-const QMM_DIMS: &[(usize, usize, usize)] =
- &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];
+const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];
fn run_kernel_test(
ctx: &MTLContext,
@@ -778,9 +696,7 @@ fn test_quant_gmv() {
for config in &configs {
for &(output_dim, input_dim) in QMV_DIMS {
- run_kernel_test(
- &ctx, 1, output_dim, input_dim, true, config, true, 1,
- );
+ run_kernel_test(&ctx, 1, output_dim, input_dim, true, config, true, 1);
}
}
}
@@ -824,9 +740,7 @@ fn test_quant_qvm() {
for config in &configs {
for &(output_dim, input_dim) in QVM_DIMS {
- run_kernel_test(
- &ctx, 1, output_dim, input_dim, false, config, true, 1,
- );
+ run_kernel_test(&ctx, 1, output_dim, input_dim, false, config, true, 1);
}
}
}
@@ -870,9 +784,7 @@ fn test_quant_gmm() {
for config in &configs {
for &(batch, output_dim, input_dim) in QMM_DIMS {
- run_kernel_test(
- &ctx, batch, output_dim, input_dim, false, config, true, 1,
- );
+ run_kernel_test(&ctx, batch, output_dim, input_dim, false, config, true, 1);
}
}
}
@@ -916,9 +828,7 @@ fn test_quant_gmm_transposed() {
for config in &configs {
for &(batch, output_dim, input_dim) in QMM_DIMS {
- run_kernel_test(
- &ctx, batch, output_dim, input_dim, true, config, true, 1,
- );
+ run_kernel_test(&ctx, batch, output_dim, input_dim, true, config, true, 1);
}
}
}
@@ -983,9 +893,7 @@ fn test_quant_matmul_perf() {
for config in &configs {
for &(batch, output_dim, input_dim) in &shapes {
- let result = run_kernel_test(
- &ctx, batch, output_dim, input_dim, false, config, false, 20,
- );
+ let result = run_kernel_test(&ctx, batch, output_dim, input_dim, false, config, false, 20);
let avg = result.elapsed / 20.0 * 1000.0; // ms
println!(
diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rs
index c4225ad4b..40c9e8dd3 100644
--- a/crates/uzu/tests/rms_norm_kernel_test.rs
+++ b/crates/uzu/tests/rms_norm_kernel_test.rs
@@ -3,29 +3,21 @@
use bytemuck;
use half::{bf16, f16};
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,
- MTLDeviceExt, MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use uzu::{
DataType,
backends::{
common::Context,
metal::{
MTLContext,
- kernel::rms_norm::{
- QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormKernel,
- RMSNormKernelType,
- },
+ kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormKernel, RMSNormKernelType},
metal_extensions::CommandBufferTimingExt,
},
},
};
// Helper trait to unify different float types for testing
-trait TestFloat:
- Copy + Clone + std::fmt::Debug + PartialEq + bytemuck::NoUninit
-{
+trait TestFloat: Copy + Clone + std::fmt::Debug + PartialEq + bytemuck::NoUninit {
fn from_f32(val: f32) -> Self;
fn to_f32(self) -> f32;
fn size_of() -> usize;
@@ -92,47 +84,26 @@ fn compute_expected_rms_norm(
DataType::F16 => {
// Convert inputs to their actual types, then to F16 for accumulation
let input_f16: Vec<f16> = match input_type {
- DataType::F32 => {
- input_data_f32.iter().map(|&x| f16::from_f32(x)).collect()
- },
- DataType::F16 => input_data_f32
- .iter()
- .map(|&x| f16::from_f32(f16::from_f32(x).to_f32()))
- .collect(),
- DataType::BF16 => input_data_f32
- .iter()
- .map(|&x| f16::from_f32(bf16::from_f32(x).to_f32()))
- .collect(),
+ DataType::F32 => input_data_f32.iter().map(|&x| f16::from_f32(x)).collect(),
+ DataType::F16 => input_data_f32.iter().map(|&x| f16::from_f32(f16::from_f32(x).to_f32())).collect(),
+ DataType::BF16 => input_data_f32.iter().map(|&x| f16::from_f32(bf16::from_f32(x).to_f32())).collect(),
_ => panic!("Unsupported input type: {:?}", input_type),
};
let scale_f16: Vec<f16> = match scale_type {
- DataType::F32 => {
- scale_data_f32.iter().map(|&x| f16::from_f32(x)).collect()
- },
- DataType::F16 => scale_data_f32
- .iter()
- .map(|&x| f16::from_f32(f16::from_f32(x).to_f32()))
- .collect(),
- DataType::BF16 => scale_data_f32
- .iter()
- .map(|&x| f16::from_f32(bf16::from_f32(x).to_f32()))
- .collect(),
+ DataType::F32 => scale_data_f32.iter().map(|&x| f16::from_f32(x)).collect(),
+ DataType::F16 => scale_data_f32.iter().map(|&x| f16::from_f32(f16::from_f32(x).to_f32())).collect(),
+ DataType::BF16 => scale_data_f32.iter().map(|&x| f16::from_f32(bf16::from_f32(x).to_f32())).collect(),
_ => panic!("Unsupported scale type: {:?}", scale_type),
};
let epsilon_f16 = f16::from_f32(epsilon);
let scale_offset_f16 = f16::from_f32(scale_offset);
// Compute mean square in F16 (matches Metal kernel logic)
- let mean_square_f16 = input_f16
- .iter()
- .map(|&x| x * x)
- .fold(f16::ZERO, |acc, x| acc + x)
+ let mean_square_f16 = input_f16.iter().map(|&x| x * x).fold(f16::ZERO, |acc, x| acc + x)
/ f16::from_f32(input_f16.len() as f32);
// Use rsqrt like the Metal kernel (inverse square root)
- let rms_norm_f16 = f16::from_f32(
- (mean_square_f16 + epsilon_f16).to_f32().sqrt().recip(),
- );
+ let rms_norm_f16 = f16::from_f32((mean_square_f16 + epsilon_f16).to_f32().sqrt().recip());
// Apply normalization and scaling based on mode (matches Metal kernel exactly)
if full_layer_mode {
@@ -164,47 +135,26 @@ fn compute_expected_rms_norm(
DataType::BF16 => {
// Convert inputs to their actual types, then to BF16 for accumulation
let input_bf16: Vec<bf16> = match input_type {
- DataType::F32 => {
- input_data_f32.iter().map(|&x| bf16::from_f32(x)).collect()
- },
- DataType::F16 => input_data_f32
- .iter()
- .map(|&x| bf16::from_f32(f16::from_f32(x).to_f32()))
- .collect(),
- DataType::BF16 => input_data_f32
- .iter()
- .map(|&x| bf16::from_f32(bf16::from_f32(x).to_f32()))
- .collect(),
+ DataType::F32 => input_data_f32.iter().map(|&x| bf16::from_f32(x)).collect(),
+ DataType::F16 => input_data_f32.iter().map(|&x| bf16::from_f32(f16::from_f32(x).to_f32())).collect(),
+ DataType::BF16 => input_data_f32.iter().map(|&x| bf16::from_f32(bf16::from_f32(x).to_f32())).collect(),
_ => panic!("Unsupported input type: {:?}", input_type),
};
let scale_bf16: Vec<bf16> = match scale_type {
- DataType::F32 => {
- scale_data_f32.iter().map(|&x| bf16::from_f32(x)).collect()
- },
- DataType::F16 => scale_data_f32
- .iter()
- .map(|&x| bf16::from_f32(f16::from_f32(x).to_f32()))
- .collect(),
- DataType::BF16 => scale_data_f32
- .iter()
- .map(|&x| bf16::from_f32(bf16::from_f32(x).to_f32()))
- .collect(),
+ DataType::F32 => scale_data_f32.iter().map(|&x| bf16::from_f32(x)).collect(),
+ DataType::F16 => scale_data_f32.iter().map(|&x| bf16::from_f32(f16::from_f32(x).to_f32())).collect(),
+ DataType::BF16 => scale_data_f32.iter().map(|&x| bf16::from_f32(bf16::from_f32(x).to_f32())).collect(),
_ => panic!("Unsupported scale type: {:?}", scale_type),
};
let epsilon_bf16 = bf16::from_f32(epsilon);
let scale_offset_bf16 = bf16::from_f32(scale_offset);
// Compute mean square in BF16 (matches Metal kernel logic)
- let mean_square_bf16 = input_bf16
- .iter()
- .map(|&x| x * x)
- .fold(bf16::ZERO, |acc, x| acc + x)
+ let mean_square_bf16 = input_bf16.iter().map(|&x| x * x).fold(bf16::ZERO, |acc, x| acc + x)
/ bf16::from_f32(input_bf16.len() as f32);
// Use rsqrt like the Metal kernel (inverse square root)
- let rms_norm_bf16 = bf16::from_f32(
- (mean_square_bf16 + epsilon_bf16).to_f32().sqrt().recip(),
- );
+ let rms_norm_bf16 = bf16::from_f32((mean_square_bf16 + epsilon_bf16).to_f32().sqrt().recip());
// Apply normalization and scaling based on mode (matches Metal kernel exactly)
if full_layer_mode {
@@ -237,33 +187,19 @@ fn compute_expected_rms_norm(
// Convert inputs to their actual types, then to F32 for accumulation
let input_f32: Vec<f32> = match input_type {
DataType::F32 => input_data_f32.to_vec(),
- DataType::F16 => input_data_f32
- .iter()
- .map(|&x| f16::from_f32(x).to_f32())
- .collect(),
- DataType::BF16 => input_data_f32
- .iter()
- .map(|&x| bf16::from_f32(x).to_f32())
- .collect(),
+ DataType::F16 => input_data_f32.iter().map(|&x| f16::from_f32(x).to_f32()).collect(),
+ DataType::BF16 => input_data_f32.iter().map(|&x| bf16::from_f32(x).to_f32()).collect(),
_ => panic!("Unsupported input type: {:?}", input_type),
};
let scale_f32: Vec<f32> = match scale_type {
DataType::F32 => scale_data_f32.to_vec(),
- DataType::F16 => scale_data_f32
- .iter()
- .map(|&x| f16::from_f32(x).to_f32())
- .collect(),
- DataType::BF16 => scale_data_f32
- .iter()
- .map(|&x| bf16::from_f32(x).to_f32())
- .collect(),
+ DataType::F16 => scale_data_f32.iter().map(|&x| f16::from_f32(x).to_f32()).collect(),
+ DataType::BF16 => scale_data_f32.iter().map(|&x| bf16::from_f32(x).to_f32()).collect(),
_ => panic!("Unsupported scale type: {:?}", scale_type),
};
// Compute mean square in F32 (matches Metal kernel logic)
- let mean_square: f32 =
- input_f32.iter().map(|&x| x * x).sum::<f32>()
- / (input_f32.len() as f32);
+ let mean_square: f32 = input_f32.iter().map(|&x| x * x).sum::<f32>() / (input_f32.len() as f32);
// Use rsqrt like the Metal kernel (inverse square root)
let rms_norm = (mean_square + epsilon).sqrt().recip();
@@ -327,17 +263,14 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
.map(|i| {
// Create a sine wave pattern with values roughly in [-2, 2] range
// This gives us reasonable values that work well with F16 precision
- 2.0 * f32::sin(i as f32 * 0.01)
- + 0.1 * (i as f32 / model_dim as f32)
+ 2.0 * f32::sin(i as f32 * 0.01) + 0.1 * (i as f32 / model_dim as f32)
})
.collect();
let scale_data_f32: Vec<f32> = vec![1.0; model_dim as usize]; // Unit scales
// Convert to target types
- let input_data: Vec<InputT> =
- input_data_f32.iter().map(|&x| InputT::from_f32(x)).collect();
- let scale_data: Vec<ScaleT> =
- scale_data_f32.iter().map(|&x| ScaleT::from_f32(x)).collect();
+ let input_data: Vec<InputT> = input_data_f32.iter().map(|&x| InputT::from_f32(x)).collect();
+ let scale_data: Vec<ScaleT> = scale_data_f32.iter().map(|&x| ScaleT::from_f32(x)).collect();
// Calculate expected output
let expected_output_f32 = compute_expected_rms_norm(
@@ -354,26 +287,17 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
// Create Metal buffers
let input_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&input_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&input_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let scale_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&scale_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buffer = mtl_context
.device
- .new_buffer(
- model_dim as usize * OutputT::size_of(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(model_dim as usize * OutputT::size_of(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
// Create RMS norm kernel
@@ -388,14 +312,9 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
.expect("Failed to create RMS norm kernel");
// Create command buffer and encode
- let command_buffer_ref = mtl_context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let _ = kernel.encode(
&compute_encoder,
@@ -418,10 +337,8 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
// Read results
let output_ptr = output_buffer.contents().as_ptr() as *const OutputT;
- let output_data =
- unsafe { std::slice::from_raw_parts(output_ptr, model_dim as usize) };
- let output_data_f32: Vec<f32> =
- output_data.iter().map(|&x| x.to_f32()).collect();
+ let output_data = unsafe { std::slice::from_raw_parts(output_ptr, model_dim as usize) };
+ let output_data_f32: Vec<f32> = output_data.iter().map(|&x| x.to_f32()).collect();
println!(
"Types: Input={:?}, Scale={:?}, Output={:?}, Accum={:?}",
@@ -432,11 +349,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
println!("Output (first 8): {:?}", &output_data_f32[..8]);
println!(
"RMS value: {}",
- f32::sqrt(
- input_data_f32.iter().map(|&x| x * x).sum::<f32>()
- / (model_dim as f32)
- + epsilon
- )
+ f32::sqrt(input_data_f32.iter().map(|&x| x * x).sum::<f32>() / (model_dim as f32) + epsilon)
);
// Verify results with tolerance for different precisions
@@ -450,9 +363,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
1e-5
};
- for (i, (&expected, &actual)) in
- expected_output_f32.iter().zip(output_data_f32.iter()).enumerate()
- {
+ for (i, (&expected, &actual)) in expected_output_f32.iter().zip(output_data_f32.iter()).enumerate() {
let diff = (expected - actual).abs();
assert!(
diff < tolerance,
@@ -494,33 +405,22 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
let input_data_f32: Vec<f32> = vec![1e-4, 2e-4, 3e-4, 4e-4]; // Scaled up for F16 precision
let scale_data_f32: Vec<f32> = vec![2.0, 0.5, 1.5, 0.8]; // Non-unit scales
- let input_data: Vec<InputT> =
- input_data_f32.iter().map(|&x| InputT::from_f32(x)).collect();
- let scale_data: Vec<ScaleT> =
- scale_data_f32.iter().map(|&x| ScaleT::from_f32(x)).collect();
+ let input_data: Vec<InputT> = input_data_f32.iter().map(|&x| InputT::from_f32(x)).collect();
+ let scale_data: Vec<ScaleT> = scale_data_f32.iter().map(|&x| ScaleT::from_f32(x)).collect();
let input_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&input_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&input_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let scale_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&scale_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buffer = mtl_context
.device
- .new_buffer(
- 4 * OutputT::size_of(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(4 * OutputT::size_of(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let kernel = RMSNormKernel::new(
@@ -533,14 +433,9 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
)
.expect("Failed to create RMS norm kernel");
- let command_buffer_ref = mtl_context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let _ = kernel.encode(
&compute_encoder,
@@ -567,13 +462,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
// Check that all values are finite (no NaN or infinity)
for (i, &value) in output_data.iter().enumerate() {
let f32_value = value.to_f32();
- assert!(
- f32_value.is_finite(),
- "Output at index {} is not finite: {} (type {:?})",
- i,
- f32_value,
- output_type
- );
+ assert!(f32_value.is_finite(), "Output at index {} is not finite: {} (type {:?})", i, f32_value, output_type);
}
println!(
@@ -585,237 +474,112 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
// All F32 accumulation tests
#[test]
fn test_rms_norm_f32_f32_f32_f32() {
- test_rms_norm_basic_typed::<f32, f32, f32>(
- DataType::F32,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_f32_f16_f32_f32() {
- test_rms_norm_basic_typed::<f32, f16, f32>(
- DataType::F32,
- DataType::F16,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f32, f16, f32>(DataType::F32, DataType::F16, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_f32_f32_f16_f32() {
- test_rms_norm_basic_typed::<f32, f32, f16>(
- DataType::F32,
- DataType::F32,
- DataType::F16,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f32, f32, f16>(DataType::F32, DataType::F32, DataType::F16, DataType::F32);
}
#[test]
fn test_rms_norm_f32_f16_f16_f32() {
- test_rms_norm_basic_typed::<f32, f16, f16>(
- DataType::F32,
- DataType::F16,
- DataType::F16,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f32, f16, f16>(DataType::F32, DataType::F16, DataType::F16, DataType::F32);
}
#[test]
fn test_rms_norm_f16_f32_f32_f32() {
- test_rms_norm_basic_typed::<f16, f32, f32>(
- DataType::F16,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f16, f32, f32>(DataType::F16, DataType::F32, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_f16_f16_f32_f32() {
- test_rms_norm_basic_typed::<f16, f16, f32>(
- DataType::F16,
- DataType::F16,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f16, f16, f32>(DataType::F16, DataType::F16, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_f16_f32_f16_f32() {
- test_rms_norm_basic_typed::<f16, f32, f16>(
- DataType::F16,
- DataType::F32,
- DataType::F16,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f16, f32, f16>(DataType::F16, DataType::F32, DataType::F16, DataType::F32);
}
#[test]
fn test_rms_norm_f16_f16_f16_f32() {
- test_rms_norm_basic_typed::<f16, f16, f16>(
- DataType::F16,
- DataType::F16,
- DataType::F16,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f16, f16, f16>(DataType::F16, DataType::F16, DataType::F16, DataType::F32);
}
// F16 accumulation tests
#[test]
fn test_rms_norm_f32_f32_f32_f16() {
- test_rms_norm_basic_typed::<f32, f32, f32>(
- DataType::F32,
- DataType::F32,
- DataType::F32,
- DataType::F16,
- );
+ test_rms_norm_basic_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F16);
}
#[test]
fn test_rms_norm_f32_f16_f32_f16() {
- test_rms_norm_basic_typed::<f32, f16, f32>(
- DataType::F32,
- DataType::F16,
- DataType::F32,
- DataType::F16,
- );
+ test_rms_norm_basic_typed::<f32, f16, f32>(DataType::F32, DataType::F16, DataType::F32, DataType::F16);
}
#[test]
fn test_rms_norm_f32_f32_f16_f16() {
- test_rms_norm_basic_typed::<f32, f32, f16>(
- DataType::F32,
- DataType::F32,
- DataType::F16,
- DataType::F16,
- );
+ test_rms_norm_basic_typed::<f32, f32, f16>(DataType::F32, DataType::F32, DataType::F16, DataType::F16);
}
#[test]
fn test_rms_norm_f32_f16_f16_f16() {
- test_rms_norm_basic_typed::<f32, f16, f16>(
- DataType::F32,
- DataType::F16,
- DataType::F16,
- DataType::F16,
- );
+ test_rms_norm_basic_typed::<f32, f16, f16>(DataType::F32, DataType::F16, DataType::F16, DataType::F16);
}
#[test]
fn test_rms_norm_f16_f32_f32_f16() {
- test_rms_norm_basic_typed::<f16, f32, f32>(
- DataType::F16,
- DataType::F32,
- DataType::F32,
- DataType::F16,
- );
+ test_rms_norm_basic_typed::<f16, f32, f32>(DataType::F16, DataType::F32, DataType::F32, DataType::F16);
}
#[test]
fn test_rms_norm_f16_f16_f32_f16() {
- test_rms_norm_basic_typed::<f16, f16, f32>(
- DataType::F16,
- DataType::F16,
- DataType::F32,
- DataType::F16,
- );
+ test_rms_norm_basic_typed::<f16, f16, f32>(DataType::F16, DataType::F16, DataType::F32, DataType::F16);
}
#[test]
fn test_rms_norm_f16_f32_f16_f16() {
- test_rms_norm_basic_typed::<f16, f32, f16>(
- DataType::F16,
- DataType::F32,
- DataType::F16,
- DataType::F16,
- );
+ test_rms_norm_basic_typed::<f16, f32, f16>(DataType::F16, DataType::F32, DataType::F16, DataType::F16);
}
#[test]
fn test_rms_norm_f16_f16_f16_f16() {
- test_rms_norm_basic_typed::<f16, f16, f16>(
- DataType::F16,
- DataType::F16,
- DataType::F16,
- DataType::F16,
- );
+ test_rms_norm_basic_typed::<f16, f16, f16>(DataType::F16, DataType::F16, DataType::F16, DataType::F16);
}
// BFloat16 tests (using f32 storage)
#[test]
fn test_rms_norm_bf16_bf16_f32_f32() {
- test_rms_norm_basic_typed::<bf16, bf16, f32>(
- DataType::BF16,
- DataType::BF16,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<bf16, bf16, f32>(DataType::BF16, DataType::BF16, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_bf16_f32_f32_f32() {
- test_rms_norm_basic_typed::<bf16, f32, f32>(
- DataType::BF16,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<bf16, f32, f32>(DataType::BF16, DataType::F32, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_bf16_f16_f32_f32() {
- test_rms_norm_basic_typed::<bf16, f16, f32>(
- DataType::BF16,
- DataType::F16,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<bf16, f16, f32>(DataType::BF16, DataType::F16, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_f32_bf16_f32_f32() {
- test_rms_norm_basic_typed::<f32, bf16, f32>(
- DataType::F32,
- DataType::BF16,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f32, bf16, f32>(DataType::F32, DataType::BF16, DataType::F32, DataType::F32);
}
// Edge case tests for critical combinations
#[test]
fn test_rms_norm_edge_f32_f32_f32_f32() {
- test_rms_norm_edge_cases_typed::<f32, f32, f32>(
- DataType::F32,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_edge_cases_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_edge_f16_f16_f16_f16() {
- test_rms_norm_edge_cases_typed::<f16, f16, f16>(
- DataType::F16,
- DataType::F16,
- DataType::F16,
- DataType::F16,
- );
+ test_rms_norm_edge_cases_typed::<f16, f16, f16>(DataType::F16, DataType::F16, DataType::F16, DataType::F16);
}
#[test]
fn test_rms_norm_edge_f32_f16_f16_f32() {
- test_rms_norm_edge_cases_typed::<f32, f16, f16>(
- DataType::F32,
- DataType::F16,
- DataType::F16,
- DataType::F32,
- );
+ test_rms_norm_edge_cases_typed::<f32, f16, f16>(DataType::F32, DataType::F16, DataType::F16, DataType::F32);
}
// Legacy test wrappers for compatibility
#[test]
fn test_rms_norm_kernel_basic() {
- test_rms_norm_basic_typed::<f32, f32, f32>(
- DataType::F32,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_basic_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F32);
}
#[test]
fn test_rms_norm_edge_cases() {
- test_rms_norm_edge_cases_typed::<f32, f32, f32>(
- DataType::F32,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- );
+ test_rms_norm_edge_cases_typed::<f32, f32, f32>(DataType::F32, DataType::F32, DataType::F32, DataType::F32);
}
fn perf_rms_norm_with_size(
@@ -863,18 +627,12 @@ fn perf_rms_norm_with_size(
// ---- Create Metal buffers ----
let input_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&input_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&input_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let scale_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&scale_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buffer = mtl_context
@@ -886,14 +644,9 @@ fn perf_rms_norm_with_size(
.expect("Failed to create buffer");
// ---- Launch and time ----
- let command_buffer_ref = mtl_context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let _ = kernel.encode(
&compute_encoder,
@@ -938,39 +691,23 @@ fn perf_rms_norm_with_size(
// ---- Sanity check results ----
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_data = unsafe {
- std::slice::from_raw_parts(
- output_ptr,
- (batch_size * model_dim) as usize,
- )
- };
+ let output_data = unsafe { std::slice::from_raw_parts(output_ptr, (batch_size * model_dim) as usize) };
// Sample check for large outputs
let sample_size = std::cmp::min(1000, output_data.len());
- for i in (0..output_data.len())
- .step_by(std::cmp::max(1, output_data.len() / sample_size))
- {
- assert!(
- output_data[i].is_finite(),
- "Output at index {} is not finite: {}",
- i,
- output_data[i]
- );
+ for i in (0..output_data.len()).step_by(std::cmp::max(1, output_data.len() / sample_size)) {
+ assert!(output_data[i].is_finite(), "Output at index {} is not finite: {}", i, output_data[i]);
}
// Check that normalization is working
- let mean_abs =
- output_data.iter().take(sample_size).map(|x| x.abs()).sum::<f32>()
- / sample_size as f32;
+ let mean_abs = output_data.iter().take(sample_size).map(|x| x.abs()).sum::<f32>() / sample_size as f32;
assert!(
mean_abs > 0.01 && mean_abs < 100.0,
"Mean absolute value {} seems unreasonable - normalization may not be working",
mean_abs
);
- println!(
- "βœ… RMS norm performance test passed with reasonable output values"
- );
+ println!("βœ… RMS norm performance test passed with reasonable output values");
}
#[test]
@@ -1024,8 +761,7 @@ fn qk_norm_test() {
let k_offset = (num_q_heads * head_dim) as usize;
for head in 0..num_kv_heads {
for dim in 0..head_dim {
- let idx =
- batch_offset + k_offset + (head * head_dim + dim) as usize;
+ let idx = batch_offset + k_offset + (head * head_dim + dim) as usize;
qkv_data[idx] = 2.0 + (head as f32) * 0.1 + (dim as f32) * 0.01;
}
}
@@ -1034,8 +770,7 @@ fn qk_norm_test() {
let v_offset = ((num_q_heads + num_kv_heads) * head_dim) as usize;
for head in 0..num_kv_heads {
for dim in 0..head_dim {
- let idx =
- batch_offset + v_offset + (head * head_dim + dim) as usize;
+ let idx = batch_offset + v_offset + (head * head_dim + dim) as usize;
qkv_data[idx] = 3.0 + (head as f32) * 0.1 + (dim as f32) * 0.01;
}
}
@@ -1046,26 +781,17 @@ fn qk_norm_test() {
// Create buffers
let qkv_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&qkv_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&qkv_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let q_scales_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&scale_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let k_scales_buffer = mtl_context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&scale_data),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&scale_data), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
// Create QK norm kernels
@@ -1093,13 +819,8 @@ fn qk_norm_test() {
// Test Q head normalization
{
- let command_buffer = mtl_context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let _ = q_kernel.encode_qk_norm(
&compute_encoder,
@@ -1108,7 +829,7 @@ fn qk_norm_test() {
scales_buffer: &q_scales_buffer,
qkv_output_buffer: &qkv_buffer,
batch_size,
- num_q_heads, // Now correctly passes actual head count
+ num_q_heads, // Now correctly passes actual head count
num_kv_heads, // Now correctly passes actual head count
head_dim,
epsilon,
@@ -1123,12 +844,8 @@ fn qk_norm_test() {
}
// Verify Q heads were normalized (should have different values now)
- let output_data = unsafe {
- std::slice::from_raw_parts(
- qkv_buffer.contents().as_ptr() as *const f32,
- qkv_data.len(),
- )
- };
+ let output_data =
+ unsafe { std::slice::from_raw_parts(qkv_buffer.contents().as_ptr() as *const f32, qkv_data.len()) };
// Check that Q heads were processed (values should be different from original)
for head in 0..num_q_heads as usize {
@@ -1164,13 +881,8 @@ fn qk_norm_test() {
// Test K head normalization
{
- let command_buffer = mtl_context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let _ = k_kernel.encode_qk_norm(
&compute_encoder,
@@ -1179,7 +891,7 @@ fn qk_norm_test() {
scales_buffer: &k_scales_buffer,
qkv_output_buffer: &qkv_buffer,
batch_size,
- num_q_heads, // Now correctly passes actual head count
+ num_q_heads, // Now correctly passes actual head count
num_kv_heads, // Now correctly passes actual head count
head_dim,
epsilon,
@@ -1194,12 +906,8 @@ fn qk_norm_test() {
}
// Verify K heads were normalized
- let final_data = unsafe {
- std::slice::from_raw_parts(
- qkv_buffer.contents().as_ptr() as *const f32,
- qkv_data.len(),
- )
- };
+ let final_data =
+ unsafe { std::slice::from_raw_parts(qkv_buffer.contents().as_ptr() as *const f32, qkv_data.len()) };
// Check that K heads were processed
for head in 0..num_kv_heads as usize {
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index 2341f5594..14a27695b 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -1,9 +1,6 @@
mod common;
use bytemuck;
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,
- MTLDeviceExt, MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use rand::{RngExt, seq::SliceRandom};
// for Vec::shuffle
use uzu::{
@@ -12,16 +9,14 @@ use uzu::{
common::{
Context,
kernel::{
- GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel,
- TopPKernel,
+ GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel, TopPKernel,
sampling::{ArgmaxStrategy, SamplingKernel},
},
},
metal::{
KernelDataType, MTLContext, Metal,
kernel::dsl::{
- GumbelMetalKernel, MinPMetalKernel, TemperatureMetalKernel,
- TopKMetalKernel, TopPMetalKernel,
+ GumbelMetalKernel, MinPMetalKernel, TemperatureMetalKernel, TopKMetalKernel, TopPMetalKernel,
},
metal_extensions::CommandBufferTimingExt,
},
@@ -38,13 +33,9 @@ fn cpu_reference_top_p(
top_p: f32,
) -> Vec<f32> {
// 1. soft‑max (unnormalised)
- let max_logit =
- row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
- let mut softmax: Vec<(usize, f32)> = row_logits
- .iter()
- .enumerate()
- .map(|(idx, &z)| (idx, (z - max_logit).exp()))
- .collect();
+ let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+ let mut softmax: Vec<(usize, f32)> =
+ row_logits.iter().enumerate().map(|(idx, &z)| (idx, (z - max_logit).exp())).collect();
let total_mass: f32 = softmax.iter().map(|(_, p)| p).sum();
// 2. sort descending
@@ -74,8 +65,7 @@ fn cpu_reference_min_p(
row_logits: &[f32],
min_p: f32,
) -> Vec<f32> {
- let max_logit =
- row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+ let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let threshold = max_logit + min_p.ln();
row_logits
@@ -102,14 +92,8 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
let batch_size = 2;
let vocab_size = 4;
- let kernel = SamplingKernel::<Metal>::new_with_strategy(
- &context,
- DataType::F32,
- batch_size,
- vocab_size,
- strategy,
- )
- .expect("Failed to create argmax kernel");
+ let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, batch_size, vocab_size, strategy)
+ .expect("Failed to create argmax kernel");
// Create test data: batch_size=2, vocab_size=4
// First batch: [1.0, 3.0, 2.0, 0.5] -> should select index 1
@@ -121,30 +105,19 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
let logits_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&test_logits),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buffer = context
.device
- .new_buffer(
- batch_size * std::mem::size_of::<u32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
// Run sampling
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
&logits_buffer,
@@ -168,19 +141,10 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
let result_ptr = output_buffer.contents().as_ptr() as *const u32;
let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };
- assert_eq!(
- results[0], 1,
- "First batch should select token 1 (highest logit: 3.0)"
- );
- assert_eq!(
- results[1], 2,
- "Second batch should select token 2 (highest logit: 2.5)"
- );
-
- println!(
- "βœ“ Argmax sampling test passed with {:?} strategy - selected tokens: {:?}",
- strategy, results
- );
+ assert_eq!(results[0], 1, "First batch should select token 1 (highest logit: 3.0)");
+ assert_eq!(results[1], 2, "Second batch should select token 2 (highest logit: 2.5)");
+
+ println!("βœ“ Argmax sampling test passed with {:?} strategy - selected tokens: {:?}", strategy, results);
}
#[test]
@@ -220,19 +184,13 @@ fn test_topp_sampling_from_prob_exact_match(
let p = (k as f32) * 0.1;
// Create kernel
- let kernel = SamplingKernel::<Metal>::new(
- &context,
- DataType::F32,
- batch_size,
- vocab_size,
- )
- .expect("Failed to create sampling kernel");
+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)
+ .expect("Failed to create sampling kernel");
// Build probability table
let low_prob = (1.0 - p) / (vocab_size - k) as f32;
let mut probs = vec![low_prob; batch_size * vocab_size];
- let mut high_prob_token_sets: Vec<HashSet<usize>> =
- Vec::with_capacity(batch_size);
+ let mut high_prob_token_sets: Vec<HashSet<usize>> = Vec::with_capacity(batch_size);
// Use std::sample equivalent (Fisher-Yates shuffle + take first k)
let mut rng = StdRng::seed_from_u64(42); // Deterministic for reproducibility
@@ -241,8 +199,7 @@ fn test_topp_sampling_from_prob_exact_match(
for b in 0..batch_size {
// Equivalent to std::sample - shuffle and take first k
all_token_ids.shuffle(&mut rng);
- let high_prob_tokens: HashSet<usize> =
- all_token_ids[..k].iter().cloned().collect();
+ let high_prob_tokens: HashSet<usize> = all_token_ids[..k].iter().cloned().collect();
// Set selected tokens to high probability
for &token_id in &high_prob_tokens {
@@ -266,10 +223,7 @@ fn test_topp_sampling_from_prob_exact_match(
// GPU buffers
let output_buf = context
.device
- .new_buffer(
- batch_size * std::mem::size_of::<u32>(),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(batch_size * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
// Counter for samples
@@ -281,30 +235,18 @@ fn test_topp_sampling_from_prob_exact_match(
// Create fresh logits buffer since kernel mutates in-place
let logits_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let seeds: Vec<u64> =
- vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
let seeds_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let cb_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
&logits_buf,
@@ -329,14 +271,10 @@ fn test_topp_sampling_from_prob_exact_match(
cb_ref.wait_until_completed();
let ptr = output_buf.contents().as_ptr() as *const u32;
- let sampled_ids =
- unsafe { std::slice::from_raw_parts(ptr, batch_size) };
+ let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };
for (i, &sampled_id) in sampled_ids.iter().enumerate() {
- assert!(
- (sampled_id as usize) < vocab_size,
- "Sampled token out of range"
- );
+ assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");
counter[i * vocab_size + sampled_id as usize] += 1;
}
}
@@ -355,10 +293,7 @@ fn test_topp_sampling_from_prob_exact_match(
}
}
- println!(
- "batch_size: {}, p: {:.1}, vocab_size: {}, accuracy test passed.",
- batch_size, p, vocab_size
- );
+ println!("batch_size: {}, p: {:.1}, vocab_size: {}, accuracy test passed.", batch_size, p, vocab_size);
}
#[test]
@@ -393,8 +328,7 @@ fn test_topp_sampling_statistical_large() {
// ===== 3. Build kernel =====
let kernel =
- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB)
- .expect("Failed to create sampling kernel");
+ SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
// ===== 4. Generate reproducible random logits =====
let mut rng = StdRng::seed_from_u64(42);
@@ -415,10 +349,7 @@ fn test_topp_sampling_statistical_large() {
// ===== 6. Allocate output & counters =====
let output_buf = context
.device
- .new_buffer(
- BATCH * std::mem::size_of::<u32>(),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let mut counters = vec![0u32; BATCH * VOCAB];
@@ -427,29 +358,18 @@ fn test_topp_sampling_statistical_large() {
// Create fresh logits buffer since kernel mutates in-place
let logits_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];
let seeds_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let cb_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -490,25 +410,15 @@ fn test_topp_sampling_statistical_large() {
if expected < 1e-12 {
continue;
} // ignore tiny
- let observed =
- (counters[b * VOCAB + j] as f32) / (NUM_DRAWS as f32);
+ let observed = (counters[b * VOCAB + j] as f32) / (NUM_DRAWS as f32);
if observed > 0.0 {
kl += observed * (observed.ln() - expected.ln());
}
}
- assert!(
- kl < TOLERANCE_KL,
- "Row {} KL {:.4} exceeded tolerance {:.3}",
- b,
- kl,
- TOLERANCE_KL
- );
+ assert!(kl < TOLERANCE_KL, "Row {} KL {:.4} exceeded tolerance {:.3}", b, kl, TOLERANCE_KL);
}
- println!(
- "βœ“ Large statistical Top‑p test passed (KL < {:.3})",
- TOLERANCE_KL
- );
+ println!("βœ“ Large statistical Top‑p test passed (KL < {:.3})", TOLERANCE_KL);
}
#[test]
@@ -533,8 +443,7 @@ fn perf_topp_128k_vocab() {
// ---- Kernel ----
let kernel =
- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB)
- .expect("Failed to create sampling kernel");
+ SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
// ---- Build random logits ----
let mut rng = StdRng::seed_from_u64(123);
@@ -545,38 +454,24 @@ fn perf_topp_128k_vocab() {
let logits_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];
let seeds_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buf = context
.device
- .new_buffer(
- BATCH * std::mem::size_of::<u32>(),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
// ---- Launch once and time ----
- let cb_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -651,14 +546,8 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
const VOCAB: usize = 128000; // 128K
// ---- Kernel ----
- let kernel = SamplingKernel::<Metal>::new_with_strategy(
- &context,
- DataType::F32,
- BATCH,
- VOCAB,
- strategy,
- )
- .expect("Failed to create Argmax kernel");
+ let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, BATCH, VOCAB, strategy)
+ .expect("Failed to create Argmax kernel");
// ---- Build random logits ----
let mut rng = StdRng::seed_from_u64(123);
@@ -669,38 +558,24 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
let logits_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];
let seeds_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buf = context
.device
- .new_buffer(
- BATCH * std::mem::size_of::<u32>(),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
// ---- Launch once and time ----
- let cb_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -756,8 +631,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
let selected_logit = row_logits[selected_token];
// Find the actual maximum logit
- let max_logit =
- row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+ let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
assert!(
(selected_logit - max_logit).abs() < 1e-5,
@@ -785,13 +659,8 @@ fn test_categorical_sampling() {
let batch_size = 2;
let vocab_size = 4;
- let kernel = SamplingKernel::<Metal>::new(
- &context,
- DataType::F32,
- batch_size,
- vocab_size,
- )
- .expect("Failed to create sampling kernel");
+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)
+ .expect("Failed to create sampling kernel");
// Create test data with different probability distributions
// First batch: [1.0, 2.0, 1.5, 0.5] -> softmax: [0.134, 0.366, 0.201, 0.082]
@@ -803,10 +672,7 @@ fn test_categorical_sampling() {
let output_buffer = context
.device
- .new_buffer(
- batch_size * std::mem::size_of::<u32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
// Run sampling multiple times to check distribution
@@ -817,30 +683,18 @@ fn test_categorical_sampling() {
// Create fresh logits buffer since kernel mutates in-place
let logits_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&test_logits),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let seeds_vec =
- vec![TEST_SAMPLING_SEED + sample_idx as u64; batch_size];
+ let seeds_vec = vec![TEST_SAMPLING_SEED + sample_idx as u64; batch_size];
let seeds_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds_vec),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -867,8 +721,7 @@ fn test_categorical_sampling() {
command_buffer_ref.wait_until_completed();
let result_ptr = output_buffer.contents().as_ptr() as *const u32;
- let results =
- unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };
+ let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };
for (batch_idx, &token) in results.iter().enumerate() {
assert!(
@@ -883,8 +736,7 @@ fn test_categorical_sampling() {
// Check that all batches produced valid outputs
for batch_idx in 0..batch_size {
- let batch_counts =
- &counts[batch_idx * vocab_size..(batch_idx + 1) * vocab_size];
+ let batch_counts = &counts[batch_idx * vocab_size..(batch_idx + 1) * vocab_size];
let total_samples: usize = batch_counts.iter().sum();
assert_eq!(
total_samples, num_samples,
@@ -909,10 +761,7 @@ fn test_categorical_sampling() {
}
}
- println!(
- "βœ“ Categorical sampling test passed - sampled {} times per batch",
- num_samples
- );
+ println!("βœ“ Categorical sampling test passed - sampled {} times per batch", num_samples);
println!(" Batch 0 counts: {:?}", &counts[0..vocab_size]);
println!(" Batch 1 counts: {:?}", &counts[vocab_size..2 * vocab_size]);
}
@@ -935,8 +784,7 @@ fn test_categorical_sampling_statistical() {
const TOLERANCE: f32 = 0.05; // 5% tolerance
let kernel =
- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB)
- .expect("Failed to create sampling kernel");
+ SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
// Generate random logits
let mut rng = StdRng::seed_from_u64(42);
@@ -949,8 +797,7 @@ fn test_categorical_sampling_statistical() {
let mut expected_probs = vec![0.0f32; BATCH * VOCAB];
for batch_idx in 0..BATCH {
let batch_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
- let max_logit =
- batch_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+ let max_logit = batch_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let mut sum_exp = 0.0f32;
for j in 0..VOCAB {
@@ -967,10 +814,7 @@ fn test_categorical_sampling_statistical() {
let output_buffer = context
.device
- .new_buffer(
- BATCH * std::mem::size_of::<u32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(BATCH * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let mut counts = vec![0; BATCH * VOCAB];
@@ -980,29 +824,18 @@ fn test_categorical_sampling_statistical() {
// Create fresh logits buffer since kernel mutates in-place
let logits_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let seeds_vec = vec![TEST_SAMPLING_SEED + sample_idx as u64; BATCH];
let seeds_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds_vec),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -1039,8 +872,7 @@ fn test_categorical_sampling_statistical() {
// Check statistical accuracy
for batch_idx in 0..BATCH {
for token_idx in 0..VOCAB {
- let observed_freq = counts[batch_idx * VOCAB + token_idx] as f32
- / NUM_SAMPLES as f32;
+ let observed_freq = counts[batch_idx * VOCAB + token_idx] as f32 / NUM_SAMPLES as f32;
let expected_freq = expected_probs[batch_idx * VOCAB + token_idx];
if expected_freq > 0.01 {
@@ -1060,10 +892,7 @@ fn test_categorical_sampling_statistical() {
}
}
- println!(
- "βœ“ Categorical statistical test passed (tolerance: {:.1}%)",
- TOLERANCE * 100.0
- );
+ println!("βœ“ Categorical statistical test passed (tolerance: {:.1}%)", TOLERANCE * 100.0);
}
#[test]
@@ -1084,8 +913,7 @@ fn perf_categorical_128k_vocab() {
const VOCAB: usize = 128000; // 128K
let kernel =
- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB)
- .expect("Failed to create sampling kernel");
+ SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
// Build random logits
let mut rng = StdRng::seed_from_u64(123);
@@ -1096,38 +924,24 @@ fn perf_categorical_128k_vocab() {
let logits_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];
let seeds_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let output_buf = context
.device
- .new_buffer(
- BATCH * std::mem::size_of::<u32>(),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
// Launch and time
- let cb_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -1198,59 +1012,35 @@ fn test_temperature_gpu_cpu_match() {
const RTOL: f32 = 1e-6;
const ATOL: f32 = 1e-6;
- let kernel =
- TemperatureMetalKernel::new(&context, KernelDataType::Float32.into())
- .expect("Failed to create temperature kernel");
+ let kernel = TemperatureMetalKernel::new(&context, KernelDataType::Float32.into())
+ .expect("Failed to create temperature kernel");
- let logits: Vec<f32> = (0..BATCH * VOCAB)
- .map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01)
- .collect();
+ let logits: Vec<f32> = (0..BATCH * VOCAB).map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01).collect();
let logits_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let processed_buffer = context
.device
- .new_buffer(
- logits.len() * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
- kernel.encode(
- &logits_buffer,
- &processed_buffer,
- BATCH as u32,
- VOCAB as u32,
- TEMPERATURE,
- &compute_encoder,
- );
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
command_buffer_ref.wait_until_completed();
let gpu_ptr = processed_buffer.contents().as_ptr() as *const f32;
- let gpu_results =
- unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };
+ let gpu_results = unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };
- for (idx, (&logit, &processed)) in
- logits.iter().zip(gpu_results.iter()).enumerate()
- {
+ for (idx, (&logit, &processed)) in logits.iter().zip(gpu_results.iter()).enumerate() {
let expected = logit / TEMPERATURE;
let abs_diff = (expected - processed).abs();
let tolerance = ATOL + RTOL * expected.abs();
@@ -1265,10 +1055,7 @@ fn test_temperature_gpu_cpu_match() {
);
}
- println!(
- "βœ“ Temperature processor gpu cpu match (temp={}, rtol={}, atol={})",
- TEMPERATURE, RTOL, ATOL
- );
+ println!("βœ“ Temperature processor gpu cpu match (temp={}, rtol={}, atol={})", TEMPERATURE, RTOL, ATOL);
}
#[test]
@@ -1287,8 +1074,7 @@ fn test_topk_gpu_cpu_match() {
const VOCAB: usize = 1024;
const TOPK: u32 = 16;
- let kernel = TopKMetalKernel::new(&context, KernelDataType::Float32.into())
- .expect("Failed to create topk kernel");
+ let kernel = TopKMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create topk kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1298,58 +1084,37 @@ fn test_topk_gpu_cpu_match() {
let logits_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let processed_buffer = context
.device
- .new_buffer(
- logits.len() * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
- kernel.encode(
- &logits_buffer,
- &processed_buffer,
- BATCH as u32,
- VOCAB as u32,
- TOPK,
- &compute_encoder,
- );
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
command_buffer_ref.wait_until_completed();
let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
- let all_results =
- unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
+ let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
for batch_idx in 0..BATCH {
let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
- let mut cpu_sorted_logits: Vec<(usize, f32)> =
- cpu_logits.iter().copied().enumerate().collect();
+ let mut cpu_sorted_logits: Vec<(usize, f32)> = cpu_logits.iter().copied().enumerate().collect();
cpu_sorted_logits.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));
let mut cpu_processed_logits: Vec<f32> = vec![f32::NEG_INFINITY; VOCAB];
for (idx, val) in cpu_sorted_logits.into_iter().take(TOPK as usize) {
cpu_processed_logits[idx] = val;
}
- let gpu_processed_logits =
- &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
+ let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
assert_eq!(cpu_processed_logits, gpu_processed_logits);
}
@@ -1373,8 +1138,7 @@ fn test_topp_gpu_cpu_match() {
const VOCAB: usize = 1024;
const TOPP: f32 = 0.9;
- let kernel = TopPMetalKernel::new(&context, KernelDataType::Float32.into())
- .expect("Failed to create topp kernel");
+ let kernel = TopPMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create topp kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1384,45 +1148,26 @@ fn test_topp_gpu_cpu_match() {
let logits_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let processed_buffer = context
.device
- .new_buffer(
- logits.len() * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
- kernel.encode(
- &logits_buffer,
- &processed_buffer,
- BATCH as u32,
- VOCAB as u32,
- TOPP,
- &compute_encoder,
- );
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
command_buffer_ref.wait_until_completed();
let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
- let all_results =
- unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
+ let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
for batch_idx in 0..BATCH {
let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
@@ -1438,8 +1183,7 @@ fn test_topp_gpu_cpu_match() {
})
.collect::<Vec<f32>>();
- let gpu_processed_logits =
- &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
+ let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
assert_eq!(&cpu_processed_logits, gpu_processed_logits);
}
@@ -1463,8 +1207,7 @@ fn test_minp_gpu_cpu_match() {
const VOCAB: usize = 1024;
const MINP: f32 = 0.1;
- let kernel = MinPMetalKernel::new(&context, KernelDataType::Float32.into())
- .expect("Failed to create minp kernel");
+ let kernel = MinPMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create minp kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1474,52 +1217,32 @@ fn test_minp_gpu_cpu_match() {
let logits_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let processed_buffer = context
.device
- .new_buffer(
- logits.len() * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
- kernel.encode(
- &logits_buffer,
- &processed_buffer,
- BATCH as u32,
- VOCAB as u32,
- MINP,
- &compute_encoder,
- );
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, MINP, &compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
command_buffer_ref.wait_until_completed();
let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
- let all_results =
- unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
+ let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
for batch_idx in 0..BATCH {
let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
let cpu_processed_logits = cpu_reference_min_p(cpu_logits, MINP);
- let gpu_processed_logits =
- &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
+ let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
assert_eq!(&cpu_processed_logits, gpu_processed_logits);
}
@@ -1544,13 +1267,8 @@ fn test_minp_sampling_exact_match(
},
};
- let kernel = SamplingKernel::<Metal>::new(
- &context,
- DataType::F32,
- batch_size,
- vocab_size,
- )
- .expect("Failed to create sampling kernel");
+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)
+ .expect("Failed to create sampling kernel");
// Build logits where some tokens have high probability and others have low
// For min_p filtering, tokens with probability < min_p * max_prob are masked
@@ -1560,16 +1278,14 @@ fn test_minp_sampling_exact_match(
let low_logit = -50.0f32; // exp(-50) β‰ˆ 0
let mut logits = vec![low_logit; batch_size * vocab_size];
- let mut high_prob_token_sets: Vec<HashSet<usize>> =
- Vec::with_capacity(batch_size);
+ let mut high_prob_token_sets: Vec<HashSet<usize>> = Vec::with_capacity(batch_size);
let mut rng = StdRng::seed_from_u64(42);
let mut all_token_ids: Vec<usize> = (0..vocab_size).collect();
for b in 0..batch_size {
all_token_ids.shuffle(&mut rng);
- let high_prob_tokens: HashSet<usize> =
- all_token_ids[..k].iter().cloned().collect();
+ let high_prob_tokens: HashSet<usize> = all_token_ids[..k].iter().cloned().collect();
for &token_id in &high_prob_tokens {
logits[b * vocab_size + token_id] = high_logit;
@@ -1579,10 +1295,7 @@ fn test_minp_sampling_exact_match(
let output_buf = context
.device
- .new_buffer(
- batch_size * std::mem::size_of::<u32>(),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(batch_size * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let num_samples = 1000;
@@ -1592,30 +1305,18 @@ fn test_minp_sampling_exact_match(
// Create fresh logits buffer since kernel mutates in-place
let logits_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let seeds: Vec<u64> =
- vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
let seeds_buf = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds),
- metal::MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let cb_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
&logits_buf,
@@ -1640,14 +1341,10 @@ fn test_minp_sampling_exact_match(
cb_ref.wait_until_completed();
let ptr = output_buf.contents().as_ptr() as *const u32;
- let sampled_ids =
- unsafe { std::slice::from_raw_parts(ptr, batch_size) };
+ let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };
for (i, &sampled_id) in sampled_ids.iter().enumerate() {
- assert!(
- (sampled_id as usize) < vocab_size,
- "Sampled token out of range"
- );
+ assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");
counter[i * vocab_size + sampled_id as usize] += 1;
}
}
@@ -1666,10 +1363,7 @@ fn test_minp_sampling_exact_match(
}
}
- println!(
- "batch_size: {}, min_p: {:.2}, vocab_size: {}, accuracy test passed.",
- batch_size, min_p, vocab_size
- );
+ println!("batch_size: {}, min_p: {:.2}, vocab_size: {}, accuracy test passed.", batch_size, min_p, vocab_size);
}
#[test]
@@ -1698,68 +1392,43 @@ fn test_gumbel_gpu_cpu_match() {
const ATOL: f32 = 1e-6;
let kernel =
- GumbelMetalKernel::new(&context, KernelDataType::Float32.into())
- .expect("Failed to create gumbel kernel");
+ GumbelMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create gumbel kernel");
let logits = vec![0.0f32; BATCH * VOCAB];
let seeds: Vec<u64> = (0_u64..BATCH as u64).collect();
let logits_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&logits),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let seeds_buffer = context
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&seeds),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&seeds), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let gumbel_logits_buffer = context
.device
- .new_buffer(
- BATCH * VOCAB * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(BATCH * VOCAB * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
- kernel.encode(
- &logits_buffer,
- &seeds_buffer,
- &gumbel_logits_buffer,
- BATCH as u32,
- VOCAB as u32,
- &compute_encoder,
- );
+ let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &seeds_buffer, &gumbel_logits_buffer, BATCH as u32, VOCAB as u32, &compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
command_buffer_ref.wait_until_completed();
let result_ptr = gumbel_logits_buffer.contents().as_ptr() as *const f32;
- let all_results =
- unsafe { std::slice::from_raw_parts(result_ptr, BATCH * VOCAB) };
+ let all_results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH * VOCAB) };
for (batch_idx, batch_seed) in seeds.iter().copied().enumerate() {
let results = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
- for (logit_idx, gpu_logit_value) in results.iter().copied().enumerate()
- {
- let cpu_logit_value =
- gumbel_float(batch_seed, revidx(logit_idx as u32));
+ for (logit_idx, gpu_logit_value) in results.iter().copied().enumerate() {
+ let cpu_logit_value = gumbel_float(batch_seed, revidx(logit_idx as u32));
let abs_diff = (cpu_logit_value - gpu_logit_value).abs();
let tolerance = ATOL + RTOL * cpu_logit_value.abs();
assert!(
@@ -1769,9 +1438,5 @@ fn test_gumbel_gpu_cpu_match() {
}
}
- println!(
- "βœ“ Gumbel cpu gpu match test passed (rtol: {:.1}%, atol: {:.1e})",
- RTOL * 100.0,
- ATOL
- );
+ println!("βœ“ Gumbel cpu gpu match test passed (rtol: {:.1}%, atol: {:.1e})", RTOL * 100.0, ATOL);
}
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index 5659e7364..f6d117228 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -4,8 +4,7 @@ use std::mem::size_of;
use bytemuck;
use metal::{
- MTLBlitCommandEncoder, MTLCommandBuffer, MTLCommandEncoder,
- MTLCommandQueue, MTLDeviceExt, MTLResourceOptions,
+ MTLBlitCommandEncoder, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions,
};
use uzu::backends::{
common::Context,
@@ -19,8 +18,7 @@ use uzu::backends::{
};
use uzu::config::Activation;
-const STORAGE_MODE: MTLResourceOptions =
- MTLResourceOptions::STORAGE_MODE_SHARED;
+const STORAGE_MODE: MTLResourceOptions = MTLResourceOptions::STORAGE_MODE_SHARED;
fn silu_scalar(x: f32) -> f32 {
let y = 1.0 / (1.0 + (-x).exp());
@@ -40,11 +38,7 @@ fn write_buffer(
data: &[f32],
) {
unsafe {
- std::ptr::copy_nonoverlapping(
- data.as_ptr(),
- buf.contents().as_ptr() as *mut f32,
- data.len(),
- );
+ std::ptr::copy_nonoverlapping(data.as_ptr(), buf.contents().as_ptr() as *mut f32, data.len());
}
}
@@ -54,22 +48,14 @@ fn read_buffer(
) -> Vec<f32> {
let mut out = vec![0.0f32; len];
unsafe {
- std::ptr::copy_nonoverlapping(
- buf.contents().as_ptr() as *const f32,
- out.as_mut_ptr(),
- len,
- );
+ std::ptr::copy_nonoverlapping(buf.contents().as_ptr() as *const f32, out.as_mut_ptr(), len);
}
out
}
fn zero_buffer(buf: &ProtocolObject<dyn MTLBuffer>) {
unsafe {
- std::ptr::write_bytes(
- buf.contents().as_ptr(),
- 0,
- buf.length() as usize,
- );
+ std::ptr::write_bytes(buf.contents().as_ptr(), 0, buf.length() as usize);
}
}
@@ -101,8 +87,7 @@ fn ssd_prefill_cpu_reference(
for dh in 0..head_dim {
let state_base = h * state_strides[0] + dh * state_strides[1];
for token in 0..suffix_len {
- let x_idx =
- token * x_strides[0] + h * x_strides[1] + dh * x_strides[2];
+ let x_idx = token * x_strides[0] + h * x_strides[1] + dh * x_strides[2];
let dt_idx = token * dt_strides[0] + h * dt_strides[1];
let cb_base = token * cb_strides[0] + group_idx * cb_strides[1];
@@ -119,8 +104,7 @@ fn ssd_prefill_cpu_reference(
let cb_idx = cb_base + s * cb_strides[2];
let b_coeff = b_data[cb_idx];
let c_coeff = c_data[cb_idx];
- let new_state = decay_val * state[state_idx]
- + dt_scaled_input * b_coeff;
+ let new_state = decay_val * state[state_idx] + dt_scaled_input * b_coeff;
state[state_idx] = new_state;
acc += new_state * c_coeff;
}
@@ -171,20 +155,13 @@ impl SSDPrefillFixture {
let total_cb = suffix_len * group_count * state_dim;
let total_state = num_heads * head_dim * state_dim;
- let x_data: Vec<f32> =
- (0..total_x).map(|i| ((i % 17) as f32) * 0.01 - 0.05).collect();
- let dt_data: Vec<f32> =
- (0..total_dt).map(|i| ((i % 13) as f32) * 0.2 - 1.5).collect();
- let b_data: Vec<f32> =
- (0..total_cb).map(|i| ((i % 11) as f32) * 0.02 - 0.05).collect();
- let c_data: Vec<f32> =
- (0..total_cb).map(|i| ((i % 19) as f32) * 0.01 - 0.02).collect();
- let d_data: Vec<f32> =
- (0..num_heads).map(|i| ((i % 3) as f32) * 0.05 - 0.05).collect();
- let z_data: Vec<f32> =
- (0..total_x).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();
- let state_init: Vec<f32> =
- (0..total_state).map(|i| ((i % 29) as f32) * 0.03 - 0.4).collect();
+ let x_data: Vec<f32> = (0..total_x).map(|i| ((i % 17) as f32) * 0.01 - 0.05).collect();
+ let dt_data: Vec<f32> = (0..total_dt).map(|i| ((i % 13) as f32) * 0.2 - 1.5).collect();
+ let b_data: Vec<f32> = (0..total_cb).map(|i| ((i % 11) as f32) * 0.02 - 0.05).collect();
+ let c_data: Vec<f32> = (0..total_cb).map(|i| ((i % 19) as f32) * 0.01 - 0.02).collect();
+ let d_data: Vec<f32> = (0..num_heads).map(|i| ((i % 3) as f32) * 0.05 - 0.05).collect();
+ let z_data: Vec<f32> = (0..total_x).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();
+ let state_init: Vec<f32> = (0..total_state).map(|i| ((i % 29) as f32) * 0.03 - 0.4).collect();
let x_strides = [num_heads * head_dim, head_dim, 1usize];
let dt_strides = [num_heads, 1usize];
@@ -224,47 +201,25 @@ fn run_prefill_kernel_mode(
) -> (Vec<f32>, Vec<f32>, Option<(Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>)>) {
let device = &ctx.device;
let x_buf = device
- .new_buffer_with_data(
- bytemuck::cast_slice(&fixture.x_data),
- STORAGE_MODE,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.x_data), STORAGE_MODE)
.expect("Failed to create buffer");
let dt_buf = device
- .new_buffer_with_data(
- bytemuck::cast_slice(&fixture.dt_data),
- STORAGE_MODE,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.dt_data), STORAGE_MODE)
.expect("Failed to create buffer");
let b_buf = device
- .new_buffer_with_data(
- bytemuck::cast_slice(&fixture.b_data),
- STORAGE_MODE,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.b_data), STORAGE_MODE)
.expect("Failed to create buffer");
let c_buf = device
- .new_buffer_with_data(
- bytemuck::cast_slice(&fixture.c_data),
- STORAGE_MODE,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.c_data), STORAGE_MODE)
.expect("Failed to create buffer");
let d_buf = device
- .new_buffer_with_data(
- bytemuck::cast_slice(&fixture.d_data),
- STORAGE_MODE,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.d_data), STORAGE_MODE)
.expect("Failed to create buffer");
let z_buf = device
- .new_buffer_with_data(
- bytemuck::cast_slice(&fixture.z_data),
- STORAGE_MODE,
- )
- .expect("Failed to create buffer");
- let state_buf = device
- .new_buffer(fixture.total_state * 4, STORAGE_MODE)
- .expect("Failed to create buffer");
- let y_buf = device
- .new_buffer(fixture.total_x * 4, STORAGE_MODE)
+ .new_buffer_with_data(bytemuck::cast_slice(&fixture.z_data), STORAGE_MODE)
.expect("Failed to create buffer");
+ let state_buf = device.new_buffer(fixture.total_state * 4, STORAGE_MODE).expect("Failed to create buffer");
+ let y_buf = device.new_buffer(fixture.total_x * 4, STORAGE_MODE).expect("Failed to create buffer");
write_buffer(&state_buf, &fixture.state_init);
zero_buffer(&y_buf);
@@ -289,13 +244,8 @@ fn run_prefill_kernel_mode(
head_dim: fixture.head_dim,
};
- let command_buffer = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(&encoder, args, mode);
encoder.end_encoding();
command_buffer.commit();
@@ -326,31 +276,19 @@ fn run_conv_scan_once(
let total_state = channels * tap_count;
let y_buf = if alias_io {
- device
- .new_buffer_with_data(bytemuck::cast_slice(x_data), STORAGE_MODE)
- .expect("Failed to create buffer")
+ device.new_buffer_with_data(bytemuck::cast_slice(x_data), STORAGE_MODE).expect("Failed to create buffer")
} else {
- let buf = device
- .new_buffer(total_x * size_of::<f32>(), STORAGE_MODE)
- .expect("Failed to create buffer");
+ let buf = device.new_buffer(total_x * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");
zero_buffer(&buf);
buf
};
- let w_buf = device
- .new_buffer_with_data(bytemuck::cast_slice(w_data), STORAGE_MODE)
- .expect("Failed to create buffer");
- let b_buf = device
- .new_buffer_with_data(bytemuck::cast_slice(b_data), STORAGE_MODE)
- .expect("Failed to create buffer");
- let state_buf = device
- .new_buffer(total_state * size_of::<f32>(), STORAGE_MODE)
- .expect("Failed to create buffer");
+ let w_buf =
+ device.new_buffer_with_data(bytemuck::cast_slice(w_data), STORAGE_MODE).expect("Failed to create buffer");
+ let b_buf =
+ device.new_buffer_with_data(bytemuck::cast_slice(b_data), STORAGE_MODE).expect("Failed to create buffer");
+ let state_buf = device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");
let scratch_buf = if use_scratch && tap_count > 0 {
- Some(
- device
- .new_buffer(total_state * size_of::<f32>(), STORAGE_MODE)
- .expect("Failed to create buffer"),
- )
+ Some(device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer"))
} else {
None
};
@@ -361,9 +299,8 @@ fn run_conv_scan_once(
}
let padded_len = tap_count + suffix_len;
- let padded_buf = device
- .new_buffer(padded_len * channels * size_of::<f32>(), STORAGE_MODE)
- .expect("Failed to create buffer");
+ let padded_buf =
+ device.new_buffer(padded_len * channels * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");
{
let mut host = vec![0.0f32; padded_len * channels];
for tap in 0..tap_count {
@@ -373,8 +310,7 @@ fn run_conv_scan_once(
}
for token in 0..suffix_len {
for ch in 0..channels {
- host[(tap_count + token) * channels + ch] =
- x_data[token * channels + ch];
+ host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];
}
}
write_buffer(&padded_buf, &host);
@@ -397,22 +333,15 @@ fn run_conv_scan_once(
proj_dim: 0,
};
- let command_buffer = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
- let encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(&encoder, args).unwrap();
encoder.end_encoding();
if let Some(ref scratch) = scratch_buf {
let bytes = channels * tap_count * size_of::<f32>();
if bytes > 0 {
- let blit = command_buffer
- .new_blit_command_encoder()
- .expect("Failed to create blit encoder");
+ let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit encoder");
blit.copy_buffer_to_buffer(scratch, 0, &state_buf, 0, bytes);
blit.end_encoding();
}
@@ -434,10 +363,8 @@ fn assert_deterministic_for_mode(mode: SSDPrefillMode) {
let kernel = SSDPrefillKernels::new(&ctx, KernelDataType::Float32).unwrap();
let fixture = SSDPrefillFixture::new();
- let (y_a, state_a, _) =
- run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
- let (y_b, state_b, _) =
- run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
+ let (y_a, state_a, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
+ let (y_b, state_b, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
assert_eq!(y_a, y_b, "Prefill outputs differ in {:?} mode", mode);
assert_eq!(state_a, state_b, "Prefill states differ in {:?} mode", mode);
@@ -470,8 +397,7 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {
fixture.state_strides,
);
- let (y_gpu, state_gpu, _) =
- run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
+ let (y_gpu, state_gpu, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
let tolerance = 5e-5f32;
let mut max_y_diff = 0.0f32;
@@ -536,9 +462,7 @@ fn conv1d_scan_is_deterministic() {
return;
};
let activation = Activation::Identity;
- let kernel =
- Conv1dScanKernel::new(&ctx, KernelDataType::Float32, &activation)
- .unwrap();
+ let kernel = Conv1dScanKernel::new(&ctx, KernelDataType::Float32, &activation).unwrap();
let suffix_len = 192usize;
let channels = 8usize;
@@ -548,15 +472,10 @@ fn conv1d_scan_is_deterministic() {
let total_x = suffix_len * channels;
let total_state = channels * tap_count;
- let x_data: Vec<f32> =
- (0..total_x).map(|i| ((i % 31) as f32) * 0.02 - 0.3).collect();
- let w_data: Vec<f32> = (0..(channels * kernel_size as usize))
- .map(|i| ((i % 17) as f32) * 0.01 - 0.04)
- .collect();
- let b_data: Vec<f32> =
- (0..channels).map(|i| ((i % 5) as f32) * 0.03 - 0.07).collect();
- let state_init: Vec<f32> =
- (0..total_state).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();
+ let x_data: Vec<f32> = (0..total_x).map(|i| ((i % 31) as f32) * 0.02 - 0.3).collect();
+ let w_data: Vec<f32> = (0..(channels * kernel_size as usize)).map(|i| ((i % 17) as f32) * 0.01 - 0.04).collect();
+ let b_data: Vec<f32> = (0..channels).map(|i| ((i % 5) as f32) * 0.03 - 0.07).collect();
+ let state_init: Vec<f32> = (0..total_state).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();
let use_scratch = tap_count > 0 && suffix_len > 1;
@@ -590,13 +509,7 @@ fn conv1d_scan_is_deterministic() {
alias_io,
);
- assert_eq!(
- first.0, second.0,
- "Conv outputs differ (alias_io={alias_io})"
- );
- assert_eq!(
- first.1, second.1,
- "Conv states differ (alias_io={alias_io})"
- );
+ assert_eq!(first.0, second.0, "Conv outputs differ (alias_io={alias_io})");
+ assert_eq!(first.1, second.1, "Conv states differ (alias_io={alias_io})");
}
}
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index 0aa225d4c..f5e27100f 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -2,10 +2,7 @@
use bytemuck;
use half::bf16;
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,
- MTLDeviceExt, MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use uzu::backends::{
common::Context,
common::kernel::SSDUpdateKernel,
@@ -50,8 +47,7 @@ fn ssm_update_ref_bf16(
let state_offset = idx * n;
let mut acc = 0.0f32;
for i in 0..n {
- let new_s = state[state_offset + i].to_f32()
- * (a[i].to_f32() * delta).exp()
+ let new_s = state[state_offset + i].to_f32() * (a[i].to_f32() * delta).exp()
+ b[cb_offset + i].to_f32() * delta * this_x;
next_state[state_offset + i] = bf16::from_f32(new_s);
acc += new_s * c[cb_offset + i].to_f32();
@@ -107,8 +103,7 @@ fn ssd_update_no_z_ref_bf16(
let mut acc = 0.0f32;
for i in 0..n {
let s_old = state[state_idx0 + i].to_f32();
- let new_s = s_old * this_decay
- + b[cb_idx0 + i].to_f32() * dt_scaled_input;
+ let new_s = s_old * this_decay + b[cb_idx0 + i].to_f32() * dt_scaled_input;
next_state[state_idx0 + i] = bf16::from_f32(new_s);
acc += new_s * c[cb_idx0 + i].to_f32();
}
@@ -162,8 +157,7 @@ fn ssd_update_ref_bf16(
let mut acc = 0.0f32;
for i in 0..n {
let s_old = state[state_idx0 + i].to_f32();
- let new_s = s_old * this_decay
- + b[cb_idx0 + i].to_f32() * dt_scaled_input;
+ let new_s = s_old * this_decay + b[cb_idx0 + i].to_f32() * dt_scaled_input;
next_state[state_idx0 + i] = bf16::from_f32(new_s);
acc += new_s * c[cb_idx0 + i].to_f32();
}
@@ -188,29 +182,15 @@ fn ssd_update_with_z_bf16() {
let g = 2usize;
let n = 8usize;
- let x: Vec<bf16> = (0..bsz * h * dh)
- .map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.2))
- .collect();
- let z: Vec<bf16> = (0..bsz * h * dh)
- .map(|i| bf16::from_f32(((i % 5) as f32) * 0.1 - 0.1))
- .collect();
- let dt: Vec<bf16> = (0..bsz * h)
- .map(|i| bf16::from_f32(((i % 5) as f32) * 0.3 - 1.0))
- .collect();
- let b: Vec<bf16> = (0..bsz * g * n)
- .map(|i| bf16::from_f32(((i % 11) as f32) * 0.02 - 0.05))
- .collect();
- let c: Vec<bf16> = (0..bsz * g * n)
- .map(|i| bf16::from_f32(((i % 13) as f32) * 0.015))
- .collect();
- let d: Vec<bf16> =
- (0..h).map(|i| bf16::from_f32(((i % 3) as f32) * 0.05)).collect();
- let state: Vec<bf16> = (0..bsz * h * dh * n)
- .map(|i| bf16::from_f32(((i % 23) as f32) * 0.01 - 0.05))
- .collect();
+ let x: Vec<bf16> = (0..bsz * h * dh).map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.2)).collect();
+ let z: Vec<bf16> = (0..bsz * h * dh).map(|i| bf16::from_f32(((i % 5) as f32) * 0.1 - 0.1)).collect();
+ let dt: Vec<bf16> = (0..bsz * h).map(|i| bf16::from_f32(((i % 5) as f32) * 0.3 - 1.0)).collect();
+ let b: Vec<bf16> = (0..bsz * g * n).map(|i| bf16::from_f32(((i % 11) as f32) * 0.02 - 0.05)).collect();
+ let c: Vec<bf16> = (0..bsz * g * n).map(|i| bf16::from_f32(((i % 13) as f32) * 0.015)).collect();
+ let d: Vec<bf16> = (0..h).map(|i| bf16::from_f32(((i % 3) as f32) * 0.05)).collect();
+ let state: Vec<bf16> = (0..bsz * h * dh * n).map(|i| bf16::from_f32(((i % 23) as f32) * 0.01 - 0.05)).collect();
- let (y_exp, ns_exp) =
- ssd_update_ref_bf16(&x, &dt, &b, &c, &d, &z, &state, bsz, h, dh, g, n);
+ let (y_exp, ns_exp) = ssd_update_ref_bf16(&x, &dt, &b, &c, &d, &z, &state, bsz, h, dh, g, n);
let y_exp_f32: Vec<f32> = y_exp.iter().map(|&v| v.to_f32()).collect();
let ns_exp_f32: Vec<f32> = ns_exp.iter().map(|&v| v.to_f32()).collect();
@@ -221,79 +201,45 @@ fn ssd_update_with_z_bf16() {
let x_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&x),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&x), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let dt_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&dt),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&dt), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let b_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&b),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&b), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let c_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&c),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&c), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let d_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&d),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&d), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let z_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&z),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&z), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let state_buf = ctx
.device
- .new_buffer_with_data(
- bytemuck::cast_slice(&state),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer_with_data(bytemuck::cast_slice(&state), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let y_buf = ctx
.device
- .new_buffer(
- bsz * h * dh * std::mem::size_of::<bf16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(bsz * h * dh * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
let ns_buf = ctx
.device
- .new_buffer(
- bsz * h * dh * n * std::mem::size_of::<bf16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
+ .new_buffer(bsz * h * dh * n * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let kernel =
- SSDUpdateMetalKernel::new(&ctx, KernelDataType::BFloat16.into())
- .unwrap();
- let cb_ref = ctx
- .command_queue
- .command_buffer()
- .expect("Failed to create command buffer");
+ let kernel = SSDUpdateMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).unwrap();
+ let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let enc = cb
- .new_compute_command_encoder()
- .expect("Failed to create compute encoder");
+ let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(
&x_buf,
&dt_buf,
@@ -323,8 +269,7 @@ fn ssd_update_with_z_bf16() {
let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h * dh) };
let y_out_f32: Vec<f32> = y_out.iter().map(|&v| v.to_f32()).collect();
let ns_ptr = ns_buf.contents().as_ptr() as *const bf16;
- let ns_out =
- unsafe { std::slice::from_raw_parts(ns_ptr, bsz * h * dh * n) };
+ let ns_out = unsafe { std::slice::from_raw_parts(ns_ptr, bsz * h * dh * n) };
let ns_out_f32: Vec<f32> = ns_out.iter().map(|&v| v.to_f32()).collect();
let tol = 2e-2;
for (i, (a, b)) in y_out_f32.iter().zip(y_exp_f32.iter()).enumerate() {
diff --git a/crates/uzu/tests/text_session_test.rs b/crates/uzu/tests/text_session_test.rs
index 42706dcc8..22be26ab6 100644
--- a/crates/uzu/tests/text_session_test.rs
+++ b/crates/uzu/tests/text_session_test.rs
@@ -29,17 +29,13 @@ fn test_text_session_base() {
#[test]
fn test_text_session_scenario() {
let system_prompt = String::from("You are a helpful assistant.");
- let user_prompts = vec![
- String::from("Tell about London"),
- String::from("Compare with New York"),
- ];
+ let user_prompts = vec![String::from("Tell about London"), String::from("Compare with New York")];
run_scenario(Some(system_prompt), user_prompts);
}
#[test]
fn test_text_session_stability() {
- let mut session =
- Session::new(build_model_path(), build_decoding_config()).unwrap();
+ let mut session = Session::new(build_model_path(), build_decoding_config()).unwrap();
println!("Index | TTFT, s | Prompt t/s | Generate t/s");
for index in 0..10 {
let input = Input::Text(build_default_text());
@@ -67,8 +63,7 @@ fn run(
decoding_config: DecodingConfig,
tokens_limit: u64,
) {
- let mut session =
- Session::new(build_model_path(), decoding_config).unwrap();
+ let mut session = Session::new(build_model_path(), decoding_config).unwrap();
let input = Input::Text(text);
let output = session
.run(
@@ -83,15 +78,9 @@ fn run(
let empty_response = String::from("None");
println!("-------------------------");
- println!(
- "{}",
- output.text.parsed.chain_of_thought.unwrap_or(empty_response.clone())
- );
+ println!("{}", output.text.parsed.chain_of_thought.unwrap_or(empty_response.clone()));
println!("-------------------------");
- println!(
- "{}",
- output.text.parsed.response.unwrap_or(empty_response.clone())
- );
+ println!("{}", output.text.parsed.response.unwrap_or(empty_response.clone()));
println!("-------------------------");
println!("{:#?}", output.stats);
println!("-------------------------");
@@ -103,8 +92,7 @@ fn run_scenario(
system_prompt: Option<String>,
user_prompts: Vec<String>,
) {
- let mut session =
- Session::new(build_model_path(), build_decoding_config()).unwrap();
+ let mut session = Session::new(build_model_path(), build_decoding_config()).unwrap();
let mut messages: Vec<Message> = vec![];
if let Some(system_prompt) = system_prompt {
diff --git a/crates/uzu/tests/tracer_test.rs b/crates/uzu/tests/tracer_test.rs
index ef3aeff20..a8928a7f2 100644
--- a/crates/uzu/tests/tracer_test.rs
+++ b/crates/uzu/tests/tracer_test.rs
@@ -15,10 +15,7 @@ fn test_tracer() {
// Ensure traces file present, otherwise skip
let traces_path = crate::common::get_traces_path();
if !traces_path.exists() {
- println!(
- "Skipping tracer test: traces file missing at {:?}",
- traces_path
- );
+ println!("Skipping tracer test: traces file missing at {:?}", traces_path);
return;
}
@@ -58,27 +55,17 @@ fn test_tracer() {
);
println!("{}: {}", result.name, text);
if !valid {
- println!(
- "{}",
- colored_text(result.metrics.message().as_str(), false)
- );
+ println!("{}", colored_text(result.metrics.message().as_str(), false));
}
}
println!("-------------------------");
println!(
"number_of_tokens_violations: {}",
colored_text(
- format!(
- "{} / {}",
- results.number_of_tokens_violations(),
- results.number_of_allowed_tokens_violations()
- )
- .as_str(),
+ format!("{} / {}", results.number_of_tokens_violations(), results.number_of_allowed_tokens_violations())
+ .as_str(),
results.is_valid(),
)
);
- println!(
- "tokens_violation_indices: {:?}",
- results.tokens_violation_indices
- );
+ println!("tokens_violation_indices: {:?}", results.tokens_violation_indices);
}
diff --git a/crates/uzu/tests/trie_test.rs b/crates/uzu/tests/trie_test.rs
index c5fcb2cf5..751135df8 100644
--- a/crates/uzu/tests/trie_test.rs
+++ b/crates/uzu/tests/trie_test.rs
@@ -22,10 +22,7 @@ fn verify_sprout(
assert_eq!(flat_trie.index(&TrieNode::new(0, None, 0)), None);
assert_eq!(flat_trie.token_ids().collect::<Vec<u64>>(), vec![0]);
assert_eq!(flat_trie.token_positions().collect::<Vec<usize>>(), vec![0]);
- assert_eq!(
- flat_trie.token_seeds().collect::<Vec<u64>>(),
- vec![expected_seed]
- );
+ assert_eq!(flat_trie.token_seeds().collect::<Vec<u64>>(), vec![expected_seed]);
}
#[test]
@@ -108,10 +105,7 @@ fn test_trie_from_speculator_stick() {
let mut hs = HashMap::new();
for i in 1..10 {
- hs.insert(
- (0..i).collect(),
- vec![(i, 0.988), (888, 0.001), (999, 0.001)].into_iter().collect(),
- );
+ hs.insert((0..i).collect(), vec![(i, 0.988), (888, 0.001), (999, 0.001)].into_iter().collect());
}
let speculator = StaticSpeculator::new(hs);
@@ -180,12 +174,7 @@ fn test_trie_from_speculator_bush() {
let rng = PRng::new(0);
let mut hs = HashMap::new();
- hs.insert(
- vec![0],
- vec![(1, 0.33), (2, 0.33), (3, 0.33), (888, 0.005), (999, 0.005)]
- .into_iter()
- .collect(),
- );
+ hs.insert(vec![0], vec![(1, 0.33), (2, 0.33), (3, 0.33), (888, 0.005), (999, 0.005)].into_iter().collect());
let speculator = StaticSpeculator::new(hs);
let trie_root = TrieNode::from_speculator(
diff --git a/rustfmt.toml b/rustfmt.toml
index 9b128290a..29d67d950 100644
--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -6,4 +6,4 @@ imports_granularity = "Crate"
fn_params_layout = "Vertical"
match_block_trailing_comma = true
use_small_heuristics = "Off"
-max_width = 80
+max_width = 120