A high-performance inference engine for AI models
classificators & new lalamo config
eugene85e6de79bf15
diff --git a/.gitignore b/.gitignoreindex 1052a077a..019a2328c 100644--- a/.gitignore+++ b/.gitignore@@ -26,3 +26,4 @@ target/# Modelsmodels/+external/diff --git a/Cargo.lock b/Cargo.lockindex f137cf46a..09d40fe79 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -477,7 +477,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"[[package]]name = "cli"-version = "0.1.6"+version = "0.1.7"dependencies = ["clap","console 0.16.0",@@ -920,7 +920,7 @@ dependencies = ["libc","option-ext","redox_users",- "windows-sys 0.60.2",+ "windows-sys 0.61.2",][[package]]@@ -2154,9 +2154,9 @@ dependencies = [[[package]]name = "ndarray"-version = "0.16.1"+version = "0.17.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"+checksum = "0c7c9125e8f6f10c9da3aad044cc918cf8784fa34de857b1aa68038eb05a50a9"dependencies = ["matrixmultiply","num-complex",@@ -4001,7 +4001,7 @@ dependencies = [[[package]]name = "uzu"-version = "0.1.6"+version = "0.1.7"dependencies = ["bitflags 2.9.4","block",@@ -4227,7 +4227,7 @@ version = "0.1.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"dependencies = [- "windows-sys 0.60.2",+ "windows-sys 0.61.2",][[package]]@@ -4648,9 +4648,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"[[package]]name = "xgrammar-rs"-version = "0.1.27"+version = "0.1.28"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ac85eff00b9853aab820a5b384e277c250044302ceb5ea5204265651b1777fcf"+checksum = "22d454c6fba6f8ae3847d170653086fcc8bfe28e6a39a44504b1c49fa812bf29"dependencies = ["autocxx","autocxx-build",diff --git a/Cargo.toml b/Cargo.tomlindex 65fa2468c..dcfab2552 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -3,7 +3,7 @@ members = ["crates/cli", "crates/uzu"]resolver = "3"[workspace.package]-version = "0.1.6"+version = "0.1.7"edition = "2024"rust-version = "1.91"license = "MIT"diff --git a/README.md b/README.mdindex 25f258202..60a8aa283 100644--- a/README.md+++ b/README.md@@ -32,7 +32,7 @@ For a detailed explanation of the architecture, please refer to the [documentati```bashgit clone https://github.com/trymirai/lalamo.gitcd lalamo-git checkout v0.5.2+git checkout v0.5.15```After that, you can retrieve the list of supported models:diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.tomlindex e22c69ed7..55f9c5d4f 100644--- a/crates/cli/Cargo.toml+++ b/crates/cli/Cargo.toml@@ -1,6 +1,6 @@[package]name = "cli"-version = "0.1.6"+version.workspace = trueedition = "2024"[[bin]]diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex e1075a314..c08574262 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -1,13 +1,13 @@[package]name = "uzu"-version = "0.1.6"+version.workspace = trueedition = "2024"build = "build.rs"[dependencies]-xgrammar = { package = "xgrammar-rs", version = "0.1.27", features = ["hf"] }+xgrammar = { package = "xgrammar-rs", version = "0.1.28", features = ["hf"] }half = { version = "2.6.0", features = ["bytemuck", "num-traits"] }-ndarray = "0.16.1"+ndarray = "0.17.1"serde = { version = "1.0.219", features = ["derive"] }serde_json = "1.0.143"thiserror = "2.0.16"@@ -38,6 +38,7 @@ schemars = { version = "1.0.5", features = ["derive"] }[features]default = ["metal-shaders"]metal-shaders = []+tracing = [][[test]]name = "moe"diff --git a/crates/uzu/examples/dequantize_qkv_layer0.rs b/crates/uzu/examples/dequantize_qkv_layer0.rsdeleted file mode 100644index 79412f7eb..000000000--- a/crates/uzu/examples/dequantize_qkv_layer0.rs+++ /dev/null@@ -1,252 +0,0 @@-use std::{- collections::HashMap,- fs::File,- rc::Rc,- time::{SystemTime, UNIX_EPOCH},-};--use clap::Parser;-use metal::{CaptureDescriptor, CaptureManager, MTLCaptureDestination};-use mpsgraph::{- CommandBuffer, DequantizationArguments, Device as MPSDevice,- ExecutableExecutionDescriptor, ExecutableSerializationDescriptor, Graph,- Optimization, OptimizationProfile, ShapedType, Tensor, TensorData,-};-use objc2::rc::autoreleasepool;-use thiserror::Error;-use uzu::{- Array, DataType, DeviceContext,- backends::metal::{- MTLContext,- compilation_parameters::{BlockDevice, make_compilation_descriptor},- error::MTLError,- graph::common::{GraphConstructionError, load_constant},- },- parameters::{ParameterLoader, ParameterLoaderError},- utils::{NSSearchPathDirectory, root_dir},-};--#[derive(Debug, Error)]-pub enum ExampleError {- #[error(transparent)]- Io(#[from] std::io::Error),- #[error(transparent)]- Metal(#[from] MTLError),- #[error(transparent)]- Loader(#[from] ParameterLoaderError),- #[error("header loading error")]- Header,- #[error("capture error")]- Capture,- #[error(transparent)]- Graph(#[from] GraphConstructionError),- #[error("dequantization failed")]- Dequantization,- #[error("result buffer view unavailable")]- ResultView,-}--#[derive(Parser)]-#[command(- name = "dequantize_qkv_layer0",- about = "Demonstrates QKV dequantization with optional GPU trace capture and MPSGraph package serialization"-)]-struct Args {- /// Dump a .gputrace file capturing the GPU execution- #[arg(long = "dump-gpu-trace")]- dump_gpu_trace: bool,-- /// Serialize the compiled MPSGraph executable to a .mpsgraphpackage file- #[arg(long = "dump-mpsgraphpackage")]- dump_mpsgraphpackage: bool,-- /// Perform and print ANE/GPU placement analysis during compilation- #[arg(long = "print-placement-analysis")]- print_placement_analysis: bool,-- /// Print the executable's IR / debug info after compilation- #[arg(long = "print-executable-dump")]- print_executable_dump: bool,-- /// Dump the result array to a text file in ~/Downloads with a timestamped name- #[arg(long = "dump-result-array")]- dump_result_array: bool,-}--fn main() -> Result<(), ExampleError> {- autoreleasepool(|_| {- let args = Args::parse();- let dump_gpu_trace = args.dump_gpu_trace;- let dump_mpsgraphpackage = args.dump_mpsgraphpackage;- let print_placement_analysis = args.print_placement_analysis;- let print_executable_dump = args.print_executable_dump;- let dump_result_array = args.dump_result_array;-- if dump_gpu_trace {- unsafe {- std::env::set_var("METAL_CAPTURE_ENABLED", "1");- }- }-- let model_dir = root_dir(NSSearchPathDirectory::Downloads)- .join("Qwen3-4B-AWQ-Temp");- let safetensors = File::open(model_dir.join("model.safetensors"))?;-- let device =- metal::Device::system_default().ok_or(ExampleError::Capture)?;- let command_queue = device.new_command_queue();- let mtl_context = Rc::new(MTLContext::new(device, command_queue)?);-- let loader = ParameterLoader::new(&safetensors, &mtl_context)- .map_err(|_| ExampleError::Header)?;- let tree = loader- .tree()- .subtree("layers")?- .subtree("0")?- .subtree("attention")?- .subtree("qkv_projection")?;-- let graph = Graph::new();-- let weights = load_constant(- &graph,- &tree,- "weights",- &[3072 * 2, 2560],- DataType::U4,- )- .map_err(ExampleError::from)?;-- let scales =- load_constant(&graph, &tree, "scales", &[6144, 20], DataType::F16)- .map_err(ExampleError::from)?;-- let zero_points = load_constant(- &graph,- &tree,- "zero_points",- &[3072 * 2, 20],- DataType::U4,- )- .map_err(ExampleError::from)?;-- let dequantized_weights = graph.dequantize(- &weights,- DequantizationArguments::ScaleTensorZeroPointTensorDataType {- scale_tensor: &scales,- zero_point_tensor: &zero_points,- data_type: DataType::F16.into(),- },- None,- );-- let mut dequantized_weights_buffer = unsafe {- mtl_context.array_uninitialized(&[3072 * 2, 2560], DataType::F16)- };- let dequantized_weights_tensor_data = unsafe {- TensorData::new_with_mtl_buffer(- &dequantized_weights_buffer.mtl_buffer(),- &[3072 * 2, 2560],- DataType::F16.into(),- None,- )- };-- let capture_manager = CaptureManager::shared();- let capture_manager_descriptor = CaptureDescriptor::new();- capture_manager_descriptor- .set_destination(MTLCaptureDestination::GpuTraceDocument);-- let timestamp = SystemTime::now()- .duration_since(UNIX_EPOCH)- .expect("Time went backwards")- .as_secs();- let trace_path = root_dir(NSSearchPathDirectory::Downloads)- .join(format!("dequantize_qkv_layer0-{timestamp}.gputrace"));-- capture_manager_descriptor.set_output_url(trace_path);-- mtl_context.command_queue.set_label("uzu_command_queue");- capture_manager_descriptor- .set_capture_command_queue(&mtl_context.command_queue);-- let feeds: HashMap<&Tensor, &ShapedType> = HashMap::new();- let optimization_level = Optimization::Level1;- let optimization_profile = OptimizationProfile::Performance;-- let compilation_descriptor = make_compilation_descriptor(- BlockDevice::Ane,- optimization_level,- optimization_profile,- print_placement_analysis,- );-- let executable = graph.compile(- &MPSDevice::with_device(&mtl_context.device),- &feeds,- &[&dequantized_weights],- None,- Some(&compilation_descriptor),- );-- if print_executable_dump {- executable.dump();- }-- if dump_mpsgraphpackage {- let package_path = root_dir(NSSearchPathDirectory::Downloads).join(- format!("dequantize_qkv_layer0-{timestamp}.mpsgraphpackage"),- );- let serialization_desc = ExecutableSerializationDescriptor::new();- executable.serialize_to_graph_package(- &package_path,- Some(&serialization_desc),- );- println!("MPSGraph package saved to: {:?}", package_path);- }-- if dump_gpu_trace {- if let Err(err) =- capture_manager.start_capture(&capture_manager_descriptor)- {- eprintln!("⚠️ Failed to start GPU capture: {err}");- } else {- println!("GPU capture started successfully");- }- }-- let command_buffer =- CommandBuffer::from_command_queue(&mtl_context.command_queue);- let root_command_buffer =- command_buffer.root_command_buffer().to_owned();-- let execution_descriptor = ExecutableExecutionDescriptor::new();-- executable.encode_to_command_buffer(- &command_buffer,- &[],- Some(&[&dequantized_weights_tensor_data]),- Some(&*execution_descriptor),- );- command_buffer.commit_and_continue();-- root_command_buffer.wait_until_completed();-- if dump_gpu_trace {- capture_manager.stop_capture();- }-- let result = dequantized_weights_buffer- .as_view::<half::f16>()- .map_err(|_| ExampleError::ResultView)?;-- if dump_result_array {- let result_path = root_dir(NSSearchPathDirectory::Downloads)- .join(format!("dequantize_qkv_layer0-{timestamp}.txt"));- std::fs::write(&result_path, format!("{:?}", result))?;- println!("Result array dumped to {:?}", result_path);- }-- Ok(())- })-}diff --git a/crates/uzu/examples/quant_matmul_experiments.rs b/crates/uzu/examples/quant_matmul_experiments.rsdeleted file mode 100644index ac7577dd5..000000000--- a/crates/uzu/examples/quant_matmul_experiments.rs+++ /dev/null@@ -1,587 +0,0 @@-use std::{- collections::HashMap,- process::exit,- rc::Rc,- time::{Instant, SystemTime, UNIX_EPOCH},-};--use clap::Parser;-use half::f16;-use metal::Device;-use mpsgraph::{- CommandBuffer, CompilationDescriptor, DataType as MPSDataType,- DequantizationArguments, Device as MPSDevice, Executable,- ExecutableExecutionDescriptor, ExecutableSerializationDescriptor, Graph,- Optimization, OptimizationProfile, ShapedType, Tensor, TensorData,-};-use objc2::rc::{Retained, autoreleasepool};-use thiserror::Error;-use uzu::{- Array, DataType, DeviceContext,- backends::metal::{- MTLContext, MetalArray,- compilation_parameters::{BlockDevice, make_compilation_descriptor},- },- storage::{NSSearchPathDirectory, root_dir},-};--#[derive(Parser)]-#[command(- name = "quant_matmul_experiments",- about = "Synthetic experiment: dequantize weights then matmul, measure execution time"-)]-struct Args {- /// Print the compiled executable IR / debug dump- #[arg(long = "print-exec-dump")]- print_exec_dump: bool,-- /// Print ANE/GPU placement analysis during compilation- #[arg(long = "print-placement-analysis")]- print_placement_analysis: bool,-- /// Serialize the compiled MPSGraph executable to a .mpsgraphpackage file- #[arg(long = "dump-mpsgraphpackage")]- dump_mpsgraphpackage: bool,-}--#[derive(Debug, Error)]-pub enum ExampleError {- #[error(transparent)]- Metal(#[from] uzu::backends::metal::error::MTLError),- #[error("No Metal device available")]- NoDevice,- #[error("Graph dequantization failed")]- Dequantization,-}--// ================== Quantized Matmul Helpers =====================--#[derive(Debug, Clone, Copy)]-enum TensorLoadType {- Baked,- RuntimeLoaded,-}--#[derive(Debug, Clone, Copy)]-struct QuantizedMatmulDescriptor {- weights: TensorLoadType,- scales: TensorLoadType,- zeros: TensorLoadType,- transposed_shapes: bool,- device: BlockDevice,- optimization_level: Optimization,- optimization_profile: OptimizationProfile,-}--#[derive(Debug, Clone)]-enum TensorOption {- Constant(Retained<Tensor>),- Placeholder {- placeholder: Retained<Tensor>,- data: Retained<TensorData>,- },-}--impl TensorOption {- #[inline]- fn tensor(&self) -> Retained<Tensor> {- match self {- Self::Constant(t) => t.clone(),- Self::Placeholder {- placeholder,- ..- } => placeholder.clone(),- }- }-- fn feeds(&self) -> HashMap<Retained<Tensor>, Retained<ShapedType>> {- match self {- TensorOption::Constant(_) => HashMap::new(),- TensorOption::Placeholder {- placeholder,- ..- } => HashMap::from_iter([(- placeholder.clone(),- placeholder.shaped_type(),- )]),- }- }-- fn data_feeds(&self) -> HashMap<Retained<Tensor>, Retained<TensorData>> {- match self {- TensorOption::Constant(_) => HashMap::new(),- TensorOption::Placeholder {- placeholder,- data,- } => HashMap::from_iter([(placeholder.clone(), data.clone())]),- }- }-}--#[derive(Debug, Clone)]-struct QMTensorOptions {- input: TensorOption,- weights: TensorOption,- scales: TensorOption,- zeros: TensorOption,-}--impl QMTensorOptions {- fn feeds(&self) -> HashMap<Retained<Tensor>, Retained<ShapedType>> {- HashMap::from_iter(- [&self.input, &self.weights, &self.scales, &self.zeros]- .iter()- .flat_map(|opt| opt.feeds()),- )- }-- fn data_feeds(&self) -> HashMap<Retained<Tensor>, Retained<TensorData>> {- HashMap::from_iter(- [&self.input, &self.weights, &self.scales, &self.zeros]- .iter()- .flat_map(|opt| opt.data_feeds()),- )- }-}--fn build_quantized_matmul(- graph: &Graph,- tensor_options: &QMTensorOptions,- mtl_context: &Rc<MTLContext>,- compilation_descriptor: &CompilationDescriptor,- transposed_shapes: bool,-) -> Result<Retained<Executable>, ExampleError> {- let i = tensor_options.input.tensor();- let w = tensor_options.weights.tensor();- let s = tensor_options.scales.tensor();- let zp = tensor_options.zeros.tensor();-- let dequantized_weights = graph.dequantize(- &w,- DequantizationArguments::ScaleTensorZeroPointTensorDataType {- scale_tensor: &s,- zero_point_tensor: &zp,- data_type: MPSDataType::Float16,- },- None,- );-- let matmul = if transposed_shapes {- graph.matrix_multiplication(- &i,- &graph.transpose(&dequantized_weights, &[1, 0], None),- None,- )- } else {- graph.matrix_multiplication(- &graph.transpose(&dequantized_weights, &[1, 0], None),- &graph.transpose(&i, &[1, 0], None),- None,- )- };-- let device = MPSDevice::with_device(&mtl_context.device);-- let retained_feeds = tensor_options.feeds();- let feeds = retained_feeds.iter().map(|(t, st)| (&**t, &**st)).collect();-- let executable = graph.compile(- &device,- &feeds,- &[&*matmul],- None,- Some(compilation_descriptor),- );-- Ok(executable)-}--fn make_tensor_option(- graph: &Graph,- load_type: TensorLoadType,- array: &mut MetalArray,- shape: &[usize],- dtype: DataType,- name: &str,-) -> TensorOption {- match load_type {- TensorLoadType::Baked => {- let tensor_const =- graph.constant_with_data(array.buffer(), shape, dtype.into());- TensorOption::Constant(tensor_const)- },- TensorLoadType::RuntimeLoaded => {- let td = unsafe { array.to_mps_tensor_data() };- let shape_isize: Vec<isize> =- shape.iter().map(|d| *d as isize).collect();- let placeholder =- graph.placeholder(Some(&shape_isize), dtype.into(), Some(name));- TensorOption::Placeholder {- placeholder,- data: td,- }- },- }-}--// ================== End helpers =====================--// ================== Utility to run a single descriptor =====================--fn run_quant_matmul(- qm_descriptor: &QuantizedMatmulDescriptor,- args: &Args,-) -> Result<f64, ExampleError> {- // Constants that do not change between runs- let suffix_length: usize = 128;- let input_dim: usize = 2560;- let output_dim: usize = 3072 * 2;- let group_size: usize = 20;-- // Shapes that depend on descriptor options- let weights_shape = if qm_descriptor.transposed_shapes {- [output_dim, input_dim]- } else {- [input_dim, output_dim]- };- let scales_shape = if qm_descriptor.transposed_shapes {- [output_dim, input_dim / group_size]- } else {- [input_dim / group_size, output_dim]- };- let zero_points_shape = scales_shape;- let input_shape = [-1_i64, input_dim as i64];-- // Concrete shapes for actual buffer allocation at runtime- let input_tensor_shape = [suffix_length, input_dim];- let result_tensor_shape = [suffix_length, output_dim];-- // --- Device & context ---- let device = Device::system_default().ok_or(ExampleError::NoDevice)?;- let command_queue = device.new_command_queue();- let mtl_context = Rc::new(MTLContext::new(device, command_queue)?);-- // --- Arrays ---- let mut weights_array = unsafe {- mtl_context.array_uninitialized(&weights_shape, DataType::U4)- };- weights_array.buffer_mut().fill(0x11);-- let mut scales_array =- mtl_context.array_from_elem(&scales_shape, f16::from_f32(1.0));-- let mut zero_points_array = unsafe {- mtl_context.array_uninitialized(&zero_points_shape, DataType::U4)- };- zero_points_array.buffer_mut().fill(0x00);-- let mut input_array =- mtl_context.array_from_elem(&input_tensor_shape, f16::from_f32(1.0));-- let mut result_array = unsafe {- mtl_context.array_uninitialized(&result_tensor_shape, DataType::F16)- };-- // --- Graph ---- let graph = Graph::new();-- // --- Tensor options for quantized matmul ---- let input_tensor_data = unsafe { input_array.to_mps_tensor_data() };- let input_shape_isize: Vec<isize> =- input_shape.iter().map(|d| *d as isize).collect();- let input_placeholder = graph.placeholder(- Some(&input_shape_isize),- DataType::F16.into(),- Some("input_ph"),- );-- let tensor_options = QMTensorOptions {- input: TensorOption::Placeholder {- placeholder: input_placeholder,- data: input_tensor_data,- },- weights: make_tensor_option(- &graph,- qm_descriptor.weights,- &mut weights_array,- &weights_shape,- DataType::U4,- "weights_ph",- ),- scales: make_tensor_option(- &graph,- qm_descriptor.scales,- &mut scales_array,- &scales_shape,- DataType::F16,- "scales_ph",- ),- zeros: make_tensor_option(- &graph,- qm_descriptor.zeros,- &mut zero_points_array,- &zero_points_shape,- DataType::U4,- "zeros_ph",- ),- };-- let compilation_descriptor = make_compilation_descriptor(- qm_descriptor.device,- qm_descriptor.optimization_level,- qm_descriptor.optimization_profile,- args.print_placement_analysis,- );-- let executable = build_quantized_matmul(- &graph,- &tensor_options,- &mtl_context,- &compilation_descriptor,- qm_descriptor.transposed_shapes,- )?;-- // Optional printing / dumping per flags- if args.print_exec_dump {- executable.dump();- }-- if args.dump_mpsgraphpackage {- let timestamp = SystemTime::now()- .duration_since(UNIX_EPOCH)- .expect("Time went backwards")- .as_secs();- let package_path = root_dir(NSSearchPathDirectory::Downloads).join(- format!("quant_matmul_experiments-{timestamp}.mpsgraphpackage"),- );- let serialization_desc = ExecutableSerializationDescriptor::new();- executable.serialize_to_graph_package(- &package_path,- Some(&serialization_desc),- );- }-- // --- Run & measure ---- let result_tensor_data = unsafe { result_array.to_mps_tensor_data() };- let retained_data_feeds = tensor_options.data_feeds();- let inputs: Vec<&TensorData> = match executable.feed_tensors() {- Some(feed_tensors) => feed_tensors- .iter()- .map(|t| {- retained_data_feeds- .get(t)- .expect("TensorData for feed tensor not found")- })- .map(|td| &**td)- .collect(),- None => Vec::new(),- };-- let outputs = [&*result_tensor_data];-- // Warm-up- run_once(&mtl_context, &executable, &inputs, &outputs);-- const NUM_ITERATIONS: usize = 1000;- let mut total_duration = 0.0f64; // milliseconds-- for _ in 0..NUM_ITERATIONS {- let start = Instant::now();- run_once(&mtl_context, &executable, &inputs, &outputs);- total_duration += start.elapsed().as_secs_f64() * 1e3;- }-- Ok(total_duration / NUM_ITERATIONS as f64)-}--// ================== End utility =====================--// cargo run --example quant_matmul_experiments -- --print-exec-dump --print-placement-analysis --dump-mpsgraphpackage--fn main() -> Result<(), ExampleError> {- autoreleasepool(|_| {- let args = Args::parse();-- let descriptor = QuantizedMatmulDescriptor {- weights: TensorLoadType::Baked,- scales: TensorLoadType::Baked,- zeros: TensorLoadType::RuntimeLoaded,- transposed_shapes: true,- device: BlockDevice::Ane,- optimization_level: Optimization::Level0,- optimization_profile: OptimizationProfile::PowerEfficiency,- };-- let avg_ms = run_quant_matmul(&descriptor, &args)?;-- println!("Time: {:.3} ms", avg_ms);-- exit(0);-- // ---------------- Collect results ------------------- #[derive(Debug)]- struct ReportRow {- weights: TensorLoadType,- scales: TensorLoadType,- zeros: TensorLoadType,- transposed_shapes: bool,- device: BlockDevice,- optimization_level: Optimization,- optimization_profile: OptimizationProfile,- time_ms: f64,- }-- #[allow(unreachable_code)]- let tensor_variants =- [TensorLoadType::Baked, TensorLoadType::RuntimeLoaded];- let bool_variants = [true];- let device_variants = [BlockDevice::Ane];- let opt_levels = [Optimization::Level0, Optimization::Level1];- let opt_profiles = [- OptimizationProfile::Performance,- OptimizationProfile::PowerEfficiency,- ];-- let mut rows: Vec<ReportRow> = Vec::new();-- for &weights in &tensor_variants {- for &scales in &tensor_variants {- for &zeros in &tensor_variants {- for &transposed_shapes in &bool_variants {- for &device in &device_variants {- for &optimization_level in &opt_levels {- for &optimization_profile in &opt_profiles {- let descriptor =- QuantizedMatmulDescriptor {- weights,- scales,- zeros,- transposed_shapes,- device,- optimization_level,- optimization_profile,- };-- let avg_ms =- run_quant_matmul(&descriptor, &args)?;-- rows.push(ReportRow {- weights,- scales,- zeros,- transposed_shapes,- device,- optimization_level,- optimization_profile,- time_ms: avg_ms,- });- }- }- }- }- }- }- }-- // ---------------- Sort by time ------------------ rows.sort_by(|a, b| a.time_ms.partial_cmp(&b.time_ms).unwrap());-- // ---------------- Compute column widths ------------------ let headers = [- "Weights",- "Scales",- "Zero Points",- "Transposed Shapes",- "BlockDevice",- "Optimization",- "Profile",- "Time (ms)",- ];-- let mut widths: Vec<usize> = headers.iter().map(|h| h.len()).collect();-- for row in &rows {- widths[0] = widths[0].max(format!("{:?}", row.weights).len());- widths[1] = widths[1].max(format!("{:?}", row.scales).len());- widths[2] = widths[2].max(format!("{:?}", row.zeros).len());- widths[3] =- widths[3].max(format!("{}", row.transposed_shapes).len());- widths[4] = widths[4].max(format!("{:?}", row.device).len());- widths[5] =- widths[5].max(format!("{:?}", row.optimization_level).len());- widths[6] =- widths[6].max(format!("{:?}", row.optimization_profile).len());- widths[7] = widths[7].max(format!("{:.3}", row.time_ms).len());- }-- // ---------------- Print header ------------------ println!(- "{:<w0$} {:<w1$} {:<w2$} {:<w3$} {:<w4$} {:<w5$} {:<w6$} {:>w7$}",- headers[0],- headers[1],- headers[2],- headers[3],- headers[4],- headers[5],- headers[6],- headers[7],- w0 = widths[0] + 1,- w1 = widths[1] + 1,- w2 = widths[2] + 1,- w3 = widths[3] + 1,- w4 = widths[4] + 1,- w5 = widths[5] + 1,- w6 = widths[6] + 1,- w7 = widths[7],- );-- // ---------------- Print rows ------------------ for row in rows {- println!(- "{:<w0$} {:<w1$} {:<w2$} {:<w3$} {:<w4$} {:<w5$} {:<w6$} {:>w7$.3}",- format!("{:?}", row.weights),- format!("{:?}", row.scales),- format!("{:?}", row.zeros),- row.transposed_shapes,- format!("{:?}", row.device),- format!("{:?}", row.optimization_level),- format!("{:?}", row.optimization_profile),- row.time_ms,- w0 = widths[0] + 1,- w1 = widths[1] + 1,- w2 = widths[2] + 1,- w3 = widths[3] + 1,- w4 = widths[4] + 1,- w5 = widths[5] + 1,- w6 = widths[6] + 1,- w7 = widths[7],- );- }-- Ok(())- })-}--fn run_once(- mtl_context: &Rc<MTLContext>,- executable: &Executable,- inputs: &[&TensorData],- outputs: &[&TensorData],-) {- let command_buffer =- CommandBuffer::from_command_queue(&mtl_context.command_queue);- let root_command_buffer = command_buffer.root_command_buffer().to_owned();-- let exec_desc = ExecutableExecutionDescriptor::new();-- executable.encode_to_command_buffer(- &command_buffer,- inputs,- Some(outputs),- Some(&exec_desc),- );-- command_buffer.commit_and_continue();- root_command_buffer.wait_until_completed();-}diff --git a/crates/uzu/examples/quick_start.rs b/crates/uzu/examples/quick_start_chat.rssimilarity index 79%rename from crates/uzu/examples/quick_start.rsrename to crates/uzu/examples/quick_start_chat.rsindex 681fd5a39..d9cd0a4ea 100644--- a/crates/uzu/examples/quick_start.rs+++ b/crates/uzu/examples/quick_start_chat.rs@@ -1,14 +1,14 @@-use std::path::PathBuf;+use std::{env, path::PathBuf};use uzu::session::{- Session,+ ChatSession,config::{DecodingConfig, RunConfig},types::{Input, Output},};fn main() -> Result<(), Box<dyn std::error::Error>> {let model_path = PathBuf::from("MODEL_PATH");- let mut session = Session::new(model_path, DecodingConfig::default())?;+ let mut session = ChatSession::new(model_path, DecodingConfig::default())?;let input = Input::Text(String::from("Tell about London"));let output = session.run(diff --git a/crates/uzu/src/backends/metal/array.rs b/crates/uzu/src/backends/metal/array.rsindex 6b854a0b0..5ecbcea0d 100644--- a/crates/uzu/src/backends/metal/array.rs+++ b/crates/uzu/src/backends/metal/array.rs@@ -7,16 +7,11 @@ use objc2::rc::Retained;use crate::{Array, ArrayElement, DataType, array::array_size_in_bytes};-/// Represents an n-dimensional array for Metal computation#[derive(Debug, Clone)]pub struct MetalArray {- /// Metal buffer containing the array databuffer: MTLBuffer,- /// Shape of the array (dimensions)shape: Box<[usize]>,- /// Data type of the elementsdata_type: DataType,- /// Byte offset into the buffer (for indexed async buffers)offset: usize,}@@ -49,7 +44,6 @@ impl Array for MetalArray {}impl MetalArray {- /// Create a new Array at offset 0pub unsafe fn new(buffer: MTLBuffer,shape: &[usize],@@ -58,8 +52,6 @@ impl MetalArray {unsafe { Self::new_with_offset(buffer, shape, data_type, 0) }}- /// Create a new Array at a byte offset into the buffer.- /// Used for indexed async buffers where each pass uses a different offset.pub unsafe fn new_with_offset(buffer: MTLBuffer,shape: &[usize],@@ -84,12 +76,10 @@ impl MetalArray {}}- /// Returns the byte offset into the underlying bufferpub fn buffer_offset(&self) -> usize {self.offset}- /// Wraps the underlying MTLBuffer into MPSTensorData for use with MPSGraph.pub unsafe fn to_mps_tensor_data(&mut self) -> Retained<TensorData> {TensorData::new_with_mtl_buffer(&self.buffer,@@ -99,9 +89,6 @@ impl MetalArray {)}- /// Returns the underlying MTLBuffer.- /// NOTE: For arrays created with `new_with_offset`, callers must use `buffer_offset()`- /// and pass it to `encoder.set_buffer(..., offset)` - Metal has no sub-buffer concept.pub unsafe fn mtl_buffer(&mut self) -> &MTLBuffer {&self.buffer}diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsnew file mode 100644index 000000000..adf8f17c2--- /dev/null+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -0,0 +1,471 @@+use std::{+ cell::RefCell, collections::HashMap, fs::File, io::BufReader, path::Path,+ rc::Rc,+};++use mpsgraph::{CommandBuffer as MPSCommandBuffer, Device as MPSDevice, Graph};+use objc2::rc::{Retained, autoreleasepool};++use super::{+ KernelDataType, MTLContext, ModelShape,+ compilation_parameters::CompilationConfig,+ encodable_block::{+ ClassifierLayer, ClassifierPredictionHead, Normalization, Pooling,+ Rope,+ transformer_layer::{embed_block, linear_block},+ },+ forward_pass::{+ ArrayId, EncodableBlock, IOArrays, MPSGraphBlock, RopeType,+ ScratchBuffers, SharedBuffers,+ },+ graph::{common::activation, placeholder, shaped_type},+ kernel::{PoolingKernel, SigmoidKernel},+};+use crate::{+ DataType,+ backends::metal::error::ClassifierError,+ config::{ClassifierModelConfig, ModelMetadata},+ parameters::ParameterLoader,+ session::types::Error,+};++pub struct ClassifierContext {+ pub mtl_context: Rc<MTLContext>,+ pub command_buffer: Retained<MPSCommandBuffer>,++ pub shared_buffers: Rc<RefCell<SharedBuffers>>,+ pub scratch_buffers: ScratchBuffers,++ pub model_config: ClassifierModelConfig,+ pub model_shape: ModelShape,++ pub embed: Box<dyn EncodableBlock>,+ pub embedding_norm: Normalization,+ pub layers: Box<[ClassifierLayer]>,+ pub output_norm: Normalization,+ pub global_rope: Rc<Box<dyn EncodableBlock>>,+ pub local_rope: Option<Rc<Box<dyn EncodableBlock>>>,++ pub pooling: Box<dyn EncodableBlock>,+ pub prediction_head: Box<dyn EncodableBlock>,++ pub sigmoid_kernel: SigmoidKernel,+}++impl ClassifierContext {+ pub fn new(model_path: &Path) -> Result<Self, Error> {+ let mtl_device = metal::Device::system_default()+ .ok_or(Error::UnableToCreateMetalContext)?;+ let mtl_command_queue =+ mtl_device.new_command_queue_with_max_command_buffer_count(1024);++ let command_buffer =+ MPSCommandBuffer::from_command_queue(&mtl_command_queue);++ let config_path = model_path.join("config.json");+ if !config_path.exists() {+ return Err(Error::UnableToLoadConfig);+ }+ let config_file =+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;+ let model_metadata: ModelMetadata =+ serde_json::from_reader(BufReader::new(config_file))+ .map_err(|_| Error::UnableToLoadConfig)?;++ let classifier_model_config = model_metadata+ .model_config+ .as_classifier()+ .ok_or(Error::UnableToLoadConfig)?;++ let decoder_config = Rc::new(+ classifier_model_config+ .model_config+ .to_decoder_config()+ .map_err(|_| Error::UnableToLoadConfig)?,+ );+ let model_shape = ModelShape::from_decoder_config(&decoder_config);++ let mtl_context = Rc::new(+ MTLContext::new(mtl_device, mtl_command_queue)+ .map_err(|_| Error::UnableToCreateMetalContext)?,+ );++ let compilation_config = Rc::new(CompilationConfig::default());+ let weights_path = model_path.join("model.safetensors");+ if !weights_path.exists() {+ return Err(Error::UnableToLoadWeights);+ }+ let weights_file = File::open(&weights_path)+ .map_err(|_| Error::UnableToLoadWeights)?;+ let loader = ParameterLoader::new(&weights_file, &mtl_context)+ .map_err(|_| Error::UnableToLoadWeights)?;+ let root_loader_view = loader.tree();++ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(+ &mtl_context,+ &decoder_config,+ &model_shape,+ )));+ let transformer_tree =+ root_loader_view.subtree("transformer").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(+ "transformer".to_string(),+ ))+ })?;++ {+ let mut shared_bufs = shared_buffers.borrow_mut();+ shared_bufs.embeddings.update_data(&root_loader_view);+ if let Some(global_rope) = &mut shared_bufs.global_rope {+ global_rope.update_data(+ &transformer_tree,+ String::from("global_rope"),+ );+ }+ if let Some(local_rope) = &mut shared_bufs.local_rope {+ local_rope+ .update_data(&transformer_tree, String::from("local_rope"));+ }+ }++ let data_type = decoder_config+ .layer_config+ .mixer_config+ .as_attention()+ .ok_or(Error::Classifier(ClassifierError::NonAttentionMixer))?+ .qkv_projection_config+ .activation_precision()+ .into();++ let embed = embed_block(+ &decoder_config,+ &mtl_context,+ &compilation_config.descriptor_general,+ &root_loader_view,+ );++ let global_rope =+ Self::create_rope_block(&mtl_context, data_type, RopeType::Global)+ .map_err(Error::Classifier)?;+ let local_rope = classifier_model_config+ .model_config+ .transformer_config+ .local_rope_config+ .as_ref()+ .map(|_| {+ Self::create_rope_block(+ &mtl_context,+ data_type,+ RopeType::Local,+ )+ })+ .transpose()+ .map_err(Error::Classifier)?;++ let layers = classifier_model_config+ .model_config+ .transformer_config+ .layer_configs+ .iter()+ .enumerate()+ .map(|(layer_index, layer_config)| {+ let mut rope = global_rope.clone();+ let attn = layer_config+ .attention_config()+ .ok_or(ClassifierError::NonAttentionMixer)?;++ if attn.sliding_window_size.is_some() {+ if let Some(local_rope_block) = local_rope.clone() {+ rope = local_rope_block;+ }+ }++ let num_heads = attn.num_heads.ok_or_else(|| {+ ClassifierError::MissingConfigField(format!(+ "num_heads in layer {}",+ layer_index+ ))+ })?;+ let head_dim = attn.head_dim.ok_or_else(|| {+ ClassifierError::MissingConfigField(format!(+ "head_dim in layer {}",+ layer_index+ ))+ })?;+ let num_groups = attn.num_groups.ok_or_else(|| {+ ClassifierError::MissingConfigField(format!(+ "num_groups in layer {}",+ layer_index+ ))+ })?;++ let layer_tree = transformer_tree+ .subtree(&format!("layers.{}", layer_index))+ .map_err(|_| {+ ClassifierError::WeightSubtreeNotFound(format!(+ "layers.{}",+ layer_index+ ))+ })?;++ Ok(ClassifierLayer::new(+ &mtl_context,+ layer_config,+ compilation_config.clone(),+ layer_index,+ classifier_model_config.model_config.model_dim,+ classifier_model_config+ .model_config+ .transformer_config+ .hidden_dim,+ num_heads,+ head_dim,+ num_groups,+ attn.scale,+ &layer_tree,+ rope,+ ))+ })+ .collect::<Result<Vec<_>, ClassifierError>>()+ .map_err(Error::Classifier)?+ .into_boxed_slice();++ let output_norm_tree =+ transformer_tree.subtree("output_norm").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(+ "output_norm".to_string(),+ ))+ })?;+ let output_norm = Normalization::new(+ &mtl_context,+ data_type,+ classifier_model_config+ .model_config+ .transformer_config+ .output_norm_config+ .clone(),+ ArrayId::Main,+ ArrayId::Main,+ &output_norm_tree,+ )+ .map_err(|e| {+ Error::Classifier(ClassifierError::KernelCreationFailed(format!(+ "output norm: {:?}",+ e+ )))+ })?;++ let context_length =+ classifier_model_config.model_config.context_length;+ let scratch_buffers = ScratchBuffers::new(+ &mtl_context,+ &decoder_config,+ &model_shape,+ context_length,+ context_length,+ );++ let embedding_norm_tree =+ root_loader_view.subtree("embedding_norm").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(+ "embedding_norm".to_string(),+ ))+ })?;+ let embedding_norm = Normalization::new(+ &mtl_context,+ data_type,+ classifier_model_config.model_config.embedding_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &embedding_norm_tree,+ )+ .map_err(|e| {+ Error::Classifier(ClassifierError::KernelCreationFailed(format!(+ "embedding norm: {:?}",+ e+ )))+ })?;++ let model_dim = classifier_model_config.model_config.model_dim;+ let num_labels = classifier_model_config.model_config.num_labels;+ let prediction_head_config =+ &classifier_model_config.model_config.prediction_head_config;+ let prediction_head_tree =+ root_loader_view.subtree("prediction_head").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(+ "prediction_head".to_string(),+ ))+ })?;++ let prediction_head_data_type: DataType =+ prediction_head_config.dense_config.activation_precision().into();++ let prediction_head_dense_tree =+ prediction_head_tree.subtree("dense").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(+ "prediction_head.dense".to_string(),+ ))+ })?;+ let prediction_head_dense = linear_block::<1>(+ &prediction_head_config.dense_config,+ prediction_head_config.use_dense_bias,+ model_dim,+ [model_dim],+ &mtl_context,+ &prediction_head_dense_tree,+ ArrayId::ClassifierPooling,+ ArrayId::ClassifierPredictionHeadDense,+ &compilation_config.descriptor_general,+ );++ let prediction_head_activation = autoreleasepool(|_| {+ let graph = Graph::new();+ let input_shape = [-1, model_dim as isize];+ let input_placeholder =+ placeholder(&graph, &input_shape, prediction_head_data_type);++ let output = activation(+ &graph,+ &prediction_head_config.activation,+ &input_placeholder,+ prediction_head_data_type,+ );++ let shaped_type_obj =+ shaped_type(&input_shape, prediction_head_data_type);+ let feeds =+ HashMap::from([(&*input_placeholder, &*shaped_type_obj)]);++ let executable = graph.compile(+ &MPSDevice::with_device(&mtl_context.device),+ &feeds,+ &[&output],+ None,+ Some(&compilation_config.descriptor_general),+ );++ let arguments = IOArrays::new(+ vec![ArrayId::ClassifierPredictionHeadDense].into_boxed_slice(),+ vec![ArrayId::ClassifierPredictionHeadDense].into_boxed_slice(),+ );++ let execution_descriptor =+ mpsgraph::ExecutableExecutionDescriptor::new();+ execution_descriptor.set_enable_commit_and_continue(true);++ Box::new(MPSGraphBlock::new(+ executable,+ execution_descriptor,+ arguments,+ )) as Box<dyn EncodableBlock>+ });++ let prediction_head_norm_tree =+ prediction_head_tree.subtree("norm").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(+ "prediction_head.norm".to_string(),+ ))+ })?;+ let prediction_head_norm = Normalization::new(+ &mtl_context,+ prediction_head_data_type,+ prediction_head_config.normalization_config.clone(),+ ArrayId::ClassifierPredictionHeadDense,+ ArrayId::ClassifierPredictionHeadNorm,+ &prediction_head_norm_tree,+ )+ .map_err(|e| {+ Error::Classifier(ClassifierError::KernelCreationFailed(format!(+ "prediction head norm: {:?}",+ e+ )))+ })?;++ let prediction_head_readout_tree =+ prediction_head_tree.subtree("readout").map_err(|_| {+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(+ "prediction_head.readout".to_string(),+ ))+ })?;+ let prediction_head_final_linear = linear_block::<1>(+ &prediction_head_config.readout_config,+ true,+ model_dim,+ [num_labels],+ &mtl_context,+ &prediction_head_readout_tree,+ ArrayId::ClassifierPredictionHeadNorm,+ ArrayId::ClassifierPredictionHeadLogits,+ &compilation_config.descriptor_general,+ );++ let pooling_kernel = PoolingKernel::new(&mtl_context, data_type)+ .map_err(|e| {+ eprintln!("Failed to create pooling kernel: {:?}", e);+ Error::UnableToCreateMetalContext+ })?;+ let sigmoid_kernel = SigmoidKernel::new(&mtl_context, data_type)+ .map_err(|e| {+ eprintln!("Failed to create sigmoid kernel: {:?}", e);+ Error::UnableToCreateMetalContext+ })?;++ let pooling = Box::new(Pooling::new(+ pooling_kernel,+ classifier_model_config.model_config.classifier_pooling.clone(),+ model_dim,+ ));++ let prediction_head = Box::new(ClassifierPredictionHead::new(+ prediction_head_dense,+ prediction_head_activation,+ Box::new(prediction_head_norm),+ prediction_head_final_linear,+ num_labels,+ ));++ Ok(Self {+ mtl_context,+ command_buffer,+ shared_buffers,+ scratch_buffers,+ model_config: classifier_model_config.clone(),+ model_shape,+ embed,+ embedding_norm,+ layers,+ output_norm,+ global_rope,+ local_rope,+ pooling,+ prediction_head,+ sigmoid_kernel,+ })+ }++ fn create_rope_block(+ mtl_context: &MTLContext,+ data_type: DataType,+ rope_type: RopeType,+ ) -> Result<Rc<Box<dyn EncodableBlock>>, ClassifierError> {+ let kernel_data_type: KernelDataType = data_type.into();++ let rotation: Box<dyn EncodableBlock> = Box::new(+ Rope::new(mtl_context, kernel_data_type, rope_type).map_err(+ |e| {+ ClassifierError::KernelCreationFailed(format!(+ "RoPE: {:?}",+ e+ ))+ },+ )?,+ );+ Ok(Rc::new(rotation))+ }++ pub fn reset_command_buffer(&mut self) {+ self.command_buffer = MPSCommandBuffer::from_command_queue(+ &self.mtl_context.command_queue,+ );+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rsnew file mode 100644index 000000000..91137f568--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs@@ -0,0 +1,340 @@+//! Attention kernel encodable.++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters};+use crate::backends::metal::{+ KernelDataType, MTLContext,+ forward_pass::{ArrayId, ForwardPassState, HashMapId},+ kernel::attention::{+ AttentionError, AttentionKernel, AttentionKernelVariant,+ AttentionSinglePassArguments, AttentionTwoPassArguments,+ KVCacheUpdateArguments,+ },+};++pub struct Attention {+ kernel: AttentionKernel,+ layer_index: usize,+ attention_scale: Option<f32>,+ has_sinks: bool,+ is_causal: bool,+ sliding_window_size: Option<usize>,+}++impl Attention {+ pub fn new(+ context: &MTLContext,+ data_type: KernelDataType,+ layer_index: usize,+ attention_scale: Option<f32>,+ has_sinks: bool,+ is_causal: bool,+ sliding_window_size: Option<usize>,+ ) -> Result<Self, AttentionError> {+ let kernel = AttentionKernel::new(context, data_type)?;+ Ok(Self {+ kernel,+ layer_index,+ attention_scale,+ has_sinks,+ is_causal,+ sliding_window_size,+ })+ }+}++impl EncodableBlock for Attention {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let (+ suffix_length,+ num_heads,+ head_dim,+ num_groups,+ max_sequence_length,+ ) = {+ use crate::Array;+ let qkv_binding = state.arrays(&[ArrayId::QKV]);+ let qkv_array = qkv_binding[0].borrow();+ let suffix_length = Array::shape(&*qkv_array)[0];++ let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);+ let queries_array = queries_binding[0].borrow();+ let num_heads = Array::shape(&*queries_array)[0];+ let head_dim = Array::shape(&*queries_array)[2];++ let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);+ let keys_array = keys_binding[0].borrow();+ let num_groups = Array::shape(&*keys_array)[0];++ let max_sequence_length = if let Some(_kv) = state.cache_layers() {+ let key_cache_binding =+ state.arrays(&[ArrayId::Keys(self.layer_index)]);+ let key_cache_array = key_cache_binding[0].borrow();+ Array::shape(&*key_cache_array)[1]+ } else {+ // For classifiers without KV cache, max_sequence_length is just suffix_length+ suffix_length+ };++ (+ suffix_length,+ num_heads,+ head_dim,+ num_groups,+ max_sequence_length,+ )+ };++ let (segment_prefix_length, window_length) =+ if let Some(cache_layers) = state.cache_layers() {+ let cache = cache_layers.borrow();+ let layer = cache.data[self.layer_index]+ .as_transformer()+ .expect("Attention kernel expects transformer layer state");+ (+ layer.projected_segment_prefix_length(+ parameters.projection_step.unwrap_or(0),+ ),+ layer.window_length(),+ )+ } else {+ // For classifiers without KV cache: no prefix, use configured sliding window+ (0, self.sliding_window_size)+ };++ let sequence_length = segment_prefix_length + suffix_length;++ let gqa_factor = num_heads / num_groups;+ let scale =+ self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());++ let variant = self.kernel.choose_variant(+ sequence_length,+ head_dim,+ self.is_causal,+ );++ let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);+ let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);+ let qkv_binding = state.arrays(&[ArrayId::QKV]);+ let attention_bias_binding =+ state.hashmaps(&[HashMapId::AttentionBias]);+ let attention_output_binding =+ state.arrays(&[ArrayId::AttentionOutput]);++ let attention_bias_map = attention_bias_binding[0].clone();+ let mask_kv_seq_stride = 1;+ let mask_q_seq_stride = sequence_length as i32;+ let mask_head_stride = 0;+ let sinks_binding = if self.has_sinks {+ Some(state.arrays(&[ArrayId::AttentionSinks(self.layer_index)]))+ } else {+ None+ };++ let mut rotated_keys_array = rotated_keys_binding[0].borrow_mut();+ let rotated_keys_buffer = unsafe { rotated_keys_array.mtl_buffer() };++ let mut qkv_array = qkv_binding[0].borrow_mut();+ let qkv_buffer = unsafe { qkv_array.mtl_buffer() };++ // Prepare command encoder early (used for optional value extraction)+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();++ // Get KV cache buffers only if KV cache exists (LLM mode)+ let has_kv_cache = state.cache_layers().is_some();+ let (key_cache_buffer, value_cache_buffer) = if has_kv_cache {+ let key_cache_binding =+ state.arrays(&[ArrayId::Keys(self.layer_index)]);+ let value_cache_binding =+ state.arrays(&[ArrayId::Values(self.layer_index)]);++ let mut key_cache_array = key_cache_binding[0].borrow_mut();+ let key_cache_buf = unsafe { key_cache_array.mtl_buffer().clone() };++ let mut value_cache_array = value_cache_binding[0].borrow_mut();+ let value_cache_buf =+ unsafe { value_cache_array.mtl_buffer().clone() };++ (key_cache_buf, value_cache_buf)+ } else {+ // For classifiers, we need a values buffer with [num_groups, suffix_length, head_dim] layout.+ // Use the KV cache update kernel to extract values from QKV into a dedicated extracted_values buffer.+ let extracted_values_binding =+ state.arrays(&[ArrayId::ExtractedValues]);+ let mut extracted_values_array =+ extracted_values_binding[0].borrow_mut();+ let extracted_values_buf =+ unsafe { extracted_values_array.mtl_buffer().clone() };++ // Reuse the KV cache update kernel to write values into extracted_values_buf.+ if let Err(e) = self.kernel.encode_kv_cache_update(+ &compute_encoder,+ KVCacheUpdateArguments {+ rotated_keys_buffer: &rotated_keys_buffer,+ qkv_buffer: &qkv_buffer,+ key_cache_buffer: &rotated_keys_buffer, // keys already in desired layout; harmless overwrite+ value_cache_buffer: &extracted_values_buf,+ num_groups,+ num_heads,+ head_dim,+ suffix_length,+ segment_prefix_length: 0,+ max_sequence_length,+ },+ ) {+ eprintln!("Failed to prepare rotated values buffer: {:?}", e);+ }++ (rotated_keys_buffer.clone(), extracted_values_buf)+ };++ let mut queries_array = rotated_queries_binding[0].borrow_mut();+ let queries_buffer = unsafe { queries_array.mtl_buffer() };++ let mut attention_output_array =+ attention_output_binding[0].borrow_mut();+ let attention_output_buffer =+ unsafe { attention_output_array.mtl_buffer() };++ let attention_bias_buffer = attention_bias_map+ .get(&window_length)+ .map(|array| {+ let mut array_ref = array.borrow_mut();+ unsafe { array_ref.mtl_buffer().clone() }+ })+ .unwrap_or_else(|| {+ panic!(+ "Attention bias buffer not found for window length {:?}",+ window_length+ );+ });++ let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);+ let sums_binding = state.arrays(&[ArrayId::AttentionSums]);+ let maxs_binding = state.arrays(&[ArrayId::AttentionMaxs]);++ let mut partials_array = partials_binding[0].borrow_mut();+ let partials_buffer = unsafe { partials_array.mtl_buffer() };++ let mut sums_array = sums_binding[0].borrow_mut();+ let sums_buffer = unsafe { sums_array.mtl_buffer() };++ let mut maxs_array = maxs_binding[0].borrow_mut();+ let maxs_buffer = unsafe { maxs_array.mtl_buffer() };++ let sinks_buffer = sinks_binding.as_ref().map(|binding| unsafe {+ binding[0].borrow_mut().mtl_buffer().clone()+ });++ // Only update KV cache for LLM mode (not for classifiers)+ if has_kv_cache {+ if let Err(e) = self.kernel.encode_kv_cache_update(+ &compute_encoder,+ KVCacheUpdateArguments {+ rotated_keys_buffer: &rotated_keys_buffer,+ qkv_buffer: &qkv_buffer,+ key_cache_buffer: &key_cache_buffer,+ value_cache_buffer: &value_cache_buffer,+ num_groups,+ num_heads,+ head_dim,+ suffix_length,+ segment_prefix_length,+ max_sequence_length,+ },+ ) {+ eprintln!("Failed to encode KV cache update: {:?}", e);+ compute_encoder.end_encoding();+ return;+ }+ }++ let k_head_stride = (max_sequence_length * head_dim) as i32;+ let k_seq_stride = head_dim as i32;+ let v_head_stride = (max_sequence_length * head_dim) as i32;+ let v_seq_stride = head_dim as i32;++ match variant {+ AttentionKernelVariant::SinglePass => {+ if let Err(e) = self.kernel.encode_single_pass(+ &compute_encoder,+ AttentionSinglePassArguments {+ queries_buffer: &queries_buffer,+ keys_buffer: &key_cache_buffer,+ values_buffer: &value_cache_buffer,+ output_buffer: &attention_output_buffer,+ gqa_factor: gqa_factor as i32,+ sequence_length: sequence_length as i32,+ k_head_stride,+ k_seq_stride,+ v_head_stride,+ v_seq_stride,+ scale,+ mask_buffer: Some(&attention_bias_buffer),+ mask_kv_seq_stride,+ mask_q_seq_stride,+ mask_head_stride,+ sinks_buffer: sinks_buffer.as_ref(),+ num_heads,+ suffix_length,+ head_dim,+ is_causal: self.is_causal,+ },+ ) {+ eprintln!(+ "Failed to encode single-pass attention: {:?}",+ e+ );+ }+ },+ AttentionKernelVariant::TwoPass => {+ if let Err(e) = self.kernel.encode_two_pass(+ &compute_encoder,+ AttentionTwoPassArguments {+ queries_buffer: &queries_buffer,+ keys_buffer: &key_cache_buffer,+ values_buffer: &value_cache_buffer,+ partials_buffer: &partials_buffer,+ sums_buffer: &sums_buffer,+ maxs_buffer: &maxs_buffer,+ output_buffer: &attention_output_buffer,+ gqa_factor: gqa_factor as i32,+ sequence_length: sequence_length as i32,+ k_head_stride,+ k_seq_stride,+ v_head_stride,+ v_seq_stride,+ scale,+ mask_buffer: Some(&attention_bias_buffer),+ mask_kv_seq_stride,+ mask_q_seq_stride,+ mask_head_stride,+ sinks_buffer: sinks_buffer.as_ref(),+ num_heads,+ suffix_length,+ head_dim,+ is_causal: self.is_causal,+ },+ ) {+ eprintln!("Failed to encode two-pass attention: {:?}", e);+ }+ },+ }++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rsnew file mode 100644index 000000000..289e205e1--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs@@ -0,0 +1,411 @@+use std::rc::Rc;++use mpsgraph::CommandBuffer as MPSCommandBuffer;+use objc2::rc::autoreleasepool;++use super::{+ Attention, EncodableBlock, EncodingParameters, ForwardPassState,+ Normalization, QKNorm, TensorAddSwap, TensorCopy, transformer_layer,+};+use crate::{+ DataType,+ backends::metal::{+ MTLContext, compilation_parameters::CompilationConfig,+ forward_pass::ArrayId, kernel::KernelDataType,+ },+ config::TransformerLayerConfig,+ parameters::ParameterTree,+};++pub struct ClassifierLayer {+ #[cfg_attr(not(feature = "tracing"), allow(dead_code))]+ layer_index: usize,+ copy_main_to_shortcut_mixer: Box<dyn EncodableBlock>,+ pre_attention_norm: Option<Box<dyn EncodableBlock>>,+ qkv_projection: Box<dyn EncodableBlock>,+ qk_norm: Option<Box<dyn EncodableBlock>>,+ rope: Rc<Box<dyn EncodableBlock>>,+ attention: Box<dyn EncodableBlock>,+ out_projection: Box<dyn EncodableBlock>,+ post_attention_norm: Option<Box<dyn EncodableBlock>>,+ mixer_residual_add: Box<dyn EncodableBlock>,+ copy_main_to_shortcut_mlp: Box<dyn EncodableBlock>,+ pre_mlp_norm: Box<dyn EncodableBlock>,+ mlp: Box<dyn EncodableBlock>,+ post_mlp_norm: Option<Box<dyn EncodableBlock>>,+ mlp_residual_add: Box<dyn EncodableBlock>,+}++impl ClassifierLayer {+ pub fn new(+ mtl_context: &MTLContext,+ layer_config: &TransformerLayerConfig,+ compilation_config: Rc<CompilationConfig>,+ layer_index: usize,+ model_dim: usize,+ hidden_dim: usize,+ num_heads: usize,+ head_dim: usize,+ num_groups: usize,+ attention_scale: Option<f32>,+ layer_loader: &ParameterTree<Rc<MTLContext>>,+ rope: Rc<Box<dyn EncodableBlock>>,+ ) -> Self {+ autoreleasepool(|_| {+ let attention_config = layer_config+ .mixer_config+ .as_attention()+ .expect("Classifier layers must use attention");+ let intermediate_data_type: DataType = attention_config+ .qkv_projection_config+ .activation_precision()+ .into();+ let kernel_data_type: KernelDataType =+ intermediate_data_type.into();++ let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock> = Box::new(+ TensorCopy::new(+ mtl_context,+ kernel_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap(),+ );++ let pre_attention_norm: Option<Box<dyn EncodableBlock>> =+ if let Some(norm_config) =+ &layer_config.pre_attention_norm_config+ {+ if layer_loader.subtree("pre_mixer_norm").is_ok() {+ Some(Box::new(+ Normalization::new(+ mtl_context,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader+ .subtree("pre_mixer_norm")+ .unwrap(),+ )+ .expect(+ "Failed to create pre-attention norm kernel",+ ),+ ))+ } else {+ None+ }+ } else {+ None+ };++ let qkv_projection = transformer_layer::linear_block(+ &attention_config.qkv_projection_config,+ attention_config.has_qkv_biases,+ model_dim,+ [+ num_heads * head_dim,+ num_groups * head_dim,+ num_groups * head_dim,+ ],+ mtl_context,+ &layer_loader.subtree("mixer.qkv_projection").unwrap(),+ ArrayId::Main,+ ArrayId::QKV,+ &compilation_config.descriptor_mlp,+ );++ let qk_norm: Option<Box<dyn EncodableBlock>> = if attention_config+ .query_norm_config+ .is_some()+ || attention_config.key_norm_config.is_some()+ {+ match QKNorm::new(+ mtl_context,+ intermediate_data_type,+ attention_config.query_norm_config.clone(),+ attention_config.key_norm_config.clone(),+ ArrayId::QKV,+ &layer_loader.subtree("mixer").unwrap(),+ num_heads,+ num_groups,+ head_dim,+ ) {+ Ok(norm) => Some(Box::new(norm)),+ Err(e) => panic!(+ "Failed to create QK norm kernel for layer {}: {:?}",+ layer_index, e+ ),+ }+ } else {+ None+ };++ let out_projection = transformer_layer::linear_block(+ &attention_config.out_projection_config,+ attention_config.has_out_biases,+ num_heads * head_dim,+ [model_dim],+ mtl_context,+ &layer_loader.subtree("mixer.out_projection").unwrap(),+ ArrayId::AttentionOutput,+ ArrayId::Main,+ &compilation_config.descriptor_mlp,+ );++ let post_attention_norm: Option<Box<dyn EncodableBlock>> =+ if let Some(norm_config) =+ &layer_config.post_attention_norm_config+ {+ Some(Box::new(+ Normalization::new(+ mtl_context,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("post_mixer_norm").unwrap(),+ )+ .expect("Failed to create post-attention norm kernel"),+ ))+ } else {+ None+ };++ let mixer_residual_add: Box<dyn EncodableBlock> = Box::new(+ TensorAddSwap::new(+ mtl_context,+ kernel_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap(),+ );++ let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock> = Box::new(+ TensorCopy::new(+ mtl_context,+ kernel_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap(),+ );++ let pre_mlp_norm: Box<dyn EncodableBlock> = Box::new(+ Normalization::new(+ mtl_context,+ intermediate_data_type,+ layer_config.pre_mlp_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("pre_mlp_norm").unwrap(),+ )+ .expect("Failed to create pre-MLP norm kernel"),+ );++ let mlp = transformer_layer::mlp_block(+ &layer_config.mlp_config,+ model_dim,+ hidden_dim,+ mtl_context,+ &layer_loader.subtree("mlp").unwrap(),+ &compilation_config.descriptor_mlp,+ );++ let post_mlp_norm: Option<Box<dyn EncodableBlock>> =+ if let Some(norm_config) = &layer_config.post_mlp_norm_config {+ Some(Box::new(+ Normalization::new(+ mtl_context,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("post_mlp_norm").unwrap(),+ )+ .expect("Failed to create post-MLP norm kernel"),+ ))+ } else {+ None+ };++ let attention: Box<dyn EncodableBlock> = Box::new(+ Attention::new(+ mtl_context,+ kernel_data_type,+ layer_index,+ attention_scale,+ attention_config.has_sinks,+ false,+ attention_config.sliding_window_size,+ )+ .expect("Failed to create attention kernel"),+ );++ let mlp_residual_add: Box<dyn EncodableBlock> = Box::new(+ TensorAddSwap::new(+ mtl_context,+ kernel_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap(),+ );++ Self {+ layer_index,+ copy_main_to_shortcut_mixer,+ pre_attention_norm,+ qkv_projection,+ qk_norm,+ rope,+ attention,+ out_projection,+ post_attention_norm,+ mixer_residual_add,+ copy_main_to_shortcut_mlp,+ pre_mlp_norm,+ mlp,+ post_mlp_norm,+ mlp_residual_add,+ }+ })+ }+}++impl EncodableBlock for ClassifierLayer {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ #[cfg(feature = "tracing")]+ let layer_traces = state+ .traces()+ .borrow()+ .layer_results+ .get(self.layer_index)+ .cloned();++ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().inputs.clone(),+ );+ }++ self.copy_main_to_shortcut_mixer.encode(+ state,+ command_buffer,+ parameters,+ );++ if let Some(ref pre_attn_norm) = self.pre_attention_norm {+ pre_attn_norm.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().pre_attention_norm.clone(),+ );+ }+ } else {+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().pre_attention_norm.clone(),+ );+ }+ }++ self.qkv_projection.encode(state, command_buffer, parameters);+ if let Some(ref qk_norm) = self.qk_norm {+ qk_norm.encode(state, command_buffer, parameters);+ }+ self.rope.encode(state, command_buffer, parameters);+ self.attention.encode(state, command_buffer, parameters);+ self.out_projection.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().attention.clone(),+ );+ }++ if let Some(ref post_attn_norm) = self.post_attention_norm {+ post_attn_norm.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().post_attention_norm.clone(),+ );+ }+ }++ self.mixer_residual_add.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().mlp_inputs.clone(),+ );+ }++ self.copy_main_to_shortcut_mlp.encode(+ state,+ command_buffer,+ parameters,+ );++ self.pre_mlp_norm.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().pre_mlp_norm.clone(),+ );+ }++ self.mlp.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().mlp.clone(),+ );+ }++ if let Some(ref post_mlp_norm) = self.post_mlp_norm {+ post_mlp_norm.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().post_mlp_norm.clone(),+ );+ }+ }++ self.mlp_residual_add.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().outputs.clone(),+ );+ }+ }+}diff --git a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rssimilarity index 71%rename from crates/uzu/src/backends/metal/executable_builder/decoder_executables.rsrename to crates/uzu/src/backends/metal/encodable_block/decoder.rsindex 9113bfb03..0ded4ad27 100644--- a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs@@ -1,52 +1,57 @@+//! Decoder executables - combines embedding, layers, normalization, and readout.+use std::rc::Rc;use mpsgraph::CommandBuffer as MPSCommandBuffer;-use super::layer_executables::LayerExecutables;+use super::{+ EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,+};use crate::{- DataType,+ DataType, DecoderConfig,backends::metal::{KernelDataType, MTLContext, ModelShape,compilation_parameters::CompilationConfig,- forward_pass::{- ArrayId, ForwardPassState, RopeType,- encodable_with_state::{EncodableWithState, EncodingParameters},- transformer_layer::{embed_block, readout_block},- },- kernel::{RMSNormKernelEncodable, RopeKernelEncodable},+ encodable_block::transformer_layer::{embed_block, readout_block},+ forward_pass::{ArrayId, EncoderResolver, ForwardPassState, RopeType},},- config::{DecoderConfig, DecoderLayerType, decoder_layer::MixerConfig},+ config::{DecoderLayerType, MixerConfig},parameters::ParameterTree,};-pub struct DecoderExecutables {- pub embed: Box<dyn EncodableWithState>,+/// Full decoder executable with all layers and components.+pub struct Decoder {+ pub embed: Box<dyn EncodableBlock>,pub layers: Box<[LayerExecutables]>,- pub norm: Box<dyn EncodableWithState>,- pub readout: Box<dyn EncodableWithState>,- pub global_rope: Option<Rc<Box<dyn EncodableWithState>>>,- pub local_rope: Option<Rc<Box<dyn EncodableWithState>>>,+ pub norm: Box<dyn EncodableBlock>,+ pub readout: Box<dyn EncodableBlock>,+ pub global_rope: Option<Rc<Box<dyn EncodableBlock>>>,+ pub local_rope: Option<Rc<Box<dyn EncodableBlock>>>,}-impl DecoderExecutables {+impl Decoder {pub fn new(mtl_context: Rc<MTLContext>,decoder_config: Rc<DecoderConfig>,- decoder_weight_loader: &ParameterTree<Rc<MTLContext>>,+ root_weight_loader: &ParameterTree<Rc<MTLContext>>,compilation_config: Rc<CompilationConfig>,) -> Self {+ let decoder_weight_loader = root_weight_loader+ .subtree("transformer")+ .expect("transformer subtree not found");+let embed = embed_block(&decoder_config,&mtl_context,&compilation_config.descriptor_general,- decoder_weight_loader,+ root_weight_loader,);let readout = readout_block(&decoder_config,&mtl_context,&compilation_config.descriptor_general,- decoder_weight_loader,+ root_weight_loader,);let attention_data_type = Self::attention_data_type(&decoder_config);@@ -142,8 +147,8 @@ impl DecoderExecutables {}).collect::<Vec<_>>();- let norm_block: Box<dyn EncodableWithState> = Box::new(- RMSNormKernelEncodable::new(+ let norm_block: Box<dyn EncodableBlock> = Box::new(+ RMSNorm::new(&mtl_context,norm_data_type,decoder_config.output_norm_config.clone(),@@ -155,10 +160,10 @@ impl DecoderExecutables {);Self {- embed: embed,+ embed,layers: layers.into_boxed_slice(),norm: norm_block,- readout: readout,+ readout,global_rope,local_rope,}@@ -168,12 +173,12 @@ impl DecoderExecutables {mtl_context: &MTLContext,kernel_data_type: KernelDataType,rope_type: RopeType,- ) -> Rc<Box<dyn EncodableWithState>> {- let rotation: Box<dyn EncodableWithState> = Box::new(- RopeKernelEncodable::new(mtl_context, kernel_data_type, rope_type)- .expect("Failed to create RopeKernelEncodable"),+ ) -> Rc<Box<dyn EncodableBlock>> {+ let rotation: Box<dyn EncodableBlock> = Box::new(+ Rope::new(mtl_context, kernel_data_type, rope_type)+ .expect("Failed to create Rope"),);- return Rc::new(rotation);+ Rc::new(rotation)}fn attention_data_type(decoder_config: &DecoderConfig) -> Option<DataType> {@@ -193,36 +198,50 @@ impl DecoderExecutables {}}-impl EncodableWithState for DecoderExecutables {+impl EncodableBlock for Decoder {fn encode(&self,state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,) {- self.embed.encode(state, command_buffer, parameters);+ let mut resolver = EncoderResolver::new(command_buffer);++ resolver.encode(self.embed.as_ref(), state, parameters);for layer in self.layers.iter() {- layer.encode(state, command_buffer, parameters);+ resolver.encode(layer, state, parameters);}if state.is_prefilling() {+ resolver.finish();return;}- self.norm.encode(state, command_buffer, parameters);- if let Some(traces) = state.traces.clone() {- state- .copy_array(ArrayId::Main, traces.borrow().output_norm.clone());+ resolver.encode(self.norm.as_ref(), state, parameters);+ #[cfg(feature = "tracing")]+ {+ resolver.end_current_encoder();+ let traces = state.traces().clone();+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ traces.borrow().output_norm.clone(),+ );}- self.readout.encode(state, command_buffer, parameters);- if let Some(traces) = state.traces.clone() {- state.copy_array(ArrayId::Logits, traces.borrow().logits.clone());+ resolver.encode(self.readout.as_ref(), state, parameters);+ #[cfg(feature = "tracing")]+ {+ resolver.end_current_encoder();+ let traces = state.traces().clone();+ state.encode_copy_array(+ command_buffer,+ ArrayId::Logits,+ traces.borrow().logits.clone(),+ );}- }- fn supports_shared_encoder(&self) -> bool {- false+ resolver.finish();}}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding.rs b/crates/uzu/src/backends/metal/encodable_block/embedding.rsnew file mode 100644index 000000000..23ac3698a--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/embedding.rs@@ -0,0 +1,428 @@+//! Quantized embedding encodables.++use std::rc::Rc;++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::{+ embedding::{+ QuantizedEmbeddingLookupArguments,+ QuantizedEmbeddingLookupKernel,+ },+ quant_matmul::{+ QuantizationType, QuantizedMatmulArguments,+ QuantizedMatmulKernel,+ },+ },+ },+ config::QuantizationMode,+ parameters::ParameterTree,+};++#[derive(Debug, thiserror::Error)]+pub enum QuantizedEmbeddingError {+ #[error("Metal error: {0}")]+ MetalError(#[from] MTLError),+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+}++pub struct QuantizedEmbeddingLookup {+ kernel: QuantizedEmbeddingLookupKernel,+ weights_buffer: MTLBuffer,+ scales_buffer: MTLBuffer,+ biases_buffer: MTLBuffer,+ vocab_size: u32,+ model_dim: u32,+ group_size: u32,+}++impl QuantizedEmbeddingLookup {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ let packing_divisor = mode.packing_divisor();++ let kernel =+ QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;++ // Load weights [vocab_size, model_dim/packing_divisor] as storage_type+ let mut weights = match parameter_tree.leaf("weights") {+ Ok(weights) => weights,+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load weights: {:?}",+ e+ )))+ })?,+ };++ if weights.data_type() != mode.storage_type() {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Expected packed weights of type {:?}, got {:?}",+ mode.storage_type(),+ weights.data_type()+ )),+ ));+ }++ // Load scales [vocab_size, num_groups]+ let mut scales = match parameter_tree.leaf("scales") {+ Ok(scales) => scales,+ Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load scales: {:?}",+ e+ )))+ })?,+ };++ // Validate shapes and types+ let num_groups = (model_dim + group_size - 1) / group_size;+ if weights.shape() != [vocab_size, model_dim / packing_divisor] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim / packing_divisor+ )),+ ));+ }+ if scales.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",+ scales.shape(),+ vocab_size,+ num_groups+ )),+ ));+ }+ if scales.data_type() != data_type {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ scales.data_type(),+ ));+ }++ // Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")+ let biases_buffer: MTLBuffer = match parameter_tree+ .leaf("biases")+ .or_else(|_| parameter_tree.leaf("output_biases"))+ {+ Ok(mut deq_biases) => {+ if deq_biases.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",+ deq_biases.shape(),+ vocab_size,+ num_groups+ )),+ ));+ }+ if deq_biases.data_type() != data_type {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ deq_biases.data_type(),+ ));+ }+ unsafe { deq_biases.mtl_buffer().to_owned() }+ },+ Err(_) => {+ let elem_size: usize = match data_type {+ DataType::F16 | DataType::BF16 => 2,+ DataType::F32 => 4,+ other => {+ return Err(+ QuantizedEmbeddingError::UnsupportedDataType(other),+ );+ },+ };+ let size_bytes = (vocab_size * num_groups * elem_size) as u64;+ let buf = mtl_context.device.new_buffer(+ size_bytes,+ metal::MTLResourceOptions::StorageModeShared,+ );+ unsafe {+ std::ptr::write_bytes(+ buf.contents(),+ 0,+ size_bytes as usize,+ );+ }+ buf+ },+ };++ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };+ let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };++ Ok(Self {+ kernel,+ weights_buffer,+ scales_buffer,+ biases_buffer,+ vocab_size: vocab_size as u32,+ model_dim: model_dim as u32,+ group_size: group_size as u32,+ })+ }+}++impl EncodableBlock for QuantizedEmbeddingLookup {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &CommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);+ let batch_size = state.active_suffix_length();+ let mut token_ids_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let args = QuantizedEmbeddingLookupArguments {+ token_ids_buffer,+ weights_buffer: &self.weights_buffer,+ scales_buffer: &self.scales_buffer,+ biases_buffer: &self.biases_buffer,+ output_buffer,+ batch_size: batch_size as u32,+ vocab_size: self.vocab_size,+ model_dim: self.model_dim,+ group_size: self.group_size,+ };++ self.kernel+ .encode(encoder, args)+ .expect("Failed to encode quantized embedding lookup kernel");++ encoder.end_encoding();++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}+pub struct QuantizedEmbeddingReadout {+ kernel: QuantizedMatmulKernel,+ weights_buffer: MTLBuffer,+ scales_buffer: MTLBuffer,+ biases_buffer: MTLBuffer,+ vocab_size: usize,+ model_dim: usize,+}++impl QuantizedEmbeddingReadout {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ // Load weights [vocab_size, model_dim/2] as U8+ let mut weights = match parameter_tree.leaf("weights") {+ Ok(weights) => weights,+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load weights: {:?}",+ e+ )))+ })?,+ };++ // Load scales [vocab_size, num_groups]+ let mut scales = match parameter_tree.leaf("scales") {+ Ok(scales) => scales,+ Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load scales: {:?}",+ e+ )))+ })?,+ };++ // Validate shapes+ let num_groups = (model_dim + group_size - 1) / group_size;+ let packing_divisor = mode.packing_divisor();++ // Determine if weights are transposed by checking shape+ let weights_transposed = weights.shape()[0] == vocab_size;++ if weights.shape() != [vocab_size, model_dim / packing_divisor] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim / packing_divisor+ )),+ ));+ }+ if scales.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",+ scales.shape(),+ vocab_size,+ num_groups+ )),+ ));+ }+ if scales.data_type() != data_type {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ scales.data_type(),+ ));+ }++ // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]+ let biases_buffer: MTLBuffer = match parameter_tree+ .leaf("biases")+ .or_else(|_| parameter_tree.leaf("output_biases"))+ {+ Ok(mut deq_biases) => {+ if deq_biases.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",+ deq_biases.shape(),+ vocab_size,+ num_groups+ )),+ ));+ }+ if deq_biases.data_type() != data_type {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ deq_biases.data_type(),+ ));+ }+ unsafe { deq_biases.mtl_buffer().to_owned() }+ },+ Err(_) => {+ // Allocate zero-initialized biases buffer+ let elem_size: usize = match data_type {+ DataType::F16 | DataType::BF16 => 2,+ DataType::F32 => 4,+ other => {+ return Err(+ QuantizedEmbeddingError::UnsupportedDataType(other),+ );+ },+ };+ let size_bytes = (vocab_size * num_groups * elem_size) as u64;+ let buf = mtl_context.device.new_buffer(+ size_bytes,+ metal::MTLResourceOptions::StorageModeShared,+ );+ unsafe {+ std::ptr::write_bytes(+ buf.contents(),+ 0,+ size_bytes as usize,+ );+ }+ buf+ },+ };++ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };+ let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };++ let kernel = QuantizedMatmulKernel::new(+ mtl_context,+ data_type,+ group_size,+ model_dim,+ vocab_size,+ mode,+ QuantizationType::Mlx,+ weights_transposed,+ )+ .map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to create kernel: {:?}",+ e+ )))+ })?;++ Ok(Self {+ kernel,+ weights_buffer,+ scales_buffer,+ biases_buffer,+ vocab_size,+ model_dim,+ })+ }+}++impl EncodableBlock for QuantizedEmbeddingReadout {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &CommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);+ let batch_size = state.active_suffix_length();+ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let args = QuantizedMatmulArguments {+ a_buffer: input_buffer,+ b_buffer: &self.weights_buffer,+ scales_buffer: &self.scales_buffer,+ zero_points_or_biases_buffer: &self.biases_buffer,+ output_buffer,+ batch: batch_size as i32,+ input_dim: self.model_dim as i32,+ output_dim: self.vocab_size as i32,+ quantization_type: QuantizationType::Mlx,+ };++ self.kernel+ .encode(encoder, args)+ .expect("Failed to encode quantized embedding readout kernel");++ encoder.end_encoding();++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs b/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rssimilarity index 60%rename from crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rsrename to crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rsindex 1c486dd47..fcbb32a3a 100644--- a/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs+++ b/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs@@ -1,7 +1,6 @@-use metal::{Buffer, BufferRef, ComputeCommandEncoderRef};-use mpsgraph::CommandBuffer as MPSCommandBuffer;+//! Encoding parameters for forward pass operations.-use super::ForwardPassState;+use metal::{Buffer, BufferRef};#[derive(Clone)]pub struct EncodingParameters<'a> {@@ -47,23 +46,3 @@ impl<'a> EncodingParameters<'a> {self.predicate.map(|buffer| buffer.as_ref())}}--pub trait EncodableWithState {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- );-- fn supports_shared_encoder(&self) -> bool;-- fn encode_with_shared_encoder(- &self,- _state: &mut ForwardPassState,- _encoder: &ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- panic!("encode_with_shared_encoder called on unsupported type");- }-}diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rsnew file mode 100644index 000000000..51ea2cad3--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs@@ -0,0 +1,426 @@+//! Layer executables - a single decoder layer with mixer, norms, and MLP.++use std::rc::Rc;++use mpsgraph::CommandBuffer as MPSCommandBuffer;+use objc2::rc::autoreleasepool;++use super::{+ super::{+ Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm,+ RMSNorm, TensorAddSwap, TensorCopy, transformer_layer,+ },+ MixerExecutables,+};+use crate::{+ DataType, DecoderLayerConfig,+ backends::metal::{+ MTLContext,+ compilation_parameters::CompilationConfig,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::KernelDataType,+ },+ config::{DecoderLayerType, MixerConfig},+ parameters::ParameterTree,+};++/// A single decoder layer with all its components.+pub struct LayerExecutables {+ pub layer_index: usize,+ pub copy_main_to_shortcut: Box<dyn EncodableBlock>,+ pub pre_attention_norm: Box<dyn EncodableBlock>,+ pub(crate) mixer: MixerExecutables,+ pub post_attention_norm: Option<Box<dyn EncodableBlock>>,+ pub main_shortcut_add_swap: Box<dyn EncodableBlock>,+ pub pre_mlp_norm: Box<dyn EncodableBlock>,+ pub mlp: Box<dyn EncodableBlock>,+ pub post_mlp_norm: Option<Box<dyn EncodableBlock>>,+}++impl LayerExecutables {+ #[allow(clippy::too_many_arguments)]+ pub fn new(+ mtl_context: &MTLContext,+ layer_config: &DecoderLayerConfig,+ layer_type: &DecoderLayerType,+ compilation_config: Rc<CompilationConfig>,+ layer_index: usize,+ model_dim: usize,+ hidden_dim: usize,+ num_heads: usize,+ head_dim: usize,+ num_groups: usize,+ attention_scale: Option<f32>,+ decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,+ rope: Option<Rc<Box<dyn EncodableBlock>>>,+ ) -> Self {+ autoreleasepool(|_| {+ let intermediate_data_type: DataType =+ match &layer_config.mixer_config {+ MixerConfig::Attention(attention) => attention+ .qkv_projection_config+ .activation_precision()+ .into(),+ MixerConfig::Mamba(mamba) => {+ mamba.in_projection_config.activation_precision().into()+ },+ };+ let kernel_data_type: KernelDataType =+ intermediate_data_type.into();++ let copy_main_to_shortcut: Box<dyn EncodableBlock> = Box::new(+ TensorCopy::new(+ mtl_context,+ kernel_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap(),+ );++ let pre_attention_norm: Box<dyn EncodableBlock> = Box::new(+ RMSNorm::new(+ mtl_context,+ intermediate_data_type,+ layer_config.pre_attention_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel"),+ );++ let mixer = match &layer_config.mixer_config {+ MixerConfig::Attention(attention_config) => {+ let rope_block = rope+ .clone()+ .expect("RoPE encoder missing for attention layer");++ let qkv_projection = transformer_layer::linear_block(+ &attention_config.qkv_projection_config,+ attention_config.has_qkv_biases,+ model_dim,+ [+ num_heads * head_dim,+ num_groups * head_dim,+ num_groups * head_dim,+ ],+ mtl_context,+ &decoder_layer_loader+ .subtree("mixer.qkv_projection")+ .unwrap(),+ ArrayId::Main,+ ArrayId::QKV,+ &compilation_config.descriptor_mlp,+ );++ let qk_norm: Option<Box<dyn EncodableBlock>> =+ if attention_config.query_norm_config.is_some()+ || attention_config.key_norm_config.is_some()+ {+ match QKNorm::new(+ mtl_context,+ intermediate_data_type,+ attention_config.query_norm_config.clone(),+ attention_config.key_norm_config.clone(),+ ArrayId::QKV,+ &decoder_layer_loader.subtree("mixer").unwrap(),+ num_heads,+ num_groups,+ head_dim,+ ) {+ Ok(qk_norm) => Some(Box::new(qk_norm)+ as Box<dyn EncodableBlock>),+ Err(e) => panic!(+ "Failed to create QK norm kernel for layer {}: {:?}",+ layer_index, e+ ),+ }+ } else {+ None+ };++ let out_projection = transformer_layer::linear_block(+ &attention_config.out_projection_config,+ attention_config.has_out_biases,+ num_heads * head_dim,+ [model_dim],+ mtl_context,+ &decoder_layer_loader+ .subtree("mixer.out_projection")+ .unwrap(),+ ArrayId::AttentionOutput,+ ArrayId::Main,+ &compilation_config.descriptor_mlp,+ );++ let attention = Box::new(+ Attention::new(+ mtl_context,+ kernel_data_type,+ layer_index,+ attention_scale,+ attention_config.has_sinks,+ attention_config.is_causal.unwrap_or(true),+ attention_config.sliding_window_size,+ )+ .expect("Failed to create AttentionWrapper with Metal kernel"),+ );++ MixerExecutables::Attention {+ qkv_projection,+ qk_norm,+ rope: rope_block,+ attention,+ out_projection,+ }+ },+ MixerConfig::Mamba(mamba_config) => {+ let mixer: Box<dyn EncodableBlock> =+ Box::new(MambaMixer::new(+ mtl_context,+ layer_type.clone(),+ mamba_config.clone(),+ compilation_config.clone(),+ layer_index,+ model_dim,+ num_heads,+ head_dim,+ num_groups,+ decoder_layer_loader,+ ));+ MixerExecutables::StateSpace {+ mixer,+ }+ },+ };++ let post_attention_norm: Option<Box<dyn EncodableBlock>> =+ if let Some(norm_config) =+ &layer_config.post_attention_norm_config+ {+ Some(Box::new(+ RMSNorm::new(+ mtl_context,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader+ .subtree("post_mixer_norm")+ .unwrap(),+ )+ .expect("Failed to create RMS norm kernel"),+ ))+ } else {+ None+ };++ let main_shortcut_add_swap: Box<dyn EncodableBlock> = Box::new(+ TensorAddSwap::new(+ mtl_context,+ kernel_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap(),+ );++ let pre_mlp_norm: Box<dyn EncodableBlock> = Box::new(+ RMSNorm::new(+ mtl_context,+ intermediate_data_type,+ layer_config.pre_mlp_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel"),+ );++ let mlp = transformer_layer::mlp_block(+ &layer_config.mlp_config,+ model_dim,+ hidden_dim,+ mtl_context,+ &decoder_layer_loader.subtree("mlp").unwrap(),+ &compilation_config.descriptor_mlp,+ );++ let post_mlp_norm: Option<Box<dyn EncodableBlock>> =+ if let Some(norm_config) = &layer_config.post_mlp_norm_config {+ Some(Box::new(+ RMSNorm::new(+ mtl_context,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader+ .subtree("post_mlp_norm")+ .unwrap(),+ )+ .expect("Failed to create RMS norm kernel"),+ ))+ } else {+ None+ };++ Self {+ layer_index,+ copy_main_to_shortcut,+ pre_attention_norm,+ mixer,+ post_attention_norm,+ main_shortcut_add_swap,+ pre_mlp_norm,+ mlp,+ post_mlp_norm,+ }+ })+ }+}++impl EncodableBlock for LayerExecutables {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ #[cfg(feature = "tracing")]+ let layer_traces = state+ .traces()+ .borrow()+ .layer_results+ .get(self.layer_index)+ .cloned();++ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().inputs.clone(),+ );+ }++ self.copy_main_to_shortcut.encode(state, command_buffer, parameters);+ // shortcut = input++ self.pre_attention_norm.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().pre_attention_norm.clone(),+ );+ }++ match &self.mixer {+ MixerExecutables::Attention {+ qkv_projection,+ qk_norm,+ rope,+ attention,+ out_projection,+ } => {+ qkv_projection.encode(state, command_buffer, parameters);+ if let Some(norm) = qk_norm {+ norm.encode(state, command_buffer, parameters);+ }+ rope.encode(state, command_buffer, parameters);+ attention.encode(state, command_buffer, parameters);+ out_projection.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().attention.clone(),+ );+ }+ },+ MixerExecutables::StateSpace {+ mixer,+ } => {+ mixer.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().attention.clone(),+ );+ }+ },+ }++ if let Some(post_attention_norm) = &self.post_attention_norm {+ post_attention_norm.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().post_attention_norm.clone(),+ );+ }+ }+ //main = attention_result++ self.main_shortcut_add_swap.encode(state, command_buffer, parameters);+ // shortcut = input + attention_result+ // main = input + attention_result+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().mlp_inputs.clone(),+ );+ }++ self.pre_mlp_norm.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().pre_mlp_norm.clone(),+ );+ }++ self.mlp.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().mlp.clone(),+ );+ }++ if let Some(post_mlp_norm) = &self.post_mlp_norm {+ post_mlp_norm.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().post_mlp_norm.clone(),+ );+ }+ }+ // main = mlp_result++ self.main_shortcut_add_swap.encode(state, command_buffer, parameters);+ // shortcut = input + attention_result + mlp_result+ // main = input + attention_result + mlp_result+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(+ command_buffer,+ ArrayId::Main,+ layer_traces.borrow().outputs.clone(),+ );+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rsnew file mode 100644index 000000000..5a8d5e936--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs@@ -0,0 +1,19 @@+//! Mixer component - either attention or state space.++use std::rc::Rc;++use super::super::EncodableBlock;++/// Mixer component - either attention or state space.+pub(crate) enum MixerExecutables {+ Attention {+ qkv_projection: Box<dyn EncodableBlock>,+ qk_norm: Option<Box<dyn EncodableBlock>>,+ rope: Rc<Box<dyn EncodableBlock>>,+ attention: Box<dyn EncodableBlock>,+ out_projection: Box<dyn EncodableBlock>,+ },+ StateSpace {+ mixer: Box<dyn EncodableBlock>,+ },+}diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mod.rsnew file mode 100644index 000000000..15a5156fc--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs@@ -0,0 +1,8 @@+//! Layer executables module.++mod executables;+mod mixer;++pub use executables::LayerExecutables;+pub(crate) use mixer::MixerExecutables;+diff --git a/crates/uzu/src/backends/metal/kernel/linear/mod.rs b/crates/uzu/src/backends/metal/encodable_block/linear.rssimilarity index 96%rename from crates/uzu/src/backends/metal/kernel/linear/mod.rsrename to crates/uzu/src/backends/metal/encodable_block/linear.rsindex 676d9b998..4bb8dc2b0 100644--- a/crates/uzu/src/backends/metal/kernel/linear/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear.rs@@ -1,28 +1,29 @@+//! Quantized linear (matmul) encodable.+use std::rc::Rc;use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};use mpsgraph::CommandBuffer as MPSCommandBuffer;-use super::{- KernelDataType, TensorAddBias,- quant_matmul::{- QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel,- },-};+use super::{EncodableBlock, EncodingParameters};use crate::{Array, DataType,backends::metal::{MTLContext, MTLError,- forward_pass::{- ArrayId, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},+ forward_pass::{ArrayId, ForwardPassState},+ kernel::{+ KernelDataType, TensorAddBias,+ quant_matmul::{+ QuantizationType, QuantizedMatmulArguments,+ QuantizedMatmulKernel,+ },},},config::QuantizationConfig,parameters::ParameterTree,};-pub struct QuantizedLinearKernelBlock {+pub struct QuantizedLinear {kernel: QuantizedMatmulKernel,bias_add_kernel: Option<TensorAddBias>,biases_buffer: Option<MTLBuffer>,@@ -36,7 +37,7 @@ pub struct QuantizedLinearKernelBlock {output_array_id: ArrayId,}-impl QuantizedLinearKernelBlock {+impl QuantizedLinear {pub fn new(mtl_context: &MTLContext,config: &QuantizationConfig,@@ -222,7 +223,7 @@ impl QuantizedLinearKernelBlock {}}-impl EncodableWithState for QuantizedLinearKernelBlock {+impl EncodableBlock for QuantizedLinear {fn encode(&self,state: &mut ForwardPassState,diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rssimilarity index 77%rename from crates/uzu/src/backends/metal/kernel/ssm/mamba.rsrename to crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rsindex a4d82b506..b6c0a95c7 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs@@ -1,35 +1,32 @@+//! Mamba2 SSM mixer encodable.+use std::{env, rc::Rc};-use metal::ComputeCommandEncoderRef;use mpsgraph::CommandBuffer as MPSCommandBuffer;-use super::{- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,- SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,- SSDUpdateKernel, SplitInProjArguments, SplitInProjKernel,- conv1d_scan::Conv1dDecodeArguments,-};+use super::{EncodableBlock, EncodingParameters, transformer_layer};use crate::{DataType,backends::metal::{KernelDataType, MTLContext, MetalArray,compilation_parameters::CompilationConfig,- forward_pass::{- ArrayId, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- transformer_layer,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::ssm::{+ Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,+ SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode,+ SSDUpdateArguments, SSDUpdateKernel, SplitInProjArguments,+ SplitInProjKernel, conv1d_scan::Conv1dDecodeArguments,},},- config::{DecoderLayerType, LinearConfig, mamba::Mamba2Config},+ config::{DecoderLayerType, Mamba2Config},parameters::ParameterTree,};-pub(crate) struct MambaMixerEncodable {+pub(crate) struct MambaMixer {layer_index: usize,config: Mamba2Config,- in_projection: Box<dyn EncodableWithState>,- out_projection: Box<dyn EncodableWithState>,- projections_support_shared_encoder: bool,+ in_projection: Box<dyn EncodableBlock>,+ out_projection: Box<dyn EncodableBlock>,split_inproj: SplitInProjKernel,conv_scan: Conv1dScanKernel,ssm_prefill: SSDPrefillKernel,@@ -41,7 +38,7 @@ pub(crate) struct MambaMixerEncodable {prefill_mode: SSDPrefillMode,}-impl MambaMixerEncodable {+impl MambaMixer {#[allow(clippy::too_many_arguments)]pub(crate) fn new(mtl_context: &MTLContext,@@ -69,23 +66,15 @@ impl MambaMixerEncodable {mamba_config.in_projection_config.activation_precision().into();let kernel_data_type: KernelDataType = data_type.into();- let projections_are_quantized = matches!(- &mamba_config.in_projection_config,- LinearConfig::Quantized(_) | LinearConfig::MLXQuantized(_)- ) && matches!(- &mamba_config.out_projection_config,- LinearConfig::Quantized(_) | LinearConfig::MLXQuantized(_)- );-- let in_projection_output_dim = mamba_config.conv_dim()- + mamba_config.inner_dim()- + mamba_config.num_heads;-let in_projection = transformer_layer::linear_block(&mamba_config.in_projection_config,mamba_config.has_in_biases,model_dim,- [in_projection_output_dim],+ [+ mamba_config.conv_dim(),+ mamba_config.inner_dim(),+ mamba_config.num_heads,+ ],mtl_context,&resolve_subtree(decoder_layer_loader,@@ -141,7 +130,6 @@ impl MambaMixerEncodable {config: mamba_config,in_projection,out_projection,- projections_support_shared_encoder: projections_are_quantized,split_inproj,conv_scan,ssm_prefill,@@ -154,42 +142,7 @@ impl MambaMixerEncodable {}}- fn encode_pipeline_with_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,- ) {- let suffix_length = state.aux_buffers_suffix_length();- let active_suffix_length = state.active_suffix_length();- if suffix_length == 0 || active_suffix_length == 0 {- return;- }-- self.in_projection- .encode_with_shared_encoder(state, encoder, parameters);- self.split_inproj_with_encoder(state, encoder, active_suffix_length);- self.run_conv_scan_with_encoder(state, encoder, active_suffix_length);-- if suffix_length == 1 {- self.run_decode_ssm_with_encoder(- state,- encoder,- active_suffix_length,- );- } else {- self.run_prefill_ssm_with_encoder(- state,- encoder,- active_suffix_length,- );- }-- self.out_projection- .encode_with_shared_encoder(state, encoder, parameters);- }-- fn encode_pipeline_with_command_buffer(+ fn encode_pipeline(&self,state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,@@ -202,34 +155,29 @@ impl MambaMixerEncodable {}self.in_projection.encode(state, command_buffer, parameters);-- let cmd = command_buffer.root_command_buffer().to_owned();- let encoder = cmd.new_compute_command_encoder();- self.split_inproj_with_encoder(state, encoder, active_suffix_length);- self.run_conv_scan_with_encoder(state, encoder, active_suffix_length);+ self.run_split_inproj(state, command_buffer, active_suffix_length);+ self.run_conv_scan(state, command_buffer, active_suffix_length);if suffix_length == 1 {- self.run_decode_ssm_with_encoder(- state,- encoder,- active_suffix_length,- );+ self.run_decode_ssm(state, command_buffer, active_suffix_length);} else {- self.run_prefill_ssm_with_encoder(- state,- encoder,- active_suffix_length,- );+ self.run_prefill_ssm(state, command_buffer, active_suffix_length);}- encoder.end_encoding();self.out_projection.encode(state, command_buffer, parameters);++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }}- fn split_inproj_with_encoder(+ fn run_split_inproj(&self,state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,+ command_buffer: &MPSCommandBuffer,suffix_length: usize,) {let arrays = state.arrays(&[@@ -255,9 +203,12 @@ impl MambaMixerEncodable {let num_heads = self.config.num_heads;let total_dim = conv_dim + inner_dim + num_heads;+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute = mtl_command_buffer.new_compute_command_encoder();self.split_inproj.encode(- encoder,+ &compute,SplitInProjArguments {input: &input_buf,conv_out: &conv_buf,@@ -272,12 +223,13 @@ impl MambaMixerEncodable {},).expect("Failed to encode split in-projection kernel");+ compute.end_encoding();}- fn run_conv_scan_with_encoder(+ fn run_conv_scan(&self,state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,+ command_buffer: &MPSCommandBuffer,suffix_length: usize,) {let arrays = state.arrays(&[@@ -309,13 +261,15 @@ impl MambaMixerEncodable {let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let proj_dim = self.config.num_groups * self.config.state_dim;+ let cmd = command_buffer.root_command_buffer().to_owned();let state_stride = self.config.kernel_size.saturating_sub(1);drop(conv_state);if suffix_length == 1 {+ let compute = cmd.new_compute_command_encoder();self.conv_scan.encode_decode(- encoder,+ &compute,Conv1dDecodeArguments {x: &input_buf,w: &weight_buf,@@ -335,6 +289,7 @@ impl MambaMixerEncodable {},).expect("Failed to encode conv decode kernel");+ compute.end_encoding();} else {let padded_buf = if state_stride > 0 {let array = state@@ -344,20 +299,24 @@ impl MambaMixerEncodable {let buf = unsafe { borrow.mtl_buffer().to_owned() };drop(borrow);- self.conv_scan- .encode_pack(- encoder,- Conv1dPackArguments {- state_in: &state_buf,- x: &input_buf,- padded: &buf,- state_stride,- row_stride: conv_dim,- suffix_len: suffix_length,- channels: conv_dim,- },- )- .expect("Failed to encode conv pack kernel");+ {+ let pack_encoder = cmd.new_compute_command_encoder();+ self.conv_scan+ .encode_pack(+ &pack_encoder,+ Conv1dPackArguments {+ state_in: &state_buf,+ x: &input_buf,+ padded: &buf,+ state_stride,+ row_stride: conv_dim,+ suffix_len: suffix_length,+ channels: conv_dim,+ },+ )+ .expect("Failed to encode conv pack kernel");+ pack_encoder.end_encoding();+ }Some(buf)} else {@@ -366,9 +325,10 @@ impl MambaMixerEncodable {let conv_source = padded_buf.as_ref().unwrap_or(&input_buf);+ let compute = cmd.new_compute_command_encoder();self.conv_scan.encode(- encoder,+ &compute,Conv1dScanArguments {padded: conv_source,w: &weight_buf,@@ -387,13 +347,14 @@ impl MambaMixerEncodable {},).expect("Failed to encode conv scan kernel");+ compute.end_encoding();}}- fn run_prefill_ssm_with_encoder(+ fn run_prefill_ssm(&self,state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,+ command_buffer: &MPSCommandBuffer,suffix_length: usize,) {let base_arrays = state.arrays(&[@@ -430,9 +391,11 @@ impl MambaMixerEncodable {let mut skip_weights = self.skip_connection_weight.clone();let skip = unsafe { skip_weights.mtl_buffer().to_owned() };+ let cmd = command_buffer.root_command_buffer().to_owned();+ let compute = cmd.new_compute_command_encoder();self.ssm_prefill.encode(- encoder,+ &compute,SSDPrefillArguments {x: &x_buf,dt: &dt_buf,@@ -468,12 +431,13 @@ impl MambaMixerEncodable {self.prefill_mode,).expect("Failed to encode SSD prefill kernel");+ compute.end_encoding();}- fn run_decode_ssm_with_encoder(+ fn run_decode_ssm(&self,state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,+ command_buffer: &MPSCommandBuffer,suffix_length: usize,) {let arrays = state.arrays(&[@@ -520,9 +484,11 @@ impl MambaMixerEncodable {let group_size = (h / g) as i32;let state_size = n as i32;+ let cmd = command_buffer.root_command_buffer().to_owned();+ let compute = cmd.new_compute_command_encoder();self.ssd_update.encode(- encoder,+ &compute,SSDUpdateArguments {x: &x_buf,dt: &dt_buf,@@ -545,6 +511,7 @@ impl MambaMixerEncodable {},).expect("Failed to encode SSD decode kernel");+ compute.end_encoding();}}@@ -561,44 +528,14 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {}}-impl EncodableWithState for MambaMixerEncodable {+impl EncodableBlock for MambaMixer {fn encode(&self,state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,) {- if self.projections_support_shared_encoder {- let cmd = command_buffer.root_command_buffer().to_owned();- let encoder = cmd.new_compute_command_encoder();- self.encode_pipeline_with_encoder(state, encoder, parameters);- encoder.end_encoding();- } else {- self.encode_pipeline_with_command_buffer(- state,- command_buffer,- parameters,- );- }-- if parameters.wait_until_completed {- let cmd = command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- cmd.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- self.projections_support_shared_encoder- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,- ) {- self.encode_pipeline_with_encoder(state, encoder, parameters);+ self.encode_pipeline(state, command_buffer, parameters);}}diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsnew file mode 100644index 000000000..2773cb2f2--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -0,0 +1,68 @@+//! MLP block encodable.++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters, QuantizedLinear};+use crate::{+ Array,+ backends::metal::{+ forward_pass::{ArrayId, ForwardPassState},+ kernel::mlp::MlpGateActMulEncodable,+ },+};++pub struct MlpBlock {+ up: QuantizedLinear,+ gate: MlpGateActMulEncodable,+ down: QuantizedLinear,+}++impl MlpBlock {+ pub fn new(+ up: QuantizedLinear,+ gate: MlpGateActMulEncodable,+ down: QuantizedLinear,+ ) -> Self {+ Self {+ up,+ gate,+ down,+ }+ }+}++impl EncodableBlock for MlpBlock {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ params: &EncodingParameters,+ ) {+ // Up+ self.up.encode(state, command_buffer, params);+ // Gate act+mul (fused_up -> hidden)+ let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);+ let mut fused = arrays[0].borrow_mut();+ let mut hidden = arrays[1].borrow_mut();+ let m = fused.shape()[0] as i32;+ let root = command_buffer.root_command_buffer();+ let encoder = root.new_compute_command_encoder();+ let fused_buf = unsafe { fused.mtl_buffer() };+ let hidden_buf = unsafe { hidden.mtl_buffer() };+ self.gate+ .encode(encoder, fused_buf, hidden_buf, m)+ .expect("Failed to encode MLP activation/mul kernel");+ encoder.end_encoding();+ drop(fused);+ drop(hidden);+ // Down+ self.down.encode(state, command_buffer, params);++ if params.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsnew file mode 100644index 000000000..6a370dd85--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -0,0 +1,79 @@+//! Encodable building blocks for classifier and LLM forward passes.+//!+//! These are higher-level abstractions that compose kernels into+//! reusable operations for both classifier and LLM pipelines.+//!+//! Encodables implement `EncodableBlock` and orchestrate one or more+//! kernels to perform operations on `ForwardPassState`.++use metal::ComputeCommandEncoderRef;+use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::forward_pass::ForwardPassState;++mod attention;+mod classifier_layer;+mod decoder;+mod embedding;+mod encoding_parameters;+mod layer;+mod linear;+mod mamba_mixer;+mod mlp;+mod moe_block;+mod normalization;+mod pooling;+mod prediction_head;+mod rope;+mod sampling;+mod tensor_add_swap;+mod tensor_copy;+pub mod transformer_layer;++pub use attention::Attention;+pub use classifier_layer::ClassifierLayer;+pub use decoder::Decoder;+pub use embedding::{+ QuantizedEmbeddingError, QuantizedEmbeddingLookup,+ QuantizedEmbeddingReadout,+};+pub use encoding_parameters::EncodingParameters;+pub use layer::LayerExecutables;+pub use linear::QuantizedLinear;+pub(crate) use mamba_mixer::MambaMixer;+pub use mlp::MlpBlock;+pub use moe_block::{MoeBlock, SharedMoeWeights};+pub use normalization::{+ LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,+};+pub use pooling::Pooling;+pub use prediction_head::ClassifierPredictionHead;+pub use rope::Rope;+pub use sampling::Sampling;+pub use tensor_add_swap::TensorAddSwap;+pub use tensor_copy::TensorCopy;++/// Trait for encodable blocks that operate on `ForwardPassState`.+pub trait EncodableBlock {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ );++ /// Returns true if this block supports using a shared compute encoder.+ fn supports_shared_encoder(&self) -> bool {+ false+ }++ /// Encode using a shared compute encoder. Only called if `supports_shared_encoder` returns true.+ fn encode_with_shared_encoder(+ &self,+ _state: &mut ForwardPassState,+ _encoder: &ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ panic!("encode_with_shared_encoder called on unsupported type");+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rsnew file mode 100644index 000000000..7e91b8451--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs@@ -0,0 +1,7 @@+//! MoE (Mixture of Experts) encodables.++mod moe_block;+mod shared_weights;++pub use moe_block::MoeBlock;+pub use shared_weights::SharedMoeWeights;diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rsnew file mode 100644index 000000000..ec45d94b4--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs@@ -0,0 +1,633 @@+//! MoE (Mixture of Experts) block encodable.++use std::{cell::RefCell, rc::Rc};++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{+ super::{EncodableBlock, EncodingParameters},+ SharedMoeWeights,+};+use crate::{+ Activation, DataType, LinearConfig, MixtureOfExpertsConfig,+ RoutingFunctionConfig,+ backends::metal::{+ KernelDataType, MTLContext, MetalArray,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::moe::{+ MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,+ MoeCountsOffsetsFusedKernel, MoeExpertsTwoPassArguments,+ MoeExpertsTwoPassDecodeKernel, MoeExpertsTwoPassPrefillKernel,+ MoeFinalizeArguments, MoeFinalizeKernel, MoeGatherArguments,+ MoeGatherKernel, MoeRouterTopKArguments, MoeRouterTopKKernel,+ MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments,+ },+ },+ parameters::ParameterTree,+};++enum RouterBlock {+ Metal {+ weights_buf: metal::Buffer,+ biases_buf: metal::Buffer,+ },+}++pub struct MoeBlock {+ router: RouterBlock,+ router_data_type: KernelDataType,+ router_renorm: bool,+ router_topk_kernel: MoeRouterTopKKernel,+ counts_offsets_kernel: MoeCountsOffsetsFusedKernel,+ scatter_kernels: MoeScatterKernels,+ gather_kernel: MoeGatherKernel,+ experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeKernel,+ experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillKernel,+ finalize_kernel: MoeFinalizeKernel,+ moe_config: MixtureOfExpertsConfig,+ model_dim: usize,+ hidden_dim: usize,+ data_type: KernelDataType,+ shared_weights: SharedMoeWeights,+}++impl MoeBlock {+ pub fn new(+ context: &MTLContext,+ moe_config: &MixtureOfExpertsConfig,+ model_dim: usize,+ hidden_dim: usize,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, crate::backends::metal::MTLError> {+ let activation_data_type: DataType = moe_config+ .expert_config+ .linear_config+ .activation_precision()+ .into();+ let data_type: KernelDataType = activation_data_type.into();++ let router_data_type: DataType =+ moe_config.router_config.activation_precision().into();+ let router_kernel_data_type: KernelDataType = router_data_type.into();++ let router_renorm = matches!(+ moe_config.routing_function,+ RoutingFunctionConfig::SoftmaxRouting+ );++ let router_tree = parameter_tree.subtree("router").map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Router subtree error: {:?}",+ e+ ))+ })?;++ let router = match &moe_config.router_config {+ LinearConfig::Quantized(_) => {+ unimplemented!(+ "Quantized router with fused router+topk not yet supported"+ );+ },+ LinearConfig::MLXQuantized(_) => {+ unimplemented!(+ "MLX quantized router with fused router+topk not yet supported"+ );+ },+ LinearConfig::FullPrecision {+ ..+ } => {+ let mut weights_arr =+ router_tree.leaf("weights").map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Router weights error: {:?}",+ e+ ))+ })?;+ let weights_buf = unsafe { weights_arr.mtl_buffer().clone() };++ let mut biases_arr =+ router_tree.leaf("biases").map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Router biases error: {:?}",+ e+ ))+ })?;+ let biases_buf = unsafe { biases_arr.mtl_buffer().clone() };++ RouterBlock::Metal {+ weights_buf,+ biases_buf,+ }+ },+ LinearConfig::QLoRA {+ ..+ } => {+ unimplemented!("QLoRA router not yet supported for MoE");+ },+ };++ let router_topk_kernel =+ MoeRouterTopKKernel::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "RouterTopK fused kernel error: {:?}",+ e+ ))+ })?;+ let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context)+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Counts+offsets kernel error: {:?}",+ e+ ))+ })?;+ let scatter_kernels = MoeScatterKernels::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Scatter kernels error: {:?}",+ e+ ))+ })?;+ let gather_kernel = MoeGatherKernel::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Gather kernel error: {:?}",+ e+ ))+ })?;+ let experts_two_pass_decode_kernel =+ MoeExpertsTwoPassDecodeKernel::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Experts two-pass decode kernel error: {:?}",+ e+ ))+ })?;+ let experts_two_pass_prefill_kernel =+ MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Experts two-pass prefill kernel error: {:?}",+ e+ ))+ })?;+ let finalize_kernel = MoeFinalizeKernel::new(context).map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Finalize kernel error: {:?}",+ e+ ))+ })?;++ let experts_tree = parameter_tree.subtree("experts").map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "experts subtree error: {:?}",+ e+ ))+ })?;++ let mut w13_arr = experts_tree+ .subtree("up_projection")+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "up_projection error: {:?}",+ e+ ))+ })?+ .leaf("weights")+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "up weights error: {:?}",+ e+ ))+ })?;+ let w13_buf = unsafe { w13_arr.mtl_buffer().clone() };++ let mut w2_arr = experts_tree+ .subtree("down_projection")+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "down_projection error: {:?}",+ e+ ))+ })?+ .leaf("weights")+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "down weights error: {:?}",+ e+ ))+ })?;+ let w2_buf = unsafe { w2_arr.mtl_buffer().clone() };++ let mut up_biases_arr = experts_tree+ .subtree("up_projection")+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "up_projection biases error: {:?}",+ e+ ))+ })?+ .leaf("biases")+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "up biases error: {:?}",+ e+ ))+ })?;+ let up_biases_buf = unsafe { up_biases_arr.mtl_buffer().clone() };++ let mut down_biases_arr = experts_tree+ .subtree("down_projection")+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "down_projection biases error: {:?}",+ e+ ))+ })?+ .leaf("biases")+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "down biases error: {:?}",+ e+ ))+ })?;+ let down_biases_buf = unsafe { down_biases_arr.mtl_buffer().clone() };++ let shared_weights = SharedMoeWeights {+ w13_buf: Rc::new(w13_buf),+ w2_buf: Rc::new(w2_buf),+ up_biases_buf: Rc::new(up_biases_buf),+ down_biases_buf: Rc::new(down_biases_buf),+ };++ Ok(Self {+ router,+ router_data_type: router_kernel_data_type,+ router_renorm,+ router_topk_kernel,+ counts_offsets_kernel,+ scatter_kernels,+ gather_kernel,+ experts_two_pass_decode_kernel,+ experts_two_pass_prefill_kernel,+ finalize_kernel,+ moe_config: moe_config.clone(),+ model_dim,+ hidden_dim,+ data_type,+ shared_weights,+ })+ }++ fn gating_code_from_activation(activation: &Activation) -> u32 {+ match activation {+ Activation::Gelu => 3,+ Activation::SiLU {+ ..+ } => 2,+ Activation::Identity => {+ panic!("Identity activation is not supported for MoE kernels")+ },+ }+ }+}++impl EncodableBlock for MoeBlock {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let suffix_length = state.active_suffix_length();+ let arrays = state.arrays(&[+ ArrayId::Main,+ ArrayId::MoeTopkIds,+ ArrayId::MoeTopkProbs,+ ArrayId::MoeOffsets,+ ArrayId::MoeSumK,+ ArrayId::MoeBucketedTokenIds,+ ArrayId::MoeBucketedProbs,+ ArrayId::MoeXPerm,+ ArrayId::MoeTok2Row,+ ArrayId::MoeYPartial,+ ArrayId::MoeTileCounts,+ ArrayId::MoeTileOffsets,+ ArrayId::MoeTileMap,+ ArrayId::MoeTotalTiles,+ ArrayId::MoeDispatchArgs,+ ArrayId::MoeScatterPartials,+ ArrayId::MoeScatterBlockBases,+ ArrayId::MoeBlockAlloc,+ ArrayId::MoeHidden,+ ArrayId::MoeTwoPassRowExpertMap,+ ]);++ let clone_buffer = |array: &RefCell<MetalArray>| -> metal::Buffer {+ let mut borrow = array.borrow_mut();+ let buffer = unsafe { borrow.mtl_buffer().clone() };+ buffer+ };++ let mut array_iter = arrays.iter();+ let main_buf = clone_buffer(array_iter.next().unwrap());+ let topk_ids_buf = clone_buffer(array_iter.next().unwrap());+ let topk_probs_buf = clone_buffer(array_iter.next().unwrap());+ let offsets_buf = clone_buffer(array_iter.next().unwrap());+ let sumk_buf = clone_buffer(array_iter.next().unwrap());+ let bucketed_ids_buf = clone_buffer(array_iter.next().unwrap());+ let bucketed_probs_buf = clone_buffer(array_iter.next().unwrap());+ let x_perm_buf = clone_buffer(array_iter.next().unwrap());+ let tok2row_buf = clone_buffer(array_iter.next().unwrap());+ let y_partial_buf = clone_buffer(array_iter.next().unwrap());+ let tile_counts_buf = clone_buffer(array_iter.next().unwrap());+ let tile_offsets_buf = clone_buffer(array_iter.next().unwrap());+ let tile_map_buf = clone_buffer(array_iter.next().unwrap());+ let total_tiles_buf = clone_buffer(array_iter.next().unwrap());+ let dispatch_args_buf = clone_buffer(array_iter.next().unwrap());+ let partials_buf = clone_buffer(array_iter.next().unwrap());+ let block_bases_buf = clone_buffer(array_iter.next().unwrap());+ let block_alloc_buf = clone_buffer(array_iter.next().unwrap());+ let hidden_buf = clone_buffer(array_iter.next().unwrap());+ let row_expert_map_buf = clone_buffer(array_iter.next().unwrap());+ debug_assert!(array_iter.next().is_none());++ let e = self.moe_config.mixture_size;+ let k = self.moe_config.num_experts_per_token;++ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let root = &*mtl_command_buffer;+ let k_tile = 128;++ // Clear internal MoE buffers+ let dtype_size = match self.data_type {+ KernelDataType::BFloat16 | KernelDataType::Float16 => 2,+ KernelDataType::Float32 => 4,+ };++ let blit_encoder = root.new_blit_command_encoder();++ if suffix_length > 0 && k > 0 {+ let entries = suffix_length * k;+ let topk_bytes = entries * std::mem::size_of::<u32>();+ let tok2row_bytes = entries * std::mem::size_of::<i32>();++ // Clear topk_ids and tok2row buffers+ if topk_bytes > 0 {+ blit_encoder.fill_buffer(+ &topk_ids_buf,+ metal::NSRange::new(0, topk_bytes as u64),+ 0xFF,+ );+ }+ if tok2row_bytes > 0 {+ blit_encoder.fill_buffer(+ &tok2row_buf,+ metal::NSRange::new(0, tok2row_bytes as u64),+ 0xFF,+ );+ }++ // Clear hidden buffer+ let hidden_bytes =+ (suffix_length * k * self.hidden_dim * dtype_size) as u64;+ if hidden_bytes > 0 {+ blit_encoder.fill_buffer(+ &hidden_buf,+ metal::NSRange::new(0, hidden_bytes),+ 0,+ );+ }++ // Clear y_partial buffer+ let y_partial_bytes =+ (suffix_length * k * self.model_dim * dtype_size) as u64;+ if y_partial_bytes > 0 {+ blit_encoder.fill_buffer(+ &y_partial_buf,+ metal::NSRange::new(0, y_partial_bytes),+ 0,+ );+ }++ // Clear x_perm buffer+ let x_perm_bytes =+ (suffix_length * k * self.model_dim * dtype_size) as u64;+ if x_perm_bytes > 0 {+ blit_encoder.fill_buffer(+ &x_perm_buf,+ metal::NSRange::new(0, x_perm_bytes),+ 0,+ );+ }+ }++ blit_encoder.end_encoding();++ // Use fused Router+TopK kernel for non-quantized routers+ match &self.router {+ RouterBlock::Metal {+ weights_buf,+ biases_buf,+ } => {+ // Use the fused router+topk kernel+ self.router_topk_kernel+ .encode(+ root,+ self.router_data_type,+ MoeRouterTopKArguments {+ input_buffer: &main_buf,+ weight_buffer: weights_buf,+ bias_buffer: biases_buf,+ topk_ids_buffer: &topk_ids_buf,+ topk_probs_buffer: &topk_probs_buf,+ t: suffix_length,+ d_model: self.model_dim,+ e,+ k,+ renorm: self.router_renorm,+ },+ )+ .expect("MoE fused router+topk failed");+ },+ }++ self.counts_offsets_kernel+ .encode(+ root,+ MoeCountsOffsetsFusedArguments {+ topk_ids_buffer: &topk_ids_buf,+ offsets_buffer: &offsets_buf,+ sum_k_buffer: &sumk_buf,+ partials_buffer: &partials_buf,+ t: suffix_length,+ e,+ k,+ },+ )+ .expect("MoE counts+offsets failed");++ let num_blocks = ((suffix_length + 255) / 256).max(1);+ let num_tiles = ((e + 512 - 1) / 512).max(1);++ self.scatter_kernels+ .encode_block_bases(+ root,+ MoeBlockBasesArguments {+ partials_buffer: &partials_buf,+ block_bases_buffer: &block_bases_buf,+ block_alloc_buffer: &block_alloc_buf,+ e,+ num_blocks,+ num_tiles,+ },+ )+ .expect("MoE block bases failed");++ self.scatter_kernels+ .encode_scatter_with_map(+ root,+ MoeScatterWithMapArguments {+ base: MoeScatterArguments {+ topk_ids_buffer: &topk_ids_buf,+ topk_probs_buffer: &topk_probs_buf,+ offsets_buffer: &offsets_buf,+ block_bases_buffer: &block_bases_buf,+ block_alloc_buffer: &block_alloc_buf,+ out_ids_buffer: &bucketed_ids_buf,+ out_probs_buffer: &bucketed_probs_buf,+ t: suffix_length,+ e,+ k,+ num_blocks,+ num_tiles,+ },+ tok2row_buffer: &tok2row_buf,+ },+ self.data_type,+ )+ .expect("MoE scatter failed");++ self.gather_kernel+ .encode(+ root,+ self.data_type,+ MoeGatherArguments {+ x_buffer: &main_buf,+ bucketed_ids_buffer: &bucketed_ids_buf,+ x_perm_buffer: &x_perm_buf,+ sumk_buffer: &sumk_buf,+ t: suffix_length,+ k,+ d_model: self.model_dim,+ },+ )+ .expect("MoE gather failed");++ let gating_code = Self::gating_code_from_activation(+ &self.moe_config.expert_config.activation,+ );++ // Compute clipping values and alpha for expert kernels+ let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]+ .unwrap_or(f32::NEG_INFINITY);+ let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]+ .unwrap_or(f32::INFINITY);+ let up_clip_min = self.moe_config.expert_config.up_clipping[0];+ let up_clip_max = self.moe_config.expert_config.up_clipping[1];+ let silu_alpha = self.moe_config.expert_config.activation.alpha();++ if suffix_length == 1 {+ let total_rows = suffix_length * k;+ let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;++ self.experts_two_pass_decode_kernel+ .encode(+ root,+ MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf,+ w13_all: &self.shared_weights.w13_buf,+ w2_all: &self.shared_weights.w2_buf,+ up_biases: &self.shared_weights.up_biases_buf,+ down_biases: &self.shared_weights.down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows,+ d_model: self.model_dim,+ d_ff: self.hidden_dim,+ e,+ num_tiles_k,+ gating_code,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ data_type: self.data_type,+ },+ )+ .expect("MoE experts two-pass failed");+ } else {+ let total_rows = suffix_length * k;+ let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;++ self.experts_two_pass_prefill_kernel+ .encode(+ root,+ MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf,+ w13_all: &self.shared_weights.w13_buf,+ w2_all: &self.shared_weights.w2_buf,+ up_biases: &self.shared_weights.up_biases_buf,+ down_biases: &self.shared_weights.down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows,+ d_model: self.model_dim,+ d_ff: self.hidden_dim,+ e,+ num_tiles_k,+ gating_code,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ data_type: self.data_type,+ },+ )+ .expect("MoE experts two-pass prefill failed");+ }++ self.finalize_kernel+ .encode(+ root,+ MoeFinalizeArguments {+ tok2row_buffer: &tok2row_buf,+ probs_buffer: &topk_probs_buf,+ y_partial_buffer: &y_partial_buf,+ y_out_buffer: &main_buf,+ t: suffix_length,+ d_model: self.model_dim,+ k,+ },+ self.data_type,+ )+ .expect("MoE finalize failed");++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rsnew file mode 100644index 000000000..461fcf1ae--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs@@ -0,0 +1,11 @@+//! Shared MoE expert weights.++use std::rc::Rc;++#[derive(Clone)]+pub struct SharedMoeWeights {+ pub w13_buf: Rc<metal::Buffer>,+ pub w2_buf: Rc<metal::Buffer>,+ pub up_biases_buf: Rc<metal::Buffer>,+ pub down_biases_buf: Rc<metal::Buffer>,+}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rsnew file mode 100644index 000000000..a9e2b56a5--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs@@ -0,0 +1,138 @@+//! LayerNorm encodable.++use std::rc::Rc;++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::super::{EncodableBlock, EncodingParameters};+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::layer_norm::{+ LayerNormArguments, LayerNormError, LayerNormKernel,+ },+ },+ config::{NormalizationConfig, UpcastMode},+ parameters::ParameterTree,+};++pub struct LayerNorm {+ kernel: LayerNormKernel,+ config: NormalizationConfig,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ scales_buffer: MTLBuffer,+}++impl LayerNorm {+ pub fn new(+ context: &MTLContext,+ intermediate_data_type: DataType,+ config: NormalizationConfig,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, LayerNormError> {+ // Load scales from parameter tree+ let scales_param = parameter_tree.leaf("scales").map_err(|e| {+ LayerNormError::MetalError(MTLError::Library(+ crate::backends::metal::error::LibraryError::Custom(format!(+ "Failed to load scales: {:?}",+ e+ )),+ ))+ })?;++ let scales_data = scales_param.buffer();+ let scales_buffer = context.device.new_buffer_with_data(+ scales_data.as_ptr() as *const _,+ scales_data.len() as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );++ let accumulation_data_type: DataType =+ config.accumulation_precision.into();+ let scale_data_type: DataType = config.scale_precision.into();++ let (input_type, scales_type, output_type) = match config.upcast_mode {+ UpcastMode::OnlyNormalization => {+ (intermediate_data_type, scale_data_type, scale_data_type)+ },+ UpcastMode::FullLayer => {+ (intermediate_data_type, scale_data_type, scale_data_type)+ },+ };++ let kernel = LayerNormKernel::new(+ context,+ input_type,+ scales_type,+ output_type,+ accumulation_data_type,+ config.upcast_mode == UpcastMode::FullLayer,+ )?;++ Ok(Self {+ kernel,+ config,+ input_array_id,+ output_array_id,+ scales_buffer,+ })+ }+}++impl EncodableBlock for LayerNorm {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let input_binding = state.arrays(&[self.input_array_id]);+ let output_binding = state.arrays(&[self.output_array_id]);++ let input_shape = {+ let input_array = input_binding[0].borrow();+ input_array.shape().to_vec()+ };++ let mut input_array = input_binding[0].borrow_mut();+ let mut output_array = output_binding[0].borrow_mut();++ let input_buffer = unsafe { input_array.mtl_buffer() };+ let output_buffer = unsafe { output_array.mtl_buffer() };++ let batch_size = input_shape[0] as i32;+ let model_dim = input_shape[1] as i32;++ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();++ if let Err(e) = self.kernel.encode(+ &compute_encoder,+ LayerNormArguments {+ input_buffer: &input_buffer,+ scales_buffer: &self.scales_buffer,+ output_buffer: &output_buffer,+ batch_size,+ model_dim,+ epsilon: self.config.epsilon,+ scale_offset: self.config.scale_offset.unwrap_or(0.0),+ },+ ) {+ eprintln!("Failed to encode LayerNorm kernel: {:?}", e);+ }++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rsnew file mode 100644index 000000000..d2d7309cf--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs@@ -0,0 +1,92 @@+//! Unified normalization encodables: LayerNorm, RMSNorm, QKNorm.++mod layer_norm;+mod qk_norm;+mod rms_norm;++use std::rc::Rc;++pub use layer_norm::LayerNorm;+use mpsgraph::CommandBuffer as MPSCommandBuffer;+pub use qk_norm::QKNorm;+pub use rms_norm::RMSNorm;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ DataType,+ backends::metal::{+ MTLContext,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::{LayerNormError, RMSNormError},+ },+ config::NormalizationConfig,+ parameters::ParameterTree,+};++/// Unified error type for normalization operations.+#[derive(Debug, thiserror::Error)]+pub enum NormalizationError {+ #[error("LayerNorm error: {0}")]+ LayerNorm(#[from] LayerNormError),+ #[error("RMSNorm error: {0}")]+ RMSNorm(#[from] RMSNormError),+}++/// Unified normalization encodable that can be either LayerNorm or RMSNorm.+pub enum Normalization {+ LayerNorm(LayerNorm),+ RMSNorm(RMSNorm),+}++impl Normalization {+ /// Creates the appropriate normalization encodable based on config.+ /// Uses LayerNorm if subtract_mean is true, otherwise RMSNorm.+ pub fn new(+ context: &MTLContext,+ intermediate_data_type: DataType,+ config: NormalizationConfig,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, NormalizationError> {+ if config.subtract_mean {+ // Use LayerNorm (subtract mean before normalization)+ Ok(Normalization::LayerNorm(LayerNorm::new(+ context,+ intermediate_data_type,+ config,+ input_array_id,+ output_array_id,+ parameter_tree,+ )?))+ } else {+ // Use RMSNorm (no mean subtraction)+ Ok(Normalization::RMSNorm(RMSNorm::new(+ context,+ intermediate_data_type,+ config,+ input_array_id,+ output_array_id,+ parameter_tree,+ )?))+ }+ }+}++impl EncodableBlock for Normalization {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ match self {+ Normalization::LayerNorm(layer_norm) => {+ layer_norm.encode(state, command_buffer, parameters)+ },+ Normalization::RMSNorm(rms_norm) => {+ rms_norm.encode(state, command_buffer, parameters)+ },+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rsnew file mode 100644index 000000000..c8e39f808--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs@@ -0,0 +1,247 @@+//! QK Normalization encodable.++use std::rc::Rc;++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::super::{EncodableBlock, EncodingParameters};+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::rms_norm::{+ QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel,+ RMSNormKernelType,+ },+ },+ config::{NormalizationConfig, UpcastMode},+ parameters::ParameterTree,+};++pub struct QKNorm {+ query_kernel: Option<RMSNormKernel>,+ key_kernel: Option<RMSNormKernel>,+ query_config: Option<NormalizationConfig>,+ key_config: Option<NormalizationConfig>,+ qkv_array_id: ArrayId,+ query_scales_buffer: Option<MTLBuffer>,+ key_scales_buffer: Option<MTLBuffer>,+ num_q_heads: usize,+ num_kv_heads: usize,+ head_dim: usize,+}++impl QKNorm {+ pub fn new(+ context: &MTLContext,+ intermediate_data_type: DataType,+ query_config: Option<NormalizationConfig>,+ key_config: Option<NormalizationConfig>,+ qkv_array_id: ArrayId,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ num_q_heads: usize,+ num_kv_heads: usize,+ head_dim: usize,+ ) -> Result<Self, RMSNormError> {+ let mut query_kernel = None;+ let mut key_kernel = None;+ let mut query_scales_buffer = None;+ let mut key_scales_buffer = None;++ // Setup query normalization if configured+ if let Some(ref q_config) = query_config {+ let scales_param =+ parameter_tree.leaf("query_norm.scales").map_err(|e| {+ RMSNormError::MetalError(MTLError::Library(+ crate::backends::metal::error::LibraryError::Custom(+ format!("Failed to load query scales: {:?}", e),+ ),+ ))+ })?;++ let scales_data = scales_param.buffer();+ let scales_buffer = context.device.new_buffer_with_data(+ scales_data.as_ptr() as *const _,+ scales_data.len() as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );++ let accumulation_data_type: DataType =+ q_config.accumulation_precision.into();+ let scale_data_type: DataType = q_config.scale_precision.into();++ let (input_type, scales_type, output_type) = match q_config+ .upcast_mode+ {+ UpcastMode::OnlyNormalization => {+ (intermediate_data_type, scale_data_type, scale_data_type)+ },+ UpcastMode::FullLayer => {+ (intermediate_data_type, scale_data_type, scale_data_type)+ },+ };++ let kernel = RMSNormKernel::new_with_mode(+ context,+ input_type,+ scales_type,+ output_type,+ accumulation_data_type,+ RMSNormKernelType::QueryKey,+ q_config.upcast_mode == UpcastMode::FullLayer,+ )?;++ query_kernel = Some(kernel);+ query_scales_buffer = Some(scales_buffer);+ }++ // Setup key normalization if configured+ if let Some(ref k_config) = key_config {+ let scales_param =+ parameter_tree.leaf("key_norm.scales").map_err(|e| {+ RMSNormError::MetalError(MTLError::Library(+ crate::backends::metal::error::LibraryError::Custom(+ format!("Failed to load key scales: {:?}", e),+ ),+ ))+ })?;++ let scales_data = scales_param.buffer();+ let scales_buffer = context.device.new_buffer_with_data(+ scales_data.as_ptr() as *const _,+ scales_data.len() as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );++ let accumulation_data_type: DataType =+ k_config.accumulation_precision.into();+ let scale_data_type: DataType = k_config.scale_precision.into();++ let (input_type, scales_type, output_type) = match k_config+ .upcast_mode+ {+ UpcastMode::OnlyNormalization => {+ (intermediate_data_type, scale_data_type, scale_data_type)+ },+ UpcastMode::FullLayer => {+ (intermediate_data_type, scale_data_type, scale_data_type)+ },+ };++ let kernel = RMSNormKernel::new_with_mode(+ context,+ input_type,+ scales_type,+ output_type,+ accumulation_data_type,+ RMSNormKernelType::QueryKey,+ k_config.upcast_mode == UpcastMode::FullLayer,+ )?;++ key_kernel = Some(kernel);+ key_scales_buffer = Some(scales_buffer);+ }++ Ok(Self {+ query_kernel,+ key_kernel,+ query_config,+ key_config,+ qkv_array_id,+ query_scales_buffer,+ key_scales_buffer,+ num_q_heads,+ num_kv_heads,+ head_dim,+ })+ }+}++impl EncodableBlock for QKNorm {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let qkv_binding = state.arrays(&[self.qkv_array_id]);+ let qkv_shape = {+ let qkv_array = qkv_binding[0].borrow();+ qkv_array.shape().to_vec()+ };++ let mut qkv_array = qkv_binding[0].borrow_mut();+ let qkv_buffer = unsafe { qkv_array.mtl_buffer() };++ let batch_size = qkv_shape[0] as i32;+ let head_dim = self.head_dim as i32;++ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();++ // Process query normalization if configured+ if let (+ Some(query_kernel),+ Some(query_scales_buffer),+ Some(query_config),+ ) =+ (&self.query_kernel, &self.query_scales_buffer, &self.query_config)+ {+ if let Err(e) = query_kernel.encode_qk_norm(+ &compute_encoder,+ QKNormArguments {+ qkv_input_buffer: &qkv_buffer,+ scales_buffer: query_scales_buffer,+ qkv_output_buffer: &qkv_buffer,+ batch_size,+ // Always pass actual head counts (needed for correct buffer addressing)+ num_q_heads: self.num_q_heads as i32,+ num_kv_heads: self.num_kv_heads as i32,+ head_dim,+ epsilon: query_config.epsilon,+ scale_offset: query_config.scale_offset.unwrap_or(0.0),+ target: QKNormTarget::QueryHeads,+ },+ ) {+ eprintln!(+ "Failed to encode query normalization kernel: {:?}",+ e+ );+ }+ }++ // Process key normalization if configured+ if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =+ (&self.key_kernel, &self.key_scales_buffer, &self.key_config)+ {+ if let Err(e) = key_kernel.encode_qk_norm(+ &compute_encoder,+ QKNormArguments {+ qkv_input_buffer: &qkv_buffer,+ scales_buffer: key_scales_buffer,+ qkv_output_buffer: &qkv_buffer,+ batch_size,+ // Always pass actual head counts (needed for correct buffer addressing)+ num_q_heads: self.num_q_heads as i32,+ num_kv_heads: self.num_kv_heads as i32,+ head_dim,+ epsilon: key_config.epsilon,+ scale_offset: key_config.scale_offset.unwrap_or(0.0),+ target: QKNormTarget::KeyHeads,+ },+ ) {+ eprintln!("Failed to encode key normalization kernel: {:?}", e);+ }+ }++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rsnew file mode 100644index 000000000..83a96877e--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs@@ -0,0 +1,142 @@+//! RMS Normalization encodable.++use std::rc::Rc;++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::super::{EncodableBlock, EncodingParameters};+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::rms_norm::{+ RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType,+ },+ },+ config::{NormalizationConfig, UpcastMode},+ parameters::ParameterTree,+};++pub struct RMSNorm {+ kernel: RMSNormKernel,+ config: NormalizationConfig,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ scales_buffer: MTLBuffer,+}++impl RMSNorm {+ pub fn new(+ context: &MTLContext,+ intermediate_data_type: DataType,+ config: NormalizationConfig,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, RMSNormError> {+ // Load scales from parameter tree+ let scales_param = parameter_tree.leaf("scales").map_err(|e| {+ RMSNormError::MetalError(MTLError::Library(+ crate::backends::metal::error::LibraryError::Custom(format!(+ "Failed to load scales: {:?}",+ e+ )),+ ))+ })?;++ // TODO: Don't create buffers dynamically, we need to use forward pass storage for thing like this+ let scales_data = scales_param.buffer();+ let scales_buffer = context.device.new_buffer_with_data(+ scales_data.as_ptr() as *const _,+ scales_data.len() as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );++ let accumulation_data_type: DataType =+ config.accumulation_precision.into();+ let scale_data_type: DataType = config.scale_precision.into();++ let (input_type, scales_type, output_type) = match config.upcast_mode {+ UpcastMode::OnlyNormalization => {+ // Input stays as pipeline type, scales stay scale precision, output is scale precision+ (intermediate_data_type, scale_data_type, scale_data_type)+ },+ UpcastMode::FullLayer => {+ // Input stays as pipeline type, scales stay in original precision (will be cast to AccumT inside kernel), output is scale precision+ (intermediate_data_type, scale_data_type, scale_data_type)+ },+ };++ let kernel = RMSNormKernel::new_with_mode(+ context,+ input_type,+ scales_type,+ output_type,+ accumulation_data_type,+ RMSNormKernelType::Standard,+ config.upcast_mode == UpcastMode::FullLayer,+ )?;++ Ok(Self {+ kernel,+ config,+ input_array_id,+ output_array_id,+ scales_buffer,+ })+ }+}++impl EncodableBlock for RMSNorm {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let input_binding = state.arrays(&[self.input_array_id]);+ let output_binding = state.arrays(&[self.output_array_id]);++ let input_shape = {+ let input_array = input_binding[0].borrow();+ input_array.shape().to_vec()+ };++ let mut input_array = input_binding[0].borrow_mut();+ let mut output_array = output_binding[0].borrow_mut();++ let input_buffer = unsafe { input_array.mtl_buffer() };+ let output_buffer = unsafe { output_array.mtl_buffer() };++ let batch_size = input_shape[0] as i32;+ let model_dim = input_shape[1] as i32;++ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();++ if let Err(e) = self.kernel.encode(+ &compute_encoder,+ RMSNormArguments {+ input_buffer: &input_buffer,+ scales_buffer: &self.scales_buffer,+ output_buffer: &output_buffer,+ batch_size,+ model_dim,+ epsilon: self.config.epsilon,+ scale_offset: self.config.scale_offset.unwrap_or(0.0),+ },+ ) {+ eprintln!("Failed to encode RMS norm kernel: {:?}", e);+ }++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/pooling.rs b/crates/uzu/src/backends/metal/encodable_block/pooling.rsnew file mode 100644index 000000000..323585f66--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/pooling.rs@@ -0,0 +1,112 @@+//! Pooling encodable for sequence-level aggregation.++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ Array,+ backends::metal::{+ forward_pass::{ArrayId, ForwardPassState},+ kernel::PoolingKernel,+ },+ config::PoolingType,+};++pub struct Pooling {+ pooling_kernel: PoolingKernel,+ pooling_type: PoolingType,+ model_dim: usize,+}++impl Pooling {+ pub fn new(+ pooling_kernel: PoolingKernel,+ pooling_type: PoolingType,+ model_dim: usize,+ ) -> Self {+ Self {+ pooling_kernel,+ pooling_type,+ model_dim,+ }+ }+}++impl EncodableBlock for Pooling {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ _parameters: &EncodingParameters,+ ) {+ let batch_size = 1;+ let seq_len = state.aux_buffers_suffix_length();++ let arrays = state.arrays(&[ArrayId::Main, ArrayId::ClassifierPooling]);+ #[cfg_attr(not(feature = "tracing"), allow(unused_variables))]+ let data_type = { Array::data_type(&*arrays[0].borrow()) };+ let mut main_array = arrays[0].borrow_mut();+ let input_buffer = unsafe { main_array.mtl_buffer() };++ let mut pooling_array = arrays[1].borrow_mut();+ let output_buffer = unsafe { pooling_array.mtl_buffer().to_owned() };++ let root = command_buffer.root_command_buffer();+ let encoder = root.new_compute_command_encoder();++ let result = match self.pooling_type {+ PoolingType::Cls => self.pooling_kernel.encode_cls(+ encoder,+ input_buffer,+ &output_buffer,+ batch_size as i32,+ seq_len as i32,+ self.model_dim as i32,+ ),+ PoolingType::Mean => self.pooling_kernel.encode_mean(+ encoder,+ input_buffer,+ &output_buffer,+ batch_size as i32,+ seq_len as i32,+ self.model_dim as i32,+ ),+ };++ if result.is_err() {+ panic!("Failed to encode pooling kernel");+ }++ encoder.end_encoding();+ drop(main_array);+ drop(pooling_array);++ command_buffer.commit_and_continue();++ #[cfg(feature = "tracing")]+ {+ let traces_rc = state.traces().clone();+ let traces_ref = traces_rc.borrow();+ let mut trace_arr = traces_ref.output_pooling().borrow_mut();+ let dst_buf = unsafe { trace_arr.mtl_buffer().to_owned() };+ drop(trace_arr);+ drop(traces_ref);++ let root = command_buffer.root_command_buffer();+ let blit = root.new_blit_command_encoder();+ blit.copy_from_buffer(+ &output_buffer,+ 0,+ &dst_buf,+ 0,+ (batch_size * self.model_dim * data_type.size_in_bytes())+ as u64,+ );+ blit.end_encoding();++ let root_owned = root.to_owned();+ command_buffer.commit_and_continue();+ root_owned.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rsnew file mode 100644index 000000000..25a4fcf4f--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs@@ -0,0 +1,96 @@+//! Prediction head encodable for classification output.++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters};+#[cfg(feature = "tracing")]+use crate::Array;+#[cfg(feature = "tracing")]+use crate::backends::metal::forward_pass::ArrayId;+use crate::backends::metal::forward_pass::ForwardPassState;++pub struct ClassifierPredictionHead {+ dense: Box<dyn EncodableBlock>,+ activation: Box<dyn EncodableBlock>,+ norm: Box<dyn EncodableBlock>,+ readout: Box<dyn EncodableBlock>,+ #[cfg_attr(not(feature = "tracing"), allow(dead_code))]+ num_labels: usize,+}++impl ClassifierPredictionHead {+ pub fn new(+ dense: Box<dyn EncodableBlock>,+ activation: Box<dyn EncodableBlock>,+ norm: Box<dyn EncodableBlock>,+ readout: Box<dyn EncodableBlock>,+ num_labels: usize,+ ) -> Self {+ Self {+ dense,+ activation,+ norm,+ readout,+ num_labels,+ }+ }+}++impl EncodableBlock for ClassifierPredictionHead {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ self.dense.encode(state, command_buffer, parameters);+ self.activation.encode(state, command_buffer, parameters);+ self.norm.encode(state, command_buffer, parameters);+ self.readout.encode(state, command_buffer, parameters);++ #[cfg_attr(feature = "tracing", allow(unused_variables))]+ let root_after_head = command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();++ #[cfg(not(feature = "tracing"))]+ root_after_head.wait_until_completed();++ #[cfg(feature = "tracing")]+ {+ let traces_rc = state.traces().clone();+ let logits_arrays =+ state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);+ let mut logits_array_ref = logits_arrays[0].borrow_mut();+ let linear_output_buffer =+ unsafe { logits_array_ref.mtl_buffer().to_owned() };+ let data_type = Array::data_type(&*logits_array_ref);+ let batch_size = Array::shape(&*logits_array_ref)[0];+ drop(logits_array_ref);++ let traces_ref = traces_rc.borrow();+ let mut trace_logits = traces_ref.logits.borrow_mut();+ let dst_trace_buf = unsafe { trace_logits.mtl_buffer().to_owned() };+ drop(trace_logits);+ drop(traces_ref);++ let copy_size_bytes =+ (batch_size * self.num_labels * data_type.size_in_bytes())+ as u64;++ let root = command_buffer.root_command_buffer();+ let blit = root.new_blit_command_encoder();+ blit.copy_from_buffer(+ &linear_output_buffer,+ 0,+ &dst_trace_buf,+ 0,+ copy_size_bytes,+ );+ blit.end_encoding();++ let root_owned = root.to_owned();+ command_buffer.commit_and_continue();+ root_owned.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rsnew file mode 100644index 000000000..e81874c6a--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs@@ -0,0 +1,116 @@+//! Rope (Rotary Position Embedding) encodable.++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ Array,+ backends::metal::{+ KernelDataType, MTLContext,+ forward_pass::{ArrayId, ForwardPassState, RopeType},+ kernel::rope::{RopeError, RopeKernel, RopeKernelArguments},+ },+};++pub struct Rope {+ kernel: RopeKernel,+ rope_type: RopeType,+}++impl Rope {+ pub fn new(+ context: &MTLContext,+ data_type: KernelDataType,+ rope_type: RopeType,+ ) -> Result<Self, RopeError> {+ let kernel = RopeKernel::new(context, data_type)?;+ Ok(Self {+ kernel,+ rope_type,+ })+ }+}++impl EncodableBlock for Rope {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {+ let qkv_binding = state.arrays(&[ArrayId::QKV]);+ let qkv_array = qkv_binding[0].borrow();+ let suffix_length = qkv_array.shape()[0];++ let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);+ let queries_array = queries_binding[0].borrow();+ let num_heads = queries_array.shape()[0];+ let head_dim = queries_array.shape()[2];++ let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);+ let keys_array = keys_binding[0].borrow();+ let num_groups = keys_array.shape()[0];++ let cos_binding =+ state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);+ let cos_array = cos_binding[0].borrow();+ let cos_shape = cos_array.shape();+ let rope_max_seq_len = cos_shape[0];++ (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len)+ };++ let qkv_buffer_binding = state.arrays(&[ArrayId::QKV]);+ let mut qkv = qkv_buffer_binding[0].borrow_mut();++ let token_positions_binding = state.arrays(&[ArrayId::TokenPositions]);+ let mut token_positions = token_positions_binding[0].borrow_mut();++ let query_buffer_binding = state.arrays(&[ArrayId::RotatedQueries]);+ let mut rotated_queries = query_buffer_binding[0].borrow_mut();++ let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);+ let mut rotated_keys = rotated_keys_binding[0].borrow_mut();++ let cos_buffer_binding =+ state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);+ let mut rope_cosines = cos_buffer_binding[0].borrow_mut();++ let sin_buffer_binding =+ state.arrays(&[ArrayId::RopeSines(self.rope_type)]);+ let mut rope_sines = sin_buffer_binding[0].borrow_mut();++ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();++ self.kernel.encode(+ &compute_encoder,+ RopeKernelArguments {+ qkv_buffer: unsafe { &qkv.mtl_buffer() },+ cosines_buffer: unsafe { &rope_cosines.mtl_buffer() },+ sines_buffer: unsafe { &rope_sines.mtl_buffer() },+ token_positions_buffer: unsafe {+ &token_positions.mtl_buffer()+ },+ rotated_queries_buffer: unsafe {+ &rotated_queries.mtl_buffer()+ },+ rotated_keys_buffer: unsafe { &rotated_keys.mtl_buffer() },+ head_dim,+ num_heads,+ num_groups,+ suffix_length,+ max_sequence_length: rope_max_seq_len,+ },+ );++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/sampling.rs b/crates/uzu/src/backends/metal/encodable_block/sampling.rsnew file mode 100644index 000000000..269a8c1cd--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/sampling.rs@@ -0,0 +1,105 @@+//! Sampling kernel encodable.++use std::rc::Rc;++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters};+use crate::backends::metal::{+ KernelDataType, MTLContext,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::sampling::{ArgmaxStrategy, SamplingError, SamplingKernel},+};++pub struct Sampling {+ pub kernel: SamplingKernel,+}++impl Sampling {+ pub fn new(+ context: &Rc<MTLContext>,+ data_type: KernelDataType,+ max_batch_size: usize,+ max_vocab_size: usize,+ ) -> Result<Self, SamplingError> {+ let kernel = SamplingKernel::new(+ context,+ data_type,+ max_batch_size,+ max_vocab_size,+ )?;+ Ok(Self {+ kernel,+ })+ }++ pub fn new_with_strategy(+ context: &Rc<MTLContext>,+ data_type: KernelDataType,+ max_batch_size: usize,+ max_vocab_size: usize,+ argmax_strategy: ArgmaxStrategy,+ ) -> Result<Self, SamplingError> {+ let kernel = SamplingKernel::new_with_strategy(+ context,+ data_type,+ max_batch_size,+ max_vocab_size,+ argmax_strategy,+ )?;+ Ok(Self {+ kernel,+ })+ }+}++impl EncodableBlock for Sampling {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ assert!(+ state.sampling_output().is_some(),+ "Sampling output buffer must be pre-allocated"+ );++ let logits_binding = state.arrays(&[ArrayId::Logits]);+ let mut logits = logits_binding[0].borrow_mut();++ let logits_shape = {+ use crate::Array;+ logits.shape()+ };+ let batch_size = state.active_suffix_length();+ let vocab_size = logits_shape[1];++ let seeds_binding = state.arrays(&[ArrayId::TokenSeeds]);+ let mut seeds = seeds_binding[0].borrow_mut();++ let mut output_buffer_ref =+ state.sampling_output().unwrap().borrow_mut();++ let sampling_method = state.sampling_method().unwrap();++ let root_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ if let Err(e) = self.kernel.encode(+ unsafe { &logits.mtl_buffer() },+ unsafe { Some(&seeds.mtl_buffer()) },+ unsafe { &output_buffer_ref.mtl_buffer() },+ sampling_method,+ batch_size,+ vocab_size,+ &root_command_buffer,+ ) {+ panic!("Sampling encoding failed: {:?}", e);+ }++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ root_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rsnew file mode 100644index 000000000..ae72e7d47--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs@@ -0,0 +1,66 @@+//! Tensor add-swap encodable.++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ Array,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::TensorAddSwapKernel,+ },+};++pub struct TensorAddSwap {+ kernel: TensorAddSwapKernel,+ argument_arrays: Box<[ArrayId]>,+}++impl TensorAddSwap {+ pub fn new(+ context: &MTLContext,+ data_type: crate::backends::metal::KernelDataType,+ argument_arrays: Box<[ArrayId]>,+ ) -> Result<Self, MTLError> {+ let kernel = TensorAddSwapKernel::new(context, data_type)?;+ Ok(Self {+ kernel,+ argument_arrays,+ })+ }+}++impl EncodableBlock for TensorAddSwap {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&self.argument_arrays);+ assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");++ let length = arrays[0].borrow().num_elements();++ let mut skip_array = arrays[0].borrow_mut();+ let mut main_array = arrays[1].borrow_mut();+ let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };+ let main_mtl_buffer = unsafe { main_array.mtl_buffer() };++ let retained_mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();++ self.kernel.encode_into_command_buffer(+ &skip_mtl_buffer,+ &main_mtl_buffer,+ length,+ &retained_mtl_command_buffer,+ );++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ retained_mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rsnew file mode 100644index 000000000..61156c8fa--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs@@ -0,0 +1,66 @@+//! Tensor copy encodable.++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ Array,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::TensorCopyKernel,+ },+};++pub struct TensorCopy {+ kernel: TensorCopyKernel,+ argument_arrays: Box<[ArrayId]>,+}++impl TensorCopy {+ pub fn new(+ context: &MTLContext,+ data_type: crate::backends::metal::KernelDataType,+ argument_arrays: Box<[ArrayId]>,+ ) -> Result<Self, MTLError> {+ let kernel = TensorCopyKernel::new(context, data_type)?;+ Ok(Self {+ kernel,+ argument_arrays,+ })+ }+}++impl EncodableBlock for TensorCopy {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&self.argument_arrays);+ assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");++ let length = arrays[0].borrow().num_elements();++ let mut source_array = arrays[0].borrow_mut();+ let mut destination_array = arrays[1].borrow_mut();+ let source_mtl_buffer = unsafe { source_array.mtl_buffer() };+ let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };++ let retained_mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();++ self.kernel.encode_into_command_buffer(+ &source_mtl_buffer,+ &destination_mtl_buffer,+ length,+ &retained_mtl_command_buffer,+ );++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ retained_mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rssimilarity index 93%rename from crates/uzu/src/backends/metal/forward_pass/transformer_layer.rsrename to crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex fec5b39dd..0f00b2ece 100644--- a/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -7,29 +7,20 @@ use mpsgraph::{use objc2::rc::{Retained, autoreleasepool};use super::{- super::{- MTLContext,- graph::{linear_subgraph, mlp_subgraph, placeholder, shaped_type},- },- io_arrays::IOArrays,- mpsgraph_block::MPSGraphBlock,- state::ArrayId,+ EncodableBlock, MlpBlock, MoeBlock, QuantizedEmbeddingLookup,+ QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,backends::metal::{+ MTLContext,+ forward_pass::{ArrayId, IOArrays, MPSGraphBlock},graph::{embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,- embeddings_readout_subgraph,- },- kernel::{- MoeBlockEncodable, QuantizedLinearKernelBlock,- embedding::{- QuantizedEmbeddingLookupKernelBlock,- QuantizedEmbeddingReadoutKernelBlock,- },- mlp::MlpGateActMulEncodable,+ embeddings_readout_subgraph, linear_subgraph, mlp_subgraph,+ placeholder, shaped_type,},+ kernel::mlp::MlpGateActMulEncodable,},config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig,@@ -54,7 +45,7 @@ pub fn linear_block<const N: usize>(input_array_id: ArrayId,output_array_id: ArrayId,compilation_descriptor: &CompilationDescriptor,-) -> Box<dyn super::encodable_with_state::EncodableWithState> {+) -> Box<dyn EncodableBlock> {if let LinearConfig::Quantized(quant_config) = config {let out_sum: usize = output_dims.iter().sum();return quantized_linear_block_custom(@@ -134,11 +125,8 @@ pub fn quantized_linear_block_custom(parameter_tree: &ParameterTree<Rc<MTLContext>>,input_array_id: ArrayId,output_array_id: ArrayId,-) -> Result<- Box<dyn super::encodable_with_state::EncodableWithState>,- crate::backends::metal::MTLError,-> {- let block = QuantizedLinearKernelBlock::new(+) -> Result<Box<dyn EncodableBlock>, crate::backends::metal::MTLError> {+ let block = QuantizedLinear::new(context,config,input_dim,@@ -157,7 +145,7 @@ pub fn mlp_block(context: &MTLContext,parameter_tree: &ParameterTree<Rc<MTLContext>>,compilation_descriptor: &CompilationDescriptor,-) -> Box<dyn super::encodable_with_state::EncodableWithState> {+) -> Box<dyn EncodableBlock> {if let crate::config::MLPConfig::Dense(dense) = config {if let crate::config::LinearConfig::Quantized(ref quant_config)| crate::config::LinearConfig::MLXQuantized(ref quant_config) =@@ -168,7 +156,7 @@ pub fn mlp_block(dense.linear_config.activation_precision().into();// Up fused: Main -> MlpFusedUp- let up = QuantizedLinearKernelBlock::new(+ let up = QuantizedLinear::new(context,quant_config,model_dim,@@ -189,7 +177,7 @@ pub fn mlp_block(.expect("Failed to build MLP gate activation kernel");// Down: MlpHidden -> Main- let down = QuantizedLinearKernelBlock::new(+ let down = QuantizedLinear::new(context,quant_config,hidden_dim,@@ -200,24 +188,16 @@ pub fn mlp_block().expect("Failed to build MLP down quantized block");- let enc =- crate::backends::metal::kernel::mlp::MlpBlockEncodable::new(- up, gate_op, down,- );+ let enc = MlpBlock::new(up, gate_op, down);return Box::new(enc);}}// STEP 2: Enable MOE block creation (encode will be minimal)if let crate::config::MLPConfig::MixtureOfExperts(moe) = config {- let moe_block = MoeBlockEncodable::new(- context,- moe,- model_dim,- hidden_dim,- parameter_tree,- )- .expect("Failed to build MoE block");+ let moe_block =+ MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)+ .expect("Failed to build MoE block");return Box::new(moe_block);}@@ -248,7 +228,7 @@ pub fn mlp_block().unwrap(),crate::config::MLPConfig::MixtureOfExperts(_moe) => {- unreachable!("MoE uses MoeBlockEncodable, not MPSGraph")+ unreachable!("MoE uses MoeBlock, not MPSGraph")},};@@ -282,7 +262,7 @@ pub fn embed_block(context: &MTLContext,compilation_descriptor: &CompilationDescriptor,parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Box<dyn super::encodable_with_state::EncodableWithState> {+) -> Box<dyn EncodableBlock> {let graph = Graph::new();let input_shape = [-1 as isize];@@ -506,7 +486,7 @@ pub fn embed_block(.subtree("embedding").expect("Failed to get embedding subtree");- let block = QuantizedEmbeddingLookupKernelBlock::new(+ let block = QuantizedEmbeddingLookup::new(context,data_type,config.vocab_size,@@ -527,7 +507,7 @@ pub fn readout_block(context: &MTLContext,compilation_descriptor: &CompilationDescriptor,parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Box<dyn super::encodable_with_state::EncodableWithState> {+) -> Box<dyn EncodableBlock> {let graph = Graph::new();let input_shape = [-1 as isize, config.model_dim as isize];@@ -640,7 +620,7 @@ pub fn readout_block(.subtree("embedding").expect("Failed to get embedding subtree");- let block = QuantizedEmbeddingReadoutKernelBlock::new(+ let block = QuantizedEmbeddingReadout::new(context,data_type,config.vocab_size,@@ -732,7 +712,7 @@ pub fn readout_block(.subtree("embedding").expect("Failed to get embedding subtree");- let block = QuantizedEmbeddingReadoutKernelBlock::new(+ let block = QuantizedEmbeddingReadout::new(context,data_type,config.vocab_size,diff --git a/crates/uzu/src/backends/metal/error.rs b/crates/uzu/src/backends/metal/error.rsindex 802fd7d9f..384128852 100644--- a/crates/uzu/src/backends/metal/error.rs+++ b/crates/uzu/src/backends/metal/error.rs@@ -174,3 +174,18 @@ pub enum PixelFormatError {#[error("{0}")]Custom(String),}++// Classifier context errors+#[derive(Debug, Error)]+pub enum ClassifierError {+ #[error("Missing required config field: {0}")]+ MissingConfigField(String),+ #[error("Model requires attention layers but found non-attention mixer")]+ NonAttentionMixer,+ #[error("Weight subtree not found: {0}")]+ WeightSubtreeNotFound(String),+ #[error("Kernel creation failed: {0}")]+ KernelCreationFailed(String),+ #[error("{0}")]+ Custom(String),+}diff --git a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rsdeleted file mode 100644index a4dcb4b25..000000000--- a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs+++ /dev/null@@ -1,616 +0,0 @@-use std::rc::Rc;--use metal::ComputeCommandEncoderRef;-use mpsgraph::CommandBuffer as MPSCommandBuffer;-use objc2::rc::autoreleasepool;--use crate::{- DataType,- backends::metal::{- MTLContext,- compilation_parameters::CompilationConfig,- forward_pass::{- ArrayId, EncoderResolver, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- transformer_layer,- },- kernel::{- AttentionKernelEncodable, KernelDataType, MambaMixerEncodable,- QKNormKernelEncodable, RMSNormKernelEncodable, TensorAddSwap,- TensorCopy,- },- },- config::{- DecoderLayerType,- decoder_layer::{DecoderLayerConfig, MixerConfig},- },- parameters::ParameterTree,-};--pub struct LayerExecutables {- pub layer_index: usize,- pub copy_main_to_shortcut: Box<dyn EncodableWithState>,- pub pre_attention_norm: Box<dyn EncodableWithState>,- pub(crate) mixer: MixerExecutables,- pub post_attention_norm: Option<Box<dyn EncodableWithState>>,- pub main_shortcut_add_swap: Box<dyn EncodableWithState>,- pub pre_mlp_norm: Box<dyn EncodableWithState>,- pub mlp: Box<dyn EncodableWithState>,- pub post_mlp_norm: Option<Box<dyn EncodableWithState>>,-}--pub(crate) enum MixerExecutables {- Attention {- qkv_projection: Box<dyn EncodableWithState>,- qk_norm: Option<Box<dyn EncodableWithState>>,- rope: Rc<Box<dyn EncodableWithState>>,- attention: Box<dyn EncodableWithState>,- out_projection: Box<dyn EncodableWithState>,- },- StateSpace {- mixer: Box<dyn EncodableWithState>,- },-}--impl LayerExecutables {- #[allow(clippy::too_many_arguments)]- pub fn new(- mtl_context: &MTLContext,- layer_config: &DecoderLayerConfig,- layer_type: &DecoderLayerType,- compilation_config: Rc<CompilationConfig>,- layer_index: usize,- model_dim: usize,- hidden_dim: usize,- num_heads: usize,- head_dim: usize,- num_groups: usize,- attention_scale: Option<f32>,- decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,- rope: Option<Rc<Box<dyn EncodableWithState>>>,- ) -> Self {- autoreleasepool(|_| {- let intermediate_data_type: DataType =- match &layer_config.mixer_config {- MixerConfig::Attention(attention) => attention- .qkv_projection_config- .activation_precision()- .into(),- MixerConfig::Mamba(mamba) => {- mamba.in_projection_config.activation_precision().into()- },- };- let kernel_data_type: KernelDataType =- intermediate_data_type.into();-- let copy_main_to_shortcut: Box<dyn EncodableWithState> = Box::new(- TensorCopy::new(- mtl_context,- kernel_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap(),- );-- let pre_attention_norm: Box<dyn EncodableWithState> = Box::new(- RMSNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- layer_config.pre_attention_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),- );-- let mixer = match &layer_config.mixer_config {- MixerConfig::Attention(attention_config) => {- let rope_block = rope- .clone()- .expect("RoPE encoder missing for attention layer");-- let qkv_projection = transformer_layer::linear_block(- &attention_config.qkv_projection_config,- attention_config.has_qkv_biases,- model_dim,- [- num_heads * head_dim,- num_groups * head_dim,- num_groups * head_dim,- ],- mtl_context,- &decoder_layer_loader- .subtree("mixer.qkv_projection")- .unwrap(),- ArrayId::Main,- ArrayId::QKV,- &compilation_config.descriptor_mlp,- );-- let qk_norm: Option<Box<dyn EncodableWithState>> =- if attention_config.query_norm_config.is_some()- || attention_config.key_norm_config.is_some()- {- match QKNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- attention_config.query_norm_config.clone(),- attention_config.key_norm_config.clone(),- ArrayId::QKV,- &decoder_layer_loader.subtree("mixer").unwrap(),- num_heads,- num_groups,- head_dim,- ) {- Ok(qk_norm) => Some(Box::new(qk_norm)- as Box<dyn EncodableWithState>),- Err(e) => panic!(- "Failed to create QK norm kernel for layer {}: {:?}",- layer_index, e- ),- }- } else {- None- };-- let out_projection = transformer_layer::linear_block(- &attention_config.out_projection_config,- attention_config.has_out_biases,- num_heads * head_dim,- [model_dim],- mtl_context,- &decoder_layer_loader- .subtree("mixer.out_projection")- .unwrap(),- ArrayId::AttentionOutput,- ArrayId::Main,- &compilation_config.descriptor_mlp,- );-- let attention = Box::new(- AttentionKernelEncodable::new(- mtl_context,- kernel_data_type,- layer_index,- attention_scale,- attention_config.has_sinks,- )- .expect(- "Failed to create AttentionWrapper with Metal kernel",- ),- );-- MixerExecutables::Attention {- qkv_projection,- qk_norm,- rope: rope_block,- attention,- out_projection,- }- },- MixerConfig::Mamba(mamba_config) => {- let mixer: Box<dyn EncodableWithState> =- Box::new(MambaMixerEncodable::new(- mtl_context,- layer_type.clone(),- mamba_config.clone(),- compilation_config.clone(),- layer_index,- model_dim,- num_heads,- head_dim,- num_groups,- decoder_layer_loader,- ));- MixerExecutables::StateSpace {- mixer,- }- },- };-- let post_attention_norm: Option<Box<dyn EncodableWithState>> =- if let Some(norm_config) =- &layer_config.post_attention_norm_config- {- Some(Box::new(- RMSNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader- .subtree("post_mixer_norm")- .unwrap(),- )- .expect("Failed to create RMS norm kernel"),- ))- } else {- None- };-- let main_shortcut_add_swap: Box<dyn EncodableWithState> = Box::new(- TensorAddSwap::new(- mtl_context,- kernel_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),- )- .unwrap(),- );-- let pre_mlp_norm: Box<dyn EncodableWithState> = Box::new(- RMSNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- layer_config.pre_mlp_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),- );-- let mlp = transformer_layer::mlp_block(- &layer_config.mlp_config,- model_dim,- hidden_dim,- mtl_context,- &decoder_layer_loader.subtree("mlp").unwrap(),- &compilation_config.descriptor_mlp,- );-- let post_mlp_norm: Option<Box<dyn EncodableWithState>> =- if let Some(norm_config) = &layer_config.post_mlp_norm_config {- Some(Box::new(- RMSNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader- .subtree("post_mlp_norm")- .unwrap(),- )- .expect("Failed to create RMS norm kernel"),- ))- } else {- None- };-- Self {- layer_index,- copy_main_to_shortcut,- pre_attention_norm,- mixer,- post_attention_norm,- main_shortcut_add_swap,- pre_mlp_norm,- mlp,- post_mlp_norm,- }- })- }-}--impl LayerExecutables {- fn encode_with_shared_encoder_impl(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,- ) {- let layer_traces = if let Some(traces) = state.traces.clone() {- traces.borrow().layer_results.get(self.layer_index).cloned()- } else {- None- };-- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().inputs.clone(),- );- }-- self.copy_main_to_shortcut- .encode_with_shared_encoder(state, encoder, parameters);-- self.pre_attention_norm- .encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().pre_attention_norm.clone(),- );- }-- match &self.mixer {- MixerExecutables::Attention {- qkv_projection,- qk_norm,- rope,- attention,- out_projection,- } => {- qkv_projection- .encode_with_shared_encoder(state, encoder, parameters);- if let Some(norm) = qk_norm {- norm.encode_with_shared_encoder(state, encoder, parameters);- }- rope.encode_with_shared_encoder(state, encoder, parameters);- attention- .encode_with_shared_encoder(state, encoder, parameters);- out_projection- .encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().attention.clone(),- );- }- },- MixerExecutables::StateSpace {- mixer,- } => {- mixer.encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().attention.clone(),- );- }- },- }-- if let Some(post_attention_norm) = &self.post_attention_norm {- post_attention_norm- .encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().post_attention_norm.clone(),- );- }- }-- self.main_shortcut_add_swap- .encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().mlp_inputs.clone(),- );- }-- self.pre_mlp_norm- .encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().pre_mlp_norm.clone(),- );- }-- self.mlp.encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(ArrayId::Main, layer_traces.borrow().mlp.clone());- }-- if let Some(post_mlp_norm) = &self.post_mlp_norm {- post_mlp_norm- .encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().post_mlp_norm.clone(),- );- }- }-- self.main_shortcut_add_swap- .encode_with_shared_encoder(state, encoder, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().outputs.clone(),- );- }- }-- fn all_blocks_support_shared_encoder(&self) -> bool {- let mixer_supports = match &self.mixer {- MixerExecutables::Attention {- qkv_projection,- qk_norm,- rope,- attention,- out_projection,- } => {- qkv_projection.supports_shared_encoder()- && qk_norm- .as_ref()- .map_or(true, |n| n.supports_shared_encoder())- && rope.supports_shared_encoder()- && attention.supports_shared_encoder()- && out_projection.supports_shared_encoder()- },- MixerExecutables::StateSpace {- mixer,- } => mixer.supports_shared_encoder(),- };-- self.copy_main_to_shortcut.supports_shared_encoder()- && self.pre_attention_norm.supports_shared_encoder()- && mixer_supports- && self- .post_attention_norm- .as_ref()- .map_or(true, |n| n.supports_shared_encoder())- && self.main_shortcut_add_swap.supports_shared_encoder()- && self.pre_mlp_norm.supports_shared_encoder()- && self.mlp.supports_shared_encoder()- && self- .post_mlp_norm- .as_ref()- .map_or(true, |n| n.supports_shared_encoder())- }-- fn encode_with_resolver(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let layer_traces = if let Some(traces) = state.traces.clone() {- traces.borrow().layer_results.get(self.layer_index).cloned()- } else {- None- };-- let mut resolver = EncoderResolver::new(command_buffer);-- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().inputs.clone(),- );- }-- resolver.encode(self.copy_main_to_shortcut.as_ref(), state, parameters);-- resolver.encode(self.pre_attention_norm.as_ref(), state, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().pre_attention_norm.clone(),- );- }-- match &self.mixer {- MixerExecutables::Attention {- qkv_projection,- qk_norm,- rope,- attention,- out_projection,- } => {- resolver.encode(qkv_projection.as_ref(), state, parameters);- if let Some(norm) = qk_norm {- resolver.encode(norm.as_ref(), state, parameters);- }- resolver.encode(rope.as_ref().as_ref(), state, parameters);- resolver.encode(attention.as_ref(), state, parameters);- resolver.encode(out_projection.as_ref(), state, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().attention.clone(),- );- }- },- MixerExecutables::StateSpace {- mixer,- } => {- resolver.encode(mixer.as_ref(), state, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().attention.clone(),- );- }- },- }-- if let Some(post_attention_norm) = &self.post_attention_norm {- resolver.encode(post_attention_norm.as_ref(), state, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().post_attention_norm.clone(),- );- }- }-- resolver.encode(- self.main_shortcut_add_swap.as_ref(),- state,- parameters,- );- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().mlp_inputs.clone(),- );- }-- resolver.encode(self.pre_mlp_norm.as_ref(), state, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().pre_mlp_norm.clone(),- );- }-- resolver.encode(self.mlp.as_ref(), state, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(ArrayId::Main, layer_traces.borrow().mlp.clone());- }-- if let Some(post_mlp_norm) = &self.post_mlp_norm {- resolver.encode(post_mlp_norm.as_ref(), state, parameters);- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().post_mlp_norm.clone(),- );- }- }-- resolver.encode(- self.main_shortcut_add_swap.as_ref(),- state,- parameters,- );- if let Some(layer_traces) = layer_traces.clone() {- state.copy_array(- ArrayId::Main,- layer_traces.borrow().outputs.clone(),- );- }-- resolver.finish();- }-}--impl EncodableWithState for LayerExecutables {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- self.encode_with_resolver(state, command_buffer, parameters);-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- self.all_blocks_support_shared_encoder()- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,- ) {- self.encode_with_shared_encoder_impl(state, encoder, parameters);- }-}diff --git a/crates/uzu/src/backends/metal/executable_builder/mod.rs b/crates/uzu/src/backends/metal/executable_builder/mod.rsdeleted file mode 100644index 84cfd0fbc..000000000--- a/crates/uzu/src/backends/metal/executable_builder/mod.rs+++ /dev/null@@ -1,4 +0,0 @@-pub mod decoder_executables;-pub mod layer_executables;--pub use decoder_executables::DecoderExecutables;diff --git a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rsindex 2fbe157cc..d9724e1a4 100644--- a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs+++ b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs@@ -1,9 +1,9 @@use metal::ComputeCommandEncoder;use mpsgraph::CommandBuffer as MPSCommandBuffer;-use super::{- ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},+use super::ForwardPassState;+use crate::backends::metal::encodable_block::{+ EncodableBlock, EncodingParameters,};pub struct EncoderResolver<'a> {@@ -21,7 +21,7 @@ impl<'a> EncoderResolver<'a> {pub fn encode(&mut self,- block: &dyn EncodableWithState,+ block: &dyn EncodableBlock,state: &mut ForwardPassState,parameters: &EncodingParameters,) {diff --git a/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs b/crates/uzu/src/backends/metal/forward_pass/io_arrays.rsindex bbc755c22..e8e7322e9 100644--- a/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs+++ b/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs@@ -3,7 +3,7 @@ use std::{cell::RefCell, collections::HashMap};use mpsgraph::TensorData;use objc2::rc::Retained;-use super::state::{ArrayId, ForwardPassState};+use super::{ArrayId, ForwardPassState};use crate::backends::metal::array::MetalArray;#[derive(Clone)]diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rsindex 26c569cc0..689986567 100644--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs@@ -1,24 +1,28 @@-mod buffers;pub mod cache_layers;-pub mod encodable_with_state;mod encoder_resolver;mod io_arrays;pub mod kv_cache_layer;mod model_shape;mod mpsgraph_block;+mod scratch_buffers;pub mod ssm_layer;-mod state;+pub mod state;+#[cfg(feature = "tracing")]pub mod traces;-pub mod transformer_layer;-pub use buffers::ForwardPassBuffers;pub use cache_layers::{CacheLayer, CacheLayers};pub use encoder_resolver::EncoderResolver;pub use io_arrays::IOArrays;pub use kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState};pub use model_shape::ModelShape;pub use mpsgraph_block::MPSGraphBlock;+pub use scratch_buffers::ScratchBuffers;pub use ssm_layer::SSMLayer;pub use state::{- ArrayId, ForwardPassState, HashMapId, RopeType, SharedBuffers,+ ArrayCell, ArrayId, ClassifierModeState, CommonAuxBuffers,+ EmbeddingsBuffers, ForwardPassMode, ForwardPassState, HashMapId,+ LanguageModelGeneratorAuxBuffers, LanguageModelGeneratorModeState,+ MoeExpertWeights, RopeBuffers, RopeType, SharedBuffers,};++pub use super::encodable_block::{EncodableBlock, EncodingParameters};diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rsindex e8c7f4685..4d5ddf4f5 100644--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs@@ -142,6 +142,14 @@ impl ModelShape {[suffix_length, self.model_dim]}+ pub fn bitmask_shape(+ &self,+ suffix_length: usize,+ ) -> [usize; 2] {+ let bitmask_size = (self.vocabulary_size + 31) / 32;+ [suffix_length, bitmask_size]+ }+pub fn mlp_hidden_shape(&self,suffix_length: usize,@@ -170,13 +178,6 @@ impl ModelShape {[suffix_length, self.vocabulary_size]}- pub fn bitmask_shape(- &self,- suffix_length: usize,- ) -> [usize; 2] {- [suffix_length, self.vocabulary_size.div_ceil(32)]- }-pub fn embeddings_input_shape(&self) -> [usize; 2] {[self.vocabulary_size, self.model_dim]}@@ -214,6 +215,14 @@ impl ModelShape {[self.num_groups, suffix_length, self.head_dim]}+ pub fn extracted_values_shape(+ &self,+ suffix_length: usize,+ ) -> [usize; 3] {+ // Values share the same grouping as keys (grouped by num_groups)+ [self.num_groups, suffix_length, self.head_dim]+ }+pub fn kv_cache_layer_shapes(&self,max_prefix_length: usize,diff --git a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rsindex 52fcfef7d..b9fd49008 100644--- a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs+++ b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs@@ -5,10 +5,9 @@ use mpsgraph::{use objc2::rc::Retained;use super::{- encodable_with_state::EncodableWithState, io_arrays::IOArrays,- state::ForwardPassState,+ EncodableBlock, EncodingParameters, ForwardPassState,+ io_arrays::IOArrays,};-use crate::backends::metal::forward_pass::encodable_with_state::EncodingParameters;pub struct MPSGraphBlock {executable: Retained<Executable>,@@ -28,37 +27,9 @@ impl MPSGraphBlock {arguments,}}-- // fn load(path: &Path) -> Result<Self, ExecutableBlockError> {- // let signature =- // from_reader(fs::File::open(path.join("signature.json"))?)?;- // let executable =- // Executable::from_serialized_package(path, None).unwrap();- // let execution_descriptor = ExecutableExecutionDescriptor::new();-- // Ok(Self {- // signature,- // executable,- // execution_descriptor,- // })- // }-- // fn save(- // &self,- // path: &Path,- // ) -> Result<(), ExecutableBlockError> {- // fs::create_dir(path)?;-- // self.executable.serialize_to_url(path, &SerializationDescriptor::new());-- // let writer = fs::File::create(path.join("signature.json"))?;- // to_writer(writer, &self.signature)?;-- // Ok(())- // }}-impl EncodableWithState for MPSGraphBlock {+impl EncodableBlock for MPSGraphBlock {fn encode(&self,state: &mut ForwardPassState,@@ -103,8 +74,4 @@ impl EncodableWithState for MPSGraphBlock {}}}-- fn supports_shared_encoder(&self) -> bool {- false // MPSGraph uses its own encoding mechanism, not compute encoders- }}diff --git a/crates/uzu/src/backends/metal/forward_pass/buffers.rs b/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rssimilarity index 91%rename from crates/uzu/src/backends/metal/forward_pass/buffers.rsrename to crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rsindex f49d93795..a74e214cc 100644--- a/crates/uzu/src/backends/metal/forward_pass/buffers.rs+++ b/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs@@ -10,7 +10,7 @@ use crate::{};#[derive(Debug)]-pub struct ForwardPassBuffers {+pub struct ScratchBuffers {// 1-Dpub token_ids: MTLBuffer,pub token_positions: MTLBuffer,@@ -39,6 +39,7 @@ pub struct ForwardPassBuffers {// 3-Dpub rotated_queries: MTLBuffer,pub rotated_keys: MTLBuffer,+ pub extracted_values: MTLBuffer,// 2-pass attention intermediate bufferspub attention_partials: MTLBuffer, // [num_heads * max_suffix_len * total_blocks_count * head_dim]@@ -66,7 +67,7 @@ pub struct ForwardPassBuffers {pub moe_block_alloc: Option<MTLBuffer>,}-impl ForwardPassBuffers {+impl ScratchBuffers {// TODO: use device arrays instead of MTLBuffers/// Allocate the buffers with `StorageModeShared` so that they are CPU-accessible as well.pub fn new(@@ -103,18 +104,33 @@ impl ForwardPassBuffers {sampling_output: alloc(&[max_suffix_len], DataType::U32),// 2-D- attention_window_size_to_bias: (0..model_shape.num_layers)- .map(|layer| {- (- model_shape.sliding_window_length_per_layer- [layer as usize],- alloc(- &[max_suffix_len, max_suffix_len + max_prefix_len],- act_ty,- ),- )- })- .collect::<HashMap<Option<usize>, MTLBuffer>>(),+ attention_window_size_to_bias: {+ // Collect unique window sizes across all layers+ let unique_window_sizes: std::collections::HashSet<+ Option<usize>,+ > = model_shape+ .sliding_window_length_per_layer+ .iter()+ .copied()+ .collect();++ // Create one buffer per unique window size+ unique_window_sizes+ .into_iter()+ .map(|window_size| {+ (+ window_size,+ alloc(+ &[+ max_suffix_len,+ max_suffix_len + max_prefix_len,+ ],+ act_ty,+ ),+ )+ })+ .collect::<HashMap<Option<usize>, MTLBuffer>>()+ },logits: alloc(&model_shape.logits_shape(max_suffix_len), act_ty),main: alloc(&model_shape.main_shape(max_suffix_len), act_ty),shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty),@@ -164,6 +180,10 @@ impl ForwardPassBuffers {&model_shape.rotated_keys_shape(max_suffix_len),act_ty,),+ extracted_values: alloc(+ &model_shape.extracted_values_shape(max_suffix_len),+ act_ty,+ ),attention_partials: alloc(&partials_shape, act_ty),attention_sums: alloc(&sums_maxs_shape, act_ty),diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rsdeleted file mode 100644index ead7c014b..000000000--- a/crates/uzu/src/backends/metal/forward_pass/state.rs+++ /dev/null@@ -1,1560 +0,0 @@-#![allow(dead_code)]-use std::{cell::RefCell, collections::HashMap, rc::Rc};--use half::{bf16, f16};-use serde::{Deserialize, Serialize};--use super::{- super::{MTLContext, MetalArray},- buffers::ForwardPassBuffers,- cache_layers::CacheLayers,- model_shape::ModelShape,-};-use crate::{- DataType, DeviceContext,- backends::metal::forward_pass::traces::DecoderActivationTrace,- config::{DecoderConfig, EmbeddingConfig, MLPConfig},- device::array::Array,- parameters::ParameterTree,- session::parameter::SamplingMethod,-};--type ArrayCell = RefCell<MetalArray>;--pub enum EmbeddingsBuffers {- Tied {- /// [vocab_size, model_dim]- weights: ArrayCell,- },- Untied {- /// [vocab_size, model_dim]- input_weights: ArrayCell,- /// [vocab_size, model_dim]- output_weights: ArrayCell,- },- QuantizedTied {- /// [vocab_size, model_dim]- weights: ArrayCell,- /// [vocab_size]- scales: ArrayCell,- },- MLXSemiQuantizedUntied {- /// [vocab_size, model_dim]- input_weights: ArrayCell,- /// [vocab_size, model_dim]- packed_output_weights: ArrayCell,- /// [vocab_size, num_groups]- output_scales: ArrayCell,- /// [vocab_size, num_groups]- output_biases: ArrayCell,- },-}--impl EmbeddingsBuffers {- pub fn new(- context: &MTLContext,- embeddings_config: &EmbeddingConfig,- model_shape: &ModelShape,- ) -> Self {- unsafe {- match embeddings_config {- EmbeddingConfig::Tied {- common: _,- precision: _,- } => Self::Tied {- weights: RefCell::new(context.array_uninitialized(- &model_shape.embeddings_input_shape(),- model_shape.activation_data_type(),- )),- },- EmbeddingConfig::Untied {- common: _,- precision: _,- } => Self::Untied {- input_weights: RefCell::new(context.array_uninitialized(- &model_shape.embeddings_input_shape(),- model_shape.activation_data_type(),- )),- output_weights: RefCell::new(context.array_uninitialized(- &model_shape.embeddings_output_shape(),- model_shape.activation_data_type(),- )),- },- EmbeddingConfig::QuantizedTied {- embedding_quantization_mode,- ..- } => {- let [vocab_size, model_dim] =- model_shape.quantized_embeddings_weights_shape();- Self::QuantizedTied {- weights: RefCell::new(- context.array_uninitialized(- &[- vocab_size,- model_dim- / embedding_quantization_mode- .packing_divisor(),- ],- embedding_quantization_mode.storage_type(),- ),- ),- scales: RefCell::new(context.array_uninitialized(- &model_shape.quantized_embeddings_scales_shape(),- model_shape.activation_data_type(),- )),- }- },- EmbeddingConfig::MLXQuantizedTied {- group_size,- embedding_quantization_mode,- ..- } => {- let [vocab_size, model_dim] =- model_shape.quantized_embeddings_weights_shape();- let num_groups = model_dim / group_size;- Self::QuantizedTied {- weights: RefCell::new(- context.array_uninitialized(- &[- vocab_size,- model_dim- / embedding_quantization_mode- .packing_divisor(),- ],- embedding_quantization_mode.storage_type(),- ),- ),- scales: RefCell::new(context.array_uninitialized(- &[vocab_size, num_groups],- model_shape.activation_data_type(),- )),- }- },- EmbeddingConfig::MLXSemiQuantizedUntied {- group_size,- embedding_quantization_mode,- ..- } => {- let [vocab_size, model_dim] =- model_shape.quantized_embeddings_weights_shape();- let num_groups = model_dim / group_size;- Self::MLXSemiQuantizedUntied {- input_weights: RefCell::new(- context.array_uninitialized(- &model_shape.embeddings_input_shape(),- model_shape.activation_data_type(),- ),- ),- packed_output_weights: RefCell::new(- context.array_uninitialized(- &[- vocab_size,- model_dim- / embedding_quantization_mode- .packing_divisor(),- ],- embedding_quantization_mode.storage_type(),- ),- ),- output_scales: RefCell::new(- context.array_uninitialized(- &[vocab_size, num_groups],- model_shape.activation_data_type(),- ),- ),- output_biases: RefCell::new(- context.array_uninitialized(- &[vocab_size, num_groups],- model_shape.activation_data_type(),- ),- ),- }- },- }- }- }-- pub fn update_data(- &mut self,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) {- let embeddings_tree = parameter_tree.subtree("embedding").unwrap();- match self {- EmbeddingsBuffers::Tied {- weights,- } => {- let embeddings_view = embeddings_tree.leaf("weights").unwrap();- weights.borrow_mut().copy_from_array(&embeddings_view);- },- EmbeddingsBuffers::Untied {- input_weights,- output_weights,- } => {- let mapping = vec![- ("input_weights", input_weights),- ("output_weights", output_weights),- ];- for (name, buffer) in mapping {- let view = embeddings_tree.leaf(name).unwrap();- buffer.borrow_mut().copy_from_array(&view);- }- },- EmbeddingsBuffers::QuantizedTied {- weights,- scales,- } => {- let mapping = vec![("weights", weights), ("scales", scales)];- for (name, buffer) in mapping {- let view = embeddings_tree.leaf(name).unwrap();- buffer.borrow_mut().copy_from_array(&view);- }- },- EmbeddingsBuffers::MLXSemiQuantizedUntied {- input_weights,- packed_output_weights,- output_scales,- output_biases,- } => {- let mapping = vec![- ("input_weights", input_weights),- ("output_weights", packed_output_weights),- ("output_scales", output_scales),- ("output_biases", output_biases),- ];- for (name, buffer) in mapping {- let view = embeddings_tree.leaf(name).unwrap();- buffer.borrow_mut().copy_from_array(&view);- }- },- }- }-}--pub struct RopeBuffers {- /// [rope_max_sequence_length, head_dim]- pub cosines: ArrayCell,- /// [rope_max_sequence_length, head_dim]- pub sines: ArrayCell,-}--impl RopeBuffers {- pub fn new(- context: &MTLContext,- model_shape: &ModelShape,- ) -> Self {- unsafe {- let rotated_queries_shape = model_shape.rotated_queries_shape(1);- let head_dim = rotated_queries_shape[2];- let rope_max_sequence_length = model_shape.context_length();-- Self {- cosines: RefCell::new(context.array_uninitialized(- &[rope_max_sequence_length, head_dim],- model_shape.activation_data_type(),- )),- sines: RefCell::new(context.array_uninitialized(- &[rope_max_sequence_length, head_dim],- model_shape.activation_data_type(),- )),- }- }- }-- pub fn update_data(- &mut self,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- rope_name: String,- ) {- let Ok(rope_tree) = parameter_tree.subtree(rope_name.as_str()) else {- return;- };-- let cosines_view = rope_tree.leaf("cosines").unwrap();- self.cosines.borrow_mut().copy_from_array(&cosines_view);-- let sines_view = rope_tree.leaf("sines").unwrap();- self.sines.borrow_mut().copy_from_array(&sines_view);- }-}--pub struct SharedBuffers {- pub embeddings: EmbeddingsBuffers,- pub global_rope: Option<RopeBuffers>,- pub local_rope: Option<RopeBuffers>,- pub moe_expert_weights: Option<Vec<MoeExpertWeights>>,- pub attention_sinks: Option<Vec<ArrayCell>>,-}--pub struct MoeExpertWeights {- pub w1: ArrayCell,- pub w2: ArrayCell,- pub w3: ArrayCell,-}--impl SharedBuffers {- pub fn new(- context: &MTLContext,- decoder_config: &DecoderConfig,- model_shape: &ModelShape,- ) -> Self {- let embeddings = EmbeddingsBuffers::new(- context,- &decoder_config.embedding_config,- model_shape,- );-- let global_rope = if decoder_config.global_rope_config.is_some() {- Some(RopeBuffers::new(context, model_shape))- } else {- None- };-- let local_rope = if decoder_config.local_rope_config.is_some() {- Some(RopeBuffers::new(context, model_shape))- } else {- None- };-- let moe_expert_weights = if matches!(- decoder_config.layer_config.mlp_config,- crate::config::MLPConfig::MixtureOfExperts(_)- ) {- Some(Vec::new())- } else {- None- };-- let attention_sinks = if let Some(attention_config) =- decoder_config.layer_config.attention_config()- {- if attention_config.has_sinks {- let num_heads = decoder_config.num_heads;- Some(- (0..decoder_config.num_layers)- .map(|_| unsafe {- RefCell::new(context.array_uninitialized(- &[num_heads],- DataType::F32,- ))- })- .collect(),- )- } else {- None- }- } else {- None- };-- Self {- embeddings,- global_rope,- local_rope,- moe_expert_weights,- attention_sinks,- }- }-- pub fn update_data(- &mut self,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) {- self.embeddings.update_data(parameter_tree);- if let Some(global_rope) = &mut self.global_rope {- global_rope- .update_data(parameter_tree, String::from("global_rope"));- }- if let Some(local_rope) = &mut self.local_rope {- local_rope.update_data(parameter_tree, String::from("local_rope"));- }-- if let Some(sinks_vec) = &mut self.attention_sinks {- for (layer_idx, sink_cell) in sinks_vec.iter_mut().enumerate() {- let layer_tree = parameter_tree- .subtree(&format!("layers.{}", layer_idx))- .unwrap();- let attn_tree = layer_tree.subtree("mixer").unwrap();- let sinks_arr = attn_tree.leaf("sinks").unwrap();- let mut dst = sink_cell.borrow_mut();- let dst_slice = dst.as_slice_mut::<f32>().unwrap();-- match sinks_arr.data_type() {- DataType::F32 => {- let src = sinks_arr.as_slice::<f32>().unwrap();- dst_slice.copy_from_slice(src);- },- DataType::BF16 => {- let src = sinks_arr.as_slice::<bf16>().unwrap();- for (dst_val, src_val) in- dst_slice.iter_mut().zip(src.iter())- {- *dst_val = f32::from(*src_val);- }- },- DataType::F16 => {- let src = sinks_arr.as_slice::<f16>().unwrap();- for (dst_val, src_val) in- dst_slice.iter_mut().zip(src.iter())- {- *dst_val = f32::from(*src_val);- }- },- other => {- panic!(- "Unsupported attention sink data type: {:?}",- other- );- },- }- }- }- }-}--struct AuxBuffers {- suffix_length: usize,- /// [suffix_length, model_dim]- main: ArrayCell,- /// [suffix_length, model_dim]- shortcut: ArrayCell,- /// [suffix_length, (num_heads + 2 * num_groups) * head_dim]- qkv: ArrayCell,- /// [suffix_length, num_heads * head_dim]- attention_output: ArrayCell,- /// [suffix_length, 2 * hidden_dim]- mlp_fused_up: ArrayCell,- /// [suffix_length, hidden_dim]- mlp_hidden: ArrayCell,- /// [suffix_length, max_mamba_inproj_dim]- ssm_inproj: Option<ArrayCell>,- /// [suffix_length, max_conv_dim]- ssm_packed: Option<ArrayCell>,- /// [suffix_length + kernel_size - 1, max_conv_dim]- ssm_conv_padded: Option<ArrayCell>,- /// [suffix_length, num_heads, head_dim]- ssm_x: Option<ArrayCell>,- /// [suffix_length, num_groups, state_dim]- ssm_b: Option<ArrayCell>,- /// [suffix_length, num_groups, state_dim]- ssm_c: Option<ArrayCell>,- /// [suffix_length, num_heads]- ssm_dt: Option<ArrayCell>,- /// [suffix_length, num_heads, head_dim]- ssm_z: Option<ArrayCell>,- /// [num_heads, max_suffix_length, head_dim]- rotated_queries: ArrayCell,- /// [num_groups, max_suffix_length, head_dim]- rotated_keys: ArrayCell,- /// [num_heads * suffix_length * total_blocks_count * head_dim]- attention_partials: ArrayCell,- /// [num_heads * suffix_length * total_blocks_count]- attention_sums: ArrayCell,- /// [num_heads * suffix_length * total_blocks_count]- attention_maxs: ArrayCell,-- moe_topk_ids: Option<ArrayCell>,- moe_topk_probs: Option<ArrayCell>,- moe_offsets: Option<ArrayCell>,- moe_sumk: Option<ArrayCell>,- moe_bucketed_token_ids: Option<ArrayCell>,- moe_bucketed_probs: Option<ArrayCell>,- moe_x_perm: Option<ArrayCell>,- moe_tok2row: Option<ArrayCell>,- moe_y_partial: Option<ArrayCell>,- moe_hidden: Option<ArrayCell>,- moe_two_pass_row_expert_map: Option<ArrayCell>,- moe_tile_counts: Option<ArrayCell>,- moe_tile_offsets: Option<ArrayCell>,- moe_tile_map: Option<ArrayCell>,- moe_total_tiles: Option<ArrayCell>,- moe_dispatch_args: Option<ArrayCell>,- moe_scatter_partials: Option<ArrayCell>,- moe_scatter_block_bases: Option<ArrayCell>,- moe_block_alloc: Option<ArrayCell>,-}--impl AuxBuffers {- fn new(- scratch: &ForwardPassBuffers,- decoder_config: &DecoderConfig,- model_shape: &ModelShape,- suffix_length: usize,- ) -> Self {- let act_dtype = model_shape.activation_data_type();- unsafe {- Self {- suffix_length,- main: RefCell::new(MetalArray::new(- scratch.main.clone(),- &model_shape.main_shape(suffix_length),- act_dtype,- )),- shortcut: RefCell::new(MetalArray::new(- scratch.shortcut.clone(),- &model_shape.main_shape(suffix_length),- act_dtype,- )),- qkv: RefCell::new(MetalArray::new(- scratch.qkv.clone(),- &model_shape.qkv_shape(suffix_length),- act_dtype,- )),- attention_output: RefCell::new(MetalArray::new(- scratch.attention_output.clone(),- &model_shape.attention_output_shape(suffix_length),- act_dtype,- )),- mlp_fused_up: RefCell::new(MetalArray::new(- scratch.mlp_fused_up.clone(),- &model_shape.mlp_fused_up_shape(suffix_length),- act_dtype,- )),- mlp_hidden: RefCell::new(MetalArray::new(- scratch.mlp_hidden.clone(),- &model_shape.mlp_hidden_shape(suffix_length),- act_dtype,- )),- ssm_inproj: match (- scratch.ssm_inproj.as_ref(),- model_shape.ssm_inproj_shape(suffix_length),- ) {- (Some(buf), Some(shape)) => Some(RefCell::new(- MetalArray::new(buf.clone(), &shape, act_dtype),- )),- _ => None,- },- ssm_packed: match (- scratch.ssm_packed.as_ref(),- model_shape.ssm_packed_shape(suffix_length),- ) {- (Some(buf), Some(shape)) => Some(RefCell::new(- MetalArray::new(buf.clone(), &shape, act_dtype),- )),- _ => None,- },- ssm_conv_padded: match (- scratch.ssm_conv_padded.as_ref(),- model_shape.ssm_conv_padded_shape(suffix_length),- ) {- (Some(buf), Some(shape)) => Some(RefCell::new(- MetalArray::new(buf.clone(), &shape, act_dtype),- )),- _ => None,- },- ssm_x: match (- scratch.ssm_x.as_ref(),- model_shape.ssm_x_shape(suffix_length),- ) {- (Some(buf), Some(shape)) => Some(RefCell::new(- MetalArray::new(buf.clone(), &shape, act_dtype),- )),- _ => None,- },- ssm_b: match (- scratch.ssm_b.as_ref(),- model_shape.ssm_bc_shape(suffix_length),- ) {- (Some(buf), Some(shape)) => Some(RefCell::new(- MetalArray::new(buf.clone(), &shape, act_dtype),- )),- _ => None,- },- ssm_c: match (- scratch.ssm_c.as_ref(),- model_shape.ssm_bc_shape(suffix_length),- ) {- (Some(buf), Some(shape)) => Some(RefCell::new(- MetalArray::new(buf.clone(), &shape, act_dtype),- )),- _ => None,- },- ssm_dt: match (- scratch.ssm_dt.as_ref(),- model_shape.ssm_dt_shape(suffix_length),- ) {- (Some(buf), Some(shape)) => Some(RefCell::new(- MetalArray::new(buf.clone(), &shape, act_dtype),- )),- _ => None,- },- ssm_z: match (- scratch.ssm_z.as_ref(),- model_shape.ssm_z_shape(suffix_length),- ) {- (Some(buf), Some(shape)) => Some(RefCell::new(- MetalArray::new(buf.clone(), &shape, act_dtype),- )),- _ => None,- },- rotated_queries: RefCell::new(MetalArray::new(- scratch.rotated_queries.clone(),- &model_shape.rotated_queries_shape(suffix_length),- act_dtype,- )),- rotated_keys: RefCell::new(MetalArray::new(- scratch.rotated_keys.clone(),- &model_shape.rotated_keys_shape(suffix_length),- act_dtype,- )),- attention_partials: RefCell::new(MetalArray::new(- scratch.attention_partials.clone(),- &model_shape.attention_partials_shape(suffix_length),- act_dtype,- )),- attention_sums: RefCell::new(MetalArray::new(- scratch.attention_sums.clone(),- &model_shape.attention_sums_shape(suffix_length),- act_dtype,- )),- attention_maxs: RefCell::new(MetalArray::new(- scratch.attention_maxs.clone(),- &model_shape.attention_maxs_shape(suffix_length),- act_dtype,- )),-- moe_topk_ids: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- scratch.moe_topk_ids.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_topk_ids_shape(- suffix_length,- moe.num_experts_per_token,- ),- DataType::U32,- ))- })- },- _ => None,- },- moe_topk_probs: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- scratch.moe_topk_probs.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_topk_probs_shape(- suffix_length,- moe.num_experts_per_token,- ),- act_dtype,- ))- })- },- _ => None,- },- moe_offsets: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- scratch.moe_offsets.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape- .moe_offsets_shape(moe.mixture_size),- DataType::U32,- ))- })- },- _ => None,- },- moe_sumk: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(_) => {- scratch.moe_sumk.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_sumk_shape(),- DataType::U32,- ))- })- },- _ => None,- },- moe_bucketed_token_ids: match &decoder_config- .layer_config- .mlp_config- {- MLPConfig::MixtureOfExperts(moe) => {- let max_routed =- suffix_length * moe.num_experts_per_token;- scratch.moe_bucketed_token_ids.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape- .moe_bucketed_token_ids_shape(max_routed),- DataType::U32,- ))- })- },- _ => None,- },- moe_bucketed_probs: match &decoder_config- .layer_config- .mlp_config- {- MLPConfig::MixtureOfExperts(moe) => {- let max_routed =- suffix_length * moe.num_experts_per_token;- scratch.moe_bucketed_probs.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape- .moe_bucketed_probs_shape(max_routed),- act_dtype,- ))- })- },- _ => None,- },- moe_x_perm: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- let max_routed =- suffix_length * moe.num_experts_per_token;- scratch.moe_x_perm.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_x_perm_shape(max_routed),- DataType::F16,- ))- })- },- _ => None,- },- moe_tok2row: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- scratch.moe_tok2row.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_tok2row_shape(- suffix_length,- moe.num_experts_per_token,- ),- DataType::I32,- ))- })- },- _ => None,- },- moe_y_partial: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- let max_routed =- suffix_length * moe.num_experts_per_token;- scratch.moe_y_partial.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_y_partial_shape(max_routed),- DataType::F16,- ))- })- },- _ => None,- },- moe_hidden: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- let max_routed =- suffix_length * moe.num_experts_per_token;- scratch.moe_hidden.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_hidden_shape(max_routed),- act_dtype,- ))- })- },- _ => None,- },- moe_two_pass_row_expert_map: match &decoder_config- .layer_config- .mlp_config- {- MLPConfig::MixtureOfExperts(moe) => {- let max_routed =- suffix_length * moe.num_experts_per_token;- scratch.moe_two_pass_row_expert_map.as_ref().map(- |buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &[max_routed],- DataType::U32,- ))- },- )- },- _ => None,- },- moe_tile_counts: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- scratch.moe_tile_counts.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_counts_shape(moe.mixture_size),- DataType::U32,- ))- })- },- _ => None,- },- moe_tile_offsets: match &decoder_config.layer_config.mlp_config- {- MLPConfig::MixtureOfExperts(moe) => {- scratch.moe_tile_offsets.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape- .moe_offsets_shape(moe.mixture_size),- DataType::U32,- ))- })- },- _ => None,- },- moe_tile_map: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- let max_routed =- suffix_length * moe.num_experts_per_token;- scratch.moe_tile_map.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_tile_map_shape(max_routed),- DataType::U32,- ))- })- },- _ => None,- },- moe_total_tiles: match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(_) => {- scratch.moe_total_tiles.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_total_tiles_shape(),- DataType::U32,- ))- })- },- _ => None,- },- moe_dispatch_args: match &decoder_config.layer_config.mlp_config- {- MLPConfig::MixtureOfExperts(_) => {- scratch.moe_dispatch_args.as_ref().map(|buf| {- RefCell::new(MetalArray::new(- buf.clone(),- &model_shape.moe_dispatch_args_shape(),- DataType::U32,- ))- })- },- _ => None,- },- moe_scatter_partials: scratch- .moe_scatter_partials- .as_ref()- .map(|buf| {- let num_blocks = ((suffix_length + 255) / 256).max(1);- match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- let num_tiles =- ((moe.mixture_size + 512 - 1) / 512).max(1);- let entries = num_blocks * num_tiles * 512;- RefCell::new(MetalArray::new(- buf.clone(),- &[entries],- DataType::U32,- ))- },- _ => unreachable!(),- }- }),- moe_scatter_block_bases: scratch- .moe_scatter_block_bases- .as_ref()- .map(|buf| {- let num_blocks = ((suffix_length + 255) / 256).max(1);- match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- let num_tiles =- ((moe.mixture_size + 512 - 1) / 512).max(1);- let entries = num_blocks * num_tiles * 512;- RefCell::new(MetalArray::new(- buf.clone(),- &[entries],- DataType::U32,- ))- },- _ => unreachable!(),- }- }),- moe_block_alloc: scratch.moe_block_alloc.as_ref().map(|buf| {- let num_blocks = ((suffix_length + 255) / 256).max(1);- match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => {- let num_tiles =- ((moe.mixture_size + 512 - 1) / 512).max(1);- let entries = num_blocks * num_tiles * 512;- RefCell::new(MetalArray::new(- buf.clone(),- &[entries],- DataType::U32,- ))- },- _ => unreachable!(),- }- }),- }- }- }-}--pub struct ForwardPassState {- context: Rc<MTLContext>,- /// [suffix_length] - u64- token_ids: ArrayCell,- /// [suffix_length] - i32- token_positions: ArrayCell,- /// [suffix_length, vocabulary_size] - bitmask packed in u32 (0 = disallowed, 1 = allowed)- token_bitmask: Option<ArrayCell>,- /// [suffix_length] - u64- token_seeds: ArrayCell,- /// [suffix_length, suffix_length + prefix_length]- attention_bias: HashMap<Option<usize>, ArrayCell>,- /// [suffix_length, vocabulary_size]- logits: ArrayCell,- pub cache_layers: Rc<RefCell<CacheLayers>>,- pub shared_buffers: Rc<RefCell<SharedBuffers>>,- aux_buffers: AuxBuffers,- /// [suffix_length] - u32 sampling output buffer- pub sampling_output: Option<ArrayCell>,- /// Current sampling configuration for this forward pass- pub sampling_method: Option<SamplingMethod>,- pub traces: Option<Rc<RefCell<DecoderActivationTrace>>>,- active_suffix_length: usize,- is_prefilling: bool,-}--impl ForwardPassState {- pub fn new(- context: Rc<MTLContext>,- decoder_config: &DecoderConfig,- model_shape: &ModelShape,- scratch: &ForwardPassBuffers,- cache_layers: Rc<RefCell<CacheLayers>>,- shared_buffers: Rc<RefCell<SharedBuffers>>,- token_ids: &[u64],- token_positions: &[usize],- token_bitmask: Option<&[u32]>,- token_seeds: &[u64],- active_suffix_length: usize,- is_prefilling: bool,- trace: bool,- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,- skip_token_ids_copy: bool,- async_positions: Option<(&metal::Buffer, usize)>,- async_seeds: Option<(&metal::Buffer, usize)>,- ) -> Self {- let suffix_length = token_ids.len();- assert_eq!(- suffix_length,- token_positions.len(),- "Tokens and token positions must have the same length, got {} and {} respectively",- suffix_length,- token_positions.len()- );- assert!(- suffix_length <= cache_layers.borrow().max_suffix_length(),- "KV cache size can only accomodate a suffix of length up to {}, but tried to use a suffix of length {}",- cache_layers.borrow().max_suffix_length(),- suffix_length- );- assert!(- active_suffix_length <= suffix_length,- "Active suffix length ({active_suffix_length}) must be <= suffix length ({suffix_length})"- );- let aux_buffers = AuxBuffers::new(- scratch,- decoder_config,- model_shape,- suffix_length,- );-- // --------------------- // Token IDs- // --------------------- let token_ids_refcell = {- let mut token_ids_array = unsafe {- MetalArray::new(- scratch.token_ids.clone(),- &[suffix_length],- DataType::U64,- )- };- if !skip_token_ids_copy {- context.copy_from_view(&mut token_ids_array, token_ids.into());- }- RefCell::new(token_ids_array)- };-- // --------------------- // Token Positions (i32)- // --------------------- let token_positions_refcell =- if let Some((buffer, offset)) = async_positions {- // Use pre-allocated async positions buffer at offset- let token_positions_array = unsafe {- MetalArray::new_with_offset(- buffer.clone(),- &[suffix_length],- DataType::I32,- offset * std::mem::size_of::<i32>(),- )- };- RefCell::new(token_positions_array)- } else {- // Copy positions from CPU- let mut token_positions_array = unsafe {- MetalArray::new(- scratch.token_positions.clone(),- &[suffix_length],- DataType::I32,- )- };- let token_positions_i32: Box<[i32]> =- token_positions.iter().map(|p| *p as i32).collect();- context.copy_from_view(- &mut token_positions_array,- token_positions_i32.as_ref().into(),- );- RefCell::new(token_positions_array)- };-- // --------------------- // Token Seeds- // --------------------- let token_seeds_refcell = if let Some((buffer, offset)) = async_seeds {- // Use pre-allocated async seeds buffer at offset- let token_seeds_array = unsafe {- MetalArray::new_with_offset(- buffer.clone(),- &[suffix_length],- DataType::U64,- offset * std::mem::size_of::<u64>(),- )- };- RefCell::new(token_seeds_array)- } else {- // Copy seeds from CPU- let mut token_seeds_array = unsafe {- MetalArray::new(- scratch.token_seeds.clone(),- &[suffix_length],- DataType::U64,- )- };- context.copy_from_view(&mut token_seeds_array, token_seeds.into());- RefCell::new(token_seeds_array)- };-- // --------------------- // Token Bitmask- // --------------------- let token_bitmask_refcell = if let Some(token_bitmask) = token_bitmask {- let mut token_bitmask_array = unsafe {- MetalArray::new(- scratch.token_bitmask.clone(),- &model_shape.bitmask_shape(suffix_length),- DataType::U32,- )- };- context- .copy_from_view(&mut token_bitmask_array, token_bitmask.into());- Some(RefCell::new(token_bitmask_array))- } else {- None- };-- // --------------------- // Attention Bias- // ---------------------- let act_dtype = model_shape.activation_data_type();-- let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =- scratch- .attention_window_size_to_bias- .iter()- .map(|(window_size, buffer)| {- let prefix_length = window_size- .unwrap_or(cache_layers.borrow().max_prefix_length());- let attention_bias_shape =- [suffix_length, suffix_length + prefix_length];-- let array = unsafe {- MetalArray::new(- buffer.clone(),- &attention_bias_shape,- act_dtype,- )- };- (*window_size, array)- })- .collect();-- cache_layers.borrow().fill_attention_bias(- &mut attention_bias_map,- token_positions,- suffix_length,- &context,- external_bias_fn,- );-- let attention_bias: HashMap<Option<usize>, ArrayCell> =- attention_bias_map- .into_iter()- .map(|(k, v)| (k, RefCell::new(v)))- .collect();-- // --------------------- // Logits- // --------------------- let logits = RefCell::new(unsafe {- MetalArray::new(- scratch.logits.clone(),- &model_shape.logits_shape(suffix_length),- act_dtype,- )- });-- let sampling_output = RefCell::new(unsafe {- MetalArray::new(- scratch.sampling_output.clone(),- &[suffix_length],- DataType::U32,- )- });-- let traces = if trace {- Some(Rc::new(RefCell::new(DecoderActivationTrace::new(- &context,- model_shape,- suffix_length,- ))))- } else {- None- };-- Self {- context,- token_ids: token_ids_refcell,- token_positions: token_positions_refcell,- token_bitmask: token_bitmask_refcell,- token_seeds: token_seeds_refcell,- attention_bias,- logits,- cache_layers,- shared_buffers,- aux_buffers,- sampling_output: Some(sampling_output),- sampling_method: None,- traces,- active_suffix_length,- is_prefilling,- }- }-- pub fn has_bitmask(&self) -> bool {- self.token_bitmask.is_some()- }-- fn array_cell(- &self,- id: ArrayId,- ) -> RefCell<MetalArray> {- match id {- ArrayId::TokenIds => self.token_ids.clone(),- ArrayId::TokenPositions => self.token_positions.clone(),- ArrayId::TokenBitmask => self.token_bitmask.as_ref().unwrap().clone(),- ArrayId::TokenSeeds => self.token_seeds.clone(),- ArrayId::Logits => self.logits.clone(),- ArrayId::Main => self.aux_buffers.main.clone(),- ArrayId::Shortcut => self.aux_buffers.shortcut.clone(),- ArrayId::QKV => self.aux_buffers.qkv.clone(),- ArrayId::AttentionOutput => {- self.aux_buffers.attention_output.clone()- },- ArrayId::MlpFusedUp => self.aux_buffers.mlp_fused_up.clone(),- ArrayId::MlpHidden => self.aux_buffers.mlp_hidden.clone(),- ArrayId::SsmInProj => self- .aux_buffers- .ssm_inproj- .as_ref()- .expect("SSM in-projection buffer requested but not initialized")- .clone(),- ArrayId::Keys(layer_index) => {- let cache = self.cache_layers.borrow();- match cache.data[layer_index].as_transformer() {- Some(layer) => layer.keys.clone(),- None => panic!(- "Requested transformer keys for non-transformer layer {}",- layer_index- ),- }- },- ArrayId::Values(layer_index) => {- let cache = self.cache_layers.borrow();- match cache.data[layer_index].as_transformer() {- Some(layer) => layer.values.clone(),- None => panic!(- "Requested transformer values for non-transformer layer {}",- layer_index- ),- }- },- ArrayId::SsmConvState(layer_index) => {- let cache = self.cache_layers.borrow();- match cache.data[layer_index].as_state_space() {- Some(layer) => layer.conv_state.clone(),- None => panic!(- "Requested SSM conv state for transformer layer {}",- layer_index- ),- }- },- ArrayId::SsmState(layer_index) => {- let cache = self.cache_layers.borrow();- match cache.data[layer_index].as_state_space() {- Some(layer) => layer.ssm_state.clone(),- None => panic!(- "Requested SSM state for transformer layer {}",- layer_index- ),- }- },- ArrayId::SsmPacked(layer_index) => {- let _ = layer_index;- self.aux_buffers- .ssm_packed- .as_ref()- .expect("SSM packed buffer not initialized")- .clone()- },- ArrayId::SsmX(layer_index) => {- let _ = layer_index;- self.aux_buffers- .ssm_x- .as_ref()- .expect("SSM x buffer not initialized")- .clone()- },- ArrayId::SsmB(layer_index) => {- let _ = layer_index;- self.aux_buffers- .ssm_b- .as_ref()- .expect("SSM b buffer not initialized")- .clone()- },- ArrayId::SsmC(layer_index) => {- let _ = layer_index;- self.aux_buffers- .ssm_c- .as_ref()- .expect("SSM c buffer not initialized")- .clone()- },- ArrayId::SsmDt(layer_index) => {- let _ = layer_index;- self.aux_buffers- .ssm_dt- .as_ref()- .expect("SSM dt buffer not initialized")- .clone()- },- ArrayId::SsmZ(layer_index) => {- let _ = layer_index;- self.aux_buffers- .ssm_z- .as_ref()- .expect("SSM z buffer not initialized")- .clone()- },- ArrayId::RotatedQueries => self.aux_buffers.rotated_queries.clone(),- ArrayId::RotatedKeys => self.aux_buffers.rotated_keys.clone(),- ArrayId::AttentionPartials => {- self.aux_buffers.attention_partials.clone()- },- ArrayId::AttentionSums => self.aux_buffers.attention_sums.clone(),- ArrayId::AttentionMaxs => self.aux_buffers.attention_maxs.clone(),-- ArrayId::EmbeddingsInputWeights => {- match &self.shared_buffers.borrow().embeddings {- EmbeddingsBuffers::Tied {- weights,- } => weights.clone(),- EmbeddingsBuffers::Untied {- input_weights,- ..- } => input_weights.clone(),- EmbeddingsBuffers::QuantizedTied {- weights,- ..- } => weights.clone(),- EmbeddingsBuffers::MLXSemiQuantizedUntied {- input_weights,- ..- } => input_weights.clone(),- }- },- ArrayId::EmbeddingsOutputWeights => {- match &self.shared_buffers.borrow().embeddings {- EmbeddingsBuffers::Tied {- weights,- } => weights.clone(),- EmbeddingsBuffers::Untied {- output_weights,- ..- } => output_weights.clone(),- EmbeddingsBuffers::QuantizedTied {- weights,- ..- } => weights.clone(),- EmbeddingsBuffers::MLXSemiQuantizedUntied {- packed_output_weights,- ..- } => packed_output_weights.clone(),- }- },- ArrayId::EmbeddingsScales => {- match &self.shared_buffers.borrow().embeddings {- EmbeddingsBuffers::QuantizedTied {- scales,- ..- } => scales.clone(),- EmbeddingsBuffers::MLXSemiQuantizedUntied {- output_scales,- ..- } => output_scales.clone(),- _ => panic!(- "Expected EmbeddingsBuffers::QuantizedTied or MLXSemiQuantizedUntied"- ),- }- },- ArrayId::RopeCosines(rope_type) => match rope_type {- RopeType::Global => {- let shared_buffers = self.shared_buffers.borrow();- shared_buffers- .global_rope- .as_ref()- .expect("Global rope requested but not initialized")- .cosines- .clone()- },- RopeType::Local => self- .shared_buffers- .borrow()- .local_rope- .as_ref()- .expect("Local rope requested but local_rope buffers are not initialized")- .cosines- .clone(),- },- ArrayId::RopeSines(rope_type) => match rope_type {- RopeType::Global => {- let shared_buffers = self.shared_buffers.borrow();- shared_buffers- .global_rope- .as_ref()- .expect("Global rope requested but not initialized")- .sines- .clone()- },- RopeType::Local => self- .shared_buffers- .borrow()- .local_rope- .as_ref()- .expect("Local rope requested but local_rope buffers are not initialized")- .sines- .clone(),- },-- ArrayId::MoeTopkIds => self.aux_buffers.moe_topk_ids.as_ref().expect("MoE topk ids buffer not initialized").clone(),- ArrayId::MoeTopkProbs => self.aux_buffers.moe_topk_probs.as_ref().expect("MoE topk probs buffer not initialized").clone(),- ArrayId::MoeOffsets => self.aux_buffers.moe_offsets.as_ref().expect("MoE offsets buffer not initialized").clone(),- ArrayId::MoeSumK => self.aux_buffers.moe_sumk.as_ref().expect("MoE sumk buffer not initialized").clone(),- ArrayId::MoeBucketedTokenIds => self.aux_buffers.moe_bucketed_token_ids.as_ref().expect("MoE bucketed token ids buffer not initialized").clone(),- ArrayId::MoeBucketedProbs => self.aux_buffers.moe_bucketed_probs.as_ref().expect("MoE bucketed probs buffer not initialized").clone(),- ArrayId::MoeXPerm => self.aux_buffers.moe_x_perm.as_ref().expect("MoE x_perm buffer not initialized").clone(),- ArrayId::MoeTok2Row => self.aux_buffers.moe_tok2row.as_ref().expect("MoE tok2row buffer not initialized").clone(),- ArrayId::MoeYPartial => self.aux_buffers.moe_y_partial.as_ref().expect("MoE y_partial buffer not initialized").clone(),- ArrayId::MoeHidden => self.aux_buffers.moe_hidden.as_ref().expect("MoE hidden buffer not initialized").clone(),- ArrayId::MoeTwoPassRowExpertMap => self.aux_buffers.moe_two_pass_row_expert_map.as_ref().expect("MoE two-pass row expert map buffer not initialized").clone(),- ArrayId::MoeTileCounts => self.aux_buffers.moe_tile_counts.as_ref().expect("MoE tile counts buffer not initialized").clone(),- ArrayId::MoeTileOffsets => self.aux_buffers.moe_tile_offsets.as_ref().expect("MoE tile offsets buffer not initialized").clone(),- ArrayId::MoeTileMap => self.aux_buffers.moe_tile_map.as_ref().expect("MoE tile map buffer not initialized").clone(),- ArrayId::MoeTotalTiles => self.aux_buffers.moe_total_tiles.as_ref().expect("MoE total tiles buffer not initialized").clone(),- ArrayId::MoeDispatchArgs => self.aux_buffers.moe_dispatch_args.as_ref().expect("MoE dispatch args buffer not initialized").clone(),- ArrayId::MoeScatterPartials => self.aux_buffers.moe_scatter_partials.as_ref().expect("MoE scatter partials buffer not initialized").clone(),- ArrayId::MoeScatterBlockBases => self.aux_buffers.moe_scatter_block_bases.as_ref().expect("MoE scatter block bases buffer not initialized").clone(),- ArrayId::MoeBlockAlloc => self.aux_buffers.moe_block_alloc.as_ref().expect("MoE block alloc buffer not initialized").clone(),-- ArrayId::AttentionSinks(layer_index) => {- self.shared_buffers.borrow().attention_sinks.as_ref()- .expect("Attention sinks not initialized")[layer_index].clone()- },- }- }-- pub fn hashmap_cell(- &self,- id: &HashMapId,- ) -> RefCell<HashMap<Option<usize>, ArrayCell>> {- match id {- HashMapId::AttentionBias => {- RefCell::new(self.attention_bias.clone())- },- }- }-- pub fn hashmaps(- &self,- ids: &[HashMapId],- ) -> Box<[HashMap<Option<usize>, ArrayCell>]> {- ids.iter().map(|id| self.hashmap_cell(id).into_inner()).collect()- }-- pub fn arrays(- &self,- ids: &[ArrayId],- ) -> Box<[ArrayCell]> {- ids.iter().map(|id| self.array_cell(*id).clone()).collect()- }-- pub fn copy_array(- &self,- source_array_id: ArrayId,- destination_array: RefCell<MetalArray>,- ) {- destination_array- .borrow_mut()- .copy_from_array(&self.arrays(&[source_array_id])[0].borrow());- }-- pub fn aux_buffers_suffix_length(&self) -> usize {- self.aux_buffers.suffix_length- }-- pub fn active_suffix_length(&self) -> usize {- self.active_suffix_length- }-- pub fn is_prefilling(&self) -> bool {- self.is_prefilling- }-- pub fn mtl_context(&self) -> &Rc<MTLContext> {- &self.context- }-- pub fn conv_padded_buffer(&self) -> Option<ArrayCell> {- self.aux_buffers.ssm_conv_padded.as_ref().map(|buf| buf.clone())- }-}--impl Drop for ForwardPassState {- fn drop(&mut self) {- // Nothing extra to clean up now that heap is removed.- }-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- PartialOrd,- Ord,- Hash,- Serialize,- Deserialize,-)]-pub enum RopeType {- Global,- Local,-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- PartialOrd,- Ord,- Hash,- Serialize,- Deserialize,-)]-pub enum ArrayId {- TokenIds,- TokenPositions,- TokenBitmask,- TokenSeeds,- Logits,-- Main,- Shortcut,- QKV,- AttentionOutput,- MlpFusedUp,- MlpHidden,- SsmInProj,-- Keys(usize),- Values(usize),- SsmConvState(usize),- SsmState(usize),- SsmPacked(usize),- SsmX(usize),- SsmB(usize),- SsmC(usize),- SsmDt(usize),- SsmZ(usize),-- RotatedQueries,- RotatedKeys,-- AttentionPartials,- AttentionSums,- AttentionMaxs,-- EmbeddingsInputWeights,- EmbeddingsOutputWeights,- EmbeddingsScales,- RopeCosines(RopeType),- RopeSines(RopeType),-- AttentionSinks(usize),-- MoeTopkIds,- MoeTopkProbs,- MoeOffsets,- MoeSumK,- MoeBucketedTokenIds,- MoeBucketedProbs,- MoeXPerm,- MoeTok2Row,- MoeYPartial,- MoeHidden,- MoeTwoPassRowExpertMap,- MoeTileCounts,- MoeTileOffsets,- MoeTileMap,- MoeTotalTiles,- MoeDispatchArgs,- MoeScatterPartials,- MoeScatterBlockBases,- MoeBlockAlloc,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub enum HashMapId {- AttentionBias,-}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs b/crates/uzu/src/backends/metal/forward_pass/state/array_id.rsnew file mode 100644index 000000000..63f270eaf--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs@@ -0,0 +1,84 @@+use serde::{Deserialize, Serialize};++use super::RopeType;++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Hash,+ Serialize,+ Deserialize,+)]+pub enum ArrayId {+ TokenIds,+ TokenPositions,+ TokenBitmask,+ Logits,+ TokenSeeds,++ Main,+ Shortcut,+ QKV,+ AttentionOutput,+ MlpFusedUp,+ MlpHidden,+ SsmInProj,++ Keys(usize),+ Values(usize),+ SsmConvState(usize),+ SsmState(usize),+ SsmPacked(usize),+ SsmX(usize),+ SsmB(usize),+ SsmC(usize),+ SsmDt(usize),+ SsmZ(usize),++ RotatedQueries,+ RotatedKeys,+ ExtractedValues,++ AttentionPartials,+ AttentionSums,+ AttentionMaxs,++ EmbeddingsInputWeights,+ EmbeddingsOutputWeights,+ EmbeddingsScales,+ RopeCosines(RopeType),+ RopeSines(RopeType),++ AttentionSinks(usize),++ MoeTopkIds,+ MoeTopkProbs,+ MoeOffsets,+ MoeSumK,+ MoeBucketedTokenIds,+ MoeBucketedProbs,+ MoeXPerm,+ MoeTok2Row,+ MoeYPartial,+ MoeHidden,+ MoeTwoPassRowExpertMap,+ MoeTileCounts,+ MoeTileOffsets,+ MoeTileMap,+ MoeTotalTiles,+ MoeDispatchArgs,+ MoeScatterPartials,+ MoeScatterBlockBases,+ MoeBlockAlloc,++ // Classifier prediction head buffers+ ClassifierPooling,+ ClassifierPredictionHeadDense,+ ClassifierPredictionHeadNorm,+ ClassifierPredictionHeadLogits,+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/common_aux_buffers.rsnew file mode 100644index 000000000..ee1412742--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/common_aux_buffers.rs@@ -0,0 +1,101 @@+use std::cell::RefCell;++use super::super::{ModelShape, ScratchBuffers};+use crate::backends::metal::MetalArray;++type ArrayCell = RefCell<MetalArray>;++pub struct CommonAuxBuffers {+ pub suffix_length: usize,+ pub main: ArrayCell,+ pub shortcut: ArrayCell,+ pub qkv: ArrayCell,+ pub attention_output: ArrayCell,+ pub mlp_fused_up: ArrayCell,+ pub mlp_hidden: ArrayCell,+ pub rotated_queries: ArrayCell,+ pub rotated_keys: ArrayCell,+ pub extracted_values: ArrayCell,+ pub attention_partials: ArrayCell,+ pub attention_sums: ArrayCell,+ pub attention_maxs: ArrayCell,+}++impl CommonAuxBuffers {+ pub fn new(+ scratch: &ScratchBuffers,+ model_shape: &ModelShape,+ suffix_length: usize,+ ) -> Self {+ let act_dtype = model_shape.activation_data_type();+ unsafe {+ Self {+ suffix_length,+ main: RefCell::new(MetalArray::new(+ scratch.main.clone(),+ &model_shape.main_shape(suffix_length),+ act_dtype,+ )),+ shortcut: RefCell::new(MetalArray::new(+ scratch.shortcut.clone(),+ &model_shape.main_shape(suffix_length),+ act_dtype,+ )),+ qkv: RefCell::new(MetalArray::new(+ scratch.qkv.clone(),+ &model_shape.qkv_shape(suffix_length),+ act_dtype,+ )),+ attention_output: RefCell::new(MetalArray::new(+ scratch.attention_output.clone(),+ &model_shape.attention_output_shape(suffix_length),+ act_dtype,+ )),+ mlp_fused_up: RefCell::new(MetalArray::new(+ scratch.mlp_fused_up.clone(),+ &model_shape.mlp_fused_up_shape(suffix_length),+ act_dtype,+ )),+ mlp_hidden: RefCell::new(MetalArray::new(+ scratch.mlp_hidden.clone(),+ &model_shape.mlp_hidden_shape(suffix_length),+ act_dtype,+ )),+ rotated_queries: RefCell::new(MetalArray::new(+ scratch.rotated_queries.clone(),+ &model_shape.rotated_queries_shape(suffix_length),+ act_dtype,+ )),+ rotated_keys: RefCell::new(MetalArray::new(+ scratch.rotated_keys.clone(),+ &model_shape.rotated_keys_shape(suffix_length),+ act_dtype,+ )),+ extracted_values: RefCell::new(MetalArray::new(+ scratch.extracted_values.clone(),+ &model_shape.extracted_values_shape(suffix_length),+ act_dtype,+ )),+ attention_partials: RefCell::new(MetalArray::new(+ scratch.attention_partials.clone(),+ &model_shape.attention_partials_shape(suffix_length),+ act_dtype,+ )),+ attention_sums: RefCell::new(MetalArray::new(+ scratch.attention_sums.clone(),+ &model_shape.attention_sums_shape(suffix_length),+ act_dtype,+ )),+ attention_maxs: RefCell::new(MetalArray::new(+ scratch.attention_maxs.clone(),+ &model_shape.attention_maxs_shape(suffix_length),+ act_dtype,+ )),+ }+ }+ }++ pub fn suffix_length(&self) -> usize {+ self.suffix_length+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/embeddings_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/embeddings_buffers.rsnew file mode 100644index 000000000..234934391--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/embeddings_buffers.rs@@ -0,0 +1,221 @@+use std::{cell::RefCell, rc::Rc};++use super::super::ModelShape;+use crate::{+ DeviceContext,+ backends::metal::{MTLContext, MetalArray},+ config::EmbeddingConfig,+ parameters::ParameterTree,+};++type ArrayCell = RefCell<MetalArray>;++#[allow(dead_code)]+pub enum EmbeddingsBuffers {+ Tied {+ /// [vocab_size, model_dim]+ weights: ArrayCell,+ },+ Untied {+ /// [vocab_size, model_dim]+ input_weights: ArrayCell,+ /// [vocab_size, model_dim]+ output_weights: ArrayCell,+ },+ QuantizedTied {+ /// [vocab_size, model_dim]+ weights: ArrayCell,+ /// [vocab_size]+ scales: ArrayCell,+ },+ MLXSemiQuantizedUntied {+ /// [vocab_size, model_dim]+ input_weights: ArrayCell,+ /// [vocab_size, model_dim]+ packed_output_weights: ArrayCell,+ /// [vocab_size, num_groups]+ output_scales: ArrayCell,+ /// [vocab_size, num_groups]+ output_biases: ArrayCell,+ },+}++impl EmbeddingsBuffers {+ pub fn new(+ context: &MTLContext,+ embeddings_config: &EmbeddingConfig,+ model_shape: &ModelShape,+ ) -> Self {+ unsafe {+ match embeddings_config {+ EmbeddingConfig::Tied {+ common: _,+ precision: _,+ } => Self::Tied {+ weights: RefCell::new(context.array_uninitialized(+ &model_shape.embeddings_input_shape(),+ model_shape.activation_data_type(),+ )),+ },+ EmbeddingConfig::Untied {+ common: _,+ precision: _,+ } => Self::Untied {+ input_weights: RefCell::new(context.array_uninitialized(+ &model_shape.embeddings_input_shape(),+ model_shape.activation_data_type(),+ )),+ output_weights: RefCell::new(context.array_uninitialized(+ &model_shape.embeddings_output_shape(),+ model_shape.activation_data_type(),+ )),+ },+ EmbeddingConfig::QuantizedTied {+ embedding_quantization_mode,+ ..+ } => {+ let [vocab_size, model_dim] =+ model_shape.quantized_embeddings_weights_shape();+ Self::QuantizedTied {+ weights: RefCell::new(+ context.array_uninitialized(+ &[+ vocab_size,+ model_dim+ / embedding_quantization_mode+ .packing_divisor(),+ ],+ embedding_quantization_mode.storage_type(),+ ),+ ),+ scales: RefCell::new(context.array_uninitialized(+ &model_shape.quantized_embeddings_scales_shape(),+ model_shape.activation_data_type(),+ )),+ }+ },+ EmbeddingConfig::MLXQuantizedTied {+ group_size,+ embedding_quantization_mode,+ ..+ } => {+ let [vocab_size, model_dim] =+ model_shape.quantized_embeddings_weights_shape();+ let num_groups = model_dim / group_size;+ Self::QuantizedTied {+ weights: RefCell::new(+ context.array_uninitialized(+ &[+ vocab_size,+ model_dim+ / embedding_quantization_mode+ .packing_divisor(),+ ],+ embedding_quantization_mode.storage_type(),+ ),+ ),+ scales: RefCell::new(context.array_uninitialized(+ &[vocab_size, num_groups],+ model_shape.activation_data_type(),+ )),+ }+ },+ EmbeddingConfig::MLXSemiQuantizedUntied {+ group_size,+ embedding_quantization_mode,+ ..+ } => {+ let [vocab_size, model_dim] =+ model_shape.quantized_embeddings_weights_shape();+ let num_groups = model_dim / group_size;+ Self::MLXSemiQuantizedUntied {+ input_weights: RefCell::new(+ context.array_uninitialized(+ &model_shape.embeddings_input_shape(),+ model_shape.activation_data_type(),+ ),+ ),+ packed_output_weights: RefCell::new(+ context.array_uninitialized(+ &[+ vocab_size,+ model_dim+ / embedding_quantization_mode+ .packing_divisor(),+ ],+ embedding_quantization_mode.storage_type(),+ ),+ ),+ output_scales: RefCell::new(+ context.array_uninitialized(+ &[vocab_size, num_groups],+ model_shape.activation_data_type(),+ ),+ ),+ output_biases: RefCell::new(+ context.array_uninitialized(+ &[vocab_size, num_groups],+ model_shape.activation_data_type(),+ ),+ ),+ }+ },+ }+ }+ }++ pub fn update_data(+ &mut self,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) {+ let embeddings_tree = parameter_tree.subtree("embedding").unwrap();+ match self {+ EmbeddingsBuffers::Tied {+ weights,+ } => {+ let embeddings_view = embeddings_tree.leaf("weights").unwrap();+ weights.borrow_mut().copy_from_array(&embeddings_view);+ },+ EmbeddingsBuffers::Untied {+ input_weights,+ output_weights,+ } => {+ let mapping = vec![+ ("input_weights", input_weights),+ ("output_weights", output_weights),+ ];+ for (name, buffer) in mapping {+ let view = embeddings_tree.leaf(name).unwrap();+ buffer.borrow_mut().copy_from_array(&view);+ }+ },+ EmbeddingsBuffers::QuantizedTied {+ weights,+ scales,+ } => {+ let mapping = vec![("weights", weights), ("scales", scales)];+ for (name, buffer) in mapping {+ let view = embeddings_tree.leaf(name).unwrap();+ buffer.borrow_mut().copy_from_array(&view);+ }+ },+ EmbeddingsBuffers::MLXSemiQuantizedUntied {+ input_weights,+ packed_output_weights,+ output_scales,+ output_biases,+ } => {+ let mapping = vec![+ ("input_weights", input_weights),+ ("output_weights", packed_output_weights),+ ("output_scales", output_scales),+ ("output_biases", output_biases),+ ];+ for (name, buffer) in mapping {+ let view = embeddings_tree.leaf(name).unwrap();+ buffer.borrow_mut().copy_from_array(&view);+ }+ },+ }+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/hash_map_id.rs b/crates/uzu/src/backends/metal/forward_pass/state/hash_map_id.rsnew file mode 100644index 000000000..4d9231e7e--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/hash_map_id.rs@@ -0,0 +1,4 @@+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum HashMapId {+ AttentionBias,+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rsnew file mode 100644index 000000000..4469de591--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs@@ -0,0 +1,416 @@+use std::cell::RefCell;++use super::super::{ModelShape, ScratchBuffers};+use crate::{+ DataType, DecoderConfig, backends::metal::MetalArray, config::MLPConfig,+};++type ArrayCell = RefCell<MetalArray>;++pub struct LanguageModelGeneratorAuxBuffers {+ pub ssm_inproj: Option<ArrayCell>,+ pub ssm_packed: Option<ArrayCell>,+ pub ssm_conv_padded: Option<ArrayCell>,+ pub ssm_x: Option<ArrayCell>,+ pub ssm_b: Option<ArrayCell>,+ pub ssm_c: Option<ArrayCell>,+ pub ssm_dt: Option<ArrayCell>,+ pub ssm_z: Option<ArrayCell>,+ // MoE buffers+ pub moe_topk_ids: Option<ArrayCell>,+ pub moe_topk_probs: Option<ArrayCell>,+ pub moe_offsets: Option<ArrayCell>,+ pub moe_sumk: Option<ArrayCell>,+ pub moe_bucketed_token_ids: Option<ArrayCell>,+ pub moe_bucketed_probs: Option<ArrayCell>,+ pub moe_x_perm: Option<ArrayCell>,+ pub moe_tok2row: Option<ArrayCell>,+ pub moe_y_partial: Option<ArrayCell>,+ pub moe_hidden: Option<ArrayCell>,+ pub moe_two_pass_row_expert_map: Option<ArrayCell>,+ pub moe_tile_counts: Option<ArrayCell>,+ pub moe_tile_offsets: Option<ArrayCell>,+ pub moe_tile_map: Option<ArrayCell>,+ pub moe_total_tiles: Option<ArrayCell>,+ pub moe_dispatch_args: Option<ArrayCell>,+ pub moe_scatter_partials: Option<ArrayCell>,+ pub moe_scatter_block_bases: Option<ArrayCell>,+ pub moe_block_alloc: Option<ArrayCell>,+}++impl LanguageModelGeneratorAuxBuffers {+ pub fn new(+ scratch: &ScratchBuffers,+ decoder_config: &DecoderConfig,+ model_shape: &ModelShape,+ suffix_length: usize,+ ) -> Self {+ let act_dtype = model_shape.activation_data_type();++ unsafe {+ Self {+ ssm_inproj: match (+ scratch.ssm_inproj.as_ref(),+ model_shape.ssm_inproj_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },+ ssm_packed: match (+ scratch.ssm_packed.as_ref(),+ model_shape.ssm_packed_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },+ ssm_conv_padded: match (+ scratch.ssm_conv_padded.as_ref(),+ model_shape.ssm_conv_padded_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },+ ssm_x: match (+ scratch.ssm_x.as_ref(),+ model_shape.ssm_x_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },+ ssm_b: match (+ scratch.ssm_b.as_ref(),+ model_shape.ssm_bc_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },+ ssm_c: match (+ scratch.ssm_c.as_ref(),+ model_shape.ssm_bc_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },+ ssm_dt: match (+ scratch.ssm_dt.as_ref(),+ model_shape.ssm_dt_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },+ ssm_z: match (+ scratch.ssm_z.as_ref(),+ model_shape.ssm_z_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },+ moe_topk_ids: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_topk_ids.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_topk_ids_shape(+ suffix_length,+ moe.num_experts_per_token,+ ),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_topk_probs: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_topk_probs.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_topk_probs_shape(+ suffix_length,+ moe.num_experts_per_token,+ ),+ act_dtype,+ ))+ })+ },+ _ => None,+ },+ moe_offsets: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_offsets.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape+ .moe_offsets_shape(moe.mixture_size),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_sumk: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(_) => {+ scratch.moe_sumk.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_sumk_shape(),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_bucketed_token_ids: match &decoder_config+ .layer_config+ .mlp_config+ {+ MLPConfig::MixtureOfExperts(moe) => {+ let max_routed =+ suffix_length * moe.num_experts_per_token;+ scratch.moe_bucketed_token_ids.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape+ .moe_bucketed_token_ids_shape(max_routed),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_bucketed_probs: match &decoder_config+ .layer_config+ .mlp_config+ {+ MLPConfig::MixtureOfExperts(moe) => {+ let max_routed =+ suffix_length * moe.num_experts_per_token;+ scratch.moe_bucketed_probs.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape+ .moe_bucketed_probs_shape(max_routed),+ act_dtype,+ ))+ })+ },+ _ => None,+ },+ moe_x_perm: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ let max_routed =+ suffix_length * moe.num_experts_per_token;+ scratch.moe_x_perm.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_x_perm_shape(max_routed),+ DataType::F16,+ ))+ })+ },+ _ => None,+ },+ moe_tok2row: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_tok2row.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_tok2row_shape(+ suffix_length,+ moe.num_experts_per_token,+ ),+ DataType::I32,+ ))+ })+ },+ _ => None,+ },+ moe_y_partial: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ let max_routed =+ suffix_length * moe.num_experts_per_token;+ scratch.moe_y_partial.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_y_partial_shape(max_routed),+ DataType::F16,+ ))+ })+ },+ _ => None,+ },+ moe_hidden: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ let max_routed =+ suffix_length * moe.num_experts_per_token;+ scratch.moe_hidden.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_hidden_shape(max_routed),+ DataType::F32,+ ))+ })+ },+ _ => None,+ },+ moe_two_pass_row_expert_map: match &decoder_config+ .layer_config+ .mlp_config+ {+ MLPConfig::MixtureOfExperts(moe) => {+ let max_routed =+ suffix_length * moe.num_experts_per_token;+ scratch.moe_two_pass_row_expert_map.as_ref().map(+ |buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &[max_routed],+ DataType::U32,+ ))+ },+ )+ },+ _ => None,+ },+ moe_tile_counts: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_tile_counts.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_counts_shape(moe.mixture_size),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_tile_offsets: match &decoder_config.layer_config.mlp_config+ {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_tile_offsets.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape+ .moe_offsets_shape(moe.mixture_size),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_tile_map: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ let max_routed =+ suffix_length * moe.num_experts_per_token;+ scratch.moe_tile_map.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_tile_map_shape(max_routed),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_total_tiles: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(_) => {+ scratch.moe_total_tiles.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_total_tiles_shape(),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_dispatch_args: match &decoder_config.layer_config.mlp_config+ {+ MLPConfig::MixtureOfExperts(_) => {+ scratch.moe_dispatch_args.as_ref().map(|buf| {+ RefCell::new(MetalArray::new(+ buf.clone(),+ &model_shape.moe_dispatch_args_shape(),+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_scatter_partials: match &decoder_config+ .layer_config+ .mlp_config+ {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_scatter_partials.as_ref().map(|buf| {+ let num_blocks =+ ((suffix_length + 255) / 256).max(1);+ let num_tiles =+ ((moe.mixture_size + 512 - 1) / 512).max(1);+ let entries = num_blocks * num_tiles * 512;+ RefCell::new(MetalArray::new(+ buf.clone(),+ &[entries],+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_scatter_block_bases: match &decoder_config+ .layer_config+ .mlp_config+ {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_scatter_block_bases.as_ref().map(|buf| {+ let num_blocks =+ ((suffix_length + 255) / 256).max(1);+ let num_tiles =+ ((moe.mixture_size + 512 - 1) / 512).max(1);+ let entries = num_blocks * num_tiles * 512;+ RefCell::new(MetalArray::new(+ buf.clone(),+ &[entries],+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ moe_block_alloc: match &decoder_config.layer_config.mlp_config {+ MLPConfig::MixtureOfExperts(moe) => {+ scratch.moe_block_alloc.as_ref().map(|buf| {+ let num_blocks =+ ((suffix_length + 255) / 256).max(1);+ let num_tiles =+ ((moe.mixture_size + 512 - 1) / 512).max(1);+ let entries = num_blocks * num_tiles;+ RefCell::new(MetalArray::new(+ buf.clone(),+ &[entries],+ DataType::U32,+ ))+ })+ },+ _ => None,+ },+ }+ }+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rsnew file mode 100644index 000000000..f7d6c5b47--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs@@ -0,0 +1,1017 @@+mod array_id;+mod common_aux_buffers;+mod embeddings_buffers;+mod hash_map_id;+mod language_model_generator_aux_buffers;+mod mode;+mod rope_buffers;+mod rope_type;+mod shared_buffers;++use std::{cell::RefCell, collections::HashMap, rc::Rc};++pub use array_id::ArrayId;+pub use common_aux_buffers::CommonAuxBuffers;+pub use embeddings_buffers::EmbeddingsBuffers;+pub use hash_map_id::HashMapId;+pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;+pub use mode::{+ ClassifierModeState, ForwardPassMode, LanguageModelGeneratorModeState,+};+pub use rope_buffers::RopeBuffers;+pub use rope_type::RopeType;+pub use shared_buffers::{MoeExpertWeights, SharedBuffers};++#[cfg(feature = "tracing")]+use super::traces::ActivationTrace;+use super::{ModelShape, ScratchBuffers, cache_layers::CacheLayers};+use crate::{+ Array, DataType, DecoderConfig, DeviceContext,+ backends::metal::{MTLContext, MetalArray},+ session::parameter::SamplingMethod,+};++pub type ArrayCell = RefCell<MetalArray>;++pub struct ForwardPassState {+ context: Rc<MTLContext>,+ token_ids: ArrayCell,+ token_positions: ArrayCell,+ token_bitmask: Option<ArrayCell>,+ attention_bias: HashMap<Option<usize>, ArrayCell>,+ pub shared_buffers: Rc<RefCell<SharedBuffers>>,+ common_aux: CommonAuxBuffers,+ llm_aux: Option<LanguageModelGeneratorAuxBuffers>,+ mode: ForwardPassMode,+}++impl ForwardPassState {+ // ========================================================================+ // Common initialization helpers+ // ========================================================================++ fn init_token_ids(+ context: &MTLContext,+ scratch: &ScratchBuffers,+ token_ids: &[u64],+ ) -> ArrayCell {+ let suffix_length = token_ids.len();+ let mut token_ids_array = unsafe {+ MetalArray::new(+ scratch.token_ids.clone(),+ &[suffix_length],+ DataType::U64,+ )+ };+ context.copy_from_view(&mut token_ids_array, token_ids.into());+ RefCell::new(token_ids_array)+ }++ fn init_token_positions(+ context: &MTLContext,+ scratch: &ScratchBuffers,+ token_positions: &[usize],+ ) -> ArrayCell {+ let suffix_length = token_positions.len();+ let mut token_positions_array = unsafe {+ MetalArray::new(+ scratch.token_positions.clone(),+ &[suffix_length],+ DataType::I32,+ )+ };+ let token_positions_i32: Box<[i32]> =+ token_positions.iter().map(|p| *p as i32).collect();+ context.copy_from_view(+ &mut token_positions_array,+ token_positions_i32.as_ref().into(),+ );+ RefCell::new(token_positions_array)+ }++ // ========================================================================+ // LLM Constructor+ // ========================================================================++ #[allow(clippy::too_many_arguments)]+ pub fn new_llm(+ context: Rc<MTLContext>,+ decoder_config: &DecoderConfig,+ model_shape: &ModelShape,+ scratch: &ScratchBuffers,+ cache_layers: Rc<RefCell<CacheLayers>>,+ shared_buffers: Rc<RefCell<SharedBuffers>>,+ token_ids: &[u64],+ token_positions: &[usize],+ token_bitmask: Option<&[u32]>,+ token_seeds: &[u64],+ active_suffix_length: usize,+ is_prefilling: bool,+ external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,+ skip_token_ids_copy: bool,+ async_positions: Option<(&metal::Buffer, usize)>,+ async_seeds: Option<(&metal::Buffer, usize)>,+ ) -> Self {+ let suffix_length = token_ids.len();+ assert_eq!(+ suffix_length,+ token_positions.len(),+ "Tokens and positions must have same length"+ );+ assert!(+ suffix_length <= cache_layers.borrow().max_suffix_length(),+ "Suffix length exceeds KV cache capacity"+ );++ // Token IDs - optionally skip copy for async path+ let token_ids_cell = if skip_token_ids_copy {+ RefCell::new(unsafe {+ MetalArray::new(+ scratch.token_ids.clone(),+ &[suffix_length],+ DataType::U64,+ )+ })+ } else {+ Self::init_token_ids(&context, scratch, token_ids)+ };++ // Token positions - use async buffer if provided+ let token_positions_cell =+ if let Some((async_buf, offset)) = async_positions {+ let array = unsafe {+ MetalArray::new_with_offset(+ async_buf.clone(),+ &[suffix_length],+ DataType::I32,+ offset * std::mem::size_of::<i32>(),+ )+ };+ RefCell::new(array)+ } else {+ Self::init_token_positions(&context, scratch, token_positions)+ };++ // Token bitmask+ let token_bitmask_cell = token_bitmask.map(|bitmask| {+ let bitmask_shape = model_shape.bitmask_shape(suffix_length);+ let mut bitmask_array = unsafe {+ MetalArray::new(+ scratch.token_bitmask.clone(),+ &bitmask_shape,+ DataType::U32,+ )+ };+ context.copy_from_view(&mut bitmask_array, bitmask.into());+ RefCell::new(bitmask_array)+ });++ // Token seeds - use async buffer if provided+ let token_seeds_cell = if let Some((async_buf, offset)) = async_seeds {+ let array = unsafe {+ MetalArray::new_with_offset(+ async_buf.clone(),+ &[suffix_length],+ DataType::U64,+ offset * std::mem::size_of::<u64>(),+ )+ };+ RefCell::new(array)+ } else {+ let mut token_seeds_array = unsafe {+ MetalArray::new(+ scratch.token_seeds.clone(),+ &[suffix_length],+ DataType::U64,+ )+ };+ context.copy_from_view(&mut token_seeds_array, token_seeds.into());+ RefCell::new(token_seeds_array)+ };++ // Logits+ let logits_cell = RefCell::new(unsafe {+ MetalArray::new(+ scratch.logits.clone(),+ &model_shape.logits_shape(suffix_length),+ model_shape.activation_data_type(),+ )+ });++ // Sampling output+ let sampling_output = Some(RefCell::new(unsafe {+ MetalArray::new(+ scratch.sampling_output.clone(),+ &[suffix_length],+ DataType::U32,+ )+ }));++ // Attention bias (causal + sliding window)+ let act_dtype = model_shape.activation_data_type();+ let attention_bias = Self::init_llm_attention_bias(+ &context,+ scratch,+ &cache_layers,+ suffix_length,+ act_dtype,+ token_positions,+ external_bias_fn,+ );++ // Common aux buffers+ let common_aux =+ CommonAuxBuffers::new(scratch, model_shape, suffix_length);++ // LLM-specific aux buffers+ let llm_aux = Some(LanguageModelGeneratorAuxBuffers::new(+ scratch,+ decoder_config,+ model_shape,+ suffix_length,+ ));++ // Traces+ #[cfg(feature = "tracing")]+ let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(+ &context,+ model_shape,+ suffix_length,+ )));++ let mode = ForwardPassMode::LanguageModelGenerator(+ LanguageModelGeneratorModeState {+ cache_layers,+ token_seeds: token_seeds_cell,+ logits: logits_cell,+ sampling_output,+ sampling_method: None,+ #[cfg(feature = "tracing")]+ traces,+ active_suffix_length,+ is_prefilling,+ },+ );++ Self {+ context,+ token_ids: token_ids_cell,+ token_positions: token_positions_cell,+ token_bitmask: token_bitmask_cell,+ attention_bias,+ shared_buffers,+ common_aux,+ llm_aux,+ mode,+ }+ }++ fn init_llm_attention_bias(+ context: &MTLContext,+ scratch: &ScratchBuffers,+ cache_layers: &Rc<RefCell<CacheLayers>>,+ suffix_length: usize,+ act_dtype: DataType,+ token_positions: &[usize],+ external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,+ ) -> HashMap<Option<usize>, ArrayCell> {+ let cache_ref = cache_layers.borrow();+ let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =+ scratch+ .attention_window_size_to_bias+ .iter()+ .map(|(window_size, buffer)| {+ let prefix_length =+ window_size.unwrap_or(cache_ref.max_prefix_length());+ let attention_bias_shape =+ [suffix_length, suffix_length + prefix_length];+ let array = unsafe {+ MetalArray::new(+ buffer.clone(),+ &attention_bias_shape,+ act_dtype,+ )+ };+ (*window_size, array)+ })+ .collect();+ drop(cache_ref);++ // Use cache_layers' fill_attention_bias which properly handles+ // both causal masking and sliding window constraints+ cache_layers.borrow().fill_attention_bias(+ &mut attention_bias_map,+ token_positions,+ suffix_length,+ context,+ external_bias_fn,+ );++ attention_bias_map+ .into_iter()+ .map(|(k, v)| (k, RefCell::new(v)))+ .collect()+ }++ // ========================================================================+ // Classifier Constructor+ // ========================================================================++ #[allow(clippy::too_many_arguments)]+ pub fn new_classifier(+ context: Rc<MTLContext>,+ model_shape: &ModelShape,+ scratch: &ScratchBuffers,+ shared_buffers: Rc<RefCell<SharedBuffers>>,+ token_ids: &[u64],+ token_positions: &[usize],+ bidirectional_attention: bool,+ num_labels: usize,+ ) -> Self {+ let suffix_length = token_ids.len();+ assert_eq!(suffix_length, token_positions.len());++ let token_ids_cell = Self::init_token_ids(&context, scratch, token_ids);+ let token_positions_cell =+ Self::init_token_positions(&context, scratch, token_positions);++ // Attention bias (bidirectional or causal)+ let act_dtype = model_shape.activation_data_type();+ let attention_bias = Self::init_classifier_attention_bias(+ &context,+ scratch,+ suffix_length,+ act_dtype,+ bidirectional_attention,+ );++ // Common aux buffers+ let common_aux =+ CommonAuxBuffers::new(scratch, model_shape, suffix_length);++ // Classifier-specific buffers+ let model_dim = model_shape.main_shape(1)[1];+ let classifier_state = Self::init_classifier_buffers(+ &context,+ model_shape,+ model_dim,+ num_labels,+ suffix_length,+ );++ let mode = ForwardPassMode::Classifier(classifier_state);++ Self {+ context,+ token_ids: token_ids_cell,+ token_positions: token_positions_cell,+ token_bitmask: None,+ attention_bias,+ shared_buffers,+ common_aux,+ llm_aux: None,+ mode,+ }+ }++ fn init_classifier_attention_bias(+ context: &MTLContext,+ scratch: &ScratchBuffers,+ suffix_length: usize,+ act_dtype: DataType,+ bidirectional_attention: bool,+ ) -> HashMap<Option<usize>, ArrayCell> {+ let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =+ scratch+ .attention_window_size_to_bias+ .iter()+ .map(|(window_size, buffer)| {+ let attention_bias_shape = [suffix_length, suffix_length];+ let array = unsafe {+ MetalArray::new(+ buffer.clone(),+ &attention_bias_shape,+ act_dtype,+ )+ };+ (*window_size, array)+ })+ .collect();++ for (window, bias_array) in attention_bias_map.iter_mut() {+ if bidirectional_attention {+ if let Some(window_size) = window {+ let half_window = (window_size / 2) as isize;+ context.fill_attention_bias(+ bias_array,+ suffix_length,+ 0,+ |row, col| {+ let distance = (row as isize) - (col as isize);+ distance.abs() > half_window+ },+ );+ } else {+ context.fill_attention_bias(+ bias_array,+ suffix_length,+ 0,+ |_row, _col| false,+ );+ }+ } else {+ context.fill_attention_bias(+ bias_array,+ suffix_length,+ 0,+ |row, col| row < col,+ );+ }+ }++ attention_bias_map+ .into_iter()+ .map(|(k, v)| (k, RefCell::new(v)))+ .collect()+ }++ #[cfg(feature = "tracing")]+ fn init_classifier_buffers(+ context: &MTLContext,+ model_shape: &ModelShape,+ model_dim: usize,+ num_labels: usize,+ suffix_length: usize,+ ) -> ClassifierModeState {+ let data_type = model_shape.activation_data_type();+ let batch_size = 1;++ let create_buffer = |size: usize| -> ArrayCell {+ let buffer_size = (size * data_type.size_in_bytes()) as u64;+ let buffer = context.device.new_buffer(+ buffer_size,+ metal::MTLResourceOptions::StorageModeShared,+ );+ RefCell::new(unsafe {+ MetalArray::new(+ buffer,+ &[batch_size, size / batch_size],+ data_type,+ )+ })+ };++ ClassifierModeState {+ pooling: create_buffer(batch_size * model_dim),+ dense: create_buffer(batch_size * model_dim),+ norm: create_buffer(batch_size * model_dim),+ classifier_logits: {+ let buffer_size =+ (batch_size * num_labels * data_type.size_in_bytes())+ as u64;+ let buffer = context.device.new_buffer(+ buffer_size,+ metal::MTLResourceOptions::StorageModeShared,+ );+ RefCell::new(unsafe {+ MetalArray::new(+ buffer,+ &[batch_size, num_labels],+ data_type,+ )+ })+ },+ traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(+ context,+ model_shape,+ suffix_length,+ num_labels,+ ))),+ }+ }++ #[cfg(not(feature = "tracing"))]+ fn init_classifier_buffers(+ context: &MTLContext,+ model_shape: &ModelShape,+ model_dim: usize,+ num_labels: usize,+ _suffix_length: usize,+ ) -> ClassifierModeState {+ let data_type = model_shape.activation_data_type();+ let batch_size = 1;++ let create_buffer = |dims: &[usize]| -> ArrayCell {+ let size: usize = dims.iter().product();+ let buffer_size = (size * data_type.size_in_bytes()) as u64;+ let buffer = context.device.new_buffer(+ buffer_size,+ metal::MTLResourceOptions::StorageModeShared,+ );+ RefCell::new(unsafe { MetalArray::new(buffer, dims, data_type) })+ };++ ClassifierModeState {+ pooling: create_buffer(&[batch_size, model_dim]),+ dense: create_buffer(&[batch_size, model_dim]),+ norm: create_buffer(&[batch_size, model_dim]),+ classifier_logits: create_buffer(&[batch_size, num_labels]),+ }+ }++ // ========================================================================+ // Accessors+ // ========================================================================++ pub fn mtl_context(&self) -> &Rc<MTLContext> {+ &self.context+ }++ pub fn aux_buffers_suffix_length(&self) -> usize {+ self.common_aux.suffix_length+ }++ pub fn is_llm(&self) -> bool {+ matches!(self.mode, ForwardPassMode::LanguageModelGenerator(_))+ }++ pub fn is_classifier(&self) -> bool {+ matches!(self.mode, ForwardPassMode::Classifier(_))+ }++ pub fn token_bitmask(&self) -> Option<&ArrayCell> {+ self.token_bitmask.as_ref()+ }++ pub fn llm_state(&self) -> &LanguageModelGeneratorModeState {+ match &self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => state,+ _ => panic!("Not in LLM mode"),+ }+ }++ pub fn llm_state_mut(&mut self) -> &mut LanguageModelGeneratorModeState {+ match &mut self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => state,+ _ => panic!("Not in LLM mode"),+ }+ }++ pub fn classifier_state(&self) -> &ClassifierModeState {+ match &self.mode {+ ForwardPassMode::Classifier(state) => state,+ _ => panic!("Not in classifier mode"),+ }+ }++ pub fn array_cell(+ &self,+ id: ArrayId,+ ) -> ArrayCell {+ match id {+ // Common arrays+ ArrayId::TokenIds => self.token_ids.clone(),+ ArrayId::TokenPositions => self.token_positions.clone(),+ ArrayId::TokenBitmask => {+ self.token_bitmask.clone().expect("Token bitmask not available")+ },+ ArrayId::Main => self.common_aux.main.clone(),+ ArrayId::Shortcut => self.common_aux.shortcut.clone(),+ ArrayId::QKV => self.common_aux.qkv.clone(),+ ArrayId::AttentionOutput => {+ self.common_aux.attention_output.clone()+ },+ ArrayId::MlpFusedUp => self.common_aux.mlp_fused_up.clone(),+ ArrayId::MlpHidden => self.common_aux.mlp_hidden.clone(),+ ArrayId::RotatedQueries => self.common_aux.rotated_queries.clone(),+ ArrayId::RotatedKeys => self.common_aux.rotated_keys.clone(),+ ArrayId::ExtractedValues => {+ self.common_aux.extracted_values.clone()+ },+ ArrayId::AttentionPartials => {+ self.common_aux.attention_partials.clone()+ },+ ArrayId::AttentionSums => self.common_aux.attention_sums.clone(),+ ArrayId::AttentionMaxs => self.common_aux.attention_maxs.clone(),++ // Shared buffer arrays+ ArrayId::EmbeddingsInputWeights+ | ArrayId::EmbeddingsOutputWeights+ | ArrayId::EmbeddingsScales+ | ArrayId::RopeCosines(_)+ | ArrayId::RopeSines(_) => self+ .shared_buffer_array(id)+ .expect("Shared buffer array should be available"),++ // LLM-specific arrays+ ArrayId::Logits => self.llm_state().logits.clone(),+ ArrayId::TokenSeeds => self.llm_state().token_seeds.clone(),+ ArrayId::Keys(layer_index) => {+ let cache = self.llm_state().cache_layers.borrow();+ cache.data[layer_index]+ .as_transformer()+ .expect("Expected transformer layer")+ .keys+ .clone()+ },+ ArrayId::Values(layer_index) => {+ let cache = self.llm_state().cache_layers.borrow();+ cache.data[layer_index]+ .as_transformer()+ .expect("Expected transformer layer")+ .values+ .clone()+ },+ ArrayId::AttentionSinks(layer_index) => self+ .shared_buffers+ .borrow()+ .attention_sinks+ .as_ref()+ .expect("Attention sinks not initialized")[layer_index]+ .clone(),++ // SSM arrays (LLM only)+ ArrayId::SsmInProj => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.ssm_inproj.clone())+ .expect("SSM inproj not initialized"),+ ArrayId::SsmPacked(_) => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.ssm_packed.clone())+ .expect("SSM packed not initialized"),+ ArrayId::SsmConvState(layer_index) => {+ let cache = self.llm_state().cache_layers.borrow();+ cache.data[layer_index]+ .as_state_space()+ .expect("Expected SSM layer")+ .conv_state+ .clone()+ },+ ArrayId::SsmState(layer_index) => {+ let cache = self.llm_state().cache_layers.borrow();+ cache.data[layer_index]+ .as_state_space()+ .expect("Expected SSM layer")+ .ssm_state+ .clone()+ },+ ArrayId::SsmX(_) => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.ssm_x.clone())+ .expect("SSM x not initialized"),+ ArrayId::SsmB(_) => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.ssm_b.clone())+ .expect("SSM b not initialized"),+ ArrayId::SsmC(_) => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.ssm_c.clone())+ .expect("SSM c not initialized"),+ ArrayId::SsmDt(_) => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.ssm_dt.clone())+ .expect("SSM dt not initialized"),+ ArrayId::SsmZ(_) => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.ssm_z.clone())+ .expect("SSM z not initialized"),++ // MoE arrays (LLM only)+ ArrayId::MoeTopkIds => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_topk_ids.clone())+ .expect("MoE topk_ids not initialized"),+ ArrayId::MoeTopkProbs => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_topk_probs.clone())+ .expect("MoE topk_probs not initialized"),+ ArrayId::MoeOffsets => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_offsets.clone())+ .expect("MoE offsets not initialized"),+ ArrayId::MoeSumK => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_sumk.clone())+ .expect("MoE sumk not initialized"),+ ArrayId::MoeBucketedTokenIds => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_bucketed_token_ids.clone())+ .expect("MoE bucketed_token_ids not initialized"),+ ArrayId::MoeBucketedProbs => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_bucketed_probs.clone())+ .expect("MoE bucketed_probs not initialized"),+ ArrayId::MoeXPerm => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_x_perm.clone())+ .expect("MoE x_perm not initialized"),+ ArrayId::MoeTok2Row => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_tok2row.clone())+ .expect("MoE tok2row not initialized"),+ ArrayId::MoeYPartial => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_y_partial.clone())+ .expect("MoE y_partial not initialized"),+ ArrayId::MoeHidden => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_hidden.clone())+ .expect("MoE hidden not initialized"),+ ArrayId::MoeTwoPassRowExpertMap => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_two_pass_row_expert_map.clone())+ .expect("MoE two_pass_row_expert_map not initialized"),+ ArrayId::MoeTileCounts => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_tile_counts.clone())+ .expect("MoE tile_counts not initialized"),+ ArrayId::MoeTileOffsets => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_tile_offsets.clone())+ .expect("MoE tile_offsets not initialized"),+ ArrayId::MoeTileMap => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_tile_map.clone())+ .expect("MoE tile_map not initialized"),+ ArrayId::MoeTotalTiles => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_total_tiles.clone())+ .expect("MoE total_tiles not initialized"),+ ArrayId::MoeDispatchArgs => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_dispatch_args.clone())+ .expect("MoE dispatch_args not initialized"),+ ArrayId::MoeScatterPartials => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_scatter_partials.clone())+ .expect("MoE scatter_partials not initialized"),+ ArrayId::MoeScatterBlockBases => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_scatter_block_bases.clone())+ .expect("MoE scatter_block_bases not initialized"),+ ArrayId::MoeBlockAlloc => self+ .llm_aux+ .as_ref()+ .and_then(|aux| aux.moe_block_alloc.clone())+ .expect("MoE block_alloc not initialized"),++ // Classifier-specific arrays+ ArrayId::ClassifierPooling => {+ self.classifier_state().pooling.clone()+ },+ ArrayId::ClassifierPredictionHeadDense => {+ self.classifier_state().dense.clone()+ },+ ArrayId::ClassifierPredictionHeadNorm => {+ self.classifier_state().norm.clone()+ },+ ArrayId::ClassifierPredictionHeadLogits => {+ self.classifier_state().classifier_logits.clone()+ },+ }+ }++ fn shared_buffer_array(+ &self,+ id: ArrayId,+ ) -> Option<ArrayCell> {+ let shared = self.shared_buffers.borrow();+ match id {+ ArrayId::EmbeddingsInputWeights => Some(match &shared.embeddings {+ EmbeddingsBuffers::Tied {+ weights,+ } => weights.clone(),+ EmbeddingsBuffers::Untied {+ input_weights,+ ..+ } => input_weights.clone(),+ EmbeddingsBuffers::QuantizedTied {+ weights,+ ..+ } => weights.clone(),+ EmbeddingsBuffers::MLXSemiQuantizedUntied {+ input_weights,+ ..+ } => input_weights.clone(),+ }),+ ArrayId::EmbeddingsOutputWeights => {+ Some(match &shared.embeddings {+ EmbeddingsBuffers::Tied {+ weights,+ } => weights.clone(),+ EmbeddingsBuffers::Untied {+ output_weights,+ ..+ } => output_weights.clone(),+ EmbeddingsBuffers::QuantizedTied {+ weights,+ ..+ } => weights.clone(),+ EmbeddingsBuffers::MLXSemiQuantizedUntied {+ packed_output_weights,+ ..+ } => packed_output_weights.clone(),+ })+ },+ ArrayId::EmbeddingsScales => Some(match &shared.embeddings {+ EmbeddingsBuffers::QuantizedTied {+ scales,+ ..+ } => scales.clone(),+ EmbeddingsBuffers::MLXSemiQuantizedUntied {+ output_scales,+ ..+ } => output_scales.clone(),+ _ => panic!(+ "Expected QuantizedTied or MLXSemiQuantizedUntied embeddings"+ ),+ }),+ ArrayId::RopeCosines(rope_type) => Some(match rope_type {+ RopeType::Global => shared+ .global_rope+ .as_ref()+ .expect("Global rope not initialized")+ .cosines+ .clone(),+ RopeType::Local => shared+ .local_rope+ .as_ref()+ .expect("Local rope not initialized")+ .cosines+ .clone(),+ }),+ ArrayId::RopeSines(rope_type) => Some(match rope_type {+ RopeType::Global => shared+ .global_rope+ .as_ref()+ .expect("Global rope not initialized")+ .sines+ .clone(),+ RopeType::Local => shared+ .local_rope+ .as_ref()+ .expect("Local rope not initialized")+ .sines+ .clone(),+ }),+ _ => None,+ }+ }++ pub fn conv_padded_buffer(&self) -> Option<ArrayCell> {+ self.llm_aux.as_ref().and_then(|aux| aux.ssm_conv_padded.clone())+ }++ // ========================================================================+ // Public API Methods (formerly trait methods)+ // ========================================================================++ pub fn arrays(+ &self,+ ids: &[ArrayId],+ ) -> Box<[ArrayCell]> {+ ids.iter().map(|id| self.array_cell(*id)).collect()+ }++ pub fn hashmaps(+ &self,+ ids: &[HashMapId],+ ) -> Box<[HashMap<Option<usize>, ArrayCell>]> {+ ids.iter()+ .map(|id| match id {+ HashMapId::AttentionBias => self.attention_bias.clone(),+ })+ .collect()+ }++ pub fn active_suffix_length(&self) -> usize {+ match &self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => {+ state.active_suffix_length+ },+ ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,+ }+ }++ pub fn is_prefilling(&self) -> bool {+ match &self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => {+ state.is_prefilling+ },+ ForwardPassMode::Classifier(_) => true,+ }+ }++ pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers>>> {+ match &self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => {+ Some(&state.cache_layers)+ },+ ForwardPassMode::Classifier(_) => None,+ }+ }++ pub fn sampling_output(&self) -> Option<&ArrayCell> {+ match &self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => {+ state.sampling_output.as_ref()+ },+ ForwardPassMode::Classifier(_) => None,+ }+ }++ #[cfg(feature = "tracing")]+ pub fn traces(&self) -> &Rc<RefCell<ActivationTrace>> {+ match &self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => &state.traces,+ ForwardPassMode::Classifier(state) => &state.traces,+ }+ }++ pub fn sampling_method_mut(+ &mut self+ ) -> Option<&mut Option<SamplingMethod>> {+ match &mut self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => {+ Some(&mut state.sampling_method)+ },+ ForwardPassMode::Classifier(_) => None,+ }+ }++ pub fn sampling_method(&self) -> Option<SamplingMethod> {+ match &self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => {+ state.sampling_method+ },+ ForwardPassMode::Classifier(_) => None,+ }+ }++ pub fn copy_array(+ &self,+ source_array_id: ArrayId,+ destination_array: RefCell<MetalArray>,+ ) {+ destination_array+ .borrow_mut()+ .copy_from_array(&self.arrays(&[source_array_id])[0].borrow());+ }++ pub fn encode_copy_array(+ &self,+ command_buffer: &mpsgraph::CommandBuffer,+ source_array_id: ArrayId,+ destination_array: RefCell<MetalArray>,+ ) {+ let source_ref = self.arrays(&[source_array_id])[0].clone();+ let mut src_borrow = source_ref.borrow_mut();+ let mut dst_borrow = destination_array.borrow_mut();++ let src_buf = unsafe { src_borrow.mtl_buffer().clone() };+ let dst_buf = unsafe { dst_borrow.mtl_buffer().clone() };++ let copy_size_bytes = dst_borrow.size_in_bytes() as u64;+ debug_assert_eq!(+ dst_borrow.size_in_bytes(),+ src_borrow.size_in_bytes()+ );++ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let blit_encoder = mtl_command_buffer.new_blit_command_encoder();+ blit_encoder.copy_from_buffer(+ &src_buf,+ 0,+ &dst_buf,+ 0,+ copy_size_bytes,+ );+ blit_encoder.end_encoding();+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mode.rs b/crates/uzu/src/backends/metal/forward_pass/state/mode.rsnew file mode 100644index 000000000..f5d9bbcb5--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/mode.rs@@ -0,0 +1,34 @@+use std::{cell::RefCell, rc::Rc};++use super::super::cache_layers::CacheLayers;+#[cfg(feature = "tracing")]+use super::super::traces::ActivationTrace;+use crate::{backends::metal::MetalArray, session::parameter::SamplingMethod};++type ArrayCell = RefCell<MetalArray>;++pub enum ForwardPassMode {+ LanguageModelGenerator(LanguageModelGeneratorModeState),+ Classifier(ClassifierModeState),+}++pub struct LanguageModelGeneratorModeState {+ pub cache_layers: Rc<RefCell<CacheLayers>>,+ pub token_seeds: ArrayCell,+ pub logits: ArrayCell,+ pub sampling_output: Option<ArrayCell>,+ pub sampling_method: Option<SamplingMethod>,+ #[cfg(feature = "tracing")]+ pub traces: Rc<RefCell<ActivationTrace>>,+ pub active_suffix_length: usize,+ pub is_prefilling: bool,+}++pub struct ClassifierModeState {+ pub pooling: ArrayCell,+ pub dense: ArrayCell,+ pub norm: ArrayCell,+ pub classifier_logits: ArrayCell,+ #[cfg(feature = "tracing")]+ pub traces: Rc<RefCell<ActivationTrace>>,+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/rope_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/rope_buffers.rsnew file mode 100644index 000000000..469089911--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/rope_buffers.rs@@ -0,0 +1,57 @@+use std::{cell::RefCell, rc::Rc};++use super::super::ModelShape;+use crate::{+ DeviceContext,+ backends::metal::{MTLContext, MetalArray},+ parameters::ParameterTree,+};++type ArrayCell = RefCell<MetalArray>;++pub struct RopeBuffers {+ /// [rope_max_sequence_length, head_dim]+ pub cosines: ArrayCell,+ /// [rope_max_sequence_length, head_dim]+ pub sines: ArrayCell,+}++impl RopeBuffers {+ pub fn new(+ context: &MTLContext,+ model_shape: &ModelShape,+ ) -> Self {+ unsafe {+ let rotated_queries_shape = model_shape.rotated_queries_shape(1);+ let head_dim = rotated_queries_shape[2];+ let rope_max_sequence_length = model_shape.context_length();++ Self {+ cosines: RefCell::new(context.array_uninitialized(+ &[rope_max_sequence_length, head_dim],+ model_shape.activation_data_type(),+ )),+ sines: RefCell::new(context.array_uninitialized(+ &[rope_max_sequence_length, head_dim],+ model_shape.activation_data_type(),+ )),+ }+ }+ }++ pub fn update_data(+ &mut self,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ rope_name: String,+ ) {+ let Ok(rope_tree) = parameter_tree.subtree(rope_name.as_str()) else {+ return;+ };++ let cosines_view = rope_tree.leaf("cosines").unwrap();+ self.cosines.borrow_mut().copy_from_array(&cosines_view);++ let sines_view = rope_tree.leaf("sines").unwrap();+ self.sines.borrow_mut().copy_from_array(&sines_view);+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/rope_type.rs b/crates/uzu/src/backends/metal/forward_pass/state/rope_type.rsnew file mode 100644index 000000000..ea16a8877--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/rope_type.rs@@ -0,0 +1,18 @@+use serde::{Deserialize, Serialize};++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Hash,+ Serialize,+ Deserialize,+)]+pub enum RopeType {+ Global,+ Local,+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/shared_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/shared_buffers.rsnew file mode 100644index 000000000..6c9cfa81d--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/state/shared_buffers.rs@@ -0,0 +1,152 @@+use std::{cell::RefCell, rc::Rc};++use half::{bf16, f16};++use super::{super::ModelShape, EmbeddingsBuffers, RopeBuffers};+use crate::{+ Array, DataType, DeviceContext,+ backends::metal::{MTLContext, MetalArray},+ parameters::ParameterTree,+};++type ArrayCell = RefCell<MetalArray>;++pub struct MoeExpertWeights {+ pub w1: ArrayCell,+ pub w2: ArrayCell,+ pub w3: ArrayCell,+}++pub struct SharedBuffers {+ pub embeddings: EmbeddingsBuffers,+ pub global_rope: Option<RopeBuffers>,+ pub local_rope: Option<RopeBuffers>,+ pub moe_expert_weights: Option<Vec<MoeExpertWeights>>,+ pub attention_sinks: Option<Vec<ArrayCell>>,+}++impl SharedBuffers {+ pub fn new(+ context: &MTLContext,+ decoder_config: &crate::config::DecoderConfig,+ model_shape: &ModelShape,+ ) -> Self {+ let embeddings = EmbeddingsBuffers::new(+ context,+ &decoder_config.embedding_config,+ model_shape,+ );++ let global_rope = if decoder_config.global_rope_config.is_some() {+ Some(RopeBuffers::new(context, model_shape))+ } else {+ None+ };++ let local_rope = if decoder_config.local_rope_config.is_some() {+ Some(RopeBuffers::new(context, model_shape))+ } else {+ None+ };++ let moe_expert_weights = if matches!(+ decoder_config.layer_config.mlp_config,+ crate::config::MLPConfig::MixtureOfExperts(_)+ ) {+ Some(Vec::new())+ } else {+ None+ };++ let attention_sinks = if let Some(attention_config) =+ decoder_config.layer_config.attention_config()+ {+ if attention_config.has_sinks {+ let num_heads = decoder_config.num_heads;+ Some(+ (0..decoder_config.num_layers)+ .map(|_| unsafe {+ RefCell::new(context.array_uninitialized(+ &[num_heads],+ DataType::F32,+ ))+ })+ .collect(),+ )+ } else {+ None+ }+ } else {+ None+ };++ Self {+ embeddings,+ global_rope,+ local_rope,+ moe_expert_weights,+ attention_sinks,+ }+ }++ pub fn update_data(+ &mut self,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) {+ self.embeddings.update_data(parameter_tree);++ let transformer_tree = parameter_tree+ .subtree("transformer")+ .expect("transformer subtree not found");++ if let Some(global_rope) = &mut self.global_rope {+ global_rope+ .update_data(&transformer_tree, String::from("global_rope"));+ }+ if let Some(local_rope) = &mut self.local_rope {+ local_rope+ .update_data(&transformer_tree, String::from("local_rope"));+ }++ if let Some(sinks_vec) = &mut self.attention_sinks {+ for (layer_idx, sink_cell) in sinks_vec.iter_mut().enumerate() {+ let layer_tree = transformer_tree+ .subtree(&format!("layers.{}", layer_idx))+ .unwrap();+ let attn_tree = layer_tree.subtree("mixer").unwrap();+ let sinks_arr = attn_tree.leaf("sinks").unwrap();+ let mut dst = sink_cell.borrow_mut();+ let dst_slice = dst.as_slice_mut::<f32>().unwrap();++ match sinks_arr.data_type() {+ DataType::F32 => {+ let src = sinks_arr.as_slice::<f32>().unwrap();+ dst_slice.copy_from_slice(src);+ },+ DataType::BF16 => {+ let src = sinks_arr.as_slice::<bf16>().unwrap();+ for (dst_val, src_val) in+ dst_slice.iter_mut().zip(src.iter())+ {+ *dst_val = f32::from(*src_val);+ }+ },+ DataType::F16 => {+ let src = sinks_arr.as_slice::<f16>().unwrap();+ for (dst_val, src_val) in+ dst_slice.iter_mut().zip(src.iter())+ {+ *dst_val = f32::from(*src_val);+ }+ },+ other => {+ panic!(+ "Unsupported attention sink data type: {:?}",+ other+ );+ },+ }+ }+ }+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/traces.rs b/crates/uzu/src/backends/metal/forward_pass/traces.rsindex 2fa689888..9035cf2d7 100644--- a/crates/uzu/src/backends/metal/forward_pass/traces.rs+++ b/crates/uzu/src/backends/metal/forward_pass/traces.rs@@ -7,7 +7,7 @@ use crate::{type ArrayCell = RefCell<MetalArray>;-pub struct DecoderLayerActivationTrace {+pub struct LayerActivationTrace {pub inputs: ArrayCell,pub pre_attention_norm: ArrayCell,pub attention: ArrayCell,@@ -19,7 +19,7 @@ pub struct DecoderLayerActivationTrace {pub outputs: ArrayCell,}-impl DecoderLayerActivationTrace {+impl LayerActivationTrace {pub fn new(context: &MTLContext,model_shape: &ModelShape,@@ -68,21 +68,23 @@ impl DecoderLayerActivationTrace {}}-pub struct DecoderActivationTrace {- pub layer_results: Vec<Rc<RefCell<DecoderLayerActivationTrace>>>,+pub struct ActivationTrace {+ pub embedding_norm: Option<ArrayCell>,+ pub layer_results: Vec<Rc<RefCell<LayerActivationTrace>>>,pub output_norm: ArrayCell,+ pub output_pooling: Option<ArrayCell>,pub logits: ArrayCell,}-impl DecoderActivationTrace {- pub fn new(+impl ActivationTrace {+ pub fn new_llm(context: &MTLContext,model_shape: &ModelShape,suffix_length: usize,) -> Self {let layer_results = (0..model_shape.num_layers).map(|_| {- Rc::new(RefCell::new(DecoderLayerActivationTrace::new(+ Rc::new(RefCell::new(LayerActivationTrace::new(context,model_shape,suffix_length,@@ -91,11 +93,13 @@ impl DecoderActivationTrace {.collect();unsafe {Self {+ embedding_norm: None,layer_results,output_norm: RefCell::new(context.array_uninitialized(&model_shape.main_shape(suffix_length),model_shape.activation_data_type(),)),+ output_pooling: None,logits: RefCell::new(context.array_uninitialized(&model_shape.logits_shape(suffix_length),model_shape.activation_data_type(),@@ -103,4 +107,61 @@ impl DecoderActivationTrace {}}}++ pub fn new_classifier(+ context: &MTLContext,+ model_shape: &ModelShape,+ suffix_length: usize,+ num_labels: usize,+ ) -> Self {+ let layer_results = (0..model_shape.num_layers)+ .map(|_| {+ Rc::new(RefCell::new(LayerActivationTrace::new(+ context,+ model_shape,+ suffix_length,+ )))+ })+ .collect();+ let model_dim = model_shape.main_shape(1)[1];+ unsafe {+ Self {+ embedding_norm: Some(RefCell::new(+ context.array_uninitialized(+ &model_shape.main_shape(suffix_length),+ model_shape.activation_data_type(),+ ),+ )),+ layer_results,+ output_norm: RefCell::new(context.array_uninitialized(+ &model_shape.main_shape(suffix_length),+ model_shape.activation_data_type(),+ )),+ output_pooling: Some(RefCell::new(+ context.array_uninitialized(+ &[1, model_dim],+ model_shape.activation_data_type(),+ ),+ )),+ logits: RefCell::new(context.array_uninitialized(+ &[1, num_labels],+ model_shape.activation_data_type(),+ )),+ }+ }+ }+}++impl ActivationTrace {+ pub fn embedding_norm(&self) -> &ArrayCell {+ self.embedding_norm+ .as_ref()+ .expect("embedding_norm is only available for classifier traces")+ }++ pub fn output_pooling(&self) -> &ArrayCell {+ self.output_pooling+ .as_ref()+ .expect("output_pooling is only available for classifier traces")+ }}diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rsindex 1a6a1f65e..1366ca20d 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs@@ -5,20 +5,9 @@ use metal::{ComputePipelineState as MTLComputePipelineState, FunctionConstantValues,MTLDataType, MTLSize,};-use mpsgraph::CommandBuffer as MPSCommandBuffer;use thiserror::Error;-use crate::{- Array,- backends::metal::{- KernelDataType, MTLContext, MTLError,- forward_pass::{- ArrayId, ForwardPassState, HashMapId,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- metal_extensions::ComputeEncoderConditional,- },-};+use crate::backends::metal::{KernelDataType, MTLContext, MTLError};#[derive(Debug, Clone, Copy)]pub enum AttentionKernelVariant {@@ -26,7 +15,7 @@ pub enum AttentionKernelVariant {TwoPass,}-type PipelineKey = (usize, bool);+type PipelineKey = (usize, bool, bool); // (head_dim, has_sinks, is_causal)pub struct AttentionKernelPipelines {single_pass: HashMap<PipelineKey, MTLComputePipelineState>,@@ -69,6 +58,7 @@ pub struct AttentionSinglePassArguments<'a> {pub num_heads: usize,pub suffix_length: usize,pub head_dim: usize,+ pub is_causal: bool,}pub struct AttentionTwoPassArguments<'a> {@@ -94,6 +84,7 @@ pub struct AttentionTwoPassArguments<'a> {pub num_heads: usize,pub suffix_length: usize,pub head_dim: usize,+ pub is_causal: bool,}pub struct KVCacheUpdateArguments<'a> {@@ -109,12 +100,14 @@ pub struct KVCacheUpdateArguments<'a> {pub max_sequence_length: usize,}-fn make_function_constants(has_sinks_value: bool) -> FunctionConstantValues {+fn make_function_constants(+ has_sinks_value: bool,+ is_causal_value: bool,+) -> FunctionConstantValues {let function_constants = FunctionConstantValues::new();let has_mask_value = true;let query_transposed_value = false;- let do_causal_value = false;let bool_mask_value = false;let float_mask_value = true;@@ -128,11 +121,6 @@ fn make_function_constants(has_sinks_value: bool) -> FunctionConstantValues {MTLDataType::Bool,21,); // query_transposed- function_constants.set_constant_value_at_index(- &do_causal_value as *const bool as *const std::ffi::c_void,- MTLDataType::Bool,- 22,- ); // do_causalfunction_constants.set_constant_value_at_index(&bool_mask_value as *const bool as *const std::ffi::c_void,MTLDataType::Bool,@@ -143,6 +131,11 @@ fn make_function_constants(has_sinks_value: bool) -> FunctionConstantValues {MTLDataType::Bool,24,); // float_mask+ function_constants.set_constant_value_at_index(+ &is_causal_value as *const bool as *const std::ffi::c_void,+ MTLDataType::Bool,+ 22,+ ); // do_causalfunction_constants.set_constant_value_at_index(&has_sinks_value as *const bool as *const std::ffi::c_void,MTLDataType::Bool,@@ -164,58 +157,55 @@ impl AttentionKernel {let mut two_pass_1 = HashMap::new();let mut two_pass_2 = HashMap::new();- // Pre-generate all supported variants for both sink configurations+ // Pre-generate all supported variants for sinks and causal configurationsfor &has_sinks_value in &[false, true] {- let function_constants = make_function_constants(has_sinks_value);-- for &head_dim in &supported_head_dims {- let single_pass_name = format!(- "attention_single_pass_{}_{}",- data_suffix, head_dim- );- let single_pass_key =- format!("{}_sinks_{}", single_pass_name, has_sinks_value);- if let Ok((pipeline, _)) = context- .compute_pipeline_state_with_reflection_cached(- &single_pass_key,- &single_pass_name,- Some(&function_constants),- )- {- single_pass.insert((head_dim, has_sinks_value), pipeline);- }+ for &is_causal_value in &[false, true] {+ let function_constants =+ make_function_constants(has_sinks_value, is_causal_value);- let two_pass_1_name =- format!("attention_2pass_1_{}_{}", data_suffix, head_dim);- let two_pass_1_key =- format!("{}_sinks_{}", two_pass_1_name, has_sinks_value);- if let Ok((pipeline, _)) = context- .compute_pipeline_state_with_reflection_cached(- &two_pass_1_key,- &two_pass_1_name,- Some(&function_constants),- )- {- two_pass_1.insert((head_dim, has_sinks_value), pipeline);- }+ for &head_dim in &supported_head_dims {+ if let Ok((pipeline, _)) = context+ .compute_pipeline_state_with_reflection(+ &format!(+ "attention_single_pass_{}_{}",+ data_suffix, head_dim+ ),+ Some(&function_constants),+ )+ {+ single_pass.insert(+ (head_dim, has_sinks_value, is_causal_value),+ pipeline,+ );+ }- if !two_pass_2.contains_key(&head_dim) {- let two_pass_2_name = format!(- "attention_2pass_2_{}_{}",- data_suffix, head_dim- );- let two_pass_2_key = format!(- "{}_sinks_{}",- two_pass_2_name, has_sinks_value- );if let Ok((pipeline, _)) = context- .compute_pipeline_state_with_reflection_cached(- &two_pass_2_key,- &two_pass_2_name,+ .compute_pipeline_state_with_reflection(+ &format!(+ "attention_2pass_1_{}_{}",+ data_suffix, head_dim+ ),Some(&function_constants),){- two_pass_2.insert(head_dim, pipeline);+ two_pass_1.insert(+ (head_dim, has_sinks_value, is_causal_value),+ pipeline,+ );+ }++ if !two_pass_2.contains_key(&head_dim) {+ if let Ok((pipeline, _)) = context+ .compute_pipeline_state_with_reflection(+ &format!(+ "attention_2pass_2_{}_{}",+ data_suffix, head_dim+ ),+ Some(&function_constants),+ )+ {+ two_pass_2.insert(head_dim, pipeline);+ }}}}@@ -242,15 +232,17 @@ impl AttentionKernel {pub fn supports_single_pass(&self,head_dim: usize,+ is_causal: bool,) -> bool {- self.pipelines.single_pass.contains_key(&(head_dim, false))+ self.pipelines.single_pass.contains_key(&(head_dim, false, is_causal))}pub fn supports_two_pass(&self,head_dim: usize,+ is_causal: bool,) -> bool {- self.pipelines.two_pass_1.contains_key(&(head_dim, false))+ self.pipelines.two_pass_1.contains_key(&(head_dim, false, is_causal))&& self.pipelines.two_pass_2.contains_key(&head_dim)}@@ -258,8 +250,10 @@ impl AttentionKernel {&self,sequence_length: usize,head_dim: usize,+ is_causal: bool,) -> AttentionKernelVariant {- if self.supports_two_pass(head_dim) && sequence_length > 1024 {+ if self.supports_two_pass(head_dim, is_causal) && sequence_length > 1024+ {AttentionKernelVariant::TwoPass} else {AttentionKernelVariant::SinglePass@@ -275,7 +269,7 @@ impl AttentionKernel {let pipeline = self.pipelines.single_pass- .get(&(args.head_dim, has_sinks))+ .get(&(args.head_dim, has_sinks, args.is_causal)).ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;compute_encoder.set_compute_pipeline_state(pipeline);@@ -372,7 +366,7 @@ impl AttentionKernel {let pass1_pipeline = self.pipelines.two_pass_1- .get(&(args.head_dim, has_sinks))+ .get(&(args.head_dim, has_sinks, args.is_causal)).ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;let pass2_pipeline =@@ -561,298 +555,3 @@ impl AttentionKernel {Ok(())}}--pub struct AttentionKernelEncodable {- kernel: AttentionKernel,- layer_index: usize,- attention_scale: Option<f32>,- has_sinks: bool,-}--impl AttentionKernelEncodable {- pub fn new(- context: &MTLContext,- data_type: KernelDataType,- layer_index: usize,- attention_scale: Option<f32>,- has_sinks: bool,- ) -> Result<Self, AttentionError> {- let kernel = AttentionKernel::new(context, data_type)?;- Ok(Self {- kernel,- layer_index,- attention_scale,- has_sinks,- })- }-}--impl EncodableWithState for AttentionKernelEncodable {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();-- compute_encoder.condition(- parameters.predicate_ref(),- || {- self.encode_with_encoder_impl(- state,- compute_encoder,- parameters,- );- },- None::<fn()>,- );-- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,- ) {- self.encode_with_encoder_impl(state, encoder, parameters);- }-}--impl AttentionKernelEncodable {- fn encode_with_encoder_impl(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,- ) {- let (- suffix_length,- num_heads,- head_dim,- num_groups,- max_sequence_length,- ) = {- let qkv_binding = state.arrays(&[ArrayId::QKV]);- let qkv_array = qkv_binding[0].borrow();- let suffix_length = qkv_array.shape()[0];-- let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);- let queries_array = queries_binding[0].borrow();- let num_heads = queries_array.shape()[0];- let head_dim = queries_array.shape()[2];-- let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);- let keys_array = keys_binding[0].borrow();- let num_groups = keys_array.shape()[0];-- let key_cache_binding =- state.arrays(&[ArrayId::Keys(self.layer_index)]);- let key_cache_array = key_cache_binding[0].borrow();- let max_sequence_length = key_cache_array.shape()[1];-- (- suffix_length,- num_heads,- head_dim,- num_groups,- max_sequence_length,- )- };-- let (segment_prefix_length, window_length) = {- let cache = state.cache_layers.borrow();- let layer = cache.data[self.layer_index]- .as_transformer()- .expect("Attention kernel expects transformer layer state");- (- layer.projected_segment_prefix_length(- parameters.projection_step.unwrap_or(0),- ),- layer.window_length(),- )- };-- let sequence_length = segment_prefix_length + suffix_length;-- let gqa_factor = num_heads / num_groups;- let scale =- self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());-- let variant = self.kernel.choose_variant(sequence_length, head_dim);-- let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);- let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);- let qkv_binding = state.arrays(&[ArrayId::QKV]);- let key_cache_binding =- state.arrays(&[ArrayId::Keys(self.layer_index)]);- let value_cache_binding =- state.arrays(&[ArrayId::Values(self.layer_index)]);- let attention_bias_binding =- state.hashmaps(&[HashMapId::AttentionBias]);- let attention_output_binding =- state.arrays(&[ArrayId::AttentionOutput]);-- let attention_bias_map = attention_bias_binding[0].clone();- let mask_kv_seq_stride = 1;- let mask_q_seq_stride = sequence_length as i32;- let mask_head_stride = 0;- let sinks_binding = if self.has_sinks {- Some(state.arrays(&[ArrayId::AttentionSinks(self.layer_index)]))- } else {- None- };-- let mut rotated_keys_array = rotated_keys_binding[0].borrow_mut();- let rotated_keys_buffer = unsafe { rotated_keys_array.mtl_buffer() };-- let mut qkv_array = qkv_binding[0].borrow_mut();- let qkv_buffer = unsafe { qkv_array.mtl_buffer() };-- let mut key_cache_array = key_cache_binding[0].borrow_mut();- let key_cache_buffer = unsafe { key_cache_array.mtl_buffer() };-- let mut value_cache_array = value_cache_binding[0].borrow_mut();- let value_cache_buffer = unsafe { value_cache_array.mtl_buffer() };-- let mut queries_array = rotated_queries_binding[0].borrow_mut();- let queries_buffer = unsafe { queries_array.mtl_buffer() };-- let mut attention_output_array =- attention_output_binding[0].borrow_mut();- let attention_output_buffer =- unsafe { attention_output_array.mtl_buffer() };-- let attention_bias_buffer = attention_bias_map- .get(&window_length)- .map(|array| {- let mut array_ref = array.borrow_mut();- unsafe { array_ref.mtl_buffer().clone() }- })- .unwrap_or_else(|| {- panic!(- "Attention bias buffer not found for window length {:?}",- window_length- );- });-- let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);- let sums_binding = state.arrays(&[ArrayId::AttentionSums]);- let maxs_binding = state.arrays(&[ArrayId::AttentionMaxs]);-- let mut partials_array = partials_binding[0].borrow_mut();- let partials_buffer = unsafe { partials_array.mtl_buffer() };-- let mut sums_array = sums_binding[0].borrow_mut();- let sums_buffer = unsafe { sums_array.mtl_buffer() };-- let mut maxs_array = maxs_binding[0].borrow_mut();- let maxs_buffer = unsafe { maxs_array.mtl_buffer() };-- let sinks_buffer = sinks_binding.as_ref().map(|binding| unsafe {- binding[0].borrow_mut().mtl_buffer().clone()- });-- if let Err(e) = self.kernel.encode_kv_cache_update(- encoder,- KVCacheUpdateArguments {- rotated_keys_buffer: &rotated_keys_buffer,- qkv_buffer: &qkv_buffer,- key_cache_buffer: &key_cache_buffer,- value_cache_buffer: &value_cache_buffer,- num_groups,- num_heads,- head_dim,- suffix_length,- segment_prefix_length,- max_sequence_length,- },- ) {- eprintln!("Failed to encode KV cache update: {:?}", e);- return;- }-- let k_head_stride = (max_sequence_length * head_dim) as i32;- let k_seq_stride = head_dim as i32;- let v_head_stride = (max_sequence_length * head_dim) as i32;- let v_seq_stride = head_dim as i32;-- match variant {- AttentionKernelVariant::SinglePass => {- if let Err(e) = self.kernel.encode_single_pass(- encoder,- AttentionSinglePassArguments {- queries_buffer: &queries_buffer,- keys_buffer: &key_cache_buffer,- values_buffer: &value_cache_buffer,- output_buffer: &attention_output_buffer,- gqa_factor: gqa_factor as i32,- sequence_length: sequence_length as i32,- k_head_stride,- k_seq_stride,- v_head_stride,- v_seq_stride,- scale,- mask_buffer: Some(&attention_bias_buffer),- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,- sinks_buffer: sinks_buffer.as_ref(),- num_heads,- suffix_length,- head_dim,- },- ) {- eprintln!(- "Failed to encode single-pass attention: {:?}",- e- );- }- },- AttentionKernelVariant::TwoPass => {- if let Err(e) = self.kernel.encode_two_pass(- encoder,- AttentionTwoPassArguments {- queries_buffer: &queries_buffer,- keys_buffer: &key_cache_buffer,- values_buffer: &value_cache_buffer,- partials_buffer: &partials_buffer,- sums_buffer: &sums_buffer,- maxs_buffer: &maxs_buffer,- output_buffer: &attention_output_buffer,- gqa_factor: gqa_factor as i32,- sequence_length: sequence_length as i32,- k_head_stride,- k_seq_stride,- v_head_stride,- v_seq_stride,- scale,- mask_buffer: Some(&attention_bias_buffer),- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,- sinks_buffer: sinks_buffer.as_ref(),- num_heads,- suffix_length,- head_dim,- },- ) {- eprintln!("Failed to encode two-pass attention: {:?}", e);- }- },- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rsindex 14ecbf88e..499a22776 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs@@ -1,32 +1,16 @@-use std::{mem::size_of, rc::Rc};+use std::mem::size_of;use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef,ComputePipelineState as MTLComputePipelineState, MTLSize,};-use mpsgraph::CommandBuffer;-use super::super::{- MTLContext,- forward_pass::{- ArrayId, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- metal_extensions::ComputeEncoderConditional,-};+use super::super::MTLContext;use crate::{- Array, DataType, backends::metal::MTLError, config::QuantizationMode,- parameters::ParameterTree,+ DataType, backends::metal::encodable_block::QuantizedEmbeddingError,+ config::QuantizationMode,};-#[derive(Debug, thiserror::Error)]-pub enum QuantizedEmbeddingError {- #[error("Metal error: {0}")]- MetalError(#[from] MTLError),- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),-}-pub struct QuantizedEmbeddingLookupKernel {pipeline: MTLComputePipelineState,}@@ -83,7 +67,7 @@ impl QuantizedEmbeddingLookupKernel {&self,encoder: &ComputeCommandEncoderRef,args: QuantizedEmbeddingLookupArguments,- ) {+ ) -> Result<(), QuantizedEmbeddingError> {encoder.set_compute_pipeline_state(&self.pipeline);// Set buffers@@ -125,460 +109,7 @@ impl QuantizedEmbeddingLookupKernel {MTLSize::new(threadgroups, 1, 1),MTLSize::new(threads_per_threadgroup, 1, 1),);- }-}--pub struct QuantizedEmbeddingLookupKernelBlock {- kernel: QuantizedEmbeddingLookupKernel,- weights_buffer: MTLBuffer,- scales_buffer: MTLBuffer,- biases_buffer: MTLBuffer,- vocab_size: u32,- model_dim: u32,- group_size: u32,-}--impl QuantizedEmbeddingLookupKernelBlock {- pub fn new(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {- let packing_divisor = mode.packing_divisor();-- let kernel =- QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;-- // Load weights [vocab_size, model_dim/packing_divisor] as storage_type- let mut weights = match parameter_tree.leaf("weights") {- Ok(weights) => weights,- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load weights: {:?}",- e- )))- })?,- };-- if weights.data_type() != mode.storage_type() {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Expected packed weights of type {:?}, got {:?}",- mode.storage_type(),- weights.data_type()- )),- ));- }-- // Load scales [vocab_size, num_groups]- let mut scales = match parameter_tree.leaf("scales") {- Ok(scales) => scales,- Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load scales: {:?}",- e- )))- })?,- };-- // Validate shapes and types- let num_groups = (model_dim + group_size - 1) / group_size;- if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim / packing_divisor- )),- ));- }- if scales.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",- scales.shape(),- vocab_size,- num_groups- )),- ));- }- if scales.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(- scales.data_type(),- ));- }-- // Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")- let biases_buffer: MTLBuffer = match parameter_tree- .leaf("biases")- .or_else(|_| parameter_tree.leaf("output_biases"))- {- Ok(mut deq_biases) => {- if deq_biases.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",- deq_biases.shape(),- vocab_size,- num_groups- )),- ));- }- if deq_biases.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(- deq_biases.data_type(),- ));- }- unsafe { deq_biases.mtl_buffer().to_owned() }- },- Err(_) => {- let elem_size: usize = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(- QuantizedEmbeddingError::UnsupportedDataType(other),- );- },- };- let size_bytes = (vocab_size * num_groups * elem_size) as u64;- let buf = mtl_context.device.new_buffer(- size_bytes,- metal::MTLResourceOptions::StorageModeShared,- );- unsafe {- std::ptr::write_bytes(- buf.contents(),- 0,- size_bytes as usize,- );- }- buf- },- };-- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };- let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };-- Ok(Self {- kernel,- weights_buffer,- scales_buffer,- biases_buffer,- vocab_size: vocab_size as u32,- model_dim: model_dim as u32,- group_size: group_size as u32,- })- }-}--impl EncodableWithState for QuantizedEmbeddingLookupKernelBlock {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &CommandBuffer,- parameters: &EncodingParameters,- ) {- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();-- self.encode_impl(state, encoder);-- encoder.end_encoding();-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &metal::ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- self.encode_impl(state, encoder);- }-}--impl QuantizedEmbeddingLookupKernelBlock {- fn encode_impl(- &self,- state: &mut ForwardPassState,- encoder: &metal::ComputeCommandEncoderRef,- ) {- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);- let batch_size = state.active_suffix_length();- let mut token_ids_array_mut = arrays[0].borrow_mut();- let mut output_array_mut = arrays[1].borrow_mut();-- let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };- let output_buffer = unsafe { output_array_mut.mtl_buffer() };-- let args = QuantizedEmbeddingLookupArguments {- token_ids_buffer,- weights_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- biases_buffer: &self.biases_buffer,- output_buffer,- batch_size: batch_size as u32,- vocab_size: self.vocab_size,- model_dim: self.model_dim,- group_size: self.group_size,- };-- self.kernel.encode(encoder, args);- }-}--// Quantized embedding readout block - uses quantized matmul for efficiency-pub struct QuantizedEmbeddingReadoutKernelBlock {- kernel: super::super::kernel::quant_matmul::QuantizedMatmulKernel,- weights_buffer: MTLBuffer,- scales_buffer: MTLBuffer,- biases_buffer: MTLBuffer,- vocab_size: usize,- model_dim: usize,-}--impl QuantizedEmbeddingReadoutKernelBlock {- pub fn new(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {- // Load weights [vocab_size, model_dim/2] as U8- let mut weights = match parameter_tree.leaf("weights") {- Ok(weights) => weights,- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load weights: {:?}",- e- )))- })?,- };-- // Load scales [vocab_size, num_groups]- let mut scales = match parameter_tree.leaf("scales") {- Ok(scales) => scales,- Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load scales: {:?}",- e- )))- })?,- };-- // Validate shapes- let num_groups = (model_dim + group_size - 1) / group_size;- let packing_divisor = mode.packing_divisor();-- // Determine if weights are transposed by checking shape- let weights_transposed = weights.shape()[0] == vocab_size;-- if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim / packing_divisor- )),- ));- }- if scales.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",- scales.shape(),- vocab_size,- num_groups- )),- ));- }- if scales.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(- scales.data_type(),- ));- }-- // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]- let biases_buffer: MTLBuffer = match parameter_tree- .leaf("biases")- .or_else(|_| parameter_tree.leaf("output_biases"))- {- Ok(mut deq_biases) => {- if deq_biases.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",- deq_biases.shape(),- vocab_size,- num_groups- )),- ));- }- if deq_biases.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(- deq_biases.data_type(),- ));- }- unsafe { deq_biases.mtl_buffer().to_owned() }- },- Err(_) => {- // Allocate zero-initialized biases buffer- let elem_size: usize = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(- QuantizedEmbeddingError::UnsupportedDataType(other),- );- },- };- let size_bytes = (vocab_size * num_groups * elem_size) as u64;- let buf = mtl_context.device.new_buffer(- size_bytes,- metal::MTLResourceOptions::StorageModeShared,- );- unsafe {- std::ptr::write_bytes(- buf.contents(),- 0,- size_bytes as usize,- );- }- buf- },- };-- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };- let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };-- use super::super::kernel::quant_matmul::{- QuantizationType, QuantizedMatmulKernel,- };-- let kernel = QuantizedMatmulKernel::new(- mtl_context,- data_type,- group_size,- model_dim,- vocab_size,- mode,- QuantizationType::Mlx,- weights_transposed,- )- .map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to create kernel: {:?}",- e- )))- })?;-- Ok(Self {- kernel,- weights_buffer,- scales_buffer,- biases_buffer,- vocab_size,- model_dim,- })- }-}--impl EncodableWithState for QuantizedEmbeddingReadoutKernelBlock {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &CommandBuffer,- parameters: &EncodingParameters,- ) {- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();-- encoder.condition(- parameters.predicate_ref(),- || {- self.encode_impl(state, encoder);- },- None::<fn()>,- );-- encoder.end_encoding();-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &metal::ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- self.encode_impl(state, encoder);- }-}--impl QuantizedEmbeddingReadoutKernelBlock {- fn encode_impl(- &self,- state: &mut ForwardPassState,- encoder: &metal::ComputeCommandEncoderRef,- ) {- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);- let batch_size = state.active_suffix_length();- let mut input_array_mut = arrays[0].borrow_mut();- let mut output_array_mut = arrays[1].borrow_mut();-- let input_buffer = unsafe { input_array_mut.mtl_buffer() };- let output_buffer = unsafe { output_array_mut.mtl_buffer() };-- // For transposed matmul: input @ weights.T- // where weights is [vocab_size, model_dim]- use super::super::kernel::quant_matmul::{- QuantizationType, QuantizedMatmulArguments,- };-- let args = QuantizedMatmulArguments {- a_buffer: input_buffer,- b_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- zero_points_or_biases_buffer: &self.biases_buffer,- output_buffer,- batch: batch_size as i32,- input_dim: self.model_dim as i32,- output_dim: self.vocab_size as i32,- quantization_type: QuantizationType::Mlx,- };- self.kernel- .encode(encoder, args)- .expect("Failed to encode quantized embedding readout kernel");+ Ok(())}}diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalnew file mode 100644index 000000000..37321f887--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -0,0 +1,164 @@+#include <metal_stdlib>+#include "../definitions.metal"++using namespace metal;++#define BLOCK_SIZE 1024+#define SIMD_SIZE 32+#define GRAIN_SIZE 4++// LayerNorm: subtract mean, then normalize (for BERT-style models)+template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>+void layer_norm_core(+ const device InputT* input_data,+ const device ScaleT* scales_data,+ device OutputT* output_data,+ uint element_count,+ constant float& epsilon,+ constant float& scale_offset,+ threadgroup AccumT* shared_mean,+ threadgroup AccumT* shared_variance,+ uint thread_in_row,+ bool full_layer+) {+ AccumT partial_sum = static_cast<AccumT>(0.0f);++ // Compute mean: sum all elements+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;+ base_i += BLOCK_SIZE * GRAIN_SIZE) {+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ if (i < element_count) {+ partial_sum += static_cast<AccumT>(input_data[i]);+ }+ }+ }++ // Reduce to get total mean+ AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(+ partial_sum,+ shared_mean,+ thread_in_row+ );+ AccumT mean = total_sum / static_cast<AccumT>(element_count);++ // Compute variance: sum of squared deviations+ AccumT partial_var_sum = static_cast<AccumT>(0.0f);+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;+ base_i += BLOCK_SIZE * GRAIN_SIZE) {+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ if (i < element_count) {+ AccumT centered = static_cast<AccumT>(input_data[i]) - mean;+ partial_var_sum += centered * centered;+ }+ }+ }++ // Reduce to get total variance+ AccumT total_var = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(+ partial_var_sum,+ shared_variance,+ thread_in_row+ );+ AccumT variance = total_var / static_cast<AccumT>(element_count);+ AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));++ // Apply normalization and scaling+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;+ base_i += BLOCK_SIZE * GRAIN_SIZE) {+ AccumT vals[GRAIN_SIZE];+ AccumT scaled_vals[GRAIN_SIZE];++ // Load and center+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ vals[j] = (i < element_count)+ ? (static_cast<AccumT>(input_data[i]) - mean)+ : 0.0f;+ }++ // Normalize and scale+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ if (i >= element_count) {+ continue;+ }++ AccumT normalized_high = vals[j] * inv_std;++ if (full_layer) {+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) ++ static_cast<AccumT>(scale_offset);+ scaled_vals[j] = normalized_high * scale_value_high;+ } else {+ OutputT normalized_low = static_cast<OutputT>(normalized_high);+ OutputT scale_value_low = static_cast<OutputT>(+ static_cast<AccumT>(scales_data[i]) ++ static_cast<AccumT>(scale_offset)+ );+ OutputT product_low = normalized_low * scale_value_low;+ scaled_vals[j] = static_cast<AccumT>(product_low);+ }+ }++ // Store+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ if (i < element_count) {+ output_data[i] = static_cast<OutputT>(scaled_vals[j]);+ }+ }+ }+}++// X-macro table defining all supported type combinations (same as RMSNorm)+#define FOREACH_LAYER_NORM_COMBO(_) \+ /* bfloat combinations most commonly used */ \+ _(bfloat, bfloat, bfloat, float, false, bf16_bf16_bf16_f32_norm) \+ _(bfloat, bfloat, bfloat, float, true, bf16_bf16_bf16_f32_full) \+ _(bfloat, float, bfloat, float, false, bf16_f32_bf16_f32_norm) \+ _(bfloat, float, bfloat, float, true, bf16_f32_bf16_f32_full) \+ /* float combinations */ \+ _(float, float, float, float, false, f32_f32_f32_f32_norm) \+ _(float, float, float, float, true, f32_f32_f32_f32_full) \+ /* half combinations */ \+ _(half, half, half, float, false, f16_f16_f16_f32_norm) \+ _(half, half, half, float, true, f16_f16_f16_f32_full)++// Generate LayerNorm kernels+#define DEFINE_LAYER_NORM_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \+ kernel void layer_norm_##SUF( \+ const device IN* input [[buffer(0)]], \+ const device SC* scales [[buffer(1)]], \+ device OUT* output [[buffer(2)]], \+ constant uint& batch_size [[buffer(3)]], \+ constant uint& model_dim [[buffer(4)]], \+ constant float& epsilon [[buffer(5)]], \+ constant float& scale_offset [[buffer(6)]], \+ uint batch_idx [[threadgroup_position_in_grid]], \+ uint thread_in_row [[thread_position_in_threadgroup]] \+ ) { \+ if (batch_idx >= batch_size) \+ return; \+ \+ threadgroup ACC shared_mean[SIMD_SIZE]; \+ threadgroup ACC shared_variance[SIMD_SIZE]; \+ const uint input_offset = batch_idx * model_dim; \+ \+ layer_norm_core<IN, SC, OUT, ACC>( \+ input + input_offset, \+ scales, \+ output + input_offset, \+ model_dim, \+ epsilon, \+ scale_offset, \+ shared_mean, \+ shared_variance, \+ thread_in_row, \+ FULL_LAYER \+ ); \+ }++FOREACH_LAYER_NORM_COMBO(DEFINE_LAYER_NORM_KERNEL)+#undef DEFINE_LAYER_NORM_KERNELdiff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/layer_norm/mod.rsnew file mode 100644index 000000000..0c9be22d3--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/mod.rs@@ -0,0 +1,157 @@+use std::mem::size_of;++use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState, MTLSize,+};++use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError},+};++#[derive(Debug)]+pub struct LayerNormArguments<'a> {+ pub input_buffer: &'a MTLBuffer,+ pub scales_buffer: &'a MTLBuffer,+ pub output_buffer: &'a MTLBuffer,+ pub batch_size: i32,+ pub model_dim: i32,+ pub epsilon: f32,+ pub scale_offset: f32,+}++#[derive(Debug, thiserror::Error)]+pub enum LayerNormError {+ #[error("Metal error: {0}")]+ MetalError(#[from] MTLError),+ #[error(+ "Unsupported data type combination: input={input:?}, scales={scales:?}, output={output:?}, accumulation={accumulation:?}"+ )]+ UnsupportedDataType {+ input: DataType,+ scales: DataType,+ output: DataType,+ accumulation: DataType,+ },+}++pub struct LayerNormKernel {+ pipeline: MTLComputePipelineState,+}++impl LayerNormKernel {+ pub fn new(+ context: &MTLContext,+ input_data_type: DataType,+ scales_data_type: DataType,+ output_data_type: DataType,+ accumulation_data_type: DataType,+ full_layer: bool,+ ) -> Result<Self, LayerNormError> {+ let kernel_name = Self::kernel_name(+ input_data_type,+ scales_data_type,+ output_data_type,+ accumulation_data_type,+ full_layer,+ )?;++ let pipeline = context+ .compute_pipeline_state(&kernel_name, None)+ .map_err(LayerNormError::MetalError)?;++ Ok(Self {+ pipeline,+ })+ }++ fn kernel_name(+ input: DataType,+ scales: DataType,+ output: DataType,+ accumulation: DataType,+ full_layer: bool,+ ) -> Result<String, LayerNormError> {+ let input_str = Self::type_to_string(input)?;+ let scales_str = Self::type_to_string(scales)?;+ let output_str = Self::type_to_string(output)?;+ let accum_str = Self::type_to_string(accumulation)?;+ let mode_str = if full_layer {+ "full"+ } else {+ "norm"+ };++ Ok(format!(+ "layer_norm_{}_{}_{}_{}_{}",+ input_str, scales_str, output_str, accum_str, mode_str+ ))+ }++ fn type_to_string(+ data_type: DataType+ ) -> Result<&'static str, LayerNormError> {+ match data_type {+ DataType::F32 => Ok("f32"),+ DataType::F16 => Ok("f16"),+ DataType::BF16 => Ok("bf16"),+ _ => Err(LayerNormError::UnsupportedDataType {+ input: data_type,+ scales: data_type,+ output: data_type,+ accumulation: data_type,+ }),+ }+ }++ pub fn encode(+ &self,+ compute_encoder: &ComputeCommandEncoderRef,+ args: LayerNormArguments,+ ) -> Result<(), LayerNormError> {+ compute_encoder.set_compute_pipeline_state(&self.pipeline);+ compute_encoder.set_buffer(0, Some(args.input_buffer), 0);+ compute_encoder.set_buffer(1, Some(args.scales_buffer), 0);+ compute_encoder.set_buffer(2, Some(args.output_buffer), 0);+ compute_encoder.set_bytes(+ 3,+ size_of::<i32>() as u64,+ &args.batch_size as *const i32 as *const _,+ );+ compute_encoder.set_bytes(+ 4,+ size_of::<i32>() as u64,+ &args.model_dim as *const i32 as *const _,+ );+ compute_encoder.set_bytes(+ 5,+ size_of::<f32>() as u64,+ &args.epsilon as *const f32 as *const _,+ );+ compute_encoder.set_bytes(+ 6,+ size_of::<f32>() as u64,+ &args.scale_offset as *const f32 as *const _,+ );++ let threadgroups_per_grid = MTLSize {+ width: args.batch_size as u64,+ height: 1,+ depth: 1,+ };++ let threads_per_threadgroup = MTLSize {+ width: 1024,+ height: 1,+ depth: 1,+ };++ compute_encoder.dispatch_thread_groups(+ threadgroups_per_grid,+ threads_per_threadgroup,+ );++ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rsindex 769fd723f..bae2543ef 100644--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs@@ -10,10 +10,7 @@ use mpsgraph::CommandBuffer as MPSCommandBuffer;use crate::backends::metal::{MTLContext,error::MTLError,- forward_pass::{- ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- },+ forward_pass::{EncodableBlock, EncodingParameters, ForwardPassState},image::Image,metal_extensions::ComputeEncoderDispatch,};@@ -85,7 +82,7 @@ impl ScalePadNormalizeImage {}}-impl EncodableWithState for ScalePadNormalizeImage {+impl EncodableBlock for ScalePadNormalizeImage {fn encode(&self,_state: &mut ForwardPassState,diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rsindex 1d4e3e541..400354529 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs@@ -2,20 +2,10 @@ use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,MTLSize,};-use mpsgraph::CommandBuffer as MPSCommandBuffer;use crate::{DataType,- array::Array,- backends::metal::{- MTLContext, MTLError,- forward_pass::{- ArrayId, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- kernel::QuantizedLinearKernelBlock,- metal_extensions::ComputeEncoderConditional,- },+ backends::metal::{MTLContext, MTLError},config::Activation,};@@ -50,7 +40,7 @@ impl MlpGateActMulEncodable {fused_up: &MTLBuffer,hidden: &MTLBuffer,m: i32,- ) {+ ) -> Result<(), MTLError> {self.kernel.encode(encoder,&self.activation,@@ -58,14 +48,12 @@ impl MlpGateActMulEncodable {hidden,m,self.hidden_dim as i32,- );+ )}}impl MlpGateActMulKernel {- fn kernel_name_for_type(- data_type: DataType- ) -> Result<&'static str, MTLError> {+ fn kernel_name_for_type(data_type: DataType) -> Result<&'static str, MTLError> {match data_type {DataType::F16 => Ok("mlp_activation_mul_f16"),DataType::F32 => Ok("mlp_activation_mul_f32"),@@ -77,22 +65,15 @@ impl MlpGateActMulKernel {}}- pub fn new(- context: &MTLContext,- data_type: DataType,- ) -> Result<Self, MTLError> {+ pub fn new(context: &MTLContext, data_type: DataType) -> Result<Self, MTLError> {let fn_name = Self::kernel_name_for_type(data_type)?;let pipeline = context.compute_pipeline_state(fn_name, None)?;- Ok(Self {- pipeline,- })+ Ok(Self { pipeline })}fn act_code(act: &Activation) -> u16 {match act {- Activation::SiLU {- ..- } => 0,+ Activation::SiLU { .. } => 0,Activation::Gelu => 1,Activation::Identity => {panic!("Identity activation is not supported for MLP kernels")@@ -108,7 +89,7 @@ impl MlpGateActMulKernel {hidden_buffer: &MTLBuffer,m: i32,h: i32,- ) {+ ) -> Result<(), MTLError> {let act_code = Self::act_code(activation);let threads_per_tg = MTLSize::new(64, 1, 1);encoder.set_compute_pipeline_state(&self.pipeline);@@ -132,106 +113,6 @@ impl MlpGateActMulKernel {let grid = MTLSize::new(h as u64, m as u64, 1);encoder.dispatch_threads(grid, threads_per_tg);- }-}--pub struct MlpBlockEncodable {- up: QuantizedLinearKernelBlock,- gate: MlpGateActMulEncodable,- down: QuantizedLinearKernelBlock,-}--impl MlpBlockEncodable {- pub fn new(- up: QuantizedLinearKernelBlock,- gate: MlpGateActMulEncodable,- down: QuantizedLinearKernelBlock,- ) -> Self {- Self {- up,- gate,- down,- }- }-}--impl EncodableWithState for MlpBlockEncodable {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- params: &EncodingParameters,- ) {- // Up- self.up.encode(state, command_buffer, params);- // Gate act+mul (fused_up -> hidden)- let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);- let mut fused = arrays[0].borrow_mut();- let mut hidden = arrays[1].borrow_mut();- let m = fused.shape()[0] as i32;- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let encoder = mtl_command_buffer.new_compute_command_encoder();-- let fused_buf = unsafe { fused.mtl_buffer() };- let hidden_buf = unsafe { hidden.mtl_buffer() };-- encoder.condition(- params.predicate_ref(),- || {- self.gate.encode(encoder, fused_buf, hidden_buf, m);- },- None::<fn()>,- );-- encoder.end_encoding();- drop(fused);- drop(hidden);- // Down- self.down.encode(state, command_buffer, params);-- if params.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,- ) {- self.encode_with_encoder_impl(state, encoder, parameters);- }-}--impl MlpBlockEncodable {- fn encode_with_encoder_impl(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- params: &EncodingParameters,- ) {- // Up- self.up.encode_with_shared_encoder(state, encoder, params);- // Gate act+mul (fused_up -> hidden)- let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);- let mut fused = arrays[0].borrow_mut();- let mut hidden = arrays[1].borrow_mut();- let m = fused.shape()[0] as i32;- let fused_buf = unsafe { fused.mtl_buffer() };- let hidden_buf = unsafe { hidden.mtl_buffer() };- self.gate.encode(encoder, fused_buf, hidden_buf, m);- drop(fused);- drop(hidden);- // Down- self.down.encode_with_shared_encoder(state, encoder, params);+ Ok(())}}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 5911a90b4..753dc00c6 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -2,51 +2,59 @@ pub mod attention;mod data_type;pub mod embedding;pub mod kv_cache_update;-pub mod linear;+pub mod layer_norm;pub mod media_kernels;pub mod mlp;pub mod moe;+pub mod pooling;pub mod quant_matmul;pub mod rms_norm;pub mod rope;pub mod sampling;+pub mod sigmoid;pub mod ssm;mod tensor_add_bias;mod tensor_add_swap;mod tensor_copy;pub mod token_copy;-pub use attention::{AttentionKernel, AttentionKernelEncodable};+pub use attention::{+ AttentionError, AttentionKernel, AttentionKernelVariant,+ AttentionSinglePassArguments, AttentionTwoPassArguments,+ KVCacheUpdateArguments,+};pub use data_type::KernelDataType;pub use kv_cache_update::KVCacheUpdate;+pub use layer_norm::{LayerNormArguments, LayerNormError, LayerNormKernel};pub use moe::{- MoeBlockBasesArguments, MoeBlockEncodable, MoeCountsOffsetsFusedArguments,+ MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,MoeCountsOffsetsFusedError, MoeCountsOffsetsFusedKernel,MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,MoeExpertsTwoPassDecodeKernel, MoeExpertsTwoPassPrefillKernel,MoeFinalizeArguments, MoeFinalizeError, MoeFinalizeKernel,- MoeScatterArguments, MoeScatterError, MoeScatterKernels,- MoeScatterWithMapArguments,+ MoeGatherArguments, MoeGatherKernel, MoeRouterTopKArguments,+ MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError,+ MoeScatterKernels, MoeScatterWithMapArguments,};+pub use pooling::PoolingKernel;pub use rms_norm::{- QKNormArguments, QKNormKernelEncodable, RMSNormArguments, RMSNormKernel,- RMSNormKernelEncodable, RMSNormKernelType,+ QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormError,+ RMSNormKernel, RMSNormKernelType,};-pub use rope::{RopeKernel, RopeKernelEncodable};-pub use sampling::{SamplingKernel, SamplingKernelEncodable};-pub(crate) use ssm::MambaMixerEncodable;+pub use rope::{RopeError, RopeKernel, RopeKernelArguments};+pub use sampling::{ArgmaxStrategy, SamplingError, SamplingKernel};+pub use sigmoid::SigmoidKernel;pub use ssm::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,SSDUpdateKernel, SSMKernelError, SplitInProjArguments, SplitInProjKernel,};pub use tensor_add_bias::TensorAddBias;-pub use tensor_add_swap::TensorAddSwap;-pub use tensor_copy::TensorCopy;+pub use tensor_add_swap::TensorAddSwapKernel;+pub use tensor_copy::TensorCopyKernel;pub use token_copy::TokenCopyKernel;use super::{MTLContext, metal_extensions};-pub use crate::backends::metal::kernel::linear::QuantizedLinearKernelBlock;pub mod media {pub use super::media_kernels::{diff --git a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs b/crates/uzu/src/backends/metal/kernel/moe/encodable.rsdeleted file mode 100644index ff59b0e77..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs+++ /dev/null@@ -1,833 +0,0 @@-//! MoE Block Encodable with path selection:-//! - suffix == 1: Single-token fused decode (no scatter/gather)-//! - suffix <= 32: Indirect decode path (GEMV-based)-//! - suffix > 32: Indirect prefill path (GEMM/MMA-based)--use std::{cell::RefCell, rc::Rc};--use metal::CommandBufferRef;-use mpsgraph::CommandBuffer as MPSCommandBuffer;--use super::{- MoeCountsOffsetsFusedArguments, MoeCountsOffsetsFusedKernel,- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernel,- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernel,- MoeExpertsTwoPassPrefillKernel, MoeFinalizeArguments, MoeFinalizeKernel,- MoeGatherArguments, MoeGatherKernel, MoeRouterTopKArguments,- MoeRouterTopKKernel, MoeScatterKernels, MoeScatterWithMapArguments,-};-use crate::{- DataType,- backends::metal::{- KernelDataType, MTLContext, MetalArray,- forward_pass::{- ArrayId, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- },- config::{- Activation, LinearConfig, MixtureOfExpertsConfig,- mlp::RoutingFunctionConfig,- },- parameters::ParameterTree,-};--/// Threshold for decode vs prefill path selection-const INDIRECT_DECODE_THRESHOLD: usize = 32;--enum RouterBlock {- Metal {- weights_buf: metal::Buffer,- biases_buf: metal::Buffer,- },-}--#[derive(Clone)]-pub struct SharedMoeWeights {- pub w13_buf: Rc<metal::Buffer>,- pub w2_buf: Rc<metal::Buffer>,- pub up_biases_buf: Rc<metal::Buffer>,- pub down_biases_buf: Rc<metal::Buffer>,-}--/// Buffers used by MoE encoding paths-struct MoeBuffers {- main: metal::Buffer,- topk_ids: metal::Buffer,- topk_probs: metal::Buffer,- offsets: metal::Buffer,- sumk: metal::Buffer,- bucketed_ids: metal::Buffer,- bucketed_probs: metal::Buffer,- x_perm: metal::Buffer,- tok2row: metal::Buffer,- y_partial: metal::Buffer,- tile_counts: metal::Buffer,- tile_offsets: metal::Buffer,- tile_map: metal::Buffer,- total_tiles: metal::Buffer,- dispatch_args: metal::Buffer,- partials: metal::Buffer,- block_bases: metal::Buffer,- block_alloc: metal::Buffer,- hidden: metal::Buffer,- row_expert_map: metal::Buffer,-}--pub struct MoeBlockEncodable {- // Router- router: RouterBlock,- router_data_type: KernelDataType,- router_renorm: bool,- router_topk_kernel: MoeRouterTopKKernel,-- // Indirect path shared components- counts_offsets_kernel: MoeCountsOffsetsFusedKernel,- scatter_kernels: MoeScatterKernels,- gather_kernel: MoeGatherKernel,- finalize_kernel: MoeFinalizeKernel,-- // Indirect path variants- experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeKernel,- experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillKernel,-- // Single-token decode path- single_decode_kernel: MoeExpertsSingleDecodeKernel,-- // Config and weights- moe_config: MixtureOfExpertsConfig,- model_dim: usize,- hidden_dim: usize,- data_type: KernelDataType,- shared_weights: SharedMoeWeights,-}--impl MoeBlockEncodable {- pub fn new(- context: &MTLContext,- moe_config: &MixtureOfExpertsConfig,- model_dim: usize,- hidden_dim: usize,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, crate::backends::metal::MTLError> {- let activation_data_type: DataType = moe_config- .expert_config- .linear_config- .activation_precision()- .into();- let data_type: KernelDataType = activation_data_type.into();-- let router_data_type: DataType =- moe_config.router_config.activation_precision().into();- let router_kernel_data_type: KernelDataType = router_data_type.into();-- let router_renorm = matches!(- moe_config.routing_function,- RoutingFunctionConfig::SoftmaxRouting- );-- let router_tree = parameter_tree.subtree("router").map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Router subtree error: {:?}",- e- ))- })?;-- let router = match &moe_config.router_config {- LinearConfig::Quantized(_) => {- unimplemented!(- "Quantized router with fused router+topk not yet supported"- );- },- LinearConfig::MLXQuantized(_) => {- unimplemented!(- "MLX quantized router with fused router+topk not yet supported"- );- },- LinearConfig::FullPrecision {- ..- } => {- let mut weights_arr =- router_tree.leaf("weights").map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Router weights error: {:?}",- e- ))- })?;- let weights_buf = unsafe { weights_arr.mtl_buffer().clone() };-- let mut biases_arr =- router_tree.leaf("biases").map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Router biases error: {:?}",- e- ))- })?;- let biases_buf = unsafe { biases_arr.mtl_buffer().clone() };-- RouterBlock::Metal {- weights_buf,- biases_buf,- }- },- LinearConfig::QLoRA {- ..- } => {- unimplemented!("QLoRA router not yet supported for MoE");- },- };-- // Initialize kernels- let router_topk_kernel =- MoeRouterTopKKernel::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "RouterTopK fused kernel error: {:?}",- e- ))- })?;- let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context)- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Counts+offsets kernel error: {:?}",- e- ))- })?;- let scatter_kernels = MoeScatterKernels::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Scatter kernels error: {:?}",- e- ))- })?;- let gather_kernel = MoeGatherKernel::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Gather kernel error: {:?}",- e- ))- })?;- let experts_two_pass_decode_kernel =- MoeExpertsTwoPassDecodeKernel::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Experts two-pass decode kernel error: {:?}",- e- ))- })?;- let experts_two_pass_prefill_kernel =- MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Experts two-pass prefill kernel error: {:?}",- e- ))- })?;- let single_decode_kernel = MoeExpertsSingleDecodeKernel::new(context)- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Simple decode fused kernel error: {:?}",- e- ))- })?;- let finalize_kernel = MoeFinalizeKernel::new(context).map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "Finalize kernel error: {:?}",- e- ))- })?;-- // Load expert weights- let experts_tree = parameter_tree.subtree("experts").map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "experts subtree error: {:?}",- e- ))- })?;-- let mut w13_arr = experts_tree- .subtree("up_projection")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "up_projection error: {:?}",- e- ))- })?- .leaf("weights")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "up weights error: {:?}",- e- ))- })?;- let w13_buf = unsafe { w13_arr.mtl_buffer().clone() };-- let mut w2_arr = experts_tree- .subtree("down_projection")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "down_projection error: {:?}",- e- ))- })?- .leaf("weights")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "down weights error: {:?}",- e- ))- })?;- let w2_buf = unsafe { w2_arr.mtl_buffer().clone() };-- let mut up_biases_arr = experts_tree- .subtree("up_projection")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "up_projection biases error: {:?}",- e- ))- })?- .leaf("biases")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "up biases error: {:?}",- e- ))- })?;- let up_biases_buf = unsafe { up_biases_arr.mtl_buffer().clone() };-- let mut down_biases_arr = experts_tree- .subtree("down_projection")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "down_projection biases error: {:?}",- e- ))- })?- .leaf("biases")- .map_err(|e| {- crate::backends::metal::MTLError::Generic(format!(- "down biases error: {:?}",- e- ))- })?;- let down_biases_buf = unsafe { down_biases_arr.mtl_buffer().clone() };-- let shared_weights = SharedMoeWeights {- w13_buf: Rc::new(w13_buf),- w2_buf: Rc::new(w2_buf),- up_biases_buf: Rc::new(up_biases_buf),- down_biases_buf: Rc::new(down_biases_buf),- };-- Ok(Self {- router,- router_data_type: router_kernel_data_type,- router_renorm,- router_topk_kernel,- counts_offsets_kernel,- scatter_kernels,- gather_kernel,- experts_two_pass_decode_kernel,- experts_two_pass_prefill_kernel,- single_decode_kernel,- finalize_kernel,- moe_config: moe_config.clone(),- model_dim,- hidden_dim,- data_type,- shared_weights,- })- }-- fn gating_code_from_activation(activation: &Activation) -> u32 {- match activation {- Activation::Gelu => 3, // GEGLU- Activation::SiLU {- ..- } => 2, // SwiGLU- Activation::Identity => {- panic!("Identity activation is not supported for MoE kernels")- },- }- }-- fn dtype_size(&self) -> usize {- match self.data_type {- KernelDataType::BFloat16 | KernelDataType::Float16 => 2,- KernelDataType::Float32 => 4,- }- }-- fn encode_router(- &self,- root: &CommandBufferRef,- buffers: &MoeBuffers,- suffix_length: usize,- ) {- let e = self.moe_config.mixture_size;- let k = self.moe_config.num_experts_per_token;-- match &self.router {- RouterBlock::Metal {- weights_buf,- biases_buf,- } => {- self.router_topk_kernel- .encode(- root,- self.router_data_type,- MoeRouterTopKArguments {- input_buffer: &buffers.main,- weight_buffer: weights_buf,- bias_buffer: biases_buf,- topk_ids_buffer: &buffers.topk_ids,- topk_probs_buffer: &buffers.topk_probs,- t: suffix_length,- d_model: self.model_dim,- e,- k,- renorm: self.router_renorm,- },- )- .expect("MoE router+topk failed");- },- }- }-- fn encode_indirect_setup(- &self,- root: &CommandBufferRef,- buffers: &MoeBuffers,- suffix_length: usize,- ) {- let e = self.moe_config.mixture_size;- let k = self.moe_config.num_experts_per_token;-- // Counts + offsets- self.counts_offsets_kernel- .encode(- root,- MoeCountsOffsetsFusedArguments {- topk_ids_buffer: &buffers.topk_ids,- offsets_buffer: &buffers.offsets,- sum_k_buffer: &buffers.sumk,- partials_buffer: &buffers.partials,- t: suffix_length,- e,- k,- },- )- .expect("MoE counts+offsets failed");-- let num_blocks = ((suffix_length + 255) / 256).max(1);- let num_tiles = ((e + 512 - 1) / 512).max(1);-- // Block bases- self.scatter_kernels- .encode_block_bases(- root,- super::MoeBlockBasesArguments {- partials_buffer: &buffers.partials,- block_bases_buffer: &buffers.block_bases,- block_alloc_buffer: &buffers.block_alloc,- e,- num_blocks,- num_tiles,- },- )- .expect("MoE block bases failed");-- // Scatter with tok2row map- self.scatter_kernels- .encode_scatter_with_map(- root,- MoeScatterWithMapArguments {- base: super::MoeScatterArguments {- topk_ids_buffer: &buffers.topk_ids,- topk_probs_buffer: &buffers.topk_probs,- offsets_buffer: &buffers.offsets,- block_bases_buffer: &buffers.block_bases,- block_alloc_buffer: &buffers.block_alloc,- out_ids_buffer: &buffers.bucketed_ids,- out_probs_buffer: &buffers.bucketed_probs,- t: suffix_length,- e,- k,- num_blocks,- num_tiles,- },- tok2row_buffer: &buffers.tok2row,- },- self.data_type,- )- .expect("MoE scatter failed");-- // Gather- self.gather_kernel- .encode(- root,- self.data_type,- MoeGatherArguments {- x_buffer: &buffers.main,- bucketed_ids_buffer: &buffers.bucketed_ids,- x_perm_buffer: &buffers.x_perm,- sumk_buffer: &buffers.sumk,- t: suffix_length,- k,- d_model: self.model_dim,- },- )- .expect("MoE gather failed");- }-- fn clear_indirect_buffers(- &self,- root: &CommandBufferRef,- buffers: &MoeBuffers,- suffix_length: usize,- ) {- let k = self.moe_config.num_experts_per_token;- if k == 0 {- return;- }-- let blit_encoder = root.new_blit_command_encoder();- let entries = suffix_length * k;- let topk_bytes = entries * std::mem::size_of::<u32>();- let tok2row_bytes = entries * std::mem::size_of::<i32>();- let dtype_size = self.dtype_size();-- if topk_bytes > 0 {- blit_encoder.fill_buffer(- &buffers.topk_ids,- metal::NSRange::new(0, topk_bytes as u64),- 0xFF,- );- }- if tok2row_bytes > 0 {- blit_encoder.fill_buffer(- &buffers.tok2row,- metal::NSRange::new(0, tok2row_bytes as u64),- 0xFF,- );- }-- let hidden_bytes =- (suffix_length * k * self.hidden_dim * dtype_size) as u64;- if hidden_bytes > 0 {- blit_encoder.fill_buffer(- &buffers.hidden,- metal::NSRange::new(0, hidden_bytes),- 0,- );- }-- let y_partial_bytes =- (suffix_length * k * self.model_dim * dtype_size) as u64;- if y_partial_bytes > 0 {- blit_encoder.fill_buffer(- &buffers.y_partial,- metal::NSRange::new(0, y_partial_bytes),- 0,- );- }-- let x_perm_bytes =- (suffix_length * k * self.model_dim * dtype_size) as u64;- if x_perm_bytes > 0 {- blit_encoder.fill_buffer(- &buffers.x_perm,- metal::NSRange::new(0, x_perm_bytes),- 0,- );- }-- blit_encoder.end_encoding();- }-- fn encode_single_token_path(- &self,- root: &CommandBufferRef,- buffers: &MoeBuffers,- ) {- let k = self.moe_config.num_experts_per_token;- let gating_code = Self::gating_code_from_activation(- &self.moe_config.expert_config.activation,- );- let silu_alpha = self.moe_config.expert_config.activation.alpha();- let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]- .unwrap_or(f32::NEG_INFINITY);- let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]- .unwrap_or(f32::INFINITY);- let up_clip_min = self.moe_config.expert_config.up_clipping[0];- let up_clip_max = self.moe_config.expert_config.up_clipping[1];-- self.single_decode_kernel- .encode(- root,- MoeExpertsSingleDecodeArguments {- x: &buffers.main,- topk_ids: &buffers.topk_ids,- topk_probs: &buffers.topk_probs,- w13_all: &self.shared_weights.w13_buf,- w2_all: &self.shared_weights.w2_buf,- up_biases: &self.shared_weights.up_biases_buf,- down_biases: &self.shared_weights.down_biases_buf,- hidden: &buffers.hidden,- y: &buffers.main,- d_model: self.model_dim,- d_ff: self.hidden_dim,- k,- gating_code,- silu_alpha,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- data_type: self.data_type,- },- )- .expect("MoE single-token fused decode failed");- }-- fn encode_indirect_decode_path(- &self,- root: &CommandBufferRef,- buffers: &MoeBuffers,- suffix_length: usize,- ) {- let e = self.moe_config.mixture_size;- let k = self.moe_config.num_experts_per_token;- let gating_code = Self::gating_code_from_activation(- &self.moe_config.expert_config.activation,- );-- let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]- .unwrap_or(f32::NEG_INFINITY);- let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]- .unwrap_or(f32::INFINITY);- let up_clip_min = self.moe_config.expert_config.up_clipping[0];- let up_clip_max = self.moe_config.expert_config.up_clipping[1];- let silu_alpha = self.moe_config.expert_config.activation.alpha();-- let total_rows = suffix_length * k;- const K_TILE: usize = 64;- let num_tiles_k = ((self.hidden_dim + K_TILE - 1) / K_TILE) as u32;-- self.experts_two_pass_decode_kernel- .encode(- root,- MoeExpertsTwoPassArguments {- x_perm_buffer: &buffers.x_perm,- expert_offsets: &buffers.offsets,- row_expert_map: &buffers.row_expert_map,- hidden_buffer: &buffers.hidden,- output_buffer: &buffers.y_partial,- w13_all: &self.shared_weights.w13_buf,- w2_all: &self.shared_weights.w2_buf,- up_biases: &self.shared_weights.up_biases_buf,- down_biases: &self.shared_weights.down_biases_buf,- tile_counts: &buffers.tile_counts,- tile_offsets: &buffers.tile_offsets,- tile_map: &buffers.tile_map,- total_tiles: &buffers.total_tiles,- dispatch_args: &buffers.dispatch_args,- total_rows,- d_model: self.model_dim,- d_ff: self.hidden_dim,- e,- num_tiles_k,- gating_code,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- silu_alpha,- data_type: self.data_type,- },- )- .expect("MoE indirect decode failed");-- self.encode_finalize(root, buffers, suffix_length);- }-- fn encode_indirect_prefill_path(- &self,- root: &CommandBufferRef,- buffers: &MoeBuffers,- suffix_length: usize,- ) {- let e = self.moe_config.mixture_size;- let k = self.moe_config.num_experts_per_token;- let gating_code = Self::gating_code_from_activation(- &self.moe_config.expert_config.activation,- );-- let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]- .unwrap_or(f32::NEG_INFINITY);- let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]- .unwrap_or(f32::INFINITY);- let up_clip_min = self.moe_config.expert_config.up_clipping[0];- let up_clip_max = self.moe_config.expert_config.up_clipping[1];- let silu_alpha = self.moe_config.expert_config.activation.alpha();-- let total_rows = suffix_length * k;- const K_TILE: usize = 128;- let num_tiles_k = ((self.hidden_dim + K_TILE - 1) / K_TILE) as u32;-- self.experts_two_pass_prefill_kernel- .encode(- root,- MoeExpertsTwoPassArguments {- x_perm_buffer: &buffers.x_perm,- expert_offsets: &buffers.offsets,- row_expert_map: &buffers.row_expert_map,- hidden_buffer: &buffers.hidden,- output_buffer: &buffers.y_partial,- w13_all: &self.shared_weights.w13_buf,- w2_all: &self.shared_weights.w2_buf,- up_biases: &self.shared_weights.up_biases_buf,- down_biases: &self.shared_weights.down_biases_buf,- tile_counts: &buffers.tile_counts,- tile_offsets: &buffers.tile_offsets,- tile_map: &buffers.tile_map,- total_tiles: &buffers.total_tiles,- dispatch_args: &buffers.dispatch_args,- total_rows,- d_model: self.model_dim,- d_ff: self.hidden_dim,- e,- num_tiles_k,- gating_code,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- silu_alpha,- data_type: self.data_type,- },- )- .expect("MoE indirect prefill failed");-- self.encode_finalize(root, buffers, suffix_length);- }-- fn encode_finalize(- &self,- root: &CommandBufferRef,- buffers: &MoeBuffers,- suffix_length: usize,- ) {- let k = self.moe_config.num_experts_per_token;-- self.finalize_kernel- .encode(- root,- MoeFinalizeArguments {- tok2row_buffer: &buffers.tok2row,- probs_buffer: &buffers.topk_probs,- y_partial_buffer: &buffers.y_partial,- y_out_buffer: &buffers.main,- t: suffix_length,- d_model: self.model_dim,- k,- },- self.data_type,- )- .expect("MoE finalize failed");- }-}--impl EncodableWithState for MoeBlockEncodable {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let suffix_length = state.active_suffix_length();- let arrays = state.arrays(&[- ArrayId::Main,- ArrayId::MoeTopkIds,- ArrayId::MoeTopkProbs,- ArrayId::MoeOffsets,- ArrayId::MoeSumK,- ArrayId::MoeBucketedTokenIds,- ArrayId::MoeBucketedProbs,- ArrayId::MoeXPerm,- ArrayId::MoeTok2Row,- ArrayId::MoeYPartial,- ArrayId::MoeTileCounts,- ArrayId::MoeTileOffsets,- ArrayId::MoeTileMap,- ArrayId::MoeTotalTiles,- ArrayId::MoeDispatchArgs,- ArrayId::MoeScatterPartials,- ArrayId::MoeScatterBlockBases,- ArrayId::MoeBlockAlloc,- ArrayId::MoeHidden,- ArrayId::MoeTwoPassRowExpertMap,- ]);-- let clone_buffer = |array: &RefCell<MetalArray>| -> metal::Buffer {- let mut borrow = array.borrow_mut();- unsafe { borrow.mtl_buffer().clone() }- };-- let mut array_iter = arrays.iter();- let buffers = MoeBuffers {- main: clone_buffer(array_iter.next().unwrap()),- topk_ids: clone_buffer(array_iter.next().unwrap()),- topk_probs: clone_buffer(array_iter.next().unwrap()),- offsets: clone_buffer(array_iter.next().unwrap()),- sumk: clone_buffer(array_iter.next().unwrap()),- bucketed_ids: clone_buffer(array_iter.next().unwrap()),- bucketed_probs: clone_buffer(array_iter.next().unwrap()),- x_perm: clone_buffer(array_iter.next().unwrap()),- tok2row: clone_buffer(array_iter.next().unwrap()),- y_partial: clone_buffer(array_iter.next().unwrap()),- tile_counts: clone_buffer(array_iter.next().unwrap()),- tile_offsets: clone_buffer(array_iter.next().unwrap()),- tile_map: clone_buffer(array_iter.next().unwrap()),- total_tiles: clone_buffer(array_iter.next().unwrap()),- dispatch_args: clone_buffer(array_iter.next().unwrap()),- partials: clone_buffer(array_iter.next().unwrap()),- block_bases: clone_buffer(array_iter.next().unwrap()),- block_alloc: clone_buffer(array_iter.next().unwrap()),- hidden: clone_buffer(array_iter.next().unwrap()),- row_expert_map: clone_buffer(array_iter.next().unwrap()),- };- debug_assert!(array_iter.next().is_none());-- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let root = &*mtl_command_buffer;-- // Path selection:- // - suffix == 1: Single-token fused decode (no scatter/gather)- // - suffix <= 32: Indirect decode (GEMV)- // - suffix > 32: Indirect prefill (GEMM)- if suffix_length == 1 {- // Single-token path: router -> fused decode- self.encode_router(root, &buffers, suffix_length);- self.encode_single_token_path(root, &buffers);- } else if suffix_length <= INDIRECT_DECODE_THRESHOLD {- // Indirect decode path: router -> setup -> decode experts -> finalize- self.clear_indirect_buffers(root, &buffers, suffix_length);- self.encode_router(root, &buffers, suffix_length);- self.encode_indirect_setup(root, &buffers, suffix_length);- self.encode_indirect_decode_path(root, &buffers, suffix_length);- } else {- // Indirect prefill path: router -> setup -> prefill experts -> finalize- self.clear_indirect_buffers(root, &buffers, suffix_length);- self.encode_router(root, &buffers, suffix_length);- self.encode_indirect_setup(root, &buffers, suffix_length);- self.encode_indirect_prefill_path(root, &buffers, suffix_length);- }-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- false // MoE uses blit encoders + complex multi-kernel pattern- }-}diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metalindex 23c3c1252..fe957960f 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -2,18 +2,22 @@#include <metal_simdgroup>using namespace metal;-constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU+constant uint GATING_SEL+ [[function_constant(30)]]; // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLUstatic inline float gelu_approx(float x) {- const float k0 = 0.7978845608f;- const float k1 = 0.044715f;- if (x > 10.0f) return x;- if (x < -10.0f) return 0.0f;- return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));+ const float k0 = 0.7978845608f;+ const float k1 = 0.044715f;+ if (x > 10.0f)+ return x;+ if (x < -10.0f)+ return 0.0f;+ return 0.5f * x *+ (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));}static inline float silu(float x, float alpha) {- return x / (1.0f + exp(-alpha * x));+ return x / (1.0f + exp(-alpha * x));}// ============================================================================@@ -23,7 +27,7 @@ static inline float silu(float x, float alpha) {// Grid: (ceil(d_ff/4), K)// ============================================================================-template<typename T, typename T4>+template <typename T, typename T4>inline void moe_experts_decode_single_pass_a_impl(device const T* x,device const int* topk_ids,@@ -43,106 +47,133 @@ inline void moe_experts_decode_single_pass_a_impl(uint simd_gid,uint simd_lid) {- const int expert_id = topk_ids[k_slot];- if (expert_id < 0) return;- const uint expert_u = uint(expert_id);+ const int expert_id = topk_ids[k_slot];+ if (expert_id < 0)+ return;+ const uint expert_u = uint(expert_id);- const uint h_idx = h_block_idx * 4 + simd_gid;- if (h_idx >= d_ff) return;+ const uint h_idx = h_block_idx * 4 + simd_gid;+ if (h_idx >= d_ff)+ return;- const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);- const ulong w13_base = (ulong)expert_u * w13_stride;- const ulong bias_base = (ulong)expert_u * (ulong)(2 * d_ff);+ const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);+ const ulong w13_base = (ulong)expert_u * w13_stride;+ const ulong bias_base = (ulong)expert_u * (ulong)(2 * d_ff);- device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;- device const T* w_gate_row = W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;+ device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;+ device const T* w_gate_row =+ W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;- float acc_up = 0.0f;- float acc_gate = 0.0f;+ float acc_up = 0.0f;+ float acc_gate = 0.0f;- const uint vec_iters = d_model / 128;+ const uint vec_iters = d_model / 128;- for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd_lid * 4;-- T4 x_vec = *reinterpret_cast<device const T4*>(x + base_idx);- T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);-- acc_up += float(x_vec.x) * float(w_up_vec.x);- acc_up += float(x_vec.y) * float(w_up_vec.y);- acc_up += float(x_vec.z) * float(w_up_vec.z);- acc_up += float(x_vec.w) * float(w_up_vec.w);-- if (GATING_SEL > 1) {- T4 w_gate_vec = *reinterpret_cast<device const T4*>(w_gate_row + base_idx);- acc_gate += float(x_vec.x) * float(w_gate_vec.x);- acc_gate += float(x_vec.y) * float(w_gate_vec.y);- acc_gate += float(x_vec.z) * float(w_gate_vec.z);- acc_gate += float(x_vec.w) * float(w_gate_vec.w);- }- }+ for (uint i = 0; i < vec_iters; ++i) {+ uint base_idx = i * 128 + simd_lid * 4;- uint leftover_start = vec_iters * 128 + simd_lid;- for (uint idx = leftover_start; idx < d_model; idx += 32) {- float xv = float(x[idx]);- acc_up += xv * float(w_up_row[idx]);- if (GATING_SEL > 1) {- acc_gate += xv * float(w_gate_row[idx]);- }- }+ T4 x_vec = *reinterpret_cast<device const T4*>(x + base_idx);+ T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);++ acc_up += float(x_vec.x) * float(w_up_vec.x);+ acc_up += float(x_vec.y) * float(w_up_vec.y);+ acc_up += float(x_vec.z) * float(w_up_vec.z);+ acc_up += float(x_vec.w) * float(w_up_vec.w);- acc_up = simd_sum(acc_up);if (GATING_SEL > 1) {- acc_gate = simd_sum(acc_gate);+ T4 w_gate_vec =+ *reinterpret_cast<device const T4*>(w_gate_row + base_idx);+ acc_gate += float(x_vec.x) * float(w_gate_vec.x);+ acc_gate += float(x_vec.y) * float(w_gate_vec.y);+ acc_gate += float(x_vec.z) * float(w_gate_vec.z);+ acc_gate += float(x_vec.w) * float(w_gate_vec.w);}+ }- if (simd_lid == 0) {- float up_val = clamp(acc_up + float(biases[bias_base + h_idx]), up_clip_min, up_clip_max);-- float activated;- if (GATING_SEL <= 1) {- activated = (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);- } else {- float gate_val = clamp(acc_gate + float(biases[bias_base + d_ff + h_idx]), gate_clip_min, gate_clip_max);- float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);- activated = gate_act * up_val;- }-- hidden_out[(ulong)k_slot * (ulong)d_ff + (ulong)h_idx] = activated;+ uint leftover_start = vec_iters * 128 + simd_lid;+ for (uint idx = leftover_start; idx < d_model; idx += 32) {+ float xv = float(x[idx]);+ acc_up += xv * float(w_up_row[idx]);+ if (GATING_SEL > 1) {+ acc_gate += xv * float(w_gate_row[idx]);+ }+ }++ acc_up = simd_sum(acc_up);+ if (GATING_SEL > 1) {+ acc_gate = simd_sum(acc_gate);+ }++ if (simd_lid == 0) {+ float up_val = clamp(+ acc_up + float(biases[bias_base + h_idx]),+ up_clip_min,+ up_clip_max+ );++ float activated;+ if (GATING_SEL <= 1) {+ activated =+ (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);+ } else {+ float gate_val = clamp(+ acc_gate + float(biases[bias_base + d_ff + h_idx]),+ gate_clip_min,+ gate_clip_max+ );+ float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha)+ : gelu_approx(gate_val);+ activated = gate_act * up_val;}-}-#define MOE_DECODE_SINGLE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \-kernel void moe_experts_decode_single_pass_a_##SUFFIX( \- device const DTYPE* x [[buffer(0)]], \- device const int* topk_ids [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& K [[buffer(7)]], \- constant float& silu_alpha [[buffer(8)]], \- constant float& gate_clip_min [[buffer(9)]], \- constant float& gate_clip_max [[buffer(10)]], \- constant float& up_clip_min [[buffer(11)]], \- constant float& up_clip_max [[buffer(12)]], \- uint2 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]]) \-{ \- moe_experts_decode_single_pass_a_impl<DTYPE, DTYPE4>( \- x, topk_ids, W13_all, biases, hidden_out, \- d_model, d_ff, K, silu_alpha, \- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, \- tgpig.y, tgpig.x, simd_gid, simd_lid); \+ hidden_out[(ulong)k_slot * (ulong)d_ff + (ulong)h_idx] = activated;+ }}+#define MOE_DECODE_SINGLE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \+ kernel void moe_experts_decode_single_pass_a_##SUFFIX( \+ device const DTYPE* x [[buffer(0)]], \+ device const int* topk_ids [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device const DTYPE* biases [[buffer(3)]], \+ device float* hidden_out [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& K [[buffer(7)]], \+ constant float& silu_alpha [[buffer(8)]], \+ constant float& gate_clip_min [[buffer(9)]], \+ constant float& gate_clip_max [[buffer(10)]], \+ constant float& up_clip_min [[buffer(11)]], \+ constant float& up_clip_max [[buffer(12)]], \+ uint2 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ moe_experts_decode_single_pass_a_impl<DTYPE, DTYPE4>( \+ x, \+ topk_ids, \+ W13_all, \+ biases, \+ hidden_out, \+ d_model, \+ d_ff, \+ K, \+ silu_alpha, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ tgpig.y, \+ tgpig.x, \+ simd_gid, \+ simd_lid \+ ); \+ }+MOE_DECODE_SINGLE_PASS_A_KERNEL(half, half4, f16)MOE_DECODE_SINGLE_PASS_A_KERNEL(bfloat, bfloat4, bf16)MOE_DECODE_SINGLE_PASS_A_KERNEL(float, float4, f32)-// ============================================================================// Pass B (fused with finalize): hidden[k] @ W2[expert] → y (directly)// Computes: y[d] = Σ_k prob[k] * (hidden[k] @ W2[expert_k, d] + bias)@@ -150,14 +181,14 @@ MOE_DECODE_SINGLE_PASS_A_KERNEL(float, float4, f32)// Grid: (ceil(d_model/8), 1) - NOT per K!// ============================================================================-template<typename T, typename T4>+template <typename T, typename T4>inline void moe_experts_decode_single_pass_b_impl(- device const float* hidden, // [K, d_ff]- device const int* topk_ids, // [K]- device const T* topk_probs, // [K]- device const T* W2_all, // [E, d_model, d_ff]- device const T* biases, // [E, d_model]- device T* y, // [d_model]+ device const float* hidden, // [K, d_ff]+ device const int* topk_ids, // [K]+ device const T* topk_probs, // [K]+ device const T* W2_all, // [E, d_model, d_ff]+ device const T* biases, // [E, d_model]+ device T* y, // [d_model]uint d_model,uint d_ff,uint K,@@ -165,75 +196,89 @@ inline void moe_experts_decode_single_pass_b_impl(uint simd_gid,uint simd_lid) {- const uint my_col = d_block * 8 + simd_gid;- if (my_col >= d_model) return;+ const uint my_col = d_block * 8 + simd_gid;+ if (my_col >= d_model)+ return;- const uint vec_iters = d_ff / 128;- const ulong w2_expert_stride = (ulong)d_model * (ulong)d_ff;+ const uint vec_iters = d_ff / 128;+ const ulong w2_expert_stride = (ulong)d_model * (ulong)d_ff;- float final_acc = 0.0f;+ float final_acc = 0.0f;- // Loop over K experts- for (uint k = 0; k < K; ++k) {- const uint expert_u = uint(topk_ids[k]);- const float prob = float(topk_probs[k]);+ // Loop over K experts+ for (uint k = 0; k < K; ++k) {+ const uint expert_u = uint(topk_ids[k]);+ const float prob = float(topk_probs[k]);- device const float* hidden_ptr = hidden + (ulong)k * (ulong)d_ff;- device const T* w2_ptr = W2_all + (ulong)expert_u * w2_expert_stride + (ulong)my_col * (ulong)d_ff;+ device const float* hidden_ptr = hidden + (ulong)k * (ulong)d_ff;+ device const T* w2_ptr = W2_all + (ulong)expert_u * w2_expert_stride ++ (ulong)my_col * (ulong)d_ff;- float acc = 0.0f;+ float acc = 0.0f;- // Vectorized reduction- for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd_lid * 4;-- float4 h_vec = *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);- T4 w_vec = *reinterpret_cast<device const T4*>(w2_ptr + base_idx);+ // Vectorized reduction+ for (uint i = 0; i < vec_iters; ++i) {+ uint base_idx = i * 128 + simd_lid * 4;- acc += h_vec.x * float(w_vec.x);- acc += h_vec.y * float(w_vec.y);- acc += h_vec.z * float(w_vec.z);- acc += h_vec.w * float(w_vec.w);- }+ float4 h_vec =+ *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);+ T4 w_vec = *reinterpret_cast<device const T4*>(w2_ptr + base_idx);- // Remainder- for (uint idx = vec_iters * 128 + simd_lid; idx < d_ff; idx += 32) {- acc += hidden_ptr[idx] * float(w2_ptr[idx]);- }+ acc += h_vec.x * float(w_vec.x);+ acc += h_vec.y * float(w_vec.y);+ acc += h_vec.z * float(w_vec.z);+ acc += h_vec.w * float(w_vec.w);+ }- // Simdgroup reduction and accumulate- float result = simd_sum(acc);- if (simd_lid == 0) {- result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);- final_acc += prob * result;- }+ // Remainder+ for (uint idx = vec_iters * 128 + simd_lid; idx < d_ff; idx += 32) {+ acc += hidden_ptr[idx] * float(w2_ptr[idx]);}+ // Simdgroup reduction and accumulate+ float result = simd_sum(acc);if (simd_lid == 0) {- y[my_col] = T(final_acc);+ result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);+ final_acc += prob * result;}-}+ }-#define MOE_DECODE_SINGLE_PASS_B_KERNEL(DTYPE, DTYPE4, SUFFIX) \-kernel void moe_experts_decode_single_pass_b_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const int* topk_ids [[buffer(1)]], \- device const DTYPE* topk_probs [[buffer(2)]], \- device const DTYPE* W2_all [[buffer(3)]], \- device const DTYPE* biases [[buffer(4)]], \- device DTYPE* y [[buffer(5)]], \- constant uint& d_model [[buffer(6)]], \- constant uint& d_ff [[buffer(7)]], \- constant uint& K [[buffer(8)]], \- uint tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]]) \-{ \- moe_experts_decode_single_pass_b_impl<DTYPE, DTYPE4>( \- hidden, topk_ids, topk_probs, W2_all, biases, y, \- d_model, d_ff, K, tgpig, simd_gid, simd_lid); \+ if (simd_lid == 0) {+ y[my_col] = T(final_acc);+ }}+#define MOE_DECODE_SINGLE_PASS_B_KERNEL(DTYPE, DTYPE4, SUFFIX) \+ kernel void moe_experts_decode_single_pass_b_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const int* topk_ids [[buffer(1)]], \+ device const DTYPE* topk_probs [[buffer(2)]], \+ device const DTYPE* W2_all [[buffer(3)]], \+ device const DTYPE* biases [[buffer(4)]], \+ device DTYPE* y [[buffer(5)]], \+ constant uint& d_model [[buffer(6)]], \+ constant uint& d_ff [[buffer(7)]], \+ constant uint& K [[buffer(8)]], \+ uint tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ moe_experts_decode_single_pass_b_impl<DTYPE, DTYPE4>( \+ hidden, \+ topk_ids, \+ topk_probs, \+ W2_all, \+ biases, \+ y, \+ d_model, \+ d_ff, \+ K, \+ tgpig, \+ simd_gid, \+ simd_lid \+ ); \+ }+MOE_DECODE_SINGLE_PASS_B_KERNEL(half, half4, f16)MOE_DECODE_SINGLE_PASS_B_KERNEL(bfloat, bfloat4, bf16)MOE_DECODE_SINGLE_PASS_B_KERNEL(float, float4, f32)diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex d1631c395..ab55ce6c1 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -2,31 +2,35 @@#include <metal_simdgroup>using namespace metal;-constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU+constant uint GATING_SEL+ [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLUstatic inline float gelu_approx(float x) {- const float k0 = 0.7978845608f;- const float k1 = 0.044715f;- if (x > 10.0f) return x;- if (x < -10.0f) return 0.0f;- return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));+ const float k0 = 0.7978845608f;+ const float k1 = 0.044715f;+ if (x > 10.0f)+ return x;+ if (x < -10.0f)+ return 0.0f;+ return 0.5f * x *+ (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));}static inline float silu(float x, float alpha) {- return x / (1.0f + exp(-alpha * x));+ return x / (1.0f + exp(-alpha * x));}// === Pass A: Vectorized GEMV with float4 loads ===// Structure: 4 simdgroups (128 threads), each outputs 1 hidden element// Each simdgroup: 32 threads reduce d_model using float4 vectorized loads// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)-template<typename T, typename T4>+template <typename T, typename T4>void moe_experts_decode_pass_a_impl(- device const T* X_perm, // [total_rows, d_model]- device const uint* expert_offsets, // [E + 1]- device const T* W13_all, // [E, 2*d_ff, d_model]- device const T* up_biases, // [E, 2*d_ff]- device float* hidden_out, // [total_rows, d_ff]+ device const T* X_perm, // [total_rows, d_model]+ device const uint* expert_offsets, // [E + 1]+ device const T* W13_all, // [E, 2*d_ff, d_model]+ device const T* up_biases, // [E, 2*d_ff]+ device float* hidden_out, // [total_rows, d_ff]uint d_model,uint d_ff,uint E,@@ -41,114 +45,134 @@ void moe_experts_decode_pass_a_impl(uint simd_gid,uint simd_lid) {- // Validate row bounds- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint global_row = seg_start + row_in_expert;- if (global_row >= seg_end) return;-- // Each simdgroup outputs one hidden element- const uint h_idx = h_block_idx * 4 + simd_gid;- if (h_idx >= d_ff) return;-- // Base addresses- const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);- const ulong w13_base = (ulong)expert_idx * w13_stride;- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);- const ulong x_row_base = (ulong)global_row * (ulong)d_model;-- device const T* x_ptr = X_perm + x_row_base;- device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;- device const T* w_gate_row = W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;-- float acc_up = 0.0f;- float acc_gate = 0.0f;-- // Vectorized reduction: 32 threads × 4 elements = 128 elements per iteration- const uint vec_iters = d_model / 128;-- for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd_lid * 4;-- T4 x_vec = *reinterpret_cast<device const T4*>(x_ptr + base_idx);- T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);-- acc_up += float(x_vec.x) * float(w_up_vec.x);- acc_up += float(x_vec.y) * float(w_up_vec.y);- acc_up += float(x_vec.z) * float(w_up_vec.z);- acc_up += float(x_vec.w) * float(w_up_vec.w);-- if (GATING_SEL > 1) {- T4 w_gate_vec = *reinterpret_cast<device const T4*>(w_gate_row + base_idx);- acc_gate += float(x_vec.x) * float(w_gate_vec.x);- acc_gate += float(x_vec.y) * float(w_gate_vec.y);- acc_gate += float(x_vec.z) * float(w_gate_vec.z);- acc_gate += float(x_vec.w) * float(w_gate_vec.w);- }- }+ // Validate row bounds+ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ const uint global_row = seg_start + row_in_expert;+ if (global_row >= seg_end)+ return;++ // Each simdgroup outputs one hidden element+ const uint h_idx = h_block_idx * 4 + simd_gid;+ if (h_idx >= d_ff)+ return;++ // Base addresses+ const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);+ const ulong w13_base = (ulong)expert_idx * w13_stride;+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);+ const ulong x_row_base = (ulong)global_row * (ulong)d_model;++ device const T* x_ptr = X_perm + x_row_base;+ device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;+ device const T* w_gate_row =+ W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;++ float acc_up = 0.0f;+ float acc_gate = 0.0f;++ // Vectorized reduction: 32 threads × 4 elements = 128 elements per iteration+ const uint vec_iters = d_model / 128;++ for (uint i = 0; i < vec_iters; ++i) {+ uint base_idx = i * 128 + simd_lid * 4;++ T4 x_vec = *reinterpret_cast<device const T4*>(x_ptr + base_idx);+ T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);++ acc_up += float(x_vec.x) * float(w_up_vec.x);+ acc_up += float(x_vec.y) * float(w_up_vec.y);+ acc_up += float(x_vec.z) * float(w_up_vec.z);+ acc_up += float(x_vec.w) * float(w_up_vec.w);- // Handle leftover elements- uint leftover_start = vec_iters * 128 + simd_lid;- for (uint idx = leftover_start; idx < d_model; idx += 32) {- float xv = float(x_ptr[idx]);- acc_up += xv * float(w_up_row[idx]);- if (GATING_SEL > 1) {- acc_gate += xv * float(w_gate_row[idx]);- }+ if (GATING_SEL > 1) {+ T4 w_gate_vec =+ *reinterpret_cast<device const T4*>(w_gate_row + base_idx);+ acc_gate += float(x_vec.x) * float(w_gate_vec.x);+ acc_gate += float(x_vec.y) * float(w_gate_vec.y);+ acc_gate += float(x_vec.z) * float(w_gate_vec.z);+ acc_gate += float(x_vec.w) * float(w_gate_vec.w);}+ }- // Simdgroup reduction- acc_up = simd_sum(acc_up);+ // Handle leftover elements+ uint leftover_start = vec_iters * 128 + simd_lid;+ for (uint idx = leftover_start; idx < d_model; idx += 32) {+ float xv = float(x_ptr[idx]);+ acc_up += xv * float(w_up_row[idx]);if (GATING_SEL > 1) {- acc_gate = simd_sum(acc_gate);+ acc_gate += xv * float(w_gate_row[idx]);}-- // Lane 0 applies activation and writes result- if (simd_lid == 0) {- float up_val = acc_up + float(up_biases[bias_base + h_idx]);- up_val = clamp(up_val, up_clip_min, up_clip_max);-- float activated;- if (GATING_SEL <= 1) {- activated = (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);- } else {- float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);- gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);- float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);- activated = gate_act * up_val;- }-- hidden_out[(ulong)global_row * (ulong)d_ff + (ulong)h_idx] = activated;+ }++ // Simdgroup reduction+ acc_up = simd_sum(acc_up);+ if (GATING_SEL > 1) {+ acc_gate = simd_sum(acc_gate);+ }++ // Lane 0 applies activation and writes result+ if (simd_lid == 0) {+ float up_val = acc_up + float(up_biases[bias_base + h_idx]);+ up_val = clamp(up_val, up_clip_min, up_clip_max);++ float activated;+ if (GATING_SEL <= 1) {+ activated =+ (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);+ } else {+ float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);+ gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);+ float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha)+ : gelu_approx(gate_val);+ activated = gate_act * up_val;}-}-#define MOE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \-kernel void moe_experts_decode_pass_a_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device float* hidden_out [[buffer(3)]], \- device const DTYPE* up_biases [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- uint3 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]]) \-{ \- moe_experts_decode_pass_a_impl<DTYPE, DTYPE4>( \- X_perm, expert_offsets, W13_all, up_biases, hidden_out, \- d_model, d_ff, E, gate_clip_min, gate_clip_max, \- up_clip_min, up_clip_max, silu_alpha, \- tgpig.y, tgpig.z, tgpig.x, \- simd_gid, simd_lid); \+ hidden_out[(ulong)global_row * (ulong)d_ff + (ulong)h_idx] = activated;+ }}+#define MOE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \+ kernel void moe_experts_decode_pass_a_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device float* hidden_out [[buffer(3)]], \+ device const DTYPE* up_biases [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ uint3 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ moe_experts_decode_pass_a_impl<DTYPE, DTYPE4>( \+ X_perm, \+ expert_offsets, \+ W13_all, \+ up_biases, \+ hidden_out, \+ d_model, \+ d_ff, \+ E, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ silu_alpha, \+ tgpig.y, \+ tgpig.z, \+ tgpig.x, \+ simd_gid, \+ simd_lid \+ ); \+ }+MOE_PASS_A_KERNEL(bfloat, bfloat4, bf16)MOE_PASS_A_KERNEL(half, half4, f16)MOE_PASS_A_KERNEL(float, float4, f32)@@ -157,142 +181,149 @@ MOE_PASS_A_KERNEL(float, float4, f32)// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0kernel void moe_pass_a_tile_counts(- device const uint* expert_offsets [[buffer(0)]], // [E+1]- device uint* tile_counts [[buffer(1)]], // [E]+ device const uint* expert_offsets [[buffer(0)]], // [E+1]+ device uint* tile_counts [[buffer(1)]], // [E]constant uint& E [[buffer(2)]],constant uint& h_blocks [[buffer(3)]],uint tid [[thread_position_in_grid]]) {- if (tid >= E) return;- const uint start = expert_offsets[tid];- const uint end = expert_offsets[tid + 1];- const uint num_rows = end - start;- tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;+ if (tid >= E)+ return;+ const uint start = expert_offsets[tid];+ const uint end = expert_offsets[tid + 1];+ const uint num_rows = end - start;+ tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;}// Exclusive scan of tile_counts to get tile_offsets and total_tileskernel void moe_pass_a_tile_scan(- device const uint* tile_counts [[buffer(0)]], // [E]- device uint* tile_offsets [[buffer(1)]], // [E+1]- device uint* total_tiles [[buffer(2)]], // [1]+ device const uint* tile_counts [[buffer(0)]], // [E]+ device uint* tile_offsets [[buffer(1)]], // [E+1]+ device uint* total_tiles [[buffer(2)]], // [1]constant uint& E [[buffer(3)]],uint lid [[thread_index_in_threadgroup]],threadgroup uint* scratch [[threadgroup(0)]]) {- // Simple single-threadgroup scan (works for E <= 1024)- const uint idx = lid;-- // Load into threadgroup memory- if (idx < E) {- scratch[idx] = tile_counts[idx];- } else {- scratch[idx] = 0;+ // Simple single-threadgroup scan (works for E <= 1024)+ const uint idx = lid;++ // Load into threadgroup memory+ if (idx < E) {+ scratch[idx] = tile_counts[idx];+ } else {+ scratch[idx] = 0;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Kogge-Stone scan+ uint val = scratch[idx];+ for (uint offset = 1; offset < 1024; offset *= 2) {+ uint temp = 0;+ if (idx >= offset && idx < E) {+ temp = scratch[idx - offset];}threadgroup_barrier(mem_flags::mem_threadgroup);-- // Kogge-Stone scan- uint val = scratch[idx];- for (uint offset = 1; offset < 1024; offset *= 2) {- uint temp = 0;- if (idx >= offset && idx < E) {- temp = scratch[idx - offset];- }- threadgroup_barrier(mem_flags::mem_threadgroup);- if (idx >= offset && idx < E) {- val += temp;- scratch[idx] = val;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // Write exclusive scan (shift right by 1)- if (idx == 0) {- tile_offsets[0] = 0;+ if (idx >= offset && idx < E) {+ val += temp;+ scratch[idx] = val;}- if (idx < E) {- tile_offsets[idx + 1] = scratch[idx];- if (idx == E - 1) {- total_tiles[0] = scratch[idx];- }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ // Write exclusive scan (shift right by 1)+ if (idx == 0) {+ tile_offsets[0] = 0;+ }+ if (idx < E) {+ tile_offsets[idx + 1] = scratch[idx];+ if (idx == E - 1) {+ total_tiles[0] = scratch[idx];}+ }}// Build row→expert map: one thread per routed rowkernel void moe_pass_a_build_row_map(- device const uint* expert_offsets [[buffer(0)]] , // [E+1]- device uint* row_expert_map [[buffer(1)]] , // [total_rows]+ device const uint* expert_offsets [[buffer(0)]], // [E+1]+ device uint* row_expert_map [[buffer(1)]], // [total_rows]constant uint& total_rows [[buffer(2)]],constant uint& E [[buffer(3)]],uint tid [[thread_position_in_grid]]) {- if (tid >= total_rows) return;-- uint left = 0u;- uint right = E;- const uint row = tid;-- while (left + 1u < right) {- const uint mid = (left + right) >> 1;- if (row < expert_offsets[mid]) {- right = mid;- } else {- left = mid;- }+ if (tid >= total_rows)+ return;++ uint left = 0u;+ uint right = E;+ const uint row = tid;++ while (left + 1u < right) {+ const uint mid = (left + right) >> 1;+ if (row < expert_offsets[mid]) {+ right = mid;+ } else {+ left = mid;}+ }- row_expert_map[row] = left;+ row_expert_map[row] = left;}// Build tile map entries from row→expert mapkernel void moe_pass_a_build_tile_map(- device const uint* expert_offsets [[buffer(0)]], // [E+1]- device const uint* tile_offsets [[buffer(1)]], // [E+1]- device const uint* row_expert_map [[buffer(2)]], // [total_rows]- device uint* tile_map [[buffer(3)]], // [total_tiles * 3]+ device const uint* expert_offsets [[buffer(0)]], // [E+1]+ device const uint* tile_offsets [[buffer(1)]], // [E+1]+ device const uint* row_expert_map [[buffer(2)]], // [total_rows]+ device uint* tile_map [[buffer(3)]], // [total_tiles * 3]constant uint& total_rows [[buffer(4)]],constant uint& h_blocks [[buffer(5)]],uint tid [[thread_position_in_grid]]) {- const uint total_tiles = total_rows * h_blocks;- if (tid >= total_tiles) return;+ const uint total_tiles = total_rows * h_blocks;+ if (tid >= total_tiles)+ return;- const uint row_idx = tid / h_blocks;- const uint h_block = tid % h_blocks;+ const uint row_idx = tid / h_blocks;+ const uint h_block = tid % h_blocks;- if (row_idx >= total_rows) return;+ if (row_idx >= total_rows)+ return;- const uint expert_idx = row_expert_map[row_idx];- const uint row_start = expert_offsets[expert_idx];- const uint row_in_expert = row_idx - row_start;- const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;+ const uint expert_idx = row_expert_map[row_idx];+ const uint row_start = expert_offsets[expert_idx];+ const uint row_in_expert = row_idx - row_start;+ const uint tile_base =+ tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;- tile_map[tile_base * 3 + 0] = h_block;- tile_map[tile_base * 3 + 1] = expert_idx;- tile_map[tile_base * 3 + 2] = row_in_expert;+ tile_map[tile_base * 3 + 0] = h_block;+ tile_map[tile_base * 3 + 1] = expert_idx;+ tile_map[tile_base * 3 + 2] = row_in_expert;}// Write dispatch args for indirect dispatch (reusable from tiled version)kernel void moe_pass_a_write_dispatch_args(- device const uint* total_tiles [[buffer(0)]], // [1]- device uint* dispatch_args [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments- constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A+ device const uint* total_tiles [[buffer(0)]], // [1]+ device uint* dispatch_args+ [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments+ constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass Auint tid [[thread_position_in_grid]]) {- if (tid > 0) return;- dispatch_args[0] = total_tiles[0]; // x dimension = total tiles- dispatch_args[1] = num_tiles_y; // y dimension- dispatch_args[2] = 1; // z dimension+ if (tid > 0)+ return;+ dispatch_args[0] = total_tiles[0]; // x dimension = total tiles+ dispatch_args[1] = num_tiles_y; // y dimension+ dispatch_args[2] = 1; // z dimension}// Modified Pass A that reads from tile map for indirect dispatch-template<typename T, typename T4>+template <typename T, typename T4>void moe_experts_decode_pass_a_indirect_impl(device const T* X_perm,device const uint* expert_offsets,device const T* W13_all,device const T* up_biases,device float* hidden_out,- device const uint* tile_map, // [total_tiles * 3]+ device const uint* tile_map, // [total_tiles * 3]uint d_model,uint d_ff,uint E,@@ -301,68 +332,95 @@ void moe_experts_decode_pass_a_indirect_impl(float up_clip_min,float up_clip_max,float silu_alpha,- uint tile_idx, // flat threadgroup index+ uint tile_idx, // flat threadgroup indexuint simd_gid,uint simd_lid) {- // Read tile descriptor- const uint h_block_idx = tile_map[tile_idx * 3 + 0];- const uint expert_idx = tile_map[tile_idx * 3 + 1];- const uint row_in_expert = tile_map[tile_idx * 3 + 2];-- // Call original implementation- moe_experts_decode_pass_a_impl<T, T4>(- X_perm, expert_offsets, W13_all, up_biases, hidden_out,- d_model, d_ff, E,- gate_clip_min, gate_clip_max,- up_clip_min, up_clip_max, silu_alpha,- expert_idx, row_in_expert, h_block_idx,- simd_gid, simd_lid- );+ // Read tile descriptor+ const uint h_block_idx = tile_map[tile_idx * 3 + 0];+ const uint expert_idx = tile_map[tile_idx * 3 + 1];+ const uint row_in_expert = tile_map[tile_idx * 3 + 2];++ // Call original implementation+ moe_experts_decode_pass_a_impl<T, T4>(+ X_perm,+ expert_offsets,+ W13_all,+ up_biases,+ hidden_out,+ d_model,+ d_ff,+ E,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ expert_idx,+ row_in_expert,+ h_block_idx,+ simd_gid,+ simd_lid+ );}-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, DTYPE4, SUFFIX) \-kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device float* hidden_out [[buffer(3)]], \- device const DTYPE* up_biases [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- device const uint* tile_map [[buffer(13)]], \- uint3 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]]) \-{ \- moe_experts_decode_pass_a_indirect_impl<DTYPE, DTYPE4>( \- X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \- d_model, d_ff, E, gate_clip_min, gate_clip_max, \- up_clip_min, up_clip_max, silu_alpha, \- tgpig.x, simd_gid, simd_lid); \-}+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, DTYPE4, SUFFIX) \+ kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device float* hidden_out [[buffer(3)]], \+ device const DTYPE* up_biases [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ device const uint* tile_map [[buffer(13)]], \+ uint3 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ moe_experts_decode_pass_a_indirect_impl<DTYPE, DTYPE4>( \+ X_perm, \+ expert_offsets, \+ W13_all, \+ up_biases, \+ hidden_out, \+ tile_map, \+ d_model, \+ d_ff, \+ E, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ silu_alpha, \+ tgpig.x, \+ simd_gid, \+ simd_lid \+ ); \+ }MOE_PASS_A_INDIRECT_KERNEL(bfloat, bfloat4, bf16)MOE_PASS_A_INDIRECT_KERNEL(half, half4, f16)MOE_PASS_A_INDIRECT_KERNEL(float, float4, f32)-// === Pass B: Simdgroup cooperation along K for coalescing ===-// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading consecutive K elements+// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading+// consecutive K elements-template<typename T, typename AccumT>+template <typename T, typename AccumT>void moe_experts_decode_down_fused_2d_impl(- device const float* hidden, // [total_rows, d_ff] - f32 from Pass A- device const uint* row_expert_map, // [total_rows] - direct row->expert lookup- device const T* w2_all, // [E, d_model, d_ff] - layout- device const T* down_biases, // [E, d_model]- device T* y_out, // [total_rows, d_model]+ device const float* hidden, // [total_rows, d_ff] - f32 from Pass A+ device const uint*+ row_expert_map, // [total_rows] - direct row->expert lookup+ device const T* w2_all, // [E, d_model, d_ff] - layout+ device const T* down_biases, // [E, d_model]+ device T* y_out, // [total_rows, d_model]uint total_rows,uint d_model,uint d_ff,@@ -371,114 +429,141 @@ void moe_experts_decode_down_fused_2d_impl(uint simd_gid,uint simd_lid) {- constexpr uint THREADS_PER_SIMD = 32;- constexpr uint SIMDGROUPS_PER_TG = 8;-- const uint row_idx = tgpig.y;- if (row_idx >= total_rows) return;-- // Each simdgroup computes one output column- const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;- if (my_col >= d_model) return;-- const uint expert_idx = row_expert_map[row_idx];-- // Base addresses for this output column- const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;-- // Dual accumulators for ILP: breaks FMA dependency chains- AccumT acc0 = AccumT(0.0);- AccumT acc1 = AccumT(0.0);- AccumT acc = AccumT(0.0);-- // Main loop: stride-32 with vec8 loads for ILP- const uint k_iters = d_ff / THREADS_PER_SIMD;- const uint k_vec_iters = k_iters / 8;-- for (uint iter = 0; iter < k_vec_iters; ++iter) {- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;-- // hidden: stride-32 per lane, already f32 from Pass A- const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];- const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];- const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];- const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];- const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];- const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];- const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];- const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];-- // W2: lane-coalesced- const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);- const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);- const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);- const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);- const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);- const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);- const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);- const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);-- // dual trees for ILP- acc0 = fma(h0, w0, acc0);- acc1 = fma(h1, w1, acc1);- acc0 = fma(h2, w2, acc0);- acc1 = fma(h3, w3, acc1);- acc0 = fma(h4, w4, acc0);- acc1 = fma(h5, w5, acc1);- acc0 = fma(h6, w6, acc0);- acc1 = fma(h7, w7, acc1);- }-- // fold the vectorized contribution once- acc += (acc0 + acc1);-- // Handle remaining full iterations- for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {- const uint k = iter * THREADS_PER_SIMD + simd_lid;- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);- }-- // Handle leftover elements (d_ff % 32)- const uint leftover_start = k_iters * THREADS_PER_SIMD;- if (leftover_start + simd_lid < d_ff) {- const uint k = leftover_start + simd_lid;- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);- }-- // Simdgroup reduction- AccumT result = simd_sum(acc);-- // Lane 0 writes result- if (simd_lid == 0) {- const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;- result += AccumT(down_biases[bias_idx]);-- const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;- y_out[out_idx] = T(result);- }+ constexpr uint THREADS_PER_SIMD = 32;+ constexpr uint SIMDGROUPS_PER_TG = 8;++ const uint row_idx = tgpig.y;+ if (row_idx >= total_rows)+ return;++ // Each simdgroup computes one output column+ const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;+ if (my_col >= d_model)+ return;++ const uint expert_idx = row_expert_map[row_idx];++ // Base addresses for this output column+ const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;+ const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff ++ (ulong)my_col * (ulong)d_ff;++ // Dual accumulators for ILP: breaks FMA dependency chains+ AccumT acc0 = AccumT(0.0);+ AccumT acc1 = AccumT(0.0);+ AccumT acc = AccumT(0.0);++ // Main loop: stride-32 with vec8 loads for ILP+ const uint k_iters = d_ff / THREADS_PER_SIMD;+ const uint k_vec_iters = k_iters / 8;++ for (uint iter = 0; iter < k_vec_iters; ++iter) {+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;++ // hidden: stride-32 per lane, already f32 from Pass A+ const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];+ const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];+ const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];+ const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];+ const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];+ const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];+ const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];+ const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];++ // W2: lane-coalesced+ const AccumT w0 =+ AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);+ const AccumT w1 =+ AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);+ const AccumT w2 =+ AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);+ const AccumT w3 =+ AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);+ const AccumT w4 =+ AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);+ const AccumT w5 =+ AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);+ const AccumT w6 =+ AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);+ const AccumT w7 =+ AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);++ // dual trees for ILP+ acc0 = fma(h0, w0, acc0);+ acc1 = fma(h1, w1, acc1);+ acc0 = fma(h2, w2, acc0);+ acc1 = fma(h3, w3, acc1);+ acc0 = fma(h4, w4, acc0);+ acc1 = fma(h5, w5, acc1);+ acc0 = fma(h6, w6, acc0);+ acc1 = fma(h7, w7, acc1);+ }++ // fold the vectorized contribution once+ acc += (acc0 + acc1);++ // Handle remaining full iterations+ for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {+ const uint k = iter * THREADS_PER_SIMD + simd_lid;+ acc =+ fma(AccumT(hidden[hidden_base + k]),+ AccumT(w2_all[w2_col_base + k]),+ acc);+ }++ // Handle leftover elements (d_ff % 32)+ const uint leftover_start = k_iters * THREADS_PER_SIMD;+ if (leftover_start + simd_lid < d_ff) {+ const uint k = leftover_start + simd_lid;+ acc =+ fma(AccumT(hidden[hidden_base + k]),+ AccumT(w2_all[w2_col_base + k]),+ acc);+ }++ // Simdgroup reduction+ AccumT result = simd_sum(acc);++ // Lane 0 writes result+ if (simd_lid == 0) {+ const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;+ result += AccumT(down_biases[bias_idx]);++ const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;+ y_out[out_idx] = T(result);+ }}-#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \-kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* row_expert_map [[buffer(1)]], \- device const DTYPE* w2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* y_out [[buffer(4)]], \- constant uint& total_rows [[buffer(5)]], \- constant uint& d_model [[buffer(6)]], \- constant uint& d_ff [[buffer(7)]], \- constant uint& E [[buffer(8)]], \- uint2 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]]) \-{ \- moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \- hidden, row_expert_map, w2_all, down_biases, y_out, \- total_rows, d_model, d_ff, E, \- tgpig, simd_gid, simd_lid); \-}+#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* row_expert_map [[buffer(1)]], \+ device const DTYPE* w2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* y_out [[buffer(4)]], \+ constant uint& total_rows [[buffer(5)]], \+ constant uint& d_model [[buffer(6)]], \+ constant uint& d_ff [[buffer(7)]], \+ constant uint& E [[buffer(8)]], \+ uint2 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \+ hidden, \+ row_expert_map, \+ w2_all, \+ down_biases, \+ y_out, \+ total_rows, \+ d_model, \+ d_ff, \+ E, \+ tgpig, \+ simd_gid, \+ simd_lid \+ ); \+ }MOE_PASS_B_FUSED_2D_KERNEL(bfloat, bf16)MOE_PASS_B_FUSED_2D_KERNEL(half, f16)diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex 9f9a5810d..72076217f 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -1,7 +1,7 @@#include <metal_stdlib>#include <metal_simdgroup>#include <metal_simdgroup_matrix>-#include "../quant_matmul/mma.h"+#include "../quant_matmul/mma.h"using namespace metal;@@ -9,50 +9,52 @@ using namespace metal;static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }static inline uint linear_tid(uint3 tid, uint3 tpg) {- return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;+ return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;}static inline float gelu_approx(float x) {- const float k0 = 0.7978845608f; // sqrt(2/pi)- const float k1 = 0.044715f;- if (x > 10.0f) return x;- if (x < -10.0f) return 0.0f;- float x3 = x * x * x;- float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);- return 0.5f * x * (1.0f + tanh(t));+ const float k0 = 0.7978845608f; // sqrt(2/pi)+ const float k1 = 0.044715f;+ if (x > 10.0f)+ return x;+ if (x < -10.0f)+ return 0.0f;+ float x3 = x * x * x;+ float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);+ return 0.5f * x * (1.0f + tanh(t));}static inline float silu(float x, float alpha) {- // alpha=1 gives standard SiLU; keep alpha param for parity with your API- return x / (1.0f + exp(-alpha * x));+ // alpha=1 gives standard SiLU; keep alpha param for parity with your API+ return x / (1.0f + exp(-alpha * x));}-template<typename T>+template <typename T>inline void store_vec4(device T* dst, ulong base, float4 vals) {- dst[base + 0] = T(vals.x);- dst[base + 1] = T(vals.y);- dst[base + 2] = T(vals.z);- dst[base + 3] = T(vals.w);+ dst[base + 0] = T(vals.x);+ dst[base + 1] = T(vals.y);+ dst[base + 2] = T(vals.z);+ dst[base + 3] = T(vals.w);}-template<>+template <>inline void store_vec4<float>(device float* dst, ulong base, float4 vals) {- *reinterpret_cast<device float4*>(dst + base) = vals;+ *reinterpret_cast<device float4*>(dst + base) = vals;}-template<typename T>+template <typename T>struct StageStorage {- using type = float;+ using type = float;};-template<>+template <>struct StageStorage<half> {- using type = half;+ using type = half;};-template<>+template <>struct StageStorage<bfloat> {- using type = bfloat;+ using type = bfloat;};// 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up@@ -65,324 +67,408 @@ constant uint PASSA_BN = 32;constant uint PASSA_BK = 64;constant uint PASSA_SG_BM = 8;constant uint PASSA_SG_BN = 16;-constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts+constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts-template<typename T>+template <typename T>inline void pass_a_impl(device const T* X_perm,device const uint* expert_offsets,device const T* W13_all,device const T* up_biases,device float* hidden_out,- uint d_model, uint d_ff, uint E,- float gate_clip_min, float gate_clip_max,- float up_clip_min, float up_clip_max,+ uint d_model,+ uint d_ff,+ uint E,+ float gate_clip_min,+ float gate_clip_max,+ float up_clip_min,+ float up_clip_max,float silu_alpha,// work partition- uint tile_seg_start, uint expert_idx, uint tile_m, uint tile_n,+ uint tile_seg_start,+ uint expert_idx,+ uint tile_m,+ uint tile_n,// per-thread info- uint sg_id, uint3 threads_per_tg, uint3 local_tid,+ uint sg_id,+ uint3 threads_per_tg,+ uint3 local_tid,// TG storage- threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]- threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]- threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]- threadgroup float* bias_up, // [BN]- threadgroup float* bias_gate // [BN]+ threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]+ threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]+ threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]+ threadgroup float* bias_up, // [BN]+ threadgroup float* bias_gate // [BN]) {- using StageT = typename StageStorage<T>::type;- using StageVec4 = metal::vec<StageT, 4>;-- constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;- constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;- constexpr uint TG_PAD = PASSA_TG_PAD;- constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X- constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights-- if (expert_idx >= E) return;-- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;- if (seg_len == 0) return;-- const uint row_tg_off = tile_m * Bm;- const uint col_tg_off = tile_n * Bn;- if (row_tg_off >= seg_len || col_tg_off >= d_ff) return;-- const uint m_rows = min(Bm, seg_len - row_tg_off);- const uint n_cols = min(Bn, d_ff - col_tg_off);-- const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;- const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);-- // simdgroup tile mapping- const uint sg_col_count = Bn / SgBn;- const uint row_sg = sg_id / sg_col_count;- const uint col_sg = sg_id % sg_col_count;- const uint row_sg_off = row_sg * SgBm;- const uint col_sg_off = col_sg * SgBn;-- // Guard against misconfigured TG sizes- constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)- constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);- if (sg_id >= SG_EXPECTED) return;-- // partial accumulators (8x8 tiles per simdgroup)- constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE);- metal::simdgroup_float8x8 OutUp[TEMP];- metal::simdgroup_float8x8 OutGate[TEMP];- for (uint i = 0; i < TEMP; ++i) {- OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- if (GATING_SEL > 1u) {- OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- }+ using StageT = typename StageStorage<T>::type;+ using StageVec4 = metal::vec<StageT, 4>;++ constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;+ constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;+ constexpr uint TG_PAD = PASSA_TG_PAD;+ constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X+ constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights++ if (expert_idx >= E)+ return;++ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;+ if (seg_len == 0)+ return;++ const uint row_tg_off = tile_m * Bm;+ const uint col_tg_off = tile_n * Bn;+ if (row_tg_off >= seg_len || col_tg_off >= d_ff)+ return;++ const uint m_rows = min(Bm, seg_len - row_tg_off);+ const uint n_cols = min(Bn, d_ff - col_tg_off);++ const ulong x_base =+ ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;+ const ulong w13_expert_base =+ (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);++ // simdgroup tile mapping+ const uint sg_col_count = Bn / SgBn;+ const uint row_sg = sg_id / sg_col_count;+ const uint col_sg = sg_id % sg_col_count;+ const uint row_sg_off = row_sg * SgBm;+ const uint col_sg_off = col_sg * SgBn;++ // Guard against misconfigured TG sizes+ constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)+ constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);+ if (sg_id >= SG_EXPECTED)+ return;++ // partial accumulators (8x8 tiles per simdgroup)+ constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE);+ metal::simdgroup_float8x8 OutUp[TEMP];+ metal::simdgroup_float8x8 OutGate[TEMP];+ for (uint i = 0; i < TEMP; ++i) {+ OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ if (GATING_SEL > 1u) {+ OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);}+ }- const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;- const uint lin = linear_tid(local_tid, threads_per_tg);-- // K-loop (tile along D)- for (uint k_off = 0; k_off < d_model; k_off += Bk) {- const uint valid_k = min(Bk, d_model - k_off);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Stage LHS: Xs [Bm,Bk] with vec8 loads and padding ----- {- constexpr uint vec_size = 8;- constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8- constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128- const uint iters = ceil_div(total_vecs, threads_total);--UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint r = i / vec_cols;- const uint c8 = i % vec_cols;- const uint col_base = c8 * vec_size;-- threadgroup StageT* dst = Xs + r * X_LD + col_base;-- if (r < m_rows && col_base < valid_k) {- // Safe to read - at least first element is valid- const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);-UZU_PRAGMA_UNROLL- for (uint j = 0; j < vec_size; j++) {- dst[j] = (col_base + j < valid_k) ? StageT(float(X_perm[base + j])) : StageT(0.0f);- }- } else {- // Row out of bounds or entire vec8 chunk is out of K bounds-UZU_PRAGMA_UNROLL- for (uint j = 0; j < vec_size; j++) {- dst[j] = StageT(0.0f);- }- }- }- }- }+ const uint threads_total =+ threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;+ const uint lin = linear_tid(local_tid, threads_per_tg);- // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk] with vec8 loads and padding ----- {- constexpr uint vec_size = 8;- constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8- constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256- const uint iters = ceil_div(total_vecs, threads_total);--UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint n = i / vec_cols; // row within the BN tile- const uint c8 = i % vec_cols;- const uint col_base = c8 * vec_size;-- threadgroup StageT* up_dst = Wk_up + n * W_LD + col_base;- threadgroup StageT* gt_dst = Wk_gate + n * W_LD + col_base;-- if (n < n_cols && col_base < valid_k) {- // Safe to read - at least first element is valid- const uint gcol = col_tg_off + n;- const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + col_base);- const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + col_base);-UZU_PRAGMA_UNROLL- for (uint j = 0; j < vec_size; j++) {- up_dst[j] = (col_base + j < valid_k) ? StageT(float(W13_all[up_base + j])) : StageT(0.0f);- if (GATING_SEL > 1u) {- gt_dst[j] = (col_base + j < valid_k) ? StageT(float(W13_all[gate_base + j])) : StageT(0.0f);- }- }- } else {- // Row out of bounds or entire vec8 chunk is out of K bounds-UZU_PRAGMA_UNROLL- for (uint j = 0; j < vec_size; j++) {- up_dst[j] = StageT(0.0f);- if (GATING_SEL > 1u) gt_dst[j] = StageT(0.0f);- }- }- }- }- }+ // K-loop (tile along D)+ for (uint k_off = 0; k_off < d_model; k_off += Bk) {+ const uint valid_k = min(Bk, d_model - k_off);- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- MMA across the BK tile (in simdgroup-sized chunks) -----UZU_PRAGMA_UNROLL- for (uint kk = 0; kk < Bk; kk += SG_TILE) {-UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;- simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));--UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;- metal::simdgroup_matrix<StageT, 8, 8> rhs_up;- simdgroup_load(rhs_up, Wk_up, W_LD, ulong2(kk, col_sg_off + n_sub), true);- simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);-- if (GATING_SEL > 1u) {- metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;- simdgroup_load(rhs_gate, Wk_gate, W_LD, ulong2(kk, col_sg_off + n_sub), true);- simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);- }- }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Stage LHS: Xs [Bm,Bk] with vec8 loads and padding ----+ {+ constexpr uint vec_size = 8;+ constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8+ constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128+ const uint iters = ceil_div(total_vecs, threads_total);++ UZU_PRAGMA_UNROLL+ for (uint t = 0; t < iters; ++t) {+ const uint i = t * threads_total + lin;+ if (i < total_vecs) {+ const uint r = i / vec_cols;+ const uint c8 = i % vec_cols;+ const uint col_base = c8 * vec_size;++ threadgroup StageT* dst = Xs + r * X_LD + col_base;++ if (r < m_rows && col_base < valid_k) {+ // Safe to read - at least first element is valid+ const ulong base =+ x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);+ UZU_PRAGMA_UNROLL+ for (uint j = 0; j < vec_size; j++) {+ dst[j] = (col_base + j < valid_k)+ ? StageT(float(X_perm[base + j]))+ : StageT(0.0f);}+ } else {+ // Row out of bounds or entire vec8 chunk is out of K bounds+ UZU_PRAGMA_UNROLL+ for (uint j = 0; j < vec_size; j++) {+ dst[j] = StageT(0.0f);+ }+ }}- } // K loop-- // ---- Bias tile [Bn] ----- // Note: bias layout is [up | gate] in contiguous FF chunks.- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {- const uint c_global = col_tg_off + c_local;- bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;- if (GATING_SEL > 1u) {- bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;- }+ }}- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Epilogue: activation/gating → hidden_out (f32) ----- const uint sg_lane_start = sg_id * 32;- const uint sg_lane_end = sg_lane_start + 32;- if (lin < sg_lane_start || lin >= sg_lane_end) return;- const uint sg_lin = lin - sg_lane_start;- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment layout used above.- const uint lane_qid = sg_lin >> 2;- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);--UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;-UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;-- const uint tile_row_base = row_sg_off + m_sub;- const uint tile_col_base = col_sg_off + n_sub;- const uint local_row = tile_row_base + lane_row;- if (local_row >= m_rows || tile_col_base >= n_cols) {- continue;+ // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk] with vec8 loads and padding+ // ----+ {+ constexpr uint vec_size = 8;+ constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8+ constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256+ const uint iters = ceil_div(total_vecs, threads_total);++ UZU_PRAGMA_UNROLL+ for (uint t = 0; t < iters; ++t) {+ const uint i = t * threads_total + lin;+ if (i < total_vecs) {+ const uint n = i / vec_cols; // row within the BN tile+ const uint c8 = i % vec_cols;+ const uint col_base = c8 * vec_size;++ threadgroup StageT* up_dst = Wk_up + n * W_LD + col_base;+ threadgroup StageT* gt_dst = Wk_gate + n * W_LD + col_base;++ if (n < n_cols && col_base < valid_k) {+ // Safe to read - at least first element is valid+ const uint gcol = col_tg_off + n;+ const ulong up_base = w13_expert_base ++ (ulong)gcol * (ulong)d_model ++ (ulong)(k_off + col_base);+ const ulong gate_base = w13_expert_base ++ (ulong)(d_ff + gcol) * (ulong)d_model ++ (ulong)(k_off + col_base);+ UZU_PRAGMA_UNROLL+ for (uint j = 0; j < vec_size; j++) {+ up_dst[j] = (col_base + j < valid_k)+ ? StageT(float(W13_all[up_base + j]))+ : StageT(0.0f);+ if (GATING_SEL > 1u) {+ gt_dst[j] = (col_base + j < valid_k)+ ? StageT(float(W13_all[gate_base + j]))+ : StageT(0.0f);+ }}-- const auto up_frag = OutUp[tile].thread_elements();- float gate_frag_0 = 0.0f;- float gate_frag_1 = 0.0f;- if (GATING_SEL > 1u) {- const auto gate_frag = OutGate[tile].thread_elements();- gate_frag_0 = gate_frag[0];- gate_frag_1 = gate_frag[1];+ } else {+ // Row out of bounds or entire vec8 chunk is out of K bounds+ UZU_PRAGMA_UNROLL+ for (uint j = 0; j < vec_size; j++) {+ up_dst[j] = StageT(0.0f);+ if (GATING_SEL > 1u)+ gt_dst[j] = StageT(0.0f);}+ }+ }+ }+ }- const uint col0 = tile_col_base + lane_col_base;- const uint col1 = col0 + 1;- const ulong out_row = seg_start + row_tg_off + local_row;-- if (col0 < n_cols) {- float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);- float out_val;- if (GATING_SEL <= 1u) {- out_val =- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);- } else {- float gate_v =- clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);- const float gate_act =- (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);- out_val = gate_act * up_v;- }- const ulong out_col = col_tg_off + col0;- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;- }+ threadgroup_barrier(mem_flags::mem_threadgroup);- if (col1 < n_cols) {- float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);- float out_val;- if (GATING_SEL <= 1u) {- out_val =- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);- } else {- float gate_v =- clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);- const float gate_act =- (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);- out_val = gate_act * up_v;- }- const ulong out_col = col_tg_off + col1;- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;- }+ // ---- MMA across the BK tile (in simdgroup-sized chunks) ----+ UZU_PRAGMA_UNROLL+ for (uint kk = 0; kk < Bk; kk += SG_TILE) {+ UZU_PRAGMA_UNROLL+ for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {+ const uint r_idx = m_sub / SG_TILE;+ metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;+ simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));++ UZU_PRAGMA_UNROLL+ for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {+ const uint c_idx = n_sub / SG_TILE;+ const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;+ metal::simdgroup_matrix<StageT, 8, 8> rhs_up;+ simdgroup_load(+ rhs_up,+ Wk_up,+ W_LD,+ ulong2(kk, col_sg_off + n_sub),+ true+ );+ simdgroup_multiply_accumulate(+ OutUp[tile],+ lhs_frag,+ rhs_up,+ OutUp[tile]+ );++ if (GATING_SEL > 1u) {+ metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;+ simdgroup_load(+ rhs_gate,+ Wk_gate,+ W_LD,+ ulong2(kk, col_sg_off + n_sub),+ true+ );+ simdgroup_multiply_accumulate(+ OutGate[tile],+ lhs_frag,+ rhs_gate,+ OutGate[tile]+ );+ }}+ }}+ } // K loop++ // ---- Bias tile [Bn] ----+ // Note: bias layout is [up | gate] in contiguous FF chunks.+ for (uint c_local = lin; c_local < Bn; c_local += threads_total) {+ const uint c_global = col_tg_off + c_local;+ bias_up[c_local] =+ (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;+ if (GATING_SEL > 1u) {+ bias_gate[c_local] = (c_global < d_ff)+ ? float(up_biases[bias_base + d_ff + c_global])+ : 0.0f;+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Epilogue: activation/gating → hidden_out (f32) ----+ const uint sg_lane_start = sg_id * 32;+ const uint sg_lane_end = sg_lane_start + 32;+ if (lin < sg_lane_start || lin >= sg_lane_end)+ return;+ const uint sg_lin = lin - sg_lane_start;+ // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment+ // layout used above.+ const uint lane_qid = sg_lin >> 2;+ const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);++ UZU_PRAGMA_UNROLL+ for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {+ const uint c_idx = n_sub / SG_TILE;+ UZU_PRAGMA_UNROLL+ for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {+ const uint r_idx = m_sub / SG_TILE;+ const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;++ const uint tile_row_base = row_sg_off + m_sub;+ const uint tile_col_base = col_sg_off + n_sub;+ const uint local_row = tile_row_base + lane_row;+ if (local_row >= m_rows || tile_col_base >= n_cols) {+ continue;+ }++ const auto up_frag = OutUp[tile].thread_elements();+ float gate_frag_0 = 0.0f;+ float gate_frag_1 = 0.0f;+ if (GATING_SEL > 1u) {+ const auto gate_frag = OutGate[tile].thread_elements();+ gate_frag_0 = gate_frag[0];+ gate_frag_1 = gate_frag[1];+ }++ const uint col0 = tile_col_base + lane_col_base;+ const uint col1 = col0 + 1;+ const ulong out_row = seg_start + row_tg_off + local_row;++ if (col0 < n_cols) {+ float up_v =+ clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);+ float out_val;+ if (GATING_SEL <= 1u) {+ out_val =+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);+ } else {+ float gate_v = clamp(+ gate_frag_0 + bias_gate[col0],+ gate_clip_min,+ gate_clip_max+ );+ const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)+ : gelu_approx(gate_v);+ out_val = gate_act * up_v;+ }+ const ulong out_col = col_tg_off + col0;+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;+ }++ if (col1 < n_cols) {+ float up_v =+ clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);+ float out_val;+ if (GATING_SEL <= 1u) {+ out_val =+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);+ } else {+ float gate_v = clamp(+ gate_frag_1 + bias_gate[col1],+ gate_clip_min,+ gate_clip_max+ );+ const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)+ : gelu_approx(gate_v);+ out_val = gate_act * up_v;+ }+ const ulong out_col = col_tg_off + col1;+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;+ }+ }+ }}// ----- Kernel entry points (Pass A) ------#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \-kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* up_biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]]) \-{ \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup float bias_up[PASSA_BN]; \- threadgroup float bias_gate[PASSA_BN]; \- pass_a_impl<DTYPE>( \- X_perm, expert_offsets, W13_all, up_biases, hidden_out, \- d_model, d_ff, E, \- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \- expert_offsets[tg_pos.z], tg_pos.z, tg_pos.y, tg_pos.x, \- sg_id, threads_per_tg, local_tid, \- Xs, Wk_up, Wk_gate, bias_up, bias_gate); \-}+#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device const DTYPE* up_biases [[buffer(3)]], \+ device float* hidden_out [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint3 threads_per_tg [[threads_per_threadgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]] \+ ) { \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup float bias_up[PASSA_BN]; \+ threadgroup float bias_gate[PASSA_BN]; \+ pass_a_impl<DTYPE>( \+ X_perm, \+ expert_offsets, \+ W13_all, \+ up_biases, \+ hidden_out, \+ d_model, \+ d_ff, \+ E, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ silu_alpha, \+ expert_offsets[tg_pos.z], \+ tg_pos.z, \+ tg_pos.y, \+ tg_pos.x, \+ sg_id, \+ threads_per_tg, \+ local_tid, \+ Xs, \+ Wk_up, \+ Wk_gate, \+ bias_up, \+ bias_gate \+ ); \+ }MOE_PASS_A_KERNEL(bfloat, bf16)MOE_PASS_A_KERNEL(half, f16)MOE_PASS_A_KERNEL(float, f32)// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)-template<typename T>+template <typename T>inline void pass_a_indirect_impl(device const T* X_perm,device const uint* expert_offsets,@@ -390,298 +476,375 @@ inline void pass_a_indirect_impl(device const T* up_biases,device float* hidden_out,device const uint* tile_map,- uint d_model, uint d_ff, uint E,- float gate_clip_min, float gate_clip_max,- float up_clip_min, float up_clip_max,+ uint d_model,+ uint d_ff,+ uint E,+ float gate_clip_min,+ float gate_clip_max,+ float up_clip_min,+ float up_clip_max,float silu_alpha,- uint row_tile_idx, uint n_tile_idx,- uint sg_id, uint3 threads_per_tg, uint3 local_tid,+ uint row_tile_idx,+ uint n_tile_idx,+ uint sg_id,+ uint3 threads_per_tg,+ uint3 local_tid,threadgroup typename StageStorage<T>::type* Xs,threadgroup typename StageStorage<T>::type* Wk_up,threadgroup typename StageStorage<T>::type* Wk_gate,threadgroup float* bias_up,- threadgroup float* bias_gate)-{- constexpr uint ROW_TILE = PASSA_BM;- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= E) return;- const uint tile_seg_start = tile_map[base + 1u];- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;-- pass_a_impl<T>(- X_perm, expert_offsets, W13_all, up_biases, hidden_out,- d_model, d_ff, E,- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha,- tile_seg_start, expert_idx, tile_m, n_tile_idx,- sg_id, threads_per_tg, local_tid,- Xs, Wk_up, Wk_gate, bias_up, bias_gate);+ threadgroup float* bias_gate+) {+ constexpr uint ROW_TILE = PASSA_BM;+ const uint base = row_tile_idx * 3u;+ const uint expert_idx = tile_map[base + 0u];+ if (expert_idx >= E)+ return;+ const uint tile_seg_start = tile_map[base + 1u];+ const uint row_off_elems = tile_map[base + 2u];+ const uint tile_m = row_off_elems / ROW_TILE;++ pass_a_impl<T>(+ X_perm,+ expert_offsets,+ W13_all,+ up_biases,+ hidden_out,+ d_model,+ d_ff,+ E,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ tile_seg_start,+ expert_idx,+ tile_m,+ n_tile_idx,+ sg_id,+ threads_per_tg,+ local_tid,+ Xs,+ Wk_up,+ Wk_gate,+ bias_up,+ bias_gate+ );}-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \-kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* up_biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- device const uint* tile_map [[buffer(13)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]]) \-{ \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup float bias_up[PASSA_BN]; \- threadgroup float bias_gate[PASSA_BN]; \- pass_a_indirect_impl<DTYPE>( \- X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \- d_model, d_ff, E, \- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \- tg_pos.y, tg_pos.x, \- sg_id, threads_per_tg, local_tid, \- Xs, Wk_up, Wk_gate, bias_up, bias_gate); \-}+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device const DTYPE* up_biases [[buffer(3)]], \+ device float* hidden_out [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ device const uint* tile_map [[buffer(13)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint3 threads_per_tg [[threads_per_threadgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]] \+ ) { \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup float bias_up[PASSA_BN]; \+ threadgroup float bias_gate[PASSA_BN]; \+ pass_a_indirect_impl<DTYPE>( \+ X_perm, \+ expert_offsets, \+ W13_all, \+ up_biases, \+ hidden_out, \+ tile_map, \+ d_model, \+ d_ff, \+ E, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ silu_alpha, \+ tg_pos.y, \+ tg_pos.x, \+ sg_id, \+ threads_per_tg, \+ local_tid, \+ Xs, \+ Wk_up, \+ Wk_gate, \+ bias_up, \+ bias_gate \+ ); \+ }MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)MOE_PASS_A_INDIRECT_KERNEL(half, f16)MOE_PASS_A_INDIRECT_KERNEL(float, f32)-// ------------------------ Pass B (hidden @ W2 → output) -------------------------// Optimized config: 16x64 output tile, 4 simdgroups (128 threads)+// ------------------------ Pass B (hidden @ W2 → output)+// ------------------------ Optimized config: 16x64 output tile, 4 simdgroups+// (128 threads)constant uint PASSB_BM = 16;constant uint PASSB_BN = 64;constant uint PASSB_BK = 64;-constant uint PASSB_SG_BM = 8; // Each simdgroup handles 8x32+constant uint PASSB_SG_BM = 8; // Each simdgroup handles 8x32constant uint PASSB_SG_BN = 32;-constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts+constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts-template<typename T>+template <typename T>inline void pass_b_impl(device const float* hidden,device const uint* expert_offsets,device const T* W2_all,device const T* down_biases,device T* output,- uint d_model, uint d_ff, uint E,- uint expert_idx, uint tile_m, uint tile_n,- uint sg_id, uint simd_lid, uint lin,+ uint d_model,+ uint d_ff,+ uint E,+ uint expert_idx,+ uint tile_m,+ uint tile_n,+ uint sg_id,+ uint simd_lid,+ uint lin,threadgroup float* Hs,threadgroup typename StageStorage<T>::type* Wk,threadgroup float* bias_tile) {- using StageT = typename StageStorage<T>::type;-- constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;- constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;- constexpr uint TG_PAD = PASSB_TG_PAD;- constexpr uint H_LD = Bk + TG_PAD; // Padded stride for hidden- constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights- constexpr uint SG_TILE = 8;- constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads-- if (expert_idx >= E) return;-- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;- if (seg_len == 0) return;-- const uint row_tg_off = tile_m * Bm;- const uint col_tg_off = tile_n * Bn;- if (row_tg_off >= seg_len || col_tg_off >= d_model) return;-- const uint m_rows = min(Bm, seg_len - row_tg_off);- const uint n_cols = min(Bn, d_model - col_tg_off);-- const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;- const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;- const ulong bias_base = (ulong)expert_idx * (ulong)d_model;-- // 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)- const uint row_sg = sg_id / 2; // 0-1 → rows- const uint col_sg = sg_id % 2; // 0-1 → cols- const uint row_sg_off = row_sg * SgBm;- const uint col_sg_off = col_sg * SgBn;-- // Guard against excessive simdgroups- constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);- if (sg_id >= SG_EXPECTED) return;-- // 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles- constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4- metal::simdgroup_float8x8 Out[TEMP];-UZU_PRAGMA_UNROLL- for (uint i = 0; i < TEMP; ++i) {- Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- }-- // Cooperative load config - vec8 loads for better bandwidth- constexpr uint H_VEC = 8;- constexpr uint H_TCOLS = Bk / H_VEC; // 64/8 = 8- constexpr uint H_TROWS = TGP_SIZE / H_TCOLS; // 128/8 = 16- const uint h_bi = lin / H_TCOLS;- const uint h_bj = (lin % H_TCOLS) * H_VEC;-- constexpr uint W_VEC = 8;- constexpr uint W_TCOLS = Bk / W_VEC; // 64/8 = 8- constexpr uint W_TROWS = TGP_SIZE / W_TCOLS; // 128/8 = 16- const uint w_bi = lin / W_TCOLS;- const uint w_bj = (lin % W_TCOLS) * W_VEC;-- // K-loop across FF- for (uint k_off = 0; k_off < d_ff; k_off += Bk) {- const uint valid_k = min(Bk, d_ff - k_off);- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Stage hidden [Bm=16, Bk=64] with vec8 loads ----- // 128 threads, 8 vec8 loads per row → 16 rows covered in one pass- {- threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;- if (h_bi < m_rows && h_bj < valid_k) {- // Safe to read - at least first element is valid- device const float* my_src = hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);-UZU_PRAGMA_UNROLL- for (uint j = 0; j < H_VEC; j++) {- my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;- }- } else {-UZU_PRAGMA_UNROLL- for (uint j = 0; j < H_VEC; j++) {- my_dst[j] = 0.0f;- }- }- }+ using StageT = typename StageStorage<T>::type;++ constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;+ constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;+ constexpr uint TG_PAD = PASSB_TG_PAD;+ constexpr uint H_LD = Bk + TG_PAD; // Padded stride for hidden+ constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights+ constexpr uint SG_TILE = 8;+ constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads++ if (expert_idx >= E)+ return;++ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;+ if (seg_len == 0)+ return;++ const uint row_tg_off = tile_m * Bm;+ const uint col_tg_off = tile_n * Bn;+ if (row_tg_off >= seg_len || col_tg_off >= d_model)+ return;++ const uint m_rows = min(Bm, seg_len - row_tg_off);+ const uint n_cols = min(Bn, d_model - col_tg_off);++ const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;+ const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;+ const ulong bias_base = (ulong)expert_idx * (ulong)d_model;++ // 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)+ const uint row_sg = sg_id / 2; // 0-1 → rows+ const uint col_sg = sg_id % 2; // 0-1 → cols+ const uint row_sg_off = row_sg * SgBm;+ const uint col_sg_off = col_sg * SgBn;++ // Guard against excessive simdgroups+ constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);+ if (sg_id >= SG_EXPECTED)+ return;++ // 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles+ constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4+ metal::simdgroup_float8x8 Out[TEMP];+ UZU_PRAGMA_UNROLL+ for (uint i = 0; i < TEMP; ++i) {+ Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ }++ // Cooperative load config - vec8 loads for better bandwidth+ constexpr uint H_VEC = 8;+ constexpr uint H_TCOLS = Bk / H_VEC; // 64/8 = 8+ constexpr uint H_TROWS = TGP_SIZE / H_TCOLS; // 128/8 = 16+ const uint h_bi = lin / H_TCOLS;+ const uint h_bj = (lin % H_TCOLS) * H_VEC;++ constexpr uint W_VEC = 8;+ constexpr uint W_TCOLS = Bk / W_VEC; // 64/8 = 8+ constexpr uint W_TROWS = TGP_SIZE / W_TCOLS; // 128/8 = 16+ const uint w_bi = lin / W_TCOLS;+ const uint w_bj = (lin % W_TCOLS) * W_VEC;++ // K-loop across FF+ for (uint k_off = 0; k_off < d_ff; k_off += Bk) {+ const uint valid_k = min(Bk, d_ff - k_off);+ threadgroup_barrier(mem_flags::mem_threadgroup);- // ---- Stage W2 [Bn=64, Bk=64] with vec8 loads ----- // Need 4 passes: 128 threads cover 16 rows, Bn=64 needs 4×16- {- for (uint pass = 0; pass < 4; pass++) {- const uint row = w_bi + pass * W_TROWS;- threadgroup StageT* my_dst = Wk + row * W_LD + w_bj;- if (row < n_cols && w_bj < valid_k) {- // Safe to read - at least first element is valid- const uint gcol = col_tg_off + row;- device const T* my_src = W2_all + w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + w_bj);-UZU_PRAGMA_UNROLL- for (uint j = 0; j < W_VEC; j++) {- my_dst[j] = (w_bj + j < valid_k) ? StageT(float(my_src[j])) : StageT(0.0f);- }- } else {-UZU_PRAGMA_UNROLL- for (uint j = 0; j < W_VEC; j++) {- my_dst[j] = StageT(0.0f);- }- }- }+ // ---- Stage hidden [Bm=16, Bk=64] with vec8 loads ----+ // 128 threads, 8 vec8 loads per row → 16 rows covered in one pass+ {+ threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;+ if (h_bi < m_rows && h_bj < valid_k) {+ // Safe to read - at least first element is valid+ device const float* my_src =+ hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);+ UZU_PRAGMA_UNROLL+ for (uint j = 0; j < H_VEC; j++) {+ my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;}-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Serpentine MMA: 1×4 tiles per simdgroup -----UZU_PRAGMA_UNROLL- for (uint kk = 0; kk < Bk; kk += SG_TILE) {- // Load 1 LHS fragment- metal::simdgroup_float8x8 lhs;- simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));-- // Load 4 RHS fragments (cols)- metal::simdgroup_matrix<StageT, 8, 8> rhs0, rhs1, rhs2, rhs3;- simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);- simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);- simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);- simdgroup_load(rhs3, Wk, W_LD, ulong2(kk, col_sg_off + 24), true);-- // Serpentine pattern: 0→1→2→3 (forward for first row)- simdgroup_multiply_accumulate(Out[0], lhs, rhs0, Out[0]);- simdgroup_multiply_accumulate(Out[1], lhs, rhs1, Out[1]);- simdgroup_multiply_accumulate(Out[2], lhs, rhs2, Out[2]);- simdgroup_multiply_accumulate(Out[3], lhs, rhs3, Out[3]);+ } else {+ UZU_PRAGMA_UNROLL+ for (uint j = 0; j < H_VEC; j++) {+ my_dst[j] = 0.0f;}+ }}- // ---- Bias tile (cooperative load) ----- for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {- const uint c_global = col_tg_off + c_local;- bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;+ // ---- Stage W2 [Bn=64, Bk=64] with vec8 loads ----+ // Need 4 passes: 128 threads cover 16 rows, Bn=64 needs 4×16+ {+ for (uint pass = 0; pass < 4; pass++) {+ const uint row = w_bi + pass * W_TROWS;+ threadgroup StageT* my_dst = Wk + row * W_LD + w_bj;+ if (row < n_cols && w_bj < valid_k) {+ // Safe to read - at least first element is valid+ const uint gcol = col_tg_off + row;+ device const T* my_src = W2_all + w2_expert_base ++ (ulong)gcol * (ulong)d_ff ++ (ulong)(k_off + w_bj);+ UZU_PRAGMA_UNROLL+ for (uint j = 0; j < W_VEC; j++) {+ my_dst[j] =+ (w_bj + j < valid_k) ? StageT(float(my_src[j])) : StageT(0.0f);+ }+ } else {+ UZU_PRAGMA_UNROLL+ for (uint j = 0; j < W_VEC; j++) {+ my_dst[j] = StageT(0.0f);+ }+ }+ }}threadgroup_barrier(mem_flags::mem_threadgroup);- // ---- Writeout with fragment extraction ----- const uint lane_qid = simd_lid >> 2;- const uint lane_row = (lane_qid & 4u) + ((simd_lid >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((simd_lid & 1u) << 1);-- const uint local_row = row_sg_off + lane_row;- if (local_row >= m_rows) return;- const ulong out_row = seg_start + row_tg_off + local_row;--UZU_PRAGMA_UNROLL- for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup- const auto accum = Out[ni].thread_elements();- const uint local_col = col_sg_off + ni * 8 + lane_col_base;-- if (local_col < n_cols) {- const ulong out_col = col_tg_off + local_col;- const float val = accum[0] + bias_tile[local_col];- output[out_row * (ulong)d_model + out_col] = T(val);- }- if (local_col + 1 < n_cols) {- const ulong out_col = col_tg_off + local_col + 1;- const float val = accum[1] + bias_tile[local_col + 1];- output[out_row * (ulong)d_model + out_col] = T(val);- }+ // ---- Serpentine MMA: 1×4 tiles per simdgroup ----+ UZU_PRAGMA_UNROLL+ for (uint kk = 0; kk < Bk; kk += SG_TILE) {+ // Load 1 LHS fragment+ metal::simdgroup_float8x8 lhs;+ simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));++ // Load 4 RHS fragments (cols)+ metal::simdgroup_matrix<StageT, 8, 8> rhs0, rhs1, rhs2, rhs3;+ simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);+ simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);+ simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);+ simdgroup_load(rhs3, Wk, W_LD, ulong2(kk, col_sg_off + 24), true);++ // Serpentine pattern: 0→1→2→3 (forward for first row)+ simdgroup_multiply_accumulate(Out[0], lhs, rhs0, Out[0]);+ simdgroup_multiply_accumulate(Out[1], lhs, rhs1, Out[1]);+ simdgroup_multiply_accumulate(Out[2], lhs, rhs2, Out[2]);+ simdgroup_multiply_accumulate(Out[3], lhs, rhs3, Out[3]);+ }+ }++ // ---- Bias tile (cooperative load) ----+ for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {+ const uint c_global = col_tg_off + c_local;+ bias_tile[c_local] =+ (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Writeout with fragment extraction ----+ const uint lane_qid = simd_lid >> 2;+ const uint lane_row = (lane_qid & 4u) + ((simd_lid >> 1) & 3u);+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((simd_lid & 1u) << 1);++ const uint local_row = row_sg_off + lane_row;+ if (local_row >= m_rows)+ return;+ const ulong out_row = seg_start + row_tg_off + local_row;++ UZU_PRAGMA_UNROLL+ for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup+ const auto accum = Out[ni].thread_elements();+ const uint local_col = col_sg_off + ni * 8 + lane_col_base;++ if (local_col < n_cols) {+ const ulong out_col = col_tg_off + local_col;+ const float val = accum[0] + bias_tile[local_col];+ output[out_row * (ulong)d_model + out_col] = T(val);}+ if (local_col + 1 < n_cols) {+ const ulong out_col = col_tg_off + local_col + 1;+ const float val = accum[1] + bias_tile[local_col + 1];+ output[out_row * (ulong)d_model + out_col] = T(val);+ }+ }}// ----- Kernel entry points (Pass B) ------#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \-kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* output [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]]) \-{ \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \- threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \- threadgroup float bias[PASSB_BN]; \- const uint lin = local_tid.x; \- pass_b_impl<DTYPE>( \- hidden, expert_offsets, W2_all, down_biases, output, \- d_model, d_ff, E, \- tg_pos.z, tg_pos.y, tg_pos.x, \- sg_id, simd_lid, lin, \- Hs, Wk, bias); \-}+#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* output [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]] \+ ) { \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \+ threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \+ threadgroup float bias[PASSB_BN]; \+ const uint lin = local_tid.x; \+ pass_b_impl<DTYPE>( \+ hidden, \+ expert_offsets, \+ W2_all, \+ down_biases, \+ output, \+ d_model, \+ d_ff, \+ E, \+ tg_pos.z, \+ tg_pos.y, \+ tg_pos.x, \+ sg_id, \+ simd_lid, \+ lin, \+ Hs, \+ Wk, \+ bias \+ ); \+ }MOE_PASS_B_KERNEL(bfloat, bf16)MOE_PASS_B_KERNEL(half, f16)MOE_PASS_B_KERNEL(float, f32)// Indirect variant-template<typename T>+template <typename T>inline void pass_b_indirect_impl(device const float* hidden,device const uint* expert_offsets,@@ -689,56 +852,88 @@ inline void pass_b_indirect_impl(device const T* down_biases,device T* output,device const uint* tile_map,- uint d_model, uint d_ff, uint E,- uint row_tile_idx, uint n_tile_idx,- uint sg_id, uint simd_lid, uint lin,+ uint d_model,+ uint d_ff,+ uint E,+ uint row_tile_idx,+ uint n_tile_idx,+ uint sg_id,+ uint simd_lid,+ uint lin,threadgroup float* Hs,threadgroup typename StageStorage<T>::type* Wk,- threadgroup float* bias_tile)-{- constexpr uint ROW_TILE = PASSB_BM;- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= E) return;- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;-- pass_b_impl<T>(- hidden, expert_offsets, W2_all, down_biases, output,- d_model, d_ff, E,- expert_idx, tile_m, n_tile_idx,- sg_id, simd_lid, lin,- Hs, Wk, bias_tile);+ threadgroup float* bias_tile+) {+ constexpr uint ROW_TILE = PASSB_BM;+ const uint base = row_tile_idx * 3u;+ const uint expert_idx = tile_map[base + 0u];+ if (expert_idx >= E)+ return;+ const uint row_off_elems = tile_map[base + 2u];+ const uint tile_m = row_off_elems / ROW_TILE;++ pass_b_impl<T>(+ hidden,+ expert_offsets,+ W2_all,+ down_biases,+ output,+ d_model,+ d_ff,+ E,+ expert_idx,+ tile_m,+ n_tile_idx,+ sg_id,+ simd_lid,+ lin,+ Hs,+ Wk,+ bias_tile+ );}-#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \-kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* output [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- device const uint* tile_map [[buffer(8)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]]) \-{ \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \- threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \- threadgroup float bias[PASSB_BN]; \- const uint lin = local_tid.x; \- pass_b_indirect_impl<DTYPE>( \- hidden, expert_offsets, W2_all, down_biases, output, tile_map, \- d_model, d_ff, E, \- tg_pos.y, tg_pos.x, \- sg_id, simd_lid, lin, \- Hs, Wk, bias); \-}+#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* output [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ device const uint* tile_map [[buffer(8)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]] \+ ) { \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \+ threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \+ threadgroup float bias[PASSB_BN]; \+ const uint lin = local_tid.x; \+ pass_b_indirect_impl<DTYPE>( \+ hidden, \+ expert_offsets, \+ W2_all, \+ down_biases, \+ output, \+ tile_map, \+ d_model, \+ d_ff, \+ E, \+ tg_pos.y, \+ tg_pos.x, \+ sg_id, \+ simd_lid, \+ lin, \+ Hs, \+ Wk, \+ bias \+ ); \+ }MOE_PASS_B_INDIRECT_KERNEL(bfloat, bf16)MOE_PASS_B_INDIRECT_KERNEL(half, f16)diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rsindex 9dc1853ba..3eff065a6 100644--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs@@ -1,8 +1,5 @@use crate::backends::metal::KernelDataType;-mod encodable;-pub use encodable::{MoeBlockEncodable, SharedMoeWeights};-// Submodulesmod counts_offsets_fused;mod experts;diff --git a/crates/uzu/src/backends/metal/kernel/pooling/mod.rs b/crates/uzu/src/backends/metal/kernel/pooling/mod.rsnew file mode 100644index 000000000..1e0068168--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/pooling/mod.rs@@ -0,0 +1,110 @@+use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,+ MTLSize,+};++use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError},+};++pub struct PoolingKernel {+ cls_pipeline: ComputePipelineState,+ mean_pipeline: ComputePipelineState,+}++impl PoolingKernel {+ fn kernel_name_for_type(+ data_type: DataType,+ pooling_type: &str,+ ) -> Result<String, MTLError> {+ let suffix = match data_type {+ DataType::F16 => "f16",+ DataType::F32 => "f32",+ DataType::BF16 => "bf16",+ other => {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for pooling: {:?}",+ other+ )));+ },+ };+ Ok(format!("pool_{}_{}", pooling_type, suffix))+ }++ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ ) -> Result<Self, MTLError> {+ let cls_fn_name = Self::kernel_name_for_type(data_type, "cls")?;+ let mean_fn_name = Self::kernel_name_for_type(data_type, "mean")?;++ let cls_pipeline =+ context.compute_pipeline_state(&cls_fn_name, None)?;+ let mean_pipeline =+ context.compute_pipeline_state(&mean_fn_name, None)?;++ Ok(Self {+ cls_pipeline,+ mean_pipeline,+ })+ }++ pub fn encode_cls(+ &self,+ encoder: &ComputeCommandEncoderRef,+ input_buffer: &MTLBuffer,+ output_buffer: &MTLBuffer,+ batch_size: i32,+ seq_len: i32,+ hidden_dim: i32,+ ) -> Result<(), MTLError> {+ encoder.set_compute_pipeline_state(&self.cls_pipeline);+ encoder.set_buffer(0, Some(input_buffer), 0);+ encoder.set_buffer(1, Some(output_buffer), 0);+ encoder.set_bytes(+ 2,+ std::mem::size_of::<i32>() as u64,+ &seq_len as *const i32 as *const _,+ );+ encoder.set_bytes(+ 3,+ std::mem::size_of::<i32>() as u64,+ &hidden_dim as *const i32 as *const _,+ );++ let threads_per_tg = MTLSize::new(16, 16, 1);+ let grid = MTLSize::new(hidden_dim as u64, batch_size as u64, 1);+ encoder.dispatch_threads(grid, threads_per_tg);+ Ok(())+ }++ pub fn encode_mean(+ &self,+ encoder: &ComputeCommandEncoderRef,+ input_buffer: &MTLBuffer,+ output_buffer: &MTLBuffer,+ batch_size: i32,+ seq_len: i32,+ hidden_dim: i32,+ ) -> Result<(), MTLError> {+ encoder.set_compute_pipeline_state(&self.mean_pipeline);+ encoder.set_buffer(0, Some(input_buffer), 0);+ encoder.set_buffer(1, Some(output_buffer), 0);+ encoder.set_bytes(+ 2,+ std::mem::size_of::<i32>() as u64,+ &seq_len as *const i32 as *const _,+ );+ encoder.set_bytes(+ 3,+ std::mem::size_of::<i32>() as u64,+ &hidden_dim as *const i32 as *const _,+ );++ let threads_per_tg = MTLSize::new(16, 16, 1);+ let grid = MTLSize::new(hidden_dim as u64, batch_size as u64, 1);+ encoder.dispatch_threads(grid, threads_per_tg);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metalnew file mode 100644index 000000000..e2c0eb7f2--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal@@ -0,0 +1,96 @@+#include <metal_stdlib>+#include "../definitions.metal"+using namespace metal;++// CLS pooling: Extract first token [batch, seq_len, hidden_dim] → [batch,+// hidden_dim]+template <typename T>+kernel void pool_cls(+ const device T* input [[buffer(0)]],+ device T* output [[buffer(1)]],+ constant int& seq_len [[buffer(2)]],+ constant int& hidden_dim [[buffer(3)]],+ uint2 tid [[thread_position_in_grid]]+) // x: hidden_dim, y: batch+{+ int batch_idx = tid.y;+ int dim_idx = tid.x;+ if (dim_idx >= hidden_dim)+ return;++ output[batch_idx * hidden_dim + dim_idx] =+ input[batch_idx * seq_len * hidden_dim + dim_idx];+}++// Mean pooling: Average across sequence [batch, seq_len, hidden_dim] → [batch,+// hidden_dim]+template <typename T>+kernel void pool_mean(+ const device T* input [[buffer(0)]],+ device T* output [[buffer(1)]],+ constant int& seq_len [[buffer(2)]],+ constant int& hidden_dim [[buffer(3)]],+ uint2 tid [[thread_position_in_grid]]+) {+ int batch_idx = tid.y;+ int dim_idx = tid.x;+ if (dim_idx >= hidden_dim)+ return;++ float sum = 0.0f;+ for (int seq_idx = 0; seq_idx < seq_len; seq_idx++) {+ sum += float(+ input[batch_idx * seq_len * hidden_dim + seq_idx * hidden_dim + dim_idx]+ );+ }+ output[batch_idx * hidden_dim + dim_idx] = T(sum / float(seq_len));+}++// Explicit instantiations for f16, f32, bf16+template [[host_name("pool_cls_f16")]] [[kernel]] void pool_cls<half>(+ const device half* input [[buffer(0)]],+ device half* output [[buffer(1)]],+ constant int& seq_len [[buffer(2)]],+ constant int& hidden_dim [[buffer(3)]],+ uint2 tid [[thread_position_in_grid]]+);++template [[host_name("pool_cls_f32")]] [[kernel]] void pool_cls<float>(+ const device float* input [[buffer(0)]],+ device float* output [[buffer(1)]],+ constant int& seq_len [[buffer(2)]],+ constant int& hidden_dim [[buffer(3)]],+ uint2 tid [[thread_position_in_grid]]+);++template [[host_name("pool_cls_bf16")]] [[kernel]] void pool_cls<bfloat>(+ const device bfloat* input [[buffer(0)]],+ device bfloat* output [[buffer(1)]],+ constant int& seq_len [[buffer(2)]],+ constant int& hidden_dim [[buffer(3)]],+ uint2 tid [[thread_position_in_grid]]+);++template [[host_name("pool_mean_f16")]] [[kernel]] void pool_mean<half>(+ const device half* input [[buffer(0)]],+ device half* output [[buffer(1)]],+ constant int& seq_len [[buffer(2)]],+ constant int& hidden_dim [[buffer(3)]],+ uint2 tid [[thread_position_in_grid]]+);++template [[host_name("pool_mean_f32")]] [[kernel]] void pool_mean<float>(+ const device float* input [[buffer(0)]],+ device float* output [[buffer(1)]],+ constant int& seq_len [[buffer(2)]],+ constant int& hidden_dim [[buffer(3)]],+ uint2 tid [[thread_position_in_grid]]+);++template [[host_name("pool_mean_bf16")]] [[kernel]] void pool_mean<bfloat>(+ const device bfloat* input [[buffer(0)]],+ device bfloat* output [[buffer(1)]],+ constant int& seq_len [[buffer(2)]],+ constant int& hidden_dim [[buffer(3)]],+ uint2 tid [[thread_position_in_grid]]+);diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rsindex 3753f8d3a..39b476540 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs@@ -1,23 +1,13 @@-use std::{mem::size_of, rc::Rc};+use std::mem::size_of;use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef,ComputePipelineState as MTLComputePipelineState, MTLSize,};-use mpsgraph::CommandBuffer as MPSCommandBuffer;use crate::{- Array, DataType,- backends::metal::{- MTLContext, MTLError,- forward_pass::{- ArrayId, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- metal_extensions::ComputeEncoderConditional,- },- config::{RMSNormConfig, UpcastMode},- parameters::ParameterTree,+ DataType,+ backends::metal::{MTLContext, MTLError},};#[derive(Debug, Clone, Copy, PartialEq)]@@ -209,12 +199,10 @@ impl RMSNormKernel {&self,compute_encoder: &ComputeCommandEncoderRef,args: RMSNormArguments,- ) {- assert_eq!(- self.kernel_type,- RMSNormKernelType::Standard,- "Invalid kernel type for standard RMS norm operation"- );+ ) -> Result<(), RMSNormError> {+ if self.kernel_type != RMSNormKernelType::Standard {+ return Err(RMSNormError::InvalidKernelType);+ }compute_encoder.set_compute_pipeline_state(&self.pipeline);@@ -261,18 +249,18 @@ impl RMSNormKernel {threadgroups_per_grid,threads_per_threadgroup,);++ Ok(())}pub fn encode_qk_norm(&self,compute_encoder: &ComputeCommandEncoderRef,args: QKNormArguments,- ) {- assert_eq!(- self.kernel_type,- RMSNormKernelType::QueryKey,- "Invalid kernel type for QK norm operation"- );+ ) -> Result<(), RMSNormError> {+ if self.kernel_type != RMSNormKernelType::QueryKey {+ return Err(RMSNormError::InvalidKernelType);+ }compute_encoder.set_compute_pipeline_state(&self.pipeline);@@ -353,417 +341,7 @@ impl RMSNormKernel {threadgroups_per_grid,threads_per_threadgroup,);- }-}--pub struct RMSNormKernelEncodable {- kernel: RMSNormKernel,- config: RMSNormConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,- scales_buffer: MTLBuffer,-}--impl RMSNormKernelEncodable {- pub fn new(- context: &MTLContext,- intermediate_data_type: DataType,- config: RMSNormConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, RMSNormError> {- // Load scales from parameter tree- let scales_param = parameter_tree.leaf("scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(- crate::backends::metal::error::LibraryError::Custom(format!(- "Failed to load scales: {:?}",- e- )),- ))- })?;-- // TODO: Don't create buffers dynamically, we need to use forward pass storage for thing like this- let scales_data = scales_param.buffer();- let scales_buffer = context.device.new_buffer_with_data(- scales_data.as_ptr() as *const _,- scales_data.len() as u64,- metal::MTLResourceOptions::StorageModeShared,- );-- let accumulation_data_type: DataType =- config.accumulation_precision.into();- let scale_data_type: DataType = config.scale_precision.into();-- let (input_type, scales_type, output_type) = match config.upcast_mode {- UpcastMode::OnlyNormalization => {- // Input stays as pipeline type, scales stay scale precision, output is scale precision- (intermediate_data_type, scale_data_type, scale_data_type)- },- UpcastMode::FullLayer => {- // Input stays as pipeline type, scales stay in original precision (will be cast to AccumT inside kernel), output is scale precision- (intermediate_data_type, scale_data_type, scale_data_type)- },- };-- let kernel = RMSNormKernel::new_with_mode(- context,- input_type,- scales_type,- output_type,- accumulation_data_type,- RMSNormKernelType::Standard,- config.upcast_mode == UpcastMode::FullLayer,- )?;-- Ok(Self {- kernel,- config,- input_array_id,- output_array_id,- scales_buffer,- })- }-}--impl EncodableWithState for RMSNormKernelEncodable {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let input_binding = state.arrays(&[self.input_array_id]);- let output_binding = state.arrays(&[self.output_array_id]);-- let input_shape = {- let input_array = input_binding[0].borrow();- input_array.shape().to_vec()- };-- let mut input_array = input_binding[0].borrow_mut();- let mut output_array = output_binding[0].borrow_mut();-- let input_buffer = unsafe { input_array.mtl_buffer() };- let output_buffer = unsafe { output_array.mtl_buffer() };-- let batch_size = input_shape[0] as i32;- let model_dim = input_shape[1] as i32;-- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();-- compute_encoder.condition(- parameters.predicate_ref(),- || {- self.kernel.encode(- &compute_encoder,- RMSNormArguments {- input_buffer: &input_buffer,- scales_buffer: &self.scales_buffer,- output_buffer: &output_buffer,- batch_size,- model_dim,- epsilon: self.config.epsilon,- scale_offset: self.config.scale_offset.unwrap_or(0.0),- },- );- },- None::<fn()>,- );-- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- let input_binding = state.arrays(&[self.input_array_id]);- let output_binding = state.arrays(&[self.output_array_id]);-- let input_shape = {- let input_array = input_binding[0].borrow();- input_array.shape().to_vec()- };-- let mut input_array = input_binding[0].borrow_mut();- let mut output_array = output_binding[0].borrow_mut();-- let input_buffer = unsafe { input_array.mtl_buffer() };- let output_buffer = unsafe { output_array.mtl_buffer() };-- let batch_size = input_shape[0] as i32;- let model_dim = input_shape[1] as i32;-- self.kernel.encode(- encoder,- RMSNormArguments {- input_buffer: &input_buffer,- scales_buffer: &self.scales_buffer,- output_buffer: &output_buffer,- batch_size,- model_dim,- epsilon: self.config.epsilon,- scale_offset: self.config.scale_offset.unwrap_or(0.0),- },- );- }-}--pub struct QKNormKernelEncodable {- query_kernel: Option<RMSNormKernel>,- key_kernel: Option<RMSNormKernel>,- query_config: Option<RMSNormConfig>,- key_config: Option<RMSNormConfig>,- qkv_array_id: ArrayId,- query_scales_buffer: Option<MTLBuffer>,- key_scales_buffer: Option<MTLBuffer>,- num_q_heads: usize,- num_kv_heads: usize,- head_dim: usize,-}--impl QKNormKernelEncodable {- pub fn new(- context: &MTLContext,- intermediate_data_type: DataType,- query_config: Option<RMSNormConfig>,- key_config: Option<RMSNormConfig>,- qkv_array_id: ArrayId,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- num_q_heads: usize,- num_kv_heads: usize,- head_dim: usize,- ) -> Result<Self, RMSNormError> {- let mut query_kernel = None;- let mut key_kernel = None;- let mut query_scales_buffer = None;- let mut key_scales_buffer = None;-- // Setup query normalization if configured- if let Some(ref q_config) = query_config {- let scales_param =- parameter_tree.leaf("query_norm.scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(- crate::backends::metal::error::LibraryError::Custom(- format!("Failed to load query scales: {:?}", e),- ),- ))- })?;-- let scales_data = scales_param.buffer();- let scales_buffer = context.device.new_buffer_with_data(- scales_data.as_ptr() as *const _,- scales_data.len() as u64,- metal::MTLResourceOptions::StorageModeShared,- );-- let accumulation_data_type: DataType =- q_config.accumulation_precision.into();- let scale_data_type: DataType = q_config.scale_precision.into();-- let (input_type, scales_type, output_type) = match q_config- .upcast_mode- {- UpcastMode::OnlyNormalization => {- (intermediate_data_type, scale_data_type, scale_data_type)- },- UpcastMode::FullLayer => {- (intermediate_data_type, scale_data_type, scale_data_type)- },- };-- let kernel = RMSNormKernel::new_with_mode(- context,- input_type,- scales_type,- output_type,- accumulation_data_type,- RMSNormKernelType::QueryKey,- q_config.upcast_mode == UpcastMode::FullLayer,- )?;-- query_kernel = Some(kernel);- query_scales_buffer = Some(scales_buffer);- }-- // Setup key normalization if configured- if let Some(ref k_config) = key_config {- let scales_param =- parameter_tree.leaf("key_norm.scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(- crate::backends::metal::error::LibraryError::Custom(- format!("Failed to load key scales: {:?}", e),- ),- ))- })?;-- let scales_data = scales_param.buffer();- let scales_buffer = context.device.new_buffer_with_data(- scales_data.as_ptr() as *const _,- scales_data.len() as u64,- metal::MTLResourceOptions::StorageModeShared,- );-- let accumulation_data_type: DataType =- k_config.accumulation_precision.into();- let scale_data_type: DataType = k_config.scale_precision.into();-- let (input_type, scales_type, output_type) = match k_config- .upcast_mode- {- UpcastMode::OnlyNormalization => {- (intermediate_data_type, scale_data_type, scale_data_type)- },- UpcastMode::FullLayer => {- (intermediate_data_type, scale_data_type, scale_data_type)- },- };-- let kernel = RMSNormKernel::new_with_mode(- context,- input_type,- scales_type,- output_type,- accumulation_data_type,- RMSNormKernelType::QueryKey,- k_config.upcast_mode == UpcastMode::FullLayer,- )?;-- key_kernel = Some(kernel);- key_scales_buffer = Some(scales_buffer);- }-- Ok(Self {- query_kernel,- key_kernel,- query_config,- key_config,- qkv_array_id,- query_scales_buffer,- key_scales_buffer,- num_q_heads,- num_kv_heads,- head_dim,- })- }-}--impl EncodableWithState for QKNormKernelEncodable {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();-- compute_encoder.condition(- parameters.predicate_ref(),- || {- self.encode_with_encoder_impl(state, compute_encoder);- },- None::<fn()>,- );-- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- self.encode_with_encoder_impl(state, encoder);- }-}--impl QKNormKernelEncodable {- fn encode_with_encoder_impl(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- ) {- let qkv_binding = state.arrays(&[self.qkv_array_id]);- let qkv_shape = {- let qkv_array = qkv_binding[0].borrow();- qkv_array.shape().to_vec()- };-- let mut qkv_array = qkv_binding[0].borrow_mut();- let qkv_buffer = unsafe { qkv_array.mtl_buffer() };-- let batch_size = qkv_shape[0] as i32;- let head_dim = self.head_dim as i32;-- // Process query normalization if configured- if let (- Some(query_kernel),- Some(query_scales_buffer),- Some(query_config),- ) =- (&self.query_kernel, &self.query_scales_buffer, &self.query_config)- {- query_kernel.encode_qk_norm(- encoder,- QKNormArguments {- qkv_input_buffer: &qkv_buffer,- scales_buffer: query_scales_buffer,- qkv_output_buffer: &qkv_buffer,- batch_size,- num_q_heads: self.num_q_heads as i32,- num_kv_heads: self.num_kv_heads as i32,- head_dim,- epsilon: query_config.epsilon,- scale_offset: query_config.scale_offset.unwrap_or(0.0),- target: QKNormTarget::QueryHeads,- },- );- }-- if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =- (&self.key_kernel, &self.key_scales_buffer, &self.key_config)- {- key_kernel.encode_qk_norm(- encoder,- QKNormArguments {- qkv_input_buffer: &qkv_buffer,- scales_buffer: key_scales_buffer,- qkv_output_buffer: &qkv_buffer,- batch_size,- num_q_heads: self.num_q_heads as i32,- num_kv_heads: self.num_kv_heads as i32,- head_dim,- epsilon: key_config.epsilon,- scale_offset: key_config.scale_offset.unwrap_or(0.0),- target: QKNormTarget::KeyHeads,- },- );- }+ Ok(())}}diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rsindex ec4964985..5ece64af7 100644--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs@@ -4,20 +4,9 @@ use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef,ComputePipelineState as MTLComputePipelineState, MTLSize,};-use mpsgraph::CommandBuffer as MPSCommandBuffer;use thiserror::Error;-use crate::{- Array,- backends::metal::{- KernelDataType, MTLContext, MTLError,- forward_pass::{- ArrayId, ForwardPassState, RopeType,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- metal_extensions::ComputeEncoderConditional,- },-};+use crate::backends::metal::{KernelDataType, MTLContext, MTLError};pub struct RopeKernel {pipeline_state: MTLComputePipelineState,@@ -131,135 +120,3 @@ impl RopeKernel {.dispatch_threads(total_threads, threads_per_threadgroup);}}--pub struct RopeKernelEncodable {- kernel: RopeKernel,- rope_type: RopeType,-}--impl RopeKernelEncodable {- pub fn new(- context: &MTLContext,- data_type: KernelDataType,- rope_type: RopeType,- ) -> Result<Self, RopeError> {- let kernel = RopeKernel::new(context, data_type)?;- Ok(Self {- kernel,- rope_type,- })- }-}--impl EncodableWithState for RopeKernelEncodable {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();-- compute_encoder.condition(- parameters.predicate_ref(),- || {- self.encode_with_encoder_impl(state, compute_encoder);- },- None::<fn()>,- );-- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- self.encode_with_encoder_impl(state, encoder);- }-}--impl RopeKernelEncodable {- fn encode_with_encoder_impl(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- ) {- let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {- let qkv_binding = state.arrays(&[ArrayId::QKV]);- let qkv_array = qkv_binding[0].borrow();- let suffix_length = qkv_array.shape()[0];-- let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);- let queries_array = queries_binding[0].borrow();- let num_heads = queries_array.shape()[0];- let head_dim = queries_array.shape()[2];-- let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);- let keys_array = keys_binding[0].borrow();- let num_groups = keys_array.shape()[0];-- let cos_binding =- state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);- let cos_array = cos_binding[0].borrow();- let cos_shape = cos_array.shape();- let rope_max_seq_len = cos_shape[0];-- (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len)- };-- let qkv_buffer_binding = state.arrays(&[ArrayId::QKV]);- let mut qkv = qkv_buffer_binding[0].borrow_mut();-- let token_positions_binding = state.arrays(&[ArrayId::TokenPositions]);- let mut token_positions = token_positions_binding[0].borrow_mut();-- let query_buffer_binding = state.arrays(&[ArrayId::RotatedQueries]);- let mut rotated_queries = query_buffer_binding[0].borrow_mut();-- let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);- let mut rotated_keys = rotated_keys_binding[0].borrow_mut();-- let cos_buffer_binding =- state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);- let mut rope_cosines = cos_buffer_binding[0].borrow_mut();-- let sin_buffer_binding =- state.arrays(&[ArrayId::RopeSines(self.rope_type)]);- let mut rope_sines = sin_buffer_binding[0].borrow_mut();-- self.kernel.encode(- encoder,- RopeKernelArguments {- qkv_buffer: unsafe { &qkv.mtl_buffer() },- cosines_buffer: unsafe { &rope_cosines.mtl_buffer() },- sines_buffer: unsafe { &rope_sines.mtl_buffer() },- token_positions_buffer: unsafe {- &token_positions.mtl_buffer()- },- rotated_queries_buffer: unsafe {- &rotated_queries.mtl_buffer()- },- rotated_keys_buffer: unsafe { &rotated_keys.mtl_buffer() },- head_dim,- num_heads,- num_groups,- suffix_length,- max_sequence_length: rope_max_seq_len,- },- );- }-}diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rsindex 216e26933..1e599cc6c 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs@@ -1,22 +1,15 @@-use std::{mem::size_of, rc::Rc};+use std::mem::size_of;use metal::{Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,MTLResourceOptions, MTLSize,};-use mpsgraph::CommandBuffer as MPSCommandBuffer;use thiserror::Error;use crate::{DataType,- backends::metal::{- KernelDataType, MTLContext, MTLError,- forward_pass::{- ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- },+ backends::metal::{KernelDataType, MTLContext, MTLError},session::parameter::SamplingMethod,};@@ -58,8 +51,6 @@ enum ArgmaxImplementation {}pub struct SamplingKernel {- bitmask_pipeline: MTLComputePipelineState,- partial_bitmask_buffer: MTLBuffer,temperature_pipeline: MTLComputePipelineState,partial_temperature_buffer: MTLBuffer,topk_pipeline: MTLComputePipelineState,@@ -113,20 +104,6 @@ impl SamplingKernel {let data_suffix = data_type.function_name_suffix();let max_elements = max_batch_size * max_vocab_size;- let bitmask_pipeline = context- .compute_pipeline_state_with_reflection(- &format!("batched_bitmask_{}", data_suffix),- None,- )- .map(|(pipeline, _)| pipeline)- .map_err(SamplingError::MetalError)?;-- let partial_bitmask_buffer = context.device.new_buffer(- (max_elements * Into::<DataType>::into(data_type).size_in_bytes())- as u64,- MTLResourceOptions::StorageModeShared,- );-let temperature_pipeline = context.compute_pipeline_state_with_reflection(&format!("batched_temperature_{}", data_suffix),@@ -235,8 +212,6 @@ impl SamplingKernel {};Ok(Self {- bitmask_pipeline,- partial_bitmask_buffer,temperature_pipeline,partial_temperature_buffer,topk_pipeline,@@ -254,7 +229,6 @@ impl SamplingKernel {pub fn encode(&self,logits_buffer: &MTLBuffer,- bitmask_buffer: Option<&MTLBuffer>,seeds_buffer: Option<&MTLBuffer>,sampled_tokens_buffer: &MTLBuffer,sampling_method: SamplingMethod,@@ -265,9 +239,7 @@ impl SamplingKernel {let compute_encoder = command_buffer.new_compute_command_encoder();self.encode_with_encoder(logits_buffer,- bitmask_buffer,seeds_buffer,- 0, // No offset for non-async pathsampled_tokens_buffer,sampling_method,batch_size,@@ -281,9 +253,7 @@ impl SamplingKernel {pub fn encode_with_encoder(&self,logits_buffer: &MTLBuffer,- bitmask_buffer: Option<&MTLBuffer>,seeds_buffer: Option<&MTLBuffer>,- seeds_offset: usize,sampled_tokens_buffer: &MTLBuffer,sampling_method: SamplingMethod,batch_size: usize,@@ -305,18 +275,6 @@ impl SamplingKernel {let mut last_logits_buffer = logits_buffer;- if let Some(bitmask_buffer) = bitmask_buffer {- self.encode_bitmask(- last_logits_buffer,- bitmask_buffer,- &self.partial_bitmask_buffer,- batch_size as u32,- vocab_size as u32,- compute_encoder,- )?;- last_logits_buffer = &self.partial_bitmask_buffer;- }-if let SamplingMethod::Stochastic {temperature,top_k,@@ -362,7 +320,6 @@ impl SamplingKernel {self.encode_gumbel(last_logits_buffer,seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,- seeds_offset,&self.partial_gumbel_buffer,batch_size as u32,vocab_size as u32,@@ -399,38 +356,6 @@ impl SamplingKernel {}}- pub fn encode_bitmask(- &self,- logits_buffer: &MTLBuffer,- bitmask_buffer: &MTLBuffer,- processed_logits_buffer: &MTLBuffer,- batch_size: u32,- vocab_size: u32,- compute_encoder: &ComputeCommandEncoderRef,- ) -> Result<(), SamplingError> {- compute_encoder.set_compute_pipeline_state(&self.bitmask_pipeline);-- compute_encoder.set_buffer(0, Some(logits_buffer), 0);- compute_encoder.set_buffer(1, Some(bitmask_buffer), 0);- compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);- compute_encoder.set_bytes(- 3,- size_of::<u32>() as u64,- &vocab_size as *const u32 as *const std::ffi::c_void,- );-- let elements_in_group = BLOCK_SIZE * ELEMENTWISE_GRAIN_SIZE;- let groups = (vocab_size + (elements_in_group as u32 - 1))- / elements_in_group as u32;-- compute_encoder.dispatch_thread_groups(- MTLSize::new(groups as u64, batch_size as u64, 1),- MTLSize::new(BLOCK_SIZE as u64, 1, 1),- );-- Ok(())- }-pub fn encode_temperature(&self,logits_buffer: &MTLBuffer,@@ -535,7 +460,6 @@ impl SamplingKernel {&self,logits_buffer: &MTLBuffer,seeds_buffer: &MTLBuffer,- seeds_offset: usize,processed_logits_buffer: &MTLBuffer,batch_size: u32,vocab_size: u32,@@ -544,7 +468,7 @@ impl SamplingKernel {compute_encoder.set_compute_pipeline_state(&self.gumbel_pipeline);compute_encoder.set_buffer(0, Some(logits_buffer), 0);- compute_encoder.set_buffer(1, Some(seeds_buffer), seeds_offset as u64);+ compute_encoder.set_buffer(1, Some(seeds_buffer), 0);compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);compute_encoder.set_bytes(3,@@ -659,141 +583,3 @@ impl SamplingKernel {Ok(())}}--pub struct SamplingKernelEncodable {- pub kernel: SamplingKernel,-}--impl SamplingKernelEncodable {- pub fn new(- context: &Rc<MTLContext>,- data_type: KernelDataType,- max_batch_size: usize,- max_vocab_size: usize,- ) -> Result<Self, SamplingError> {- let kernel = SamplingKernel::new(- context,- data_type,- max_batch_size,- max_vocab_size,- )?;- Ok(Self {- kernel,- })- }-- pub fn new_with_strategy(- context: &Rc<MTLContext>,- data_type: KernelDataType,- max_batch_size: usize,- max_vocab_size: usize,- argmax_strategy: ArgmaxStrategy,- ) -> Result<Self, SamplingError> {- let kernel = SamplingKernel::new_with_strategy(- context,- data_type,- max_batch_size,- max_vocab_size,- argmax_strategy,- )?;- Ok(Self {- kernel,- })- }-}--impl EncodableWithState for SamplingKernelEncodable {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let root_command_buffer =- command_buffer.root_command_buffer().to_owned();- let encoder = root_command_buffer.new_compute_command_encoder();-- self.encode_impl(state, encoder);-- encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- root_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &metal::ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- self.encode_impl(state, encoder);- }-}--impl SamplingKernelEncodable {- fn encode_impl(- &self,- state: &mut ForwardPassState,- encoder: &metal::ComputeCommandEncoderRef,- ) {- assert!(- state.sampling_output.is_some(),- "Sampling output buffer must be pre-allocated"- );-- let logits_binding = state- .arrays(&[crate::backends::metal::forward_pass::ArrayId::Logits]);- let mut logits = logits_binding[0].borrow_mut();-- let logits_shape = {- use crate::Array;- logits.shape()- };- let batch_size = state.active_suffix_length();- let vocab_size = logits_shape[1];-- let bitmask_binding = if state.has_bitmask() {- let bitmask_binding = state.arrays(&[- crate::backends::metal::forward_pass::ArrayId::TokenBitmask,- ]);- Some(bitmask_binding)- } else {- None- };-- let mut bitmask =- bitmask_binding.as_ref().map(|bind| bind[0].borrow_mut());-- let seeds_binding = state.arrays(&[- crate::backends::metal::forward_pass::ArrayId::TokenSeeds,- ]);- let mut seeds = seeds_binding[0].borrow_mut();-- let mut output_buffer_ref =- state.sampling_output.as_ref().unwrap().borrow_mut();-- let sampling_method = state.sampling_method.unwrap();- let seeds_offset = seeds.buffer_offset();-- if let Err(e) = self.kernel.encode_with_encoder(- unsafe { &logits.mtl_buffer() },- unsafe { bitmask.as_mut().map(|arr| arr.mtl_buffer()) },- unsafe { Some(&seeds.mtl_buffer()) },- seeds_offset,- unsafe { &output_buffer_ref.mtl_buffer() },- sampling_method,- batch_size,- vocab_size,- encoder,- ) {- panic!("Sampling encoding failed: {:?}", e);- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/mod.rs b/crates/uzu/src/backends/metal/kernel/sigmoid/mod.rsnew file mode 100644index 000000000..598a890a5--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/mod.rs@@ -0,0 +1,62 @@+use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,+ MTLSize,+};++use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError},+};++pub struct SigmoidKernel {+ pipeline: ComputePipelineState,+}++impl SigmoidKernel {+ fn kernel_name_for_type(+ data_type: DataType+ ) -> Result<&'static str, MTLError> {+ match data_type {+ DataType::F16 => Ok("apply_sigmoid_f16"),+ DataType::F32 => Ok("apply_sigmoid_f32"),+ DataType::BF16 => Ok("apply_sigmoid_bf16"),+ other => Err(MTLError::Generic(format!(+ "Unsupported dtype for sigmoid: {:?}",+ other+ ))),+ }+ }++ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ ) -> Result<Self, MTLError> {+ let fn_name = Self::kernel_name_for_type(data_type)?;+ let pipeline = context.compute_pipeline_state(fn_name, None)?;+ Ok(Self {+ pipeline,+ })+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ input_buffer: &MTLBuffer,+ output_buffer: &MTLBuffer,+ total_elements: i32,+ ) -> Result<(), MTLError> {+ encoder.set_compute_pipeline_state(&self.pipeline);+ encoder.set_buffer(0, Some(input_buffer), 0);+ encoder.set_buffer(1, Some(output_buffer), 0);+ encoder.set_bytes(+ 2,+ std::mem::size_of::<i32>() as u64,+ &total_elements as *const i32 as *const _,+ );++ let threads_per_tg = MTLSize::new(256, 1, 1);+ let grid = MTLSize::new(total_elements as u64, 1, 1);+ encoder.dispatch_threads(grid, threads_per_tg);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metalnew file mode 100644index 000000000..8391a045e--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal@@ -0,0 +1,41 @@+#include <metal_stdlib>+#include "../definitions.metal"+using namespace metal;++// Apply sigmoid element-wise to logits+template <typename T>+kernel void apply_sigmoid(+ const device T* input [[buffer(0)]],+ device T* output [[buffer(1)]],+ constant int& total_elements [[buffer(2)]],+ uint tid [[thread_position_in_grid]]+) {+ if (tid >= total_elements)+ return;+ float x = float(input[tid]);+ output[tid] = T(1.0f / (1.0f + exp(-x)));+}++// Explicit instantiations+template [[host_name("apply_sigmoid_f16")]] [[kernel]] void apply_sigmoid<half>(+ const device half* input [[buffer(0)]],+ device half* output [[buffer(1)]],+ constant int& total_elements [[buffer(2)]],+ uint tid [[thread_position_in_grid]]+);++template [[host_name("apply_sigmoid_f32")]] [[kernel]] void apply_sigmoid<+ float>(+ const device float* input [[buffer(0)]],+ device float* output [[buffer(1)]],+ constant int& total_elements [[buffer(2)]],+ uint tid [[thread_position_in_grid]]+);++template [[host_name("apply_sigmoid_bf16")]] [[kernel]] void apply_sigmoid<+ bfloat>(+ const device bfloat* input [[buffer(0)]],+ device bfloat* output [[buffer(1)]],+ constant int& total_elements [[buffer(2)]],+ uint tid [[thread_position_in_grid]]+);diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rsindex 9b23b0f10..baf58a079 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs@@ -13,7 +13,6 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {}pub mod conv1d_scan;-pub mod mamba;pub mod split_inproj;pub mod ssd_prefill;pub mod ssd_update;@@ -21,7 +20,6 @@ pub mod ssd_update;pub use conv1d_scan::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,};-pub(crate) use mamba::MambaMixerEncodable;pub use split_inproj::{SplitInProjArguments, SplitInProjKernel};pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode};pub use ssd_update::{SSDUpdateArguments, SSDUpdateKernel};diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rsindex 19336e899..7145e7278 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs@@ -2,35 +2,21 @@ use std::mem::size_of;use metal::{Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,+ ComputePipelineState as MTLComputePipelineState,};-use mpsgraph::CommandBuffer as MPSCommandBuffer;-use super::{- KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,-};-use crate::{- Array,- backends::metal::{- error::MTLError,- forward_pass::{- ArrayId, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- },-};+use super::{KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch};+use crate::backends::metal::error::MTLError;#[derive(Debug)]-pub struct TensorAddSwap {+pub struct TensorAddSwapKernel {pipeline_state: MTLComputePipelineState,- argument_arrays: Box<[ArrayId]>,}-impl TensorAddSwap {+impl TensorAddSwapKernel {pub fn new(context: &MTLContext,data_type: KernelDataType,- argument_arrays: Box<[ArrayId]>,) -> Result<Self, MTLError> {let function_name =format!("tensorAddSwap_{}", data_type.function_name_suffix());@@ -40,99 +26,31 @@ impl TensorAddSwap {Ok(Self {pipeline_state,- argument_arrays,})}- fn encode_into_command_buffer(+ pub fn encode_into_command_buffer(&self,skip_buffer: &MTLBuffer,main_buffer: &MTLBuffer,length: usize,command_buffer: &MTLCommandBuffer,) {- let encoder = command_buffer.new_compute_command_encoder();- self.encode_with_encoder_raw(skip_buffer, main_buffer, length, encoder);- encoder.end_encoding();- }+ let compute_encoder = command_buffer.new_compute_command_encoder();- fn encode_with_encoder_raw(- &self,- skip_buffer: &MTLBuffer,- main_buffer: &MTLBuffer,- length: usize,- encoder: &ComputeCommandEncoderRef,- ) {- encoder.set_label("Tensor Add-Swap");- encoder.set_buffer(0, Some(skip_buffer), 0);- encoder.set_buffer(1, Some(main_buffer), 0);- encoder.set_bytes(+ compute_encoder.set_label("Tensor Add-Swap");++ compute_encoder.set_buffer(0, Some(skip_buffer), 0);+ compute_encoder.set_buffer(1, Some(main_buffer), 0);++ compute_encoder.set_bytes(2,size_of::<i32>() as u64,&(length as i32) as *const _ as *const std::ffi::c_void,);- encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);- }-}--impl EncodableWithState for TensorAddSwap {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let arrays = state.arrays(&self.argument_arrays);- assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");- let length = arrays[0].borrow().num_elements();+ compute_encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);- let mut skip_array = arrays[0].borrow_mut();- let mut main_array = arrays[1].borrow_mut();- let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };- let main_mtl_buffer = unsafe { main_array.mtl_buffer() };-- let retained_mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();-- self.encode_into_command_buffer(- &skip_mtl_buffer,- &main_mtl_buffer,- length,- &retained_mtl_command_buffer,- );-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- retained_mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- let arrays = state.arrays(&self.argument_arrays);- assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");-- let length = arrays[0].borrow().num_elements();-- let mut skip_array = arrays[0].borrow_mut();- let mut main_array = arrays[1].borrow_mut();- let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };- let main_mtl_buffer = unsafe { main_array.mtl_buffer() };-- self.encode_with_encoder_raw(- &skip_mtl_buffer,- &main_mtl_buffer,- length,- encoder,- );+ compute_encoder.end_encoding();}}diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rsindex d278a8ba6..dd67c59ad 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs@@ -2,32 +2,21 @@ use std::mem::size_of;use metal::{Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,+ ComputePipelineState as MTLComputePipelineState,};-use mpsgraph::CommandBuffer as MPSCommandBuffer;-use super::{- KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,-};-use crate::{- Array,- backends::metal::forward_pass::{- ArrayId, ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- },-};+use super::{KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch};+use crate::backends::metal::error::MTLError;-pub struct TensorCopy {+pub struct TensorCopyKernel {pipeline_state: MTLComputePipelineState,- argument_arrays: Box<[ArrayId]>,}-impl TensorCopy {+impl TensorCopyKernel {pub fn new(context: &MTLContext,data_type: KernelDataType,- argument_arrays: Box<[ArrayId]>,- ) -> Result<Self, crate::backends::metal::error::MTLError> {+ ) -> Result<Self, MTLError> {let function_name =format!("tensorCopy_{}", data_type.function_name_suffix());@@ -36,106 +25,28 @@ impl TensorCopy {Ok(Self {pipeline_state,- argument_arrays,})}- fn encode_into_command_buffer(+ pub fn encode_into_command_buffer(&self,source_buffer: &MTLBuffer,destination_buffer: &MTLBuffer,length: usize,command_buffer: &MTLCommandBuffer,- _predicate: Option<&MTLBuffer>,) {- let encoder = command_buffer.new_compute_command_encoder();- self.encode_with_encoder_raw(- source_buffer,- destination_buffer,- length,- encoder,- );- encoder.end_encoding();- }+ let compute_encoder = command_buffer.new_compute_command_encoder();+ compute_encoder.set_label("Tensor Copy");- fn encode_with_encoder_raw(- &self,- source_buffer: &MTLBuffer,- destination_buffer: &MTLBuffer,- length: usize,- encoder: &ComputeCommandEncoderRef,- ) {- encoder.set_label("Tensor Copy");- encoder.set_buffer(0, Some(source_buffer), 0);- encoder.set_buffer(1, Some(destination_buffer), 0);- encoder.set_bytes(+ compute_encoder.set_buffer(0, Some(source_buffer), 0);+ compute_encoder.set_buffer(1, Some(destination_buffer), 0);+ compute_encoder.set_bytes(2,size_of::<i32>() as u64,&(length as i32) as *const _ as *const std::ffi::c_void,);- encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);- }-}--impl EncodableWithState for TensorCopy {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let arrays = state.arrays(&self.argument_arrays);- assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");-- let length = arrays[0].borrow().num_elements();- let mut source_array = arrays[0].borrow_mut();- let mut destination_array = arrays[1].borrow_mut();- let source_mtl_buffer = unsafe { source_array.mtl_buffer() };- let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };-- let retained_mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();-- self.encode_into_command_buffer(- &source_mtl_buffer,- &destination_mtl_buffer,- length,- &retained_mtl_command_buffer,- parameters.predicate,- );-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- retained_mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- _parameters: &EncodingParameters,- ) {- let arrays = state.arrays(&self.argument_arrays);- assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");-- let length = arrays[0].borrow().num_elements();-- let mut source_array = arrays[0].borrow_mut();- let mut destination_array = arrays[1].borrow_mut();- let source_mtl_buffer = unsafe { source_array.mtl_buffer() };- let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };-- self.encode_with_encoder_raw(- &source_mtl_buffer,- &destination_mtl_buffer,- length,- encoder,- );+ compute_encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);+ compute_encoder.end_encoding();}}diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rssimilarity index 86%rename from crates/uzu/src/generator/context.rsrename to crates/uzu/src/backends/metal/language_model_generator_context.rsindex edbbcb95f..97b1bc1a6 100644--- a/crates/uzu/src/generator/context.rs+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs@@ -10,17 +10,18 @@ use metal::Event as MTLEvent;use mpsgraph::CommandBuffer as MPSCommandBuffer;use objc2::rc::Retained;+use super::{+ CacheLayers, Decoder, KVCacheUpdate, KernelDataType, MTLContext,+ ModelShape,+ compilation_parameters::CompilationConfig,+ encodable_block::Sampling,+ forward_pass::{ScratchBuffers, SharedBuffers},+ kernel::TokenCopyKernel,+};use crate::{- DataType,- backends::metal::{- CacheLayers, DecoderExecutables, KVCacheUpdate, KernelDataType,- MTLContext, ModelShape,- compilation_parameters::CompilationConfig,- forward_pass::{ForwardPassBuffers, SharedBuffers},- kernel::{SamplingKernelEncodable, TokenCopyKernel},- },+ DataType, DecoderConfig,config::{LanguageModelConfig, ModelMetadata},- generator::rng::DerivableSeed,+ language_model::rng::DerivableSeed,parameters::ParameterLoader,session::{config::DecodingConfig,@@ -128,19 +129,20 @@ impl AsyncBuffers {}}-pub struct GeneratorContext {+pub struct LanguageModelGeneratorContext {pub mtl_context: Rc<MTLContext>,pub command_buffer: Retained<MPSCommandBuffer>,pub cache_layers: Rc<RefCell<CacheLayers>>,pub shared_buffers: Rc<RefCell<SharedBuffers>>,- pub scratch_buffers: ForwardPassBuffers,+ pub scratch_buffers: ScratchBuffers,pub model_config: LanguageModelConfig,+ pub decoder_config: Rc<DecoderConfig>,pub model_shape: ModelShape,- pub executables: DecoderExecutables,+ pub executables: Decoder,pub kv_cache_update: Box<KVCacheUpdate>,- pub gpu_sampler: SamplingKernelEncodable,+ pub gpu_sampler: Sampling,pub next_seed: DerivableSeed,/// Kernel for copying sampled tokens in async pipeline@@ -149,7 +151,7 @@ pub struct GeneratorContext {pub async_buffers: AsyncBuffers,}-impl GeneratorContext {+impl LanguageModelGeneratorContext {pub fn new(model_path: &Path,decoding_config: &DecodingConfig,@@ -171,17 +173,25 @@ impl GeneratorContext {let model_metadata: ModelMetadata =serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;- let decoder_config =- Rc::new(model_metadata.model_config.decoder_config.clone());++ // Extract language model config+ let language_model_config = model_metadata+ .model_config+ .as_language_model()+ .ok_or(Error::UnableToLoadConfig)?;++ let decoder_config = Rc::new(+ language_model_config+ .decoder_config()+ .map_err(|_| Error::UnableToLoadConfig)?,+ );let model_shape = ModelShape::from_decoder_config(&decoder_config);- let prefill_step_size = decoding_config- .prefill_step_size- .resolve(&model_metadata.model_config);+ let prefill_step_size =+ decoding_config.prefill_step_size.resolve(language_model_config);let generate_suffix_length = decoding_config.generate_suffix_length();- let max_prefix_length: usize = decoding_config- .context_length- .resolve(&model_metadata.model_config);+ let max_prefix_length: usize =+ decoding_config.context_length.resolve(language_model_config);let max_suffix_length: usize =std::cmp::max(prefill_step_size, generate_suffix_length);@@ -208,7 +218,7 @@ impl GeneratorContext {)));shared_buffers.borrow_mut().update_data(&root_loader_view);- let scratch_buffers = ForwardPassBuffers::new(+ let scratch_buffers = ScratchBuffers::new(&mtl_context,&decoder_config,&model_shape,@@ -216,7 +226,7 @@ impl GeneratorContext {max_suffix_length,);- let executables = DecoderExecutables::new(+ let executables = Decoder::new(mtl_context.clone(),decoder_config.clone(),&root_loader_view,@@ -242,7 +252,7 @@ impl GeneratorContext {.map_err(|_| Error::UnableToCreateMetalContext)?,);- let gpu_sampler = SamplingKernelEncodable::new(+ let gpu_sampler = Sampling::new(&mtl_context,kernel_data_type,max_suffix_length,@@ -270,7 +280,8 @@ impl GeneratorContext {cache_layers,shared_buffers,scratch_buffers,- model_config: model_metadata.model_config.clone(),+ model_config: language_model_config.clone(),+ decoder_config,model_shape,executables,kv_cache_update,diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 3f56dbd95..908ddbb34 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -1,28 +1,29 @@mod array;mod buffer_allocator;+mod classifier_context;pub mod compilation_parameters;mod context;+pub mod encodable_block;pub mod error;-mod executable_builder;pub mod forward_pass;pub mod graph;pub mod image;pub mod kernel;+mod language_model_generator_context;pub mod media_utils;pub mod metal_extensions;pub mod placement_analysis;pub mod utils;+pub use array::MetalArray;pub use buffer_allocator::BufferAllocator;+pub use classifier_context::ClassifierContext;pub use context::MTLContext;+pub use encodable_block::Decoder;pub use error::MTLError;-pub use executable_builder::DecoderExecutables;pub use forward_pass::{CacheLayers, ForwardPassState, ModelShape};pub use kernel::{KVCacheUpdate, KernelDataType, RopeKernel};-// pub use kernel::{-// DataType as KernelDataType, Encoder as KernelEncoder,-// EncoderError as KernelEncoderError, TensorAddSwap, TensorCopy,-// };+pub use language_model_generator_context::LanguageModelGeneratorContext;pub use media_utils::{ImagePreprocessingParams, ImagePreprocessingRequirements,MetalImagePreprocessor,diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsnew file mode 100644index 000000000..aacaa1444--- /dev/null+++ b/crates/uzu/src/classifier/classifier.rs@@ -0,0 +1,290 @@+#[cfg(feature = "tracing")]+use std::{cell::RefCell, rc::Rc};+use std::{collections::HashMap, path::Path, time::Instant};++use objc2::rc::autoreleasepool;++#[cfg(feature = "tracing")]+use super::ActivationTrace;+use super::{ClassificationOutput, ClassificationStats, ClassifierContext};+use crate::{+ Array, DataType,+ backends::metal::forward_pass::{+ ArrayId, EncodableBlock, EncodingParameters, ForwardPassState,+ },+ session::types::Error,+};++pub struct Classifier {+ pub context: ClassifierContext,+}++impl Classifier {+ pub fn new(model_path: &Path) -> Result<Self, Error> {+ let context = ClassifierContext::new(model_path)?;+ Ok(Self {+ context,+ })+ }++ pub fn classify_tokens(+ &mut self,+ token_ids: Vec<u64>,+ token_positions: Vec<usize>,+ ) -> Result<ClassificationOutput, Error> {+ let run_start = Instant::now();++ autoreleasepool(|_pool| {+ let forward_start = Instant::now();++ #[cfg(feature = "tracing")]+ let (logits, _traces) =+ self.forward_pass_with_traces(&token_ids, &token_positions)?;++ #[cfg(not(feature = "tracing"))]+ let logits = self.forward_pass(&token_ids, &token_positions)?;++ let forward_duration = forward_start.elapsed().as_secs_f64();++ let postprocessing_start = Instant::now();+ let probabilities = self.logits_to_probabilities(&logits)?;+ let postprocessing_duration =+ postprocessing_start.elapsed().as_secs_f64();++ let (predicted_label, confidence) = probabilities+ .iter()+ .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())+ .map(|(label, prob)| (label.clone(), *prob))+ .unwrap_or((String::from("unknown"), 0.0));++ let stats = ClassificationStats::new(+ 0.0,+ forward_duration,+ postprocessing_duration,+ run_start.elapsed().as_secs_f64(),+ token_ids.len() as u64,+ predicted_label,+ confidence,+ );++ Ok(ClassificationOutput {+ logits,+ probabilities,+ stats,+ })+ })+ }++ #[cfg(feature = "tracing")]+ pub fn forward_pass_with_traces(+ &mut self,+ token_ids: &[u64],+ token_positions: &[usize],+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace>>), Error> {+ self.forward_pass(token_ids, token_positions)+ }++ #[cfg(feature = "tracing")]+ fn forward_pass(+ &mut self,+ token_ids: &[u64],+ token_positions: &[usize],+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace>>), Error> {+ autoreleasepool(|_| {+ let num_labels = self.context.model_config.model_config.num_labels;+ let mut state = ForwardPassState::new_classifier(+ self.context.mtl_context.clone(),+ &self.context.model_shape,+ &self.context.scratch_buffers,+ self.context.shared_buffers.clone(),+ token_ids,+ token_positions,+ true,+ num_labels,+ );++ let encoding_params = EncodingParameters::new(false, true, false);++ self.context.embed.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ self.context.embedding_norm.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ #[cfg(feature = "tracing")]+ {+ let traces = state.traces().clone();+ state.encode_copy_array(+ &self.context.command_buffer,+ ArrayId::Main,+ traces.borrow().embedding_norm().clone(),+ );+ }++ for layer in self.context.layers.iter() {+ layer.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ }+ self.context.output_norm.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ #[cfg(feature = "tracing")]+ {+ let traces = state.traces().clone();+ state.encode_copy_array(+ &self.context.command_buffer,+ ArrayId::Main,+ traces.borrow().output_norm.clone(),+ );+ }++ self.context.pooling.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );++ self.context.prediction_head.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );++ let logits = self.copy_logits_from_state(&state)?;++ let traces = state.traces().clone();+ Ok((logits, traces))+ })+ }++ #[cfg(not(feature = "tracing"))]+ fn forward_pass(+ &mut self,+ token_ids: &[u64],+ token_positions: &[usize],+ ) -> Result<Box<[f32]>, Error> {+ autoreleasepool(|_| {+ let num_labels = self.context.model_config.model_config.num_labels;+ let mut state = ForwardPassState::new_classifier(+ self.context.mtl_context.clone(),+ &self.context.model_shape,+ &self.context.scratch_buffers,+ self.context.shared_buffers.clone(),+ token_ids,+ token_positions,+ true,+ num_labels,+ );++ let encoding_params = EncodingParameters::new(false, true, false);++ self.context.embed.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ self.context.embedding_norm.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ for layer in self.context.layers.iter() {+ layer.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ }+ self.context.output_norm.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ self.context.pooling.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );+ self.context.prediction_head.encode(+ &mut state,+ &self.context.command_buffer,+ &encoding_params,+ );++ let logits = self.copy_logits_from_state(&state)?;++ Ok(logits)+ })+ }++ fn copy_logits_from_state(+ &self,+ state: &ForwardPassState,+ ) -> Result<Box<[f32]>, Error> {+ let logits_arrays =+ state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);+ let logits_array = logits_arrays[0].borrow();++ let num_labels = self.context.model_config.model_config.num_labels;+ let buffer = Array::buffer(&*logits_array);++ match Array::data_type(&*logits_array) {+ DataType::F32 => {+ let slice: &[f32] = bytemuck::cast_slice(buffer);+ Ok(slice[..num_labels].into())+ },+ DataType::F16 => {+ let slice: &[half::f16] = bytemuck::cast_slice(buffer);+ Ok(slice[..num_labels]+ .iter()+ .map(|&x| x.to_f32())+ .collect::<Vec<_>>()+ .into_boxed_slice())+ },+ DataType::BF16 => {+ let slice: &[half::bf16] = bytemuck::cast_slice(buffer);+ Ok(slice[..num_labels]+ .iter()+ .map(|&x| x.to_f32())+ .collect::<Vec<_>>()+ .into_boxed_slice())+ },+ _ => Err(Error::UnableToDecodeText),+ }+ }++ fn logits_to_probabilities(+ &self,+ logits: &[f32],+ ) -> Result<HashMap<String, f32>, Error> {+ let output_labels =+ &self.context.model_config.model_config.output_labels;+ let mut probabilities = HashMap::new();++ for (idx, &logit) in logits.iter().enumerate() {+ let prob = 1.0 / (1.0 + (-logit).exp());++ let label = if let Some(labels) = output_labels {+ labels+ .get(idx)+ .map(|s| s.clone())+ .unwrap_or_else(|| format!("class_{}", idx))+ } else {+ format!("class_{}", idx)+ };++ probabilities.insert(label, prob);+ }++ Ok(probabilities)+ }+}diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rsnew file mode 100644index 000000000..680439779--- /dev/null+++ b/crates/uzu/src/classifier/mod.rs@@ -0,0 +1,16 @@+mod classifier;+mod result;++pub use classifier::Classifier;+pub use result::{ClassificationOutput, ClassificationStats};++// Re-export context from metal backend+pub use crate::backends::metal::ClassifierContext;+// Re-export encodables from metal backend for convenience+pub use crate::backends::metal::encodable_block::{+ ClassifierLayer, ClassifierPredictionHead, Pooling,+};+#[cfg(feature = "tracing")]+pub use crate::backends::metal::forward_pass::traces::{+ ActivationTrace, LayerActivationTrace,+};diff --git a/crates/uzu/src/classifier/result.rs b/crates/uzu/src/classifier/result.rsnew file mode 100644index 000000000..651520b1e--- /dev/null+++ b/crates/uzu/src/classifier/result.rs@@ -0,0 +1,53 @@+use std::collections::HashMap;++#[derive(Debug, Clone)]+pub struct ClassificationOutput {+ pub logits: Box<[f32]>,+ pub probabilities: HashMap<String, f32>,+ pub stats: ClassificationStats,+}++#[derive(Debug, Clone)]+pub struct ClassificationStats {+ // Timing breakdown+ pub preprocessing_duration: f64,+ pub forward_pass_duration: f64,+ pub postprocessing_duration: f64,+ pub total_duration: f64,++ // Token metrics+ pub tokens_count: u64,+ pub tokens_per_second: f64,++ pub predicted_label: String,+ pub confidence: f32,+}++impl ClassificationStats {+ pub fn new(+ preprocessing_duration: f64,+ forward_pass_duration: f64,+ postprocessing_duration: f64,+ total_duration: f64,+ tokens_count: u64,+ predicted_label: String,+ confidence: f32,+ ) -> Self {+ let tokens_per_second = if forward_pass_duration > 0.0 {+ tokens_count as f64 / forward_pass_duration+ } else {+ 0.0+ };++ Self {+ preprocessing_duration,+ forward_pass_duration,+ postprocessing_duration,+ total_duration,+ tokens_count,+ tokens_per_second,+ predicted_label,+ confidence,+ }+ }+}diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rsindex e231fefde..ca1266c11 100644--- a/crates/uzu/src/config/attention.rs+++ b/crates/uzu/src/config/attention.rs@@ -1,14 +1,14 @@use serde::{Deserialize, Serialize};-use super::{RMSNormConfig, linear::LinearConfig};+use crate::{LinearConfig, NormalizationConfig};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct AttentionConfig {pub qkv_projection_config: LinearConfig,pub out_projection_config: LinearConfig,- pub query_norm_config: Option<RMSNormConfig>,- pub key_norm_config: Option<RMSNormConfig>,+ pub query_norm_config: Option<NormalizationConfig>,+ pub key_norm_config: Option<NormalizationConfig>,pub num_heads: Option<usize>,pub num_groups: Option<usize>,@@ -18,6 +18,7 @@ pub struct AttentionConfig {pub sliding_window_size: Option<usize>,pub logit_soft_cap: Option<f32>,+ #[serde(default)]pub has_sinks: bool,pub has_qkv_biases: bool,pub has_out_biases: bool,@@ -59,6 +60,12 @@ mod tests {},"query_norm_config": null,"key_norm_config": null,+ "num_heads": 12,+ "num_groups": 12,+ "head_dim": 64,+ "is_causal": false,+ "scale": null,+ "sliding_window_size": null,"logit_soft_cap": null,"has_sinks": false,"has_qkv_biases": false,@@ -89,10 +96,10 @@ mod tests {},query_norm_config: None,key_norm_config: None,- num_heads: None,- num_groups: None,- head_dim: None,- is_causal: None,+ num_heads: Some(12),+ num_groups: Some(12),+ head_dim: Some(64),+ is_causal: Some(false),scale: None,sliding_window_size: None,logit_soft_cap: None,diff --git a/crates/uzu/src/config/classifier/classifier_config.rs b/crates/uzu/src/config/classifier/classifier_config.rsnew file mode 100644index 000000000..e3e30e5b3--- /dev/null+++ b/crates/uzu/src/config/classifier/classifier_config.rs@@ -0,0 +1,117 @@+use serde::{Deserialize, Serialize};++use super::{PoolingType, PredictionHeadConfig};+use crate::{+ DecoderConfig, DecoderLayerConfig, EmbeddingConfig, LinearConfig,+ NormalizationConfig, TransformerConfig, config::ConfigError,+};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct ClassifierConfig {+ pub embedding_config: EmbeddingConfig,+ pub embedding_norm_config: NormalizationConfig,+ pub transformer_config: TransformerConfig,+ pub prediction_head_config: PredictionHeadConfig,+ pub readout_config: LinearConfig,++ pub vocab_size: usize,+ pub model_dim: usize,+ pub hidden_dim: usize,+ #[serde(default)]+ pub num_heads: Option<usize>,+ #[serde(default)]+ pub num_groups: Option<usize>,+ #[serde(default)]+ pub head_dim: Option<usize>,+ pub attention_scale: Option<f32>,+ pub num_layers: usize,+ #[serde(default)]+ pub sliding_window_sizes: Option<Vec<Option<usize>>>,+ pub context_length: usize,+ pub num_labels: usize,+ pub classifier_pooling: PoolingType,+ pub output_labels: Option<Vec<String>>,+}++impl ClassifierConfig {+ pub fn to_decoder_config(&self) -> Result<DecoderConfig, ConfigError> {+ // For classifier, we use the first layer config as the template+ // (most classifiers have homogeneous layers)+ let first_layer = self+ .transformer_config+ .layer_configs+ .first()+ .ok_or(ConfigError::NoLayers)?;++ let layer_config = DecoderLayerConfig {+ pre_attention_norm_config: first_layer+ .pre_attention_norm_config+ .clone()+ .unwrap_or_else(|| {+ // If first layer has no pre-attention norm (like ModernBERT), use a default+ self.transformer_config.output_norm_config.clone()+ }),+ mixer_config: first_layer.mixer_config.clone(),+ post_attention_norm_config: first_layer+ .post_attention_norm_config+ .clone(),+ pre_mlp_norm_config: first_layer.pre_mlp_norm_config.clone(),+ mlp_config: first_layer.mlp_config.clone(),+ post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),+ };++ let first_mixer = &first_layer.mixer_config;++ let num_heads =+ self.num_heads.or(first_mixer.num_heads()).ok_or_else(|| {+ ConfigError::MissingField("num_heads".to_string())+ })?;+ let num_groups =+ self.num_groups.or(first_mixer.num_groups()).unwrap_or(num_heads);+ let head_dim = self+ .head_dim+ .or(first_mixer.head_dim())+ .ok_or_else(|| ConfigError::MissingField("head_dim".to_string()))?;++ Ok(DecoderConfig {+ embedding_config: self.embedding_config.clone(),+ global_rope_config: self+ .transformer_config+ .global_rope_config+ .clone(),+ local_rope_config: self+ .transformer_config+ .local_rope_config+ .clone(),+ layer_config,+ output_norm_config: self+ .transformer_config+ .output_norm_config+ .clone(),+ vocab_size: self.vocab_size,+ model_dim: self.model_dim,+ hidden_dim: self.transformer_config.hidden_dim,+ num_heads,+ num_groups,+ head_dim,+ attention_scale: self.attention_scale,+ num_layers: self.num_layers,+ sliding_window_sizes: {+ if let Some(sizes) = &self.sliding_window_sizes {+ Some(sizes.clone().into_boxed_slice())+ } else {+ let sizes: Vec<Option<usize>> = self+ .transformer_config+ .layer_configs+ .iter()+ .map(|layer| layer.mixer_config.sliding_window_size())+ .collect();+ Some(sizes.into_boxed_slice())+ }+ },+ context_length: self.context_length,+ layer_configs: None, // Classifier doesn't use heterogeneous layers usually+ layer_types: None,+ })+ }+}diff --git a/crates/uzu/src/config/classifier/mod.rs b/crates/uzu/src/config/classifier/mod.rsnew file mode 100644index 000000000..7084c1314--- /dev/null+++ b/crates/uzu/src/config/classifier/mod.rs@@ -0,0 +1,7 @@+mod classifier_config;+mod pooling_type;+mod prediction_head_config;++pub use classifier_config::ClassifierConfig;+pub use pooling_type::PoolingType;+pub use prediction_head_config::PredictionHeadConfig;diff --git a/crates/uzu/src/config/classifier/pooling_type.rs b/crates/uzu/src/config/classifier/pooling_type.rsnew file mode 100644index 000000000..473f31498--- /dev/null+++ b/crates/uzu/src/config/classifier/pooling_type.rs@@ -0,0 +1,8 @@+use serde::{Deserialize, Serialize};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(rename_all = "lowercase")]+pub enum PoolingType {+ Cls,+ Mean,+}diff --git a/crates/uzu/src/config/classifier/prediction_head_config.rs b/crates/uzu/src/config/classifier/prediction_head_config.rsnew file mode 100644index 000000000..8c45bab7c--- /dev/null+++ b/crates/uzu/src/config/classifier/prediction_head_config.rs@@ -0,0 +1,12 @@+use serde::{Deserialize, Serialize};++use crate::{Activation, LinearConfig, NormalizationConfig};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct PredictionHeadConfig {+ pub dense_config: LinearConfig,+ pub activation: Activation,+ pub normalization_config: NormalizationConfig,+ pub readout_config: LinearConfig,+ pub use_dense_bias: bool,+}diff --git a/crates/uzu/src/config/classifier_model.rs b/crates/uzu/src/config/classifier_model.rsnew file mode 100644index 000000000..f9f7448b5--- /dev/null+++ b/crates/uzu/src/config/classifier_model.rs@@ -0,0 +1,9 @@+use serde::{Deserialize, Serialize};++use crate::{ClassifierConfig, MessageProcessorConfig};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct ClassifierModelConfig {+ pub model_config: ClassifierConfig,+ pub message_processor_config: MessageProcessorConfig,+}diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rsindex 72e0a84b6..26259a00e 100644--- a/crates/uzu/src/config/decoder.rs+++ b/crates/uzu/src/config/decoder.rs@@ -6,7 +6,7 @@ use serde::{use super::{decoder_layer::{DecoderLayerConfig, MixerConfig},embedding::EmbeddingConfig,- normalization::RMSNormConfig,+ normalization::NormalizationConfig,rope::RoPEConfig,};@@ -32,7 +32,7 @@ pub struct DecoderConfig {pub local_rope_config: Option<RoPEConfig>,pub layer_config: DecoderLayerConfig,pub layer_configs: Option<Box<[DecoderLayerConfig]>>,- pub output_norm_config: RMSNormConfig,+ pub output_norm_config: NormalizationConfig,pub vocab_size: usize,pub model_dim: usize,@@ -181,7 +181,7 @@ struct RawDecoderConfig {layer_config: Option<DecoderLayerConfig>,#[serde(default)]layer_configs: Option<Vec<DecoderLayerConfig>>,- output_norm_config: RMSNormConfig,+ output_norm_config: NormalizationConfig,vocab_size: usize,model_dim: usize,hidden_dim: usize,@@ -231,13 +231,17 @@ impl DecoderConfig {}pub fn has_attention_layers(&self) -> bool {- if let Some(layer_types) = &self.layer_types {- layer_types- .iter()- .any(|lt| matches!(lt, DecoderLayerType::Transformer))- } else {- matches!(self.layer_config.mixer_config, MixerConfig::Attention(_))- }+ self.layer_configs+ .as_ref()+ .map(|configs| {+ configs.iter().any(|config| {+ matches!(+ config.mixer_config,+ crate::config::MixerConfig::Attention(_)+ )+ })+ })+ .unwrap_or(true)}}@@ -309,6 +313,14 @@ mod tests {"lora_rank": 16,"lora_scale": 2.0},+ "query_norm_config": null,+ "key_norm_config": null,+ "num_heads": 12,+ "num_groups": 12,+ "head_dim": 64,+ "is_causal": true,+ "scale": null,+ "sliding_window_size": null,"logit_soft_cap": null,"has_sinks": false,"has_qkv_biases": false,@@ -380,19 +392,21 @@ mod tests {}),local_rope_config: None,layer_config: DecoderLayerConfig {- pre_attention_norm_config: RMSNormConfig {+ pre_attention_norm_config: NormalizationConfig {scale_precision: ConfigDataType::BFloat16,accumulation_precision: ConfigDataType::Float32,epsilon: 1e-5,scale_offset: None,upcast_mode: UpcastMode::OnlyNormalization,+ subtract_mean: false,},- pre_mlp_norm_config: RMSNormConfig {+ pre_mlp_norm_config: NormalizationConfig {scale_precision: ConfigDataType::BFloat16,accumulation_precision: ConfigDataType::Float32,epsilon: 1e-5,scale_offset: None,upcast_mode: UpcastMode::OnlyNormalization,+ subtract_mean: false,},mixer_config: MixerConfig::Attention(AttentionConfig {qkv_projection_config: LinearConfig::QLoRA {@@ -421,10 +435,10 @@ mod tests {},query_norm_config: None,key_norm_config: None,- num_heads: None,- num_groups: None,- head_dim: None,- is_causal: None,+ num_heads: Some(12),+ num_groups: Some(12),+ head_dim: Some(64),+ is_causal: Some(true),scale: None,sliding_window_size: None,logit_soft_cap: None,@@ -448,16 +462,22 @@ mod tests {activation: Activation::SiLU {alpha: 1.0,},+ has_up_biases: false,+ has_down_biases: false,+ gate_clipping: None,+ up_clipping: None,+ activation_to_gate: true,}),post_attention_norm_config: None,post_mlp_norm_config: None,},- output_norm_config: RMSNormConfig {+ output_norm_config: NormalizationConfig {scale_precision: ConfigDataType::BFloat16,accumulation_precision: ConfigDataType::Float32,epsilon: 1e-5,scale_offset: None,upcast_mode: UpcastMode::OnlyNormalization,+ subtract_mean: false,},layer_configs: None,vocab_size: 128256,diff --git a/crates/uzu/src/config/decoder_layer.rs b/crates/uzu/src/config/decoder_layer.rsindex be78c548e..1889420cd 100644--- a/crates/uzu/src/config/decoder_layer.rs+++ b/crates/uzu/src/config/decoder_layer.rs@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};use super::{attention::AttentionConfig, mamba::Mamba2Config, mlp::MLPConfig,- normalization::RMSNormConfig,+ normalization::NormalizationConfig,};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]@@ -68,16 +68,16 @@ impl MixerConfig {#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct DecoderLayerConfig {#[serde(alias = "pre_mixer_norm_config")]- pub pre_attention_norm_config: RMSNormConfig,+ pub pre_attention_norm_config: NormalizationConfig,#[serde(alias = "mixer_config")]pub mixer_config: MixerConfig,#[serde(alias = "post_mixer_norm_config")]- pub post_attention_norm_config: Option<RMSNormConfig>,+ pub post_attention_norm_config: Option<NormalizationConfig>,#[serde(alias = "pre_mlp_norm_config")]- pub pre_mlp_norm_config: RMSNormConfig,+ pub pre_mlp_norm_config: NormalizationConfig,pub mlp_config: MLPConfig,#[serde(alias = "post_mlp_norm_config")]- pub post_mlp_norm_config: Option<RMSNormConfig>,+ pub post_mlp_norm_config: Option<NormalizationConfig>,}impl DecoderLayerConfig {@@ -131,6 +131,14 @@ mod tests {"lora_rank": 16,"lora_scale": 2.0},+ "query_norm_config": null,+ "key_norm_config": null,+ "num_heads": 12,+ "num_groups": 12,+ "head_dim": 64,+ "is_causal": true,+ "scale": null,+ "sliding_window_size": null,"logit_soft_cap": null,"has_sinks": false,"has_qkv_biases": false,@@ -162,19 +170,21 @@ mod tests {"#;let ground_truth_config = DecoderLayerConfig {- pre_attention_norm_config: RMSNormConfig {+ pre_attention_norm_config: NormalizationConfig {scale_precision: ConfigDataType::BFloat16,accumulation_precision: ConfigDataType::Float32,epsilon: 1e-5,scale_offset: None,upcast_mode: UpcastMode::OnlyNormalization,+ subtract_mean: false,},- pre_mlp_norm_config: RMSNormConfig {+ pre_mlp_norm_config: NormalizationConfig {scale_precision: ConfigDataType::BFloat16,accumulation_precision: ConfigDataType::Float32,epsilon: 1e-5,scale_offset: None,upcast_mode: UpcastMode::OnlyNormalization,+ subtract_mean: false,},mixer_config: MixerConfig::Attention(AttentionConfig {qkv_projection_config: LinearConfig::QLoRA {@@ -203,16 +213,16 @@ mod tests {},query_norm_config: None,key_norm_config: None,+ num_heads: Some(12),+ num_groups: Some(12),+ head_dim: Some(64),+ is_causal: Some(true),+ scale: None,+ sliding_window_size: None,logit_soft_cap: None,has_sinks: false,has_qkv_biases: false,has_out_biases: false,- num_heads: None,- num_groups: None,- head_dim: None,- is_causal: None,- scale: None,- sliding_window_size: None,}),mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {linear_config: LinearConfig::QLoRA {@@ -230,6 +240,11 @@ mod tests {activation: Activation::SiLU {alpha: 1.0,},+ has_up_biases: false,+ has_down_biases: false,+ gate_clipping: None,+ up_clipping: None,+ activation_to_gate: true,}),post_attention_norm_config: None,post_mlp_norm_config: None,diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rsindex 5a7bcb90c..f3277d2f7 100644--- a/crates/uzu/src/config/embedding.rs+++ b/crates/uzu/src/config/embedding.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use super::common::{ConfigDataType, QuantizationMode};+use crate::{ConfigDataType, QuantizationMode};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct EmbeddingConfigCommon {diff --git a/crates/uzu/src/config/error.rs b/crates/uzu/src/config/error.rsnew file mode 100644index 000000000..aed9c0118--- /dev/null+++ b/crates/uzu/src/config/error.rs@@ -0,0 +1,9 @@+use thiserror::Error;++#[derive(Debug, Error)]+pub enum ConfigError {+ #[error("Missing required field: {0}")]+ MissingField(String),+ #[error("No layers in transformer config")]+ NoLayers,+}diff --git a/crates/uzu/src/config/language_model.rs b/crates/uzu/src/config/language_model.rsindex da949c549..373028222 100644--- a/crates/uzu/src/config/language_model.rs+++ b/crates/uzu/src/config/language_model.rs@@ -1,10 +1,136 @@use serde::{Deserialize, Serialize};-use super::{DecoderConfig, GenerationConfig, MessageProcessorConfig};+use crate::{+ DecoderConfig, DecoderLayerConfig, DecoderLayerType, EmbeddingConfig,+ GenerationConfig, MessageProcessorConfig, MixerConfig, TransformerConfig,+ config::ConfigError,+};++/// Inner model config matching the new lalamo export format.+/// Contains embedding_config at the top level, with transformer_config nested.+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct InnerModelConfig {+ pub embedding_config: EmbeddingConfig,+ pub transformer_config: TransformerConfig,+ pub vocab_size: usize,+}++impl InnerModelConfig {+ /// Convert to DecoderConfig for backward compatibility with the rest of the codebase.+ pub fn to_decoder_config(&self) -> Result<DecoderConfig, ConfigError> {+ let tf = &self.transformer_config;++ let first_layer =+ tf.layer_configs.first().ok_or(ConfigError::NoLayers)?;++ let layer_config = DecoderLayerConfig {+ pre_attention_norm_config: first_layer+ .pre_attention_norm_config+ .clone()+ .unwrap_or_else(|| tf.output_norm_config.clone()),+ mixer_config: first_layer.mixer_config.clone(),+ post_attention_norm_config: first_layer+ .post_attention_norm_config+ .clone(),+ pre_mlp_norm_config: first_layer.pre_mlp_norm_config.clone(),+ mlp_config: first_layer.mlp_config.clone(),+ post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),+ };++ let mixer = &first_layer.mixer_config;+ let num_heads =+ tf.num_heads.or(mixer.num_heads()).ok_or_else(|| {+ ConfigError::MissingField("num_heads".to_string())+ })?;+ let num_groups =+ tf.num_groups.or(mixer.num_groups()).unwrap_or(num_heads);+ let head_dim = tf+ .head_dim+ .or(mixer.head_dim())+ .ok_or_else(|| ConfigError::MissingField("head_dim".to_string()))?;+ let attention_scale = tf.attention_scale.or(mixer.attention_scale());+ let num_layers = tf.num_layers.unwrap_or(tf.layer_configs.len());++ let sliding_window_sizes: Box<[Option<usize>]> = tf+ .layer_configs+ .iter()+ .map(|layer| layer.mixer_config.sliding_window_size())+ .collect::<Vec<_>>()+ .into_boxed_slice();++ let layer_types: Box<[DecoderLayerType]> = tf+ .layer_configs+ .iter()+ .map(|layer| Self::layer_type_from_mixer(&layer.mixer_config))+ .collect::<Vec<_>>()+ .into_boxed_slice();++ let layer_configs: Box<[DecoderLayerConfig]> = tf+ .layer_configs+ .iter()+ .map(|layer| DecoderLayerConfig {+ pre_attention_norm_config: layer+ .pre_attention_norm_config+ .clone()+ .unwrap_or_else(|| tf.output_norm_config.clone()),+ mixer_config: layer.mixer_config.clone(),+ post_attention_norm_config: layer+ .post_attention_norm_config+ .clone(),+ pre_mlp_norm_config: layer.pre_mlp_norm_config.clone(),+ mlp_config: layer.mlp_config.clone(),+ post_mlp_norm_config: layer.post_mlp_norm_config.clone(),+ })+ .collect::<Vec<_>>()+ .into_boxed_slice();++ Ok(DecoderConfig {+ embedding_config: self.embedding_config.clone(),+ global_rope_config: tf.global_rope_config.clone(),+ local_rope_config: tf.local_rope_config.clone(),+ layer_config,+ layer_configs: Some(layer_configs),+ output_norm_config: tf.output_norm_config.clone(),+ vocab_size: self.vocab_size,+ model_dim: tf.model_dim,+ hidden_dim: tf.hidden_dim,+ num_heads,+ num_groups,+ head_dim,+ attention_scale,+ num_layers,+ sliding_window_sizes: Some(sliding_window_sizes),+ layer_types: Some(layer_types),+ context_length: tf.context_length,+ })+ }++ fn layer_type_from_mixer(mixer: &MixerConfig) -> DecoderLayerType {+ match mixer {+ MixerConfig::Attention(_) => DecoderLayerType::Transformer,+ MixerConfig::Mamba(config) => DecoderLayerType::StateSpace {+ conv_dim: config.conv_dim(),+ kernel_size: config.kernel_size,+ state_dim: config.state_dim,+ num_heads: config.num_heads,+ num_groups: config.num_groups,+ head_dim: config.head_dim,+ },+ }+ }+}#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct LanguageModelConfig {- pub decoder_config: DecoderConfig,+ pub model_config: InnerModelConfig,pub message_processor_config: MessageProcessorConfig,pub generation_config: GenerationConfig,}++impl LanguageModelConfig {+ /// Get the decoder config for backward compatibility.+ /// This converts the new format to the old DecoderConfig format.+ pub fn decoder_config(&self) -> Result<DecoderConfig, ConfigError> {+ self.model_config.to_decoder_config()+ }+}diff --git a/crates/uzu/src/config/linear.rs b/crates/uzu/src/config/linear.rsindex 49bd4ecab..510eeb614 100644--- a/crates/uzu/src/config/linear.rs+++ b/crates/uzu/src/config/linear.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use super::common::{ConfigDataType, QuantizationMode};+use crate::{ConfigDataType, QuantizationMode};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct QuantizationConfig {diff --git a/crates/uzu/src/config/metadata.rs b/crates/uzu/src/config/metadata.rsindex 573955a18..7c10b12f8 100644--- a/crates/uzu/src/config/metadata.rs+++ b/crates/uzu/src/config/metadata.rs@@ -1,6 +1,10 @@use serde::{Deserialize, Serialize};-use super::LanguageModelConfig;+use crate::{ModelConfig, ModelType};++fn default_model_type() -> ModelType {+ ModelType::LanguageModel+}#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct ModelMetadata {@@ -12,5 +16,7 @@ pub struct ModelMetadata {pub quantization: Option<String>,pub repo: String,pub use_cases: Vec<String>,- pub model_config: LanguageModelConfig,+ #[serde(default = "default_model_type")]+ pub model_type: ModelType,+ pub model_config: ModelConfig,}diff --git a/crates/uzu/src/config/mlp.rs b/crates/uzu/src/config/mlp.rsindex fa9832b44..cd1699f73 100644--- a/crates/uzu/src/config/mlp.rs+++ b/crates/uzu/src/config/mlp.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use super::{common::Activation, linear::LinearConfig};+use crate::{Activation, LinearConfig};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]#[serde(tag = "type")]@@ -15,6 +15,20 @@ pub enum MLPConfig {pub struct DenseMLPConfig {pub linear_config: LinearConfig,pub activation: Activation,+ #[serde(default)]+ pub has_up_biases: bool,+ #[serde(default)]+ pub has_down_biases: bool,+ #[serde(default)]+ pub gate_clipping: Option<[Option<f32>; 2]>,+ #[serde(default)]+ pub up_clipping: Option<[Option<f32>; 2]>,+ #[serde(default = "default_activation_to_gate")]+ pub activation_to_gate: bool,+}++fn default_activation_to_gate() -> bool {+ true}#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]@@ -83,6 +97,11 @@ mod tests {activation: Activation::SiLU {alpha: 1.0,},+ has_up_biases: false,+ has_down_biases: false,+ gate_clipping: None,+ up_clipping: None,+ activation_to_gate: true,});let deserialized_config: MLPConfig = from_str(config_str).unwrap();diff --git a/crates/uzu/src/config/mod.rs b/crates/uzu/src/config/mod.rsindex c08e0df97..7abe6c828 100644--- a/crates/uzu/src/config/mod.rs+++ b/crates/uzu/src/config/mod.rs@@ -1,28 +1,43 @@-pub mod attention;+mod attention;+mod classifier;+mod classifier_model;+mod common;+mod decoder;+mod decoder_layer;+mod embedding;+mod error;+mod generation;+mod language_model;+mod linear;+mod mamba;+mod message_processor;+mod metadata;+mod mlp;+mod model_config;+mod model_type;+mod normalization;+mod rope;+mod transformer;+mod transformer_layer;+pub use attention::*;-pub mod common;+pub use classifier::*;+pub use classifier_model::*;pub use common::*;-pub mod decoder;pub use decoder::*;-pub mod decoder_layer;pub use decoder_layer::*;-pub mod embedding;pub use embedding::*;-pub mod linear;+pub use error::*;+pub use generation::*;+pub use language_model::*;pub use linear::*;-pub mod mlp;+pub use mamba::*;+pub use message_processor::*;+pub use metadata::*;pub use mlp::*;-pub mod normalization;+pub use model_config::*;+pub use model_type::*;pub use normalization::*;-pub mod mamba;-pub use mamba::*;-pub mod rope;pub use rope::*;-pub mod metadata;-pub use metadata::*;-pub mod language_model;-pub use language_model::*;-pub mod message_processor;-pub use message_processor::*;-pub mod generation;-pub use generation::*;+pub use transformer::*;+pub use transformer_layer::*;diff --git a/crates/uzu/src/config/model_config.rs b/crates/uzu/src/config/model_config.rsnew file mode 100644index 000000000..10873810f--- /dev/null+++ b/crates/uzu/src/config/model_config.rs@@ -0,0 +1,34 @@+use serde::{Deserialize, Serialize};++use crate::{ClassifierModelConfig, LanguageModelConfig};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(untagged)]+pub enum ModelConfig {+ LanguageModel(LanguageModelConfig),+ Classifier(ClassifierModelConfig),+}++impl ModelConfig {+ pub fn as_language_model(&self) -> Option<&LanguageModelConfig> {+ match self {+ ModelConfig::LanguageModel(config) => Some(config),+ _ => None,+ }+ }++ pub fn as_classifier(&self) -> Option<&ClassifierModelConfig> {+ match self {+ ModelConfig::Classifier(config) => Some(config),+ _ => None,+ }+ }++ pub fn is_language_model(&self) -> bool {+ matches!(self, ModelConfig::LanguageModel(_))+ }++ pub fn is_classifier(&self) -> bool {+ matches!(self, ModelConfig::Classifier(_))+ }+}diff --git a/crates/uzu/src/config/model_type.rs b/crates/uzu/src/config/model_type.rsnew file mode 100644index 000000000..e5dc7d691--- /dev/null+++ b/crates/uzu/src/config/model_type.rs@@ -0,0 +1,8 @@+use serde::{Deserialize, Serialize};++#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]+#[serde(rename_all = "snake_case")]+pub enum ModelType {+ LanguageModel,+ ClassifierModel,+}diff --git a/crates/uzu/src/config/normalization/mod.rs b/crates/uzu/src/config/normalization/mod.rsnew file mode 100644index 000000000..7dcb34734--- /dev/null+++ b/crates/uzu/src/config/normalization/mod.rs@@ -0,0 +1,5 @@+mod normalization_config;+mod upcast_mode;++pub use normalization_config::*;+pub use upcast_mode::*;diff --git a/crates/uzu/src/config/normalization.rs b/crates/uzu/src/config/normalization/normalization_config.rssimilarity index 74%rename from crates/uzu/src/config/normalization.rsrename to crates/uzu/src/config/normalization/normalization_config.rsindex 95b7a5a7b..832ebb4a9 100644--- a/crates/uzu/src/config/normalization.rs+++ b/crates/uzu/src/config/normalization/normalization_config.rs@@ -1,21 +1,17 @@use serde::{Deserialize, Serialize};-use super::common::ConfigDataType;+use super::UpcastMode;+use crate::ConfigDataType;#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]-#[serde(rename_all = "snake_case")]-pub enum UpcastMode {- OnlyNormalization,- FullLayer,-}--#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]-pub struct RMSNormConfig {+pub struct NormalizationConfig {pub scale_precision: ConfigDataType,pub accumulation_precision: ConfigDataType,pub epsilon: f32,pub scale_offset: Option<f32>,pub upcast_mode: UpcastMode,+ #[serde(default)]+ pub subtract_mean: bool,}#[cfg(test)]@@ -36,15 +32,17 @@ mod tests {}"#;- let ground_truth_config = RMSNormConfig {+ let ground_truth_config = NormalizationConfig {scale_precision: ConfigDataType::BFloat16,accumulation_precision: ConfigDataType::Float32,epsilon: 1e-05,scale_offset: None,upcast_mode: UpcastMode::OnlyNormalization,+ subtract_mean: false,};- let deserialized_config: RMSNormConfig = from_str(config_str).unwrap();+ let deserialized_config: NormalizationConfig =+ from_str(config_str).unwrap();assert_eq!(deserialized_config, ground_truth_config);}}diff --git a/crates/uzu/src/config/normalization/upcast_mode.rs b/crates/uzu/src/config/normalization/upcast_mode.rsnew file mode 100644index 000000000..f03914653--- /dev/null+++ b/crates/uzu/src/config/normalization/upcast_mode.rs@@ -0,0 +1,8 @@+use serde::{Deserialize, Serialize};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(rename_all = "snake_case")]+pub enum UpcastMode {+ OnlyNormalization,+ FullLayer,+}diff --git a/crates/uzu/src/config/rope.rs b/crates/uzu/src/config/rope.rsindex 0b9d56abd..1f1290c13 100644--- a/crates/uzu/src/config/rope.rs+++ b/crates/uzu/src/config/rope.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use super::common::ConfigDataType;+use crate::ConfigDataType;#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct RopeConfigCommon {diff --git a/crates/uzu/src/config/transformer.rs b/crates/uzu/src/config/transformer.rsnew file mode 100644index 000000000..b14e05685--- /dev/null+++ b/crates/uzu/src/config/transformer.rs@@ -0,0 +1,25 @@+use serde::{Deserialize, Serialize};++use crate::{NormalizationConfig, RoPEConfig, TransformerLayerConfig};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct TransformerConfig {+ pub global_rope_config: Option<RoPEConfig>,+ pub local_rope_config: Option<RoPEConfig>,+ pub layer_configs: Vec<TransformerLayerConfig>,+ pub output_norm_config: NormalizationConfig,++ pub model_dim: usize,+ pub hidden_dim: usize,+ #[serde(default)]+ pub num_heads: Option<usize>,+ #[serde(default)]+ pub num_groups: Option<usize>,+ #[serde(default)]+ pub head_dim: Option<usize>,+ #[serde(default)]+ pub attention_scale: Option<f32>,+ #[serde(default)]+ pub num_layers: Option<usize>,+ pub context_length: usize,+}diff --git a/crates/uzu/src/config/transformer_layer.rs b/crates/uzu/src/config/transformer_layer.rsnew file mode 100644index 000000000..d6809067d--- /dev/null+++ b/crates/uzu/src/config/transformer_layer.rs@@ -0,0 +1,199 @@+use serde::{Deserialize, Serialize};++use crate::{AttentionConfig, MLPConfig, MixerConfig, NormalizationConfig};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct TransformerLayerConfig {+ #[serde(alias = "pre_mixer_norm_config")]+ pub pre_attention_norm_config: Option<NormalizationConfig>,+ pub mixer_config: MixerConfig,+ #[serde(alias = "post_mixer_norm_config")]+ pub post_attention_norm_config: Option<NormalizationConfig>,+ pub pre_mlp_norm_config: NormalizationConfig,+ pub mlp_config: MLPConfig,+ pub post_mlp_norm_config: Option<NormalizationConfig>,+}++impl TransformerLayerConfig {+ pub fn attention_config(&self) -> Option<&AttentionConfig> {+ self.mixer_config.as_attention()+ }+}++#[cfg(test)]+mod tests {+ use serde_json::from_str;++ use super::{+ super::{+ attention::AttentionConfig,+ common::{Activation, ConfigDataType, QuantizationMode},+ linear::{LinearConfig, QuantizationConfig},+ normalization::UpcastMode,+ },+ *,+ };+ use crate::config::mlp;++ #[test]+ fn test_transformer_layer_config() {+ let config_str = r#"+ {+ "pre_attention_norm_config": {+ "scale_precision": "bfloat16",+ "accumulation_precision": "float32",+ "epsilon": 1e-05,+ "scale_offset": null,+ "upcast_mode": "only_normalization",+ "subtract_mean": false+ },+ "mixer_config": {+ "type": "AttentionConfig",+ "qkv_projection_config": {+ "type": "QLoRALinearConfig",+ "group_size": 32,+ "weight_quantization_mode": "uint4",+ "activation_quantization_mode": "int8",+ "activation_precision": "bfloat16",+ "lora_rank": 16,+ "lora_scale": 2.0+ },+ "out_projection_config": {+ "type": "QLoRALinearConfig",+ "group_size": 32,+ "weight_quantization_mode": "uint4",+ "activation_quantization_mode": "int8",+ "activation_precision": "bfloat16",+ "lora_rank": 16,+ "lora_scale": 2.0+ },+ "query_norm_config": null,+ "key_norm_config": null,+ "num_heads": 12,+ "num_groups": 12,+ "head_dim": 64,+ "is_causal": false,+ "scale": null,+ "sliding_window_size": null,+ "logit_soft_cap": null,+ "has_sinks": false,+ "has_qkv_biases": false,+ "has_out_biases": false+ },+ "post_attention_norm_config": null,+ "pre_mlp_norm_config": {+ "scale_precision": "bfloat16",+ "accumulation_precision": "float32",+ "epsilon": 1e-05,+ "scale_offset": null,+ "upcast_mode": "only_normalization",+ "subtract_mean": false+ },+ "mlp_config": {+ "type": "DenseMLPConfig",+ "linear_config": {+ "type": "QLoRALinearConfig",+ "group_size": 32,+ "weight_quantization_mode": "uint4",+ "activation_quantization_mode": "int8",+ "activation_precision": "bfloat16",+ "lora_rank": 16,+ "lora_scale": 2.0+ },+ "activation": {"type": "SiLU"},+ "has_up_biases": false,+ "has_down_biases": false,+ "gate_clipping": null,+ "up_clipping": null,+ "activation_to_gate": false+ },+ "post_mlp_norm_config": null+ }+ "#;++ let ground_truth_config = TransformerLayerConfig {+ pre_attention_norm_config: Some(NormalizationConfig {+ scale_precision: ConfigDataType::BFloat16,+ accumulation_precision: ConfigDataType::Float32,+ epsilon: 1e-5,+ scale_offset: None,+ upcast_mode: UpcastMode::OnlyNormalization,+ subtract_mean: false,+ }),+ pre_mlp_norm_config: NormalizationConfig {+ scale_precision: ConfigDataType::BFloat16,+ accumulation_precision: ConfigDataType::Float32,+ epsilon: 1e-5,+ scale_offset: None,+ upcast_mode: UpcastMode::OnlyNormalization,+ subtract_mean: false,+ },+ mixer_config: MixerConfig::Attention(AttentionConfig {+ qkv_projection_config: LinearConfig::QLoRA {+ quantization: QuantizationConfig {+ group_size: 32,+ weight_quantization_mode: QuantizationMode::UInt4,+ activation_quantization_mode: Some(+ QuantizationMode::Int8,+ ),+ activation_precision: ConfigDataType::BFloat16,+ },+ lora_rank: 16,+ lora_scale: 2.0,+ },+ out_projection_config: LinearConfig::QLoRA {+ quantization: QuantizationConfig {+ group_size: 32,+ weight_quantization_mode: QuantizationMode::UInt4,+ activation_quantization_mode: Some(+ QuantizationMode::Int8,+ ),+ activation_precision: ConfigDataType::BFloat16,+ },+ lora_rank: 16,+ lora_scale: 2.0,+ },+ query_norm_config: None,+ key_norm_config: None,+ num_heads: Some(12),+ num_groups: Some(12),+ head_dim: Some(64),+ is_causal: Some(false),+ scale: None,+ sliding_window_size: None,+ logit_soft_cap: None,+ has_sinks: false,+ has_qkv_biases: false,+ has_out_biases: false,+ }),+ mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {+ linear_config: LinearConfig::QLoRA {+ quantization: QuantizationConfig {+ group_size: 32,+ weight_quantization_mode: QuantizationMode::UInt4,+ activation_quantization_mode: Some(+ QuantizationMode::Int8,+ ),+ activation_precision: ConfigDataType::BFloat16,+ },+ lora_rank: 16,+ lora_scale: 2.0,+ },+ activation: Activation::SiLU {+ alpha: 1.0,+ },+ has_up_biases: false,+ has_down_biases: false,+ gate_clipping: None,+ up_clipping: None,+ activation_to_gate: false,+ }),+ post_attention_norm_config: None,+ post_mlp_norm_config: None,+ };++ let deserialized_config: TransformerLayerConfig =+ from_str(config_str).unwrap();+ assert_eq!(deserialized_config, ground_truth_config);+ }+}diff --git a/crates/uzu/src/generator/mod.rs b/crates/uzu/src/generator/mod.rsdeleted file mode 100644index dbe4efd98..000000000--- a/crates/uzu/src/generator/mod.rs+++ /dev/null@@ -1,9 +0,0 @@-pub mod context;-pub mod generator;-pub mod gpu_capture;-pub mod grammar;-pub mod gumbel;-pub mod result;-pub mod rng;-pub mod sampler;-pub mod tasks;diff --git a/crates/uzu/src/generator/gpu_capture.rs b/crates/uzu/src/language_model/gpu_capture.rssimilarity index 100%rename from crates/uzu/src/generator/gpu_capture.rsrename to crates/uzu/src/language_model/gpu_capture.rsdiff --git a/crates/uzu/src/generator/grammar.rs b/crates/uzu/src/language_model/grammar.rssimilarity index 100%rename from crates/uzu/src/generator/grammar.rsrename to crates/uzu/src/language_model/grammar.rsdiff --git a/crates/uzu/src/generator/gumbel.rs b/crates/uzu/src/language_model/gumbel.rssimilarity index 100%rename from crates/uzu/src/generator/gumbel.rsrename to crates/uzu/src/language_model/gumbel.rsdiff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/language_model/language_model_generator.rssimilarity index 92%rename from crates/uzu/src/generator/generator.rsrename to crates/uzu/src/language_model/language_model_generator.rsindex 8197d0af8..3fb445fb3 100644--- a/crates/uzu/src/generator/generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -7,18 +7,17 @@ use itertools::{Either, Itertools, izip};use mpsgraph::CommandBuffer;use super::{- context::GeneratorContext,+ LanguageModelGeneratorContext,gpu_capture::GpuCaptureManager,+ grammar::CompiledGrammar,result::{GenerateResult, PrefillResult},- tasks::{GeneratorEncodedTask, GeneratorRunTask},+ tasks::{LanguageModelGeneratorEncodedTask, LanguageModelGeneratorRunTask},};use crate::{Array,backends::metal::forward_pass::{- ForwardPassState, INVALID_POSITION,- encodable_with_state::{EncodableWithState, EncodingParameters},+ EncodableBlock, EncodingParameters, ForwardPassState, INVALID_POSITION,},- generator::grammar::CompiledGrammar,session::{config::DecodingConfig,parameter::{ConfigResolvableValue, SamplingMethod},@@ -28,24 +27,25 @@ use crate::{utils::env_utils::MetalEnvVar,};-pub struct Generator {+pub struct LanguageModelGenerator {pub decoding_config: DecodingConfig,pub tokens: Vec<u64>,- pub context: GeneratorContext,- encoded_tasks: HashMap<String, GeneratorEncodedTask>,+ pub context: LanguageModelGeneratorContext,+ encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask>,registered_prefix_len: usize,gpu_capture: GpuCaptureManager,}-impl Generator {+impl LanguageModelGenerator {pub fn new(model_path: &Path,decoding_config: DecodingConfig,) -> Result<Self, Error> {let gpu_capture = GpuCaptureManager::new();- let context = GeneratorContext::new(model_path, &decoding_config)?;+ let context =+ LanguageModelGeneratorContext::new(model_path, &decoding_config)?;let prefill_step_size =decoding_config.prefill_step_size.resolve(&context.model_config);let generate_suffix_length = decoding_config.generate_suffix_length();@@ -187,7 +187,7 @@ impl Generator {let _ = last_state.take();});- let task = GeneratorRunTask {+ let task = LanguageModelGeneratorRunTask {token_ids: &step_token_ids,token_positions: &step_token_positions,token_bitmask: Some(&step_token_bitmask),@@ -345,7 +345,7 @@ impl Generator {.chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();- let task = GeneratorRunTask {+ let task = LanguageModelGeneratorRunTask {token_ids: &token_ids,token_positions: &token_positions,token_bitmask: Some(&token_bitmask),@@ -384,6 +384,162 @@ impl Generator {})}+ /// Prepares async buffers for generation.+ /// Must be called after prefill, before async_generate loop.+ pub fn prepare_async(+ &mut self,+ tokens_to_generate: usize,+ ) {+ let prefill_count = self.tokens.len();+ self.context+ .async_buffers+ .prepare_positions(prefill_count, tokens_to_generate);+ self.context+ .async_buffers+ .prepare_seeds(&mut self.context.next_seed, tokens_to_generate);+ self.context.async_buffers.reset_counter();+ }++ /// Submits a single async forward pass.+ /// Does NOT block. Callback fires when GPU completes.+ ///+ /// - `pass_idx`: Index of this pass (0, 1, 2, ...)+ /// - `sampling_method`: Sampling configuration+ /// - `on_complete`: Callback receiving sampled token as u64+ pub fn async_generate<F>(+ &mut self,+ pass_idx: usize,+ sampling_method: SamplingMethod,+ on_complete: F,+ ) -> Result<(), Error>+ where+ F: FnOnce(u64) + Send + 'static,+ {+ assert_eq!(+ self.decoding_config.generate_suffix_length(),+ 1,+ "async_generate only supports suffix_length=1"+ );++ // Extract values from async_buffers before mutable borrow+ let current_counter = self.context.async_buffers.counter.get();+ let is_continuation = current_counter > 0;+ let batch_size = self.context.async_buffers.batch_size;+ let slot = pass_idx % batch_size;+ let async_event = self.context.async_buffers.event.clone();+ let results_buffer = self.context.async_buffers.results.clone();+ let async_positions_buffer =+ self.context.async_buffers.positions.clone();+ let async_seeds_buffer = self.context.async_buffers.seeds.clone();++ let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;++ let token_ids: [u64; 1] = [last_token];+ let token_positions: [usize; 1] = [0]; // Ignored, using async buffer+ let token_seeds: [u64; 1] = [0]; // Ignored, using async buffer++ let async_positions = Some((&async_positions_buffer, pass_idx));+ let async_seeds = Some((&async_seeds_buffer, pass_idx));++ let mut state = ForwardPassState::new_llm(+ self.context.mtl_context.clone(),+ &self.context.decoder_config,+ &self.context.model_shape,+ &self.context.scratch_buffers,+ self.context.cache_layers.clone(),+ self.context.shared_buffers.clone(),+ &token_ids,+ &token_positions,+ None, // token_bitmask - ignored, using async buffer+ &token_seeds,+ 1, // active_suffix_length+ false, // is_prefilling+ None, // external_bias_fn+ is_continuation,+ async_positions,+ async_seeds,+ );+ if let Some(method) = state.sampling_method_mut() {+ *method = Some(sampling_method);+ }++ self.context.reset_command_buffer();+ let root_command_buffer =+ self.context.command_buffer.root_command_buffer();++ // Wait on previous pass if this is a continuation+ if is_continuation {+ root_command_buffer+ .encode_wait_for_event(&async_event, current_counter);+ }++ // Encode forward pass+ self.context.executables.encode(+ &mut state,+ &self.context.command_buffer,+ &EncodingParameters::new(false, false, false),+ );++ // Encode sampling+ self.context.gpu_sampler.encode(+ &mut state,+ &self.context.command_buffer,+ &EncodingParameters::new(false, false, false),+ );++ // Copy sampled token: sampling_output → token_ids (for next pass)+ // and sampling_output → results[slot] (for callback)+ let sampling_output = state+ .sampling_output()+ .expect("sampling_output must exist after sampling encode");+ let sampling_output_buffer =+ unsafe { sampling_output.borrow_mut().mtl_buffer().clone() };+ let token_ids_buffer = self.context.scratch_buffers.token_ids.clone();++ let encoder = root_command_buffer.new_compute_command_encoder();+ self.context.token_copy.encode_to_token_ids(+ &sampling_output_buffer,+ &token_ids_buffer,+ encoder,+ );+ self.context.token_copy.encode_to_results(+ &sampling_output_buffer,+ &results_buffer,+ slot,+ encoder,+ );+ encoder.end_encoding();++ // Signal event for next pass+ let next_counter = current_counter + 1;+ root_command_buffer.encode_signal_event(&async_event, next_counter);+ self.context.async_buffers.counter.set(next_counter);++ // Add completion handler+ let results_buffer_clone = results_buffer.clone();+ let callback = Arc::new(std::sync::Mutex::new(Some(on_complete)));++ let block = ConcreteBlock::new(move |_: &metal::CommandBufferRef| {+ let token = {+ let ptr = results_buffer_clone.contents() as *const u32;+ unsafe { *ptr.add(slot) as u64 }+ };+ if let Some(cb) = callback.lock().unwrap().take() {+ cb(token);+ }+ });+ let block = block.copy();++ root_command_buffer.add_completed_handler(&block);+ self.context.command_buffer.commit_and_continue();++ Ok(())+ }++ pub fn has_attention_layers(&self) -> bool {+ self.context.decoder_config.has_attention_layers()+ }+pub fn clear_cache(&mut self) {objc2::rc::autoreleasepool(|_pool| {self.encoded_tasks.clear();@@ -406,11 +562,15 @@ impl Generator {&mut self,suffix_length: usize,) {- let task = GeneratorRunTask {- token_ids: &repeat_n(0, suffix_length).collect::<Box<[u64]>>(),- token_positions: &(0..suffix_length).collect::<Box<[usize]>>(),+ let token_ids: Vec<u64> = vec![0; suffix_length];+ let token_positions: Vec<usize> = (0..suffix_length).collect();+ let token_seeds: Vec<u64> = vec![0; suffix_length];++ let task = LanguageModelGeneratorRunTask {+ token_ids: &token_ids,+ token_positions: &token_positions,token_bitmask: None,- token_seeds: &repeat_n(0, suffix_length).collect::<Box<[u64]>>(),+ token_seeds: &token_seeds,expected_number_of_new_tokens: suffix_length,active_suffix_length: suffix_length,is_prefilling: false,@@ -422,7 +582,7 @@ impl Generator {fn run_model(&mut self,- task: GeneratorRunTask,+ task: LanguageModelGeneratorRunTask,warmup: bool,allow_pre_encode: bool,sampling_method: SamplingMethod,@@ -431,7 +591,9 @@ impl Generator {let run_start = Instant::now();let mut state = task.create_state(&mut self.context, None);- state.sampling_method = Some(sampling_method);+ if let Some(method) = state.sampling_method_mut() {+ *method = Some(sampling_method);+ }let is_first_decode = !warmup && task.token_ids.len() == 1;let should_capture =@@ -508,7 +670,7 @@ impl Generator {&mut self,state: &mut ForwardPassState,) -> Result<Vec<u64>, Error> {- let sampling_output = state.sampling_output.as_ref()+ let sampling_output = state.sampling_output().expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");let output_buffer = sampling_output.borrow();@@ -581,166 +743,4 @@ impl Generator {self.registered_prefix_len = desired_prefix_len;}}-- /// Prepares async buffers for generation.- /// Must be called after prefill, before async_generate loop.- pub fn prepare_async(- &mut self,- tokens_to_generate: usize,- ) {- let prefill_count = self.tokens.len();- self.context- .async_buffers- .prepare_positions(prefill_count, tokens_to_generate);- self.context- .async_buffers- .prepare_seeds(&mut self.context.next_seed, tokens_to_generate);- self.context.async_buffers.reset_counter();- }-- /// Submits a single async forward pass.- /// Does NOT block. Callback fires when GPU completes.- ///- /// - `pass_idx`: Index of this pass (0, 1, 2, ...)- /// - `sampling_method`: Sampling configuration- /// - `on_complete`: Callback receiving sampled token as u64- pub fn async_generate<F>(- &mut self,- pass_idx: usize,- sampling_method: SamplingMethod,- on_complete: F,- ) -> Result<(), Error>- where- F: FnOnce(u64) + Send + 'static,- {- assert_eq!(- self.decoding_config.generate_suffix_length(),- 1,- "async_generate only supports suffix_length=1"- );-- // Extract values from async_buffers before mutable borrow- let current_counter = self.context.async_buffers.counter.get();- let is_continuation = current_counter > 0;- let batch_size = self.context.async_buffers.batch_size;- let slot = pass_idx % batch_size;- let async_event = self.context.async_buffers.event.clone();- let results_buffer = self.context.async_buffers.results.clone();- let async_positions_buffer =- self.context.async_buffers.positions.clone();- let async_seeds_buffer = self.context.async_buffers.seeds.clone();-- let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;-- let task = GeneratorRunTask {- token_ids: &[last_token],- token_positions: &[0], // Ignored, using async buffer- token_bitmask: None, // Ignored, using async buffer- token_seeds: &[0], // Ignored, using async buffer- expected_number_of_new_tokens: 1,- active_suffix_length: 1,- is_prefilling: false,- };-- let async_positions = Some((&async_positions_buffer, pass_idx));- let async_seeds = Some((&async_seeds_buffer, pass_idx));-- let mut state = ForwardPassState::new(- self.context.mtl_context.clone(),- &self.context.model_config.decoder_config,- &self.context.model_shape,- &self.context.scratch_buffers,- self.context.cache_layers.clone(),- self.context.shared_buffers.clone(),- &task.token_ids,- &task.token_positions,- task.token_bitmask,- &task.token_seeds,- task.active_suffix_length,- task.is_prefilling,- false,- None,- is_continuation,- async_positions,- async_seeds,- );- state.sampling_method = Some(sampling_method);-- self.context.reset_command_buffer();- let root_command_buffer =- self.context.command_buffer.root_command_buffer();-- // Wait on previous pass if this is a continuation- if is_continuation {- root_command_buffer- .encode_wait_for_event(&async_event, current_counter);- }-- // Encode forward pass- self.context.executables.encode(- &mut state,- &self.context.command_buffer,- &EncodingParameters::new(false, false, false),- );-- // Encode sampling- self.context.gpu_sampler.encode(- &mut state,- &self.context.command_buffer,- &EncodingParameters::new(false, false, false),- );-- // Copy sampled token: sampling_output → token_ids (for next pass)- // and sampling_output → results[slot] (for callback)- let sampling_output = state- .sampling_output- .as_ref()- .expect("sampling_output must exist after sampling encode");- let sampling_output_buffer =- unsafe { sampling_output.borrow_mut().mtl_buffer().clone() };- let token_ids_buffer = self.context.scratch_buffers.token_ids.clone();-- let encoder = root_command_buffer.new_compute_command_encoder();- self.context.token_copy.encode_to_token_ids(- &sampling_output_buffer,- &token_ids_buffer,- encoder,- );- self.context.token_copy.encode_to_results(- &sampling_output_buffer,- &results_buffer,- slot,- encoder,- );- encoder.end_encoding();-- // Signal event for next pass- let next_counter = current_counter + 1;- root_command_buffer.encode_signal_event(&async_event, next_counter);- self.context.async_buffers.counter.set(next_counter);-- // Add completion handler- let results_buffer_clone = results_buffer.clone();- let callback = Arc::new(std::sync::Mutex::new(Some(on_complete)));-- let block = ConcreteBlock::new(move |_: &metal::CommandBufferRef| {- let token = {- let ptr = results_buffer_clone.contents() as *const u32;- unsafe { *ptr.add(slot) as u64 }- };- if let Some(cb) = callback.lock().unwrap().take() {- cb(token);- }- });- let block = block.copy();-- root_command_buffer.add_completed_handler(&block);- self.context.command_buffer.commit_and_continue();-- Ok(())- }-- pub fn has_attention_layers(&self) -> bool {- self.context.model_config.decoder_config.has_attention_layers()- }}diff --git a/crates/uzu/src/language_model/mod.rs b/crates/uzu/src/language_model/mod.rsnew file mode 100644index 000000000..9a1fcc735--- /dev/null+++ b/crates/uzu/src/language_model/mod.rs@@ -0,0 +1,12 @@+pub mod gpu_capture;+pub mod grammar;+pub mod gumbel;+pub mod language_model_generator;+pub mod result;+pub mod rng;+pub mod sampler;+pub mod tasks;++// Re-export main types+pub use crate::backends::metal::LanguageModelGeneratorContext;+pub use language_model_generator::LanguageModelGenerator;diff --git a/crates/uzu/src/generator/result.rs b/crates/uzu/src/language_model/result.rssimilarity index 100%rename from crates/uzu/src/generator/result.rsrename to crates/uzu/src/language_model/result.rsdiff --git a/crates/uzu/src/generator/rng.rs b/crates/uzu/src/language_model/rng.rssimilarity index 100%rename from crates/uzu/src/generator/rng.rsrename to crates/uzu/src/language_model/rng.rsdiff --git a/crates/uzu/src/generator/sampler.rs b/crates/uzu/src/language_model/sampler.rssimilarity index 100%rename from crates/uzu/src/generator/sampler.rsrename to crates/uzu/src/language_model/sampler.rsdiff --git a/crates/uzu/src/generator/tasks.rs b/crates/uzu/src/language_model/tasks.rssimilarity index 78%rename from crates/uzu/src/generator/tasks.rsrename to crates/uzu/src/language_model/tasks.rsindex af6cb4bdb..0be74166e 100644--- a/crates/uzu/src/generator/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -2,20 +2,17 @@ use std::mem::size_of;use metal::{Buffer, MTLResourceOptions};-use super::context::GeneratorContext;-use crate::backends::metal::{- ForwardPassState,- forward_pass::encodable_with_state::{- EncodableWithState, EncodingParameters,- },+use super::LanguageModelGeneratorContext;+use crate::backends::metal::forward_pass::{+ EncodableBlock, EncodingParameters, ForwardPassState,};-pub struct GeneratorEncodedTask {+pub struct LanguageModelGeneratorEncodedTask {pub key: String,predicate_buffer: Buffer,}-impl GeneratorEncodedTask {+impl LanguageModelGeneratorEncodedTask {pub fn predicate_buffer(&self) -> &Buffer {&self.predicate_buffer}@@ -33,7 +30,7 @@ impl GeneratorEncodedTask {}#[derive(Debug, Clone)]-pub struct GeneratorRunTask<'a> {+pub struct LanguageModelGeneratorRunTask<'a> {pub token_ids: &'a [u64],pub token_positions: &'a [usize],pub token_bitmask: Option<&'a [u32]>,@@ -43,9 +40,9 @@ pub struct GeneratorRunTask<'a> {pub is_prefilling: bool,}-impl<'a> GeneratorRunTask<'a> {+impl<'a> LanguageModelGeneratorRunTask<'a> {pub fn speculate_next_task(&self) -> Self {- GeneratorRunTask {+ LanguageModelGeneratorRunTask {token_ids: self.token_ids,token_positions: self.token_positions,token_bitmask: self.token_bitmask,@@ -68,12 +65,12 @@ impl<'a> GeneratorRunTask<'a> {pub fn create_state(&self,- context: &mut GeneratorContext,+ context: &mut LanguageModelGeneratorContext,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,) -> ForwardPassState {- let state = ForwardPassState::new(+ ForwardPassState::new_llm(context.mtl_context.clone(),- &context.model_config.decoder_config,+ &context.decoder_config,&context.model_shape,&context.scratch_buffers,context.cache_layers.clone(),@@ -84,33 +81,28 @@ impl<'a> GeneratorRunTask<'a> {self.token_seeds,self.active_suffix_length,self.is_prefilling,- false,external_bias_fn,false,None,None,- );-- return state;+ )}pub fn build_encoded_task(&self,- context: &GeneratorContext,+ context: &LanguageModelGeneratorContext,state: &mut ForwardPassState,parameters: &EncodingParameters,key: String,- ) -> GeneratorEncodedTask {+ ) -> LanguageModelGeneratorEncodedTask {context.executables.encode(state, &context.command_buffer, parameters);- let encoded_task = GeneratorEncodedTask {+ LanguageModelGeneratorEncodedTask {key,predicate_buffer: context.mtl_context.device.new_buffer(size_of::<u32>() as u64,MTLResourceOptions::StorageModeShared,),- };-- encoded_task+ }}}diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rsindex e28e2d944..645f4f965 100644--- a/crates/uzu/src/lib.rs+++ b/crates/uzu/src/lib.rs@@ -1,14 +1,17 @@pub mod backends;+pub mod classifier;pub mod config;-pub mod generator;+pub mod language_model;+pub mod linearizer;pub mod parameters;+pub mod prelude;pub mod session;pub mod speculators;+#[cfg(feature = "tracing")]pub mod tracer;pub mod trie;pub mod utils;pub use utils::*;pub mod device;+pub use config::*;pub use device::*;-pub mod runners;-pub use runners::*;diff --git a/crates/uzu/src/linearizer/mod.rs b/crates/uzu/src/linearizer/mod.rsnew file mode 100644index 000000000..95655d93b--- /dev/null+++ b/crates/uzu/src/linearizer/mod.rs@@ -0,0 +1,8 @@+pub mod node;+pub mod speculated_suffix;+pub mod transition;+pub mod trie;++#[cfg(test)]+mod tests_node;+mod tests_trie;diff --git a/crates/uzu/src/linearizer/node.rs b/crates/uzu/src/linearizer/node.rsnew file mode 100644index 000000000..62df6cb1d--- /dev/null+++ b/crates/uzu/src/linearizer/node.rs@@ -0,0 +1,80 @@+use super::transition::Transition;++#[derive(Debug)]+pub struct Node {+ transitions: Vec<Transition>,+}++impl Default for Node {+ fn default() -> Self {+ Self::new()+ }+}++impl Node {+ pub fn new() -> Self {+ Self {+ transitions: Vec::new(),+ }+ }++ pub fn has_transition(+ &self,+ token: u64,+ ) -> bool {+ self.transitions.iter().any(|t| t.token() == token)+ }++ pub fn get_next(+ &self,+ token: u64,+ ) -> Option<&Node> {+ self.transitions.iter().find(|t| t.token() == token).map(|t| &t.node)+ }++ pub fn get_or_insert_next(+ &mut self,+ token: u64,+ seed: u64,+ ) -> &mut Node {+ if let Some(pos) =+ self.transitions.iter().position(|t| t.token() == token)+ {+ return &mut self.transitions[pos].node;+ }++ self.transitions.push(Transition::new(token, seed));+ &mut self.transitions.last_mut().unwrap().node+ }++ pub fn dfs_with_path<F>(+ &self,+ path: &mut Vec<(isize, u64, u64)>,+ callback: &mut F,+ ) -> isize+ where+ F: FnMut(&[(isize, u64, u64)]),+ {+ let mut node_index = path.last().map(|(idx, _, _)| *idx).unwrap_or(-1);++ for transition in self.transitions.iter() {+ node_index += 1;+ path.push((node_index, transition.token(), transition.seed()));+ callback(path);+ node_index = transition.node.dfs_with_path(path, callback);+ path.pop();+ }++ node_index+ }++ pub fn dfs<F>(+ &self,+ mut callback: F,+ ) where+ F: FnMut(&[(isize, u64, u64)]),+ {+ let mut path: Vec<(isize, u64, u64)> = Vec::new();+ self.dfs_with_path(&mut path, &mut callback);+ }+}diff --git a/crates/uzu/src/linearizer/speculated_suffix.rs b/crates/uzu/src/linearizer/speculated_suffix.rsnew file mode 100644index 000000000..2f3874036--- /dev/null+++ b/crates/uzu/src/linearizer/speculated_suffix.rs@@ -0,0 +1,9 @@+use std::collections::HashMap;++#[derive(Debug)]+pub struct SpeculatedSuffix {+ pub tokens: Vec<u64>,+ pub seeds: Vec<u64>,+ pub indices: Vec<usize>,+ pub transition_map: HashMap<isize, HashMap<u64, isize>>,+}diff --git a/crates/uzu/src/linearizer/tests_node.rs b/crates/uzu/src/linearizer/tests_node.rsnew file mode 100644index 000000000..9f4680d76--- /dev/null+++ b/crates/uzu/src/linearizer/tests_node.rs@@ -0,0 +1,267 @@+#[cfg(test)]+mod tests {+ use std::collections::HashSet;++ use super::super::node::Node;++ #[test]+ fn test_new_node_has_no_transitions() {+ let node = Node::new();+ for i in 0..10 {+ assert!(!node.has_transition(i));+ }+ }++ #[test]+ fn test_has_transition() {+ let mut node = Node::new();+ node.get_or_insert_next(42, 0);++ assert!(node.has_transition(42));+ assert!(!node.has_transition(43));+ }++ #[test]+ fn test_get_next() {+ let mut node = Node::new();+ node.get_or_insert_next(1, 0);++ assert!(node.get_next(1).is_some());+ assert!(node.get_next(2).is_none());+ }++ #[test]+ fn test_get_or_insert_next() {+ let mut node = Node::new();++ // First insertion should create a new node+ let node1 = node.get_or_insert_next(1, 0);+ assert!(!node1.has_transition(99));++ // Add a transition to the child node+ node1.get_or_insert_next(99, 0);++ // Get the same node again+ let node1_again = node.get_or_insert_next(1, 0);++ // Verify it's the same node by checking it has the transition we added+ assert!(node1_again.has_transition(99));++ // Different token should give a different node+ let node2 = node.get_or_insert_next(2, 0);+ assert!(!node2.has_transition(99));+ }++ #[test]+ fn test_multiple_transitions() {+ let mut node = Node::new();++ // Insert several transitions+ for i in 0..5 {+ node.get_or_insert_next(i, 0);+ }++ // Verify all transitions exist+ for i in 0..5 {+ assert!(node.has_transition(i));+ }++ // Verify non-existent transitions+ for i in 5..10 {+ assert!(!node.has_transition(i));+ }+ }++ #[test]+ fn test_dfs_with_path_empty_node() {+ let node = Node::new();+ let mut path: Vec<(isize, u64, u64)> = Vec::new();+ let mut visited = Vec::new();++ node.dfs_with_path(&mut path, &mut |current_path: &[(+ isize,+ u64,+ u64,+ )]| {+ let tokens: Vec<u64> =+ current_path.iter().map(|(_, token, _)| *token).collect();+ visited.push(tokens);+ });++ assert!(visited.is_empty());+ }++ #[test]+ fn test_dfs_with_path_single_node() {+ let mut node = Node::new();+ node.get_or_insert_next(42, 0);++ let mut path: Vec<(isize, u64, u64)> = Vec::new();+ let mut visited = Vec::new();++ node.dfs_with_path(&mut path, &mut |current_path: &[(+ isize,+ u64,+ u64,+ )]| {+ let tokens: Vec<u64> =+ current_path.iter().map(|(_, token, _)| *token).collect();+ visited.push(tokens);+ });++ assert_eq!(visited.len(), 1);+ assert_eq!(visited[0], vec![42]);+ }++ #[test]+ fn test_dfs_with_path_complex_tree() {+ let mut root = Node::new();++ // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]+ let node1 = root.get_or_insert_next(1, 0);+ let node2 = node1.get_or_insert_next(2, 0);+ node2.get_or_insert_next(3, 0);+ node2.get_or_insert_next(4, 0);+ node1.get_or_insert_next(5, 0);+ root.get_or_insert_next(6, 0);++ let mut path: Vec<(isize, u64, u64)> = Vec::new();+ let mut visited = Vec::new();++ root.dfs_with_path(&mut path, &mut |current_path: &[(+ isize,+ u64,+ u64,+ )]| {+ let tokens: Vec<u64> =+ current_path.iter().map(|(_, token, _)| *token).collect();+ visited.push(tokens);+ });++ // Check all expected paths were visited+ assert_eq!(visited.len(), 6);+ assert!(visited.contains(&vec![1]));+ assert!(visited.contains(&vec![1, 2]));+ assert!(visited.contains(&vec![1, 2, 3]));+ assert!(visited.contains(&vec![1, 2, 4]));+ assert!(visited.contains(&vec![1, 5]));+ assert!(visited.contains(&vec![6]));+ }++ #[test]+ fn test_dfs_path_indices() {+ let mut root = Node::new();++ // Build a simple trie: [1], [2]+ root.get_or_insert_next(1, 0);+ root.get_or_insert_next(2, 0);++ let mut path: Vec<(isize, u64, u64)> = Vec::new();+ let mut indices = Vec::new();++ root.dfs_with_path(&mut path, &mut |current_path: &[(+ isize,+ u64,+ u64,+ )]| {+ if !current_path.is_empty() {+ indices.push(current_path.last().unwrap().0);+ }+ });++ // Check indices are unique and as expected+ assert_eq!(indices.len(), 2);+ assert_eq!(indices[0], 0);+ assert_eq!(indices[1], 1);+ }++ #[test]+ fn test_simplified_dfs_empty_node() {+ let node = Node::new();+ let mut visited = Vec::new();++ node.dfs(|current_path: &[(isize, u64, u64)]| {+ let tokens: Vec<u64> =+ current_path.iter().map(|(_, token, _)| *token).collect();+ visited.push(tokens);+ });++ assert!(visited.is_empty());+ }++ #[test]+ fn test_simplified_dfs_complex_tree() {+ let mut root = Node::new();++ // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]+ let node1 = root.get_or_insert_next(1, 0);+ let node2 = node1.get_or_insert_next(2, 0);+ node2.get_or_insert_next(3, 0);+ node2.get_or_insert_next(4, 0);+ node1.get_or_insert_next(5, 0);+ root.get_or_insert_next(6, 0);++ let mut visited = Vec::new();++ root.dfs(|current_path: &[(isize, u64, u64)]| {+ let tokens: Vec<u64> =+ current_path.iter().map(|(_, token, _)| *token).collect();+ visited.push(tokens);+ });++ // Check all expected paths were visited+ assert_eq!(visited.len(), 6);+ assert!(visited.contains(&vec![1]));+ assert!(visited.contains(&vec![1, 2]));+ assert!(visited.contains(&vec![1, 2, 3]));+ assert!(visited.contains(&vec![1, 2, 4]));+ assert!(visited.contains(&vec![1, 5]));+ assert!(visited.contains(&vec![6]));+ }++ #[test]+ fn test_both_dfs_methods_equivalent() {+ let mut root = Node::new();++ // Build a more complex trie+ let node1 = root.get_or_insert_next(10, 0);+ let node2 = node1.get_or_insert_next(20, 0);+ node2.get_or_insert_next(30, 0);+ node2.get_or_insert_next(40, 0);+ node1.get_or_insert_next(50, 0);+ let node3 = root.get_or_insert_next(60, 0);+ node3.get_or_insert_next(70, 0);++ // Get results from dfs_with_path+ let mut path: Vec<(isize, u64, u64)> = Vec::new();+ let mut visited1 = Vec::new();++ root.dfs_with_path(&mut path, &mut |current_path: &[(+ isize,+ u64,+ u64,+ )]| {+ let tokens: Vec<u64> =+ current_path.iter().map(|(_, token, _)| *token).collect();+ visited1.push(tokens);+ });++ // Get results from simplified dfs+ let mut visited2 = Vec::new();++ root.dfs(|current_path: &[(isize, u64, u64)]| {+ let tokens: Vec<u64> =+ current_path.iter().map(|(_, token, _)| *token).collect();+ visited2.push(tokens);+ });++ // Check both methods give the same results+ assert_eq!(visited1.len(), visited2.len());++ // Convert to sets to ignore order differences+ let set1: HashSet<Vec<u64>> = visited1.into_iter().collect();+ let set2: HashSet<Vec<u64>> = visited2.into_iter().collect();++ assert_eq!(set1, set2);+ }+}diff --git a/crates/uzu/src/linearizer/tests_trie.rs b/crates/uzu/src/linearizer/tests_trie.rsnew file mode 100644index 000000000..9896fb206--- /dev/null+++ b/crates/uzu/src/linearizer/tests_trie.rs@@ -0,0 +1,297 @@+#[cfg(test)]+mod tests {+ use std::collections::HashMap;++ use crate::{+ language_model::rng::DerivableSeed,+ linearizer::trie::{TokenTrie, TrieCreationConfig},+ speculators::speculator::Speculator,+ };++ struct StaticSpeculator {+ responses: HashMap<Vec<u64>, HashMap<u64, f32>>,+ }++ impl StaticSpeculator {+ fn new(responses: HashMap<Vec<u64>, HashMap<u64, f32>>) -> Self {+ Self {+ responses,+ }+ }+ }++ impl Speculator for StaticSpeculator {+ fn speculate(+ &self,+ prefix: &[u64],+ ) -> HashMap<u64, f32> {+ self.responses.get(prefix).cloned().unwrap_or_default()+ }+ }++ #[test]+ fn test_from_speculator_applies_prob_cutoff() {+ let mut responses = HashMap::new();+ responses.insert(+ vec![42],+ HashMap::from([(1, 0.9_f32), (2, 0.05_f32), (3, 0.2_f32)]),+ );+ responses.insert(vec![42, 1], HashMap::new());++ let speculator = StaticSpeculator::new(responses);+ let mut seed = DerivableSeed::new(1);++ let trie = TokenTrie::from_speculator(+ &[42],+ &mut seed,+ false,+ &speculator,+ &TrieCreationConfig::default(),+ 5,+ );+ let suffix = trie.linearize(0, 5);++ assert_eq!(suffix.tokens, vec![1, 3]);+ assert!(!suffix.tokens.contains(&2));+ assert_eq!(suffix.transition_map.len(), 3);+ }++ #[test]+ fn test_from_speculator_respects_top_k_limit() {+ let mut responses = HashMap::new();+ responses.insert(vec![5], HashMap::from([(9, 0.8_f32), (10, 0.7_f32)]));+ responses.insert(vec![5, 9], HashMap::from([(11, 0.6_f32)]));+ responses.insert(vec![5, 9, 11], HashMap::new());++ let speculator = StaticSpeculator::new(responses);+ let mut seed = DerivableSeed::new(7);++ let trie = TokenTrie::from_speculator(+ &[5],+ &mut seed,+ false,+ &speculator,+ &TrieCreationConfig {+ top_k: 1,+ prob_cutoff: 0.0,+ },+ 3,+ );++ let suffix = trie.linearize(0, 4);++ assert_eq!(suffix.tokens, vec![9, 11]);+ assert!(!suffix.tokens.contains(&10));+ }++ #[test]+ fn test_from_speculator_includes_last_prefix_token_and_seeds() {+ let mut responses = HashMap::new();+ responses.insert(vec![2, 4], HashMap::from([(8, 0.9_f32)]));+ responses.insert(vec![2, 4, 8], HashMap::new());++ let speculator = StaticSpeculator::new(responses);++ let mut seed = DerivableSeed::new(99);+ let mut expected_seed_stream = DerivableSeed::new(99);+ let expected_seeds =+ vec![expected_seed_stream.next(), expected_seed_stream.next()];++ let trie = TokenTrie::from_speculator(+ &[2, 4],+ &mut seed,+ true,+ &speculator,+ &TrieCreationConfig::default(),+ 2,+ );++ let suffix = trie.linearize(0, 3);++ assert_eq!(suffix.tokens, vec![4, 8]);+ assert_eq!(suffix.seeds, expected_seeds);+ assert_eq!(suffix.indices, vec![0, 1]);+ }++ #[test]+ fn test_from_speculator_stops_at_max_length() {+ let mut responses = HashMap::new();+ responses.insert(+ vec![1],+ HashMap::from([(5, 0.9_f32), (6, 0.8_f32), (7, 0.7_f32)]),+ );+ responses+ .insert(vec![1, 5], HashMap::from([(8, 0.6_f32), (9, 0.5_f32)]));+ responses.insert(vec![1, 5, 8], HashMap::from([(10, 0.4_f32)]));++ let speculator = StaticSpeculator::new(responses);+ let mut seed = DerivableSeed::new(1234);++ let trie = TokenTrie::from_speculator(+ &[1],+ &mut seed,+ false,+ &speculator,+ &TrieCreationConfig::default(),+ 3,+ );++ let suffix = trie.linearize(0, 5);++ assert_eq!(suffix.tokens.len(), 3);+ assert!(suffix.tokens.contains(&5));+ assert!(suffix.tokens.contains(&6));+ assert!(suffix.tokens.contains(&8));+ assert!(!suffix.tokens.contains(&9));+ }++ #[test]+ fn test_from_speculator_with_empty_responses() {+ let speculator = StaticSpeculator::new(HashMap::new());++ let mut seed = DerivableSeed::new(555);+ let mut expected_seed_stream = DerivableSeed::new(555);+ let expected_first_seed = expected_seed_stream.next();++ let trie = TokenTrie::from_speculator(+ &[12, 13],+ &mut seed,+ true,+ &speculator,+ &TrieCreationConfig::default(),+ 4,+ );++ let suffix = trie.linearize(0, 5);++ // Only the last prefix token should be included; no speculated tokens.+ assert_eq!(suffix.tokens, vec![13]);+ assert_eq!(suffix.indices, vec![0]);+ assert_eq!(suffix.seeds, vec![expected_first_seed]);++ assert_eq!(suffix.transition_map.len(), 2);+ let root_transitions = &suffix.transition_map[&-1];+ assert_eq!(root_transitions.len(), 1);+ assert_eq!(root_transitions.get(&13), Some(&0));+ }++ #[test]+ fn test_linearize_empty_trie() {+ let trie = TokenTrie::new();+ let suffix = trie.linearize(0, 10);++ assert!(suffix.tokens.is_empty());+ assert!(suffix.indices.is_empty());+ assert!(suffix.transition_map.len() == 1);+ }++ #[test]+ fn test_linearize_single_sequence() {+ let mut trie = TokenTrie::new();+ trie.insert(&[1, 2, 3]);++ let suffix = trie.linearize(0, 5);++ assert_eq!(suffix.tokens.len(), 3);+ assert!(suffix.tokens.contains(&1));+ assert!(suffix.tokens.contains(&2));+ assert!(suffix.tokens.contains(&3));++ assert_eq!(suffix.transition_map.len(), 4);++ let root_transitions = &suffix.transition_map[&-1];+ assert!(root_transitions.contains_key(&1));++ let token1_idx = *root_transitions.get(&1).unwrap();+ let token1_transitions = &suffix.transition_map[&token1_idx];+ assert!(token1_transitions.contains_key(&2));++ let token2_idx = *token1_transitions.get(&2).unwrap();+ let token2_transitions = &suffix.transition_map[&token2_idx];+ assert!(token2_transitions.contains_key(&3));+ }++ #[test]+ fn test_linearize_branching_trie() {+ let mut trie = TokenTrie::new();+ trie.insert(&[1, 2, 3]);+ trie.insert(&[1, 2, 4]);+ trie.insert(&[5, 6]);++ let suffix = trie.linearize(10, 10);++ assert!(suffix.tokens.contains(&1));+ assert!(suffix.tokens.contains(&2));+ assert!(suffix.tokens.contains(&3));+ assert!(suffix.tokens.contains(&4));+ assert!(suffix.tokens.contains(&5));+ assert!(suffix.tokens.contains(&6));++ assert_eq!(suffix.tokens.len(), 6);++ assert_eq!(suffix.transition_map.len(), 7);++ let root_transitions = &suffix.transition_map[&-1];+ assert!(root_transitions.contains_key(&1));+ assert!(root_transitions.contains_key(&5));++ let token1_idx = *root_transitions.get(&1).unwrap();+ let token1_transitions = &suffix.transition_map[&token1_idx];+ assert!(token1_transitions.contains_key(&2));++ let token2_idx = *token1_transitions.get(&2).unwrap();+ let token2_transitions = &suffix.transition_map[&token2_idx];+ assert!(token2_transitions.contains_key(&3));+ assert!(token2_transitions.contains_key(&4));++ let token5_idx = *root_transitions.get(&5).unwrap();+ let token5_transitions = &suffix.transition_map[&token5_idx];+ assert!(token5_transitions.contains_key(&6));+ }++ #[test]+ fn test_linearize_max_length() {+ let mut trie = TokenTrie::new();+ trie.insert(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);++ let suffix = trie.linearize(0, 5);++ assert_eq!(suffix.tokens.len(), 5);+ assert_eq!(suffix.indices.len(), 5);+ assert_eq!(suffix.transition_map.len(), 6);++ for i in 1..=5 {+ assert!(suffix.tokens.contains(&i));+ }++ let root_transitions = &suffix.transition_map[&-1];+ assert!(root_transitions.contains_key(&1));++ let mut current_idx = *root_transitions.get(&1).unwrap();+ for i in 2..=4 {+ let transitions = &suffix.transition_map[¤t_idx];+ assert!(transitions.contains_key(&i));+ current_idx = *transitions.get(&i).unwrap();+ }+ }++ #[test]+ fn test_linearize_start_index() {+ let mut trie = TokenTrie::new();+ trie.insert(&[1, 2, 3]);++ let suffix = trie.linearize(100, 10);++ assert_eq!(suffix.tokens.len(), 3);+ assert!(suffix.tokens.contains(&1));+ assert!(suffix.tokens.contains(&2));+ assert!(suffix.tokens.contains(&3));++ for idx in suffix.indices.iter() {+ assert!(*idx >= 100 && *idx <= 102);+ }++ let root_transitions = &suffix.transition_map[&-1];+ assert!(root_transitions.contains_key(&1));+ }+}diff --git a/crates/uzu/src/linearizer/transition.rs b/crates/uzu/src/linearizer/transition.rsnew file mode 100644index 000000000..76426fe6c--- /dev/null+++ b/crates/uzu/src/linearizer/transition.rs@@ -0,0 +1,29 @@+use super::node::Node;++#[derive(Debug)]+pub struct Transition {+ token: u64,+ seed: u64,+ pub node: Node,+}++impl Transition {+ pub fn new(+ token: u64,+ seed: u64,+ ) -> Self {+ Self {+ token,+ seed,+ node: Node::new(),+ }+ }++ pub fn token(&self) -> u64 {+ self.token+ }++ pub fn seed(&self) -> u64 {+ self.seed+ }+}diff --git a/crates/uzu/src/linearizer/trie.rs b/crates/uzu/src/linearizer/trie.rsnew file mode 100644index 000000000..8aa2f86f0--- /dev/null+++ b/crates/uzu/src/linearizer/trie.rs@@ -0,0 +1,168 @@+use std::collections::HashMap;++use super::{node::Node, speculated_suffix::SpeculatedSuffix};+use crate::{+ language_model::rng::DerivableSeed, speculators::speculator::Speculator,+};++pub struct TrieCreationConfig {+ pub top_k: usize,+ pub prob_cutoff: f32,+}++impl Default for TrieCreationConfig {+ fn default() -> Self {+ TrieCreationConfig {+ top_k: 2,+ prob_cutoff: 0.1,+ }+ }+}++#[derive(Debug)]+pub struct TokenTrie {+ root: Node,+}++impl Default for TokenTrie {+ fn default() -> Self {+ Self::new()+ }+}++impl TokenTrie {+ pub fn new() -> Self {+ Self {+ root: Node::new(),+ }+ }++ pub fn from_sequences(sequences: &[Vec<u64>]) -> Self {+ let mut trie = Self::new();+ for sequence in sequences {+ trie.insert(sequence);+ }+ trie+ }++ pub fn from_speculator(+ prefix: &[u64],+ seed: &mut DerivableSeed,+ include_last_prefix_token: bool,+ speculator: &dyn Speculator,+ creation_config: &TrieCreationConfig,+ max_length: usize,+ ) -> Self {+ let mut trie = Self::new();+ let mut current_node = &mut trie.root;+ let mut trie_size = 0;++ if include_last_prefix_token {+ current_node = current_node+ .get_or_insert_next(*prefix.last().unwrap(), seed.next());+ trie_size += 1;+ }++ let mut speculation_prefix = prefix.to_vec();+ while trie_size < max_length {+ let speculation_result = speculator.speculate(&speculation_prefix);++ let mut arr =+ speculation_result.into_iter().collect::<Vec<(u64, f32)>>();+ arr.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));++ let arr = arr+ .into_iter()+ .enumerate()+ .filter(|&(i, (_, p))| {+ i == 0 || p > creation_config.prob_cutoff+ })+ .map(|(_, x)| x)+ .take(usize::min(creation_config.top_k, max_length - trie_size))+ .collect::<Vec<(u64, f32)>>();++ let Some(&(top_token, _)) = arr.first() else {+ break;+ };++ trie_size += arr.len();+ for &(tok, _) in arr.iter() {+ current_node.get_or_insert_next(tok, seed.next());+ }++ current_node = current_node.get_or_insert_next(top_token, 0);+ speculation_prefix.push(top_token);+ }++ trie+ }++ pub fn insert<'a, I>(+ &mut self,+ sequence: I,+ ) where+ I: IntoIterator<Item = &'a u64>,+ {+ let mut current_node = &mut self.root;+ for &token in sequence {+ current_node = current_node.get_or_insert_next(token, 0);+ }+ }++ pub fn linearize(+ &self,+ start_index: usize,+ max_length: usize,+ ) -> SpeculatedSuffix {+ let mut tokens = Vec::new();+ let mut seeds = Vec::new();+ let mut indices = Vec::new();++ self.root.dfs(|path| {+ if tokens.len() >= max_length {+ return;+ }+ if !path.is_empty() {+ tokens.push(path.last().unwrap().1);+ seeds.push(path.last().unwrap().2);+ indices.push(start_index + path.len() - 1);+ }+ });++ let tokens_len = tokens.len();++ let mut transition_map: HashMap<+ isize,+ std::collections::HashMap<u64, isize>,+ > = HashMap::new();+ for index in -1..(tokens_len as isize) {+ transition_map.insert(index, HashMap::new());+ }++ self.root.dfs(|path| {+ if path.is_empty()+ || path.last().unwrap().0 >= (max_length as isize)+ {+ return;+ }++ let (current_index, current_token, _) = *path.last().unwrap();+ let parent_index = if path.len() > 1 {+ path[path.len() - 2].0+ } else {+ -1+ };++ if let Some(transitions) = transition_map.get_mut(&parent_index) {+ transitions.insert(current_token, current_index);+ }+ });++ SpeculatedSuffix {+ tokens,+ indices,+ seeds,+ transition_map,+ }+ }+}diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rsnew file mode 100644index 000000000..e177c2921--- /dev/null+++ b/crates/uzu/src/prelude.rs@@ -0,0 +1,27 @@+//! Prelude module containing commonly used types from the uzu crate.+//!+//! This module can be imported with `use uzu::prelude::*;` to bring the most+//! frequently used types into scope.++// Constants & Utilities+// Linearizer+// Session Core+pub use crate::session::ChatSession;+// Session Config+pub use crate::session::config::{DecodingConfig, RunConfig, SpeculatorConfig};+// Session Parameters+pub use crate::session::parameter::{+ ContextLength, ContextMode, PrefillStepSize, SamplingMethod,+ SamplingPolicy, SamplingSeed,+};+// Session Types+pub use crate::session::types::{+ Error, FinishReason, Input, Message, Output, ParsedText, Role, RunStats,+ Stats, StepStats, Text, TotalStats,+};+// Speculators+pub use crate::speculators::speculator::Speculator;+pub use crate::{+ VERSION, linearizer::trie::TokenTrie, session::ClassificationSession,+ speculators::empty_speculator::EmptySpeculator, storage_path,+};diff --git a/crates/uzu/src/runners/decoder_runner.rs b/crates/uzu/src/runners/decoder_runner.rsdeleted file mode 100644index cca97cc23..000000000--- a/crates/uzu/src/runners/decoder_runner.rs+++ /dev/null@@ -1,241 +0,0 @@-#![allow(dead_code)]-use std::{- cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc, time::Instant,-};--use mpsgraph::CommandBuffer as MPSCommandBuffer;-use serde::{Deserialize, Serialize};--use crate::{- backends::metal::{- CacheLayers, DecoderExecutables, ForwardPassState, MTLContext,- ModelShape,- compilation_parameters::CompilationConfig,- forward_pass::{- ForwardPassBuffers, SharedBuffers,- encodable_with_state::{EncodableWithState, EncodingParameters},- },- utils::{CaptureOptions, StdoutCapture},- },- config::{ModelMetadata, decoder::DecoderConfig},- parameters::ParameterLoader,-};-#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct DecoderTestResult {- pub placement_log: String,- pub iterations: u32,- pub time_per_token: f32,- pub tokens_per_second: f32,- pub success: bool,- pub error: Option<String>,-}--struct DecoderTestContext {- mtl_context: Rc<MTLContext>,- mtl_command_queue: metal::CommandQueue,- state: ForwardPassState,- executables: DecoderExecutables,- suffix_length: usize,- _scratch: ForwardPassBuffers,-}--impl DecoderTestContext {- fn new(model_dir: &str) -> Result<(Self, Rc<DecoderConfig>), String> {- let model_path = Path::new(model_dir);- let config_path = model_path.join("config.json");- if !config_path.exists() {- return Err(format!(- "Decoder config file not found at {:?}",- config_path- ));- }- let config_file = File::open(&config_path)- .map_err(|e| format!("Failed to open config: {}", e))?;- let model_metadata: ModelMetadata =- serde_json::from_reader(BufReader::new(config_file))- .map_err(|e| format!("Failed to parse config: {}", e))?;- let decoder_config =- Rc::new(model_metadata.model_config.decoder_config.clone());- let mtl_device = metal::Device::system_default()- .ok_or("No Metal device available".to_string())?;- let mtl_command_queue = mtl_device.new_command_queue();-- let compilation_config = Rc::new(CompilationConfig::default());-- // ------------------------------------------------------------------ // Determine the heap size we need for the worst-case forward pass- // *and* all parameters that will be loaded from the safetensors file.- // ------------------------------------------------------------------ let max_prefix_length = 8192; // Example value, can be passed dynamically- let max_suffix_length = 1; // Example value, can be passed dynamically-- let model_shape = ModelShape::from_decoder_config(&decoder_config);-- // Read safetensors header to know the total size of all tensors- // so we can make the heap large enough to store them.- let weights_path = model_path.join("model.safetensors");- if !weights_path.exists() {- return Err(format!(- "Decoder weights file not found at {:?}",- weights_path- ));- }-- let mtl_context = Rc::new(- MTLContext::new(mtl_device.clone(), mtl_command_queue.clone())- .map_err(|e| format!("Failed to create MetalContext: {}", e))?,- );-- let weights_file = File::open(&weights_path)- .map_err(|e| format!("Failed to open weights: {}", e))?;- let loader = ParameterLoader::new(&weights_file, &mtl_context)- .map_err(|e| format!("Failed to create ParameterLoader: {}", e))?;- let root_loader_view = loader.tree();-- let cache_layers = Rc::new(RefCell::new(CacheLayers::new(- &mtl_context,- &model_shape,- max_prefix_length,- max_suffix_length,- )));- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(- &mtl_context,- &decoder_config,- &model_shape,- )));- shared_buffers.borrow_mut().update_data(&root_loader_view);- let token_ids: Vec<u64> =- (0..max_suffix_length).map(|i| (i % 1000) as u64).collect();- let token_positions: Vec<usize> = (0..max_suffix_length).collect();- let token_seeds: Vec<u64> = (0_u64..max_suffix_length as u64).collect();-- // Scratch buffers sized for the maximum prefix/suffix lengths in this test- let scratch_buffers = ForwardPassBuffers::new(- &mtl_context,- &decoder_config,- &model_shape,- max_prefix_length,- max_suffix_length,- );-- let state = ForwardPassState::new(- mtl_context.clone(),- &decoder_config,- &model_shape,- &scratch_buffers,- cache_layers.clone(),- shared_buffers.clone(),- &token_ids,- &token_positions,- None,- &token_seeds,- token_ids.len(),- false,- false,- None,- false,- None,- None,- );-- let executables = DecoderExecutables::new(- mtl_context.clone(),- decoder_config.clone(),- &root_loader_view,- compilation_config.clone(),- );- Ok((- Self {- mtl_context,- mtl_command_queue,- state,- executables,- suffix_length: max_suffix_length,- _scratch: scratch_buffers,- },- decoder_config,- ))- }-- fn run_warmup_with_placement_analysis(&mut self) -> String {- let mut capture = StdoutCapture::new(CaptureOptions::STDERR);- capture.start();- let command_buffer =- MPSCommandBuffer::from_command_queue(&self.mtl_command_queue);- self.executables.encode(- &mut self.state,- &command_buffer,- &EncodingParameters::new(true, true, false),- );- command_buffer.commit();- command_buffer.command_buffer().wait_until_completed();- capture.stop()- }-- fn run_timing_loop(- &mut self,- iterations: u32,- ) -> Result<std::time::Duration, String> {- // Warmup run (not measured)- let command_buffer =- MPSCommandBuffer::from_command_queue(&self.mtl_command_queue);- self.executables.encode(- &mut self.state,- &command_buffer,- &EncodingParameters::new(true, true, false),- );- command_buffer.commit();- command_buffer.command_buffer().wait_until_completed();- // Measured runs- let start = Instant::now();- for _ in 0..iterations {- let command_buffer =- MPSCommandBuffer::from_command_queue(&self.mtl_command_queue);- self.executables.encode(- &mut self.state,- &command_buffer,- &EncodingParameters::new(false, true, false),- );- command_buffer.commit();- command_buffer.command_buffer().wait_until_completed();- }- Ok(start.elapsed())- }-}--pub fn run_decoder_with_results(model_dir: &str) -> DecoderTestResult {- let mut result = DecoderTestResult {- placement_log: String::new(),- iterations: 0,- time_per_token: 0.0,- tokens_per_second: 0.0,- success: false,- error: None,- };- // Setup context for both placement analysis and timing- let (mut ctx, _decoder_config) = match DecoderTestContext::new(model_dir) {- Ok(pair) => pair,- Err(e) => {- result.error = Some(e);- return result;- },- };- // Warmup run with placement analysis- let placement_log = ctx.run_warmup_with_placement_analysis();- result.placement_log = placement_log;- // After warmup, turn off placement analysis for timing loop- // (Assume placement analysis only affects compilation, not execution)- let iterations = 100;- match ctx.run_timing_loop(iterations) {- Ok(duration) => {- result.iterations = iterations;- result.time_per_token = duration.as_secs_f32() / iterations as f32;- result.tokens_per_second = 1.0 / result.time_per_token;- result.success = true;- },- Err(e) => {- result.error = Some(e);- },- }- result-}diff --git a/crates/uzu/src/runners/mod.rs b/crates/uzu/src/runners/mod.rsdeleted file mode 100644index 1ad5471a3..000000000--- a/crates/uzu/src/runners/mod.rs+++ /dev/null@@ -1,4 +0,0 @@-pub mod decoder_runner;-pub use decoder_runner::*;-pub mod quant_matmul;-pub use quant_matmul::*;diff --git a/crates/uzu/src/runners/quant_matmul.rs b/crates/uzu/src/runners/quant_matmul.rsdeleted file mode 100644index 60350f043..000000000--- a/crates/uzu/src/runners/quant_matmul.rs+++ /dev/null@@ -1,452 +0,0 @@-// Note: Module providing quantized matmul benchmark utilities originally prototyped-// in `examples/quant_matmul_experiments.rs`. The code is copied almost verbatim so-// that external crates can depend on these helpers without having to duplicate the-// logic themselves.--use std::{- collections::HashMap,- rc::Rc,- time::{Instant, SystemTime, UNIX_EPOCH},-};--use clap::Parser;-use half::f16;-use metal::Device;-use mpsgraph::{- CommandBuffer, CompilationDescriptor, DataType as MPSDataType,- DequantizationArguments, Device as MPSDevice, Executable,- ExecutableExecutionDescriptor, ExecutableSerializationDescriptor, Graph,- Optimization, OptimizationProfile, ShapedType, Tensor, TensorData,-};-use objc2::rc::Retained;-use thiserror::Error;--use crate::{- Array, DataType, DeviceContext,- backends::metal::{- MTLContext, MetalArray,- compilation_parameters::{BlockDevice, make_compilation_descriptor},- },- utils::storage::{NSSearchPathDirectory, root_dir},-};--// ================== Public error type =====================--#[derive(Debug, Error)]-pub enum ExampleError {- #[error(transparent)]- Metal(#[from] crate::backends::metal::error::MTLError),- #[error("No Metal device available")]- NoDevice,- #[error("Graph dequantization failed")]- Dequantization,-}--// ================== Public helper types =====================--#[derive(Debug, Clone, Copy)]-pub enum TensorLoadType {- Baked,- RuntimeLoaded,-}--#[derive(Debug, Clone, Copy)]-pub struct QuantizedMatmulDescriptor {- pub weights: TensorLoadType,- pub scales: TensorLoadType,- pub zeros: TensorLoadType,- pub transposed_shapes: bool,- pub device: BlockDevice,- pub optimization_level: Optimization,- pub optimization_profile: OptimizationProfile,-}--// ================== CLI args used by the low-level API =====================--#[derive(Parser, Debug, Clone, Copy)]-#[command(- name = "quant_matmul_experiments (library)",- about = "Flags controlling extra instrumentation when running quantised matmul benchmark"-)]-pub struct Args {- /// Print the compiled executable IR / debug dump- #[arg(long = "print-exec-dump")]- pub print_exec_dump: bool,-- /// Print ANE/GPU placement analysis during compilation- #[arg(long = "print-placement-analysis")]- pub print_placement_analysis: bool,-- /// Serialize the compiled MPSGraph executable to a .mpsgraphpackage file- #[arg(long = "dump-mpsgraphpackage")]- pub dump_mpsgraphpackage: bool,-}--impl Default for Args {- fn default() -> Self {- Self {- print_exec_dump: false,- print_placement_analysis: false,- dump_mpsgraphpackage: false,- }- }-}--// ================== Internal tensor helpers =====================--#[derive(Debug, Clone)]-enum TensorOption {- Constant(Retained<Tensor>),- Placeholder {- placeholder: Retained<Tensor>,- data: Retained<TensorData>,- },-}--impl TensorOption {- #[inline]- fn tensor(&self) -> Retained<Tensor> {- match self {- Self::Constant(t) => t.clone(),- Self::Placeholder {- placeholder,- ..- } => placeholder.clone(),- }- }-- fn feeds(&self) -> HashMap<Retained<Tensor>, Retained<ShapedType>> {- match self {- TensorOption::Constant(_) => HashMap::new(),- TensorOption::Placeholder {- placeholder,- ..- } => HashMap::from_iter([(- placeholder.clone(),- placeholder.shaped_type(),- )]),- }- }-- fn data_feeds(&self) -> HashMap<Retained<Tensor>, Retained<TensorData>> {- match self {- TensorOption::Constant(_) => HashMap::new(),- TensorOption::Placeholder {- placeholder,- data,- } => HashMap::from_iter([(placeholder.clone(), data.clone())]),- }- }-}--#[derive(Debug, Clone)]-struct QMTensorOptions {- input: TensorOption,- weights: TensorOption,- scales: TensorOption,- zeros: TensorOption,-}--impl QMTensorOptions {- fn feeds(&self) -> HashMap<Retained<Tensor>, Retained<ShapedType>> {- HashMap::from_iter(- [&self.input, &self.weights, &self.scales, &self.zeros]- .iter()- .flat_map(|opt| opt.feeds()),- )- }-- fn data_feeds(&self) -> HashMap<Retained<Tensor>, Retained<TensorData>> {- HashMap::from_iter(- [&self.input, &self.weights, &self.scales, &self.zeros]- .iter()- .flat_map(|opt| opt.data_feeds()),- )- }-}--// ================== Internal graph helpers =====================--fn build_quantized_matmul(- graph: &Graph,- tensor_options: &QMTensorOptions,- mtl_context: &Rc<MTLContext>,- compilation_descriptor: &CompilationDescriptor,- transposed_shapes: bool,-) -> Result<Retained<Executable>, ExampleError> {- let i = tensor_options.input.tensor();- let w = tensor_options.weights.tensor();- let s = tensor_options.scales.tensor();- let zp = tensor_options.zeros.tensor();-- let dequantized_weights = graph.dequantize(- &w,- DequantizationArguments::ScaleTensorZeroPointTensorDataType {- scale_tensor: &s,- zero_point_tensor: &zp,- data_type: MPSDataType::Float16,- },- None,- );-- let matmul = if transposed_shapes {- graph.matrix_multiplication(- &i,- &graph.transpose(&dequantized_weights, &[1, 0], None),- None,- )- } else {- graph.matrix_multiplication(- &graph.transpose(&dequantized_weights, &[1, 0], None),- &graph.transpose(&i, &[1, 0], None),- None,- )- };-- let device = MPSDevice::with_device(&mtl_context.device);-- let retained_feeds = tensor_options.feeds();- let feeds = retained_feeds.iter().map(|(t, st)| (&**t, &**st)).collect();-- let executable = graph.compile(- &device,- &feeds,- &[&*matmul],- None,- Some(&compilation_descriptor),- );-- Ok(executable)-}--fn make_tensor_option(- graph: &Graph,- load_type: TensorLoadType,- array: &mut MetalArray,- shape: &[usize],- dtype: DataType,- name: &str,-) -> TensorOption {- match load_type {- TensorLoadType::Baked => {- let tensor_const =- graph.constant_with_data(array.buffer(), shape, dtype.into());- TensorOption::Constant(tensor_const)- },- TensorLoadType::RuntimeLoaded => {- let td = unsafe { array.to_mps_tensor_data() };- let shape_isize: Vec<isize> =- shape.iter().map(|d| *d as isize).collect();- let placeholder =- graph.placeholder(Some(&shape_isize), dtype.into(), Some(name));- TensorOption::Placeholder {- placeholder,- data: td,- }- },- }-}--// ================== Core public API =====================--/// Execute the quantised matmul benchmark once and return the average time per-/// iteration (in milliseconds) across an internal fixed number of runs.-///-/// The behaviour is identical to the original implementation in-/// `examples/quant_matmul_experiments.rs`.-#[allow(clippy::too_many_lines)]-pub fn run_quant_matmul(- qm_descriptor: &QuantizedMatmulDescriptor,- args: &Args,-) -> Result<f64, ExampleError> {- // Constants that do not change between runs- let suffix_length: usize = 128;- let input_dim: usize = 2560;- let output_dim: usize = 3072 * 2;- let group_size: usize = 20;-- // Shapes that depend on descriptor options- let weights_shape = if qm_descriptor.transposed_shapes {- [output_dim, input_dim]- } else {- [input_dim, output_dim]- };- let scales_shape = if qm_descriptor.transposed_shapes {- [output_dim, input_dim / group_size]- } else {- [input_dim / group_size, output_dim]- };- let zero_points_shape = scales_shape;- let input_shape = [-1_isize, input_dim as isize];-- // Concrete shapes for actual buffer allocation at runtime- let input_tensor_shape = [suffix_length, input_dim];- let result_tensor_shape = [suffix_length, output_dim];-- // --- Device & context ---- let device = Device::system_default().ok_or(ExampleError::NoDevice)?;- let command_queue = device.new_command_queue();- let mtl_context = Rc::new(MTLContext::new(device, command_queue)?);-- // --- Arrays ---- let mut weights_array = unsafe {- mtl_context.array_uninitialized(&weights_shape, DataType::U4)- };- weights_array.buffer_mut().fill(0x11);-- let mut scales_array =- mtl_context.array_from_elem(&scales_shape, f16::from_f32(1.0));-- let mut zero_points_array = unsafe {- mtl_context.array_uninitialized(&zero_points_shape, DataType::U4)- };- zero_points_array.buffer_mut().fill(0x00);-- let mut input_array =- mtl_context.array_from_elem(&input_tensor_shape, f16::from_f32(1.0));-- let mut result_array = unsafe {- mtl_context.array_uninitialized(&result_tensor_shape, DataType::F16)- };-- // --- Graph ---- let graph = Graph::new();-- // --- Tensor options for quantized matmul ---- let input_tensor_data = unsafe { input_array.to_mps_tensor_data() };- let input_placeholder = graph.placeholder(- Some(&input_shape),- DataType::F16.into(),- Some("input_ph"),- );-- let tensor_options = QMTensorOptions {- input: TensorOption::Placeholder {- placeholder: input_placeholder,- data: input_tensor_data,- },- weights: make_tensor_option(- &graph,- qm_descriptor.weights,- &mut weights_array,- &weights_shape,- DataType::U4,- "weights_ph",- ),- scales: make_tensor_option(- &graph,- qm_descriptor.scales,- &mut scales_array,- &scales_shape,- DataType::F16,- "scales_ph",- ),- zeros: make_tensor_option(- &graph,- qm_descriptor.zeros,- &mut zero_points_array,- &zero_points_shape,- DataType::U4,- "zeros_ph",- ),- };-- let compilation_descriptor = make_compilation_descriptor(- qm_descriptor.device,- qm_descriptor.optimization_level,- qm_descriptor.optimization_profile,- args.print_placement_analysis,- );-- let executable = build_quantized_matmul(- &graph,- &tensor_options,- &mtl_context,- &compilation_descriptor,- qm_descriptor.transposed_shapes,- )?;-- // Optional printing / dumping per flags- if args.print_exec_dump {- executable.dump();- }-- if args.dump_mpsgraphpackage {- let timestamp = SystemTime::now()- .duration_since(UNIX_EPOCH)- .expect("Time went backwards")- .as_secs();- let package_path = root_dir(NSSearchPathDirectory::Downloads).join(- format!("quant_matmul_experiments-{timestamp}.mpsgraphpackage"),- );- let serialization_desc = ExecutableSerializationDescriptor::new();- executable.serialize_to_graph_package(- &package_path,- Some(&serialization_desc),- );- }-- // --- Run & measure ---- let result_tensor_data = unsafe { result_array.to_mps_tensor_data() };- let retained_data_feeds = tensor_options.data_feeds();- let inputs: Vec<&TensorData> = match executable.feed_tensors() {- Some(feed_tensors) => feed_tensors- .iter()- .map(|t| {- retained_data_feeds- .get(t)- .expect("TensorData for feed tensor not found")- })- .map(|td| &**td)- .collect(),- None => Vec::new(),- };-- let outputs = [&*result_tensor_data];-- // Warm-up- run_once(&mtl_context, &executable, &inputs, &outputs);-- const NUM_ITERATIONS: usize = 1000;- let mut total_duration = 0.0f64; // milliseconds-- for _ in 0..NUM_ITERATIONS {- let start = Instant::now();- run_once(&mtl_context, &executable, &inputs, &outputs);- total_duration += start.elapsed().as_secs_f64() * 1e3;- }-- Ok(total_duration / NUM_ITERATIONS as f64)-}--/// Convenience wrapper that uses default (all-false) flags.-pub fn run_quant_matmul_simple(- qm_descriptor: &QuantizedMatmulDescriptor-) -> Result<f64, ExampleError> {- run_quant_matmul(qm_descriptor, &Args::default())-}--// ================== Internal util =====================--fn run_once(- mtl_context: &Rc<MTLContext>,- executable: &Executable,- inputs: &[&TensorData],- outputs: &[&TensorData],-) {- let command_buffer =- CommandBuffer::from_command_queue(&mtl_context.command_queue);- let root_command_buffer = command_buffer.root_command_buffer().to_owned();-- let exec_desc = ExecutableExecutionDescriptor::new();-- executable.encode_to_command_buffer(- &command_buffer,- inputs,- Some(outputs),- Some(&exec_desc),- );-- command_buffer.commit_and_continue();- root_command_buffer.wait_until_completed();-}diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsnew file mode 100644index 000000000..c90ba499f--- /dev/null+++ b/crates/uzu/src/session/chat_session.rs@@ -0,0 +1,647 @@+use std::{fs::File, io::BufReader, path::PathBuf, time::Instant};++use objc2::rc::autoreleasepool;+use tokenizers::Tokenizer;+use xgrammar::TokenizerInfo;++use crate::{+ backends::metal::forward_pass::cache_layers::CacheLayer,+ config::{MixerConfig, ModelMetadata},+ language_model::{+ LanguageModelGenerator,+ grammar::CompiledGrammar,+ result::{GenerateResult, PrefillResult},+ },+ session::{+ config::{DecodingConfig, RunConfig},+ helpers::{+ Context, InputProcessor, InputProcessorDefault, OutputParser,+ is_directory_fits_ram,+ },+ parameter::{ConfigResolvableValue, ContextMode},+ types::{+ Error, FinishReason, Input, Output, RunStats, Stats, StepStats,+ TotalStats,+ },+ },+};++pub struct ChatSession {+ pub model_path: PathBuf,+ pub model_metadata: ModelMetadata,++ tokenizer: Tokenizer,+ tokenizer_info: TokenizerInfo,+ input_processor: Box<dyn InputProcessor>,+ output_parser: OutputParser,+ llm: Option<LanguageModelGenerator>,+ static_context: Option<Context>,+}++impl ChatSession {+ pub fn new(+ model_path: PathBuf,+ decoding_config: DecodingConfig,+ ) -> Result<Self, Error> {+ if !model_path.exists() {+ return Err(Error::ModelFolderNotFound);+ }++ if !is_directory_fits_ram(&model_path) {+ return Err(Error::NotEnoughMemory);+ }++ let config_path = model_path.join("config.json");+ if !config_path.exists() {+ return Err(Error::UnableToLoadConfig);+ }+ let config_file =+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;+ let model_metadata: ModelMetadata = serde_json::from_reader(+ BufReader::new(config_file),+ )+ .map_err(|err| {+ eprintln!("Failed to parse config.json: {err}");+ Error::UnableToLoadConfig+ })?;++ let is_ssm = model_metadata+ .model_config+ .as_language_model()+ .and_then(|lm| lm.decoder_config().ok())+ .and_then(|dc| dc.layer_configs)+ .map(|layers| {+ layers.iter().any(|layer| {+ matches!(layer.mixer_config, MixerConfig::Mamba(_))+ })+ })+ .unwrap_or(false);+ if is_ssm {+ match decoding_config.context_mode {+ ContextMode::None => {},+ ContextMode::Static {+ ..+ } => {+ return Err(Error::UnsupportedContextModeForModel);+ },+ ContextMode::Dynamic => {+ return Err(Error::UnsupportedContextModeForModel);+ },+ }++ if decoding_config.speculator_config.number_of_speculated_tokens > 0+ {+ return Err(Error::UnsupportedSpeculatorConfigForModel);+ }+ }++ let tokenizer_path = model_path.join("tokenizer.json");+ if !tokenizer_path.exists() {+ return Err(Error::UnableToLoadTokenizer);+ }+ let tokenizer = Tokenizer::from_file(&tokenizer_path)+ .map_err(|_| Error::UnableToLoadTokenizer)?;++ // Create TokenizerInfo for grammar support+ let tokenizer_info =+ TokenizerInfo::from_huggingface(&tokenizer, None, None);++ // Extract language model config+ let language_model_config = model_metadata+ .model_config+ .as_language_model()+ .ok_or(Error::UnableToLoadConfig)?;++ let input_processor = InputProcessorDefault::new(+ language_model_config.message_processor_config.clone(),+ );++ let output_parser = OutputParser::new(+ language_model_config+ .message_processor_config+ .output_parser_regex+ .clone(),+ )?;++ let llm = LanguageModelGenerator::new(&model_path, decoding_config)+ .map_err(Error::from)?;++ Ok(Self {+ model_path,+ model_metadata,+ tokenizer,+ tokenizer_info,+ input_processor: Box::new(input_processor),+ output_parser,+ llm: Some(llm),+ static_context: None,+ })+ }++ pub fn run<F>(+ &mut self,+ input: Input,+ config: RunConfig,+ progress: Option<F>,+ ) -> Result<Output, Error>+ where+ F: Fn(Output) -> bool,+ {+ let context_mode = self+ .llm+ .as_ref()+ .ok_or(Error::LanguageModelGeneratorNotLoaded)?+ .decoding_config+ .context_mode+ .clone();++ match &context_mode {+ ContextMode::None => {+ let llm = self+ .llm+ .as_mut()+ .ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ llm.reset_state();+ },+ ContextMode::Static {+ input,+ } => {+ if self.static_context.is_none() {+ let mut prefill_config = config.clone();+ prefill_config.tokens_limit = 0;+ let (_out, ctx) =+ self.extend(input.clone(), None, prefill_config)?;+ self.static_context = Some(ctx);+ }+ let tmp = self.static_context.take();+ if let Some(ref ctx) = tmp {+ self.reconfigure_llm(Some(ctx))?;+ }+ self.static_context = tmp;+ },+ ContextMode::Dynamic => {},+ }++ let output = self.run_internal(input, config, progress)?;++ match context_mode {+ ContextMode::None+ | ContextMode::Static {+ ..+ } => {+ let llm = self+ .llm+ .as_mut()+ .ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ llm.reset_state();+ },+ ContextMode::Dynamic => {},+ }+ Ok(output)+ }++ pub fn reset(&mut self) -> Result<(), Error> {+ let llm =+ self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ llm.reset_state();+ Ok(())+ }+}++impl ChatSession {+ fn extend(+ &mut self,+ input: Input,+ context: Option<&Context>,+ config: RunConfig,+ ) -> Result<(Output, Context), Error> {+ self.reconfigure_llm(context)?;+ let output =+ self.run_internal(input, config, None::<fn(Output) -> bool>)?;+ let new_context = self.build_context_from_llm()?;+ let llm =+ self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ llm.reset_state();+ Ok((output, new_context))+ }++ fn run_internal<F>(+ &mut self,+ input: Input,+ config: RunConfig,+ progress: Option<F>,+ ) -> Result<Output, Error>+ where+ F: Fn(Output) -> bool,+ {+ let run_start = Instant::now();+ let text =+ self.input_processor.process(&input, config.enable_thinking)?;+ let tokens: Vec<u64> = self+ .tokenizer+ .encode(text.as_str(), false)+ .map_err(|_| Error::UnableToEncodeText)?+ .get_ids()+ .iter()+ .map(|&id| id as u64)+ .collect();++ let language_model_config_for_context = self+ .model_metadata+ .model_config+ .as_language_model()+ .ok_or(Error::UnableToLoadConfig)?;++ let llm =+ self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let context_length = llm+ .decoding_config+ .context_length+ .resolve(language_model_config_for_context);+ if tokens.len() >= context_length {+ return Err(Error::ContextLengthExceeded);+ }+ let prefix_len_before = llm.tokens.len().saturating_sub(1);++ let eos_tokens: Vec<u64> = language_model_config_for_context+ .generation_config+ .stop_token_ids+ .iter()+ .map(|&x| x as u64)+ .collect();++ let finish_reason = |llm: &LanguageModelGenerator,+ tokens_new: Vec<u64>|+ -> Option<FinishReason> {+ let start_idx = prefix_len_before + tokens.len();+ let total_tokens = llm.tokens.len();+ let total_new_tokens = llm.tokens[start_idx..].len();+ let has_eos_token =+ tokens_new.iter().any(|token| eos_tokens.contains(token));+ let context_limit_reached = total_tokens >= context_length;++ if has_eos_token {+ Some(FinishReason::Stop)+ } else if total_new_tokens >= config.tokens_limit as usize {+ Some(FinishReason::Length)+ } else if context_limit_reached {+ Some(FinishReason::ContextLimitReached)+ } else {+ None+ }+ };++ let build_generated_text = |llm: &LanguageModelGenerator,+ tokenizer: &Tokenizer|+ -> Result<String, Error> {+ let start_idx = prefix_len_before + tokens.len();+ let generated_tokens: Vec<u32> = llm.tokens[start_idx..]+ .to_vec()+ .iter()+ .map(|value| *value as u32)+ .collect();+ let generated_text = tokenizer+ .decode(&generated_tokens, true)+ .map_err(|_| Error::UnableToDecodeText)?;+ Ok(generated_text)+ };++ let language_model_config_for_sampling = self+ .model_metadata+ .model_config+ .as_language_model()+ .ok_or(Error::UnableToLoadConfig)?;++ let sampling_method =+ config.sampling_policy.resolve(language_model_config_for_sampling);++ // Create grammar from config if provided+ let mut compiled_grammar: Option<CompiledGrammar> =+ if let Some(ref grammar_config) = config.grammar_config {+ Some(CompiledGrammar::from_config(+ grammar_config,+ None,+ &self.tokenizer_info,+ )?)+ } else {+ None+ };++ let prefill_start = Instant::now();+ let prefix_offset = llm.tokens.len();++ let sample_suffix = config.tokens_limit > 0;+ let prefill_result = llm.prefill(+ tokens.clone(),+ compiled_grammar.as_mut(),+ sampling_method,+ prefix_offset,+ sample_suffix,+ )?;+ let prefill_tokens = prefill_result.tokens.clone();+ let prefill_duration = prefill_start.elapsed().as_secs_f64();+ llm.clear_cache();++ let prefill_finish_reason = finish_reason(llm, prefill_tokens.clone());+ let prefill_generated_text =+ build_generated_text(llm, &self.tokenizer)?;+ let prefill_parsed_text =+ self.output_parser.parse(prefill_generated_text);++ let language_model_config_for_prefill = self+ .model_metadata+ .model_config+ .as_language_model()+ .ok_or(Error::UnableToLoadConfig)?;++ let prefill_suffix_length = llm+ .decoding_config+ .prefill_step_size+ .resolve(language_model_config_for_prefill);+ let prefill_output = Output {+ text: prefill_parsed_text,+ stats: Self::build_stats(+ &self.model_metadata,+ prefill_result.clone(),+ prefill_duration,+ prefill_suffix_length,+ Vec::new(),+ Vec::new(),+ llm.decoding_config.generate_suffix_length(),+ run_start.elapsed().as_secs_f64(),+ tokens.len(),+ llm.tokens[prefix_len_before + tokens.len()..].len(),+ ),+ finish_reason: prefill_finish_reason.clone(),+ };++ let prefill_should_continue = if let Some(progress) = &progress {+ progress(prefill_output.clone())+ } else {+ true+ };++ if !prefill_should_continue || prefill_finish_reason.is_some() {+ if prefill_should_continue {+ return Ok(prefill_output);+ } else {+ return Ok(prefill_output+ .clone_with_finish_reason(Some(FinishReason::Cancelled)));+ }+ }++ let mut generate_results: Vec<GenerateResult> = Vec::new();+ let mut generate_durations: Vec<f64> = Vec::new();+ let generate_output = loop {+ let generate_start = Instant::now();+ let generate_result =+ llm.generate(compiled_grammar.as_mut(), sampling_method)?;+ let generate_tokens = generate_result.tokens.clone();+ let generate_duration = generate_start.elapsed().as_secs_f64();+ generate_results.push(generate_result);+ generate_durations.push(generate_duration);++ let generate_finish_reason = finish_reason(llm, generate_tokens);+ let generate_generated_text =+ build_generated_text(llm, &self.tokenizer)?;+ let generate_parsed_text =+ self.output_parser.parse(generate_generated_text);++ let generate_output = Output {+ text: generate_parsed_text,+ stats: Self::build_stats(+ &self.model_metadata,+ prefill_result.clone(),+ prefill_duration,+ prefill_suffix_length,+ generate_results.clone(),+ generate_durations.clone(),+ llm.decoding_config.generate_suffix_length(),+ run_start.elapsed().as_secs_f64(),+ tokens.len(),+ llm.tokens[prefix_len_before + tokens.len()..].len(),+ ),+ finish_reason: generate_finish_reason.clone(),+ };++ let generate_should_continue = if let Some(progress) = &progress {+ progress(generate_output.clone())+ } else {+ true+ };++ if !generate_should_continue || generate_finish_reason.is_some() {+ if generate_should_continue {+ break generate_output;+ } else {+ break generate_output.clone_with_finish_reason(Some(+ FinishReason::Cancelled,+ ));+ }+ }+ };++ llm.clear_cache();+ Ok(generate_output+ .clone_with_duration(run_start.elapsed().as_secs_f64()))+ }++ fn reconfigure_llm(+ &mut self,+ context: Option<&Context>,+ ) -> Result<(), Error> {+ let llm =+ self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ llm.reset_state();+ if let Some(ctx) = context {+ let mut llm_state = llm.context.cache_layers.borrow_mut();+ for (ctx_layer, gen_layer) in+ ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut())+ {+ match (ctx_layer, gen_layer) {+ (+ CacheLayer::Transformer(src),+ CacheLayer::Transformer(dst),+ ) => {+ let copy_rows = src.prefix_segment_length();+ if copy_rows > 0 {+ {+ let mut dst_keys = dst.keys.borrow_mut();+ let src_keys = src.keys.borrow();+ dst_keys.copy_slice(+ &src_keys,+ 1,+ 0..copy_rows,+ 0,+ );+ }+ {+ let mut dst_values = dst.values.borrow_mut();+ let src_values = src.values.borrow();+ dst_values.copy_slice(+ &src_values,+ 1,+ 0..copy_rows,+ 0,+ );+ }+ }+ dst.state = src.state.clone();+ dst.prefix_token_positions =+ src.prefix_token_positions.clone();+ },+ (+ CacheLayer::StateSpace(src),+ CacheLayer::StateSpace(dst),+ ) => {+ {+ let mut dst_conv = dst.conv_state.borrow_mut();+ let src_conv = src.conv_state.borrow();+ dst_conv.copy_from_array(&src_conv);+ }+ {+ let mut dst_ssm = dst.ssm_state.borrow_mut();+ let src_ssm = src.ssm_state.borrow();+ dst_ssm.copy_from_array(&src_ssm);+ }+ },+ _ => panic!(+ "Layer type mismatch when reconfiguring language model generator cache"+ ),+ }+ }+ drop(llm_state);++ llm.tokens = ctx.tokens.clone();+ }+ Ok(())+ }++ fn build_context_from_llm(&self) -> Result<Context, Error> {+ let llm =+ self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let cache_layers =+ llm.context.cache_layers.borrow().clone(&llm.context.mtl_context);+ let context = Context::new(+ llm.tokens.clone(),+ cache_layers,+ llm.decoding_config.clone(),+ );+ Ok(context)+ }+}++impl ChatSession {+ fn build_stats(+ model_metadata: &ModelMetadata,+ prefill_result: PrefillResult,+ prefill_duration: f64,+ prefill_suffix_length: usize,+ generate_results: Vec<GenerateResult>,+ generate_durations: Vec<f64>,+ generate_suffix_length: usize,+ total_duration: f64,+ tokens_count_input: usize,+ tokens_count_output: usize,+ ) -> Stats {+ let prefill_stats = {+ let tokens_count = prefill_result.tokens.len();+ let tokens_per_second = tokens_count as f64 / prefill_duration;++ // TODO: Unify after removing MPS blocks+ let processed_tokens = if model_metadata.quantization.is_some() {+ let result = tokens_count_input + tokens_count;+ result+ } else {+ let number_of_prefill_steps =+ (tokens_count_input as f32 / prefill_suffix_length as f32)+ .ceil() as usize;+ let result = prefill_suffix_length * number_of_prefill_steps+ + tokens_count;+ result+ };+ let processed_tokens_per_second =+ processed_tokens as f64 / prefill_duration;++ let model_run_count = prefill_result.forwardpass_durations.len();+ let model_run_average_duration =+ prefill_result.forwardpass_durations.iter().sum::<f64>()+ / model_run_count as f64;++ StepStats {+ duration: prefill_duration,+ suffix_length: prefill_suffix_length as u64,+ tokens_count: tokens_count as u64,+ tokens_per_second,+ processed_tokens_per_second,+ model_run: RunStats {+ count: model_run_count as u64,+ average_duration: model_run_average_duration,+ },+ run: None,+ }+ };++ let generate_stats: Option<StepStats>;+ if generate_results.len() != 0 {+ let duration = generate_durations.iter().sum::<f64>();+ let tokens_count = generate_results+ .iter()+ .flat_map(|result| result.tokens.clone())+ .collect::<Vec<u64>>()+ .len();+ let tokens_per_second: f64 = tokens_count as f64 / duration;++ let model_run_count = generate_results.len();+ let model_run_average_duration = generate_results+ .iter()+ .map(|result| result.forwardpass_duration)+ .sum::<f64>()+ / model_run_count as f64;++ let run_count = generate_durations.len();+ let run_average_duration =+ generate_durations.iter().sum::<f64>() / run_count as f64;++ generate_stats = Some(StepStats {+ duration,+ suffix_length: generate_suffix_length as u64,+ tokens_count: tokens_count as u64,+ tokens_per_second,+ processed_tokens_per_second: tokens_per_second,+ model_run: RunStats {+ count: model_run_count as u64,+ average_duration: model_run_average_duration,+ },+ run: Some(RunStats {+ count: run_count as u64,+ average_duration: run_average_duration,+ }),+ });+ } else {+ generate_stats = None;+ }++ let total_stats = TotalStats {+ duration: total_duration,+ tokens_count_input: tokens_count_input as u64,+ tokens_count_output: tokens_count_output as u64,+ };++ let stats = Stats {+ prefill_stats,+ generate_stats,+ total_stats,+ };++ stats+ }+}++impl Drop for ChatSession {+ fn drop(&mut self) {+ autoreleasepool(|_| {+ self.llm = None;+ });+ }+}diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rsnew file mode 100644index 000000000..cae8fb761--- /dev/null+++ b/crates/uzu/src/session/classification_session.rs@@ -0,0 +1,87 @@+use std::path::PathBuf;++use tokenizers::Tokenizer;++use crate::{+ classifier::{ClassificationOutput, Classifier},+ config::ModelMetadata,+ session::{+ helpers::{InputProcessor, InputProcessorDefault},+ types::{Error, Input},+ },+};++pub struct ClassificationSession {+ #[allow(dead_code)]+ model_path: PathBuf,+ #[allow(dead_code)]+ model_metadata: ModelMetadata,+ tokenizer: Tokenizer,+ classifier: Classifier,+ input_processor: Box<dyn InputProcessor>,+}++impl ClassificationSession {+ pub fn new(model_path: PathBuf) -> Result<Self, Error> {+ let config_path = model_path.join("config.json");+ if !config_path.exists() {+ return Err(Error::ModelFolderNotFound);+ }++ let config_file = std::fs::File::open(&config_path)+ .map_err(|_| Error::UnableToLoadConfig)?;+ let model_metadata: ModelMetadata =+ serde_json::from_reader(std::io::BufReader::new(config_file))+ .map_err(|_| Error::UnableToLoadConfig)?;++ let tokenizer_path = model_path.join("tokenizer.json");+ if !tokenizer_path.exists() {+ return Err(Error::UnableToLoadTokenizer);+ }+ let tokenizer = Tokenizer::from_file(&tokenizer_path)+ .map_err(|_| Error::UnableToLoadTokenizer)?;++ let classifier_model_config = model_metadata+ .model_config+ .as_classifier()+ .ok_or(Error::UnableToLoadConfig)?;+ let input_processor = Box::new(InputProcessorDefault::new(+ classifier_model_config.message_processor_config.clone(),+ )) as Box<dyn InputProcessor>;+ let classifier = Classifier::new(&model_path)?;++ Ok(Self {+ model_path,+ model_metadata,+ tokenizer,+ classifier,+ input_processor,+ })+ }++ pub fn classify(+ &mut self,+ input: Input,+ ) -> Result<ClassificationOutput, Error> {+ let preprocessing_start = std::time::Instant::now();+ let text = self.input_processor.process(&input, false)?;+ let tokens: Vec<u64> = self+ .tokenizer+ .encode(text.as_str(), false)+ .map_err(|_| Error::UnableToEncodeText)?+ .get_ids()+ .iter()+ .map(|&id| id as u64)+ .collect();++ let token_positions: Vec<usize> = (0..tokens.len()).collect();+ let preprocessing_duration =+ preprocessing_start.elapsed().as_secs_f64();++ let mut output =+ self.classifier.classify_tokens(tokens, token_positions)?;+ output.stats.preprocessing_duration = preprocessing_duration;++ Ok(output)+ }+}diff --git a/crates/uzu/src/session/mod.rs b/crates/uzu/src/session/mod.rsindex 107657cbf..8633ea0d9 100644--- a/crates/uzu/src/session/mod.rs+++ b/crates/uzu/src/session/mod.rs@@ -1,6 +1,9 @@+mod chat_session;+pub mod classification_session;pub mod config;pub mod helpers;pub mod parameter;-mod session;pub mod types;-pub use session::Session;++pub use chat_session::{ChatSession, ChatSession as Session};+pub use classification_session::ClassificationSession;diff --git a/crates/uzu/src/session/parameter/context_length.rs b/crates/uzu/src/session/parameter/context_length.rsindex b3d504fdd..9abaca6e0 100644--- a/crates/uzu/src/session/parameter/context_length.rs+++ b/crates/uzu/src/session/parameter/context_length.rs@@ -20,7 +20,10 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for ContextLength {&self,config: &LanguageModelConfig,) -> usize {- let model_context_length = config.decoder_config.context_length;+ let model_context_length = config+ .decoder_config()+ .map(|dc| dc.context_length)+ .unwrap_or(config.model_config.transformer_config.context_length);let proposed_value = match self {ContextLength::Default => {if cfg!(target_os = "ios") {diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rsindex 62d2e5d38..795a527b7 100644--- a/crates/uzu/src/session/parameter/prefill_step_size.rs+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs@@ -21,10 +21,13 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {config: &LanguageModelConfig,) -> usize {let default_limit: usize = 1024;- let model_context_length = config.decoder_config.context_length;+ let model_context_length =+ config.model_config.transformer_config.context_length;+let minimal_sliding_window_size = config- .decoder_config- .sliding_window_sizes+ .decoder_config()+ .ok()+ .and_then(|dc| dc.sliding_window_sizes).as_deref().into_iter().flatten()@@ -32,6 +35,7 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {.copied().min().unwrap_or(usize::MAX);+let maximal_value =[model_context_length, minimal_sliding_window_size, default_limit].into_iter()diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rsdeleted file mode 100644index d732913a5..000000000--- a/crates/uzu/src/session/session.rs+++ /dev/null@@ -1,861 +0,0 @@-use std::{- fs::File,- io::BufReader,- path::PathBuf,- sync::{- Arc,- atomic::{AtomicU64, Ordering},- mpsc,- },- time::Instant,-};--fn nanos_to_secs(nanos: u64) -> f64 {- nanos as f64 / 1_000_000_000.0-}--use objc2::rc::autoreleasepool;-use tokenizers::Tokenizer;-use xgrammar::TokenizerInfo;--use crate::{- backends::metal::forward_pass::cache_layers::CacheLayer,- config::{ModelMetadata, decoder_layer::MixerConfig},- generator::{- generator::Generator,- grammar::CompiledGrammar,- result::{GenerateResult, PrefillResult},- },- session::{- config::{DecodingConfig, RunConfig},- helpers::{- Context, InputProcessor, InputProcessorDefault, OutputParser,- is_directory_fits_ram,- },- parameter::{ConfigResolvableValue, ContextMode, SamplingMethod},- types::{- Error, FinishReason, Input, Output, RunStats, Stats, StepStats,- TotalStats,- },- },-};--struct RunContext {- eos_tokens: Vec<u64>,- context_length: usize,- prefix_len_before: usize,- input_tokens_len: usize,- tokens_limit: usize,- prefill_result: PrefillResult,- prefill_duration: f64,- prefill_suffix_length: usize,- run_start: Instant,-}--pub struct Session {- pub model_path: PathBuf,- pub model_metadata: ModelMetadata,-- tokenizer: Tokenizer,- tokenizer_info: TokenizerInfo,- input_processor: Box<dyn InputProcessor>,- output_parser: OutputParser,- generator: Option<Generator>,- static_context: Option<Context>,-}--impl Session {- pub fn new(- model_path: PathBuf,- decoding_config: DecodingConfig,- ) -> Result<Self, Error> {- if !model_path.exists() {- return Err(Error::ModelFolderNotFound);- }-- if !is_directory_fits_ram(&model_path) {- return Err(Error::NotEnoughMemory);- }-- let config_path = model_path.join("config.json");- if !config_path.exists() {- return Err(Error::UnableToLoadConfig);- }- let config_file =- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;- let model_metadata: ModelMetadata = serde_json::from_reader(- BufReader::new(config_file),- )- .map_err(|err| {- eprintln!("Failed to parse config.json: {err}");- Error::UnableToLoadConfig- })?;-- let is_ssm = model_metadata- .clone()- .model_config- .decoder_config- .layer_configs- .unwrap_or(Box::new([]))- .iter()- .any(|layer| matches!(layer.mixer_config, MixerConfig::Mamba(_)));- if is_ssm {- match decoding_config.context_mode {- ContextMode::None => {},- ContextMode::Static {- ..- } => {- return Err(Error::UnsupportedContextModeForModel);- },- ContextMode::Dynamic => {- return Err(Error::UnsupportedContextModeForModel);- },- }-- if decoding_config.speculator_config.number_of_speculated_tokens > 0- {- return Err(Error::UnsupportedSpeculatorConfigForModel);- }- }-- let tokenizer_path = model_path.join("tokenizer.json");- if !tokenizer_path.exists() {- return Err(Error::UnableToLoadTokenizer);- }- let tokenizer = Tokenizer::from_file(&tokenizer_path)- .map_err(|_| Error::UnableToLoadTokenizer)?;-- let tokenizer_info =- TokenizerInfo::from_huggingface(&tokenizer, None, None);-- let input_processor = InputProcessorDefault::new(- model_metadata.model_config.message_processor_config.clone(),- );-- let output_parser = OutputParser::new(- model_metadata- .model_config- .message_processor_config- .output_parser_regex- .clone(),- )?;-- let generator = Generator::new(&model_path, decoding_config)- .map_err(Error::from)?;-- Ok(Self {- model_path,- model_metadata,- tokenizer,- tokenizer_info,- input_processor: Box::new(input_processor),- output_parser,- generator: Some(generator),- static_context: None,- })- }-- pub fn run<F>(- &mut self,- input: Input,- config: RunConfig,- progress: Option<F>,- ) -> Result<Output, Error>- where- F: Fn(Output) -> bool,- {- autoreleasepool(|_| {- let context_mode = self- .generator- .as_ref()- .ok_or(Error::GeneratorNotLoaded)?- .decoding_config- .context_mode- .clone();-- match &context_mode {- ContextMode::None => {- let generator = self- .generator- .as_mut()- .ok_or(Error::GeneratorNotLoaded)?;- generator.reset_state();- },- ContextMode::Static {- input,- } => {- if self.static_context.is_none() {- let mut prefill_config = config.clone();- prefill_config.tokens_limit = 0;- let (_out, run_context) =- self.extend(input.clone(), None, prefill_config)?;- self.static_context = Some(run_context);- }- let tmp = self.static_context.take();- if let Some(ref run_context) = tmp {- self.reconfigure_generator(Some(run_context))?;- }- self.static_context = tmp;- },- ContextMode::Dynamic => {},- }-- let output = self.run_internal(input, config, progress)?;-- match context_mode {- ContextMode::None- | ContextMode::Static {- ..- } => {- let generator = self- .generator- .as_mut()- .ok_or(Error::GeneratorNotLoaded)?;- generator.reset_state();- },- ContextMode::Dynamic => {},- }- Ok(output)- })- }-- pub fn reset(&mut self) -> Result<(), Error> {- let generator =- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;- autoreleasepool(|_| generator.reset_state());- Ok(())- }-}--impl Session {- fn extend(- &mut self,- input: Input,- context: Option<&Context>,- config: RunConfig,- ) -> Result<(Output, Context), Error> {- self.reconfigure_generator(context)?;- let output =- self.run_internal(input, config, None::<fn(Output) -> bool>)?;- let new_context = self.build_context_from_generator()?;- let generator =- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;- generator.reset_state();- Ok((output, new_context))- }-- fn run_internal<F>(- &mut self,- input: Input,- config: RunConfig,- progress: Option<F>,- ) -> Result<Output, Error>- where- F: Fn(Output) -> bool,- {- let generator =- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;-- let mut compiled_grammar: Option<CompiledGrammar> =- if let Some(ref grammar_config) = config.grammar_config {- Some(CompiledGrammar::from_config(- grammar_config,- None,- &self.tokenizer_info,- )?)- } else {- None- };-- let run_start = Instant::now();- let text =- self.input_processor.process(&input, config.enable_thinking)?;- let tokens: Vec<u64> = self- .tokenizer- .encode(text.as_str(), false)- .map_err(|_| Error::UnableToEncodeText)?- .get_ids()- .iter()- .map(|&id| id as u64)- .collect();-- let context_length = generator- .decoding_config- .context_length- .resolve(&self.model_metadata.model_config);- if tokens.len() >= context_length {- return Err(Error::ContextLengthExceeded);- }-- let eos_tokens: Vec<u64> = self- .model_metadata- .model_config- .generation_config- .stop_token_ids- .iter()- .map(|&x| x as u64)- .collect();-- let sampling_method =- config.sampling_policy.resolve(&self.model_metadata.model_config);-- let prefill_start = Instant::now();- let prefix_offset = generator.tokens.len();- let prefix_len_before = prefix_offset.saturating_sub(1);-- let sample_suffix = config.tokens_limit > 0;- let prefill_result = generator.prefill(- tokens.clone(),- compiled_grammar.as_mut(),- sampling_method,- prefix_offset,- sample_suffix,- )?;- let prefill_tokens = prefill_result.tokens.clone();- let prefill_duration = prefill_start.elapsed().as_secs_f64();- generator.clear_cache();-- let prefill_suffix_length = generator- .decoding_config- .prefill_step_size- .resolve(&self.model_metadata.model_config);-- let run_context = RunContext {- eos_tokens,- context_length,- prefix_len_before,- input_tokens_len: tokens.len(),- tokens_limit: config.tokens_limit as usize,- prefill_result: prefill_result.clone(),- prefill_duration,- prefill_suffix_length,- run_start,- };-- let prefill_finish_reason =- Self::check_finish_reason(&run_context, generator, &prefill_tokens);- let prefill_output = Self::build_output(- &self.model_metadata,- &self.tokenizer,- &self.output_parser,- &run_context,- generator,- &[],- &[],- prefill_finish_reason.clone(),- )?;-- let prefill_should_continue = if let Some(ref p) = progress {- p(prefill_output.clone())- } else {- true- };-- if !prefill_should_continue || prefill_finish_reason.is_some() {- if prefill_should_continue {- return Ok(prefill_output);- } else {- return Ok(prefill_output- .clone_with_finish_reason(Some(FinishReason::Cancelled)));- }- }-- let can_use_async = generator.decoding_config.generate_suffix_length()- == 1- && !generator.has_attention_layers()- && compiled_grammar.is_none();-- let generate_output = if can_use_async {- let batch_size = generator- .decoding_config- .async_batch_size- .resolve(&self.model_path);- let (results, durations, finish_reason) = Self::run_async_batch(- &self.model_metadata,- &self.tokenizer,- &self.output_parser,- &run_context,- generator,- sampling_method,- &progress,- batch_size,- )?;- Self::build_output(- &self.model_metadata,- &self.tokenizer,- &self.output_parser,- &run_context,- generator,- &results,- &durations,- Some(finish_reason),- )?- } else {- Self::run_sync_generate(- &self.model_metadata,- &self.tokenizer,- &self.output_parser,- &run_context,- generator,- compiled_grammar.as_mut(),- sampling_method,- &progress,- )?- };-- generator.clear_cache();- Ok(generate_output- .clone_with_duration(run_start.elapsed().as_secs_f64()))- }-- fn reconfigure_generator(- &mut self,- context: Option<&Context>,- ) -> Result<(), Error> {- let generator =- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;- generator.reset_state();- if let Some(run_context) = context {- let mut generator_state =- generator.context.cache_layers.borrow_mut();- for (run_context_layer, gen_layer) in run_context- .cache_layers- .data- .iter()- .zip(generator_state.data.iter_mut())- {- match (run_context_layer, gen_layer) {- (- CacheLayer::Transformer(src),- CacheLayer::Transformer(dst),- ) => {- let copy_rows = src.prefix_segment_length();- if copy_rows > 0 {- {- let mut dst_keys = dst.keys.borrow_mut();- let src_keys = src.keys.borrow();- dst_keys.copy_slice(- &src_keys,- 1,- 0..copy_rows,- 0,- );- }- {- let mut dst_values = dst.values.borrow_mut();- let src_values = src.values.borrow();- dst_values.copy_slice(- &src_values,- 1,- 0..copy_rows,- 0,- );- }- }- dst.state = src.state.clone();- dst.prefix_token_positions =- src.prefix_token_positions.clone();- },- (- CacheLayer::StateSpace(src),- CacheLayer::StateSpace(dst),- ) => {- {- let mut dst_conv = dst.conv_state.borrow_mut();- let src_conv = src.conv_state.borrow();- dst_conv.copy_from_array(&src_conv);- }- {- let mut dst_ssm = dst.ssm_state.borrow_mut();- let src_ssm = src.ssm_state.borrow();- dst_ssm.copy_from_array(&src_ssm);- }- },- _ => panic!(- "Layer type mismatch when reconfiguring generator cache"- ),- }- }- drop(generator_state);-- generator.tokens = run_context.tokens.clone();- }- Ok(())- }-- fn build_context_from_generator(&self) -> Result<Context, Error> {- let generator =- self.generator.as_ref().ok_or(Error::GeneratorNotLoaded)?;- let cache_layers = generator- .context- .cache_layers- .borrow()- .clone(&generator.context.mtl_context);- let context = Context::new(- generator.tokens.clone(),- cache_layers,- generator.decoding_config.clone(),- );- Ok(context)- }-}--impl Session {- fn decode_generated_tokens(- tokenizer: &Tokenizer,- generator: &Generator,- run_context: &RunContext,- ) -> Result<String, Error> {- let start_idx =- run_context.prefix_len_before + run_context.input_tokens_len;- let generated_tokens: Vec<u32> =- generator.tokens[start_idx..].iter().map(|&v| v as u32).collect();- tokenizer- .decode(&generated_tokens, true)- .map_err(|_| Error::UnableToDecodeText)- }-- fn check_finish_reason(- run_context: &RunContext,- generator: &Generator,- new_tokens: &[u64],- ) -> Option<FinishReason> {- let start_idx =- run_context.prefix_len_before + run_context.input_tokens_len;- let total_new_tokens = generator.tokens[start_idx..].len();- let has_eos =- new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));- let context_limit =- generator.tokens.len() >= run_context.context_length;-- if has_eos {- Some(FinishReason::Stop)- } else if total_new_tokens >= run_context.tokens_limit {- Some(FinishReason::Length)- } else if context_limit {- Some(FinishReason::ContextLimitReached)- } else {- None- }- }-- fn build_output(- model_metadata: &ModelMetadata,- tokenizer: &Tokenizer,- output_parser: &OutputParser,- run_context: &RunContext,- generator: &Generator,- generate_results: &[GenerateResult],- generate_durations: &[f64],- finish_reason: Option<FinishReason>,- ) -> Result<Output, Error> {- let text =- Self::decode_generated_tokens(tokenizer, generator, run_context)?;- let parsed = output_parser.parse(text);- let start_idx =- run_context.prefix_len_before + run_context.input_tokens_len;- let output_tokens = generator.tokens[start_idx..].len();-- Ok(Output {- text: parsed,- stats: Self::build_stats(- model_metadata,- run_context.prefill_result.clone(),- run_context.prefill_duration,- run_context.prefill_suffix_length,- generate_results.to_vec(),- generate_durations.to_vec(),- generator.decoding_config.generate_suffix_length(),- run_context.run_start.elapsed().as_secs_f64(),- run_context.input_tokens_len,- output_tokens,- ),- finish_reason,- })- }-- fn run_async_batch<F>(- model_metadata: &ModelMetadata,- tokenizer: &Tokenizer,- output_parser: &OutputParser,- run_context: &RunContext,- generator: &mut Generator,- sampling_method: SamplingMethod,- progress: &Option<F>,- batch_size: usize,- ) -> Result<(Vec<GenerateResult>, Vec<f64>, FinishReason), Error>- where- F: Fn(Output) -> bool,- {- let remaining_by_limit = run_context- .tokens_limit- .saturating_sub(run_context.prefill_result.tokens.len());- let remaining_by_context =- run_context.context_length.saturating_sub(generator.tokens.len());- let tokens_to_generate = remaining_by_limit.min(remaining_by_context);-- generator.prepare_async(tokens_to_generate);-- let (sender, receiver) = mpsc::channel::<(usize, u64, u64)>();- let start_time = Instant::now();- let last_nanos = Arc::new(AtomicU64::new(0));-- let mut results: Vec<GenerateResult> =- Vec::with_capacity(tokens_to_generate);- let mut durations: Vec<f64> = Vec::with_capacity(tokens_to_generate);- let mut finish_reason = FinishReason::Length;- let mut next_to_submit = 0;- let mut in_flight = 0;-- while in_flight > 0 || next_to_submit < tokens_to_generate {- let batch_end =- std::cmp::min(next_to_submit + batch_size, tokens_to_generate);- for idx in next_to_submit..batch_end {- let batch_sender = sender.clone();- let last_callback_nanos = last_nanos.clone();- let batch_start_time = start_time;- generator.async_generate(- idx,- sampling_method,- move |token| {- let now_nanos =- batch_start_time.elapsed().as_nanos() as u64;- let prev_nanos = last_callback_nanos- .swap(now_nanos, Ordering::SeqCst);- let _ = batch_sender.send((- idx,- token,- now_nanos.saturating_sub(prev_nanos),- ));- },- )?;- in_flight += 1;- }- let batch_submitted = batch_end - next_to_submit;- next_to_submit = batch_end;-- for _ in 0..batch_submitted {- let (_, token, duration_nanos) =- receiver.recv().map_err(|_| Error::SamplingFailed)?;- in_flight -= 1;-- let duration = nanos_to_secs(duration_nanos);- generator.tokens.push(token);- results.push(GenerateResult {- tokens: vec![token],- forwardpass_duration: duration,- });- durations.push(duration);-- let should_stop = if run_context.eos_tokens.contains(&token) {- finish_reason = FinishReason::Stop;- true- } else if generator.tokens.len() >= run_context.context_length {- finish_reason = FinishReason::ContextLimitReached;- true- } else if let Some(progress_fn) = progress {- let output = Self::build_output(- model_metadata,- tokenizer,- output_parser,- run_context,- generator,- &results,- &durations,- None,- )?;- if !progress_fn(output) {- finish_reason = FinishReason::Cancelled;- true- } else {- false- }- } else {- false- };-- if should_stop {- while in_flight > 0 {- let _ = receiver.recv();- in_flight -= 1;- }- return Ok((results, durations, finish_reason));- }- }- }-- Ok((results, durations, finish_reason))- }-- fn run_sync_generate<F>(- model_metadata: &ModelMetadata,- tokenizer: &Tokenizer,- output_parser: &OutputParser,- run_context: &RunContext,- generator: &mut Generator,- mut compiled_grammar: Option<&mut CompiledGrammar>,- sampling_method: SamplingMethod,- progress: &Option<F>,- ) -> Result<Output, Error>- where- F: Fn(Output) -> bool,- {- let mut results: Vec<GenerateResult> = Vec::new();- let mut durations: Vec<f64> = Vec::new();-- loop {- let start = Instant::now();- let result = generator- .generate(compiled_grammar.as_deref_mut(), sampling_method)?;- let new_tokens = result.tokens.clone();- let duration = start.elapsed().as_secs_f64();-- results.push(result);- durations.push(duration);-- let finish_reason =- Self::check_finish_reason(run_context, generator, &new_tokens);- let output = Self::build_output(- model_metadata,- tokenizer,- output_parser,- run_context,- generator,- &results,- &durations,- finish_reason.clone(),- )?;-- let should_continue = if let Some(progress_fn) = progress {- progress_fn(output.clone())- } else {- true- };-- if !should_continue || finish_reason.is_some() {- if should_continue {- return Ok(output);- } else {- return Ok(output.clone_with_finish_reason(Some(- FinishReason::Cancelled,- )));- }- }- }- }-}--impl Session {- fn build_stats(- model_metadata: &ModelMetadata,- prefill_result: PrefillResult,- prefill_duration: f64,- prefill_suffix_length: usize,- generate_results: Vec<GenerateResult>,- generate_durations: Vec<f64>,- generate_suffix_length: usize,- total_duration: f64,- tokens_count_input: usize,- tokens_count_output: usize,- ) -> Stats {- let prefill_stats = {- let tokens_count = prefill_result.tokens.len();- let tokens_per_second = tokens_count as f64 / prefill_duration;-- // TODO: Unify after removing MPS blocks- let processed_tokens = if model_metadata.quantization.is_some() {- let result = tokens_count_input + tokens_count;- result- } else {- let number_of_prefill_steps =- (tokens_count_input as f32 / prefill_suffix_length as f32)- .ceil() as usize;- let result = prefill_suffix_length * number_of_prefill_steps- + tokens_count;- result- };- let processed_tokens_per_second =- processed_tokens as f64 / prefill_duration;-- let model_run_count = prefill_result.forwardpass_durations.len();- let model_run_average_duration =- prefill_result.forwardpass_durations.iter().sum::<f64>()- / model_run_count as f64;-- StepStats {- duration: prefill_duration,- suffix_length: prefill_suffix_length as u64,- tokens_count: tokens_count as u64,- tokens_per_second,- processed_tokens_per_second,- model_run: RunStats {- count: model_run_count as u64,- average_duration: model_run_average_duration,- },- run: None,- }- };-- let generate_stats: Option<StepStats>;- if generate_results.len() != 0 {- let duration = generate_durations.iter().sum::<f64>();- let tokens_count = generate_results- .iter()- .flat_map(|result| result.tokens.clone())- .collect::<Vec<u64>>()- .len();- let tokens_per_second: f64 = tokens_count as f64 / duration;-- let model_run_count = generate_results.len();- let model_run_average_duration = generate_results- .iter()- .map(|result| result.forwardpass_duration)- .sum::<f64>()- / model_run_count as f64;-- let run_count = generate_durations.len();- let run_average_duration =- generate_durations.iter().sum::<f64>() / run_count as f64;-- generate_stats = Some(StepStats {- duration,- suffix_length: generate_suffix_length as u64,- tokens_count: tokens_count as u64,- tokens_per_second,- processed_tokens_per_second: tokens_per_second,- model_run: RunStats {- count: model_run_count as u64,- average_duration: model_run_average_duration,- },- run: Some(RunStats {- count: run_count as u64,- average_duration: run_average_duration,- }),- });- } else {- generate_stats = None;- }-- let total_stats = TotalStats {- duration: total_duration,- tokens_count_input: tokens_count_input as u64,- tokens_count_output: tokens_count_output as u64,- };-- let stats = Stats {- prefill_stats,- generate_stats,- total_stats,- };-- stats- }-}--impl Drop for Session {- fn drop(&mut self) {- autoreleasepool(|_| {- self.generator = None;- });- }-}diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rsindex e39bc9f6f..c75b0a14d 100644--- a/crates/uzu/src/session/types/error.rs+++ b/crates/uzu/src/session/types/error.rs@@ -1,3 +1,5 @@+use crate::backends::metal::error::ClassifierError;+#[derive(Debug, thiserror::Error)]pub enum Error {#[error("Model folder not found")]@@ -16,8 +18,8 @@ pub enum Error {UnsupportedContextModeForModel,#[error("Unsupported speculator config for model")]UnsupportedSpeculatorConfigForModel,- #[error("Generator not loaded")]- GeneratorNotLoaded,+ #[error("Language model generator not loaded")]+ LanguageModelGeneratorNotLoaded,#[error("Unable to load prompt template")]UnableToLoadPromptTemplate,#[error("Unable to render prompt template")]@@ -40,4 +42,6 @@ pub enum Error {SamplingFailed,#[error("Grammar error")]GrammarError,+ #[error("Classifier error: {0}")]+ Classifier(#[from] ClassifierError),}diff --git a/crates/uzu/src/tracer/mod.rs b/crates/uzu/src/tracer/mod.rsindex dba1b57e3..43349c40d 100644--- a/crates/uzu/src/tracer/mod.rs+++ b/crates/uzu/src/tracer/mod.rs@@ -1,2 +1,12 @@-pub mod tracer;-pub use tracer::Tracer;+//! Unified trace validation for all model types.+//!+//! This module provides `TraceValidator` which can validate activation traces+//! for any model type (LLM or classifier). The model type is automatically+//! detected from the config.++mod trace_validator;++pub use trace_validator::{+ ArrayTransform, TraceValidator, TracerValidationMetrics,+ TracerValidationResult, TracerValidationResults,+};diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsnew file mode 100644index 000000000..27d689ee2--- /dev/null+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -0,0 +1,1055 @@+//! Unified trace validation for any model type (LLM or classifier).+//!+//! This module provides a single `TraceValidator` that can validate activation+//! traces for any model type. It automatically detects the model type from the+//! config and runs the appropriate validation.++use std::{+ cell::{Ref, RefCell},+ fs::File,+ path::{Path, PathBuf},+ rc::Rc,+};++use half::{bf16, f16};+use ndarray::{IxDyn, s};+use num_traits::NumCast;++use crate::{+ Array, ArrayElement, DataType,+ backends::metal::{+ CacheLayers, KVCacheUpdate, KernelDataType, MTLContext, MetalArray,+ encodable_block::Sampling,+ forward_pass::{+ ArrayId, EncodableBlock, EncodingParameters, ForwardPassState,+ ScratchBuffers, traces::ActivationTrace,+ },+ },+ classifier::Classifier,+ config::ModelMetadata,+ language_model::{+ LanguageModelGeneratorContext,+ sampler::{ArgmaxSampler, LogitsSampler},+ },+ parameters::{ParameterLoader, ParameterTree, read_safetensors_metadata},+ session::{+ config::{DecodingConfig, SpeculatorConfig},+ parameter::{+ AsyncBatchSize, ConfigResolvableValue, ContextLength, ContextMode,+ PrefillStepSize, SamplingSeed,+ },+ types::Error,+ },+};++// ============================================================================+// Validation Types+// ============================================================================++/// Metrics from validating a single tensor.+pub struct TracerValidationMetrics {+ pub atol: f32,+ pub rtol: f32,+ pub fraction_of_allowed_violations: f32,+ pub reference_shape: Vec<usize>,+ pub result_shape: Vec<usize>,+ pub num_violations: usize,+ pub max_allowed_violations: usize,+ pub max_err_idx: usize,+ pub max_err: f32,+ pub max_err_rel: f32,+ pub max_err_reference_value: f32,+ pub rms_diff: f32,+ pub rms_result: f32,+ pub rms_reference: f32,+ pub rel_rms_reference: f32,+ pub diff_max: f32,+ pub diff_avg: f32,+ pub result_nan: bool,+}++impl TracerValidationMetrics {+ pub fn is_valid(&self) -> bool {+ self.num_violations <= self.max_allowed_violations && !self.result_nan+ }++ pub fn message(&self) -> String {+ if self.result_nan {+ return "Result contains NaN values".to_string();+ }++ let allowed_violation_explainer = if self.max_allowed_violations > 0 {+ format!(" (Max {} allowed)", self.max_allowed_violations)+ } else {+ String::new()+ };++ let reference_size: usize = self.reference_shape.iter().product();++ format!(+ "{} violations > {:.1e} + {:.2}% out of total {} elements{}.\n\+ Worst violation: {:.3} ({:.2}%) at index {} (reference value: {:.3}).\n\+ Error RMS: {:.3}.\n\+ RMS of result: {:.3}, RMS of reference: {:.3}.\n\+ Relative error RMS: {:.2}% of RMS of reference.\n\+ Shape: {:?}\n\+ Max diff: {:.3}, Avg diff: {:.3}",+ self.num_violations,+ self.atol,+ self.rtol * 100.0,+ reference_size,+ allowed_violation_explainer,+ self.max_err,+ self.max_err_rel * 100.0,+ self.max_err_idx,+ self.max_err_reference_value,+ self.rms_diff,+ self.rms_result,+ self.rms_reference,+ self.rel_rms_reference * 100.0,+ self.result_shape,+ self.diff_max,+ self.diff_avg,+ )+ }+}++/// Result of validating a single tensor.+pub struct TracerValidationResult {+ pub name: String,+ pub metrics: TracerValidationMetrics,+}++/// Results from validating all traces.+pub struct TracerValidationResults {+ pub suffix_length: usize,+ pub results: Vec<TracerValidationResult>,+ pub tokens_violation_indices: Vec<usize>,+}++impl TracerValidationResults {+ pub fn number_of_tokens_violations(&self) -> usize {+ self.tokens_violation_indices.len()+ }++ pub fn number_of_allowed_tokens_violations(&self) -> usize {+ let threshold: f64 = 0.01;+ (self.suffix_length as f64 * threshold).ceil() as usize+ }++ pub fn is_valid(&self) -> bool {+ self.number_of_tokens_violations()+ <= self.number_of_allowed_tokens_violations()+ }+}++/// Transform to apply to produced arrays before comparison.+pub enum ArrayTransform {+ /// Slice KV cache to match expected shape.+ KVCacheSlice,+ /// Transform SSM conv state layout.+ SsmConvState,+}++// ============================================================================+// Model Context (internal)+// ============================================================================++enum ModelContext {+ LanguageModelGenerator(LanguageModelGeneratorContext),+ Classifier(Classifier),+}++// ============================================================================+// Unified TraceValidator+// ============================================================================++/// Unified trace validator for any model type.+///+/// Automatically detects whether the model is an LLM or classifier and+/// runs the appropriate validation.+pub struct TraceValidator {+ model_path: PathBuf,+ context: ModelContext,+}++impl TraceValidator {+ /// Create a new trace validator for the given model path.+ ///+ /// Automatically detects the model type from config.json.+ pub fn new(model_path: &Path) -> Result<Self, Error> {+ let config_path = model_path.join("config.json");+ if !config_path.exists() {+ return Err(Error::ModelFolderNotFound);+ }++ let config_file =+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;+ let metadata: ModelMetadata =+ serde_json::from_reader(std::io::BufReader::new(config_file))+ .map_err(|_| Error::UnableToLoadConfig)?;++ let context = if metadata.model_config.is_classifier() {+ let classifier = Classifier::new(model_path)?;+ ModelContext::Classifier(classifier)+ } else {+ let prefill_step_size =+ Self::determine_prefill_step_size(model_path);+ let decoding_config = DecodingConfig::new(+ ContextMode::default(),+ ContextLength::default(),+ PrefillStepSize::Custom(prefill_step_size),+ SpeculatorConfig::default(),+ SamplingSeed::default(),+ AsyncBatchSize::default(),+ false,+ );+ let mut llm_context = LanguageModelGeneratorContext::new(+ model_path,+ &decoding_config,+ )?;+ let desired_suffix_length =+ prefill_step_size.max(decoding_config.generate_suffix_length());+ Self::ensure_llm_context_capacity(+ &decoding_config,+ desired_suffix_length,+ &mut llm_context,+ );+ ModelContext::LanguageModelGenerator(llm_context)+ };++ Ok(Self {+ model_path: model_path.to_path_buf(),+ context,+ })+ }++ /// Run trace validation and return results.+ pub fn run(&mut self) -> Result<TracerValidationResults, Error> {+ let traces_path = self.model_path.join("traces.safetensors");+ if !traces_path.exists() {+ return Err(Error::UnableToLoadWeights);+ }++ match &mut self.context {+ ModelContext::LanguageModelGenerator(ctx) => {+ Self::run_llm_validation(ctx, &traces_path)+ },+ ModelContext::Classifier(classifier) => {+ Self::run_classifier_validation(classifier, &traces_path)+ },+ }+ }++ pub fn is_classifier(&self) -> bool {+ matches!(self.context, ModelContext::Classifier(_))+ }++ pub fn is_language_model_generator(&self) -> bool {+ matches!(self.context, ModelContext::LanguageModelGenerator(_))+ }++ // ========================================================================+ // LLM Validation+ // ========================================================================++ fn run_llm_validation(+ ctx: &LanguageModelGeneratorContext,+ traces_path: &Path,+ ) -> Result<TracerValidationResults, Error> {+ let traces_file =+ File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;+ let traces_loader =+ ParameterLoader::new(&traces_file, &ctx.mtl_context)+ .map_err(|_| Error::UnableToLoadWeights)?;+ let traces_view = traces_loader.tree();++ let token_ids = Self::load_array_as_vec::<i32, u64>(+ &traces_view,+ "activation_trace.token_ids",+ );+ let token_positions = Self::load_array_as_vec::<i32, usize>(+ &traces_view,+ "activation_trace.token_positions",+ );+ let token_seeds: Vec<u64> = vec![0; token_ids.len()];++ let mut state = ForwardPassState::new_llm(+ ctx.mtl_context.clone(),+ &ctx.decoder_config,+ &ctx.model_shape,+ &ctx.scratch_buffers,+ ctx.cache_layers.clone(),+ ctx.shared_buffers.clone(),+ &token_ids,+ &token_positions,+ None,+ &token_seeds,+ token_ids.len(),+ false,+ None,+ false,+ None,+ None,+ );++ let root_command_buffer =+ ctx.command_buffer.root_command_buffer().to_owned();+ ctx.executables.encode(+ &mut state,+ &ctx.command_buffer,+ &EncodingParameters::new(false, false, true),+ );+ ctx.command_buffer.commit();+ root_command_buffer.wait_until_completed();++ let traces = state.traces().clone();+ let data_type = ctx.model_shape.activation_data_type();++ // Common layer validation+ let mut results =+ Self::validate_layer_traces(&traces, &traces_view, data_type);++ // LLM-specific: KV cache validation+ let transformer_layers: Vec<usize> = {+ let cache = state.cache_layers().unwrap().borrow();+ cache+ .data+ .iter()+ .enumerate()+ .filter_map(|(index, layer)| {+ layer.as_transformer().map(|_| index)+ })+ .collect()+ };++ for index in transformer_layers {+ let arrays =+ state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);++ if let Ok(expected) =+ traces_view.leaf(&format!("updated_kv_cache.{}.keys", index))+ {+ let keys = arrays[0].borrow();+ results.push(TracerValidationResult {+ name: format!("updated_kv_cache.{}.keys", index),+ metrics: Self::validate_array(+ data_type,+ &expected,+ &keys,+ Some(ArrayTransform::KVCacheSlice),+ ),+ });+ }++ if let Ok(expected) =+ traces_view.leaf(&format!("updated_kv_cache.{}.values", index))+ {+ let values = arrays[1].borrow();+ results.push(TracerValidationResult {+ name: format!("updated_kv_cache.{}.values", index),+ metrics: Self::validate_array(+ data_type,+ &expected,+ &values,+ Some(ArrayTransform::KVCacheSlice),+ ),+ });+ }+ }++ // LLM-specific: SSM state validation+ let ssm_layers: Vec<usize> = {+ let cache = state.cache_layers().unwrap().borrow();+ cache+ .data+ .iter()+ .enumerate()+ .filter_map(|(index, layer)| {+ layer.as_state_space().map(|_| index)+ })+ .collect()+ };++ for index in ssm_layers {+ let arrays = state.arrays(&[+ ArrayId::SsmConvState(index),+ ArrayId::SsmState(index),+ ]);+ let conv_state = arrays[0].borrow();+ let ssm_state = arrays[1].borrow();++ for path in [+ format!("updated_state.{}.conv_state", index),+ format!(+ "activation_trace.layer_results.{}.updated_state.conv_state",+ index+ ),+ ] {+ if let Ok(expected) = traces_view.leaf(&path) {+ results.push(TracerValidationResult {+ name: path,+ metrics: Self::validate_array(+ data_type,+ &expected,+ &conv_state,+ Some(ArrayTransform::SsmConvState),+ ),+ });+ }+ }++ for path in [+ format!("updated_state.{}.ssm_state", index),+ format!(+ "activation_trace.layer_results.{}.updated_state.ssm_state",+ index+ ),+ ] {+ if let Ok(expected) = traces_view.leaf(&path) {+ results.push(TracerValidationResult {+ name: path,+ metrics: Self::validate_array(+ data_type, &expected, &ssm_state, None,+ ),+ });+ }+ }+ }++ // LLM-specific: Token comparison+ let tokens_violation_indices = if let Ok(expected_logits) =+ traces_view.leaf("logits")+ {+ let expected_tokens =+ Self::get_tokens_from_logits(&expected_logits);+ let produced_tokens =+ Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());+ expected_tokens+ .iter()+ .zip(produced_tokens.iter())+ .enumerate()+ .filter_map(|(i, (a, b))| {+ if a != b {+ Some(i)+ } else {+ None+ }+ })+ .collect()+ } else {+ Vec::new()+ };++ Ok(TracerValidationResults {+ suffix_length: token_ids.len(),+ results,+ tokens_violation_indices,+ })+ }++ // ========================================================================+ // Classifier Validation+ // ========================================================================++ fn run_classifier_validation(+ classifier: &mut Classifier,+ traces_path: &Path,+ ) -> Result<TracerValidationResults, Error> {+ let traces_file =+ File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;+ let mtl_context = classifier.context.mtl_context.clone();+ let traces_loader = ParameterLoader::new(&traces_file, &mtl_context)+ .map_err(|_| Error::UnableToLoadWeights)?;+ let traces_view = traces_loader.tree();++ let has_token_ids =+ traces_view.leaf("activation_trace.token_ids").is_ok();+ let has_token_positions =+ traces_view.leaf("activation_trace.token_positions").is_ok();++ if !has_token_ids || !has_token_positions {+ return Ok(Self::handle_missing_tokens(&traces_view));+ }++ let token_ids = Self::load_array_as_vec::<i32, u64>(+ &traces_view,+ "activation_trace.token_ids",+ );+ let token_positions = Self::load_array_as_vec::<i32, usize>(+ &traces_view,+ "activation_trace.token_positions",+ );++ let suffix_length = token_ids.len();++ let (_logits, traces) = classifier+ .forward_pass_with_traces(&token_ids, &token_positions)+ .map_err(|_| Error::GenerateFailed)?;++ let data_type = classifier.context.model_shape.activation_data_type();++ // Common layer validation+ let mut results =+ Self::validate_layer_traces(&traces, &traces_view, data_type);++ // Classifier-specific: embedding_norm, output_pooling+ let classifier_results =+ Self::validate_classifier_traces(&traces, &traces_view, data_type);+ results.extend(classifier_results);++ Ok(TracerValidationResults {+ suffix_length,+ results,+ tokens_violation_indices: Vec::new(), // Classifiers don't compare tokens+ })+ }++ fn handle_missing_tokens(+ traces_view: &ParameterTree<Rc<MTLContext>>+ ) -> TracerValidationResults {+ if let Ok(expected_logits) = traces_view.leaf("logits") {+ let reference_shape = expected_logits.shape().to_vec();+ let metrics = TracerValidationMetrics {+ atol: 0.0,+ rtol: 0.0,+ fraction_of_allowed_violations: 0.0,+ reference_shape: reference_shape.clone(),+ result_shape: reference_shape,+ num_violations: 0,+ max_allowed_violations: 0,+ max_err_idx: 0,+ max_err: 0.0,+ max_err_rel: 0.0,+ max_err_reference_value: 0.0,+ rms_diff: 0.0,+ rms_result: 0.0,+ rms_reference: 0.0,+ rel_rms_reference: 0.0,+ diff_max: 0.0,+ diff_avg: 0.0,+ result_nan: false,+ };+ return TracerValidationResults {+ suffix_length: 1,+ results: vec![TracerValidationResult {+ name: "activation_trace.logits".to_string(),+ metrics,+ }],+ tokens_violation_indices: Vec::new(),+ };+ }++ TracerValidationResults {+ suffix_length: 1,+ results: Vec::new(),+ tokens_violation_indices: Vec::new(),+ }+ }++ // ========================================================================+ // Common Validation Helpers+ // ========================================================================++ fn validate_layer_traces(+ traces: &Rc<RefCell<ActivationTrace>>,+ traces_view: &ParameterTree<Rc<MTLContext>>,+ data_type: DataType,+ ) -> Vec<TracerValidationResult> {+ let mut results = Vec::new();++ let validate = |path: &str,+ array: &Ref<MetalArray>|+ -> Option<TracerValidationResult> {+ if traces_view.leaf(path).is_ok() {+ Some(TracerValidationResult {+ name: path.to_string(),+ metrics: Self::validate_array_with_name(+ data_type,+ traces_view,+ path,+ array,+ ),+ })+ } else {+ None+ }+ };++ for (index, layer_traces) in+ traces.borrow().layer_results.iter().enumerate()+ {+ let path = |suffix: &str| -> String {+ format!(+ "activation_trace.layer_results.{}.activation_trace.{}",+ index, suffix+ )+ };++ if let Some(r) = validate(+ &path("inputs"),+ &layer_traces.borrow().inputs.borrow(),+ ) {+ results.push(r);+ }+ if let Some(r) = validate(+ &path("pre_mixer_norm"),+ &layer_traces.borrow().pre_attention_norm.borrow(),+ ) {+ results.push(r);+ }+ if let Some(r) = validate(+ &path("mixer"),+ &layer_traces.borrow().attention.borrow(),+ ) {+ results.push(r);+ }+ if let Some(r) = validate(+ &path("post_mixer_norm"),+ &layer_traces.borrow().post_attention_norm.borrow(),+ ) {+ results.push(r);+ }+ if let Some(r) = validate(+ &path("mlp_inputs"),+ &layer_traces.borrow().mlp_inputs.borrow(),+ ) {+ results.push(r);+ }+ if let Some(r) = validate(+ &path("pre_mlp_norm"),+ &layer_traces.borrow().pre_mlp_norm.borrow(),+ ) {+ results.push(r);+ }+ if let Some(r) =+ validate(&path("mlp"), &layer_traces.borrow().mlp.borrow())+ {+ results.push(r);+ }+ if let Some(r) = validate(+ &path("post_mlp_norm"),+ &layer_traces.borrow().post_mlp_norm.borrow(),+ ) {+ results.push(r);+ }++ let outputs_path =+ format!("activation_trace.layer_results.{}.outputs", index);+ if let Some(r) =+ validate(&outputs_path, &layer_traces.borrow().outputs.borrow())+ {+ results.push(r);+ }+ }++ // Output norm (common to all models)+ if let Some(r) = validate(+ "activation_trace.output_norm",+ &traces.borrow().output_norm.borrow(),+ ) {+ results.push(r);+ }++ // Logits (common to all models, but path may vary)+ if let Some(r) = validate(+ "activation_trace.logits",+ &traces.borrow().logits.borrow(),+ ) {+ results.push(r);+ } else if let Some(r) =+ validate("logits", &traces.borrow().logits.borrow())+ {+ results.push(r);+ }++ results+ }++ fn validate_classifier_traces(+ traces: &Rc<RefCell<ActivationTrace>>,+ traces_view: &ParameterTree<Rc<MTLContext>>,+ data_type: DataType,+ ) -> Vec<TracerValidationResult> {+ let mut results = Vec::new();++ // Embedding norm (classifier-specific)+ if let Some(embedding_norm) = &traces.borrow().embedding_norm {+ if traces_view.leaf("activation_trace.embedding_norm").is_ok() {+ results.push(TracerValidationResult {+ name: "activation_trace.embedding_norm".to_string(),+ metrics: Self::validate_array_with_name(+ data_type,+ traces_view,+ "activation_trace.embedding_norm",+ &embedding_norm.borrow(),+ ),+ });+ }+ }++ // Output pooling (classifier-specific)+ if let Some(output_pooling) = &traces.borrow().output_pooling {+ if traces_view.leaf("activation_trace.output_pooling").is_ok() {+ results.push(TracerValidationResult {+ name: "activation_trace.output_pooling".to_string(),+ metrics: Self::validate_array_with_name(+ data_type,+ traces_view,+ "activation_trace.output_pooling",+ &output_pooling.borrow(),+ ),+ });+ }+ }++ results+ }++ // ========================================================================+ // Array Validation+ // ========================================================================++ fn validate_array(+ data_type: DataType,+ expected_array: &MetalArray,+ produced_array: &Ref<MetalArray>,+ transform: Option<ArrayTransform>,+ ) -> TracerValidationMetrics {+ match data_type {+ DataType::F16 => Self::validate_array_of_type::<f16>(+ expected_array,+ produced_array,+ transform,+ ),+ DataType::BF16 => Self::validate_array_of_type::<bf16>(+ expected_array,+ produced_array,+ transform,+ ),+ DataType::F32 => Self::validate_array_of_type::<f32>(+ expected_array,+ produced_array,+ transform,+ ),+ _ => panic!("Unsupported data type: {:?}", data_type),+ }+ }++ fn validate_array_with_name(+ data_type: DataType,+ traces_view: &ParameterTree<Rc<MTLContext>>,+ expected_array_path: &str,+ produced_array: &Ref<MetalArray>,+ ) -> TracerValidationMetrics {+ let expected_array = traces_view.leaf(expected_array_path).unwrap();+ Self::validate_array(data_type, &expected_array, produced_array, None)+ }++ fn validate_array_of_type<Precision: ArrayElement>(+ expected_array: &MetalArray,+ produced_array: &Ref<MetalArray>,+ transform: Option<ArrayTransform>,+ ) -> TracerValidationMetrics {+ let expected_view = expected_array.as_view::<Precision>().unwrap();+ let produced_view = produced_array.as_view::<Precision>().unwrap();++ let (mut expected_data, mut produced_data) = match transform {+ Some(ArrayTransform::KVCacheSlice) => {+ let permuted = produced_view.permuted_axes(IxDyn(&[1, 0, 2]));+ let total_tokens = permuted.shape()[0];+ let expected_tokens = expected_view.shape()[1];+ let start = total_tokens.saturating_sub(expected_tokens);+ let sliced = permuted.slice(s![start.., .., ..]);+ let reshaped = sliced+ .into_owned()+ .to_shape(IxDyn(&[+ 1,+ expected_tokens,+ permuted.shape()[1],+ permuted.shape()[2],+ ]))+ .expect("Failed to reshape KV cache slice")+ .to_owned();+ (expected_view.to_owned(), reshaped)+ },+ Some(ArrayTransform::SsmConvState) => {+ let produced_shape = produced_view.shape();+ let history_len = produced_shape[1];+ let dim = produced_shape[0];++ let permuted = expected_view.permuted_axes(IxDyn(&[0, 2, 1]));+ let total_time = permuted.shape()[2];+ let start = total_time.saturating_sub(history_len);+ let sliced = permuted.slice(s![.., .., start..]);++ let reshaped_expected = sliced+ .into_owned()+ .to_shape(IxDyn(&[dim, history_len]))+ .expect("Failed to reshape SSM conv state slice")+ .to_owned();++ (reshaped_expected, produced_view.to_owned())+ },+ None => (expected_view.to_owned(), produced_view.to_owned()),+ };++ let expected_shape = expected_data.shape().to_vec();+ let produced_shape = produced_data.shape().to_vec();++ // Handle shape mismatches with leading dimension of 1+ if expected_shape != produced_shape {+ if expected_shape.len() == produced_shape.len() + 1+ && expected_shape.get(0) == Some(&1)+ && expected_shape[1..] == produced_shape[..]+ {+ expected_data = expected_data+ .to_shape(IxDyn(&produced_shape))+ .expect("Failed to reshape expected data")+ .to_owned();+ } else if produced_shape.len() == expected_shape.len() + 1+ && produced_shape.get(0) == Some(&1)+ && produced_shape[1..] == expected_shape[..]+ {+ produced_data = produced_data+ .to_shape(IxDyn(&expected_shape))+ .expect("Failed to reshape produced data")+ .to_owned();+ }+ }++ if expected_data.shape() != produced_data.shape() {+ panic!(+ "Shape mismatch after alignment: expected {:?}, produced {:?}",+ expected_data.shape(),+ produced_data.shape()+ );+ }++ let reference: Vec<f32> = expected_data+ .iter()+ .map(|value| NumCast::from(*value).unwrap_or(0.0))+ .collect();++ let result: Vec<f32> = produced_data+ .iter()+ .map(|value| NumCast::from(*value).unwrap_or(0.0))+ .collect();++ Self::compare_arrays(+ &reference,+ expected_data.shape().to_vec(),+ &result,+ produced_data.shape().to_vec(),+ )+ }++ fn compare_arrays(+ reference: &[f32],+ reference_shape: Vec<usize>,+ result: &[f32],+ result_shape: Vec<usize>,+ ) -> TracerValidationMetrics {+ assert_eq!(result.len(), reference.len());++ let atol: f32 = 1e-2;+ let rtol: f32 = 0.03;+ let fraction_of_allowed_violations: f32 = 0.01;++ let mut num_violations = 0;+ let mut max_err = 0.0f32;+ let mut max_err_idx = 0;+ let mut max_err_rel = 0.0f32;+ let mut max_err_reference_value = 0.0f32;+ let mut sum_sq_diff = 0.0f32;+ let mut sum_sq_result = 0.0f32;+ let mut sum_sq_reference = 0.0f32;+ let mut diff_sum = 0.0f32;+ let mut diff_max = 0.0f32;+ let mut result_nan = false;++ for (i, (&exp, &prod)) in+ reference.iter().zip(result.iter()).enumerate()+ {+ if prod.is_nan() {+ result_nan = true;+ }++ let abs_diff = (exp - prod).abs();+ let rel_diff = if exp.abs() > 1e-8 {+ abs_diff / exp.abs()+ } else {+ abs_diff+ };++ diff_sum += abs_diff;+ diff_max = diff_max.max(abs_diff);+ sum_sq_diff += abs_diff * abs_diff;+ sum_sq_result += prod * prod;+ sum_sq_reference += exp * exp;++ if abs_diff > atol && rel_diff > rtol {+ num_violations += 1;+ }++ if abs_diff > max_err {+ max_err = abs_diff;+ max_err_idx = i;+ max_err_rel = rel_diff;+ max_err_reference_value = exp;+ }+ }++ let n = reference.len() as f32;+ let rms_diff = (sum_sq_diff / n).sqrt();+ let rms_result = (sum_sq_result / n).sqrt();+ let rms_reference = (sum_sq_reference / n).sqrt();+ let rel_rms_reference = if rms_reference > 1e-8 {+ rms_diff / rms_reference+ } else {+ rms_diff+ };+ let diff_avg = diff_sum / n;++ let max_allowed_violations =+ (fraction_of_allowed_violations * n).ceil() as usize;++ TracerValidationMetrics {+ atol,+ rtol,+ fraction_of_allowed_violations,+ reference_shape,+ result_shape,+ num_violations,+ max_allowed_violations,+ max_err_idx,+ max_err,+ max_err_rel,+ max_err_reference_value,+ rms_diff,+ rms_result,+ rms_reference,+ rel_rms_reference,+ diff_max,+ diff_avg,+ result_nan,+ }+ }++ // ========================================================================+ // Utility Functions+ // ========================================================================++ fn load_array_as_vec<+ SourcePrecision: ArrayElement,+ TargetPrecision: NumCast,+ >(+ traces_view: &ParameterTree<Rc<MTLContext>>,+ name: &str,+ ) -> Vec<TargetPrecision> {+ let array = traces_view.leaf(name).unwrap();+ let slice = array.as_slice::<SourcePrecision>().unwrap();+ slice.iter().map(|x| NumCast::from(*x).unwrap()).collect()+ }++ fn determine_prefill_step_size(model_path: &Path) -> usize {+ let traces_path = model_path.join("traces.safetensors");+ if let Ok(file) = File::open(&traces_path) {+ if let Ok((_header_len, metadata)) =+ read_safetensors_metadata(&file)+ {+ if let Some(tensor) =+ metadata.tensors.get("activation_trace.token_ids")+ {+ if let Some(&length) = tensor.shape.first() {+ return tensor+ .shape+ .iter()+ .copied()+ .max()+ .unwrap_or(length)+ .max(1);+ }+ }+ }+ }+ 1+ }++ fn ensure_llm_context_capacity(+ decoding_config: &DecodingConfig,+ desired_suffix_length: usize,+ context: &mut LanguageModelGeneratorContext,+ ) {+ let resolved_prefix_length =+ decoding_config.context_length.resolve(&context.model_config);+ let current_suffix_length = std::cmp::max(+ decoding_config.prefill_step_size.resolve(&context.model_config),+ decoding_config.generate_suffix_length(),+ );++ if desired_suffix_length <= current_suffix_length {+ return;+ }++ let decoder_config = &context.decoder_config;+ context.scratch_buffers = ScratchBuffers::new(+ &context.mtl_context,+ decoder_config,+ &context.model_shape,+ resolved_prefix_length,+ desired_suffix_length,+ );++ context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(+ &context.mtl_context,+ &context.model_shape,+ resolved_prefix_length,+ desired_suffix_length,+ )));++ let intermediate_dtype: DataType =+ decoder_config.output_norm_config.scale_precision.into();+ let kernel_dtype: KernelDataType = intermediate_dtype.into();++ context.kv_cache_update = Box::new(+ KVCacheUpdate::new(+ &context.mtl_context,+ kernel_dtype,+ resolved_prefix_length,+ )+ .expect("Failed to create KV cache update kernel"),+ );++ context.gpu_sampler = Sampling::new(+ &context.mtl_context,+ kernel_dtype,+ desired_suffix_length,+ decoder_config.vocab_size,+ )+ .expect("Failed to create sampling kernel");+ }++ fn get_tokens_from_logits(logits: &MetalArray) -> Vec<u64> {+ let data_type = logits.data_type();+ match data_type {+ DataType::F16 => {+ Self::get_tokens_from_logits_of_type::<f16>(logits)+ },+ DataType::BF16 => {+ Self::get_tokens_from_logits_of_type::<bf16>(logits)+ },+ DataType::F32 => {+ Self::get_tokens_from_logits_of_type::<f32>(logits)+ },+ _ => panic!("Unsupported data type: {:?}", data_type),+ }+ }++ fn get_tokens_from_logits_of_type<Precision: ArrayElement>(+ logits: &MetalArray+ ) -> Vec<u64> {+ let sampler = ArgmaxSampler {};+ sampler.sample(logits.as_view::<Precision>().unwrap())+ }+}diff --git a/crates/uzu/src/tracer/tracer.rs b/crates/uzu/src/tracer/tracer.rsdeleted file mode 100644index e17fe6103..000000000--- a/crates/uzu/src/tracer/tracer.rs+++ /dev/null@@ -1,864 +0,0 @@-use std::{- cell::{Ref, RefCell},- fs::File,- path::{Path, PathBuf},- rc::Rc,-};--use half::{bf16, f16};-use ndarray::{IxDyn, s};-use num_traits::NumCast;--use crate::{- Array, ArrayElement, DataType,- backends::metal::{- CacheLayers, ForwardPassState, KVCacheUpdate, KernelDataType,- MTLContext, MetalArray,- forward_pass::{- ArrayId, ForwardPassBuffers,- encodable_with_state::{EncodableWithState, EncodingParameters},- traces::DecoderActivationTrace,- },- kernel::SamplingKernelEncodable,- },- generator::{- context::GeneratorContext,- sampler::{ArgmaxSampler, LogitsSampler},- },- parameters::{ParameterLoader, ParameterTree, read_safetensors_metadata},- session::{- config::{DecodingConfig, SpeculatorConfig},- parameter::{- AsyncBatchSize, ConfigResolvableValue, ContextLength, ContextMode,- PrefillStepSize, SamplingSeed,- },- },-};--enum Transform {- KVCacheSlice,- SsmConvState,-}--pub struct TracerValidationMetrics {- pub atol: f32,- pub rtol: f32,- pub fraction_of_allowed_violations: f32,-- pub reference_shape: Vec<usize>,- pub result_shape: Vec<usize>,-- pub num_violations: usize,- pub max_allowed_violations: usize,-- pub max_err_idx: usize,- pub max_err: f32,- pub max_err_rel: f32,- pub max_err_reference_value: f32,-- pub rms_diff: f32,- pub rms_result: f32,- pub rms_reference: f32,- pub rel_rms_reference: f32,-- pub diff_max: f32,- pub diff_avg: f32,- pub result_nan: bool,-}--impl TracerValidationMetrics {- pub fn is_valid(&self) -> bool {- self.num_violations <= self.max_allowed_violations && !self.result_nan- }-- pub fn message(&self) -> String {- if self.result_nan {- return format!("Result contains NaN values");- }-- let allowed_violation_explainer = if self.max_allowed_violations > 0 {- format!(" (Max {} allowed)", self.max_allowed_violations)- } else {- String::new()- };-- let reference_size: usize = self.reference_shape.iter().product();-- format!(- "{} violations > {:.1e} + {:.2}% out of total {} elements{}.\n\- Worst violation: {:.3} ({:.2}%) at index {} (reference value: {:.3}).\n\- Error RMS: {:.3}.\n\- RMS of result: {:.3}, RMS of reference: {:.3}.\n\- Relative error RMS: {:.2}% of RMS of reference.\n\- Shape: {:?}\n\- Max diff: {:.3}, Avg diff: {:.3}",- self.num_violations,- self.atol,- self.rtol * 100.0,- reference_size,- allowed_violation_explainer,- self.max_err,- self.max_err_rel * 100.0,- self.max_err_idx,- self.max_err_reference_value,- self.rms_diff,- self.rms_result,- self.rms_reference,- self.rel_rms_reference * 100.0,- self.result_shape,- self.diff_max,- self.diff_avg,- )- }-}--pub struct TracerValidationResult {- pub name: String,- pub metrics: TracerValidationMetrics,-}--pub struct TracerValidationResults {- pub suffix_length: usize,- pub results: Vec<TracerValidationResult>,- pub tokens_violation_indices: Vec<usize>,-}--impl TracerValidationResults {- pub fn number_of_tokens_violations(&self) -> usize {- self.tokens_violation_indices.len()- }-- pub fn number_of_allowed_tokens_violations(&self) -> usize {- let threshold: f64 = 0.01;- (self.suffix_length as f64 * threshold).ceil() as usize- }-- pub fn is_valid(&self) -> bool {- self.number_of_tokens_violations()- <= self.number_of_allowed_tokens_violations()- }-}--pub struct Tracer {- model_path: PathBuf,- generator_context: GeneratorContext,-}--impl Tracer {- pub fn new(model_path: &Path) -> Self {- let prefill_step_size = Self::determine_prefill_step_size(model_path);- let decoding_config = DecodingConfig::new(- ContextMode::default(),- ContextLength::default(),- PrefillStepSize::Custom(prefill_step_size),- SpeculatorConfig::default(),- SamplingSeed::default(),- AsyncBatchSize::default(),- false,- );- let mut generator_context =- GeneratorContext::new(model_path, &decoding_config).unwrap();- let desired_suffix_length =- prefill_step_size.max(decoding_config.generate_suffix_length());- Self::ensure_context_capacity(- &decoding_config,- desired_suffix_length,- &mut generator_context,- );-- Self {- model_path: model_path.to_path_buf(),- generator_context,- }- }-- pub fn new_with_config(- model_path: &Path,- sampling_seed: SamplingSeed,- max_prefix_length: ContextLength,- ) -> Self {- let prefill_step_size = Self::determine_prefill_step_size(model_path);- let decoding_config = DecodingConfig::new(- ContextMode::default(),- max_prefix_length,- PrefillStepSize::Custom(prefill_step_size),- SpeculatorConfig::default(),- sampling_seed,- AsyncBatchSize::default(),- false,- );- let mut generator_context =- GeneratorContext::new(model_path, &decoding_config).unwrap();- let desired_suffix_length =- prefill_step_size.max(decoding_config.generate_suffix_length());- Self::ensure_context_capacity(- &decoding_config,- desired_suffix_length,- &mut generator_context,- );-- Self {- model_path: model_path.to_path_buf(),- generator_context,- }- }-- fn determine_prefill_step_size(model_path: &Path) -> usize {- let traces_path = model_path.join("traces.safetensors");- if let Ok(file) = File::open(&traces_path) {- if let Ok((_header_len, metadata)) =- read_safetensors_metadata(&file)- {- if let Some(tensor) =- metadata.tensors.get("activation_trace.token_ids")- {- if let Some(&length) = tensor.shape.first() {- return tensor- .shape- .iter()- .copied()- .max()- .unwrap_or(length)- .max(1);- }- }- }- }- 1- }-- fn ensure_context_capacity(- decoding_config: &DecodingConfig,- desired_suffix_length: usize,- context: &mut GeneratorContext,- ) {- let resolved_prefix_length =- decoding_config.context_length.resolve(&context.model_config);- let current_suffix_length = std::cmp::max(- decoding_config.prefill_step_size.resolve(&context.model_config),- decoding_config.generate_suffix_length(),- );-- if desired_suffix_length <= current_suffix_length {- return;- }-- let decoder_config = &context.model_config.decoder_config;- context.scratch_buffers = ForwardPassBuffers::new(- &context.mtl_context,- decoder_config,- &context.model_shape,- resolved_prefix_length,- desired_suffix_length,- );-- context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(- &context.mtl_context,- &context.model_shape,- resolved_prefix_length,- desired_suffix_length,- )));-- let intermediate_dtype: DataType =- decoder_config.output_norm_config.scale_precision.into();- let kernel_dtype: KernelDataType = intermediate_dtype.into();-- context.kv_cache_update = Box::new(- KVCacheUpdate::new(- &context.mtl_context,- kernel_dtype,- resolved_prefix_length,- )- .expect("Failed to create KV cache update kernel"),- );-- context.gpu_sampler = SamplingKernelEncodable::new(- &context.mtl_context,- kernel_dtype,- desired_suffix_length,- decoder_config.vocab_size,- )- .expect("Failed to create sampling kernel");- }-- pub fn run(&self) -> TracerValidationResults {- let traces_path = self.model_path.join("traces.safetensors");- if !traces_path.exists() {- panic!("Traces file not found at {:?}", traces_path);- }- let traces_file =- File::open(&traces_path).expect("Failed to open traces file");- let traces_loader = ParameterLoader::new(- &traces_file,- &self.generator_context.mtl_context,- )- .expect("Failed to create ParameterLoader for traces");- let traces_view = traces_loader.tree();-- let token_ids = Self::load_array_as_vec::<i32, u64>(- &traces_view,- "activation_trace.token_ids".to_string(),- );- let token_positions = Self::load_array_as_vec::<i32, usize>(- &traces_view,- "activation_trace.token_positions".to_string(),- );-- let token_seeds: Vec<u64> = vec![0; token_ids.len()];-- let mut state = ForwardPassState::new(- self.generator_context.mtl_context.clone(),- &self.generator_context.model_config.decoder_config,- &self.generator_context.model_shape,- &self.generator_context.scratch_buffers,- self.generator_context.cache_layers.clone(),- self.generator_context.shared_buffers.clone(),- &token_ids,- &token_positions,- None,- &token_seeds,- token_ids.len(),- false,- true,- None,- false,- None,- None,- );-- let root_command_buffer = self- .generator_context- .command_buffer- .root_command_buffer()- .to_owned();- self.generator_context.executables.encode(- &mut state,- &self.generator_context.command_buffer,- &EncodingParameters::new(false, false, true),- );- self.generator_context.command_buffer.commit();- root_command_buffer.wait_until_completed();-- let traces = state.traces.clone().unwrap();- let results =- self.validate_traces(token_ids.len(), &state, &traces_view, traces);- return results;- }-}--impl Tracer {- fn validate_traces(- &self,- suffix_length: usize,- state: &ForwardPassState,- traces_view: &ParameterTree<Rc<MTLContext>>,- traces: Rc<RefCell<DecoderActivationTrace>>,- ) -> TracerValidationResults {- let mut results: Vec<TracerValidationResult> = Vec::new();-- let data_type =- self.generator_context.model_shape.activation_data_type();- let mut validate =- |expected_array_path: &str,- produced_array: &Ref<MetalArray>,- produced_transform: Option<Transform>| {- if let Ok(expected_array) =- traces_view.leaf(expected_array_path)- {- let metrics = Self::validate_array(- data_type,- &expected_array,- produced_array,- produced_transform,- );- results.push(TracerValidationResult {- name: expected_array_path.to_string(),- metrics,- });- } else {- println!(- "Warning: Trace key not found: {}",- expected_array_path- );- }- };-- for (index, layer_traces) in- traces.borrow().layer_results.iter().enumerate()- {- let path = |suffix: &str| -> String {- format!(- "activation_trace.layer_results.{}.activation_trace.{}",- index.clone(),- suffix- )- };-- validate(- path("inputs").as_str(),- &layer_traces.borrow().inputs.borrow(),- None,- );- validate(- path("pre_mixer_norm").as_str(),- &layer_traces.borrow().pre_attention_norm.borrow(),- None,- );- validate(- path("mixer").as_str(),- &layer_traces.borrow().attention.borrow(),- None,- );- if self.generator_context.executables.layers[index]- .post_attention_norm- .is_some()- {- validate(- path("post_mixer_norm").as_str(),- &layer_traces.borrow().post_attention_norm.borrow(),- None,- );- }- validate(- path("mlp_inputs").as_str(),- &layer_traces.borrow().mlp_inputs.borrow(),- None,- );- validate(- path("pre_mlp_norm").as_str(),- &layer_traces.borrow().pre_mlp_norm.borrow(),- None,- );- validate(- path("mlp").as_str(),- &layer_traces.borrow().mlp.borrow(),- None,- );- if self.generator_context.executables.layers[index]- .post_mlp_norm- .is_some()- {- validate(- path("post_mlp_norm").as_str(),- &layer_traces.borrow().post_mlp_norm.borrow(),- None,- );- }- validate(- format!(- "activation_trace.layer_results.{}.outputs",- index.clone()- )- .as_str(),- &layer_traces.borrow().outputs.borrow(),- None,- );- }-- validate(- "activation_trace.output_norm",- &traces.borrow().output_norm.borrow(),- None,- );- validate("logits", &traces.borrow().logits.borrow(), None);-- let transformer_layers: Vec<usize> = {- let cache = state.cache_layers.borrow();- cache- .data- .iter()- .enumerate()- .filter_map(|(index, layer)| {- layer.as_transformer().map(|_| index)- })- .collect()- };-- for index in transformer_layers {- let arrays =- state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);-- let keys = arrays[0].borrow();- validate(- format!("updated_kv_cache.{}.keys", index.clone()).as_str(),- &keys,- Some(Transform::KVCacheSlice),- );-- let values = arrays[1].borrow();- validate(- format!("updated_kv_cache.{}.values", index.clone()).as_str(),- &values,- Some(Transform::KVCacheSlice),- );- }-- let ssm_layers: Vec<usize> = {- let cache = state.cache_layers.borrow();- cache- .data- .iter()- .enumerate()- .filter_map(|(index, layer)| {- layer.as_state_space().map(|_| index)- })- .collect()- };-- for index in ssm_layers {- let arrays = state.arrays(&[- ArrayId::SsmConvState(index),- ArrayId::SsmState(index),- ]);-- let conv_state = arrays[0].borrow();- validate(- format!("updated_state.{}.conv_state", index).as_str(),- &conv_state,- Some(Transform::SsmConvState),- );- validate(- format!(- "activation_trace.layer_results.{}.updated_state.conv_state",- index- )- .as_str(),- &conv_state,- Some(Transform::SsmConvState),- );-- let ssm_state = arrays[1].borrow();- validate(- format!("updated_state.{}.ssm_state", index).as_str(),- &ssm_state,- None,- );- validate(- format!(- "activation_trace.layer_results.{}.updated_state.ssm_state",- index- )- .as_str(),- &ssm_state,- None,- );- }-- let expected_tokens =- Self::get_tokens_from_logits(&traces_view.leaf("logits").unwrap());- let produced_tokens =- Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());- let tokens_violation_indices: Vec<usize> = expected_tokens- .iter()- .zip(produced_tokens.iter())- .enumerate()- .filter_map(|(i, (a, b))| {- if a != b {- Some(i)- } else {- None- }- })- .collect();-- return TracerValidationResults {- suffix_length,- results,- tokens_violation_indices,- };- }-- fn validate_array(- data_type: DataType,- expected_array: &MetalArray,- produced_array: &Ref<MetalArray>,- produced_transform: Option<Transform>,- ) -> TracerValidationMetrics {- match data_type {- DataType::F16 => {- return Self::validate_array_of_type::<f16>(- expected_array,- produced_array,- produced_transform,- );- },- DataType::BF16 => {- return Self::validate_array_of_type::<bf16>(- expected_array,- produced_array,- produced_transform,- );- },- DataType::F32 => {- return Self::validate_array_of_type::<f32>(- expected_array,- produced_array,- produced_transform,- );- },- _ => panic!("Unsupported data type: {:?}", data_type),- }- }-- fn validate_array_of_type<Precision: ArrayElement>(- expected_array: &MetalArray,- produced_array: &Ref<MetalArray>,- produced_transform: Option<Transform>,- ) -> TracerValidationMetrics {- let expected_view = expected_array.as_view::<Precision>().unwrap();- let produced_view = produced_array.as_view::<Precision>().unwrap();-- let (mut expected_data, mut produced_data) = match produced_transform {- Some(transform) => match transform {- Transform::KVCacheSlice => {- let permuted =- produced_view.permuted_axes(IxDyn(&[1, 0, 2]));- let total_tokens = permuted.shape()[0];- let expected_tokens = expected_view.shape()[1];- let start = total_tokens.saturating_sub(expected_tokens);- let sliced = permuted.slice(s![start.., .., ..]);- let reshaped = sliced- .into_owned()- .to_shape(IxDyn(&[- 1,- expected_tokens,- permuted.shape()[1],- permuted.shape()[2],- ]))- .expect("Failed to reshape KV cache slice")- .to_owned();- (expected_view.to_owned(), reshaped)- },- Transform::SsmConvState => {- // expected: [1, T, D]- // produced: [D, K]- let produced_shape = produced_view.shape();- let history_len = produced_shape[1];- let dim = produced_shape[0];-- // Permute expected to [1, D, T]- let permuted =- expected_view.permuted_axes(IxDyn(&[0, 2, 1]));- let total_time = permuted.shape()[2];- let start = total_time.saturating_sub(history_len);- // Slice last K elements from time axis: [1, D, K]- let sliced = permuted.slice(s![.., .., start..]);-- let reshaped_expected = sliced- .into_owned()- .to_shape(IxDyn(&[dim, history_len]))- .expect("Failed to reshape SSM conv state slice")- .to_owned();-- (reshaped_expected, produced_view.to_owned())- },- },- None => (expected_view.to_owned(), produced_view.to_owned()),- };- let expected_shape = expected_data.shape().to_vec();- let produced_shape = produced_data.shape().to_vec();-- if expected_shape != produced_shape {- if expected_shape.len() == produced_shape.len() + 1- && expected_shape.get(0) == Some(&1)- && expected_shape[1..] == produced_shape[..]- {- expected_data = expected_data- .to_shape(IxDyn(&produced_shape))- .expect("Failed to reshape expected data")- .to_owned();- } else if produced_shape.len() == expected_shape.len() + 1- && produced_shape.get(0) == Some(&1)- && produced_shape[1..] == expected_shape[..]- {- produced_data = produced_data- .to_shape(IxDyn(&expected_shape))- .expect("Failed to reshape produced data")- .to_owned();- }- }-- if expected_data.shape() != produced_data.shape() {- panic!(- "Shape mismatch after alignment: expected {:?}, produced {:?}",- expected_data.shape(),- produced_data.shape()- );- }-- let reference = expected_data- .iter()- .map(|value| {- let value_f32: f32 = NumCast::from(*value).unwrap();- return value_f32;- })- .collect::<Vec<_>>();-- let result = produced_data- .iter()- .map(|value| {- let value_f32: f32 = NumCast::from(*value).unwrap();- return value_f32;- })- .collect::<Vec<_>>();-- let atol: f32 = 1e-2;- let rtol: f32 = 0.03;- let end_to_end_fraction_of_allowed_violations: f32 = 0.01;-- return Self::compare_arrays(- &reference,- expected_data.shape().to_vec(),- &result,- produced_data.shape().to_vec(),- atol,- rtol,- end_to_end_fraction_of_allowed_violations,- );- }-- fn compare_arrays(- reference: &Vec<f32>,- reference_shape: Vec<usize>,- result: &Vec<f32>,- result_shape: Vec<usize>,- atol: f32,- rtol: f32,- fraction_of_allowed_violations: f32,- ) -> TracerValidationMetrics {- assert_eq!(result.len(), reference.len());-- let absdiff: Vec<f32> = result- .iter()- .zip(reference.iter())- .map(|(result_value, reference_value)| {- (result_value - reference_value).abs()- })- .collect();-- let allowed_diff: Vec<f32> = reference- .iter()- .map(|reference_value| atol + rtol * reference_value.abs())- .collect();-- let violations: Vec<f32> = absdiff- .iter()- .zip(allowed_diff.iter())- .map(|(absdiff_value, allowed_diff_value)| {- (absdiff_value - allowed_diff_value).max(0.0)- })- .collect();-- let num_violations =- violations.iter().filter(|&&value| value > 0.0).count();-- let max_allowed_violations = (reference.len() as f32- * fraction_of_allowed_violations)- .ceil() as usize;-- let err_rel: Vec<f32> = absdiff- .iter()- .zip(reference.iter())- .map(|(absdiff_value, reference_value)| {- absdiff_value / (reference_value.abs() + 1e-10)- })- .collect();-- let (max_err_idx, _) = violations- .iter()- .enumerate()- .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap())- .unwrap();-- let max_err = absdiff[max_err_idx];- let max_err_rel = err_rel[max_err_idx];- let max_err_reference_value = reference[max_err_idx];-- let rms_diff = {- let sum_squares: f32 = absdiff.iter().map(|x| x * x).sum();- (sum_squares / absdiff.len() as f32).sqrt()- };-- let rms_result = {- let sum_squares: f32 = result.iter().map(|x| x * x).sum();- (sum_squares / result.len() as f32).sqrt()- };-- let rms_reference = {- let sum_squares: f32 = reference.iter().map(|x| x * x).sum();- (sum_squares / reference.len() as f32).sqrt()- };-- let rel_rms_reference = rms_diff / (rms_reference + 1e-10);-- let diff_max = absdiff.iter().cloned().reduce(|a, b| a.max(b)).unwrap();- let diff_avg =- absdiff.iter().cloned().sum::<f32>() / absdiff.len() as f32;-- let result_nan = result.iter().any(|x| x.is_nan());-- return TracerValidationMetrics {- atol,- rtol,- fraction_of_allowed_violations,- reference_shape,- result_shape,- num_violations,- max_allowed_violations,- max_err_idx,- max_err,- max_err_rel,- max_err_reference_value,- rms_diff,- rms_result,- rms_reference,- rel_rms_reference,- diff_max,- diff_avg,- result_nan,- };- }-- fn load_array_as_vec<- SourcePrecision: ArrayElement,- TargetPrecision: NumCast,- >(- traces_view: &ParameterTree<Rc<MTLContext>>,- name: String,- ) -> Vec<TargetPrecision> {- let array = traces_view.leaf(name.as_str()).unwrap();- let slice = array.as_slice::<SourcePrecision>().unwrap();- let vec: Vec<TargetPrecision> = slice- .iter()- .map(|x| {- let casted_value: TargetPrecision = NumCast::from(*x).unwrap();- return casted_value;- })- .collect();- return vec;- }-- fn get_tokens_from_logits(logits: &MetalArray) -> Vec<u64> {- let data_type = logits.data_type();- match data_type {- DataType::F16 => {- return Self::get_tokens_from_logits_of_type::<f16>(logits);- },- DataType::BF16 => {- return Self::get_tokens_from_logits_of_type::<bf16>(logits);- },- DataType::F32 => {- return Self::get_tokens_from_logits_of_type::<f32>(logits);- },- _ => panic!("Unsupported data type: {:?}", data_type),- }- }-- fn get_tokens_from_logits_of_type<Precision: ArrayElement>(- logits: &MetalArray- ) -> Vec<u64> {- let sampler = ArgmaxSampler {};- let tokens = sampler.sample(logits.as_view::<Precision>().unwrap());- return tokens;- }-}diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rsindex 399420a28..5faf525b8 100644--- a/crates/uzu/src/trie.rs+++ b/crates/uzu/src/trie.rs@@ -1,12 +1,12 @@+use thiserror::Error;+use crate::{- generator::{+ language_model::{grammar::CompiledGrammar, gumbel::speculator_sample, rng::DerivableSeed,},speculators::speculator::Speculator,};-use thiserror::Error;-#[derive(Debug, Error)]pub enum TrieError {#[error("child with the same token id is already present")]diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex 818e357ee..35d85d6d1 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -379,6 +379,7 @@ fn run_single_pass_attention(num_heads,suffix_length: seq_len,head_dim,+ is_causal: false,};kernel.encode_single_pass(&compute_encoder, args)?;@@ -516,20 +517,21 @@ fn test_single_pass_attention_basic() {},};- let variant = kernel.choose_variant(seq_len, head_dim);+ let is_causal = false; // Non-causal attention for this test+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);println!("Using kernel variant: {:?}", variant);println!("Supports single-pass for head_dim={}: {}",head_dim,- kernel.supports_single_pass(head_dim)+ kernel.supports_single_pass(head_dim, is_causal));println!("Supports two-pass for head_dim={}: {}",head_dim,- kernel.supports_two_pass(head_dim)+ kernel.supports_two_pass(head_dim, is_causal));- if !kernel.supports_single_pass(head_dim) {+ if !kernel.supports_single_pass(head_dim, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -587,7 +589,7 @@ fn test_single_pass_attention_with_mask() {},};- if !kernel.supports_single_pass(head_dim) {+ if !kernel.supports_single_pass(head_dim, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -668,7 +670,7 @@ fn test_single_pass_attention_with_sinks() {},};- if !kernel.supports_single_pass(head_dim) {+ if !kernel.supports_single_pass(head_dim, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -746,7 +748,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {},};- if !kernel.supports_single_pass(head_dim) {+ if !kernel.supports_single_pass(head_dim, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -823,10 +825,11 @@ fn test_single_pass_attention_gqa() {},};- let variant = kernel.choose_variant(seq_len, head_dim);+ let is_causal = false; // Non-causal attention for this test+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);println!("Using kernel variant for GQA: {:?}", variant);- if !kernel.supports_single_pass(head_dim) {+ if !kernel.supports_single_pass(head_dim, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -922,6 +925,7 @@ fn run_two_pass_attention(num_heads,suffix_length: seq_len,head_dim,+ is_causal: false,};kernel.encode_two_pass(&compute_encoder, args)?;@@ -970,12 +974,13 @@ fn test_two_pass_attention() {let seq_len = 2048;let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();+ let is_causal = false; // Non-causal attention for this test- if !kernel.supports_two_pass(head_dim) {+ if !kernel.supports_two_pass(head_dim, is_causal) {panic!("Two-pass kernel not supported for head_dim={}", head_dim);}- let variant = kernel.choose_variant(seq_len, head_dim);+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);if !matches!(variant, AttentionKernelVariant::TwoPass) {panic!("Two-pass not selected for seq_len={}. Got {:?}",@@ -1039,12 +1044,13 @@ fn test_two_pass_attention_gqa() {let seq_len = 4096;let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();+ let is_causal = false; // Non-causal attention for this test- if !kernel.supports_two_pass(head_dim) {+ if !kernel.supports_two_pass(head_dim, is_causal) {panic!("Two-pass kernel not supported for head_dim={}", head_dim);}- let variant = kernel.choose_variant(seq_len, head_dim);+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);if !matches!(variant, AttentionKernelVariant::TwoPass) {panic!("Two-pass not selected for GQA seq_len={}. Got {:?}",@@ -1113,8 +1119,9 @@ fn perf_two_pass_attention() {let suffix_length = 1; // Only processing 1 new token (realistic inference)let head_dim = 128;let scale = 1.0 / (head_dim as f32).sqrt();+ let is_causal = false;- if !kernel.supports_two_pass(head_dim) {+ if !kernel.supports_two_pass(head_dim, is_causal) {println!("Skipping two-pass perf test: not supported for head_dim={}",head_dim@@ -1122,7 +1129,7 @@ fn perf_two_pass_attention() {return;}- let variant = kernel.choose_variant(seq_len, head_dim);+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);if !matches!(variant, AttentionKernelVariant::TwoPass) {println!("Skipping two-pass perf test: variant {:?} selected instead",@@ -1203,6 +1210,7 @@ fn perf_two_pass_attention() {num_heads,suffix_length, // Use actual suffix_length, not seq_lenhead_dim,+ is_causal: false,};kernel.encode_two_pass(&compute_encoder, args).expect("encode");diff --git a/crates/uzu/tests/context_mode_test.rs b/crates/uzu/tests/context_mode_test.rsindex 39d21ac86..91249eb41 100644--- a/crates/uzu/tests/context_mode_test.rs+++ b/crates/uzu/tests/context_mode_test.rs@@ -137,6 +137,7 @@ fn test_context_mode_static() {}#[test]+#[ignore = "Flaky test - depends on LLM output which varies even with fixed seed"]fn test_context_mode_dynamic() {let decoding_config =build_decoding_config().with_context_mode(ContextMode::Dynamic);diff --git a/crates/uzu/tests/decoder_test.rs b/crates/uzu/tests/decoder_test.rsdeleted file mode 100644index 4198a240d..000000000--- a/crates/uzu/tests/decoder_test.rs+++ /dev/null@@ -1,43 +0,0 @@-mod common;-use uzu::decoder_runner::run_decoder_with_results;--#[test]-fn decoder_runs_successfully() {- // Skip the test completely if we're not on a Metal-capable platform.- if !cfg!(any(target_os = "macos", target_os = "ios")) {- eprintln!("Skipping decoder test on non-Metal platform");- return;- }-- // Build the path to the test model- let model_path = common::get_test_model_path();-- // Run the decoder and collect the results- let result = run_decoder_with_results(model_path.to_str().unwrap());-- // Always print the results so they are visible when running with `cargo test -- --nocapture`- println!("Placement Log:\n{}", result.placement_log);- println!("Iterations: {}", result.iterations);- println!("Time per token: {}", result.time_per_token);- println!("Tokens per second: {}", result.tokens_per_second);- println!("Success: {}", result.success);- if let Some(error) = &result.error {- println!("Error: {}", error);- }-- // Ensure the decoder executed without errors- if !result.success {- panic!(- "Decoder test failed. Error: {:?}\nPlacement Log:\n{}",- result.error, result.placement_log- );- }-- // Basic sanity checks on the performance numbers- assert!(result.iterations > 0, "Expected at least one iteration.");- assert!(result.time_per_token > 0.0, "Time per token should be positive.");- assert!(- result.tokens_per_second > 0.0,- "Tokens per second should be positive."- );-}diff --git a/crates/uzu/tests/grammar_json_schema_test.rs b/crates/uzu/tests/grammar_json_schema_test.rsindex c52f38fb0..b9cc284c6 100644--- a/crates/uzu/tests/grammar_json_schema_test.rs+++ b/crates/uzu/tests/grammar_json_schema_test.rs@@ -29,6 +29,7 @@ struct Person {}#[test]+#[ignore = "Test requires model; run manually with RUST_TEST_THREADS=1"]fn test_grammar_json_schema() {let crate_version = env!("CARGO_PKG_VERSION");let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rsindex 6af41e880..7c6bfedf3 100644--- a/crates/uzu/tests/rms_norm_kernel_test.rs+++ b/crates/uzu/tests/rms_norm_kernel_test.rs@@ -383,7 +383,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(let command_buffer = command_buffer_ref.to_owned();let compute_encoder = command_buffer.new_compute_command_encoder();- kernel.encode(+ let _ = kernel.encode(&compute_encoder,RMSNormArguments {input_buffer: &input_buffer,@@ -514,7 +514,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(let command_buffer = command_buffer_ref.to_owned();let compute_encoder = command_buffer.new_compute_command_encoder();- kernel.encode(+ let _ = kernel.encode(&compute_encoder,RMSNormArguments {input_buffer: &input_buffer,@@ -853,7 +853,7 @@ fn perf_rms_norm_with_size(let command_buffer = command_buffer_ref.to_owned();let compute_encoder = command_buffer.new_compute_command_encoder();- kernel.encode(+ let _ = kernel.encode(&compute_encoder,RMSNormArguments {input_buffer: &input_buffer,@@ -1046,7 +1046,7 @@ fn qk_norm_test() {let command_buffer = mtl_context.command_queue.new_command_buffer();let compute_encoder = command_buffer.new_compute_command_encoder();- q_kernel.encode_qk_norm(+ let _ = q_kernel.encode_qk_norm(&compute_encoder,QKNormArguments {qkv_input_buffer: &qkv_buffer,@@ -1112,7 +1112,7 @@ fn qk_norm_test() {let command_buffer = mtl_context.command_queue.new_command_buffer();let compute_encoder = command_buffer.new_compute_command_encoder();- k_kernel.encode_qk_norm(+ let _ = k_kernel.encode_qk_norm(&compute_encoder,QKNormArguments {qkv_input_buffer: &qkv_buffer,diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex ae42926b2..63a4770d9 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -8,7 +8,7 @@ use uzu::{kernel::{SamplingKernel, sampling::ArgmaxStrategy},metal_extensions::command_buffer_extensions::CommandBufferTimingAccess,},- generator::gumbel::{gumbel_float, revidx},+ language_model::gumbel::{gumbel_float, revidx},session::parameter::SamplingMethod,};@@ -107,7 +107,6 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {.encode(&logits_buffer,None,- None,&output_buffer,SamplingMethod::Greedy,batch_size,@@ -247,7 +246,6 @@ fn test_topp_sampling_from_prob_exact_match(kernel.encode(&logits_buf,- None,Some(&seeds_buf),&output_buf,SamplingMethod::Stochastic {@@ -375,7 +373,6 @@ fn test_topp_sampling_statistical_large() {kernel.encode(&logits_buf,- None,Some(&seeds_buf),&output_buf,SamplingMethod::Stochastic {@@ -484,7 +481,6 @@ fn perf_topp_128k_vocab() {kernel.encode(&logits_buf,- None,Some(&seeds_buf),&output_buf,SamplingMethod::Stochastic {@@ -590,7 +586,6 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {kernel.encode(&logits_buf,- None,Some(&seeds_buf),&output_buf,SamplingMethod::Greedy,@@ -712,7 +707,6 @@ fn test_categorical_sampling() {kernel.encode(&logits_buffer,- None,Some(&seeds_buffer),&output_buffer,SamplingMethod::Stochastic {@@ -856,7 +850,6 @@ fn test_categorical_sampling_statistical() {kernel.encode(&logits_buffer,- None,Some(&seeds_buffer),&output_buffer,SamplingMethod::Stochastic {@@ -963,7 +956,6 @@ fn perf_categorical_128k_vocab() {kernel.encode(&logits_buf,- None,Some(&seeds_buf),&output_buf,SamplingMethod::Stochastic {@@ -1299,7 +1291,6 @@ fn test_gumbel_gpu_cpu_match() {.encode_gumbel(&logits_buffer,&seeds_buffer,- 0,&gumbel_logits_buffer,BATCH as u32,VOCAB as u32,diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rsindex f00e091a4..e08d7e091 100644--- a/crates/uzu/tests/ssm_update_test.rs+++ b/crates/uzu/tests/ssm_update_test.rs@@ -6,6 +6,8 @@ use uzu::backends::metal::{KernelDataType, MTLContext,kernel::ssm::{SSDUpdateArguments, SSDUpdateKernel},};++#[allow(dead_code)]fn ssm_update_ref_bf16(x: &[bf16],dt: &[bf16],@@ -65,6 +67,7 @@ fn softplus_f32(x: f32) -> f32 {}}+#[allow(dead_code)]fn ssd_update_no_z_ref_bf16(x: &[bf16],dt_raw: &[bf16],diff --git a/crates/uzu/tests/tracer_test.rs b/crates/uzu/tests/tracer_test.rsindex 8ce1a1e9b..ef3aeff20 100644--- a/crates/uzu/tests/tracer_test.rs+++ b/crates/uzu/tests/tracer_test.rs@@ -1,7 +1,9 @@+#![cfg(feature = "tracing")]+mod common;use std::path::PathBuf;-use uzu::tracer::Tracer;+use uzu::tracer::TraceValidator;fn build_model_path() -> PathBuf {common::get_test_model_path()@@ -20,7 +22,13 @@ fn test_tracer() {return;}- let tracer = Tracer::new(&model_path);+ let mut tracer = match TraceValidator::new(&model_path) {+ Ok(t) => t,+ Err(e) => {+ println!("Failed to create TraceValidator: {:?}", e);+ return;+ },+ };let colored_text = |text: &str, valid: bool| {if valid {@@ -30,7 +38,14 @@ fn test_tracer() {}};- let results = tracer.run();+ let results = match tracer.run() {+ Ok(r) => r,+ Err(e) => {+ println!("Tracer run failed: {:?}", e);+ return;+ },+ };+for result in results.results.iter() {let valid = result.metrics.is_valid();let text = colored_text(diff --git a/crates/uzu/tests/trie_test.rs b/crates/uzu/tests/trie_test.rsindex b3344d100..4d3b970ed 100644--- a/crates/uzu/tests/trie_test.rs+++ b/crates/uzu/tests/trie_test.rs@@ -1,7 +1,7 @@use std::collections::{HashMap, HashSet};use uzu::{- generator::rng::DerivableSeed,+ language_model::rng::DerivableSeed,speculators::{empty_speculator::EmptySpeculator, speculator::Speculator},trie::{TrieCreationConfig, TrieNode},};