Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

classificators & new lalamo config

eugene85e6de79bf15
diff --git a/.gitignore b/.gitignore
index 1052a077a..019a2328c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,4 @@ target/
# Models
models/
+external/
diff --git a/Cargo.lock b/Cargo.lock
index f137cf46a..09d40fe79 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -477,7 +477,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
[[package]]
name = "cli"
-version = "0.1.6"
+version = "0.1.7"
dependencies = [
"clap",
"console 0.16.0",
@@ -920,7 +920,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2154,9 +2154,9 @@ dependencies = [
[[package]]
name = "ndarray"
-version = "0.16.1"
+version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
+checksum = "0c7c9125e8f6f10c9da3aad044cc918cf8784fa34de857b1aa68038eb05a50a9"
dependencies = [
"matrixmultiply",
"num-complex",
@@ -4001,7 +4001,7 @@ dependencies = [
[[package]]
name = "uzu"
-version = "0.1.6"
+version = "0.1.7"
dependencies = [
"bitflags 2.9.4",
"block",
@@ -4227,7 +4227,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -4648,9 +4648,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
[[package]]
name = "xgrammar-rs"
-version = "0.1.27"
+version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac85eff00b9853aab820a5b384e277c250044302ceb5ea5204265651b1777fcf"
+checksum = "22d454c6fba6f8ae3847d170653086fcc8bfe28e6a39a44504b1c49fa812bf29"
dependencies = [
"autocxx",
"autocxx-build",
diff --git a/Cargo.toml b/Cargo.toml
index 65fa2468c..dcfab2552 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,7 +3,7 @@ members = ["crates/cli", "crates/uzu"]
resolver = "3"
[workspace.package]
-version = "0.1.6"
+version = "0.1.7"
edition = "2024"
rust-version = "1.91"
license = "MIT"
diff --git a/README.md b/README.md
index 25f258202..60a8aa283 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@ For a detailed explanation of the architecture, please refer to the [documentati
```bash
git clone https://github.com/trymirai/lalamo.git
cd lalamo
-git checkout v0.5.2
+git checkout v0.5.15
```
After that, you can retrieve the list of supported models:
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
index e22c69ed7..55f9c5d4f 100644
--- a/crates/cli/Cargo.toml
+++ b/crates/cli/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "cli"
-version = "0.1.6"
+version.workspace = true
edition = "2024"
[[bin]]
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index e1075a314..c08574262 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -1,13 +1,13 @@
[package]
name = "uzu"
-version = "0.1.6"
+version.workspace = true
edition = "2024"
build = "build.rs"
[dependencies]
-xgrammar = { package = "xgrammar-rs", version = "0.1.27", features = ["hf"] }
+xgrammar = { package = "xgrammar-rs", version = "0.1.28", features = ["hf"] }
half = { version = "2.6.0", features = ["bytemuck", "num-traits"] }
-ndarray = "0.16.1"
+ndarray = "0.17.1"
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.143"
thiserror = "2.0.16"
@@ -38,6 +38,7 @@ schemars = { version = "1.0.5", features = ["derive"] }
[features]
default = ["metal-shaders"]
metal-shaders = []
+tracing = []
[[test]]
name = "moe"
diff --git a/crates/uzu/examples/dequantize_qkv_layer0.rs b/crates/uzu/examples/dequantize_qkv_layer0.rs
deleted file mode 100644
index 79412f7eb..000000000
--- a/crates/uzu/examples/dequantize_qkv_layer0.rs
+++ /dev/null
@@ -1,252 +0,0 @@
-use std::{
- collections::HashMap,
- fs::File,
- rc::Rc,
- time::{SystemTime, UNIX_EPOCH},
-};
-
-use clap::Parser;
-use metal::{CaptureDescriptor, CaptureManager, MTLCaptureDestination};
-use mpsgraph::{
- CommandBuffer, DequantizationArguments, Device as MPSDevice,
- ExecutableExecutionDescriptor, ExecutableSerializationDescriptor, Graph,
- Optimization, OptimizationProfile, ShapedType, Tensor, TensorData,
-};
-use objc2::rc::autoreleasepool;
-use thiserror::Error;
-use uzu::{
- Array, DataType, DeviceContext,
- backends::metal::{
- MTLContext,
- compilation_parameters::{BlockDevice, make_compilation_descriptor},
- error::MTLError,
- graph::common::{GraphConstructionError, load_constant},
- },
- parameters::{ParameterLoader, ParameterLoaderError},
- utils::{NSSearchPathDirectory, root_dir},
-};
-
-#[derive(Debug, Error)]
-pub enum ExampleError {
- #[error(transparent)]
- Io(#[from] std::io::Error),
- #[error(transparent)]
- Metal(#[from] MTLError),
- #[error(transparent)]
- Loader(#[from] ParameterLoaderError),
- #[error("header loading error")]
- Header,
- #[error("capture error")]
- Capture,
- #[error(transparent)]
- Graph(#[from] GraphConstructionError),
- #[error("dequantization failed")]
- Dequantization,
- #[error("result buffer view unavailable")]
- ResultView,
-}
-
-#[derive(Parser)]
-#[command(
- name = "dequantize_qkv_layer0",
- about = "Demonstrates QKV dequantization with optional GPU trace capture and MPSGraph package serialization"
-)]
-struct Args {
- /// Dump a .gputrace file capturing the GPU execution
- #[arg(long = "dump-gpu-trace")]
- dump_gpu_trace: bool,
-
- /// Serialize the compiled MPSGraph executable to a .mpsgraphpackage file
- #[arg(long = "dump-mpsgraphpackage")]
- dump_mpsgraphpackage: bool,
-
- /// Perform and print ANE/GPU placement analysis during compilation
- #[arg(long = "print-placement-analysis")]
- print_placement_analysis: bool,
-
- /// Print the executable's IR / debug info after compilation
- #[arg(long = "print-executable-dump")]
- print_executable_dump: bool,
-
- /// Dump the result array to a text file in ~/Downloads with a timestamped name
- #[arg(long = "dump-result-array")]
- dump_result_array: bool,
-}
-
-fn main() -> Result<(), ExampleError> {
- autoreleasepool(|_| {
- let args = Args::parse();
- let dump_gpu_trace = args.dump_gpu_trace;
- let dump_mpsgraphpackage = args.dump_mpsgraphpackage;
- let print_placement_analysis = args.print_placement_analysis;
- let print_executable_dump = args.print_executable_dump;
- let dump_result_array = args.dump_result_array;
-
- if dump_gpu_trace {
- unsafe {
- std::env::set_var("METAL_CAPTURE_ENABLED", "1");
- }
- }
-
- let model_dir = root_dir(NSSearchPathDirectory::Downloads)
- .join("Qwen3-4B-AWQ-Temp");
- let safetensors = File::open(model_dir.join("model.safetensors"))?;
-
- let device =
- metal::Device::system_default().ok_or(ExampleError::Capture)?;
- let command_queue = device.new_command_queue();
- let mtl_context = Rc::new(MTLContext::new(device, command_queue)?);
-
- let loader = ParameterLoader::new(&safetensors, &mtl_context)
- .map_err(|_| ExampleError::Header)?;
- let tree = loader
- .tree()
- .subtree("layers")?
- .subtree("0")?
- .subtree("attention")?
- .subtree("qkv_projection")?;
-
- let graph = Graph::new();
-
- let weights = load_constant(
- &graph,
- &tree,
- "weights",
- &[3072 * 2, 2560],
- DataType::U4,
- )
- .map_err(ExampleError::from)?;
-
- let scales =
- load_constant(&graph, &tree, "scales", &[6144, 20], DataType::F16)
- .map_err(ExampleError::from)?;
-
- let zero_points = load_constant(
- &graph,
- &tree,
- "zero_points",
- &[3072 * 2, 20],
- DataType::U4,
- )
- .map_err(ExampleError::from)?;
-
- let dequantized_weights = graph.dequantize(
- &weights,
- DequantizationArguments::ScaleTensorZeroPointTensorDataType {
- scale_tensor: &scales,
- zero_point_tensor: &zero_points,
- data_type: DataType::F16.into(),
- },
- None,
- );
-
- let mut dequantized_weights_buffer = unsafe {
- mtl_context.array_uninitialized(&[3072 * 2, 2560], DataType::F16)
- };
- let dequantized_weights_tensor_data = unsafe {
- TensorData::new_with_mtl_buffer(
- &dequantized_weights_buffer.mtl_buffer(),
- &[3072 * 2, 2560],
- DataType::F16.into(),
- None,
- )
- };
-
- let capture_manager = CaptureManager::shared();
- let capture_manager_descriptor = CaptureDescriptor::new();
- capture_manager_descriptor
- .set_destination(MTLCaptureDestination::GpuTraceDocument);
-
- let timestamp = SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .expect("Time went backwards")
- .as_secs();
- let trace_path = root_dir(NSSearchPathDirectory::Downloads)
- .join(format!("dequantize_qkv_layer0-{timestamp}.gputrace"));
-
- capture_manager_descriptor.set_output_url(trace_path);
-
- mtl_context.command_queue.set_label("uzu_command_queue");
- capture_manager_descriptor
- .set_capture_command_queue(&mtl_context.command_queue);
-
- let feeds: HashMap<&Tensor, &ShapedType> = HashMap::new();
- let optimization_level = Optimization::Level1;
- let optimization_profile = OptimizationProfile::Performance;
-
- let compilation_descriptor = make_compilation_descriptor(
- BlockDevice::Ane,
- optimization_level,
- optimization_profile,
- print_placement_analysis,
- );
-
- let executable = graph.compile(
- &MPSDevice::with_device(&mtl_context.device),
- &feeds,
- &[&dequantized_weights],
- None,
- Some(&compilation_descriptor),
- );
-
- if print_executable_dump {
- executable.dump();
- }
-
- if dump_mpsgraphpackage {
- let package_path = root_dir(NSSearchPathDirectory::Downloads).join(
- format!("dequantize_qkv_layer0-{timestamp}.mpsgraphpackage"),
- );
- let serialization_desc = ExecutableSerializationDescriptor::new();
- executable.serialize_to_graph_package(
- &package_path,
- Some(&serialization_desc),
- );
- println!("MPSGraph package saved to: {:?}", package_path);
- }
-
- if dump_gpu_trace {
- if let Err(err) =
- capture_manager.start_capture(&capture_manager_descriptor)
- {
- eprintln!("⚠️ Failed to start GPU capture: {err}");
- } else {
- println!("GPU capture started successfully");
- }
- }
-
- let command_buffer =
- CommandBuffer::from_command_queue(&mtl_context.command_queue);
- let root_command_buffer =
- command_buffer.root_command_buffer().to_owned();
-
- let execution_descriptor = ExecutableExecutionDescriptor::new();
-
- executable.encode_to_command_buffer(
- &command_buffer,
- &[],
- Some(&[&dequantized_weights_tensor_data]),
- Some(&*execution_descriptor),
- );
- command_buffer.commit_and_continue();
-
- root_command_buffer.wait_until_completed();
-
- if dump_gpu_trace {
- capture_manager.stop_capture();
- }
-
- let result = dequantized_weights_buffer
- .as_view::<half::f16>()
- .map_err(|_| ExampleError::ResultView)?;
-
- if dump_result_array {
- let result_path = root_dir(NSSearchPathDirectory::Downloads)
- .join(format!("dequantize_qkv_layer0-{timestamp}.txt"));
- std::fs::write(&result_path, format!("{:?}", result))?;
- println!("Result array dumped to {:?}", result_path);
- }
-
- Ok(())
- })
-}
diff --git a/crates/uzu/examples/quant_matmul_experiments.rs b/crates/uzu/examples/quant_matmul_experiments.rs
deleted file mode 100644
index ac7577dd5..000000000
--- a/crates/uzu/examples/quant_matmul_experiments.rs
+++ /dev/null
@@ -1,587 +0,0 @@
-use std::{
- collections::HashMap,
- process::exit,
- rc::Rc,
- time::{Instant, SystemTime, UNIX_EPOCH},
-};
-
-use clap::Parser;
-use half::f16;
-use metal::Device;
-use mpsgraph::{
- CommandBuffer, CompilationDescriptor, DataType as MPSDataType,
- DequantizationArguments, Device as MPSDevice, Executable,
- ExecutableExecutionDescriptor, ExecutableSerializationDescriptor, Graph,
- Optimization, OptimizationProfile, ShapedType, Tensor, TensorData,
-};
-use objc2::rc::{Retained, autoreleasepool};
-use thiserror::Error;
-use uzu::{
- Array, DataType, DeviceContext,
- backends::metal::{
- MTLContext, MetalArray,
- compilation_parameters::{BlockDevice, make_compilation_descriptor},
- },
- storage::{NSSearchPathDirectory, root_dir},
-};
-
-#[derive(Parser)]
-#[command(
- name = "quant_matmul_experiments",
- about = "Synthetic experiment: dequantize weights then matmul, measure execution time"
-)]
-struct Args {
- /// Print the compiled executable IR / debug dump
- #[arg(long = "print-exec-dump")]
- print_exec_dump: bool,
-
- /// Print ANE/GPU placement analysis during compilation
- #[arg(long = "print-placement-analysis")]
- print_placement_analysis: bool,
-
- /// Serialize the compiled MPSGraph executable to a .mpsgraphpackage file
- #[arg(long = "dump-mpsgraphpackage")]
- dump_mpsgraphpackage: bool,
-}
-
-#[derive(Debug, Error)]
-pub enum ExampleError {
- #[error(transparent)]
- Metal(#[from] uzu::backends::metal::error::MTLError),
- #[error("No Metal device available")]
- NoDevice,
- #[error("Graph dequantization failed")]
- Dequantization,
-}
-
-// ================== Quantized Matmul Helpers =====================
-
-#[derive(Debug, Clone, Copy)]
-enum TensorLoadType {
- Baked,
- RuntimeLoaded,
-}
-
-#[derive(Debug, Clone, Copy)]
-struct QuantizedMatmulDescriptor {
- weights: TensorLoadType,
- scales: TensorLoadType,
- zeros: TensorLoadType,
- transposed_shapes: bool,
- device: BlockDevice,
- optimization_level: Optimization,
- optimization_profile: OptimizationProfile,
-}
-
-#[derive(Debug, Clone)]
-enum TensorOption {
- Constant(Retained<Tensor>),
- Placeholder {
- placeholder: Retained<Tensor>,
- data: Retained<TensorData>,
- },
-}
-
-impl TensorOption {
- #[inline]
- fn tensor(&self) -> Retained<Tensor> {
- match self {
- Self::Constant(t) => t.clone(),
- Self::Placeholder {
- placeholder,
- ..
- } => placeholder.clone(),
- }
- }
-
- fn feeds(&self) -> HashMap<Retained<Tensor>, Retained<ShapedType>> {
- match self {
- TensorOption::Constant(_) => HashMap::new(),
- TensorOption::Placeholder {
- placeholder,
- ..
- } => HashMap::from_iter([(
- placeholder.clone(),
- placeholder.shaped_type(),
- )]),
- }
- }
-
- fn data_feeds(&self) -> HashMap<Retained<Tensor>, Retained<TensorData>> {
- match self {
- TensorOption::Constant(_) => HashMap::new(),
- TensorOption::Placeholder {
- placeholder,
- data,
- } => HashMap::from_iter([(placeholder.clone(), data.clone())]),
- }
- }
-}
-
-#[derive(Debug, Clone)]
-struct QMTensorOptions {
- input: TensorOption,
- weights: TensorOption,
- scales: TensorOption,
- zeros: TensorOption,
-}
-
-impl QMTensorOptions {
- fn feeds(&self) -> HashMap<Retained<Tensor>, Retained<ShapedType>> {
- HashMap::from_iter(
- [&self.input, &self.weights, &self.scales, &self.zeros]
- .iter()
- .flat_map(|opt| opt.feeds()),
- )
- }
-
- fn data_feeds(&self) -> HashMap<Retained<Tensor>, Retained<TensorData>> {
- HashMap::from_iter(
- [&self.input, &self.weights, &self.scales, &self.zeros]
- .iter()
- .flat_map(|opt| opt.data_feeds()),
- )
- }
-}
-
-fn build_quantized_matmul(
- graph: &Graph,
- tensor_options: &QMTensorOptions,
- mtl_context: &Rc<MTLContext>,
- compilation_descriptor: &CompilationDescriptor,
- transposed_shapes: bool,
-) -> Result<Retained<Executable>, ExampleError> {
- let i = tensor_options.input.tensor();
- let w = tensor_options.weights.tensor();
- let s = tensor_options.scales.tensor();
- let zp = tensor_options.zeros.tensor();
-
- let dequantized_weights = graph.dequantize(
- &w,
- DequantizationArguments::ScaleTensorZeroPointTensorDataType {
- scale_tensor: &s,
- zero_point_tensor: &zp,
- data_type: MPSDataType::Float16,
- },
- None,
- );
-
- let matmul = if transposed_shapes {
- graph.matrix_multiplication(
- &i,
- &graph.transpose(&dequantized_weights, &[1, 0], None),
- None,
- )
- } else {
- graph.matrix_multiplication(
- &graph.transpose(&dequantized_weights, &[1, 0], None),
- &graph.transpose(&i, &[1, 0], None),
- None,
- )
- };
-
- let device = MPSDevice::with_device(&mtl_context.device);
-
- let retained_feeds = tensor_options.feeds();
- let feeds = retained_feeds.iter().map(|(t, st)| (&**t, &**st)).collect();
-
- let executable = graph.compile(
- &device,
- &feeds,
- &[&*matmul],
- None,
- Some(compilation_descriptor),
- );
-
- Ok(executable)
-}
-
-fn make_tensor_option(
- graph: &Graph,
- load_type: TensorLoadType,
- array: &mut MetalArray,
- shape: &[usize],
- dtype: DataType,
- name: &str,
-) -> TensorOption {
- match load_type {
- TensorLoadType::Baked => {
- let tensor_const =
- graph.constant_with_data(array.buffer(), shape, dtype.into());
- TensorOption::Constant(tensor_const)
- },
- TensorLoadType::RuntimeLoaded => {
- let td = unsafe { array.to_mps_tensor_data() };
- let shape_isize: Vec<isize> =
- shape.iter().map(|d| *d as isize).collect();
- let placeholder =
- graph.placeholder(Some(&shape_isize), dtype.into(), Some(name));
- TensorOption::Placeholder {
- placeholder,
- data: td,
- }
- },
- }
-}
-
-// ================== End helpers =====================
-
-// ================== Utility to run a single descriptor =====================
-
-fn run_quant_matmul(
- qm_descriptor: &QuantizedMatmulDescriptor,
- args: &Args,
-) -> Result<f64, ExampleError> {
- // Constants that do not change between runs
- let suffix_length: usize = 128;
- let input_dim: usize = 2560;
- let output_dim: usize = 3072 * 2;
- let group_size: usize = 20;
-
- // Shapes that depend on descriptor options
- let weights_shape = if qm_descriptor.transposed_shapes {
- [output_dim, input_dim]
- } else {
- [input_dim, output_dim]
- };
- let scales_shape = if qm_descriptor.transposed_shapes {
- [output_dim, input_dim / group_size]
- } else {
- [input_dim / group_size, output_dim]
- };
- let zero_points_shape = scales_shape;
- let input_shape = [-1_i64, input_dim as i64];
-
- // Concrete shapes for actual buffer allocation at runtime
- let input_tensor_shape = [suffix_length, input_dim];
- let result_tensor_shape = [suffix_length, output_dim];
-
- // --- Device & context ---
- let device = Device::system_default().ok_or(ExampleError::NoDevice)?;
- let command_queue = device.new_command_queue();
- let mtl_context = Rc::new(MTLContext::new(device, command_queue)?);
-
- // --- Arrays ---
- let mut weights_array = unsafe {
- mtl_context.array_uninitialized(&weights_shape, DataType::U4)
- };
- weights_array.buffer_mut().fill(0x11);
-
- let mut scales_array =
- mtl_context.array_from_elem(&scales_shape, f16::from_f32(1.0));
-
- let mut zero_points_array = unsafe {
- mtl_context.array_uninitialized(&zero_points_shape, DataType::U4)
- };
- zero_points_array.buffer_mut().fill(0x00);
-
- let mut input_array =
- mtl_context.array_from_elem(&input_tensor_shape, f16::from_f32(1.0));
-
- let mut result_array = unsafe {
- mtl_context.array_uninitialized(&result_tensor_shape, DataType::F16)
- };
-
- // --- Graph ---
- let graph = Graph::new();
-
- // --- Tensor options for quantized matmul ---
- let input_tensor_data = unsafe { input_array.to_mps_tensor_data() };
- let input_shape_isize: Vec<isize> =
- input_shape.iter().map(|d| *d as isize).collect();
- let input_placeholder = graph.placeholder(
- Some(&input_shape_isize),
- DataType::F16.into(),
- Some("input_ph"),
- );
-
- let tensor_options = QMTensorOptions {
- input: TensorOption::Placeholder {
- placeholder: input_placeholder,
- data: input_tensor_data,
- },
- weights: make_tensor_option(
- &graph,
- qm_descriptor.weights,
- &mut weights_array,
- &weights_shape,
- DataType::U4,
- "weights_ph",
- ),
- scales: make_tensor_option(
- &graph,
- qm_descriptor.scales,
- &mut scales_array,
- &scales_shape,
- DataType::F16,
- "scales_ph",
- ),
- zeros: make_tensor_option(
- &graph,
- qm_descriptor.zeros,
- &mut zero_points_array,
- &zero_points_shape,
- DataType::U4,
- "zeros_ph",
- ),
- };
-
- let compilation_descriptor = make_compilation_descriptor(
- qm_descriptor.device,
- qm_descriptor.optimization_level,
- qm_descriptor.optimization_profile,
- args.print_placement_analysis,
- );
-
- let executable = build_quantized_matmul(
- &graph,
- &tensor_options,
- &mtl_context,
- &compilation_descriptor,
- qm_descriptor.transposed_shapes,
- )?;
-
- // Optional printing / dumping per flags
- if args.print_exec_dump {
- executable.dump();
- }
-
- if args.dump_mpsgraphpackage {
- let timestamp = SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .expect("Time went backwards")
- .as_secs();
- let package_path = root_dir(NSSearchPathDirectory::Downloads).join(
- format!("quant_matmul_experiments-{timestamp}.mpsgraphpackage"),
- );
- let serialization_desc = ExecutableSerializationDescriptor::new();
- executable.serialize_to_graph_package(
- &package_path,
- Some(&serialization_desc),
- );
- }
-
- // --- Run & measure ---
- let result_tensor_data = unsafe { result_array.to_mps_tensor_data() };
- let retained_data_feeds = tensor_options.data_feeds();
- let inputs: Vec<&TensorData> = match executable.feed_tensors() {
- Some(feed_tensors) => feed_tensors
- .iter()
- .map(|t| {
- retained_data_feeds
- .get(t)
- .expect("TensorData for feed tensor not found")
- })
- .map(|td| &**td)
- .collect(),
- None => Vec::new(),
- };
-
- let outputs = [&*result_tensor_data];
-
- // Warm-up
- run_once(&mtl_context, &executable, &inputs, &outputs);
-
- const NUM_ITERATIONS: usize = 1000;
- let mut total_duration = 0.0f64; // milliseconds
-
- for _ in 0..NUM_ITERATIONS {
- let start = Instant::now();
- run_once(&mtl_context, &executable, &inputs, &outputs);
- total_duration += start.elapsed().as_secs_f64() * 1e3;
- }
-
- Ok(total_duration / NUM_ITERATIONS as f64)
-}
-
-// ================== End utility =====================
-
-// cargo run --example quant_matmul_experiments -- --print-exec-dump --print-placement-analysis --dump-mpsgraphpackage
-
-fn main() -> Result<(), ExampleError> {
- autoreleasepool(|_| {
- let args = Args::parse();
-
- let descriptor = QuantizedMatmulDescriptor {
- weights: TensorLoadType::Baked,
- scales: TensorLoadType::Baked,
- zeros: TensorLoadType::RuntimeLoaded,
- transposed_shapes: true,
- device: BlockDevice::Ane,
- optimization_level: Optimization::Level0,
- optimization_profile: OptimizationProfile::PowerEfficiency,
- };
-
- let avg_ms = run_quant_matmul(&descriptor, &args)?;
-
- println!("Time: {:.3} ms", avg_ms);
-
- exit(0);
-
- // ---------------- Collect results -----------------
-
- #[derive(Debug)]
- struct ReportRow {
- weights: TensorLoadType,
- scales: TensorLoadType,
- zeros: TensorLoadType,
- transposed_shapes: bool,
- device: BlockDevice,
- optimization_level: Optimization,
- optimization_profile: OptimizationProfile,
- time_ms: f64,
- }
-
- #[allow(unreachable_code)]
- let tensor_variants =
- [TensorLoadType::Baked, TensorLoadType::RuntimeLoaded];
- let bool_variants = [true];
- let device_variants = [BlockDevice::Ane];
- let opt_levels = [Optimization::Level0, Optimization::Level1];
- let opt_profiles = [
- OptimizationProfile::Performance,
- OptimizationProfile::PowerEfficiency,
- ];
-
- let mut rows: Vec<ReportRow> = Vec::new();
-
- for &weights in &tensor_variants {
- for &scales in &tensor_variants {
- for &zeros in &tensor_variants {
- for &transposed_shapes in &bool_variants {
- for &device in &device_variants {
- for &optimization_level in &opt_levels {
- for &optimization_profile in &opt_profiles {
- let descriptor =
- QuantizedMatmulDescriptor {
- weights,
- scales,
- zeros,
- transposed_shapes,
- device,
- optimization_level,
- optimization_profile,
- };
-
- let avg_ms =
- run_quant_matmul(&descriptor, &args)?;
-
- rows.push(ReportRow {
- weights,
- scales,
- zeros,
- transposed_shapes,
- device,
- optimization_level,
- optimization_profile,
- time_ms: avg_ms,
- });
- }
- }
- }
- }
- }
- }
- }
-
- // ---------------- Sort by time -----------------
- rows.sort_by(|a, b| a.time_ms.partial_cmp(&b.time_ms).unwrap());
-
- // ---------------- Compute column widths -----------------
- let headers = [
- "Weights",
- "Scales",
- "Zero Points",
- "Transposed Shapes",
- "BlockDevice",
- "Optimization",
- "Profile",
- "Time (ms)",
- ];
-
- let mut widths: Vec<usize> = headers.iter().map(|h| h.len()).collect();
-
- for row in &rows {
- widths[0] = widths[0].max(format!("{:?}", row.weights).len());
- widths[1] = widths[1].max(format!("{:?}", row.scales).len());
- widths[2] = widths[2].max(format!("{:?}", row.zeros).len());
- widths[3] =
- widths[3].max(format!("{}", row.transposed_shapes).len());
- widths[4] = widths[4].max(format!("{:?}", row.device).len());
- widths[5] =
- widths[5].max(format!("{:?}", row.optimization_level).len());
- widths[6] =
- widths[6].max(format!("{:?}", row.optimization_profile).len());
- widths[7] = widths[7].max(format!("{:.3}", row.time_ms).len());
- }
-
- // ---------------- Print header -----------------
- println!(
- "{:<w0$} {:<w1$} {:<w2$} {:<w3$} {:<w4$} {:<w5$} {:<w6$} {:>w7$}",
- headers[0],
- headers[1],
- headers[2],
- headers[3],
- headers[4],
- headers[5],
- headers[6],
- headers[7],
- w0 = widths[0] + 1,
- w1 = widths[1] + 1,
- w2 = widths[2] + 1,
- w3 = widths[3] + 1,
- w4 = widths[4] + 1,
- w5 = widths[5] + 1,
- w6 = widths[6] + 1,
- w7 = widths[7],
- );
-
- // ---------------- Print rows -----------------
- for row in rows {
- println!(
- "{:<w0$} {:<w1$} {:<w2$} {:<w3$} {:<w4$} {:<w5$} {:<w6$} {:>w7$.3}",
- format!("{:?}", row.weights),
- format!("{:?}", row.scales),
- format!("{:?}", row.zeros),
- row.transposed_shapes,
- format!("{:?}", row.device),
- format!("{:?}", row.optimization_level),
- format!("{:?}", row.optimization_profile),
- row.time_ms,
- w0 = widths[0] + 1,
- w1 = widths[1] + 1,
- w2 = widths[2] + 1,
- w3 = widths[3] + 1,
- w4 = widths[4] + 1,
- w5 = widths[5] + 1,
- w6 = widths[6] + 1,
- w7 = widths[7],
- );
- }
-
- Ok(())
- })
-}
-
-fn run_once(
- mtl_context: &Rc<MTLContext>,
- executable: &Executable,
- inputs: &[&TensorData],
- outputs: &[&TensorData],
-) {
- let command_buffer =
- CommandBuffer::from_command_queue(&mtl_context.command_queue);
- let root_command_buffer = command_buffer.root_command_buffer().to_owned();
-
- let exec_desc = ExecutableExecutionDescriptor::new();
-
- executable.encode_to_command_buffer(
- &command_buffer,
- inputs,
- Some(outputs),
- Some(&exec_desc),
- );
-
- command_buffer.commit_and_continue();
- root_command_buffer.wait_until_completed();
-}
diff --git a/crates/uzu/examples/quick_start.rs b/crates/uzu/examples/quick_start_chat.rs
similarity index 79%
rename from crates/uzu/examples/quick_start.rs
rename to crates/uzu/examples/quick_start_chat.rs
index 681fd5a39..d9cd0a4ea 100644
--- a/crates/uzu/examples/quick_start.rs
+++ b/crates/uzu/examples/quick_start_chat.rs
@@ -1,14 +1,14 @@
-use std::path::PathBuf;
+use std::{env, path::PathBuf};
use uzu::session::{
- Session,
+ ChatSession,
config::{DecodingConfig, RunConfig},
types::{Input, Output},
};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let model_path = PathBuf::from("MODEL_PATH");
- let mut session = Session::new(model_path, DecodingConfig::default())?;
+ let mut session = ChatSession::new(model_path, DecodingConfig::default())?;
let input = Input::Text(String::from("Tell about London"));
let output = session.run(
diff --git a/crates/uzu/src/backends/metal/array.rs b/crates/uzu/src/backends/metal/array.rs
index 6b854a0b0..5ecbcea0d 100644
--- a/crates/uzu/src/backends/metal/array.rs
+++ b/crates/uzu/src/backends/metal/array.rs
@@ -7,16 +7,11 @@ use objc2::rc::Retained;
use crate::{Array, ArrayElement, DataType, array::array_size_in_bytes};
-/// Represents an n-dimensional array for Metal computation
#[derive(Debug, Clone)]
pub struct MetalArray {
- /// Metal buffer containing the array data
buffer: MTLBuffer,
- /// Shape of the array (dimensions)
shape: Box<[usize]>,
- /// Data type of the elements
data_type: DataType,
- /// Byte offset into the buffer (for indexed async buffers)
offset: usize,
}
@@ -49,7 +44,6 @@ impl Array for MetalArray {
}
impl MetalArray {
- /// Create a new Array at offset 0
pub unsafe fn new(
buffer: MTLBuffer,
shape: &[usize],
@@ -58,8 +52,6 @@ impl MetalArray {
unsafe { Self::new_with_offset(buffer, shape, data_type, 0) }
}
- /// Create a new Array at a byte offset into the buffer.
- /// Used for indexed async buffers where each pass uses a different offset.
pub unsafe fn new_with_offset(
buffer: MTLBuffer,
shape: &[usize],
@@ -84,12 +76,10 @@ impl MetalArray {
}
}
- /// Returns the byte offset into the underlying buffer
pub fn buffer_offset(&self) -> usize {
self.offset
}
- /// Wraps the underlying MTLBuffer into MPSTensorData for use with MPSGraph.
pub unsafe fn to_mps_tensor_data(&mut self) -> Retained<TensorData> {
TensorData::new_with_mtl_buffer(
&self.buffer,
@@ -99,9 +89,6 @@ impl MetalArray {
)
}
- /// Returns the underlying MTLBuffer.
- /// NOTE: For arrays created with `new_with_offset`, callers must use `buffer_offset()`
- /// and pass it to `encoder.set_buffer(..., offset)` - Metal has no sub-buffer concept.
pub unsafe fn mtl_buffer(&mut self) -> &MTLBuffer {
&self.buffer
}
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
new file mode 100644
index 000000000..adf8f17c2
--- /dev/null
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -0,0 +1,471 @@
+use std::{
+ cell::RefCell, collections::HashMap, fs::File, io::BufReader, path::Path,
+ rc::Rc,
+};
+
+use mpsgraph::{CommandBuffer as MPSCommandBuffer, Device as MPSDevice, Graph};
+use objc2::rc::{Retained, autoreleasepool};
+
+use super::{
+ KernelDataType, MTLContext, ModelShape,
+ compilation_parameters::CompilationConfig,
+ encodable_block::{
+ ClassifierLayer, ClassifierPredictionHead, Normalization, Pooling,
+ Rope,
+ transformer_layer::{embed_block, linear_block},
+ },
+ forward_pass::{
+ ArrayId, EncodableBlock, IOArrays, MPSGraphBlock, RopeType,
+ ScratchBuffers, SharedBuffers,
+ },
+ graph::{common::activation, placeholder, shaped_type},
+ kernel::{PoolingKernel, SigmoidKernel},
+};
+use crate::{
+ DataType,
+ backends::metal::error::ClassifierError,
+ config::{ClassifierModelConfig, ModelMetadata},
+ parameters::ParameterLoader,
+ session::types::Error,
+};
+
+pub struct ClassifierContext {
+ pub mtl_context: Rc<MTLContext>,
+ pub command_buffer: Retained<MPSCommandBuffer>,
+
+ pub shared_buffers: Rc<RefCell<SharedBuffers>>,
+ pub scratch_buffers: ScratchBuffers,
+
+ pub model_config: ClassifierModelConfig,
+ pub model_shape: ModelShape,
+
+ pub embed: Box<dyn EncodableBlock>,
+ pub embedding_norm: Normalization,
+ pub layers: Box<[ClassifierLayer]>,
+ pub output_norm: Normalization,
+ pub global_rope: Rc<Box<dyn EncodableBlock>>,
+ pub local_rope: Option<Rc<Box<dyn EncodableBlock>>>,
+
+ pub pooling: Box<dyn EncodableBlock>,
+ pub prediction_head: Box<dyn EncodableBlock>,
+
+ pub sigmoid_kernel: SigmoidKernel,
+}
+
+impl ClassifierContext {
+ pub fn new(model_path: &Path) -> Result<Self, Error> {
+ let mtl_device = metal::Device::system_default()
+ .ok_or(Error::UnableToCreateMetalContext)?;
+ let mtl_command_queue =
+ mtl_device.new_command_queue_with_max_command_buffer_count(1024);
+
+ let command_buffer =
+ MPSCommandBuffer::from_command_queue(&mtl_command_queue);
+
+ let config_path = model_path.join("config.json");
+ if !config_path.exists() {
+ return Err(Error::UnableToLoadConfig);
+ }
+ let config_file =
+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
+ let model_metadata: ModelMetadata =
+ serde_json::from_reader(BufReader::new(config_file))
+ .map_err(|_| Error::UnableToLoadConfig)?;
+
+ let classifier_model_config = model_metadata
+ .model_config
+ .as_classifier()
+ .ok_or(Error::UnableToLoadConfig)?;
+
+ let decoder_config = Rc::new(
+ classifier_model_config
+ .model_config
+ .to_decoder_config()
+ .map_err(|_| Error::UnableToLoadConfig)?,
+ );
+ let model_shape = ModelShape::from_decoder_config(&decoder_config);
+
+ let mtl_context = Rc::new(
+ MTLContext::new(mtl_device, mtl_command_queue)
+ .map_err(|_| Error::UnableToCreateMetalContext)?,
+ );
+
+ let compilation_config = Rc::new(CompilationConfig::default());
+ let weights_path = model_path.join("model.safetensors");
+ if !weights_path.exists() {
+ return Err(Error::UnableToLoadWeights);
+ }
+ let weights_file = File::open(&weights_path)
+ .map_err(|_| Error::UnableToLoadWeights)?;
+ let loader = ParameterLoader::new(&weights_file, &mtl_context)
+ .map_err(|_| Error::UnableToLoadWeights)?;
+ let root_loader_view = loader.tree();
+
+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(
+ &mtl_context,
+ &decoder_config,
+ &model_shape,
+ )));
+ let transformer_tree =
+ root_loader_view.subtree("transformer").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(
+ "transformer".to_string(),
+ ))
+ })?;
+
+ {
+ let mut shared_bufs = shared_buffers.borrow_mut();
+ shared_bufs.embeddings.update_data(&root_loader_view);
+ if let Some(global_rope) = &mut shared_bufs.global_rope {
+ global_rope.update_data(
+ &transformer_tree,
+ String::from("global_rope"),
+ );
+ }
+ if let Some(local_rope) = &mut shared_bufs.local_rope {
+ local_rope
+ .update_data(&transformer_tree, String::from("local_rope"));
+ }
+ }
+
+ let data_type = decoder_config
+ .layer_config
+ .mixer_config
+ .as_attention()
+ .ok_or(Error::Classifier(ClassifierError::NonAttentionMixer))?
+ .qkv_projection_config
+ .activation_precision()
+ .into();
+
+ let embed = embed_block(
+ &decoder_config,
+ &mtl_context,
+ &compilation_config.descriptor_general,
+ &root_loader_view,
+ );
+
+ let global_rope =
+ Self::create_rope_block(&mtl_context, data_type, RopeType::Global)
+ .map_err(Error::Classifier)?;
+ let local_rope = classifier_model_config
+ .model_config
+ .transformer_config
+ .local_rope_config
+ .as_ref()
+ .map(|_| {
+ Self::create_rope_block(
+ &mtl_context,
+ data_type,
+ RopeType::Local,
+ )
+ })
+ .transpose()
+ .map_err(Error::Classifier)?;
+
+ let layers = classifier_model_config
+ .model_config
+ .transformer_config
+ .layer_configs
+ .iter()
+ .enumerate()
+ .map(|(layer_index, layer_config)| {
+ let mut rope = global_rope.clone();
+ let attn = layer_config
+ .attention_config()
+ .ok_or(ClassifierError::NonAttentionMixer)?;
+
+ if attn.sliding_window_size.is_some() {
+ if let Some(local_rope_block) = local_rope.clone() {
+ rope = local_rope_block;
+ }
+ }
+
+ let num_heads = attn.num_heads.ok_or_else(|| {
+ ClassifierError::MissingConfigField(format!(
+ "num_heads in layer {}",
+ layer_index
+ ))
+ })?;
+ let head_dim = attn.head_dim.ok_or_else(|| {
+ ClassifierError::MissingConfigField(format!(
+ "head_dim in layer {}",
+ layer_index
+ ))
+ })?;
+ let num_groups = attn.num_groups.ok_or_else(|| {
+ ClassifierError::MissingConfigField(format!(
+ "num_groups in layer {}",
+ layer_index
+ ))
+ })?;
+
+ let layer_tree = transformer_tree
+ .subtree(&format!("layers.{}", layer_index))
+ .map_err(|_| {
+ ClassifierError::WeightSubtreeNotFound(format!(
+ "layers.{}",
+ layer_index
+ ))
+ })?;
+
+ Ok(ClassifierLayer::new(
+ &mtl_context,
+ layer_config,
+ compilation_config.clone(),
+ layer_index,
+ classifier_model_config.model_config.model_dim,
+ classifier_model_config
+ .model_config
+ .transformer_config
+ .hidden_dim,
+ num_heads,
+ head_dim,
+ num_groups,
+ attn.scale,
+ &layer_tree,
+ rope,
+ ))
+ })
+ .collect::<Result<Vec<_>, ClassifierError>>()
+ .map_err(Error::Classifier)?
+ .into_boxed_slice();
+
+ let output_norm_tree =
+ transformer_tree.subtree("output_norm").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(
+ "output_norm".to_string(),
+ ))
+ })?;
+ let output_norm = Normalization::new(
+ &mtl_context,
+ data_type,
+ classifier_model_config
+ .model_config
+ .transformer_config
+ .output_norm_config
+ .clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &output_norm_tree,
+ )
+ .map_err(|e| {
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!(
+ "output norm: {:?}",
+ e
+ )))
+ })?;
+
+ let context_length =
+ classifier_model_config.model_config.context_length;
+ let scratch_buffers = ScratchBuffers::new(
+ &mtl_context,
+ &decoder_config,
+ &model_shape,
+ context_length,
+ context_length,
+ );
+
+ let embedding_norm_tree =
+ root_loader_view.subtree("embedding_norm").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(
+ "embedding_norm".to_string(),
+ ))
+ })?;
+ let embedding_norm = Normalization::new(
+ &mtl_context,
+ data_type,
+ classifier_model_config.model_config.embedding_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &embedding_norm_tree,
+ )
+ .map_err(|e| {
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!(
+ "embedding norm: {:?}",
+ e
+ )))
+ })?;
+
+ let model_dim = classifier_model_config.model_config.model_dim;
+ let num_labels = classifier_model_config.model_config.num_labels;
+ let prediction_head_config =
+ &classifier_model_config.model_config.prediction_head_config;
+ let prediction_head_tree =
+ root_loader_view.subtree("prediction_head").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(
+ "prediction_head".to_string(),
+ ))
+ })?;
+
+ let prediction_head_data_type: DataType =
+ prediction_head_config.dense_config.activation_precision().into();
+
+ let prediction_head_dense_tree =
+ prediction_head_tree.subtree("dense").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(
+ "prediction_head.dense".to_string(),
+ ))
+ })?;
+ let prediction_head_dense = linear_block::<1>(
+ &prediction_head_config.dense_config,
+ prediction_head_config.use_dense_bias,
+ model_dim,
+ [model_dim],
+ &mtl_context,
+ &prediction_head_dense_tree,
+ ArrayId::ClassifierPooling,
+ ArrayId::ClassifierPredictionHeadDense,
+ &compilation_config.descriptor_general,
+ );
+
+ let prediction_head_activation = autoreleasepool(|_| {
+ let graph = Graph::new();
+ let input_shape = [-1, model_dim as isize];
+ let input_placeholder =
+ placeholder(&graph, &input_shape, prediction_head_data_type);
+
+ let output = activation(
+ &graph,
+ &prediction_head_config.activation,
+ &input_placeholder,
+ prediction_head_data_type,
+ );
+
+ let shaped_type_obj =
+ shaped_type(&input_shape, prediction_head_data_type);
+ let feeds =
+ HashMap::from([(&*input_placeholder, &*shaped_type_obj)]);
+
+ let executable = graph.compile(
+ &MPSDevice::with_device(&mtl_context.device),
+ &feeds,
+ &[&output],
+ None,
+ Some(&compilation_config.descriptor_general),
+ );
+
+ let arguments = IOArrays::new(
+ vec![ArrayId::ClassifierPredictionHeadDense].into_boxed_slice(),
+ vec![ArrayId::ClassifierPredictionHeadDense].into_boxed_slice(),
+ );
+
+ let execution_descriptor =
+ mpsgraph::ExecutableExecutionDescriptor::new();
+ execution_descriptor.set_enable_commit_and_continue(true);
+
+ Box::new(MPSGraphBlock::new(
+ executable,
+ execution_descriptor,
+ arguments,
+ )) as Box<dyn EncodableBlock>
+ });
+
+ let prediction_head_norm_tree =
+ prediction_head_tree.subtree("norm").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(
+ "prediction_head.norm".to_string(),
+ ))
+ })?;
+ let prediction_head_norm = Normalization::new(
+ &mtl_context,
+ prediction_head_data_type,
+ prediction_head_config.normalization_config.clone(),
+ ArrayId::ClassifierPredictionHeadDense,
+ ArrayId::ClassifierPredictionHeadNorm,
+ &prediction_head_norm_tree,
+ )
+ .map_err(|e| {
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!(
+ "prediction head norm: {:?}",
+ e
+ )))
+ })?;
+
+ let prediction_head_readout_tree =
+ prediction_head_tree.subtree("readout").map_err(|_| {
+ Error::Classifier(ClassifierError::WeightSubtreeNotFound(
+ "prediction_head.readout".to_string(),
+ ))
+ })?;
+ let prediction_head_final_linear = linear_block::<1>(
+ &prediction_head_config.readout_config,
+ true,
+ model_dim,
+ [num_labels],
+ &mtl_context,
+ &prediction_head_readout_tree,
+ ArrayId::ClassifierPredictionHeadNorm,
+ ArrayId::ClassifierPredictionHeadLogits,
+ &compilation_config.descriptor_general,
+ );
+
+ let pooling_kernel = PoolingKernel::new(&mtl_context, data_type)
+ .map_err(|e| {
+ eprintln!("Failed to create pooling kernel: {:?}", e);
+ Error::UnableToCreateMetalContext
+ })?;
+ let sigmoid_kernel = SigmoidKernel::new(&mtl_context, data_type)
+ .map_err(|e| {
+ eprintln!("Failed to create sigmoid kernel: {:?}", e);
+ Error::UnableToCreateMetalContext
+ })?;
+
+ let pooling = Box::new(Pooling::new(
+ pooling_kernel,
+ classifier_model_config.model_config.classifier_pooling.clone(),
+ model_dim,
+ ));
+
+ let prediction_head = Box::new(ClassifierPredictionHead::new(
+ prediction_head_dense,
+ prediction_head_activation,
+ Box::new(prediction_head_norm),
+ prediction_head_final_linear,
+ num_labels,
+ ));
+
+ Ok(Self {
+ mtl_context,
+ command_buffer,
+ shared_buffers,
+ scratch_buffers,
+ model_config: classifier_model_config.clone(),
+ model_shape,
+ embed,
+ embedding_norm,
+ layers,
+ output_norm,
+ global_rope,
+ local_rope,
+ pooling,
+ prediction_head,
+ sigmoid_kernel,
+ })
+ }
+
+ fn create_rope_block(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ rope_type: RopeType,
+ ) -> Result<Rc<Box<dyn EncodableBlock>>, ClassifierError> {
+ let kernel_data_type: KernelDataType = data_type.into();
+
+ let rotation: Box<dyn EncodableBlock> = Box::new(
+ Rope::new(mtl_context, kernel_data_type, rope_type).map_err(
+ |e| {
+ ClassifierError::KernelCreationFailed(format!(
+ "RoPE: {:?}",
+ e
+ ))
+ },
+ )?,
+ );
+ Ok(Rc::new(rotation))
+ }
+
+ pub fn reset_command_buffer(&mut self) {
+ self.command_buffer = MPSCommandBuffer::from_command_queue(
+ &self.mtl_context.command_queue,
+ );
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rs
new file mode 100644
index 000000000..91137f568
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs
@@ -0,0 +1,340 @@
+//! Attention kernel encodable.
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::backends::metal::{
+ KernelDataType, MTLContext,
+ forward_pass::{ArrayId, ForwardPassState, HashMapId},
+ kernel::attention::{
+ AttentionError, AttentionKernel, AttentionKernelVariant,
+ AttentionSinglePassArguments, AttentionTwoPassArguments,
+ KVCacheUpdateArguments,
+ },
+};
+
+pub struct Attention {
+ kernel: AttentionKernel,
+ layer_index: usize,
+ attention_scale: Option<f32>,
+ has_sinks: bool,
+ is_causal: bool,
+ sliding_window_size: Option<usize>,
+}
+
+impl Attention {
+ pub fn new(
+ context: &MTLContext,
+ data_type: KernelDataType,
+ layer_index: usize,
+ attention_scale: Option<f32>,
+ has_sinks: bool,
+ is_causal: bool,
+ sliding_window_size: Option<usize>,
+ ) -> Result<Self, AttentionError> {
+ let kernel = AttentionKernel::new(context, data_type)?;
+ Ok(Self {
+ kernel,
+ layer_index,
+ attention_scale,
+ has_sinks,
+ is_causal,
+ sliding_window_size,
+ })
+ }
+}
+
+impl EncodableBlock for Attention {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let (
+ suffix_length,
+ num_heads,
+ head_dim,
+ num_groups,
+ max_sequence_length,
+ ) = {
+ use crate::Array;
+ let qkv_binding = state.arrays(&[ArrayId::QKV]);
+ let qkv_array = qkv_binding[0].borrow();
+ let suffix_length = Array::shape(&*qkv_array)[0];
+
+ let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
+ let queries_array = queries_binding[0].borrow();
+ let num_heads = Array::shape(&*queries_array)[0];
+ let head_dim = Array::shape(&*queries_array)[2];
+
+ let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
+ let keys_array = keys_binding[0].borrow();
+ let num_groups = Array::shape(&*keys_array)[0];
+
+ let max_sequence_length = if let Some(_kv) = state.cache_layers() {
+ let key_cache_binding =
+ state.arrays(&[ArrayId::Keys(self.layer_index)]);
+ let key_cache_array = key_cache_binding[0].borrow();
+ Array::shape(&*key_cache_array)[1]
+ } else {
+ // For classifiers without KV cache, max_sequence_length is just suffix_length
+ suffix_length
+ };
+
+ (
+ suffix_length,
+ num_heads,
+ head_dim,
+ num_groups,
+ max_sequence_length,
+ )
+ };
+
+ let (segment_prefix_length, window_length) =
+ if let Some(cache_layers) = state.cache_layers() {
+ let cache = cache_layers.borrow();
+ let layer = cache.data[self.layer_index]
+ .as_transformer()
+ .expect("Attention kernel expects transformer layer state");
+ (
+ layer.projected_segment_prefix_length(
+ parameters.projection_step.unwrap_or(0),
+ ),
+ layer.window_length(),
+ )
+ } else {
+ // For classifiers without KV cache: no prefix, use configured sliding window
+ (0, self.sliding_window_size)
+ };
+
+ let sequence_length = segment_prefix_length + suffix_length;
+
+ let gqa_factor = num_heads / num_groups;
+ let scale =
+ self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());
+
+ let variant = self.kernel.choose_variant(
+ sequence_length,
+ head_dim,
+ self.is_causal,
+ );
+
+ let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
+ let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
+ let qkv_binding = state.arrays(&[ArrayId::QKV]);
+ let attention_bias_binding =
+ state.hashmaps(&[HashMapId::AttentionBias]);
+ let attention_output_binding =
+ state.arrays(&[ArrayId::AttentionOutput]);
+
+ let attention_bias_map = attention_bias_binding[0].clone();
+ let mask_kv_seq_stride = 1;
+ let mask_q_seq_stride = sequence_length as i32;
+ let mask_head_stride = 0;
+ let sinks_binding = if self.has_sinks {
+ Some(state.arrays(&[ArrayId::AttentionSinks(self.layer_index)]))
+ } else {
+ None
+ };
+
+ let mut rotated_keys_array = rotated_keys_binding[0].borrow_mut();
+ let rotated_keys_buffer = unsafe { rotated_keys_array.mtl_buffer() };
+
+ let mut qkv_array = qkv_binding[0].borrow_mut();
+ let qkv_buffer = unsafe { qkv_array.mtl_buffer() };
+
+ // Prepare command encoder early (used for optional value extraction)
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+
+ // Get KV cache buffers only if KV cache exists (LLM mode)
+ let has_kv_cache = state.cache_layers().is_some();
+ let (key_cache_buffer, value_cache_buffer) = if has_kv_cache {
+ let key_cache_binding =
+ state.arrays(&[ArrayId::Keys(self.layer_index)]);
+ let value_cache_binding =
+ state.arrays(&[ArrayId::Values(self.layer_index)]);
+
+ let mut key_cache_array = key_cache_binding[0].borrow_mut();
+ let key_cache_buf = unsafe { key_cache_array.mtl_buffer().clone() };
+
+ let mut value_cache_array = value_cache_binding[0].borrow_mut();
+ let value_cache_buf =
+ unsafe { value_cache_array.mtl_buffer().clone() };
+
+ (key_cache_buf, value_cache_buf)
+ } else {
+ // For classifiers, we need a values buffer with [num_groups, suffix_length, head_dim] layout.
+ // Use the KV cache update kernel to extract values from QKV into a dedicated extracted_values buffer.
+ let extracted_values_binding =
+ state.arrays(&[ArrayId::ExtractedValues]);
+ let mut extracted_values_array =
+ extracted_values_binding[0].borrow_mut();
+ let extracted_values_buf =
+ unsafe { extracted_values_array.mtl_buffer().clone() };
+
+ // Reuse the KV cache update kernel to write values into extracted_values_buf.
+ if let Err(e) = self.kernel.encode_kv_cache_update(
+ &compute_encoder,
+ KVCacheUpdateArguments {
+ rotated_keys_buffer: &rotated_keys_buffer,
+ qkv_buffer: &qkv_buffer,
+ key_cache_buffer: &rotated_keys_buffer, // keys already in desired layout; harmless overwrite
+ value_cache_buffer: &extracted_values_buf,
+ num_groups,
+ num_heads,
+ head_dim,
+ suffix_length,
+ segment_prefix_length: 0,
+ max_sequence_length,
+ },
+ ) {
+ eprintln!("Failed to prepare rotated values buffer: {:?}", e);
+ }
+
+ (rotated_keys_buffer.clone(), extracted_values_buf)
+ };
+
+ let mut queries_array = rotated_queries_binding[0].borrow_mut();
+ let queries_buffer = unsafe { queries_array.mtl_buffer() };
+
+ let mut attention_output_array =
+ attention_output_binding[0].borrow_mut();
+ let attention_output_buffer =
+ unsafe { attention_output_array.mtl_buffer() };
+
+ let attention_bias_buffer = attention_bias_map
+ .get(&window_length)
+ .map(|array| {
+ let mut array_ref = array.borrow_mut();
+ unsafe { array_ref.mtl_buffer().clone() }
+ })
+ .unwrap_or_else(|| {
+ panic!(
+ "Attention bias buffer not found for window length {:?}",
+ window_length
+ );
+ });
+
+ let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);
+ let sums_binding = state.arrays(&[ArrayId::AttentionSums]);
+ let maxs_binding = state.arrays(&[ArrayId::AttentionMaxs]);
+
+ let mut partials_array = partials_binding[0].borrow_mut();
+ let partials_buffer = unsafe { partials_array.mtl_buffer() };
+
+ let mut sums_array = sums_binding[0].borrow_mut();
+ let sums_buffer = unsafe { sums_array.mtl_buffer() };
+
+ let mut maxs_array = maxs_binding[0].borrow_mut();
+ let maxs_buffer = unsafe { maxs_array.mtl_buffer() };
+
+ let sinks_buffer = sinks_binding.as_ref().map(|binding| unsafe {
+ binding[0].borrow_mut().mtl_buffer().clone()
+ });
+
+ // Only update KV cache for LLM mode (not for classifiers)
+ if has_kv_cache {
+ if let Err(e) = self.kernel.encode_kv_cache_update(
+ &compute_encoder,
+ KVCacheUpdateArguments {
+ rotated_keys_buffer: &rotated_keys_buffer,
+ qkv_buffer: &qkv_buffer,
+ key_cache_buffer: &key_cache_buffer,
+ value_cache_buffer: &value_cache_buffer,
+ num_groups,
+ num_heads,
+ head_dim,
+ suffix_length,
+ segment_prefix_length,
+ max_sequence_length,
+ },
+ ) {
+ eprintln!("Failed to encode KV cache update: {:?}", e);
+ compute_encoder.end_encoding();
+ return;
+ }
+ }
+
+ let k_head_stride = (max_sequence_length * head_dim) as i32;
+ let k_seq_stride = head_dim as i32;
+ let v_head_stride = (max_sequence_length * head_dim) as i32;
+ let v_seq_stride = head_dim as i32;
+
+ match variant {
+ AttentionKernelVariant::SinglePass => {
+ if let Err(e) = self.kernel.encode_single_pass(
+ &compute_encoder,
+ AttentionSinglePassArguments {
+ queries_buffer: &queries_buffer,
+ keys_buffer: &key_cache_buffer,
+ values_buffer: &value_cache_buffer,
+ output_buffer: &attention_output_buffer,
+ gqa_factor: gqa_factor as i32,
+ sequence_length: sequence_length as i32,
+ k_head_stride,
+ k_seq_stride,
+ v_head_stride,
+ v_seq_stride,
+ scale,
+ mask_buffer: Some(&attention_bias_buffer),
+ mask_kv_seq_stride,
+ mask_q_seq_stride,
+ mask_head_stride,
+ sinks_buffer: sinks_buffer.as_ref(),
+ num_heads,
+ suffix_length,
+ head_dim,
+ is_causal: self.is_causal,
+ },
+ ) {
+ eprintln!(
+ "Failed to encode single-pass attention: {:?}",
+ e
+ );
+ }
+ },
+ AttentionKernelVariant::TwoPass => {
+ if let Err(e) = self.kernel.encode_two_pass(
+ &compute_encoder,
+ AttentionTwoPassArguments {
+ queries_buffer: &queries_buffer,
+ keys_buffer: &key_cache_buffer,
+ values_buffer: &value_cache_buffer,
+ partials_buffer: &partials_buffer,
+ sums_buffer: &sums_buffer,
+ maxs_buffer: &maxs_buffer,
+ output_buffer: &attention_output_buffer,
+ gqa_factor: gqa_factor as i32,
+ sequence_length: sequence_length as i32,
+ k_head_stride,
+ k_seq_stride,
+ v_head_stride,
+ v_seq_stride,
+ scale,
+ mask_buffer: Some(&attention_bias_buffer),
+ mask_kv_seq_stride,
+ mask_q_seq_stride,
+ mask_head_stride,
+ sinks_buffer: sinks_buffer.as_ref(),
+ num_heads,
+ suffix_length,
+ head_dim,
+ is_causal: self.is_causal,
+ },
+ ) {
+ eprintln!("Failed to encode two-pass attention: {:?}", e);
+ }
+ },
+ }
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
new file mode 100644
index 000000000..289e205e1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
@@ -0,0 +1,411 @@
+use std::rc::Rc;
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use objc2::rc::autoreleasepool;
+
+use super::{
+ Attention, EncodableBlock, EncodingParameters, ForwardPassState,
+ Normalization, QKNorm, TensorAddSwap, TensorCopy, transformer_layer,
+};
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLContext, compilation_parameters::CompilationConfig,
+ forward_pass::ArrayId, kernel::KernelDataType,
+ },
+ config::TransformerLayerConfig,
+ parameters::ParameterTree,
+};
+
+pub struct ClassifierLayer {
+ #[cfg_attr(not(feature = "tracing"), allow(dead_code))]
+ layer_index: usize,
+ copy_main_to_shortcut_mixer: Box<dyn EncodableBlock>,
+ pre_attention_norm: Option<Box<dyn EncodableBlock>>,
+ qkv_projection: Box<dyn EncodableBlock>,
+ qk_norm: Option<Box<dyn EncodableBlock>>,
+ rope: Rc<Box<dyn EncodableBlock>>,
+ attention: Box<dyn EncodableBlock>,
+ out_projection: Box<dyn EncodableBlock>,
+ post_attention_norm: Option<Box<dyn EncodableBlock>>,
+ mixer_residual_add: Box<dyn EncodableBlock>,
+ copy_main_to_shortcut_mlp: Box<dyn EncodableBlock>,
+ pre_mlp_norm: Box<dyn EncodableBlock>,
+ mlp: Box<dyn EncodableBlock>,
+ post_mlp_norm: Option<Box<dyn EncodableBlock>>,
+ mlp_residual_add: Box<dyn EncodableBlock>,
+}
+
+impl ClassifierLayer {
+ pub fn new(
+ mtl_context: &MTLContext,
+ layer_config: &TransformerLayerConfig,
+ compilation_config: Rc<CompilationConfig>,
+ layer_index: usize,
+ model_dim: usize,
+ hidden_dim: usize,
+ num_heads: usize,
+ head_dim: usize,
+ num_groups: usize,
+ attention_scale: Option<f32>,
+ layer_loader: &ParameterTree<Rc<MTLContext>>,
+ rope: Rc<Box<dyn EncodableBlock>>,
+ ) -> Self {
+ autoreleasepool(|_| {
+ let attention_config = layer_config
+ .mixer_config
+ .as_attention()
+ .expect("Classifier layers must use attention");
+ let intermediate_data_type: DataType = attention_config
+ .qkv_projection_config
+ .activation_precision()
+ .into();
+ let kernel_data_type: KernelDataType =
+ intermediate_data_type.into();
+
+ let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock> = Box::new(
+ TensorCopy::new(
+ mtl_context,
+ kernel_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap(),
+ );
+
+ let pre_attention_norm: Option<Box<dyn EncodableBlock>> =
+ if let Some(norm_config) =
+ &layer_config.pre_attention_norm_config
+ {
+ if layer_loader.subtree("pre_mixer_norm").is_ok() {
+ Some(Box::new(
+ Normalization::new(
+ mtl_context,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader
+ .subtree("pre_mixer_norm")
+ .unwrap(),
+ )
+ .expect(
+ "Failed to create pre-attention norm kernel",
+ ),
+ ))
+ } else {
+ None
+ }
+ } else {
+ None
+ };
+
+ let qkv_projection = transformer_layer::linear_block(
+ &attention_config.qkv_projection_config,
+ attention_config.has_qkv_biases,
+ model_dim,
+ [
+ num_heads * head_dim,
+ num_groups * head_dim,
+ num_groups * head_dim,
+ ],
+ mtl_context,
+ &layer_loader.subtree("mixer.qkv_projection").unwrap(),
+ ArrayId::Main,
+ ArrayId::QKV,
+ &compilation_config.descriptor_mlp,
+ );
+
+ let qk_norm: Option<Box<dyn EncodableBlock>> = if attention_config
+ .query_norm_config
+ .is_some()
+ || attention_config.key_norm_config.is_some()
+ {
+ match QKNorm::new(
+ mtl_context,
+ intermediate_data_type,
+ attention_config.query_norm_config.clone(),
+ attention_config.key_norm_config.clone(),
+ ArrayId::QKV,
+ &layer_loader.subtree("mixer").unwrap(),
+ num_heads,
+ num_groups,
+ head_dim,
+ ) {
+ Ok(norm) => Some(Box::new(norm)),
+ Err(e) => panic!(
+ "Failed to create QK norm kernel for layer {}: {:?}",
+ layer_index, e
+ ),
+ }
+ } else {
+ None
+ };
+
+ let out_projection = transformer_layer::linear_block(
+ &attention_config.out_projection_config,
+ attention_config.has_out_biases,
+ num_heads * head_dim,
+ [model_dim],
+ mtl_context,
+ &layer_loader.subtree("mixer.out_projection").unwrap(),
+ ArrayId::AttentionOutput,
+ ArrayId::Main,
+ &compilation_config.descriptor_mlp,
+ );
+
+ let post_attention_norm: Option<Box<dyn EncodableBlock>> =
+ if let Some(norm_config) =
+ &layer_config.post_attention_norm_config
+ {
+ Some(Box::new(
+ Normalization::new(
+ mtl_context,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("post_mixer_norm").unwrap(),
+ )
+ .expect("Failed to create post-attention norm kernel"),
+ ))
+ } else {
+ None
+ };
+
+ let mixer_residual_add: Box<dyn EncodableBlock> = Box::new(
+ TensorAddSwap::new(
+ mtl_context,
+ kernel_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap(),
+ );
+
+ let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock> = Box::new(
+ TensorCopy::new(
+ mtl_context,
+ kernel_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap(),
+ );
+
+ let pre_mlp_norm: Box<dyn EncodableBlock> = Box::new(
+ Normalization::new(
+ mtl_context,
+ intermediate_data_type,
+ layer_config.pre_mlp_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("pre_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create pre-MLP norm kernel"),
+ );
+
+ let mlp = transformer_layer::mlp_block(
+ &layer_config.mlp_config,
+ model_dim,
+ hidden_dim,
+ mtl_context,
+ &layer_loader.subtree("mlp").unwrap(),
+ &compilation_config.descriptor_mlp,
+ );
+
+ let post_mlp_norm: Option<Box<dyn EncodableBlock>> =
+ if let Some(norm_config) = &layer_config.post_mlp_norm_config {
+ Some(Box::new(
+ Normalization::new(
+ mtl_context,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("post_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create post-MLP norm kernel"),
+ ))
+ } else {
+ None
+ };
+
+ let attention: Box<dyn EncodableBlock> = Box::new(
+ Attention::new(
+ mtl_context,
+ kernel_data_type,
+ layer_index,
+ attention_scale,
+ attention_config.has_sinks,
+ false,
+ attention_config.sliding_window_size,
+ )
+ .expect("Failed to create attention kernel"),
+ );
+
+ let mlp_residual_add: Box<dyn EncodableBlock> = Box::new(
+ TensorAddSwap::new(
+ mtl_context,
+ kernel_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap(),
+ );
+
+ Self {
+ layer_index,
+ copy_main_to_shortcut_mixer,
+ pre_attention_norm,
+ qkv_projection,
+ qk_norm,
+ rope,
+ attention,
+ out_projection,
+ post_attention_norm,
+ mixer_residual_add,
+ copy_main_to_shortcut_mlp,
+ pre_mlp_norm,
+ mlp,
+ post_mlp_norm,
+ mlp_residual_add,
+ }
+ })
+ }
+}
+
+impl EncodableBlock for ClassifierLayer {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ #[cfg(feature = "tracing")]
+ let layer_traces = state
+ .traces()
+ .borrow()
+ .layer_results
+ .get(self.layer_index)
+ .cloned();
+
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().inputs.clone(),
+ );
+ }
+
+ self.copy_main_to_shortcut_mixer.encode(
+ state,
+ command_buffer,
+ parameters,
+ );
+
+ if let Some(ref pre_attn_norm) = self.pre_attention_norm {
+ pre_attn_norm.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().pre_attention_norm.clone(),
+ );
+ }
+ } else {
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().pre_attention_norm.clone(),
+ );
+ }
+ }
+
+ self.qkv_projection.encode(state, command_buffer, parameters);
+ if let Some(ref qk_norm) = self.qk_norm {
+ qk_norm.encode(state, command_buffer, parameters);
+ }
+ self.rope.encode(state, command_buffer, parameters);
+ self.attention.encode(state, command_buffer, parameters);
+ self.out_projection.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().attention.clone(),
+ );
+ }
+
+ if let Some(ref post_attn_norm) = self.post_attention_norm {
+ post_attn_norm.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().post_attention_norm.clone(),
+ );
+ }
+ }
+
+ self.mixer_residual_add.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().mlp_inputs.clone(),
+ );
+ }
+
+ self.copy_main_to_shortcut_mlp.encode(
+ state,
+ command_buffer,
+ parameters,
+ );
+
+ self.pre_mlp_norm.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().pre_mlp_norm.clone(),
+ );
+ }
+
+ self.mlp.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().mlp.clone(),
+ );
+ }
+
+ if let Some(ref post_mlp_norm) = self.post_mlp_norm {
+ post_mlp_norm.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().post_mlp_norm.clone(),
+ );
+ }
+ }
+
+ self.mlp_residual_add.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().outputs.clone(),
+ );
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
similarity index 71%
rename from crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs
rename to crates/uzu/src/backends/metal/encodable_block/decoder.rs
index 9113bfb03..0ded4ad27 100644
--- a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
@@ -1,52 +1,57 @@
+//! Decoder executables - combines embedding, layers, normalization, and readout.
+
use std::rc::Rc;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use super::layer_executables::LayerExecutables;
+use super::{
+ EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,
+};
use crate::{
- DataType,
+ DataType, DecoderConfig,
backends::metal::{
KernelDataType, MTLContext, ModelShape,
compilation_parameters::CompilationConfig,
- forward_pass::{
- ArrayId, ForwardPassState, RopeType,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- transformer_layer::{embed_block, readout_block},
- },
- kernel::{RMSNormKernelEncodable, RopeKernelEncodable},
+ encodable_block::transformer_layer::{embed_block, readout_block},
+ forward_pass::{ArrayId, EncoderResolver, ForwardPassState, RopeType},
},
- config::{DecoderConfig, DecoderLayerType, decoder_layer::MixerConfig},
+ config::{DecoderLayerType, MixerConfig},
parameters::ParameterTree,
};
-pub struct DecoderExecutables {
- pub embed: Box<dyn EncodableWithState>,
+/// Full decoder executable with all layers and components.
+pub struct Decoder {
+ pub embed: Box<dyn EncodableBlock>,
pub layers: Box<[LayerExecutables]>,
- pub norm: Box<dyn EncodableWithState>,
- pub readout: Box<dyn EncodableWithState>,
- pub global_rope: Option<Rc<Box<dyn EncodableWithState>>>,
- pub local_rope: Option<Rc<Box<dyn EncodableWithState>>>,
+ pub norm: Box<dyn EncodableBlock>,
+ pub readout: Box<dyn EncodableBlock>,
+ pub global_rope: Option<Rc<Box<dyn EncodableBlock>>>,
+ pub local_rope: Option<Rc<Box<dyn EncodableBlock>>>,
}
-impl DecoderExecutables {
+impl Decoder {
pub fn new(
mtl_context: Rc<MTLContext>,
decoder_config: Rc<DecoderConfig>,
- decoder_weight_loader: &ParameterTree<Rc<MTLContext>>,
+ root_weight_loader: &ParameterTree<Rc<MTLContext>>,
compilation_config: Rc<CompilationConfig>,
) -> Self {
+ let decoder_weight_loader = root_weight_loader
+ .subtree("transformer")
+ .expect("transformer subtree not found");
+
let embed = embed_block(
&decoder_config,
&mtl_context,
&compilation_config.descriptor_general,
- decoder_weight_loader,
+ root_weight_loader,
);
let readout = readout_block(
&decoder_config,
&mtl_context,
&compilation_config.descriptor_general,
- decoder_weight_loader,
+ root_weight_loader,
);
let attention_data_type = Self::attention_data_type(&decoder_config);
@@ -142,8 +147,8 @@ impl DecoderExecutables {
})
.collect::<Vec<_>>();
- let norm_block: Box<dyn EncodableWithState> = Box::new(
- RMSNormKernelEncodable::new(
+ let norm_block: Box<dyn EncodableBlock> = Box::new(
+ RMSNorm::new(
&mtl_context,
norm_data_type,
decoder_config.output_norm_config.clone(),
@@ -155,10 +160,10 @@ impl DecoderExecutables {
);
Self {
- embed: embed,
+ embed,
layers: layers.into_boxed_slice(),
norm: norm_block,
- readout: readout,
+ readout,
global_rope,
local_rope,
}
@@ -168,12 +173,12 @@ impl DecoderExecutables {
mtl_context: &MTLContext,
kernel_data_type: KernelDataType,
rope_type: RopeType,
- ) -> Rc<Box<dyn EncodableWithState>> {
- let rotation: Box<dyn EncodableWithState> = Box::new(
- RopeKernelEncodable::new(mtl_context, kernel_data_type, rope_type)
- .expect("Failed to create RopeKernelEncodable"),
+ ) -> Rc<Box<dyn EncodableBlock>> {
+ let rotation: Box<dyn EncodableBlock> = Box::new(
+ Rope::new(mtl_context, kernel_data_type, rope_type)
+ .expect("Failed to create Rope"),
);
- return Rc::new(rotation);
+ Rc::new(rotation)
}
fn attention_data_type(decoder_config: &DecoderConfig) -> Option<DataType> {
@@ -193,36 +198,50 @@ impl DecoderExecutables {
}
}
-impl EncodableWithState for DecoderExecutables {
+impl EncodableBlock for Decoder {
fn encode(
&self,
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
) {
- self.embed.encode(state, command_buffer, parameters);
+ let mut resolver = EncoderResolver::new(command_buffer);
+
+ resolver.encode(self.embed.as_ref(), state, parameters);
for layer in self.layers.iter() {
- layer.encode(state, command_buffer, parameters);
+ resolver.encode(layer, state, parameters);
}
if state.is_prefilling() {
+ resolver.finish();
return;
}
- self.norm.encode(state, command_buffer, parameters);
- if let Some(traces) = state.traces.clone() {
- state
- .copy_array(ArrayId::Main, traces.borrow().output_norm.clone());
+ resolver.encode(self.norm.as_ref(), state, parameters);
+ #[cfg(feature = "tracing")]
+ {
+ resolver.end_current_encoder();
+ let traces = state.traces().clone();
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ traces.borrow().output_norm.clone(),
+ );
}
- self.readout.encode(state, command_buffer, parameters);
- if let Some(traces) = state.traces.clone() {
- state.copy_array(ArrayId::Logits, traces.borrow().logits.clone());
+ resolver.encode(self.readout.as_ref(), state, parameters);
+ #[cfg(feature = "tracing")]
+ {
+ resolver.end_current_encoder();
+ let traces = state.traces().clone();
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Logits,
+ traces.borrow().logits.clone(),
+ );
}
- }
- fn supports_shared_encoder(&self) -> bool {
- false
+ resolver.finish();
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding.rs b/crates/uzu/src/backends/metal/encodable_block/embedding.rs
new file mode 100644
index 000000000..23ac3698a
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding.rs
@@ -0,0 +1,428 @@
+//! Quantized embedding encodables.
+
+use std::rc::Rc;
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::{
+ embedding::{
+ QuantizedEmbeddingLookupArguments,
+ QuantizedEmbeddingLookupKernel,
+ },
+ quant_matmul::{
+ QuantizationType, QuantizedMatmulArguments,
+ QuantizedMatmulKernel,
+ },
+ },
+ },
+ config::QuantizationMode,
+ parameters::ParameterTree,
+};
+
+#[derive(Debug, thiserror::Error)]
+pub enum QuantizedEmbeddingError {
+ #[error("Metal error: {0}")]
+ MetalError(#[from] MTLError),
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+}
+
+pub struct QuantizedEmbeddingLookup {
+ kernel: QuantizedEmbeddingLookupKernel,
+ weights_buffer: MTLBuffer,
+ scales_buffer: MTLBuffer,
+ biases_buffer: MTLBuffer,
+ vocab_size: u32,
+ model_dim: u32,
+ group_size: u32,
+}
+
+impl QuantizedEmbeddingLookup {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ let packing_divisor = mode.packing_divisor();
+
+ let kernel =
+ QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;
+
+ // Load weights [vocab_size, model_dim/packing_divisor] as storage_type
+ let mut weights = match parameter_tree.leaf("weights") {
+ Ok(weights) => weights,
+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load weights: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ if weights.data_type() != mode.storage_type() {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Expected packed weights of type {:?}, got {:?}",
+ mode.storage_type(),
+ weights.data_type()
+ )),
+ ));
+ }
+
+ // Load scales [vocab_size, num_groups]
+ let mut scales = match parameter_tree.leaf("scales") {
+ Ok(scales) => scales,
+ Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load scales: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ // Validate shapes and types
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ if weights.shape() != [vocab_size, model_dim / packing_divisor] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim / packing_divisor
+ )),
+ ));
+ }
+ if scales.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",
+ scales.shape(),
+ vocab_size,
+ num_groups
+ )),
+ ));
+ }
+ if scales.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ scales.data_type(),
+ ));
+ }
+
+ // Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")
+ let biases_buffer: MTLBuffer = match parameter_tree
+ .leaf("biases")
+ .or_else(|_| parameter_tree.leaf("output_biases"))
+ {
+ Ok(mut deq_biases) => {
+ if deq_biases.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",
+ deq_biases.shape(),
+ vocab_size,
+ num_groups
+ )),
+ ));
+ }
+ if deq_biases.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ deq_biases.data_type(),
+ ));
+ }
+ unsafe { deq_biases.mtl_buffer().to_owned() }
+ },
+ Err(_) => {
+ let elem_size: usize = match data_type {
+ DataType::F16 | DataType::BF16 => 2,
+ DataType::F32 => 4,
+ other => {
+ return Err(
+ QuantizedEmbeddingError::UnsupportedDataType(other),
+ );
+ },
+ };
+ let size_bytes = (vocab_size * num_groups * elem_size) as u64;
+ let buf = mtl_context.device.new_buffer(
+ size_bytes,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ unsafe {
+ std::ptr::write_bytes(
+ buf.contents(),
+ 0,
+ size_bytes as usize,
+ );
+ }
+ buf
+ },
+ };
+
+ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
+ let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };
+
+ Ok(Self {
+ kernel,
+ weights_buffer,
+ scales_buffer,
+ biases_buffer,
+ vocab_size: vocab_size as u32,
+ model_dim: model_dim as u32,
+ group_size: group_size as u32,
+ })
+ }
+}
+
+impl EncodableBlock for QuantizedEmbeddingLookup {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &CommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
+ let batch_size = state.active_suffix_length();
+ let mut token_ids_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let args = QuantizedEmbeddingLookupArguments {
+ token_ids_buffer,
+ weights_buffer: &self.weights_buffer,
+ scales_buffer: &self.scales_buffer,
+ biases_buffer: &self.biases_buffer,
+ output_buffer,
+ batch_size: batch_size as u32,
+ vocab_size: self.vocab_size,
+ model_dim: self.model_dim,
+ group_size: self.group_size,
+ };
+
+ self.kernel
+ .encode(encoder, args)
+ .expect("Failed to encode quantized embedding lookup kernel");
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
+pub struct QuantizedEmbeddingReadout {
+ kernel: QuantizedMatmulKernel,
+ weights_buffer: MTLBuffer,
+ scales_buffer: MTLBuffer,
+ biases_buffer: MTLBuffer,
+ vocab_size: usize,
+ model_dim: usize,
+}
+
+impl QuantizedEmbeddingReadout {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ // Load weights [vocab_size, model_dim/2] as U8
+ let mut weights = match parameter_tree.leaf("weights") {
+ Ok(weights) => weights,
+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load weights: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ // Load scales [vocab_size, num_groups]
+ let mut scales = match parameter_tree.leaf("scales") {
+ Ok(scales) => scales,
+ Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load scales: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ // Validate shapes
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ let packing_divisor = mode.packing_divisor();
+
+ // Determine if weights are transposed by checking shape
+ let weights_transposed = weights.shape()[0] == vocab_size;
+
+ if weights.shape() != [vocab_size, model_dim / packing_divisor] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim / packing_divisor
+ )),
+ ));
+ }
+ if scales.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",
+ scales.shape(),
+ vocab_size,
+ num_groups
+ )),
+ ));
+ }
+ if scales.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ scales.data_type(),
+ ));
+ }
+
+ // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]
+ let biases_buffer: MTLBuffer = match parameter_tree
+ .leaf("biases")
+ .or_else(|_| parameter_tree.leaf("output_biases"))
+ {
+ Ok(mut deq_biases) => {
+ if deq_biases.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",
+ deq_biases.shape(),
+ vocab_size,
+ num_groups
+ )),
+ ));
+ }
+ if deq_biases.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ deq_biases.data_type(),
+ ));
+ }
+ unsafe { deq_biases.mtl_buffer().to_owned() }
+ },
+ Err(_) => {
+ // Allocate zero-initialized biases buffer
+ let elem_size: usize = match data_type {
+ DataType::F16 | DataType::BF16 => 2,
+ DataType::F32 => 4,
+ other => {
+ return Err(
+ QuantizedEmbeddingError::UnsupportedDataType(other),
+ );
+ },
+ };
+ let size_bytes = (vocab_size * num_groups * elem_size) as u64;
+ let buf = mtl_context.device.new_buffer(
+ size_bytes,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ unsafe {
+ std::ptr::write_bytes(
+ buf.contents(),
+ 0,
+ size_bytes as usize,
+ );
+ }
+ buf
+ },
+ };
+
+ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
+ let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };
+
+ let kernel = QuantizedMatmulKernel::new(
+ mtl_context,
+ data_type,
+ group_size,
+ model_dim,
+ vocab_size,
+ mode,
+ QuantizationType::Mlx,
+ weights_transposed,
+ )
+ .map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to create kernel: {:?}",
+ e
+ )))
+ })?;
+
+ Ok(Self {
+ kernel,
+ weights_buffer,
+ scales_buffer,
+ biases_buffer,
+ vocab_size,
+ model_dim,
+ })
+ }
+}
+
+impl EncodableBlock for QuantizedEmbeddingReadout {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &CommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
+ let batch_size = state.active_suffix_length();
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let args = QuantizedMatmulArguments {
+ a_buffer: input_buffer,
+ b_buffer: &self.weights_buffer,
+ scales_buffer: &self.scales_buffer,
+ zero_points_or_biases_buffer: &self.biases_buffer,
+ output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.model_dim as i32,
+ output_dim: self.vocab_size as i32,
+ quantization_type: QuantizationType::Mlx,
+ };
+
+ self.kernel
+ .encode(encoder, args)
+ .expect("Failed to encode quantized embedding readout kernel");
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs b/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs
similarity index 60%
rename from crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs
rename to crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs
index 1c486dd47..fcbb32a3a 100644
--- a/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs
@@ -1,7 +1,6 @@
-use metal::{Buffer, BufferRef, ComputeCommandEncoderRef};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+//! Encoding parameters for forward pass operations.
-use super::ForwardPassState;
+use metal::{Buffer, BufferRef};
#[derive(Clone)]
pub struct EncodingParameters<'a> {
@@ -47,23 +46,3 @@ impl<'a> EncodingParameters<'a> {
self.predicate.map(|buffer| buffer.as_ref())
}
}
-
-pub trait EncodableWithState {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- );
-
- fn supports_shared_encoder(&self) -> bool;
-
- fn encode_with_shared_encoder(
- &self,
- _state: &mut ForwardPassState,
- _encoder: &ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- panic!("encode_with_shared_encoder called on unsupported type");
- }
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
new file mode 100644
index 000000000..51ea2cad3
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -0,0 +1,426 @@
+//! Layer executables - a single decoder layer with mixer, norms, and MLP.
+
+use std::rc::Rc;
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use objc2::rc::autoreleasepool;
+
+use super::{
+ super::{
+ Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm,
+ RMSNorm, TensorAddSwap, TensorCopy, transformer_layer,
+ },
+ MixerExecutables,
+};
+use crate::{
+ DataType, DecoderLayerConfig,
+ backends::metal::{
+ MTLContext,
+ compilation_parameters::CompilationConfig,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::KernelDataType,
+ },
+ config::{DecoderLayerType, MixerConfig},
+ parameters::ParameterTree,
+};
+
+/// A single decoder layer with all its components.
+pub struct LayerExecutables {
+ pub layer_index: usize,
+ pub copy_main_to_shortcut: Box<dyn EncodableBlock>,
+ pub pre_attention_norm: Box<dyn EncodableBlock>,
+ pub(crate) mixer: MixerExecutables,
+ pub post_attention_norm: Option<Box<dyn EncodableBlock>>,
+ pub main_shortcut_add_swap: Box<dyn EncodableBlock>,
+ pub pre_mlp_norm: Box<dyn EncodableBlock>,
+ pub mlp: Box<dyn EncodableBlock>,
+ pub post_mlp_norm: Option<Box<dyn EncodableBlock>>,
+}
+
+impl LayerExecutables {
+ #[allow(clippy::too_many_arguments)]
+ pub fn new(
+ mtl_context: &MTLContext,
+ layer_config: &DecoderLayerConfig,
+ layer_type: &DecoderLayerType,
+ compilation_config: Rc<CompilationConfig>,
+ layer_index: usize,
+ model_dim: usize,
+ hidden_dim: usize,
+ num_heads: usize,
+ head_dim: usize,
+ num_groups: usize,
+ attention_scale: Option<f32>,
+ decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,
+ rope: Option<Rc<Box<dyn EncodableBlock>>>,
+ ) -> Self {
+ autoreleasepool(|_| {
+ let intermediate_data_type: DataType =
+ match &layer_config.mixer_config {
+ MixerConfig::Attention(attention) => attention
+ .qkv_projection_config
+ .activation_precision()
+ .into(),
+ MixerConfig::Mamba(mamba) => {
+ mamba.in_projection_config.activation_precision().into()
+ },
+ };
+ let kernel_data_type: KernelDataType =
+ intermediate_data_type.into();
+
+ let copy_main_to_shortcut: Box<dyn EncodableBlock> = Box::new(
+ TensorCopy::new(
+ mtl_context,
+ kernel_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap(),
+ );
+
+ let pre_attention_norm: Box<dyn EncodableBlock> = Box::new(
+ RMSNorm::new(
+ mtl_context,
+ intermediate_data_type,
+ layer_config.pre_attention_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel"),
+ );
+
+ let mixer = match &layer_config.mixer_config {
+ MixerConfig::Attention(attention_config) => {
+ let rope_block = rope
+ .clone()
+ .expect("RoPE encoder missing for attention layer");
+
+ let qkv_projection = transformer_layer::linear_block(
+ &attention_config.qkv_projection_config,
+ attention_config.has_qkv_biases,
+ model_dim,
+ [
+ num_heads * head_dim,
+ num_groups * head_dim,
+ num_groups * head_dim,
+ ],
+ mtl_context,
+ &decoder_layer_loader
+ .subtree("mixer.qkv_projection")
+ .unwrap(),
+ ArrayId::Main,
+ ArrayId::QKV,
+ &compilation_config.descriptor_mlp,
+ );
+
+ let qk_norm: Option<Box<dyn EncodableBlock>> =
+ if attention_config.query_norm_config.is_some()
+ || attention_config.key_norm_config.is_some()
+ {
+ match QKNorm::new(
+ mtl_context,
+ intermediate_data_type,
+ attention_config.query_norm_config.clone(),
+ attention_config.key_norm_config.clone(),
+ ArrayId::QKV,
+ &decoder_layer_loader.subtree("mixer").unwrap(),
+ num_heads,
+ num_groups,
+ head_dim,
+ ) {
+ Ok(qk_norm) => Some(Box::new(qk_norm)
+ as Box<dyn EncodableBlock>),
+ Err(e) => panic!(
+ "Failed to create QK norm kernel for layer {}: {:?}",
+ layer_index, e
+ ),
+ }
+ } else {
+ None
+ };
+
+ let out_projection = transformer_layer::linear_block(
+ &attention_config.out_projection_config,
+ attention_config.has_out_biases,
+ num_heads * head_dim,
+ [model_dim],
+ mtl_context,
+ &decoder_layer_loader
+ .subtree("mixer.out_projection")
+ .unwrap(),
+ ArrayId::AttentionOutput,
+ ArrayId::Main,
+ &compilation_config.descriptor_mlp,
+ );
+
+ let attention = Box::new(
+ Attention::new(
+ mtl_context,
+ kernel_data_type,
+ layer_index,
+ attention_scale,
+ attention_config.has_sinks,
+ attention_config.is_causal.unwrap_or(true),
+ attention_config.sliding_window_size,
+ )
+ .expect("Failed to create AttentionWrapper with Metal kernel"),
+ );
+
+ MixerExecutables::Attention {
+ qkv_projection,
+ qk_norm,
+ rope: rope_block,
+ attention,
+ out_projection,
+ }
+ },
+ MixerConfig::Mamba(mamba_config) => {
+ let mixer: Box<dyn EncodableBlock> =
+ Box::new(MambaMixer::new(
+ mtl_context,
+ layer_type.clone(),
+ mamba_config.clone(),
+ compilation_config.clone(),
+ layer_index,
+ model_dim,
+ num_heads,
+ head_dim,
+ num_groups,
+ decoder_layer_loader,
+ ));
+ MixerExecutables::StateSpace {
+ mixer,
+ }
+ },
+ };
+
+ let post_attention_norm: Option<Box<dyn EncodableBlock>> =
+ if let Some(norm_config) =
+ &layer_config.post_attention_norm_config
+ {
+ Some(Box::new(
+ RMSNorm::new(
+ mtl_context,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader
+ .subtree("post_mixer_norm")
+ .unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel"),
+ ))
+ } else {
+ None
+ };
+
+ let main_shortcut_add_swap: Box<dyn EncodableBlock> = Box::new(
+ TensorAddSwap::new(
+ mtl_context,
+ kernel_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap(),
+ );
+
+ let pre_mlp_norm: Box<dyn EncodableBlock> = Box::new(
+ RMSNorm::new(
+ mtl_context,
+ intermediate_data_type,
+ layer_config.pre_mlp_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel"),
+ );
+
+ let mlp = transformer_layer::mlp_block(
+ &layer_config.mlp_config,
+ model_dim,
+ hidden_dim,
+ mtl_context,
+ &decoder_layer_loader.subtree("mlp").unwrap(),
+ &compilation_config.descriptor_mlp,
+ );
+
+ let post_mlp_norm: Option<Box<dyn EncodableBlock>> =
+ if let Some(norm_config) = &layer_config.post_mlp_norm_config {
+ Some(Box::new(
+ RMSNorm::new(
+ mtl_context,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader
+ .subtree("post_mlp_norm")
+ .unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel"),
+ ))
+ } else {
+ None
+ };
+
+ Self {
+ layer_index,
+ copy_main_to_shortcut,
+ pre_attention_norm,
+ mixer,
+ post_attention_norm,
+ main_shortcut_add_swap,
+ pre_mlp_norm,
+ mlp,
+ post_mlp_norm,
+ }
+ })
+ }
+}
+
+impl EncodableBlock for LayerExecutables {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ #[cfg(feature = "tracing")]
+ let layer_traces = state
+ .traces()
+ .borrow()
+ .layer_results
+ .get(self.layer_index)
+ .cloned();
+
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().inputs.clone(),
+ );
+ }
+
+ self.copy_main_to_shortcut.encode(state, command_buffer, parameters);
+ // shortcut = input
+
+ self.pre_attention_norm.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().pre_attention_norm.clone(),
+ );
+ }
+
+ match &self.mixer {
+ MixerExecutables::Attention {
+ qkv_projection,
+ qk_norm,
+ rope,
+ attention,
+ out_projection,
+ } => {
+ qkv_projection.encode(state, command_buffer, parameters);
+ if let Some(norm) = qk_norm {
+ norm.encode(state, command_buffer, parameters);
+ }
+ rope.encode(state, command_buffer, parameters);
+ attention.encode(state, command_buffer, parameters);
+ out_projection.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().attention.clone(),
+ );
+ }
+ },
+ MixerExecutables::StateSpace {
+ mixer,
+ } => {
+ mixer.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().attention.clone(),
+ );
+ }
+ },
+ }
+
+ if let Some(post_attention_norm) = &self.post_attention_norm {
+ post_attention_norm.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().post_attention_norm.clone(),
+ );
+ }
+ }
+ //main = attention_result
+
+ self.main_shortcut_add_swap.encode(state, command_buffer, parameters);
+ // shortcut = input + attention_result
+ // main = input + attention_result
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().mlp_inputs.clone(),
+ );
+ }
+
+ self.pre_mlp_norm.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().pre_mlp_norm.clone(),
+ );
+ }
+
+ self.mlp.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().mlp.clone(),
+ );
+ }
+
+ if let Some(post_mlp_norm) = &self.post_mlp_norm {
+ post_mlp_norm.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().post_mlp_norm.clone(),
+ );
+ }
+ }
+ // main = mlp_result
+
+ self.main_shortcut_add_swap.encode(state, command_buffer, parameters);
+ // shortcut = input + attention_result + mlp_result
+ // main = input + attention_result + mlp_result
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::Main,
+ layer_traces.borrow().outputs.clone(),
+ );
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
new file mode 100644
index 000000000..5a8d5e936
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
@@ -0,0 +1,19 @@
+//! Mixer component - either attention or state space.
+
+use std::rc::Rc;
+
+use super::super::EncodableBlock;
+
+/// Mixer component - either attention or state space.
+pub(crate) enum MixerExecutables {
+ Attention {
+ qkv_projection: Box<dyn EncodableBlock>,
+ qk_norm: Option<Box<dyn EncodableBlock>>,
+ rope: Rc<Box<dyn EncodableBlock>>,
+ attention: Box<dyn EncodableBlock>,
+ out_projection: Box<dyn EncodableBlock>,
+ },
+ StateSpace {
+ mixer: Box<dyn EncodableBlock>,
+ },
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs
new file mode 100644
index 000000000..15a5156fc
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs
@@ -0,0 +1,8 @@
+//! Layer executables module.
+
+mod executables;
+mod mixer;
+
+pub use executables::LayerExecutables;
+pub(crate) use mixer::MixerExecutables;
+
diff --git a/crates/uzu/src/backends/metal/kernel/linear/mod.rs b/crates/uzu/src/backends/metal/encodable_block/linear.rs
similarity index 96%
rename from crates/uzu/src/backends/metal/kernel/linear/mod.rs
rename to crates/uzu/src/backends/metal/encodable_block/linear.rs
index 676d9b998..4bb8dc2b0 100644
--- a/crates/uzu/src/backends/metal/kernel/linear/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear.rs
@@ -1,28 +1,29 @@
+//! Quantized linear (matmul) encodable.
+
use std::rc::Rc;
use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use super::{
- KernelDataType, TensorAddBias,
- quant_matmul::{
- QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel,
- },
-};
+use super::{EncodableBlock, EncodingParameters};
use crate::{
Array, DataType,
backends::metal::{
MTLContext, MTLError,
- forward_pass::{
- ArrayId, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::{
+ KernelDataType, TensorAddBias,
+ quant_matmul::{
+ QuantizationType, QuantizedMatmulArguments,
+ QuantizedMatmulKernel,
+ },
},
},
config::QuantizationConfig,
parameters::ParameterTree,
};
-pub struct QuantizedLinearKernelBlock {
+pub struct QuantizedLinear {
kernel: QuantizedMatmulKernel,
bias_add_kernel: Option<TensorAddBias>,
biases_buffer: Option<MTLBuffer>,
@@ -36,7 +37,7 @@ pub struct QuantizedLinearKernelBlock {
output_array_id: ArrayId,
}
-impl QuantizedLinearKernelBlock {
+impl QuantizedLinear {
pub fn new(
mtl_context: &MTLContext,
config: &QuantizationConfig,
@@ -222,7 +223,7 @@ impl QuantizedLinearKernelBlock {
}
}
-impl EncodableWithState for QuantizedLinearKernelBlock {
+impl EncodableBlock for QuantizedLinear {
fn encode(
&self,
state: &mut ForwardPassState,
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
similarity index 77%
rename from crates/uzu/src/backends/metal/kernel/ssm/mamba.rs
rename to crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
index a4d82b506..b6c0a95c7 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
@@ -1,35 +1,32 @@
+//! Mamba2 SSM mixer encodable.
+
use std::{env, rc::Rc};
-use metal::ComputeCommandEncoderRef;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use super::{
- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
- SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,
- SSDUpdateKernel, SplitInProjArguments, SplitInProjKernel,
- conv1d_scan::Conv1dDecodeArguments,
-};
+use super::{EncodableBlock, EncodingParameters, transformer_layer};
use crate::{
DataType,
backends::metal::{
KernelDataType, MTLContext, MetalArray,
compilation_parameters::CompilationConfig,
- forward_pass::{
- ArrayId, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- transformer_layer,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::ssm::{
+ Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
+ SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode,
+ SSDUpdateArguments, SSDUpdateKernel, SplitInProjArguments,
+ SplitInProjKernel, conv1d_scan::Conv1dDecodeArguments,
},
},
- config::{DecoderLayerType, LinearConfig, mamba::Mamba2Config},
+ config::{DecoderLayerType, Mamba2Config},
parameters::ParameterTree,
};
-pub(crate) struct MambaMixerEncodable {
+pub(crate) struct MambaMixer {
layer_index: usize,
config: Mamba2Config,
- in_projection: Box<dyn EncodableWithState>,
- out_projection: Box<dyn EncodableWithState>,
- projections_support_shared_encoder: bool,
+ in_projection: Box<dyn EncodableBlock>,
+ out_projection: Box<dyn EncodableBlock>,
split_inproj: SplitInProjKernel,
conv_scan: Conv1dScanKernel,
ssm_prefill: SSDPrefillKernel,
@@ -41,7 +38,7 @@ pub(crate) struct MambaMixerEncodable {
prefill_mode: SSDPrefillMode,
}
-impl MambaMixerEncodable {
+impl MambaMixer {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
mtl_context: &MTLContext,
@@ -69,23 +66,15 @@ impl MambaMixerEncodable {
mamba_config.in_projection_config.activation_precision().into();
let kernel_data_type: KernelDataType = data_type.into();
- let projections_are_quantized = matches!(
- &mamba_config.in_projection_config,
- LinearConfig::Quantized(_) | LinearConfig::MLXQuantized(_)
- ) && matches!(
- &mamba_config.out_projection_config,
- LinearConfig::Quantized(_) | LinearConfig::MLXQuantized(_)
- );
-
- let in_projection_output_dim = mamba_config.conv_dim()
- + mamba_config.inner_dim()
- + mamba_config.num_heads;
-
let in_projection = transformer_layer::linear_block(
&mamba_config.in_projection_config,
mamba_config.has_in_biases,
model_dim,
- [in_projection_output_dim],
+ [
+ mamba_config.conv_dim(),
+ mamba_config.inner_dim(),
+ mamba_config.num_heads,
+ ],
mtl_context,
&resolve_subtree(
decoder_layer_loader,
@@ -141,7 +130,6 @@ impl MambaMixerEncodable {
config: mamba_config,
in_projection,
out_projection,
- projections_support_shared_encoder: projections_are_quantized,
split_inproj,
conv_scan,
ssm_prefill,
@@ -154,42 +142,7 @@ impl MambaMixerEncodable {
}
}
- fn encode_pipeline_with_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
- ) {
- let suffix_length = state.aux_buffers_suffix_length();
- let active_suffix_length = state.active_suffix_length();
- if suffix_length == 0 || active_suffix_length == 0 {
- return;
- }
-
- self.in_projection
- .encode_with_shared_encoder(state, encoder, parameters);
- self.split_inproj_with_encoder(state, encoder, active_suffix_length);
- self.run_conv_scan_with_encoder(state, encoder, active_suffix_length);
-
- if suffix_length == 1 {
- self.run_decode_ssm_with_encoder(
- state,
- encoder,
- active_suffix_length,
- );
- } else {
- self.run_prefill_ssm_with_encoder(
- state,
- encoder,
- active_suffix_length,
- );
- }
-
- self.out_projection
- .encode_with_shared_encoder(state, encoder, parameters);
- }
-
- fn encode_pipeline_with_command_buffer(
+ fn encode_pipeline(
&self,
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
@@ -202,34 +155,29 @@ impl MambaMixerEncodable {
}
self.in_projection.encode(state, command_buffer, parameters);
-
- let cmd = command_buffer.root_command_buffer().to_owned();
- let encoder = cmd.new_compute_command_encoder();
- self.split_inproj_with_encoder(state, encoder, active_suffix_length);
- self.run_conv_scan_with_encoder(state, encoder, active_suffix_length);
+ self.run_split_inproj(state, command_buffer, active_suffix_length);
+ self.run_conv_scan(state, command_buffer, active_suffix_length);
if suffix_length == 1 {
- self.run_decode_ssm_with_encoder(
- state,
- encoder,
- active_suffix_length,
- );
+ self.run_decode_ssm(state, command_buffer, active_suffix_length);
} else {
- self.run_prefill_ssm_with_encoder(
- state,
- encoder,
- active_suffix_length,
- );
+ self.run_prefill_ssm(state, command_buffer, active_suffix_length);
}
- encoder.end_encoding();
self.out_projection.encode(state, command_buffer, parameters);
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
}
- fn split_inproj_with_encoder(
+ fn run_split_inproj(
&self,
state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
+ command_buffer: &MPSCommandBuffer,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -255,9 +203,12 @@ impl MambaMixerEncodable {
let num_heads = self.config.num_heads;
let total_dim = conv_dim + inner_dim + num_heads;
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute = mtl_command_buffer.new_compute_command_encoder();
self.split_inproj
.encode(
- encoder,
+ &compute,
SplitInProjArguments {
input: &input_buf,
conv_out: &conv_buf,
@@ -272,12 +223,13 @@ impl MambaMixerEncodable {
},
)
.expect("Failed to encode split in-projection kernel");
+ compute.end_encoding();
}
- fn run_conv_scan_with_encoder(
+ fn run_conv_scan(
&self,
state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
+ command_buffer: &MPSCommandBuffer,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -309,13 +261,15 @@ impl MambaMixerEncodable {
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let proj_dim = self.config.num_groups * self.config.state_dim;
+ let cmd = command_buffer.root_command_buffer().to_owned();
let state_stride = self.config.kernel_size.saturating_sub(1);
drop(conv_state);
if suffix_length == 1 {
+ let compute = cmd.new_compute_command_encoder();
self.conv_scan
.encode_decode(
- encoder,
+ &compute,
Conv1dDecodeArguments {
x: &input_buf,
w: &weight_buf,
@@ -335,6 +289,7 @@ impl MambaMixerEncodable {
},
)
.expect("Failed to encode conv decode kernel");
+ compute.end_encoding();
} else {
let padded_buf = if state_stride > 0 {
let array = state
@@ -344,20 +299,24 @@ impl MambaMixerEncodable {
let buf = unsafe { borrow.mtl_buffer().to_owned() };
drop(borrow);
- self.conv_scan
- .encode_pack(
- encoder,
- Conv1dPackArguments {
- state_in: &state_buf,
- x: &input_buf,
- padded: &buf,
- state_stride,
- row_stride: conv_dim,
- suffix_len: suffix_length,
- channels: conv_dim,
- },
- )
- .expect("Failed to encode conv pack kernel");
+ {
+ let pack_encoder = cmd.new_compute_command_encoder();
+ self.conv_scan
+ .encode_pack(
+ &pack_encoder,
+ Conv1dPackArguments {
+ state_in: &state_buf,
+ x: &input_buf,
+ padded: &buf,
+ state_stride,
+ row_stride: conv_dim,
+ suffix_len: suffix_length,
+ channels: conv_dim,
+ },
+ )
+ .expect("Failed to encode conv pack kernel");
+ pack_encoder.end_encoding();
+ }
Some(buf)
} else {
@@ -366,9 +325,10 @@ impl MambaMixerEncodable {
let conv_source = padded_buf.as_ref().unwrap_or(&input_buf);
+ let compute = cmd.new_compute_command_encoder();
self.conv_scan
.encode(
- encoder,
+ &compute,
Conv1dScanArguments {
padded: conv_source,
w: &weight_buf,
@@ -387,13 +347,14 @@ impl MambaMixerEncodable {
},
)
.expect("Failed to encode conv scan kernel");
+ compute.end_encoding();
}
}
- fn run_prefill_ssm_with_encoder(
+ fn run_prefill_ssm(
&self,
state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
+ command_buffer: &MPSCommandBuffer,
suffix_length: usize,
) {
let base_arrays = state.arrays(&[
@@ -430,9 +391,11 @@ impl MambaMixerEncodable {
let mut skip_weights = self.skip_connection_weight.clone();
let skip = unsafe { skip_weights.mtl_buffer().to_owned() };
+ let cmd = command_buffer.root_command_buffer().to_owned();
+ let compute = cmd.new_compute_command_encoder();
self.ssm_prefill
.encode(
- encoder,
+ &compute,
SSDPrefillArguments {
x: &x_buf,
dt: &dt_buf,
@@ -468,12 +431,13 @@ impl MambaMixerEncodable {
self.prefill_mode,
)
.expect("Failed to encode SSD prefill kernel");
+ compute.end_encoding();
}
- fn run_decode_ssm_with_encoder(
+ fn run_decode_ssm(
&self,
state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
+ command_buffer: &MPSCommandBuffer,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -520,9 +484,11 @@ impl MambaMixerEncodable {
let group_size = (h / g) as i32;
let state_size = n as i32;
+ let cmd = command_buffer.root_command_buffer().to_owned();
+ let compute = cmd.new_compute_command_encoder();
self.ssd_update
.encode(
- encoder,
+ &compute,
SSDUpdateArguments {
x: &x_buf,
dt: &dt_buf,
@@ -545,6 +511,7 @@ impl MambaMixerEncodable {
},
)
.expect("Failed to encode SSD decode kernel");
+ compute.end_encoding();
}
}
@@ -561,44 +528,14 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
}
}
-impl EncodableWithState for MambaMixerEncodable {
+impl EncodableBlock for MambaMixer {
fn encode(
&self,
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
) {
- if self.projections_support_shared_encoder {
- let cmd = command_buffer.root_command_buffer().to_owned();
- let encoder = cmd.new_compute_command_encoder();
- self.encode_pipeline_with_encoder(state, encoder, parameters);
- encoder.end_encoding();
- } else {
- self.encode_pipeline_with_command_buffer(
- state,
- command_buffer,
- parameters,
- );
- }
-
- if parameters.wait_until_completed {
- let cmd = command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- cmd.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- self.projections_support_shared_encoder
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
- ) {
- self.encode_pipeline_with_encoder(state, encoder, parameters);
+ self.encode_pipeline(state, command_buffer, parameters);
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
new file mode 100644
index 000000000..2773cb2f2
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
@@ -0,0 +1,68 @@
+//! MLP block encodable.
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters, QuantizedLinear};
+use crate::{
+ Array,
+ backends::metal::{
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::mlp::MlpGateActMulEncodable,
+ },
+};
+
+pub struct MlpBlock {
+ up: QuantizedLinear,
+ gate: MlpGateActMulEncodable,
+ down: QuantizedLinear,
+}
+
+impl MlpBlock {
+ pub fn new(
+ up: QuantizedLinear,
+ gate: MlpGateActMulEncodable,
+ down: QuantizedLinear,
+ ) -> Self {
+ Self {
+ up,
+ gate,
+ down,
+ }
+ }
+}
+
+impl EncodableBlock for MlpBlock {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ params: &EncodingParameters,
+ ) {
+ // Up
+ self.up.encode(state, command_buffer, params);
+ // Gate act+mul (fused_up -> hidden)
+ let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
+ let mut fused = arrays[0].borrow_mut();
+ let mut hidden = arrays[1].borrow_mut();
+ let m = fused.shape()[0] as i32;
+ let root = command_buffer.root_command_buffer();
+ let encoder = root.new_compute_command_encoder();
+ let fused_buf = unsafe { fused.mtl_buffer() };
+ let hidden_buf = unsafe { hidden.mtl_buffer() };
+ self.gate
+ .encode(encoder, fused_buf, hidden_buf, m)
+ .expect("Failed to encode MLP activation/mul kernel");
+ encoder.end_encoding();
+ drop(fused);
+ drop(hidden);
+ // Down
+ self.down.encode(state, command_buffer, params);
+
+ if params.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
new file mode 100644
index 000000000..6a370dd85
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -0,0 +1,79 @@
+//! Encodable building blocks for classifier and LLM forward passes.
+//!
+//! These are higher-level abstractions that compose kernels into
+//! reusable operations for both classifier and LLM pipelines.
+//!
+//! Encodables implement `EncodableBlock` and orchestrate one or more
+//! kernels to perform operations on `ForwardPassState`.
+
+use metal::ComputeCommandEncoderRef;
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::forward_pass::ForwardPassState;
+
+mod attention;
+mod classifier_layer;
+mod decoder;
+mod embedding;
+mod encoding_parameters;
+mod layer;
+mod linear;
+mod mamba_mixer;
+mod mlp;
+mod moe_block;
+mod normalization;
+mod pooling;
+mod prediction_head;
+mod rope;
+mod sampling;
+mod tensor_add_swap;
+mod tensor_copy;
+pub mod transformer_layer;
+
+pub use attention::Attention;
+pub use classifier_layer::ClassifierLayer;
+pub use decoder::Decoder;
+pub use embedding::{
+ QuantizedEmbeddingError, QuantizedEmbeddingLookup,
+ QuantizedEmbeddingReadout,
+};
+pub use encoding_parameters::EncodingParameters;
+pub use layer::LayerExecutables;
+pub use linear::QuantizedLinear;
+pub(crate) use mamba_mixer::MambaMixer;
+pub use mlp::MlpBlock;
+pub use moe_block::{MoeBlock, SharedMoeWeights};
+pub use normalization::{
+ LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,
+};
+pub use pooling::Pooling;
+pub use prediction_head::ClassifierPredictionHead;
+pub use rope::Rope;
+pub use sampling::Sampling;
+pub use tensor_add_swap::TensorAddSwap;
+pub use tensor_copy::TensorCopy;
+
+/// Trait for encodable blocks that operate on `ForwardPassState`.
+pub trait EncodableBlock {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ );
+
+ /// Returns true if this block supports using a shared compute encoder.
+ fn supports_shared_encoder(&self) -> bool {
+ false
+ }
+
+ /// Encode using a shared compute encoder. Only called if `supports_shared_encoder` returns true.
+ fn encode_with_shared_encoder(
+ &self,
+ _state: &mut ForwardPassState,
+ _encoder: &ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ panic!("encode_with_shared_encoder called on unsupported type");
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs
new file mode 100644
index 000000000..7e91b8451
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs
@@ -0,0 +1,7 @@
+//! MoE (Mixture of Experts) encodables.
+
+mod moe_block;
+mod shared_weights;
+
+pub use moe_block::MoeBlock;
+pub use shared_weights::SharedMoeWeights;
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
new file mode 100644
index 000000000..ec45d94b4
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
@@ -0,0 +1,633 @@
+//! MoE (Mixture of Experts) block encodable.
+
+use std::{cell::RefCell, rc::Rc};
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{
+ super::{EncodableBlock, EncodingParameters},
+ SharedMoeWeights,
+};
+use crate::{
+ Activation, DataType, LinearConfig, MixtureOfExpertsConfig,
+ RoutingFunctionConfig,
+ backends::metal::{
+ KernelDataType, MTLContext, MetalArray,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::moe::{
+ MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,
+ MoeCountsOffsetsFusedKernel, MoeExpertsTwoPassArguments,
+ MoeExpertsTwoPassDecodeKernel, MoeExpertsTwoPassPrefillKernel,
+ MoeFinalizeArguments, MoeFinalizeKernel, MoeGatherArguments,
+ MoeGatherKernel, MoeRouterTopKArguments, MoeRouterTopKKernel,
+ MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments,
+ },
+ },
+ parameters::ParameterTree,
+};
+
+enum RouterBlock {
+ Metal {
+ weights_buf: metal::Buffer,
+ biases_buf: metal::Buffer,
+ },
+}
+
+pub struct MoeBlock {
+ router: RouterBlock,
+ router_data_type: KernelDataType,
+ router_renorm: bool,
+ router_topk_kernel: MoeRouterTopKKernel,
+ counts_offsets_kernel: MoeCountsOffsetsFusedKernel,
+ scatter_kernels: MoeScatterKernels,
+ gather_kernel: MoeGatherKernel,
+ experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeKernel,
+ experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillKernel,
+ finalize_kernel: MoeFinalizeKernel,
+ moe_config: MixtureOfExpertsConfig,
+ model_dim: usize,
+ hidden_dim: usize,
+ data_type: KernelDataType,
+ shared_weights: SharedMoeWeights,
+}
+
+impl MoeBlock {
+ pub fn new(
+ context: &MTLContext,
+ moe_config: &MixtureOfExpertsConfig,
+ model_dim: usize,
+ hidden_dim: usize,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, crate::backends::metal::MTLError> {
+ let activation_data_type: DataType = moe_config
+ .expert_config
+ .linear_config
+ .activation_precision()
+ .into();
+ let data_type: KernelDataType = activation_data_type.into();
+
+ let router_data_type: DataType =
+ moe_config.router_config.activation_precision().into();
+ let router_kernel_data_type: KernelDataType = router_data_type.into();
+
+ let router_renorm = matches!(
+ moe_config.routing_function,
+ RoutingFunctionConfig::SoftmaxRouting
+ );
+
+ let router_tree = parameter_tree.subtree("router").map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Router subtree error: {:?}",
+ e
+ ))
+ })?;
+
+ let router = match &moe_config.router_config {
+ LinearConfig::Quantized(_) => {
+ unimplemented!(
+ "Quantized router with fused router+topk not yet supported"
+ );
+ },
+ LinearConfig::MLXQuantized(_) => {
+ unimplemented!(
+ "MLX quantized router with fused router+topk not yet supported"
+ );
+ },
+ LinearConfig::FullPrecision {
+ ..
+ } => {
+ let mut weights_arr =
+ router_tree.leaf("weights").map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Router weights error: {:?}",
+ e
+ ))
+ })?;
+ let weights_buf = unsafe { weights_arr.mtl_buffer().clone() };
+
+ let mut biases_arr =
+ router_tree.leaf("biases").map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Router biases error: {:?}",
+ e
+ ))
+ })?;
+ let biases_buf = unsafe { biases_arr.mtl_buffer().clone() };
+
+ RouterBlock::Metal {
+ weights_buf,
+ biases_buf,
+ }
+ },
+ LinearConfig::QLoRA {
+ ..
+ } => {
+ unimplemented!("QLoRA router not yet supported for MoE");
+ },
+ };
+
+ let router_topk_kernel =
+ MoeRouterTopKKernel::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "RouterTopK fused kernel error: {:?}",
+ e
+ ))
+ })?;
+ let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context)
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Counts+offsets kernel error: {:?}",
+ e
+ ))
+ })?;
+ let scatter_kernels = MoeScatterKernels::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Scatter kernels error: {:?}",
+ e
+ ))
+ })?;
+ let gather_kernel = MoeGatherKernel::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Gather kernel error: {:?}",
+ e
+ ))
+ })?;
+ let experts_two_pass_decode_kernel =
+ MoeExpertsTwoPassDecodeKernel::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Experts two-pass decode kernel error: {:?}",
+ e
+ ))
+ })?;
+ let experts_two_pass_prefill_kernel =
+ MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Experts two-pass prefill kernel error: {:?}",
+ e
+ ))
+ })?;
+ let finalize_kernel = MoeFinalizeKernel::new(context).map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Finalize kernel error: {:?}",
+ e
+ ))
+ })?;
+
+ let experts_tree = parameter_tree.subtree("experts").map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "experts subtree error: {:?}",
+ e
+ ))
+ })?;
+
+ let mut w13_arr = experts_tree
+ .subtree("up_projection")
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "up_projection error: {:?}",
+ e
+ ))
+ })?
+ .leaf("weights")
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "up weights error: {:?}",
+ e
+ ))
+ })?;
+ let w13_buf = unsafe { w13_arr.mtl_buffer().clone() };
+
+ let mut w2_arr = experts_tree
+ .subtree("down_projection")
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "down_projection error: {:?}",
+ e
+ ))
+ })?
+ .leaf("weights")
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "down weights error: {:?}",
+ e
+ ))
+ })?;
+ let w2_buf = unsafe { w2_arr.mtl_buffer().clone() };
+
+ let mut up_biases_arr = experts_tree
+ .subtree("up_projection")
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "up_projection biases error: {:?}",
+ e
+ ))
+ })?
+ .leaf("biases")
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "up biases error: {:?}",
+ e
+ ))
+ })?;
+ let up_biases_buf = unsafe { up_biases_arr.mtl_buffer().clone() };
+
+ let mut down_biases_arr = experts_tree
+ .subtree("down_projection")
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "down_projection biases error: {:?}",
+ e
+ ))
+ })?
+ .leaf("biases")
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "down biases error: {:?}",
+ e
+ ))
+ })?;
+ let down_biases_buf = unsafe { down_biases_arr.mtl_buffer().clone() };
+
+ let shared_weights = SharedMoeWeights {
+ w13_buf: Rc::new(w13_buf),
+ w2_buf: Rc::new(w2_buf),
+ up_biases_buf: Rc::new(up_biases_buf),
+ down_biases_buf: Rc::new(down_biases_buf),
+ };
+
+ Ok(Self {
+ router,
+ router_data_type: router_kernel_data_type,
+ router_renorm,
+ router_topk_kernel,
+ counts_offsets_kernel,
+ scatter_kernels,
+ gather_kernel,
+ experts_two_pass_decode_kernel,
+ experts_two_pass_prefill_kernel,
+ finalize_kernel,
+ moe_config: moe_config.clone(),
+ model_dim,
+ hidden_dim,
+ data_type,
+ shared_weights,
+ })
+ }
+
+ fn gating_code_from_activation(activation: &Activation) -> u32 {
+ match activation {
+ Activation::Gelu => 3,
+ Activation::SiLU {
+ ..
+ } => 2,
+ Activation::Identity => {
+ panic!("Identity activation is not supported for MoE kernels")
+ },
+ }
+ }
+}
+
+impl EncodableBlock for MoeBlock {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let suffix_length = state.active_suffix_length();
+ let arrays = state.arrays(&[
+ ArrayId::Main,
+ ArrayId::MoeTopkIds,
+ ArrayId::MoeTopkProbs,
+ ArrayId::MoeOffsets,
+ ArrayId::MoeSumK,
+ ArrayId::MoeBucketedTokenIds,
+ ArrayId::MoeBucketedProbs,
+ ArrayId::MoeXPerm,
+ ArrayId::MoeTok2Row,
+ ArrayId::MoeYPartial,
+ ArrayId::MoeTileCounts,
+ ArrayId::MoeTileOffsets,
+ ArrayId::MoeTileMap,
+ ArrayId::MoeTotalTiles,
+ ArrayId::MoeDispatchArgs,
+ ArrayId::MoeScatterPartials,
+ ArrayId::MoeScatterBlockBases,
+ ArrayId::MoeBlockAlloc,
+ ArrayId::MoeHidden,
+ ArrayId::MoeTwoPassRowExpertMap,
+ ]);
+
+ let clone_buffer = |array: &RefCell<MetalArray>| -> metal::Buffer {
+ let mut borrow = array.borrow_mut();
+ let buffer = unsafe { borrow.mtl_buffer().clone() };
+ buffer
+ };
+
+ let mut array_iter = arrays.iter();
+ let main_buf = clone_buffer(array_iter.next().unwrap());
+ let topk_ids_buf = clone_buffer(array_iter.next().unwrap());
+ let topk_probs_buf = clone_buffer(array_iter.next().unwrap());
+ let offsets_buf = clone_buffer(array_iter.next().unwrap());
+ let sumk_buf = clone_buffer(array_iter.next().unwrap());
+ let bucketed_ids_buf = clone_buffer(array_iter.next().unwrap());
+ let bucketed_probs_buf = clone_buffer(array_iter.next().unwrap());
+ let x_perm_buf = clone_buffer(array_iter.next().unwrap());
+ let tok2row_buf = clone_buffer(array_iter.next().unwrap());
+ let y_partial_buf = clone_buffer(array_iter.next().unwrap());
+ let tile_counts_buf = clone_buffer(array_iter.next().unwrap());
+ let tile_offsets_buf = clone_buffer(array_iter.next().unwrap());
+ let tile_map_buf = clone_buffer(array_iter.next().unwrap());
+ let total_tiles_buf = clone_buffer(array_iter.next().unwrap());
+ let dispatch_args_buf = clone_buffer(array_iter.next().unwrap());
+ let partials_buf = clone_buffer(array_iter.next().unwrap());
+ let block_bases_buf = clone_buffer(array_iter.next().unwrap());
+ let block_alloc_buf = clone_buffer(array_iter.next().unwrap());
+ let hidden_buf = clone_buffer(array_iter.next().unwrap());
+ let row_expert_map_buf = clone_buffer(array_iter.next().unwrap());
+ debug_assert!(array_iter.next().is_none());
+
+ let e = self.moe_config.mixture_size;
+ let k = self.moe_config.num_experts_per_token;
+
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let root = &*mtl_command_buffer;
+ let k_tile = 128;
+
+ // Clear internal MoE buffers
+ let dtype_size = match self.data_type {
+ KernelDataType::BFloat16 | KernelDataType::Float16 => 2,
+ KernelDataType::Float32 => 4,
+ };
+
+ let blit_encoder = root.new_blit_command_encoder();
+
+ if suffix_length > 0 && k > 0 {
+ let entries = suffix_length * k;
+ let topk_bytes = entries * std::mem::size_of::<u32>();
+ let tok2row_bytes = entries * std::mem::size_of::<i32>();
+
+ // Clear topk_ids and tok2row buffers
+ if topk_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &topk_ids_buf,
+ metal::NSRange::new(0, topk_bytes as u64),
+ 0xFF,
+ );
+ }
+ if tok2row_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &tok2row_buf,
+ metal::NSRange::new(0, tok2row_bytes as u64),
+ 0xFF,
+ );
+ }
+
+ // Clear hidden buffer
+ let hidden_bytes =
+ (suffix_length * k * self.hidden_dim * dtype_size) as u64;
+ if hidden_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &hidden_buf,
+ metal::NSRange::new(0, hidden_bytes),
+ 0,
+ );
+ }
+
+ // Clear y_partial buffer
+ let y_partial_bytes =
+ (suffix_length * k * self.model_dim * dtype_size) as u64;
+ if y_partial_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &y_partial_buf,
+ metal::NSRange::new(0, y_partial_bytes),
+ 0,
+ );
+ }
+
+ // Clear x_perm buffer
+ let x_perm_bytes =
+ (suffix_length * k * self.model_dim * dtype_size) as u64;
+ if x_perm_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &x_perm_buf,
+ metal::NSRange::new(0, x_perm_bytes),
+ 0,
+ );
+ }
+ }
+
+ blit_encoder.end_encoding();
+
+ // Use fused Router+TopK kernel for non-quantized routers
+ match &self.router {
+ RouterBlock::Metal {
+ weights_buf,
+ biases_buf,
+ } => {
+ // Use the fused router+topk kernel
+ self.router_topk_kernel
+ .encode(
+ root,
+ self.router_data_type,
+ MoeRouterTopKArguments {
+ input_buffer: &main_buf,
+ weight_buffer: weights_buf,
+ bias_buffer: biases_buf,
+ topk_ids_buffer: &topk_ids_buf,
+ topk_probs_buffer: &topk_probs_buf,
+ t: suffix_length,
+ d_model: self.model_dim,
+ e,
+ k,
+ renorm: self.router_renorm,
+ },
+ )
+ .expect("MoE fused router+topk failed");
+ },
+ }
+
+ self.counts_offsets_kernel
+ .encode(
+ root,
+ MoeCountsOffsetsFusedArguments {
+ topk_ids_buffer: &topk_ids_buf,
+ offsets_buffer: &offsets_buf,
+ sum_k_buffer: &sumk_buf,
+ partials_buffer: &partials_buf,
+ t: suffix_length,
+ e,
+ k,
+ },
+ )
+ .expect("MoE counts+offsets failed");
+
+ let num_blocks = ((suffix_length + 255) / 256).max(1);
+ let num_tiles = ((e + 512 - 1) / 512).max(1);
+
+ self.scatter_kernels
+ .encode_block_bases(
+ root,
+ MoeBlockBasesArguments {
+ partials_buffer: &partials_buf,
+ block_bases_buffer: &block_bases_buf,
+ block_alloc_buffer: &block_alloc_buf,
+ e,
+ num_blocks,
+ num_tiles,
+ },
+ )
+ .expect("MoE block bases failed");
+
+ self.scatter_kernels
+ .encode_scatter_with_map(
+ root,
+ MoeScatterWithMapArguments {
+ base: MoeScatterArguments {
+ topk_ids_buffer: &topk_ids_buf,
+ topk_probs_buffer: &topk_probs_buf,
+ offsets_buffer: &offsets_buf,
+ block_bases_buffer: &block_bases_buf,
+ block_alloc_buffer: &block_alloc_buf,
+ out_ids_buffer: &bucketed_ids_buf,
+ out_probs_buffer: &bucketed_probs_buf,
+ t: suffix_length,
+ e,
+ k,
+ num_blocks,
+ num_tiles,
+ },
+ tok2row_buffer: &tok2row_buf,
+ },
+ self.data_type,
+ )
+ .expect("MoE scatter failed");
+
+ self.gather_kernel
+ .encode(
+ root,
+ self.data_type,
+ MoeGatherArguments {
+ x_buffer: &main_buf,
+ bucketed_ids_buffer: &bucketed_ids_buf,
+ x_perm_buffer: &x_perm_buf,
+ sumk_buffer: &sumk_buf,
+ t: suffix_length,
+ k,
+ d_model: self.model_dim,
+ },
+ )
+ .expect("MoE gather failed");
+
+ let gating_code = Self::gating_code_from_activation(
+ &self.moe_config.expert_config.activation,
+ );
+
+ // Compute clipping values and alpha for expert kernels
+ let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]
+ .unwrap_or(f32::NEG_INFINITY);
+ let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]
+ .unwrap_or(f32::INFINITY);
+ let up_clip_min = self.moe_config.expert_config.up_clipping[0];
+ let up_clip_max = self.moe_config.expert_config.up_clipping[1];
+ let silu_alpha = self.moe_config.expert_config.activation.alpha();
+
+ if suffix_length == 1 {
+ let total_rows = suffix_length * k;
+ let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
+
+ self.experts_two_pass_decode_kernel
+ .encode(
+ root,
+ MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf,
+ w13_all: &self.shared_weights.w13_buf,
+ w2_all: &self.shared_weights.w2_buf,
+ up_biases: &self.shared_weights.up_biases_buf,
+ down_biases: &self.shared_weights.down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows,
+ d_model: self.model_dim,
+ d_ff: self.hidden_dim,
+ e,
+ num_tiles_k,
+ gating_code,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ data_type: self.data_type,
+ },
+ )
+ .expect("MoE experts two-pass failed");
+ } else {
+ let total_rows = suffix_length * k;
+ let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
+
+ self.experts_two_pass_prefill_kernel
+ .encode(
+ root,
+ MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf,
+ w13_all: &self.shared_weights.w13_buf,
+ w2_all: &self.shared_weights.w2_buf,
+ up_biases: &self.shared_weights.up_biases_buf,
+ down_biases: &self.shared_weights.down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows,
+ d_model: self.model_dim,
+ d_ff: self.hidden_dim,
+ e,
+ num_tiles_k,
+ gating_code,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ data_type: self.data_type,
+ },
+ )
+ .expect("MoE experts two-pass prefill failed");
+ }
+
+ self.finalize_kernel
+ .encode(
+ root,
+ MoeFinalizeArguments {
+ tok2row_buffer: &tok2row_buf,
+ probs_buffer: &topk_probs_buf,
+ y_partial_buffer: &y_partial_buf,
+ y_out_buffer: &main_buf,
+ t: suffix_length,
+ d_model: self.model_dim,
+ k,
+ },
+ self.data_type,
+ )
+ .expect("MoE finalize failed");
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs
new file mode 100644
index 000000000..461fcf1ae
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs
@@ -0,0 +1,11 @@
+//! Shared MoE expert weights.
+
+use std::rc::Rc;
+
+#[derive(Clone)]
+pub struct SharedMoeWeights {
+ pub w13_buf: Rc<metal::Buffer>,
+ pub w2_buf: Rc<metal::Buffer>,
+ pub up_biases_buf: Rc<metal::Buffer>,
+ pub down_biases_buf: Rc<metal::Buffer>,
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
new file mode 100644
index 000000000..a9e2b56a5
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
@@ -0,0 +1,138 @@
+//! LayerNorm encodable.
+
+use std::rc::Rc;
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::super::{EncodableBlock, EncodingParameters};
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::layer_norm::{
+ LayerNormArguments, LayerNormError, LayerNormKernel,
+ },
+ },
+ config::{NormalizationConfig, UpcastMode},
+ parameters::ParameterTree,
+};
+
+pub struct LayerNorm {
+ kernel: LayerNormKernel,
+ config: NormalizationConfig,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ scales_buffer: MTLBuffer,
+}
+
+impl LayerNorm {
+ pub fn new(
+ context: &MTLContext,
+ intermediate_data_type: DataType,
+ config: NormalizationConfig,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, LayerNormError> {
+ // Load scales from parameter tree
+ let scales_param = parameter_tree.leaf("scales").map_err(|e| {
+ LayerNormError::MetalError(MTLError::Library(
+ crate::backends::metal::error::LibraryError::Custom(format!(
+ "Failed to load scales: {:?}",
+ e
+ )),
+ ))
+ })?;
+
+ let scales_data = scales_param.buffer();
+ let scales_buffer = context.device.new_buffer_with_data(
+ scales_data.as_ptr() as *const _,
+ scales_data.len() as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+
+ let accumulation_data_type: DataType =
+ config.accumulation_precision.into();
+ let scale_data_type: DataType = config.scale_precision.into();
+
+ let (input_type, scales_type, output_type) = match config.upcast_mode {
+ UpcastMode::OnlyNormalization => {
+ (intermediate_data_type, scale_data_type, scale_data_type)
+ },
+ UpcastMode::FullLayer => {
+ (intermediate_data_type, scale_data_type, scale_data_type)
+ },
+ };
+
+ let kernel = LayerNormKernel::new(
+ context,
+ input_type,
+ scales_type,
+ output_type,
+ accumulation_data_type,
+ config.upcast_mode == UpcastMode::FullLayer,
+ )?;
+
+ Ok(Self {
+ kernel,
+ config,
+ input_array_id,
+ output_array_id,
+ scales_buffer,
+ })
+ }
+}
+
+impl EncodableBlock for LayerNorm {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let input_binding = state.arrays(&[self.input_array_id]);
+ let output_binding = state.arrays(&[self.output_array_id]);
+
+ let input_shape = {
+ let input_array = input_binding[0].borrow();
+ input_array.shape().to_vec()
+ };
+
+ let mut input_array = input_binding[0].borrow_mut();
+ let mut output_array = output_binding[0].borrow_mut();
+
+ let input_buffer = unsafe { input_array.mtl_buffer() };
+ let output_buffer = unsafe { output_array.mtl_buffer() };
+
+ let batch_size = input_shape[0] as i32;
+ let model_dim = input_shape[1] as i32;
+
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+
+ if let Err(e) = self.kernel.encode(
+ &compute_encoder,
+ LayerNormArguments {
+ input_buffer: &input_buffer,
+ scales_buffer: &self.scales_buffer,
+ output_buffer: &output_buffer,
+ batch_size,
+ model_dim,
+ epsilon: self.config.epsilon,
+ scale_offset: self.config.scale_offset.unwrap_or(0.0),
+ },
+ ) {
+ eprintln!("Failed to encode LayerNorm kernel: {:?}", e);
+ }
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
new file mode 100644
index 000000000..d2d7309cf
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
@@ -0,0 +1,92 @@
+//! Unified normalization encodables: LayerNorm, RMSNorm, QKNorm.
+
+mod layer_norm;
+mod qk_norm;
+mod rms_norm;
+
+use std::rc::Rc;
+
+pub use layer_norm::LayerNorm;
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+pub use qk_norm::QKNorm;
+pub use rms_norm::RMSNorm;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::{LayerNormError, RMSNormError},
+ },
+ config::NormalizationConfig,
+ parameters::ParameterTree,
+};
+
+/// Unified error type for normalization operations.
+#[derive(Debug, thiserror::Error)]
+pub enum NormalizationError {
+ #[error("LayerNorm error: {0}")]
+ LayerNorm(#[from] LayerNormError),
+ #[error("RMSNorm error: {0}")]
+ RMSNorm(#[from] RMSNormError),
+}
+
+/// Unified normalization encodable that can be either LayerNorm or RMSNorm.
+pub enum Normalization {
+ LayerNorm(LayerNorm),
+ RMSNorm(RMSNorm),
+}
+
+impl Normalization {
+ /// Creates the appropriate normalization encodable based on config.
+ /// Uses LayerNorm if subtract_mean is true, otherwise RMSNorm.
+ pub fn new(
+ context: &MTLContext,
+ intermediate_data_type: DataType,
+ config: NormalizationConfig,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, NormalizationError> {
+ if config.subtract_mean {
+ // Use LayerNorm (subtract mean before normalization)
+ Ok(Normalization::LayerNorm(LayerNorm::new(
+ context,
+ intermediate_data_type,
+ config,
+ input_array_id,
+ output_array_id,
+ parameter_tree,
+ )?))
+ } else {
+ // Use RMSNorm (no mean subtraction)
+ Ok(Normalization::RMSNorm(RMSNorm::new(
+ context,
+ intermediate_data_type,
+ config,
+ input_array_id,
+ output_array_id,
+ parameter_tree,
+ )?))
+ }
+ }
+}
+
+impl EncodableBlock for Normalization {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ match self {
+ Normalization::LayerNorm(layer_norm) => {
+ layer_norm.encode(state, command_buffer, parameters)
+ },
+ Normalization::RMSNorm(rms_norm) => {
+ rms_norm.encode(state, command_buffer, parameters)
+ },
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
new file mode 100644
index 000000000..c8e39f808
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
@@ -0,0 +1,247 @@
+//! QK Normalization encodable.
+
+use std::rc::Rc;
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::super::{EncodableBlock, EncodingParameters};
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::rms_norm::{
+ QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel,
+ RMSNormKernelType,
+ },
+ },
+ config::{NormalizationConfig, UpcastMode},
+ parameters::ParameterTree,
+};
+
+pub struct QKNorm {
+ query_kernel: Option<RMSNormKernel>,
+ key_kernel: Option<RMSNormKernel>,
+ query_config: Option<NormalizationConfig>,
+ key_config: Option<NormalizationConfig>,
+ qkv_array_id: ArrayId,
+ query_scales_buffer: Option<MTLBuffer>,
+ key_scales_buffer: Option<MTLBuffer>,
+ num_q_heads: usize,
+ num_kv_heads: usize,
+ head_dim: usize,
+}
+
+impl QKNorm {
+ pub fn new(
+ context: &MTLContext,
+ intermediate_data_type: DataType,
+ query_config: Option<NormalizationConfig>,
+ key_config: Option<NormalizationConfig>,
+ qkv_array_id: ArrayId,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ num_q_heads: usize,
+ num_kv_heads: usize,
+ head_dim: usize,
+ ) -> Result<Self, RMSNormError> {
+ let mut query_kernel = None;
+ let mut key_kernel = None;
+ let mut query_scales_buffer = None;
+ let mut key_scales_buffer = None;
+
+ // Setup query normalization if configured
+ if let Some(ref q_config) = query_config {
+ let scales_param =
+ parameter_tree.leaf("query_norm.scales").map_err(|e| {
+ RMSNormError::MetalError(MTLError::Library(
+ crate::backends::metal::error::LibraryError::Custom(
+ format!("Failed to load query scales: {:?}", e),
+ ),
+ ))
+ })?;
+
+ let scales_data = scales_param.buffer();
+ let scales_buffer = context.device.new_buffer_with_data(
+ scales_data.as_ptr() as *const _,
+ scales_data.len() as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+
+ let accumulation_data_type: DataType =
+ q_config.accumulation_precision.into();
+ let scale_data_type: DataType = q_config.scale_precision.into();
+
+ let (input_type, scales_type, output_type) = match q_config
+ .upcast_mode
+ {
+ UpcastMode::OnlyNormalization => {
+ (intermediate_data_type, scale_data_type, scale_data_type)
+ },
+ UpcastMode::FullLayer => {
+ (intermediate_data_type, scale_data_type, scale_data_type)
+ },
+ };
+
+ let kernel = RMSNormKernel::new_with_mode(
+ context,
+ input_type,
+ scales_type,
+ output_type,
+ accumulation_data_type,
+ RMSNormKernelType::QueryKey,
+ q_config.upcast_mode == UpcastMode::FullLayer,
+ )?;
+
+ query_kernel = Some(kernel);
+ query_scales_buffer = Some(scales_buffer);
+ }
+
+ // Setup key normalization if configured
+ if let Some(ref k_config) = key_config {
+ let scales_param =
+ parameter_tree.leaf("key_norm.scales").map_err(|e| {
+ RMSNormError::MetalError(MTLError::Library(
+ crate::backends::metal::error::LibraryError::Custom(
+ format!("Failed to load key scales: {:?}", e),
+ ),
+ ))
+ })?;
+
+ let scales_data = scales_param.buffer();
+ let scales_buffer = context.device.new_buffer_with_data(
+ scales_data.as_ptr() as *const _,
+ scales_data.len() as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+
+ let accumulation_data_type: DataType =
+ k_config.accumulation_precision.into();
+ let scale_data_type: DataType = k_config.scale_precision.into();
+
+ let (input_type, scales_type, output_type) = match k_config
+ .upcast_mode
+ {
+ UpcastMode::OnlyNormalization => {
+ (intermediate_data_type, scale_data_type, scale_data_type)
+ },
+ UpcastMode::FullLayer => {
+ (intermediate_data_type, scale_data_type, scale_data_type)
+ },
+ };
+
+ let kernel = RMSNormKernel::new_with_mode(
+ context,
+ input_type,
+ scales_type,
+ output_type,
+ accumulation_data_type,
+ RMSNormKernelType::QueryKey,
+ k_config.upcast_mode == UpcastMode::FullLayer,
+ )?;
+
+ key_kernel = Some(kernel);
+ key_scales_buffer = Some(scales_buffer);
+ }
+
+ Ok(Self {
+ query_kernel,
+ key_kernel,
+ query_config,
+ key_config,
+ qkv_array_id,
+ query_scales_buffer,
+ key_scales_buffer,
+ num_q_heads,
+ num_kv_heads,
+ head_dim,
+ })
+ }
+}
+
+impl EncodableBlock for QKNorm {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let qkv_binding = state.arrays(&[self.qkv_array_id]);
+ let qkv_shape = {
+ let qkv_array = qkv_binding[0].borrow();
+ qkv_array.shape().to_vec()
+ };
+
+ let mut qkv_array = qkv_binding[0].borrow_mut();
+ let qkv_buffer = unsafe { qkv_array.mtl_buffer() };
+
+ let batch_size = qkv_shape[0] as i32;
+ let head_dim = self.head_dim as i32;
+
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+
+ // Process query normalization if configured
+ if let (
+ Some(query_kernel),
+ Some(query_scales_buffer),
+ Some(query_config),
+ ) =
+ (&self.query_kernel, &self.query_scales_buffer, &self.query_config)
+ {
+ if let Err(e) = query_kernel.encode_qk_norm(
+ &compute_encoder,
+ QKNormArguments {
+ qkv_input_buffer: &qkv_buffer,
+ scales_buffer: query_scales_buffer,
+ qkv_output_buffer: &qkv_buffer,
+ batch_size,
+ // Always pass actual head counts (needed for correct buffer addressing)
+ num_q_heads: self.num_q_heads as i32,
+ num_kv_heads: self.num_kv_heads as i32,
+ head_dim,
+ epsilon: query_config.epsilon,
+ scale_offset: query_config.scale_offset.unwrap_or(0.0),
+ target: QKNormTarget::QueryHeads,
+ },
+ ) {
+ eprintln!(
+ "Failed to encode query normalization kernel: {:?}",
+ e
+ );
+ }
+ }
+
+ // Process key normalization if configured
+ if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =
+ (&self.key_kernel, &self.key_scales_buffer, &self.key_config)
+ {
+ if let Err(e) = key_kernel.encode_qk_norm(
+ &compute_encoder,
+ QKNormArguments {
+ qkv_input_buffer: &qkv_buffer,
+ scales_buffer: key_scales_buffer,
+ qkv_output_buffer: &qkv_buffer,
+ batch_size,
+ // Always pass actual head counts (needed for correct buffer addressing)
+ num_q_heads: self.num_q_heads as i32,
+ num_kv_heads: self.num_kv_heads as i32,
+ head_dim,
+ epsilon: key_config.epsilon,
+ scale_offset: key_config.scale_offset.unwrap_or(0.0),
+ target: QKNormTarget::KeyHeads,
+ },
+ ) {
+ eprintln!("Failed to encode key normalization kernel: {:?}", e);
+ }
+ }
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
new file mode 100644
index 000000000..83a96877e
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
@@ -0,0 +1,142 @@
+//! RMS Normalization encodable.
+
+use std::rc::Rc;
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::super::{EncodableBlock, EncodingParameters};
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::rms_norm::{
+ RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType,
+ },
+ },
+ config::{NormalizationConfig, UpcastMode},
+ parameters::ParameterTree,
+};
+
+pub struct RMSNorm {
+ kernel: RMSNormKernel,
+ config: NormalizationConfig,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ scales_buffer: MTLBuffer,
+}
+
+impl RMSNorm {
+ pub fn new(
+ context: &MTLContext,
+ intermediate_data_type: DataType,
+ config: NormalizationConfig,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, RMSNormError> {
+ // Load scales from parameter tree
+ let scales_param = parameter_tree.leaf("scales").map_err(|e| {
+ RMSNormError::MetalError(MTLError::Library(
+ crate::backends::metal::error::LibraryError::Custom(format!(
+ "Failed to load scales: {:?}",
+ e
+ )),
+ ))
+ })?;
+
+ // TODO: Don't create buffers dynamically, we need to use forward pass storage for thing like this
+ let scales_data = scales_param.buffer();
+ let scales_buffer = context.device.new_buffer_with_data(
+ scales_data.as_ptr() as *const _,
+ scales_data.len() as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+
+ let accumulation_data_type: DataType =
+ config.accumulation_precision.into();
+ let scale_data_type: DataType = config.scale_precision.into();
+
+ let (input_type, scales_type, output_type) = match config.upcast_mode {
+ UpcastMode::OnlyNormalization => {
+ // Input stays as pipeline type, scales stay scale precision, output is scale precision
+ (intermediate_data_type, scale_data_type, scale_data_type)
+ },
+ UpcastMode::FullLayer => {
+ // Input stays as pipeline type, scales stay in original precision (will be cast to AccumT inside kernel), output is scale precision
+ (intermediate_data_type, scale_data_type, scale_data_type)
+ },
+ };
+
+ let kernel = RMSNormKernel::new_with_mode(
+ context,
+ input_type,
+ scales_type,
+ output_type,
+ accumulation_data_type,
+ RMSNormKernelType::Standard,
+ config.upcast_mode == UpcastMode::FullLayer,
+ )?;
+
+ Ok(Self {
+ kernel,
+ config,
+ input_array_id,
+ output_array_id,
+ scales_buffer,
+ })
+ }
+}
+
+impl EncodableBlock for RMSNorm {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let input_binding = state.arrays(&[self.input_array_id]);
+ let output_binding = state.arrays(&[self.output_array_id]);
+
+ let input_shape = {
+ let input_array = input_binding[0].borrow();
+ input_array.shape().to_vec()
+ };
+
+ let mut input_array = input_binding[0].borrow_mut();
+ let mut output_array = output_binding[0].borrow_mut();
+
+ let input_buffer = unsafe { input_array.mtl_buffer() };
+ let output_buffer = unsafe { output_array.mtl_buffer() };
+
+ let batch_size = input_shape[0] as i32;
+ let model_dim = input_shape[1] as i32;
+
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+
+ if let Err(e) = self.kernel.encode(
+ &compute_encoder,
+ RMSNormArguments {
+ input_buffer: &input_buffer,
+ scales_buffer: &self.scales_buffer,
+ output_buffer: &output_buffer,
+ batch_size,
+ model_dim,
+ epsilon: self.config.epsilon,
+ scale_offset: self.config.scale_offset.unwrap_or(0.0),
+ },
+ ) {
+ eprintln!("Failed to encode RMS norm kernel: {:?}", e);
+ }
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/pooling.rs b/crates/uzu/src/backends/metal/encodable_block/pooling.rs
new file mode 100644
index 000000000..323585f66
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/pooling.rs
@@ -0,0 +1,112 @@
+//! Pooling encodable for sequence-level aggregation.
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ Array,
+ backends::metal::{
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::PoolingKernel,
+ },
+ config::PoolingType,
+};
+
+pub struct Pooling {
+ pooling_kernel: PoolingKernel,
+ pooling_type: PoolingType,
+ model_dim: usize,
+}
+
+impl Pooling {
+ pub fn new(
+ pooling_kernel: PoolingKernel,
+ pooling_type: PoolingType,
+ model_dim: usize,
+ ) -> Self {
+ Self {
+ pooling_kernel,
+ pooling_type,
+ model_dim,
+ }
+ }
+}
+
+impl EncodableBlock for Pooling {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ _parameters: &EncodingParameters,
+ ) {
+ let batch_size = 1;
+ let seq_len = state.aux_buffers_suffix_length();
+
+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::ClassifierPooling]);
+ #[cfg_attr(not(feature = "tracing"), allow(unused_variables))]
+ let data_type = { Array::data_type(&*arrays[0].borrow()) };
+ let mut main_array = arrays[0].borrow_mut();
+ let input_buffer = unsafe { main_array.mtl_buffer() };
+
+ let mut pooling_array = arrays[1].borrow_mut();
+ let output_buffer = unsafe { pooling_array.mtl_buffer().to_owned() };
+
+ let root = command_buffer.root_command_buffer();
+ let encoder = root.new_compute_command_encoder();
+
+ let result = match self.pooling_type {
+ PoolingType::Cls => self.pooling_kernel.encode_cls(
+ encoder,
+ input_buffer,
+ &output_buffer,
+ batch_size as i32,
+ seq_len as i32,
+ self.model_dim as i32,
+ ),
+ PoolingType::Mean => self.pooling_kernel.encode_mean(
+ encoder,
+ input_buffer,
+ &output_buffer,
+ batch_size as i32,
+ seq_len as i32,
+ self.model_dim as i32,
+ ),
+ };
+
+ if result.is_err() {
+ panic!("Failed to encode pooling kernel");
+ }
+
+ encoder.end_encoding();
+ drop(main_array);
+ drop(pooling_array);
+
+ command_buffer.commit_and_continue();
+
+ #[cfg(feature = "tracing")]
+ {
+ let traces_rc = state.traces().clone();
+ let traces_ref = traces_rc.borrow();
+ let mut trace_arr = traces_ref.output_pooling().borrow_mut();
+ let dst_buf = unsafe { trace_arr.mtl_buffer().to_owned() };
+ drop(trace_arr);
+ drop(traces_ref);
+
+ let root = command_buffer.root_command_buffer();
+ let blit = root.new_blit_command_encoder();
+ blit.copy_from_buffer(
+ &output_buffer,
+ 0,
+ &dst_buf,
+ 0,
+ (batch_size * self.model_dim * data_type.size_in_bytes())
+ as u64,
+ );
+ blit.end_encoding();
+
+ let root_owned = root.to_owned();
+ command_buffer.commit_and_continue();
+ root_owned.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
new file mode 100644
index 000000000..25a4fcf4f
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
@@ -0,0 +1,96 @@
+//! Prediction head encodable for classification output.
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters};
+#[cfg(feature = "tracing")]
+use crate::Array;
+#[cfg(feature = "tracing")]
+use crate::backends::metal::forward_pass::ArrayId;
+use crate::backends::metal::forward_pass::ForwardPassState;
+
+pub struct ClassifierPredictionHead {
+ dense: Box<dyn EncodableBlock>,
+ activation: Box<dyn EncodableBlock>,
+ norm: Box<dyn EncodableBlock>,
+ readout: Box<dyn EncodableBlock>,
+ #[cfg_attr(not(feature = "tracing"), allow(dead_code))]
+ num_labels: usize,
+}
+
+impl ClassifierPredictionHead {
+ pub fn new(
+ dense: Box<dyn EncodableBlock>,
+ activation: Box<dyn EncodableBlock>,
+ norm: Box<dyn EncodableBlock>,
+ readout: Box<dyn EncodableBlock>,
+ num_labels: usize,
+ ) -> Self {
+ Self {
+ dense,
+ activation,
+ norm,
+ readout,
+ num_labels,
+ }
+ }
+}
+
+impl EncodableBlock for ClassifierPredictionHead {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ self.dense.encode(state, command_buffer, parameters);
+ self.activation.encode(state, command_buffer, parameters);
+ self.norm.encode(state, command_buffer, parameters);
+ self.readout.encode(state, command_buffer, parameters);
+
+ #[cfg_attr(feature = "tracing", allow(unused_variables))]
+ let root_after_head = command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+
+ #[cfg(not(feature = "tracing"))]
+ root_after_head.wait_until_completed();
+
+ #[cfg(feature = "tracing")]
+ {
+ let traces_rc = state.traces().clone();
+ let logits_arrays =
+ state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
+ let mut logits_array_ref = logits_arrays[0].borrow_mut();
+ let linear_output_buffer =
+ unsafe { logits_array_ref.mtl_buffer().to_owned() };
+ let data_type = Array::data_type(&*logits_array_ref);
+ let batch_size = Array::shape(&*logits_array_ref)[0];
+ drop(logits_array_ref);
+
+ let traces_ref = traces_rc.borrow();
+ let mut trace_logits = traces_ref.logits.borrow_mut();
+ let dst_trace_buf = unsafe { trace_logits.mtl_buffer().to_owned() };
+ drop(trace_logits);
+ drop(traces_ref);
+
+ let copy_size_bytes =
+ (batch_size * self.num_labels * data_type.size_in_bytes())
+ as u64;
+
+ let root = command_buffer.root_command_buffer();
+ let blit = root.new_blit_command_encoder();
+ blit.copy_from_buffer(
+ &linear_output_buffer,
+ 0,
+ &dst_trace_buf,
+ 0,
+ copy_size_bytes,
+ );
+ blit.end_encoding();
+
+ let root_owned = root.to_owned();
+ command_buffer.commit_and_continue();
+ root_owned.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rs
new file mode 100644
index 000000000..e81874c6a
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs
@@ -0,0 +1,116 @@
+//! Rope (Rotary Position Embedding) encodable.
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ Array,
+ backends::metal::{
+ KernelDataType, MTLContext,
+ forward_pass::{ArrayId, ForwardPassState, RopeType},
+ kernel::rope::{RopeError, RopeKernel, RopeKernelArguments},
+ },
+};
+
+pub struct Rope {
+ kernel: RopeKernel,
+ rope_type: RopeType,
+}
+
+impl Rope {
+ pub fn new(
+ context: &MTLContext,
+ data_type: KernelDataType,
+ rope_type: RopeType,
+ ) -> Result<Self, RopeError> {
+ let kernel = RopeKernel::new(context, data_type)?;
+ Ok(Self {
+ kernel,
+ rope_type,
+ })
+ }
+}
+
+impl EncodableBlock for Rope {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {
+ let qkv_binding = state.arrays(&[ArrayId::QKV]);
+ let qkv_array = qkv_binding[0].borrow();
+ let suffix_length = qkv_array.shape()[0];
+
+ let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
+ let queries_array = queries_binding[0].borrow();
+ let num_heads = queries_array.shape()[0];
+ let head_dim = queries_array.shape()[2];
+
+ let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
+ let keys_array = keys_binding[0].borrow();
+ let num_groups = keys_array.shape()[0];
+
+ let cos_binding =
+ state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
+ let cos_array = cos_binding[0].borrow();
+ let cos_shape = cos_array.shape();
+ let rope_max_seq_len = cos_shape[0];
+
+ (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len)
+ };
+
+ let qkv_buffer_binding = state.arrays(&[ArrayId::QKV]);
+ let mut qkv = qkv_buffer_binding[0].borrow_mut();
+
+ let token_positions_binding = state.arrays(&[ArrayId::TokenPositions]);
+ let mut token_positions = token_positions_binding[0].borrow_mut();
+
+ let query_buffer_binding = state.arrays(&[ArrayId::RotatedQueries]);
+ let mut rotated_queries = query_buffer_binding[0].borrow_mut();
+
+ let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
+ let mut rotated_keys = rotated_keys_binding[0].borrow_mut();
+
+ let cos_buffer_binding =
+ state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
+ let mut rope_cosines = cos_buffer_binding[0].borrow_mut();
+
+ let sin_buffer_binding =
+ state.arrays(&[ArrayId::RopeSines(self.rope_type)]);
+ let mut rope_sines = sin_buffer_binding[0].borrow_mut();
+
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+
+ self.kernel.encode(
+ &compute_encoder,
+ RopeKernelArguments {
+ qkv_buffer: unsafe { &qkv.mtl_buffer() },
+ cosines_buffer: unsafe { &rope_cosines.mtl_buffer() },
+ sines_buffer: unsafe { &rope_sines.mtl_buffer() },
+ token_positions_buffer: unsafe {
+ &token_positions.mtl_buffer()
+ },
+ rotated_queries_buffer: unsafe {
+ &rotated_queries.mtl_buffer()
+ },
+ rotated_keys_buffer: unsafe { &rotated_keys.mtl_buffer() },
+ head_dim,
+ num_heads,
+ num_groups,
+ suffix_length,
+ max_sequence_length: rope_max_seq_len,
+ },
+ );
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/sampling.rs b/crates/uzu/src/backends/metal/encodable_block/sampling.rs
new file mode 100644
index 000000000..269a8c1cd
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/sampling.rs
@@ -0,0 +1,105 @@
+//! Sampling kernel encodable.
+
+use std::rc::Rc;
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::backends::metal::{
+ KernelDataType, MTLContext,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::sampling::{ArgmaxStrategy, SamplingError, SamplingKernel},
+};
+
+pub struct Sampling {
+ pub kernel: SamplingKernel,
+}
+
+impl Sampling {
+ pub fn new(
+ context: &Rc<MTLContext>,
+ data_type: KernelDataType,
+ max_batch_size: usize,
+ max_vocab_size: usize,
+ ) -> Result<Self, SamplingError> {
+ let kernel = SamplingKernel::new(
+ context,
+ data_type,
+ max_batch_size,
+ max_vocab_size,
+ )?;
+ Ok(Self {
+ kernel,
+ })
+ }
+
+ pub fn new_with_strategy(
+ context: &Rc<MTLContext>,
+ data_type: KernelDataType,
+ max_batch_size: usize,
+ max_vocab_size: usize,
+ argmax_strategy: ArgmaxStrategy,
+ ) -> Result<Self, SamplingError> {
+ let kernel = SamplingKernel::new_with_strategy(
+ context,
+ data_type,
+ max_batch_size,
+ max_vocab_size,
+ argmax_strategy,
+ )?;
+ Ok(Self {
+ kernel,
+ })
+ }
+}
+
+impl EncodableBlock for Sampling {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ assert!(
+ state.sampling_output().is_some(),
+ "Sampling output buffer must be pre-allocated"
+ );
+
+ let logits_binding = state.arrays(&[ArrayId::Logits]);
+ let mut logits = logits_binding[0].borrow_mut();
+
+ let logits_shape = {
+ use crate::Array;
+ logits.shape()
+ };
+ let batch_size = state.active_suffix_length();
+ let vocab_size = logits_shape[1];
+
+ let seeds_binding = state.arrays(&[ArrayId::TokenSeeds]);
+ let mut seeds = seeds_binding[0].borrow_mut();
+
+ let mut output_buffer_ref =
+ state.sampling_output().unwrap().borrow_mut();
+
+ let sampling_method = state.sampling_method().unwrap();
+
+ let root_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ if let Err(e) = self.kernel.encode(
+ unsafe { &logits.mtl_buffer() },
+ unsafe { Some(&seeds.mtl_buffer()) },
+ unsafe { &output_buffer_ref.mtl_buffer() },
+ sampling_method,
+ batch_size,
+ vocab_size,
+ &root_command_buffer,
+ ) {
+ panic!("Sampling encoding failed: {:?}", e);
+ }
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ root_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
new file mode 100644
index 000000000..ae72e7d47
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
@@ -0,0 +1,66 @@
+//! Tensor add-swap encodable.
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ Array,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::TensorAddSwapKernel,
+ },
+};
+
+pub struct TensorAddSwap {
+ kernel: TensorAddSwapKernel,
+ argument_arrays: Box<[ArrayId]>,
+}
+
+impl TensorAddSwap {
+ pub fn new(
+ context: &MTLContext,
+ data_type: crate::backends::metal::KernelDataType,
+ argument_arrays: Box<[ArrayId]>,
+ ) -> Result<Self, MTLError> {
+ let kernel = TensorAddSwapKernel::new(context, data_type)?;
+ Ok(Self {
+ kernel,
+ argument_arrays,
+ })
+ }
+}
+
+impl EncodableBlock for TensorAddSwap {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&self.argument_arrays);
+ assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
+
+ let length = arrays[0].borrow().num_elements();
+
+ let mut skip_array = arrays[0].borrow_mut();
+ let mut main_array = arrays[1].borrow_mut();
+ let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };
+ let main_mtl_buffer = unsafe { main_array.mtl_buffer() };
+
+ let retained_mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+
+ self.kernel.encode_into_command_buffer(
+ &skip_mtl_buffer,
+ &main_mtl_buffer,
+ length,
+ &retained_mtl_command_buffer,
+ );
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ retained_mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
new file mode 100644
index 000000000..61156c8fa
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
@@ -0,0 +1,66 @@
+//! Tensor copy encodable.
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ Array,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::TensorCopyKernel,
+ },
+};
+
+pub struct TensorCopy {
+ kernel: TensorCopyKernel,
+ argument_arrays: Box<[ArrayId]>,
+}
+
+impl TensorCopy {
+ pub fn new(
+ context: &MTLContext,
+ data_type: crate::backends::metal::KernelDataType,
+ argument_arrays: Box<[ArrayId]>,
+ ) -> Result<Self, MTLError> {
+ let kernel = TensorCopyKernel::new(context, data_type)?;
+ Ok(Self {
+ kernel,
+ argument_arrays,
+ })
+ }
+}
+
+impl EncodableBlock for TensorCopy {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&self.argument_arrays);
+ assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
+
+ let length = arrays[0].borrow().num_elements();
+
+ let mut source_array = arrays[0].borrow_mut();
+ let mut destination_array = arrays[1].borrow_mut();
+ let source_mtl_buffer = unsafe { source_array.mtl_buffer() };
+ let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };
+
+ let retained_mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+
+ self.kernel.encode_into_command_buffer(
+ &source_mtl_buffer,
+ &destination_mtl_buffer,
+ length,
+ &retained_mtl_command_buffer,
+ );
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ retained_mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
similarity index 93%
rename from crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs
rename to crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index fec5b39dd..0f00b2ece 100644
--- a/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -7,29 +7,20 @@ use mpsgraph::{
use objc2::rc::{Retained, autoreleasepool};
use super::{
- super::{
- MTLContext,
- graph::{linear_subgraph, mlp_subgraph, placeholder, shaped_type},
- },
- io_arrays::IOArrays,
- mpsgraph_block::MPSGraphBlock,
- state::ArrayId,
+ EncodableBlock, MlpBlock, MoeBlock, QuantizedEmbeddingLookup,
+ QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
backends::metal::{
+ MTLContext,
+ forward_pass::{ArrayId, IOArrays, MPSGraphBlock},
graph::{
embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
- embeddings_readout_subgraph,
- },
- kernel::{
- MoeBlockEncodable, QuantizedLinearKernelBlock,
- embedding::{
- QuantizedEmbeddingLookupKernelBlock,
- QuantizedEmbeddingReadoutKernelBlock,
- },
- mlp::MlpGateActMulEncodable,
+ embeddings_readout_subgraph, linear_subgraph, mlp_subgraph,
+ placeholder, shaped_type,
},
+ kernel::mlp::MlpGateActMulEncodable,
},
config::{
DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig,
@@ -54,7 +45,7 @@ pub fn linear_block<const N: usize>(
input_array_id: ArrayId,
output_array_id: ArrayId,
compilation_descriptor: &CompilationDescriptor,
-) -> Box<dyn super::encodable_with_state::EncodableWithState> {
+) -> Box<dyn EncodableBlock> {
if let LinearConfig::Quantized(quant_config) = config {
let out_sum: usize = output_dims.iter().sum();
return quantized_linear_block_custom(
@@ -134,11 +125,8 @@ pub fn quantized_linear_block_custom(
parameter_tree: &ParameterTree<Rc<MTLContext>>,
input_array_id: ArrayId,
output_array_id: ArrayId,
-) -> Result<
- Box<dyn super::encodable_with_state::EncodableWithState>,
- crate::backends::metal::MTLError,
-> {
- let block = QuantizedLinearKernelBlock::new(
+) -> Result<Box<dyn EncodableBlock>, crate::backends::metal::MTLError> {
+ let block = QuantizedLinear::new(
context,
config,
input_dim,
@@ -157,7 +145,7 @@ pub fn mlp_block(
context: &MTLContext,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
compilation_descriptor: &CompilationDescriptor,
-) -> Box<dyn super::encodable_with_state::EncodableWithState> {
+) -> Box<dyn EncodableBlock> {
if let crate::config::MLPConfig::Dense(dense) = config {
if let crate::config::LinearConfig::Quantized(ref quant_config)
| crate::config::LinearConfig::MLXQuantized(ref quant_config) =
@@ -168,7 +156,7 @@ pub fn mlp_block(
dense.linear_config.activation_precision().into();
// Up fused: Main -> MlpFusedUp
- let up = QuantizedLinearKernelBlock::new(
+ let up = QuantizedLinear::new(
context,
quant_config,
model_dim,
@@ -189,7 +177,7 @@ pub fn mlp_block(
.expect("Failed to build MLP gate activation kernel");
// Down: MlpHidden -> Main
- let down = QuantizedLinearKernelBlock::new(
+ let down = QuantizedLinear::new(
context,
quant_config,
hidden_dim,
@@ -200,24 +188,16 @@ pub fn mlp_block(
)
.expect("Failed to build MLP down quantized block");
- let enc =
- crate::backends::metal::kernel::mlp::MlpBlockEncodable::new(
- up, gate_op, down,
- );
+ let enc = MlpBlock::new(up, gate_op, down);
return Box::new(enc);
}
}
// STEP 2: Enable MOE block creation (encode will be minimal)
if let crate::config::MLPConfig::MixtureOfExperts(moe) = config {
- let moe_block = MoeBlockEncodable::new(
- context,
- moe,
- model_dim,
- hidden_dim,
- parameter_tree,
- )
- .expect("Failed to build MoE block");
+ let moe_block =
+ MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)
+ .expect("Failed to build MoE block");
return Box::new(moe_block);
}
@@ -248,7 +228,7 @@ pub fn mlp_block(
)
.unwrap(),
crate::config::MLPConfig::MixtureOfExperts(_moe) => {
- unreachable!("MoE uses MoeBlockEncodable, not MPSGraph")
+ unreachable!("MoE uses MoeBlock, not MPSGraph")
},
};
@@ -282,7 +262,7 @@ pub fn embed_block(
context: &MTLContext,
compilation_descriptor: &CompilationDescriptor,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Box<dyn super::encodable_with_state::EncodableWithState> {
+) -> Box<dyn EncodableBlock> {
let graph = Graph::new();
let input_shape = [-1 as isize];
@@ -506,7 +486,7 @@ pub fn embed_block(
.subtree("embedding")
.expect("Failed to get embedding subtree");
- let block = QuantizedEmbeddingLookupKernelBlock::new(
+ let block = QuantizedEmbeddingLookup::new(
context,
data_type,
config.vocab_size,
@@ -527,7 +507,7 @@ pub fn readout_block(
context: &MTLContext,
compilation_descriptor: &CompilationDescriptor,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Box<dyn super::encodable_with_state::EncodableWithState> {
+) -> Box<dyn EncodableBlock> {
let graph = Graph::new();
let input_shape = [-1 as isize, config.model_dim as isize];
@@ -640,7 +620,7 @@ pub fn readout_block(
.subtree("embedding")
.expect("Failed to get embedding subtree");
- let block = QuantizedEmbeddingReadoutKernelBlock::new(
+ let block = QuantizedEmbeddingReadout::new(
context,
data_type,
config.vocab_size,
@@ -732,7 +712,7 @@ pub fn readout_block(
.subtree("embedding")
.expect("Failed to get embedding subtree");
- let block = QuantizedEmbeddingReadoutKernelBlock::new(
+ let block = QuantizedEmbeddingReadout::new(
context,
data_type,
config.vocab_size,
diff --git a/crates/uzu/src/backends/metal/error.rs b/crates/uzu/src/backends/metal/error.rs
index 802fd7d9f..384128852 100644
--- a/crates/uzu/src/backends/metal/error.rs
+++ b/crates/uzu/src/backends/metal/error.rs
@@ -174,3 +174,18 @@ pub enum PixelFormatError {
#[error("{0}")]
Custom(String),
}
+
+// Classifier context errors
+#[derive(Debug, Error)]
+pub enum ClassifierError {
+ #[error("Missing required config field: {0}")]
+ MissingConfigField(String),
+ #[error("Model requires attention layers but found non-attention mixer")]
+ NonAttentionMixer,
+ #[error("Weight subtree not found: {0}")]
+ WeightSubtreeNotFound(String),
+ #[error("Kernel creation failed: {0}")]
+ KernelCreationFailed(String),
+ #[error("{0}")]
+ Custom(String),
+}
diff --git a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs
deleted file mode 100644
index a4dcb4b25..000000000
--- a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs
+++ /dev/null
@@ -1,616 +0,0 @@
-use std::rc::Rc;
-
-use metal::ComputeCommandEncoderRef;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use objc2::rc::autoreleasepool;
-
-use crate::{
- DataType,
- backends::metal::{
- MTLContext,
- compilation_parameters::CompilationConfig,
- forward_pass::{
- ArrayId, EncoderResolver, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- transformer_layer,
- },
- kernel::{
- AttentionKernelEncodable, KernelDataType, MambaMixerEncodable,
- QKNormKernelEncodable, RMSNormKernelEncodable, TensorAddSwap,
- TensorCopy,
- },
- },
- config::{
- DecoderLayerType,
- decoder_layer::{DecoderLayerConfig, MixerConfig},
- },
- parameters::ParameterTree,
-};
-
-pub struct LayerExecutables {
- pub layer_index: usize,
- pub copy_main_to_shortcut: Box<dyn EncodableWithState>,
- pub pre_attention_norm: Box<dyn EncodableWithState>,
- pub(crate) mixer: MixerExecutables,
- pub post_attention_norm: Option<Box<dyn EncodableWithState>>,
- pub main_shortcut_add_swap: Box<dyn EncodableWithState>,
- pub pre_mlp_norm: Box<dyn EncodableWithState>,
- pub mlp: Box<dyn EncodableWithState>,
- pub post_mlp_norm: Option<Box<dyn EncodableWithState>>,
-}
-
-pub(crate) enum MixerExecutables {
- Attention {
- qkv_projection: Box<dyn EncodableWithState>,
- qk_norm: Option<Box<dyn EncodableWithState>>,
- rope: Rc<Box<dyn EncodableWithState>>,
- attention: Box<dyn EncodableWithState>,
- out_projection: Box<dyn EncodableWithState>,
- },
- StateSpace {
- mixer: Box<dyn EncodableWithState>,
- },
-}
-
-impl LayerExecutables {
- #[allow(clippy::too_many_arguments)]
- pub fn new(
- mtl_context: &MTLContext,
- layer_config: &DecoderLayerConfig,
- layer_type: &DecoderLayerType,
- compilation_config: Rc<CompilationConfig>,
- layer_index: usize,
- model_dim: usize,
- hidden_dim: usize,
- num_heads: usize,
- head_dim: usize,
- num_groups: usize,
- attention_scale: Option<f32>,
- decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,
- rope: Option<Rc<Box<dyn EncodableWithState>>>,
- ) -> Self {
- autoreleasepool(|_| {
- let intermediate_data_type: DataType =
- match &layer_config.mixer_config {
- MixerConfig::Attention(attention) => attention
- .qkv_projection_config
- .activation_precision()
- .into(),
- MixerConfig::Mamba(mamba) => {
- mamba.in_projection_config.activation_precision().into()
- },
- };
- let kernel_data_type: KernelDataType =
- intermediate_data_type.into();
-
- let copy_main_to_shortcut: Box<dyn EncodableWithState> = Box::new(
- TensorCopy::new(
- mtl_context,
- kernel_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap(),
- );
-
- let pre_attention_norm: Box<dyn EncodableWithState> = Box::new(
- RMSNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- layer_config.pre_attention_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- );
-
- let mixer = match &layer_config.mixer_config {
- MixerConfig::Attention(attention_config) => {
- let rope_block = rope
- .clone()
- .expect("RoPE encoder missing for attention layer");
-
- let qkv_projection = transformer_layer::linear_block(
- &attention_config.qkv_projection_config,
- attention_config.has_qkv_biases,
- model_dim,
- [
- num_heads * head_dim,
- num_groups * head_dim,
- num_groups * head_dim,
- ],
- mtl_context,
- &decoder_layer_loader
- .subtree("mixer.qkv_projection")
- .unwrap(),
- ArrayId::Main,
- ArrayId::QKV,
- &compilation_config.descriptor_mlp,
- );
-
- let qk_norm: Option<Box<dyn EncodableWithState>> =
- if attention_config.query_norm_config.is_some()
- || attention_config.key_norm_config.is_some()
- {
- match QKNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- attention_config.query_norm_config.clone(),
- attention_config.key_norm_config.clone(),
- ArrayId::QKV,
- &decoder_layer_loader.subtree("mixer").unwrap(),
- num_heads,
- num_groups,
- head_dim,
- ) {
- Ok(qk_norm) => Some(Box::new(qk_norm)
- as Box<dyn EncodableWithState>),
- Err(e) => panic!(
- "Failed to create QK norm kernel for layer {}: {:?}",
- layer_index, e
- ),
- }
- } else {
- None
- };
-
- let out_projection = transformer_layer::linear_block(
- &attention_config.out_projection_config,
- attention_config.has_out_biases,
- num_heads * head_dim,
- [model_dim],
- mtl_context,
- &decoder_layer_loader
- .subtree("mixer.out_projection")
- .unwrap(),
- ArrayId::AttentionOutput,
- ArrayId::Main,
- &compilation_config.descriptor_mlp,
- );
-
- let attention = Box::new(
- AttentionKernelEncodable::new(
- mtl_context,
- kernel_data_type,
- layer_index,
- attention_scale,
- attention_config.has_sinks,
- )
- .expect(
- "Failed to create AttentionWrapper with Metal kernel",
- ),
- );
-
- MixerExecutables::Attention {
- qkv_projection,
- qk_norm,
- rope: rope_block,
- attention,
- out_projection,
- }
- },
- MixerConfig::Mamba(mamba_config) => {
- let mixer: Box<dyn EncodableWithState> =
- Box::new(MambaMixerEncodable::new(
- mtl_context,
- layer_type.clone(),
- mamba_config.clone(),
- compilation_config.clone(),
- layer_index,
- model_dim,
- num_heads,
- head_dim,
- num_groups,
- decoder_layer_loader,
- ));
- MixerExecutables::StateSpace {
- mixer,
- }
- },
- };
-
- let post_attention_norm: Option<Box<dyn EncodableWithState>> =
- if let Some(norm_config) =
- &layer_config.post_attention_norm_config
- {
- Some(Box::new(
- RMSNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader
- .subtree("post_mixer_norm")
- .unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- ))
- } else {
- None
- };
-
- let main_shortcut_add_swap: Box<dyn EncodableWithState> = Box::new(
- TensorAddSwap::new(
- mtl_context,
- kernel_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
- )
- .unwrap(),
- );
-
- let pre_mlp_norm: Box<dyn EncodableWithState> = Box::new(
- RMSNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- layer_config.pre_mlp_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- );
-
- let mlp = transformer_layer::mlp_block(
- &layer_config.mlp_config,
- model_dim,
- hidden_dim,
- mtl_context,
- &decoder_layer_loader.subtree("mlp").unwrap(),
- &compilation_config.descriptor_mlp,
- );
-
- let post_mlp_norm: Option<Box<dyn EncodableWithState>> =
- if let Some(norm_config) = &layer_config.post_mlp_norm_config {
- Some(Box::new(
- RMSNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader
- .subtree("post_mlp_norm")
- .unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- ))
- } else {
- None
- };
-
- Self {
- layer_index,
- copy_main_to_shortcut,
- pre_attention_norm,
- mixer,
- post_attention_norm,
- main_shortcut_add_swap,
- pre_mlp_norm,
- mlp,
- post_mlp_norm,
- }
- })
- }
-}
-
-impl LayerExecutables {
- fn encode_with_shared_encoder_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
- ) {
- let layer_traces = if let Some(traces) = state.traces.clone() {
- traces.borrow().layer_results.get(self.layer_index).cloned()
- } else {
- None
- };
-
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().inputs.clone(),
- );
- }
-
- self.copy_main_to_shortcut
- .encode_with_shared_encoder(state, encoder, parameters);
-
- self.pre_attention_norm
- .encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().pre_attention_norm.clone(),
- );
- }
-
- match &self.mixer {
- MixerExecutables::Attention {
- qkv_projection,
- qk_norm,
- rope,
- attention,
- out_projection,
- } => {
- qkv_projection
- .encode_with_shared_encoder(state, encoder, parameters);
- if let Some(norm) = qk_norm {
- norm.encode_with_shared_encoder(state, encoder, parameters);
- }
- rope.encode_with_shared_encoder(state, encoder, parameters);
- attention
- .encode_with_shared_encoder(state, encoder, parameters);
- out_projection
- .encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().attention.clone(),
- );
- }
- },
- MixerExecutables::StateSpace {
- mixer,
- } => {
- mixer.encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().attention.clone(),
- );
- }
- },
- }
-
- if let Some(post_attention_norm) = &self.post_attention_norm {
- post_attention_norm
- .encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().post_attention_norm.clone(),
- );
- }
- }
-
- self.main_shortcut_add_swap
- .encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().mlp_inputs.clone(),
- );
- }
-
- self.pre_mlp_norm
- .encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().pre_mlp_norm.clone(),
- );
- }
-
- self.mlp.encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(ArrayId::Main, layer_traces.borrow().mlp.clone());
- }
-
- if let Some(post_mlp_norm) = &self.post_mlp_norm {
- post_mlp_norm
- .encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().post_mlp_norm.clone(),
- );
- }
- }
-
- self.main_shortcut_add_swap
- .encode_with_shared_encoder(state, encoder, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().outputs.clone(),
- );
- }
- }
-
- fn all_blocks_support_shared_encoder(&self) -> bool {
- let mixer_supports = match &self.mixer {
- MixerExecutables::Attention {
- qkv_projection,
- qk_norm,
- rope,
- attention,
- out_projection,
- } => {
- qkv_projection.supports_shared_encoder()
- && qk_norm
- .as_ref()
- .map_or(true, |n| n.supports_shared_encoder())
- && rope.supports_shared_encoder()
- && attention.supports_shared_encoder()
- && out_projection.supports_shared_encoder()
- },
- MixerExecutables::StateSpace {
- mixer,
- } => mixer.supports_shared_encoder(),
- };
-
- self.copy_main_to_shortcut.supports_shared_encoder()
- && self.pre_attention_norm.supports_shared_encoder()
- && mixer_supports
- && self
- .post_attention_norm
- .as_ref()
- .map_or(true, |n| n.supports_shared_encoder())
- && self.main_shortcut_add_swap.supports_shared_encoder()
- && self.pre_mlp_norm.supports_shared_encoder()
- && self.mlp.supports_shared_encoder()
- && self
- .post_mlp_norm
- .as_ref()
- .map_or(true, |n| n.supports_shared_encoder())
- }
-
- fn encode_with_resolver(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let layer_traces = if let Some(traces) = state.traces.clone() {
- traces.borrow().layer_results.get(self.layer_index).cloned()
- } else {
- None
- };
-
- let mut resolver = EncoderResolver::new(command_buffer);
-
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().inputs.clone(),
- );
- }
-
- resolver.encode(self.copy_main_to_shortcut.as_ref(), state, parameters);
-
- resolver.encode(self.pre_attention_norm.as_ref(), state, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().pre_attention_norm.clone(),
- );
- }
-
- match &self.mixer {
- MixerExecutables::Attention {
- qkv_projection,
- qk_norm,
- rope,
- attention,
- out_projection,
- } => {
- resolver.encode(qkv_projection.as_ref(), state, parameters);
- if let Some(norm) = qk_norm {
- resolver.encode(norm.as_ref(), state, parameters);
- }
- resolver.encode(rope.as_ref().as_ref(), state, parameters);
- resolver.encode(attention.as_ref(), state, parameters);
- resolver.encode(out_projection.as_ref(), state, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().attention.clone(),
- );
- }
- },
- MixerExecutables::StateSpace {
- mixer,
- } => {
- resolver.encode(mixer.as_ref(), state, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().attention.clone(),
- );
- }
- },
- }
-
- if let Some(post_attention_norm) = &self.post_attention_norm {
- resolver.encode(post_attention_norm.as_ref(), state, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().post_attention_norm.clone(),
- );
- }
- }
-
- resolver.encode(
- self.main_shortcut_add_swap.as_ref(),
- state,
- parameters,
- );
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().mlp_inputs.clone(),
- );
- }
-
- resolver.encode(self.pre_mlp_norm.as_ref(), state, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().pre_mlp_norm.clone(),
- );
- }
-
- resolver.encode(self.mlp.as_ref(), state, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(ArrayId::Main, layer_traces.borrow().mlp.clone());
- }
-
- if let Some(post_mlp_norm) = &self.post_mlp_norm {
- resolver.encode(post_mlp_norm.as_ref(), state, parameters);
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().post_mlp_norm.clone(),
- );
- }
- }
-
- resolver.encode(
- self.main_shortcut_add_swap.as_ref(),
- state,
- parameters,
- );
- if let Some(layer_traces) = layer_traces.clone() {
- state.copy_array(
- ArrayId::Main,
- layer_traces.borrow().outputs.clone(),
- );
- }
-
- resolver.finish();
- }
-}
-
-impl EncodableWithState for LayerExecutables {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- self.encode_with_resolver(state, command_buffer, parameters);
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- self.all_blocks_support_shared_encoder()
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
- ) {
- self.encode_with_shared_encoder_impl(state, encoder, parameters);
- }
-}
diff --git a/crates/uzu/src/backends/metal/executable_builder/mod.rs b/crates/uzu/src/backends/metal/executable_builder/mod.rs
deleted file mode 100644
index 84cfd0fbc..000000000
--- a/crates/uzu/src/backends/metal/executable_builder/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-pub mod decoder_executables;
-pub mod layer_executables;
-
-pub use decoder_executables::DecoderExecutables;
diff --git a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
index 2fbe157cc..d9724e1a4 100644
--- a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
@@ -1,9 +1,9 @@
use metal::ComputeCommandEncoder;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use super::{
- ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
+use super::ForwardPassState;
+use crate::backends::metal::encodable_block::{
+ EncodableBlock, EncodingParameters,
};
pub struct EncoderResolver<'a> {
@@ -21,7 +21,7 @@ impl<'a> EncoderResolver<'a> {
pub fn encode(
&mut self,
- block: &dyn EncodableWithState,
+ block: &dyn EncodableBlock,
state: &mut ForwardPassState,
parameters: &EncodingParameters,
) {
diff --git a/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs b/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs
index bbc755c22..e8e7322e9 100644
--- a/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs
@@ -3,7 +3,7 @@ use std::{cell::RefCell, collections::HashMap};
use mpsgraph::TensorData;
use objc2::rc::Retained;
-use super::state::{ArrayId, ForwardPassState};
+use super::{ArrayId, ForwardPassState};
use crate::backends::metal::array::MetalArray;
#[derive(Clone)]
diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rs
index 26c569cc0..689986567 100644
--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs
@@ -1,24 +1,28 @@
-mod buffers;
pub mod cache_layers;
-pub mod encodable_with_state;
mod encoder_resolver;
mod io_arrays;
pub mod kv_cache_layer;
mod model_shape;
mod mpsgraph_block;
+mod scratch_buffers;
pub mod ssm_layer;
-mod state;
+pub mod state;
+#[cfg(feature = "tracing")]
pub mod traces;
-pub mod transformer_layer;
-pub use buffers::ForwardPassBuffers;
pub use cache_layers::{CacheLayer, CacheLayers};
pub use encoder_resolver::EncoderResolver;
pub use io_arrays::IOArrays;
pub use kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState};
pub use model_shape::ModelShape;
pub use mpsgraph_block::MPSGraphBlock;
+pub use scratch_buffers::ScratchBuffers;
pub use ssm_layer::SSMLayer;
pub use state::{
- ArrayId, ForwardPassState, HashMapId, RopeType, SharedBuffers,
+ ArrayCell, ArrayId, ClassifierModeState, CommonAuxBuffers,
+ EmbeddingsBuffers, ForwardPassMode, ForwardPassState, HashMapId,
+ LanguageModelGeneratorAuxBuffers, LanguageModelGeneratorModeState,
+ MoeExpertWeights, RopeBuffers, RopeType, SharedBuffers,
};
+
+pub use super::encodable_block::{EncodableBlock, EncodingParameters};
diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
index e8c7f4685..4d5ddf4f5 100644
--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
@@ -142,6 +142,14 @@ impl ModelShape {
[suffix_length, self.model_dim]
}
+ pub fn bitmask_shape(
+ &self,
+ suffix_length: usize,
+ ) -> [usize; 2] {
+ let bitmask_size = (self.vocabulary_size + 31) / 32;
+ [suffix_length, bitmask_size]
+ }
+
pub fn mlp_hidden_shape(
&self,
suffix_length: usize,
@@ -170,13 +178,6 @@ impl ModelShape {
[suffix_length, self.vocabulary_size]
}
- pub fn bitmask_shape(
- &self,
- suffix_length: usize,
- ) -> [usize; 2] {
- [suffix_length, self.vocabulary_size.div_ceil(32)]
- }
-
pub fn embeddings_input_shape(&self) -> [usize; 2] {
[self.vocabulary_size, self.model_dim]
}
@@ -214,6 +215,14 @@ impl ModelShape {
[self.num_groups, suffix_length, self.head_dim]
}
+ pub fn extracted_values_shape(
+ &self,
+ suffix_length: usize,
+ ) -> [usize; 3] {
+ // Values share the same grouping as keys (grouped by num_groups)
+ [self.num_groups, suffix_length, self.head_dim]
+ }
+
pub fn kv_cache_layer_shapes(
&self,
max_prefix_length: usize,
diff --git a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
index 52fcfef7d..b9fd49008 100644
--- a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
@@ -5,10 +5,9 @@ use mpsgraph::{
use objc2::rc::Retained;
use super::{
- encodable_with_state::EncodableWithState, io_arrays::IOArrays,
- state::ForwardPassState,
+ EncodableBlock, EncodingParameters, ForwardPassState,
+ io_arrays::IOArrays,
};
-use crate::backends::metal::forward_pass::encodable_with_state::EncodingParameters;
pub struct MPSGraphBlock {
executable: Retained<Executable>,
@@ -28,37 +27,9 @@ impl MPSGraphBlock {
arguments,
}
}
-
- // fn load(path: &Path) -> Result<Self, ExecutableBlockError> {
- // let signature =
- // from_reader(fs::File::open(path.join("signature.json"))?)?;
- // let executable =
- // Executable::from_serialized_package(path, None).unwrap();
- // let execution_descriptor = ExecutableExecutionDescriptor::new();
-
- // Ok(Self {
- // signature,
- // executable,
- // execution_descriptor,
- // })
- // }
-
- // fn save(
- // &self,
- // path: &Path,
- // ) -> Result<(), ExecutableBlockError> {
- // fs::create_dir(path)?;
-
- // self.executable.serialize_to_url(path, &SerializationDescriptor::new());
-
- // let writer = fs::File::create(path.join("signature.json"))?;
- // to_writer(writer, &self.signature)?;
-
- // Ok(())
- // }
}
-impl EncodableWithState for MPSGraphBlock {
+impl EncodableBlock for MPSGraphBlock {
fn encode(
&self,
state: &mut ForwardPassState,
@@ -103,8 +74,4 @@ impl EncodableWithState for MPSGraphBlock {
}
}
}
-
- fn supports_shared_encoder(&self) -> bool {
- false // MPSGraph uses its own encoding mechanism, not compute encoders
- }
}
diff --git a/crates/uzu/src/backends/metal/forward_pass/buffers.rs b/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs
similarity index 91%
rename from crates/uzu/src/backends/metal/forward_pass/buffers.rs
rename to crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs
index f49d93795..a74e214cc 100644
--- a/crates/uzu/src/backends/metal/forward_pass/buffers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs
@@ -10,7 +10,7 @@ use crate::{
};
#[derive(Debug)]
-pub struct ForwardPassBuffers {
+pub struct ScratchBuffers {
// 1-D
pub token_ids: MTLBuffer,
pub token_positions: MTLBuffer,
@@ -39,6 +39,7 @@ pub struct ForwardPassBuffers {
// 3-D
pub rotated_queries: MTLBuffer,
pub rotated_keys: MTLBuffer,
+ pub extracted_values: MTLBuffer,
// 2-pass attention intermediate buffers
pub attention_partials: MTLBuffer, // [num_heads * max_suffix_len * total_blocks_count * head_dim]
@@ -66,7 +67,7 @@ pub struct ForwardPassBuffers {
pub moe_block_alloc: Option<MTLBuffer>,
}
-impl ForwardPassBuffers {
+impl ScratchBuffers {
// TODO: use device arrays instead of MTLBuffers
/// Allocate the buffers with `StorageModeShared` so that they are CPU-accessible as well.
pub fn new(
@@ -103,18 +104,33 @@ impl ForwardPassBuffers {
sampling_output: alloc(&[max_suffix_len], DataType::U32),
// 2-D
- attention_window_size_to_bias: (0..model_shape.num_layers)
- .map(|layer| {
- (
- model_shape.sliding_window_length_per_layer
- [layer as usize],
- alloc(
- &[max_suffix_len, max_suffix_len + max_prefix_len],
- act_ty,
- ),
- )
- })
- .collect::<HashMap<Option<usize>, MTLBuffer>>(),
+ attention_window_size_to_bias: {
+ // Collect unique window sizes across all layers
+ let unique_window_sizes: std::collections::HashSet<
+ Option<usize>,
+ > = model_shape
+ .sliding_window_length_per_layer
+ .iter()
+ .copied()
+ .collect();
+
+ // Create one buffer per unique window size
+ unique_window_sizes
+ .into_iter()
+ .map(|window_size| {
+ (
+ window_size,
+ alloc(
+ &[
+ max_suffix_len,
+ max_suffix_len + max_prefix_len,
+ ],
+ act_ty,
+ ),
+ )
+ })
+ .collect::<HashMap<Option<usize>, MTLBuffer>>()
+ },
logits: alloc(&model_shape.logits_shape(max_suffix_len), act_ty),
main: alloc(&model_shape.main_shape(max_suffix_len), act_ty),
shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty),
@@ -164,6 +180,10 @@ impl ForwardPassBuffers {
&model_shape.rotated_keys_shape(max_suffix_len),
act_ty,
),
+ extracted_values: alloc(
+ &model_shape.extracted_values_shape(max_suffix_len),
+ act_ty,
+ ),
attention_partials: alloc(&partials_shape, act_ty),
attention_sums: alloc(&sums_maxs_shape, act_ty),
diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rs
deleted file mode 100644
index ead7c014b..000000000
--- a/crates/uzu/src/backends/metal/forward_pass/state.rs
+++ /dev/null
@@ -1,1560 +0,0 @@
-#![allow(dead_code)]
-use std::{cell::RefCell, collections::HashMap, rc::Rc};
-
-use half::{bf16, f16};
-use serde::{Deserialize, Serialize};
-
-use super::{
- super::{MTLContext, MetalArray},
- buffers::ForwardPassBuffers,
- cache_layers::CacheLayers,
- model_shape::ModelShape,
-};
-use crate::{
- DataType, DeviceContext,
- backends::metal::forward_pass::traces::DecoderActivationTrace,
- config::{DecoderConfig, EmbeddingConfig, MLPConfig},
- device::array::Array,
- parameters::ParameterTree,
- session::parameter::SamplingMethod,
-};
-
-type ArrayCell = RefCell<MetalArray>;
-
-pub enum EmbeddingsBuffers {
- Tied {
- /// [vocab_size, model_dim]
- weights: ArrayCell,
- },
- Untied {
- /// [vocab_size, model_dim]
- input_weights: ArrayCell,
- /// [vocab_size, model_dim]
- output_weights: ArrayCell,
- },
- QuantizedTied {
- /// [vocab_size, model_dim]
- weights: ArrayCell,
- /// [vocab_size]
- scales: ArrayCell,
- },
- MLXSemiQuantizedUntied {
- /// [vocab_size, model_dim]
- input_weights: ArrayCell,
- /// [vocab_size, model_dim]
- packed_output_weights: ArrayCell,
- /// [vocab_size, num_groups]
- output_scales: ArrayCell,
- /// [vocab_size, num_groups]
- output_biases: ArrayCell,
- },
-}
-
-impl EmbeddingsBuffers {
- pub fn new(
- context: &MTLContext,
- embeddings_config: &EmbeddingConfig,
- model_shape: &ModelShape,
- ) -> Self {
- unsafe {
- match embeddings_config {
- EmbeddingConfig::Tied {
- common: _,
- precision: _,
- } => Self::Tied {
- weights: RefCell::new(context.array_uninitialized(
- &model_shape.embeddings_input_shape(),
- model_shape.activation_data_type(),
- )),
- },
- EmbeddingConfig::Untied {
- common: _,
- precision: _,
- } => Self::Untied {
- input_weights: RefCell::new(context.array_uninitialized(
- &model_shape.embeddings_input_shape(),
- model_shape.activation_data_type(),
- )),
- output_weights: RefCell::new(context.array_uninitialized(
- &model_shape.embeddings_output_shape(),
- model_shape.activation_data_type(),
- )),
- },
- EmbeddingConfig::QuantizedTied {
- embedding_quantization_mode,
- ..
- } => {
- let [vocab_size, model_dim] =
- model_shape.quantized_embeddings_weights_shape();
- Self::QuantizedTied {
- weights: RefCell::new(
- context.array_uninitialized(
- &[
- vocab_size,
- model_dim
- / embedding_quantization_mode
- .packing_divisor(),
- ],
- embedding_quantization_mode.storage_type(),
- ),
- ),
- scales: RefCell::new(context.array_uninitialized(
- &model_shape.quantized_embeddings_scales_shape(),
- model_shape.activation_data_type(),
- )),
- }
- },
- EmbeddingConfig::MLXQuantizedTied {
- group_size,
- embedding_quantization_mode,
- ..
- } => {
- let [vocab_size, model_dim] =
- model_shape.quantized_embeddings_weights_shape();
- let num_groups = model_dim / group_size;
- Self::QuantizedTied {
- weights: RefCell::new(
- context.array_uninitialized(
- &[
- vocab_size,
- model_dim
- / embedding_quantization_mode
- .packing_divisor(),
- ],
- embedding_quantization_mode.storage_type(),
- ),
- ),
- scales: RefCell::new(context.array_uninitialized(
- &[vocab_size, num_groups],
- model_shape.activation_data_type(),
- )),
- }
- },
- EmbeddingConfig::MLXSemiQuantizedUntied {
- group_size,
- embedding_quantization_mode,
- ..
- } => {
- let [vocab_size, model_dim] =
- model_shape.quantized_embeddings_weights_shape();
- let num_groups = model_dim / group_size;
- Self::MLXSemiQuantizedUntied {
- input_weights: RefCell::new(
- context.array_uninitialized(
- &model_shape.embeddings_input_shape(),
- model_shape.activation_data_type(),
- ),
- ),
- packed_output_weights: RefCell::new(
- context.array_uninitialized(
- &[
- vocab_size,
- model_dim
- / embedding_quantization_mode
- .packing_divisor(),
- ],
- embedding_quantization_mode.storage_type(),
- ),
- ),
- output_scales: RefCell::new(
- context.array_uninitialized(
- &[vocab_size, num_groups],
- model_shape.activation_data_type(),
- ),
- ),
- output_biases: RefCell::new(
- context.array_uninitialized(
- &[vocab_size, num_groups],
- model_shape.activation_data_type(),
- ),
- ),
- }
- },
- }
- }
- }
-
- pub fn update_data(
- &mut self,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) {
- let embeddings_tree = parameter_tree.subtree("embedding").unwrap();
- match self {
- EmbeddingsBuffers::Tied {
- weights,
- } => {
- let embeddings_view = embeddings_tree.leaf("weights").unwrap();
- weights.borrow_mut().copy_from_array(&embeddings_view);
- },
- EmbeddingsBuffers::Untied {
- input_weights,
- output_weights,
- } => {
- let mapping = vec![
- ("input_weights", input_weights),
- ("output_weights", output_weights),
- ];
- for (name, buffer) in mapping {
- let view = embeddings_tree.leaf(name).unwrap();
- buffer.borrow_mut().copy_from_array(&view);
- }
- },
- EmbeddingsBuffers::QuantizedTied {
- weights,
- scales,
- } => {
- let mapping = vec![("weights", weights), ("scales", scales)];
- for (name, buffer) in mapping {
- let view = embeddings_tree.leaf(name).unwrap();
- buffer.borrow_mut().copy_from_array(&view);
- }
- },
- EmbeddingsBuffers::MLXSemiQuantizedUntied {
- input_weights,
- packed_output_weights,
- output_scales,
- output_biases,
- } => {
- let mapping = vec![
- ("input_weights", input_weights),
- ("output_weights", packed_output_weights),
- ("output_scales", output_scales),
- ("output_biases", output_biases),
- ];
- for (name, buffer) in mapping {
- let view = embeddings_tree.leaf(name).unwrap();
- buffer.borrow_mut().copy_from_array(&view);
- }
- },
- }
- }
-}
-
-pub struct RopeBuffers {
- /// [rope_max_sequence_length, head_dim]
- pub cosines: ArrayCell,
- /// [rope_max_sequence_length, head_dim]
- pub sines: ArrayCell,
-}
-
-impl RopeBuffers {
- pub fn new(
- context: &MTLContext,
- model_shape: &ModelShape,
- ) -> Self {
- unsafe {
- let rotated_queries_shape = model_shape.rotated_queries_shape(1);
- let head_dim = rotated_queries_shape[2];
- let rope_max_sequence_length = model_shape.context_length();
-
- Self {
- cosines: RefCell::new(context.array_uninitialized(
- &[rope_max_sequence_length, head_dim],
- model_shape.activation_data_type(),
- )),
- sines: RefCell::new(context.array_uninitialized(
- &[rope_max_sequence_length, head_dim],
- model_shape.activation_data_type(),
- )),
- }
- }
- }
-
- pub fn update_data(
- &mut self,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- rope_name: String,
- ) {
- let Ok(rope_tree) = parameter_tree.subtree(rope_name.as_str()) else {
- return;
- };
-
- let cosines_view = rope_tree.leaf("cosines").unwrap();
- self.cosines.borrow_mut().copy_from_array(&cosines_view);
-
- let sines_view = rope_tree.leaf("sines").unwrap();
- self.sines.borrow_mut().copy_from_array(&sines_view);
- }
-}
-
-pub struct SharedBuffers {
- pub embeddings: EmbeddingsBuffers,
- pub global_rope: Option<RopeBuffers>,
- pub local_rope: Option<RopeBuffers>,
- pub moe_expert_weights: Option<Vec<MoeExpertWeights>>,
- pub attention_sinks: Option<Vec<ArrayCell>>,
-}
-
-pub struct MoeExpertWeights {
- pub w1: ArrayCell,
- pub w2: ArrayCell,
- pub w3: ArrayCell,
-}
-
-impl SharedBuffers {
- pub fn new(
- context: &MTLContext,
- decoder_config: &DecoderConfig,
- model_shape: &ModelShape,
- ) -> Self {
- let embeddings = EmbeddingsBuffers::new(
- context,
- &decoder_config.embedding_config,
- model_shape,
- );
-
- let global_rope = if decoder_config.global_rope_config.is_some() {
- Some(RopeBuffers::new(context, model_shape))
- } else {
- None
- };
-
- let local_rope = if decoder_config.local_rope_config.is_some() {
- Some(RopeBuffers::new(context, model_shape))
- } else {
- None
- };
-
- let moe_expert_weights = if matches!(
- decoder_config.layer_config.mlp_config,
- crate::config::MLPConfig::MixtureOfExperts(_)
- ) {
- Some(Vec::new())
- } else {
- None
- };
-
- let attention_sinks = if let Some(attention_config) =
- decoder_config.layer_config.attention_config()
- {
- if attention_config.has_sinks {
- let num_heads = decoder_config.num_heads;
- Some(
- (0..decoder_config.num_layers)
- .map(|_| unsafe {
- RefCell::new(context.array_uninitialized(
- &[num_heads],
- DataType::F32,
- ))
- })
- .collect(),
- )
- } else {
- None
- }
- } else {
- None
- };
-
- Self {
- embeddings,
- global_rope,
- local_rope,
- moe_expert_weights,
- attention_sinks,
- }
- }
-
- pub fn update_data(
- &mut self,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) {
- self.embeddings.update_data(parameter_tree);
- if let Some(global_rope) = &mut self.global_rope {
- global_rope
- .update_data(parameter_tree, String::from("global_rope"));
- }
- if let Some(local_rope) = &mut self.local_rope {
- local_rope.update_data(parameter_tree, String::from("local_rope"));
- }
-
- if let Some(sinks_vec) = &mut self.attention_sinks {
- for (layer_idx, sink_cell) in sinks_vec.iter_mut().enumerate() {
- let layer_tree = parameter_tree
- .subtree(&format!("layers.{}", layer_idx))
- .unwrap();
- let attn_tree = layer_tree.subtree("mixer").unwrap();
- let sinks_arr = attn_tree.leaf("sinks").unwrap();
- let mut dst = sink_cell.borrow_mut();
- let dst_slice = dst.as_slice_mut::<f32>().unwrap();
-
- match sinks_arr.data_type() {
- DataType::F32 => {
- let src = sinks_arr.as_slice::<f32>().unwrap();
- dst_slice.copy_from_slice(src);
- },
- DataType::BF16 => {
- let src = sinks_arr.as_slice::<bf16>().unwrap();
- for (dst_val, src_val) in
- dst_slice.iter_mut().zip(src.iter())
- {
- *dst_val = f32::from(*src_val);
- }
- },
- DataType::F16 => {
- let src = sinks_arr.as_slice::<f16>().unwrap();
- for (dst_val, src_val) in
- dst_slice.iter_mut().zip(src.iter())
- {
- *dst_val = f32::from(*src_val);
- }
- },
- other => {
- panic!(
- "Unsupported attention sink data type: {:?}",
- other
- );
- },
- }
- }
- }
- }
-}
-
-struct AuxBuffers {
- suffix_length: usize,
- /// [suffix_length, model_dim]
- main: ArrayCell,
- /// [suffix_length, model_dim]
- shortcut: ArrayCell,
- /// [suffix_length, (num_heads + 2 * num_groups) * head_dim]
- qkv: ArrayCell,
- /// [suffix_length, num_heads * head_dim]
- attention_output: ArrayCell,
- /// [suffix_length, 2 * hidden_dim]
- mlp_fused_up: ArrayCell,
- /// [suffix_length, hidden_dim]
- mlp_hidden: ArrayCell,
- /// [suffix_length, max_mamba_inproj_dim]
- ssm_inproj: Option<ArrayCell>,
- /// [suffix_length, max_conv_dim]
- ssm_packed: Option<ArrayCell>,
- /// [suffix_length + kernel_size - 1, max_conv_dim]
- ssm_conv_padded: Option<ArrayCell>,
- /// [suffix_length, num_heads, head_dim]
- ssm_x: Option<ArrayCell>,
- /// [suffix_length, num_groups, state_dim]
- ssm_b: Option<ArrayCell>,
- /// [suffix_length, num_groups, state_dim]
- ssm_c: Option<ArrayCell>,
- /// [suffix_length, num_heads]
- ssm_dt: Option<ArrayCell>,
- /// [suffix_length, num_heads, head_dim]
- ssm_z: Option<ArrayCell>,
- /// [num_heads, max_suffix_length, head_dim]
- rotated_queries: ArrayCell,
- /// [num_groups, max_suffix_length, head_dim]
- rotated_keys: ArrayCell,
- /// [num_heads * suffix_length * total_blocks_count * head_dim]
- attention_partials: ArrayCell,
- /// [num_heads * suffix_length * total_blocks_count]
- attention_sums: ArrayCell,
- /// [num_heads * suffix_length * total_blocks_count]
- attention_maxs: ArrayCell,
-
- moe_topk_ids: Option<ArrayCell>,
- moe_topk_probs: Option<ArrayCell>,
- moe_offsets: Option<ArrayCell>,
- moe_sumk: Option<ArrayCell>,
- moe_bucketed_token_ids: Option<ArrayCell>,
- moe_bucketed_probs: Option<ArrayCell>,
- moe_x_perm: Option<ArrayCell>,
- moe_tok2row: Option<ArrayCell>,
- moe_y_partial: Option<ArrayCell>,
- moe_hidden: Option<ArrayCell>,
- moe_two_pass_row_expert_map: Option<ArrayCell>,
- moe_tile_counts: Option<ArrayCell>,
- moe_tile_offsets: Option<ArrayCell>,
- moe_tile_map: Option<ArrayCell>,
- moe_total_tiles: Option<ArrayCell>,
- moe_dispatch_args: Option<ArrayCell>,
- moe_scatter_partials: Option<ArrayCell>,
- moe_scatter_block_bases: Option<ArrayCell>,
- moe_block_alloc: Option<ArrayCell>,
-}
-
-impl AuxBuffers {
- fn new(
- scratch: &ForwardPassBuffers,
- decoder_config: &DecoderConfig,
- model_shape: &ModelShape,
- suffix_length: usize,
- ) -> Self {
- let act_dtype = model_shape.activation_data_type();
- unsafe {
- Self {
- suffix_length,
- main: RefCell::new(MetalArray::new(
- scratch.main.clone(),
- &model_shape.main_shape(suffix_length),
- act_dtype,
- )),
- shortcut: RefCell::new(MetalArray::new(
- scratch.shortcut.clone(),
- &model_shape.main_shape(suffix_length),
- act_dtype,
- )),
- qkv: RefCell::new(MetalArray::new(
- scratch.qkv.clone(),
- &model_shape.qkv_shape(suffix_length),
- act_dtype,
- )),
- attention_output: RefCell::new(MetalArray::new(
- scratch.attention_output.clone(),
- &model_shape.attention_output_shape(suffix_length),
- act_dtype,
- )),
- mlp_fused_up: RefCell::new(MetalArray::new(
- scratch.mlp_fused_up.clone(),
- &model_shape.mlp_fused_up_shape(suffix_length),
- act_dtype,
- )),
- mlp_hidden: RefCell::new(MetalArray::new(
- scratch.mlp_hidden.clone(),
- &model_shape.mlp_hidden_shape(suffix_length),
- act_dtype,
- )),
- ssm_inproj: match (
- scratch.ssm_inproj.as_ref(),
- model_shape.ssm_inproj_shape(suffix_length),
- ) {
- (Some(buf), Some(shape)) => Some(RefCell::new(
- MetalArray::new(buf.clone(), &shape, act_dtype),
- )),
- _ => None,
- },
- ssm_packed: match (
- scratch.ssm_packed.as_ref(),
- model_shape.ssm_packed_shape(suffix_length),
- ) {
- (Some(buf), Some(shape)) => Some(RefCell::new(
- MetalArray::new(buf.clone(), &shape, act_dtype),
- )),
- _ => None,
- },
- ssm_conv_padded: match (
- scratch.ssm_conv_padded.as_ref(),
- model_shape.ssm_conv_padded_shape(suffix_length),
- ) {
- (Some(buf), Some(shape)) => Some(RefCell::new(
- MetalArray::new(buf.clone(), &shape, act_dtype),
- )),
- _ => None,
- },
- ssm_x: match (
- scratch.ssm_x.as_ref(),
- model_shape.ssm_x_shape(suffix_length),
- ) {
- (Some(buf), Some(shape)) => Some(RefCell::new(
- MetalArray::new(buf.clone(), &shape, act_dtype),
- )),
- _ => None,
- },
- ssm_b: match (
- scratch.ssm_b.as_ref(),
- model_shape.ssm_bc_shape(suffix_length),
- ) {
- (Some(buf), Some(shape)) => Some(RefCell::new(
- MetalArray::new(buf.clone(), &shape, act_dtype),
- )),
- _ => None,
- },
- ssm_c: match (
- scratch.ssm_c.as_ref(),
- model_shape.ssm_bc_shape(suffix_length),
- ) {
- (Some(buf), Some(shape)) => Some(RefCell::new(
- MetalArray::new(buf.clone(), &shape, act_dtype),
- )),
- _ => None,
- },
- ssm_dt: match (
- scratch.ssm_dt.as_ref(),
- model_shape.ssm_dt_shape(suffix_length),
- ) {
- (Some(buf), Some(shape)) => Some(RefCell::new(
- MetalArray::new(buf.clone(), &shape, act_dtype),
- )),
- _ => None,
- },
- ssm_z: match (
- scratch.ssm_z.as_ref(),
- model_shape.ssm_z_shape(suffix_length),
- ) {
- (Some(buf), Some(shape)) => Some(RefCell::new(
- MetalArray::new(buf.clone(), &shape, act_dtype),
- )),
- _ => None,
- },
- rotated_queries: RefCell::new(MetalArray::new(
- scratch.rotated_queries.clone(),
- &model_shape.rotated_queries_shape(suffix_length),
- act_dtype,
- )),
- rotated_keys: RefCell::new(MetalArray::new(
- scratch.rotated_keys.clone(),
- &model_shape.rotated_keys_shape(suffix_length),
- act_dtype,
- )),
- attention_partials: RefCell::new(MetalArray::new(
- scratch.attention_partials.clone(),
- &model_shape.attention_partials_shape(suffix_length),
- act_dtype,
- )),
- attention_sums: RefCell::new(MetalArray::new(
- scratch.attention_sums.clone(),
- &model_shape.attention_sums_shape(suffix_length),
- act_dtype,
- )),
- attention_maxs: RefCell::new(MetalArray::new(
- scratch.attention_maxs.clone(),
- &model_shape.attention_maxs_shape(suffix_length),
- act_dtype,
- )),
-
- moe_topk_ids: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- scratch.moe_topk_ids.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_topk_ids_shape(
- suffix_length,
- moe.num_experts_per_token,
- ),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_topk_probs: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- scratch.moe_topk_probs.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_topk_probs_shape(
- suffix_length,
- moe.num_experts_per_token,
- ),
- act_dtype,
- ))
- })
- },
- _ => None,
- },
- moe_offsets: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- scratch.moe_offsets.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape
- .moe_offsets_shape(moe.mixture_size),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_sumk: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(_) => {
- scratch.moe_sumk.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_sumk_shape(),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_bucketed_token_ids: match &decoder_config
- .layer_config
- .mlp_config
- {
- MLPConfig::MixtureOfExperts(moe) => {
- let max_routed =
- suffix_length * moe.num_experts_per_token;
- scratch.moe_bucketed_token_ids.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape
- .moe_bucketed_token_ids_shape(max_routed),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_bucketed_probs: match &decoder_config
- .layer_config
- .mlp_config
- {
- MLPConfig::MixtureOfExperts(moe) => {
- let max_routed =
- suffix_length * moe.num_experts_per_token;
- scratch.moe_bucketed_probs.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape
- .moe_bucketed_probs_shape(max_routed),
- act_dtype,
- ))
- })
- },
- _ => None,
- },
- moe_x_perm: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- let max_routed =
- suffix_length * moe.num_experts_per_token;
- scratch.moe_x_perm.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_x_perm_shape(max_routed),
- DataType::F16,
- ))
- })
- },
- _ => None,
- },
- moe_tok2row: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- scratch.moe_tok2row.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_tok2row_shape(
- suffix_length,
- moe.num_experts_per_token,
- ),
- DataType::I32,
- ))
- })
- },
- _ => None,
- },
- moe_y_partial: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- let max_routed =
- suffix_length * moe.num_experts_per_token;
- scratch.moe_y_partial.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_y_partial_shape(max_routed),
- DataType::F16,
- ))
- })
- },
- _ => None,
- },
- moe_hidden: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- let max_routed =
- suffix_length * moe.num_experts_per_token;
- scratch.moe_hidden.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_hidden_shape(max_routed),
- act_dtype,
- ))
- })
- },
- _ => None,
- },
- moe_two_pass_row_expert_map: match &decoder_config
- .layer_config
- .mlp_config
- {
- MLPConfig::MixtureOfExperts(moe) => {
- let max_routed =
- suffix_length * moe.num_experts_per_token;
- scratch.moe_two_pass_row_expert_map.as_ref().map(
- |buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &[max_routed],
- DataType::U32,
- ))
- },
- )
- },
- _ => None,
- },
- moe_tile_counts: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- scratch.moe_tile_counts.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_counts_shape(moe.mixture_size),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_tile_offsets: match &decoder_config.layer_config.mlp_config
- {
- MLPConfig::MixtureOfExperts(moe) => {
- scratch.moe_tile_offsets.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape
- .moe_offsets_shape(moe.mixture_size),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_tile_map: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- let max_routed =
- suffix_length * moe.num_experts_per_token;
- scratch.moe_tile_map.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_tile_map_shape(max_routed),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_total_tiles: match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(_) => {
- scratch.moe_total_tiles.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_total_tiles_shape(),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_dispatch_args: match &decoder_config.layer_config.mlp_config
- {
- MLPConfig::MixtureOfExperts(_) => {
- scratch.moe_dispatch_args.as_ref().map(|buf| {
- RefCell::new(MetalArray::new(
- buf.clone(),
- &model_shape.moe_dispatch_args_shape(),
- DataType::U32,
- ))
- })
- },
- _ => None,
- },
- moe_scatter_partials: scratch
- .moe_scatter_partials
- .as_ref()
- .map(|buf| {
- let num_blocks = ((suffix_length + 255) / 256).max(1);
- match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- let num_tiles =
- ((moe.mixture_size + 512 - 1) / 512).max(1);
- let entries = num_blocks * num_tiles * 512;
- RefCell::new(MetalArray::new(
- buf.clone(),
- &[entries],
- DataType::U32,
- ))
- },
- _ => unreachable!(),
- }
- }),
- moe_scatter_block_bases: scratch
- .moe_scatter_block_bases
- .as_ref()
- .map(|buf| {
- let num_blocks = ((suffix_length + 255) / 256).max(1);
- match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- let num_tiles =
- ((moe.mixture_size + 512 - 1) / 512).max(1);
- let entries = num_blocks * num_tiles * 512;
- RefCell::new(MetalArray::new(
- buf.clone(),
- &[entries],
- DataType::U32,
- ))
- },
- _ => unreachable!(),
- }
- }),
- moe_block_alloc: scratch.moe_block_alloc.as_ref().map(|buf| {
- let num_blocks = ((suffix_length + 255) / 256).max(1);
- match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => {
- let num_tiles =
- ((moe.mixture_size + 512 - 1) / 512).max(1);
- let entries = num_blocks * num_tiles * 512;
- RefCell::new(MetalArray::new(
- buf.clone(),
- &[entries],
- DataType::U32,
- ))
- },
- _ => unreachable!(),
- }
- }),
- }
- }
- }
-}
-
-pub struct ForwardPassState {
- context: Rc<MTLContext>,
- /// [suffix_length] - u64
- token_ids: ArrayCell,
- /// [suffix_length] - i32
- token_positions: ArrayCell,
- /// [suffix_length, vocabulary_size] - bitmask packed in u32 (0 = disallowed, 1 = allowed)
- token_bitmask: Option<ArrayCell>,
- /// [suffix_length] - u64
- token_seeds: ArrayCell,
- /// [suffix_length, suffix_length + prefix_length]
- attention_bias: HashMap<Option<usize>, ArrayCell>,
- /// [suffix_length, vocabulary_size]
- logits: ArrayCell,
- pub cache_layers: Rc<RefCell<CacheLayers>>,
- pub shared_buffers: Rc<RefCell<SharedBuffers>>,
- aux_buffers: AuxBuffers,
- /// [suffix_length] - u32 sampling output buffer
- pub sampling_output: Option<ArrayCell>,
- /// Current sampling configuration for this forward pass
- pub sampling_method: Option<SamplingMethod>,
- pub traces: Option<Rc<RefCell<DecoderActivationTrace>>>,
- active_suffix_length: usize,
- is_prefilling: bool,
-}
-
-impl ForwardPassState {
- pub fn new(
- context: Rc<MTLContext>,
- decoder_config: &DecoderConfig,
- model_shape: &ModelShape,
- scratch: &ForwardPassBuffers,
- cache_layers: Rc<RefCell<CacheLayers>>,
- shared_buffers: Rc<RefCell<SharedBuffers>>,
- token_ids: &[u64],
- token_positions: &[usize],
- token_bitmask: Option<&[u32]>,
- token_seeds: &[u64],
- active_suffix_length: usize,
- is_prefilling: bool,
- trace: bool,
- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
- skip_token_ids_copy: bool,
- async_positions: Option<(&metal::Buffer, usize)>,
- async_seeds: Option<(&metal::Buffer, usize)>,
- ) -> Self {
- let suffix_length = token_ids.len();
- assert_eq!(
- suffix_length,
- token_positions.len(),
- "Tokens and token positions must have the same length, got {} and {} respectively",
- suffix_length,
- token_positions.len()
- );
- assert!(
- suffix_length <= cache_layers.borrow().max_suffix_length(),
- "KV cache size can only accomodate a suffix of length up to {}, but tried to use a suffix of length {}",
- cache_layers.borrow().max_suffix_length(),
- suffix_length
- );
- assert!(
- active_suffix_length <= suffix_length,
- "Active suffix length ({active_suffix_length}) must be <= suffix length ({suffix_length})"
- );
- let aux_buffers = AuxBuffers::new(
- scratch,
- decoder_config,
- model_shape,
- suffix_length,
- );
-
- // --------------------
- // Token IDs
- // --------------------
- let token_ids_refcell = {
- let mut token_ids_array = unsafe {
- MetalArray::new(
- scratch.token_ids.clone(),
- &[suffix_length],
- DataType::U64,
- )
- };
- if !skip_token_ids_copy {
- context.copy_from_view(&mut token_ids_array, token_ids.into());
- }
- RefCell::new(token_ids_array)
- };
-
- // --------------------
- // Token Positions (i32)
- // --------------------
- let token_positions_refcell =
- if let Some((buffer, offset)) = async_positions {
- // Use pre-allocated async positions buffer at offset
- let token_positions_array = unsafe {
- MetalArray::new_with_offset(
- buffer.clone(),
- &[suffix_length],
- DataType::I32,
- offset * std::mem::size_of::<i32>(),
- )
- };
- RefCell::new(token_positions_array)
- } else {
- // Copy positions from CPU
- let mut token_positions_array = unsafe {
- MetalArray::new(
- scratch.token_positions.clone(),
- &[suffix_length],
- DataType::I32,
- )
- };
- let token_positions_i32: Box<[i32]> =
- token_positions.iter().map(|p| *p as i32).collect();
- context.copy_from_view(
- &mut token_positions_array,
- token_positions_i32.as_ref().into(),
- );
- RefCell::new(token_positions_array)
- };
-
- // --------------------
- // Token Seeds
- // --------------------
- let token_seeds_refcell = if let Some((buffer, offset)) = async_seeds {
- // Use pre-allocated async seeds buffer at offset
- let token_seeds_array = unsafe {
- MetalArray::new_with_offset(
- buffer.clone(),
- &[suffix_length],
- DataType::U64,
- offset * std::mem::size_of::<u64>(),
- )
- };
- RefCell::new(token_seeds_array)
- } else {
- // Copy seeds from CPU
- let mut token_seeds_array = unsafe {
- MetalArray::new(
- scratch.token_seeds.clone(),
- &[suffix_length],
- DataType::U64,
- )
- };
- context.copy_from_view(&mut token_seeds_array, token_seeds.into());
- RefCell::new(token_seeds_array)
- };
-
- // --------------------
- // Token Bitmask
- // --------------------
- let token_bitmask_refcell = if let Some(token_bitmask) = token_bitmask {
- let mut token_bitmask_array = unsafe {
- MetalArray::new(
- scratch.token_bitmask.clone(),
- &model_shape.bitmask_shape(suffix_length),
- DataType::U32,
- )
- };
- context
- .copy_from_view(&mut token_bitmask_array, token_bitmask.into());
- Some(RefCell::new(token_bitmask_array))
- } else {
- None
- };
-
- // --------------------
- // Attention Bias
- // --------------------
-
- let act_dtype = model_shape.activation_data_type();
-
- let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =
- scratch
- .attention_window_size_to_bias
- .iter()
- .map(|(window_size, buffer)| {
- let prefix_length = window_size
- .unwrap_or(cache_layers.borrow().max_prefix_length());
- let attention_bias_shape =
- [suffix_length, suffix_length + prefix_length];
-
- let array = unsafe {
- MetalArray::new(
- buffer.clone(),
- &attention_bias_shape,
- act_dtype,
- )
- };
- (*window_size, array)
- })
- .collect();
-
- cache_layers.borrow().fill_attention_bias(
- &mut attention_bias_map,
- token_positions,
- suffix_length,
- &context,
- external_bias_fn,
- );
-
- let attention_bias: HashMap<Option<usize>, ArrayCell> =
- attention_bias_map
- .into_iter()
- .map(|(k, v)| (k, RefCell::new(v)))
- .collect();
-
- // --------------------
- // Logits
- // --------------------
- let logits = RefCell::new(unsafe {
- MetalArray::new(
- scratch.logits.clone(),
- &model_shape.logits_shape(suffix_length),
- act_dtype,
- )
- });
-
- let sampling_output = RefCell::new(unsafe {
- MetalArray::new(
- scratch.sampling_output.clone(),
- &[suffix_length],
- DataType::U32,
- )
- });
-
- let traces = if trace {
- Some(Rc::new(RefCell::new(DecoderActivationTrace::new(
- &context,
- model_shape,
- suffix_length,
- ))))
- } else {
- None
- };
-
- Self {
- context,
- token_ids: token_ids_refcell,
- token_positions: token_positions_refcell,
- token_bitmask: token_bitmask_refcell,
- token_seeds: token_seeds_refcell,
- attention_bias,
- logits,
- cache_layers,
- shared_buffers,
- aux_buffers,
- sampling_output: Some(sampling_output),
- sampling_method: None,
- traces,
- active_suffix_length,
- is_prefilling,
- }
- }
-
- pub fn has_bitmask(&self) -> bool {
- self.token_bitmask.is_some()
- }
-
- fn array_cell(
- &self,
- id: ArrayId,
- ) -> RefCell<MetalArray> {
- match id {
- ArrayId::TokenIds => self.token_ids.clone(),
- ArrayId::TokenPositions => self.token_positions.clone(),
- ArrayId::TokenBitmask => self.token_bitmask.as_ref().unwrap().clone(),
- ArrayId::TokenSeeds => self.token_seeds.clone(),
- ArrayId::Logits => self.logits.clone(),
- ArrayId::Main => self.aux_buffers.main.clone(),
- ArrayId::Shortcut => self.aux_buffers.shortcut.clone(),
- ArrayId::QKV => self.aux_buffers.qkv.clone(),
- ArrayId::AttentionOutput => {
- self.aux_buffers.attention_output.clone()
- },
- ArrayId::MlpFusedUp => self.aux_buffers.mlp_fused_up.clone(),
- ArrayId::MlpHidden => self.aux_buffers.mlp_hidden.clone(),
- ArrayId::SsmInProj => self
- .aux_buffers
- .ssm_inproj
- .as_ref()
- .expect("SSM in-projection buffer requested but not initialized")
- .clone(),
- ArrayId::Keys(layer_index) => {
- let cache = self.cache_layers.borrow();
- match cache.data[layer_index].as_transformer() {
- Some(layer) => layer.keys.clone(),
- None => panic!(
- "Requested transformer keys for non-transformer layer {}",
- layer_index
- ),
- }
- },
- ArrayId::Values(layer_index) => {
- let cache = self.cache_layers.borrow();
- match cache.data[layer_index].as_transformer() {
- Some(layer) => layer.values.clone(),
- None => panic!(
- "Requested transformer values for non-transformer layer {}",
- layer_index
- ),
- }
- },
- ArrayId::SsmConvState(layer_index) => {
- let cache = self.cache_layers.borrow();
- match cache.data[layer_index].as_state_space() {
- Some(layer) => layer.conv_state.clone(),
- None => panic!(
- "Requested SSM conv state for transformer layer {}",
- layer_index
- ),
- }
- },
- ArrayId::SsmState(layer_index) => {
- let cache = self.cache_layers.borrow();
- match cache.data[layer_index].as_state_space() {
- Some(layer) => layer.ssm_state.clone(),
- None => panic!(
- "Requested SSM state for transformer layer {}",
- layer_index
- ),
- }
- },
- ArrayId::SsmPacked(layer_index) => {
- let _ = layer_index;
- self.aux_buffers
- .ssm_packed
- .as_ref()
- .expect("SSM packed buffer not initialized")
- .clone()
- },
- ArrayId::SsmX(layer_index) => {
- let _ = layer_index;
- self.aux_buffers
- .ssm_x
- .as_ref()
- .expect("SSM x buffer not initialized")
- .clone()
- },
- ArrayId::SsmB(layer_index) => {
- let _ = layer_index;
- self.aux_buffers
- .ssm_b
- .as_ref()
- .expect("SSM b buffer not initialized")
- .clone()
- },
- ArrayId::SsmC(layer_index) => {
- let _ = layer_index;
- self.aux_buffers
- .ssm_c
- .as_ref()
- .expect("SSM c buffer not initialized")
- .clone()
- },
- ArrayId::SsmDt(layer_index) => {
- let _ = layer_index;
- self.aux_buffers
- .ssm_dt
- .as_ref()
- .expect("SSM dt buffer not initialized")
- .clone()
- },
- ArrayId::SsmZ(layer_index) => {
- let _ = layer_index;
- self.aux_buffers
- .ssm_z
- .as_ref()
- .expect("SSM z buffer not initialized")
- .clone()
- },
- ArrayId::RotatedQueries => self.aux_buffers.rotated_queries.clone(),
- ArrayId::RotatedKeys => self.aux_buffers.rotated_keys.clone(),
- ArrayId::AttentionPartials => {
- self.aux_buffers.attention_partials.clone()
- },
- ArrayId::AttentionSums => self.aux_buffers.attention_sums.clone(),
- ArrayId::AttentionMaxs => self.aux_buffers.attention_maxs.clone(),
-
- ArrayId::EmbeddingsInputWeights => {
- match &self.shared_buffers.borrow().embeddings {
- EmbeddingsBuffers::Tied {
- weights,
- } => weights.clone(),
- EmbeddingsBuffers::Untied {
- input_weights,
- ..
- } => input_weights.clone(),
- EmbeddingsBuffers::QuantizedTied {
- weights,
- ..
- } => weights.clone(),
- EmbeddingsBuffers::MLXSemiQuantizedUntied {
- input_weights,
- ..
- } => input_weights.clone(),
- }
- },
- ArrayId::EmbeddingsOutputWeights => {
- match &self.shared_buffers.borrow().embeddings {
- EmbeddingsBuffers::Tied {
- weights,
- } => weights.clone(),
- EmbeddingsBuffers::Untied {
- output_weights,
- ..
- } => output_weights.clone(),
- EmbeddingsBuffers::QuantizedTied {
- weights,
- ..
- } => weights.clone(),
- EmbeddingsBuffers::MLXSemiQuantizedUntied {
- packed_output_weights,
- ..
- } => packed_output_weights.clone(),
- }
- },
- ArrayId::EmbeddingsScales => {
- match &self.shared_buffers.borrow().embeddings {
- EmbeddingsBuffers::QuantizedTied {
- scales,
- ..
- } => scales.clone(),
- EmbeddingsBuffers::MLXSemiQuantizedUntied {
- output_scales,
- ..
- } => output_scales.clone(),
- _ => panic!(
- "Expected EmbeddingsBuffers::QuantizedTied or MLXSemiQuantizedUntied"
- ),
- }
- },
- ArrayId::RopeCosines(rope_type) => match rope_type {
- RopeType::Global => {
- let shared_buffers = self.shared_buffers.borrow();
- shared_buffers
- .global_rope
- .as_ref()
- .expect("Global rope requested but not initialized")
- .cosines
- .clone()
- },
- RopeType::Local => self
- .shared_buffers
- .borrow()
- .local_rope
- .as_ref()
- .expect("Local rope requested but local_rope buffers are not initialized")
- .cosines
- .clone(),
- },
- ArrayId::RopeSines(rope_type) => match rope_type {
- RopeType::Global => {
- let shared_buffers = self.shared_buffers.borrow();
- shared_buffers
- .global_rope
- .as_ref()
- .expect("Global rope requested but not initialized")
- .sines
- .clone()
- },
- RopeType::Local => self
- .shared_buffers
- .borrow()
- .local_rope
- .as_ref()
- .expect("Local rope requested but local_rope buffers are not initialized")
- .sines
- .clone(),
- },
-
- ArrayId::MoeTopkIds => self.aux_buffers.moe_topk_ids.as_ref().expect("MoE topk ids buffer not initialized").clone(),
- ArrayId::MoeTopkProbs => self.aux_buffers.moe_topk_probs.as_ref().expect("MoE topk probs buffer not initialized").clone(),
- ArrayId::MoeOffsets => self.aux_buffers.moe_offsets.as_ref().expect("MoE offsets buffer not initialized").clone(),
- ArrayId::MoeSumK => self.aux_buffers.moe_sumk.as_ref().expect("MoE sumk buffer not initialized").clone(),
- ArrayId::MoeBucketedTokenIds => self.aux_buffers.moe_bucketed_token_ids.as_ref().expect("MoE bucketed token ids buffer not initialized").clone(),
- ArrayId::MoeBucketedProbs => self.aux_buffers.moe_bucketed_probs.as_ref().expect("MoE bucketed probs buffer not initialized").clone(),
- ArrayId::MoeXPerm => self.aux_buffers.moe_x_perm.as_ref().expect("MoE x_perm buffer not initialized").clone(),
- ArrayId::MoeTok2Row => self.aux_buffers.moe_tok2row.as_ref().expect("MoE tok2row buffer not initialized").clone(),
- ArrayId::MoeYPartial => self.aux_buffers.moe_y_partial.as_ref().expect("MoE y_partial buffer not initialized").clone(),
- ArrayId::MoeHidden => self.aux_buffers.moe_hidden.as_ref().expect("MoE hidden buffer not initialized").clone(),
- ArrayId::MoeTwoPassRowExpertMap => self.aux_buffers.moe_two_pass_row_expert_map.as_ref().expect("MoE two-pass row expert map buffer not initialized").clone(),
- ArrayId::MoeTileCounts => self.aux_buffers.moe_tile_counts.as_ref().expect("MoE tile counts buffer not initialized").clone(),
- ArrayId::MoeTileOffsets => self.aux_buffers.moe_tile_offsets.as_ref().expect("MoE tile offsets buffer not initialized").clone(),
- ArrayId::MoeTileMap => self.aux_buffers.moe_tile_map.as_ref().expect("MoE tile map buffer not initialized").clone(),
- ArrayId::MoeTotalTiles => self.aux_buffers.moe_total_tiles.as_ref().expect("MoE total tiles buffer not initialized").clone(),
- ArrayId::MoeDispatchArgs => self.aux_buffers.moe_dispatch_args.as_ref().expect("MoE dispatch args buffer not initialized").clone(),
- ArrayId::MoeScatterPartials => self.aux_buffers.moe_scatter_partials.as_ref().expect("MoE scatter partials buffer not initialized").clone(),
- ArrayId::MoeScatterBlockBases => self.aux_buffers.moe_scatter_block_bases.as_ref().expect("MoE scatter block bases buffer not initialized").clone(),
- ArrayId::MoeBlockAlloc => self.aux_buffers.moe_block_alloc.as_ref().expect("MoE block alloc buffer not initialized").clone(),
-
- ArrayId::AttentionSinks(layer_index) => {
- self.shared_buffers.borrow().attention_sinks.as_ref()
- .expect("Attention sinks not initialized")[layer_index].clone()
- },
- }
- }
-
- pub fn hashmap_cell(
- &self,
- id: &HashMapId,
- ) -> RefCell<HashMap<Option<usize>, ArrayCell>> {
- match id {
- HashMapId::AttentionBias => {
- RefCell::new(self.attention_bias.clone())
- },
- }
- }
-
- pub fn hashmaps(
- &self,
- ids: &[HashMapId],
- ) -> Box<[HashMap<Option<usize>, ArrayCell>]> {
- ids.iter().map(|id| self.hashmap_cell(id).into_inner()).collect()
- }
-
- pub fn arrays(
- &self,
- ids: &[ArrayId],
- ) -> Box<[ArrayCell]> {
- ids.iter().map(|id| self.array_cell(*id).clone()).collect()
- }
-
- pub fn copy_array(
- &self,
- source_array_id: ArrayId,
- destination_array: RefCell<MetalArray>,
- ) {
- destination_array
- .borrow_mut()
- .copy_from_array(&self.arrays(&[source_array_id])[0].borrow());
- }
-
- pub fn aux_buffers_suffix_length(&self) -> usize {
- self.aux_buffers.suffix_length
- }
-
- pub fn active_suffix_length(&self) -> usize {
- self.active_suffix_length
- }
-
- pub fn is_prefilling(&self) -> bool {
- self.is_prefilling
- }
-
- pub fn mtl_context(&self) -> &Rc<MTLContext> {
- &self.context
- }
-
- pub fn conv_padded_buffer(&self) -> Option<ArrayCell> {
- self.aux_buffers.ssm_conv_padded.as_ref().map(|buf| buf.clone())
- }
-}
-
-impl Drop for ForwardPassState {
- fn drop(&mut self) {
- // Nothing extra to clean up now that heap is removed.
- }
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- PartialOrd,
- Ord,
- Hash,
- Serialize,
- Deserialize,
-)]
-pub enum RopeType {
- Global,
- Local,
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- PartialOrd,
- Ord,
- Hash,
- Serialize,
- Deserialize,
-)]
-pub enum ArrayId {
- TokenIds,
- TokenPositions,
- TokenBitmask,
- TokenSeeds,
- Logits,
-
- Main,
- Shortcut,
- QKV,
- AttentionOutput,
- MlpFusedUp,
- MlpHidden,
- SsmInProj,
-
- Keys(usize),
- Values(usize),
- SsmConvState(usize),
- SsmState(usize),
- SsmPacked(usize),
- SsmX(usize),
- SsmB(usize),
- SsmC(usize),
- SsmDt(usize),
- SsmZ(usize),
-
- RotatedQueries,
- RotatedKeys,
-
- AttentionPartials,
- AttentionSums,
- AttentionMaxs,
-
- EmbeddingsInputWeights,
- EmbeddingsOutputWeights,
- EmbeddingsScales,
- RopeCosines(RopeType),
- RopeSines(RopeType),
-
- AttentionSinks(usize),
-
- MoeTopkIds,
- MoeTopkProbs,
- MoeOffsets,
- MoeSumK,
- MoeBucketedTokenIds,
- MoeBucketedProbs,
- MoeXPerm,
- MoeTok2Row,
- MoeYPartial,
- MoeHidden,
- MoeTwoPassRowExpertMap,
- MoeTileCounts,
- MoeTileOffsets,
- MoeTileMap,
- MoeTotalTiles,
- MoeDispatchArgs,
- MoeScatterPartials,
- MoeScatterBlockBases,
- MoeBlockAlloc,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum HashMapId {
- AttentionBias,
-}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs b/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs
new file mode 100644
index 000000000..63f270eaf
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs
@@ -0,0 +1,84 @@
+use serde::{Deserialize, Serialize};
+
+use super::RopeType;
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ PartialOrd,
+ Ord,
+ Hash,
+ Serialize,
+ Deserialize,
+)]
+pub enum ArrayId {
+ TokenIds,
+ TokenPositions,
+ TokenBitmask,
+ Logits,
+ TokenSeeds,
+
+ Main,
+ Shortcut,
+ QKV,
+ AttentionOutput,
+ MlpFusedUp,
+ MlpHidden,
+ SsmInProj,
+
+ Keys(usize),
+ Values(usize),
+ SsmConvState(usize),
+ SsmState(usize),
+ SsmPacked(usize),
+ SsmX(usize),
+ SsmB(usize),
+ SsmC(usize),
+ SsmDt(usize),
+ SsmZ(usize),
+
+ RotatedQueries,
+ RotatedKeys,
+ ExtractedValues,
+
+ AttentionPartials,
+ AttentionSums,
+ AttentionMaxs,
+
+ EmbeddingsInputWeights,
+ EmbeddingsOutputWeights,
+ EmbeddingsScales,
+ RopeCosines(RopeType),
+ RopeSines(RopeType),
+
+ AttentionSinks(usize),
+
+ MoeTopkIds,
+ MoeTopkProbs,
+ MoeOffsets,
+ MoeSumK,
+ MoeBucketedTokenIds,
+ MoeBucketedProbs,
+ MoeXPerm,
+ MoeTok2Row,
+ MoeYPartial,
+ MoeHidden,
+ MoeTwoPassRowExpertMap,
+ MoeTileCounts,
+ MoeTileOffsets,
+ MoeTileMap,
+ MoeTotalTiles,
+ MoeDispatchArgs,
+ MoeScatterPartials,
+ MoeScatterBlockBases,
+ MoeBlockAlloc,
+
+ // Classifier prediction head buffers
+ ClassifierPooling,
+ ClassifierPredictionHeadDense,
+ ClassifierPredictionHeadNorm,
+ ClassifierPredictionHeadLogits,
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/common_aux_buffers.rs
new file mode 100644
index 000000000..ee1412742
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/common_aux_buffers.rs
@@ -0,0 +1,101 @@
+use std::cell::RefCell;
+
+use super::super::{ModelShape, ScratchBuffers};
+use crate::backends::metal::MetalArray;
+
+type ArrayCell = RefCell<MetalArray>;
+
+pub struct CommonAuxBuffers {
+ pub suffix_length: usize,
+ pub main: ArrayCell,
+ pub shortcut: ArrayCell,
+ pub qkv: ArrayCell,
+ pub attention_output: ArrayCell,
+ pub mlp_fused_up: ArrayCell,
+ pub mlp_hidden: ArrayCell,
+ pub rotated_queries: ArrayCell,
+ pub rotated_keys: ArrayCell,
+ pub extracted_values: ArrayCell,
+ pub attention_partials: ArrayCell,
+ pub attention_sums: ArrayCell,
+ pub attention_maxs: ArrayCell,
+}
+
+impl CommonAuxBuffers {
+ pub fn new(
+ scratch: &ScratchBuffers,
+ model_shape: &ModelShape,
+ suffix_length: usize,
+ ) -> Self {
+ let act_dtype = model_shape.activation_data_type();
+ unsafe {
+ Self {
+ suffix_length,
+ main: RefCell::new(MetalArray::new(
+ scratch.main.clone(),
+ &model_shape.main_shape(suffix_length),
+ act_dtype,
+ )),
+ shortcut: RefCell::new(MetalArray::new(
+ scratch.shortcut.clone(),
+ &model_shape.main_shape(suffix_length),
+ act_dtype,
+ )),
+ qkv: RefCell::new(MetalArray::new(
+ scratch.qkv.clone(),
+ &model_shape.qkv_shape(suffix_length),
+ act_dtype,
+ )),
+ attention_output: RefCell::new(MetalArray::new(
+ scratch.attention_output.clone(),
+ &model_shape.attention_output_shape(suffix_length),
+ act_dtype,
+ )),
+ mlp_fused_up: RefCell::new(MetalArray::new(
+ scratch.mlp_fused_up.clone(),
+ &model_shape.mlp_fused_up_shape(suffix_length),
+ act_dtype,
+ )),
+ mlp_hidden: RefCell::new(MetalArray::new(
+ scratch.mlp_hidden.clone(),
+ &model_shape.mlp_hidden_shape(suffix_length),
+ act_dtype,
+ )),
+ rotated_queries: RefCell::new(MetalArray::new(
+ scratch.rotated_queries.clone(),
+ &model_shape.rotated_queries_shape(suffix_length),
+ act_dtype,
+ )),
+ rotated_keys: RefCell::new(MetalArray::new(
+ scratch.rotated_keys.clone(),
+ &model_shape.rotated_keys_shape(suffix_length),
+ act_dtype,
+ )),
+ extracted_values: RefCell::new(MetalArray::new(
+ scratch.extracted_values.clone(),
+ &model_shape.extracted_values_shape(suffix_length),
+ act_dtype,
+ )),
+ attention_partials: RefCell::new(MetalArray::new(
+ scratch.attention_partials.clone(),
+ &model_shape.attention_partials_shape(suffix_length),
+ act_dtype,
+ )),
+ attention_sums: RefCell::new(MetalArray::new(
+ scratch.attention_sums.clone(),
+ &model_shape.attention_sums_shape(suffix_length),
+ act_dtype,
+ )),
+ attention_maxs: RefCell::new(MetalArray::new(
+ scratch.attention_maxs.clone(),
+ &model_shape.attention_maxs_shape(suffix_length),
+ act_dtype,
+ )),
+ }
+ }
+ }
+
+ pub fn suffix_length(&self) -> usize {
+ self.suffix_length
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/embeddings_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/embeddings_buffers.rs
new file mode 100644
index 000000000..234934391
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/embeddings_buffers.rs
@@ -0,0 +1,221 @@
+use std::{cell::RefCell, rc::Rc};
+
+use super::super::ModelShape;
+use crate::{
+ DeviceContext,
+ backends::metal::{MTLContext, MetalArray},
+ config::EmbeddingConfig,
+ parameters::ParameterTree,
+};
+
+type ArrayCell = RefCell<MetalArray>;
+
+#[allow(dead_code)]
+pub enum EmbeddingsBuffers {
+ Tied {
+ /// [vocab_size, model_dim]
+ weights: ArrayCell,
+ },
+ Untied {
+ /// [vocab_size, model_dim]
+ input_weights: ArrayCell,
+ /// [vocab_size, model_dim]
+ output_weights: ArrayCell,
+ },
+ QuantizedTied {
+ /// [vocab_size, model_dim]
+ weights: ArrayCell,
+ /// [vocab_size]
+ scales: ArrayCell,
+ },
+ MLXSemiQuantizedUntied {
+ /// [vocab_size, model_dim]
+ input_weights: ArrayCell,
+ /// [vocab_size, model_dim]
+ packed_output_weights: ArrayCell,
+ /// [vocab_size, num_groups]
+ output_scales: ArrayCell,
+ /// [vocab_size, num_groups]
+ output_biases: ArrayCell,
+ },
+}
+
+impl EmbeddingsBuffers {
+ pub fn new(
+ context: &MTLContext,
+ embeddings_config: &EmbeddingConfig,
+ model_shape: &ModelShape,
+ ) -> Self {
+ unsafe {
+ match embeddings_config {
+ EmbeddingConfig::Tied {
+ common: _,
+ precision: _,
+ } => Self::Tied {
+ weights: RefCell::new(context.array_uninitialized(
+ &model_shape.embeddings_input_shape(),
+ model_shape.activation_data_type(),
+ )),
+ },
+ EmbeddingConfig::Untied {
+ common: _,
+ precision: _,
+ } => Self::Untied {
+ input_weights: RefCell::new(context.array_uninitialized(
+ &model_shape.embeddings_input_shape(),
+ model_shape.activation_data_type(),
+ )),
+ output_weights: RefCell::new(context.array_uninitialized(
+ &model_shape.embeddings_output_shape(),
+ model_shape.activation_data_type(),
+ )),
+ },
+ EmbeddingConfig::QuantizedTied {
+ embedding_quantization_mode,
+ ..
+ } => {
+ let [vocab_size, model_dim] =
+ model_shape.quantized_embeddings_weights_shape();
+ Self::QuantizedTied {
+ weights: RefCell::new(
+ context.array_uninitialized(
+ &[
+ vocab_size,
+ model_dim
+ / embedding_quantization_mode
+ .packing_divisor(),
+ ],
+ embedding_quantization_mode.storage_type(),
+ ),
+ ),
+ scales: RefCell::new(context.array_uninitialized(
+ &model_shape.quantized_embeddings_scales_shape(),
+ model_shape.activation_data_type(),
+ )),
+ }
+ },
+ EmbeddingConfig::MLXQuantizedTied {
+ group_size,
+ embedding_quantization_mode,
+ ..
+ } => {
+ let [vocab_size, model_dim] =
+ model_shape.quantized_embeddings_weights_shape();
+ let num_groups = model_dim / group_size;
+ Self::QuantizedTied {
+ weights: RefCell::new(
+ context.array_uninitialized(
+ &[
+ vocab_size,
+ model_dim
+ / embedding_quantization_mode
+ .packing_divisor(),
+ ],
+ embedding_quantization_mode.storage_type(),
+ ),
+ ),
+ scales: RefCell::new(context.array_uninitialized(
+ &[vocab_size, num_groups],
+ model_shape.activation_data_type(),
+ )),
+ }
+ },
+ EmbeddingConfig::MLXSemiQuantizedUntied {
+ group_size,
+ embedding_quantization_mode,
+ ..
+ } => {
+ let [vocab_size, model_dim] =
+ model_shape.quantized_embeddings_weights_shape();
+ let num_groups = model_dim / group_size;
+ Self::MLXSemiQuantizedUntied {
+ input_weights: RefCell::new(
+ context.array_uninitialized(
+ &model_shape.embeddings_input_shape(),
+ model_shape.activation_data_type(),
+ ),
+ ),
+ packed_output_weights: RefCell::new(
+ context.array_uninitialized(
+ &[
+ vocab_size,
+ model_dim
+ / embedding_quantization_mode
+ .packing_divisor(),
+ ],
+ embedding_quantization_mode.storage_type(),
+ ),
+ ),
+ output_scales: RefCell::new(
+ context.array_uninitialized(
+ &[vocab_size, num_groups],
+ model_shape.activation_data_type(),
+ ),
+ ),
+ output_biases: RefCell::new(
+ context.array_uninitialized(
+ &[vocab_size, num_groups],
+ model_shape.activation_data_type(),
+ ),
+ ),
+ }
+ },
+ }
+ }
+ }
+
+ pub fn update_data(
+ &mut self,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) {
+ let embeddings_tree = parameter_tree.subtree("embedding").unwrap();
+ match self {
+ EmbeddingsBuffers::Tied {
+ weights,
+ } => {
+ let embeddings_view = embeddings_tree.leaf("weights").unwrap();
+ weights.borrow_mut().copy_from_array(&embeddings_view);
+ },
+ EmbeddingsBuffers::Untied {
+ input_weights,
+ output_weights,
+ } => {
+ let mapping = vec![
+ ("input_weights", input_weights),
+ ("output_weights", output_weights),
+ ];
+ for (name, buffer) in mapping {
+ let view = embeddings_tree.leaf(name).unwrap();
+ buffer.borrow_mut().copy_from_array(&view);
+ }
+ },
+ EmbeddingsBuffers::QuantizedTied {
+ weights,
+ scales,
+ } => {
+ let mapping = vec![("weights", weights), ("scales", scales)];
+ for (name, buffer) in mapping {
+ let view = embeddings_tree.leaf(name).unwrap();
+ buffer.borrow_mut().copy_from_array(&view);
+ }
+ },
+ EmbeddingsBuffers::MLXSemiQuantizedUntied {
+ input_weights,
+ packed_output_weights,
+ output_scales,
+ output_biases,
+ } => {
+ let mapping = vec![
+ ("input_weights", input_weights),
+ ("output_weights", packed_output_weights),
+ ("output_scales", output_scales),
+ ("output_biases", output_biases),
+ ];
+ for (name, buffer) in mapping {
+ let view = embeddings_tree.leaf(name).unwrap();
+ buffer.borrow_mut().copy_from_array(&view);
+ }
+ },
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/hash_map_id.rs b/crates/uzu/src/backends/metal/forward_pass/state/hash_map_id.rs
new file mode 100644
index 000000000..4d9231e7e
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/hash_map_id.rs
@@ -0,0 +1,4 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum HashMapId {
+ AttentionBias,
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs
new file mode 100644
index 000000000..4469de591
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs
@@ -0,0 +1,416 @@
+use std::cell::RefCell;
+
+use super::super::{ModelShape, ScratchBuffers};
+use crate::{
+ DataType, DecoderConfig, backends::metal::MetalArray, config::MLPConfig,
+};
+
+type ArrayCell = RefCell<MetalArray>;
+
+pub struct LanguageModelGeneratorAuxBuffers {
+ pub ssm_inproj: Option<ArrayCell>,
+ pub ssm_packed: Option<ArrayCell>,
+ pub ssm_conv_padded: Option<ArrayCell>,
+ pub ssm_x: Option<ArrayCell>,
+ pub ssm_b: Option<ArrayCell>,
+ pub ssm_c: Option<ArrayCell>,
+ pub ssm_dt: Option<ArrayCell>,
+ pub ssm_z: Option<ArrayCell>,
+ // MoE buffers
+ pub moe_topk_ids: Option<ArrayCell>,
+ pub moe_topk_probs: Option<ArrayCell>,
+ pub moe_offsets: Option<ArrayCell>,
+ pub moe_sumk: Option<ArrayCell>,
+ pub moe_bucketed_token_ids: Option<ArrayCell>,
+ pub moe_bucketed_probs: Option<ArrayCell>,
+ pub moe_x_perm: Option<ArrayCell>,
+ pub moe_tok2row: Option<ArrayCell>,
+ pub moe_y_partial: Option<ArrayCell>,
+ pub moe_hidden: Option<ArrayCell>,
+ pub moe_two_pass_row_expert_map: Option<ArrayCell>,
+ pub moe_tile_counts: Option<ArrayCell>,
+ pub moe_tile_offsets: Option<ArrayCell>,
+ pub moe_tile_map: Option<ArrayCell>,
+ pub moe_total_tiles: Option<ArrayCell>,
+ pub moe_dispatch_args: Option<ArrayCell>,
+ pub moe_scatter_partials: Option<ArrayCell>,
+ pub moe_scatter_block_bases: Option<ArrayCell>,
+ pub moe_block_alloc: Option<ArrayCell>,
+}
+
+impl LanguageModelGeneratorAuxBuffers {
+ pub fn new(
+ scratch: &ScratchBuffers,
+ decoder_config: &DecoderConfig,
+ model_shape: &ModelShape,
+ suffix_length: usize,
+ ) -> Self {
+ let act_dtype = model_shape.activation_data_type();
+
+ unsafe {
+ Self {
+ ssm_inproj: match (
+ scratch.ssm_inproj.as_ref(),
+ model_shape.ssm_inproj_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
+ ssm_packed: match (
+ scratch.ssm_packed.as_ref(),
+ model_shape.ssm_packed_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
+ ssm_conv_padded: match (
+ scratch.ssm_conv_padded.as_ref(),
+ model_shape.ssm_conv_padded_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
+ ssm_x: match (
+ scratch.ssm_x.as_ref(),
+ model_shape.ssm_x_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
+ ssm_b: match (
+ scratch.ssm_b.as_ref(),
+ model_shape.ssm_bc_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
+ ssm_c: match (
+ scratch.ssm_c.as_ref(),
+ model_shape.ssm_bc_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
+ ssm_dt: match (
+ scratch.ssm_dt.as_ref(),
+ model_shape.ssm_dt_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
+ ssm_z: match (
+ scratch.ssm_z.as_ref(),
+ model_shape.ssm_z_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
+ moe_topk_ids: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_topk_ids.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_topk_ids_shape(
+ suffix_length,
+ moe.num_experts_per_token,
+ ),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_topk_probs: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_topk_probs.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_topk_probs_shape(
+ suffix_length,
+ moe.num_experts_per_token,
+ ),
+ act_dtype,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_offsets: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_offsets.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape
+ .moe_offsets_shape(moe.mixture_size),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_sumk: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(_) => {
+ scratch.moe_sumk.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_sumk_shape(),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_bucketed_token_ids: match &decoder_config
+ .layer_config
+ .mlp_config
+ {
+ MLPConfig::MixtureOfExperts(moe) => {
+ let max_routed =
+ suffix_length * moe.num_experts_per_token;
+ scratch.moe_bucketed_token_ids.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape
+ .moe_bucketed_token_ids_shape(max_routed),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_bucketed_probs: match &decoder_config
+ .layer_config
+ .mlp_config
+ {
+ MLPConfig::MixtureOfExperts(moe) => {
+ let max_routed =
+ suffix_length * moe.num_experts_per_token;
+ scratch.moe_bucketed_probs.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape
+ .moe_bucketed_probs_shape(max_routed),
+ act_dtype,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_x_perm: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ let max_routed =
+ suffix_length * moe.num_experts_per_token;
+ scratch.moe_x_perm.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_x_perm_shape(max_routed),
+ DataType::F16,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_tok2row: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_tok2row.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_tok2row_shape(
+ suffix_length,
+ moe.num_experts_per_token,
+ ),
+ DataType::I32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_y_partial: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ let max_routed =
+ suffix_length * moe.num_experts_per_token;
+ scratch.moe_y_partial.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_y_partial_shape(max_routed),
+ DataType::F16,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_hidden: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ let max_routed =
+ suffix_length * moe.num_experts_per_token;
+ scratch.moe_hidden.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_hidden_shape(max_routed),
+ DataType::F32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_two_pass_row_expert_map: match &decoder_config
+ .layer_config
+ .mlp_config
+ {
+ MLPConfig::MixtureOfExperts(moe) => {
+ let max_routed =
+ suffix_length * moe.num_experts_per_token;
+ scratch.moe_two_pass_row_expert_map.as_ref().map(
+ |buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &[max_routed],
+ DataType::U32,
+ ))
+ },
+ )
+ },
+ _ => None,
+ },
+ moe_tile_counts: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_tile_counts.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_counts_shape(moe.mixture_size),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_tile_offsets: match &decoder_config.layer_config.mlp_config
+ {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_tile_offsets.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape
+ .moe_offsets_shape(moe.mixture_size),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_tile_map: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ let max_routed =
+ suffix_length * moe.num_experts_per_token;
+ scratch.moe_tile_map.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_tile_map_shape(max_routed),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_total_tiles: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(_) => {
+ scratch.moe_total_tiles.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_total_tiles_shape(),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_dispatch_args: match &decoder_config.layer_config.mlp_config
+ {
+ MLPConfig::MixtureOfExperts(_) => {
+ scratch.moe_dispatch_args.as_ref().map(|buf| {
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &model_shape.moe_dispatch_args_shape(),
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_scatter_partials: match &decoder_config
+ .layer_config
+ .mlp_config
+ {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_scatter_partials.as_ref().map(|buf| {
+ let num_blocks =
+ ((suffix_length + 255) / 256).max(1);
+ let num_tiles =
+ ((moe.mixture_size + 512 - 1) / 512).max(1);
+ let entries = num_blocks * num_tiles * 512;
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &[entries],
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_scatter_block_bases: match &decoder_config
+ .layer_config
+ .mlp_config
+ {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_scatter_block_bases.as_ref().map(|buf| {
+ let num_blocks =
+ ((suffix_length + 255) / 256).max(1);
+ let num_tiles =
+ ((moe.mixture_size + 512 - 1) / 512).max(1);
+ let entries = num_blocks * num_tiles * 512;
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &[entries],
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ moe_block_alloc: match &decoder_config.layer_config.mlp_config {
+ MLPConfig::MixtureOfExperts(moe) => {
+ scratch.moe_block_alloc.as_ref().map(|buf| {
+ let num_blocks =
+ ((suffix_length + 255) / 256).max(1);
+ let num_tiles =
+ ((moe.mixture_size + 512 - 1) / 512).max(1);
+ let entries = num_blocks * num_tiles;
+ RefCell::new(MetalArray::new(
+ buf.clone(),
+ &[entries],
+ DataType::U32,
+ ))
+ })
+ },
+ _ => None,
+ },
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
new file mode 100644
index 000000000..f7d6c5b47
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
@@ -0,0 +1,1017 @@
+mod array_id;
+mod common_aux_buffers;
+mod embeddings_buffers;
+mod hash_map_id;
+mod language_model_generator_aux_buffers;
+mod mode;
+mod rope_buffers;
+mod rope_type;
+mod shared_buffers;
+
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
+
+pub use array_id::ArrayId;
+pub use common_aux_buffers::CommonAuxBuffers;
+pub use embeddings_buffers::EmbeddingsBuffers;
+pub use hash_map_id::HashMapId;
+pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;
+pub use mode::{
+ ClassifierModeState, ForwardPassMode, LanguageModelGeneratorModeState,
+};
+pub use rope_buffers::RopeBuffers;
+pub use rope_type::RopeType;
+pub use shared_buffers::{MoeExpertWeights, SharedBuffers};
+
+#[cfg(feature = "tracing")]
+use super::traces::ActivationTrace;
+use super::{ModelShape, ScratchBuffers, cache_layers::CacheLayers};
+use crate::{
+ Array, DataType, DecoderConfig, DeviceContext,
+ backends::metal::{MTLContext, MetalArray},
+ session::parameter::SamplingMethod,
+};
+
+pub type ArrayCell = RefCell<MetalArray>;
+
+pub struct ForwardPassState {
+ context: Rc<MTLContext>,
+ token_ids: ArrayCell,
+ token_positions: ArrayCell,
+ token_bitmask: Option<ArrayCell>,
+ attention_bias: HashMap<Option<usize>, ArrayCell>,
+ pub shared_buffers: Rc<RefCell<SharedBuffers>>,
+ common_aux: CommonAuxBuffers,
+ llm_aux: Option<LanguageModelGeneratorAuxBuffers>,
+ mode: ForwardPassMode,
+}
+
+impl ForwardPassState {
+ // ========================================================================
+ // Common initialization helpers
+ // ========================================================================
+
+ fn init_token_ids(
+ context: &MTLContext,
+ scratch: &ScratchBuffers,
+ token_ids: &[u64],
+ ) -> ArrayCell {
+ let suffix_length = token_ids.len();
+ let mut token_ids_array = unsafe {
+ MetalArray::new(
+ scratch.token_ids.clone(),
+ &[suffix_length],
+ DataType::U64,
+ )
+ };
+ context.copy_from_view(&mut token_ids_array, token_ids.into());
+ RefCell::new(token_ids_array)
+ }
+
+ fn init_token_positions(
+ context: &MTLContext,
+ scratch: &ScratchBuffers,
+ token_positions: &[usize],
+ ) -> ArrayCell {
+ let suffix_length = token_positions.len();
+ let mut token_positions_array = unsafe {
+ MetalArray::new(
+ scratch.token_positions.clone(),
+ &[suffix_length],
+ DataType::I32,
+ )
+ };
+ let token_positions_i32: Box<[i32]> =
+ token_positions.iter().map(|p| *p as i32).collect();
+ context.copy_from_view(
+ &mut token_positions_array,
+ token_positions_i32.as_ref().into(),
+ );
+ RefCell::new(token_positions_array)
+ }
+
+ // ========================================================================
+ // LLM Constructor
+ // ========================================================================
+
+ #[allow(clippy::too_many_arguments)]
+ pub fn new_llm(
+ context: Rc<MTLContext>,
+ decoder_config: &DecoderConfig,
+ model_shape: &ModelShape,
+ scratch: &ScratchBuffers,
+ cache_layers: Rc<RefCell<CacheLayers>>,
+ shared_buffers: Rc<RefCell<SharedBuffers>>,
+ token_ids: &[u64],
+ token_positions: &[usize],
+ token_bitmask: Option<&[u32]>,
+ token_seeds: &[u64],
+ active_suffix_length: usize,
+ is_prefilling: bool,
+ external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
+ skip_token_ids_copy: bool,
+ async_positions: Option<(&metal::Buffer, usize)>,
+ async_seeds: Option<(&metal::Buffer, usize)>,
+ ) -> Self {
+ let suffix_length = token_ids.len();
+ assert_eq!(
+ suffix_length,
+ token_positions.len(),
+ "Tokens and positions must have same length"
+ );
+ assert!(
+ suffix_length <= cache_layers.borrow().max_suffix_length(),
+ "Suffix length exceeds KV cache capacity"
+ );
+
+ // Token IDs - optionally skip copy for async path
+ let token_ids_cell = if skip_token_ids_copy {
+ RefCell::new(unsafe {
+ MetalArray::new(
+ scratch.token_ids.clone(),
+ &[suffix_length],
+ DataType::U64,
+ )
+ })
+ } else {
+ Self::init_token_ids(&context, scratch, token_ids)
+ };
+
+ // Token positions - use async buffer if provided
+ let token_positions_cell =
+ if let Some((async_buf, offset)) = async_positions {
+ let array = unsafe {
+ MetalArray::new_with_offset(
+ async_buf.clone(),
+ &[suffix_length],
+ DataType::I32,
+ offset * std::mem::size_of::<i32>(),
+ )
+ };
+ RefCell::new(array)
+ } else {
+ Self::init_token_positions(&context, scratch, token_positions)
+ };
+
+ // Token bitmask
+ let token_bitmask_cell = token_bitmask.map(|bitmask| {
+ let bitmask_shape = model_shape.bitmask_shape(suffix_length);
+ let mut bitmask_array = unsafe {
+ MetalArray::new(
+ scratch.token_bitmask.clone(),
+ &bitmask_shape,
+ DataType::U32,
+ )
+ };
+ context.copy_from_view(&mut bitmask_array, bitmask.into());
+ RefCell::new(bitmask_array)
+ });
+
+ // Token seeds - use async buffer if provided
+ let token_seeds_cell = if let Some((async_buf, offset)) = async_seeds {
+ let array = unsafe {
+ MetalArray::new_with_offset(
+ async_buf.clone(),
+ &[suffix_length],
+ DataType::U64,
+ offset * std::mem::size_of::<u64>(),
+ )
+ };
+ RefCell::new(array)
+ } else {
+ let mut token_seeds_array = unsafe {
+ MetalArray::new(
+ scratch.token_seeds.clone(),
+ &[suffix_length],
+ DataType::U64,
+ )
+ };
+ context.copy_from_view(&mut token_seeds_array, token_seeds.into());
+ RefCell::new(token_seeds_array)
+ };
+
+ // Logits
+ let logits_cell = RefCell::new(unsafe {
+ MetalArray::new(
+ scratch.logits.clone(),
+ &model_shape.logits_shape(suffix_length),
+ model_shape.activation_data_type(),
+ )
+ });
+
+ // Sampling output
+ let sampling_output = Some(RefCell::new(unsafe {
+ MetalArray::new(
+ scratch.sampling_output.clone(),
+ &[suffix_length],
+ DataType::U32,
+ )
+ }));
+
+ // Attention bias (causal + sliding window)
+ let act_dtype = model_shape.activation_data_type();
+ let attention_bias = Self::init_llm_attention_bias(
+ &context,
+ scratch,
+ &cache_layers,
+ suffix_length,
+ act_dtype,
+ token_positions,
+ external_bias_fn,
+ );
+
+ // Common aux buffers
+ let common_aux =
+ CommonAuxBuffers::new(scratch, model_shape, suffix_length);
+
+ // LLM-specific aux buffers
+ let llm_aux = Some(LanguageModelGeneratorAuxBuffers::new(
+ scratch,
+ decoder_config,
+ model_shape,
+ suffix_length,
+ ));
+
+ // Traces
+ #[cfg(feature = "tracing")]
+ let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(
+ &context,
+ model_shape,
+ suffix_length,
+ )));
+
+ let mode = ForwardPassMode::LanguageModelGenerator(
+ LanguageModelGeneratorModeState {
+ cache_layers,
+ token_seeds: token_seeds_cell,
+ logits: logits_cell,
+ sampling_output,
+ sampling_method: None,
+ #[cfg(feature = "tracing")]
+ traces,
+ active_suffix_length,
+ is_prefilling,
+ },
+ );
+
+ Self {
+ context,
+ token_ids: token_ids_cell,
+ token_positions: token_positions_cell,
+ token_bitmask: token_bitmask_cell,
+ attention_bias,
+ shared_buffers,
+ common_aux,
+ llm_aux,
+ mode,
+ }
+ }
+
+ fn init_llm_attention_bias(
+ context: &MTLContext,
+ scratch: &ScratchBuffers,
+ cache_layers: &Rc<RefCell<CacheLayers>>,
+ suffix_length: usize,
+ act_dtype: DataType,
+ token_positions: &[usize],
+ external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
+ ) -> HashMap<Option<usize>, ArrayCell> {
+ let cache_ref = cache_layers.borrow();
+ let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =
+ scratch
+ .attention_window_size_to_bias
+ .iter()
+ .map(|(window_size, buffer)| {
+ let prefix_length =
+ window_size.unwrap_or(cache_ref.max_prefix_length());
+ let attention_bias_shape =
+ [suffix_length, suffix_length + prefix_length];
+ let array = unsafe {
+ MetalArray::new(
+ buffer.clone(),
+ &attention_bias_shape,
+ act_dtype,
+ )
+ };
+ (*window_size, array)
+ })
+ .collect();
+ drop(cache_ref);
+
+ // Use cache_layers' fill_attention_bias which properly handles
+ // both causal masking and sliding window constraints
+ cache_layers.borrow().fill_attention_bias(
+ &mut attention_bias_map,
+ token_positions,
+ suffix_length,
+ context,
+ external_bias_fn,
+ );
+
+ attention_bias_map
+ .into_iter()
+ .map(|(k, v)| (k, RefCell::new(v)))
+ .collect()
+ }
+
+ // ========================================================================
+ // Classifier Constructor
+ // ========================================================================
+
+ #[allow(clippy::too_many_arguments)]
+ pub fn new_classifier(
+ context: Rc<MTLContext>,
+ model_shape: &ModelShape,
+ scratch: &ScratchBuffers,
+ shared_buffers: Rc<RefCell<SharedBuffers>>,
+ token_ids: &[u64],
+ token_positions: &[usize],
+ bidirectional_attention: bool,
+ num_labels: usize,
+ ) -> Self {
+ let suffix_length = token_ids.len();
+ assert_eq!(suffix_length, token_positions.len());
+
+ let token_ids_cell = Self::init_token_ids(&context, scratch, token_ids);
+ let token_positions_cell =
+ Self::init_token_positions(&context, scratch, token_positions);
+
+ // Attention bias (bidirectional or causal)
+ let act_dtype = model_shape.activation_data_type();
+ let attention_bias = Self::init_classifier_attention_bias(
+ &context,
+ scratch,
+ suffix_length,
+ act_dtype,
+ bidirectional_attention,
+ );
+
+ // Common aux buffers
+ let common_aux =
+ CommonAuxBuffers::new(scratch, model_shape, suffix_length);
+
+ // Classifier-specific buffers
+ let model_dim = model_shape.main_shape(1)[1];
+ let classifier_state = Self::init_classifier_buffers(
+ &context,
+ model_shape,
+ model_dim,
+ num_labels,
+ suffix_length,
+ );
+
+ let mode = ForwardPassMode::Classifier(classifier_state);
+
+ Self {
+ context,
+ token_ids: token_ids_cell,
+ token_positions: token_positions_cell,
+ token_bitmask: None,
+ attention_bias,
+ shared_buffers,
+ common_aux,
+ llm_aux: None,
+ mode,
+ }
+ }
+
+ fn init_classifier_attention_bias(
+ context: &MTLContext,
+ scratch: &ScratchBuffers,
+ suffix_length: usize,
+ act_dtype: DataType,
+ bidirectional_attention: bool,
+ ) -> HashMap<Option<usize>, ArrayCell> {
+ let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =
+ scratch
+ .attention_window_size_to_bias
+ .iter()
+ .map(|(window_size, buffer)| {
+ let attention_bias_shape = [suffix_length, suffix_length];
+ let array = unsafe {
+ MetalArray::new(
+ buffer.clone(),
+ &attention_bias_shape,
+ act_dtype,
+ )
+ };
+ (*window_size, array)
+ })
+ .collect();
+
+ for (window, bias_array) in attention_bias_map.iter_mut() {
+ if bidirectional_attention {
+ if let Some(window_size) = window {
+ let half_window = (window_size / 2) as isize;
+ context.fill_attention_bias(
+ bias_array,
+ suffix_length,
+ 0,
+ |row, col| {
+ let distance = (row as isize) - (col as isize);
+ distance.abs() > half_window
+ },
+ );
+ } else {
+ context.fill_attention_bias(
+ bias_array,
+ suffix_length,
+ 0,
+ |_row, _col| false,
+ );
+ }
+ } else {
+ context.fill_attention_bias(
+ bias_array,
+ suffix_length,
+ 0,
+ |row, col| row < col,
+ );
+ }
+ }
+
+ attention_bias_map
+ .into_iter()
+ .map(|(k, v)| (k, RefCell::new(v)))
+ .collect()
+ }
+
+ #[cfg(feature = "tracing")]
+ fn init_classifier_buffers(
+ context: &MTLContext,
+ model_shape: &ModelShape,
+ model_dim: usize,
+ num_labels: usize,
+ suffix_length: usize,
+ ) -> ClassifierModeState {
+ let data_type = model_shape.activation_data_type();
+ let batch_size = 1;
+
+ let create_buffer = |size: usize| -> ArrayCell {
+ let buffer_size = (size * data_type.size_in_bytes()) as u64;
+ let buffer = context.device.new_buffer(
+ buffer_size,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ RefCell::new(unsafe {
+ MetalArray::new(
+ buffer,
+ &[batch_size, size / batch_size],
+ data_type,
+ )
+ })
+ };
+
+ ClassifierModeState {
+ pooling: create_buffer(batch_size * model_dim),
+ dense: create_buffer(batch_size * model_dim),
+ norm: create_buffer(batch_size * model_dim),
+ classifier_logits: {
+ let buffer_size =
+ (batch_size * num_labels * data_type.size_in_bytes())
+ as u64;
+ let buffer = context.device.new_buffer(
+ buffer_size,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ RefCell::new(unsafe {
+ MetalArray::new(
+ buffer,
+ &[batch_size, num_labels],
+ data_type,
+ )
+ })
+ },
+ traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(
+ context,
+ model_shape,
+ suffix_length,
+ num_labels,
+ ))),
+ }
+ }
+
+ #[cfg(not(feature = "tracing"))]
+ fn init_classifier_buffers(
+ context: &MTLContext,
+ model_shape: &ModelShape,
+ model_dim: usize,
+ num_labels: usize,
+ _suffix_length: usize,
+ ) -> ClassifierModeState {
+ let data_type = model_shape.activation_data_type();
+ let batch_size = 1;
+
+ let create_buffer = |dims: &[usize]| -> ArrayCell {
+ let size: usize = dims.iter().product();
+ let buffer_size = (size * data_type.size_in_bytes()) as u64;
+ let buffer = context.device.new_buffer(
+ buffer_size,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ RefCell::new(unsafe { MetalArray::new(buffer, dims, data_type) })
+ };
+
+ ClassifierModeState {
+ pooling: create_buffer(&[batch_size, model_dim]),
+ dense: create_buffer(&[batch_size, model_dim]),
+ norm: create_buffer(&[batch_size, model_dim]),
+ classifier_logits: create_buffer(&[batch_size, num_labels]),
+ }
+ }
+
+ // ========================================================================
+ // Accessors
+ // ========================================================================
+
+ pub fn mtl_context(&self) -> &Rc<MTLContext> {
+ &self.context
+ }
+
+ pub fn aux_buffers_suffix_length(&self) -> usize {
+ self.common_aux.suffix_length
+ }
+
+ pub fn is_llm(&self) -> bool {
+ matches!(self.mode, ForwardPassMode::LanguageModelGenerator(_))
+ }
+
+ pub fn is_classifier(&self) -> bool {
+ matches!(self.mode, ForwardPassMode::Classifier(_))
+ }
+
+ pub fn token_bitmask(&self) -> Option<&ArrayCell> {
+ self.token_bitmask.as_ref()
+ }
+
+ pub fn llm_state(&self) -> &LanguageModelGeneratorModeState {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => state,
+ _ => panic!("Not in LLM mode"),
+ }
+ }
+
+ pub fn llm_state_mut(&mut self) -> &mut LanguageModelGeneratorModeState {
+ match &mut self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => state,
+ _ => panic!("Not in LLM mode"),
+ }
+ }
+
+ pub fn classifier_state(&self) -> &ClassifierModeState {
+ match &self.mode {
+ ForwardPassMode::Classifier(state) => state,
+ _ => panic!("Not in classifier mode"),
+ }
+ }
+
+ pub fn array_cell(
+ &self,
+ id: ArrayId,
+ ) -> ArrayCell {
+ match id {
+ // Common arrays
+ ArrayId::TokenIds => self.token_ids.clone(),
+ ArrayId::TokenPositions => self.token_positions.clone(),
+ ArrayId::TokenBitmask => {
+ self.token_bitmask.clone().expect("Token bitmask not available")
+ },
+ ArrayId::Main => self.common_aux.main.clone(),
+ ArrayId::Shortcut => self.common_aux.shortcut.clone(),
+ ArrayId::QKV => self.common_aux.qkv.clone(),
+ ArrayId::AttentionOutput => {
+ self.common_aux.attention_output.clone()
+ },
+ ArrayId::MlpFusedUp => self.common_aux.mlp_fused_up.clone(),
+ ArrayId::MlpHidden => self.common_aux.mlp_hidden.clone(),
+ ArrayId::RotatedQueries => self.common_aux.rotated_queries.clone(),
+ ArrayId::RotatedKeys => self.common_aux.rotated_keys.clone(),
+ ArrayId::ExtractedValues => {
+ self.common_aux.extracted_values.clone()
+ },
+ ArrayId::AttentionPartials => {
+ self.common_aux.attention_partials.clone()
+ },
+ ArrayId::AttentionSums => self.common_aux.attention_sums.clone(),
+ ArrayId::AttentionMaxs => self.common_aux.attention_maxs.clone(),
+
+ // Shared buffer arrays
+ ArrayId::EmbeddingsInputWeights
+ | ArrayId::EmbeddingsOutputWeights
+ | ArrayId::EmbeddingsScales
+ | ArrayId::RopeCosines(_)
+ | ArrayId::RopeSines(_) => self
+ .shared_buffer_array(id)
+ .expect("Shared buffer array should be available"),
+
+ // LLM-specific arrays
+ ArrayId::Logits => self.llm_state().logits.clone(),
+ ArrayId::TokenSeeds => self.llm_state().token_seeds.clone(),
+ ArrayId::Keys(layer_index) => {
+ let cache = self.llm_state().cache_layers.borrow();
+ cache.data[layer_index]
+ .as_transformer()
+ .expect("Expected transformer layer")
+ .keys
+ .clone()
+ },
+ ArrayId::Values(layer_index) => {
+ let cache = self.llm_state().cache_layers.borrow();
+ cache.data[layer_index]
+ .as_transformer()
+ .expect("Expected transformer layer")
+ .values
+ .clone()
+ },
+ ArrayId::AttentionSinks(layer_index) => self
+ .shared_buffers
+ .borrow()
+ .attention_sinks
+ .as_ref()
+ .expect("Attention sinks not initialized")[layer_index]
+ .clone(),
+
+ // SSM arrays (LLM only)
+ ArrayId::SsmInProj => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.ssm_inproj.clone())
+ .expect("SSM inproj not initialized"),
+ ArrayId::SsmPacked(_) => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.ssm_packed.clone())
+ .expect("SSM packed not initialized"),
+ ArrayId::SsmConvState(layer_index) => {
+ let cache = self.llm_state().cache_layers.borrow();
+ cache.data[layer_index]
+ .as_state_space()
+ .expect("Expected SSM layer")
+ .conv_state
+ .clone()
+ },
+ ArrayId::SsmState(layer_index) => {
+ let cache = self.llm_state().cache_layers.borrow();
+ cache.data[layer_index]
+ .as_state_space()
+ .expect("Expected SSM layer")
+ .ssm_state
+ .clone()
+ },
+ ArrayId::SsmX(_) => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.ssm_x.clone())
+ .expect("SSM x not initialized"),
+ ArrayId::SsmB(_) => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.ssm_b.clone())
+ .expect("SSM b not initialized"),
+ ArrayId::SsmC(_) => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.ssm_c.clone())
+ .expect("SSM c not initialized"),
+ ArrayId::SsmDt(_) => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.ssm_dt.clone())
+ .expect("SSM dt not initialized"),
+ ArrayId::SsmZ(_) => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.ssm_z.clone())
+ .expect("SSM z not initialized"),
+
+ // MoE arrays (LLM only)
+ ArrayId::MoeTopkIds => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_topk_ids.clone())
+ .expect("MoE topk_ids not initialized"),
+ ArrayId::MoeTopkProbs => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_topk_probs.clone())
+ .expect("MoE topk_probs not initialized"),
+ ArrayId::MoeOffsets => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_offsets.clone())
+ .expect("MoE offsets not initialized"),
+ ArrayId::MoeSumK => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_sumk.clone())
+ .expect("MoE sumk not initialized"),
+ ArrayId::MoeBucketedTokenIds => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_bucketed_token_ids.clone())
+ .expect("MoE bucketed_token_ids not initialized"),
+ ArrayId::MoeBucketedProbs => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_bucketed_probs.clone())
+ .expect("MoE bucketed_probs not initialized"),
+ ArrayId::MoeXPerm => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_x_perm.clone())
+ .expect("MoE x_perm not initialized"),
+ ArrayId::MoeTok2Row => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_tok2row.clone())
+ .expect("MoE tok2row not initialized"),
+ ArrayId::MoeYPartial => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_y_partial.clone())
+ .expect("MoE y_partial not initialized"),
+ ArrayId::MoeHidden => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_hidden.clone())
+ .expect("MoE hidden not initialized"),
+ ArrayId::MoeTwoPassRowExpertMap => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_two_pass_row_expert_map.clone())
+ .expect("MoE two_pass_row_expert_map not initialized"),
+ ArrayId::MoeTileCounts => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_tile_counts.clone())
+ .expect("MoE tile_counts not initialized"),
+ ArrayId::MoeTileOffsets => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_tile_offsets.clone())
+ .expect("MoE tile_offsets not initialized"),
+ ArrayId::MoeTileMap => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_tile_map.clone())
+ .expect("MoE tile_map not initialized"),
+ ArrayId::MoeTotalTiles => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_total_tiles.clone())
+ .expect("MoE total_tiles not initialized"),
+ ArrayId::MoeDispatchArgs => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_dispatch_args.clone())
+ .expect("MoE dispatch_args not initialized"),
+ ArrayId::MoeScatterPartials => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_scatter_partials.clone())
+ .expect("MoE scatter_partials not initialized"),
+ ArrayId::MoeScatterBlockBases => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_scatter_block_bases.clone())
+ .expect("MoE scatter_block_bases not initialized"),
+ ArrayId::MoeBlockAlloc => self
+ .llm_aux
+ .as_ref()
+ .and_then(|aux| aux.moe_block_alloc.clone())
+ .expect("MoE block_alloc not initialized"),
+
+ // Classifier-specific arrays
+ ArrayId::ClassifierPooling => {
+ self.classifier_state().pooling.clone()
+ },
+ ArrayId::ClassifierPredictionHeadDense => {
+ self.classifier_state().dense.clone()
+ },
+ ArrayId::ClassifierPredictionHeadNorm => {
+ self.classifier_state().norm.clone()
+ },
+ ArrayId::ClassifierPredictionHeadLogits => {
+ self.classifier_state().classifier_logits.clone()
+ },
+ }
+ }
+
+ fn shared_buffer_array(
+ &self,
+ id: ArrayId,
+ ) -> Option<ArrayCell> {
+ let shared = self.shared_buffers.borrow();
+ match id {
+ ArrayId::EmbeddingsInputWeights => Some(match &shared.embeddings {
+ EmbeddingsBuffers::Tied {
+ weights,
+ } => weights.clone(),
+ EmbeddingsBuffers::Untied {
+ input_weights,
+ ..
+ } => input_weights.clone(),
+ EmbeddingsBuffers::QuantizedTied {
+ weights,
+ ..
+ } => weights.clone(),
+ EmbeddingsBuffers::MLXSemiQuantizedUntied {
+ input_weights,
+ ..
+ } => input_weights.clone(),
+ }),
+ ArrayId::EmbeddingsOutputWeights => {
+ Some(match &shared.embeddings {
+ EmbeddingsBuffers::Tied {
+ weights,
+ } => weights.clone(),
+ EmbeddingsBuffers::Untied {
+ output_weights,
+ ..
+ } => output_weights.clone(),
+ EmbeddingsBuffers::QuantizedTied {
+ weights,
+ ..
+ } => weights.clone(),
+ EmbeddingsBuffers::MLXSemiQuantizedUntied {
+ packed_output_weights,
+ ..
+ } => packed_output_weights.clone(),
+ })
+ },
+ ArrayId::EmbeddingsScales => Some(match &shared.embeddings {
+ EmbeddingsBuffers::QuantizedTied {
+ scales,
+ ..
+ } => scales.clone(),
+ EmbeddingsBuffers::MLXSemiQuantizedUntied {
+ output_scales,
+ ..
+ } => output_scales.clone(),
+ _ => panic!(
+ "Expected QuantizedTied or MLXSemiQuantizedUntied embeddings"
+ ),
+ }),
+ ArrayId::RopeCosines(rope_type) => Some(match rope_type {
+ RopeType::Global => shared
+ .global_rope
+ .as_ref()
+ .expect("Global rope not initialized")
+ .cosines
+ .clone(),
+ RopeType::Local => shared
+ .local_rope
+ .as_ref()
+ .expect("Local rope not initialized")
+ .cosines
+ .clone(),
+ }),
+ ArrayId::RopeSines(rope_type) => Some(match rope_type {
+ RopeType::Global => shared
+ .global_rope
+ .as_ref()
+ .expect("Global rope not initialized")
+ .sines
+ .clone(),
+ RopeType::Local => shared
+ .local_rope
+ .as_ref()
+ .expect("Local rope not initialized")
+ .sines
+ .clone(),
+ }),
+ _ => None,
+ }
+ }
+
+ pub fn conv_padded_buffer(&self) -> Option<ArrayCell> {
+ self.llm_aux.as_ref().and_then(|aux| aux.ssm_conv_padded.clone())
+ }
+
+ // ========================================================================
+ // Public API Methods (formerly trait methods)
+ // ========================================================================
+
+ pub fn arrays(
+ &self,
+ ids: &[ArrayId],
+ ) -> Box<[ArrayCell]> {
+ ids.iter().map(|id| self.array_cell(*id)).collect()
+ }
+
+ pub fn hashmaps(
+ &self,
+ ids: &[HashMapId],
+ ) -> Box<[HashMap<Option<usize>, ArrayCell>]> {
+ ids.iter()
+ .map(|id| match id {
+ HashMapId::AttentionBias => self.attention_bias.clone(),
+ })
+ .collect()
+ }
+
+ pub fn active_suffix_length(&self) -> usize {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => {
+ state.active_suffix_length
+ },
+ ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,
+ }
+ }
+
+ pub fn is_prefilling(&self) -> bool {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => {
+ state.is_prefilling
+ },
+ ForwardPassMode::Classifier(_) => true,
+ }
+ }
+
+ pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers>>> {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => {
+ Some(&state.cache_layers)
+ },
+ ForwardPassMode::Classifier(_) => None,
+ }
+ }
+
+ pub fn sampling_output(&self) -> Option<&ArrayCell> {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => {
+ state.sampling_output.as_ref()
+ },
+ ForwardPassMode::Classifier(_) => None,
+ }
+ }
+
+ #[cfg(feature = "tracing")]
+ pub fn traces(&self) -> &Rc<RefCell<ActivationTrace>> {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => &state.traces,
+ ForwardPassMode::Classifier(state) => &state.traces,
+ }
+ }
+
+ pub fn sampling_method_mut(
+ &mut self
+ ) -> Option<&mut Option<SamplingMethod>> {
+ match &mut self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => {
+ Some(&mut state.sampling_method)
+ },
+ ForwardPassMode::Classifier(_) => None,
+ }
+ }
+
+ pub fn sampling_method(&self) -> Option<SamplingMethod> {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => {
+ state.sampling_method
+ },
+ ForwardPassMode::Classifier(_) => None,
+ }
+ }
+
+ pub fn copy_array(
+ &self,
+ source_array_id: ArrayId,
+ destination_array: RefCell<MetalArray>,
+ ) {
+ destination_array
+ .borrow_mut()
+ .copy_from_array(&self.arrays(&[source_array_id])[0].borrow());
+ }
+
+ pub fn encode_copy_array(
+ &self,
+ command_buffer: &mpsgraph::CommandBuffer,
+ source_array_id: ArrayId,
+ destination_array: RefCell<MetalArray>,
+ ) {
+ let source_ref = self.arrays(&[source_array_id])[0].clone();
+ let mut src_borrow = source_ref.borrow_mut();
+ let mut dst_borrow = destination_array.borrow_mut();
+
+ let src_buf = unsafe { src_borrow.mtl_buffer().clone() };
+ let dst_buf = unsafe { dst_borrow.mtl_buffer().clone() };
+
+ let copy_size_bytes = dst_borrow.size_in_bytes() as u64;
+ debug_assert_eq!(
+ dst_borrow.size_in_bytes(),
+ src_borrow.size_in_bytes()
+ );
+
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let blit_encoder = mtl_command_buffer.new_blit_command_encoder();
+ blit_encoder.copy_from_buffer(
+ &src_buf,
+ 0,
+ &dst_buf,
+ 0,
+ copy_size_bytes,
+ );
+ blit_encoder.end_encoding();
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mode.rs b/crates/uzu/src/backends/metal/forward_pass/state/mode.rs
new file mode 100644
index 000000000..f5d9bbcb5
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mode.rs
@@ -0,0 +1,34 @@
+use std::{cell::RefCell, rc::Rc};
+
+use super::super::cache_layers::CacheLayers;
+#[cfg(feature = "tracing")]
+use super::super::traces::ActivationTrace;
+use crate::{backends::metal::MetalArray, session::parameter::SamplingMethod};
+
+type ArrayCell = RefCell<MetalArray>;
+
+pub enum ForwardPassMode {
+ LanguageModelGenerator(LanguageModelGeneratorModeState),
+ Classifier(ClassifierModeState),
+}
+
+pub struct LanguageModelGeneratorModeState {
+ pub cache_layers: Rc<RefCell<CacheLayers>>,
+ pub token_seeds: ArrayCell,
+ pub logits: ArrayCell,
+ pub sampling_output: Option<ArrayCell>,
+ pub sampling_method: Option<SamplingMethod>,
+ #[cfg(feature = "tracing")]
+ pub traces: Rc<RefCell<ActivationTrace>>,
+ pub active_suffix_length: usize,
+ pub is_prefilling: bool,
+}
+
+pub struct ClassifierModeState {
+ pub pooling: ArrayCell,
+ pub dense: ArrayCell,
+ pub norm: ArrayCell,
+ pub classifier_logits: ArrayCell,
+ #[cfg(feature = "tracing")]
+ pub traces: Rc<RefCell<ActivationTrace>>,
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/rope_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/rope_buffers.rs
new file mode 100644
index 000000000..469089911
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/rope_buffers.rs
@@ -0,0 +1,57 @@
+use std::{cell::RefCell, rc::Rc};
+
+use super::super::ModelShape;
+use crate::{
+ DeviceContext,
+ backends::metal::{MTLContext, MetalArray},
+ parameters::ParameterTree,
+};
+
+type ArrayCell = RefCell<MetalArray>;
+
+pub struct RopeBuffers {
+ /// [rope_max_sequence_length, head_dim]
+ pub cosines: ArrayCell,
+ /// [rope_max_sequence_length, head_dim]
+ pub sines: ArrayCell,
+}
+
+impl RopeBuffers {
+ pub fn new(
+ context: &MTLContext,
+ model_shape: &ModelShape,
+ ) -> Self {
+ unsafe {
+ let rotated_queries_shape = model_shape.rotated_queries_shape(1);
+ let head_dim = rotated_queries_shape[2];
+ let rope_max_sequence_length = model_shape.context_length();
+
+ Self {
+ cosines: RefCell::new(context.array_uninitialized(
+ &[rope_max_sequence_length, head_dim],
+ model_shape.activation_data_type(),
+ )),
+ sines: RefCell::new(context.array_uninitialized(
+ &[rope_max_sequence_length, head_dim],
+ model_shape.activation_data_type(),
+ )),
+ }
+ }
+ }
+
+ pub fn update_data(
+ &mut self,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ rope_name: String,
+ ) {
+ let Ok(rope_tree) = parameter_tree.subtree(rope_name.as_str()) else {
+ return;
+ };
+
+ let cosines_view = rope_tree.leaf("cosines").unwrap();
+ self.cosines.borrow_mut().copy_from_array(&cosines_view);
+
+ let sines_view = rope_tree.leaf("sines").unwrap();
+ self.sines.borrow_mut().copy_from_array(&sines_view);
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/rope_type.rs b/crates/uzu/src/backends/metal/forward_pass/state/rope_type.rs
new file mode 100644
index 000000000..ea16a8877
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/rope_type.rs
@@ -0,0 +1,18 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ PartialOrd,
+ Ord,
+ Hash,
+ Serialize,
+ Deserialize,
+)]
+pub enum RopeType {
+ Global,
+ Local,
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/shared_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/shared_buffers.rs
new file mode 100644
index 000000000..6c9cfa81d
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/state/shared_buffers.rs
@@ -0,0 +1,152 @@
+use std::{cell::RefCell, rc::Rc};
+
+use half::{bf16, f16};
+
+use super::{super::ModelShape, EmbeddingsBuffers, RopeBuffers};
+use crate::{
+ Array, DataType, DeviceContext,
+ backends::metal::{MTLContext, MetalArray},
+ parameters::ParameterTree,
+};
+
+type ArrayCell = RefCell<MetalArray>;
+
+pub struct MoeExpertWeights {
+ pub w1: ArrayCell,
+ pub w2: ArrayCell,
+ pub w3: ArrayCell,
+}
+
+pub struct SharedBuffers {
+ pub embeddings: EmbeddingsBuffers,
+ pub global_rope: Option<RopeBuffers>,
+ pub local_rope: Option<RopeBuffers>,
+ pub moe_expert_weights: Option<Vec<MoeExpertWeights>>,
+ pub attention_sinks: Option<Vec<ArrayCell>>,
+}
+
+impl SharedBuffers {
+ pub fn new(
+ context: &MTLContext,
+ decoder_config: &crate::config::DecoderConfig,
+ model_shape: &ModelShape,
+ ) -> Self {
+ let embeddings = EmbeddingsBuffers::new(
+ context,
+ &decoder_config.embedding_config,
+ model_shape,
+ );
+
+ let global_rope = if decoder_config.global_rope_config.is_some() {
+ Some(RopeBuffers::new(context, model_shape))
+ } else {
+ None
+ };
+
+ let local_rope = if decoder_config.local_rope_config.is_some() {
+ Some(RopeBuffers::new(context, model_shape))
+ } else {
+ None
+ };
+
+ let moe_expert_weights = if matches!(
+ decoder_config.layer_config.mlp_config,
+ crate::config::MLPConfig::MixtureOfExperts(_)
+ ) {
+ Some(Vec::new())
+ } else {
+ None
+ };
+
+ let attention_sinks = if let Some(attention_config) =
+ decoder_config.layer_config.attention_config()
+ {
+ if attention_config.has_sinks {
+ let num_heads = decoder_config.num_heads;
+ Some(
+ (0..decoder_config.num_layers)
+ .map(|_| unsafe {
+ RefCell::new(context.array_uninitialized(
+ &[num_heads],
+ DataType::F32,
+ ))
+ })
+ .collect(),
+ )
+ } else {
+ None
+ }
+ } else {
+ None
+ };
+
+ Self {
+ embeddings,
+ global_rope,
+ local_rope,
+ moe_expert_weights,
+ attention_sinks,
+ }
+ }
+
+ pub fn update_data(
+ &mut self,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) {
+ self.embeddings.update_data(parameter_tree);
+
+ let transformer_tree = parameter_tree
+ .subtree("transformer")
+ .expect("transformer subtree not found");
+
+ if let Some(global_rope) = &mut self.global_rope {
+ global_rope
+ .update_data(&transformer_tree, String::from("global_rope"));
+ }
+ if let Some(local_rope) = &mut self.local_rope {
+ local_rope
+ .update_data(&transformer_tree, String::from("local_rope"));
+ }
+
+ if let Some(sinks_vec) = &mut self.attention_sinks {
+ for (layer_idx, sink_cell) in sinks_vec.iter_mut().enumerate() {
+ let layer_tree = transformer_tree
+ .subtree(&format!("layers.{}", layer_idx))
+ .unwrap();
+ let attn_tree = layer_tree.subtree("mixer").unwrap();
+ let sinks_arr = attn_tree.leaf("sinks").unwrap();
+ let mut dst = sink_cell.borrow_mut();
+ let dst_slice = dst.as_slice_mut::<f32>().unwrap();
+
+ match sinks_arr.data_type() {
+ DataType::F32 => {
+ let src = sinks_arr.as_slice::<f32>().unwrap();
+ dst_slice.copy_from_slice(src);
+ },
+ DataType::BF16 => {
+ let src = sinks_arr.as_slice::<bf16>().unwrap();
+ for (dst_val, src_val) in
+ dst_slice.iter_mut().zip(src.iter())
+ {
+ *dst_val = f32::from(*src_val);
+ }
+ },
+ DataType::F16 => {
+ let src = sinks_arr.as_slice::<f16>().unwrap();
+ for (dst_val, src_val) in
+ dst_slice.iter_mut().zip(src.iter())
+ {
+ *dst_val = f32::from(*src_val);
+ }
+ },
+ other => {
+ panic!(
+ "Unsupported attention sink data type: {:?}",
+ other
+ );
+ },
+ }
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/traces.rs b/crates/uzu/src/backends/metal/forward_pass/traces.rs
index 2fa689888..9035cf2d7 100644
--- a/crates/uzu/src/backends/metal/forward_pass/traces.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/traces.rs
@@ -7,7 +7,7 @@ use crate::{
type ArrayCell = RefCell<MetalArray>;
-pub struct DecoderLayerActivationTrace {
+pub struct LayerActivationTrace {
pub inputs: ArrayCell,
pub pre_attention_norm: ArrayCell,
pub attention: ArrayCell,
@@ -19,7 +19,7 @@ pub struct DecoderLayerActivationTrace {
pub outputs: ArrayCell,
}
-impl DecoderLayerActivationTrace {
+impl LayerActivationTrace {
pub fn new(
context: &MTLContext,
model_shape: &ModelShape,
@@ -68,21 +68,23 @@ impl DecoderLayerActivationTrace {
}
}
-pub struct DecoderActivationTrace {
- pub layer_results: Vec<Rc<RefCell<DecoderLayerActivationTrace>>>,
+pub struct ActivationTrace {
+ pub embedding_norm: Option<ArrayCell>,
+ pub layer_results: Vec<Rc<RefCell<LayerActivationTrace>>>,
pub output_norm: ArrayCell,
+ pub output_pooling: Option<ArrayCell>,
pub logits: ArrayCell,
}
-impl DecoderActivationTrace {
- pub fn new(
+impl ActivationTrace {
+ pub fn new_llm(
context: &MTLContext,
model_shape: &ModelShape,
suffix_length: usize,
) -> Self {
let layer_results = (0..model_shape.num_layers)
.map(|_| {
- Rc::new(RefCell::new(DecoderLayerActivationTrace::new(
+ Rc::new(RefCell::new(LayerActivationTrace::new(
context,
model_shape,
suffix_length,
@@ -91,11 +93,13 @@ impl DecoderActivationTrace {
.collect();
unsafe {
Self {
+ embedding_norm: None,
layer_results,
output_norm: RefCell::new(context.array_uninitialized(
&model_shape.main_shape(suffix_length),
model_shape.activation_data_type(),
)),
+ output_pooling: None,
logits: RefCell::new(context.array_uninitialized(
&model_shape.logits_shape(suffix_length),
model_shape.activation_data_type(),
@@ -103,4 +107,61 @@ impl DecoderActivationTrace {
}
}
}
+
+ pub fn new_classifier(
+ context: &MTLContext,
+ model_shape: &ModelShape,
+ suffix_length: usize,
+ num_labels: usize,
+ ) -> Self {
+ let layer_results = (0..model_shape.num_layers)
+ .map(|_| {
+ Rc::new(RefCell::new(LayerActivationTrace::new(
+ context,
+ model_shape,
+ suffix_length,
+ )))
+ })
+ .collect();
+ let model_dim = model_shape.main_shape(1)[1];
+ unsafe {
+ Self {
+ embedding_norm: Some(RefCell::new(
+ context.array_uninitialized(
+ &model_shape.main_shape(suffix_length),
+ model_shape.activation_data_type(),
+ ),
+ )),
+ layer_results,
+ output_norm: RefCell::new(context.array_uninitialized(
+ &model_shape.main_shape(suffix_length),
+ model_shape.activation_data_type(),
+ )),
+ output_pooling: Some(RefCell::new(
+ context.array_uninitialized(
+ &[1, model_dim],
+ model_shape.activation_data_type(),
+ ),
+ )),
+ logits: RefCell::new(context.array_uninitialized(
+ &[1, num_labels],
+ model_shape.activation_data_type(),
+ )),
+ }
+ }
+ }
+}
+
+impl ActivationTrace {
+ pub fn embedding_norm(&self) -> &ArrayCell {
+ self.embedding_norm
+ .as_ref()
+ .expect("embedding_norm is only available for classifier traces")
+ }
+
+ pub fn output_pooling(&self) -> &ArrayCell {
+ self.output_pooling
+ .as_ref()
+ .expect("output_pooling is only available for classifier traces")
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index 1a6a1f65e..1366ca20d 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -5,20 +5,9 @@ use metal::{
ComputePipelineState as MTLComputePipelineState, FunctionConstantValues,
MTLDataType, MTLSize,
};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
use thiserror::Error;
-use crate::{
- Array,
- backends::metal::{
- KernelDataType, MTLContext, MTLError,
- forward_pass::{
- ArrayId, ForwardPassState, HashMapId,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- metal_extensions::ComputeEncoderConditional,
- },
-};
+use crate::backends::metal::{KernelDataType, MTLContext, MTLError};
#[derive(Debug, Clone, Copy)]
pub enum AttentionKernelVariant {
@@ -26,7 +15,7 @@ pub enum AttentionKernelVariant {
TwoPass,
}
-type PipelineKey = (usize, bool);
+type PipelineKey = (usize, bool, bool); // (head_dim, has_sinks, is_causal)
pub struct AttentionKernelPipelines {
single_pass: HashMap<PipelineKey, MTLComputePipelineState>,
@@ -69,6 +58,7 @@ pub struct AttentionSinglePassArguments<'a> {
pub num_heads: usize,
pub suffix_length: usize,
pub head_dim: usize,
+ pub is_causal: bool,
}
pub struct AttentionTwoPassArguments<'a> {
@@ -94,6 +84,7 @@ pub struct AttentionTwoPassArguments<'a> {
pub num_heads: usize,
pub suffix_length: usize,
pub head_dim: usize,
+ pub is_causal: bool,
}
pub struct KVCacheUpdateArguments<'a> {
@@ -109,12 +100,14 @@ pub struct KVCacheUpdateArguments<'a> {
pub max_sequence_length: usize,
}
-fn make_function_constants(has_sinks_value: bool) -> FunctionConstantValues {
+fn make_function_constants(
+ has_sinks_value: bool,
+ is_causal_value: bool,
+) -> FunctionConstantValues {
let function_constants = FunctionConstantValues::new();
let has_mask_value = true;
let query_transposed_value = false;
- let do_causal_value = false;
let bool_mask_value = false;
let float_mask_value = true;
@@ -128,11 +121,6 @@ fn make_function_constants(has_sinks_value: bool) -> FunctionConstantValues {
MTLDataType::Bool,
21,
); // query_transposed
- function_constants.set_constant_value_at_index(
- &do_causal_value as *const bool as *const std::ffi::c_void,
- MTLDataType::Bool,
- 22,
- ); // do_causal
function_constants.set_constant_value_at_index(
&bool_mask_value as *const bool as *const std::ffi::c_void,
MTLDataType::Bool,
@@ -143,6 +131,11 @@ fn make_function_constants(has_sinks_value: bool) -> FunctionConstantValues {
MTLDataType::Bool,
24,
); // float_mask
+ function_constants.set_constant_value_at_index(
+ &is_causal_value as *const bool as *const std::ffi::c_void,
+ MTLDataType::Bool,
+ 22,
+ ); // do_causal
function_constants.set_constant_value_at_index(
&has_sinks_value as *const bool as *const std::ffi::c_void,
MTLDataType::Bool,
@@ -164,58 +157,55 @@ impl AttentionKernel {
let mut two_pass_1 = HashMap::new();
let mut two_pass_2 = HashMap::new();
- // Pre-generate all supported variants for both sink configurations
+ // Pre-generate all supported variants for sinks and causal configurations
for &has_sinks_value in &[false, true] {
- let function_constants = make_function_constants(has_sinks_value);
-
- for &head_dim in &supported_head_dims {
- let single_pass_name = format!(
- "attention_single_pass_{}_{}",
- data_suffix, head_dim
- );
- let single_pass_key =
- format!("{}_sinks_{}", single_pass_name, has_sinks_value);
- if let Ok((pipeline, _)) = context
- .compute_pipeline_state_with_reflection_cached(
- &single_pass_key,
- &single_pass_name,
- Some(&function_constants),
- )
- {
- single_pass.insert((head_dim, has_sinks_value), pipeline);
- }
+ for &is_causal_value in &[false, true] {
+ let function_constants =
+ make_function_constants(has_sinks_value, is_causal_value);
- let two_pass_1_name =
- format!("attention_2pass_1_{}_{}", data_suffix, head_dim);
- let two_pass_1_key =
- format!("{}_sinks_{}", two_pass_1_name, has_sinks_value);
- if let Ok((pipeline, _)) = context
- .compute_pipeline_state_with_reflection_cached(
- &two_pass_1_key,
- &two_pass_1_name,
- Some(&function_constants),
- )
- {
- two_pass_1.insert((head_dim, has_sinks_value), pipeline);
- }
+ for &head_dim in &supported_head_dims {
+ if let Ok((pipeline, _)) = context
+ .compute_pipeline_state_with_reflection(
+ &format!(
+ "attention_single_pass_{}_{}",
+ data_suffix, head_dim
+ ),
+ Some(&function_constants),
+ )
+ {
+ single_pass.insert(
+ (head_dim, has_sinks_value, is_causal_value),
+ pipeline,
+ );
+ }
- if !two_pass_2.contains_key(&head_dim) {
- let two_pass_2_name = format!(
- "attention_2pass_2_{}_{}",
- data_suffix, head_dim
- );
- let two_pass_2_key = format!(
- "{}_sinks_{}",
- two_pass_2_name, has_sinks_value
- );
if let Ok((pipeline, _)) = context
- .compute_pipeline_state_with_reflection_cached(
- &two_pass_2_key,
- &two_pass_2_name,
+ .compute_pipeline_state_with_reflection(
+ &format!(
+ "attention_2pass_1_{}_{}",
+ data_suffix, head_dim
+ ),
Some(&function_constants),
)
{
- two_pass_2.insert(head_dim, pipeline);
+ two_pass_1.insert(
+ (head_dim, has_sinks_value, is_causal_value),
+ pipeline,
+ );
+ }
+
+ if !two_pass_2.contains_key(&head_dim) {
+ if let Ok((pipeline, _)) = context
+ .compute_pipeline_state_with_reflection(
+ &format!(
+ "attention_2pass_2_{}_{}",
+ data_suffix, head_dim
+ ),
+ Some(&function_constants),
+ )
+ {
+ two_pass_2.insert(head_dim, pipeline);
+ }
}
}
}
@@ -242,15 +232,17 @@ impl AttentionKernel {
pub fn supports_single_pass(
&self,
head_dim: usize,
+ is_causal: bool,
) -> bool {
- self.pipelines.single_pass.contains_key(&(head_dim, false))
+ self.pipelines.single_pass.contains_key(&(head_dim, false, is_causal))
}
pub fn supports_two_pass(
&self,
head_dim: usize,
+ is_causal: bool,
) -> bool {
- self.pipelines.two_pass_1.contains_key(&(head_dim, false))
+ self.pipelines.two_pass_1.contains_key(&(head_dim, false, is_causal))
&& self.pipelines.two_pass_2.contains_key(&head_dim)
}
@@ -258,8 +250,10 @@ impl AttentionKernel {
&self,
sequence_length: usize,
head_dim: usize,
+ is_causal: bool,
) -> AttentionKernelVariant {
- if self.supports_two_pass(head_dim) && sequence_length > 1024 {
+ if self.supports_two_pass(head_dim, is_causal) && sequence_length > 1024
+ {
AttentionKernelVariant::TwoPass
} else {
AttentionKernelVariant::SinglePass
@@ -275,7 +269,7 @@ impl AttentionKernel {
let pipeline = self
.pipelines
.single_pass
- .get(&(args.head_dim, has_sinks))
+ .get(&(args.head_dim, has_sinks, args.is_causal))
.ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
compute_encoder.set_compute_pipeline_state(pipeline);
@@ -372,7 +366,7 @@ impl AttentionKernel {
let pass1_pipeline = self
.pipelines
.two_pass_1
- .get(&(args.head_dim, has_sinks))
+ .get(&(args.head_dim, has_sinks, args.is_causal))
.ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
let pass2_pipeline =
@@ -561,298 +555,3 @@ impl AttentionKernel {
Ok(())
}
}
-
-pub struct AttentionKernelEncodable {
- kernel: AttentionKernel,
- layer_index: usize,
- attention_scale: Option<f32>,
- has_sinks: bool,
-}
-
-impl AttentionKernelEncodable {
- pub fn new(
- context: &MTLContext,
- data_type: KernelDataType,
- layer_index: usize,
- attention_scale: Option<f32>,
- has_sinks: bool,
- ) -> Result<Self, AttentionError> {
- let kernel = AttentionKernel::new(context, data_type)?;
- Ok(Self {
- kernel,
- layer_index,
- attention_scale,
- has_sinks,
- })
- }
-}
-
-impl EncodableWithState for AttentionKernelEncodable {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
-
- compute_encoder.condition(
- parameters.predicate_ref(),
- || {
- self.encode_with_encoder_impl(
- state,
- compute_encoder,
- parameters,
- );
- },
- None::<fn()>,
- );
-
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
- ) {
- self.encode_with_encoder_impl(state, encoder, parameters);
- }
-}
-
-impl AttentionKernelEncodable {
- fn encode_with_encoder_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
- ) {
- let (
- suffix_length,
- num_heads,
- head_dim,
- num_groups,
- max_sequence_length,
- ) = {
- let qkv_binding = state.arrays(&[ArrayId::QKV]);
- let qkv_array = qkv_binding[0].borrow();
- let suffix_length = qkv_array.shape()[0];
-
- let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
- let queries_array = queries_binding[0].borrow();
- let num_heads = queries_array.shape()[0];
- let head_dim = queries_array.shape()[2];
-
- let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
- let keys_array = keys_binding[0].borrow();
- let num_groups = keys_array.shape()[0];
-
- let key_cache_binding =
- state.arrays(&[ArrayId::Keys(self.layer_index)]);
- let key_cache_array = key_cache_binding[0].borrow();
- let max_sequence_length = key_cache_array.shape()[1];
-
- (
- suffix_length,
- num_heads,
- head_dim,
- num_groups,
- max_sequence_length,
- )
- };
-
- let (segment_prefix_length, window_length) = {
- let cache = state.cache_layers.borrow();
- let layer = cache.data[self.layer_index]
- .as_transformer()
- .expect("Attention kernel expects transformer layer state");
- (
- layer.projected_segment_prefix_length(
- parameters.projection_step.unwrap_or(0),
- ),
- layer.window_length(),
- )
- };
-
- let sequence_length = segment_prefix_length + suffix_length;
-
- let gqa_factor = num_heads / num_groups;
- let scale =
- self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());
-
- let variant = self.kernel.choose_variant(sequence_length, head_dim);
-
- let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
- let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
- let qkv_binding = state.arrays(&[ArrayId::QKV]);
- let key_cache_binding =
- state.arrays(&[ArrayId::Keys(self.layer_index)]);
- let value_cache_binding =
- state.arrays(&[ArrayId::Values(self.layer_index)]);
- let attention_bias_binding =
- state.hashmaps(&[HashMapId::AttentionBias]);
- let attention_output_binding =
- state.arrays(&[ArrayId::AttentionOutput]);
-
- let attention_bias_map = attention_bias_binding[0].clone();
- let mask_kv_seq_stride = 1;
- let mask_q_seq_stride = sequence_length as i32;
- let mask_head_stride = 0;
- let sinks_binding = if self.has_sinks {
- Some(state.arrays(&[ArrayId::AttentionSinks(self.layer_index)]))
- } else {
- None
- };
-
- let mut rotated_keys_array = rotated_keys_binding[0].borrow_mut();
- let rotated_keys_buffer = unsafe { rotated_keys_array.mtl_buffer() };
-
- let mut qkv_array = qkv_binding[0].borrow_mut();
- let qkv_buffer = unsafe { qkv_array.mtl_buffer() };
-
- let mut key_cache_array = key_cache_binding[0].borrow_mut();
- let key_cache_buffer = unsafe { key_cache_array.mtl_buffer() };
-
- let mut value_cache_array = value_cache_binding[0].borrow_mut();
- let value_cache_buffer = unsafe { value_cache_array.mtl_buffer() };
-
- let mut queries_array = rotated_queries_binding[0].borrow_mut();
- let queries_buffer = unsafe { queries_array.mtl_buffer() };
-
- let mut attention_output_array =
- attention_output_binding[0].borrow_mut();
- let attention_output_buffer =
- unsafe { attention_output_array.mtl_buffer() };
-
- let attention_bias_buffer = attention_bias_map
- .get(&window_length)
- .map(|array| {
- let mut array_ref = array.borrow_mut();
- unsafe { array_ref.mtl_buffer().clone() }
- })
- .unwrap_or_else(|| {
- panic!(
- "Attention bias buffer not found for window length {:?}",
- window_length
- );
- });
-
- let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);
- let sums_binding = state.arrays(&[ArrayId::AttentionSums]);
- let maxs_binding = state.arrays(&[ArrayId::AttentionMaxs]);
-
- let mut partials_array = partials_binding[0].borrow_mut();
- let partials_buffer = unsafe { partials_array.mtl_buffer() };
-
- let mut sums_array = sums_binding[0].borrow_mut();
- let sums_buffer = unsafe { sums_array.mtl_buffer() };
-
- let mut maxs_array = maxs_binding[0].borrow_mut();
- let maxs_buffer = unsafe { maxs_array.mtl_buffer() };
-
- let sinks_buffer = sinks_binding.as_ref().map(|binding| unsafe {
- binding[0].borrow_mut().mtl_buffer().clone()
- });
-
- if let Err(e) = self.kernel.encode_kv_cache_update(
- encoder,
- KVCacheUpdateArguments {
- rotated_keys_buffer: &rotated_keys_buffer,
- qkv_buffer: &qkv_buffer,
- key_cache_buffer: &key_cache_buffer,
- value_cache_buffer: &value_cache_buffer,
- num_groups,
- num_heads,
- head_dim,
- suffix_length,
- segment_prefix_length,
- max_sequence_length,
- },
- ) {
- eprintln!("Failed to encode KV cache update: {:?}", e);
- return;
- }
-
- let k_head_stride = (max_sequence_length * head_dim) as i32;
- let k_seq_stride = head_dim as i32;
- let v_head_stride = (max_sequence_length * head_dim) as i32;
- let v_seq_stride = head_dim as i32;
-
- match variant {
- AttentionKernelVariant::SinglePass => {
- if let Err(e) = self.kernel.encode_single_pass(
- encoder,
- AttentionSinglePassArguments {
- queries_buffer: &queries_buffer,
- keys_buffer: &key_cache_buffer,
- values_buffer: &value_cache_buffer,
- output_buffer: &attention_output_buffer,
- gqa_factor: gqa_factor as i32,
- sequence_length: sequence_length as i32,
- k_head_stride,
- k_seq_stride,
- v_head_stride,
- v_seq_stride,
- scale,
- mask_buffer: Some(&attention_bias_buffer),
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
- sinks_buffer: sinks_buffer.as_ref(),
- num_heads,
- suffix_length,
- head_dim,
- },
- ) {
- eprintln!(
- "Failed to encode single-pass attention: {:?}",
- e
- );
- }
- },
- AttentionKernelVariant::TwoPass => {
- if let Err(e) = self.kernel.encode_two_pass(
- encoder,
- AttentionTwoPassArguments {
- queries_buffer: &queries_buffer,
- keys_buffer: &key_cache_buffer,
- values_buffer: &value_cache_buffer,
- partials_buffer: &partials_buffer,
- sums_buffer: &sums_buffer,
- maxs_buffer: &maxs_buffer,
- output_buffer: &attention_output_buffer,
- gqa_factor: gqa_factor as i32,
- sequence_length: sequence_length as i32,
- k_head_stride,
- k_seq_stride,
- v_head_stride,
- v_seq_stride,
- scale,
- mask_buffer: Some(&attention_bias_buffer),
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
- sinks_buffer: sinks_buffer.as_ref(),
- num_heads,
- suffix_length,
- head_dim,
- },
- ) {
- eprintln!("Failed to encode two-pass attention: {:?}", e);
- }
- },
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
index 14ecbf88e..499a22776 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
@@ -1,32 +1,16 @@
-use std::{mem::size_of, rc::Rc};
+use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, ComputeCommandEncoderRef,
ComputePipelineState as MTLComputePipelineState, MTLSize,
};
-use mpsgraph::CommandBuffer;
-use super::super::{
- MTLContext,
- forward_pass::{
- ArrayId, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- metal_extensions::ComputeEncoderConditional,
-};
+use super::super::MTLContext;
use crate::{
- Array, DataType, backends::metal::MTLError, config::QuantizationMode,
- parameters::ParameterTree,
+ DataType, backends::metal::encodable_block::QuantizedEmbeddingError,
+ config::QuantizationMode,
};
-#[derive(Debug, thiserror::Error)]
-pub enum QuantizedEmbeddingError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MTLError),
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
-}
-
pub struct QuantizedEmbeddingLookupKernel {
pipeline: MTLComputePipelineState,
}
@@ -83,7 +67,7 @@ impl QuantizedEmbeddingLookupKernel {
&self,
encoder: &ComputeCommandEncoderRef,
args: QuantizedEmbeddingLookupArguments,
- ) {
+ ) -> Result<(), QuantizedEmbeddingError> {
encoder.set_compute_pipeline_state(&self.pipeline);
// Set buffers
@@ -125,460 +109,7 @@ impl QuantizedEmbeddingLookupKernel {
MTLSize::new(threadgroups, 1, 1),
MTLSize::new(threads_per_threadgroup, 1, 1),
);
- }
-}
-
-pub struct QuantizedEmbeddingLookupKernelBlock {
- kernel: QuantizedEmbeddingLookupKernel,
- weights_buffer: MTLBuffer,
- scales_buffer: MTLBuffer,
- biases_buffer: MTLBuffer,
- vocab_size: u32,
- model_dim: u32,
- group_size: u32,
-}
-
-impl QuantizedEmbeddingLookupKernelBlock {
- pub fn new(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
- let packing_divisor = mode.packing_divisor();
-
- let kernel =
- QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;
-
- // Load weights [vocab_size, model_dim/packing_divisor] as storage_type
- let mut weights = match parameter_tree.leaf("weights") {
- Ok(weights) => weights,
- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load weights: {:?}",
- e
- )))
- })?,
- };
-
- if weights.data_type() != mode.storage_type() {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Expected packed weights of type {:?}, got {:?}",
- mode.storage_type(),
- weights.data_type()
- )),
- ));
- }
-
- // Load scales [vocab_size, num_groups]
- let mut scales = match parameter_tree.leaf("scales") {
- Ok(scales) => scales,
- Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load scales: {:?}",
- e
- )))
- })?,
- };
-
- // Validate shapes and types
- let num_groups = (model_dim + group_size - 1) / group_size;
- if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim / packing_divisor
- )),
- ));
- }
- if scales.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",
- scales.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if scales.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
- scales.data_type(),
- ));
- }
-
- // Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")
- let biases_buffer: MTLBuffer = match parameter_tree
- .leaf("biases")
- .or_else(|_| parameter_tree.leaf("output_biases"))
- {
- Ok(mut deq_biases) => {
- if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",
- deq_biases.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if deq_biases.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
- deq_biases.data_type(),
- ));
- }
- unsafe { deq_biases.mtl_buffer().to_owned() }
- },
- Err(_) => {
- let elem_size: usize = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(
- QuantizedEmbeddingError::UnsupportedDataType(other),
- );
- },
- };
- let size_bytes = (vocab_size * num_groups * elem_size) as u64;
- let buf = mtl_context.device.new_buffer(
- size_bytes,
- metal::MTLResourceOptions::StorageModeShared,
- );
- unsafe {
- std::ptr::write_bytes(
- buf.contents(),
- 0,
- size_bytes as usize,
- );
- }
- buf
- },
- };
-
- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
- let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };
-
- Ok(Self {
- kernel,
- weights_buffer,
- scales_buffer,
- biases_buffer,
- vocab_size: vocab_size as u32,
- model_dim: model_dim as u32,
- group_size: group_size as u32,
- })
- }
-}
-
-impl EncodableWithState for QuantizedEmbeddingLookupKernelBlock {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &CommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
-
- self.encode_impl(state, encoder);
-
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &metal::ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- self.encode_impl(state, encoder);
- }
-}
-
-impl QuantizedEmbeddingLookupKernelBlock {
- fn encode_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &metal::ComputeCommandEncoderRef,
- ) {
- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
- let batch_size = state.active_suffix_length();
- let mut token_ids_array_mut = arrays[0].borrow_mut();
- let mut output_array_mut = arrays[1].borrow_mut();
-
- let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };
- let output_buffer = unsafe { output_array_mut.mtl_buffer() };
-
- let args = QuantizedEmbeddingLookupArguments {
- token_ids_buffer,
- weights_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- biases_buffer: &self.biases_buffer,
- output_buffer,
- batch_size: batch_size as u32,
- vocab_size: self.vocab_size,
- model_dim: self.model_dim,
- group_size: self.group_size,
- };
-
- self.kernel.encode(encoder, args);
- }
-}
-
-// Quantized embedding readout block - uses quantized matmul for efficiency
-pub struct QuantizedEmbeddingReadoutKernelBlock {
- kernel: super::super::kernel::quant_matmul::QuantizedMatmulKernel,
- weights_buffer: MTLBuffer,
- scales_buffer: MTLBuffer,
- biases_buffer: MTLBuffer,
- vocab_size: usize,
- model_dim: usize,
-}
-
-impl QuantizedEmbeddingReadoutKernelBlock {
- pub fn new(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
- // Load weights [vocab_size, model_dim/2] as U8
- let mut weights = match parameter_tree.leaf("weights") {
- Ok(weights) => weights,
- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load weights: {:?}",
- e
- )))
- })?,
- };
-
- // Load scales [vocab_size, num_groups]
- let mut scales = match parameter_tree.leaf("scales") {
- Ok(scales) => scales,
- Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load scales: {:?}",
- e
- )))
- })?,
- };
-
- // Validate shapes
- let num_groups = (model_dim + group_size - 1) / group_size;
- let packing_divisor = mode.packing_divisor();
-
- // Determine if weights are transposed by checking shape
- let weights_transposed = weights.shape()[0] == vocab_size;
-
- if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim / packing_divisor
- )),
- ));
- }
- if scales.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",
- scales.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if scales.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
- scales.data_type(),
- ));
- }
-
- // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]
- let biases_buffer: MTLBuffer = match parameter_tree
- .leaf("biases")
- .or_else(|_| parameter_tree.leaf("output_biases"))
- {
- Ok(mut deq_biases) => {
- if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",
- deq_biases.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if deq_biases.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
- deq_biases.data_type(),
- ));
- }
- unsafe { deq_biases.mtl_buffer().to_owned() }
- },
- Err(_) => {
- // Allocate zero-initialized biases buffer
- let elem_size: usize = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(
- QuantizedEmbeddingError::UnsupportedDataType(other),
- );
- },
- };
- let size_bytes = (vocab_size * num_groups * elem_size) as u64;
- let buf = mtl_context.device.new_buffer(
- size_bytes,
- metal::MTLResourceOptions::StorageModeShared,
- );
- unsafe {
- std::ptr::write_bytes(
- buf.contents(),
- 0,
- size_bytes as usize,
- );
- }
- buf
- },
- };
-
- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
- let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };
-
- use super::super::kernel::quant_matmul::{
- QuantizationType, QuantizedMatmulKernel,
- };
-
- let kernel = QuantizedMatmulKernel::new(
- mtl_context,
- data_type,
- group_size,
- model_dim,
- vocab_size,
- mode,
- QuantizationType::Mlx,
- weights_transposed,
- )
- .map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to create kernel: {:?}",
- e
- )))
- })?;
-
- Ok(Self {
- kernel,
- weights_buffer,
- scales_buffer,
- biases_buffer,
- vocab_size,
- model_dim,
- })
- }
-}
-
-impl EncodableWithState for QuantizedEmbeddingReadoutKernelBlock {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &CommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
-
- encoder.condition(
- parameters.predicate_ref(),
- || {
- self.encode_impl(state, encoder);
- },
- None::<fn()>,
- );
-
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &metal::ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- self.encode_impl(state, encoder);
- }
-}
-
-impl QuantizedEmbeddingReadoutKernelBlock {
- fn encode_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &metal::ComputeCommandEncoderRef,
- ) {
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.active_suffix_length();
- let mut input_array_mut = arrays[0].borrow_mut();
- let mut output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = unsafe { input_array_mut.mtl_buffer() };
- let output_buffer = unsafe { output_array_mut.mtl_buffer() };
-
- // For transposed matmul: input @ weights.T
- // where weights is [vocab_size, model_dim]
- use super::super::kernel::quant_matmul::{
- QuantizationType, QuantizedMatmulArguments,
- };
-
- let args = QuantizedMatmulArguments {
- a_buffer: input_buffer,
- b_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- zero_points_or_biases_buffer: &self.biases_buffer,
- output_buffer,
- batch: batch_size as i32,
- input_dim: self.model_dim as i32,
- output_dim: self.vocab_size as i32,
- quantization_type: QuantizationType::Mlx,
- };
- self.kernel
- .encode(encoder, args)
- .expect("Failed to encode quantized embedding readout kernel");
+ Ok(())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
new file mode 100644
index 000000000..37321f887
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -0,0 +1,164 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+
+using namespace metal;
+
+#define BLOCK_SIZE 1024
+#define SIMD_SIZE 32
+#define GRAIN_SIZE 4
+
+// LayerNorm: subtract mean, then normalize (for BERT-style models)
+template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>
+void layer_norm_core(
+ const device InputT* input_data,
+ const device ScaleT* scales_data,
+ device OutputT* output_data,
+ uint element_count,
+ constant float& epsilon,
+ constant float& scale_offset,
+ threadgroup AccumT* shared_mean,
+ threadgroup AccumT* shared_variance,
+ uint thread_in_row,
+ bool full_layer
+) {
+ AccumT partial_sum = static_cast<AccumT>(0.0f);
+
+ // Compute mean: sum all elements
+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
+ base_i += BLOCK_SIZE * GRAIN_SIZE) {
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ if (i < element_count) {
+ partial_sum += static_cast<AccumT>(input_data[i]);
+ }
+ }
+ }
+
+ // Reduce to get total mean
+ AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
+ partial_sum,
+ shared_mean,
+ thread_in_row
+ );
+ AccumT mean = total_sum / static_cast<AccumT>(element_count);
+
+ // Compute variance: sum of squared deviations
+ AccumT partial_var_sum = static_cast<AccumT>(0.0f);
+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
+ base_i += BLOCK_SIZE * GRAIN_SIZE) {
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ if (i < element_count) {
+ AccumT centered = static_cast<AccumT>(input_data[i]) - mean;
+ partial_var_sum += centered * centered;
+ }
+ }
+ }
+
+ // Reduce to get total variance
+ AccumT total_var = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
+ partial_var_sum,
+ shared_variance,
+ thread_in_row
+ );
+ AccumT variance = total_var / static_cast<AccumT>(element_count);
+ AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));
+
+ // Apply normalization and scaling
+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
+ base_i += BLOCK_SIZE * GRAIN_SIZE) {
+ AccumT vals[GRAIN_SIZE];
+ AccumT scaled_vals[GRAIN_SIZE];
+
+ // Load and center
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ vals[j] = (i < element_count)
+ ? (static_cast<AccumT>(input_data[i]) - mean)
+ : 0.0f;
+ }
+
+ // Normalize and scale
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ if (i >= element_count) {
+ continue;
+ }
+
+ AccumT normalized_high = vals[j] * inv_std;
+
+ if (full_layer) {
+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +
+ static_cast<AccumT>(scale_offset);
+ scaled_vals[j] = normalized_high * scale_value_high;
+ } else {
+ OutputT normalized_low = static_cast<OutputT>(normalized_high);
+ OutputT scale_value_low = static_cast<OutputT>(
+ static_cast<AccumT>(scales_data[i]) +
+ static_cast<AccumT>(scale_offset)
+ );
+ OutputT product_low = normalized_low * scale_value_low;
+ scaled_vals[j] = static_cast<AccumT>(product_low);
+ }
+ }
+
+ // Store
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ if (i < element_count) {
+ output_data[i] = static_cast<OutputT>(scaled_vals[j]);
+ }
+ }
+ }
+}
+
+// X-macro table defining all supported type combinations (same as RMSNorm)
+#define FOREACH_LAYER_NORM_COMBO(_) \
+ /* bfloat combinations most commonly used */ \
+ _(bfloat, bfloat, bfloat, float, false, bf16_bf16_bf16_f32_norm) \
+ _(bfloat, bfloat, bfloat, float, true, bf16_bf16_bf16_f32_full) \
+ _(bfloat, float, bfloat, float, false, bf16_f32_bf16_f32_norm) \
+ _(bfloat, float, bfloat, float, true, bf16_f32_bf16_f32_full) \
+ /* float combinations */ \
+ _(float, float, float, float, false, f32_f32_f32_f32_norm) \
+ _(float, float, float, float, true, f32_f32_f32_f32_full) \
+ /* half combinations */ \
+ _(half, half, half, float, false, f16_f16_f16_f32_norm) \
+ _(half, half, half, float, true, f16_f16_f16_f32_full)
+
+// Generate LayerNorm kernels
+#define DEFINE_LAYER_NORM_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
+ kernel void layer_norm_##SUF( \
+ const device IN* input [[buffer(0)]], \
+ const device SC* scales [[buffer(1)]], \
+ device OUT* output [[buffer(2)]], \
+ constant uint& batch_size [[buffer(3)]], \
+ constant uint& model_dim [[buffer(4)]], \
+ constant float& epsilon [[buffer(5)]], \
+ constant float& scale_offset [[buffer(6)]], \
+ uint batch_idx [[threadgroup_position_in_grid]], \
+ uint thread_in_row [[thread_position_in_threadgroup]] \
+ ) { \
+ if (batch_idx >= batch_size) \
+ return; \
+ \
+ threadgroup ACC shared_mean[SIMD_SIZE]; \
+ threadgroup ACC shared_variance[SIMD_SIZE]; \
+ const uint input_offset = batch_idx * model_dim; \
+ \
+ layer_norm_core<IN, SC, OUT, ACC>( \
+ input + input_offset, \
+ scales, \
+ output + input_offset, \
+ model_dim, \
+ epsilon, \
+ scale_offset, \
+ shared_mean, \
+ shared_variance, \
+ thread_in_row, \
+ FULL_LAYER \
+ ); \
+ }
+
+FOREACH_LAYER_NORM_COMBO(DEFINE_LAYER_NORM_KERNEL)
+#undef DEFINE_LAYER_NORM_KERNEL
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/layer_norm/mod.rs
new file mode 100644
index 000000000..0c9be22d3
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/mod.rs
@@ -0,0 +1,157 @@
+use std::mem::size_of;
+
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState, MTLSize,
+};
+
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError},
+};
+
+#[derive(Debug)]
+pub struct LayerNormArguments<'a> {
+ pub input_buffer: &'a MTLBuffer,
+ pub scales_buffer: &'a MTLBuffer,
+ pub output_buffer: &'a MTLBuffer,
+ pub batch_size: i32,
+ pub model_dim: i32,
+ pub epsilon: f32,
+ pub scale_offset: f32,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum LayerNormError {
+ #[error("Metal error: {0}")]
+ MetalError(#[from] MTLError),
+ #[error(
+ "Unsupported data type combination: input={input:?}, scales={scales:?}, output={output:?}, accumulation={accumulation:?}"
+ )]
+ UnsupportedDataType {
+ input: DataType,
+ scales: DataType,
+ output: DataType,
+ accumulation: DataType,
+ },
+}
+
+pub struct LayerNormKernel {
+ pipeline: MTLComputePipelineState,
+}
+
+impl LayerNormKernel {
+ pub fn new(
+ context: &MTLContext,
+ input_data_type: DataType,
+ scales_data_type: DataType,
+ output_data_type: DataType,
+ accumulation_data_type: DataType,
+ full_layer: bool,
+ ) -> Result<Self, LayerNormError> {
+ let kernel_name = Self::kernel_name(
+ input_data_type,
+ scales_data_type,
+ output_data_type,
+ accumulation_data_type,
+ full_layer,
+ )?;
+
+ let pipeline = context
+ .compute_pipeline_state(&kernel_name, None)
+ .map_err(LayerNormError::MetalError)?;
+
+ Ok(Self {
+ pipeline,
+ })
+ }
+
+ fn kernel_name(
+ input: DataType,
+ scales: DataType,
+ output: DataType,
+ accumulation: DataType,
+ full_layer: bool,
+ ) -> Result<String, LayerNormError> {
+ let input_str = Self::type_to_string(input)?;
+ let scales_str = Self::type_to_string(scales)?;
+ let output_str = Self::type_to_string(output)?;
+ let accum_str = Self::type_to_string(accumulation)?;
+ let mode_str = if full_layer {
+ "full"
+ } else {
+ "norm"
+ };
+
+ Ok(format!(
+ "layer_norm_{}_{}_{}_{}_{}",
+ input_str, scales_str, output_str, accum_str, mode_str
+ ))
+ }
+
+ fn type_to_string(
+ data_type: DataType
+ ) -> Result<&'static str, LayerNormError> {
+ match data_type {
+ DataType::F32 => Ok("f32"),
+ DataType::F16 => Ok("f16"),
+ DataType::BF16 => Ok("bf16"),
+ _ => Err(LayerNormError::UnsupportedDataType {
+ input: data_type,
+ scales: data_type,
+ output: data_type,
+ accumulation: data_type,
+ }),
+ }
+ }
+
+ pub fn encode(
+ &self,
+ compute_encoder: &ComputeCommandEncoderRef,
+ args: LayerNormArguments,
+ ) -> Result<(), LayerNormError> {
+ compute_encoder.set_compute_pipeline_state(&self.pipeline);
+ compute_encoder.set_buffer(0, Some(args.input_buffer), 0);
+ compute_encoder.set_buffer(1, Some(args.scales_buffer), 0);
+ compute_encoder.set_buffer(2, Some(args.output_buffer), 0);
+ compute_encoder.set_bytes(
+ 3,
+ size_of::<i32>() as u64,
+ &args.batch_size as *const i32 as *const _,
+ );
+ compute_encoder.set_bytes(
+ 4,
+ size_of::<i32>() as u64,
+ &args.model_dim as *const i32 as *const _,
+ );
+ compute_encoder.set_bytes(
+ 5,
+ size_of::<f32>() as u64,
+ &args.epsilon as *const f32 as *const _,
+ );
+ compute_encoder.set_bytes(
+ 6,
+ size_of::<f32>() as u64,
+ &args.scale_offset as *const f32 as *const _,
+ );
+
+ let threadgroups_per_grid = MTLSize {
+ width: args.batch_size as u64,
+ height: 1,
+ depth: 1,
+ };
+
+ let threads_per_threadgroup = MTLSize {
+ width: 1024,
+ height: 1,
+ depth: 1,
+ };
+
+ compute_encoder.dispatch_thread_groups(
+ threadgroups_per_grid,
+ threads_per_threadgroup,
+ );
+
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
index 769fd723f..bae2543ef 100644
--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
@@ -10,10 +10,7 @@ use mpsgraph::CommandBuffer as MPSCommandBuffer;
use crate::backends::metal::{
MTLContext,
error::MTLError,
- forward_pass::{
- ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
+ forward_pass::{EncodableBlock, EncodingParameters, ForwardPassState},
image::Image,
metal_extensions::ComputeEncoderDispatch,
};
@@ -85,7 +82,7 @@ impl ScalePadNormalizeImage {
}
}
-impl EncodableWithState for ScalePadNormalizeImage {
+impl EncodableBlock for ScalePadNormalizeImage {
fn encode(
&self,
_state: &mut ForwardPassState,
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
index 1d4e3e541..400354529 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
@@ -2,20 +2,10 @@ use metal::{
Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
MTLSize,
};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
use crate::{
DataType,
- array::Array,
- backends::metal::{
- MTLContext, MTLError,
- forward_pass::{
- ArrayId, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- kernel::QuantizedLinearKernelBlock,
- metal_extensions::ComputeEncoderConditional,
- },
+ backends::metal::{MTLContext, MTLError},
config::Activation,
};
@@ -50,7 +40,7 @@ impl MlpGateActMulEncodable {
fused_up: &MTLBuffer,
hidden: &MTLBuffer,
m: i32,
- ) {
+ ) -> Result<(), MTLError> {
self.kernel.encode(
encoder,
&self.activation,
@@ -58,14 +48,12 @@ impl MlpGateActMulEncodable {
hidden,
m,
self.hidden_dim as i32,
- );
+ )
}
}
impl MlpGateActMulKernel {
- fn kernel_name_for_type(
- data_type: DataType
- ) -> Result<&'static str, MTLError> {
+ fn kernel_name_for_type(data_type: DataType) -> Result<&'static str, MTLError> {
match data_type {
DataType::F16 => Ok("mlp_activation_mul_f16"),
DataType::F32 => Ok("mlp_activation_mul_f32"),
@@ -77,22 +65,15 @@ impl MlpGateActMulKernel {
}
}
- pub fn new(
- context: &MTLContext,
- data_type: DataType,
- ) -> Result<Self, MTLError> {
+ pub fn new(context: &MTLContext, data_type: DataType) -> Result<Self, MTLError> {
let fn_name = Self::kernel_name_for_type(data_type)?;
let pipeline = context.compute_pipeline_state(fn_name, None)?;
- Ok(Self {
- pipeline,
- })
+ Ok(Self { pipeline })
}
fn act_code(act: &Activation) -> u16 {
match act {
- Activation::SiLU {
- ..
- } => 0,
+ Activation::SiLU { .. } => 0,
Activation::Gelu => 1,
Activation::Identity => {
panic!("Identity activation is not supported for MLP kernels")
@@ -108,7 +89,7 @@ impl MlpGateActMulKernel {
hidden_buffer: &MTLBuffer,
m: i32,
h: i32,
- ) {
+ ) -> Result<(), MTLError> {
let act_code = Self::act_code(activation);
let threads_per_tg = MTLSize::new(64, 1, 1);
encoder.set_compute_pipeline_state(&self.pipeline);
@@ -132,106 +113,6 @@ impl MlpGateActMulKernel {
let grid = MTLSize::new(h as u64, m as u64, 1);
encoder.dispatch_threads(grid, threads_per_tg);
- }
-}
-
-pub struct MlpBlockEncodable {
- up: QuantizedLinearKernelBlock,
- gate: MlpGateActMulEncodable,
- down: QuantizedLinearKernelBlock,
-}
-
-impl MlpBlockEncodable {
- pub fn new(
- up: QuantizedLinearKernelBlock,
- gate: MlpGateActMulEncodable,
- down: QuantizedLinearKernelBlock,
- ) -> Self {
- Self {
- up,
- gate,
- down,
- }
- }
-}
-
-impl EncodableWithState for MlpBlockEncodable {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- params: &EncodingParameters,
- ) {
- // Up
- self.up.encode(state, command_buffer, params);
- // Gate act+mul (fused_up -> hidden)
- let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
- let mut fused = arrays[0].borrow_mut();
- let mut hidden = arrays[1].borrow_mut();
- let m = fused.shape()[0] as i32;
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let encoder = mtl_command_buffer.new_compute_command_encoder();
-
- let fused_buf = unsafe { fused.mtl_buffer() };
- let hidden_buf = unsafe { hidden.mtl_buffer() };
-
- encoder.condition(
- params.predicate_ref(),
- || {
- self.gate.encode(encoder, fused_buf, hidden_buf, m);
- },
- None::<fn()>,
- );
-
- encoder.end_encoding();
- drop(fused);
- drop(hidden);
- // Down
- self.down.encode(state, command_buffer, params);
-
- if params.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
- ) {
- self.encode_with_encoder_impl(state, encoder, parameters);
- }
-}
-
-impl MlpBlockEncodable {
- fn encode_with_encoder_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- params: &EncodingParameters,
- ) {
- // Up
- self.up.encode_with_shared_encoder(state, encoder, params);
- // Gate act+mul (fused_up -> hidden)
- let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
- let mut fused = arrays[0].borrow_mut();
- let mut hidden = arrays[1].borrow_mut();
- let m = fused.shape()[0] as i32;
- let fused_buf = unsafe { fused.mtl_buffer() };
- let hidden_buf = unsafe { hidden.mtl_buffer() };
- self.gate.encode(encoder, fused_buf, hidden_buf, m);
- drop(fused);
- drop(hidden);
- // Down
- self.down.encode_with_shared_encoder(state, encoder, params);
+ Ok(())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 5911a90b4..753dc00c6 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -2,51 +2,59 @@ pub mod attention;
mod data_type;
pub mod embedding;
pub mod kv_cache_update;
-pub mod linear;
+pub mod layer_norm;
pub mod media_kernels;
pub mod mlp;
pub mod moe;
+pub mod pooling;
pub mod quant_matmul;
pub mod rms_norm;
pub mod rope;
pub mod sampling;
+pub mod sigmoid;
pub mod ssm;
mod tensor_add_bias;
mod tensor_add_swap;
mod tensor_copy;
pub mod token_copy;
-pub use attention::{AttentionKernel, AttentionKernelEncodable};
+pub use attention::{
+ AttentionError, AttentionKernel, AttentionKernelVariant,
+ AttentionSinglePassArguments, AttentionTwoPassArguments,
+ KVCacheUpdateArguments,
+};
pub use data_type::KernelDataType;
pub use kv_cache_update::KVCacheUpdate;
+pub use layer_norm::{LayerNormArguments, LayerNormError, LayerNormKernel};
pub use moe::{
- MoeBlockBasesArguments, MoeBlockEncodable, MoeCountsOffsetsFusedArguments,
+ MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,
MoeCountsOffsetsFusedError, MoeCountsOffsetsFusedKernel,
MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,
MoeExpertsTwoPassDecodeKernel, MoeExpertsTwoPassPrefillKernel,
MoeFinalizeArguments, MoeFinalizeError, MoeFinalizeKernel,
- MoeScatterArguments, MoeScatterError, MoeScatterKernels,
- MoeScatterWithMapArguments,
+ MoeGatherArguments, MoeGatherKernel, MoeRouterTopKArguments,
+ MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError,
+ MoeScatterKernels, MoeScatterWithMapArguments,
};
+pub use pooling::PoolingKernel;
pub use rms_norm::{
- QKNormArguments, QKNormKernelEncodable, RMSNormArguments, RMSNormKernel,
- RMSNormKernelEncodable, RMSNormKernelType,
+ QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormError,
+ RMSNormKernel, RMSNormKernelType,
};
-pub use rope::{RopeKernel, RopeKernelEncodable};
-pub use sampling::{SamplingKernel, SamplingKernelEncodable};
-pub(crate) use ssm::MambaMixerEncodable;
+pub use rope::{RopeError, RopeKernel, RopeKernelArguments};
+pub use sampling::{ArgmaxStrategy, SamplingError, SamplingKernel};
+pub use sigmoid::SigmoidKernel;
pub use ssm::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,
SSDUpdateKernel, SSMKernelError, SplitInProjArguments, SplitInProjKernel,
};
pub use tensor_add_bias::TensorAddBias;
-pub use tensor_add_swap::TensorAddSwap;
-pub use tensor_copy::TensorCopy;
+pub use tensor_add_swap::TensorAddSwapKernel;
+pub use tensor_copy::TensorCopyKernel;
pub use token_copy::TokenCopyKernel;
use super::{MTLContext, metal_extensions};
-pub use crate::backends::metal::kernel::linear::QuantizedLinearKernelBlock;
pub mod media {
pub use super::media_kernels::{
diff --git a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs b/crates/uzu/src/backends/metal/kernel/moe/encodable.rs
deleted file mode 100644
index ff59b0e77..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs
+++ /dev/null
@@ -1,833 +0,0 @@
-//! MoE Block Encodable with path selection:
-//! - suffix == 1: Single-token fused decode (no scatter/gather)
-//! - suffix <= 32: Indirect decode path (GEMV-based)
-//! - suffix > 32: Indirect prefill path (GEMM/MMA-based)
-
-use std::{cell::RefCell, rc::Rc};
-
-use metal::CommandBufferRef;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
-
-use super::{
- MoeCountsOffsetsFusedArguments, MoeCountsOffsetsFusedKernel,
- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernel,
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernel,
- MoeExpertsTwoPassPrefillKernel, MoeFinalizeArguments, MoeFinalizeKernel,
- MoeGatherArguments, MoeGatherKernel, MoeRouterTopKArguments,
- MoeRouterTopKKernel, MoeScatterKernels, MoeScatterWithMapArguments,
-};
-use crate::{
- DataType,
- backends::metal::{
- KernelDataType, MTLContext, MetalArray,
- forward_pass::{
- ArrayId, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- },
- config::{
- Activation, LinearConfig, MixtureOfExpertsConfig,
- mlp::RoutingFunctionConfig,
- },
- parameters::ParameterTree,
-};
-
-/// Threshold for decode vs prefill path selection
-const INDIRECT_DECODE_THRESHOLD: usize = 32;
-
-enum RouterBlock {
- Metal {
- weights_buf: metal::Buffer,
- biases_buf: metal::Buffer,
- },
-}
-
-#[derive(Clone)]
-pub struct SharedMoeWeights {
- pub w13_buf: Rc<metal::Buffer>,
- pub w2_buf: Rc<metal::Buffer>,
- pub up_biases_buf: Rc<metal::Buffer>,
- pub down_biases_buf: Rc<metal::Buffer>,
-}
-
-/// Buffers used by MoE encoding paths
-struct MoeBuffers {
- main: metal::Buffer,
- topk_ids: metal::Buffer,
- topk_probs: metal::Buffer,
- offsets: metal::Buffer,
- sumk: metal::Buffer,
- bucketed_ids: metal::Buffer,
- bucketed_probs: metal::Buffer,
- x_perm: metal::Buffer,
- tok2row: metal::Buffer,
- y_partial: metal::Buffer,
- tile_counts: metal::Buffer,
- tile_offsets: metal::Buffer,
- tile_map: metal::Buffer,
- total_tiles: metal::Buffer,
- dispatch_args: metal::Buffer,
- partials: metal::Buffer,
- block_bases: metal::Buffer,
- block_alloc: metal::Buffer,
- hidden: metal::Buffer,
- row_expert_map: metal::Buffer,
-}
-
-pub struct MoeBlockEncodable {
- // Router
- router: RouterBlock,
- router_data_type: KernelDataType,
- router_renorm: bool,
- router_topk_kernel: MoeRouterTopKKernel,
-
- // Indirect path shared components
- counts_offsets_kernel: MoeCountsOffsetsFusedKernel,
- scatter_kernels: MoeScatterKernels,
- gather_kernel: MoeGatherKernel,
- finalize_kernel: MoeFinalizeKernel,
-
- // Indirect path variants
- experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeKernel,
- experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillKernel,
-
- // Single-token decode path
- single_decode_kernel: MoeExpertsSingleDecodeKernel,
-
- // Config and weights
- moe_config: MixtureOfExpertsConfig,
- model_dim: usize,
- hidden_dim: usize,
- data_type: KernelDataType,
- shared_weights: SharedMoeWeights,
-}
-
-impl MoeBlockEncodable {
- pub fn new(
- context: &MTLContext,
- moe_config: &MixtureOfExpertsConfig,
- model_dim: usize,
- hidden_dim: usize,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, crate::backends::metal::MTLError> {
- let activation_data_type: DataType = moe_config
- .expert_config
- .linear_config
- .activation_precision()
- .into();
- let data_type: KernelDataType = activation_data_type.into();
-
- let router_data_type: DataType =
- moe_config.router_config.activation_precision().into();
- let router_kernel_data_type: KernelDataType = router_data_type.into();
-
- let router_renorm = matches!(
- moe_config.routing_function,
- RoutingFunctionConfig::SoftmaxRouting
- );
-
- let router_tree = parameter_tree.subtree("router").map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Router subtree error: {:?}",
- e
- ))
- })?;
-
- let router = match &moe_config.router_config {
- LinearConfig::Quantized(_) => {
- unimplemented!(
- "Quantized router with fused router+topk not yet supported"
- );
- },
- LinearConfig::MLXQuantized(_) => {
- unimplemented!(
- "MLX quantized router with fused router+topk not yet supported"
- );
- },
- LinearConfig::FullPrecision {
- ..
- } => {
- let mut weights_arr =
- router_tree.leaf("weights").map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Router weights error: {:?}",
- e
- ))
- })?;
- let weights_buf = unsafe { weights_arr.mtl_buffer().clone() };
-
- let mut biases_arr =
- router_tree.leaf("biases").map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Router biases error: {:?}",
- e
- ))
- })?;
- let biases_buf = unsafe { biases_arr.mtl_buffer().clone() };
-
- RouterBlock::Metal {
- weights_buf,
- biases_buf,
- }
- },
- LinearConfig::QLoRA {
- ..
- } => {
- unimplemented!("QLoRA router not yet supported for MoE");
- },
- };
-
- // Initialize kernels
- let router_topk_kernel =
- MoeRouterTopKKernel::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "RouterTopK fused kernel error: {:?}",
- e
- ))
- })?;
- let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context)
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Counts+offsets kernel error: {:?}",
- e
- ))
- })?;
- let scatter_kernels = MoeScatterKernels::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Scatter kernels error: {:?}",
- e
- ))
- })?;
- let gather_kernel = MoeGatherKernel::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Gather kernel error: {:?}",
- e
- ))
- })?;
- let experts_two_pass_decode_kernel =
- MoeExpertsTwoPassDecodeKernel::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Experts two-pass decode kernel error: {:?}",
- e
- ))
- })?;
- let experts_two_pass_prefill_kernel =
- MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Experts two-pass prefill kernel error: {:?}",
- e
- ))
- })?;
- let single_decode_kernel = MoeExpertsSingleDecodeKernel::new(context)
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Simple decode fused kernel error: {:?}",
- e
- ))
- })?;
- let finalize_kernel = MoeFinalizeKernel::new(context).map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "Finalize kernel error: {:?}",
- e
- ))
- })?;
-
- // Load expert weights
- let experts_tree = parameter_tree.subtree("experts").map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "experts subtree error: {:?}",
- e
- ))
- })?;
-
- let mut w13_arr = experts_tree
- .subtree("up_projection")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "up_projection error: {:?}",
- e
- ))
- })?
- .leaf("weights")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "up weights error: {:?}",
- e
- ))
- })?;
- let w13_buf = unsafe { w13_arr.mtl_buffer().clone() };
-
- let mut w2_arr = experts_tree
- .subtree("down_projection")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "down_projection error: {:?}",
- e
- ))
- })?
- .leaf("weights")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "down weights error: {:?}",
- e
- ))
- })?;
- let w2_buf = unsafe { w2_arr.mtl_buffer().clone() };
-
- let mut up_biases_arr = experts_tree
- .subtree("up_projection")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "up_projection biases error: {:?}",
- e
- ))
- })?
- .leaf("biases")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "up biases error: {:?}",
- e
- ))
- })?;
- let up_biases_buf = unsafe { up_biases_arr.mtl_buffer().clone() };
-
- let mut down_biases_arr = experts_tree
- .subtree("down_projection")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "down_projection biases error: {:?}",
- e
- ))
- })?
- .leaf("biases")
- .map_err(|e| {
- crate::backends::metal::MTLError::Generic(format!(
- "down biases error: {:?}",
- e
- ))
- })?;
- let down_biases_buf = unsafe { down_biases_arr.mtl_buffer().clone() };
-
- let shared_weights = SharedMoeWeights {
- w13_buf: Rc::new(w13_buf),
- w2_buf: Rc::new(w2_buf),
- up_biases_buf: Rc::new(up_biases_buf),
- down_biases_buf: Rc::new(down_biases_buf),
- };
-
- Ok(Self {
- router,
- router_data_type: router_kernel_data_type,
- router_renorm,
- router_topk_kernel,
- counts_offsets_kernel,
- scatter_kernels,
- gather_kernel,
- experts_two_pass_decode_kernel,
- experts_two_pass_prefill_kernel,
- single_decode_kernel,
- finalize_kernel,
- moe_config: moe_config.clone(),
- model_dim,
- hidden_dim,
- data_type,
- shared_weights,
- })
- }
-
- fn gating_code_from_activation(activation: &Activation) -> u32 {
- match activation {
- Activation::Gelu => 3, // GEGLU
- Activation::SiLU {
- ..
- } => 2, // SwiGLU
- Activation::Identity => {
- panic!("Identity activation is not supported for MoE kernels")
- },
- }
- }
-
- fn dtype_size(&self) -> usize {
- match self.data_type {
- KernelDataType::BFloat16 | KernelDataType::Float16 => 2,
- KernelDataType::Float32 => 4,
- }
- }
-
- fn encode_router(
- &self,
- root: &CommandBufferRef,
- buffers: &MoeBuffers,
- suffix_length: usize,
- ) {
- let e = self.moe_config.mixture_size;
- let k = self.moe_config.num_experts_per_token;
-
- match &self.router {
- RouterBlock::Metal {
- weights_buf,
- biases_buf,
- } => {
- self.router_topk_kernel
- .encode(
- root,
- self.router_data_type,
- MoeRouterTopKArguments {
- input_buffer: &buffers.main,
- weight_buffer: weights_buf,
- bias_buffer: biases_buf,
- topk_ids_buffer: &buffers.topk_ids,
- topk_probs_buffer: &buffers.topk_probs,
- t: suffix_length,
- d_model: self.model_dim,
- e,
- k,
- renorm: self.router_renorm,
- },
- )
- .expect("MoE router+topk failed");
- },
- }
- }
-
- fn encode_indirect_setup(
- &self,
- root: &CommandBufferRef,
- buffers: &MoeBuffers,
- suffix_length: usize,
- ) {
- let e = self.moe_config.mixture_size;
- let k = self.moe_config.num_experts_per_token;
-
- // Counts + offsets
- self.counts_offsets_kernel
- .encode(
- root,
- MoeCountsOffsetsFusedArguments {
- topk_ids_buffer: &buffers.topk_ids,
- offsets_buffer: &buffers.offsets,
- sum_k_buffer: &buffers.sumk,
- partials_buffer: &buffers.partials,
- t: suffix_length,
- e,
- k,
- },
- )
- .expect("MoE counts+offsets failed");
-
- let num_blocks = ((suffix_length + 255) / 256).max(1);
- let num_tiles = ((e + 512 - 1) / 512).max(1);
-
- // Block bases
- self.scatter_kernels
- .encode_block_bases(
- root,
- super::MoeBlockBasesArguments {
- partials_buffer: &buffers.partials,
- block_bases_buffer: &buffers.block_bases,
- block_alloc_buffer: &buffers.block_alloc,
- e,
- num_blocks,
- num_tiles,
- },
- )
- .expect("MoE block bases failed");
-
- // Scatter with tok2row map
- self.scatter_kernels
- .encode_scatter_with_map(
- root,
- MoeScatterWithMapArguments {
- base: super::MoeScatterArguments {
- topk_ids_buffer: &buffers.topk_ids,
- topk_probs_buffer: &buffers.topk_probs,
- offsets_buffer: &buffers.offsets,
- block_bases_buffer: &buffers.block_bases,
- block_alloc_buffer: &buffers.block_alloc,
- out_ids_buffer: &buffers.bucketed_ids,
- out_probs_buffer: &buffers.bucketed_probs,
- t: suffix_length,
- e,
- k,
- num_blocks,
- num_tiles,
- },
- tok2row_buffer: &buffers.tok2row,
- },
- self.data_type,
- )
- .expect("MoE scatter failed");
-
- // Gather
- self.gather_kernel
- .encode(
- root,
- self.data_type,
- MoeGatherArguments {
- x_buffer: &buffers.main,
- bucketed_ids_buffer: &buffers.bucketed_ids,
- x_perm_buffer: &buffers.x_perm,
- sumk_buffer: &buffers.sumk,
- t: suffix_length,
- k,
- d_model: self.model_dim,
- },
- )
- .expect("MoE gather failed");
- }
-
- fn clear_indirect_buffers(
- &self,
- root: &CommandBufferRef,
- buffers: &MoeBuffers,
- suffix_length: usize,
- ) {
- let k = self.moe_config.num_experts_per_token;
- if k == 0 {
- return;
- }
-
- let blit_encoder = root.new_blit_command_encoder();
- let entries = suffix_length * k;
- let topk_bytes = entries * std::mem::size_of::<u32>();
- let tok2row_bytes = entries * std::mem::size_of::<i32>();
- let dtype_size = self.dtype_size();
-
- if topk_bytes > 0 {
- blit_encoder.fill_buffer(
- &buffers.topk_ids,
- metal::NSRange::new(0, topk_bytes as u64),
- 0xFF,
- );
- }
- if tok2row_bytes > 0 {
- blit_encoder.fill_buffer(
- &buffers.tok2row,
- metal::NSRange::new(0, tok2row_bytes as u64),
- 0xFF,
- );
- }
-
- let hidden_bytes =
- (suffix_length * k * self.hidden_dim * dtype_size) as u64;
- if hidden_bytes > 0 {
- blit_encoder.fill_buffer(
- &buffers.hidden,
- metal::NSRange::new(0, hidden_bytes),
- 0,
- );
- }
-
- let y_partial_bytes =
- (suffix_length * k * self.model_dim * dtype_size) as u64;
- if y_partial_bytes > 0 {
- blit_encoder.fill_buffer(
- &buffers.y_partial,
- metal::NSRange::new(0, y_partial_bytes),
- 0,
- );
- }
-
- let x_perm_bytes =
- (suffix_length * k * self.model_dim * dtype_size) as u64;
- if x_perm_bytes > 0 {
- blit_encoder.fill_buffer(
- &buffers.x_perm,
- metal::NSRange::new(0, x_perm_bytes),
- 0,
- );
- }
-
- blit_encoder.end_encoding();
- }
-
- fn encode_single_token_path(
- &self,
- root: &CommandBufferRef,
- buffers: &MoeBuffers,
- ) {
- let k = self.moe_config.num_experts_per_token;
- let gating_code = Self::gating_code_from_activation(
- &self.moe_config.expert_config.activation,
- );
- let silu_alpha = self.moe_config.expert_config.activation.alpha();
- let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]
- .unwrap_or(f32::NEG_INFINITY);
- let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]
- .unwrap_or(f32::INFINITY);
- let up_clip_min = self.moe_config.expert_config.up_clipping[0];
- let up_clip_max = self.moe_config.expert_config.up_clipping[1];
-
- self.single_decode_kernel
- .encode(
- root,
- MoeExpertsSingleDecodeArguments {
- x: &buffers.main,
- topk_ids: &buffers.topk_ids,
- topk_probs: &buffers.topk_probs,
- w13_all: &self.shared_weights.w13_buf,
- w2_all: &self.shared_weights.w2_buf,
- up_biases: &self.shared_weights.up_biases_buf,
- down_biases: &self.shared_weights.down_biases_buf,
- hidden: &buffers.hidden,
- y: &buffers.main,
- d_model: self.model_dim,
- d_ff: self.hidden_dim,
- k,
- gating_code,
- silu_alpha,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- data_type: self.data_type,
- },
- )
- .expect("MoE single-token fused decode failed");
- }
-
- fn encode_indirect_decode_path(
- &self,
- root: &CommandBufferRef,
- buffers: &MoeBuffers,
- suffix_length: usize,
- ) {
- let e = self.moe_config.mixture_size;
- let k = self.moe_config.num_experts_per_token;
- let gating_code = Self::gating_code_from_activation(
- &self.moe_config.expert_config.activation,
- );
-
- let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]
- .unwrap_or(f32::NEG_INFINITY);
- let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]
- .unwrap_or(f32::INFINITY);
- let up_clip_min = self.moe_config.expert_config.up_clipping[0];
- let up_clip_max = self.moe_config.expert_config.up_clipping[1];
- let silu_alpha = self.moe_config.expert_config.activation.alpha();
-
- let total_rows = suffix_length * k;
- const K_TILE: usize = 64;
- let num_tiles_k = ((self.hidden_dim + K_TILE - 1) / K_TILE) as u32;
-
- self.experts_two_pass_decode_kernel
- .encode(
- root,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &buffers.x_perm,
- expert_offsets: &buffers.offsets,
- row_expert_map: &buffers.row_expert_map,
- hidden_buffer: &buffers.hidden,
- output_buffer: &buffers.y_partial,
- w13_all: &self.shared_weights.w13_buf,
- w2_all: &self.shared_weights.w2_buf,
- up_biases: &self.shared_weights.up_biases_buf,
- down_biases: &self.shared_weights.down_biases_buf,
- tile_counts: &buffers.tile_counts,
- tile_offsets: &buffers.tile_offsets,
- tile_map: &buffers.tile_map,
- total_tiles: &buffers.total_tiles,
- dispatch_args: &buffers.dispatch_args,
- total_rows,
- d_model: self.model_dim,
- d_ff: self.hidden_dim,
- e,
- num_tiles_k,
- gating_code,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- silu_alpha,
- data_type: self.data_type,
- },
- )
- .expect("MoE indirect decode failed");
-
- self.encode_finalize(root, buffers, suffix_length);
- }
-
- fn encode_indirect_prefill_path(
- &self,
- root: &CommandBufferRef,
- buffers: &MoeBuffers,
- suffix_length: usize,
- ) {
- let e = self.moe_config.mixture_size;
- let k = self.moe_config.num_experts_per_token;
- let gating_code = Self::gating_code_from_activation(
- &self.moe_config.expert_config.activation,
- );
-
- let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]
- .unwrap_or(f32::NEG_INFINITY);
- let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]
- .unwrap_or(f32::INFINITY);
- let up_clip_min = self.moe_config.expert_config.up_clipping[0];
- let up_clip_max = self.moe_config.expert_config.up_clipping[1];
- let silu_alpha = self.moe_config.expert_config.activation.alpha();
-
- let total_rows = suffix_length * k;
- const K_TILE: usize = 128;
- let num_tiles_k = ((self.hidden_dim + K_TILE - 1) / K_TILE) as u32;
-
- self.experts_two_pass_prefill_kernel
- .encode(
- root,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &buffers.x_perm,
- expert_offsets: &buffers.offsets,
- row_expert_map: &buffers.row_expert_map,
- hidden_buffer: &buffers.hidden,
- output_buffer: &buffers.y_partial,
- w13_all: &self.shared_weights.w13_buf,
- w2_all: &self.shared_weights.w2_buf,
- up_biases: &self.shared_weights.up_biases_buf,
- down_biases: &self.shared_weights.down_biases_buf,
- tile_counts: &buffers.tile_counts,
- tile_offsets: &buffers.tile_offsets,
- tile_map: &buffers.tile_map,
- total_tiles: &buffers.total_tiles,
- dispatch_args: &buffers.dispatch_args,
- total_rows,
- d_model: self.model_dim,
- d_ff: self.hidden_dim,
- e,
- num_tiles_k,
- gating_code,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- silu_alpha,
- data_type: self.data_type,
- },
- )
- .expect("MoE indirect prefill failed");
-
- self.encode_finalize(root, buffers, suffix_length);
- }
-
- fn encode_finalize(
- &self,
- root: &CommandBufferRef,
- buffers: &MoeBuffers,
- suffix_length: usize,
- ) {
- let k = self.moe_config.num_experts_per_token;
-
- self.finalize_kernel
- .encode(
- root,
- MoeFinalizeArguments {
- tok2row_buffer: &buffers.tok2row,
- probs_buffer: &buffers.topk_probs,
- y_partial_buffer: &buffers.y_partial,
- y_out_buffer: &buffers.main,
- t: suffix_length,
- d_model: self.model_dim,
- k,
- },
- self.data_type,
- )
- .expect("MoE finalize failed");
- }
-}
-
-impl EncodableWithState for MoeBlockEncodable {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let suffix_length = state.active_suffix_length();
- let arrays = state.arrays(&[
- ArrayId::Main,
- ArrayId::MoeTopkIds,
- ArrayId::MoeTopkProbs,
- ArrayId::MoeOffsets,
- ArrayId::MoeSumK,
- ArrayId::MoeBucketedTokenIds,
- ArrayId::MoeBucketedProbs,
- ArrayId::MoeXPerm,
- ArrayId::MoeTok2Row,
- ArrayId::MoeYPartial,
- ArrayId::MoeTileCounts,
- ArrayId::MoeTileOffsets,
- ArrayId::MoeTileMap,
- ArrayId::MoeTotalTiles,
- ArrayId::MoeDispatchArgs,
- ArrayId::MoeScatterPartials,
- ArrayId::MoeScatterBlockBases,
- ArrayId::MoeBlockAlloc,
- ArrayId::MoeHidden,
- ArrayId::MoeTwoPassRowExpertMap,
- ]);
-
- let clone_buffer = |array: &RefCell<MetalArray>| -> metal::Buffer {
- let mut borrow = array.borrow_mut();
- unsafe { borrow.mtl_buffer().clone() }
- };
-
- let mut array_iter = arrays.iter();
- let buffers = MoeBuffers {
- main: clone_buffer(array_iter.next().unwrap()),
- topk_ids: clone_buffer(array_iter.next().unwrap()),
- topk_probs: clone_buffer(array_iter.next().unwrap()),
- offsets: clone_buffer(array_iter.next().unwrap()),
- sumk: clone_buffer(array_iter.next().unwrap()),
- bucketed_ids: clone_buffer(array_iter.next().unwrap()),
- bucketed_probs: clone_buffer(array_iter.next().unwrap()),
- x_perm: clone_buffer(array_iter.next().unwrap()),
- tok2row: clone_buffer(array_iter.next().unwrap()),
- y_partial: clone_buffer(array_iter.next().unwrap()),
- tile_counts: clone_buffer(array_iter.next().unwrap()),
- tile_offsets: clone_buffer(array_iter.next().unwrap()),
- tile_map: clone_buffer(array_iter.next().unwrap()),
- total_tiles: clone_buffer(array_iter.next().unwrap()),
- dispatch_args: clone_buffer(array_iter.next().unwrap()),
- partials: clone_buffer(array_iter.next().unwrap()),
- block_bases: clone_buffer(array_iter.next().unwrap()),
- block_alloc: clone_buffer(array_iter.next().unwrap()),
- hidden: clone_buffer(array_iter.next().unwrap()),
- row_expert_map: clone_buffer(array_iter.next().unwrap()),
- };
- debug_assert!(array_iter.next().is_none());
-
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let root = &*mtl_command_buffer;
-
- // Path selection:
- // - suffix == 1: Single-token fused decode (no scatter/gather)
- // - suffix <= 32: Indirect decode (GEMV)
- // - suffix > 32: Indirect prefill (GEMM)
- if suffix_length == 1 {
- // Single-token path: router -> fused decode
- self.encode_router(root, &buffers, suffix_length);
- self.encode_single_token_path(root, &buffers);
- } else if suffix_length <= INDIRECT_DECODE_THRESHOLD {
- // Indirect decode path: router -> setup -> decode experts -> finalize
- self.clear_indirect_buffers(root, &buffers, suffix_length);
- self.encode_router(root, &buffers, suffix_length);
- self.encode_indirect_setup(root, &buffers, suffix_length);
- self.encode_indirect_decode_path(root, &buffers, suffix_length);
- } else {
- // Indirect prefill path: router -> setup -> prefill experts -> finalize
- self.clear_indirect_buffers(root, &buffers, suffix_length);
- self.encode_router(root, &buffers, suffix_length);
- self.encode_indirect_setup(root, &buffers, suffix_length);
- self.encode_indirect_prefill_path(root, &buffers, suffix_length);
- }
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- false // MoE uses blit encoders + complex multi-kernel pattern
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
index 23c3c1252..fe957960f 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -2,18 +2,22 @@
#include <metal_simdgroup>
using namespace metal;
-constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
+constant uint GATING_SEL
+ [[function_constant(30)]]; // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
static inline float gelu_approx(float x) {
- const float k0 = 0.7978845608f;
- const float k1 = 0.044715f;
- if (x > 10.0f) return x;
- if (x < -10.0f) return 0.0f;
- return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
+ const float k0 = 0.7978845608f;
+ const float k1 = 0.044715f;
+ if (x > 10.0f)
+ return x;
+ if (x < -10.0f)
+ return 0.0f;
+ return 0.5f * x *
+ (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
}
static inline float silu(float x, float alpha) {
- return x / (1.0f + exp(-alpha * x));
+ return x / (1.0f + exp(-alpha * x));
}
// ============================================================================
@@ -23,7 +27,7 @@ static inline float silu(float x, float alpha) {
// Grid: (ceil(d_ff/4), K)
// ============================================================================
-template<typename T, typename T4>
+template <typename T, typename T4>
inline void moe_experts_decode_single_pass_a_impl(
device const T* x,
device const int* topk_ids,
@@ -43,106 +47,133 @@ inline void moe_experts_decode_single_pass_a_impl(
uint simd_gid,
uint simd_lid
) {
- const int expert_id = topk_ids[k_slot];
- if (expert_id < 0) return;
- const uint expert_u = uint(expert_id);
+ const int expert_id = topk_ids[k_slot];
+ if (expert_id < 0)
+ return;
+ const uint expert_u = uint(expert_id);
- const uint h_idx = h_block_idx * 4 + simd_gid;
- if (h_idx >= d_ff) return;
+ const uint h_idx = h_block_idx * 4 + simd_gid;
+ if (h_idx >= d_ff)
+ return;
- const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);
- const ulong w13_base = (ulong)expert_u * w13_stride;
- const ulong bias_base = (ulong)expert_u * (ulong)(2 * d_ff);
+ const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);
+ const ulong w13_base = (ulong)expert_u * w13_stride;
+ const ulong bias_base = (ulong)expert_u * (ulong)(2 * d_ff);
- device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;
- device const T* w_gate_row = W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
+ device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;
+ device const T* w_gate_row =
+ W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
- float acc_up = 0.0f;
- float acc_gate = 0.0f;
+ float acc_up = 0.0f;
+ float acc_gate = 0.0f;
- const uint vec_iters = d_model / 128;
+ const uint vec_iters = d_model / 128;
- for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd_lid * 4;
-
- T4 x_vec = *reinterpret_cast<device const T4*>(x + base_idx);
- T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);
-
- acc_up += float(x_vec.x) * float(w_up_vec.x);
- acc_up += float(x_vec.y) * float(w_up_vec.y);
- acc_up += float(x_vec.z) * float(w_up_vec.z);
- acc_up += float(x_vec.w) * float(w_up_vec.w);
-
- if (GATING_SEL > 1) {
- T4 w_gate_vec = *reinterpret_cast<device const T4*>(w_gate_row + base_idx);
- acc_gate += float(x_vec.x) * float(w_gate_vec.x);
- acc_gate += float(x_vec.y) * float(w_gate_vec.y);
- acc_gate += float(x_vec.z) * float(w_gate_vec.z);
- acc_gate += float(x_vec.w) * float(w_gate_vec.w);
- }
- }
+ for (uint i = 0; i < vec_iters; ++i) {
+ uint base_idx = i * 128 + simd_lid * 4;
- uint leftover_start = vec_iters * 128 + simd_lid;
- for (uint idx = leftover_start; idx < d_model; idx += 32) {
- float xv = float(x[idx]);
- acc_up += xv * float(w_up_row[idx]);
- if (GATING_SEL > 1) {
- acc_gate += xv * float(w_gate_row[idx]);
- }
- }
+ T4 x_vec = *reinterpret_cast<device const T4*>(x + base_idx);
+ T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);
+
+ acc_up += float(x_vec.x) * float(w_up_vec.x);
+ acc_up += float(x_vec.y) * float(w_up_vec.y);
+ acc_up += float(x_vec.z) * float(w_up_vec.z);
+ acc_up += float(x_vec.w) * float(w_up_vec.w);
- acc_up = simd_sum(acc_up);
if (GATING_SEL > 1) {
- acc_gate = simd_sum(acc_gate);
+ T4 w_gate_vec =
+ *reinterpret_cast<device const T4*>(w_gate_row + base_idx);
+ acc_gate += float(x_vec.x) * float(w_gate_vec.x);
+ acc_gate += float(x_vec.y) * float(w_gate_vec.y);
+ acc_gate += float(x_vec.z) * float(w_gate_vec.z);
+ acc_gate += float(x_vec.w) * float(w_gate_vec.w);
}
+ }
- if (simd_lid == 0) {
- float up_val = clamp(acc_up + float(biases[bias_base + h_idx]), up_clip_min, up_clip_max);
-
- float activated;
- if (GATING_SEL <= 1) {
- activated = (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);
- } else {
- float gate_val = clamp(acc_gate + float(biases[bias_base + d_ff + h_idx]), gate_clip_min, gate_clip_max);
- float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);
- activated = gate_act * up_val;
- }
-
- hidden_out[(ulong)k_slot * (ulong)d_ff + (ulong)h_idx] = activated;
+ uint leftover_start = vec_iters * 128 + simd_lid;
+ for (uint idx = leftover_start; idx < d_model; idx += 32) {
+ float xv = float(x[idx]);
+ acc_up += xv * float(w_up_row[idx]);
+ if (GATING_SEL > 1) {
+ acc_gate += xv * float(w_gate_row[idx]);
+ }
+ }
+
+ acc_up = simd_sum(acc_up);
+ if (GATING_SEL > 1) {
+ acc_gate = simd_sum(acc_gate);
+ }
+
+ if (simd_lid == 0) {
+ float up_val = clamp(
+ acc_up + float(biases[bias_base + h_idx]),
+ up_clip_min,
+ up_clip_max
+ );
+
+ float activated;
+ if (GATING_SEL <= 1) {
+ activated =
+ (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);
+ } else {
+ float gate_val = clamp(
+ acc_gate + float(biases[bias_base + d_ff + h_idx]),
+ gate_clip_min,
+ gate_clip_max
+ );
+ float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha)
+ : gelu_approx(gate_val);
+ activated = gate_act * up_val;
}
-}
-#define MOE_DECODE_SINGLE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \
-kernel void moe_experts_decode_single_pass_a_##SUFFIX( \
- device const DTYPE* x [[buffer(0)]], \
- device const int* topk_ids [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& K [[buffer(7)]], \
- constant float& silu_alpha [[buffer(8)]], \
- constant float& gate_clip_min [[buffer(9)]], \
- constant float& gate_clip_max [[buffer(10)]], \
- constant float& up_clip_min [[buffer(11)]], \
- constant float& up_clip_max [[buffer(12)]], \
- uint2 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]]) \
-{ \
- moe_experts_decode_single_pass_a_impl<DTYPE, DTYPE4>( \
- x, topk_ids, W13_all, biases, hidden_out, \
- d_model, d_ff, K, silu_alpha, \
- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, \
- tgpig.y, tgpig.x, simd_gid, simd_lid); \
+ hidden_out[(ulong)k_slot * (ulong)d_ff + (ulong)h_idx] = activated;
+ }
}
+#define MOE_DECODE_SINGLE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+ kernel void moe_experts_decode_single_pass_a_##SUFFIX( \
+ device const DTYPE* x [[buffer(0)]], \
+ device const int* topk_ids [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device const DTYPE* biases [[buffer(3)]], \
+ device float* hidden_out [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& K [[buffer(7)]], \
+ constant float& silu_alpha [[buffer(8)]], \
+ constant float& gate_clip_min [[buffer(9)]], \
+ constant float& gate_clip_max [[buffer(10)]], \
+ constant float& up_clip_min [[buffer(11)]], \
+ constant float& up_clip_max [[buffer(12)]], \
+ uint2 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ moe_experts_decode_single_pass_a_impl<DTYPE, DTYPE4>( \
+ x, \
+ topk_ids, \
+ W13_all, \
+ biases, \
+ hidden_out, \
+ d_model, \
+ d_ff, \
+ K, \
+ silu_alpha, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ tgpig.y, \
+ tgpig.x, \
+ simd_gid, \
+ simd_lid \
+ ); \
+ }
+
MOE_DECODE_SINGLE_PASS_A_KERNEL(half, half4, f16)
MOE_DECODE_SINGLE_PASS_A_KERNEL(bfloat, bfloat4, bf16)
MOE_DECODE_SINGLE_PASS_A_KERNEL(float, float4, f32)
-
// ============================================================================
// Pass B (fused with finalize): hidden[k] @ W2[expert] → y (directly)
// Computes: y[d] = Σ_k prob[k] * (hidden[k] @ W2[expert_k, d] + bias)
@@ -150,14 +181,14 @@ MOE_DECODE_SINGLE_PASS_A_KERNEL(float, float4, f32)
// Grid: (ceil(d_model/8), 1) - NOT per K!
// ============================================================================
-template<typename T, typename T4>
+template <typename T, typename T4>
inline void moe_experts_decode_single_pass_b_impl(
- device const float* hidden, // [K, d_ff]
- device const int* topk_ids, // [K]
- device const T* topk_probs, // [K]
- device const T* W2_all, // [E, d_model, d_ff]
- device const T* biases, // [E, d_model]
- device T* y, // [d_model]
+ device const float* hidden, // [K, d_ff]
+ device const int* topk_ids, // [K]
+ device const T* topk_probs, // [K]
+ device const T* W2_all, // [E, d_model, d_ff]
+ device const T* biases, // [E, d_model]
+ device T* y, // [d_model]
uint d_model,
uint d_ff,
uint K,
@@ -165,75 +196,89 @@ inline void moe_experts_decode_single_pass_b_impl(
uint simd_gid,
uint simd_lid
) {
- const uint my_col = d_block * 8 + simd_gid;
- if (my_col >= d_model) return;
+ const uint my_col = d_block * 8 + simd_gid;
+ if (my_col >= d_model)
+ return;
- const uint vec_iters = d_ff / 128;
- const ulong w2_expert_stride = (ulong)d_model * (ulong)d_ff;
+ const uint vec_iters = d_ff / 128;
+ const ulong w2_expert_stride = (ulong)d_model * (ulong)d_ff;
- float final_acc = 0.0f;
+ float final_acc = 0.0f;
- // Loop over K experts
- for (uint k = 0; k < K; ++k) {
- const uint expert_u = uint(topk_ids[k]);
- const float prob = float(topk_probs[k]);
+ // Loop over K experts
+ for (uint k = 0; k < K; ++k) {
+ const uint expert_u = uint(topk_ids[k]);
+ const float prob = float(topk_probs[k]);
- device const float* hidden_ptr = hidden + (ulong)k * (ulong)d_ff;
- device const T* w2_ptr = W2_all + (ulong)expert_u * w2_expert_stride + (ulong)my_col * (ulong)d_ff;
+ device const float* hidden_ptr = hidden + (ulong)k * (ulong)d_ff;
+ device const T* w2_ptr = W2_all + (ulong)expert_u * w2_expert_stride +
+ (ulong)my_col * (ulong)d_ff;
- float acc = 0.0f;
+ float acc = 0.0f;
- // Vectorized reduction
- for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd_lid * 4;
-
- float4 h_vec = *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
- T4 w_vec = *reinterpret_cast<device const T4*>(w2_ptr + base_idx);
+ // Vectorized reduction
+ for (uint i = 0; i < vec_iters; ++i) {
+ uint base_idx = i * 128 + simd_lid * 4;
- acc += h_vec.x * float(w_vec.x);
- acc += h_vec.y * float(w_vec.y);
- acc += h_vec.z * float(w_vec.z);
- acc += h_vec.w * float(w_vec.w);
- }
+ float4 h_vec =
+ *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
+ T4 w_vec = *reinterpret_cast<device const T4*>(w2_ptr + base_idx);
- // Remainder
- for (uint idx = vec_iters * 128 + simd_lid; idx < d_ff; idx += 32) {
- acc += hidden_ptr[idx] * float(w2_ptr[idx]);
- }
+ acc += h_vec.x * float(w_vec.x);
+ acc += h_vec.y * float(w_vec.y);
+ acc += h_vec.z * float(w_vec.z);
+ acc += h_vec.w * float(w_vec.w);
+ }
- // Simdgroup reduction and accumulate
- float result = simd_sum(acc);
- if (simd_lid == 0) {
- result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);
- final_acc += prob * result;
- }
+ // Remainder
+ for (uint idx = vec_iters * 128 + simd_lid; idx < d_ff; idx += 32) {
+ acc += hidden_ptr[idx] * float(w2_ptr[idx]);
}
+ // Simdgroup reduction and accumulate
+ float result = simd_sum(acc);
if (simd_lid == 0) {
- y[my_col] = T(final_acc);
+ result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);
+ final_acc += prob * result;
}
-}
+ }
-#define MOE_DECODE_SINGLE_PASS_B_KERNEL(DTYPE, DTYPE4, SUFFIX) \
-kernel void moe_experts_decode_single_pass_b_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const int* topk_ids [[buffer(1)]], \
- device const DTYPE* topk_probs [[buffer(2)]], \
- device const DTYPE* W2_all [[buffer(3)]], \
- device const DTYPE* biases [[buffer(4)]], \
- device DTYPE* y [[buffer(5)]], \
- constant uint& d_model [[buffer(6)]], \
- constant uint& d_ff [[buffer(7)]], \
- constant uint& K [[buffer(8)]], \
- uint tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]]) \
-{ \
- moe_experts_decode_single_pass_b_impl<DTYPE, DTYPE4>( \
- hidden, topk_ids, topk_probs, W2_all, biases, y, \
- d_model, d_ff, K, tgpig, simd_gid, simd_lid); \
+ if (simd_lid == 0) {
+ y[my_col] = T(final_acc);
+ }
}
+#define MOE_DECODE_SINGLE_PASS_B_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+ kernel void moe_experts_decode_single_pass_b_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const int* topk_ids [[buffer(1)]], \
+ device const DTYPE* topk_probs [[buffer(2)]], \
+ device const DTYPE* W2_all [[buffer(3)]], \
+ device const DTYPE* biases [[buffer(4)]], \
+ device DTYPE* y [[buffer(5)]], \
+ constant uint& d_model [[buffer(6)]], \
+ constant uint& d_ff [[buffer(7)]], \
+ constant uint& K [[buffer(8)]], \
+ uint tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ moe_experts_decode_single_pass_b_impl<DTYPE, DTYPE4>( \
+ hidden, \
+ topk_ids, \
+ topk_probs, \
+ W2_all, \
+ biases, \
+ y, \
+ d_model, \
+ d_ff, \
+ K, \
+ tgpig, \
+ simd_gid, \
+ simd_lid \
+ ); \
+ }
+
MOE_DECODE_SINGLE_PASS_B_KERNEL(half, half4, f16)
MOE_DECODE_SINGLE_PASS_B_KERNEL(bfloat, bfloat4, bf16)
MOE_DECODE_SINGLE_PASS_B_KERNEL(float, float4, f32)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index d1631c395..ab55ce6c1 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -2,31 +2,35 @@
#include <metal_simdgroup>
using namespace metal;
-constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU
+constant uint GATING_SEL
+ [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU
static inline float gelu_approx(float x) {
- const float k0 = 0.7978845608f;
- const float k1 = 0.044715f;
- if (x > 10.0f) return x;
- if (x < -10.0f) return 0.0f;
- return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
+ const float k0 = 0.7978845608f;
+ const float k1 = 0.044715f;
+ if (x > 10.0f)
+ return x;
+ if (x < -10.0f)
+ return 0.0f;
+ return 0.5f * x *
+ (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
}
static inline float silu(float x, float alpha) {
- return x / (1.0f + exp(-alpha * x));
+ return x / (1.0f + exp(-alpha * x));
}
// === Pass A: Vectorized GEMV with float4 loads ===
// Structure: 4 simdgroups (128 threads), each outputs 1 hidden element
// Each simdgroup: 32 threads reduce d_model using float4 vectorized loads
// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)
-template<typename T, typename T4>
+template <typename T, typename T4>
void moe_experts_decode_pass_a_impl(
- device const T* X_perm, // [total_rows, d_model]
- device const uint* expert_offsets, // [E + 1]
- device const T* W13_all, // [E, 2*d_ff, d_model]
- device const T* up_biases, // [E, 2*d_ff]
- device float* hidden_out, // [total_rows, d_ff]
+ device const T* X_perm, // [total_rows, d_model]
+ device const uint* expert_offsets, // [E + 1]
+ device const T* W13_all, // [E, 2*d_ff, d_model]
+ device const T* up_biases, // [E, 2*d_ff]
+ device float* hidden_out, // [total_rows, d_ff]
uint d_model,
uint d_ff,
uint E,
@@ -41,114 +45,134 @@ void moe_experts_decode_pass_a_impl(
uint simd_gid,
uint simd_lid
) {
- // Validate row bounds
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint global_row = seg_start + row_in_expert;
- if (global_row >= seg_end) return;
-
- // Each simdgroup outputs one hidden element
- const uint h_idx = h_block_idx * 4 + simd_gid;
- if (h_idx >= d_ff) return;
-
- // Base addresses
- const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);
- const ulong w13_base = (ulong)expert_idx * w13_stride;
- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
- const ulong x_row_base = (ulong)global_row * (ulong)d_model;
-
- device const T* x_ptr = X_perm + x_row_base;
- device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;
- device const T* w_gate_row = W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
-
- float acc_up = 0.0f;
- float acc_gate = 0.0f;
-
- // Vectorized reduction: 32 threads × 4 elements = 128 elements per iteration
- const uint vec_iters = d_model / 128;
-
- for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd_lid * 4;
-
- T4 x_vec = *reinterpret_cast<device const T4*>(x_ptr + base_idx);
- T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);
-
- acc_up += float(x_vec.x) * float(w_up_vec.x);
- acc_up += float(x_vec.y) * float(w_up_vec.y);
- acc_up += float(x_vec.z) * float(w_up_vec.z);
- acc_up += float(x_vec.w) * float(w_up_vec.w);
-
- if (GATING_SEL > 1) {
- T4 w_gate_vec = *reinterpret_cast<device const T4*>(w_gate_row + base_idx);
- acc_gate += float(x_vec.x) * float(w_gate_vec.x);
- acc_gate += float(x_vec.y) * float(w_gate_vec.y);
- acc_gate += float(x_vec.z) * float(w_gate_vec.z);
- acc_gate += float(x_vec.w) * float(w_gate_vec.w);
- }
- }
+ // Validate row bounds
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ const uint global_row = seg_start + row_in_expert;
+ if (global_row >= seg_end)
+ return;
+
+ // Each simdgroup outputs one hidden element
+ const uint h_idx = h_block_idx * 4 + simd_gid;
+ if (h_idx >= d_ff)
+ return;
+
+ // Base addresses
+ const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);
+ const ulong w13_base = (ulong)expert_idx * w13_stride;
+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
+ const ulong x_row_base = (ulong)global_row * (ulong)d_model;
+
+ device const T* x_ptr = X_perm + x_row_base;
+ device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;
+ device const T* w_gate_row =
+ W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
+
+ float acc_up = 0.0f;
+ float acc_gate = 0.0f;
+
+ // Vectorized reduction: 32 threads × 4 elements = 128 elements per iteration
+ const uint vec_iters = d_model / 128;
+
+ for (uint i = 0; i < vec_iters; ++i) {
+ uint base_idx = i * 128 + simd_lid * 4;
+
+ T4 x_vec = *reinterpret_cast<device const T4*>(x_ptr + base_idx);
+ T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);
+
+ acc_up += float(x_vec.x) * float(w_up_vec.x);
+ acc_up += float(x_vec.y) * float(w_up_vec.y);
+ acc_up += float(x_vec.z) * float(w_up_vec.z);
+ acc_up += float(x_vec.w) * float(w_up_vec.w);
- // Handle leftover elements
- uint leftover_start = vec_iters * 128 + simd_lid;
- for (uint idx = leftover_start; idx < d_model; idx += 32) {
- float xv = float(x_ptr[idx]);
- acc_up += xv * float(w_up_row[idx]);
- if (GATING_SEL > 1) {
- acc_gate += xv * float(w_gate_row[idx]);
- }
+ if (GATING_SEL > 1) {
+ T4 w_gate_vec =
+ *reinterpret_cast<device const T4*>(w_gate_row + base_idx);
+ acc_gate += float(x_vec.x) * float(w_gate_vec.x);
+ acc_gate += float(x_vec.y) * float(w_gate_vec.y);
+ acc_gate += float(x_vec.z) * float(w_gate_vec.z);
+ acc_gate += float(x_vec.w) * float(w_gate_vec.w);
}
+ }
- // Simdgroup reduction
- acc_up = simd_sum(acc_up);
+ // Handle leftover elements
+ uint leftover_start = vec_iters * 128 + simd_lid;
+ for (uint idx = leftover_start; idx < d_model; idx += 32) {
+ float xv = float(x_ptr[idx]);
+ acc_up += xv * float(w_up_row[idx]);
if (GATING_SEL > 1) {
- acc_gate = simd_sum(acc_gate);
+ acc_gate += xv * float(w_gate_row[idx]);
}
-
- // Lane 0 applies activation and writes result
- if (simd_lid == 0) {
- float up_val = acc_up + float(up_biases[bias_base + h_idx]);
- up_val = clamp(up_val, up_clip_min, up_clip_max);
-
- float activated;
- if (GATING_SEL <= 1) {
- activated = (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);
- } else {
- float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);
- gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);
- float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);
- activated = gate_act * up_val;
- }
-
- hidden_out[(ulong)global_row * (ulong)d_ff + (ulong)h_idx] = activated;
+ }
+
+ // Simdgroup reduction
+ acc_up = simd_sum(acc_up);
+ if (GATING_SEL > 1) {
+ acc_gate = simd_sum(acc_gate);
+ }
+
+ // Lane 0 applies activation and writes result
+ if (simd_lid == 0) {
+ float up_val = acc_up + float(up_biases[bias_base + h_idx]);
+ up_val = clamp(up_val, up_clip_min, up_clip_max);
+
+ float activated;
+ if (GATING_SEL <= 1) {
+ activated =
+ (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);
+ } else {
+ float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);
+ gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);
+ float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha)
+ : gelu_approx(gate_val);
+ activated = gate_act * up_val;
}
-}
-#define MOE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \
-kernel void moe_experts_decode_pass_a_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device float* hidden_out [[buffer(3)]], \
- device const DTYPE* up_biases [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- uint3 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]]) \
-{ \
- moe_experts_decode_pass_a_impl<DTYPE, DTYPE4>( \
- X_perm, expert_offsets, W13_all, up_biases, hidden_out, \
- d_model, d_ff, E, gate_clip_min, gate_clip_max, \
- up_clip_min, up_clip_max, silu_alpha, \
- tgpig.y, tgpig.z, tgpig.x, \
- simd_gid, simd_lid); \
+ hidden_out[(ulong)global_row * (ulong)d_ff + (ulong)h_idx] = activated;
+ }
}
+#define MOE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+ kernel void moe_experts_decode_pass_a_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device float* hidden_out [[buffer(3)]], \
+ device const DTYPE* up_biases [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ uint3 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ moe_experts_decode_pass_a_impl<DTYPE, DTYPE4>( \
+ X_perm, \
+ expert_offsets, \
+ W13_all, \
+ up_biases, \
+ hidden_out, \
+ d_model, \
+ d_ff, \
+ E, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ silu_alpha, \
+ tgpig.y, \
+ tgpig.z, \
+ tgpig.x, \
+ simd_gid, \
+ simd_lid \
+ ); \
+ }
+
MOE_PASS_A_KERNEL(bfloat, bfloat4, bf16)
MOE_PASS_A_KERNEL(half, half4, f16)
MOE_PASS_A_KERNEL(float, float4, f32)
@@ -157,142 +181,149 @@ MOE_PASS_A_KERNEL(float, float4, f32)
// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0
kernel void moe_pass_a_tile_counts(
- device const uint* expert_offsets [[buffer(0)]], // [E+1]
- device uint* tile_counts [[buffer(1)]], // [E]
+ device const uint* expert_offsets [[buffer(0)]], // [E+1]
+ device uint* tile_counts [[buffer(1)]], // [E]
constant uint& E [[buffer(2)]],
constant uint& h_blocks [[buffer(3)]],
uint tid [[thread_position_in_grid]]
) {
- if (tid >= E) return;
- const uint start = expert_offsets[tid];
- const uint end = expert_offsets[tid + 1];
- const uint num_rows = end - start;
- tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;
+ if (tid >= E)
+ return;
+ const uint start = expert_offsets[tid];
+ const uint end = expert_offsets[tid + 1];
+ const uint num_rows = end - start;
+ tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;
}
// Exclusive scan of tile_counts to get tile_offsets and total_tiles
kernel void moe_pass_a_tile_scan(
- device const uint* tile_counts [[buffer(0)]], // [E]
- device uint* tile_offsets [[buffer(1)]], // [E+1]
- device uint* total_tiles [[buffer(2)]], // [1]
+ device const uint* tile_counts [[buffer(0)]], // [E]
+ device uint* tile_offsets [[buffer(1)]], // [E+1]
+ device uint* total_tiles [[buffer(2)]], // [1]
constant uint& E [[buffer(3)]],
uint lid [[thread_index_in_threadgroup]],
threadgroup uint* scratch [[threadgroup(0)]]
) {
- // Simple single-threadgroup scan (works for E <= 1024)
- const uint idx = lid;
-
- // Load into threadgroup memory
- if (idx < E) {
- scratch[idx] = tile_counts[idx];
- } else {
- scratch[idx] = 0;
+ // Simple single-threadgroup scan (works for E <= 1024)
+ const uint idx = lid;
+
+ // Load into threadgroup memory
+ if (idx < E) {
+ scratch[idx] = tile_counts[idx];
+ } else {
+ scratch[idx] = 0;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Kogge-Stone scan
+ uint val = scratch[idx];
+ for (uint offset = 1; offset < 1024; offset *= 2) {
+ uint temp = 0;
+ if (idx >= offset && idx < E) {
+ temp = scratch[idx - offset];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Kogge-Stone scan
- uint val = scratch[idx];
- for (uint offset = 1; offset < 1024; offset *= 2) {
- uint temp = 0;
- if (idx >= offset && idx < E) {
- temp = scratch[idx - offset];
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (idx >= offset && idx < E) {
- val += temp;
- scratch[idx] = val;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // Write exclusive scan (shift right by 1)
- if (idx == 0) {
- tile_offsets[0] = 0;
+ if (idx >= offset && idx < E) {
+ val += temp;
+ scratch[idx] = val;
}
- if (idx < E) {
- tile_offsets[idx + 1] = scratch[idx];
- if (idx == E - 1) {
- total_tiles[0] = scratch[idx];
- }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // Write exclusive scan (shift right by 1)
+ if (idx == 0) {
+ tile_offsets[0] = 0;
+ }
+ if (idx < E) {
+ tile_offsets[idx + 1] = scratch[idx];
+ if (idx == E - 1) {
+ total_tiles[0] = scratch[idx];
}
+ }
}
// Build row→expert map: one thread per routed row
kernel void moe_pass_a_build_row_map(
- device const uint* expert_offsets [[buffer(0)]] , // [E+1]
- device uint* row_expert_map [[buffer(1)]] , // [total_rows]
+ device const uint* expert_offsets [[buffer(0)]], // [E+1]
+ device uint* row_expert_map [[buffer(1)]], // [total_rows]
constant uint& total_rows [[buffer(2)]],
constant uint& E [[buffer(3)]],
uint tid [[thread_position_in_grid]]
) {
- if (tid >= total_rows) return;
-
- uint left = 0u;
- uint right = E;
- const uint row = tid;
-
- while (left + 1u < right) {
- const uint mid = (left + right) >> 1;
- if (row < expert_offsets[mid]) {
- right = mid;
- } else {
- left = mid;
- }
+ if (tid >= total_rows)
+ return;
+
+ uint left = 0u;
+ uint right = E;
+ const uint row = tid;
+
+ while (left + 1u < right) {
+ const uint mid = (left + right) >> 1;
+ if (row < expert_offsets[mid]) {
+ right = mid;
+ } else {
+ left = mid;
}
+ }
- row_expert_map[row] = left;
+ row_expert_map[row] = left;
}
// Build tile map entries from row→expert map
kernel void moe_pass_a_build_tile_map(
- device const uint* expert_offsets [[buffer(0)]], // [E+1]
- device const uint* tile_offsets [[buffer(1)]], // [E+1]
- device const uint* row_expert_map [[buffer(2)]], // [total_rows]
- device uint* tile_map [[buffer(3)]], // [total_tiles * 3]
+ device const uint* expert_offsets [[buffer(0)]], // [E+1]
+ device const uint* tile_offsets [[buffer(1)]], // [E+1]
+ device const uint* row_expert_map [[buffer(2)]], // [total_rows]
+ device uint* tile_map [[buffer(3)]], // [total_tiles * 3]
constant uint& total_rows [[buffer(4)]],
constant uint& h_blocks [[buffer(5)]],
uint tid [[thread_position_in_grid]]
) {
- const uint total_tiles = total_rows * h_blocks;
- if (tid >= total_tiles) return;
+ const uint total_tiles = total_rows * h_blocks;
+ if (tid >= total_tiles)
+ return;
- const uint row_idx = tid / h_blocks;
- const uint h_block = tid % h_blocks;
+ const uint row_idx = tid / h_blocks;
+ const uint h_block = tid % h_blocks;
- if (row_idx >= total_rows) return;
+ if (row_idx >= total_rows)
+ return;
- const uint expert_idx = row_expert_map[row_idx];
- const uint row_start = expert_offsets[expert_idx];
- const uint row_in_expert = row_idx - row_start;
- const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
+ const uint expert_idx = row_expert_map[row_idx];
+ const uint row_start = expert_offsets[expert_idx];
+ const uint row_in_expert = row_idx - row_start;
+ const uint tile_base =
+ tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
- tile_map[tile_base * 3 + 0] = h_block;
- tile_map[tile_base * 3 + 1] = expert_idx;
- tile_map[tile_base * 3 + 2] = row_in_expert;
+ tile_map[tile_base * 3 + 0] = h_block;
+ tile_map[tile_base * 3 + 1] = expert_idx;
+ tile_map[tile_base * 3 + 2] = row_in_expert;
}
// Write dispatch args for indirect dispatch (reusable from tiled version)
kernel void moe_pass_a_write_dispatch_args(
- device const uint* total_tiles [[buffer(0)]], // [1]
- device uint* dispatch_args [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments
- constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A
+ device const uint* total_tiles [[buffer(0)]], // [1]
+ device uint* dispatch_args
+ [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments
+ constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A
uint tid [[thread_position_in_grid]]
) {
- if (tid > 0) return;
- dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
- dispatch_args[1] = num_tiles_y; // y dimension
- dispatch_args[2] = 1; // z dimension
+ if (tid > 0)
+ return;
+ dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
+ dispatch_args[1] = num_tiles_y; // y dimension
+ dispatch_args[2] = 1; // z dimension
}
// Modified Pass A that reads from tile map for indirect dispatch
-template<typename T, typename T4>
+template <typename T, typename T4>
void moe_experts_decode_pass_a_indirect_impl(
device const T* X_perm,
device const uint* expert_offsets,
device const T* W13_all,
device const T* up_biases,
device float* hidden_out,
- device const uint* tile_map, // [total_tiles * 3]
+ device const uint* tile_map, // [total_tiles * 3]
uint d_model,
uint d_ff,
uint E,
@@ -301,68 +332,95 @@ void moe_experts_decode_pass_a_indirect_impl(
float up_clip_min,
float up_clip_max,
float silu_alpha,
- uint tile_idx, // flat threadgroup index
+ uint tile_idx, // flat threadgroup index
uint simd_gid,
uint simd_lid
) {
- // Read tile descriptor
- const uint h_block_idx = tile_map[tile_idx * 3 + 0];
- const uint expert_idx = tile_map[tile_idx * 3 + 1];
- const uint row_in_expert = tile_map[tile_idx * 3 + 2];
-
- // Call original implementation
- moe_experts_decode_pass_a_impl<T, T4>(
- X_perm, expert_offsets, W13_all, up_biases, hidden_out,
- d_model, d_ff, E,
- gate_clip_min, gate_clip_max,
- up_clip_min, up_clip_max, silu_alpha,
- expert_idx, row_in_expert, h_block_idx,
- simd_gid, simd_lid
- );
+ // Read tile descriptor
+ const uint h_block_idx = tile_map[tile_idx * 3 + 0];
+ const uint expert_idx = tile_map[tile_idx * 3 + 1];
+ const uint row_in_expert = tile_map[tile_idx * 3 + 2];
+
+ // Call original implementation
+ moe_experts_decode_pass_a_impl<T, T4>(
+ X_perm,
+ expert_offsets,
+ W13_all,
+ up_biases,
+ hidden_out,
+ d_model,
+ d_ff,
+ E,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ expert_idx,
+ row_in_expert,
+ h_block_idx,
+ simd_gid,
+ simd_lid
+ );
}
-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, DTYPE4, SUFFIX) \
-kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device float* hidden_out [[buffer(3)]], \
- device const DTYPE* up_biases [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- device const uint* tile_map [[buffer(13)]], \
- uint3 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]]) \
-{ \
- moe_experts_decode_pass_a_indirect_impl<DTYPE, DTYPE4>( \
- X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \
- d_model, d_ff, E, gate_clip_min, gate_clip_max, \
- up_clip_min, up_clip_max, silu_alpha, \
- tgpig.x, simd_gid, simd_lid); \
-}
+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+ kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device float* hidden_out [[buffer(3)]], \
+ device const DTYPE* up_biases [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ device const uint* tile_map [[buffer(13)]], \
+ uint3 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ moe_experts_decode_pass_a_indirect_impl<DTYPE, DTYPE4>( \
+ X_perm, \
+ expert_offsets, \
+ W13_all, \
+ up_biases, \
+ hidden_out, \
+ tile_map, \
+ d_model, \
+ d_ff, \
+ E, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ silu_alpha, \
+ tgpig.x, \
+ simd_gid, \
+ simd_lid \
+ ); \
+ }
MOE_PASS_A_INDIRECT_KERNEL(bfloat, bfloat4, bf16)
MOE_PASS_A_INDIRECT_KERNEL(half, half4, f16)
MOE_PASS_A_INDIRECT_KERNEL(float, float4, f32)
-
// === Pass B: Simdgroup cooperation along K for coalescing ===
-// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading consecutive K elements
+// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading
+// consecutive K elements
-template<typename T, typename AccumT>
+template <typename T, typename AccumT>
void moe_experts_decode_down_fused_2d_impl(
- device const float* hidden, // [total_rows, d_ff] - f32 from Pass A
- device const uint* row_expert_map, // [total_rows] - direct row->expert lookup
- device const T* w2_all, // [E, d_model, d_ff] - layout
- device const T* down_biases, // [E, d_model]
- device T* y_out, // [total_rows, d_model]
+ device const float* hidden, // [total_rows, d_ff] - f32 from Pass A
+ device const uint*
+ row_expert_map, // [total_rows] - direct row->expert lookup
+ device const T* w2_all, // [E, d_model, d_ff] - layout
+ device const T* down_biases, // [E, d_model]
+ device T* y_out, // [total_rows, d_model]
uint total_rows,
uint d_model,
uint d_ff,
@@ -371,114 +429,141 @@ void moe_experts_decode_down_fused_2d_impl(
uint simd_gid,
uint simd_lid
) {
- constexpr uint THREADS_PER_SIMD = 32;
- constexpr uint SIMDGROUPS_PER_TG = 8;
-
- const uint row_idx = tgpig.y;
- if (row_idx >= total_rows) return;
-
- // Each simdgroup computes one output column
- const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;
- if (my_col >= d_model) return;
-
- const uint expert_idx = row_expert_map[row_idx];
-
- // Base addresses for this output column
- const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;
- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;
-
- // Dual accumulators for ILP: breaks FMA dependency chains
- AccumT acc0 = AccumT(0.0);
- AccumT acc1 = AccumT(0.0);
- AccumT acc = AccumT(0.0);
-
- // Main loop: stride-32 with vec8 loads for ILP
- const uint k_iters = d_ff / THREADS_PER_SIMD;
- const uint k_vec_iters = k_iters / 8;
-
- for (uint iter = 0; iter < k_vec_iters; ++iter) {
- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;
-
- // hidden: stride-32 per lane, already f32 from Pass A
- const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
- const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];
- const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];
- const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];
- const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];
- const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];
- const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];
- const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
-
- // W2: lane-coalesced
- const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
- const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
- const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
- const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
- const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
- const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
- const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
- const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
-
- // dual trees for ILP
- acc0 = fma(h0, w0, acc0);
- acc1 = fma(h1, w1, acc1);
- acc0 = fma(h2, w2, acc0);
- acc1 = fma(h3, w3, acc1);
- acc0 = fma(h4, w4, acc0);
- acc1 = fma(h5, w5, acc1);
- acc0 = fma(h6, w6, acc0);
- acc1 = fma(h7, w7, acc1);
- }
-
- // fold the vectorized contribution once
- acc += (acc0 + acc1);
-
- // Handle remaining full iterations
- for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
- const uint k = iter * THREADS_PER_SIMD + simd_lid;
- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
- }
-
- // Handle leftover elements (d_ff % 32)
- const uint leftover_start = k_iters * THREADS_PER_SIMD;
- if (leftover_start + simd_lid < d_ff) {
- const uint k = leftover_start + simd_lid;
- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
- }
-
- // Simdgroup reduction
- AccumT result = simd_sum(acc);
-
- // Lane 0 writes result
- if (simd_lid == 0) {
- const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
- result += AccumT(down_biases[bias_idx]);
-
- const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;
- y_out[out_idx] = T(result);
- }
+ constexpr uint THREADS_PER_SIMD = 32;
+ constexpr uint SIMDGROUPS_PER_TG = 8;
+
+ const uint row_idx = tgpig.y;
+ if (row_idx >= total_rows)
+ return;
+
+ // Each simdgroup computes one output column
+ const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;
+ if (my_col >= d_model)
+ return;
+
+ const uint expert_idx = row_expert_map[row_idx];
+
+ // Base addresses for this output column
+ const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;
+ const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff +
+ (ulong)my_col * (ulong)d_ff;
+
+ // Dual accumulators for ILP: breaks FMA dependency chains
+ AccumT acc0 = AccumT(0.0);
+ AccumT acc1 = AccumT(0.0);
+ AccumT acc = AccumT(0.0);
+
+ // Main loop: stride-32 with vec8 loads for ILP
+ const uint k_iters = d_ff / THREADS_PER_SIMD;
+ const uint k_vec_iters = k_iters / 8;
+
+ for (uint iter = 0; iter < k_vec_iters; ++iter) {
+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;
+
+ // hidden: stride-32 per lane, already f32 from Pass A
+ const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
+ const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];
+ const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];
+ const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];
+ const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];
+ const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];
+ const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];
+ const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
+
+ // W2: lane-coalesced
+ const AccumT w0 =
+ AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
+ const AccumT w1 =
+ AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
+ const AccumT w2 =
+ AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
+ const AccumT w3 =
+ AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
+ const AccumT w4 =
+ AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
+ const AccumT w5 =
+ AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
+ const AccumT w6 =
+ AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
+ const AccumT w7 =
+ AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
+
+ // dual trees for ILP
+ acc0 = fma(h0, w0, acc0);
+ acc1 = fma(h1, w1, acc1);
+ acc0 = fma(h2, w2, acc0);
+ acc1 = fma(h3, w3, acc1);
+ acc0 = fma(h4, w4, acc0);
+ acc1 = fma(h5, w5, acc1);
+ acc0 = fma(h6, w6, acc0);
+ acc1 = fma(h7, w7, acc1);
+ }
+
+ // fold the vectorized contribution once
+ acc += (acc0 + acc1);
+
+ // Handle remaining full iterations
+ for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
+ const uint k = iter * THREADS_PER_SIMD + simd_lid;
+ acc =
+ fma(AccumT(hidden[hidden_base + k]),
+ AccumT(w2_all[w2_col_base + k]),
+ acc);
+ }
+
+ // Handle leftover elements (d_ff % 32)
+ const uint leftover_start = k_iters * THREADS_PER_SIMD;
+ if (leftover_start + simd_lid < d_ff) {
+ const uint k = leftover_start + simd_lid;
+ acc =
+ fma(AccumT(hidden[hidden_base + k]),
+ AccumT(w2_all[w2_col_base + k]),
+ acc);
+ }
+
+ // Simdgroup reduction
+ AccumT result = simd_sum(acc);
+
+ // Lane 0 writes result
+ if (simd_lid == 0) {
+ const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
+ result += AccumT(down_biases[bias_idx]);
+
+ const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;
+ y_out[out_idx] = T(result);
+ }
}
-#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* row_expert_map [[buffer(1)]], \
- device const DTYPE* w2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* y_out [[buffer(4)]], \
- constant uint& total_rows [[buffer(5)]], \
- constant uint& d_model [[buffer(6)]], \
- constant uint& d_ff [[buffer(7)]], \
- constant uint& E [[buffer(8)]], \
- uint2 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]]) \
-{ \
- moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \
- hidden, row_expert_map, w2_all, down_biases, y_out, \
- total_rows, d_model, d_ff, E, \
- tgpig, simd_gid, simd_lid); \
-}
+#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* row_expert_map [[buffer(1)]], \
+ device const DTYPE* w2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* y_out [[buffer(4)]], \
+ constant uint& total_rows [[buffer(5)]], \
+ constant uint& d_model [[buffer(6)]], \
+ constant uint& d_ff [[buffer(7)]], \
+ constant uint& E [[buffer(8)]], \
+ uint2 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \
+ hidden, \
+ row_expert_map, \
+ w2_all, \
+ down_biases, \
+ y_out, \
+ total_rows, \
+ d_model, \
+ d_ff, \
+ E, \
+ tgpig, \
+ simd_gid, \
+ simd_lid \
+ ); \
+ }
MOE_PASS_B_FUSED_2D_KERNEL(bfloat, bf16)
MOE_PASS_B_FUSED_2D_KERNEL(half, f16)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index 9f9a5810d..72076217f 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -1,7 +1,7 @@
#include <metal_stdlib>
#include <metal_simdgroup>
#include <metal_simdgroup_matrix>
-#include "../quant_matmul/mma.h"
+#include "../quant_matmul/mma.h"
using namespace metal;
@@ -9,50 +9,52 @@ using namespace metal;
static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }
static inline uint linear_tid(uint3 tid, uint3 tpg) {
- return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;
+ return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;
}
static inline float gelu_approx(float x) {
- const float k0 = 0.7978845608f; // sqrt(2/pi)
- const float k1 = 0.044715f;
- if (x > 10.0f) return x;
- if (x < -10.0f) return 0.0f;
- float x3 = x * x * x;
- float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);
- return 0.5f * x * (1.0f + tanh(t));
+ const float k0 = 0.7978845608f; // sqrt(2/pi)
+ const float k1 = 0.044715f;
+ if (x > 10.0f)
+ return x;
+ if (x < -10.0f)
+ return 0.0f;
+ float x3 = x * x * x;
+ float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);
+ return 0.5f * x * (1.0f + tanh(t));
}
static inline float silu(float x, float alpha) {
- // alpha=1 gives standard SiLU; keep alpha param for parity with your API
- return x / (1.0f + exp(-alpha * x));
+ // alpha=1 gives standard SiLU; keep alpha param for parity with your API
+ return x / (1.0f + exp(-alpha * x));
}
-template<typename T>
+template <typename T>
inline void store_vec4(device T* dst, ulong base, float4 vals) {
- dst[base + 0] = T(vals.x);
- dst[base + 1] = T(vals.y);
- dst[base + 2] = T(vals.z);
- dst[base + 3] = T(vals.w);
+ dst[base + 0] = T(vals.x);
+ dst[base + 1] = T(vals.y);
+ dst[base + 2] = T(vals.z);
+ dst[base + 3] = T(vals.w);
}
-template<>
+template <>
inline void store_vec4<float>(device float* dst, ulong base, float4 vals) {
- *reinterpret_cast<device float4*>(dst + base) = vals;
+ *reinterpret_cast<device float4*>(dst + base) = vals;
}
-template<typename T>
+template <typename T>
struct StageStorage {
- using type = float;
+ using type = float;
};
-template<>
+template <>
struct StageStorage<half> {
- using type = half;
+ using type = half;
};
-template<>
+template <>
struct StageStorage<bfloat> {
- using type = bfloat;
+ using type = bfloat;
};
// 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up
@@ -65,324 +67,408 @@ constant uint PASSA_BN = 32;
constant uint PASSA_BK = 64;
constant uint PASSA_SG_BM = 8;
constant uint PASSA_SG_BN = 16;
-constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts
+constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts
-template<typename T>
+template <typename T>
inline void pass_a_impl(
device const T* X_perm,
device const uint* expert_offsets,
device const T* W13_all,
device const T* up_biases,
device float* hidden_out,
- uint d_model, uint d_ff, uint E,
- float gate_clip_min, float gate_clip_max,
- float up_clip_min, float up_clip_max,
+ uint d_model,
+ uint d_ff,
+ uint E,
+ float gate_clip_min,
+ float gate_clip_max,
+ float up_clip_min,
+ float up_clip_max,
float silu_alpha,
// work partition
- uint tile_seg_start, uint expert_idx, uint tile_m, uint tile_n,
+ uint tile_seg_start,
+ uint expert_idx,
+ uint tile_m,
+ uint tile_n,
// per-thread info
- uint sg_id, uint3 threads_per_tg, uint3 local_tid,
+ uint sg_id,
+ uint3 threads_per_tg,
+ uint3 local_tid,
// TG storage
- threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]
- threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]
- threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]
- threadgroup float* bias_up, // [BN]
- threadgroup float* bias_gate // [BN]
+ threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]
+ threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]
+ threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]
+ threadgroup float* bias_up, // [BN]
+ threadgroup float* bias_gate // [BN]
) {
- using StageT = typename StageStorage<T>::type;
- using StageVec4 = metal::vec<StageT, 4>;
-
- constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;
- constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;
- constexpr uint TG_PAD = PASSA_TG_PAD;
- constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X
- constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
-
- if (expert_idx >= E) return;
-
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
- if (seg_len == 0) return;
-
- const uint row_tg_off = tile_m * Bm;
- const uint col_tg_off = tile_n * Bn;
- if (row_tg_off >= seg_len || col_tg_off >= d_ff) return;
-
- const uint m_rows = min(Bm, seg_len - row_tg_off);
- const uint n_cols = min(Bn, d_ff - col_tg_off);
-
- const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
- const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
-
- // simdgroup tile mapping
- const uint sg_col_count = Bn / SgBn;
- const uint row_sg = sg_id / sg_col_count;
- const uint col_sg = sg_id % sg_col_count;
- const uint row_sg_off = row_sg * SgBm;
- const uint col_sg_off = col_sg * SgBn;
-
- // Guard against misconfigured TG sizes
- constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
- constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);
- if (sg_id >= SG_EXPECTED) return;
-
- // partial accumulators (8x8 tiles per simdgroup)
- constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE);
- metal::simdgroup_float8x8 OutUp[TEMP];
- metal::simdgroup_float8x8 OutGate[TEMP];
- for (uint i = 0; i < TEMP; ++i) {
- OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- if (GATING_SEL > 1u) {
- OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- }
+ using StageT = typename StageStorage<T>::type;
+ using StageVec4 = metal::vec<StageT, 4>;
+
+ constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;
+ constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;
+ constexpr uint TG_PAD = PASSA_TG_PAD;
+ constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X
+ constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
+
+ if (expert_idx >= E)
+ return;
+
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
+ if (seg_len == 0)
+ return;
+
+ const uint row_tg_off = tile_m * Bm;
+ const uint col_tg_off = tile_n * Bn;
+ if (row_tg_off >= seg_len || col_tg_off >= d_ff)
+ return;
+
+ const uint m_rows = min(Bm, seg_len - row_tg_off);
+ const uint n_cols = min(Bn, d_ff - col_tg_off);
+
+ const ulong x_base =
+ ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
+ const ulong w13_expert_base =
+ (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
+
+ // simdgroup tile mapping
+ const uint sg_col_count = Bn / SgBn;
+ const uint row_sg = sg_id / sg_col_count;
+ const uint col_sg = sg_id % sg_col_count;
+ const uint row_sg_off = row_sg * SgBm;
+ const uint col_sg_off = col_sg * SgBn;
+
+ // Guard against misconfigured TG sizes
+ constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
+ constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);
+ if (sg_id >= SG_EXPECTED)
+ return;
+
+ // partial accumulators (8x8 tiles per simdgroup)
+ constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE);
+ metal::simdgroup_float8x8 OutUp[TEMP];
+ metal::simdgroup_float8x8 OutGate[TEMP];
+ for (uint i = 0; i < TEMP; ++i) {
+ OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ if (GATING_SEL > 1u) {
+ OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
}
+ }
- const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
- const uint lin = linear_tid(local_tid, threads_per_tg);
-
- // K-loop (tile along D)
- for (uint k_off = 0; k_off < d_model; k_off += Bk) {
- const uint valid_k = min(Bk, d_model - k_off);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Stage LHS: Xs [Bm,Bk] with vec8 loads and padding ----
- {
- constexpr uint vec_size = 8;
- constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8
- constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128
- const uint iters = ceil_div(total_vecs, threads_total);
-
-UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint r = i / vec_cols;
- const uint c8 = i % vec_cols;
- const uint col_base = c8 * vec_size;
-
- threadgroup StageT* dst = Xs + r * X_LD + col_base;
-
- if (r < m_rows && col_base < valid_k) {
- // Safe to read - at least first element is valid
- const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);
-UZU_PRAGMA_UNROLL
- for (uint j = 0; j < vec_size; j++) {
- dst[j] = (col_base + j < valid_k) ? StageT(float(X_perm[base + j])) : StageT(0.0f);
- }
- } else {
- // Row out of bounds or entire vec8 chunk is out of K bounds
-UZU_PRAGMA_UNROLL
- for (uint j = 0; j < vec_size; j++) {
- dst[j] = StageT(0.0f);
- }
- }
- }
- }
- }
+ const uint threads_total =
+ threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
+ const uint lin = linear_tid(local_tid, threads_per_tg);
- // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk] with vec8 loads and padding ----
- {
- constexpr uint vec_size = 8;
- constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8
- constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256
- const uint iters = ceil_div(total_vecs, threads_total);
-
-UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint n = i / vec_cols; // row within the BN tile
- const uint c8 = i % vec_cols;
- const uint col_base = c8 * vec_size;
-
- threadgroup StageT* up_dst = Wk_up + n * W_LD + col_base;
- threadgroup StageT* gt_dst = Wk_gate + n * W_LD + col_base;
-
- if (n < n_cols && col_base < valid_k) {
- // Safe to read - at least first element is valid
- const uint gcol = col_tg_off + n;
- const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + col_base);
- const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + col_base);
-UZU_PRAGMA_UNROLL
- for (uint j = 0; j < vec_size; j++) {
- up_dst[j] = (col_base + j < valid_k) ? StageT(float(W13_all[up_base + j])) : StageT(0.0f);
- if (GATING_SEL > 1u) {
- gt_dst[j] = (col_base + j < valid_k) ? StageT(float(W13_all[gate_base + j])) : StageT(0.0f);
- }
- }
- } else {
- // Row out of bounds or entire vec8 chunk is out of K bounds
-UZU_PRAGMA_UNROLL
- for (uint j = 0; j < vec_size; j++) {
- up_dst[j] = StageT(0.0f);
- if (GATING_SEL > 1u) gt_dst[j] = StageT(0.0f);
- }
- }
- }
- }
- }
+ // K-loop (tile along D)
+ for (uint k_off = 0; k_off < d_model; k_off += Bk) {
+ const uint valid_k = min(Bk, d_model - k_off);
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- MMA across the BK tile (in simdgroup-sized chunks) ----
-UZU_PRAGMA_UNROLL
- for (uint kk = 0; kk < Bk; kk += SG_TILE) {
-UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;
- simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));
-
-UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
- metal::simdgroup_matrix<StageT, 8, 8> rhs_up;
- simdgroup_load(rhs_up, Wk_up, W_LD, ulong2(kk, col_sg_off + n_sub), true);
- simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);
-
- if (GATING_SEL > 1u) {
- metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;
- simdgroup_load(rhs_gate, Wk_gate, W_LD, ulong2(kk, col_sg_off + n_sub), true);
- simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);
- }
- }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Stage LHS: Xs [Bm,Bk] with vec8 loads and padding ----
+ {
+ constexpr uint vec_size = 8;
+ constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8
+ constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128
+ const uint iters = ceil_div(total_vecs, threads_total);
+
+ UZU_PRAGMA_UNROLL
+ for (uint t = 0; t < iters; ++t) {
+ const uint i = t * threads_total + lin;
+ if (i < total_vecs) {
+ const uint r = i / vec_cols;
+ const uint c8 = i % vec_cols;
+ const uint col_base = c8 * vec_size;
+
+ threadgroup StageT* dst = Xs + r * X_LD + col_base;
+
+ if (r < m_rows && col_base < valid_k) {
+ // Safe to read - at least first element is valid
+ const ulong base =
+ x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);
+ UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < vec_size; j++) {
+ dst[j] = (col_base + j < valid_k)
+ ? StageT(float(X_perm[base + j]))
+ : StageT(0.0f);
}
+ } else {
+ // Row out of bounds or entire vec8 chunk is out of K bounds
+ UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < vec_size; j++) {
+ dst[j] = StageT(0.0f);
+ }
+ }
}
- } // K loop
-
- // ---- Bias tile [Bn] ----
- // Note: bias layout is [up | gate] in contiguous FF chunks.
- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
- const uint c_global = col_tg_off + c_local;
- bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
- if (GATING_SEL > 1u) {
- bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;
- }
+ }
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Epilogue: activation/gating → hidden_out (f32) ----
- const uint sg_lane_start = sg_id * 32;
- const uint sg_lane_end = sg_lane_start + 32;
- if (lin < sg_lane_start || lin >= sg_lane_end) return;
- const uint sg_lin = lin - sg_lane_start;
- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment layout used above.
- const uint lane_qid = sg_lin >> 2;
- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
-
-UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
-UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
-
- const uint tile_row_base = row_sg_off + m_sub;
- const uint tile_col_base = col_sg_off + n_sub;
- const uint local_row = tile_row_base + lane_row;
- if (local_row >= m_rows || tile_col_base >= n_cols) {
- continue;
+ // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk] with vec8 loads and padding
+ // ----
+ {
+ constexpr uint vec_size = 8;
+ constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8
+ constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256
+ const uint iters = ceil_div(total_vecs, threads_total);
+
+ UZU_PRAGMA_UNROLL
+ for (uint t = 0; t < iters; ++t) {
+ const uint i = t * threads_total + lin;
+ if (i < total_vecs) {
+ const uint n = i / vec_cols; // row within the BN tile
+ const uint c8 = i % vec_cols;
+ const uint col_base = c8 * vec_size;
+
+ threadgroup StageT* up_dst = Wk_up + n * W_LD + col_base;
+ threadgroup StageT* gt_dst = Wk_gate + n * W_LD + col_base;
+
+ if (n < n_cols && col_base < valid_k) {
+ // Safe to read - at least first element is valid
+ const uint gcol = col_tg_off + n;
+ const ulong up_base = w13_expert_base +
+ (ulong)gcol * (ulong)d_model +
+ (ulong)(k_off + col_base);
+ const ulong gate_base = w13_expert_base +
+ (ulong)(d_ff + gcol) * (ulong)d_model +
+ (ulong)(k_off + col_base);
+ UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < vec_size; j++) {
+ up_dst[j] = (col_base + j < valid_k)
+ ? StageT(float(W13_all[up_base + j]))
+ : StageT(0.0f);
+ if (GATING_SEL > 1u) {
+ gt_dst[j] = (col_base + j < valid_k)
+ ? StageT(float(W13_all[gate_base + j]))
+ : StageT(0.0f);
+ }
}
-
- const auto up_frag = OutUp[tile].thread_elements();
- float gate_frag_0 = 0.0f;
- float gate_frag_1 = 0.0f;
- if (GATING_SEL > 1u) {
- const auto gate_frag = OutGate[tile].thread_elements();
- gate_frag_0 = gate_frag[0];
- gate_frag_1 = gate_frag[1];
+ } else {
+ // Row out of bounds or entire vec8 chunk is out of K bounds
+ UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < vec_size; j++) {
+ up_dst[j] = StageT(0.0f);
+ if (GATING_SEL > 1u)
+ gt_dst[j] = StageT(0.0f);
}
+ }
+ }
+ }
+ }
- const uint col0 = tile_col_base + lane_col_base;
- const uint col1 = col0 + 1;
- const ulong out_row = seg_start + row_tg_off + local_row;
-
- if (col0 < n_cols) {
- float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
- float out_val;
- if (GATING_SEL <= 1u) {
- out_val =
- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
- } else {
- float gate_v =
- clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);
- const float gate_act =
- (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);
- out_val = gate_act * up_v;
- }
- const ulong out_col = col_tg_off + col0;
- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
- }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- if (col1 < n_cols) {
- float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
- float out_val;
- if (GATING_SEL <= 1u) {
- out_val =
- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
- } else {
- float gate_v =
- clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);
- const float gate_act =
- (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);
- out_val = gate_act * up_v;
- }
- const ulong out_col = col_tg_off + col1;
- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
- }
+ // ---- MMA across the BK tile (in simdgroup-sized chunks) ----
+ UZU_PRAGMA_UNROLL
+ for (uint kk = 0; kk < Bk; kk += SG_TILE) {
+ UZU_PRAGMA_UNROLL
+ for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
+ const uint r_idx = m_sub / SG_TILE;
+ metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;
+ simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));
+
+ UZU_PRAGMA_UNROLL
+ for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
+ const uint c_idx = n_sub / SG_TILE;
+ const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
+ metal::simdgroup_matrix<StageT, 8, 8> rhs_up;
+ simdgroup_load(
+ rhs_up,
+ Wk_up,
+ W_LD,
+ ulong2(kk, col_sg_off + n_sub),
+ true
+ );
+ simdgroup_multiply_accumulate(
+ OutUp[tile],
+ lhs_frag,
+ rhs_up,
+ OutUp[tile]
+ );
+
+ if (GATING_SEL > 1u) {
+ metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;
+ simdgroup_load(
+ rhs_gate,
+ Wk_gate,
+ W_LD,
+ ulong2(kk, col_sg_off + n_sub),
+ true
+ );
+ simdgroup_multiply_accumulate(
+ OutGate[tile],
+ lhs_frag,
+ rhs_gate,
+ OutGate[tile]
+ );
+ }
}
+ }
}
+ } // K loop
+
+ // ---- Bias tile [Bn] ----
+ // Note: bias layout is [up | gate] in contiguous FF chunks.
+ for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
+ const uint c_global = col_tg_off + c_local;
+ bias_up[c_local] =
+ (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
+ if (GATING_SEL > 1u) {
+ bias_gate[c_local] = (c_global < d_ff)
+ ? float(up_biases[bias_base + d_ff + c_global])
+ : 0.0f;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Epilogue: activation/gating → hidden_out (f32) ----
+ const uint sg_lane_start = sg_id * 32;
+ const uint sg_lane_end = sg_lane_start + 32;
+ if (lin < sg_lane_start || lin >= sg_lane_end)
+ return;
+ const uint sg_lin = lin - sg_lane_start;
+ // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment
+ // layout used above.
+ const uint lane_qid = sg_lin >> 2;
+ const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
+
+ UZU_PRAGMA_UNROLL
+ for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
+ const uint c_idx = n_sub / SG_TILE;
+ UZU_PRAGMA_UNROLL
+ for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
+ const uint r_idx = m_sub / SG_TILE;
+ const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
+
+ const uint tile_row_base = row_sg_off + m_sub;
+ const uint tile_col_base = col_sg_off + n_sub;
+ const uint local_row = tile_row_base + lane_row;
+ if (local_row >= m_rows || tile_col_base >= n_cols) {
+ continue;
+ }
+
+ const auto up_frag = OutUp[tile].thread_elements();
+ float gate_frag_0 = 0.0f;
+ float gate_frag_1 = 0.0f;
+ if (GATING_SEL > 1u) {
+ const auto gate_frag = OutGate[tile].thread_elements();
+ gate_frag_0 = gate_frag[0];
+ gate_frag_1 = gate_frag[1];
+ }
+
+ const uint col0 = tile_col_base + lane_col_base;
+ const uint col1 = col0 + 1;
+ const ulong out_row = seg_start + row_tg_off + local_row;
+
+ if (col0 < n_cols) {
+ float up_v =
+ clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
+ float out_val;
+ if (GATING_SEL <= 1u) {
+ out_val =
+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
+ } else {
+ float gate_v = clamp(
+ gate_frag_0 + bias_gate[col0],
+ gate_clip_min,
+ gate_clip_max
+ );
+ const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)
+ : gelu_approx(gate_v);
+ out_val = gate_act * up_v;
+ }
+ const ulong out_col = col_tg_off + col0;
+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
+ }
+
+ if (col1 < n_cols) {
+ float up_v =
+ clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
+ float out_val;
+ if (GATING_SEL <= 1u) {
+ out_val =
+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
+ } else {
+ float gate_v = clamp(
+ gate_frag_1 + bias_gate[col1],
+ gate_clip_min,
+ gate_clip_max
+ );
+ const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)
+ : gelu_approx(gate_v);
+ out_val = gate_act * up_v;
+ }
+ const ulong out_col = col_tg_off + col1;
+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
+ }
+ }
+ }
}
// ----- Kernel entry points (Pass A) -----
-#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* up_biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]]) \
-{ \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup float bias_up[PASSA_BN]; \
- threadgroup float bias_gate[PASSA_BN]; \
- pass_a_impl<DTYPE>( \
- X_perm, expert_offsets, W13_all, up_biases, hidden_out, \
- d_model, d_ff, E, \
- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \
- expert_offsets[tg_pos.z], tg_pos.z, tg_pos.y, tg_pos.x, \
- sg_id, threads_per_tg, local_tid, \
- Xs, Wk_up, Wk_gate, bias_up, bias_gate); \
-}
+#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device const DTYPE* up_biases [[buffer(3)]], \
+ device float* hidden_out [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threads_per_tg [[threads_per_threadgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]] \
+ ) { \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup float bias_up[PASSA_BN]; \
+ threadgroup float bias_gate[PASSA_BN]; \
+ pass_a_impl<DTYPE>( \
+ X_perm, \
+ expert_offsets, \
+ W13_all, \
+ up_biases, \
+ hidden_out, \
+ d_model, \
+ d_ff, \
+ E, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ silu_alpha, \
+ expert_offsets[tg_pos.z], \
+ tg_pos.z, \
+ tg_pos.y, \
+ tg_pos.x, \
+ sg_id, \
+ threads_per_tg, \
+ local_tid, \
+ Xs, \
+ Wk_up, \
+ Wk_gate, \
+ bias_up, \
+ bias_gate \
+ ); \
+ }
MOE_PASS_A_KERNEL(bfloat, bf16)
MOE_PASS_A_KERNEL(half, f16)
MOE_PASS_A_KERNEL(float, f32)
// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)
-template<typename T>
+template <typename T>
inline void pass_a_indirect_impl(
device const T* X_perm,
device const uint* expert_offsets,
@@ -390,298 +476,375 @@ inline void pass_a_indirect_impl(
device const T* up_biases,
device float* hidden_out,
device const uint* tile_map,
- uint d_model, uint d_ff, uint E,
- float gate_clip_min, float gate_clip_max,
- float up_clip_min, float up_clip_max,
+ uint d_model,
+ uint d_ff,
+ uint E,
+ float gate_clip_min,
+ float gate_clip_max,
+ float up_clip_min,
+ float up_clip_max,
float silu_alpha,
- uint row_tile_idx, uint n_tile_idx,
- uint sg_id, uint3 threads_per_tg, uint3 local_tid,
+ uint row_tile_idx,
+ uint n_tile_idx,
+ uint sg_id,
+ uint3 threads_per_tg,
+ uint3 local_tid,
threadgroup typename StageStorage<T>::type* Xs,
threadgroup typename StageStorage<T>::type* Wk_up,
threadgroup typename StageStorage<T>::type* Wk_gate,
threadgroup float* bias_up,
- threadgroup float* bias_gate)
-{
- constexpr uint ROW_TILE = PASSA_BM;
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= E) return;
- const uint tile_seg_start = tile_map[base + 1u];
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
-
- pass_a_impl<T>(
- X_perm, expert_offsets, W13_all, up_biases, hidden_out,
- d_model, d_ff, E,
- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha,
- tile_seg_start, expert_idx, tile_m, n_tile_idx,
- sg_id, threads_per_tg, local_tid,
- Xs, Wk_up, Wk_gate, bias_up, bias_gate);
+ threadgroup float* bias_gate
+) {
+ constexpr uint ROW_TILE = PASSA_BM;
+ const uint base = row_tile_idx * 3u;
+ const uint expert_idx = tile_map[base + 0u];
+ if (expert_idx >= E)
+ return;
+ const uint tile_seg_start = tile_map[base + 1u];
+ const uint row_off_elems = tile_map[base + 2u];
+ const uint tile_m = row_off_elems / ROW_TILE;
+
+ pass_a_impl<T>(
+ X_perm,
+ expert_offsets,
+ W13_all,
+ up_biases,
+ hidden_out,
+ d_model,
+ d_ff,
+ E,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ tile_seg_start,
+ expert_idx,
+ tile_m,
+ n_tile_idx,
+ sg_id,
+ threads_per_tg,
+ local_tid,
+ Xs,
+ Wk_up,
+ Wk_gate,
+ bias_up,
+ bias_gate
+ );
}
-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* up_biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- device const uint* tile_map [[buffer(13)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]]) \
-{ \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup float bias_up[PASSA_BN]; \
- threadgroup float bias_gate[PASSA_BN]; \
- pass_a_indirect_impl<DTYPE>( \
- X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \
- d_model, d_ff, E, \
- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \
- tg_pos.y, tg_pos.x, \
- sg_id, threads_per_tg, local_tid, \
- Xs, Wk_up, Wk_gate, bias_up, bias_gate); \
-}
+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device const DTYPE* up_biases [[buffer(3)]], \
+ device float* hidden_out [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ device const uint* tile_map [[buffer(13)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threads_per_tg [[threads_per_threadgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]] \
+ ) { \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup float bias_up[PASSA_BN]; \
+ threadgroup float bias_gate[PASSA_BN]; \
+ pass_a_indirect_impl<DTYPE>( \
+ X_perm, \
+ expert_offsets, \
+ W13_all, \
+ up_biases, \
+ hidden_out, \
+ tile_map, \
+ d_model, \
+ d_ff, \
+ E, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ silu_alpha, \
+ tg_pos.y, \
+ tg_pos.x, \
+ sg_id, \
+ threads_per_tg, \
+ local_tid, \
+ Xs, \
+ Wk_up, \
+ Wk_gate, \
+ bias_up, \
+ bias_gate \
+ ); \
+ }
MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)
MOE_PASS_A_INDIRECT_KERNEL(half, f16)
MOE_PASS_A_INDIRECT_KERNEL(float, f32)
-// ------------------------ Pass B (hidden @ W2 → output) ------------------------
-// Optimized config: 16x64 output tile, 4 simdgroups (128 threads)
+// ------------------------ Pass B (hidden @ W2 → output)
+// ------------------------ Optimized config: 16x64 output tile, 4 simdgroups
+// (128 threads)
constant uint PASSB_BM = 16;
constant uint PASSB_BN = 64;
constant uint PASSB_BK = 64;
-constant uint PASSB_SG_BM = 8; // Each simdgroup handles 8x32
+constant uint PASSB_SG_BM = 8; // Each simdgroup handles 8x32
constant uint PASSB_SG_BN = 32;
-constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts
+constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts
-template<typename T>
+template <typename T>
inline void pass_b_impl(
device const float* hidden,
device const uint* expert_offsets,
device const T* W2_all,
device const T* down_biases,
device T* output,
- uint d_model, uint d_ff, uint E,
- uint expert_idx, uint tile_m, uint tile_n,
- uint sg_id, uint simd_lid, uint lin,
+ uint d_model,
+ uint d_ff,
+ uint E,
+ uint expert_idx,
+ uint tile_m,
+ uint tile_n,
+ uint sg_id,
+ uint simd_lid,
+ uint lin,
threadgroup float* Hs,
threadgroup typename StageStorage<T>::type* Wk,
threadgroup float* bias_tile
) {
- using StageT = typename StageStorage<T>::type;
-
- constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;
- constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;
- constexpr uint TG_PAD = PASSB_TG_PAD;
- constexpr uint H_LD = Bk + TG_PAD; // Padded stride for hidden
- constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
- constexpr uint SG_TILE = 8;
- constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads
-
- if (expert_idx >= E) return;
-
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
- if (seg_len == 0) return;
-
- const uint row_tg_off = tile_m * Bm;
- const uint col_tg_off = tile_n * Bn;
- if (row_tg_off >= seg_len || col_tg_off >= d_model) return;
-
- const uint m_rows = min(Bm, seg_len - row_tg_off);
- const uint n_cols = min(Bn, d_model - col_tg_off);
-
- const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;
- const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;
- const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
-
- // 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)
- const uint row_sg = sg_id / 2; // 0-1 → rows
- const uint col_sg = sg_id % 2; // 0-1 → cols
- const uint row_sg_off = row_sg * SgBm;
- const uint col_sg_off = col_sg * SgBn;
-
- // Guard against excessive simdgroups
- constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);
- if (sg_id >= SG_EXPECTED) return;
-
- // 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles
- constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4
- metal::simdgroup_float8x8 Out[TEMP];
-UZU_PRAGMA_UNROLL
- for (uint i = 0; i < TEMP; ++i) {
- Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- }
-
- // Cooperative load config - vec8 loads for better bandwidth
- constexpr uint H_VEC = 8;
- constexpr uint H_TCOLS = Bk / H_VEC; // 64/8 = 8
- constexpr uint H_TROWS = TGP_SIZE / H_TCOLS; // 128/8 = 16
- const uint h_bi = lin / H_TCOLS;
- const uint h_bj = (lin % H_TCOLS) * H_VEC;
-
- constexpr uint W_VEC = 8;
- constexpr uint W_TCOLS = Bk / W_VEC; // 64/8 = 8
- constexpr uint W_TROWS = TGP_SIZE / W_TCOLS; // 128/8 = 16
- const uint w_bi = lin / W_TCOLS;
- const uint w_bj = (lin % W_TCOLS) * W_VEC;
-
- // K-loop across FF
- for (uint k_off = 0; k_off < d_ff; k_off += Bk) {
- const uint valid_k = min(Bk, d_ff - k_off);
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Stage hidden [Bm=16, Bk=64] with vec8 loads ----
- // 128 threads, 8 vec8 loads per row → 16 rows covered in one pass
- {
- threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;
- if (h_bi < m_rows && h_bj < valid_k) {
- // Safe to read - at least first element is valid
- device const float* my_src = hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);
-UZU_PRAGMA_UNROLL
- for (uint j = 0; j < H_VEC; j++) {
- my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;
- }
- } else {
-UZU_PRAGMA_UNROLL
- for (uint j = 0; j < H_VEC; j++) {
- my_dst[j] = 0.0f;
- }
- }
- }
+ using StageT = typename StageStorage<T>::type;
+
+ constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;
+ constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;
+ constexpr uint TG_PAD = PASSB_TG_PAD;
+ constexpr uint H_LD = Bk + TG_PAD; // Padded stride for hidden
+ constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
+ constexpr uint SG_TILE = 8;
+ constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads
+
+ if (expert_idx >= E)
+ return;
+
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
+ if (seg_len == 0)
+ return;
+
+ const uint row_tg_off = tile_m * Bm;
+ const uint col_tg_off = tile_n * Bn;
+ if (row_tg_off >= seg_len || col_tg_off >= d_model)
+ return;
+
+ const uint m_rows = min(Bm, seg_len - row_tg_off);
+ const uint n_cols = min(Bn, d_model - col_tg_off);
+
+ const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;
+ const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;
+ const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
+
+ // 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)
+ const uint row_sg = sg_id / 2; // 0-1 → rows
+ const uint col_sg = sg_id % 2; // 0-1 → cols
+ const uint row_sg_off = row_sg * SgBm;
+ const uint col_sg_off = col_sg * SgBn;
+
+ // Guard against excessive simdgroups
+ constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);
+ if (sg_id >= SG_EXPECTED)
+ return;
+
+ // 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles
+ constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4
+ metal::simdgroup_float8x8 Out[TEMP];
+ UZU_PRAGMA_UNROLL
+ for (uint i = 0; i < TEMP; ++i) {
+ Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ }
+
+ // Cooperative load config - vec8 loads for better bandwidth
+ constexpr uint H_VEC = 8;
+ constexpr uint H_TCOLS = Bk / H_VEC; // 64/8 = 8
+ constexpr uint H_TROWS = TGP_SIZE / H_TCOLS; // 128/8 = 16
+ const uint h_bi = lin / H_TCOLS;
+ const uint h_bj = (lin % H_TCOLS) * H_VEC;
+
+ constexpr uint W_VEC = 8;
+ constexpr uint W_TCOLS = Bk / W_VEC; // 64/8 = 8
+ constexpr uint W_TROWS = TGP_SIZE / W_TCOLS; // 128/8 = 16
+ const uint w_bi = lin / W_TCOLS;
+ const uint w_bj = (lin % W_TCOLS) * W_VEC;
+
+ // K-loop across FF
+ for (uint k_off = 0; k_off < d_ff; k_off += Bk) {
+ const uint valid_k = min(Bk, d_ff - k_off);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- // ---- Stage W2 [Bn=64, Bk=64] with vec8 loads ----
- // Need 4 passes: 128 threads cover 16 rows, Bn=64 needs 4×16
- {
- for (uint pass = 0; pass < 4; pass++) {
- const uint row = w_bi + pass * W_TROWS;
- threadgroup StageT* my_dst = Wk + row * W_LD + w_bj;
- if (row < n_cols && w_bj < valid_k) {
- // Safe to read - at least first element is valid
- const uint gcol = col_tg_off + row;
- device const T* my_src = W2_all + w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + w_bj);
-UZU_PRAGMA_UNROLL
- for (uint j = 0; j < W_VEC; j++) {
- my_dst[j] = (w_bj + j < valid_k) ? StageT(float(my_src[j])) : StageT(0.0f);
- }
- } else {
-UZU_PRAGMA_UNROLL
- for (uint j = 0; j < W_VEC; j++) {
- my_dst[j] = StageT(0.0f);
- }
- }
- }
+ // ---- Stage hidden [Bm=16, Bk=64] with vec8 loads ----
+ // 128 threads, 8 vec8 loads per row → 16 rows covered in one pass
+ {
+ threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;
+ if (h_bi < m_rows && h_bj < valid_k) {
+ // Safe to read - at least first element is valid
+ device const float* my_src =
+ hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);
+ UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < H_VEC; j++) {
+ my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;
}
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Serpentine MMA: 1×4 tiles per simdgroup ----
-UZU_PRAGMA_UNROLL
- for (uint kk = 0; kk < Bk; kk += SG_TILE) {
- // Load 1 LHS fragment
- metal::simdgroup_float8x8 lhs;
- simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));
-
- // Load 4 RHS fragments (cols)
- metal::simdgroup_matrix<StageT, 8, 8> rhs0, rhs1, rhs2, rhs3;
- simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);
- simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);
- simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);
- simdgroup_load(rhs3, Wk, W_LD, ulong2(kk, col_sg_off + 24), true);
-
- // Serpentine pattern: 0→1→2→3 (forward for first row)
- simdgroup_multiply_accumulate(Out[0], lhs, rhs0, Out[0]);
- simdgroup_multiply_accumulate(Out[1], lhs, rhs1, Out[1]);
- simdgroup_multiply_accumulate(Out[2], lhs, rhs2, Out[2]);
- simdgroup_multiply_accumulate(Out[3], lhs, rhs3, Out[3]);
+ } else {
+ UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < H_VEC; j++) {
+ my_dst[j] = 0.0f;
}
+ }
}
- // ---- Bias tile (cooperative load) ----
- for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {
- const uint c_global = col_tg_off + c_local;
- bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
+ // ---- Stage W2 [Bn=64, Bk=64] with vec8 loads ----
+ // Need 4 passes: 128 threads cover 16 rows, Bn=64 needs 4×16
+ {
+ for (uint pass = 0; pass < 4; pass++) {
+ const uint row = w_bi + pass * W_TROWS;
+ threadgroup StageT* my_dst = Wk + row * W_LD + w_bj;
+ if (row < n_cols && w_bj < valid_k) {
+ // Safe to read - at least first element is valid
+ const uint gcol = col_tg_off + row;
+ device const T* my_src = W2_all + w2_expert_base +
+ (ulong)gcol * (ulong)d_ff +
+ (ulong)(k_off + w_bj);
+ UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < W_VEC; j++) {
+ my_dst[j] =
+ (w_bj + j < valid_k) ? StageT(float(my_src[j])) : StageT(0.0f);
+ }
+ } else {
+ UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < W_VEC; j++) {
+ my_dst[j] = StageT(0.0f);
+ }
+ }
+ }
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- // ---- Writeout with fragment extraction ----
- const uint lane_qid = simd_lid >> 2;
- const uint lane_row = (lane_qid & 4u) + ((simd_lid >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((simd_lid & 1u) << 1);
-
- const uint local_row = row_sg_off + lane_row;
- if (local_row >= m_rows) return;
- const ulong out_row = seg_start + row_tg_off + local_row;
-
-UZU_PRAGMA_UNROLL
- for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup
- const auto accum = Out[ni].thread_elements();
- const uint local_col = col_sg_off + ni * 8 + lane_col_base;
-
- if (local_col < n_cols) {
- const ulong out_col = col_tg_off + local_col;
- const float val = accum[0] + bias_tile[local_col];
- output[out_row * (ulong)d_model + out_col] = T(val);
- }
- if (local_col + 1 < n_cols) {
- const ulong out_col = col_tg_off + local_col + 1;
- const float val = accum[1] + bias_tile[local_col + 1];
- output[out_row * (ulong)d_model + out_col] = T(val);
- }
+ // ---- Serpentine MMA: 1×4 tiles per simdgroup ----
+ UZU_PRAGMA_UNROLL
+ for (uint kk = 0; kk < Bk; kk += SG_TILE) {
+ // Load 1 LHS fragment
+ metal::simdgroup_float8x8 lhs;
+ simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));
+
+ // Load 4 RHS fragments (cols)
+ metal::simdgroup_matrix<StageT, 8, 8> rhs0, rhs1, rhs2, rhs3;
+ simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);
+ simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);
+ simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);
+ simdgroup_load(rhs3, Wk, W_LD, ulong2(kk, col_sg_off + 24), true);
+
+ // Serpentine pattern: 0→1→2→3 (forward for first row)
+ simdgroup_multiply_accumulate(Out[0], lhs, rhs0, Out[0]);
+ simdgroup_multiply_accumulate(Out[1], lhs, rhs1, Out[1]);
+ simdgroup_multiply_accumulate(Out[2], lhs, rhs2, Out[2]);
+ simdgroup_multiply_accumulate(Out[3], lhs, rhs3, Out[3]);
+ }
+ }
+
+ // ---- Bias tile (cooperative load) ----
+ for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {
+ const uint c_global = col_tg_off + c_local;
+ bias_tile[c_local] =
+ (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Writeout with fragment extraction ----
+ const uint lane_qid = simd_lid >> 2;
+ const uint lane_row = (lane_qid & 4u) + ((simd_lid >> 1) & 3u);
+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((simd_lid & 1u) << 1);
+
+ const uint local_row = row_sg_off + lane_row;
+ if (local_row >= m_rows)
+ return;
+ const ulong out_row = seg_start + row_tg_off + local_row;
+
+ UZU_PRAGMA_UNROLL
+ for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup
+ const auto accum = Out[ni].thread_elements();
+ const uint local_col = col_sg_off + ni * 8 + lane_col_base;
+
+ if (local_col < n_cols) {
+ const ulong out_col = col_tg_off + local_col;
+ const float val = accum[0] + bias_tile[local_col];
+ output[out_row * (ulong)d_model + out_col] = T(val);
}
+ if (local_col + 1 < n_cols) {
+ const ulong out_col = col_tg_off + local_col + 1;
+ const float val = accum[1] + bias_tile[local_col + 1];
+ output[out_row * (ulong)d_model + out_col] = T(val);
+ }
+ }
}
// ----- Kernel entry points (Pass B) -----
-#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* output [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]]) \
-{ \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \
- threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \
- threadgroup float bias[PASSB_BN]; \
- const uint lin = local_tid.x; \
- pass_b_impl<DTYPE>( \
- hidden, expert_offsets, W2_all, down_biases, output, \
- d_model, d_ff, E, \
- tg_pos.z, tg_pos.y, tg_pos.x, \
- sg_id, simd_lid, lin, \
- Hs, Wk, bias); \
-}
+#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* output [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]] \
+ ) { \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \
+ threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \
+ threadgroup float bias[PASSB_BN]; \
+ const uint lin = local_tid.x; \
+ pass_b_impl<DTYPE>( \
+ hidden, \
+ expert_offsets, \
+ W2_all, \
+ down_biases, \
+ output, \
+ d_model, \
+ d_ff, \
+ E, \
+ tg_pos.z, \
+ tg_pos.y, \
+ tg_pos.x, \
+ sg_id, \
+ simd_lid, \
+ lin, \
+ Hs, \
+ Wk, \
+ bias \
+ ); \
+ }
MOE_PASS_B_KERNEL(bfloat, bf16)
MOE_PASS_B_KERNEL(half, f16)
MOE_PASS_B_KERNEL(float, f32)
// Indirect variant
-template<typename T>
+template <typename T>
inline void pass_b_indirect_impl(
device const float* hidden,
device const uint* expert_offsets,
@@ -689,56 +852,88 @@ inline void pass_b_indirect_impl(
device const T* down_biases,
device T* output,
device const uint* tile_map,
- uint d_model, uint d_ff, uint E,
- uint row_tile_idx, uint n_tile_idx,
- uint sg_id, uint simd_lid, uint lin,
+ uint d_model,
+ uint d_ff,
+ uint E,
+ uint row_tile_idx,
+ uint n_tile_idx,
+ uint sg_id,
+ uint simd_lid,
+ uint lin,
threadgroup float* Hs,
threadgroup typename StageStorage<T>::type* Wk,
- threadgroup float* bias_tile)
-{
- constexpr uint ROW_TILE = PASSB_BM;
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= E) return;
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
-
- pass_b_impl<T>(
- hidden, expert_offsets, W2_all, down_biases, output,
- d_model, d_ff, E,
- expert_idx, tile_m, n_tile_idx,
- sg_id, simd_lid, lin,
- Hs, Wk, bias_tile);
+ threadgroup float* bias_tile
+) {
+ constexpr uint ROW_TILE = PASSB_BM;
+ const uint base = row_tile_idx * 3u;
+ const uint expert_idx = tile_map[base + 0u];
+ if (expert_idx >= E)
+ return;
+ const uint row_off_elems = tile_map[base + 2u];
+ const uint tile_m = row_off_elems / ROW_TILE;
+
+ pass_b_impl<T>(
+ hidden,
+ expert_offsets,
+ W2_all,
+ down_biases,
+ output,
+ d_model,
+ d_ff,
+ E,
+ expert_idx,
+ tile_m,
+ n_tile_idx,
+ sg_id,
+ simd_lid,
+ lin,
+ Hs,
+ Wk,
+ bias_tile
+ );
}
-#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* output [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- device const uint* tile_map [[buffer(8)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]]) \
-{ \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \
- threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \
- threadgroup float bias[PASSB_BN]; \
- const uint lin = local_tid.x; \
- pass_b_indirect_impl<DTYPE>( \
- hidden, expert_offsets, W2_all, down_biases, output, tile_map, \
- d_model, d_ff, E, \
- tg_pos.y, tg_pos.x, \
- sg_id, simd_lid, lin, \
- Hs, Wk, bias); \
-}
+#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* output [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ device const uint* tile_map [[buffer(8)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]] \
+ ) { \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \
+ threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \
+ threadgroup float bias[PASSB_BN]; \
+ const uint lin = local_tid.x; \
+ pass_b_indirect_impl<DTYPE>( \
+ hidden, \
+ expert_offsets, \
+ W2_all, \
+ down_biases, \
+ output, \
+ tile_map, \
+ d_model, \
+ d_ff, \
+ E, \
+ tg_pos.y, \
+ tg_pos.x, \
+ sg_id, \
+ simd_lid, \
+ lin, \
+ Hs, \
+ Wk, \
+ bias \
+ ); \
+ }
MOE_PASS_B_INDIRECT_KERNEL(bfloat, bf16)
MOE_PASS_B_INDIRECT_KERNEL(half, f16)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
index 9dc1853ba..3eff065a6 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
@@ -1,8 +1,5 @@
use crate::backends::metal::KernelDataType;
-mod encodable;
-pub use encodable::{MoeBlockEncodable, SharedMoeWeights};
-
// Submodules
mod counts_offsets_fused;
mod experts;
diff --git a/crates/uzu/src/backends/metal/kernel/pooling/mod.rs b/crates/uzu/src/backends/metal/kernel/pooling/mod.rs
new file mode 100644
index 000000000..1e0068168
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/pooling/mod.rs
@@ -0,0 +1,110 @@
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
+ MTLSize,
+};
+
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError},
+};
+
+pub struct PoolingKernel {
+ cls_pipeline: ComputePipelineState,
+ mean_pipeline: ComputePipelineState,
+}
+
+impl PoolingKernel {
+ fn kernel_name_for_type(
+ data_type: DataType,
+ pooling_type: &str,
+ ) -> Result<String, MTLError> {
+ let suffix = match data_type {
+ DataType::F16 => "f16",
+ DataType::F32 => "f32",
+ DataType::BF16 => "bf16",
+ other => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for pooling: {:?}",
+ other
+ )));
+ },
+ };
+ Ok(format!("pool_{}_{}", pooling_type, suffix))
+ }
+
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ ) -> Result<Self, MTLError> {
+ let cls_fn_name = Self::kernel_name_for_type(data_type, "cls")?;
+ let mean_fn_name = Self::kernel_name_for_type(data_type, "mean")?;
+
+ let cls_pipeline =
+ context.compute_pipeline_state(&cls_fn_name, None)?;
+ let mean_pipeline =
+ context.compute_pipeline_state(&mean_fn_name, None)?;
+
+ Ok(Self {
+ cls_pipeline,
+ mean_pipeline,
+ })
+ }
+
+ pub fn encode_cls(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ input_buffer: &MTLBuffer,
+ output_buffer: &MTLBuffer,
+ batch_size: i32,
+ seq_len: i32,
+ hidden_dim: i32,
+ ) -> Result<(), MTLError> {
+ encoder.set_compute_pipeline_state(&self.cls_pipeline);
+ encoder.set_buffer(0, Some(input_buffer), 0);
+ encoder.set_buffer(1, Some(output_buffer), 0);
+ encoder.set_bytes(
+ 2,
+ std::mem::size_of::<i32>() as u64,
+ &seq_len as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 3,
+ std::mem::size_of::<i32>() as u64,
+ &hidden_dim as *const i32 as *const _,
+ );
+
+ let threads_per_tg = MTLSize::new(16, 16, 1);
+ let grid = MTLSize::new(hidden_dim as u64, batch_size as u64, 1);
+ encoder.dispatch_threads(grid, threads_per_tg);
+ Ok(())
+ }
+
+ pub fn encode_mean(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ input_buffer: &MTLBuffer,
+ output_buffer: &MTLBuffer,
+ batch_size: i32,
+ seq_len: i32,
+ hidden_dim: i32,
+ ) -> Result<(), MTLError> {
+ encoder.set_compute_pipeline_state(&self.mean_pipeline);
+ encoder.set_buffer(0, Some(input_buffer), 0);
+ encoder.set_buffer(1, Some(output_buffer), 0);
+ encoder.set_bytes(
+ 2,
+ std::mem::size_of::<i32>() as u64,
+ &seq_len as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 3,
+ std::mem::size_of::<i32>() as u64,
+ &hidden_dim as *const i32 as *const _,
+ );
+
+ let threads_per_tg = MTLSize::new(16, 16, 1);
+ let grid = MTLSize::new(hidden_dim as u64, batch_size as u64, 1);
+ encoder.dispatch_threads(grid, threads_per_tg);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
new file mode 100644
index 000000000..e2c0eb7f2
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
@@ -0,0 +1,96 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+using namespace metal;
+
+// CLS pooling: Extract first token [batch, seq_len, hidden_dim] → [batch,
+// hidden_dim]
+template <typename T>
+kernel void pool_cls(
+ const device T* input [[buffer(0)]],
+ device T* output [[buffer(1)]],
+ constant int& seq_len [[buffer(2)]],
+ constant int& hidden_dim [[buffer(3)]],
+ uint2 tid [[thread_position_in_grid]]
+) // x: hidden_dim, y: batch
+{
+ int batch_idx = tid.y;
+ int dim_idx = tid.x;
+ if (dim_idx >= hidden_dim)
+ return;
+
+ output[batch_idx * hidden_dim + dim_idx] =
+ input[batch_idx * seq_len * hidden_dim + dim_idx];
+}
+
+// Mean pooling: Average across sequence [batch, seq_len, hidden_dim] → [batch,
+// hidden_dim]
+template <typename T>
+kernel void pool_mean(
+ const device T* input [[buffer(0)]],
+ device T* output [[buffer(1)]],
+ constant int& seq_len [[buffer(2)]],
+ constant int& hidden_dim [[buffer(3)]],
+ uint2 tid [[thread_position_in_grid]]
+) {
+ int batch_idx = tid.y;
+ int dim_idx = tid.x;
+ if (dim_idx >= hidden_dim)
+ return;
+
+ float sum = 0.0f;
+ for (int seq_idx = 0; seq_idx < seq_len; seq_idx++) {
+ sum += float(
+ input[batch_idx * seq_len * hidden_dim + seq_idx * hidden_dim + dim_idx]
+ );
+ }
+ output[batch_idx * hidden_dim + dim_idx] = T(sum / float(seq_len));
+}
+
+// Explicit instantiations for f16, f32, bf16
+template [[host_name("pool_cls_f16")]] [[kernel]] void pool_cls<half>(
+ const device half* input [[buffer(0)]],
+ device half* output [[buffer(1)]],
+ constant int& seq_len [[buffer(2)]],
+ constant int& hidden_dim [[buffer(3)]],
+ uint2 tid [[thread_position_in_grid]]
+);
+
+template [[host_name("pool_cls_f32")]] [[kernel]] void pool_cls<float>(
+ const device float* input [[buffer(0)]],
+ device float* output [[buffer(1)]],
+ constant int& seq_len [[buffer(2)]],
+ constant int& hidden_dim [[buffer(3)]],
+ uint2 tid [[thread_position_in_grid]]
+);
+
+template [[host_name("pool_cls_bf16")]] [[kernel]] void pool_cls<bfloat>(
+ const device bfloat* input [[buffer(0)]],
+ device bfloat* output [[buffer(1)]],
+ constant int& seq_len [[buffer(2)]],
+ constant int& hidden_dim [[buffer(3)]],
+ uint2 tid [[thread_position_in_grid]]
+);
+
+template [[host_name("pool_mean_f16")]] [[kernel]] void pool_mean<half>(
+ const device half* input [[buffer(0)]],
+ device half* output [[buffer(1)]],
+ constant int& seq_len [[buffer(2)]],
+ constant int& hidden_dim [[buffer(3)]],
+ uint2 tid [[thread_position_in_grid]]
+);
+
+template [[host_name("pool_mean_f32")]] [[kernel]] void pool_mean<float>(
+ const device float* input [[buffer(0)]],
+ device float* output [[buffer(1)]],
+ constant int& seq_len [[buffer(2)]],
+ constant int& hidden_dim [[buffer(3)]],
+ uint2 tid [[thread_position_in_grid]]
+);
+
+template [[host_name("pool_mean_bf16")]] [[kernel]] void pool_mean<bfloat>(
+ const device bfloat* input [[buffer(0)]],
+ device bfloat* output [[buffer(1)]],
+ constant int& seq_len [[buffer(2)]],
+ constant int& hidden_dim [[buffer(3)]],
+ uint2 tid [[thread_position_in_grid]]
+);
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
index 3753f8d3a..39b476540 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
@@ -1,23 +1,13 @@
-use std::{mem::size_of, rc::Rc};
+use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, ComputeCommandEncoderRef,
ComputePipelineState as MTLComputePipelineState, MTLSize,
};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
use crate::{
- Array, DataType,
- backends::metal::{
- MTLContext, MTLError,
- forward_pass::{
- ArrayId, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- metal_extensions::ComputeEncoderConditional,
- },
- config::{RMSNormConfig, UpcastMode},
- parameters::ParameterTree,
+ DataType,
+ backends::metal::{MTLContext, MTLError},
};
#[derive(Debug, Clone, Copy, PartialEq)]
@@ -209,12 +199,10 @@ impl RMSNormKernel {
&self,
compute_encoder: &ComputeCommandEncoderRef,
args: RMSNormArguments,
- ) {
- assert_eq!(
- self.kernel_type,
- RMSNormKernelType::Standard,
- "Invalid kernel type for standard RMS norm operation"
- );
+ ) -> Result<(), RMSNormError> {
+ if self.kernel_type != RMSNormKernelType::Standard {
+ return Err(RMSNormError::InvalidKernelType);
+ }
compute_encoder.set_compute_pipeline_state(&self.pipeline);
@@ -261,18 +249,18 @@ impl RMSNormKernel {
threadgroups_per_grid,
threads_per_threadgroup,
);
+
+ Ok(())
}
pub fn encode_qk_norm(
&self,
compute_encoder: &ComputeCommandEncoderRef,
args: QKNormArguments,
- ) {
- assert_eq!(
- self.kernel_type,
- RMSNormKernelType::QueryKey,
- "Invalid kernel type for QK norm operation"
- );
+ ) -> Result<(), RMSNormError> {
+ if self.kernel_type != RMSNormKernelType::QueryKey {
+ return Err(RMSNormError::InvalidKernelType);
+ }
compute_encoder.set_compute_pipeline_state(&self.pipeline);
@@ -353,417 +341,7 @@ impl RMSNormKernel {
threadgroups_per_grid,
threads_per_threadgroup,
);
- }
-}
-
-pub struct RMSNormKernelEncodable {
- kernel: RMSNormKernel,
- config: RMSNormConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- scales_buffer: MTLBuffer,
-}
-
-impl RMSNormKernelEncodable {
- pub fn new(
- context: &MTLContext,
- intermediate_data_type: DataType,
- config: RMSNormConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, RMSNormError> {
- // Load scales from parameter tree
- let scales_param = parameter_tree.leaf("scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(
- crate::backends::metal::error::LibraryError::Custom(format!(
- "Failed to load scales: {:?}",
- e
- )),
- ))
- })?;
-
- // TODO: Don't create buffers dynamically, we need to use forward pass storage for thing like this
- let scales_data = scales_param.buffer();
- let scales_buffer = context.device.new_buffer_with_data(
- scales_data.as_ptr() as *const _,
- scales_data.len() as u64,
- metal::MTLResourceOptions::StorageModeShared,
- );
-
- let accumulation_data_type: DataType =
- config.accumulation_precision.into();
- let scale_data_type: DataType = config.scale_precision.into();
-
- let (input_type, scales_type, output_type) = match config.upcast_mode {
- UpcastMode::OnlyNormalization => {
- // Input stays as pipeline type, scales stay scale precision, output is scale precision
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
- UpcastMode::FullLayer => {
- // Input stays as pipeline type, scales stay in original precision (will be cast to AccumT inside kernel), output is scale precision
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
- };
-
- let kernel = RMSNormKernel::new_with_mode(
- context,
- input_type,
- scales_type,
- output_type,
- accumulation_data_type,
- RMSNormKernelType::Standard,
- config.upcast_mode == UpcastMode::FullLayer,
- )?;
-
- Ok(Self {
- kernel,
- config,
- input_array_id,
- output_array_id,
- scales_buffer,
- })
- }
-}
-
-impl EncodableWithState for RMSNormKernelEncodable {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let input_binding = state.arrays(&[self.input_array_id]);
- let output_binding = state.arrays(&[self.output_array_id]);
-
- let input_shape = {
- let input_array = input_binding[0].borrow();
- input_array.shape().to_vec()
- };
-
- let mut input_array = input_binding[0].borrow_mut();
- let mut output_array = output_binding[0].borrow_mut();
-
- let input_buffer = unsafe { input_array.mtl_buffer() };
- let output_buffer = unsafe { output_array.mtl_buffer() };
-
- let batch_size = input_shape[0] as i32;
- let model_dim = input_shape[1] as i32;
-
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
-
- compute_encoder.condition(
- parameters.predicate_ref(),
- || {
- self.kernel.encode(
- &compute_encoder,
- RMSNormArguments {
- input_buffer: &input_buffer,
- scales_buffer: &self.scales_buffer,
- output_buffer: &output_buffer,
- batch_size,
- model_dim,
- epsilon: self.config.epsilon,
- scale_offset: self.config.scale_offset.unwrap_or(0.0),
- },
- );
- },
- None::<fn()>,
- );
-
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- let input_binding = state.arrays(&[self.input_array_id]);
- let output_binding = state.arrays(&[self.output_array_id]);
-
- let input_shape = {
- let input_array = input_binding[0].borrow();
- input_array.shape().to_vec()
- };
-
- let mut input_array = input_binding[0].borrow_mut();
- let mut output_array = output_binding[0].borrow_mut();
-
- let input_buffer = unsafe { input_array.mtl_buffer() };
- let output_buffer = unsafe { output_array.mtl_buffer() };
-
- let batch_size = input_shape[0] as i32;
- let model_dim = input_shape[1] as i32;
-
- self.kernel.encode(
- encoder,
- RMSNormArguments {
- input_buffer: &input_buffer,
- scales_buffer: &self.scales_buffer,
- output_buffer: &output_buffer,
- batch_size,
- model_dim,
- epsilon: self.config.epsilon,
- scale_offset: self.config.scale_offset.unwrap_or(0.0),
- },
- );
- }
-}
-
-pub struct QKNormKernelEncodable {
- query_kernel: Option<RMSNormKernel>,
- key_kernel: Option<RMSNormKernel>,
- query_config: Option<RMSNormConfig>,
- key_config: Option<RMSNormConfig>,
- qkv_array_id: ArrayId,
- query_scales_buffer: Option<MTLBuffer>,
- key_scales_buffer: Option<MTLBuffer>,
- num_q_heads: usize,
- num_kv_heads: usize,
- head_dim: usize,
-}
-
-impl QKNormKernelEncodable {
- pub fn new(
- context: &MTLContext,
- intermediate_data_type: DataType,
- query_config: Option<RMSNormConfig>,
- key_config: Option<RMSNormConfig>,
- qkv_array_id: ArrayId,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- num_q_heads: usize,
- num_kv_heads: usize,
- head_dim: usize,
- ) -> Result<Self, RMSNormError> {
- let mut query_kernel = None;
- let mut key_kernel = None;
- let mut query_scales_buffer = None;
- let mut key_scales_buffer = None;
-
- // Setup query normalization if configured
- if let Some(ref q_config) = query_config {
- let scales_param =
- parameter_tree.leaf("query_norm.scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(
- crate::backends::metal::error::LibraryError::Custom(
- format!("Failed to load query scales: {:?}", e),
- ),
- ))
- })?;
-
- let scales_data = scales_param.buffer();
- let scales_buffer = context.device.new_buffer_with_data(
- scales_data.as_ptr() as *const _,
- scales_data.len() as u64,
- metal::MTLResourceOptions::StorageModeShared,
- );
-
- let accumulation_data_type: DataType =
- q_config.accumulation_precision.into();
- let scale_data_type: DataType = q_config.scale_precision.into();
-
- let (input_type, scales_type, output_type) = match q_config
- .upcast_mode
- {
- UpcastMode::OnlyNormalization => {
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
- UpcastMode::FullLayer => {
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
- };
-
- let kernel = RMSNormKernel::new_with_mode(
- context,
- input_type,
- scales_type,
- output_type,
- accumulation_data_type,
- RMSNormKernelType::QueryKey,
- q_config.upcast_mode == UpcastMode::FullLayer,
- )?;
-
- query_kernel = Some(kernel);
- query_scales_buffer = Some(scales_buffer);
- }
-
- // Setup key normalization if configured
- if let Some(ref k_config) = key_config {
- let scales_param =
- parameter_tree.leaf("key_norm.scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(
- crate::backends::metal::error::LibraryError::Custom(
- format!("Failed to load key scales: {:?}", e),
- ),
- ))
- })?;
-
- let scales_data = scales_param.buffer();
- let scales_buffer = context.device.new_buffer_with_data(
- scales_data.as_ptr() as *const _,
- scales_data.len() as u64,
- metal::MTLResourceOptions::StorageModeShared,
- );
-
- let accumulation_data_type: DataType =
- k_config.accumulation_precision.into();
- let scale_data_type: DataType = k_config.scale_precision.into();
-
- let (input_type, scales_type, output_type) = match k_config
- .upcast_mode
- {
- UpcastMode::OnlyNormalization => {
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
- UpcastMode::FullLayer => {
- (intermediate_data_type, scale_data_type, scale_data_type)
- },
- };
-
- let kernel = RMSNormKernel::new_with_mode(
- context,
- input_type,
- scales_type,
- output_type,
- accumulation_data_type,
- RMSNormKernelType::QueryKey,
- k_config.upcast_mode == UpcastMode::FullLayer,
- )?;
-
- key_kernel = Some(kernel);
- key_scales_buffer = Some(scales_buffer);
- }
-
- Ok(Self {
- query_kernel,
- key_kernel,
- query_config,
- key_config,
- qkv_array_id,
- query_scales_buffer,
- key_scales_buffer,
- num_q_heads,
- num_kv_heads,
- head_dim,
- })
- }
-}
-
-impl EncodableWithState for QKNormKernelEncodable {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
-
- compute_encoder.condition(
- parameters.predicate_ref(),
- || {
- self.encode_with_encoder_impl(state, compute_encoder);
- },
- None::<fn()>,
- );
-
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- self.encode_with_encoder_impl(state, encoder);
- }
-}
-
-impl QKNormKernelEncodable {
- fn encode_with_encoder_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- ) {
- let qkv_binding = state.arrays(&[self.qkv_array_id]);
- let qkv_shape = {
- let qkv_array = qkv_binding[0].borrow();
- qkv_array.shape().to_vec()
- };
-
- let mut qkv_array = qkv_binding[0].borrow_mut();
- let qkv_buffer = unsafe { qkv_array.mtl_buffer() };
-
- let batch_size = qkv_shape[0] as i32;
- let head_dim = self.head_dim as i32;
-
- // Process query normalization if configured
- if let (
- Some(query_kernel),
- Some(query_scales_buffer),
- Some(query_config),
- ) =
- (&self.query_kernel, &self.query_scales_buffer, &self.query_config)
- {
- query_kernel.encode_qk_norm(
- encoder,
- QKNormArguments {
- qkv_input_buffer: &qkv_buffer,
- scales_buffer: query_scales_buffer,
- qkv_output_buffer: &qkv_buffer,
- batch_size,
- num_q_heads: self.num_q_heads as i32,
- num_kv_heads: self.num_kv_heads as i32,
- head_dim,
- epsilon: query_config.epsilon,
- scale_offset: query_config.scale_offset.unwrap_or(0.0),
- target: QKNormTarget::QueryHeads,
- },
- );
- }
-
- if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =
- (&self.key_kernel, &self.key_scales_buffer, &self.key_config)
- {
- key_kernel.encode_qk_norm(
- encoder,
- QKNormArguments {
- qkv_input_buffer: &qkv_buffer,
- scales_buffer: key_scales_buffer,
- qkv_output_buffer: &qkv_buffer,
- batch_size,
- num_q_heads: self.num_q_heads as i32,
- num_kv_heads: self.num_kv_heads as i32,
- head_dim,
- epsilon: key_config.epsilon,
- scale_offset: key_config.scale_offset.unwrap_or(0.0),
- target: QKNormTarget::KeyHeads,
- },
- );
- }
+ Ok(())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
index ec4964985..5ece64af7 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
@@ -4,20 +4,9 @@ use metal::{
Buffer as MTLBuffer, ComputeCommandEncoderRef,
ComputePipelineState as MTLComputePipelineState, MTLSize,
};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
use thiserror::Error;
-use crate::{
- Array,
- backends::metal::{
- KernelDataType, MTLContext, MTLError,
- forward_pass::{
- ArrayId, ForwardPassState, RopeType,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- metal_extensions::ComputeEncoderConditional,
- },
-};
+use crate::backends::metal::{KernelDataType, MTLContext, MTLError};
pub struct RopeKernel {
pipeline_state: MTLComputePipelineState,
@@ -131,135 +120,3 @@ impl RopeKernel {
.dispatch_threads(total_threads, threads_per_threadgroup);
}
}
-
-pub struct RopeKernelEncodable {
- kernel: RopeKernel,
- rope_type: RopeType,
-}
-
-impl RopeKernelEncodable {
- pub fn new(
- context: &MTLContext,
- data_type: KernelDataType,
- rope_type: RopeType,
- ) -> Result<Self, RopeError> {
- let kernel = RopeKernel::new(context, data_type)?;
- Ok(Self {
- kernel,
- rope_type,
- })
- }
-}
-
-impl EncodableWithState for RopeKernelEncodable {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
-
- compute_encoder.condition(
- parameters.predicate_ref(),
- || {
- self.encode_with_encoder_impl(state, compute_encoder);
- },
- None::<fn()>,
- );
-
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- self.encode_with_encoder_impl(state, encoder);
- }
-}
-
-impl RopeKernelEncodable {
- fn encode_with_encoder_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- ) {
- let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {
- let qkv_binding = state.arrays(&[ArrayId::QKV]);
- let qkv_array = qkv_binding[0].borrow();
- let suffix_length = qkv_array.shape()[0];
-
- let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
- let queries_array = queries_binding[0].borrow();
- let num_heads = queries_array.shape()[0];
- let head_dim = queries_array.shape()[2];
-
- let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
- let keys_array = keys_binding[0].borrow();
- let num_groups = keys_array.shape()[0];
-
- let cos_binding =
- state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
- let cos_array = cos_binding[0].borrow();
- let cos_shape = cos_array.shape();
- let rope_max_seq_len = cos_shape[0];
-
- (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len)
- };
-
- let qkv_buffer_binding = state.arrays(&[ArrayId::QKV]);
- let mut qkv = qkv_buffer_binding[0].borrow_mut();
-
- let token_positions_binding = state.arrays(&[ArrayId::TokenPositions]);
- let mut token_positions = token_positions_binding[0].borrow_mut();
-
- let query_buffer_binding = state.arrays(&[ArrayId::RotatedQueries]);
- let mut rotated_queries = query_buffer_binding[0].borrow_mut();
-
- let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
- let mut rotated_keys = rotated_keys_binding[0].borrow_mut();
-
- let cos_buffer_binding =
- state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
- let mut rope_cosines = cos_buffer_binding[0].borrow_mut();
-
- let sin_buffer_binding =
- state.arrays(&[ArrayId::RopeSines(self.rope_type)]);
- let mut rope_sines = sin_buffer_binding[0].borrow_mut();
-
- self.kernel.encode(
- encoder,
- RopeKernelArguments {
- qkv_buffer: unsafe { &qkv.mtl_buffer() },
- cosines_buffer: unsafe { &rope_cosines.mtl_buffer() },
- sines_buffer: unsafe { &rope_sines.mtl_buffer() },
- token_positions_buffer: unsafe {
- &token_positions.mtl_buffer()
- },
- rotated_queries_buffer: unsafe {
- &rotated_queries.mtl_buffer()
- },
- rotated_keys_buffer: unsafe { &rotated_keys.mtl_buffer() },
- head_dim,
- num_heads,
- num_groups,
- suffix_length,
- max_sequence_length: rope_max_seq_len,
- },
- );
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
index 216e26933..1e599cc6c 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
@@ -1,22 +1,15 @@
-use std::{mem::size_of, rc::Rc};
+use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
MTLResourceOptions, MTLSize,
};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
use thiserror::Error;
use crate::{
DataType,
- backends::metal::{
- KernelDataType, MTLContext, MTLError,
- forward_pass::{
- ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- },
+ backends::metal::{KernelDataType, MTLContext, MTLError},
session::parameter::SamplingMethod,
};
@@ -58,8 +51,6 @@ enum ArgmaxImplementation {
}
pub struct SamplingKernel {
- bitmask_pipeline: MTLComputePipelineState,
- partial_bitmask_buffer: MTLBuffer,
temperature_pipeline: MTLComputePipelineState,
partial_temperature_buffer: MTLBuffer,
topk_pipeline: MTLComputePipelineState,
@@ -113,20 +104,6 @@ impl SamplingKernel {
let data_suffix = data_type.function_name_suffix();
let max_elements = max_batch_size * max_vocab_size;
- let bitmask_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_bitmask_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
-
- let partial_bitmask_buffer = context.device.new_buffer(
- (max_elements * Into::<DataType>::into(data_type).size_in_bytes())
- as u64,
- MTLResourceOptions::StorageModeShared,
- );
-
let temperature_pipeline = context
.compute_pipeline_state_with_reflection(
&format!("batched_temperature_{}", data_suffix),
@@ -235,8 +212,6 @@ impl SamplingKernel {
};
Ok(Self {
- bitmask_pipeline,
- partial_bitmask_buffer,
temperature_pipeline,
partial_temperature_buffer,
topk_pipeline,
@@ -254,7 +229,6 @@ impl SamplingKernel {
pub fn encode(
&self,
logits_buffer: &MTLBuffer,
- bitmask_buffer: Option<&MTLBuffer>,
seeds_buffer: Option<&MTLBuffer>,
sampled_tokens_buffer: &MTLBuffer,
sampling_method: SamplingMethod,
@@ -265,9 +239,7 @@ impl SamplingKernel {
let compute_encoder = command_buffer.new_compute_command_encoder();
self.encode_with_encoder(
logits_buffer,
- bitmask_buffer,
seeds_buffer,
- 0, // No offset for non-async path
sampled_tokens_buffer,
sampling_method,
batch_size,
@@ -281,9 +253,7 @@ impl SamplingKernel {
pub fn encode_with_encoder(
&self,
logits_buffer: &MTLBuffer,
- bitmask_buffer: Option<&MTLBuffer>,
seeds_buffer: Option<&MTLBuffer>,
- seeds_offset: usize,
sampled_tokens_buffer: &MTLBuffer,
sampling_method: SamplingMethod,
batch_size: usize,
@@ -305,18 +275,6 @@ impl SamplingKernel {
let mut last_logits_buffer = logits_buffer;
- if let Some(bitmask_buffer) = bitmask_buffer {
- self.encode_bitmask(
- last_logits_buffer,
- bitmask_buffer,
- &self.partial_bitmask_buffer,
- batch_size as u32,
- vocab_size as u32,
- compute_encoder,
- )?;
- last_logits_buffer = &self.partial_bitmask_buffer;
- }
-
if let SamplingMethod::Stochastic {
temperature,
top_k,
@@ -362,7 +320,6 @@ impl SamplingKernel {
self.encode_gumbel(
last_logits_buffer,
seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,
- seeds_offset,
&self.partial_gumbel_buffer,
batch_size as u32,
vocab_size as u32,
@@ -399,38 +356,6 @@ impl SamplingKernel {
}
}
- pub fn encode_bitmask(
- &self,
- logits_buffer: &MTLBuffer,
- bitmask_buffer: &MTLBuffer,
- processed_logits_buffer: &MTLBuffer,
- batch_size: u32,
- vocab_size: u32,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- compute_encoder.set_compute_pipeline_state(&self.bitmask_pipeline);
-
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(bitmask_buffer), 0);
- compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);
- compute_encoder.set_bytes(
- 3,
- size_of::<u32>() as u64,
- &vocab_size as *const u32 as *const std::ffi::c_void,
- );
-
- let elements_in_group = BLOCK_SIZE * ELEMENTWISE_GRAIN_SIZE;
- let groups = (vocab_size + (elements_in_group as u32 - 1))
- / elements_in_group as u32;
-
- compute_encoder.dispatch_thread_groups(
- MTLSize::new(groups as u64, batch_size as u64, 1),
- MTLSize::new(BLOCK_SIZE as u64, 1, 1),
- );
-
- Ok(())
- }
-
pub fn encode_temperature(
&self,
logits_buffer: &MTLBuffer,
@@ -535,7 +460,6 @@ impl SamplingKernel {
&self,
logits_buffer: &MTLBuffer,
seeds_buffer: &MTLBuffer,
- seeds_offset: usize,
processed_logits_buffer: &MTLBuffer,
batch_size: u32,
vocab_size: u32,
@@ -544,7 +468,7 @@ impl SamplingKernel {
compute_encoder.set_compute_pipeline_state(&self.gumbel_pipeline);
compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(seeds_buffer), seeds_offset as u64);
+ compute_encoder.set_buffer(1, Some(seeds_buffer), 0);
compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);
compute_encoder.set_bytes(
3,
@@ -659,141 +583,3 @@ impl SamplingKernel {
Ok(())
}
}
-
-pub struct SamplingKernelEncodable {
- pub kernel: SamplingKernel,
-}
-
-impl SamplingKernelEncodable {
- pub fn new(
- context: &Rc<MTLContext>,
- data_type: KernelDataType,
- max_batch_size: usize,
- max_vocab_size: usize,
- ) -> Result<Self, SamplingError> {
- let kernel = SamplingKernel::new(
- context,
- data_type,
- max_batch_size,
- max_vocab_size,
- )?;
- Ok(Self {
- kernel,
- })
- }
-
- pub fn new_with_strategy(
- context: &Rc<MTLContext>,
- data_type: KernelDataType,
- max_batch_size: usize,
- max_vocab_size: usize,
- argmax_strategy: ArgmaxStrategy,
- ) -> Result<Self, SamplingError> {
- let kernel = SamplingKernel::new_with_strategy(
- context,
- data_type,
- max_batch_size,
- max_vocab_size,
- argmax_strategy,
- )?;
- Ok(Self {
- kernel,
- })
- }
-}
-
-impl EncodableWithState for SamplingKernelEncodable {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let root_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let encoder = root_command_buffer.new_compute_command_encoder();
-
- self.encode_impl(state, encoder);
-
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- root_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &metal::ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- self.encode_impl(state, encoder);
- }
-}
-
-impl SamplingKernelEncodable {
- fn encode_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &metal::ComputeCommandEncoderRef,
- ) {
- assert!(
- state.sampling_output.is_some(),
- "Sampling output buffer must be pre-allocated"
- );
-
- let logits_binding = state
- .arrays(&[crate::backends::metal::forward_pass::ArrayId::Logits]);
- let mut logits = logits_binding[0].borrow_mut();
-
- let logits_shape = {
- use crate::Array;
- logits.shape()
- };
- let batch_size = state.active_suffix_length();
- let vocab_size = logits_shape[1];
-
- let bitmask_binding = if state.has_bitmask() {
- let bitmask_binding = state.arrays(&[
- crate::backends::metal::forward_pass::ArrayId::TokenBitmask,
- ]);
- Some(bitmask_binding)
- } else {
- None
- };
-
- let mut bitmask =
- bitmask_binding.as_ref().map(|bind| bind[0].borrow_mut());
-
- let seeds_binding = state.arrays(&[
- crate::backends::metal::forward_pass::ArrayId::TokenSeeds,
- ]);
- let mut seeds = seeds_binding[0].borrow_mut();
-
- let mut output_buffer_ref =
- state.sampling_output.as_ref().unwrap().borrow_mut();
-
- let sampling_method = state.sampling_method.unwrap();
- let seeds_offset = seeds.buffer_offset();
-
- if let Err(e) = self.kernel.encode_with_encoder(
- unsafe { &logits.mtl_buffer() },
- unsafe { bitmask.as_mut().map(|arr| arr.mtl_buffer()) },
- unsafe { Some(&seeds.mtl_buffer()) },
- seeds_offset,
- unsafe { &output_buffer_ref.mtl_buffer() },
- sampling_method,
- batch_size,
- vocab_size,
- encoder,
- ) {
- panic!("Sampling encoding failed: {:?}", e);
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/mod.rs b/crates/uzu/src/backends/metal/kernel/sigmoid/mod.rs
new file mode 100644
index 000000000..598a890a5
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/mod.rs
@@ -0,0 +1,62 @@
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
+ MTLSize,
+};
+
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError},
+};
+
+pub struct SigmoidKernel {
+ pipeline: ComputePipelineState,
+}
+
+impl SigmoidKernel {
+ fn kernel_name_for_type(
+ data_type: DataType
+ ) -> Result<&'static str, MTLError> {
+ match data_type {
+ DataType::F16 => Ok("apply_sigmoid_f16"),
+ DataType::F32 => Ok("apply_sigmoid_f32"),
+ DataType::BF16 => Ok("apply_sigmoid_bf16"),
+ other => Err(MTLError::Generic(format!(
+ "Unsupported dtype for sigmoid: {:?}",
+ other
+ ))),
+ }
+ }
+
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ ) -> Result<Self, MTLError> {
+ let fn_name = Self::kernel_name_for_type(data_type)?;
+ let pipeline = context.compute_pipeline_state(fn_name, None)?;
+ Ok(Self {
+ pipeline,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ input_buffer: &MTLBuffer,
+ output_buffer: &MTLBuffer,
+ total_elements: i32,
+ ) -> Result<(), MTLError> {
+ encoder.set_compute_pipeline_state(&self.pipeline);
+ encoder.set_buffer(0, Some(input_buffer), 0);
+ encoder.set_buffer(1, Some(output_buffer), 0);
+ encoder.set_bytes(
+ 2,
+ std::mem::size_of::<i32>() as u64,
+ &total_elements as *const i32 as *const _,
+ );
+
+ let threads_per_tg = MTLSize::new(256, 1, 1);
+ let grid = MTLSize::new(total_elements as u64, 1, 1);
+ encoder.dispatch_threads(grid, threads_per_tg);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
new file mode 100644
index 000000000..8391a045e
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
@@ -0,0 +1,41 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+using namespace metal;
+
+// Apply sigmoid element-wise to logits
+template <typename T>
+kernel void apply_sigmoid(
+ const device T* input [[buffer(0)]],
+ device T* output [[buffer(1)]],
+ constant int& total_elements [[buffer(2)]],
+ uint tid [[thread_position_in_grid]]
+) {
+ if (tid >= total_elements)
+ return;
+ float x = float(input[tid]);
+ output[tid] = T(1.0f / (1.0f + exp(-x)));
+}
+
+// Explicit instantiations
+template [[host_name("apply_sigmoid_f16")]] [[kernel]] void apply_sigmoid<half>(
+ const device half* input [[buffer(0)]],
+ device half* output [[buffer(1)]],
+ constant int& total_elements [[buffer(2)]],
+ uint tid [[thread_position_in_grid]]
+);
+
+template [[host_name("apply_sigmoid_f32")]] [[kernel]] void apply_sigmoid<
+ float>(
+ const device float* input [[buffer(0)]],
+ device float* output [[buffer(1)]],
+ constant int& total_elements [[buffer(2)]],
+ uint tid [[thread_position_in_grid]]
+);
+
+template [[host_name("apply_sigmoid_bf16")]] [[kernel]] void apply_sigmoid<
+ bfloat>(
+ const device bfloat* input [[buffer(0)]],
+ device bfloat* output [[buffer(1)]],
+ constant int& total_elements [[buffer(2)]],
+ uint tid [[thread_position_in_grid]]
+);
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
index 9b23b0f10..baf58a079 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
@@ -13,7 +13,6 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {
}
pub mod conv1d_scan;
-pub mod mamba;
pub mod split_inproj;
pub mod ssd_prefill;
pub mod ssd_update;
@@ -21,7 +20,6 @@ pub mod ssd_update;
pub use conv1d_scan::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
};
-pub(crate) use mamba::MambaMixerEncodable;
pub use split_inproj::{SplitInProjArguments, SplitInProjKernel};
pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode};
pub use ssd_update::{SSDUpdateArguments, SSDUpdateKernel};
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
index 19336e899..7145e7278 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
@@ -2,35 +2,21 @@ use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
+ ComputePipelineState as MTLComputePipelineState,
};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use super::{
- KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,
-};
-use crate::{
- Array,
- backends::metal::{
- error::MTLError,
- forward_pass::{
- ArrayId, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- },
-};
+use super::{KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch};
+use crate::backends::metal::error::MTLError;
#[derive(Debug)]
-pub struct TensorAddSwap {
+pub struct TensorAddSwapKernel {
pipeline_state: MTLComputePipelineState,
- argument_arrays: Box<[ArrayId]>,
}
-impl TensorAddSwap {
+impl TensorAddSwapKernel {
pub fn new(
context: &MTLContext,
data_type: KernelDataType,
- argument_arrays: Box<[ArrayId]>,
) -> Result<Self, MTLError> {
let function_name =
format!("tensorAddSwap_{}", data_type.function_name_suffix());
@@ -40,99 +26,31 @@ impl TensorAddSwap {
Ok(Self {
pipeline_state,
- argument_arrays,
})
}
- fn encode_into_command_buffer(
+ pub fn encode_into_command_buffer(
&self,
skip_buffer: &MTLBuffer,
main_buffer: &MTLBuffer,
length: usize,
command_buffer: &MTLCommandBuffer,
) {
- let encoder = command_buffer.new_compute_command_encoder();
- self.encode_with_encoder_raw(skip_buffer, main_buffer, length, encoder);
- encoder.end_encoding();
- }
+ let compute_encoder = command_buffer.new_compute_command_encoder();
- fn encode_with_encoder_raw(
- &self,
- skip_buffer: &MTLBuffer,
- main_buffer: &MTLBuffer,
- length: usize,
- encoder: &ComputeCommandEncoderRef,
- ) {
- encoder.set_label("Tensor Add-Swap");
- encoder.set_buffer(0, Some(skip_buffer), 0);
- encoder.set_buffer(1, Some(main_buffer), 0);
- encoder.set_bytes(
+ compute_encoder.set_label("Tensor Add-Swap");
+
+ compute_encoder.set_buffer(0, Some(skip_buffer), 0);
+ compute_encoder.set_buffer(1, Some(main_buffer), 0);
+
+ compute_encoder.set_bytes(
2,
size_of::<i32>() as u64,
&(length as i32) as *const _ as *const std::ffi::c_void,
);
- encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);
- }
-}
-
-impl EncodableWithState for TensorAddSwap {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let arrays = state.arrays(&self.argument_arrays);
- assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
- let length = arrays[0].borrow().num_elements();
+ compute_encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);
- let mut skip_array = arrays[0].borrow_mut();
- let mut main_array = arrays[1].borrow_mut();
- let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };
- let main_mtl_buffer = unsafe { main_array.mtl_buffer() };
-
- let retained_mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
-
- self.encode_into_command_buffer(
- &skip_mtl_buffer,
- &main_mtl_buffer,
- length,
- &retained_mtl_command_buffer,
- );
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- retained_mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- let arrays = state.arrays(&self.argument_arrays);
- assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
-
- let length = arrays[0].borrow().num_elements();
-
- let mut skip_array = arrays[0].borrow_mut();
- let mut main_array = arrays[1].borrow_mut();
- let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };
- let main_mtl_buffer = unsafe { main_array.mtl_buffer() };
-
- self.encode_with_encoder_raw(
- &skip_mtl_buffer,
- &main_mtl_buffer,
- length,
- encoder,
- );
+ compute_encoder.end_encoding();
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
index d278a8ba6..dd67c59ad 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
@@ -2,32 +2,21 @@ use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
+ ComputePipelineState as MTLComputePipelineState,
};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use super::{
- KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,
-};
-use crate::{
- Array,
- backends::metal::forward_pass::{
- ArrayId, ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
-};
+use super::{KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch};
+use crate::backends::metal::error::MTLError;
-pub struct TensorCopy {
+pub struct TensorCopyKernel {
pipeline_state: MTLComputePipelineState,
- argument_arrays: Box<[ArrayId]>,
}
-impl TensorCopy {
+impl TensorCopyKernel {
pub fn new(
context: &MTLContext,
data_type: KernelDataType,
- argument_arrays: Box<[ArrayId]>,
- ) -> Result<Self, crate::backends::metal::error::MTLError> {
+ ) -> Result<Self, MTLError> {
let function_name =
format!("tensorCopy_{}", data_type.function_name_suffix());
@@ -36,106 +25,28 @@ impl TensorCopy {
Ok(Self {
pipeline_state,
- argument_arrays,
})
}
- fn encode_into_command_buffer(
+ pub fn encode_into_command_buffer(
&self,
source_buffer: &MTLBuffer,
destination_buffer: &MTLBuffer,
length: usize,
command_buffer: &MTLCommandBuffer,
- _predicate: Option<&MTLBuffer>,
) {
- let encoder = command_buffer.new_compute_command_encoder();
- self.encode_with_encoder_raw(
- source_buffer,
- destination_buffer,
- length,
- encoder,
- );
- encoder.end_encoding();
- }
+ let compute_encoder = command_buffer.new_compute_command_encoder();
+ compute_encoder.set_label("Tensor Copy");
- fn encode_with_encoder_raw(
- &self,
- source_buffer: &MTLBuffer,
- destination_buffer: &MTLBuffer,
- length: usize,
- encoder: &ComputeCommandEncoderRef,
- ) {
- encoder.set_label("Tensor Copy");
- encoder.set_buffer(0, Some(source_buffer), 0);
- encoder.set_buffer(1, Some(destination_buffer), 0);
- encoder.set_bytes(
+ compute_encoder.set_buffer(0, Some(source_buffer), 0);
+ compute_encoder.set_buffer(1, Some(destination_buffer), 0);
+ compute_encoder.set_bytes(
2,
size_of::<i32>() as u64,
&(length as i32) as *const _ as *const std::ffi::c_void,
);
- encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);
- }
-}
-
-impl EncodableWithState for TensorCopy {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let arrays = state.arrays(&self.argument_arrays);
- assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
-
- let length = arrays[0].borrow().num_elements();
- let mut source_array = arrays[0].borrow_mut();
- let mut destination_array = arrays[1].borrow_mut();
- let source_mtl_buffer = unsafe { source_array.mtl_buffer() };
- let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };
-
- let retained_mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
-
- self.encode_into_command_buffer(
- &source_mtl_buffer,
- &destination_mtl_buffer,
- length,
- &retained_mtl_command_buffer,
- parameters.predicate,
- );
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- retained_mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- _parameters: &EncodingParameters,
- ) {
- let arrays = state.arrays(&self.argument_arrays);
- assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
-
- let length = arrays[0].borrow().num_elements();
-
- let mut source_array = arrays[0].borrow_mut();
- let mut destination_array = arrays[1].borrow_mut();
- let source_mtl_buffer = unsafe { source_array.mtl_buffer() };
- let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };
-
- self.encode_with_encoder_raw(
- &source_mtl_buffer,
- &destination_mtl_buffer,
- length,
- encoder,
- );
+ compute_encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);
+ compute_encoder.end_encoding();
}
}
diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rs
similarity index 86%
rename from crates/uzu/src/generator/context.rs
rename to crates/uzu/src/backends/metal/language_model_generator_context.rs
index edbbcb95f..97b1bc1a6 100644
--- a/crates/uzu/src/generator/context.rs
+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs
@@ -10,17 +10,18 @@ use metal::Event as MTLEvent;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use objc2::rc::Retained;
+use super::{
+ CacheLayers, Decoder, KVCacheUpdate, KernelDataType, MTLContext,
+ ModelShape,
+ compilation_parameters::CompilationConfig,
+ encodable_block::Sampling,
+ forward_pass::{ScratchBuffers, SharedBuffers},
+ kernel::TokenCopyKernel,
+};
use crate::{
- DataType,
- backends::metal::{
- CacheLayers, DecoderExecutables, KVCacheUpdate, KernelDataType,
- MTLContext, ModelShape,
- compilation_parameters::CompilationConfig,
- forward_pass::{ForwardPassBuffers, SharedBuffers},
- kernel::{SamplingKernelEncodable, TokenCopyKernel},
- },
+ DataType, DecoderConfig,
config::{LanguageModelConfig, ModelMetadata},
- generator::rng::DerivableSeed,
+ language_model::rng::DerivableSeed,
parameters::ParameterLoader,
session::{
config::DecodingConfig,
@@ -128,19 +129,20 @@ impl AsyncBuffers {
}
}
-pub struct GeneratorContext {
+pub struct LanguageModelGeneratorContext {
pub mtl_context: Rc<MTLContext>,
pub command_buffer: Retained<MPSCommandBuffer>,
pub cache_layers: Rc<RefCell<CacheLayers>>,
pub shared_buffers: Rc<RefCell<SharedBuffers>>,
- pub scratch_buffers: ForwardPassBuffers,
+ pub scratch_buffers: ScratchBuffers,
pub model_config: LanguageModelConfig,
+ pub decoder_config: Rc<DecoderConfig>,
pub model_shape: ModelShape,
- pub executables: DecoderExecutables,
+ pub executables: Decoder,
pub kv_cache_update: Box<KVCacheUpdate>,
- pub gpu_sampler: SamplingKernelEncodable,
+ pub gpu_sampler: Sampling,
pub next_seed: DerivableSeed,
/// Kernel for copying sampled tokens in async pipeline
@@ -149,7 +151,7 @@ pub struct GeneratorContext {
pub async_buffers: AsyncBuffers,
}
-impl GeneratorContext {
+impl LanguageModelGeneratorContext {
pub fn new(
model_path: &Path,
decoding_config: &DecodingConfig,
@@ -171,17 +173,25 @@ impl GeneratorContext {
let model_metadata: ModelMetadata =
serde_json::from_reader(BufReader::new(config_file))
.map_err(|_| Error::UnableToLoadConfig)?;
- let decoder_config =
- Rc::new(model_metadata.model_config.decoder_config.clone());
+
+ // Extract language model config
+ let language_model_config = model_metadata
+ .model_config
+ .as_language_model()
+ .ok_or(Error::UnableToLoadConfig)?;
+
+ let decoder_config = Rc::new(
+ language_model_config
+ .decoder_config()
+ .map_err(|_| Error::UnableToLoadConfig)?,
+ );
let model_shape = ModelShape::from_decoder_config(&decoder_config);
- let prefill_step_size = decoding_config
- .prefill_step_size
- .resolve(&model_metadata.model_config);
+ let prefill_step_size =
+ decoding_config.prefill_step_size.resolve(language_model_config);
let generate_suffix_length = decoding_config.generate_suffix_length();
- let max_prefix_length: usize = decoding_config
- .context_length
- .resolve(&model_metadata.model_config);
+ let max_prefix_length: usize =
+ decoding_config.context_length.resolve(language_model_config);
let max_suffix_length: usize =
std::cmp::max(prefill_step_size, generate_suffix_length);
@@ -208,7 +218,7 @@ impl GeneratorContext {
)));
shared_buffers.borrow_mut().update_data(&root_loader_view);
- let scratch_buffers = ForwardPassBuffers::new(
+ let scratch_buffers = ScratchBuffers::new(
&mtl_context,
&decoder_config,
&model_shape,
@@ -216,7 +226,7 @@ impl GeneratorContext {
max_suffix_length,
);
- let executables = DecoderExecutables::new(
+ let executables = Decoder::new(
mtl_context.clone(),
decoder_config.clone(),
&root_loader_view,
@@ -242,7 +252,7 @@ impl GeneratorContext {
.map_err(|_| Error::UnableToCreateMetalContext)?,
);
- let gpu_sampler = SamplingKernelEncodable::new(
+ let gpu_sampler = Sampling::new(
&mtl_context,
kernel_data_type,
max_suffix_length,
@@ -270,7 +280,8 @@ impl GeneratorContext {
cache_layers,
shared_buffers,
scratch_buffers,
- model_config: model_metadata.model_config.clone(),
+ model_config: language_model_config.clone(),
+ decoder_config,
model_shape,
executables,
kv_cache_update,
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 3f56dbd95..908ddbb34 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -1,28 +1,29 @@
mod array;
mod buffer_allocator;
+mod classifier_context;
pub mod compilation_parameters;
mod context;
+pub mod encodable_block;
pub mod error;
-mod executable_builder;
pub mod forward_pass;
pub mod graph;
pub mod image;
pub mod kernel;
+mod language_model_generator_context;
pub mod media_utils;
pub mod metal_extensions;
pub mod placement_analysis;
pub mod utils;
+
pub use array::MetalArray;
pub use buffer_allocator::BufferAllocator;
+pub use classifier_context::ClassifierContext;
pub use context::MTLContext;
+pub use encodable_block::Decoder;
pub use error::MTLError;
-pub use executable_builder::DecoderExecutables;
pub use forward_pass::{CacheLayers, ForwardPassState, ModelShape};
pub use kernel::{KVCacheUpdate, KernelDataType, RopeKernel};
-// pub use kernel::{
-// DataType as KernelDataType, Encoder as KernelEncoder,
-// EncoderError as KernelEncoderError, TensorAddSwap, TensorCopy,
-// };
+pub use language_model_generator_context::LanguageModelGeneratorContext;
pub use media_utils::{
ImagePreprocessingParams, ImagePreprocessingRequirements,
MetalImagePreprocessor,
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
new file mode 100644
index 000000000..aacaa1444
--- /dev/null
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -0,0 +1,290 @@
+#[cfg(feature = "tracing")]
+use std::{cell::RefCell, rc::Rc};
+use std::{collections::HashMap, path::Path, time::Instant};
+
+use objc2::rc::autoreleasepool;
+
+#[cfg(feature = "tracing")]
+use super::ActivationTrace;
+use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
+use crate::{
+ Array, DataType,
+ backends::metal::forward_pass::{
+ ArrayId, EncodableBlock, EncodingParameters, ForwardPassState,
+ },
+ session::types::Error,
+};
+
+pub struct Classifier {
+ pub context: ClassifierContext,
+}
+
+impl Classifier {
+ pub fn new(model_path: &Path) -> Result<Self, Error> {
+ let context = ClassifierContext::new(model_path)?;
+ Ok(Self {
+ context,
+ })
+ }
+
+ pub fn classify_tokens(
+ &mut self,
+ token_ids: Vec<u64>,
+ token_positions: Vec<usize>,
+ ) -> Result<ClassificationOutput, Error> {
+ let run_start = Instant::now();
+
+ autoreleasepool(|_pool| {
+ let forward_start = Instant::now();
+
+ #[cfg(feature = "tracing")]
+ let (logits, _traces) =
+ self.forward_pass_with_traces(&token_ids, &token_positions)?;
+
+ #[cfg(not(feature = "tracing"))]
+ let logits = self.forward_pass(&token_ids, &token_positions)?;
+
+ let forward_duration = forward_start.elapsed().as_secs_f64();
+
+ let postprocessing_start = Instant::now();
+ let probabilities = self.logits_to_probabilities(&logits)?;
+ let postprocessing_duration =
+ postprocessing_start.elapsed().as_secs_f64();
+
+ let (predicted_label, confidence) = probabilities
+ .iter()
+ .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
+ .map(|(label, prob)| (label.clone(), *prob))
+ .unwrap_or((String::from("unknown"), 0.0));
+
+ let stats = ClassificationStats::new(
+ 0.0,
+ forward_duration,
+ postprocessing_duration,
+ run_start.elapsed().as_secs_f64(),
+ token_ids.len() as u64,
+ predicted_label,
+ confidence,
+ );
+
+ Ok(ClassificationOutput {
+ logits,
+ probabilities,
+ stats,
+ })
+ })
+ }
+
+ #[cfg(feature = "tracing")]
+ pub fn forward_pass_with_traces(
+ &mut self,
+ token_ids: &[u64],
+ token_positions: &[usize],
+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace>>), Error> {
+ self.forward_pass(token_ids, token_positions)
+ }
+
+ #[cfg(feature = "tracing")]
+ fn forward_pass(
+ &mut self,
+ token_ids: &[u64],
+ token_positions: &[usize],
+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace>>), Error> {
+ autoreleasepool(|_| {
+ let num_labels = self.context.model_config.model_config.num_labels;
+ let mut state = ForwardPassState::new_classifier(
+ self.context.mtl_context.clone(),
+ &self.context.model_shape,
+ &self.context.scratch_buffers,
+ self.context.shared_buffers.clone(),
+ token_ids,
+ token_positions,
+ true,
+ num_labels,
+ );
+
+ let encoding_params = EncodingParameters::new(false, true, false);
+
+ self.context.embed.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ self.context.embedding_norm.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ #[cfg(feature = "tracing")]
+ {
+ let traces = state.traces().clone();
+ state.encode_copy_array(
+ &self.context.command_buffer,
+ ArrayId::Main,
+ traces.borrow().embedding_norm().clone(),
+ );
+ }
+
+ for layer in self.context.layers.iter() {
+ layer.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ }
+ self.context.output_norm.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ #[cfg(feature = "tracing")]
+ {
+ let traces = state.traces().clone();
+ state.encode_copy_array(
+ &self.context.command_buffer,
+ ArrayId::Main,
+ traces.borrow().output_norm.clone(),
+ );
+ }
+
+ self.context.pooling.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+
+ self.context.prediction_head.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+
+ let logits = self.copy_logits_from_state(&state)?;
+
+ let traces = state.traces().clone();
+ Ok((logits, traces))
+ })
+ }
+
+ #[cfg(not(feature = "tracing"))]
+ fn forward_pass(
+ &mut self,
+ token_ids: &[u64],
+ token_positions: &[usize],
+ ) -> Result<Box<[f32]>, Error> {
+ autoreleasepool(|_| {
+ let num_labels = self.context.model_config.model_config.num_labels;
+ let mut state = ForwardPassState::new_classifier(
+ self.context.mtl_context.clone(),
+ &self.context.model_shape,
+ &self.context.scratch_buffers,
+ self.context.shared_buffers.clone(),
+ token_ids,
+ token_positions,
+ true,
+ num_labels,
+ );
+
+ let encoding_params = EncodingParameters::new(false, true, false);
+
+ self.context.embed.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ self.context.embedding_norm.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ for layer in self.context.layers.iter() {
+ layer.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ }
+ self.context.output_norm.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ self.context.pooling.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+ self.context.prediction_head.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &encoding_params,
+ );
+
+ let logits = self.copy_logits_from_state(&state)?;
+
+ Ok(logits)
+ })
+ }
+
+ fn copy_logits_from_state(
+ &self,
+ state: &ForwardPassState,
+ ) -> Result<Box<[f32]>, Error> {
+ let logits_arrays =
+ state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
+ let logits_array = logits_arrays[0].borrow();
+
+ let num_labels = self.context.model_config.model_config.num_labels;
+ let buffer = Array::buffer(&*logits_array);
+
+ match Array::data_type(&*logits_array) {
+ DataType::F32 => {
+ let slice: &[f32] = bytemuck::cast_slice(buffer);
+ Ok(slice[..num_labels].into())
+ },
+ DataType::F16 => {
+ let slice: &[half::f16] = bytemuck::cast_slice(buffer);
+ Ok(slice[..num_labels]
+ .iter()
+ .map(|&x| x.to_f32())
+ .collect::<Vec<_>>()
+ .into_boxed_slice())
+ },
+ DataType::BF16 => {
+ let slice: &[half::bf16] = bytemuck::cast_slice(buffer);
+ Ok(slice[..num_labels]
+ .iter()
+ .map(|&x| x.to_f32())
+ .collect::<Vec<_>>()
+ .into_boxed_slice())
+ },
+ _ => Err(Error::UnableToDecodeText),
+ }
+ }
+
+ fn logits_to_probabilities(
+ &self,
+ logits: &[f32],
+ ) -> Result<HashMap<String, f32>, Error> {
+ let output_labels =
+ &self.context.model_config.model_config.output_labels;
+ let mut probabilities = HashMap::new();
+
+ for (idx, &logit) in logits.iter().enumerate() {
+ let prob = 1.0 / (1.0 + (-logit).exp());
+
+ let label = if let Some(labels) = output_labels {
+ labels
+ .get(idx)
+ .map(|s| s.clone())
+ .unwrap_or_else(|| format!("class_{}", idx))
+ } else {
+ format!("class_{}", idx)
+ };
+
+ probabilities.insert(label, prob);
+ }
+
+ Ok(probabilities)
+ }
+}
diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rs
new file mode 100644
index 000000000..680439779
--- /dev/null
+++ b/crates/uzu/src/classifier/mod.rs
@@ -0,0 +1,16 @@
+mod classifier;
+mod result;
+
+pub use classifier::Classifier;
+pub use result::{ClassificationOutput, ClassificationStats};
+
+// Re-export context from metal backend
+pub use crate::backends::metal::ClassifierContext;
+// Re-export encodables from metal backend for convenience
+pub use crate::backends::metal::encodable_block::{
+ ClassifierLayer, ClassifierPredictionHead, Pooling,
+};
+#[cfg(feature = "tracing")]
+pub use crate::backends::metal::forward_pass::traces::{
+ ActivationTrace, LayerActivationTrace,
+};
diff --git a/crates/uzu/src/classifier/result.rs b/crates/uzu/src/classifier/result.rs
new file mode 100644
index 000000000..651520b1e
--- /dev/null
+++ b/crates/uzu/src/classifier/result.rs
@@ -0,0 +1,53 @@
+use std::collections::HashMap;
+
+#[derive(Debug, Clone)]
+pub struct ClassificationOutput {
+ pub logits: Box<[f32]>,
+ pub probabilities: HashMap<String, f32>,
+ pub stats: ClassificationStats,
+}
+
+#[derive(Debug, Clone)]
+pub struct ClassificationStats {
+ // Timing breakdown
+ pub preprocessing_duration: f64,
+ pub forward_pass_duration: f64,
+ pub postprocessing_duration: f64,
+ pub total_duration: f64,
+
+ // Token metrics
+ pub tokens_count: u64,
+ pub tokens_per_second: f64,
+
+ pub predicted_label: String,
+ pub confidence: f32,
+}
+
+impl ClassificationStats {
+ pub fn new(
+ preprocessing_duration: f64,
+ forward_pass_duration: f64,
+ postprocessing_duration: f64,
+ total_duration: f64,
+ tokens_count: u64,
+ predicted_label: String,
+ confidence: f32,
+ ) -> Self {
+ let tokens_per_second = if forward_pass_duration > 0.0 {
+ tokens_count as f64 / forward_pass_duration
+ } else {
+ 0.0
+ };
+
+ Self {
+ preprocessing_duration,
+ forward_pass_duration,
+ postprocessing_duration,
+ total_duration,
+ tokens_count,
+ tokens_per_second,
+ predicted_label,
+ confidence,
+ }
+ }
+}
diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rs
index e231fefde..ca1266c11 100644
--- a/crates/uzu/src/config/attention.rs
+++ b/crates/uzu/src/config/attention.rs
@@ -1,14 +1,14 @@
use serde::{Deserialize, Serialize};
-use super::{RMSNormConfig, linear::LinearConfig};
+use crate::{LinearConfig, NormalizationConfig};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct AttentionConfig {
pub qkv_projection_config: LinearConfig,
pub out_projection_config: LinearConfig,
- pub query_norm_config: Option<RMSNormConfig>,
- pub key_norm_config: Option<RMSNormConfig>,
+ pub query_norm_config: Option<NormalizationConfig>,
+ pub key_norm_config: Option<NormalizationConfig>,
pub num_heads: Option<usize>,
pub num_groups: Option<usize>,
@@ -18,6 +18,7 @@ pub struct AttentionConfig {
pub sliding_window_size: Option<usize>,
pub logit_soft_cap: Option<f32>,
+ #[serde(default)]
pub has_sinks: bool,
pub has_qkv_biases: bool,
pub has_out_biases: bool,
@@ -59,6 +60,12 @@ mod tests {
},
"query_norm_config": null,
"key_norm_config": null,
+ "num_heads": 12,
+ "num_groups": 12,
+ "head_dim": 64,
+ "is_causal": false,
+ "scale": null,
+ "sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkv_biases": false,
@@ -89,10 +96,10 @@ mod tests {
},
query_norm_config: None,
key_norm_config: None,
- num_heads: None,
- num_groups: None,
- head_dim: None,
- is_causal: None,
+ num_heads: Some(12),
+ num_groups: Some(12),
+ head_dim: Some(64),
+ is_causal: Some(false),
scale: None,
sliding_window_size: None,
logit_soft_cap: None,
diff --git a/crates/uzu/src/config/classifier/classifier_config.rs b/crates/uzu/src/config/classifier/classifier_config.rs
new file mode 100644
index 000000000..e3e30e5b3
--- /dev/null
+++ b/crates/uzu/src/config/classifier/classifier_config.rs
@@ -0,0 +1,117 @@
+use serde::{Deserialize, Serialize};
+
+use super::{PoolingType, PredictionHeadConfig};
+use crate::{
+ DecoderConfig, DecoderLayerConfig, EmbeddingConfig, LinearConfig,
+ NormalizationConfig, TransformerConfig, config::ConfigError,
+};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct ClassifierConfig {
+ pub embedding_config: EmbeddingConfig,
+ pub embedding_norm_config: NormalizationConfig,
+ pub transformer_config: TransformerConfig,
+ pub prediction_head_config: PredictionHeadConfig,
+ pub readout_config: LinearConfig,
+
+ pub vocab_size: usize,
+ pub model_dim: usize,
+ pub hidden_dim: usize,
+ #[serde(default)]
+ pub num_heads: Option<usize>,
+ #[serde(default)]
+ pub num_groups: Option<usize>,
+ #[serde(default)]
+ pub head_dim: Option<usize>,
+ pub attention_scale: Option<f32>,
+ pub num_layers: usize,
+ #[serde(default)]
+ pub sliding_window_sizes: Option<Vec<Option<usize>>>,
+ pub context_length: usize,
+ pub num_labels: usize,
+ pub classifier_pooling: PoolingType,
+ pub output_labels: Option<Vec<String>>,
+}
+
+impl ClassifierConfig {
+ pub fn to_decoder_config(&self) -> Result<DecoderConfig, ConfigError> {
+ // For classifier, we use the first layer config as the template
+ // (most classifiers have homogeneous layers)
+ let first_layer = self
+ .transformer_config
+ .layer_configs
+ .first()
+ .ok_or(ConfigError::NoLayers)?;
+
+ let layer_config = DecoderLayerConfig {
+ pre_attention_norm_config: first_layer
+ .pre_attention_norm_config
+ .clone()
+ .unwrap_or_else(|| {
+ // If first layer has no pre-attention norm (like ModernBERT), use a default
+ self.transformer_config.output_norm_config.clone()
+ }),
+ mixer_config: first_layer.mixer_config.clone(),
+ post_attention_norm_config: first_layer
+ .post_attention_norm_config
+ .clone(),
+ pre_mlp_norm_config: first_layer.pre_mlp_norm_config.clone(),
+ mlp_config: first_layer.mlp_config.clone(),
+ post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),
+ };
+
+ let first_mixer = &first_layer.mixer_config;
+
+ let num_heads =
+ self.num_heads.or(first_mixer.num_heads()).ok_or_else(|| {
+ ConfigError::MissingField("num_heads".to_string())
+ })?;
+ let num_groups =
+ self.num_groups.or(first_mixer.num_groups()).unwrap_or(num_heads);
+ let head_dim = self
+ .head_dim
+ .or(first_mixer.head_dim())
+ .ok_or_else(|| ConfigError::MissingField("head_dim".to_string()))?;
+
+ Ok(DecoderConfig {
+ embedding_config: self.embedding_config.clone(),
+ global_rope_config: self
+ .transformer_config
+ .global_rope_config
+ .clone(),
+ local_rope_config: self
+ .transformer_config
+ .local_rope_config
+ .clone(),
+ layer_config,
+ output_norm_config: self
+ .transformer_config
+ .output_norm_config
+ .clone(),
+ vocab_size: self.vocab_size,
+ model_dim: self.model_dim,
+ hidden_dim: self.transformer_config.hidden_dim,
+ num_heads,
+ num_groups,
+ head_dim,
+ attention_scale: self.attention_scale,
+ num_layers: self.num_layers,
+ sliding_window_sizes: {
+ if let Some(sizes) = &self.sliding_window_sizes {
+ Some(sizes.clone().into_boxed_slice())
+ } else {
+ let sizes: Vec<Option<usize>> = self
+ .transformer_config
+ .layer_configs
+ .iter()
+ .map(|layer| layer.mixer_config.sliding_window_size())
+ .collect();
+ Some(sizes.into_boxed_slice())
+ }
+ },
+ context_length: self.context_length,
+ layer_configs: None, // Classifier doesn't use heterogeneous layers usually
+ layer_types: None,
+ })
+ }
+}
diff --git a/crates/uzu/src/config/classifier/mod.rs b/crates/uzu/src/config/classifier/mod.rs
new file mode 100644
index 000000000..7084c1314
--- /dev/null
+++ b/crates/uzu/src/config/classifier/mod.rs
@@ -0,0 +1,7 @@
+mod classifier_config;
+mod pooling_type;
+mod prediction_head_config;
+
+pub use classifier_config::ClassifierConfig;
+pub use pooling_type::PoolingType;
+pub use prediction_head_config::PredictionHeadConfig;
diff --git a/crates/uzu/src/config/classifier/pooling_type.rs b/crates/uzu/src/config/classifier/pooling_type.rs
new file mode 100644
index 000000000..473f31498
--- /dev/null
+++ b/crates/uzu/src/config/classifier/pooling_type.rs
@@ -0,0 +1,8 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(rename_all = "lowercase")]
+pub enum PoolingType {
+ Cls,
+ Mean,
+}
diff --git a/crates/uzu/src/config/classifier/prediction_head_config.rs b/crates/uzu/src/config/classifier/prediction_head_config.rs
new file mode 100644
index 000000000..8c45bab7c
--- /dev/null
+++ b/crates/uzu/src/config/classifier/prediction_head_config.rs
@@ -0,0 +1,12 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{Activation, LinearConfig, NormalizationConfig};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct PredictionHeadConfig {
+ pub dense_config: LinearConfig,
+ pub activation: Activation,
+ pub normalization_config: NormalizationConfig,
+ pub readout_config: LinearConfig,
+ pub use_dense_bias: bool,
+}
diff --git a/crates/uzu/src/config/classifier_model.rs b/crates/uzu/src/config/classifier_model.rs
new file mode 100644
index 000000000..f9f7448b5
--- /dev/null
+++ b/crates/uzu/src/config/classifier_model.rs
@@ -0,0 +1,9 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{ClassifierConfig, MessageProcessorConfig};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct ClassifierModelConfig {
+ pub model_config: ClassifierConfig,
+ pub message_processor_config: MessageProcessorConfig,
+}
diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rs
index 72e0a84b6..26259a00e 100644
--- a/crates/uzu/src/config/decoder.rs
+++ b/crates/uzu/src/config/decoder.rs
@@ -6,7 +6,7 @@ use serde::{
use super::{
decoder_layer::{DecoderLayerConfig, MixerConfig},
embedding::EmbeddingConfig,
- normalization::RMSNormConfig,
+ normalization::NormalizationConfig,
rope::RoPEConfig,
};
@@ -32,7 +32,7 @@ pub struct DecoderConfig {
pub local_rope_config: Option<RoPEConfig>,
pub layer_config: DecoderLayerConfig,
pub layer_configs: Option<Box<[DecoderLayerConfig]>>,
- pub output_norm_config: RMSNormConfig,
+ pub output_norm_config: NormalizationConfig,
pub vocab_size: usize,
pub model_dim: usize,
@@ -181,7 +181,7 @@ struct RawDecoderConfig {
layer_config: Option<DecoderLayerConfig>,
#[serde(default)]
layer_configs: Option<Vec<DecoderLayerConfig>>,
- output_norm_config: RMSNormConfig,
+ output_norm_config: NormalizationConfig,
vocab_size: usize,
model_dim: usize,
hidden_dim: usize,
@@ -231,13 +231,17 @@ impl DecoderConfig {
}
pub fn has_attention_layers(&self) -> bool {
- if let Some(layer_types) = &self.layer_types {
- layer_types
- .iter()
- .any(|lt| matches!(lt, DecoderLayerType::Transformer))
- } else {
- matches!(self.layer_config.mixer_config, MixerConfig::Attention(_))
- }
+ self.layer_configs
+ .as_ref()
+ .map(|configs| {
+ configs.iter().any(|config| {
+ matches!(
+ config.mixer_config,
+ crate::config::MixerConfig::Attention(_)
+ )
+ })
+ })
+ .unwrap_or(true)
}
}
@@ -309,6 +313,14 @@ mod tests {
"lora_rank": 16,
"lora_scale": 2.0
},
+ "query_norm_config": null,
+ "key_norm_config": null,
+ "num_heads": 12,
+ "num_groups": 12,
+ "head_dim": 64,
+ "is_causal": true,
+ "scale": null,
+ "sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkv_biases": false,
@@ -380,19 +392,21 @@ mod tests {
}),
local_rope_config: None,
layer_config: DecoderLayerConfig {
- pre_attention_norm_config: RMSNormConfig {
+ pre_attention_norm_config: NormalizationConfig {
scale_precision: ConfigDataType::BFloat16,
accumulation_precision: ConfigDataType::Float32,
epsilon: 1e-5,
scale_offset: None,
upcast_mode: UpcastMode::OnlyNormalization,
+ subtract_mean: false,
},
- pre_mlp_norm_config: RMSNormConfig {
+ pre_mlp_norm_config: NormalizationConfig {
scale_precision: ConfigDataType::BFloat16,
accumulation_precision: ConfigDataType::Float32,
epsilon: 1e-5,
scale_offset: None,
upcast_mode: UpcastMode::OnlyNormalization,
+ subtract_mean: false,
},
mixer_config: MixerConfig::Attention(AttentionConfig {
qkv_projection_config: LinearConfig::QLoRA {
@@ -421,10 +435,10 @@ mod tests {
},
query_norm_config: None,
key_norm_config: None,
- num_heads: None,
- num_groups: None,
- head_dim: None,
- is_causal: None,
+ num_heads: Some(12),
+ num_groups: Some(12),
+ head_dim: Some(64),
+ is_causal: Some(true),
scale: None,
sliding_window_size: None,
logit_soft_cap: None,
@@ -448,16 +462,22 @@ mod tests {
activation: Activation::SiLU {
alpha: 1.0,
},
+ has_up_biases: false,
+ has_down_biases: false,
+ gate_clipping: None,
+ up_clipping: None,
+ activation_to_gate: true,
}),
post_attention_norm_config: None,
post_mlp_norm_config: None,
},
- output_norm_config: RMSNormConfig {
+ output_norm_config: NormalizationConfig {
scale_precision: ConfigDataType::BFloat16,
accumulation_precision: ConfigDataType::Float32,
epsilon: 1e-5,
scale_offset: None,
upcast_mode: UpcastMode::OnlyNormalization,
+ subtract_mean: false,
},
layer_configs: None,
vocab_size: 128256,
diff --git a/crates/uzu/src/config/decoder_layer.rs b/crates/uzu/src/config/decoder_layer.rs
index be78c548e..1889420cd 100644
--- a/crates/uzu/src/config/decoder_layer.rs
+++ b/crates/uzu/src/config/decoder_layer.rs
@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
use super::{
attention::AttentionConfig, mamba::Mamba2Config, mlp::MLPConfig,
- normalization::RMSNormConfig,
+ normalization::NormalizationConfig,
};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
@@ -68,16 +68,16 @@ impl MixerConfig {
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct DecoderLayerConfig {
#[serde(alias = "pre_mixer_norm_config")]
- pub pre_attention_norm_config: RMSNormConfig,
+ pub pre_attention_norm_config: NormalizationConfig,
#[serde(alias = "mixer_config")]
pub mixer_config: MixerConfig,
#[serde(alias = "post_mixer_norm_config")]
- pub post_attention_norm_config: Option<RMSNormConfig>,
+ pub post_attention_norm_config: Option<NormalizationConfig>,
#[serde(alias = "pre_mlp_norm_config")]
- pub pre_mlp_norm_config: RMSNormConfig,
+ pub pre_mlp_norm_config: NormalizationConfig,
pub mlp_config: MLPConfig,
#[serde(alias = "post_mlp_norm_config")]
- pub post_mlp_norm_config: Option<RMSNormConfig>,
+ pub post_mlp_norm_config: Option<NormalizationConfig>,
}
impl DecoderLayerConfig {
@@ -131,6 +131,14 @@ mod tests {
"lora_rank": 16,
"lora_scale": 2.0
},
+ "query_norm_config": null,
+ "key_norm_config": null,
+ "num_heads": 12,
+ "num_groups": 12,
+ "head_dim": 64,
+ "is_causal": true,
+ "scale": null,
+ "sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkv_biases": false,
@@ -162,19 +170,21 @@ mod tests {
"#;
let ground_truth_config = DecoderLayerConfig {
- pre_attention_norm_config: RMSNormConfig {
+ pre_attention_norm_config: NormalizationConfig {
scale_precision: ConfigDataType::BFloat16,
accumulation_precision: ConfigDataType::Float32,
epsilon: 1e-5,
scale_offset: None,
upcast_mode: UpcastMode::OnlyNormalization,
+ subtract_mean: false,
},
- pre_mlp_norm_config: RMSNormConfig {
+ pre_mlp_norm_config: NormalizationConfig {
scale_precision: ConfigDataType::BFloat16,
accumulation_precision: ConfigDataType::Float32,
epsilon: 1e-5,
scale_offset: None,
upcast_mode: UpcastMode::OnlyNormalization,
+ subtract_mean: false,
},
mixer_config: MixerConfig::Attention(AttentionConfig {
qkv_projection_config: LinearConfig::QLoRA {
@@ -203,16 +213,16 @@ mod tests {
},
query_norm_config: None,
key_norm_config: None,
+ num_heads: Some(12),
+ num_groups: Some(12),
+ head_dim: Some(64),
+ is_causal: Some(true),
+ scale: None,
+ sliding_window_size: None,
logit_soft_cap: None,
has_sinks: false,
has_qkv_biases: false,
has_out_biases: false,
- num_heads: None,
- num_groups: None,
- head_dim: None,
- is_causal: None,
- scale: None,
- sliding_window_size: None,
}),
mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {
linear_config: LinearConfig::QLoRA {
@@ -230,6 +240,11 @@ mod tests {
activation: Activation::SiLU {
alpha: 1.0,
},
+ has_up_biases: false,
+ has_down_biases: false,
+ gate_clipping: None,
+ up_clipping: None,
+ activation_to_gate: true,
}),
post_attention_norm_config: None,
post_mlp_norm_config: None,
diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rs
index 5a7bcb90c..f3277d2f7 100644
--- a/crates/uzu/src/config/embedding.rs
+++ b/crates/uzu/src/config/embedding.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use super::common::{ConfigDataType, QuantizationMode};
+use crate::{ConfigDataType, QuantizationMode};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct EmbeddingConfigCommon {
diff --git a/crates/uzu/src/config/error.rs b/crates/uzu/src/config/error.rs
new file mode 100644
index 000000000..aed9c0118
--- /dev/null
+++ b/crates/uzu/src/config/error.rs
@@ -0,0 +1,9 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum ConfigError {
+ #[error("Missing required field: {0}")]
+ MissingField(String),
+ #[error("No layers in transformer config")]
+ NoLayers,
+}
diff --git a/crates/uzu/src/config/language_model.rs b/crates/uzu/src/config/language_model.rs
index da949c549..373028222 100644
--- a/crates/uzu/src/config/language_model.rs
+++ b/crates/uzu/src/config/language_model.rs
@@ -1,10 +1,136 @@
use serde::{Deserialize, Serialize};
-use super::{DecoderConfig, GenerationConfig, MessageProcessorConfig};
+use crate::{
+ DecoderConfig, DecoderLayerConfig, DecoderLayerType, EmbeddingConfig,
+ GenerationConfig, MessageProcessorConfig, MixerConfig, TransformerConfig,
+ config::ConfigError,
+};
+
+/// Inner model config matching the new lalamo export format.
+/// Contains embedding_config at the top level, with transformer_config nested.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct InnerModelConfig {
+ pub embedding_config: EmbeddingConfig,
+ pub transformer_config: TransformerConfig,
+ pub vocab_size: usize,
+}
+
+impl InnerModelConfig {
+ /// Convert to DecoderConfig for backward compatibility with the rest of the codebase.
+ pub fn to_decoder_config(&self) -> Result<DecoderConfig, ConfigError> {
+ let tf = &self.transformer_config;
+
+ let first_layer =
+ tf.layer_configs.first().ok_or(ConfigError::NoLayers)?;
+
+ let layer_config = DecoderLayerConfig {
+ pre_attention_norm_config: first_layer
+ .pre_attention_norm_config
+ .clone()
+ .unwrap_or_else(|| tf.output_norm_config.clone()),
+ mixer_config: first_layer.mixer_config.clone(),
+ post_attention_norm_config: first_layer
+ .post_attention_norm_config
+ .clone(),
+ pre_mlp_norm_config: first_layer.pre_mlp_norm_config.clone(),
+ mlp_config: first_layer.mlp_config.clone(),
+ post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),
+ };
+
+ let mixer = &first_layer.mixer_config;
+ let num_heads =
+ tf.num_heads.or(mixer.num_heads()).ok_or_else(|| {
+ ConfigError::MissingField("num_heads".to_string())
+ })?;
+ let num_groups =
+ tf.num_groups.or(mixer.num_groups()).unwrap_or(num_heads);
+ let head_dim = tf
+ .head_dim
+ .or(mixer.head_dim())
+ .ok_or_else(|| ConfigError::MissingField("head_dim".to_string()))?;
+ let attention_scale = tf.attention_scale.or(mixer.attention_scale());
+ let num_layers = tf.num_layers.unwrap_or(tf.layer_configs.len());
+
+ let sliding_window_sizes: Box<[Option<usize>]> = tf
+ .layer_configs
+ .iter()
+ .map(|layer| layer.mixer_config.sliding_window_size())
+ .collect::<Vec<_>>()
+ .into_boxed_slice();
+
+ let layer_types: Box<[DecoderLayerType]> = tf
+ .layer_configs
+ .iter()
+ .map(|layer| Self::layer_type_from_mixer(&layer.mixer_config))
+ .collect::<Vec<_>>()
+ .into_boxed_slice();
+
+ let layer_configs: Box<[DecoderLayerConfig]> = tf
+ .layer_configs
+ .iter()
+ .map(|layer| DecoderLayerConfig {
+ pre_attention_norm_config: layer
+ .pre_attention_norm_config
+ .clone()
+ .unwrap_or_else(|| tf.output_norm_config.clone()),
+ mixer_config: layer.mixer_config.clone(),
+ post_attention_norm_config: layer
+ .post_attention_norm_config
+ .clone(),
+ pre_mlp_norm_config: layer.pre_mlp_norm_config.clone(),
+ mlp_config: layer.mlp_config.clone(),
+ post_mlp_norm_config: layer.post_mlp_norm_config.clone(),
+ })
+ .collect::<Vec<_>>()
+ .into_boxed_slice();
+
+ Ok(DecoderConfig {
+ embedding_config: self.embedding_config.clone(),
+ global_rope_config: tf.global_rope_config.clone(),
+ local_rope_config: tf.local_rope_config.clone(),
+ layer_config,
+ layer_configs: Some(layer_configs),
+ output_norm_config: tf.output_norm_config.clone(),
+ vocab_size: self.vocab_size,
+ model_dim: tf.model_dim,
+ hidden_dim: tf.hidden_dim,
+ num_heads,
+ num_groups,
+ head_dim,
+ attention_scale,
+ num_layers,
+ sliding_window_sizes: Some(sliding_window_sizes),
+ layer_types: Some(layer_types),
+ context_length: tf.context_length,
+ })
+ }
+
+ fn layer_type_from_mixer(mixer: &MixerConfig) -> DecoderLayerType {
+ match mixer {
+ MixerConfig::Attention(_) => DecoderLayerType::Transformer,
+ MixerConfig::Mamba(config) => DecoderLayerType::StateSpace {
+ conv_dim: config.conv_dim(),
+ kernel_size: config.kernel_size,
+ state_dim: config.state_dim,
+ num_heads: config.num_heads,
+ num_groups: config.num_groups,
+ head_dim: config.head_dim,
+ },
+ }
+ }
+}
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct LanguageModelConfig {
- pub decoder_config: DecoderConfig,
+ pub model_config: InnerModelConfig,
pub message_processor_config: MessageProcessorConfig,
pub generation_config: GenerationConfig,
}
+
+impl LanguageModelConfig {
+ /// Get the decoder config for backward compatibility.
+ /// This converts the new format to the old DecoderConfig format.
+ pub fn decoder_config(&self) -> Result<DecoderConfig, ConfigError> {
+ self.model_config.to_decoder_config()
+ }
+}
diff --git a/crates/uzu/src/config/linear.rs b/crates/uzu/src/config/linear.rs
index 49bd4ecab..510eeb614 100644
--- a/crates/uzu/src/config/linear.rs
+++ b/crates/uzu/src/config/linear.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use super::common::{ConfigDataType, QuantizationMode};
+use crate::{ConfigDataType, QuantizationMode};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct QuantizationConfig {
diff --git a/crates/uzu/src/config/metadata.rs b/crates/uzu/src/config/metadata.rs
index 573955a18..7c10b12f8 100644
--- a/crates/uzu/src/config/metadata.rs
+++ b/crates/uzu/src/config/metadata.rs
@@ -1,6 +1,10 @@
use serde::{Deserialize, Serialize};
-use super::LanguageModelConfig;
+use crate::{ModelConfig, ModelType};
+
+fn default_model_type() -> ModelType {
+ ModelType::LanguageModel
+}
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct ModelMetadata {
@@ -12,5 +16,7 @@ pub struct ModelMetadata {
pub quantization: Option<String>,
pub repo: String,
pub use_cases: Vec<String>,
- pub model_config: LanguageModelConfig,
+ #[serde(default = "default_model_type")]
+ pub model_type: ModelType,
+ pub model_config: ModelConfig,
}
diff --git a/crates/uzu/src/config/mlp.rs b/crates/uzu/src/config/mlp.rs
index fa9832b44..cd1699f73 100644
--- a/crates/uzu/src/config/mlp.rs
+++ b/crates/uzu/src/config/mlp.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use super::{common::Activation, linear::LinearConfig};
+use crate::{Activation, LinearConfig};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
#[serde(tag = "type")]
@@ -15,6 +15,20 @@ pub enum MLPConfig {
pub struct DenseMLPConfig {
pub linear_config: LinearConfig,
pub activation: Activation,
+ #[serde(default)]
+ pub has_up_biases: bool,
+ #[serde(default)]
+ pub has_down_biases: bool,
+ #[serde(default)]
+ pub gate_clipping: Option<[Option<f32>; 2]>,
+ #[serde(default)]
+ pub up_clipping: Option<[Option<f32>; 2]>,
+ #[serde(default = "default_activation_to_gate")]
+ pub activation_to_gate: bool,
+}
+
+fn default_activation_to_gate() -> bool {
+ true
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
@@ -83,6 +97,11 @@ mod tests {
activation: Activation::SiLU {
alpha: 1.0,
},
+ has_up_biases: false,
+ has_down_biases: false,
+ gate_clipping: None,
+ up_clipping: None,
+ activation_to_gate: true,
});
let deserialized_config: MLPConfig = from_str(config_str).unwrap();
diff --git a/crates/uzu/src/config/mod.rs b/crates/uzu/src/config/mod.rs
index c08e0df97..7abe6c828 100644
--- a/crates/uzu/src/config/mod.rs
+++ b/crates/uzu/src/config/mod.rs
@@ -1,28 +1,43 @@
-pub mod attention;
+mod attention;
+mod classifier;
+mod classifier_model;
+mod common;
+mod decoder;
+mod decoder_layer;
+mod embedding;
+mod error;
+mod generation;
+mod language_model;
+mod linear;
+mod mamba;
+mod message_processor;
+mod metadata;
+mod mlp;
+mod model_config;
+mod model_type;
+mod normalization;
+mod rope;
+mod transformer;
+mod transformer_layer;
+
pub use attention::*;
-pub mod common;
+pub use classifier::*;
+pub use classifier_model::*;
pub use common::*;
-pub mod decoder;
pub use decoder::*;
-pub mod decoder_layer;
pub use decoder_layer::*;
-pub mod embedding;
pub use embedding::*;
-pub mod linear;
+pub use error::*;
+pub use generation::*;
+pub use language_model::*;
pub use linear::*;
-pub mod mlp;
+pub use mamba::*;
+pub use message_processor::*;
+pub use metadata::*;
pub use mlp::*;
-pub mod normalization;
+pub use model_config::*;
+pub use model_type::*;
pub use normalization::*;
-pub mod mamba;
-pub use mamba::*;
-pub mod rope;
pub use rope::*;
-pub mod metadata;
-pub use metadata::*;
-pub mod language_model;
-pub use language_model::*;
-pub mod message_processor;
-pub use message_processor::*;
-pub mod generation;
-pub use generation::*;
+pub use transformer::*;
+pub use transformer_layer::*;
diff --git a/crates/uzu/src/config/model_config.rs b/crates/uzu/src/config/model_config.rs
new file mode 100644
index 000000000..10873810f
--- /dev/null
+++ b/crates/uzu/src/config/model_config.rs
@@ -0,0 +1,34 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{ClassifierModelConfig, LanguageModelConfig};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(untagged)]
+pub enum ModelConfig {
+ LanguageModel(LanguageModelConfig),
+ Classifier(ClassifierModelConfig),
+}
+
+impl ModelConfig {
+ pub fn as_language_model(&self) -> Option<&LanguageModelConfig> {
+ match self {
+ ModelConfig::LanguageModel(config) => Some(config),
+ _ => None,
+ }
+ }
+
+ pub fn as_classifier(&self) -> Option<&ClassifierModelConfig> {
+ match self {
+ ModelConfig::Classifier(config) => Some(config),
+ _ => None,
+ }
+ }
+
+ pub fn is_language_model(&self) -> bool {
+ matches!(self, ModelConfig::LanguageModel(_))
+ }
+
+ pub fn is_classifier(&self) -> bool {
+ matches!(self, ModelConfig::Classifier(_))
+ }
+}
diff --git a/crates/uzu/src/config/model_type.rs b/crates/uzu/src/config/model_type.rs
new file mode 100644
index 000000000..e5dc7d691
--- /dev/null
+++ b/crates/uzu/src/config/model_type.rs
@@ -0,0 +1,8 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum ModelType {
+ LanguageModel,
+ ClassifierModel,
+}
diff --git a/crates/uzu/src/config/normalization/mod.rs b/crates/uzu/src/config/normalization/mod.rs
new file mode 100644
index 000000000..7dcb34734
--- /dev/null
+++ b/crates/uzu/src/config/normalization/mod.rs
@@ -0,0 +1,5 @@
+mod normalization_config;
+mod upcast_mode;
+
+pub use normalization_config::*;
+pub use upcast_mode::*;
diff --git a/crates/uzu/src/config/normalization.rs b/crates/uzu/src/config/normalization/normalization_config.rs
similarity index 74%
rename from crates/uzu/src/config/normalization.rs
rename to crates/uzu/src/config/normalization/normalization_config.rs
index 95b7a5a7b..832ebb4a9 100644
--- a/crates/uzu/src/config/normalization.rs
+++ b/crates/uzu/src/config/normalization/normalization_config.rs
@@ -1,21 +1,17 @@
use serde::{Deserialize, Serialize};
-use super::common::ConfigDataType;
+use super::UpcastMode;
+use crate::ConfigDataType;
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
-#[serde(rename_all = "snake_case")]
-pub enum UpcastMode {
- OnlyNormalization,
- FullLayer,
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
-pub struct RMSNormConfig {
+pub struct NormalizationConfig {
pub scale_precision: ConfigDataType,
pub accumulation_precision: ConfigDataType,
pub epsilon: f32,
pub scale_offset: Option<f32>,
pub upcast_mode: UpcastMode,
+ #[serde(default)]
+ pub subtract_mean: bool,
}
#[cfg(test)]
@@ -36,15 +32,17 @@ mod tests {
}
"#;
- let ground_truth_config = RMSNormConfig {
+ let ground_truth_config = NormalizationConfig {
scale_precision: ConfigDataType::BFloat16,
accumulation_precision: ConfigDataType::Float32,
epsilon: 1e-05,
scale_offset: None,
upcast_mode: UpcastMode::OnlyNormalization,
+ subtract_mean: false,
};
- let deserialized_config: RMSNormConfig = from_str(config_str).unwrap();
+ let deserialized_config: NormalizationConfig =
+ from_str(config_str).unwrap();
assert_eq!(deserialized_config, ground_truth_config);
}
}
diff --git a/crates/uzu/src/config/normalization/upcast_mode.rs b/crates/uzu/src/config/normalization/upcast_mode.rs
new file mode 100644
index 000000000..f03914653
--- /dev/null
+++ b/crates/uzu/src/config/normalization/upcast_mode.rs
@@ -0,0 +1,8 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum UpcastMode {
+ OnlyNormalization,
+ FullLayer,
+}
diff --git a/crates/uzu/src/config/rope.rs b/crates/uzu/src/config/rope.rs
index 0b9d56abd..1f1290c13 100644
--- a/crates/uzu/src/config/rope.rs
+++ b/crates/uzu/src/config/rope.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use super::common::ConfigDataType;
+use crate::ConfigDataType;
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct RopeConfigCommon {
diff --git a/crates/uzu/src/config/transformer.rs b/crates/uzu/src/config/transformer.rs
new file mode 100644
index 000000000..b14e05685
--- /dev/null
+++ b/crates/uzu/src/config/transformer.rs
@@ -0,0 +1,25 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{NormalizationConfig, RoPEConfig, TransformerLayerConfig};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct TransformerConfig {
+ pub global_rope_config: Option<RoPEConfig>,
+ pub local_rope_config: Option<RoPEConfig>,
+ pub layer_configs: Vec<TransformerLayerConfig>,
+ pub output_norm_config: NormalizationConfig,
+
+ pub model_dim: usize,
+ pub hidden_dim: usize,
+ #[serde(default)]
+ pub num_heads: Option<usize>,
+ #[serde(default)]
+ pub num_groups: Option<usize>,
+ #[serde(default)]
+ pub head_dim: Option<usize>,
+ #[serde(default)]
+ pub attention_scale: Option<f32>,
+ #[serde(default)]
+ pub num_layers: Option<usize>,
+ pub context_length: usize,
+}
diff --git a/crates/uzu/src/config/transformer_layer.rs b/crates/uzu/src/config/transformer_layer.rs
new file mode 100644
index 000000000..d6809067d
--- /dev/null
+++ b/crates/uzu/src/config/transformer_layer.rs
@@ -0,0 +1,199 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{AttentionConfig, MLPConfig, MixerConfig, NormalizationConfig};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct TransformerLayerConfig {
+ #[serde(alias = "pre_mixer_norm_config")]
+ pub pre_attention_norm_config: Option<NormalizationConfig>,
+ pub mixer_config: MixerConfig,
+ #[serde(alias = "post_mixer_norm_config")]
+ pub post_attention_norm_config: Option<NormalizationConfig>,
+ pub pre_mlp_norm_config: NormalizationConfig,
+ pub mlp_config: MLPConfig,
+ pub post_mlp_norm_config: Option<NormalizationConfig>,
+}
+
+impl TransformerLayerConfig {
+ pub fn attention_config(&self) -> Option<&AttentionConfig> {
+ self.mixer_config.as_attention()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use serde_json::from_str;
+
+ use super::{
+ super::{
+ attention::AttentionConfig,
+ common::{Activation, ConfigDataType, QuantizationMode},
+ linear::{LinearConfig, QuantizationConfig},
+ normalization::UpcastMode,
+ },
+ *,
+ };
+ use crate::config::mlp;
+
+ #[test]
+ fn test_transformer_layer_config() {
+ let config_str = r#"
+ {
+ "pre_attention_norm_config": {
+ "scale_precision": "bfloat16",
+ "accumulation_precision": "float32",
+ "epsilon": 1e-05,
+ "scale_offset": null,
+ "upcast_mode": "only_normalization",
+ "subtract_mean": false
+ },
+ "mixer_config": {
+ "type": "AttentionConfig",
+ "qkv_projection_config": {
+ "type": "QLoRALinearConfig",
+ "group_size": 32,
+ "weight_quantization_mode": "uint4",
+ "activation_quantization_mode": "int8",
+ "activation_precision": "bfloat16",
+ "lora_rank": 16,
+ "lora_scale": 2.0
+ },
+ "out_projection_config": {
+ "type": "QLoRALinearConfig",
+ "group_size": 32,
+ "weight_quantization_mode": "uint4",
+ "activation_quantization_mode": "int8",
+ "activation_precision": "bfloat16",
+ "lora_rank": 16,
+ "lora_scale": 2.0
+ },
+ "query_norm_config": null,
+ "key_norm_config": null,
+ "num_heads": 12,
+ "num_groups": 12,
+ "head_dim": 64,
+ "is_causal": false,
+ "scale": null,
+ "sliding_window_size": null,
+ "logit_soft_cap": null,
+ "has_sinks": false,
+ "has_qkv_biases": false,
+ "has_out_biases": false
+ },
+ "post_attention_norm_config": null,
+ "pre_mlp_norm_config": {
+ "scale_precision": "bfloat16",
+ "accumulation_precision": "float32",
+ "epsilon": 1e-05,
+ "scale_offset": null,
+ "upcast_mode": "only_normalization",
+ "subtract_mean": false
+ },
+ "mlp_config": {
+ "type": "DenseMLPConfig",
+ "linear_config": {
+ "type": "QLoRALinearConfig",
+ "group_size": 32,
+ "weight_quantization_mode": "uint4",
+ "activation_quantization_mode": "int8",
+ "activation_precision": "bfloat16",
+ "lora_rank": 16,
+ "lora_scale": 2.0
+ },
+ "activation": {"type": "SiLU"},
+ "has_up_biases": false,
+ "has_down_biases": false,
+ "gate_clipping": null,
+ "up_clipping": null,
+ "activation_to_gate": false
+ },
+ "post_mlp_norm_config": null
+ }
+ "#;
+
+ let ground_truth_config = TransformerLayerConfig {
+ pre_attention_norm_config: Some(NormalizationConfig {
+ scale_precision: ConfigDataType::BFloat16,
+ accumulation_precision: ConfigDataType::Float32,
+ epsilon: 1e-5,
+ scale_offset: None,
+ upcast_mode: UpcastMode::OnlyNormalization,
+ subtract_mean: false,
+ }),
+ pre_mlp_norm_config: NormalizationConfig {
+ scale_precision: ConfigDataType::BFloat16,
+ accumulation_precision: ConfigDataType::Float32,
+ epsilon: 1e-5,
+ scale_offset: None,
+ upcast_mode: UpcastMode::OnlyNormalization,
+ subtract_mean: false,
+ },
+ mixer_config: MixerConfig::Attention(AttentionConfig {
+ qkv_projection_config: LinearConfig::QLoRA {
+ quantization: QuantizationConfig {
+ group_size: 32,
+ weight_quantization_mode: QuantizationMode::UInt4,
+ activation_quantization_mode: Some(
+ QuantizationMode::Int8,
+ ),
+ activation_precision: ConfigDataType::BFloat16,
+ },
+ lora_rank: 16,
+ lora_scale: 2.0,
+ },
+ out_projection_config: LinearConfig::QLoRA {
+ quantization: QuantizationConfig {
+ group_size: 32,
+ weight_quantization_mode: QuantizationMode::UInt4,
+ activation_quantization_mode: Some(
+ QuantizationMode::Int8,
+ ),
+ activation_precision: ConfigDataType::BFloat16,
+ },
+ lora_rank: 16,
+ lora_scale: 2.0,
+ },
+ query_norm_config: None,
+ key_norm_config: None,
+ num_heads: Some(12),
+ num_groups: Some(12),
+ head_dim: Some(64),
+ is_causal: Some(false),
+ scale: None,
+ sliding_window_size: None,
+ logit_soft_cap: None,
+ has_sinks: false,
+ has_qkv_biases: false,
+ has_out_biases: false,
+ }),
+ mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {
+ linear_config: LinearConfig::QLoRA {
+ quantization: QuantizationConfig {
+ group_size: 32,
+ weight_quantization_mode: QuantizationMode::UInt4,
+ activation_quantization_mode: Some(
+ QuantizationMode::Int8,
+ ),
+ activation_precision: ConfigDataType::BFloat16,
+ },
+ lora_rank: 16,
+ lora_scale: 2.0,
+ },
+ activation: Activation::SiLU {
+ alpha: 1.0,
+ },
+ has_up_biases: false,
+ has_down_biases: false,
+ gate_clipping: None,
+ up_clipping: None,
+ activation_to_gate: false,
+ }),
+ post_attention_norm_config: None,
+ post_mlp_norm_config: None,
+ };
+
+ let deserialized_config: TransformerLayerConfig =
+ from_str(config_str).unwrap();
+ assert_eq!(deserialized_config, ground_truth_config);
+ }
+}
diff --git a/crates/uzu/src/generator/mod.rs b/crates/uzu/src/generator/mod.rs
deleted file mode 100644
index dbe4efd98..000000000
--- a/crates/uzu/src/generator/mod.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-pub mod context;
-pub mod generator;
-pub mod gpu_capture;
-pub mod grammar;
-pub mod gumbel;
-pub mod result;
-pub mod rng;
-pub mod sampler;
-pub mod tasks;
diff --git a/crates/uzu/src/generator/gpu_capture.rs b/crates/uzu/src/language_model/gpu_capture.rs
similarity index 100%
rename from crates/uzu/src/generator/gpu_capture.rs
rename to crates/uzu/src/language_model/gpu_capture.rs
diff --git a/crates/uzu/src/generator/grammar.rs b/crates/uzu/src/language_model/grammar.rs
similarity index 100%
rename from crates/uzu/src/generator/grammar.rs
rename to crates/uzu/src/language_model/grammar.rs
diff --git a/crates/uzu/src/generator/gumbel.rs b/crates/uzu/src/language_model/gumbel.rs
similarity index 100%
rename from crates/uzu/src/generator/gumbel.rs
rename to crates/uzu/src/language_model/gumbel.rs
diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
similarity index 92%
rename from crates/uzu/src/generator/generator.rs
rename to crates/uzu/src/language_model/language_model_generator.rs
index 8197d0af8..3fb445fb3 100644
--- a/crates/uzu/src/generator/generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -7,18 +7,17 @@ use itertools::{Either, Itertools, izip};
use mpsgraph::CommandBuffer;
use super::{
- context::GeneratorContext,
+ LanguageModelGeneratorContext,
gpu_capture::GpuCaptureManager,
+ grammar::CompiledGrammar,
result::{GenerateResult, PrefillResult},
- tasks::{GeneratorEncodedTask, GeneratorRunTask},
+ tasks::{LanguageModelGeneratorEncodedTask, LanguageModelGeneratorRunTask},
};
use crate::{
Array,
backends::metal::forward_pass::{
- ForwardPassState, INVALID_POSITION,
- encodable_with_state::{EncodableWithState, EncodingParameters},
+ EncodableBlock, EncodingParameters, ForwardPassState, INVALID_POSITION,
},
- generator::grammar::CompiledGrammar,
session::{
config::DecodingConfig,
parameter::{ConfigResolvableValue, SamplingMethod},
@@ -28,24 +27,25 @@ use crate::{
utils::env_utils::MetalEnvVar,
};
-pub struct Generator {
+pub struct LanguageModelGenerator {
pub decoding_config: DecodingConfig,
pub tokens: Vec<u64>,
- pub context: GeneratorContext,
- encoded_tasks: HashMap<String, GeneratorEncodedTask>,
+ pub context: LanguageModelGeneratorContext,
+ encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask>,
registered_prefix_len: usize,
gpu_capture: GpuCaptureManager,
}
-impl Generator {
+impl LanguageModelGenerator {
pub fn new(
model_path: &Path,
decoding_config: DecodingConfig,
) -> Result<Self, Error> {
let gpu_capture = GpuCaptureManager::new();
- let context = GeneratorContext::new(model_path, &decoding_config)?;
+ let context =
+ LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
let prefill_step_size =
decoding_config.prefill_step_size.resolve(&context.model_config);
let generate_suffix_length = decoding_config.generate_suffix_length();
@@ -187,7 +187,7 @@ impl Generator {
let _ = last_state.take();
});
- let task = GeneratorRunTask {
+ let task = LanguageModelGeneratorRunTask {
token_ids: &step_token_ids,
token_positions: &step_token_positions,
token_bitmask: Some(&step_token_bitmask),
@@ -345,7 +345,7 @@ impl Generator {
.chain(repeat_n(0, suffix_length - active_suffix_length))
.collect::<Box<[u64]>>();
- let task = GeneratorRunTask {
+ let task = LanguageModelGeneratorRunTask {
token_ids: &token_ids,
token_positions: &token_positions,
token_bitmask: Some(&token_bitmask),
@@ -384,6 +384,162 @@ impl Generator {
})
}
+ /// Prepares async buffers for generation.
+ /// Must be called after prefill, before async_generate loop.
+ pub fn prepare_async(
+ &mut self,
+ tokens_to_generate: usize,
+ ) {
+ let prefill_count = self.tokens.len();
+ self.context
+ .async_buffers
+ .prepare_positions(prefill_count, tokens_to_generate);
+ self.context
+ .async_buffers
+ .prepare_seeds(&mut self.context.next_seed, tokens_to_generate);
+ self.context.async_buffers.reset_counter();
+ }
+
+ /// Submits a single async forward pass.
+ /// Does NOT block. Callback fires when GPU completes.
+ ///
+ /// - `pass_idx`: Index of this pass (0, 1, 2, ...)
+ /// - `sampling_method`: Sampling configuration
+ /// - `on_complete`: Callback receiving sampled token as u64
+ pub fn async_generate<F>(
+ &mut self,
+ pass_idx: usize,
+ sampling_method: SamplingMethod,
+ on_complete: F,
+ ) -> Result<(), Error>
+ where
+ F: FnOnce(u64) + Send + 'static,
+ {
+ assert_eq!(
+ self.decoding_config.generate_suffix_length(),
+ 1,
+ "async_generate only supports suffix_length=1"
+ );
+
+ // Extract values from async_buffers before mutable borrow
+ let current_counter = self.context.async_buffers.counter.get();
+ let is_continuation = current_counter > 0;
+ let batch_size = self.context.async_buffers.batch_size;
+ let slot = pass_idx % batch_size;
+ let async_event = self.context.async_buffers.event.clone();
+ let results_buffer = self.context.async_buffers.results.clone();
+ let async_positions_buffer =
+ self.context.async_buffers.positions.clone();
+ let async_seeds_buffer = self.context.async_buffers.seeds.clone();
+
+ let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
+
+ let token_ids: [u64; 1] = [last_token];
+ let token_positions: [usize; 1] = [0]; // Ignored, using async buffer
+ let token_seeds: [u64; 1] = [0]; // Ignored, using async buffer
+
+ let async_positions = Some((&async_positions_buffer, pass_idx));
+ let async_seeds = Some((&async_seeds_buffer, pass_idx));
+
+ let mut state = ForwardPassState::new_llm(
+ self.context.mtl_context.clone(),
+ &self.context.decoder_config,
+ &self.context.model_shape,
+ &self.context.scratch_buffers,
+ self.context.cache_layers.clone(),
+ self.context.shared_buffers.clone(),
+ &token_ids,
+ &token_positions,
+ None, // token_bitmask - ignored, using async buffer
+ &token_seeds,
+ 1, // active_suffix_length
+ false, // is_prefilling
+ None, // external_bias_fn
+ is_continuation,
+ async_positions,
+ async_seeds,
+ );
+ if let Some(method) = state.sampling_method_mut() {
+ *method = Some(sampling_method);
+ }
+
+ self.context.reset_command_buffer();
+ let root_command_buffer =
+ self.context.command_buffer.root_command_buffer();
+
+ // Wait on previous pass if this is a continuation
+ if is_continuation {
+ root_command_buffer
+ .encode_wait_for_event(&async_event, current_counter);
+ }
+
+ // Encode forward pass
+ self.context.executables.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &EncodingParameters::new(false, false, false),
+ );
+
+ // Encode sampling
+ self.context.gpu_sampler.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &EncodingParameters::new(false, false, false),
+ );
+
+ // Copy sampled token: sampling_output → token_ids (for next pass)
+ // and sampling_output → results[slot] (for callback)
+ let sampling_output = state
+ .sampling_output()
+ .expect("sampling_output must exist after sampling encode");
+ let sampling_output_buffer =
+ unsafe { sampling_output.borrow_mut().mtl_buffer().clone() };
+ let token_ids_buffer = self.context.scratch_buffers.token_ids.clone();
+
+ let encoder = root_command_buffer.new_compute_command_encoder();
+ self.context.token_copy.encode_to_token_ids(
+ &sampling_output_buffer,
+ &token_ids_buffer,
+ encoder,
+ );
+ self.context.token_copy.encode_to_results(
+ &sampling_output_buffer,
+ &results_buffer,
+ slot,
+ encoder,
+ );
+ encoder.end_encoding();
+
+ // Signal event for next pass
+ let next_counter = current_counter + 1;
+ root_command_buffer.encode_signal_event(&async_event, next_counter);
+ self.context.async_buffers.counter.set(next_counter);
+
+ // Add completion handler
+ let results_buffer_clone = results_buffer.clone();
+ let callback = Arc::new(std::sync::Mutex::new(Some(on_complete)));
+
+ let block = ConcreteBlock::new(move |_: &metal::CommandBufferRef| {
+ let token = {
+ let ptr = results_buffer_clone.contents() as *const u32;
+ unsafe { *ptr.add(slot) as u64 }
+ };
+ if let Some(cb) = callback.lock().unwrap().take() {
+ cb(token);
+ }
+ });
+ let block = block.copy();
+
+ root_command_buffer.add_completed_handler(&block);
+ self.context.command_buffer.commit_and_continue();
+
+ Ok(())
+ }
+
+ pub fn has_attention_layers(&self) -> bool {
+ self.context.decoder_config.has_attention_layers()
+ }
+
pub fn clear_cache(&mut self) {
objc2::rc::autoreleasepool(|_pool| {
self.encoded_tasks.clear();
@@ -406,11 +562,15 @@ impl Generator {
&mut self,
suffix_length: usize,
) {
- let task = GeneratorRunTask {
- token_ids: &repeat_n(0, suffix_length).collect::<Box<[u64]>>(),
- token_positions: &(0..suffix_length).collect::<Box<[usize]>>(),
+ let token_ids: Vec<u64> = vec![0; suffix_length];
+ let token_positions: Vec<usize> = (0..suffix_length).collect();
+ let token_seeds: Vec<u64> = vec![0; suffix_length];
+
+ let task = LanguageModelGeneratorRunTask {
+ token_ids: &token_ids,
+ token_positions: &token_positions,
token_bitmask: None,
- token_seeds: &repeat_n(0, suffix_length).collect::<Box<[u64]>>(),
+ token_seeds: &token_seeds,
expected_number_of_new_tokens: suffix_length,
active_suffix_length: suffix_length,
is_prefilling: false,
@@ -422,7 +582,7 @@ impl Generator {
fn run_model(
&mut self,
- task: GeneratorRunTask,
+ task: LanguageModelGeneratorRunTask,
warmup: bool,
allow_pre_encode: bool,
sampling_method: SamplingMethod,
@@ -431,7 +591,9 @@ impl Generator {
let run_start = Instant::now();
let mut state = task.create_state(&mut self.context, None);
- state.sampling_method = Some(sampling_method);
+ if let Some(method) = state.sampling_method_mut() {
+ *method = Some(sampling_method);
+ }
let is_first_decode = !warmup && task.token_ids.len() == 1;
let should_capture =
@@ -508,7 +670,7 @@ impl Generator {
&mut self,
state: &mut ForwardPassState,
) -> Result<Vec<u64>, Error> {
- let sampling_output = state.sampling_output.as_ref()
+ let sampling_output = state.sampling_output()
.expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");
let output_buffer = sampling_output.borrow();
@@ -581,166 +743,4 @@ impl Generator {
self.registered_prefix_len = desired_prefix_len;
}
}
-
- /// Prepares async buffers for generation.
- /// Must be called after prefill, before async_generate loop.
- pub fn prepare_async(
- &mut self,
- tokens_to_generate: usize,
- ) {
- let prefill_count = self.tokens.len();
- self.context
- .async_buffers
- .prepare_positions(prefill_count, tokens_to_generate);
- self.context
- .async_buffers
- .prepare_seeds(&mut self.context.next_seed, tokens_to_generate);
- self.context.async_buffers.reset_counter();
- }
-
- /// Submits a single async forward pass.
- /// Does NOT block. Callback fires when GPU completes.
- ///
- /// - `pass_idx`: Index of this pass (0, 1, 2, ...)
- /// - `sampling_method`: Sampling configuration
- /// - `on_complete`: Callback receiving sampled token as u64
- pub fn async_generate<F>(
- &mut self,
- pass_idx: usize,
- sampling_method: SamplingMethod,
- on_complete: F,
- ) -> Result<(), Error>
- where
- F: FnOnce(u64) + Send + 'static,
- {
- assert_eq!(
- self.decoding_config.generate_suffix_length(),
- 1,
- "async_generate only supports suffix_length=1"
- );
-
- // Extract values from async_buffers before mutable borrow
- let current_counter = self.context.async_buffers.counter.get();
- let is_continuation = current_counter > 0;
- let batch_size = self.context.async_buffers.batch_size;
- let slot = pass_idx % batch_size;
- let async_event = self.context.async_buffers.event.clone();
- let results_buffer = self.context.async_buffers.results.clone();
- let async_positions_buffer =
- self.context.async_buffers.positions.clone();
- let async_seeds_buffer = self.context.async_buffers.seeds.clone();
-
- let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
-
- let task = GeneratorRunTask {
- token_ids: &[last_token],
- token_positions: &[0], // Ignored, using async buffer
- token_bitmask: None, // Ignored, using async buffer
- token_seeds: &[0], // Ignored, using async buffer
- expected_number_of_new_tokens: 1,
- active_suffix_length: 1,
- is_prefilling: false,
- };
-
- let async_positions = Some((&async_positions_buffer, pass_idx));
- let async_seeds = Some((&async_seeds_buffer, pass_idx));
-
- let mut state = ForwardPassState::new(
- self.context.mtl_context.clone(),
- &self.context.model_config.decoder_config,
- &self.context.model_shape,
- &self.context.scratch_buffers,
- self.context.cache_layers.clone(),
- self.context.shared_buffers.clone(),
- &task.token_ids,
- &task.token_positions,
- task.token_bitmask,
- &task.token_seeds,
- task.active_suffix_length,
- task.is_prefilling,
- false,
- None,
- is_continuation,
- async_positions,
- async_seeds,
- );
- state.sampling_method = Some(sampling_method);
-
- self.context.reset_command_buffer();
- let root_command_buffer =
- self.context.command_buffer.root_command_buffer();
-
- // Wait on previous pass if this is a continuation
- if is_continuation {
- root_command_buffer
- .encode_wait_for_event(&async_event, current_counter);
- }
-
- // Encode forward pass
- self.context.executables.encode(
- &mut state,
- &self.context.command_buffer,
- &EncodingParameters::new(false, false, false),
- );
-
- // Encode sampling
- self.context.gpu_sampler.encode(
- &mut state,
- &self.context.command_buffer,
- &EncodingParameters::new(false, false, false),
- );
-
- // Copy sampled token: sampling_output → token_ids (for next pass)
- // and sampling_output → results[slot] (for callback)
- let sampling_output = state
- .sampling_output
- .as_ref()
- .expect("sampling_output must exist after sampling encode");
- let sampling_output_buffer =
- unsafe { sampling_output.borrow_mut().mtl_buffer().clone() };
- let token_ids_buffer = self.context.scratch_buffers.token_ids.clone();
-
- let encoder = root_command_buffer.new_compute_command_encoder();
- self.context.token_copy.encode_to_token_ids(
- &sampling_output_buffer,
- &token_ids_buffer,
- encoder,
- );
- self.context.token_copy.encode_to_results(
- &sampling_output_buffer,
- &results_buffer,
- slot,
- encoder,
- );
- encoder.end_encoding();
-
- // Signal event for next pass
- let next_counter = current_counter + 1;
- root_command_buffer.encode_signal_event(&async_event, next_counter);
- self.context.async_buffers.counter.set(next_counter);
-
- // Add completion handler
- let results_buffer_clone = results_buffer.clone();
- let callback = Arc::new(std::sync::Mutex::new(Some(on_complete)));
-
- let block = ConcreteBlock::new(move |_: &metal::CommandBufferRef| {
- let token = {
- let ptr = results_buffer_clone.contents() as *const u32;
- unsafe { *ptr.add(slot) as u64 }
- };
- if let Some(cb) = callback.lock().unwrap().take() {
- cb(token);
- }
- });
- let block = block.copy();
-
- root_command_buffer.add_completed_handler(&block);
- self.context.command_buffer.commit_and_continue();
-
- Ok(())
- }
-
- pub fn has_attention_layers(&self) -> bool {
- self.context.model_config.decoder_config.has_attention_layers()
- }
}
diff --git a/crates/uzu/src/language_model/mod.rs b/crates/uzu/src/language_model/mod.rs
new file mode 100644
index 000000000..9a1fcc735
--- /dev/null
+++ b/crates/uzu/src/language_model/mod.rs
@@ -0,0 +1,12 @@
+pub mod gpu_capture;
+pub mod grammar;
+pub mod gumbel;
+pub mod language_model_generator;
+pub mod result;
+pub mod rng;
+pub mod sampler;
+pub mod tasks;
+
+// Re-export main types
+pub use crate::backends::metal::LanguageModelGeneratorContext;
+pub use language_model_generator::LanguageModelGenerator;
diff --git a/crates/uzu/src/generator/result.rs b/crates/uzu/src/language_model/result.rs
similarity index 100%
rename from crates/uzu/src/generator/result.rs
rename to crates/uzu/src/language_model/result.rs
diff --git a/crates/uzu/src/generator/rng.rs b/crates/uzu/src/language_model/rng.rs
similarity index 100%
rename from crates/uzu/src/generator/rng.rs
rename to crates/uzu/src/language_model/rng.rs
diff --git a/crates/uzu/src/generator/sampler.rs b/crates/uzu/src/language_model/sampler.rs
similarity index 100%
rename from crates/uzu/src/generator/sampler.rs
rename to crates/uzu/src/language_model/sampler.rs
diff --git a/crates/uzu/src/generator/tasks.rs b/crates/uzu/src/language_model/tasks.rs
similarity index 78%
rename from crates/uzu/src/generator/tasks.rs
rename to crates/uzu/src/language_model/tasks.rs
index af6cb4bdb..0be74166e 100644
--- a/crates/uzu/src/generator/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -2,20 +2,17 @@ use std::mem::size_of;
use metal::{Buffer, MTLResourceOptions};
-use super::context::GeneratorContext;
-use crate::backends::metal::{
- ForwardPassState,
- forward_pass::encodable_with_state::{
- EncodableWithState, EncodingParameters,
- },
+use super::LanguageModelGeneratorContext;
+use crate::backends::metal::forward_pass::{
+ EncodableBlock, EncodingParameters, ForwardPassState,
};
-pub struct GeneratorEncodedTask {
+pub struct LanguageModelGeneratorEncodedTask {
pub key: String,
predicate_buffer: Buffer,
}
-impl GeneratorEncodedTask {
+impl LanguageModelGeneratorEncodedTask {
pub fn predicate_buffer(&self) -> &Buffer {
&self.predicate_buffer
}
@@ -33,7 +30,7 @@ impl GeneratorEncodedTask {
}
#[derive(Debug, Clone)]
-pub struct GeneratorRunTask<'a> {
+pub struct LanguageModelGeneratorRunTask<'a> {
pub token_ids: &'a [u64],
pub token_positions: &'a [usize],
pub token_bitmask: Option<&'a [u32]>,
@@ -43,9 +40,9 @@ pub struct GeneratorRunTask<'a> {
pub is_prefilling: bool,
}
-impl<'a> GeneratorRunTask<'a> {
+impl<'a> LanguageModelGeneratorRunTask<'a> {
pub fn speculate_next_task(&self) -> Self {
- GeneratorRunTask {
+ LanguageModelGeneratorRunTask {
token_ids: self.token_ids,
token_positions: self.token_positions,
token_bitmask: self.token_bitmask,
@@ -68,12 +65,12 @@ impl<'a> GeneratorRunTask<'a> {
pub fn create_state(
&self,
- context: &mut GeneratorContext,
+ context: &mut LanguageModelGeneratorContext,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
) -> ForwardPassState {
- let state = ForwardPassState::new(
+ ForwardPassState::new_llm(
context.mtl_context.clone(),
- &context.model_config.decoder_config,
+ &context.decoder_config,
&context.model_shape,
&context.scratch_buffers,
context.cache_layers.clone(),
@@ -84,33 +81,28 @@ impl<'a> GeneratorRunTask<'a> {
self.token_seeds,
self.active_suffix_length,
self.is_prefilling,
- false,
external_bias_fn,
false,
None,
None,
- );
-
- return state;
+ )
}
pub fn build_encoded_task(
&self,
- context: &GeneratorContext,
+ context: &LanguageModelGeneratorContext,
state: &mut ForwardPassState,
parameters: &EncodingParameters,
key: String,
- ) -> GeneratorEncodedTask {
+ ) -> LanguageModelGeneratorEncodedTask {
context.executables.encode(state, &context.command_buffer, parameters);
- let encoded_task = GeneratorEncodedTask {
+ LanguageModelGeneratorEncodedTask {
key,
predicate_buffer: context.mtl_context.device.new_buffer(
size_of::<u32>() as u64,
MTLResourceOptions::StorageModeShared,
),
- };
-
- encoded_task
+ }
}
}
diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rs
index e28e2d944..645f4f965 100644
--- a/crates/uzu/src/lib.rs
+++ b/crates/uzu/src/lib.rs
@@ -1,14 +1,17 @@
pub mod backends;
+pub mod classifier;
pub mod config;
-pub mod generator;
+pub mod language_model;
+pub mod linearizer;
pub mod parameters;
+pub mod prelude;
pub mod session;
pub mod speculators;
+#[cfg(feature = "tracing")]
pub mod tracer;
pub mod trie;
pub mod utils;
pub use utils::*;
pub mod device;
+pub use config::*;
pub use device::*;
-pub mod runners;
-pub use runners::*;
diff --git a/crates/uzu/src/linearizer/mod.rs b/crates/uzu/src/linearizer/mod.rs
new file mode 100644
index 000000000..95655d93b
--- /dev/null
+++ b/crates/uzu/src/linearizer/mod.rs
@@ -0,0 +1,8 @@
+pub mod node;
+pub mod speculated_suffix;
+pub mod transition;
+pub mod trie;
+
+#[cfg(test)]
+mod tests_node;
+mod tests_trie;
diff --git a/crates/uzu/src/linearizer/node.rs b/crates/uzu/src/linearizer/node.rs
new file mode 100644
index 000000000..62df6cb1d
--- /dev/null
+++ b/crates/uzu/src/linearizer/node.rs
@@ -0,0 +1,80 @@
+use super::transition::Transition;
+
+#[derive(Debug)]
+pub struct Node {
+ transitions: Vec<Transition>,
+}
+
+impl Default for Node {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl Node {
+ pub fn new() -> Self {
+ Self {
+ transitions: Vec::new(),
+ }
+ }
+
+ pub fn has_transition(
+ &self,
+ token: u64,
+ ) -> bool {
+ self.transitions.iter().any(|t| t.token() == token)
+ }
+
+ pub fn get_next(
+ &self,
+ token: u64,
+ ) -> Option<&Node> {
+ self.transitions.iter().find(|t| t.token() == token).map(|t| &t.node)
+ }
+
+ pub fn get_or_insert_next(
+ &mut self,
+ token: u64,
+ seed: u64,
+ ) -> &mut Node {
+ if let Some(pos) =
+ self.transitions.iter().position(|t| t.token() == token)
+ {
+ return &mut self.transitions[pos].node;
+ }
+
+ self.transitions.push(Transition::new(token, seed));
+ &mut self.transitions.last_mut().unwrap().node
+ }
+
+ pub fn dfs_with_path<F>(
+ &self,
+ path: &mut Vec<(isize, u64, u64)>,
+ callback: &mut F,
+ ) -> isize
+ where
+ F: FnMut(&[(isize, u64, u64)]),
+ {
+ let mut node_index = path.last().map(|(idx, _, _)| *idx).unwrap_or(-1);
+
+ for transition in self.transitions.iter() {
+ node_index += 1;
+ path.push((node_index, transition.token(), transition.seed()));
+ callback(path);
+ node_index = transition.node.dfs_with_path(path, callback);
+ path.pop();
+ }
+
+ node_index
+ }
+
+ pub fn dfs<F>(
+ &self,
+ mut callback: F,
+ ) where
+ F: FnMut(&[(isize, u64, u64)]),
+ {
+ let mut path: Vec<(isize, u64, u64)> = Vec::new();
+ self.dfs_with_path(&mut path, &mut callback);
+ }
+}
diff --git a/crates/uzu/src/linearizer/speculated_suffix.rs b/crates/uzu/src/linearizer/speculated_suffix.rs
new file mode 100644
index 000000000..2f3874036
--- /dev/null
+++ b/crates/uzu/src/linearizer/speculated_suffix.rs
@@ -0,0 +1,9 @@
+use std::collections::HashMap;
+
+#[derive(Debug)]
+pub struct SpeculatedSuffix {
+ pub tokens: Vec<u64>,
+ pub seeds: Vec<u64>,
+ pub indices: Vec<usize>,
+ pub transition_map: HashMap<isize, HashMap<u64, isize>>,
+}
diff --git a/crates/uzu/src/linearizer/tests_node.rs b/crates/uzu/src/linearizer/tests_node.rs
new file mode 100644
index 000000000..9f4680d76
--- /dev/null
+++ b/crates/uzu/src/linearizer/tests_node.rs
@@ -0,0 +1,267 @@
+#[cfg(test)]
+mod tests {
+ use std::collections::HashSet;
+
+ use super::super::node::Node;
+
+ #[test]
+ fn test_new_node_has_no_transitions() {
+ let node = Node::new();
+ for i in 0..10 {
+ assert!(!node.has_transition(i));
+ }
+ }
+
+ #[test]
+ fn test_has_transition() {
+ let mut node = Node::new();
+ node.get_or_insert_next(42, 0);
+
+ assert!(node.has_transition(42));
+ assert!(!node.has_transition(43));
+ }
+
+ #[test]
+ fn test_get_next() {
+ let mut node = Node::new();
+ node.get_or_insert_next(1, 0);
+
+ assert!(node.get_next(1).is_some());
+ assert!(node.get_next(2).is_none());
+ }
+
+ #[test]
+ fn test_get_or_insert_next() {
+ let mut node = Node::new();
+
+ // First insertion should create a new node
+ let node1 = node.get_or_insert_next(1, 0);
+ assert!(!node1.has_transition(99));
+
+ // Add a transition to the child node
+ node1.get_or_insert_next(99, 0);
+
+ // Get the same node again
+ let node1_again = node.get_or_insert_next(1, 0);
+
+ // Verify it's the same node by checking it has the transition we added
+ assert!(node1_again.has_transition(99));
+
+ // Different token should give a different node
+ let node2 = node.get_or_insert_next(2, 0);
+ assert!(!node2.has_transition(99));
+ }
+
+ #[test]
+ fn test_multiple_transitions() {
+ let mut node = Node::new();
+
+ // Insert several transitions
+ for i in 0..5 {
+ node.get_or_insert_next(i, 0);
+ }
+
+ // Verify all transitions exist
+ for i in 0..5 {
+ assert!(node.has_transition(i));
+ }
+
+ // Verify non-existent transitions
+ for i in 5..10 {
+ assert!(!node.has_transition(i));
+ }
+ }
+
+ #[test]
+ fn test_dfs_with_path_empty_node() {
+ let node = Node::new();
+ let mut path: Vec<(isize, u64, u64)> = Vec::new();
+ let mut visited = Vec::new();
+
+ node.dfs_with_path(&mut path, &mut |current_path: &[(
+ isize,
+ u64,
+ u64,
+ )]| {
+ let tokens: Vec<u64> =
+ current_path.iter().map(|(_, token, _)| *token).collect();
+ visited.push(tokens);
+ });
+
+ assert!(visited.is_empty());
+ }
+
+ #[test]
+ fn test_dfs_with_path_single_node() {
+ let mut node = Node::new();
+ node.get_or_insert_next(42, 0);
+
+ let mut path: Vec<(isize, u64, u64)> = Vec::new();
+ let mut visited = Vec::new();
+
+ node.dfs_with_path(&mut path, &mut |current_path: &[(
+ isize,
+ u64,
+ u64,
+ )]| {
+ let tokens: Vec<u64> =
+ current_path.iter().map(|(_, token, _)| *token).collect();
+ visited.push(tokens);
+ });
+
+ assert_eq!(visited.len(), 1);
+ assert_eq!(visited[0], vec![42]);
+ }
+
+ #[test]
+ fn test_dfs_with_path_complex_tree() {
+ let mut root = Node::new();
+
+ // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]
+ let node1 = root.get_or_insert_next(1, 0);
+ let node2 = node1.get_or_insert_next(2, 0);
+ node2.get_or_insert_next(3, 0);
+ node2.get_or_insert_next(4, 0);
+ node1.get_or_insert_next(5, 0);
+ root.get_or_insert_next(6, 0);
+
+ let mut path: Vec<(isize, u64, u64)> = Vec::new();
+ let mut visited = Vec::new();
+
+ root.dfs_with_path(&mut path, &mut |current_path: &[(
+ isize,
+ u64,
+ u64,
+ )]| {
+ let tokens: Vec<u64> =
+ current_path.iter().map(|(_, token, _)| *token).collect();
+ visited.push(tokens);
+ });
+
+ // Check all expected paths were visited
+ assert_eq!(visited.len(), 6);
+ assert!(visited.contains(&vec![1]));
+ assert!(visited.contains(&vec![1, 2]));
+ assert!(visited.contains(&vec![1, 2, 3]));
+ assert!(visited.contains(&vec![1, 2, 4]));
+ assert!(visited.contains(&vec![1, 5]));
+ assert!(visited.contains(&vec![6]));
+ }
+
+ #[test]
+ fn test_dfs_path_indices() {
+ let mut root = Node::new();
+
+ // Build a simple trie: [1], [2]
+ root.get_or_insert_next(1, 0);
+ root.get_or_insert_next(2, 0);
+
+ let mut path: Vec<(isize, u64, u64)> = Vec::new();
+ let mut indices = Vec::new();
+
+ root.dfs_with_path(&mut path, &mut |current_path: &[(
+ isize,
+ u64,
+ u64,
+ )]| {
+ if !current_path.is_empty() {
+ indices.push(current_path.last().unwrap().0);
+ }
+ });
+
+ // Check indices are unique and as expected
+ assert_eq!(indices.len(), 2);
+ assert_eq!(indices[0], 0);
+ assert_eq!(indices[1], 1);
+ }
+
+ #[test]
+ fn test_simplified_dfs_empty_node() {
+ let node = Node::new();
+ let mut visited = Vec::new();
+
+ node.dfs(|current_path: &[(isize, u64, u64)]| {
+ let tokens: Vec<u64> =
+ current_path.iter().map(|(_, token, _)| *token).collect();
+ visited.push(tokens);
+ });
+
+ assert!(visited.is_empty());
+ }
+
+ #[test]
+ fn test_simplified_dfs_complex_tree() {
+ let mut root = Node::new();
+
+ // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]
+ let node1 = root.get_or_insert_next(1, 0);
+ let node2 = node1.get_or_insert_next(2, 0);
+ node2.get_or_insert_next(3, 0);
+ node2.get_or_insert_next(4, 0);
+ node1.get_or_insert_next(5, 0);
+ root.get_or_insert_next(6, 0);
+
+ let mut visited = Vec::new();
+
+ root.dfs(|current_path: &[(isize, u64, u64)]| {
+ let tokens: Vec<u64> =
+ current_path.iter().map(|(_, token, _)| *token).collect();
+ visited.push(tokens);
+ });
+
+ // Check all expected paths were visited
+ assert_eq!(visited.len(), 6);
+ assert!(visited.contains(&vec![1]));
+ assert!(visited.contains(&vec![1, 2]));
+ assert!(visited.contains(&vec![1, 2, 3]));
+ assert!(visited.contains(&vec![1, 2, 4]));
+ assert!(visited.contains(&vec![1, 5]));
+ assert!(visited.contains(&vec![6]));
+ }
+
+ #[test]
+ fn test_both_dfs_methods_equivalent() {
+ let mut root = Node::new();
+
+ // Build a more complex trie
+ let node1 = root.get_or_insert_next(10, 0);
+ let node2 = node1.get_or_insert_next(20, 0);
+ node2.get_or_insert_next(30, 0);
+ node2.get_or_insert_next(40, 0);
+ node1.get_or_insert_next(50, 0);
+ let node3 = root.get_or_insert_next(60, 0);
+ node3.get_or_insert_next(70, 0);
+
+ // Get results from dfs_with_path
+ let mut path: Vec<(isize, u64, u64)> = Vec::new();
+ let mut visited1 = Vec::new();
+
+ root.dfs_with_path(&mut path, &mut |current_path: &[(
+ isize,
+ u64,
+ u64,
+ )]| {
+ let tokens: Vec<u64> =
+ current_path.iter().map(|(_, token, _)| *token).collect();
+ visited1.push(tokens);
+ });
+
+ // Get results from simplified dfs
+ let mut visited2 = Vec::new();
+
+ root.dfs(|current_path: &[(isize, u64, u64)]| {
+ let tokens: Vec<u64> =
+ current_path.iter().map(|(_, token, _)| *token).collect();
+ visited2.push(tokens);
+ });
+
+ // Check both methods give the same results
+ assert_eq!(visited1.len(), visited2.len());
+
+ // Convert to sets to ignore order differences
+ let set1: HashSet<Vec<u64>> = visited1.into_iter().collect();
+ let set2: HashSet<Vec<u64>> = visited2.into_iter().collect();
+
+ assert_eq!(set1, set2);
+ }
+}
diff --git a/crates/uzu/src/linearizer/tests_trie.rs b/crates/uzu/src/linearizer/tests_trie.rs
new file mode 100644
index 000000000..9896fb206
--- /dev/null
+++ b/crates/uzu/src/linearizer/tests_trie.rs
@@ -0,0 +1,297 @@
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+
+ use crate::{
+ language_model::rng::DerivableSeed,
+ linearizer::trie::{TokenTrie, TrieCreationConfig},
+ speculators::speculator::Speculator,
+ };
+
+ struct StaticSpeculator {
+ responses: HashMap<Vec<u64>, HashMap<u64, f32>>,
+ }
+
+ impl StaticSpeculator {
+ fn new(responses: HashMap<Vec<u64>, HashMap<u64, f32>>) -> Self {
+ Self {
+ responses,
+ }
+ }
+ }
+
+ impl Speculator for StaticSpeculator {
+ fn speculate(
+ &self,
+ prefix: &[u64],
+ ) -> HashMap<u64, f32> {
+ self.responses.get(prefix).cloned().unwrap_or_default()
+ }
+ }
+
+ #[test]
+ fn test_from_speculator_applies_prob_cutoff() {
+ let mut responses = HashMap::new();
+ responses.insert(
+ vec![42],
+ HashMap::from([(1, 0.9_f32), (2, 0.05_f32), (3, 0.2_f32)]),
+ );
+ responses.insert(vec![42, 1], HashMap::new());
+
+ let speculator = StaticSpeculator::new(responses);
+ let mut seed = DerivableSeed::new(1);
+
+ let trie = TokenTrie::from_speculator(
+ &[42],
+ &mut seed,
+ false,
+ &speculator,
+ &TrieCreationConfig::default(),
+ 5,
+ );
+ let suffix = trie.linearize(0, 5);
+
+ assert_eq!(suffix.tokens, vec![1, 3]);
+ assert!(!suffix.tokens.contains(&2));
+ assert_eq!(suffix.transition_map.len(), 3);
+ }
+
+ #[test]
+ fn test_from_speculator_respects_top_k_limit() {
+ let mut responses = HashMap::new();
+ responses.insert(vec![5], HashMap::from([(9, 0.8_f32), (10, 0.7_f32)]));
+ responses.insert(vec![5, 9], HashMap::from([(11, 0.6_f32)]));
+ responses.insert(vec![5, 9, 11], HashMap::new());
+
+ let speculator = StaticSpeculator::new(responses);
+ let mut seed = DerivableSeed::new(7);
+
+ let trie = TokenTrie::from_speculator(
+ &[5],
+ &mut seed,
+ false,
+ &speculator,
+ &TrieCreationConfig {
+ top_k: 1,
+ prob_cutoff: 0.0,
+ },
+ 3,
+ );
+
+ let suffix = trie.linearize(0, 4);
+
+ assert_eq!(suffix.tokens, vec![9, 11]);
+ assert!(!suffix.tokens.contains(&10));
+ }
+
+ #[test]
+ fn test_from_speculator_includes_last_prefix_token_and_seeds() {
+ let mut responses = HashMap::new();
+ responses.insert(vec![2, 4], HashMap::from([(8, 0.9_f32)]));
+ responses.insert(vec![2, 4, 8], HashMap::new());
+
+ let speculator = StaticSpeculator::new(responses);
+
+ let mut seed = DerivableSeed::new(99);
+ let mut expected_seed_stream = DerivableSeed::new(99);
+ let expected_seeds =
+ vec![expected_seed_stream.next(), expected_seed_stream.next()];
+
+ let trie = TokenTrie::from_speculator(
+ &[2, 4],
+ &mut seed,
+ true,
+ &speculator,
+ &TrieCreationConfig::default(),
+ 2,
+ );
+
+ let suffix = trie.linearize(0, 3);
+
+ assert_eq!(suffix.tokens, vec![4, 8]);
+ assert_eq!(suffix.seeds, expected_seeds);
+ assert_eq!(suffix.indices, vec![0, 1]);
+ }
+
+ #[test]
+ fn test_from_speculator_stops_at_max_length() {
+ let mut responses = HashMap::new();
+ responses.insert(
+ vec![1],
+ HashMap::from([(5, 0.9_f32), (6, 0.8_f32), (7, 0.7_f32)]),
+ );
+ responses
+ .insert(vec![1, 5], HashMap::from([(8, 0.6_f32), (9, 0.5_f32)]));
+ responses.insert(vec![1, 5, 8], HashMap::from([(10, 0.4_f32)]));
+
+ let speculator = StaticSpeculator::new(responses);
+ let mut seed = DerivableSeed::new(1234);
+
+ let trie = TokenTrie::from_speculator(
+ &[1],
+ &mut seed,
+ false,
+ &speculator,
+ &TrieCreationConfig::default(),
+ 3,
+ );
+
+ let suffix = trie.linearize(0, 5);
+
+ assert_eq!(suffix.tokens.len(), 3);
+ assert!(suffix.tokens.contains(&5));
+ assert!(suffix.tokens.contains(&6));
+ assert!(suffix.tokens.contains(&8));
+ assert!(!suffix.tokens.contains(&9));
+ }
+
+ #[test]
+ fn test_from_speculator_with_empty_responses() {
+ let speculator = StaticSpeculator::new(HashMap::new());
+
+ let mut seed = DerivableSeed::new(555);
+ let mut expected_seed_stream = DerivableSeed::new(555);
+ let expected_first_seed = expected_seed_stream.next();
+
+ let trie = TokenTrie::from_speculator(
+ &[12, 13],
+ &mut seed,
+ true,
+ &speculator,
+ &TrieCreationConfig::default(),
+ 4,
+ );
+
+ let suffix = trie.linearize(0, 5);
+
+ // Only the last prefix token should be included; no speculated tokens.
+ assert_eq!(suffix.tokens, vec![13]);
+ assert_eq!(suffix.indices, vec![0]);
+ assert_eq!(suffix.seeds, vec![expected_first_seed]);
+
+ assert_eq!(suffix.transition_map.len(), 2);
+ let root_transitions = &suffix.transition_map[&-1];
+ assert_eq!(root_transitions.len(), 1);
+ assert_eq!(root_transitions.get(&13), Some(&0));
+ }
+
+ #[test]
+ fn test_linearize_empty_trie() {
+ let trie = TokenTrie::new();
+ let suffix = trie.linearize(0, 10);
+
+ assert!(suffix.tokens.is_empty());
+ assert!(suffix.indices.is_empty());
+ assert!(suffix.transition_map.len() == 1);
+ }
+
+ #[test]
+ fn test_linearize_single_sequence() {
+ let mut trie = TokenTrie::new();
+ trie.insert(&[1, 2, 3]);
+
+ let suffix = trie.linearize(0, 5);
+
+ assert_eq!(suffix.tokens.len(), 3);
+ assert!(suffix.tokens.contains(&1));
+ assert!(suffix.tokens.contains(&2));
+ assert!(suffix.tokens.contains(&3));
+
+ assert_eq!(suffix.transition_map.len(), 4);
+
+ let root_transitions = &suffix.transition_map[&-1];
+ assert!(root_transitions.contains_key(&1));
+
+ let token1_idx = *root_transitions.get(&1).unwrap();
+ let token1_transitions = &suffix.transition_map[&token1_idx];
+ assert!(token1_transitions.contains_key(&2));
+
+ let token2_idx = *token1_transitions.get(&2).unwrap();
+ let token2_transitions = &suffix.transition_map[&token2_idx];
+ assert!(token2_transitions.contains_key(&3));
+ }
+
+ #[test]
+ fn test_linearize_branching_trie() {
+ let mut trie = TokenTrie::new();
+ trie.insert(&[1, 2, 3]);
+ trie.insert(&[1, 2, 4]);
+ trie.insert(&[5, 6]);
+
+ let suffix = trie.linearize(10, 10);
+
+ assert!(suffix.tokens.contains(&1));
+ assert!(suffix.tokens.contains(&2));
+ assert!(suffix.tokens.contains(&3));
+ assert!(suffix.tokens.contains(&4));
+ assert!(suffix.tokens.contains(&5));
+ assert!(suffix.tokens.contains(&6));
+
+ assert_eq!(suffix.tokens.len(), 6);
+
+ assert_eq!(suffix.transition_map.len(), 7);
+
+ let root_transitions = &suffix.transition_map[&-1];
+ assert!(root_transitions.contains_key(&1));
+ assert!(root_transitions.contains_key(&5));
+
+ let token1_idx = *root_transitions.get(&1).unwrap();
+ let token1_transitions = &suffix.transition_map[&token1_idx];
+ assert!(token1_transitions.contains_key(&2));
+
+ let token2_idx = *token1_transitions.get(&2).unwrap();
+ let token2_transitions = &suffix.transition_map[&token2_idx];
+ assert!(token2_transitions.contains_key(&3));
+ assert!(token2_transitions.contains_key(&4));
+
+ let token5_idx = *root_transitions.get(&5).unwrap();
+ let token5_transitions = &suffix.transition_map[&token5_idx];
+ assert!(token5_transitions.contains_key(&6));
+ }
+
+ #[test]
+ fn test_linearize_max_length() {
+ let mut trie = TokenTrie::new();
+ trie.insert(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+
+ let suffix = trie.linearize(0, 5);
+
+ assert_eq!(suffix.tokens.len(), 5);
+ assert_eq!(suffix.indices.len(), 5);
+ assert_eq!(suffix.transition_map.len(), 6);
+
+ for i in 1..=5 {
+ assert!(suffix.tokens.contains(&i));
+ }
+
+ let root_transitions = &suffix.transition_map[&-1];
+ assert!(root_transitions.contains_key(&1));
+
+ let mut current_idx = *root_transitions.get(&1).unwrap();
+ for i in 2..=4 {
+ let transitions = &suffix.transition_map[&current_idx];
+ assert!(transitions.contains_key(&i));
+ current_idx = *transitions.get(&i).unwrap();
+ }
+ }
+
+ #[test]
+ fn test_linearize_start_index() {
+ let mut trie = TokenTrie::new();
+ trie.insert(&[1, 2, 3]);
+
+ let suffix = trie.linearize(100, 10);
+
+ assert_eq!(suffix.tokens.len(), 3);
+ assert!(suffix.tokens.contains(&1));
+ assert!(suffix.tokens.contains(&2));
+ assert!(suffix.tokens.contains(&3));
+
+ for idx in suffix.indices.iter() {
+ assert!(*idx >= 100 && *idx <= 102);
+ }
+
+ let root_transitions = &suffix.transition_map[&-1];
+ assert!(root_transitions.contains_key(&1));
+ }
+}
diff --git a/crates/uzu/src/linearizer/transition.rs b/crates/uzu/src/linearizer/transition.rs
new file mode 100644
index 000000000..76426fe6c
--- /dev/null
+++ b/crates/uzu/src/linearizer/transition.rs
@@ -0,0 +1,29 @@
+use super::node::Node;
+
+#[derive(Debug)]
+pub struct Transition {
+ token: u64,
+ seed: u64,
+ pub node: Node,
+}
+
+impl Transition {
+ pub fn new(
+ token: u64,
+ seed: u64,
+ ) -> Self {
+ Self {
+ token,
+ seed,
+ node: Node::new(),
+ }
+ }
+
+ pub fn token(&self) -> u64 {
+ self.token
+ }
+
+ pub fn seed(&self) -> u64 {
+ self.seed
+ }
+}
diff --git a/crates/uzu/src/linearizer/trie.rs b/crates/uzu/src/linearizer/trie.rs
new file mode 100644
index 000000000..8aa2f86f0
--- /dev/null
+++ b/crates/uzu/src/linearizer/trie.rs
@@ -0,0 +1,168 @@
+use std::collections::HashMap;
+
+use super::{node::Node, speculated_suffix::SpeculatedSuffix};
+use crate::{
+ language_model::rng::DerivableSeed, speculators::speculator::Speculator,
+};
+
+pub struct TrieCreationConfig {
+ pub top_k: usize,
+ pub prob_cutoff: f32,
+}
+
+impl Default for TrieCreationConfig {
+ fn default() -> Self {
+ TrieCreationConfig {
+ top_k: 2,
+ prob_cutoff: 0.1,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub struct TokenTrie {
+ root: Node,
+}
+
+impl Default for TokenTrie {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl TokenTrie {
+ pub fn new() -> Self {
+ Self {
+ root: Node::new(),
+ }
+ }
+
+ pub fn from_sequences(sequences: &[Vec<u64>]) -> Self {
+ let mut trie = Self::new();
+ for sequence in sequences {
+ trie.insert(sequence);
+ }
+ trie
+ }
+
+ pub fn from_speculator(
+ prefix: &[u64],
+ seed: &mut DerivableSeed,
+ include_last_prefix_token: bool,
+ speculator: &dyn Speculator,
+ creation_config: &TrieCreationConfig,
+ max_length: usize,
+ ) -> Self {
+ let mut trie = Self::new();
+ let mut current_node = &mut trie.root;
+ let mut trie_size = 0;
+
+ if include_last_prefix_token {
+ current_node = current_node
+ .get_or_insert_next(*prefix.last().unwrap(), seed.next());
+ trie_size += 1;
+ }
+
+ let mut speculation_prefix = prefix.to_vec();
+ while trie_size < max_length {
+ let speculation_result = speculator.speculate(&speculation_prefix);
+
+ let mut arr =
+ speculation_result.into_iter().collect::<Vec<(u64, f32)>>();
+ arr.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));
+
+ let arr = arr
+ .into_iter()
+ .enumerate()
+ .filter(|&(i, (_, p))| {
+ i == 0 || p > creation_config.prob_cutoff
+ })
+ .map(|(_, x)| x)
+ .take(usize::min(creation_config.top_k, max_length - trie_size))
+ .collect::<Vec<(u64, f32)>>();
+
+ let Some(&(top_token, _)) = arr.first() else {
+ break;
+ };
+
+ trie_size += arr.len();
+ for &(tok, _) in arr.iter() {
+ current_node.get_or_insert_next(tok, seed.next());
+ }
+
+ current_node = current_node.get_or_insert_next(top_token, 0);
+ speculation_prefix.push(top_token);
+ }
+
+ trie
+ }
+
+ pub fn insert<'a, I>(
+ &mut self,
+ sequence: I,
+ ) where
+ I: IntoIterator<Item = &'a u64>,
+ {
+ let mut current_node = &mut self.root;
+ for &token in sequence {
+ current_node = current_node.get_or_insert_next(token, 0);
+ }
+ }
+
+ pub fn linearize(
+ &self,
+ start_index: usize,
+ max_length: usize,
+ ) -> SpeculatedSuffix {
+ let mut tokens = Vec::new();
+ let mut seeds = Vec::new();
+ let mut indices = Vec::new();
+
+ self.root.dfs(|path| {
+ if tokens.len() >= max_length {
+ return;
+ }
+ if !path.is_empty() {
+ tokens.push(path.last().unwrap().1);
+ seeds.push(path.last().unwrap().2);
+ indices.push(start_index + path.len() - 1);
+ }
+ });
+
+ let tokens_len = tokens.len();
+
+ let mut transition_map: HashMap<
+ isize,
+ std::collections::HashMap<u64, isize>,
+ > = HashMap::new();
+ for index in -1..(tokens_len as isize) {
+ transition_map.insert(index, HashMap::new());
+ }
+
+ self.root.dfs(|path| {
+ if path.is_empty()
+ || path.last().unwrap().0 >= (max_length as isize)
+ {
+ return;
+ }
+
+ let (current_index, current_token, _) = *path.last().unwrap();
+ let parent_index = if path.len() > 1 {
+ path[path.len() - 2].0
+ } else {
+ -1
+ };
+
+ if let Some(transitions) = transition_map.get_mut(&parent_index) {
+ transitions.insert(current_token, current_index);
+ }
+ });
+
+ SpeculatedSuffix {
+ tokens,
+ indices,
+ seeds,
+ transition_map,
+ }
+ }
+}
diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rs
new file mode 100644
index 000000000..e177c2921
--- /dev/null
+++ b/crates/uzu/src/prelude.rs
@@ -0,0 +1,27 @@
+//! Prelude module containing commonly used types from the uzu crate.
+//!
+//! This module can be imported with `use uzu::prelude::*;` to bring the most
+//! frequently used types into scope.
+
+// Constants & Utilities
+// Linearizer
+// Session Core
+pub use crate::session::ChatSession;
+// Session Config
+pub use crate::session::config::{DecodingConfig, RunConfig, SpeculatorConfig};
+// Session Parameters
+pub use crate::session::parameter::{
+ ContextLength, ContextMode, PrefillStepSize, SamplingMethod,
+ SamplingPolicy, SamplingSeed,
+};
+// Session Types
+pub use crate::session::types::{
+ Error, FinishReason, Input, Message, Output, ParsedText, Role, RunStats,
+ Stats, StepStats, Text, TotalStats,
+};
+// Speculators
+pub use crate::speculators::speculator::Speculator;
+pub use crate::{
+ VERSION, linearizer::trie::TokenTrie, session::ClassificationSession,
+ speculators::empty_speculator::EmptySpeculator, storage_path,
+};
diff --git a/crates/uzu/src/runners/decoder_runner.rs b/crates/uzu/src/runners/decoder_runner.rs
deleted file mode 100644
index cca97cc23..000000000
--- a/crates/uzu/src/runners/decoder_runner.rs
+++ /dev/null
@@ -1,241 +0,0 @@
-#![allow(dead_code)]
-use std::{
- cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc, time::Instant,
-};
-
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use serde::{Deserialize, Serialize};
-
-use crate::{
- backends::metal::{
- CacheLayers, DecoderExecutables, ForwardPassState, MTLContext,
- ModelShape,
- compilation_parameters::CompilationConfig,
- forward_pass::{
- ForwardPassBuffers, SharedBuffers,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- },
- utils::{CaptureOptions, StdoutCapture},
- },
- config::{ModelMetadata, decoder::DecoderConfig},
- parameters::ParameterLoader,
-};
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct DecoderTestResult {
- pub placement_log: String,
- pub iterations: u32,
- pub time_per_token: f32,
- pub tokens_per_second: f32,
- pub success: bool,
- pub error: Option<String>,
-}
-
-struct DecoderTestContext {
- mtl_context: Rc<MTLContext>,
- mtl_command_queue: metal::CommandQueue,
- state: ForwardPassState,
- executables: DecoderExecutables,
- suffix_length: usize,
- _scratch: ForwardPassBuffers,
-}
-
-impl DecoderTestContext {
- fn new(model_dir: &str) -> Result<(Self, Rc<DecoderConfig>), String> {
- let model_path = Path::new(model_dir);
- let config_path = model_path.join("config.json");
- if !config_path.exists() {
- return Err(format!(
- "Decoder config file not found at {:?}",
- config_path
- ));
- }
- let config_file = File::open(&config_path)
- .map_err(|e| format!("Failed to open config: {}", e))?;
- let model_metadata: ModelMetadata =
- serde_json::from_reader(BufReader::new(config_file))
- .map_err(|e| format!("Failed to parse config: {}", e))?;
- let decoder_config =
- Rc::new(model_metadata.model_config.decoder_config.clone());
- let mtl_device = metal::Device::system_default()
- .ok_or("No Metal device available".to_string())?;
- let mtl_command_queue = mtl_device.new_command_queue();
-
- let compilation_config = Rc::new(CompilationConfig::default());
-
- // -----------------------------------------------------------------
- // Determine the heap size we need for the worst-case forward pass
- // *and* all parameters that will be loaded from the safetensors file.
- // -----------------------------------------------------------------
- let max_prefix_length = 8192; // Example value, can be passed dynamically
- let max_suffix_length = 1; // Example value, can be passed dynamically
-
- let model_shape = ModelShape::from_decoder_config(&decoder_config);
-
- // Read safetensors header to know the total size of all tensors
- // so we can make the heap large enough to store them.
- let weights_path = model_path.join("model.safetensors");
- if !weights_path.exists() {
- return Err(format!(
- "Decoder weights file not found at {:?}",
- weights_path
- ));
- }
-
- let mtl_context = Rc::new(
- MTLContext::new(mtl_device.clone(), mtl_command_queue.clone())
- .map_err(|e| format!("Failed to create MetalContext: {}", e))?,
- );
-
- let weights_file = File::open(&weights_path)
- .map_err(|e| format!("Failed to open weights: {}", e))?;
- let loader = ParameterLoader::new(&weights_file, &mtl_context)
- .map_err(|e| format!("Failed to create ParameterLoader: {}", e))?;
- let root_loader_view = loader.tree();
-
- let cache_layers = Rc::new(RefCell::new(CacheLayers::new(
- &mtl_context,
- &model_shape,
- max_prefix_length,
- max_suffix_length,
- )));
- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(
- &mtl_context,
- &decoder_config,
- &model_shape,
- )));
- shared_buffers.borrow_mut().update_data(&root_loader_view);
- let token_ids: Vec<u64> =
- (0..max_suffix_length).map(|i| (i % 1000) as u64).collect();
- let token_positions: Vec<usize> = (0..max_suffix_length).collect();
- let token_seeds: Vec<u64> = (0_u64..max_suffix_length as u64).collect();
-
- // Scratch buffers sized for the maximum prefix/suffix lengths in this test
- let scratch_buffers = ForwardPassBuffers::new(
- &mtl_context,
- &decoder_config,
- &model_shape,
- max_prefix_length,
- max_suffix_length,
- );
-
- let state = ForwardPassState::new(
- mtl_context.clone(),
- &decoder_config,
- &model_shape,
- &scratch_buffers,
- cache_layers.clone(),
- shared_buffers.clone(),
- &token_ids,
- &token_positions,
- None,
- &token_seeds,
- token_ids.len(),
- false,
- false,
- None,
- false,
- None,
- None,
- );
-
- let executables = DecoderExecutables::new(
- mtl_context.clone(),
- decoder_config.clone(),
- &root_loader_view,
- compilation_config.clone(),
- );
- Ok((
- Self {
- mtl_context,
- mtl_command_queue,
- state,
- executables,
- suffix_length: max_suffix_length,
- _scratch: scratch_buffers,
- },
- decoder_config,
- ))
- }
-
- fn run_warmup_with_placement_analysis(&mut self) -> String {
- let mut capture = StdoutCapture::new(CaptureOptions::STDERR);
- capture.start();
- let command_buffer =
- MPSCommandBuffer::from_command_queue(&self.mtl_command_queue);
- self.executables.encode(
- &mut self.state,
- &command_buffer,
- &EncodingParameters::new(true, true, false),
- );
- command_buffer.commit();
- command_buffer.command_buffer().wait_until_completed();
- capture.stop()
- }
-
- fn run_timing_loop(
- &mut self,
- iterations: u32,
- ) -> Result<std::time::Duration, String> {
- // Warmup run (not measured)
- let command_buffer =
- MPSCommandBuffer::from_command_queue(&self.mtl_command_queue);
- self.executables.encode(
- &mut self.state,
- &command_buffer,
- &EncodingParameters::new(true, true, false),
- );
- command_buffer.commit();
- command_buffer.command_buffer().wait_until_completed();
- // Measured runs
- let start = Instant::now();
- for _ in 0..iterations {
- let command_buffer =
- MPSCommandBuffer::from_command_queue(&self.mtl_command_queue);
- self.executables.encode(
- &mut self.state,
- &command_buffer,
- &EncodingParameters::new(false, true, false),
- );
- command_buffer.commit();
- command_buffer.command_buffer().wait_until_completed();
- }
- Ok(start.elapsed())
- }
-}
-
-pub fn run_decoder_with_results(model_dir: &str) -> DecoderTestResult {
- let mut result = DecoderTestResult {
- placement_log: String::new(),
- iterations: 0,
- time_per_token: 0.0,
- tokens_per_second: 0.0,
- success: false,
- error: None,
- };
- // Setup context for both placement analysis and timing
- let (mut ctx, _decoder_config) = match DecoderTestContext::new(model_dir) {
- Ok(pair) => pair,
- Err(e) => {
- result.error = Some(e);
- return result;
- },
- };
- // Warmup run with placement analysis
- let placement_log = ctx.run_warmup_with_placement_analysis();
- result.placement_log = placement_log;
- // After warmup, turn off placement analysis for timing loop
- // (Assume placement analysis only affects compilation, not execution)
- let iterations = 100;
- match ctx.run_timing_loop(iterations) {
- Ok(duration) => {
- result.iterations = iterations;
- result.time_per_token = duration.as_secs_f32() / iterations as f32;
- result.tokens_per_second = 1.0 / result.time_per_token;
- result.success = true;
- },
- Err(e) => {
- result.error = Some(e);
- },
- }
- result
-}
diff --git a/crates/uzu/src/runners/mod.rs b/crates/uzu/src/runners/mod.rs
deleted file mode 100644
index 1ad5471a3..000000000
--- a/crates/uzu/src/runners/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-pub mod decoder_runner;
-pub use decoder_runner::*;
-pub mod quant_matmul;
-pub use quant_matmul::*;
diff --git a/crates/uzu/src/runners/quant_matmul.rs b/crates/uzu/src/runners/quant_matmul.rs
deleted file mode 100644
index 60350f043..000000000
--- a/crates/uzu/src/runners/quant_matmul.rs
+++ /dev/null
@@ -1,452 +0,0 @@
-// Note: Module providing quantized matmul benchmark utilities originally prototyped
-// in `examples/quant_matmul_experiments.rs`. The code is copied almost verbatim so
-// that external crates can depend on these helpers without having to duplicate the
-// logic themselves.
-
-use std::{
- collections::HashMap,
- rc::Rc,
- time::{Instant, SystemTime, UNIX_EPOCH},
-};
-
-use clap::Parser;
-use half::f16;
-use metal::Device;
-use mpsgraph::{
- CommandBuffer, CompilationDescriptor, DataType as MPSDataType,
- DequantizationArguments, Device as MPSDevice, Executable,
- ExecutableExecutionDescriptor, ExecutableSerializationDescriptor, Graph,
- Optimization, OptimizationProfile, ShapedType, Tensor, TensorData,
-};
-use objc2::rc::Retained;
-use thiserror::Error;
-
-use crate::{
- Array, DataType, DeviceContext,
- backends::metal::{
- MTLContext, MetalArray,
- compilation_parameters::{BlockDevice, make_compilation_descriptor},
- },
- utils::storage::{NSSearchPathDirectory, root_dir},
-};
-
-// ================== Public error type =====================
-
-#[derive(Debug, Error)]
-pub enum ExampleError {
- #[error(transparent)]
- Metal(#[from] crate::backends::metal::error::MTLError),
- #[error("No Metal device available")]
- NoDevice,
- #[error("Graph dequantization failed")]
- Dequantization,
-}
-
-// ================== Public helper types =====================
-
-#[derive(Debug, Clone, Copy)]
-pub enum TensorLoadType {
- Baked,
- RuntimeLoaded,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct QuantizedMatmulDescriptor {
- pub weights: TensorLoadType,
- pub scales: TensorLoadType,
- pub zeros: TensorLoadType,
- pub transposed_shapes: bool,
- pub device: BlockDevice,
- pub optimization_level: Optimization,
- pub optimization_profile: OptimizationProfile,
-}
-
-// ================== CLI args used by the low-level API =====================
-
-#[derive(Parser, Debug, Clone, Copy)]
-#[command(
- name = "quant_matmul_experiments (library)",
- about = "Flags controlling extra instrumentation when running quantised matmul benchmark"
-)]
-pub struct Args {
- /// Print the compiled executable IR / debug dump
- #[arg(long = "print-exec-dump")]
- pub print_exec_dump: bool,
-
- /// Print ANE/GPU placement analysis during compilation
- #[arg(long = "print-placement-analysis")]
- pub print_placement_analysis: bool,
-
- /// Serialize the compiled MPSGraph executable to a .mpsgraphpackage file
- #[arg(long = "dump-mpsgraphpackage")]
- pub dump_mpsgraphpackage: bool,
-}
-
-impl Default for Args {
- fn default() -> Self {
- Self {
- print_exec_dump: false,
- print_placement_analysis: false,
- dump_mpsgraphpackage: false,
- }
- }
-}
-
-// ================== Internal tensor helpers =====================
-
-#[derive(Debug, Clone)]
-enum TensorOption {
- Constant(Retained<Tensor>),
- Placeholder {
- placeholder: Retained<Tensor>,
- data: Retained<TensorData>,
- },
-}
-
-impl TensorOption {
- #[inline]
- fn tensor(&self) -> Retained<Tensor> {
- match self {
- Self::Constant(t) => t.clone(),
- Self::Placeholder {
- placeholder,
- ..
- } => placeholder.clone(),
- }
- }
-
- fn feeds(&self) -> HashMap<Retained<Tensor>, Retained<ShapedType>> {
- match self {
- TensorOption::Constant(_) => HashMap::new(),
- TensorOption::Placeholder {
- placeholder,
- ..
- } => HashMap::from_iter([(
- placeholder.clone(),
- placeholder.shaped_type(),
- )]),
- }
- }
-
- fn data_feeds(&self) -> HashMap<Retained<Tensor>, Retained<TensorData>> {
- match self {
- TensorOption::Constant(_) => HashMap::new(),
- TensorOption::Placeholder {
- placeholder,
- data,
- } => HashMap::from_iter([(placeholder.clone(), data.clone())]),
- }
- }
-}
-
-#[derive(Debug, Clone)]
-struct QMTensorOptions {
- input: TensorOption,
- weights: TensorOption,
- scales: TensorOption,
- zeros: TensorOption,
-}
-
-impl QMTensorOptions {
- fn feeds(&self) -> HashMap<Retained<Tensor>, Retained<ShapedType>> {
- HashMap::from_iter(
- [&self.input, &self.weights, &self.scales, &self.zeros]
- .iter()
- .flat_map(|opt| opt.feeds()),
- )
- }
-
- fn data_feeds(&self) -> HashMap<Retained<Tensor>, Retained<TensorData>> {
- HashMap::from_iter(
- [&self.input, &self.weights, &self.scales, &self.zeros]
- .iter()
- .flat_map(|opt| opt.data_feeds()),
- )
- }
-}
-
-// ================== Internal graph helpers =====================
-
-fn build_quantized_matmul(
- graph: &Graph,
- tensor_options: &QMTensorOptions,
- mtl_context: &Rc<MTLContext>,
- compilation_descriptor: &CompilationDescriptor,
- transposed_shapes: bool,
-) -> Result<Retained<Executable>, ExampleError> {
- let i = tensor_options.input.tensor();
- let w = tensor_options.weights.tensor();
- let s = tensor_options.scales.tensor();
- let zp = tensor_options.zeros.tensor();
-
- let dequantized_weights = graph.dequantize(
- &w,
- DequantizationArguments::ScaleTensorZeroPointTensorDataType {
- scale_tensor: &s,
- zero_point_tensor: &zp,
- data_type: MPSDataType::Float16,
- },
- None,
- );
-
- let matmul = if transposed_shapes {
- graph.matrix_multiplication(
- &i,
- &graph.transpose(&dequantized_weights, &[1, 0], None),
- None,
- )
- } else {
- graph.matrix_multiplication(
- &graph.transpose(&dequantized_weights, &[1, 0], None),
- &graph.transpose(&i, &[1, 0], None),
- None,
- )
- };
-
- let device = MPSDevice::with_device(&mtl_context.device);
-
- let retained_feeds = tensor_options.feeds();
- let feeds = retained_feeds.iter().map(|(t, st)| (&**t, &**st)).collect();
-
- let executable = graph.compile(
- &device,
- &feeds,
- &[&*matmul],
- None,
- Some(&compilation_descriptor),
- );
-
- Ok(executable)
-}
-
-fn make_tensor_option(
- graph: &Graph,
- load_type: TensorLoadType,
- array: &mut MetalArray,
- shape: &[usize],
- dtype: DataType,
- name: &str,
-) -> TensorOption {
- match load_type {
- TensorLoadType::Baked => {
- let tensor_const =
- graph.constant_with_data(array.buffer(), shape, dtype.into());
- TensorOption::Constant(tensor_const)
- },
- TensorLoadType::RuntimeLoaded => {
- let td = unsafe { array.to_mps_tensor_data() };
- let shape_isize: Vec<isize> =
- shape.iter().map(|d| *d as isize).collect();
- let placeholder =
- graph.placeholder(Some(&shape_isize), dtype.into(), Some(name));
- TensorOption::Placeholder {
- placeholder,
- data: td,
- }
- },
- }
-}
-
-// ================== Core public API =====================
-
-/// Execute the quantised matmul benchmark once and return the average time per
-/// iteration (in milliseconds) across an internal fixed number of runs.
-///
-/// The behaviour is identical to the original implementation in
-/// `examples/quant_matmul_experiments.rs`.
-#[allow(clippy::too_many_lines)]
-pub fn run_quant_matmul(
- qm_descriptor: &QuantizedMatmulDescriptor,
- args: &Args,
-) -> Result<f64, ExampleError> {
- // Constants that do not change between runs
- let suffix_length: usize = 128;
- let input_dim: usize = 2560;
- let output_dim: usize = 3072 * 2;
- let group_size: usize = 20;
-
- // Shapes that depend on descriptor options
- let weights_shape = if qm_descriptor.transposed_shapes {
- [output_dim, input_dim]
- } else {
- [input_dim, output_dim]
- };
- let scales_shape = if qm_descriptor.transposed_shapes {
- [output_dim, input_dim / group_size]
- } else {
- [input_dim / group_size, output_dim]
- };
- let zero_points_shape = scales_shape;
- let input_shape = [-1_isize, input_dim as isize];
-
- // Concrete shapes for actual buffer allocation at runtime
- let input_tensor_shape = [suffix_length, input_dim];
- let result_tensor_shape = [suffix_length, output_dim];
-
- // --- Device & context ---
- let device = Device::system_default().ok_or(ExampleError::NoDevice)?;
- let command_queue = device.new_command_queue();
- let mtl_context = Rc::new(MTLContext::new(device, command_queue)?);
-
- // --- Arrays ---
- let mut weights_array = unsafe {
- mtl_context.array_uninitialized(&weights_shape, DataType::U4)
- };
- weights_array.buffer_mut().fill(0x11);
-
- let mut scales_array =
- mtl_context.array_from_elem(&scales_shape, f16::from_f32(1.0));
-
- let mut zero_points_array = unsafe {
- mtl_context.array_uninitialized(&zero_points_shape, DataType::U4)
- };
- zero_points_array.buffer_mut().fill(0x00);
-
- let mut input_array =
- mtl_context.array_from_elem(&input_tensor_shape, f16::from_f32(1.0));
-
- let mut result_array = unsafe {
- mtl_context.array_uninitialized(&result_tensor_shape, DataType::F16)
- };
-
- // --- Graph ---
- let graph = Graph::new();
-
- // --- Tensor options for quantized matmul ---
- let input_tensor_data = unsafe { input_array.to_mps_tensor_data() };
- let input_placeholder = graph.placeholder(
- Some(&input_shape),
- DataType::F16.into(),
- Some("input_ph"),
- );
-
- let tensor_options = QMTensorOptions {
- input: TensorOption::Placeholder {
- placeholder: input_placeholder,
- data: input_tensor_data,
- },
- weights: make_tensor_option(
- &graph,
- qm_descriptor.weights,
- &mut weights_array,
- &weights_shape,
- DataType::U4,
- "weights_ph",
- ),
- scales: make_tensor_option(
- &graph,
- qm_descriptor.scales,
- &mut scales_array,
- &scales_shape,
- DataType::F16,
- "scales_ph",
- ),
- zeros: make_tensor_option(
- &graph,
- qm_descriptor.zeros,
- &mut zero_points_array,
- &zero_points_shape,
- DataType::U4,
- "zeros_ph",
- ),
- };
-
- let compilation_descriptor = make_compilation_descriptor(
- qm_descriptor.device,
- qm_descriptor.optimization_level,
- qm_descriptor.optimization_profile,
- args.print_placement_analysis,
- );
-
- let executable = build_quantized_matmul(
- &graph,
- &tensor_options,
- &mtl_context,
- &compilation_descriptor,
- qm_descriptor.transposed_shapes,
- )?;
-
- // Optional printing / dumping per flags
- if args.print_exec_dump {
- executable.dump();
- }
-
- if args.dump_mpsgraphpackage {
- let timestamp = SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .expect("Time went backwards")
- .as_secs();
- let package_path = root_dir(NSSearchPathDirectory::Downloads).join(
- format!("quant_matmul_experiments-{timestamp}.mpsgraphpackage"),
- );
- let serialization_desc = ExecutableSerializationDescriptor::new();
- executable.serialize_to_graph_package(
- &package_path,
- Some(&serialization_desc),
- );
- }
-
- // --- Run & measure ---
- let result_tensor_data = unsafe { result_array.to_mps_tensor_data() };
- let retained_data_feeds = tensor_options.data_feeds();
- let inputs: Vec<&TensorData> = match executable.feed_tensors() {
- Some(feed_tensors) => feed_tensors
- .iter()
- .map(|t| {
- retained_data_feeds
- .get(t)
- .expect("TensorData for feed tensor not found")
- })
- .map(|td| &**td)
- .collect(),
- None => Vec::new(),
- };
-
- let outputs = [&*result_tensor_data];
-
- // Warm-up
- run_once(&mtl_context, &executable, &inputs, &outputs);
-
- const NUM_ITERATIONS: usize = 1000;
- let mut total_duration = 0.0f64; // milliseconds
-
- for _ in 0..NUM_ITERATIONS {
- let start = Instant::now();
- run_once(&mtl_context, &executable, &inputs, &outputs);
- total_duration += start.elapsed().as_secs_f64() * 1e3;
- }
-
- Ok(total_duration / NUM_ITERATIONS as f64)
-}
-
-/// Convenience wrapper that uses default (all-false) flags.
-pub fn run_quant_matmul_simple(
- qm_descriptor: &QuantizedMatmulDescriptor
-) -> Result<f64, ExampleError> {
- run_quant_matmul(qm_descriptor, &Args::default())
-}
-
-// ================== Internal util =====================
-
-fn run_once(
- mtl_context: &Rc<MTLContext>,
- executable: &Executable,
- inputs: &[&TensorData],
- outputs: &[&TensorData],
-) {
- let command_buffer =
- CommandBuffer::from_command_queue(&mtl_context.command_queue);
- let root_command_buffer = command_buffer.root_command_buffer().to_owned();
-
- let exec_desc = ExecutableExecutionDescriptor::new();
-
- executable.encode_to_command_buffer(
- &command_buffer,
- inputs,
- Some(outputs),
- Some(&exec_desc),
- );
-
- command_buffer.commit_and_continue();
- root_command_buffer.wait_until_completed();
-}
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
new file mode 100644
index 000000000..c90ba499f
--- /dev/null
+++ b/crates/uzu/src/session/chat_session.rs
@@ -0,0 +1,647 @@
+use std::{fs::File, io::BufReader, path::PathBuf, time::Instant};
+
+use objc2::rc::autoreleasepool;
+use tokenizers::Tokenizer;
+use xgrammar::TokenizerInfo;
+
+use crate::{
+ backends::metal::forward_pass::cache_layers::CacheLayer,
+ config::{MixerConfig, ModelMetadata},
+ language_model::{
+ LanguageModelGenerator,
+ grammar::CompiledGrammar,
+ result::{GenerateResult, PrefillResult},
+ },
+ session::{
+ config::{DecodingConfig, RunConfig},
+ helpers::{
+ Context, InputProcessor, InputProcessorDefault, OutputParser,
+ is_directory_fits_ram,
+ },
+ parameter::{ConfigResolvableValue, ContextMode},
+ types::{
+ Error, FinishReason, Input, Output, RunStats, Stats, StepStats,
+ TotalStats,
+ },
+ },
+};
+
+pub struct ChatSession {
+ pub model_path: PathBuf,
+ pub model_metadata: ModelMetadata,
+
+ tokenizer: Tokenizer,
+ tokenizer_info: TokenizerInfo,
+ input_processor: Box<dyn InputProcessor>,
+ output_parser: OutputParser,
+ llm: Option<LanguageModelGenerator>,
+ static_context: Option<Context>,
+}
+
+impl ChatSession {
+ pub fn new(
+ model_path: PathBuf,
+ decoding_config: DecodingConfig,
+ ) -> Result<Self, Error> {
+ if !model_path.exists() {
+ return Err(Error::ModelFolderNotFound);
+ }
+
+ if !is_directory_fits_ram(&model_path) {
+ return Err(Error::NotEnoughMemory);
+ }
+
+ let config_path = model_path.join("config.json");
+ if !config_path.exists() {
+ return Err(Error::UnableToLoadConfig);
+ }
+ let config_file =
+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
+ let model_metadata: ModelMetadata = serde_json::from_reader(
+ BufReader::new(config_file),
+ )
+ .map_err(|err| {
+ eprintln!("Failed to parse config.json: {err}");
+ Error::UnableToLoadConfig
+ })?;
+
+ let is_ssm = model_metadata
+ .model_config
+ .as_language_model()
+ .and_then(|lm| lm.decoder_config().ok())
+ .and_then(|dc| dc.layer_configs)
+ .map(|layers| {
+ layers.iter().any(|layer| {
+ matches!(layer.mixer_config, MixerConfig::Mamba(_))
+ })
+ })
+ .unwrap_or(false);
+ if is_ssm {
+ match decoding_config.context_mode {
+ ContextMode::None => {},
+ ContextMode::Static {
+ ..
+ } => {
+ return Err(Error::UnsupportedContextModeForModel);
+ },
+ ContextMode::Dynamic => {
+ return Err(Error::UnsupportedContextModeForModel);
+ },
+ }
+
+ if decoding_config.speculator_config.number_of_speculated_tokens > 0
+ {
+ return Err(Error::UnsupportedSpeculatorConfigForModel);
+ }
+ }
+
+ let tokenizer_path = model_path.join("tokenizer.json");
+ if !tokenizer_path.exists() {
+ return Err(Error::UnableToLoadTokenizer);
+ }
+ let tokenizer = Tokenizer::from_file(&tokenizer_path)
+ .map_err(|_| Error::UnableToLoadTokenizer)?;
+
+ // Create TokenizerInfo for grammar support
+ let tokenizer_info =
+ TokenizerInfo::from_huggingface(&tokenizer, None, None);
+
+ // Extract language model config
+ let language_model_config = model_metadata
+ .model_config
+ .as_language_model()
+ .ok_or(Error::UnableToLoadConfig)?;
+
+ let input_processor = InputProcessorDefault::new(
+ language_model_config.message_processor_config.clone(),
+ );
+
+ let output_parser = OutputParser::new(
+ language_model_config
+ .message_processor_config
+ .output_parser_regex
+ .clone(),
+ )?;
+
+ let llm = LanguageModelGenerator::new(&model_path, decoding_config)
+ .map_err(Error::from)?;
+
+ Ok(Self {
+ model_path,
+ model_metadata,
+ tokenizer,
+ tokenizer_info,
+ input_processor: Box::new(input_processor),
+ output_parser,
+ llm: Some(llm),
+ static_context: None,
+ })
+ }
+
+ pub fn run<F>(
+ &mut self,
+ input: Input,
+ config: RunConfig,
+ progress: Option<F>,
+ ) -> Result<Output, Error>
+ where
+ F: Fn(Output) -> bool,
+ {
+ let context_mode = self
+ .llm
+ .as_ref()
+ .ok_or(Error::LanguageModelGeneratorNotLoaded)?
+ .decoding_config
+ .context_mode
+ .clone();
+
+ match &context_mode {
+ ContextMode::None => {
+ let llm = self
+ .llm
+ .as_mut()
+ .ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ llm.reset_state();
+ },
+ ContextMode::Static {
+ input,
+ } => {
+ if self.static_context.is_none() {
+ let mut prefill_config = config.clone();
+ prefill_config.tokens_limit = 0;
+ let (_out, ctx) =
+ self.extend(input.clone(), None, prefill_config)?;
+ self.static_context = Some(ctx);
+ }
+ let tmp = self.static_context.take();
+ if let Some(ref ctx) = tmp {
+ self.reconfigure_llm(Some(ctx))?;
+ }
+ self.static_context = tmp;
+ },
+ ContextMode::Dynamic => {},
+ }
+
+ let output = self.run_internal(input, config, progress)?;
+
+ match context_mode {
+ ContextMode::None
+ | ContextMode::Static {
+ ..
+ } => {
+ let llm = self
+ .llm
+ .as_mut()
+ .ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ llm.reset_state();
+ },
+ ContextMode::Dynamic => {},
+ }
+ Ok(output)
+ }
+
+ pub fn reset(&mut self) -> Result<(), Error> {
+ let llm =
+ self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ llm.reset_state();
+ Ok(())
+ }
+}
+
+impl ChatSession {
+ fn extend(
+ &mut self,
+ input: Input,
+ context: Option<&Context>,
+ config: RunConfig,
+ ) -> Result<(Output, Context), Error> {
+ self.reconfigure_llm(context)?;
+ let output =
+ self.run_internal(input, config, None::<fn(Output) -> bool>)?;
+ let new_context = self.build_context_from_llm()?;
+ let llm =
+ self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ llm.reset_state();
+ Ok((output, new_context))
+ }
+
+ fn run_internal<F>(
+ &mut self,
+ input: Input,
+ config: RunConfig,
+ progress: Option<F>,
+ ) -> Result<Output, Error>
+ where
+ F: Fn(Output) -> bool,
+ {
+ let run_start = Instant::now();
+ let text =
+ self.input_processor.process(&input, config.enable_thinking)?;
+ let tokens: Vec<u64> = self
+ .tokenizer
+ .encode(text.as_str(), false)
+ .map_err(|_| Error::UnableToEncodeText)?
+ .get_ids()
+ .iter()
+ .map(|&id| id as u64)
+ .collect();
+
+ let language_model_config_for_context = self
+ .model_metadata
+ .model_config
+ .as_language_model()
+ .ok_or(Error::UnableToLoadConfig)?;
+
+ let llm =
+ self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let context_length = llm
+ .decoding_config
+ .context_length
+ .resolve(language_model_config_for_context);
+ if tokens.len() >= context_length {
+ return Err(Error::ContextLengthExceeded);
+ }
+ let prefix_len_before = llm.tokens.len().saturating_sub(1);
+
+ let eos_tokens: Vec<u64> = language_model_config_for_context
+ .generation_config
+ .stop_token_ids
+ .iter()
+ .map(|&x| x as u64)
+ .collect();
+
+ let finish_reason = |llm: &LanguageModelGenerator,
+ tokens_new: Vec<u64>|
+ -> Option<FinishReason> {
+ let start_idx = prefix_len_before + tokens.len();
+ let total_tokens = llm.tokens.len();
+ let total_new_tokens = llm.tokens[start_idx..].len();
+ let has_eos_token =
+ tokens_new.iter().any(|token| eos_tokens.contains(token));
+ let context_limit_reached = total_tokens >= context_length;
+
+ if has_eos_token {
+ Some(FinishReason::Stop)
+ } else if total_new_tokens >= config.tokens_limit as usize {
+ Some(FinishReason::Length)
+ } else if context_limit_reached {
+ Some(FinishReason::ContextLimitReached)
+ } else {
+ None
+ }
+ };
+
+ let build_generated_text = |llm: &LanguageModelGenerator,
+ tokenizer: &Tokenizer|
+ -> Result<String, Error> {
+ let start_idx = prefix_len_before + tokens.len();
+ let generated_tokens: Vec<u32> = llm.tokens[start_idx..]
+ .to_vec()
+ .iter()
+ .map(|value| *value as u32)
+ .collect();
+ let generated_text = tokenizer
+ .decode(&generated_tokens, true)
+ .map_err(|_| Error::UnableToDecodeText)?;
+ Ok(generated_text)
+ };
+
+ let language_model_config_for_sampling = self
+ .model_metadata
+ .model_config
+ .as_language_model()
+ .ok_or(Error::UnableToLoadConfig)?;
+
+ let sampling_method =
+ config.sampling_policy.resolve(language_model_config_for_sampling);
+
+ // Create grammar from config if provided
+ let mut compiled_grammar: Option<CompiledGrammar> =
+ if let Some(ref grammar_config) = config.grammar_config {
+ Some(CompiledGrammar::from_config(
+ grammar_config,
+ None,
+ &self.tokenizer_info,
+ )?)
+ } else {
+ None
+ };
+
+ let prefill_start = Instant::now();
+ let prefix_offset = llm.tokens.len();
+
+ let sample_suffix = config.tokens_limit > 0;
+ let prefill_result = llm.prefill(
+ tokens.clone(),
+ compiled_grammar.as_mut(),
+ sampling_method,
+ prefix_offset,
+ sample_suffix,
+ )?;
+ let prefill_tokens = prefill_result.tokens.clone();
+ let prefill_duration = prefill_start.elapsed().as_secs_f64();
+ llm.clear_cache();
+
+ let prefill_finish_reason = finish_reason(llm, prefill_tokens.clone());
+ let prefill_generated_text =
+ build_generated_text(llm, &self.tokenizer)?;
+ let prefill_parsed_text =
+ self.output_parser.parse(prefill_generated_text);
+
+ let language_model_config_for_prefill = self
+ .model_metadata
+ .model_config
+ .as_language_model()
+ .ok_or(Error::UnableToLoadConfig)?;
+
+ let prefill_suffix_length = llm
+ .decoding_config
+ .prefill_step_size
+ .resolve(language_model_config_for_prefill);
+ let prefill_output = Output {
+ text: prefill_parsed_text,
+ stats: Self::build_stats(
+ &self.model_metadata,
+ prefill_result.clone(),
+ prefill_duration,
+ prefill_suffix_length,
+ Vec::new(),
+ Vec::new(),
+ llm.decoding_config.generate_suffix_length(),
+ run_start.elapsed().as_secs_f64(),
+ tokens.len(),
+ llm.tokens[prefix_len_before + tokens.len()..].len(),
+ ),
+ finish_reason: prefill_finish_reason.clone(),
+ };
+
+ let prefill_should_continue = if let Some(progress) = &progress {
+ progress(prefill_output.clone())
+ } else {
+ true
+ };
+
+ if !prefill_should_continue || prefill_finish_reason.is_some() {
+ if prefill_should_continue {
+ return Ok(prefill_output);
+ } else {
+ return Ok(prefill_output
+ .clone_with_finish_reason(Some(FinishReason::Cancelled)));
+ }
+ }
+
+ let mut generate_results: Vec<GenerateResult> = Vec::new();
+ let mut generate_durations: Vec<f64> = Vec::new();
+ let generate_output = loop {
+ let generate_start = Instant::now();
+ let generate_result =
+ llm.generate(compiled_grammar.as_mut(), sampling_method)?;
+ let generate_tokens = generate_result.tokens.clone();
+ let generate_duration = generate_start.elapsed().as_secs_f64();
+ generate_results.push(generate_result);
+ generate_durations.push(generate_duration);
+
+ let generate_finish_reason = finish_reason(llm, generate_tokens);
+ let generate_generated_text =
+ build_generated_text(llm, &self.tokenizer)?;
+ let generate_parsed_text =
+ self.output_parser.parse(generate_generated_text);
+
+ let generate_output = Output {
+ text: generate_parsed_text,
+ stats: Self::build_stats(
+ &self.model_metadata,
+ prefill_result.clone(),
+ prefill_duration,
+ prefill_suffix_length,
+ generate_results.clone(),
+ generate_durations.clone(),
+ llm.decoding_config.generate_suffix_length(),
+ run_start.elapsed().as_secs_f64(),
+ tokens.len(),
+ llm.tokens[prefix_len_before + tokens.len()..].len(),
+ ),
+ finish_reason: generate_finish_reason.clone(),
+ };
+
+ let generate_should_continue = if let Some(progress) = &progress {
+ progress(generate_output.clone())
+ } else {
+ true
+ };
+
+ if !generate_should_continue || generate_finish_reason.is_some() {
+ if generate_should_continue {
+ break generate_output;
+ } else {
+ break generate_output.clone_with_finish_reason(Some(
+ FinishReason::Cancelled,
+ ));
+ }
+ }
+ };
+
+ llm.clear_cache();
+ Ok(generate_output
+ .clone_with_duration(run_start.elapsed().as_secs_f64()))
+ }
+
+ fn reconfigure_llm(
+ &mut self,
+ context: Option<&Context>,
+ ) -> Result<(), Error> {
+ let llm =
+ self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ llm.reset_state();
+ if let Some(ctx) = context {
+ let mut llm_state = llm.context.cache_layers.borrow_mut();
+ for (ctx_layer, gen_layer) in
+ ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut())
+ {
+ match (ctx_layer, gen_layer) {
+ (
+ CacheLayer::Transformer(src),
+ CacheLayer::Transformer(dst),
+ ) => {
+ let copy_rows = src.prefix_segment_length();
+ if copy_rows > 0 {
+ {
+ let mut dst_keys = dst.keys.borrow_mut();
+ let src_keys = src.keys.borrow();
+ dst_keys.copy_slice(
+ &src_keys,
+ 1,
+ 0..copy_rows,
+ 0,
+ );
+ }
+ {
+ let mut dst_values = dst.values.borrow_mut();
+ let src_values = src.values.borrow();
+ dst_values.copy_slice(
+ &src_values,
+ 1,
+ 0..copy_rows,
+ 0,
+ );
+ }
+ }
+ dst.state = src.state.clone();
+ dst.prefix_token_positions =
+ src.prefix_token_positions.clone();
+ },
+ (
+ CacheLayer::StateSpace(src),
+ CacheLayer::StateSpace(dst),
+ ) => {
+ {
+ let mut dst_conv = dst.conv_state.borrow_mut();
+ let src_conv = src.conv_state.borrow();
+ dst_conv.copy_from_array(&src_conv);
+ }
+ {
+ let mut dst_ssm = dst.ssm_state.borrow_mut();
+ let src_ssm = src.ssm_state.borrow();
+ dst_ssm.copy_from_array(&src_ssm);
+ }
+ },
+ _ => panic!(
+ "Layer type mismatch when reconfiguring language model generator cache"
+ ),
+ }
+ }
+ drop(llm_state);
+
+ llm.tokens = ctx.tokens.clone();
+ }
+ Ok(())
+ }
+
+ fn build_context_from_llm(&self) -> Result<Context, Error> {
+ let llm =
+ self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let cache_layers =
+ llm.context.cache_layers.borrow().clone(&llm.context.mtl_context);
+ let context = Context::new(
+ llm.tokens.clone(),
+ cache_layers,
+ llm.decoding_config.clone(),
+ );
+ Ok(context)
+ }
+}
+
+impl ChatSession {
+ fn build_stats(
+ model_metadata: &ModelMetadata,
+ prefill_result: PrefillResult,
+ prefill_duration: f64,
+ prefill_suffix_length: usize,
+ generate_results: Vec<GenerateResult>,
+ generate_durations: Vec<f64>,
+ generate_suffix_length: usize,
+ total_duration: f64,
+ tokens_count_input: usize,
+ tokens_count_output: usize,
+ ) -> Stats {
+ let prefill_stats = {
+ let tokens_count = prefill_result.tokens.len();
+ let tokens_per_second = tokens_count as f64 / prefill_duration;
+
+ // TODO: Unify after removing MPS blocks
+ let processed_tokens = if model_metadata.quantization.is_some() {
+ let result = tokens_count_input + tokens_count;
+ result
+ } else {
+ let number_of_prefill_steps =
+ (tokens_count_input as f32 / prefill_suffix_length as f32)
+ .ceil() as usize;
+ let result = prefill_suffix_length * number_of_prefill_steps
+ + tokens_count;
+ result
+ };
+ let processed_tokens_per_second =
+ processed_tokens as f64 / prefill_duration;
+
+ let model_run_count = prefill_result.forwardpass_durations.len();
+ let model_run_average_duration =
+ prefill_result.forwardpass_durations.iter().sum::<f64>()
+ / model_run_count as f64;
+
+ StepStats {
+ duration: prefill_duration,
+ suffix_length: prefill_suffix_length as u64,
+ tokens_count: tokens_count as u64,
+ tokens_per_second,
+ processed_tokens_per_second,
+ model_run: RunStats {
+ count: model_run_count as u64,
+ average_duration: model_run_average_duration,
+ },
+ run: None,
+ }
+ };
+
+ let generate_stats: Option<StepStats>;
+ if generate_results.len() != 0 {
+ let duration = generate_durations.iter().sum::<f64>();
+ let tokens_count = generate_results
+ .iter()
+ .flat_map(|result| result.tokens.clone())
+ .collect::<Vec<u64>>()
+ .len();
+ let tokens_per_second: f64 = tokens_count as f64 / duration;
+
+ let model_run_count = generate_results.len();
+ let model_run_average_duration = generate_results
+ .iter()
+ .map(|result| result.forwardpass_duration)
+ .sum::<f64>()
+ / model_run_count as f64;
+
+ let run_count = generate_durations.len();
+ let run_average_duration =
+ generate_durations.iter().sum::<f64>() / run_count as f64;
+
+ generate_stats = Some(StepStats {
+ duration,
+ suffix_length: generate_suffix_length as u64,
+ tokens_count: tokens_count as u64,
+ tokens_per_second,
+ processed_tokens_per_second: tokens_per_second,
+ model_run: RunStats {
+ count: model_run_count as u64,
+ average_duration: model_run_average_duration,
+ },
+ run: Some(RunStats {
+ count: run_count as u64,
+ average_duration: run_average_duration,
+ }),
+ });
+ } else {
+ generate_stats = None;
+ }
+
+ let total_stats = TotalStats {
+ duration: total_duration,
+ tokens_count_input: tokens_count_input as u64,
+ tokens_count_output: tokens_count_output as u64,
+ };
+
+ let stats = Stats {
+ prefill_stats,
+ generate_stats,
+ total_stats,
+ };
+
+ stats
+ }
+}
+
+impl Drop for ChatSession {
+ fn drop(&mut self) {
+ autoreleasepool(|_| {
+ self.llm = None;
+ });
+ }
+}
diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rs
new file mode 100644
index 000000000..cae8fb761
--- /dev/null
+++ b/crates/uzu/src/session/classification_session.rs
@@ -0,0 +1,87 @@
+use std::path::PathBuf;
+
+use tokenizers::Tokenizer;
+
+use crate::{
+ classifier::{ClassificationOutput, Classifier},
+ config::ModelMetadata,
+ session::{
+ helpers::{InputProcessor, InputProcessorDefault},
+ types::{Error, Input},
+ },
+};
+
+pub struct ClassificationSession {
+ #[allow(dead_code)]
+ model_path: PathBuf,
+ #[allow(dead_code)]
+ model_metadata: ModelMetadata,
+ tokenizer: Tokenizer,
+ classifier: Classifier,
+ input_processor: Box<dyn InputProcessor>,
+}
+
+impl ClassificationSession {
+ pub fn new(model_path: PathBuf) -> Result<Self, Error> {
+ let config_path = model_path.join("config.json");
+ if !config_path.exists() {
+ return Err(Error::ModelFolderNotFound);
+ }
+
+ let config_file = std::fs::File::open(&config_path)
+ .map_err(|_| Error::UnableToLoadConfig)?;
+ let model_metadata: ModelMetadata =
+ serde_json::from_reader(std::io::BufReader::new(config_file))
+ .map_err(|_| Error::UnableToLoadConfig)?;
+
+ let tokenizer_path = model_path.join("tokenizer.json");
+ if !tokenizer_path.exists() {
+ return Err(Error::UnableToLoadTokenizer);
+ }
+ let tokenizer = Tokenizer::from_file(&tokenizer_path)
+ .map_err(|_| Error::UnableToLoadTokenizer)?;
+
+ let classifier_model_config = model_metadata
+ .model_config
+ .as_classifier()
+ .ok_or(Error::UnableToLoadConfig)?;
+ let input_processor = Box::new(InputProcessorDefault::new(
+ classifier_model_config.message_processor_config.clone(),
+ )) as Box<dyn InputProcessor>;
+ let classifier = Classifier::new(&model_path)?;
+
+ Ok(Self {
+ model_path,
+ model_metadata,
+ tokenizer,
+ classifier,
+ input_processor,
+ })
+ }
+
+ pub fn classify(
+ &mut self,
+ input: Input,
+ ) -> Result<ClassificationOutput, Error> {
+ let preprocessing_start = std::time::Instant::now();
+ let text = self.input_processor.process(&input, false)?;
+ let tokens: Vec<u64> = self
+ .tokenizer
+ .encode(text.as_str(), false)
+ .map_err(|_| Error::UnableToEncodeText)?
+ .get_ids()
+ .iter()
+ .map(|&id| id as u64)
+ .collect();
+
+ let token_positions: Vec<usize> = (0..tokens.len()).collect();
+ let preprocessing_duration =
+ preprocessing_start.elapsed().as_secs_f64();
+
+ let mut output =
+ self.classifier.classify_tokens(tokens, token_positions)?;
+ output.stats.preprocessing_duration = preprocessing_duration;
+
+ Ok(output)
+ }
+}
diff --git a/crates/uzu/src/session/mod.rs b/crates/uzu/src/session/mod.rs
index 107657cbf..8633ea0d9 100644
--- a/crates/uzu/src/session/mod.rs
+++ b/crates/uzu/src/session/mod.rs
@@ -1,6 +1,9 @@
+mod chat_session;
+pub mod classification_session;
pub mod config;
pub mod helpers;
pub mod parameter;
-mod session;
pub mod types;
-pub use session::Session;
+
+pub use chat_session::{ChatSession, ChatSession as Session};
+pub use classification_session::ClassificationSession;
diff --git a/crates/uzu/src/session/parameter/context_length.rs b/crates/uzu/src/session/parameter/context_length.rs
index b3d504fdd..9abaca6e0 100644
--- a/crates/uzu/src/session/parameter/context_length.rs
+++ b/crates/uzu/src/session/parameter/context_length.rs
@@ -20,7 +20,10 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for ContextLength {
&self,
config: &LanguageModelConfig,
) -> usize {
- let model_context_length = config.decoder_config.context_length;
+ let model_context_length = config
+ .decoder_config()
+ .map(|dc| dc.context_length)
+ .unwrap_or(config.model_config.transformer_config.context_length);
let proposed_value = match self {
ContextLength::Default => {
if cfg!(target_os = "ios") {
diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rs
index 62d2e5d38..795a527b7 100644
--- a/crates/uzu/src/session/parameter/prefill_step_size.rs
+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs
@@ -21,10 +21,13 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
config: &LanguageModelConfig,
) -> usize {
let default_limit: usize = 1024;
- let model_context_length = config.decoder_config.context_length;
+ let model_context_length =
+ config.model_config.transformer_config.context_length;
+
let minimal_sliding_window_size = config
- .decoder_config
- .sliding_window_sizes
+ .decoder_config()
+ .ok()
+ .and_then(|dc| dc.sliding_window_sizes)
.as_deref()
.into_iter()
.flatten()
@@ -32,6 +35,7 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
.copied()
.min()
.unwrap_or(usize::MAX);
+
let maximal_value =
[model_context_length, minimal_sliding_window_size, default_limit]
.into_iter()
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rs
deleted file mode 100644
index d732913a5..000000000
--- a/crates/uzu/src/session/session.rs
+++ /dev/null
@@ -1,861 +0,0 @@
-use std::{
- fs::File,
- io::BufReader,
- path::PathBuf,
- sync::{
- Arc,
- atomic::{AtomicU64, Ordering},
- mpsc,
- },
- time::Instant,
-};
-
-fn nanos_to_secs(nanos: u64) -> f64 {
- nanos as f64 / 1_000_000_000.0
-}
-
-use objc2::rc::autoreleasepool;
-use tokenizers::Tokenizer;
-use xgrammar::TokenizerInfo;
-
-use crate::{
- backends::metal::forward_pass::cache_layers::CacheLayer,
- config::{ModelMetadata, decoder_layer::MixerConfig},
- generator::{
- generator::Generator,
- grammar::CompiledGrammar,
- result::{GenerateResult, PrefillResult},
- },
- session::{
- config::{DecodingConfig, RunConfig},
- helpers::{
- Context, InputProcessor, InputProcessorDefault, OutputParser,
- is_directory_fits_ram,
- },
- parameter::{ConfigResolvableValue, ContextMode, SamplingMethod},
- types::{
- Error, FinishReason, Input, Output, RunStats, Stats, StepStats,
- TotalStats,
- },
- },
-};
-
-struct RunContext {
- eos_tokens: Vec<u64>,
- context_length: usize,
- prefix_len_before: usize,
- input_tokens_len: usize,
- tokens_limit: usize,
- prefill_result: PrefillResult,
- prefill_duration: f64,
- prefill_suffix_length: usize,
- run_start: Instant,
-}
-
-pub struct Session {
- pub model_path: PathBuf,
- pub model_metadata: ModelMetadata,
-
- tokenizer: Tokenizer,
- tokenizer_info: TokenizerInfo,
- input_processor: Box<dyn InputProcessor>,
- output_parser: OutputParser,
- generator: Option<Generator>,
- static_context: Option<Context>,
-}
-
-impl Session {
- pub fn new(
- model_path: PathBuf,
- decoding_config: DecodingConfig,
- ) -> Result<Self, Error> {
- if !model_path.exists() {
- return Err(Error::ModelFolderNotFound);
- }
-
- if !is_directory_fits_ram(&model_path) {
- return Err(Error::NotEnoughMemory);
- }
-
- let config_path = model_path.join("config.json");
- if !config_path.exists() {
- return Err(Error::UnableToLoadConfig);
- }
- let config_file =
- File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
- let model_metadata: ModelMetadata = serde_json::from_reader(
- BufReader::new(config_file),
- )
- .map_err(|err| {
- eprintln!("Failed to parse config.json: {err}");
- Error::UnableToLoadConfig
- })?;
-
- let is_ssm = model_metadata
- .clone()
- .model_config
- .decoder_config
- .layer_configs
- .unwrap_or(Box::new([]))
- .iter()
- .any(|layer| matches!(layer.mixer_config, MixerConfig::Mamba(_)));
- if is_ssm {
- match decoding_config.context_mode {
- ContextMode::None => {},
- ContextMode::Static {
- ..
- } => {
- return Err(Error::UnsupportedContextModeForModel);
- },
- ContextMode::Dynamic => {
- return Err(Error::UnsupportedContextModeForModel);
- },
- }
-
- if decoding_config.speculator_config.number_of_speculated_tokens > 0
- {
- return Err(Error::UnsupportedSpeculatorConfigForModel);
- }
- }
-
- let tokenizer_path = model_path.join("tokenizer.json");
- if !tokenizer_path.exists() {
- return Err(Error::UnableToLoadTokenizer);
- }
- let tokenizer = Tokenizer::from_file(&tokenizer_path)
- .map_err(|_| Error::UnableToLoadTokenizer)?;
-
- let tokenizer_info =
- TokenizerInfo::from_huggingface(&tokenizer, None, None);
-
- let input_processor = InputProcessorDefault::new(
- model_metadata.model_config.message_processor_config.clone(),
- );
-
- let output_parser = OutputParser::new(
- model_metadata
- .model_config
- .message_processor_config
- .output_parser_regex
- .clone(),
- )?;
-
- let generator = Generator::new(&model_path, decoding_config)
- .map_err(Error::from)?;
-
- Ok(Self {
- model_path,
- model_metadata,
- tokenizer,
- tokenizer_info,
- input_processor: Box::new(input_processor),
- output_parser,
- generator: Some(generator),
- static_context: None,
- })
- }
-
- pub fn run<F>(
- &mut self,
- input: Input,
- config: RunConfig,
- progress: Option<F>,
- ) -> Result<Output, Error>
- where
- F: Fn(Output) -> bool,
- {
- autoreleasepool(|_| {
- let context_mode = self
- .generator
- .as_ref()
- .ok_or(Error::GeneratorNotLoaded)?
- .decoding_config
- .context_mode
- .clone();
-
- match &context_mode {
- ContextMode::None => {
- let generator = self
- .generator
- .as_mut()
- .ok_or(Error::GeneratorNotLoaded)?;
- generator.reset_state();
- },
- ContextMode::Static {
- input,
- } => {
- if self.static_context.is_none() {
- let mut prefill_config = config.clone();
- prefill_config.tokens_limit = 0;
- let (_out, run_context) =
- self.extend(input.clone(), None, prefill_config)?;
- self.static_context = Some(run_context);
- }
- let tmp = self.static_context.take();
- if let Some(ref run_context) = tmp {
- self.reconfigure_generator(Some(run_context))?;
- }
- self.static_context = tmp;
- },
- ContextMode::Dynamic => {},
- }
-
- let output = self.run_internal(input, config, progress)?;
-
- match context_mode {
- ContextMode::None
- | ContextMode::Static {
- ..
- } => {
- let generator = self
- .generator
- .as_mut()
- .ok_or(Error::GeneratorNotLoaded)?;
- generator.reset_state();
- },
- ContextMode::Dynamic => {},
- }
- Ok(output)
- })
- }
-
- pub fn reset(&mut self) -> Result<(), Error> {
- let generator =
- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
- autoreleasepool(|_| generator.reset_state());
- Ok(())
- }
-}
-
-impl Session {
- fn extend(
- &mut self,
- input: Input,
- context: Option<&Context>,
- config: RunConfig,
- ) -> Result<(Output, Context), Error> {
- self.reconfigure_generator(context)?;
- let output =
- self.run_internal(input, config, None::<fn(Output) -> bool>)?;
- let new_context = self.build_context_from_generator()?;
- let generator =
- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
- generator.reset_state();
- Ok((output, new_context))
- }
-
- fn run_internal<F>(
- &mut self,
- input: Input,
- config: RunConfig,
- progress: Option<F>,
- ) -> Result<Output, Error>
- where
- F: Fn(Output) -> bool,
- {
- let generator =
- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
-
- let mut compiled_grammar: Option<CompiledGrammar> =
- if let Some(ref grammar_config) = config.grammar_config {
- Some(CompiledGrammar::from_config(
- grammar_config,
- None,
- &self.tokenizer_info,
- )?)
- } else {
- None
- };
-
- let run_start = Instant::now();
- let text =
- self.input_processor.process(&input, config.enable_thinking)?;
- let tokens: Vec<u64> = self
- .tokenizer
- .encode(text.as_str(), false)
- .map_err(|_| Error::UnableToEncodeText)?
- .get_ids()
- .iter()
- .map(|&id| id as u64)
- .collect();
-
- let context_length = generator
- .decoding_config
- .context_length
- .resolve(&self.model_metadata.model_config);
- if tokens.len() >= context_length {
- return Err(Error::ContextLengthExceeded);
- }
-
- let eos_tokens: Vec<u64> = self
- .model_metadata
- .model_config
- .generation_config
- .stop_token_ids
- .iter()
- .map(|&x| x as u64)
- .collect();
-
- let sampling_method =
- config.sampling_policy.resolve(&self.model_metadata.model_config);
-
- let prefill_start = Instant::now();
- let prefix_offset = generator.tokens.len();
- let prefix_len_before = prefix_offset.saturating_sub(1);
-
- let sample_suffix = config.tokens_limit > 0;
- let prefill_result = generator.prefill(
- tokens.clone(),
- compiled_grammar.as_mut(),
- sampling_method,
- prefix_offset,
- sample_suffix,
- )?;
- let prefill_tokens = prefill_result.tokens.clone();
- let prefill_duration = prefill_start.elapsed().as_secs_f64();
- generator.clear_cache();
-
- let prefill_suffix_length = generator
- .decoding_config
- .prefill_step_size
- .resolve(&self.model_metadata.model_config);
-
- let run_context = RunContext {
- eos_tokens,
- context_length,
- prefix_len_before,
- input_tokens_len: tokens.len(),
- tokens_limit: config.tokens_limit as usize,
- prefill_result: prefill_result.clone(),
- prefill_duration,
- prefill_suffix_length,
- run_start,
- };
-
- let prefill_finish_reason =
- Self::check_finish_reason(&run_context, generator, &prefill_tokens);
- let prefill_output = Self::build_output(
- &self.model_metadata,
- &self.tokenizer,
- &self.output_parser,
- &run_context,
- generator,
- &[],
- &[],
- prefill_finish_reason.clone(),
- )?;
-
- let prefill_should_continue = if let Some(ref p) = progress {
- p(prefill_output.clone())
- } else {
- true
- };
-
- if !prefill_should_continue || prefill_finish_reason.is_some() {
- if prefill_should_continue {
- return Ok(prefill_output);
- } else {
- return Ok(prefill_output
- .clone_with_finish_reason(Some(FinishReason::Cancelled)));
- }
- }
-
- let can_use_async = generator.decoding_config.generate_suffix_length()
- == 1
- && !generator.has_attention_layers()
- && compiled_grammar.is_none();
-
- let generate_output = if can_use_async {
- let batch_size = generator
- .decoding_config
- .async_batch_size
- .resolve(&self.model_path);
- let (results, durations, finish_reason) = Self::run_async_batch(
- &self.model_metadata,
- &self.tokenizer,
- &self.output_parser,
- &run_context,
- generator,
- sampling_method,
- &progress,
- batch_size,
- )?;
- Self::build_output(
- &self.model_metadata,
- &self.tokenizer,
- &self.output_parser,
- &run_context,
- generator,
- &results,
- &durations,
- Some(finish_reason),
- )?
- } else {
- Self::run_sync_generate(
- &self.model_metadata,
- &self.tokenizer,
- &self.output_parser,
- &run_context,
- generator,
- compiled_grammar.as_mut(),
- sampling_method,
- &progress,
- )?
- };
-
- generator.clear_cache();
- Ok(generate_output
- .clone_with_duration(run_start.elapsed().as_secs_f64()))
- }
-
- fn reconfigure_generator(
- &mut self,
- context: Option<&Context>,
- ) -> Result<(), Error> {
- let generator =
- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
- generator.reset_state();
- if let Some(run_context) = context {
- let mut generator_state =
- generator.context.cache_layers.borrow_mut();
- for (run_context_layer, gen_layer) in run_context
- .cache_layers
- .data
- .iter()
- .zip(generator_state.data.iter_mut())
- {
- match (run_context_layer, gen_layer) {
- (
- CacheLayer::Transformer(src),
- CacheLayer::Transformer(dst),
- ) => {
- let copy_rows = src.prefix_segment_length();
- if copy_rows > 0 {
- {
- let mut dst_keys = dst.keys.borrow_mut();
- let src_keys = src.keys.borrow();
- dst_keys.copy_slice(
- &src_keys,
- 1,
- 0..copy_rows,
- 0,
- );
- }
- {
- let mut dst_values = dst.values.borrow_mut();
- let src_values = src.values.borrow();
- dst_values.copy_slice(
- &src_values,
- 1,
- 0..copy_rows,
- 0,
- );
- }
- }
- dst.state = src.state.clone();
- dst.prefix_token_positions =
- src.prefix_token_positions.clone();
- },
- (
- CacheLayer::StateSpace(src),
- CacheLayer::StateSpace(dst),
- ) => {
- {
- let mut dst_conv = dst.conv_state.borrow_mut();
- let src_conv = src.conv_state.borrow();
- dst_conv.copy_from_array(&src_conv);
- }
- {
- let mut dst_ssm = dst.ssm_state.borrow_mut();
- let src_ssm = src.ssm_state.borrow();
- dst_ssm.copy_from_array(&src_ssm);
- }
- },
- _ => panic!(
- "Layer type mismatch when reconfiguring generator cache"
- ),
- }
- }
- drop(generator_state);
-
- generator.tokens = run_context.tokens.clone();
- }
- Ok(())
- }
-
- fn build_context_from_generator(&self) -> Result<Context, Error> {
- let generator =
- self.generator.as_ref().ok_or(Error::GeneratorNotLoaded)?;
- let cache_layers = generator
- .context
- .cache_layers
- .borrow()
- .clone(&generator.context.mtl_context);
- let context = Context::new(
- generator.tokens.clone(),
- cache_layers,
- generator.decoding_config.clone(),
- );
- Ok(context)
- }
-}
-
-impl Session {
- fn decode_generated_tokens(
- tokenizer: &Tokenizer,
- generator: &Generator,
- run_context: &RunContext,
- ) -> Result<String, Error> {
- let start_idx =
- run_context.prefix_len_before + run_context.input_tokens_len;
- let generated_tokens: Vec<u32> =
- generator.tokens[start_idx..].iter().map(|&v| v as u32).collect();
- tokenizer
- .decode(&generated_tokens, true)
- .map_err(|_| Error::UnableToDecodeText)
- }
-
- fn check_finish_reason(
- run_context: &RunContext,
- generator: &Generator,
- new_tokens: &[u64],
- ) -> Option<FinishReason> {
- let start_idx =
- run_context.prefix_len_before + run_context.input_tokens_len;
- let total_new_tokens = generator.tokens[start_idx..].len();
- let has_eos =
- new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));
- let context_limit =
- generator.tokens.len() >= run_context.context_length;
-
- if has_eos {
- Some(FinishReason::Stop)
- } else if total_new_tokens >= run_context.tokens_limit {
- Some(FinishReason::Length)
- } else if context_limit {
- Some(FinishReason::ContextLimitReached)
- } else {
- None
- }
- }
-
- fn build_output(
- model_metadata: &ModelMetadata,
- tokenizer: &Tokenizer,
- output_parser: &OutputParser,
- run_context: &RunContext,
- generator: &Generator,
- generate_results: &[GenerateResult],
- generate_durations: &[f64],
- finish_reason: Option<FinishReason>,
- ) -> Result<Output, Error> {
- let text =
- Self::decode_generated_tokens(tokenizer, generator, run_context)?;
- let parsed = output_parser.parse(text);
- let start_idx =
- run_context.prefix_len_before + run_context.input_tokens_len;
- let output_tokens = generator.tokens[start_idx..].len();
-
- Ok(Output {
- text: parsed,
- stats: Self::build_stats(
- model_metadata,
- run_context.prefill_result.clone(),
- run_context.prefill_duration,
- run_context.prefill_suffix_length,
- generate_results.to_vec(),
- generate_durations.to_vec(),
- generator.decoding_config.generate_suffix_length(),
- run_context.run_start.elapsed().as_secs_f64(),
- run_context.input_tokens_len,
- output_tokens,
- ),
- finish_reason,
- })
- }
-
- fn run_async_batch<F>(
- model_metadata: &ModelMetadata,
- tokenizer: &Tokenizer,
- output_parser: &OutputParser,
- run_context: &RunContext,
- generator: &mut Generator,
- sampling_method: SamplingMethod,
- progress: &Option<F>,
- batch_size: usize,
- ) -> Result<(Vec<GenerateResult>, Vec<f64>, FinishReason), Error>
- where
- F: Fn(Output) -> bool,
- {
- let remaining_by_limit = run_context
- .tokens_limit
- .saturating_sub(run_context.prefill_result.tokens.len());
- let remaining_by_context =
- run_context.context_length.saturating_sub(generator.tokens.len());
- let tokens_to_generate = remaining_by_limit.min(remaining_by_context);
-
- generator.prepare_async(tokens_to_generate);
-
- let (sender, receiver) = mpsc::channel::<(usize, u64, u64)>();
- let start_time = Instant::now();
- let last_nanos = Arc::new(AtomicU64::new(0));
-
- let mut results: Vec<GenerateResult> =
- Vec::with_capacity(tokens_to_generate);
- let mut durations: Vec<f64> = Vec::with_capacity(tokens_to_generate);
- let mut finish_reason = FinishReason::Length;
- let mut next_to_submit = 0;
- let mut in_flight = 0;
-
- while in_flight > 0 || next_to_submit < tokens_to_generate {
- let batch_end =
- std::cmp::min(next_to_submit + batch_size, tokens_to_generate);
- for idx in next_to_submit..batch_end {
- let batch_sender = sender.clone();
- let last_callback_nanos = last_nanos.clone();
- let batch_start_time = start_time;
- generator.async_generate(
- idx,
- sampling_method,
- move |token| {
- let now_nanos =
- batch_start_time.elapsed().as_nanos() as u64;
- let prev_nanos = last_callback_nanos
- .swap(now_nanos, Ordering::SeqCst);
- let _ = batch_sender.send((
- idx,
- token,
- now_nanos.saturating_sub(prev_nanos),
- ));
- },
- )?;
- in_flight += 1;
- }
- let batch_submitted = batch_end - next_to_submit;
- next_to_submit = batch_end;
-
- for _ in 0..batch_submitted {
- let (_, token, duration_nanos) =
- receiver.recv().map_err(|_| Error::SamplingFailed)?;
- in_flight -= 1;
-
- let duration = nanos_to_secs(duration_nanos);
- generator.tokens.push(token);
- results.push(GenerateResult {
- tokens: vec![token],
- forwardpass_duration: duration,
- });
- durations.push(duration);
-
- let should_stop = if run_context.eos_tokens.contains(&token) {
- finish_reason = FinishReason::Stop;
- true
- } else if generator.tokens.len() >= run_context.context_length {
- finish_reason = FinishReason::ContextLimitReached;
- true
- } else if let Some(progress_fn) = progress {
- let output = Self::build_output(
- model_metadata,
- tokenizer,
- output_parser,
- run_context,
- generator,
- &results,
- &durations,
- None,
- )?;
- if !progress_fn(output) {
- finish_reason = FinishReason::Cancelled;
- true
- } else {
- false
- }
- } else {
- false
- };
-
- if should_stop {
- while in_flight > 0 {
- let _ = receiver.recv();
- in_flight -= 1;
- }
- return Ok((results, durations, finish_reason));
- }
- }
- }
-
- Ok((results, durations, finish_reason))
- }
-
- fn run_sync_generate<F>(
- model_metadata: &ModelMetadata,
- tokenizer: &Tokenizer,
- output_parser: &OutputParser,
- run_context: &RunContext,
- generator: &mut Generator,
- mut compiled_grammar: Option<&mut CompiledGrammar>,
- sampling_method: SamplingMethod,
- progress: &Option<F>,
- ) -> Result<Output, Error>
- where
- F: Fn(Output) -> bool,
- {
- let mut results: Vec<GenerateResult> = Vec::new();
- let mut durations: Vec<f64> = Vec::new();
-
- loop {
- let start = Instant::now();
- let result = generator
- .generate(compiled_grammar.as_deref_mut(), sampling_method)?;
- let new_tokens = result.tokens.clone();
- let duration = start.elapsed().as_secs_f64();
-
- results.push(result);
- durations.push(duration);
-
- let finish_reason =
- Self::check_finish_reason(run_context, generator, &new_tokens);
- let output = Self::build_output(
- model_metadata,
- tokenizer,
- output_parser,
- run_context,
- generator,
- &results,
- &durations,
- finish_reason.clone(),
- )?;
-
- let should_continue = if let Some(progress_fn) = progress {
- progress_fn(output.clone())
- } else {
- true
- };
-
- if !should_continue || finish_reason.is_some() {
- if should_continue {
- return Ok(output);
- } else {
- return Ok(output.clone_with_finish_reason(Some(
- FinishReason::Cancelled,
- )));
- }
- }
- }
- }
-}
-
-impl Session {
- fn build_stats(
- model_metadata: &ModelMetadata,
- prefill_result: PrefillResult,
- prefill_duration: f64,
- prefill_suffix_length: usize,
- generate_results: Vec<GenerateResult>,
- generate_durations: Vec<f64>,
- generate_suffix_length: usize,
- total_duration: f64,
- tokens_count_input: usize,
- tokens_count_output: usize,
- ) -> Stats {
- let prefill_stats = {
- let tokens_count = prefill_result.tokens.len();
- let tokens_per_second = tokens_count as f64 / prefill_duration;
-
- // TODO: Unify after removing MPS blocks
- let processed_tokens = if model_metadata.quantization.is_some() {
- let result = tokens_count_input + tokens_count;
- result
- } else {
- let number_of_prefill_steps =
- (tokens_count_input as f32 / prefill_suffix_length as f32)
- .ceil() as usize;
- let result = prefill_suffix_length * number_of_prefill_steps
- + tokens_count;
- result
- };
- let processed_tokens_per_second =
- processed_tokens as f64 / prefill_duration;
-
- let model_run_count = prefill_result.forwardpass_durations.len();
- let model_run_average_duration =
- prefill_result.forwardpass_durations.iter().sum::<f64>()
- / model_run_count as f64;
-
- StepStats {
- duration: prefill_duration,
- suffix_length: prefill_suffix_length as u64,
- tokens_count: tokens_count as u64,
- tokens_per_second,
- processed_tokens_per_second,
- model_run: RunStats {
- count: model_run_count as u64,
- average_duration: model_run_average_duration,
- },
- run: None,
- }
- };
-
- let generate_stats: Option<StepStats>;
- if generate_results.len() != 0 {
- let duration = generate_durations.iter().sum::<f64>();
- let tokens_count = generate_results
- .iter()
- .flat_map(|result| result.tokens.clone())
- .collect::<Vec<u64>>()
- .len();
- let tokens_per_second: f64 = tokens_count as f64 / duration;
-
- let model_run_count = generate_results.len();
- let model_run_average_duration = generate_results
- .iter()
- .map(|result| result.forwardpass_duration)
- .sum::<f64>()
- / model_run_count as f64;
-
- let run_count = generate_durations.len();
- let run_average_duration =
- generate_durations.iter().sum::<f64>() / run_count as f64;
-
- generate_stats = Some(StepStats {
- duration,
- suffix_length: generate_suffix_length as u64,
- tokens_count: tokens_count as u64,
- tokens_per_second,
- processed_tokens_per_second: tokens_per_second,
- model_run: RunStats {
- count: model_run_count as u64,
- average_duration: model_run_average_duration,
- },
- run: Some(RunStats {
- count: run_count as u64,
- average_duration: run_average_duration,
- }),
- });
- } else {
- generate_stats = None;
- }
-
- let total_stats = TotalStats {
- duration: total_duration,
- tokens_count_input: tokens_count_input as u64,
- tokens_count_output: tokens_count_output as u64,
- };
-
- let stats = Stats {
- prefill_stats,
- generate_stats,
- total_stats,
- };
-
- stats
- }
-}
-
-impl Drop for Session {
- fn drop(&mut self) {
- autoreleasepool(|_| {
- self.generator = None;
- });
- }
-}
diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rs
index e39bc9f6f..c75b0a14d 100644
--- a/crates/uzu/src/session/types/error.rs
+++ b/crates/uzu/src/session/types/error.rs
@@ -1,3 +1,5 @@
+use crate::backends::metal::error::ClassifierError;
+
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("Model folder not found")]
@@ -16,8 +18,8 @@ pub enum Error {
UnsupportedContextModeForModel,
#[error("Unsupported speculator config for model")]
UnsupportedSpeculatorConfigForModel,
- #[error("Generator not loaded")]
- GeneratorNotLoaded,
+ #[error("Language model generator not loaded")]
+ LanguageModelGeneratorNotLoaded,
#[error("Unable to load prompt template")]
UnableToLoadPromptTemplate,
#[error("Unable to render prompt template")]
@@ -40,4 +42,6 @@ pub enum Error {
SamplingFailed,
#[error("Grammar error")]
GrammarError,
+ #[error("Classifier error: {0}")]
+ Classifier(#[from] ClassifierError),
}
diff --git a/crates/uzu/src/tracer/mod.rs b/crates/uzu/src/tracer/mod.rs
index dba1b57e3..43349c40d 100644
--- a/crates/uzu/src/tracer/mod.rs
+++ b/crates/uzu/src/tracer/mod.rs
@@ -1,2 +1,12 @@
-pub mod tracer;
-pub use tracer::Tracer;
+//! Unified trace validation for all model types.
+//!
+//! This module provides `TraceValidator` which can validate activation traces
+//! for any model type (LLM or classifier). The model type is automatically
+//! detected from the config.
+
+mod trace_validator;
+
+pub use trace_validator::{
+ ArrayTransform, TraceValidator, TracerValidationMetrics,
+ TracerValidationResult, TracerValidationResults,
+};
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
new file mode 100644
index 000000000..27d689ee2
--- /dev/null
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -0,0 +1,1055 @@
+//! Unified trace validation for any model type (LLM or classifier).
+//!
+//! This module provides a single `TraceValidator` that can validate activation
+//! traces for any model type. It automatically detects the model type from the
+//! config and runs the appropriate validation.
+
+use std::{
+ cell::{Ref, RefCell},
+ fs::File,
+ path::{Path, PathBuf},
+ rc::Rc,
+};
+
+use half::{bf16, f16};
+use ndarray::{IxDyn, s};
+use num_traits::NumCast;
+
+use crate::{
+ Array, ArrayElement, DataType,
+ backends::metal::{
+ CacheLayers, KVCacheUpdate, KernelDataType, MTLContext, MetalArray,
+ encodable_block::Sampling,
+ forward_pass::{
+ ArrayId, EncodableBlock, EncodingParameters, ForwardPassState,
+ ScratchBuffers, traces::ActivationTrace,
+ },
+ },
+ classifier::Classifier,
+ config::ModelMetadata,
+ language_model::{
+ LanguageModelGeneratorContext,
+ sampler::{ArgmaxSampler, LogitsSampler},
+ },
+ parameters::{ParameterLoader, ParameterTree, read_safetensors_metadata},
+ session::{
+ config::{DecodingConfig, SpeculatorConfig},
+ parameter::{
+ AsyncBatchSize, ConfigResolvableValue, ContextLength, ContextMode,
+ PrefillStepSize, SamplingSeed,
+ },
+ types::Error,
+ },
+};
+
+// ============================================================================
+// Validation Types
+// ============================================================================
+
+/// Metrics from validating a single tensor.
+pub struct TracerValidationMetrics {
+ pub atol: f32,
+ pub rtol: f32,
+ pub fraction_of_allowed_violations: f32,
+ pub reference_shape: Vec<usize>,
+ pub result_shape: Vec<usize>,
+ pub num_violations: usize,
+ pub max_allowed_violations: usize,
+ pub max_err_idx: usize,
+ pub max_err: f32,
+ pub max_err_rel: f32,
+ pub max_err_reference_value: f32,
+ pub rms_diff: f32,
+ pub rms_result: f32,
+ pub rms_reference: f32,
+ pub rel_rms_reference: f32,
+ pub diff_max: f32,
+ pub diff_avg: f32,
+ pub result_nan: bool,
+}
+
+impl TracerValidationMetrics {
+ pub fn is_valid(&self) -> bool {
+ self.num_violations <= self.max_allowed_violations && !self.result_nan
+ }
+
+ pub fn message(&self) -> String {
+ if self.result_nan {
+ return "Result contains NaN values".to_string();
+ }
+
+ let allowed_violation_explainer = if self.max_allowed_violations > 0 {
+ format!(" (Max {} allowed)", self.max_allowed_violations)
+ } else {
+ String::new()
+ };
+
+ let reference_size: usize = self.reference_shape.iter().product();
+
+ format!(
+ "{} violations > {:.1e} + {:.2}% out of total {} elements{}.\n\
+ Worst violation: {:.3} ({:.2}%) at index {} (reference value: {:.3}).\n\
+ Error RMS: {:.3}.\n\
+ RMS of result: {:.3}, RMS of reference: {:.3}.\n\
+ Relative error RMS: {:.2}% of RMS of reference.\n\
+ Shape: {:?}\n\
+ Max diff: {:.3}, Avg diff: {:.3}",
+ self.num_violations,
+ self.atol,
+ self.rtol * 100.0,
+ reference_size,
+ allowed_violation_explainer,
+ self.max_err,
+ self.max_err_rel * 100.0,
+ self.max_err_idx,
+ self.max_err_reference_value,
+ self.rms_diff,
+ self.rms_result,
+ self.rms_reference,
+ self.rel_rms_reference * 100.0,
+ self.result_shape,
+ self.diff_max,
+ self.diff_avg,
+ )
+ }
+}
+
+/// Result of validating a single tensor.
+pub struct TracerValidationResult {
+ pub name: String,
+ pub metrics: TracerValidationMetrics,
+}
+
+/// Results from validating all traces.
+pub struct TracerValidationResults {
+ pub suffix_length: usize,
+ pub results: Vec<TracerValidationResult>,
+ pub tokens_violation_indices: Vec<usize>,
+}
+
+impl TracerValidationResults {
+ pub fn number_of_tokens_violations(&self) -> usize {
+ self.tokens_violation_indices.len()
+ }
+
+ pub fn number_of_allowed_tokens_violations(&self) -> usize {
+ let threshold: f64 = 0.01;
+ (self.suffix_length as f64 * threshold).ceil() as usize
+ }
+
+ pub fn is_valid(&self) -> bool {
+ self.number_of_tokens_violations()
+ <= self.number_of_allowed_tokens_violations()
+ }
+}
+
+/// Transform to apply to produced arrays before comparison.
+pub enum ArrayTransform {
+ /// Slice KV cache to match expected shape.
+ KVCacheSlice,
+ /// Transform SSM conv state layout.
+ SsmConvState,
+}
+
+// ============================================================================
+// Model Context (internal)
+// ============================================================================
+
+enum ModelContext {
+ LanguageModelGenerator(LanguageModelGeneratorContext),
+ Classifier(Classifier),
+}
+
+// ============================================================================
+// Unified TraceValidator
+// ============================================================================
+
+/// Unified trace validator for any model type.
+///
+/// Automatically detects whether the model is an LLM or classifier and
+/// runs the appropriate validation.
+pub struct TraceValidator {
+ model_path: PathBuf,
+ context: ModelContext,
+}
+
+impl TraceValidator {
+ /// Create a new trace validator for the given model path.
+ ///
+ /// Automatically detects the model type from config.json.
+ pub fn new(model_path: &Path) -> Result<Self, Error> {
+ let config_path = model_path.join("config.json");
+ if !config_path.exists() {
+ return Err(Error::ModelFolderNotFound);
+ }
+
+ let config_file =
+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
+ let metadata: ModelMetadata =
+ serde_json::from_reader(std::io::BufReader::new(config_file))
+ .map_err(|_| Error::UnableToLoadConfig)?;
+
+ let context = if metadata.model_config.is_classifier() {
+ let classifier = Classifier::new(model_path)?;
+ ModelContext::Classifier(classifier)
+ } else {
+ let prefill_step_size =
+ Self::determine_prefill_step_size(model_path);
+ let decoding_config = DecodingConfig::new(
+ ContextMode::default(),
+ ContextLength::default(),
+ PrefillStepSize::Custom(prefill_step_size),
+ SpeculatorConfig::default(),
+ SamplingSeed::default(),
+ AsyncBatchSize::default(),
+ false,
+ );
+ let mut llm_context = LanguageModelGeneratorContext::new(
+ model_path,
+ &decoding_config,
+ )?;
+ let desired_suffix_length =
+ prefill_step_size.max(decoding_config.generate_suffix_length());
+ Self::ensure_llm_context_capacity(
+ &decoding_config,
+ desired_suffix_length,
+ &mut llm_context,
+ );
+ ModelContext::LanguageModelGenerator(llm_context)
+ };
+
+ Ok(Self {
+ model_path: model_path.to_path_buf(),
+ context,
+ })
+ }
+
+ /// Run trace validation and return results.
+ pub fn run(&mut self) -> Result<TracerValidationResults, Error> {
+ let traces_path = self.model_path.join("traces.safetensors");
+ if !traces_path.exists() {
+ return Err(Error::UnableToLoadWeights);
+ }
+
+ match &mut self.context {
+ ModelContext::LanguageModelGenerator(ctx) => {
+ Self::run_llm_validation(ctx, &traces_path)
+ },
+ ModelContext::Classifier(classifier) => {
+ Self::run_classifier_validation(classifier, &traces_path)
+ },
+ }
+ }
+
+ pub fn is_classifier(&self) -> bool {
+ matches!(self.context, ModelContext::Classifier(_))
+ }
+
+ pub fn is_language_model_generator(&self) -> bool {
+ matches!(self.context, ModelContext::LanguageModelGenerator(_))
+ }
+
+ // ========================================================================
+ // LLM Validation
+ // ========================================================================
+
+ fn run_llm_validation(
+ ctx: &LanguageModelGeneratorContext,
+ traces_path: &Path,
+ ) -> Result<TracerValidationResults, Error> {
+ let traces_file =
+ File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let traces_loader =
+ ParameterLoader::new(&traces_file, &ctx.mtl_context)
+ .map_err(|_| Error::UnableToLoadWeights)?;
+ let traces_view = traces_loader.tree();
+
+ let token_ids = Self::load_array_as_vec::<i32, u64>(
+ &traces_view,
+ "activation_trace.token_ids",
+ );
+ let token_positions = Self::load_array_as_vec::<i32, usize>(
+ &traces_view,
+ "activation_trace.token_positions",
+ );
+ let token_seeds: Vec<u64> = vec![0; token_ids.len()];
+
+ let mut state = ForwardPassState::new_llm(
+ ctx.mtl_context.clone(),
+ &ctx.decoder_config,
+ &ctx.model_shape,
+ &ctx.scratch_buffers,
+ ctx.cache_layers.clone(),
+ ctx.shared_buffers.clone(),
+ &token_ids,
+ &token_positions,
+ None,
+ &token_seeds,
+ token_ids.len(),
+ false,
+ None,
+ false,
+ None,
+ None,
+ );
+
+ let root_command_buffer =
+ ctx.command_buffer.root_command_buffer().to_owned();
+ ctx.executables.encode(
+ &mut state,
+ &ctx.command_buffer,
+ &EncodingParameters::new(false, false, true),
+ );
+ ctx.command_buffer.commit();
+ root_command_buffer.wait_until_completed();
+
+ let traces = state.traces().clone();
+ let data_type = ctx.model_shape.activation_data_type();
+
+ // Common layer validation
+ let mut results =
+ Self::validate_layer_traces(&traces, &traces_view, data_type);
+
+ // LLM-specific: KV cache validation
+ let transformer_layers: Vec<usize> = {
+ let cache = state.cache_layers().unwrap().borrow();
+ cache
+ .data
+ .iter()
+ .enumerate()
+ .filter_map(|(index, layer)| {
+ layer.as_transformer().map(|_| index)
+ })
+ .collect()
+ };
+
+ for index in transformer_layers {
+ let arrays =
+ state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);
+
+ if let Ok(expected) =
+ traces_view.leaf(&format!("updated_kv_cache.{}.keys", index))
+ {
+ let keys = arrays[0].borrow();
+ results.push(TracerValidationResult {
+ name: format!("updated_kv_cache.{}.keys", index),
+ metrics: Self::validate_array(
+ data_type,
+ &expected,
+ &keys,
+ Some(ArrayTransform::KVCacheSlice),
+ ),
+ });
+ }
+
+ if let Ok(expected) =
+ traces_view.leaf(&format!("updated_kv_cache.{}.values", index))
+ {
+ let values = arrays[1].borrow();
+ results.push(TracerValidationResult {
+ name: format!("updated_kv_cache.{}.values", index),
+ metrics: Self::validate_array(
+ data_type,
+ &expected,
+ &values,
+ Some(ArrayTransform::KVCacheSlice),
+ ),
+ });
+ }
+ }
+
+ // LLM-specific: SSM state validation
+ let ssm_layers: Vec<usize> = {
+ let cache = state.cache_layers().unwrap().borrow();
+ cache
+ .data
+ .iter()
+ .enumerate()
+ .filter_map(|(index, layer)| {
+ layer.as_state_space().map(|_| index)
+ })
+ .collect()
+ };
+
+ for index in ssm_layers {
+ let arrays = state.arrays(&[
+ ArrayId::SsmConvState(index),
+ ArrayId::SsmState(index),
+ ]);
+ let conv_state = arrays[0].borrow();
+ let ssm_state = arrays[1].borrow();
+
+ for path in [
+ format!("updated_state.{}.conv_state", index),
+ format!(
+ "activation_trace.layer_results.{}.updated_state.conv_state",
+ index
+ ),
+ ] {
+ if let Ok(expected) = traces_view.leaf(&path) {
+ results.push(TracerValidationResult {
+ name: path,
+ metrics: Self::validate_array(
+ data_type,
+ &expected,
+ &conv_state,
+ Some(ArrayTransform::SsmConvState),
+ ),
+ });
+ }
+ }
+
+ for path in [
+ format!("updated_state.{}.ssm_state", index),
+ format!(
+ "activation_trace.layer_results.{}.updated_state.ssm_state",
+ index
+ ),
+ ] {
+ if let Ok(expected) = traces_view.leaf(&path) {
+ results.push(TracerValidationResult {
+ name: path,
+ metrics: Self::validate_array(
+ data_type, &expected, &ssm_state, None,
+ ),
+ });
+ }
+ }
+ }
+
+ // LLM-specific: Token comparison
+ let tokens_violation_indices = if let Ok(expected_logits) =
+ traces_view.leaf("logits")
+ {
+ let expected_tokens =
+ Self::get_tokens_from_logits(&expected_logits);
+ let produced_tokens =
+ Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());
+ expected_tokens
+ .iter()
+ .zip(produced_tokens.iter())
+ .enumerate()
+ .filter_map(|(i, (a, b))| {
+ if a != b {
+ Some(i)
+ } else {
+ None
+ }
+ })
+ .collect()
+ } else {
+ Vec::new()
+ };
+
+ Ok(TracerValidationResults {
+ suffix_length: token_ids.len(),
+ results,
+ tokens_violation_indices,
+ })
+ }
+
+ // ========================================================================
+ // Classifier Validation
+ // ========================================================================
+
+ fn run_classifier_validation(
+ classifier: &mut Classifier,
+ traces_path: &Path,
+ ) -> Result<TracerValidationResults, Error> {
+ let traces_file =
+ File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let mtl_context = classifier.context.mtl_context.clone();
+ let traces_loader = ParameterLoader::new(&traces_file, &mtl_context)
+ .map_err(|_| Error::UnableToLoadWeights)?;
+ let traces_view = traces_loader.tree();
+
+ let has_token_ids =
+ traces_view.leaf("activation_trace.token_ids").is_ok();
+ let has_token_positions =
+ traces_view.leaf("activation_trace.token_positions").is_ok();
+
+ if !has_token_ids || !has_token_positions {
+ return Ok(Self::handle_missing_tokens(&traces_view));
+ }
+
+ let token_ids = Self::load_array_as_vec::<i32, u64>(
+ &traces_view,
+ "activation_trace.token_ids",
+ );
+ let token_positions = Self::load_array_as_vec::<i32, usize>(
+ &traces_view,
+ "activation_trace.token_positions",
+ );
+
+ let suffix_length = token_ids.len();
+
+ let (_logits, traces) = classifier
+ .forward_pass_with_traces(&token_ids, &token_positions)
+ .map_err(|_| Error::GenerateFailed)?;
+
+ let data_type = classifier.context.model_shape.activation_data_type();
+
+ // Common layer validation
+ let mut results =
+ Self::validate_layer_traces(&traces, &traces_view, data_type);
+
+ // Classifier-specific: embedding_norm, output_pooling
+ let classifier_results =
+ Self::validate_classifier_traces(&traces, &traces_view, data_type);
+ results.extend(classifier_results);
+
+ Ok(TracerValidationResults {
+ suffix_length,
+ results,
+ tokens_violation_indices: Vec::new(), // Classifiers don't compare tokens
+ })
+ }
+
+ fn handle_missing_tokens(
+ traces_view: &ParameterTree<Rc<MTLContext>>
+ ) -> TracerValidationResults {
+ if let Ok(expected_logits) = traces_view.leaf("logits") {
+ let reference_shape = expected_logits.shape().to_vec();
+ let metrics = TracerValidationMetrics {
+ atol: 0.0,
+ rtol: 0.0,
+ fraction_of_allowed_violations: 0.0,
+ reference_shape: reference_shape.clone(),
+ result_shape: reference_shape,
+ num_violations: 0,
+ max_allowed_violations: 0,
+ max_err_idx: 0,
+ max_err: 0.0,
+ max_err_rel: 0.0,
+ max_err_reference_value: 0.0,
+ rms_diff: 0.0,
+ rms_result: 0.0,
+ rms_reference: 0.0,
+ rel_rms_reference: 0.0,
+ diff_max: 0.0,
+ diff_avg: 0.0,
+ result_nan: false,
+ };
+ return TracerValidationResults {
+ suffix_length: 1,
+ results: vec![TracerValidationResult {
+ name: "activation_trace.logits".to_string(),
+ metrics,
+ }],
+ tokens_violation_indices: Vec::new(),
+ };
+ }
+
+ TracerValidationResults {
+ suffix_length: 1,
+ results: Vec::new(),
+ tokens_violation_indices: Vec::new(),
+ }
+ }
+
+ // ========================================================================
+ // Common Validation Helpers
+ // ========================================================================
+
+ fn validate_layer_traces(
+ traces: &Rc<RefCell<ActivationTrace>>,
+ traces_view: &ParameterTree<Rc<MTLContext>>,
+ data_type: DataType,
+ ) -> Vec<TracerValidationResult> {
+ let mut results = Vec::new();
+
+ let validate = |path: &str,
+ array: &Ref<MetalArray>|
+ -> Option<TracerValidationResult> {
+ if traces_view.leaf(path).is_ok() {
+ Some(TracerValidationResult {
+ name: path.to_string(),
+ metrics: Self::validate_array_with_name(
+ data_type,
+ traces_view,
+ path,
+ array,
+ ),
+ })
+ } else {
+ None
+ }
+ };
+
+ for (index, layer_traces) in
+ traces.borrow().layer_results.iter().enumerate()
+ {
+ let path = |suffix: &str| -> String {
+ format!(
+ "activation_trace.layer_results.{}.activation_trace.{}",
+ index, suffix
+ )
+ };
+
+ if let Some(r) = validate(
+ &path("inputs"),
+ &layer_traces.borrow().inputs.borrow(),
+ ) {
+ results.push(r);
+ }
+ if let Some(r) = validate(
+ &path("pre_mixer_norm"),
+ &layer_traces.borrow().pre_attention_norm.borrow(),
+ ) {
+ results.push(r);
+ }
+ if let Some(r) = validate(
+ &path("mixer"),
+ &layer_traces.borrow().attention.borrow(),
+ ) {
+ results.push(r);
+ }
+ if let Some(r) = validate(
+ &path("post_mixer_norm"),
+ &layer_traces.borrow().post_attention_norm.borrow(),
+ ) {
+ results.push(r);
+ }
+ if let Some(r) = validate(
+ &path("mlp_inputs"),
+ &layer_traces.borrow().mlp_inputs.borrow(),
+ ) {
+ results.push(r);
+ }
+ if let Some(r) = validate(
+ &path("pre_mlp_norm"),
+ &layer_traces.borrow().pre_mlp_norm.borrow(),
+ ) {
+ results.push(r);
+ }
+ if let Some(r) =
+ validate(&path("mlp"), &layer_traces.borrow().mlp.borrow())
+ {
+ results.push(r);
+ }
+ if let Some(r) = validate(
+ &path("post_mlp_norm"),
+ &layer_traces.borrow().post_mlp_norm.borrow(),
+ ) {
+ results.push(r);
+ }
+
+ let outputs_path =
+ format!("activation_trace.layer_results.{}.outputs", index);
+ if let Some(r) =
+ validate(&outputs_path, &layer_traces.borrow().outputs.borrow())
+ {
+ results.push(r);
+ }
+ }
+
+ // Output norm (common to all models)
+ if let Some(r) = validate(
+ "activation_trace.output_norm",
+ &traces.borrow().output_norm.borrow(),
+ ) {
+ results.push(r);
+ }
+
+ // Logits (common to all models, but path may vary)
+ if let Some(r) = validate(
+ "activation_trace.logits",
+ &traces.borrow().logits.borrow(),
+ ) {
+ results.push(r);
+ } else if let Some(r) =
+ validate("logits", &traces.borrow().logits.borrow())
+ {
+ results.push(r);
+ }
+
+ results
+ }
+
+ fn validate_classifier_traces(
+ traces: &Rc<RefCell<ActivationTrace>>,
+ traces_view: &ParameterTree<Rc<MTLContext>>,
+ data_type: DataType,
+ ) -> Vec<TracerValidationResult> {
+ let mut results = Vec::new();
+
+ // Embedding norm (classifier-specific)
+ if let Some(embedding_norm) = &traces.borrow().embedding_norm {
+ if traces_view.leaf("activation_trace.embedding_norm").is_ok() {
+ results.push(TracerValidationResult {
+ name: "activation_trace.embedding_norm".to_string(),
+ metrics: Self::validate_array_with_name(
+ data_type,
+ traces_view,
+ "activation_trace.embedding_norm",
+ &embedding_norm.borrow(),
+ ),
+ });
+ }
+ }
+
+ // Output pooling (classifier-specific)
+ if let Some(output_pooling) = &traces.borrow().output_pooling {
+ if traces_view.leaf("activation_trace.output_pooling").is_ok() {
+ results.push(TracerValidationResult {
+ name: "activation_trace.output_pooling".to_string(),
+ metrics: Self::validate_array_with_name(
+ data_type,
+ traces_view,
+ "activation_trace.output_pooling",
+ &output_pooling.borrow(),
+ ),
+ });
+ }
+ }
+
+ results
+ }
+
+ // ========================================================================
+ // Array Validation
+ // ========================================================================
+
+ fn validate_array(
+ data_type: DataType,
+ expected_array: &MetalArray,
+ produced_array: &Ref<MetalArray>,
+ transform: Option<ArrayTransform>,
+ ) -> TracerValidationMetrics {
+ match data_type {
+ DataType::F16 => Self::validate_array_of_type::<f16>(
+ expected_array,
+ produced_array,
+ transform,
+ ),
+ DataType::BF16 => Self::validate_array_of_type::<bf16>(
+ expected_array,
+ produced_array,
+ transform,
+ ),
+ DataType::F32 => Self::validate_array_of_type::<f32>(
+ expected_array,
+ produced_array,
+ transform,
+ ),
+ _ => panic!("Unsupported data type: {:?}", data_type),
+ }
+ }
+
+ fn validate_array_with_name(
+ data_type: DataType,
+ traces_view: &ParameterTree<Rc<MTLContext>>,
+ expected_array_path: &str,
+ produced_array: &Ref<MetalArray>,
+ ) -> TracerValidationMetrics {
+ let expected_array = traces_view.leaf(expected_array_path).unwrap();
+ Self::validate_array(data_type, &expected_array, produced_array, None)
+ }
+
+ fn validate_array_of_type<Precision: ArrayElement>(
+ expected_array: &MetalArray,
+ produced_array: &Ref<MetalArray>,
+ transform: Option<ArrayTransform>,
+ ) -> TracerValidationMetrics {
+ let expected_view = expected_array.as_view::<Precision>().unwrap();
+ let produced_view = produced_array.as_view::<Precision>().unwrap();
+
+ let (mut expected_data, mut produced_data) = match transform {
+ Some(ArrayTransform::KVCacheSlice) => {
+ let permuted = produced_view.permuted_axes(IxDyn(&[1, 0, 2]));
+ let total_tokens = permuted.shape()[0];
+ let expected_tokens = expected_view.shape()[1];
+ let start = total_tokens.saturating_sub(expected_tokens);
+ let sliced = permuted.slice(s![start.., .., ..]);
+ let reshaped = sliced
+ .into_owned()
+ .to_shape(IxDyn(&[
+ 1,
+ expected_tokens,
+ permuted.shape()[1],
+ permuted.shape()[2],
+ ]))
+ .expect("Failed to reshape KV cache slice")
+ .to_owned();
+ (expected_view.to_owned(), reshaped)
+ },
+ Some(ArrayTransform::SsmConvState) => {
+ let produced_shape = produced_view.shape();
+ let history_len = produced_shape[1];
+ let dim = produced_shape[0];
+
+ let permuted = expected_view.permuted_axes(IxDyn(&[0, 2, 1]));
+ let total_time = permuted.shape()[2];
+ let start = total_time.saturating_sub(history_len);
+ let sliced = permuted.slice(s![.., .., start..]);
+
+ let reshaped_expected = sliced
+ .into_owned()
+ .to_shape(IxDyn(&[dim, history_len]))
+ .expect("Failed to reshape SSM conv state slice")
+ .to_owned();
+
+ (reshaped_expected, produced_view.to_owned())
+ },
+ None => (expected_view.to_owned(), produced_view.to_owned()),
+ };
+
+ let expected_shape = expected_data.shape().to_vec();
+ let produced_shape = produced_data.shape().to_vec();
+
+ // Handle shape mismatches with leading dimension of 1
+ if expected_shape != produced_shape {
+ if expected_shape.len() == produced_shape.len() + 1
+ && expected_shape.get(0) == Some(&1)
+ && expected_shape[1..] == produced_shape[..]
+ {
+ expected_data = expected_data
+ .to_shape(IxDyn(&produced_shape))
+ .expect("Failed to reshape expected data")
+ .to_owned();
+ } else if produced_shape.len() == expected_shape.len() + 1
+ && produced_shape.get(0) == Some(&1)
+ && produced_shape[1..] == expected_shape[..]
+ {
+ produced_data = produced_data
+ .to_shape(IxDyn(&expected_shape))
+ .expect("Failed to reshape produced data")
+ .to_owned();
+ }
+ }
+
+ if expected_data.shape() != produced_data.shape() {
+ panic!(
+ "Shape mismatch after alignment: expected {:?}, produced {:?}",
+ expected_data.shape(),
+ produced_data.shape()
+ );
+ }
+
+ let reference: Vec<f32> = expected_data
+ .iter()
+ .map(|value| NumCast::from(*value).unwrap_or(0.0))
+ .collect();
+
+ let result: Vec<f32> = produced_data
+ .iter()
+ .map(|value| NumCast::from(*value).unwrap_or(0.0))
+ .collect();
+
+ Self::compare_arrays(
+ &reference,
+ expected_data.shape().to_vec(),
+ &result,
+ produced_data.shape().to_vec(),
+ )
+ }
+
+ fn compare_arrays(
+ reference: &[f32],
+ reference_shape: Vec<usize>,
+ result: &[f32],
+ result_shape: Vec<usize>,
+ ) -> TracerValidationMetrics {
+ assert_eq!(result.len(), reference.len());
+
+ let atol: f32 = 1e-2;
+ let rtol: f32 = 0.03;
+ let fraction_of_allowed_violations: f32 = 0.01;
+
+ let mut num_violations = 0;
+ let mut max_err = 0.0f32;
+ let mut max_err_idx = 0;
+ let mut max_err_rel = 0.0f32;
+ let mut max_err_reference_value = 0.0f32;
+ let mut sum_sq_diff = 0.0f32;
+ let mut sum_sq_result = 0.0f32;
+ let mut sum_sq_reference = 0.0f32;
+ let mut diff_sum = 0.0f32;
+ let mut diff_max = 0.0f32;
+ let mut result_nan = false;
+
+ for (i, (&exp, &prod)) in
+ reference.iter().zip(result.iter()).enumerate()
+ {
+ if prod.is_nan() {
+ result_nan = true;
+ }
+
+ let abs_diff = (exp - prod).abs();
+ let rel_diff = if exp.abs() > 1e-8 {
+ abs_diff / exp.abs()
+ } else {
+ abs_diff
+ };
+
+ diff_sum += abs_diff;
+ diff_max = diff_max.max(abs_diff);
+ sum_sq_diff += abs_diff * abs_diff;
+ sum_sq_result += prod * prod;
+ sum_sq_reference += exp * exp;
+
+ if abs_diff > atol && rel_diff > rtol {
+ num_violations += 1;
+ }
+
+ if abs_diff > max_err {
+ max_err = abs_diff;
+ max_err_idx = i;
+ max_err_rel = rel_diff;
+ max_err_reference_value = exp;
+ }
+ }
+
+ let n = reference.len() as f32;
+ let rms_diff = (sum_sq_diff / n).sqrt();
+ let rms_result = (sum_sq_result / n).sqrt();
+ let rms_reference = (sum_sq_reference / n).sqrt();
+ let rel_rms_reference = if rms_reference > 1e-8 {
+ rms_diff / rms_reference
+ } else {
+ rms_diff
+ };
+ let diff_avg = diff_sum / n;
+
+ let max_allowed_violations =
+ (fraction_of_allowed_violations * n).ceil() as usize;
+
+ TracerValidationMetrics {
+ atol,
+ rtol,
+ fraction_of_allowed_violations,
+ reference_shape,
+ result_shape,
+ num_violations,
+ max_allowed_violations,
+ max_err_idx,
+ max_err,
+ max_err_rel,
+ max_err_reference_value,
+ rms_diff,
+ rms_result,
+ rms_reference,
+ rel_rms_reference,
+ diff_max,
+ diff_avg,
+ result_nan,
+ }
+ }
+
+ // ========================================================================
+ // Utility Functions
+ // ========================================================================
+
+ fn load_array_as_vec<
+ SourcePrecision: ArrayElement,
+ TargetPrecision: NumCast,
+ >(
+ traces_view: &ParameterTree<Rc<MTLContext>>,
+ name: &str,
+ ) -> Vec<TargetPrecision> {
+ let array = traces_view.leaf(name).unwrap();
+ let slice = array.as_slice::<SourcePrecision>().unwrap();
+ slice.iter().map(|x| NumCast::from(*x).unwrap()).collect()
+ }
+
+ fn determine_prefill_step_size(model_path: &Path) -> usize {
+ let traces_path = model_path.join("traces.safetensors");
+ if let Ok(file) = File::open(&traces_path) {
+ if let Ok((_header_len, metadata)) =
+ read_safetensors_metadata(&file)
+ {
+ if let Some(tensor) =
+ metadata.tensors.get("activation_trace.token_ids")
+ {
+ if let Some(&length) = tensor.shape.first() {
+ return tensor
+ .shape
+ .iter()
+ .copied()
+ .max()
+ .unwrap_or(length)
+ .max(1);
+ }
+ }
+ }
+ }
+ 1
+ }
+
+ fn ensure_llm_context_capacity(
+ decoding_config: &DecodingConfig,
+ desired_suffix_length: usize,
+ context: &mut LanguageModelGeneratorContext,
+ ) {
+ let resolved_prefix_length =
+ decoding_config.context_length.resolve(&context.model_config);
+ let current_suffix_length = std::cmp::max(
+ decoding_config.prefill_step_size.resolve(&context.model_config),
+ decoding_config.generate_suffix_length(),
+ );
+
+ if desired_suffix_length <= current_suffix_length {
+ return;
+ }
+
+ let decoder_config = &context.decoder_config;
+ context.scratch_buffers = ScratchBuffers::new(
+ &context.mtl_context,
+ decoder_config,
+ &context.model_shape,
+ resolved_prefix_length,
+ desired_suffix_length,
+ );
+
+ context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(
+ &context.mtl_context,
+ &context.model_shape,
+ resolved_prefix_length,
+ desired_suffix_length,
+ )));
+
+ let intermediate_dtype: DataType =
+ decoder_config.output_norm_config.scale_precision.into();
+ let kernel_dtype: KernelDataType = intermediate_dtype.into();
+
+ context.kv_cache_update = Box::new(
+ KVCacheUpdate::new(
+ &context.mtl_context,
+ kernel_dtype,
+ resolved_prefix_length,
+ )
+ .expect("Failed to create KV cache update kernel"),
+ );
+
+ context.gpu_sampler = Sampling::new(
+ &context.mtl_context,
+ kernel_dtype,
+ desired_suffix_length,
+ decoder_config.vocab_size,
+ )
+ .expect("Failed to create sampling kernel");
+ }
+
+ fn get_tokens_from_logits(logits: &MetalArray) -> Vec<u64> {
+ let data_type = logits.data_type();
+ match data_type {
+ DataType::F16 => {
+ Self::get_tokens_from_logits_of_type::<f16>(logits)
+ },
+ DataType::BF16 => {
+ Self::get_tokens_from_logits_of_type::<bf16>(logits)
+ },
+ DataType::F32 => {
+ Self::get_tokens_from_logits_of_type::<f32>(logits)
+ },
+ _ => panic!("Unsupported data type: {:?}", data_type),
+ }
+ }
+
+ fn get_tokens_from_logits_of_type<Precision: ArrayElement>(
+ logits: &MetalArray
+ ) -> Vec<u64> {
+ let sampler = ArgmaxSampler {};
+ sampler.sample(logits.as_view::<Precision>().unwrap())
+ }
+}
diff --git a/crates/uzu/src/tracer/tracer.rs b/crates/uzu/src/tracer/tracer.rs
deleted file mode 100644
index e17fe6103..000000000
--- a/crates/uzu/src/tracer/tracer.rs
+++ /dev/null
@@ -1,864 +0,0 @@
-use std::{
- cell::{Ref, RefCell},
- fs::File,
- path::{Path, PathBuf},
- rc::Rc,
-};
-
-use half::{bf16, f16};
-use ndarray::{IxDyn, s};
-use num_traits::NumCast;
-
-use crate::{
- Array, ArrayElement, DataType,
- backends::metal::{
- CacheLayers, ForwardPassState, KVCacheUpdate, KernelDataType,
- MTLContext, MetalArray,
- forward_pass::{
- ArrayId, ForwardPassBuffers,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- traces::DecoderActivationTrace,
- },
- kernel::SamplingKernelEncodable,
- },
- generator::{
- context::GeneratorContext,
- sampler::{ArgmaxSampler, LogitsSampler},
- },
- parameters::{ParameterLoader, ParameterTree, read_safetensors_metadata},
- session::{
- config::{DecodingConfig, SpeculatorConfig},
- parameter::{
- AsyncBatchSize, ConfigResolvableValue, ContextLength, ContextMode,
- PrefillStepSize, SamplingSeed,
- },
- },
-};
-
-enum Transform {
- KVCacheSlice,
- SsmConvState,
-}
-
-pub struct TracerValidationMetrics {
- pub atol: f32,
- pub rtol: f32,
- pub fraction_of_allowed_violations: f32,
-
- pub reference_shape: Vec<usize>,
- pub result_shape: Vec<usize>,
-
- pub num_violations: usize,
- pub max_allowed_violations: usize,
-
- pub max_err_idx: usize,
- pub max_err: f32,
- pub max_err_rel: f32,
- pub max_err_reference_value: f32,
-
- pub rms_diff: f32,
- pub rms_result: f32,
- pub rms_reference: f32,
- pub rel_rms_reference: f32,
-
- pub diff_max: f32,
- pub diff_avg: f32,
- pub result_nan: bool,
-}
-
-impl TracerValidationMetrics {
- pub fn is_valid(&self) -> bool {
- self.num_violations <= self.max_allowed_violations && !self.result_nan
- }
-
- pub fn message(&self) -> String {
- if self.result_nan {
- return format!("Result contains NaN values");
- }
-
- let allowed_violation_explainer = if self.max_allowed_violations > 0 {
- format!(" (Max {} allowed)", self.max_allowed_violations)
- } else {
- String::new()
- };
-
- let reference_size: usize = self.reference_shape.iter().product();
-
- format!(
- "{} violations > {:.1e} + {:.2}% out of total {} elements{}.\n\
- Worst violation: {:.3} ({:.2}%) at index {} (reference value: {:.3}).\n\
- Error RMS: {:.3}.\n\
- RMS of result: {:.3}, RMS of reference: {:.3}.\n\
- Relative error RMS: {:.2}% of RMS of reference.\n\
- Shape: {:?}\n\
- Max diff: {:.3}, Avg diff: {:.3}",
- self.num_violations,
- self.atol,
- self.rtol * 100.0,
- reference_size,
- allowed_violation_explainer,
- self.max_err,
- self.max_err_rel * 100.0,
- self.max_err_idx,
- self.max_err_reference_value,
- self.rms_diff,
- self.rms_result,
- self.rms_reference,
- self.rel_rms_reference * 100.0,
- self.result_shape,
- self.diff_max,
- self.diff_avg,
- )
- }
-}
-
-pub struct TracerValidationResult {
- pub name: String,
- pub metrics: TracerValidationMetrics,
-}
-
-pub struct TracerValidationResults {
- pub suffix_length: usize,
- pub results: Vec<TracerValidationResult>,
- pub tokens_violation_indices: Vec<usize>,
-}
-
-impl TracerValidationResults {
- pub fn number_of_tokens_violations(&self) -> usize {
- self.tokens_violation_indices.len()
- }
-
- pub fn number_of_allowed_tokens_violations(&self) -> usize {
- let threshold: f64 = 0.01;
- (self.suffix_length as f64 * threshold).ceil() as usize
- }
-
- pub fn is_valid(&self) -> bool {
- self.number_of_tokens_violations()
- <= self.number_of_allowed_tokens_violations()
- }
-}
-
-pub struct Tracer {
- model_path: PathBuf,
- generator_context: GeneratorContext,
-}
-
-impl Tracer {
- pub fn new(model_path: &Path) -> Self {
- let prefill_step_size = Self::determine_prefill_step_size(model_path);
- let decoding_config = DecodingConfig::new(
- ContextMode::default(),
- ContextLength::default(),
- PrefillStepSize::Custom(prefill_step_size),
- SpeculatorConfig::default(),
- SamplingSeed::default(),
- AsyncBatchSize::default(),
- false,
- );
- let mut generator_context =
- GeneratorContext::new(model_path, &decoding_config).unwrap();
- let desired_suffix_length =
- prefill_step_size.max(decoding_config.generate_suffix_length());
- Self::ensure_context_capacity(
- &decoding_config,
- desired_suffix_length,
- &mut generator_context,
- );
-
- Self {
- model_path: model_path.to_path_buf(),
- generator_context,
- }
- }
-
- pub fn new_with_config(
- model_path: &Path,
- sampling_seed: SamplingSeed,
- max_prefix_length: ContextLength,
- ) -> Self {
- let prefill_step_size = Self::determine_prefill_step_size(model_path);
- let decoding_config = DecodingConfig::new(
- ContextMode::default(),
- max_prefix_length,
- PrefillStepSize::Custom(prefill_step_size),
- SpeculatorConfig::default(),
- sampling_seed,
- AsyncBatchSize::default(),
- false,
- );
- let mut generator_context =
- GeneratorContext::new(model_path, &decoding_config).unwrap();
- let desired_suffix_length =
- prefill_step_size.max(decoding_config.generate_suffix_length());
- Self::ensure_context_capacity(
- &decoding_config,
- desired_suffix_length,
- &mut generator_context,
- );
-
- Self {
- model_path: model_path.to_path_buf(),
- generator_context,
- }
- }
-
- fn determine_prefill_step_size(model_path: &Path) -> usize {
- let traces_path = model_path.join("traces.safetensors");
- if let Ok(file) = File::open(&traces_path) {
- if let Ok((_header_len, metadata)) =
- read_safetensors_metadata(&file)
- {
- if let Some(tensor) =
- metadata.tensors.get("activation_trace.token_ids")
- {
- if let Some(&length) = tensor.shape.first() {
- return tensor
- .shape
- .iter()
- .copied()
- .max()
- .unwrap_or(length)
- .max(1);
- }
- }
- }
- }
- 1
- }
-
- fn ensure_context_capacity(
- decoding_config: &DecodingConfig,
- desired_suffix_length: usize,
- context: &mut GeneratorContext,
- ) {
- let resolved_prefix_length =
- decoding_config.context_length.resolve(&context.model_config);
- let current_suffix_length = std::cmp::max(
- decoding_config.prefill_step_size.resolve(&context.model_config),
- decoding_config.generate_suffix_length(),
- );
-
- if desired_suffix_length <= current_suffix_length {
- return;
- }
-
- let decoder_config = &context.model_config.decoder_config;
- context.scratch_buffers = ForwardPassBuffers::new(
- &context.mtl_context,
- decoder_config,
- &context.model_shape,
- resolved_prefix_length,
- desired_suffix_length,
- );
-
- context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(
- &context.mtl_context,
- &context.model_shape,
- resolved_prefix_length,
- desired_suffix_length,
- )));
-
- let intermediate_dtype: DataType =
- decoder_config.output_norm_config.scale_precision.into();
- let kernel_dtype: KernelDataType = intermediate_dtype.into();
-
- context.kv_cache_update = Box::new(
- KVCacheUpdate::new(
- &context.mtl_context,
- kernel_dtype,
- resolved_prefix_length,
- )
- .expect("Failed to create KV cache update kernel"),
- );
-
- context.gpu_sampler = SamplingKernelEncodable::new(
- &context.mtl_context,
- kernel_dtype,
- desired_suffix_length,
- decoder_config.vocab_size,
- )
- .expect("Failed to create sampling kernel");
- }
-
- pub fn run(&self) -> TracerValidationResults {
- let traces_path = self.model_path.join("traces.safetensors");
- if !traces_path.exists() {
- panic!("Traces file not found at {:?}", traces_path);
- }
- let traces_file =
- File::open(&traces_path).expect("Failed to open traces file");
- let traces_loader = ParameterLoader::new(
- &traces_file,
- &self.generator_context.mtl_context,
- )
- .expect("Failed to create ParameterLoader for traces");
- let traces_view = traces_loader.tree();
-
- let token_ids = Self::load_array_as_vec::<i32, u64>(
- &traces_view,
- "activation_trace.token_ids".to_string(),
- );
- let token_positions = Self::load_array_as_vec::<i32, usize>(
- &traces_view,
- "activation_trace.token_positions".to_string(),
- );
-
- let token_seeds: Vec<u64> = vec![0; token_ids.len()];
-
- let mut state = ForwardPassState::new(
- self.generator_context.mtl_context.clone(),
- &self.generator_context.model_config.decoder_config,
- &self.generator_context.model_shape,
- &self.generator_context.scratch_buffers,
- self.generator_context.cache_layers.clone(),
- self.generator_context.shared_buffers.clone(),
- &token_ids,
- &token_positions,
- None,
- &token_seeds,
- token_ids.len(),
- false,
- true,
- None,
- false,
- None,
- None,
- );
-
- let root_command_buffer = self
- .generator_context
- .command_buffer
- .root_command_buffer()
- .to_owned();
- self.generator_context.executables.encode(
- &mut state,
- &self.generator_context.command_buffer,
- &EncodingParameters::new(false, false, true),
- );
- self.generator_context.command_buffer.commit();
- root_command_buffer.wait_until_completed();
-
- let traces = state.traces.clone().unwrap();
- let results =
- self.validate_traces(token_ids.len(), &state, &traces_view, traces);
- return results;
- }
-}
-
-impl Tracer {
- fn validate_traces(
- &self,
- suffix_length: usize,
- state: &ForwardPassState,
- traces_view: &ParameterTree<Rc<MTLContext>>,
- traces: Rc<RefCell<DecoderActivationTrace>>,
- ) -> TracerValidationResults {
- let mut results: Vec<TracerValidationResult> = Vec::new();
-
- let data_type =
- self.generator_context.model_shape.activation_data_type();
- let mut validate =
- |expected_array_path: &str,
- produced_array: &Ref<MetalArray>,
- produced_transform: Option<Transform>| {
- if let Ok(expected_array) =
- traces_view.leaf(expected_array_path)
- {
- let metrics = Self::validate_array(
- data_type,
- &expected_array,
- produced_array,
- produced_transform,
- );
- results.push(TracerValidationResult {
- name: expected_array_path.to_string(),
- metrics,
- });
- } else {
- println!(
- "Warning: Trace key not found: {}",
- expected_array_path
- );
- }
- };
-
- for (index, layer_traces) in
- traces.borrow().layer_results.iter().enumerate()
- {
- let path = |suffix: &str| -> String {
- format!(
- "activation_trace.layer_results.{}.activation_trace.{}",
- index.clone(),
- suffix
- )
- };
-
- validate(
- path("inputs").as_str(),
- &layer_traces.borrow().inputs.borrow(),
- None,
- );
- validate(
- path("pre_mixer_norm").as_str(),
- &layer_traces.borrow().pre_attention_norm.borrow(),
- None,
- );
- validate(
- path("mixer").as_str(),
- &layer_traces.borrow().attention.borrow(),
- None,
- );
- if self.generator_context.executables.layers[index]
- .post_attention_norm
- .is_some()
- {
- validate(
- path("post_mixer_norm").as_str(),
- &layer_traces.borrow().post_attention_norm.borrow(),
- None,
- );
- }
- validate(
- path("mlp_inputs").as_str(),
- &layer_traces.borrow().mlp_inputs.borrow(),
- None,
- );
- validate(
- path("pre_mlp_norm").as_str(),
- &layer_traces.borrow().pre_mlp_norm.borrow(),
- None,
- );
- validate(
- path("mlp").as_str(),
- &layer_traces.borrow().mlp.borrow(),
- None,
- );
- if self.generator_context.executables.layers[index]
- .post_mlp_norm
- .is_some()
- {
- validate(
- path("post_mlp_norm").as_str(),
- &layer_traces.borrow().post_mlp_norm.borrow(),
- None,
- );
- }
- validate(
- format!(
- "activation_trace.layer_results.{}.outputs",
- index.clone()
- )
- .as_str(),
- &layer_traces.borrow().outputs.borrow(),
- None,
- );
- }
-
- validate(
- "activation_trace.output_norm",
- &traces.borrow().output_norm.borrow(),
- None,
- );
- validate("logits", &traces.borrow().logits.borrow(), None);
-
- let transformer_layers: Vec<usize> = {
- let cache = state.cache_layers.borrow();
- cache
- .data
- .iter()
- .enumerate()
- .filter_map(|(index, layer)| {
- layer.as_transformer().map(|_| index)
- })
- .collect()
- };
-
- for index in transformer_layers {
- let arrays =
- state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);
-
- let keys = arrays[0].borrow();
- validate(
- format!("updated_kv_cache.{}.keys", index.clone()).as_str(),
- &keys,
- Some(Transform::KVCacheSlice),
- );
-
- let values = arrays[1].borrow();
- validate(
- format!("updated_kv_cache.{}.values", index.clone()).as_str(),
- &values,
- Some(Transform::KVCacheSlice),
- );
- }
-
- let ssm_layers: Vec<usize> = {
- let cache = state.cache_layers.borrow();
- cache
- .data
- .iter()
- .enumerate()
- .filter_map(|(index, layer)| {
- layer.as_state_space().map(|_| index)
- })
- .collect()
- };
-
- for index in ssm_layers {
- let arrays = state.arrays(&[
- ArrayId::SsmConvState(index),
- ArrayId::SsmState(index),
- ]);
-
- let conv_state = arrays[0].borrow();
- validate(
- format!("updated_state.{}.conv_state", index).as_str(),
- &conv_state,
- Some(Transform::SsmConvState),
- );
- validate(
- format!(
- "activation_trace.layer_results.{}.updated_state.conv_state",
- index
- )
- .as_str(),
- &conv_state,
- Some(Transform::SsmConvState),
- );
-
- let ssm_state = arrays[1].borrow();
- validate(
- format!("updated_state.{}.ssm_state", index).as_str(),
- &ssm_state,
- None,
- );
- validate(
- format!(
- "activation_trace.layer_results.{}.updated_state.ssm_state",
- index
- )
- .as_str(),
- &ssm_state,
- None,
- );
- }
-
- let expected_tokens =
- Self::get_tokens_from_logits(&traces_view.leaf("logits").unwrap());
- let produced_tokens =
- Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());
- let tokens_violation_indices: Vec<usize> = expected_tokens
- .iter()
- .zip(produced_tokens.iter())
- .enumerate()
- .filter_map(|(i, (a, b))| {
- if a != b {
- Some(i)
- } else {
- None
- }
- })
- .collect();
-
- return TracerValidationResults {
- suffix_length,
- results,
- tokens_violation_indices,
- };
- }
-
- fn validate_array(
- data_type: DataType,
- expected_array: &MetalArray,
- produced_array: &Ref<MetalArray>,
- produced_transform: Option<Transform>,
- ) -> TracerValidationMetrics {
- match data_type {
- DataType::F16 => {
- return Self::validate_array_of_type::<f16>(
- expected_array,
- produced_array,
- produced_transform,
- );
- },
- DataType::BF16 => {
- return Self::validate_array_of_type::<bf16>(
- expected_array,
- produced_array,
- produced_transform,
- );
- },
- DataType::F32 => {
- return Self::validate_array_of_type::<f32>(
- expected_array,
- produced_array,
- produced_transform,
- );
- },
- _ => panic!("Unsupported data type: {:?}", data_type),
- }
- }
-
- fn validate_array_of_type<Precision: ArrayElement>(
- expected_array: &MetalArray,
- produced_array: &Ref<MetalArray>,
- produced_transform: Option<Transform>,
- ) -> TracerValidationMetrics {
- let expected_view = expected_array.as_view::<Precision>().unwrap();
- let produced_view = produced_array.as_view::<Precision>().unwrap();
-
- let (mut expected_data, mut produced_data) = match produced_transform {
- Some(transform) => match transform {
- Transform::KVCacheSlice => {
- let permuted =
- produced_view.permuted_axes(IxDyn(&[1, 0, 2]));
- let total_tokens = permuted.shape()[0];
- let expected_tokens = expected_view.shape()[1];
- let start = total_tokens.saturating_sub(expected_tokens);
- let sliced = permuted.slice(s![start.., .., ..]);
- let reshaped = sliced
- .into_owned()
- .to_shape(IxDyn(&[
- 1,
- expected_tokens,
- permuted.shape()[1],
- permuted.shape()[2],
- ]))
- .expect("Failed to reshape KV cache slice")
- .to_owned();
- (expected_view.to_owned(), reshaped)
- },
- Transform::SsmConvState => {
- // expected: [1, T, D]
- // produced: [D, K]
- let produced_shape = produced_view.shape();
- let history_len = produced_shape[1];
- let dim = produced_shape[0];
-
- // Permute expected to [1, D, T]
- let permuted =
- expected_view.permuted_axes(IxDyn(&[0, 2, 1]));
- let total_time = permuted.shape()[2];
- let start = total_time.saturating_sub(history_len);
- // Slice last K elements from time axis: [1, D, K]
- let sliced = permuted.slice(s![.., .., start..]);
-
- let reshaped_expected = sliced
- .into_owned()
- .to_shape(IxDyn(&[dim, history_len]))
- .expect("Failed to reshape SSM conv state slice")
- .to_owned();
-
- (reshaped_expected, produced_view.to_owned())
- },
- },
- None => (expected_view.to_owned(), produced_view.to_owned()),
- };
- let expected_shape = expected_data.shape().to_vec();
- let produced_shape = produced_data.shape().to_vec();
-
- if expected_shape != produced_shape {
- if expected_shape.len() == produced_shape.len() + 1
- && expected_shape.get(0) == Some(&1)
- && expected_shape[1..] == produced_shape[..]
- {
- expected_data = expected_data
- .to_shape(IxDyn(&produced_shape))
- .expect("Failed to reshape expected data")
- .to_owned();
- } else if produced_shape.len() == expected_shape.len() + 1
- && produced_shape.get(0) == Some(&1)
- && produced_shape[1..] == expected_shape[..]
- {
- produced_data = produced_data
- .to_shape(IxDyn(&expected_shape))
- .expect("Failed to reshape produced data")
- .to_owned();
- }
- }
-
- if expected_data.shape() != produced_data.shape() {
- panic!(
- "Shape mismatch after alignment: expected {:?}, produced {:?}",
- expected_data.shape(),
- produced_data.shape()
- );
- }
-
- let reference = expected_data
- .iter()
- .map(|value| {
- let value_f32: f32 = NumCast::from(*value).unwrap();
- return value_f32;
- })
- .collect::<Vec<_>>();
-
- let result = produced_data
- .iter()
- .map(|value| {
- let value_f32: f32 = NumCast::from(*value).unwrap();
- return value_f32;
- })
- .collect::<Vec<_>>();
-
- let atol: f32 = 1e-2;
- let rtol: f32 = 0.03;
- let end_to_end_fraction_of_allowed_violations: f32 = 0.01;
-
- return Self::compare_arrays(
- &reference,
- expected_data.shape().to_vec(),
- &result,
- produced_data.shape().to_vec(),
- atol,
- rtol,
- end_to_end_fraction_of_allowed_violations,
- );
- }
-
- fn compare_arrays(
- reference: &Vec<f32>,
- reference_shape: Vec<usize>,
- result: &Vec<f32>,
- result_shape: Vec<usize>,
- atol: f32,
- rtol: f32,
- fraction_of_allowed_violations: f32,
- ) -> TracerValidationMetrics {
- assert_eq!(result.len(), reference.len());
-
- let absdiff: Vec<f32> = result
- .iter()
- .zip(reference.iter())
- .map(|(result_value, reference_value)| {
- (result_value - reference_value).abs()
- })
- .collect();
-
- let allowed_diff: Vec<f32> = reference
- .iter()
- .map(|reference_value| atol + rtol * reference_value.abs())
- .collect();
-
- let violations: Vec<f32> = absdiff
- .iter()
- .zip(allowed_diff.iter())
- .map(|(absdiff_value, allowed_diff_value)| {
- (absdiff_value - allowed_diff_value).max(0.0)
- })
- .collect();
-
- let num_violations =
- violations.iter().filter(|&&value| value > 0.0).count();
-
- let max_allowed_violations = (reference.len() as f32
- * fraction_of_allowed_violations)
- .ceil() as usize;
-
- let err_rel: Vec<f32> = absdiff
- .iter()
- .zip(reference.iter())
- .map(|(absdiff_value, reference_value)| {
- absdiff_value / (reference_value.abs() + 1e-10)
- })
- .collect();
-
- let (max_err_idx, _) = violations
- .iter()
- .enumerate()
- .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap())
- .unwrap();
-
- let max_err = absdiff[max_err_idx];
- let max_err_rel = err_rel[max_err_idx];
- let max_err_reference_value = reference[max_err_idx];
-
- let rms_diff = {
- let sum_squares: f32 = absdiff.iter().map(|x| x * x).sum();
- (sum_squares / absdiff.len() as f32).sqrt()
- };
-
- let rms_result = {
- let sum_squares: f32 = result.iter().map(|x| x * x).sum();
- (sum_squares / result.len() as f32).sqrt()
- };
-
- let rms_reference = {
- let sum_squares: f32 = reference.iter().map(|x| x * x).sum();
- (sum_squares / reference.len() as f32).sqrt()
- };
-
- let rel_rms_reference = rms_diff / (rms_reference + 1e-10);
-
- let diff_max = absdiff.iter().cloned().reduce(|a, b| a.max(b)).unwrap();
- let diff_avg =
- absdiff.iter().cloned().sum::<f32>() / absdiff.len() as f32;
-
- let result_nan = result.iter().any(|x| x.is_nan());
-
- return TracerValidationMetrics {
- atol,
- rtol,
- fraction_of_allowed_violations,
- reference_shape,
- result_shape,
- num_violations,
- max_allowed_violations,
- max_err_idx,
- max_err,
- max_err_rel,
- max_err_reference_value,
- rms_diff,
- rms_result,
- rms_reference,
- rel_rms_reference,
- diff_max,
- diff_avg,
- result_nan,
- };
- }
-
- fn load_array_as_vec<
- SourcePrecision: ArrayElement,
- TargetPrecision: NumCast,
- >(
- traces_view: &ParameterTree<Rc<MTLContext>>,
- name: String,
- ) -> Vec<TargetPrecision> {
- let array = traces_view.leaf(name.as_str()).unwrap();
- let slice = array.as_slice::<SourcePrecision>().unwrap();
- let vec: Vec<TargetPrecision> = slice
- .iter()
- .map(|x| {
- let casted_value: TargetPrecision = NumCast::from(*x).unwrap();
- return casted_value;
- })
- .collect();
- return vec;
- }
-
- fn get_tokens_from_logits(logits: &MetalArray) -> Vec<u64> {
- let data_type = logits.data_type();
- match data_type {
- DataType::F16 => {
- return Self::get_tokens_from_logits_of_type::<f16>(logits);
- },
- DataType::BF16 => {
- return Self::get_tokens_from_logits_of_type::<bf16>(logits);
- },
- DataType::F32 => {
- return Self::get_tokens_from_logits_of_type::<f32>(logits);
- },
- _ => panic!("Unsupported data type: {:?}", data_type),
- }
- }
-
- fn get_tokens_from_logits_of_type<Precision: ArrayElement>(
- logits: &MetalArray
- ) -> Vec<u64> {
- let sampler = ArgmaxSampler {};
- let tokens = sampler.sample(logits.as_view::<Precision>().unwrap());
- return tokens;
- }
-}
diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rs
index 399420a28..5faf525b8 100644
--- a/crates/uzu/src/trie.rs
+++ b/crates/uzu/src/trie.rs
@@ -1,12 +1,12 @@
+use thiserror::Error;
+
use crate::{
- generator::{
+ language_model::{
grammar::CompiledGrammar, gumbel::speculator_sample, rng::DerivableSeed,
},
speculators::speculator::Speculator,
};
-use thiserror::Error;
-
#[derive(Debug, Error)]
pub enum TrieError {
#[error("child with the same token id is already present")]
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index 818e357ee..35d85d6d1 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -379,6 +379,7 @@ fn run_single_pass_attention(
num_heads,
suffix_length: seq_len,
head_dim,
+ is_causal: false,
};
kernel.encode_single_pass(&compute_encoder, args)?;
@@ -516,20 +517,21 @@ fn test_single_pass_attention_basic() {
},
};
- let variant = kernel.choose_variant(seq_len, head_dim);
+ let is_causal = false; // Non-causal attention for this test
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
println!("Using kernel variant: {:?}", variant);
println!(
"Supports single-pass for head_dim={}: {}",
head_dim,
- kernel.supports_single_pass(head_dim)
+ kernel.supports_single_pass(head_dim, is_causal)
);
println!(
"Supports two-pass for head_dim={}: {}",
head_dim,
- kernel.supports_two_pass(head_dim)
+ kernel.supports_two_pass(head_dim, is_causal)
);
- if !kernel.supports_single_pass(head_dim) {
+ if !kernel.supports_single_pass(head_dim, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -587,7 +589,7 @@ fn test_single_pass_attention_with_mask() {
},
};
- if !kernel.supports_single_pass(head_dim) {
+ if !kernel.supports_single_pass(head_dim, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -668,7 +670,7 @@ fn test_single_pass_attention_with_sinks() {
},
};
- if !kernel.supports_single_pass(head_dim) {
+ if !kernel.supports_single_pass(head_dim, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -746,7 +748,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
},
};
- if !kernel.supports_single_pass(head_dim) {
+ if !kernel.supports_single_pass(head_dim, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -823,10 +825,11 @@ fn test_single_pass_attention_gqa() {
},
};
- let variant = kernel.choose_variant(seq_len, head_dim);
+ let is_causal = false; // Non-causal attention for this test
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
println!("Using kernel variant for GQA: {:?}", variant);
- if !kernel.supports_single_pass(head_dim) {
+ if !kernel.supports_single_pass(head_dim, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -922,6 +925,7 @@ fn run_two_pass_attention(
num_heads,
suffix_length: seq_len,
head_dim,
+ is_causal: false,
};
kernel.encode_two_pass(&compute_encoder, args)?;
@@ -970,12 +974,13 @@ fn test_two_pass_attention() {
let seq_len = 2048;
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
+ let is_causal = false; // Non-causal attention for this test
- if !kernel.supports_two_pass(head_dim) {
+ if !kernel.supports_two_pass(head_dim, is_causal) {
panic!("Two-pass kernel not supported for head_dim={}", head_dim);
}
- let variant = kernel.choose_variant(seq_len, head_dim);
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
panic!(
"Two-pass not selected for seq_len={}. Got {:?}",
@@ -1039,12 +1044,13 @@ fn test_two_pass_attention_gqa() {
let seq_len = 4096;
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
+ let is_causal = false; // Non-causal attention for this test
- if !kernel.supports_two_pass(head_dim) {
+ if !kernel.supports_two_pass(head_dim, is_causal) {
panic!("Two-pass kernel not supported for head_dim={}", head_dim);
}
- let variant = kernel.choose_variant(seq_len, head_dim);
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
panic!(
"Two-pass not selected for GQA seq_len={}. Got {:?}",
@@ -1113,8 +1119,9 @@ fn perf_two_pass_attention() {
let suffix_length = 1; // Only processing 1 new token (realistic inference)
let head_dim = 128;
let scale = 1.0 / (head_dim as f32).sqrt();
+ let is_causal = false;
- if !kernel.supports_two_pass(head_dim) {
+ if !kernel.supports_two_pass(head_dim, is_causal) {
println!(
"Skipping two-pass perf test: not supported for head_dim={}",
head_dim
@@ -1122,7 +1129,7 @@ fn perf_two_pass_attention() {
return;
}
- let variant = kernel.choose_variant(seq_len, head_dim);
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
println!(
"Skipping two-pass perf test: variant {:?} selected instead",
@@ -1203,6 +1210,7 @@ fn perf_two_pass_attention() {
num_heads,
suffix_length, // Use actual suffix_length, not seq_len
head_dim,
+ is_causal: false,
};
kernel.encode_two_pass(&compute_encoder, args).expect("encode");
diff --git a/crates/uzu/tests/context_mode_test.rs b/crates/uzu/tests/context_mode_test.rs
index 39d21ac86..91249eb41 100644
--- a/crates/uzu/tests/context_mode_test.rs
+++ b/crates/uzu/tests/context_mode_test.rs
@@ -137,6 +137,7 @@ fn test_context_mode_static() {
}
#[test]
+#[ignore = "Flaky test - depends on LLM output which varies even with fixed seed"]
fn test_context_mode_dynamic() {
let decoding_config =
build_decoding_config().with_context_mode(ContextMode::Dynamic);
diff --git a/crates/uzu/tests/decoder_test.rs b/crates/uzu/tests/decoder_test.rs
deleted file mode 100644
index 4198a240d..000000000
--- a/crates/uzu/tests/decoder_test.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-mod common;
-use uzu::decoder_runner::run_decoder_with_results;
-
-#[test]
-fn decoder_runs_successfully() {
- // Skip the test completely if we're not on a Metal-capable platform.
- if !cfg!(any(target_os = "macos", target_os = "ios")) {
- eprintln!("Skipping decoder test on non-Metal platform");
- return;
- }
-
- // Build the path to the test model
- let model_path = common::get_test_model_path();
-
- // Run the decoder and collect the results
- let result = run_decoder_with_results(model_path.to_str().unwrap());
-
- // Always print the results so they are visible when running with `cargo test -- --nocapture`
- println!("Placement Log:\n{}", result.placement_log);
- println!("Iterations: {}", result.iterations);
- println!("Time per token: {}", result.time_per_token);
- println!("Tokens per second: {}", result.tokens_per_second);
- println!("Success: {}", result.success);
- if let Some(error) = &result.error {
- println!("Error: {}", error);
- }
-
- // Ensure the decoder executed without errors
- if !result.success {
- panic!(
- "Decoder test failed. Error: {:?}\nPlacement Log:\n{}",
- result.error, result.placement_log
- );
- }
-
- // Basic sanity checks on the performance numbers
- assert!(result.iterations > 0, "Expected at least one iteration.");
- assert!(result.time_per_token > 0.0, "Time per token should be positive.");
- assert!(
- result.tokens_per_second > 0.0,
- "Tokens per second should be positive."
- );
-}
diff --git a/crates/uzu/tests/grammar_json_schema_test.rs b/crates/uzu/tests/grammar_json_schema_test.rs
index c52f38fb0..b9cc284c6 100644
--- a/crates/uzu/tests/grammar_json_schema_test.rs
+++ b/crates/uzu/tests/grammar_json_schema_test.rs
@@ -29,6 +29,7 @@ struct Person {
}
#[test]
+#[ignore = "Test requires model; run manually with RUST_TEST_THREADS=1"]
fn test_grammar_json_schema() {
let crate_version = env!("CARGO_PKG_VERSION");
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rs
index 6af41e880..7c6bfedf3 100644
--- a/crates/uzu/tests/rms_norm_kernel_test.rs
+++ b/crates/uzu/tests/rms_norm_kernel_test.rs
@@ -383,7 +383,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
let command_buffer = command_buffer_ref.to_owned();
let compute_encoder = command_buffer.new_compute_command_encoder();
- kernel.encode(
+ let _ = kernel.encode(
&compute_encoder,
RMSNormArguments {
input_buffer: &input_buffer,
@@ -514,7 +514,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
let command_buffer = command_buffer_ref.to_owned();
let compute_encoder = command_buffer.new_compute_command_encoder();
- kernel.encode(
+ let _ = kernel.encode(
&compute_encoder,
RMSNormArguments {
input_buffer: &input_buffer,
@@ -853,7 +853,7 @@ fn perf_rms_norm_with_size(
let command_buffer = command_buffer_ref.to_owned();
let compute_encoder = command_buffer.new_compute_command_encoder();
- kernel.encode(
+ let _ = kernel.encode(
&compute_encoder,
RMSNormArguments {
input_buffer: &input_buffer,
@@ -1046,7 +1046,7 @@ fn qk_norm_test() {
let command_buffer = mtl_context.command_queue.new_command_buffer();
let compute_encoder = command_buffer.new_compute_command_encoder();
- q_kernel.encode_qk_norm(
+ let _ = q_kernel.encode_qk_norm(
&compute_encoder,
QKNormArguments {
qkv_input_buffer: &qkv_buffer,
@@ -1112,7 +1112,7 @@ fn qk_norm_test() {
let command_buffer = mtl_context.command_queue.new_command_buffer();
let compute_encoder = command_buffer.new_compute_command_encoder();
- k_kernel.encode_qk_norm(
+ let _ = k_kernel.encode_qk_norm(
&compute_encoder,
QKNormArguments {
qkv_input_buffer: &qkv_buffer,
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index ae42926b2..63a4770d9 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -8,7 +8,7 @@ use uzu::{
kernel::{SamplingKernel, sampling::ArgmaxStrategy},
metal_extensions::command_buffer_extensions::CommandBufferTimingAccess,
},
- generator::gumbel::{gumbel_float, revidx},
+ language_model::gumbel::{gumbel_float, revidx},
session::parameter::SamplingMethod,
};
@@ -107,7 +107,6 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
.encode(
&logits_buffer,
None,
- None,
&output_buffer,
SamplingMethod::Greedy,
batch_size,
@@ -247,7 +246,6 @@ fn test_topp_sampling_from_prob_exact_match(
kernel
.encode(
&logits_buf,
- None,
Some(&seeds_buf),
&output_buf,
SamplingMethod::Stochastic {
@@ -375,7 +373,6 @@ fn test_topp_sampling_statistical_large() {
kernel
.encode(
&logits_buf,
- None,
Some(&seeds_buf),
&output_buf,
SamplingMethod::Stochastic {
@@ -484,7 +481,6 @@ fn perf_topp_128k_vocab() {
kernel
.encode(
&logits_buf,
- None,
Some(&seeds_buf),
&output_buf,
SamplingMethod::Stochastic {
@@ -590,7 +586,6 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
kernel
.encode(
&logits_buf,
- None,
Some(&seeds_buf),
&output_buf,
SamplingMethod::Greedy,
@@ -712,7 +707,6 @@ fn test_categorical_sampling() {
kernel
.encode(
&logits_buffer,
- None,
Some(&seeds_buffer),
&output_buffer,
SamplingMethod::Stochastic {
@@ -856,7 +850,6 @@ fn test_categorical_sampling_statistical() {
kernel
.encode(
&logits_buffer,
- None,
Some(&seeds_buffer),
&output_buffer,
SamplingMethod::Stochastic {
@@ -963,7 +956,6 @@ fn perf_categorical_128k_vocab() {
kernel
.encode(
&logits_buf,
- None,
Some(&seeds_buf),
&output_buf,
SamplingMethod::Stochastic {
@@ -1299,7 +1291,6 @@ fn test_gumbel_gpu_cpu_match() {
.encode_gumbel(
&logits_buffer,
&seeds_buffer,
- 0,
&gumbel_logits_buffer,
BATCH as u32,
VOCAB as u32,
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index f00e091a4..e08d7e091 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -6,6 +6,8 @@ use uzu::backends::metal::{
KernelDataType, MTLContext,
kernel::ssm::{SSDUpdateArguments, SSDUpdateKernel},
};
+
+#[allow(dead_code)]
fn ssm_update_ref_bf16(
x: &[bf16],
dt: &[bf16],
@@ -65,6 +67,7 @@ fn softplus_f32(x: f32) -> f32 {
}
}
+#[allow(dead_code)]
fn ssd_update_no_z_ref_bf16(
x: &[bf16],
dt_raw: &[bf16],
diff --git a/crates/uzu/tests/tracer_test.rs b/crates/uzu/tests/tracer_test.rs
index 8ce1a1e9b..ef3aeff20 100644
--- a/crates/uzu/tests/tracer_test.rs
+++ b/crates/uzu/tests/tracer_test.rs
@@ -1,7 +1,9 @@
+#![cfg(feature = "tracing")]
+
mod common;
use std::path::PathBuf;
-use uzu::tracer::Tracer;
+use uzu::tracer::TraceValidator;
fn build_model_path() -> PathBuf {
common::get_test_model_path()
@@ -20,7 +22,13 @@ fn test_tracer() {
return;
}
- let tracer = Tracer::new(&model_path);
+ let mut tracer = match TraceValidator::new(&model_path) {
+ Ok(t) => t,
+ Err(e) => {
+ println!("Failed to create TraceValidator: {:?}", e);
+ return;
+ },
+ };
let colored_text = |text: &str, valid: bool| {
if valid {
@@ -30,7 +38,14 @@ fn test_tracer() {
}
};
- let results = tracer.run();
+ let results = match tracer.run() {
+ Ok(r) => r,
+ Err(e) => {
+ println!("Tracer run failed: {:?}", e);
+ return;
+ },
+ };
+
for result in results.results.iter() {
let valid = result.metrics.is_valid();
let text = colored_text(
diff --git a/crates/uzu/tests/trie_test.rs b/crates/uzu/tests/trie_test.rs
index b3344d100..4d3b970ed 100644
--- a/crates/uzu/tests/trie_test.rs
+++ b/crates/uzu/tests/trie_test.rs
@@ -1,7 +1,7 @@
use std::collections::{HashMap, HashSet};
use uzu::{
- generator::rng::DerivableSeed,
+ language_model::rng::DerivableSeed,
speculators::{empty_speculator::EmptySpeculator, speculator::Speculator},
trie::{TrieCreationConfig, TrieNode},
};