A high-performance inference engine for AI models
Unify Gemm (#411)
eugene5c3dc00df293
diff --git a/Cargo.lock b/Cargo.lockindex 9e471853a..3fd5ea8d9 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -559,6 +559,7 @@ dependencies = ["anyhow","async-trait","bindgen",+ "bitflags 2.11.1","blake3","bytemuck","bytesize",diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.tomlindex f33baca5f..7134ac07e 100644--- a/crates/backend-uzu/Cargo.toml+++ b/crates/backend-uzu/Cargo.toml@@ -11,6 +11,8 @@ build = "build/main.rs"[dependencies]proc-macros.workspace = true+bitflags.workspace = true+derive_more.workspace = trueshoji.workspace = truehalf.workspace = truendarray.workspace = true@@ -111,7 +113,7 @@ path = "tests/unit/kernel/test_entry.rs"[[bench]]name = "kernel"-path = "tests/unit/kernel/bench_entry.rs"+path = "tests/performance/kernel/bench_entry.rs"[[test]]name = "performance"diff --git a/crates/backend-uzu/build/common/enum_paths.rs b/crates/backend-uzu/build/common/enum_paths.rsindex 90880ba3a..991991808 100644--- a/crates/backend-uzu/build/common/enum_paths.rs+++ b/crates/backend-uzu/build/common/enum_paths.rs@@ -5,35 +5,52 @@ use syn::{Type, TypePath, visit_mut::VisitMut};use super::gpu_types::{GpuType, GpuTypeName, GpuTypePath, GpuTypes};+#[derive(Copy, Clone, Debug, PartialEq, Eq)]+pub enum GpuTypeKind {+ Enum,+ OptionSet,+}++#[derive(Clone)]+struct GpuTypeEntry {+ path: GpuTypePath,+ #[allow(dead_code)]+ kind: GpuTypeKind,+}+#[derive(Clone)]pub struct EnumPaths {- short_name_to_full_path: HashMap<GpuTypeName, GpuTypePath>,+ short_name_to_entry: HashMap<GpuTypeName, GpuTypeEntry>,}impl EnumPaths {pub fn from_gpu_types(gpu_types: &GpuTypes) -> anyhow::Result<Self> {- let mut short_name_to_full_path = HashMap::new();+ let mut short_name_to_entry = HashMap::new();for file in gpu_types.files.iter() {for ty in file.types.iter() {- if let GpuType::Enum(enum_type) = ty {- let name = GpuTypeName::from(enum_type.name.as_ref());- let path = GpuTypePath::from(format!(- "crate::backends::common::gpu_types::{}::{}",- file.name, enum_type.name- ));- match short_name_to_full_path.entry(name) {- Entry::Occupied(occupied) => {- bail!("gpu type `{}` is duplicated", occupied.key())- },- Entry::Vacant(vacant) => {- vacant.insert(path);- },- }+ let (name_str, kind) = match ty {+ GpuType::Enum(enum_type) => (enum_type.name.as_ref(), GpuTypeKind::Enum),+ GpuType::OptionSet(option_set) => (option_set.name.as_ref(), GpuTypeKind::OptionSet),+ GpuType::Struct(_) => continue,+ };+ let name = GpuTypeName::from(name_str);+ let path =+ GpuTypePath::from(format!("crate::backends::common::gpu_types::{}::{}", file.name, name_str));+ match short_name_to_entry.entry(name) {+ Entry::Occupied(occupied) => {+ bail!("gpu type `{}` is duplicated", occupied.key())+ },+ Entry::Vacant(vacant) => {+ vacant.insert(GpuTypeEntry {+ path,+ kind,+ });+ },}}}Ok(Self {- short_name_to_full_path,+ short_name_to_entry,})}@@ -41,7 +58,15 @@ impl EnumPaths {&self,short_name: &str,) -> Option<&str> {- self.short_name_to_full_path.get(short_name).map(|path| &**path)+ self.short_name_to_entry.get(short_name).map(|entry| &*entry.path)+ }++ #[allow(dead_code)]+ pub fn kind_for(+ &self,+ short_name: &str,+ ) -> Option<GpuTypeKind> {+ self.short_name_to_entry.get(short_name).map(|entry| entry.kind)}pub fn canonicalize_type(diff --git a/crates/backend-uzu/build/common/gpu_types/item_option_set.rs b/crates/backend-uzu/build/common/gpu_types/item_option_set.rsnew file mode 100644index 000000000..2bce9e2ea--- /dev/null+++ b/crates/backend-uzu/build/common/gpu_types/item_option_set.rs@@ -0,0 +1,76 @@+use anyhow::Context;+use proc_macro2::TokenStream;+use quote::ToTokens;+use syn::{+ Attribute, Expr, Ident, Token, Visibility, braced,+ parse::{Parse, ParseStream},+};++#[derive(Debug)]+pub struct GpuTypeOptionSetVariant {+ pub name: String,+ /// Raw expression (e.g. `1 << 0`) emitted verbatim into C++.+ pub value_expression: String,+}++#[derive(Debug)]+pub struct GpuTypeOptionSet {+ pub name: String,+ pub underlying_type: String,+ pub variants: Vec<GpuTypeOptionSetVariant>,+}++impl GpuTypeOptionSet {+ /// Parse the token-stream inside a `bitflags! { … }` macro invocation.+ pub fn parse(tokens: TokenStream) -> anyhow::Result<Self> {+ let block: BitflagsBlock = syn::parse2(tokens).context("Cannot parse bitflags! contents")?;+ Ok(Self {+ name: block.name.to_string(),+ underlying_type: block.underlying.to_string(),+ variants: block+ .variants+ .into_iter()+ .map(|(name, expression)| GpuTypeOptionSetVariant {+ name: name.to_string(),+ value_expression: expression.into_token_stream().to_string(),+ })+ .collect(),+ })+ }+}++struct BitflagsBlock {+ name: Ident,+ underlying: Ident,+ variants: Vec<(Ident, Expr)>,+}++impl Parse for BitflagsBlock {+ fn parse(input: ParseStream) -> syn::Result<Self> {+ let _attrs = input.call(Attribute::parse_outer)?;+ let _vis: Visibility = input.parse()?;+ let _struct_kw: Token![struct] = input.parse()?;+ let name: Ident = input.parse()?;+ let _colon: Token![:] = input.parse()?;+ let underlying: Ident = input.parse()?;++ let body;+ braced!(body in input);++ let mut variants = Vec::new();+ while !body.is_empty() {+ let _const_kw: Token![const] = body.parse()?;+ let variant_name: Ident = body.parse()?;+ let _equals: Token![=] = body.parse()?;+ let expression: Expr = body.parse()?;+ let _semi: Token![;] = body.parse()?;+ variants.push((variant_name, expression));+ }++ Ok(Self {+ name,+ underlying,+ variants,+ })+ }+}diff --git a/crates/backend-uzu/build/common/gpu_types/mod.rs b/crates/backend-uzu/build/common/gpu_types/mod.rsindex b1b5cdbff..c03b4b98b 100644--- a/crates/backend-uzu/build/common/gpu_types/mod.rs+++ b/crates/backend-uzu/build/common/gpu_types/mod.rs@@ -11,9 +11,11 @@ use syn::{Attribute, Ident, Item};use walkdir::WalkDir;mod item_enum;+mod item_option_set;mod item_struct;pub use item_enum::GpuTypeEnum;+pub use item_option_set::{GpuTypeOptionSet, GpuTypeOptionSetVariant};pub use item_struct::{GpuTypeStruct, GpuTypeStructFieldType};#[derive(Clone, Debug, PartialEq, Eq, Hash, From, AsRef, Deref, Display)]@@ -47,6 +49,7 @@ fn ensure_repr_c(attrs: &[Attribute]) -> anyhow::Result<()> {pub enum GpuType {Enum(GpuTypeEnum),Struct(GpuTypeStruct),+ OptionSet(GpuTypeOptionSet),}#[derive(Debug)]@@ -68,6 +71,9 @@ impl GpuTypeFile {.filter_map(|item| match item {Item::Enum(item) => Some(GpuTypeEnum::parse(item).map(GpuType::Enum)),Item::Struct(item) => Some(GpuTypeStruct::parse(item).map(GpuType::Struct)),+ Item::Macro(item) if item.mac.path.is_ident("bitflags") => {+ Some(GpuTypeOptionSet::parse(item.mac.tokens).map(GpuType::OptionSet))+ },_ => None,}).collect::<anyhow::Result<Box<[GpuType]>>>()?;diff --git a/crates/backend-uzu/build/metal/bindgen/specialize.rs b/crates/backend-uzu/build/metal/bindgen/specialize.rsindex e4e1fdb13..4f8715d3c 100644--- a/crates/backend-uzu/build/metal/bindgen/specialize.rs+++ b/crates/backend-uzu/build/metal/bindgen/specialize.rs@@ -1,17 +1,18 @@use anyhow::{Context, Result};use proc_macro2::TokenStream;use quote::{format_ident, quote};-use syn::{Ident, Type, parse_quote};+use syn::{Ident, Type};use super::super::{ast::{MetalArgumentType, MetalKernelInfo},- enum_path_rewrite::is_enum_c_type,+ enum_path_rewrite::gpu_type_kind_for_c_type,};-use crate::common::enum_paths::EnumPaths;+use crate::common::enum_paths::{EnumPaths, GpuTypeKind};enum SpecializeLowering {Direct,- CastTo(Type),+ Enum,+ OptionSet,}struct SpecializeArgument {@@ -45,10 +46,10 @@ pub fn parse(format!("specialize type `{}` in kernel `{}` cannot be parsed", rust_type_text, kernel_name)})?;let function_constant_index = base_function_constant_index.unwrap_or(0) + offset;- let lowering = if is_enum_c_type(enum_paths, &argument.c_type) {- SpecializeLowering::CastTo(parse_quote!(u32))- } else {- SpecializeLowering::Direct+ let lowering = match gpu_type_kind_for_c_type(enum_paths, &argument.c_type) {+ Some(GpuTypeKind::Enum) => SpecializeLowering::Enum,+ Some(GpuTypeKind::OptionSet) => SpecializeLowering::OptionSet,+ None => SpecializeLowering::Direct,};Ok(SpecializeArgument {name,@@ -90,8 +91,11 @@ impl SpecializeEmission {SpecializeLowering::Direct => {quote! { function_constants.set_value(&#name, #index); }},- SpecializeLowering::CastTo(wire_type) => {- quote! { function_constants.set_value(&(#name as #wire_type), #index); }+ SpecializeLowering::Enum => {+ quote! { function_constants.set_value(&(#name as u32), #index); }+ },+ SpecializeLowering::OptionSet => {+ quote! { function_constants.set_value(&#name.bits(), #index); }},}})@@ -115,7 +119,7 @@ impl SpecializeEmission {return quote! { &entry_name };}let argument_names: Vec<&Ident> = self.arguments.iter().map(|argument| &argument.name).collect();- let format_string = format!("{{}}{}", "_{}".repeat(argument_names.len()));+ let format_string = format!("{{}}{}", "_{:?}".repeat(argument_names.len()));quote! { &format!(#format_string, &entry_name #(, #argument_names)*) }}}diff --git a/crates/backend-uzu/build/metal/enum_path_rewrite.rs b/crates/backend-uzu/build/metal/enum_path_rewrite.rsindex 50a1e689f..27e17dc17 100644--- a/crates/backend-uzu/build/metal/enum_path_rewrite.rs+++ b/crates/backend-uzu/build/metal/enum_path_rewrite.rs@@ -3,21 +3,30 @@ use proc_macro2::TokenStream;use quote::ToTokens;use syn::{Expr, Path, parse_quote};-use crate::common::{enum_paths::EnumPaths, expr_rewrite::rewrite_paths_with};+use crate::common::{+ enum_paths::{EnumPaths, GpuTypeKind},+ expr_rewrite::rewrite_paths_with,+};-pub fn is_enum_c_type(+pub fn gpu_type_kind_for_c_type(enum_paths: &EnumPaths,c_type: &str,-) -> bool {+) -> Option<GpuTypeKind> {let trimmed = c_type.trim_start_matches("const ").trim();- let Some(uzu_path) = trimmed.strip_prefix("uzu::") else {- return false;- };- let Some(short_name) = uzu_path.rsplit("::").next() else {- return false;- };+ let uzu_path = trimmed.strip_prefix("uzu::")?;+ let short_name = uzu_path.rsplit("::").next()?;let expected_rust_path = format!("crate::backends::common::gpu_types::{uzu_path}");- enum_paths.full_path_for(short_name) == Some(expected_rust_path.as_str())+ if enum_paths.full_path_for(short_name) != Some(expected_rust_path.as_str()) {+ return None;+ }+ enum_paths.kind_for(short_name)+}++pub fn is_enum_c_type(+ enum_paths: &EnumPaths,+ c_type: &str,+) -> bool {+ gpu_type_kind_for_c_type(enum_paths, c_type).is_some()}pub fn rewrite_for_rust(diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rsnew file mode 100644index 000000000..86903d05c--- /dev/null+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs@@ -0,0 +1,44 @@+use crate::common::gpu_types::GpuTypeOptionSet;++fn r2c(ty: &str) -> anyhow::Result<&'static str> {+ Ok(match ty {+ "u8" => "uint8_t",+ "u16" => "uint16_t",+ "u32" => "uint",+ "u64" => "uint64_t",+ "i8" => "int8_t",+ "i16" => "int16_t",+ "i32" => "int",+ "i64" => "int64_t",+ unknown => anyhow::bail!("Unsupported option-set underlying type: {unknown}"),+ })+}++/// Emit a Metal-side option-set as a struct wrapping the underlying primitive,+/// with named static constants for each flag and a `contains()` helper.+/// Implicit conversions from the underlying type allow a `uint` function+/// constant to be passed where the struct is expected.+pub fn gpu_type_gen_option_set(option_set: &GpuTypeOptionSet) -> anyhow::Result<String> {+ let name = &option_set.name;+ let underlying_c = r2c(&option_set.underlying_type)?;++ let variants = option_set+ .variants+ .iter()+ .map(|variant| {+ format!(" static constant constexpr {underlying_c} {} = {};\n", variant.name, variant.value_expression)+ })+ .collect::<String>();++ Ok(format!(+ "struct {name} {{\n\+ \x20 {underlying_c} raw_value;\n\+ \x20 constexpr {name}() thread : raw_value(0) {{}}\n\+ \x20 constexpr {name}({underlying_c} __dsl_v) thread : raw_value(__dsl_v) {{}}\n\+ {variants}\+ \x20 constexpr bool contains({underlying_c} flag) const thread {{\n\+ \x20\x20\x20 return (raw_value & flag) != 0;\n\+ \x20 }}\n\+ }};"+ ))+}diff --git a/crates/backend-uzu/build/metal/gpu_types/mod.rs b/crates/backend-uzu/build/metal/gpu_types/mod.rsindex 14e3b25ab..f98c7ebea 100644--- a/crates/backend-uzu/build/metal/gpu_types/mod.rs+++ b/crates/backend-uzu/build/metal/gpu_types/mod.rs@@ -6,9 +6,11 @@ use itertools::Itertools;use crate::common::gpu_types::{GpuType, GpuTypeFile, GpuTypes};mod gpu_type_enum;+mod gpu_type_option_set;mod gpu_type_struct;use gpu_type_enum::gpu_type_gen_enum;+use gpu_type_option_set::gpu_type_gen_option_set;use gpu_type_struct::gpu_type_gen_struct;pub async fn gpu_type_gen(@@ -38,6 +40,8 @@ async fn gpu_type_gen_file(.with_context(|| format!("Failed to generate bindings for {gpu_type_enum:?}")),GpuType::Struct(gpu_type_struct) => gpu_type_gen_struct(gpu_type_struct).with_context(|| format!("Failed to generate bindings for {gpu_type_struct:?}")),+ GpuType::OptionSet(gpu_type_option_set) => gpu_type_gen_option_set(gpu_type_option_set)+ .with_context(|| format!("Failed to generate bindings for {gpu_type_option_set:?}")),}).process_results(|mut it| it.join("\n\n"))?;diff --git a/crates/backend-uzu/build/metal/gpu_types/template.ht b/crates/backend-uzu/build/metal/gpu_types/template.htindex 8696754a5..ed7f31935 100644--- a/crates/backend-uzu/build/metal/gpu_types/template.ht+++ b/crates/backend-uzu/build/metal/gpu_types/template.ht@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/{module_name}.rs - do not edit manually+// Auto-generated from gpu_types/{module_name} - do not edit manually#pragma once#include <metal_stdlib>diff --git a/crates/backend-uzu/build/metal/wrapper.rs b/crates/backend-uzu/build/metal/wrapper.rsindex cc2280daf..6ee119cbf 100644--- a/crates/backend-uzu/build/metal/wrapper.rs+++ b/crates/backend-uzu/build/metal/wrapper.rs@@ -211,7 +211,7 @@ fn kernel_wrappers(wrapper_arguments.push("uint3 __dsl_axis_idx [[thread_position_in_grid]]".into());}- if kernel.has_groups() {+ if kernel.has_groups() || kernel.has_thread_context() {wrapper_arguments.push("uint3 __dsl_group_idx [[threadgroup_position_in_grid]]".into());}@@ -257,7 +257,7 @@ fn kernel_wrappers(MetalArgumentType::Threads(_) => {format!("__dsl_thread_idx.{}", thread_axis_letters.next().unwrap())},- MetalArgumentType::ThreadContext => "ThreadContext { .simdgroup_index = __dsl_simd_lane_idx, .threadgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),+ MetalArgumentType::ThreadContext => "ThreadContext { .simd_lane_id = __dsl_simd_lane_idx, .simdgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup, .threadgroup_position = __dsl_group_idx }".into(),}).collect::<Vec<_>>().join(", ")diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime.rs b/crates/backend-uzu/src/audio/nanocodec/runtime.rsindex 65304cde7..8ad4a33c1 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime.rs@@ -22,8 +22,8 @@ use crate::{},},config::{- ConfigDataType, DecoderConfig, DescriptAudioCodecConfig, EmbeddingConfig, EmbeddingConfigCommon,- NormalizationConfig, TtsAudioDecoderConfig, TtsConfig,+ DecoderConfig, DescriptAudioCodecConfig, EmbeddingConfig, EmbeddingConfigCommon, NormalizationConfig,+ TtsAudioDecoderConfig, TtsConfig,},encodable_block::{Decoder, LayerExecutables, RMSNorm},forward_pass::{model_shape::ModelShape, state::SharedBuffers},diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rsindex 6f75dde5a..e066650fe 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs@@ -1,7 +1,7 @@use super::*;pub(super) fn resolve_descript_audio_codec_vocoder_data_type(- top_level_precision: ConfigDataType,+ top_level_precision: DataType,config: &DescriptAudioCodecConfig,) -> AudioResult<DataType> {let resolved_precision = top_level_precision;diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rsnew file mode 100644index 000000000..a12f8652f--- /dev/null+++ b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs@@ -0,0 +1,50 @@+use bitflags::bitflags;+use derive_more::Display;++#[repr(C)]+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]+pub enum GemmInputPrologueKind {+ FullPrecision,+ ExternalRht,+}++#[repr(C)]+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]+pub enum GemmOutputTransformKind {+ Store,+ Scale,+ Accumulate,+ Bias,+ Rht,+ ScaleAccumulate,+ ScaleAccumulateBias,+ ScaleAccumulateBiasRht,+}++#[repr(C)]+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]+pub enum GemmWeightPrologueKind {+ FullPrecision,+ ScaleBiasDequant,+ ScaleZeroPointDequant,+}++#[repr(C)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct GemmTilingConfig {+ pub threadgroup_m: u32,+ pub threadgroup_n: u32,+ pub threadgroup_k: u32,+ pub simdgroups_m: u32,+ pub simdgroups_n: u32,+}++bitflags! {+ #[repr(transparent)]+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+ pub struct GemmAlignment: u32 {+ const M = 1 << 0;+ const N = 1 << 1;+ const K = 1 << 2;+ }+}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/matmul.rs b/crates/backend-uzu/src/backends/common/gpu_types/matmul.rsindex 26089436f..ebb3a2442 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/matmul.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/matmul.rs@@ -11,7 +11,7 @@ pub struct GemmParams {pub leading_dimension_d: u32,pub threadgroups_per_column: u32,pub threadgroups_per_row: u32,- pub swizzle_log: u32,pub aligned_inner_iterations: u32,pub use_morton: bool,+ pub ab_scale: f32,}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs b/crates/backend-uzu/src/backends/common/gpu_types/mod.rsindex cb4a46b70..8d39a391d 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs@@ -6,6 +6,7 @@pub mod activation_type;pub mod argmax;pub mod attention;+pub mod gemm;pub mod kv_cache_update;pub mod matmul;pub mod quantization;diff --git a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rsindex 38445caf1..ea72aba6e 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs@@ -1,66 +1,42 @@+use derive_more::Display;use proc_macros::uzu_config;use crate::DataType;#[repr(C)]-#[derive(Copy, Eq)]+#[derive(Display, Copy, Eq, Hash)]#[uzu_config]-#[serde(rename_all = "lowercase")]pub enum QuantizationMode {- UINT4,- INT8,- UINT8,+ #[serde(rename = "uint4")]+ U4,+ #[serde(rename = "int8")]+ I8,+ #[serde(rename = "uint8")]+ U8,}impl QuantizationMode {pub fn packing_divisor(&self) -> usize {- match self {- QuantizationMode::UINT4 => 2,- QuantizationMode::INT8 => 1,- QuantizationMode::UINT8 => 1,- }+ let bits = DataType::from(*self).size_in_bits();+ assert_eq!(8 % bits, 0, "QuantizationMode bit width ({bits}) must divide 8 evenly");+ 8 / bits}pub fn storage_type(&self) -> DataType {match self {- QuantizationMode::UINT4 => DataType::U8,- QuantizationMode::INT8 => DataType::I8,- QuantizationMode::UINT8 => DataType::U8,+ QuantizationMode::U4 => DataType::U8,+ QuantizationMode::I8 => DataType::I8,+ QuantizationMode::U8 => DataType::U8,}}-- pub fn to_u32(&self) -> u32 {- (*self).into()- }}impl From<QuantizationMode> for DataType {fn from(val: QuantizationMode) -> Self {match val {- QuantizationMode::UINT4 => DataType::U4,- QuantizationMode::INT8 => DataType::I8,- QuantizationMode::UINT8 => DataType::U8,- }- }-}--impl From<u32> for QuantizationMode {- fn from(val: u32) -> Self {- match val {- 0 => QuantizationMode::UINT4,- 1 => QuantizationMode::INT8,- 2 => QuantizationMode::UINT8,- _ => panic!("Invalid QuantizationMode value: {val}"),- }- }-}--impl Into<u32> for QuantizationMode {- fn into(self) -> u32 {- match self {- QuantizationMode::UINT4 => 0,- QuantizationMode::INT8 => 1,- QuantizationMode::UINT8 => 2,+ QuantizationMode::U4 => DataType::U4,+ QuantizationMode::I8 => DataType::I8,+ QuantizationMode::U8 => DataType::U8,}}}diff --git a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rsindex 39c345e26..75c1acdd0 100644--- a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs@@ -97,8 +97,8 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {}let bits = match configuration.mode {- QuantizationMode::UINT4 => 4,- QuantizationMode::INT8 | QuantizationMode::UINT8 => 8,+ QuantizationMode::U4 => 4,+ QuantizationMode::I8 | QuantizationMode::U8 => 8,};let group_size = configuration.group_size as u32;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex 44be4fa5e..f63a9dae9 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -21,15 +21,9 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(group_size: u32,#[allow(unused)]#[specialize]- quant_mode: u32,+ quantization_mode: QuantizationMode,) {- let quant_mode = QuantizationMode::from(quant_mode);-- let packing_divisor: u32 = if quant_mode == QuantizationMode::UINT4 {- 2- } else {- 1- };+ let packing_divisor = quantization_mode.packing_divisor() as u32;let weights_stride = model_dim / packing_divisor;let num_groups = (model_dim + group_size - 1) / group_size;@@ -49,8 +43,8 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(let scale = *scales.add((token_id as u32 * num_groups + group_idx) as usize);let bias = *biases.add((token_id as u32 * num_groups + group_idx) as usize);- let quantized_value: i32 = match quant_mode {- QuantizationMode::UINT4 => {+ let quantized_value: i32 = match quantization_mode {+ QuantizationMode::U4 => {let byte_idx = (token_id as u32 * weights_stride + dim_idx / 2) as usize;let packed = *weights.add(byte_idx);if (dim_idx & 1) == 0 {@@ -59,12 +53,12 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(((packed >> 4) & 0x0F) as i32}},- QuantizationMode::INT8 => {+ QuantizationMode::I8 => {let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;let weights_i8 = weights as *const i8;*weights_i8.add(elem_idx) as i32},- QuantizationMode::UINT8 => {+ QuantizationMode::U8 => {let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;*weights.add(elem_idx) as i32},diff --git a/crates/backend-uzu/src/backends/metal/dense_buffer.rs b/crates/backend-uzu/src/backends/metal/dense_buffer.rsindex bba78219e..761821331 100644--- a/crates/backend-uzu/src/backends/metal/dense_buffer.rs+++ b/crates/backend-uzu/src/backends/metal/dense_buffer.rs@@ -14,7 +14,7 @@ impl Buffer for Retained<ProtocolObject<dyn MTLBuffer>> {}fn size(&self) -> usize {- (**self).length()+ self.length()}}diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rsindex a14e3459a..250432ce7 100644--- a/crates/backend-uzu/src/backends/metal/error.rs+++ b/crates/backend-uzu/src/backends/metal/error.rs@@ -1,5 +1,7 @@use thiserror::Error;+use crate::backends::metal::kernel::matmul::gemm::GemmSpecializationError;+#[derive(Debug, Error)]pub enum MetalError {#[error("Cannot open device")]@@ -24,6 +26,8 @@ pub enum MetalError {CannotCreateFunction,#[error("Cannot create pipeline state: {0}")]CannotCreatePipelineState(String),+ #[error("Invalid GEMM specialization: {0:?}")]+ InvalidGemmSpecialization(GemmSpecializationError),#[error("Can not allocate buffer with size={0}")]SparseBufferAlloc(usize),#[error("Can not allocate heap with size={0} and page size={1}")]diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex a4ac61a72..fd40c931f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -139,27 +139,11 @@ PUBLIC KERNEL(AttentionGemm)(const int key_source_stride = int(params.k_strides[2]);const int value_source_stride = int(params.v_strides[2]);- thread QueryLoader query_loader(- q,- query_source_stride,- query_shared,- thread_context.threadgroup_index,- thread_context.simdgroup_index- );- thread KeyLoader key_loader(- k,- key_source_stride,- key_shared,- thread_context.threadgroup_index,- thread_context.simdgroup_index- );- thread ValueLoader value_loader(- v,- value_source_stride,- value_shared,- thread_context.threadgroup_index,- thread_context.simdgroup_index- );+ thread QueryLoader+ query_loader(q, query_source_stride, query_shared, thread_context);+ thread KeyLoader key_loader(k, key_source_stride, key_shared, thread_context);+ thread ValueLoader+ value_loader(v, value_source_stride, value_shared, thread_context);TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));@@ -224,13 +208,13 @@ PUBLIC KERNEL(AttentionGemm)(// Lane coordinates and pointer offsetsconst short2 lane_coordinates =SimdgroupMultiplyAccumulateType::get_lane_coordinates(- thread_context.simdgroup_index+ thread_context.simd_lane_id);const short lane_row = lane_coordinates.y;const short lane_col = lane_coordinates.x;const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS *- short(thread_context.threadgroup_index);+ short(thread_context.simdgroup_index);const short query_shared_offset =(simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex 368cc31e6..e9c89c2f7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -74,16 +74,16 @@ PUBLIC KERNEL(AttentionSinglePass)(: suffix_position + q_seq_idx;queries += q_offset * HEAD_DIM +- thread_context.simdgroup_index * qk_elements_per_thread;+ thread_context.simd_lane_id * qk_elements_per_thread;keys += kv_head_idx * k_head_stride +- thread_context.threadgroup_index * k_seq_stride +- thread_context.simdgroup_index * qk_elements_per_thread;+ thread_context.simdgroup_index * k_seq_stride ++ thread_context.simd_lane_id * qk_elements_per_thread;values += kv_head_idx * v_head_stride +- thread_context.threadgroup_index * v_seq_stride +- thread_context.simdgroup_index * value_elements_per_thread;+ thread_context.simdgroup_index * v_seq_stride ++ thread_context.simd_lane_id * value_elements_per_thread;out += o_offset * value_dim +- thread_context.threadgroup_index * value_elements_per_thread;+ thread_context.simdgroup_index * value_elements_per_thread;// Read the query and 0 the output accumulatorfor (int i = 0; i < qk_elements_per_thread; i++) {@@ -95,7 +95,7 @@ PUBLIC KERNEL(AttentionSinglePass)(U max_score = -INFINITY;U sum_exp_score = 0;- if (has_sinks && thread_context.threadgroup_index == 0) {+ if (has_sinks && thread_context.simdgroup_index == 0) {const int num_q_heads = static_cast<int>(num_heads);int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);@@ -103,7 +103,7 @@ PUBLIC KERNEL(AttentionSinglePass)(}// For each key- for (uint i = thread_context.threadgroup_index; i < sequence_length;+ for (uint i = thread_context.simdgroup_index; i < sequence_length;i += SEQUENCE_BLOCK_SIZE) {if (should_use_key(ring_params,@@ -151,27 +151,27 @@ PUBLIC KERNEL(AttentionSinglePass)(}// Each thread has a partial part of the output so we need to combine them.- if (thread_context.simdgroup_index == 0) {- shared_max_scores[thread_context.threadgroup_index] = max_score;- shared_sum_exp_scores[thread_context.threadgroup_index] = sum_exp_score;+ if (thread_context.simd_lane_id == 0) {+ shared_max_scores[thread_context.simdgroup_index] = max_score;+ shared_sum_exp_scores[thread_context.simdgroup_index] = sum_exp_score;}threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = shared_max_scores[thread_context.simdgroup_index];+ max_score = shared_max_scores[thread_context.simd_lane_id];U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);sum_exp_score =- simd_sum(shared_sum_exp_scores[thread_context.simdgroup_index] * factor);+ simd_sum(shared_sum_exp_scores[thread_context.simd_lane_id] * factor);// Now we need to aggregate all the outputsfor (int i = 0; i < value_elements_per_thread; i++) {shared_outputs- [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +- thread_context.threadgroup_index] = o[i];+ [thread_context.simd_lane_id * HEAD_BLOCK_SIZE ++ thread_context.simdgroup_index] = o[i];threadgroup_barrier(mem_flags::mem_threadgroup);o[i] = simd_sum(shared_outputs- [thread_context.threadgroup_index * HEAD_BLOCK_SIZE +- thread_context.simdgroup_index] *+ [thread_context.simdgroup_index * HEAD_BLOCK_SIZE ++ thread_context.simd_lane_id] *factor) /sum_exp_score;@@ -179,7 +179,7 @@ PUBLIC KERNEL(AttentionSinglePass)(}// And write the output- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {for (int i = 0; i < value_elements_per_thread; i++) {out[i] = static_cast<T>(o[i]);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex 3c36971e8..713629426 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -57,7 +57,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(// Flat query head mapping — fixed hardware shape, dynamic logical mappingconst uint head_idx = head_group_idx * SIMDGROUPS_PER_THREADGROUP +- thread_context.threadgroup_index;+ thread_context.simdgroup_index;if (head_idx >= num_heads)return;@@ -74,14 +74,14 @@ PUBLIC KERNEL(AttentionTwoPass1)(thread U q[elements_per_thread];thread U o[elements_per_thread] = {0};- queries += q_offset * HEAD_DIM +- thread_context.simdgroup_index * elements_per_thread;+ queries +=+ q_offset * HEAD_DIM + thread_context.simd_lane_id * elements_per_thread;keys += kv_head_idx * k_head_stride + block_idx * k_seq_stride +- thread_context.simdgroup_index * elements_per_thread;+ thread_context.simd_lane_id * elements_per_thread;values += kv_head_idx * v_head_stride + block_idx * v_seq_stride +- thread_context.simdgroup_index * elements_per_thread;+ thread_context.simd_lane_id * elements_per_thread;out += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM +- thread_context.simdgroup_index * elements_per_thread;+ thread_context.simd_lane_id * elements_per_thread;sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;@@ -135,7 +135,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(}// Write partial output- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {sums[0] = sum_exp_score;maxs[0] = max_score;}@@ -170,7 +170,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(// Find global max across all blocks (each lane handles strided blocks)U max_score = -1e9;- for (uint b = thread_context.simdgroup_index; b < TOTAL_BLOCKS_COUNT;+ for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT;b += METAL_SIMD_SIZE) {max_score = max(max_score, maxs[b]);}@@ -178,7 +178,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(// Find global sum across all blocksU sum_exp = 0;- for (uint b = thread_context.simdgroup_index; b < TOTAL_BLOCKS_COUNT;+ for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT;b += METAL_SIMD_SIZE) {sum_exp += sums[b] * fast::exp(maxs[b] - new_max);}@@ -189,7 +189,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(thread U o[elements_per_thread] = {0};for (uint chunk = 0; chunk < TOTAL_BLOCKS_COUNT; chunk += METAL_SIMD_SIZE) {- uint block_idx = chunk + thread_context.simdgroup_index;+ uint block_idx = chunk + thread_context.simd_lane_id;U factor = (block_idx < TOTAL_BLOCKS_COUNT)? fast::exp(maxs[block_idx] - new_max): 0;@@ -197,7 +197,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(const device float* block_partials =partials + o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +block_idx * HEAD_DIM +- thread_context.threadgroup_index * elements_per_thread;+ thread_context.simdgroup_index * elements_per_thread;for (uint i = 0; i < elements_per_thread; i++) {U val = (block_idx < TOTAL_BLOCKS_COUNT) ? block_partials[i] * factor : 0;@@ -206,9 +206,9 @@ PUBLIC KERNEL(AttentionTwoPass2)(}// Write the output- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {out += o_offset * HEAD_DIM +- thread_context.threadgroup_index * elements_per_thread;+ thread_context.simdgroup_index * elements_per_thread;for (uint i = 0; i < elements_per_thread; i++) {out[i] = static_cast<T>(o[i] / sum_exp_score);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h b/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.hindex f0fa45ae7..4c0691786 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h@@ -4,8 +4,13 @@using namespace metal;struct ThreadContext {- uint simdgroup_index;- uint threadgroup_index;- uint simdgroup_size;- uint simdgroups_per_threadgroup;+ uint simd_lane_id; // [[thread_index_in_simdgroup]] — 0..simdgroup_size+ uint simdgroup_index; // [[simdgroup_index_in_threadgroup]] —+ // 0..simdgroups_per_threadgroup+ uint simdgroup_size; // [[threads_per_simdgroup]] — typically 32+ uint simdgroups_per_threadgroup; // [[simdgroups_per_threadgroup]]+ uint3 threadgroup_position; // [[threadgroup_position_in_grid]]+ uint3 threadgroup_size;+ uint3 threadgroup_count;+ uint3 grid_size;};diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.hindex 34a0bf229..df37462bc 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h@@ -38,23 +38,23 @@ static typename Op::value_type threadgroup_cooperative_reduce() {// Phase 1: reduce within each simdgrouptypename Op::value_type local = Op::simd_reduce(value);- if (thread_context.simdgroup_index == 0) {- shared[thread_context.threadgroup_index] = local;+ if (thread_context.simd_lane_id == 0) {+ shared[thread_context.simdgroup_index] = local;}threadgroup_barrier(mem_flags::mem_threadgroup);// Phase 2: first simdgroup reduces across simdgropus- if (thread_context.threadgroup_index == 0) {+ if (thread_context.simdgroup_index == 0) {typename Op::value_type total =- thread_context.simdgroup_index <+ thread_context.simd_lane_id <(BLOCK_SIZE + thread_context.simdgroup_size - 1) /thread_context.simdgroup_size- ? shared[thread_context.simdgroup_index]+ ? shared[thread_context.simd_lane_id]: Op::identity;total = Op::simd_reduce(total);- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {shared[0] = total;}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex f06629847..d0852fb1d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -17,13 +17,10 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (constant uint32_t& model_dim,constant float& input_scale,const uint32_t group_size SPECIALIZE,- const uint32_t quant_mode SPECIALIZE,+ const QuantizationMode quantization_mode SPECIALIZE,const uint dim_idx AXIS(model_dim, 16),const uint batch_idx AXIS(batch_size, 16)) {- const QuantizationMode quantization_mode = QuantizationMode(- quant_mode- ); // TODO: should be accepted as a kernel argumentconst uint thread_position_in_grid = batch_idx * model_dim + dim_idx;const uint64_t token_id = token_ids[batch_idx];if (token_id >= vocab_size) {@@ -38,12 +35,12 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (const T bias = biases[token_id * num_groups + group_idx];const uint packing_divisor =- quantization_mode == QuantizationMode::UINT4 ? 2 : 1;+ quantization_mode == QuantizationMode::U4 ? 2 : 1;const uint weights_stride = model_dim / packing_divisor;int quantized_value = 0;switch (quantization_mode) {- case QuantizationMode::UINT4: {+ case QuantizationMode::U4: {const uint byte_idx = token_id * weights_stride + (dim_idx / 2);const uint8_t packed = weights[byte_idx];if ((dim_idx & 1) == 0) {@@ -53,14 +50,14 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (}break;};- case QuantizationMode::INT8: {+ case QuantizationMode::I8: {const uint elem_idx = token_id * weights_stride + dim_idx;const device int8_t* weights_i8 =reinterpret_cast<const device int8_t*>(weights);quantized_value = int(weights_i8[elem_idx]);break;};- case QuantizationMode::UINT8: {+ case QuantizationMode::U8: {const uint elem_idx = token_id * weights_stride + dim_idx;quantized_value = int(weights[elem_idx]);break;diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.hindex c5f89ed64..eda99e0a2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/activation_type.rs - do not edit manually+// Auto-generated from gpu_types/activation_type - do not edit manually#pragma once#include <metal_stdlib>diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/argmax.h b/crates/backend-uzu/src/backends/metal/kernel/generated/argmax.hindex 3b818d559..a67743173 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/argmax.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/argmax.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/argmax.rs - do not edit manually+// Auto-generated from gpu_types/argmax - do not edit manually#pragma once#include <metal_stdlib>diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h b/crates/backend-uzu/src/backends/metal/kernel/generated/attention.hindex 30680f516..55951bc81 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/attention.rs - do not edit manually+// Auto-generated from gpu_types/attention - do not edit manually#pragma once#include <metal_stdlib>diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.hnew file mode 100644index 000000000..b21071fa8--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h@@ -0,0 +1,49 @@+// Auto-generated from gpu_types/gemm - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::gemm {+enum class GemmInputPrologueKind : uint32_t {+ FullPrecision = 0,+ ExternalRht = 1,+};++enum class GemmOutputTransformKind : uint32_t {+ Store = 0,+ Scale = 1,+ Accumulate = 2,+ Bias = 3,+ Rht = 4,+ ScaleAccumulate = 5,+ ScaleAccumulateBias = 6,+ ScaleAccumulateBiasRht = 7,+};++enum class GemmWeightPrologueKind : uint32_t {+ FullPrecision = 0,+ ScaleBiasDequant = 1,+ ScaleZeroPointDequant = 2,+};++typedef struct {+ uint32_t threadgroup_m;+ uint32_t threadgroup_n;+ uint32_t threadgroup_k;+ uint32_t simdgroups_m;+ uint32_t simdgroups_n;+} GemmTilingConfig;++struct GemmAlignment {+ uint raw_value;+ constexpr GemmAlignment() thread : raw_value(0) {}+ constexpr GemmAlignment(uint __dsl_v) thread : raw_value(__dsl_v) {}+ static constant constexpr uint M = 1 << 0;+ static constant constexpr uint N = 1 << 1;+ static constant constexpr uint K = 1 << 2;+ constexpr bool contains(uint flag) const thread {+ return (raw_value & flag) != 0;+ }+};+} // namespace uzu::gemmdiff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/kv_cache_update.h b/crates/backend-uzu/src/backends/metal/kernel/generated/kv_cache_update.hindex c42228faa..b4a175e60 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/kv_cache_update.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/kv_cache_update.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/kv_cache_update.rs - do not edit manually+// Auto-generated from gpu_types/kv_cache_update - do not edit manually#pragma once#include <metal_stdlib>diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/matmul.h b/crates/backend-uzu/src/backends/metal/kernel/generated/matmul.hindex 400808477..072833b93 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/matmul.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/matmul.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/matmul.rs - do not edit manually+// Auto-generated from gpu_types/matmul - do not edit manually#pragma once#include <metal_stdlib>@@ -14,8 +14,8 @@ typedef struct {uint32_t leading_dimension_d;uint32_t threadgroups_per_column;uint32_t threadgroups_per_row;- uint32_t swizzle_log;uint32_t aligned_inner_iterations;bool use_morton;+ float ab_scale;} GemmParams;} // namespace uzu::matmuldiff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization.h b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization.hindex 0315c17d3..01c1ffba7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/quantization.rs - do not edit manually+// Auto-generated from gpu_types/quantization - do not edit manually#pragma once#include <metal_stdlib>@@ -6,8 +6,8 @@ using namespace metal;namespace uzu::quantization {enum class QuantizationMode : uint32_t {- UINT4 = 0,- INT8 = 1,- UINT8 = 2,+ U4 = 0,+ I8 = 1,+ U8 = 2,};} // namespace uzu::quantizationdiff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.hindex 98bd012aa..3a265d899 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/quantization_method.rs - do not edit manually+// Auto-generated from gpu_types/quantization_method - do not edit manually#pragma once#include <metal_stdlib>diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/ring.h b/crates/backend-uzu/src/backends/metal/kernel/generated/ring.hindex a10f4c220..003e6f971 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/ring.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/ring.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/ring.rs - do not edit manually+// Auto-generated from gpu_types/ring - do not edit manually#pragma once#include <metal_stdlib>diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/trie.h b/crates/backend-uzu/src/backends/metal/kernel/generated/trie.hindex 0ccfaf5d1..80b04ee4b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/trie.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/trie.h@@ -1,4 +1,4 @@-// Auto-generated from gpu_types/trie.rs - do not edit manually+// Auto-generated from gpu_types/trie - do not edit manually#pragma once#include <metal_stdlib>diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.hdeleted file mode 100644index 278600759..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h+++ /dev/null@@ -1,100 +0,0 @@-#pragma once--#include "../../common/integral_constant.h"-#include "../../common/thread_context.h"-#include "defines.h"--using namespace metal;--namespace uzu {-namespace matmul {--///////////////////////////////////////////////////////////////////////////////-// AluFragmentOps - stateless traits/ops for 8x8 ALU matmul (simdgroup_matrix)-///////////////////////////////////////////////////////////////////////////////--template <typename T>-struct AluFragmentOps {- METAL_CONST ushort FRAGMENT_ROWS = 8;- METAL_CONST ushort FRAGMENT_COLS = 8;- METAL_CONST ushort ELEMENTS_PER_THREAD =- (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;- METAL_CONST ushort THREAD_ELEMENT_ROWS = 1;- METAL_CONST ushort THREAD_ELEMENT_COLS = 2;- METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = 1;-- static_assert(- THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,- "AluFragmentOps shape is not consistent with element count"- );-- typedef metal::simdgroup_matrix<T, FRAGMENT_ROWS, FRAGMENT_COLS>- SimdgroupMatrixType;-- template <typename U>- using ThreadVector = metal::vec<U, ELEMENTS_PER_THREAD>;-- METAL_FUNC static constexpr void multiply_accumulate(- thread ThreadVector<T>& D,- thread ThreadVector<T>& A,- thread ThreadVector<T>& B,- thread ThreadVector<T>& C- ) {- SimdgroupMatrixType D_mat;- SimdgroupMatrixType A_mat;- SimdgroupMatrixType B_mat;- SimdgroupMatrixType C_mat;-- reinterpret_cast<thread ThreadVector<T>&>(A_mat.thread_elements()) = A;- reinterpret_cast<thread ThreadVector<T>&>(B_mat.thread_elements()) = B;- reinterpret_cast<thread ThreadVector<T>&>(C_mat.thread_elements()) = C;-- simdgroup_multiply_accumulate(D_mat, A_mat, B_mat, C_mat);-- D = reinterpret_cast<thread ThreadVector<T>&>(D_mat.thread_elements());- }-- template <class OutputTile, class LeftTile, class RightTile>- METAL_FUNC static void tile_matmul(- thread OutputTile& output,- thread LeftTile& left,- thread RightTile& right- ) {- constexpr ushort tile_m = OutputTile::TILE_ROWS;- constexpr ushort tile_n = OutputTile::TILE_COLS;- constexpr ushort tile_k = LeftTile::TILE_COLS;-- static_assert(- tile_m == LeftTile::TILE_ROWS,- "tile matmul: M dimensions do not match"- );- static_assert(- tile_n == RightTile::TILE_COLS,- "tile matmul: N dimensions do not match"- );- static_assert(- tile_k == RightTile::TILE_ROWS,- "tile matmul: K dimensions do not match"- );-- METAL_PRAGMA_UNROLL- for (ushort row = 0; row < tile_m; ++row) {- METAL_PRAGMA_UNROLL- for (ushort col = 0; col < tile_n; ++col) {- const ushort column_serpentine = (row % 2) ? (tile_n - 1 - col) : col;- METAL_PRAGMA_UNROLL- for (ushort k = 0; k < tile_k; ++k) {- multiply_accumulate(- output.fragment_at(row, column_serpentine),- left.fragment_at(row, k),- right.fragment_at(k, column_serpentine),- output.fragment_at(row, column_serpentine)- );- }- }- }- }-};--} // namespace matmul-} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.hindex b67370a2a..3f2973d06 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h@@ -60,11 +60,11 @@ struct Fragment {METAL_FUNC static constexpr short2 get_position(const thread ThreadContext& thread_context) {- const ushort simdgroup_index = ushort(thread_context.simdgroup_index);- const short quad = simdgroup_index / 4;- const short row = (quad & 4) + (simdgroup_index / 2) % 4;+ const ushort simd_lane_id = ushort(thread_context.simd_lane_id);+ const short quad = simd_lane_id / 4;+ const short row = (quad & 4) + (simd_lane_id / 2) % 4;const short col =- ((quad & 2) + simdgroup_index % 2) * Ops::THREAD_ELEMENT_COLS;+ ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;return short2{col, row};}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.hdeleted file mode 100644index 6b1251c0a..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h+++ /dev/null@@ -1,355 +0,0 @@-#pragma once--#include "loader.h"-#include "threadgroup_tile.h"-#include "../../generated/matmul.h"--using namespace metal;--namespace uzu {-namespace matmul {--///////////////////////////////////////////////////////////////////////////////-// ThreadgroupGemm Kernel-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- typename U,- int BLOCK_ROWS,- int BLOCK_COLS,- int BLOCK_DEPTH,- int SIMDGROUPS_PER_ROW,- int SIMDGROUPS_PER_COLUMN,- bool transpose_a,- bool transpose_b,- bool MN_aligned,- bool K_aligned,- typename AccumulatorType = float,- typename Epilogue = TransformNone<U, AccumulatorType>>-struct ThreadgroupGemm {- METAL_CONST ushort THREADGROUP_PADDING_A = 16 / sizeof(T);- METAL_CONST ushort THREADGROUP_PADDING_B = 16 / sizeof(T);- METAL_CONST ushort THREADGROUP_MEMORY_SIZE_A =- transpose_a ? BLOCK_DEPTH * (BLOCK_ROWS + THREADGROUP_PADDING_A)- : BLOCK_ROWS * (BLOCK_DEPTH + THREADGROUP_PADDING_A);- METAL_CONST ushort THREADGROUP_MEMORY_SIZE_B =- transpose_b ? BLOCK_COLS * (BLOCK_DEPTH + THREADGROUP_PADDING_B)- : BLOCK_DEPTH * (BLOCK_COLS + THREADGROUP_PADDING_B);- METAL_CONST ushort THREADGROUP_MEMORY_SIZE =- THREADGROUP_MEMORY_SIZE_A + THREADGROUP_MEMORY_SIZE_B;-- METAL_CONST ushort THREADGROUP_SIZE =- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;-- using LoaderAType = ThreadgroupLoader<- T,- transpose_a ? BLOCK_DEPTH : BLOCK_ROWS,- transpose_a ? BLOCK_ROWS : BLOCK_DEPTH,- transpose_a ? BLOCK_ROWS + THREADGROUP_PADDING_A- : BLOCK_DEPTH + THREADGROUP_PADDING_A,- !transpose_a,- THREADGROUP_SIZE>;- using LoaderBType = ThreadgroupLoader<- T,- transpose_b ? BLOCK_COLS : BLOCK_DEPTH,- transpose_b ? BLOCK_DEPTH : BLOCK_COLS,- transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B- : BLOCK_COLS + THREADGROUP_PADDING_B,- transpose_b,- THREADGROUP_SIZE>;- using ThreadgroupTileType = ThreadgroupTile<- T,- U,- BLOCK_ROWS,- BLOCK_COLS,- BLOCK_DEPTH,- SIMDGROUPS_PER_ROW,- SIMDGROUPS_PER_COLUMN,- transpose_a,- transpose_b,- transpose_a ? BLOCK_ROWS + THREADGROUP_PADDING_A- : BLOCK_DEPTH + THREADGROUP_PADDING_A,- transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B- : BLOCK_COLS + THREADGROUP_PADDING_B,- AccumulatorType,- Epilogue>;-- template <bool M_aligned, bool N_aligned, bool K_aligned_>- static METAL_FUNC void gemm_loop(- threadgroup T* a_shared,- threadgroup T* b_shared,- const int aligned_k_iterations,- thread LoaderAType& loader_a,- thread LoaderBType& loader_b,- thread ThreadgroupTileType& threadgroup_tile,- thread const ushort& threadgroup_block_rows,- thread const ushort& threadgroup_block_cols,- thread const ushort& leftover_block_depth- ) {- short2 tile_dimensions_a =- transpose_a ? short2(threadgroup_block_rows, BLOCK_DEPTH)- : short2(BLOCK_DEPTH, threadgroup_block_rows);- short2 tile_dimensions_b =- transpose_b ? short2(BLOCK_DEPTH, threadgroup_block_cols)- : short2(threadgroup_block_cols, BLOCK_DEPTH);-- for (int k = 0; k < aligned_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- if (M_aligned) {- loader_a.load_unsafe();- } else {- loader_a.load_safe(tile_dimensions_a);- }-- if (N_aligned) {- loader_b.load_unsafe();- } else {- loader_b.load_safe(tile_dimensions_b);- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- threadgroup_tile.multiply_accumulate(a_shared, b_shared);-- loader_a.next();- loader_b.next();- }-- if (!K_aligned_) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- short2 last_tile_dimensions_a =- transpose_a ? short2(threadgroup_block_rows, leftover_block_depth)- : short2(leftover_block_depth, threadgroup_block_rows);- short2 last_tile_dimensions_b =- transpose_b ? short2(leftover_block_depth, threadgroup_block_cols)- : short2(threadgroup_block_cols, leftover_block_depth);-- loader_a.load_safe(last_tile_dimensions_a);- loader_b.load_safe(last_tile_dimensions_b);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- threadgroup_tile.multiply_accumulate(a_shared, b_shared);- }- }-- static METAL_FUNC void run(- const device T* a,- const device T* b,- device U* d,- const constant GemmParams* params,- const constant float& ab_scale,- const bool is_accumulate,- threadgroup T* a_shared,- threadgroup T* b_shared,- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint2 threadgroup_position [[threadgroup_position_in_grid]],- uint3 thread_position [[thread_position_in_threadgroup]]- ) {- (void)thread_position;-- const int swizzle_stride = pow2(params->swizzle_log);- const int swizzled_row_id = threadgroup_position.y * swizzle_stride +- (threadgroup_position.x % swizzle_stride);- const int swizzled_col_id = threadgroup_position.x / swizzle_stride;-- if (params->threadgroups_per_row <= swizzled_col_id ||- params->threadgroups_per_column <= swizzled_row_id) {- return;- }-- threadgroup_barrier(mem_flags::mem_none);-- const int output_row = swizzled_row_id * BLOCK_ROWS;- const int output_col = swizzled_col_id * BLOCK_COLS;- const size_t output_row_long = size_t(output_row);- const size_t output_col_long = size_t(output_col);-- a += transpose_a ? output_row_long- : output_row_long * params->leading_dimension_a;- b += transpose_b ? output_col_long * params->leading_dimension_b- : output_col_long;- d += output_row_long * params->leading_dimension_d + output_col_long;-- thread LoaderAType loader_a(- a,- params->leading_dimension_a,- a_shared,- simd_group_id,- simd_lane_id- );- thread LoaderBType loader_b(- b,- params->leading_dimension_b,- b_shared,- simd_group_id,- simd_lane_id- );-- thread ThreadgroupTileType threadgroup_tile(simd_group_id, simd_lane_id);-- TransformScaleAccumulate<AccumulatorType, AccumulatorType> epilogue(- ab_scale,- is_accumulate ? 1.0f : 0.0f- );-- ///////////////////////////////////////////////////////////////////////////- // MNK aligned loop- if (MN_aligned) {- for (int k = 0; k < params->aligned_inner_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_a.load_unsafe();- loader_b.load_unsafe();-- threadgroup_barrier(mem_flags::mem_threadgroup);-- threadgroup_tile.multiply_accumulate(a_shared, b_shared);-- loader_a.next();- loader_b.next();- }-- threadgroup_barrier(mem_flags::mem_none);-- if (!K_aligned) {- int leftover_block_depth =- params->K - params->aligned_inner_iterations * BLOCK_DEPTH;- short2 tile_dimensions_a =- transpose_a ? short2(BLOCK_ROWS, leftover_block_depth)- : short2(leftover_block_depth, BLOCK_ROWS);- short2 tile_dimensions_b =- transpose_b ? short2(leftover_block_depth, BLOCK_COLS)- : short2(BLOCK_COLS, leftover_block_depth);-- loader_a.load_safe(tile_dimensions_a);- loader_b.load_safe(tile_dimensions_b);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- threadgroup_tile.multiply_accumulate(a_shared, b_shared);- }-- threadgroup_tile- .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);- threadgroup_tile.store_result(d, params->leading_dimension_d);- return;- }- ///////////////////////////////////////////////////////////////////////////- // MN unaligned loop- else {- ushort threadgroup_block_rows =- min(BLOCK_ROWS, ((int)params->M) - output_row);- ushort threadgroup_block_cols =- min(BLOCK_COLS, ((int)params->N) - output_col);- ushort leftover_block_depth =- params->K - params->aligned_inner_iterations * BLOCK_DEPTH;-- if (threadgroup_block_rows == BLOCK_ROWS &&- threadgroup_block_cols == BLOCK_COLS) {- gemm_loop<true, true, K_aligned>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- threadgroup_tile,- threadgroup_block_rows,- threadgroup_block_cols,- leftover_block_depth- );-- threadgroup_tile- .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);- threadgroup_tile.store_result(d, params->leading_dimension_d);- return;-- } else if (threadgroup_block_cols == BLOCK_COLS) {- gemm_loop<false, true, K_aligned>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- threadgroup_tile,- threadgroup_block_rows,- threadgroup_block_cols,- leftover_block_depth- );-- threadgroup_tile.apply_epilogue_safe(- d,- params->leading_dimension_d,- 1,- short2(threadgroup_block_cols, threadgroup_block_rows),- epilogue- );- threadgroup_tile.store_result_safe(- d,- params->leading_dimension_d,- short2(threadgroup_block_cols, threadgroup_block_rows)- );- return;-- } else if (threadgroup_block_rows == BLOCK_ROWS) {- gemm_loop<true, false, K_aligned>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- threadgroup_tile,- threadgroup_block_rows,- threadgroup_block_cols,- leftover_block_depth- );-- threadgroup_tile.apply_epilogue_safe(- d,- params->leading_dimension_d,- 1,- short2(threadgroup_block_cols, threadgroup_block_rows),- epilogue- );- threadgroup_tile.store_result_safe(- d,- params->leading_dimension_d,- short2(threadgroup_block_cols, threadgroup_block_rows)- );- return;-- } else {- gemm_loop<false, false, K_aligned>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- threadgroup_tile,- threadgroup_block_rows,- threadgroup_block_cols,- leftover_block_depth- );-- threadgroup_tile.apply_epilogue_safe(- d,- params->leading_dimension_d,- 1,- short2(threadgroup_block_cols, threadgroup_block_rows),- epilogue- );- threadgroup_tile.store_result_safe(- d,- params->leading_dimension_d,- short2(threadgroup_block_cols, threadgroup_block_rows)- );- return;- }- }- }-};--} // namespace matmul-} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.hdeleted file mode 100644index 77cc9092d..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h+++ /dev/null@@ -1,204 +0,0 @@-#pragma once--#include "../../common/defines.h"-#include "../../common/integral_constant.h"-using namespace uzu;-#include "fragment.h"-#include "mxu_fragment_ops.h"-#include "mxu_gemm_loop.h"-#include "../../generated/matmul.h"--using namespace metal;--namespace uzu {-namespace matmul {--template <- typename T,- ushort BLOCK_ROWS,- ushort BLOCK_COLS,- ushort SIMDGROUPS_PER_ROW,- ushort SIMDGROUPS_PER_COLUMN,- bool APPLY_AB_SCALE,- bool IS_ACCUMULATE>-struct GemmMppCore {- METAL_CONST ushort SIMDGROUP_BLOCK_M = BLOCK_ROWS / SIMDGROUPS_PER_ROW;- METAL_CONST ushort SIMDGROUP_BLOCK_N = BLOCK_COLS / SIMDGROUPS_PER_COLUMN;-- METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;- METAL_CONST ushort BLOCK_K = 256;-- METAL_CONST ushort TILES_M =- SIMDGROUP_BLOCK_M / MxuFragmentOps::FRAGMENT_ROWS;- METAL_CONST ushort TILES_N =- SIMDGROUP_BLOCK_N / MxuFragmentOps::FRAGMENT_COLS;-- using AccumulatorType = float;-- static METAL_FUNC void run(- const device T* left_matrix,- const device T* right_matrix,- device T* output_matrix,- const constant GemmParams* params,- const bool align_m,- const bool align_n,- const bool align_k,- const float ab_scale,- uint simd_group_id,- uint2 threadgroup_position,- const thread ThreadContext& thread_context- ) {- uint tile_id_x, tile_id_y;- if (params->use_morton) {- uint linear_id = threadgroup_position.x;- uint morton_x = linear_id;- uint morton_y = linear_id >> 1;- morton_x &= 0x55555555u;- morton_x = (morton_x | (morton_x >> 1)) & 0x33333333u;- morton_x = (morton_x | (morton_x >> 2)) & 0x0F0F0F0Fu;- morton_x = (morton_x | (morton_x >> 4)) & 0x00FF00FFu;- morton_x = (morton_x | (morton_x >> 8)) & 0x0000FFFFu;- morton_y &= 0x55555555u;- morton_y = (morton_y | (morton_y >> 1)) & 0x33333333u;- morton_y = (morton_y | (morton_y >> 2)) & 0x0F0F0F0Fu;- morton_y = (morton_y | (morton_y >> 4)) & 0x00FF00FFu;- morton_y = (morton_y | (morton_y >> 8)) & 0x0000FFFFu;- tile_id_x = morton_x;- tile_id_y = morton_y;- } else {- tile_id_x = threadgroup_position.x;- tile_id_y = threadgroup_position.y;- }-- if (tile_id_x >= params->threadgroups_per_row ||- tile_id_y >= params->threadgroups_per_column) {- return;- }-- const uint block_row_start = tile_id_y * BLOCK_ROWS;- const uint block_col_start = tile_id_x * BLOCK_COLS;- const size_t block_row_start_long = size_t(block_row_start);- const size_t block_col_start_long = size_t(block_col_start);-- const device T* left_block_ptr =- left_matrix + block_row_start_long * params->leading_dimension_a;- const device T* right_block_ptr =- right_matrix + block_col_start_long * params->leading_dimension_b;-- const ushort tile_row_offset =- SIMDGROUP_BLOCK_M * (simd_group_id / SIMDGROUPS_PER_COLUMN);- const ushort tile_col_offset =- SIMDGROUP_BLOCK_N * (simd_group_id % SIMDGROUPS_PER_COLUMN);-- device T* output_ptr =- output_matrix + block_row_start_long * params->leading_dimension_d +- block_col_start_long + tile_row_offset * params->leading_dimension_d +- tile_col_offset;-- const short simdgroup_limit_m =- align_m- ? SIMDGROUP_BLOCK_M- : short(- min(int(SIMDGROUP_BLOCK_M),- int(params->M) - int(block_row_start + tile_row_offset))- );- const short simdgroup_limit_n =- align_n- ? SIMDGROUP_BLOCK_N- : short(- min(int(SIMDGROUP_BLOCK_N),- int(params->N) - int(block_col_start + tile_col_offset))- );-- const device T* left_simdgroup_ptr =- left_block_ptr + size_t(tile_row_offset) * params->leading_dimension_a;- const device T* right_simdgroup_ptr =- right_block_ptr +- size_t(tile_col_offset) * int(params->leading_dimension_b);-- const int aligned_k_iterations = int(params->K) / int(BLOCK_K);-- dispatch_bool(align_k, [&](auto aligned_k) {- dispatch_bool(- align_m || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),- [&](auto aligned_m) {- dispatch_bool(- align_n || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),- [&](auto aligned_n) {- auto accumulator_tile = gemm_loop<- T,- SIMDGROUP_BLOCK_M,- SIMDGROUP_BLOCK_N,- SIMDGROUP_BLOCK_K,- BLOCK_K,- false,- true,- aligned_m.value,- aligned_n.value,- aligned_k.value,- AccumulatorType>(- left_simdgroup_ptr,- right_simdgroup_ptr,- int(params->leading_dimension_a),- int(params->leading_dimension_b),- int(params->K),- aligned_k_iterations,- simdgroup_limit_m,- simdgroup_limit_n,- thread_context- );-- if constexpr (APPLY_AB_SCALE) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;- i++) {- accumulator_tile.elements()[i] *=- AccumulatorType(ab_scale);- }- }-- if constexpr (IS_ACCUMULATE) {- Fragment<T, TILES_M, TILES_N, MxuFragmentOps>- existing_output(thread_context);- if constexpr (aligned_m.value && aligned_n.value) {- existing_output.load(- output_ptr,- int(params->leading_dimension_d)- );- } else {- existing_output.load_safe(- output_ptr,- int(params->leading_dimension_d),- short2(simdgroup_limit_n, simdgroup_limit_m)- );- }- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;- i++) {- accumulator_tile.elements()[i] +=- AccumulatorType(existing_output.elements()[i]);- }- }-- if constexpr (aligned_m.value && aligned_n.value) {- accumulator_tile.store(- output_ptr,- int(params->leading_dimension_d)- );- } else {- accumulator_tile.store_safe(- output_ptr,- int(params->leading_dimension_d),- short2(simdgroup_limit_n, simdgroup_limit_m)- );- }- }- );- }- );- });- }-};--} // namespace matmul-} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.hindex 9fa6d3048..c56a6f0bb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h@@ -1,6 +1,7 @@#pragma once#include "../../common/defines.h"+#include "../../common/thread_context.h"using namespace metal;@@ -44,14 +45,16 @@ struct ThreadgroupLoader {const device T* source_ptr,const int source_leading_dim,threadgroup T* destination_ptr,- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]+ const thread ThreadContext& thread_context): source_leading_dimension(source_leading_dim),tile_stride(REDUCTION_DIMENSION ? BLOCK_COLS : BLOCK_ROWS * source_leading_dim),- thread_index(simd_group_id * METAL_SIMD_SIZE + simd_lane_id),+ thread_index(+ thread_context.simdgroup_index * METAL_SIMD_SIZE ++ thread_context.simd_lane_id+ ),block_row_index(thread_index / THREAD_COLS),block_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),destination(diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.hindex a00c9b4ec..609f3bb40 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h@@ -60,6 +60,8 @@ METAL_FUNC auto gemm_loop(thread_context);+ volatile int mxu_iteration_fence;+const int left_offset =transpose_a ? inner_k * leading_dimension_a : inner_k;const int right_offset =@@ -98,6 +100,8 @@ METAL_FUNC auto gemm_loop(left_tile,right_tile);++ (void)mxu_iteration_fence;}left_ptr += transpose_a ? (BLOCK_K * leading_dimension_a) : BLOCK_K;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex 082cd7284..bbb06f838 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -1,5 +1,6 @@#pragma once+#include "../../common/thread_context.h"#include "simdgroup_fragment.h"using namespace metal;@@ -80,17 +81,18 @@ struct ThreadgroupTile {ushort a_shared_offset;ushort b_shared_offset;- METAL_FUNC ThreadgroupTile(- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]- ) {+ METAL_FUNC ThreadgroupTile(const thread ThreadContext& thread_context) {ushort tile_row_base =- SIMDGROUP_BLOCK_SIZE * (simd_group_id / SIMDGROUPS_PER_COLUMN);+ SIMDGROUP_BLOCK_SIZE *+ (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);ushort tile_col_base =- SIMDGROUP_BLOCK_SIZE * (simd_group_id % SIMDGROUPS_PER_COLUMN);+ SIMDGROUP_BLOCK_SIZE *+ (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);const ushort2 simdgroup_coordinates = ushort2(- SimdgroupMultiplyAccumulateType::get_lane_coordinates(simd_lane_id)+ SimdgroupMultiplyAccumulateType::get_lane_coordinates(+ thread_context.simd_lane_id+ ));simdgroup_row_offset = simdgroup_coordinates.y;simdgroup_col_offset = simdgroup_coordinates.x;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metaldeleted file mode 100644index 02ea47e3a..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal+++ /dev/null@@ -1,65 +0,0 @@-#include "../common/dsl.h"-#include "../common/thread_context.h"--#include "common/gemm.h"--using namespace uzu::matmul;--#define GEMM_MAX_THREADGROUP_A 2560-#define GEMM_MAX_THREADGROUP_B 1536--template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint BLOCK_DEPTH, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN, bool MN_ALIGNED, bool K_ALIGNED, bool TRANSPOSE_B>-VARIANTS(T, float, half, bfloat)-VARIANTS(BLOCK_ROWS, 32, 64)-VARIANTS(BLOCK_COLS, 32, 64)-VARIANTS(BLOCK_DEPTH, 8, 16, 32)-VARIANTS(SIMDGROUPS_PER_ROW, 1, 2, 4)-VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)-VARIANTS(MN_ALIGNED, false, true)-VARIANTS(K_ALIGNED, false, true)-VARIANTS(TRANSPOSE_B, false, true)-CONSTRAINT(max(BLOCK_ROWS, BLOCK_COLS) <= 32 * SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN)-KERNEL(MatmulGemm)(- const device T* a,- const device T* b,- device T* d,- const constant uzu::matmul::GemmParams* params,- const constant uint& group_count_x,- const constant uint& group_count_y,- const constant float& ab_scale,- const bool is_accumulate SPECIALIZE,- threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],- threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],- const uint group_x GROUPS(group_count_x),- const uint group_y GROUPS(group_count_y),- const uint thread_x THREADS(32),- const uint thread_y THREADS(2),- const uint thread_z THREADS(2),- const ThreadContext thread_context-) {- ThreadgroupGemm<- T,- T,- BLOCK_ROWS,- BLOCK_COLS,- BLOCK_DEPTH,- SIMDGROUPS_PER_ROW,- SIMDGROUPS_PER_COLUMN,- false, // transpose_a- TRANSPOSE_B,- MN_ALIGNED,- K_ALIGNED,- float>::- run(a,- b,- d,- params,- ab_scale,- is_accumulate,- a_shared,- b_shared,- thread_context.simdgroup_index,- thread_context.threadgroup_index,- uint2(group_x, group_y),- uint3(thread_x, thread_y, thread_z));-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rsdeleted file mode 100644index 96a346750..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs+++ /dev/null@@ -1,176 +0,0 @@-use crate::{- DataType,- backends::{- common::{Context, kernel::matmul::MatmulArgumentC},- metal::{Metal, context::MetalContext},- },-};--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct GemmSpecialization {- pub block_rows: u32,- pub block_cols: u32,- pub block_depth: u32,- pub simdgroups_per_row: u32,- pub simdgroups_per_column: u32,- pub align_mn: bool,- pub align_k: bool,- pub transpose_b: bool,- pub is_accumulate: bool,-}--impl GemmSpecialization {- pub fn precompile_configs(data_type: DataType) -> &'static [Self] {- match data_type {- DataType::BF16 => &[- Self {- block_rows: 64,- block_cols: 32,- block_depth: 32,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_mn: false,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 32,- block_depth: 32,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_mn: true,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 64,- block_depth: 16,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_mn: false,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 64,- block_depth: 16,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_mn: true,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 64,- block_depth: 16,- simdgroups_per_row: 1,- simdgroups_per_column: 2,- align_mn: true,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- ],- DataType::F16 => &[- Self {- block_rows: 64,- block_cols: 64,- block_depth: 16,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_mn: true,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 64,- block_depth: 16,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_mn: false,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- ],- DataType::F32 => &[- Self {- block_rows: 32,- block_cols: 64,- block_depth: 16,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_mn: false,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- Self {- block_rows: 32,- block_cols: 64,- block_depth: 16,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_mn: true,- align_k: true,- transpose_b: true,- is_accumulate: false,- },- ],- _ => &[],- }- }-- pub fn select(- context: &MetalContext,- data_type: DataType,- batch_dim: u32,- input_dim: u32,- output_dim: u32,- transpose_b: bool,- c: &MatmulArgumentC<Metal>,- ) -> Self {- let overall_work_elements = batch_dim * output_dim;- let is_float32 = matches!(data_type, DataType::F32);- let prefer_half_or_tf32 = !is_float32 || MetalContext::tf32_enabled();-- let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column) =- if context.is_high_performance() && overall_work_elements >= (1 << 20) {- if prefer_half_or_tf32 {- if 2 * std::cmp::max(batch_dim, output_dim) > input_dim {- (64, 64, 16, 2, 2)- } else {- (64, 32, 32, 2, 2)- }- } else {- (32, 64, 16, 2, 2)- }- } else if prefer_half_or_tf32 {- (64, 32, 32, 2, 2)- } else {- (32, 64, 16, 2, 2)- };-- Self {- block_rows,- block_cols,- block_depth,- simdgroups_per_row,- simdgroups_per_column,- align_mn: (batch_dim % block_rows) == 0 && (output_dim % block_cols) == 0,- align_k: (input_dim % block_depth) == 0,- transpose_b,- is_accumulate: matches!(c, MatmulArgumentC::Accumulate),- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.hnew file mode 100644index 000000000..84b37753a--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h@@ -0,0 +1,60 @@+#pragma once++#include "../../common/defines.h"+#include "../../../generated/matmul.h"++using namespace metal;++namespace uzu {+namespace gemm {++static METAL_FUNC uint morton_expand_bits(uint x) {+ x &= 0x55555555u;+ x = (x | (x >> 1)) & 0x33333333u;+ x = (x | (x >> 2)) & 0x0F0F0F0Fu;+ x = (x | (x >> 4)) & 0x00FF00FFu;+ x = (x | (x >> 8)) & 0x0000FFFFu;+ return x;+}++// Morton-unpacks a 1D dispatch index (`fp.rs` sets `group_count_y = 1`) into+// 2D.+static METAL_FUNC uint2 morton_tile_id(uint2 threadgroup_position) {+ return uint2(+ morton_expand_bits(threadgroup_position.x),+ morton_expand_bits(threadgroup_position.x >> 1)+ );+}++static METAL_FUNC uint2 tile_id(+ uint2 threadgroup_position,+ const constant uzu::matmul::GemmParams* params+) {+ if (params->use_morton) {+ return morton_tile_id(threadgroup_position);+ }+ return threadgroup_position;+}++// `out_of_bounds` accounts for morton-padded tile ids past the actual grid.+template <uint THREADGROUP_BLOCK_M, uint THREADGROUP_BLOCK_N>+struct ThreadgroupTileGeometry {+ uint2 tile_id;+ uint block_row_start;+ uint block_col_start;+ bool out_of_bounds;++ static METAL_FUNC ThreadgroupTileGeometry+ compute(uint2 tile_id, const constant uzu::matmul::GemmParams* params) {+ ThreadgroupTileGeometry geometry;+ geometry.tile_id = tile_id;+ geometry.out_of_bounds = (tile_id.x >= params->threadgroups_per_row) ||+ (tile_id.y >= params->threadgroups_per_column);+ geometry.block_row_start = tile_id.y * THREADGROUP_BLOCK_M;+ geometry.block_col_start = tile_id.x * THREADGROUP_BLOCK_N;+ return geometry;+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hnew file mode 100644index 000000000..3e62895df--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -0,0 +1,198 @@+#pragma once++#include "../../../common/integral_constant.h"+#include "../../../common/thread_context.h"+#include "../../common/fragment.h"+#include "../../common/mxu_fragment_ops.h"+#include "../../common/mxu_gemm_loop.h"+#include "../../../generated/matmul.h"+#include "../generated/gemm.h"+#include "block_geometry.h"++using namespace metal;++namespace uzu {+namespace gemm {++// MPP `matmul2d_descriptor.transpose_right` drives `TRANSPOSE_B` (MSL+// Spec 6.18).+template <+ typename T,+ ushort THREADGROUP_BLOCK_M,+ ushort THREADGROUP_BLOCK_N,+ ushort SIMDGROUPS_PER_ROW,+ ushort SIMDGROUPS_PER_COLUMN,+ bool TRANSPOSE_B>+struct MxuMmaCore {+ METAL_CONST ushort SIMDGROUP_BLOCK_M =+ THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;+ METAL_CONST ushort SIMDGROUP_BLOCK_N =+ THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;+ METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;+ METAL_CONST ushort THREADGROUP_BLOCK_K = 256;+ METAL_CONST ushort TILES_M =+ SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;+ METAL_CONST ushort TILES_N =+ SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;++ using AccumulatorType = float;++ static METAL_FUNC void run(+ const device T* a,+ const device T* b,+ device T* d,+ const constant uzu::matmul::GemmParams* params,+ GemmAlignment alignment,+ GemmOutputTransformKind output_transform,+ const thread ThreadContext& thread_context+ ) {+ const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);+ const auto geometry =+ ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::+ compute(tile, params);+ if (geometry.out_of_bounds) {+ return;+ }++ const size_t block_row = size_t(geometry.block_row_start);+ const size_t block_col = size_t(geometry.block_col_start);++ const device T* a_block = a + block_row * params->leading_dimension_a;+ const device T* b_block =+ b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);++ const ushort tile_row_offset =+ SIMDGROUP_BLOCK_M *+ (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);+ const ushort tile_col_offset =+ SIMDGROUP_BLOCK_N *+ (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);++ device T* d_simdgroup =+ d + block_row * params->leading_dimension_d + block_col ++ tile_row_offset * params->leading_dimension_d + tile_col_offset;++ const short simdgroup_limit_m =+ alignment.contains(GemmAlignment::M)+ ? SIMDGROUP_BLOCK_M+ : short(+ min(int(SIMDGROUP_BLOCK_M),+ int(params->M) -+ int(geometry.block_row_start + tile_row_offset))+ );+ const short simdgroup_limit_n =+ alignment.contains(GemmAlignment::N)+ ? SIMDGROUP_BLOCK_N+ : short(+ min(int(SIMDGROUP_BLOCK_N),+ int(params->N) -+ int(geometry.block_col_start + tile_col_offset))+ );++ const device T* a_simdgroup =+ a_block + size_t(tile_row_offset) * params->leading_dimension_a;+ const device T* b_simdgroup =+ b_block + (TRANSPOSE_B ? size_t(tile_col_offset) *+ int(params->leading_dimension_b)+ : size_t(tile_col_offset));++ const int aligned_k_iterations = int(params->K) / int(THREADGROUP_BLOCK_K);++ const bool apply_scale =+ output_transform == GemmOutputTransformKind::Scale ||+ output_transform == GemmOutputTransformKind::ScaleAccumulate;+ const bool apply_accumulate =+ output_transform == GemmOutputTransformKind::Accumulate ||+ output_transform == GemmOutputTransformKind::ScaleAccumulate;++ dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {+ dispatch_bool(+ alignment.contains(GemmAlignment::M) ||+ (simdgroup_limit_m == SIMDGROUP_BLOCK_M),+ [&](auto aligned_m) {+ dispatch_bool(+ alignment.contains(GemmAlignment::N) ||+ (simdgroup_limit_n == SIMDGROUP_BLOCK_N),+ [&](auto aligned_n) {+ auto accumulator_tile = uzu::matmul::gemm_loop<+ T,+ SIMDGROUP_BLOCK_M,+ SIMDGROUP_BLOCK_N,+ SIMDGROUP_BLOCK_K,+ THREADGROUP_BLOCK_K,+ false,+ TRANSPOSE_B,+ aligned_m.value,+ aligned_n.value,+ aligned_k.value,+ AccumulatorType>(+ a_simdgroup,+ b_simdgroup,+ int(params->leading_dimension_a),+ int(params->leading_dimension_b),+ int(params->K),+ aligned_k_iterations,+ simdgroup_limit_m,+ simdgroup_limit_n,+ thread_context+ );++ if (apply_scale) {+ const AccumulatorType scale =+ AccumulatorType(params->ab_scale);+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;+ i++) {+ accumulator_tile.elements()[i] *= scale;+ }+ }++ if (apply_accumulate) {+ uzu::matmul::Fragment<+ T,+ TILES_M,+ TILES_N,+ uzu::matmul::MxuFragmentOps>+ existing_output(thread_context);+ if constexpr (aligned_m.value && aligned_n.value) {+ existing_output.load(+ d_simdgroup,+ int(params->leading_dimension_d)+ );+ } else {+ existing_output.load_safe(+ d_simdgroup,+ int(params->leading_dimension_d),+ short2(simdgroup_limit_n, simdgroup_limit_m)+ );+ }+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;+ i++) {+ accumulator_tile.elements()[i] +=+ AccumulatorType(existing_output.elements()[i]);+ }+ }++ if constexpr (aligned_m.value && aligned_n.value) {+ accumulator_tile.store(+ d_simdgroup,+ int(params->leading_dimension_d)+ );+ } else {+ accumulator_tile.store_safe(+ d_simdgroup,+ int(params->leading_dimension_d),+ short2(simdgroup_limit_n, simdgroup_limit_m)+ );+ }+ }+ );+ }+ );+ });+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hnew file mode 100644index 000000000..10fd83532--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -0,0 +1,229 @@+#pragma once++#include "../../../common/integral_constant.h"+#include "../../../common/thread_context.h"+#include "../../common/defines.h"+#include "../../common/loader.h"+#include "../../common/threadgroup_tile.h"+#include "../../../generated/matmul.h"+#include "../generated/gemm.h"+#include "block_geometry.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <+ typename T,+ int THREADGROUP_BLOCK_M,+ int THREADGROUP_BLOCK_N,+ int THREADGROUP_BLOCK_K,+ int SIMDGROUPS_PER_ROW,+ int SIMDGROUPS_PER_COLUMN,+ bool TRANSPOSE_B>+struct SimdgroupMmaCore {+ METAL_CONST ushort PADDING_A = 16 / sizeof(T);+ METAL_CONST ushort PADDING_B = 16 / sizeof(T);+ METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;+ METAL_CONST ushort SHARED_STRIDE_B =+ (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;+ METAL_CONST ushort THREADGROUP_THREADS =+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;++ using ALoader = uzu::matmul::ThreadgroupLoader<+ T,+ THREADGROUP_BLOCK_M,+ THREADGROUP_BLOCK_K,+ SHARED_STRIDE_A,+ true,+ THREADGROUP_THREADS>;+ using BLoader = uzu::matmul::ThreadgroupLoader<+ T,+ TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,+ TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N,+ SHARED_STRIDE_B,+ TRANSPOSE_B,+ THREADGROUP_THREADS>;+ using TileAccumulator = uzu::matmul::ThreadgroupTile<+ T,+ T,+ THREADGROUP_BLOCK_M,+ THREADGROUP_BLOCK_N,+ THREADGROUP_BLOCK_K,+ SIMDGROUPS_PER_ROW,+ SIMDGROUPS_PER_COLUMN,+ false,+ TRANSPOSE_B,+ SHARED_STRIDE_A,+ SHARED_STRIDE_B,+ float,+ uzu::matmul::TransformNone<T, float>>;++ template <bool M_aligned, bool N_aligned, bool K_aligned>+ static METAL_FUNC void k_loop(+ threadgroup T* a_shared,+ threadgroup T* b_shared,+ const int aligned_k_iterations,+ thread ALoader& loader_a,+ thread BLoader& loader_b,+ thread TileAccumulator& accumulator,+ thread const ushort& tile_block_rows,+ thread const ushort& tile_block_cols,+ thread const ushort& leftover_block_depth+ ) {+ short2 tile_dimensions_a = short2(THREADGROUP_BLOCK_K, tile_block_rows);+ short2 tile_dimensions_b =+ TRANSPOSE_B ? short2(THREADGROUP_BLOCK_K, tile_block_cols)+ : short2(tile_block_cols, THREADGROUP_BLOCK_K);++ for (int k = 0; k < aligned_k_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if constexpr (M_aligned) {+ loader_a.load_unsafe();+ } else {+ loader_a.load_safe(tile_dimensions_a);+ }+ if constexpr (N_aligned) {+ loader_b.load_unsafe();+ } else {+ loader_b.load_safe(tile_dimensions_b);+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);+ accumulator.multiply_accumulate(a_shared, b_shared);++ loader_a.next();+ loader_b.next();+ }++ if constexpr (!K_aligned) {+ threadgroup_barrier(mem_flags::mem_threadgroup);++ short2 last_tile_dimensions_a =+ short2(leftover_block_depth, tile_block_rows);+ short2 last_tile_dimensions_b =+ TRANSPOSE_B ? short2(leftover_block_depth, tile_block_cols)+ : short2(tile_block_cols, leftover_block_depth);++ loader_a.load_safe(last_tile_dimensions_a);+ loader_b.load_safe(last_tile_dimensions_b);++ threadgroup_barrier(mem_flags::mem_threadgroup);+ accumulator.multiply_accumulate(a_shared, b_shared);+ }+ }++ static METAL_FUNC void run(+ const device T* a,+ const device T* b,+ device T* d,+ const constant uzu::matmul::GemmParams* params,+ GemmAlignment alignment,+ GemmOutputTransformKind output_transform,+ threadgroup T* a_shared,+ threadgroup T* b_shared,+ const thread ThreadContext& thread_context+ ) {+ const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);+ const auto geometry =+ ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::+ compute(tile, params);+ if (geometry.out_of_bounds) {+ return;+ }++ threadgroup_barrier(mem_flags::mem_none);++ const size_t block_row = size_t(geometry.block_row_start);+ const size_t block_col = size_t(geometry.block_col_start);++ a += block_row * params->leading_dimension_a;+ b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;+ d += block_row * params->leading_dimension_d + block_col;++ thread ALoader+ loader_a(a, params->leading_dimension_a, a_shared, thread_context);+ thread BLoader+ loader_b(b, params->leading_dimension_b, b_shared, thread_context);+ thread TileAccumulator accumulator(thread_context);++ const ushort tile_block_rows =+ min(THREADGROUP_BLOCK_M,+ ((int)params->M) - int(geometry.block_row_start));+ const ushort tile_block_cols =+ min(THREADGROUP_BLOCK_N,+ ((int)params->N) - int(geometry.block_col_start));+ const ushort leftover_block_depth =+ params->K - params->aligned_inner_iterations * THREADGROUP_BLOCK_K;++ const bool needs_epilogue =+ output_transform != GemmOutputTransformKind::Store;+ const float alpha =+ (output_transform == GemmOutputTransformKind::Scale ||+ output_transform == GemmOutputTransformKind::ScaleAccumulate)+ ? params->ab_scale+ : 1.0f;+ const float beta =+ (output_transform == GemmOutputTransformKind::Accumulate ||+ output_transform == GemmOutputTransformKind::ScaleAccumulate)+ ? 1.0f+ : 0.0f;+ uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);++ dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {+ dispatch_bool(+ alignment.contains(GemmAlignment::M) ||+ (tile_block_rows == THREADGROUP_BLOCK_M),+ [&](auto aligned_m) {+ dispatch_bool(+ alignment.contains(GemmAlignment::N) ||+ (tile_block_cols == THREADGROUP_BLOCK_N),+ [&](auto aligned_n) {+ k_loop<aligned_m.value, aligned_n.value, aligned_k.value>(+ a_shared,+ b_shared,+ params->aligned_inner_iterations,+ loader_a,+ loader_b,+ accumulator,+ tile_block_rows,+ tile_block_cols,+ leftover_block_depth+ );+ if constexpr (aligned_m.value && aligned_n.value) {+ if (needs_epilogue) {+ accumulator.apply_epilogue(+ d,+ params->leading_dimension_d,+ 1,+ epilogue+ );+ }+ accumulator.store_result(d, params->leading_dimension_d);+ } else {+ if (needs_epilogue) {+ accumulator.apply_epilogue_safe(+ d,+ params->leading_dimension_d,+ 1,+ short2(tile_block_cols, tile_block_rows),+ epilogue+ );+ }+ accumulator.store_result_safe(+ d,+ params->leading_dimension_d,+ short2(tile_block_cols, tile_block_rows)+ );+ }+ }+ );+ }+ );+ });+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rsnew file mode 100644index 000000000..57e38cbbb--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs@@ -0,0 +1,247 @@+use crate::{+ DataType,+ backends::common::{+ Allocation, Backend,+ gpu_types::{+ GemmParams, QuantizationMode,+ gemm::{+ GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig, GemmWeightPrologueKind,+ },+ },+ },+};++pub struct GemmAlignmentAxes {+ pub m: bool,+ pub n: bool,+ pub k: bool,+}++impl GemmAlignment {+ pub fn from_axes(axes: GemmAlignmentAxes) -> Self {+ let mut bits = Self::empty();+ bits.set(Self::M, axes.m);+ bits.set(Self::N, axes.n);+ bits.set(Self::K, axes.k);+ bits+ }+}++pub enum GemmWeights<'a, B: Backend> {+ FullPrecision {+ weights: &'a Allocation<B>,+ },+ ScaleBias {+ weights: &'a Allocation<B>,+ scales: &'a Allocation<B>,+ biases: &'a Allocation<B>,+ mode: QuantizationMode,+ group_size: u32,+ },+ ScaleZeroPoint {+ weights: &'a Allocation<B>,+ scales: &'a Allocation<B>,+ zero_points: &'a Allocation<B>,+ mode: QuantizationMode,+ group_size: u32,+ },+}++impl<B: Backend> GemmWeights<'_, B> {+ pub fn weight_prologue(&self) -> GemmWeightPrologueKind {+ match self {+ Self::FullPrecision {+ ..+ } => GemmWeightPrologueKind::FullPrecision,+ Self::ScaleBias {+ ..+ } => GemmWeightPrologueKind::ScaleBiasDequant,+ Self::ScaleZeroPoint {+ ..+ } => GemmWeightPrologueKind::ScaleZeroPointDequant,+ }+ }++ pub fn bits_per_weight(&self) -> u32 {+ match self {+ Self::FullPrecision {+ ..+ } => 0,+ Self::ScaleBias {+ mode,+ ..+ }+ | Self::ScaleZeroPoint {+ mode,+ ..+ } => DataType::from(*mode).size_in_bits() as u32,+ }+ }++ pub fn group_size(&self) -> u32 {+ match self {+ Self::FullPrecision {+ ..+ } => 0,+ Self::ScaleBias {+ group_size,+ ..+ }+ | Self::ScaleZeroPoint {+ group_size,+ ..+ } => *group_size,+ }+ }+}++pub struct GemmDispatch<'a, B: Backend> {+ pub tiling_config: GemmTilingConfig,+ pub input_prologue: GemmInputPrologueKind,+ pub use_mxu: bool,+ pub output_transform: GemmOutputTransformKind,+ pub alignment: GemmAlignment,+ pub transpose_b: bool,+ pub a: &'a Allocation<B>,+ pub a_offset: usize,+ pub b: GemmWeights<'a, B>,+ pub b_offset: usize,+ pub d: &'a mut Allocation<B>,+ pub params: GemmParams,+ pub group_count_x: u32,+ pub group_count_y: u32,+}++impl<B: Backend> GemmDispatch<'_, B> {+ pub(crate) fn specialization(&self) -> GemmSpecialization {+ GemmSpecialization {+ tiling_config: self.tiling_config,+ input_prologue: self.input_prologue,+ use_mxu: self.use_mxu,+ output_transform: self.output_transform,+ alignment: self.alignment,+ transpose_b: self.transpose_b,+ weight_prologue: self.b.weight_prologue(),+ bits_per_weight: self.b.bits_per_weight(),+ group_size: self.b.group_size(),+ }+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub(crate) struct GemmSpecialization {+ pub(crate) tiling_config: GemmTilingConfig,+ pub(crate) input_prologue: GemmInputPrologueKind,+ pub(crate) use_mxu: bool,+ pub(crate) output_transform: GemmOutputTransformKind,+ pub(crate) alignment: GemmAlignment,+ pub(crate) transpose_b: bool,+ pub(crate) weight_prologue: GemmWeightPrologueKind,+ pub(crate) bits_per_weight: u32,+ pub(crate) group_size: u32,+}++impl GemmSpecialization {+ pub(crate) fn validate(&self) -> Result<(), GemmSpecializationError> {+ if self.group_size != 0 && self.tiling_config.threadgroup_k > self.group_size {+ return Err(GemmSpecializationError::ThreadgroupKExceedsGroupSize {+ threadgroup_k: self.tiling_config.threadgroup_k,+ group_size: self.group_size,+ });+ }+ Ok(())+ }++ pub(crate) fn precompile_configs(data_type: DataType) -> Vec<Self> {+ let mut out = Vec::new();+ for &(threadgroup_m, threadgroup_n, threadgroup_k) in simdgroup_tile_set(data_type) {+ for align_mn in [true, false] {+ let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {+ m: align_mn,+ n: align_mn,+ k: true,+ });+ out.push(Self {+ tiling_config: GemmTilingConfig {+ threadgroup_m,+ threadgroup_n,+ threadgroup_k,+ simdgroups_m: 2,+ simdgroups_n: 2,+ },+ input_prologue: GemmInputPrologueKind::FullPrecision,+ use_mxu: false,+ output_transform: GemmOutputTransformKind::Store,+ alignment,+ transpose_b: true,+ weight_prologue: GemmWeightPrologueKind::FullPrecision,+ bits_per_weight: 0,+ group_size: 0,+ });+ }+ }++ for &(threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) in mxu_tile_set(data_type) {+ for align_m in [true, false] {+ for align_n in [true, false] {+ for align_k in [true, false] {+ let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {+ m: align_m,+ n: align_n,+ k: align_k,+ });+ for output_transform in [+ GemmOutputTransformKind::Store,+ GemmOutputTransformKind::Scale,+ GemmOutputTransformKind::Accumulate,+ GemmOutputTransformKind::ScaleAccumulate,+ ] {+ out.push(Self {+ tiling_config: GemmTilingConfig {+ threadgroup_m,+ threadgroup_n,+ threadgroup_k: 32,+ simdgroups_m,+ simdgroups_n,+ },+ input_prologue: GemmInputPrologueKind::FullPrecision,+ use_mxu: true,+ output_transform,+ alignment,+ transpose_b: true,+ weight_prologue: GemmWeightPrologueKind::FullPrecision,+ bits_per_weight: 0,+ group_size: 0,+ });+ }+ }+ }+ }+ }+ out+ }+}++fn simdgroup_tile_set(data_type: DataType) -> &'static [(u32, u32, u32)] {+ match data_type {+ DataType::BF16 => &[(64, 32, 32), (64, 64, 16)],+ DataType::F16 => &[(64, 64, 16), (64, 32, 32)],+ DataType::F32 => &[(32, 64, 16)],+ _ => &[],+ }+}++fn mxu_tile_set(data_type: DataType) -> &'static [(u32, u32, u32, u32)] {+ if !matches!(data_type, DataType::F16 | DataType::BF16) {+ return &[];+ }+ &[(64, 64, 2, 2), (32, 64, 2, 2), (64, 32, 4, 1), (128, 128, 4, 4)]+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum GemmSpecializationError {+ ThreadgroupKExceedsGroupSize {+ threadgroup_k: u32,+ group_size: u32,+ },+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rsnew file mode 100644index 000000000..4f39ce56b--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs@@ -0,0 +1,176 @@+use super::{GemmAlignmentAxes, GemmDispatch, GemmKernel, GemmWeights, MXU_THREADGROUP_BLOCK_K};+use crate::{+ DataType,+ backends::{+ common::{+ Encoder,+ gpu_types::{+ GemmParams,+ gemm::{GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig},+ },+ kernel::{+ TensorAddBiasKernel,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError},+ },+ },+ metal::{Metal, context::MetalContext, kernel::TensorAddBiasMetalKernel, metal_extensions::DeviceExt},+ },+};++pub(crate) fn encode(+ gemm: &mut GemmKernel,+ bias_add: &mut TensorAddBiasMetalKernel,+ data_type: DataType,+ context: &MetalContext,+ encoder: &mut Encoder<Metal>,+ arguments: MatmulArguments<Metal>,+ use_mxu: bool,+) -> Result<(), MatmulError<Metal>> {+ if use_mxu && !context.device.supports_mxu() {+ return Err(MatmulError::UnsupportedDataType(data_type));+ }++ let tile = if use_mxu {+ select_mxu_tile(&arguments)+ } else {+ select_simdgroup_tile(data_type, &arguments)+ };+ let k_block = if use_mxu {+ MXU_THREADGROUP_BLOCK_K+ } else {+ tile.threadgroup_k+ };++ let threadgroups_per_row = arguments.output_dim.div_ceil(tile.threadgroup_n);+ let threadgroups_per_column = arguments.batch_dim.div_ceil(tile.threadgroup_m);++ let (use_morton, group_count_x, group_count_y) = if use_mxu {+ let max_dim = threadgroups_per_row.max(threadgroups_per_column);+ let min_dim = threadgroups_per_row.min(threadgroups_per_column);+ let morton_dim = max_dim.next_power_of_two();+ let morton_total = morton_dim.saturating_mul(morton_dim);+ let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);+ let use_morton = min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);+ if use_morton {+ (true, morton_total, 1)+ } else {+ (false, threadgroups_per_row, threadgroups_per_column)+ }+ } else {+ (false, threadgroups_per_row, threadgroups_per_column)+ };++ let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {+ m: arguments.batch_dim % tile.threadgroup_m == 0,+ n: arguments.output_dim % tile.threadgroup_n == 0,+ k: arguments.input_dim % k_block == 0,+ });+ let output_transform = output_transform_from(&arguments);++ let default_ldb = if arguments.b_transpose {+ arguments.input_dim+ } else {+ arguments.output_dim+ };+ let params = GemmParams {+ M: arguments.batch_dim,+ N: arguments.output_dim,+ K: arguments.input_dim,+ leading_dimension_a: arguments.input_dim,+ leading_dimension_b: arguments.b_leading_dimension.unwrap_or(default_ldb),+ leading_dimension_d: arguments.output_dim,+ threadgroups_per_row,+ threadgroups_per_column,+ aligned_inner_iterations: arguments.input_dim / k_block,+ use_morton,+ ab_scale: arguments.ab_scale,+ };++ let dispatch = GemmDispatch {+ tiling_config: tile,+ input_prologue: GemmInputPrologueKind::FullPrecision,+ use_mxu,+ output_transform,+ alignment,+ transpose_b: arguments.b_transpose,+ a: arguments.a,+ a_offset: arguments.a_offset,+ b: GemmWeights::FullPrecision {+ weights: arguments.b,+ },+ b_offset: arguments.b_offset,+ d: arguments.d,+ params,+ group_count_x,+ group_count_y,+ };++ gemm.encode(context, dispatch, encoder).map_err(MatmulError::BackendError)?;++ if let MatmulArgumentC::Bias(bias) = arguments.c {+ bias_add.encode(+ None::<&<Metal as crate::backends::common::Backend>::DenseBuffer>,+ bias,+ arguments.d,+ arguments.output_dim,+ arguments.batch_dim * arguments.output_dim,+ encoder,+ );+ }++ Ok(())+}++fn select_simdgroup_tile(+ data_type: DataType,+ arguments: &MatmulArguments<Metal>,+) -> GemmTilingConfig {+ let (threadgroup_m, threadgroup_n, threadgroup_k) = match data_type {+ DataType::F32 => (32u32, 64u32, 16u32),+ _ => {+ if 2 * arguments.batch_dim.max(arguments.output_dim) > arguments.input_dim {+ (64, 64, 16)+ } else {+ (64, 32, 32)+ }+ },+ };+ GemmTilingConfig {+ threadgroup_m,+ threadgroup_n,+ threadgroup_k,+ simdgroups_m: 2,+ simdgroups_n: 2,+ }+}++fn select_mxu_tile(arguments: &MatmulArguments<Metal>) -> GemmTilingConfig {+ let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) =+ if arguments.batch_dim >= 256 && arguments.output_dim >= 128 {+ (128u32, 128u32, 4u32, 4u32)+ } else if arguments.output_dim < 64 {+ (64u32, 32u32, 4u32, 1u32)+ } else if arguments.batch_dim < 64 {+ (32u32, 64u32, 2u32, 2u32)+ } else {+ (64u32, 64u32, 2u32, 2u32)+ };+ GemmTilingConfig {+ threadgroup_m,+ threadgroup_n,+ threadgroup_k: 32,+ simdgroups_m,+ simdgroups_n,+ }+}++fn output_transform_from(arguments: &MatmulArguments<Metal>) -> GemmOutputTransformKind {+ let scale = arguments.ab_scale != 1.0;+ let accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);+ match (scale, accumulate) {+ (false, false) => GemmOutputTransformKind::Store,+ (true, false) => GemmOutputTransformKind::Scale,+ (false, true) => GemmOutputTransformKind::Accumulate,+ (true, true) => GemmOutputTransformKind::ScaleAccumulate,+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalnew file mode 100644index 000000000..00ad9e2d7--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -0,0 +1,106 @@+#include "../../common/dsl.h"+#include "../../common/defines.h"+#include "../../common/thread_context.h"+#include "../generated/gemm.h"++#include "common/mxu_mma_core.h"+#include "common/simdgroup_mma_core.h"++using namespace metal;+using namespace uzu::gemm;++#define GEMM_MAX_THREADGROUP_A 2560+#define GEMM_MAX_THREADGROUP_B 1536++template <+ typename T,+ uint THREADGROUP_BLOCK_M,+ uint THREADGROUP_BLOCK_N,+ uint THREADGROUP_BLOCK_K,+ uint SIMDGROUPS_M,+ uint SIMDGROUPS_N,+ bool TRANSPOSE_B,+ bool USE_MXU>+VARIANTS(T, float, half, bfloat)+VARIANTS(THREADGROUP_BLOCK_M, 32, 64, 128)+VARIANTS(THREADGROUP_BLOCK_N, 32, 64, 128)+VARIANTS(THREADGROUP_BLOCK_K, 16, 32)+VARIANTS(SIMDGROUPS_M, 1, 2, 4)+VARIANTS(SIMDGROUPS_N, 1, 2, 4)+VARIANTS(TRANSPOSE_B, false, true)+VARIANTS(USE_MXU, false, true)+CONSTRAINT(!USE_MXU || THREADGROUP_BLOCK_M % SIMDGROUPS_M == 0)+CONSTRAINT(!USE_MXU || THREADGROUP_BLOCK_N % SIMDGROUPS_N == 0)+CONSTRAINT(!USE_MXU || (THREADGROUP_BLOCK_M / SIMDGROUPS_M) % 16 == 0)+CONSTRAINT(!USE_MXU || (THREADGROUP_BLOCK_N / SIMDGROUPS_N) % 16 == 0)+CONSTRAINT(USE_MXU || max(THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N) <= 32 * SIMDGROUPS_M * SIMDGROUPS_N)+KERNEL(Gemm)(+ const device T* a,+ const device uint8_t* b_packed,+ device T* d,+ const device T* scales+ OPTIONAL(weight_prologue != GemmWeightPrologueKind::FullPrecision),+ const device T* biases+ OPTIONAL(weight_prologue == GemmWeightPrologueKind::ScaleBiasDequant),+ const device uint8_t* zero_points+ OPTIONAL(weight_prologue == GemmWeightPrologueKind::ScaleZeroPointDequant),+ const constant uzu::matmul::GemmParams* params,+ const constant uint& group_count_x,+ const constant uint& group_count_y,+ const GemmInputPrologueKind input_prologue SPECIALIZE,+ const GemmWeightPrologueKind weight_prologue SPECIALIZE,+ const GemmOutputTransformKind output_transform SPECIALIZE,+ const GemmAlignment alignment SPECIALIZE,+ const uint bits_per_weight SPECIALIZE,+ const uint group_size SPECIALIZE,+ threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],+ threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],+ const uint group_x GROUPS(group_count_x),+ const uint group_y GROUPS(group_count_y),+ const uint thread_x THREADS(METAL_SIMD_SIZE),+ const uint thread_y THREADS(SIMDGROUPS_N),+ const uint thread_z THREADS(SIMDGROUPS_M),+ const ThreadContext thread_context+) {+ (void)scales;+ (void)biases;+ (void)zero_points;+ (void)bits_per_weight;+ (void)group_size;+ (void)group_x;+ (void)group_y;+ (void)thread_x;+ (void)thread_y;+ (void)thread_z;+ (void)input_prologue;+ (void)weight_prologue;+ const device T* b = reinterpret_cast<const device T*>(b_packed);+ if constexpr (USE_MXU) {+ MxuMmaCore<+ T,+ THREADGROUP_BLOCK_M,+ THREADGROUP_BLOCK_N,+ SIMDGROUPS_M,+ SIMDGROUPS_N,+ TRANSPOSE_B>::+ run(a, b, d, params, alignment, output_transform, thread_context);+ } else {+ SimdgroupMmaCore<+ T,+ THREADGROUP_BLOCK_M,+ THREADGROUP_BLOCK_N,+ THREADGROUP_BLOCK_K,+ SIMDGROUPS_M,+ SIMDGROUPS_N,+ TRANSPOSE_B>::+ run(a,+ b,+ d,+ params,+ alignment,+ output_transform,+ a_shared,+ b_shared,+ thread_context);+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsnew file mode 100644index 000000000..44a925987--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -0,0 +1,115 @@+use std::collections::{HashMap, hash_map::Entry};++use super::dispatch::{GemmDispatch, GemmSpecialization, GemmWeights};+use crate::{+ DataType,+ backends::{+ common::{+ Encoder,+ gpu_types::gemm::{GemmInputPrologueKind, GemmWeightPrologueKind},+ },+ metal::{Metal, context::MetalContext, error::MetalError, kernel::GemmMetalKernel},+ },+};++pub(crate) struct GemmKernel {+ data_type: DataType,+ kernels: HashMap<GemmSpecialization, GemmMetalKernel>,+}++impl GemmKernel {+ pub(crate) fn new(+ context: &MetalContext,+ data_type: DataType,+ ) -> Result<Self, MetalError> {+ let mut kernel = Self {+ data_type,+ kernels: HashMap::new(),+ };+ for specialization in GemmSpecialization::precompile_configs(data_type) {+ kernel.get_or_create(context, specialization)?;+ }+ Ok(kernel)+ }++ fn get_or_create(+ &mut self,+ context: &MetalContext,+ specialization: GemmSpecialization,+ ) -> Result<&GemmMetalKernel, MetalError> {+ match self.kernels.entry(specialization) {+ Entry::Occupied(entry) => Ok(entry.into_mut()),+ Entry::Vacant(entry) => {+ let kernel = GemmMetalKernel::new(+ context,+ self.data_type,+ specialization.tiling_config.threadgroup_m,+ specialization.tiling_config.threadgroup_n,+ specialization.tiling_config.threadgroup_k,+ specialization.tiling_config.simdgroups_m,+ specialization.tiling_config.simdgroups_n,+ specialization.transpose_b,+ specialization.use_mxu,+ specialization.input_prologue,+ specialization.weight_prologue,+ specialization.output_transform,+ specialization.alignment,+ specialization.bits_per_weight,+ specialization.group_size,+ )?;+ Ok(entry.insert(kernel))+ },+ }+ }++ pub(crate) fn encode(+ &mut self,+ context: &MetalContext,+ dispatch: GemmDispatch<'_, Metal>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), MetalError> {+ let specialization = dispatch.specialization();+ assert_eq!(+ specialization.input_prologue,+ GemmInputPrologueKind::FullPrecision,+ "unified GEMM only implements FullPrecision input prologue today",+ );+ assert_eq!(+ specialization.weight_prologue,+ GemmWeightPrologueKind::FullPrecision,+ "unified GEMM only implements FullPrecision weight prologue today",+ );+ specialization.validate().map_err(MetalError::InvalidGemmSpecialization)?;+ let kernel = self.get_or_create(context, specialization)?;+ let (b, scales, biases, zero_points) = match &dispatch.b {+ GemmWeights::FullPrecision {+ weights,+ } => (*weights, None, None, None),+ GemmWeights::ScaleBias {+ weights,+ scales,+ biases,+ ..+ } => (*weights, Some(*scales), Some(*biases), None),+ GemmWeights::ScaleZeroPoint {+ weights,+ scales,+ zero_points,+ ..+ } => (*weights, Some(*scales), None, Some(*zero_points)),+ };+ kernel.encode(+ (dispatch.a, dispatch.a_offset),+ (b, dispatch.b_offset),+ dispatch.d,+ scales,+ biases,+ zero_points,+ std::slice::from_ref(&dispatch.params),+ dispatch.group_count_x,+ dispatch.group_count_y,+ encoder,+ );+ Ok(())+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rsnew file mode 100644index 000000000..3dc90617f--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -0,0 +1,13 @@+mod dispatch;+pub(crate) mod fp;+mod kernel;+pub(crate) mod quant;++pub use dispatch::GemmSpecializationError;+pub(crate) use dispatch::{GemmAlignmentAxes, GemmDispatch, GemmWeights};+pub(crate) use kernel::GemmKernel;++pub(crate) use crate::backends::common::gpu_types::gemm::GemmInputPrologueKind;++// Matches `MxuMmaCore::THREADGROUP_BLOCK_K` in `common/mxu_mma_core.h`.+pub(crate) const MXU_THREADGROUP_BLOCK_K: u32 = 256;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rsnew file mode 100644index 000000000..54617b5c1--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs@@ -0,0 +1,96 @@+use super::{GemmAlignmentAxes, GemmDispatch, GemmInputPrologueKind, GemmKernel, GemmWeights};+use crate::backends::{+ common::{+ Encoder,+ gpu_types::{+ GemmParams, QuantizationMethod,+ gemm::{GemmAlignment, GemmOutputTransformKind, GemmTilingConfig},+ },+ kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError},+ },+ metal::{Metal, context::MetalContext},+};++pub(crate) fn encode(+ gemm: &mut GemmKernel,+ context: &MetalContext,+ configuration: &QuantizedMatmulConfiguration,+ arguments: QuantizedMatmulArguments<Metal>,+ encoder: &mut Encoder<Metal>,+) -> Result<(), QuantizedMatmulError<Metal>> {+ let tile = select_tile(configuration, arguments.batch_dim as u32);+ let group_size = configuration.group_size as u32;+ let batch_dim = arguments.batch_dim as u32;+ let input_dim = configuration.input_dim as u32;+ let output_dim = configuration.output_dim as u32;+ let threadgroups_per_row = output_dim.div_ceil(tile.threadgroup_n);+ let threadgroups_per_column = batch_dim.div_ceil(tile.threadgroup_m);+ let dispatch = GemmDispatch {+ tiling_config: tile,+ input_prologue: GemmInputPrologueKind::FullPrecision,+ use_mxu: false,+ output_transform: GemmOutputTransformKind::Store,+ alignment: GemmAlignment::from_axes(GemmAlignmentAxes {+ m: batch_dim % tile.threadgroup_m == 0,+ n: output_dim % tile.threadgroup_n == 0,+ k: input_dim % tile.threadgroup_k == 0,+ }),+ transpose_b: true,+ a: arguments.a,+ a_offset: arguments.a_offset,+ b: match configuration.quantization_method {+ QuantizationMethod::ScaleBias => GemmWeights::ScaleBias {+ weights: arguments.b,+ scales: arguments.scales,+ biases: arguments.zero_points_or_biases,+ mode: configuration.mode,+ group_size,+ },+ QuantizationMethod::ScaleZeroPoint => GemmWeights::ScaleZeroPoint {+ weights: arguments.b,+ scales: arguments.scales,+ zero_points: arguments.zero_points_or_biases,+ mode: configuration.mode,+ group_size,+ },+ },+ b_offset: 0,+ d: arguments.output,+ params: GemmParams {+ M: batch_dim,+ N: output_dim,+ K: input_dim,+ leading_dimension_a: input_dim,+ leading_dimension_b: input_dim,+ leading_dimension_d: output_dim,+ threadgroups_per_row,+ threadgroups_per_column,+ aligned_inner_iterations: input_dim / tile.threadgroup_k,+ use_morton: false,+ ab_scale: 1.0,+ },+ group_count_x: threadgroups_per_row,+ group_count_y: threadgroups_per_column,+ };+ gemm.encode(context, dispatch, encoder).map_err(QuantizedMatmulError::BackendError)+}++fn select_tile(+ configuration: &QuantizedMatmulConfiguration,+ batch_dim: u32,+) -> GemmTilingConfig {+ let group_size = configuration.group_size as u32;+ let threadgroup_k = group_size.min(32);+ let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) = if batch_dim < 32 {+ (32u32, 32u32, 1u32, 1u32)+ } else {+ (64u32, 64u32, 2u32, 2u32)+ };+ GemmTilingConfig {+ threadgroup_m,+ threadgroup_n,+ threadgroup_k,+ simdgroups_m,+ simdgroups_n,+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metaldeleted file mode 100644index 6fc7399d7..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal+++ /dev/null@@ -1,62 +0,0 @@-#include "../common/dsl.h"-#include "../common/thread_context.h"--#include "common/gemm_mpp_core.h"--using namespace uzu::matmul;--template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN>-VARIANTS(T, float, half, bfloat)-VARIANTS(BLOCK_ROWS, 32, 64, 128)-VARIANTS(BLOCK_COLS, 32, 64, 128)-VARIANTS(SIMDGROUPS_PER_ROW, 2, 4)-VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2, 4)-CONSTRAINT(- (BLOCK_ROWS == 32 && BLOCK_COLS == 64 && SIMDGROUPS_PER_ROW == 2 && SIMDGROUPS_PER_COLUMN == 2) ||- (BLOCK_ROWS == 64 && BLOCK_COLS == 32 && SIMDGROUPS_PER_ROW == 4 && SIMDGROUPS_PER_COLUMN == 1) ||- (BLOCK_ROWS == 64 && BLOCK_COLS == 64 && SIMDGROUPS_PER_ROW == 2 && SIMDGROUPS_PER_COLUMN == 2) ||- (BLOCK_ROWS == 128 && BLOCK_COLS == 128 && SIMDGROUPS_PER_ROW == 4 && SIMDGROUPS_PER_COLUMN == 4))-KERNEL(MatmulGemmMpp)(- const device T* left_matrix,- const device T* right_matrix,- device T* output_matrix,- const constant uzu::matmul::GemmParams* params,- const constant uint& group_count_x,- const constant uint& group_count_y,- const constant float& ab_scale,- const bool align_m SPECIALIZE,- const bool align_n SPECIALIZE,- const bool align_k SPECIALIZE,- const bool apply_ab_scale SPECIALIZE,- const bool is_accumulate SPECIALIZE,- const uint group_x GROUPS(group_count_x),- const uint group_y GROUPS(group_count_y),- const uint thread_x THREADS(32),- const uint thread_y THREADS(SIMDGROUPS_PER_ROW),- const uint thread_z THREADS(SIMDGROUPS_PER_COLUMN),- const ThreadContext thread_context-) {- dispatch_bool(apply_ab_scale, [&](auto scale_enabled) {- dispatch_bool(is_accumulate, [&](auto accumulate_enabled) {- GemmMppCore<- T,- BLOCK_ROWS,- BLOCK_COLS,- SIMDGROUPS_PER_ROW,- SIMDGROUPS_PER_COLUMN,- scale_enabled.value,- accumulate_enabled.value>::- run(left_matrix,- right_matrix,- output_matrix,- params,- align_m,- align_n,- align_k,- ab_scale,- thread_context.threadgroup_index,- uint2(group_x, group_y),- thread_context);- });- });-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rsdeleted file mode 100644index a957795d0..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs+++ /dev/null@@ -1,130 +0,0 @@-use std::sync::OnceLock;--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct GemmMppSpecialization {- pub block_rows: u32,- pub block_cols: u32,- pub simdgroups_per_row: u32,- pub simdgroups_per_column: u32,- pub align_m: bool,- pub align_n: bool,- pub align_k: bool,- pub apply_ab_scale: bool,- pub is_accumulate: bool,-}--const fn config(- block_rows: u32,- block_cols: u32,- simdgroups_per_row: u32,- simdgroups_per_column: u32,- align_m: bool,- align_n: bool,- is_accumulate: bool,-) -> GemmMppSpecialization {- GemmMppSpecialization {- block_rows,- block_cols,- simdgroups_per_row,- simdgroups_per_column,- align_m,- align_n,- align_k: false,- apply_ab_scale: false,- is_accumulate,- }-}--static GEMM_MPP_PRECOMPILE: OnceLock<&'static [GemmMppSpecialization]> = OnceLock::new();--impl GemmMppSpecialization {- pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {- if !matches!(data_type, crate::DataType::F16 | crate::DataType::BF16) {- return &[];- }- GEMM_MPP_PRECOMPILE.get_or_init(|| {- const BASE: &[GemmMppSpecialization] = &[- config(64, 64, 2, 2, true, true, false),- config(64, 64, 2, 2, false, true, false),- config(64, 64, 2, 2, true, false, false),- config(64, 64, 2, 2, false, false, false),- config(32, 64, 2, 2, true, true, false),- config(32, 64, 2, 2, false, true, false),- config(64, 32, 4, 1, true, true, false),- config(64, 32, 4, 1, true, false, false),- config(128, 128, 4, 4, true, true, false),- config(128, 128, 4, 4, false, true, false),- config(128, 128, 4, 4, true, false, false),- config(128, 128, 4, 4, false, false, false),- config(64, 64, 2, 2, true, true, true),- config(64, 64, 2, 2, false, true, true),- config(64, 64, 2, 2, true, false, true),- config(64, 64, 2, 2, false, false, true),- config(32, 64, 2, 2, true, true, true),- config(32, 64, 2, 2, false, true, true),- config(64, 32, 4, 1, true, true, true),- config(64, 32, 4, 1, true, false, true),- config(128, 128, 4, 4, true, true, true),- config(128, 128, 4, 4, false, true, true),- config(128, 128, 4, 4, true, false, true),- config(128, 128, 4, 4, false, false, true),- ];-- let mut expanded = Vec::with_capacity(BASE.len() * 4);- for &base in BASE {- expanded.push(Self {- align_k: true,- apply_ab_scale: true,- ..base- });- expanded.push(Self {- align_k: true,- apply_ab_scale: false,- ..base- });- expanded.push(Self {- align_k: false,- apply_ab_scale: true,- ..base- });- expanded.push(Self {- align_k: false,- apply_ab_scale: false,- ..base- });- }-- Box::leak(expanded.into_boxed_slice())- })- }-- pub fn select(- m: u32,- n: u32,- k: u32,- is_accumulate: bool,- apply_ab_scale: bool,- ) -> Self {- let (block_rows, block_cols, simdgroups_per_row, simdgroups_per_column) = if m >= 128 && n >= 128 {- (128, 128, 4u32, 4u32)- } else if n < 64 {- (64, 32, 4u32, 1u32)- } else if m < 64 {- (32, 64, 2u32, 2u32)- } else {- (64, 64, 2u32, 2u32)- };-- Self {- block_rows,- block_cols,- simdgroups_per_row,- simdgroups_per_column,- align_m: (m % block_rows) == 0,- align_n: (n % block_cols) == 0,- align_k: (k % 256) == 0,- apply_ab_scale,- is_accumulate,- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rsnew file mode 100644index 000000000..90441c3db--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs@@ -0,0 +1,59 @@+use super::{kernel::GemvKernel, spec::GemvSpecialization};+use crate::backends::{+ common::{+ Encoder,+ kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError},+ },+ metal::Metal,+};++pub(crate) fn encode(+ kernel: &mut GemvKernel,+ encoder: &mut Encoder<Metal>,+ arguments: MatmulArguments<Metal>,+) -> Result<(), MatmulError<Metal>> {+ let MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ ab_scale,+ c,+ d,+ batch_dim,+ input_dim,+ output_dim,+ } = arguments;+ assert!(b_transpose, "encode_gemv does not support b_transpose=false");+ assert!(b_offset == 0, "encode_gemv does not support nonzero b_offset");+ assert!(+ b_leading_dimension.is_none_or(|ld| ld == input_dim),+ "encode_gemv does not support custom b_leading_dimension"+ );++ let (is_accumulate, output_bias) = match c {+ MatmulArgumentC::Accumulate => (true, None),+ MatmulArgumentC::Bias(bias) => (false, Some(bias)),+ MatmulArgumentC::None => (false, None),+ };++ let specialization = GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());++ kernel.get_or_create(encoder.context(), specialization)?.encode(+ b,+ (a, a_offset),+ output_bias,+ d,+ input_dim,+ output_dim,+ input_dim,+ ab_scale,+ batch_dim,+ specialization.output_rows_per_threadgroup(),+ encoder,+ );++ Ok(())+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalsimilarity index 94%rename from crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.metalrename to crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalindex b08eb3e73..c601f6fff 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -1,8 +1,8 @@#include <metal_stdlib>-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../common/thread_context.h"+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../common/thread_context.h"#include <metal_simdgroup>@@ -47,7 +47,7 @@ KERNEL(MatmulGemv)(}// Simdgroup guard — kill excess simdgroups beyond what this tile uses- if (thread_context.threadgroup_index >= tg_simd_rows * tg_simd_cols) {+ if (thread_context.simdgroup_index >= tg_simd_rows * tg_simd_cols) {return;}@@ -68,26 +68,26 @@ KERNEL(MatmulGemv)(const uint thread_row_in_simdgroup =sg_thread_cols != 32- ? uint(thread_context.simdgroup_index) / uint(sg_thread_cols)+ ? uint(thread_context.simd_lane_id) / uint(sg_thread_cols): 0;const int thread_col_in_simdgroup =sg_thread_cols != 32- ? uint(thread_context.simdgroup_index) % uint(sg_thread_cols)- : uint(thread_context.simdgroup_index);+ ? uint(thread_context.simd_lane_id) % uint(sg_thread_cols)+ : uint(thread_context.simd_lane_id);const int simdgroup_column_index =- tg_simd_cols != 1 ? uint(thread_context.threadgroup_index % tg_simd_cols)+ tg_simd_cols != 1 ? uint(thread_context.simdgroup_index % tg_simd_cols): 0;const uint simdgroup_row_thread_base =tg_simd_cols != 1? uint(sg_thread_rows) *- uint(thread_context.threadgroup_index / tg_simd_cols)- : uint(sg_thread_rows) * uint(thread_context.threadgroup_index);+ uint(thread_context.simdgroup_index / tg_simd_cols)+ : uint(sg_thread_rows) * uint(thread_context.simdgroup_index);const uint simdgroup_col_thread_base =tg_simd_cols != 1? uint(sg_thread_cols) *- uint(thread_context.threadgroup_index % tg_simd_cols)+ uint(thread_context.simdgroup_index % tg_simd_cols): 0;uint output_block_row_offset =diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsnew file mode 100644index 000000000..03d67e584--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -0,0 +1,57 @@+use std::collections::{HashMap, hash_map::Entry};++use super::spec::GemvSpecialization;+use crate::{+ DataType,+ backends::{+ common::kernel::matmul::MatmulError,+ metal::{Metal, context::MetalContext, kernel::MatmulGemvMetalKernel},+ },+};++pub(crate) struct GemvKernel {+ data_type: DataType,+ pipelines: HashMap<GemvSpecialization, MatmulGemvMetalKernel>,+}++impl GemvKernel {+ pub(crate) fn new(+ context: &MetalContext,+ data_type: DataType,+ ) -> Result<Self, MatmulError<Metal>> {+ let mut kernel = Self {+ data_type,+ pipelines: HashMap::new(),+ };+ for &config in GemvSpecialization::precompile_configs(data_type) {+ kernel.get_or_create(context, config)?;+ }+ Ok(kernel)+ }++ pub(crate) fn get_or_create(+ &mut self,+ context: &MetalContext,+ specialization: GemvSpecialization,+ ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {+ match self.pipelines.entry(specialization) {+ Entry::Occupied(entry) => Ok(entry.into_mut()),+ Entry::Vacant(entry) => {+ let kernel = MatmulGemvMetalKernel::new(+ context,+ self.data_type,+ specialization.threadgroup_rows,+ specialization.threadgroup_cols,+ specialization.threads_per_simdgroup_row,+ specialization.threads_per_simdgroup_col,+ specialization.elements_per_thread_row,+ specialization.elements_per_thread_col,+ specialization.is_accumulate,+ specialization.is_bias,+ )+ .map_err(MatmulError::BackendError)?;+ Ok(entry.insert(kernel))+ },+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rsnew file mode 100644index 000000000..48df5aef2--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -0,0 +1,5 @@+pub(crate) mod fp;+mod kernel;+mod spec;++pub(crate) use kernel::GemvKernel;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rssimilarity index 100%rename from crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.rsrename to crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rsdiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 1939c7c37..464bbd0d8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,44 +1,32 @@-use std::{- collections::{HashMap, hash_map::Entry},- sync::OnceLock,-};+pub mod gemm;+pub mod gemv;+pub mod quant;++use std::sync::OnceLock;+use self::{gemm::GemmKernel, gemv::GemvKernel};use crate::{DataType,backends::{common::{- Allocation, Encoder,- gpu_types::GemmParams,+ Encoder,kernel::{TensorAddBiasKernel,- matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},- },- },- metal::{- Metal,- context::MetalContext,- kernel::{- MatmulGemmMetalKernel, MatmulGemmMppMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel,+ matmul::{MatmulArguments, MatmulError, MatmulKernel},},- metal_extensions::DeviceExt,},+ metal::{Metal, context::MetalContext, kernel::TensorAddBiasMetalKernel, metal_extensions::DeviceExt},},};-mod gemm;-mod gemm_mpp;-mod gemv;-pub struct MatmulMetalKernel {data_type: DataType,- gemv_kernels: HashMap<gemv::GemvSpecialization, MatmulGemvMetalKernel>,- gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,- gemm_mpp_kernels: HashMap<gemm_mpp::GemmMppSpecialization, MatmulGemmMppMetalKernel>,- bias_add: TensorAddBiasMetalKernel,+ gemv: GemvKernel,+ pub(crate) gemm: GemmKernel,+ pub(crate) bias_add: TensorAddBiasMetalKernel,}const DEFAULT_GEMV_MAX_BATCH: u32 = 8;-static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();fn max_gemv_batch_threshold() -> u32 {@@ -48,286 +36,12 @@ fn max_gemv_batch_threshold() -> u32 {}impl MatmulMetalKernel {- fn is_mpp_eligible(+ fn is_mxu_eligible(&self,context: &MetalContext,) -> bool {context.device.supports_mxu() && matches!(self.data_type, DataType::F16 | DataType::BF16)}-- fn get_or_create_gemv(- &mut self,- context: &MetalContext,- specialization: gemv::GemvSpecialization,- ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {- match self.gemv_kernels.entry(specialization) {- Entry::Occupied(entry) => Ok(entry.into_mut()),- Entry::Vacant(entry) => {- let kernel = MatmulGemvMetalKernel::new(- context,- self.data_type,- specialization.threadgroup_rows,- specialization.threadgroup_cols,- specialization.threads_per_simdgroup_row,- specialization.threads_per_simdgroup_col,- specialization.elements_per_thread_row,- specialization.elements_per_thread_col,- specialization.is_accumulate,- specialization.is_bias,- )- .map_err(MatmulError::BackendError)?;- Ok(entry.insert(kernel))- },- }- }-- fn get_or_create_gemm(- &mut self,- context: &MetalContext,- specialization: gemm::GemmSpecialization,- ) -> Result<&MatmulGemmMetalKernel, MatmulError<Metal>> {- match self.gemm_kernels.entry(specialization) {- Entry::Occupied(entry) => Ok(entry.into_mut()),- Entry::Vacant(entry) => {- let kernel = MatmulGemmMetalKernel::new(- context,- self.data_type,- specialization.block_rows as u32,- specialization.block_cols as u32,- specialization.block_depth as u32,- specialization.simdgroups_per_row as u32,- specialization.simdgroups_per_column as u32,- specialization.align_mn,- specialization.align_k,- specialization.transpose_b,- specialization.is_accumulate,- )- .map_err(MatmulError::BackendError)?;- Ok(entry.insert(kernel))- },- }- }-- fn get_or_create_gemm_mpp(- &mut self,- context: &MetalContext,- specialization: gemm_mpp::GemmMppSpecialization,- ) -> Result<&MatmulGemmMppMetalKernel, MatmulError<Metal>> {- match self.gemm_mpp_kernels.entry(specialization) {- Entry::Occupied(entry) => Ok(entry.into_mut()),- Entry::Vacant(entry) => {- let kernel = MatmulGemmMppMetalKernel::new(- context,- self.data_type,- specialization.block_rows,- specialization.block_cols,- specialization.simdgroups_per_row,- specialization.simdgroups_per_column,- specialization.align_m,- specialization.align_n,- specialization.align_k,- specialization.apply_ab_scale,- specialization.is_accumulate,- )- .map_err(MatmulError::BackendError)?;- Ok(entry.insert(kernel))- },- }- }-- fn encode_gemv(- &mut self,- encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal>,- ) -> Result<(), MatmulError<Metal>> {- let MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- ab_scale,- c,- d,- batch_dim,- input_dim,- output_dim,- } = arguments;- assert!(b_transpose, "encode_gemv does not yet support b_transpose=false");-- let (is_accumulate, output_bias) = match c {- MatmulArgumentC::Accumulate => (true, None),- MatmulArgumentC::Bias(bias) => (false, Some(bias)),- MatmulArgumentC::None => (false, None),- };-- let specialization =- gemv::GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());-- self.get_or_create_gemv(encoder.context(), specialization)?.encode(- (b, b_offset),- (a, a_offset),- output_bias,- d,- input_dim,- output_dim,- b_leading_dimension.unwrap_or(input_dim),- ab_scale,- batch_dim,- specialization.output_rows_per_threadgroup(),- encoder,- );-- Ok(())- }-- fn encode_gemm(- &mut self,- encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal>,- ) -> Result<(), MatmulError<Metal>> {- let MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- ab_scale,- c,- d,- batch_dim,- input_dim,- output_dim,- } = arguments;-- let specialization = gemm::GemmSpecialization::select(- encoder.context(),- self.data_type,- batch_dim,- input_dim,- output_dim,- b_transpose,- &c,- );-- let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);- let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);-- let default_ldb = if b_transpose {- input_dim- } else {- output_dim- };- let params = GemmParams {- M: batch_dim,- N: output_dim,- K: input_dim,- leading_dimension_a: input_dim,- leading_dimension_b: b_leading_dimension.unwrap_or(default_ldb),- leading_dimension_d: output_dim,- threadgroups_per_row,- threadgroups_per_column,- swizzle_log: 0,- aligned_inner_iterations: input_dim / specialization.block_depth,- use_morton: false,- };-- let kernel = self.get_or_create_gemm(encoder.context(), specialization)?;-- kernel.encode(- (a, a_offset),- (b, b_offset),- &mut *d,- std::slice::from_ref(¶ms),- threadgroups_per_row,- threadgroups_per_column,- ab_scale,- encoder,- );-- if let MatmulArgumentC::Bias(bias) = c {- self.bias_add.encode(None::<&Allocation<Metal>>, bias, d, output_dim, batch_dim * output_dim, encoder);- }-- Ok(())- }-- fn encode_gemm_mpp(- &mut self,- encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal>,- ) -> Result<(), MatmulError<Metal>> {- let MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- ab_scale,- c,- d,- batch_dim,- input_dim,- output_dim,- } = arguments;- assert!(b_transpose, "encode_gemm_mpp does not yet support b_transpose=false");- let is_accumulate = matches!(c, MatmulArgumentC::Accumulate);- let specialization =- gemm_mpp::GemmMppSpecialization::select(batch_dim, output_dim, input_dim, is_accumulate, ab_scale != 1.0);-- let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);- let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);-- let max_threadgroups_per_dim = threadgroups_per_row.max(threadgroups_per_column);- let min_threadgroups_per_dim = threadgroups_per_row.min(threadgroups_per_column);- let morton_dim = max_threadgroups_per_dim.next_power_of_two();- let morton_total = morton_dim.saturating_mul(morton_dim);- let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);- // Morton ordering improves cache locality but requires padding to a square power-of-two grid.- // Allow up to 4x overhead (idle threadgroups) before the padding cost outweighs the benefit.- let use_morton = min_threadgroups_per_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);-- let params = GemmParams {- M: batch_dim,- N: output_dim,- K: input_dim,- leading_dimension_a: input_dim,- leading_dimension_b: b_leading_dimension.unwrap_or(input_dim),- leading_dimension_d: output_dim,- threadgroups_per_row,- threadgroups_per_column,- swizzle_log: 0,- aligned_inner_iterations: 0,- use_morton,- };-- let (group_count_x, group_count_y) = if use_morton {- (morton_total, 1_u32)- } else {- (threadgroups_per_row, threadgroups_per_column)- };-- let kernel = self.get_or_create_gemm_mpp(encoder.context(), specialization)?;- kernel.encode(- (a, a_offset),- (b, b_offset),- &mut *d,- std::slice::from_ref(¶ms),- group_count_x,- group_count_y,- ab_scale,- encoder,- );-- if let MatmulArgumentC::Bias(bias) = c {- self.bias_add.encode(None::<&Allocation<Metal>>, bias, d, output_dim, batch_dim * output_dim, encoder);- }-- Ok(())- }}/// Explicit dispatch paths for testing individual kernels independent of production routing.@@ -336,7 +50,7 @@ pub enum MatmulDispatchPath {Auto,Gemv,Gemm,- GemmMpp,+ GemmMxu,}impl MatmulMetalKernel {@@ -346,11 +60,20 @@ impl MatmulMetalKernel {encoder: &mut Encoder<Metal>,path: MatmulDispatchPath,) {+ let context = encoder.context();match path {MatmulDispatchPath::Auto => self.encode(arguments, encoder),- MatmulDispatchPath::Gemv => self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV"),- MatmulDispatchPath::Gemm => self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM"),- MatmulDispatchPath::GemmMpp => self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP"),+ MatmulDispatchPath::Gemv => {+ gemv::fp::encode(&mut self.gemv, encoder, arguments).expect("Failed to encode GEMV")+ },+ MatmulDispatchPath::Gemm => {+ gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, false)+ .expect("Failed to encode Gemm")+ },+ MatmulDispatchPath::GemmMxu => {+ gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, true)+ .expect("Failed to encode GemmMxu")+ },}}}@@ -367,28 +90,15 @@ impl MatmulKernel for MatmulMetalKernel {}let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true).map_err(MatmulError::BackendError)?;+ let gemm = GemmKernel::new(context, data_type).map_err(MatmulError::BackendError)?;+ let gemv = GemvKernel::new(context, data_type)?;- let mut kernel = Self {+ Ok(Self {data_type,- gemv_kernels: HashMap::new(),- gemm_kernels: HashMap::new(),- gemm_mpp_kernels: HashMap::new(),+ gemv,+ gemm,bias_add,- };-- for &config in gemv::GemvSpecialization::precompile_configs(data_type) {- kernel.get_or_create_gemv(context, config)?;- }- for &config in gemm::GemmSpecialization::precompile_configs(data_type) {- kernel.get_or_create_gemm(context, config)?;- }- if context.device.supports_mxu() {- for &config in gemm_mpp::GemmMppSpecialization::precompile_configs(data_type) {- kernel.get_or_create_gemm_mpp(context, config)?;- }- }-- Ok(kernel)+ })}fn encode(@@ -396,14 +106,20 @@ impl MatmulKernel for MatmulMetalKernel {arguments: MatmulArguments<Metal>,encoder: &mut Encoder<Metal>,) {- if !arguments.b_transpose {- self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM kernel");- } else if arguments.batch_dim <= max_gemv_batch_threshold() {- self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV kernel");- } else if self.is_mpp_eligible(encoder.context()) {- self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP kernel");- } else {- self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM kernel");+ // gemv only handles the canonical B layout; non-canonical inputs go to GEMM.+ let context = encoder.context();+ let gemv_eligible = arguments.b_transpose+ && arguments.b_offset == 0+ && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.input_dim)+ && arguments.batch_dim <= max_gemv_batch_threshold();++ if gemv_eligible {+ gemv::fp::encode(&mut self.gemv, encoder, arguments).expect("Failed to encode GEMV kernel");+ return;}++ let use_mxu = self.is_mxu_eligible(context);+ gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, use_mxu)+ .expect("Failed to encode GEMM");}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rsnew file mode 100644index 000000000..6ccbbcd74--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs@@ -0,0 +1,37 @@+use super::{MatmulMetalKernel, gemm};+use crate::backends::{+ common::{+ Encoder,+ kernel::quant_matmul::{+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,+ QuantizedMatmulKernelEncodable,+ },+ },+ metal::{Metal, context::MetalContext},+};++#[derive(Debug, Clone, Copy)]+pub enum QuantizedMatmulDispatchPath {+ Auto,+ Gemm,+}++pub fn encode_quantized_matmul_with_path(+ context: &MetalContext,+ matmul: &mut MatmulMetalKernel,+ encodable: &QuantizedMatmulKernelEncodable<Metal>,+ configuration: &QuantizedMatmulConfiguration,+ arguments: QuantizedMatmulArguments<Metal>,+ encoder: &mut Encoder<Metal>,+ path: QuantizedMatmulDispatchPath,+) -> Result<(), QuantizedMatmulError<Metal>> {+ match path {+ QuantizedMatmulDispatchPath::Auto => {+ encodable.encode(encoder, arguments);+ Ok(())+ },+ QuantizedMatmulDispatchPath::Gemm => {+ gemm::quant::encode(&mut matmul.gemm, context, configuration, arguments, encoder)+ },+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metalindex 429651362..75e029e9b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -37,7 +37,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(return;const uint expert_u = uint(expert_id);- const uint h_idx = h_block_idx * 4 + thread_context.threadgroup_index;+ const uint h_idx = h_block_idx * 4 + thread_context.simdgroup_index;if (h_idx >= d_ff)return;@@ -55,7 +55,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(const uint vec_iters = d_model / 128;for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + thread_context.simdgroup_index * 4;+ uint base_idx = i * 128 + thread_context.simd_lane_id * 4;device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);device const T* w_up_vec2 =@@ -75,7 +75,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(}}- uint leftover_start = vec_iters * 128 + thread_context.simdgroup_index;+ uint leftover_start = vec_iters * 128 + thread_context.simd_lane_id;for (uint idx = leftover_start; idx < d_model; idx += 32) {float xv = float(x[idx]);acc_up += xv * float(w_up_row[idx]);@@ -89,7 +89,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(acc_gate = simd_sum(acc_gate);}- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {float up_val = clamp(acc_up + float(biases[bias_base + h_idx]),up_clip_min,@@ -138,7 +138,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(const uint d_block GROUPS(d_model.div_ceil(8)),const uint tid THREADS(256)) {- const uint my_col = d_block * 8 + thread_context.threadgroup_index;+ const uint my_col = d_block * 8 + thread_context.simdgroup_index;if (my_col >= d_model)return;@@ -160,7 +160,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(// Vectorized reductionfor (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + thread_context.simdgroup_index * 4;+ uint base_idx = i * 128 + thread_context.simd_lane_id * 4;float4 h_vec =*reinterpret_cast<device const float4*>(hidden_ptr + base_idx);@@ -173,21 +173,20 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(}// Remainder- for (uint idx = vec_iters * 128 + thread_context.simdgroup_index;- idx < d_ff;+ for (uint idx = vec_iters * 128 + thread_context.simd_lane_id; idx < d_ff;idx += 32) {acc += hidden_ptr[idx] * float(w2_ptr[idx]);}// Simdgroup reduction and accumulatefloat result = simd_sum(acc);- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);final_acc += prob * result;}}- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {y[my_col] = T(final_acc);}}\ No newline at end of filediff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex c573e94ba..4a46fe2a0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -44,7 +44,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(return;// Each simdgroup outputs one hidden element- const uint h_idx = h_block_idx * 4 + thread_context.threadgroup_index;+ const uint h_idx = h_block_idx * 4 + thread_context.simdgroup_index;if (h_idx >= d_ff)return;@@ -66,7 +66,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(const uint vec_iters = d_model / 128;for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + thread_context.simdgroup_index * 4;+ uint base_idx = i * 128 + thread_context.simd_lane_id * 4;device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);device const T* w_up_vec =@@ -87,7 +87,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(}// Handle leftover elements- uint leftover_start = vec_iters * 128 + thread_context.simdgroup_index;+ uint leftover_start = vec_iters * 128 + thread_context.simd_lane_id;for (uint idx = leftover_start; idx < d_model; idx += 32) {float xv = float(x_ptr[idx]);acc_up += xv * float(w_up_row[idx]);@@ -103,7 +103,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(}// Lane 0 applies activation and writes result- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {float up_val = acc_up + float(up_biases[bias_base + h_idx]);up_val = clamp(up_val, up_clip_min, up_clip_max);@@ -152,7 +152,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Each simdgroup computes one output columnconst uint my_col =- tgpig_x * SIMDGROUPS_PER_TG + thread_context.threadgroup_index;+ tgpig_x * SIMDGROUPS_PER_TG + thread_context.simdgroup_index;if (my_col >= d_model)return;@@ -174,7 +174,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(for (uint iter = 0; iter < k_vec_iters; ++iter) {const uint k_base =- iter * (8 * THREADS_PER_SIMD) + thread_context.simdgroup_index;+ iter * (8 * THREADS_PER_SIMD) + thread_context.simd_lane_id;// hidden: stride-32 per lane, already f32 from Pass Aconst AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];@@ -220,7 +220,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Handle remaining full iterationsfor (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {- const uint k = iter * THREADS_PER_SIMD + thread_context.simdgroup_index;+ const uint k = iter * THREADS_PER_SIMD + thread_context.simd_lane_id;acc =fma(AccumT(hidden[hidden_base + k]),AccumT(w2_all[w2_col_base + k]),@@ -229,8 +229,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Handle leftover elements (d_ff % 32)const uint leftover_start = k_iters * THREADS_PER_SIMD;- if (leftover_start + thread_context.simdgroup_index < d_ff) {- const uint k = leftover_start + thread_context.simdgroup_index;+ if (leftover_start + thread_context.simd_lane_id < d_ff) {+ const uint k = leftover_start + thread_context.simd_lane_id;acc =fma(AccumT(hidden[hidden_base + k]),AccumT(w2_all[w2_col_base + k]),@@ -241,7 +241,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(AccumT result = simd_sum(acc);// Lane 0 writes result- if (thread_context.simdgroup_index == 0) {+ if (thread_context.simd_lane_id == 0) {const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;result += AccumT(down_biases[bias_idx]);diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex 2c46971e0..c6b8cffae 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -106,15 +106,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(// simdgroup tile mappingconst uint sg_col_count = Bn / SgBn;- const uint row_sg = thread_context.threadgroup_index / sg_col_count;- const uint col_sg = thread_context.threadgroup_index % sg_col_count;+ const uint row_sg = thread_context.simdgroup_index / sg_col_count;+ const uint col_sg = thread_context.simdgroup_index % sg_col_count;const uint row_sg_off = row_sg * SgBm;const uint col_sg_off = col_sg * SgBn;// Guard against misconfigured TG sizesconstexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);- if (thread_context.threadgroup_index >= SG_EXPECTED)+ if (thread_context.simdgroup_index >= SG_EXPECTED)return;// partial accumulators (8x8 tiles per simdgroup)@@ -295,7 +295,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Epilogue: activation/gating → hidden_out (f32) ----- const uint sg_lane_start = thread_context.threadgroup_index * 32;+ const uint sg_lane_start = thread_context.simdgroup_index * 32;const uint sg_lane_end = sg_lane_start + 32;if (lin < sg_lane_start || lin >= sg_lane_end)return;@@ -448,14 +448,14 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(const ulong bias_base = (ulong)expert_idx * (ulong)d_model;// 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)- const uint row_sg = thread_context.threadgroup_index / 2; // 0-1 → rows- const uint col_sg = thread_context.threadgroup_index % 2; // 0-1 → cols+ const uint row_sg = thread_context.simdgroup_index / 2; // 0-1 → rows+ const uint col_sg = thread_context.simdgroup_index % 2; // 0-1 → colsconst uint row_sg_off = row_sg * SgBm;const uint col_sg_off = col_sg * SgBn;// Guard against excessive simdgroupsconstexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);- if (thread_context.threadgroup_index >= SG_EXPECTED)+ if (thread_context.simdgroup_index >= SG_EXPECTED)return;// 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles@@ -563,11 +563,11 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Writeout with fragment extraction ----- const uint lane_qid = thread_context.simdgroup_index >> 2;+ const uint lane_qid = thread_context.simd_lane_id >> 2;const uint lane_row =- (lane_qid & 4u) + ((thread_context.simdgroup_index >> 1) & 3u);+ (lane_qid & 4u) + ((thread_context.simd_lane_id >> 1) & 3u);const uint lane_col_base =- ((lane_qid & 2u) * 2u) + ((thread_context.simdgroup_index & 1u) * 2u);+ ((lane_qid & 2u) * 2u) + ((thread_context.simd_lane_id & 1u) * 2u);const uint local_row = row_sg_off + lane_row;if (local_row >= m_rows)diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metalindex cbb1a161e..78eca3add 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -52,12 +52,12 @@ PUBLIC KERNEL(MoeRouterTopK)(}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint row = thread_context.threadgroup_index; row < e;+ for (uint row = thread_context.simdgroup_index; row < e;row += thread_context.simdgroups_per_threadgroup) {const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;float4 accum4 = float4(0.0f);- for (uint c = thread_context.simdgroup_index; c < vecs; c += 32u) {+ for (uint c = thread_context.simd_lane_id; c < vecs; c += 32u) {const float4 wv = float4(w_vec[c * 4 + 0],w_vec[c * 4 + 1],diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex 3eade2617..860fb1332 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -65,7 +65,7 @@ inline void moe_scatter_buckets_impl(if (ue >= e0 && ue < e0 + tile_e) {const uint te = ue - e0;atomic_fetch_add_explicit(- &sg_counts[thread_context.threadgroup_index * TILE_E + te],+ &sg_counts[thread_context.simdgroup_index * TILE_E + te],1u,memory_order_relaxed);@@ -104,7 +104,7 @@ inline void moe_scatter_buckets_impl(// Phase 3: deterministic write with 32-lane sequencing, no device atomicsfor (uint step = 0; step < thread_context.simdgroup_size; ++step) {- if (thread_context.simdgroup_index == step) {+ if (thread_context.simd_lane_id == step) {const uint t = t_start + lid;if (t < t_end) {const uint base = t * K;@@ -118,7 +118,7 @@ inline void moe_scatter_buckets_impl(const uint base_idx_block =(block_id * num_tiles + tile_id) * TILE_E + te;const uint idx_sg_te =- thread_context.threadgroup_index * TILE_E + te;+ thread_context.simdgroup_index * TILE_E + te;const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te],1u,diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rsindex 478f2add5..d00c639d3 100644--- a/crates/backend-uzu/src/backends/metal/mod.rs+++ b/crates/backend-uzu/src/backends/metal/mod.rs@@ -13,5 +13,8 @@ pub use backend::Metal;pub use buffer::BufferDowncastExt;pub use context::MetalContext;pub use device_capabilities::MetalDeviceCapabilities;-pub use kernel::matmul::MatmulDispatchPath;+pub use kernel::matmul::{+ MatmulDispatchPath,+ quant::{QuantizedMatmulDispatchPath, encode_quantized_matmul_with_path},+};pub use metal_extensions::{DeviceExt, DeviceGeneration};diff --git a/crates/backend-uzu/src/config/attention.rs b/crates/backend-uzu/src/config/attention.rsindex cf665ee2a..1326dc828 100644--- a/crates/backend-uzu/src/config/attention.rs+++ b/crates/backend-uzu/src/config/attention.rs@@ -1,6 +1,7 @@use proc_macros::uzu_config;-use super::{ConfigDataType, LinearConfig, NormalizationConfig, UpcastMode};+use super::{LinearConfig, NormalizationConfig, UpcastMode};+use crate::DataType;pub const VALUE_NORM_EPSILON: f32 = 1e-6;@@ -34,7 +35,7 @@ impl AttentionConfig {}Some(NormalizationConfig {scale_precision: self.qkv_projection_config.activation_precision(),- accumulation_precision: ConfigDataType::Float32,+ accumulation_precision: DataType::F32,epsilon: VALUE_NORM_EPSILON,scale_offset: None,upcast_mode: UpcastMode::FullLayer,diff --git a/crates/backend-uzu/src/config/common.rs b/crates/backend-uzu/src/config/common.rsdeleted file mode 100644index 4d9429729..000000000--- a/crates/backend-uzu/src/config/common.rs+++ /dev/null@@ -1,40 +0,0 @@-use proc_macros::uzu_config;--use crate::DataType;--#[derive(Eq, Copy)]-#[uzu_config]-#[serde(rename = "DataType")]-#[serde(rename_all = "lowercase")]-pub enum ConfigDataType {- BFloat16,- Float16,- Float32,- Int4,- Int8,-}--impl From<ConfigDataType> for DataType {- fn from(val: ConfigDataType) -> Self {- match val {- ConfigDataType::BFloat16 => DataType::BF16,- ConfigDataType::Float16 => DataType::F16,- ConfigDataType::Float32 => DataType::F32,- ConfigDataType::Int4 => DataType::I4,- ConfigDataType::Int8 => DataType::I8,- }- }-}--impl From<DataType> for ConfigDataType {- fn from(dtype: DataType) -> Self {- match dtype {- DataType::BF16 => ConfigDataType::BFloat16,- DataType::F16 => ConfigDataType::Float16,- DataType::F32 => ConfigDataType::Float32,- DataType::I4 => ConfigDataType::Int4,- DataType::I8 => ConfigDataType::Int8,- _ => panic!("Unsupported data type: {0:?}", dtype),- }- }-}diff --git a/crates/backend-uzu/src/config/embedding.rs b/crates/backend-uzu/src/config/embedding.rsindex 61ba7cd40..80f43d8e4 100644--- a/crates/backend-uzu/src/config/embedding.rs+++ b/crates/backend-uzu/src/config/embedding.rs@@ -1,6 +1,6 @@use proc_macros::uzu_config;-use crate::{ConfigDataType, backends::common::gpu_types::QuantizationMode};+use crate::{DataType, backends::common::gpu_types::QuantizationMode};#[uzu_config]pub struct EmbeddingConfigCommon {@@ -15,40 +15,40 @@ pub enum EmbeddingConfig {Tied {#[serde(flatten)]common: EmbeddingConfigCommon,- precision: ConfigDataType,+ precision: DataType,},#[serde(rename = "UntiedEmbeddingConfig")]Untied {#[serde(flatten)]common: EmbeddingConfigCommon,- precision: ConfigDataType,+ precision: DataType,},- #[serde(rename = "MLXQuantizedTiedEmbeddingConfig")]- MLXQuantizedTied {+ #[serde(rename = "ScaleBiasQuantizedTiedEmbeddingConfig")]+ ScaleBiasQuantizedTied {#[serde(flatten)]common: EmbeddingConfigCommon,group_size: usize,embedding_quantization_mode: QuantizationMode,activation_quantization_mode: Option<QuantizationMode>,- activation_precision: ConfigDataType,+ activation_precision: DataType,},- #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]- MLXQuantizedUntied {+ #[serde(rename = "ScaleBiasQuantizedUntiedEmbeddingConfig")]+ ScaleBiasQuantizedUntied {#[serde(flatten)]common: EmbeddingConfigCommon,group_size: usize,embedding_quantization_mode: QuantizationMode,activation_quantization_mode: Option<QuantizationMode>,- activation_precision: ConfigDataType,+ activation_precision: DataType,},- #[serde(rename = "MLXSemiQuantizedUntiedEmbeddingConfig")]- MLXSemiQuantizedUntied {+ #[serde(rename = "ScaleBiasSemiQuantizedUntiedEmbeddingConfig")]+ ScaleBiasSemiQuantizedUntied {#[serde(flatten)]common: EmbeddingConfigCommon,group_size: usize,embedding_quantization_mode: QuantizationMode,activation_quantization_mode: Option<QuantizationMode>,- activation_precision: ConfigDataType,+ activation_precision: DataType,},}@@ -63,22 +63,22 @@ impl EmbeddingConfig {common,..}- | EmbeddingConfig::MLXQuantizedTied {+ | EmbeddingConfig::ScaleBiasQuantizedTied {common,..}- | EmbeddingConfig::MLXQuantizedUntied {+ | EmbeddingConfig::ScaleBiasQuantizedUntied {common,..}- | EmbeddingConfig::MLXSemiQuantizedUntied {+ | EmbeddingConfig::ScaleBiasSemiQuantizedUntied {common,..} => common,}}- pub fn activation_precision(&self) -> ConfigDataType {+ pub fn activation_precision(&self) -> DataType {match self {EmbeddingConfig::Tied {precision,@@ -88,15 +88,15 @@ impl EmbeddingConfig {precision,..} => *precision,- EmbeddingConfig::MLXQuantizedTied {+ EmbeddingConfig::ScaleBiasQuantizedTied {activation_precision,..}- | EmbeddingConfig::MLXQuantizedUntied {+ | EmbeddingConfig::ScaleBiasQuantizedUntied {activation_precision,..}- | EmbeddingConfig::MLXSemiQuantizedUntied {+ | EmbeddingConfig::ScaleBiasSemiQuantizedUntied {activation_precision,..} => *activation_precision,diff --git a/crates/backend-uzu/src/config/linear.rs b/crates/backend-uzu/src/config/linear.rsindex 29acbe71c..4c2e4948e 100644--- a/crates/backend-uzu/src/config/linear.rs+++ b/crates/backend-uzu/src/config/linear.rs@@ -1,13 +1,13 @@use proc_macros::uzu_config;-use crate::{ConfigDataType, backends::common::gpu_types::QuantizationMode};+use crate::{DataType, backends::common::gpu_types::QuantizationMode};#[uzu_config]pub struct QuantizationConfig {pub group_size: usize,pub weight_quantization_mode: QuantizationMode,pub activation_quantization_mode: Option<QuantizationMode>,- pub activation_precision: ConfigDataType,+ pub activation_precision: DataType,}#[uzu_config]@@ -15,12 +15,12 @@ pub struct QuantizationConfig {pub enum LinearConfig {#[serde(rename = "FullPrecisionLinearConfig")]FullPrecision {- precision: ConfigDataType,+ precision: DataType,},#[serde(rename = "GroupQuantizedLinearConfig")]Quantized(QuantizationConfig),#[serde(rename = "MLXQuantizedLinearConfig")]- MLXQuantized(QuantizationConfig),+ ScaleBiasQuantized(QuantizationConfig),#[serde(rename = "QLoRALinearConfig")]QLoRA {#[serde(flatten)]@@ -36,13 +36,13 @@ pub enum LinearConfig {}impl LinearConfig {- pub fn activation_precision(&self) -> ConfigDataType {+ pub fn activation_precision(&self) -> DataType {match self {LinearConfig::FullPrecision {precision,} => *precision,LinearConfig::Quantized(quantization) => quantization.activation_precision,- LinearConfig::MLXQuantized(quantization) => quantization.activation_precision,+ LinearConfig::ScaleBiasQuantized(quantization) => quantization.activation_precision,LinearConfig::QLoRA {quantization,..diff --git a/crates/backend-uzu/src/config/mixer.rs b/crates/backend-uzu/src/config/mixer.rsindex a720378b8..56fc990f1 100644--- a/crates/backend-uzu/src/config/mixer.rs+++ b/crates/backend-uzu/src/config/mixer.rs@@ -1,6 +1,7 @@use proc_macros::uzu_config;-use super::{AttentionConfig, ConfigDataType, DeltaNetAttentionConfig, Mamba2Config, ShortConvConfig};+use super::{AttentionConfig, DeltaNetAttentionConfig, Mamba2Config, ShortConvConfig};+use crate::DataType;#[uzu_config]#[serde(tag = "type")]@@ -44,7 +45,7 @@ impl MixerConfig {}}- pub fn activation_precision(&self) -> ConfigDataType {+ pub fn activation_precision(&self) -> DataType {match self {MixerConfig::Attention(c) => c.qkv_projection_config.activation_precision(),MixerConfig::Mamba(c) => c.in_projection_config.activation_precision(),diff --git a/crates/backend-uzu/src/config/mod.rs b/crates/backend-uzu/src/config/mod.rsindex 399b09cd3..65b81eb82 100644--- a/crates/backend-uzu/src/config/mod.rs+++ b/crates/backend-uzu/src/config/mod.rs@@ -1,7 +1,6 @@mod attention;mod classifier;mod classifier_model;-mod common;mod decoder;mod delta_net;mod embedding;@@ -25,7 +24,6 @@ mod tts_model;pub use attention::*;pub use classifier::*;pub use classifier_model::*;-pub use common::*;pub use decoder::*;pub use delta_net::*;pub use embedding::*;diff --git a/crates/backend-uzu/src/config/normalization/normalization_config.rs b/crates/backend-uzu/src/config/normalization/normalization_config.rsindex 8937bf290..212e5a1a9 100644--- a/crates/backend-uzu/src/config/normalization/normalization_config.rs+++ b/crates/backend-uzu/src/config/normalization/normalization_config.rs@@ -1,12 +1,12 @@use proc_macros::uzu_config;use super::UpcastMode;-use crate::ConfigDataType;+use crate::DataType;#[uzu_config]pub struct NormalizationConfig {- pub scale_precision: ConfigDataType,- pub accumulation_precision: ConfigDataType,+ pub scale_precision: DataType,+ pub accumulation_precision: DataType,pub epsilon: f32,pub scale_offset: Option<f32>,pub upcast_mode: UpcastMode,diff --git a/crates/backend-uzu/src/config/rope.rs b/crates/backend-uzu/src/config/rope.rsindex ff96eab49..31a1a4375 100644--- a/crates/backend-uzu/src/config/rope.rs+++ b/crates/backend-uzu/src/config/rope.rs@@ -1,10 +1,10 @@use proc_macros::uzu_config;-use crate::ConfigDataType;+use crate::DataType;#[uzu_config]pub struct RopeConfigCommon {- pub precision: ConfigDataType,+ pub precision: DataType,pub base: f32,pub max_sequence_length: usize,pub head_dim: usize,diff --git a/crates/backend-uzu/src/config/short_conv.rs b/crates/backend-uzu/src/config/short_conv.rsindex cf196f65b..d12b53555 100644--- a/crates/backend-uzu/src/config/short_conv.rs+++ b/crates/backend-uzu/src/config/short_conv.rs@@ -1,10 +1,11 @@use proc_macros::uzu_config;-use super::{common::ConfigDataType, linear::LinearConfig};+use super::linear::LinearConfig;+use crate::DataType;#[uzu_config]pub struct SeparableCausalConvConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub has_biases: bool,}diff --git a/crates/backend-uzu/src/config/tts_model.rs b/crates/backend-uzu/src/config/tts_model.rsindex b34f9ebfb..4851cb7b3 100644--- a/crates/backend-uzu/src/config/tts_model.rs+++ b/crates/backend-uzu/src/config/tts_model.rs@@ -4,7 +4,7 @@ use std::path::Path;use proc_macros::uzu_config;use super::{EmbeddingConfig, EmbeddingConfigCommon, NormalizationConfig, TransformerConfig};-use crate::{ConfigDataType, backends::common::ActivationConfig};+use crate::{DataType, backends::common::ActivationConfig};#[uzu_config]pub struct TtsMessageProcessorConfig {@@ -17,7 +17,7 @@ pub struct TtsConfig {pub text_decoder_config: TtsTextDecoderConfig,pub audio_decoder_config: TtsAudioDecoderConfig,pub vocoder_config: NoopVocoderConfig,- pub activation_precision: ConfigDataType,+ pub activation_precision: DataType,}#[uzu_config]@@ -50,21 +50,21 @@ pub struct FishAudioTextDecoderConfig {pub num_codebooks: usize,pub max_seq_len: usize,pub scale_codebook_embeddings: bool,- pub precision: ConfigDataType,+ pub precision: DataType,pub short_logits_size: usize,pub repeat_window_size: usize,}#[uzu_config]pub struct FullPrecisionLinearConfig {- pub precision: ConfigDataType,+ pub precision: DataType,}#[uzu_config]pub struct TiedEmbeddingConfig {#[serde(flatten)]pub common: EmbeddingConfigCommon,- pub precision: ConfigDataType,+ pub precision: DataType,}impl TiedEmbeddingConfig {@@ -86,7 +86,7 @@ pub enum TtsAudioDecoderConfig {#[uzu_config]pub struct DescriptAudioCodecConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub quantizer_config: DescriptAudioQuantizerConfig,pub decoder_config: DescriptAudioDacDecoderConfig,pub samplerate: u32,@@ -104,7 +104,7 @@ pub struct DescriptAudioCodecConfig {#[uzu_config]pub struct DescriptAudioDacDecoderConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub conv_config: DescriptAudioCausalConv1dConfig,pub snake_config: DescriptAudioSnake1dConfig,pub decoder_block_config: DescriptAudioDacDecoderBlockConfig,@@ -113,7 +113,7 @@ pub struct DescriptAudioDacDecoderConfig {#[uzu_config]pub struct DescriptAudioDacDecoderBlockConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub snake_config: DescriptAudioSnake1dConfig,pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,pub res_unit_config: DescriptAudioResidualUnitConfig,@@ -122,7 +122,7 @@ pub struct DescriptAudioDacDecoderBlockConfig {#[uzu_config]pub struct DescriptAudioResidualUnitConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub snake_config: DescriptAudioSnake1dConfig,pub conv_config: DescriptAudioCausalConv1dConfig,pub causal: bool,@@ -130,24 +130,24 @@ pub struct DescriptAudioResidualUnitConfig {#[uzu_config]pub struct DescriptAudioCausalConv1dConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub has_biases: bool,}#[uzu_config]pub struct DescriptAudioCausalTransposeConv1dConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub has_biases: bool,}#[uzu_config]pub struct DescriptAudioSnake1dConfig {- pub precision: ConfigDataType,+ pub precision: DataType,}#[uzu_config]pub struct DescriptAudioQuantizerConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub semantic_quantizer_config: DescriptAudioResidualVectorQuantizeConfig,pub quantizer_config: DescriptAudioResidualVectorQuantizeConfig,pub post_module_config: TransformerConfig,@@ -156,13 +156,13 @@ pub struct DescriptAudioQuantizerConfig {#[uzu_config]pub struct DescriptAudioResidualVectorQuantizeConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub vq_config: DescriptAudioVectorQuantizeConfig,}#[uzu_config]pub struct DescriptAudioVectorQuantizeConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub codebook_config: TiedEmbeddingConfig,pub out_proj_config: FullPrecisionLinearConfig,}@@ -174,14 +174,14 @@ pub struct DescriptAudioUpsamplerConfig {#[uzu_config]pub struct DescriptAudioUpsamplingBlockConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,pub convnext_config: DescriptAudioConvNeXtConfig,}#[uzu_config]pub struct DescriptAudioConvNeXtConfig {- pub precision: ConfigDataType,+ pub precision: DataType,pub activation: ActivationConfig,pub dwconv_config: DescriptAudioCausalConv1dConfig,pub norm_config: NormalizationConfig,diff --git a/crates/backend-uzu/src/data_type.rs b/crates/backend-uzu/src/data_type.rsindex c4621c4e9..5af111f1a 100644--- a/crates/backend-uzu/src/data_type.rs+++ b/crates/backend-uzu/src/data_type.rs@@ -1,25 +1,41 @@use bytemuck::Pod;use half::{bf16, f16};use num_traits::NumCast;+use proc_macros::uzu_config;-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Copy, Clone)]+#[derive(Eq, PartialOrd, Ord, Copy)]+#[uzu_config]pub enum DataType {// Floating point+ #[serde(rename = "bfloat16")]BF16,+ #[serde(rename = "float16")]F16,+ #[serde(rename = "float32")]F32,+ #[serde(rename = "float64")]F64,// Sub-byte integers+ #[serde(rename = "int4")]I4,+ #[serde(rename = "uint4")]U4,// Normal integers+ #[serde(rename = "int8")]I8,+ #[serde(rename = "uint8")]U8,+ #[serde(rename = "int16")]I16,+ #[serde(rename = "uint16")]U16,+ #[serde(rename = "int32")]I32,+ #[serde(rename = "uint32")]U32,+ #[serde(rename = "int64")]I64,+ #[serde(rename = "uint64")]U64,}diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 0a7047244..1b0897e02 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -223,7 +223,7 @@ impl<B: Backend> Embedding<B> {},}},- EmbeddingConfig::MLXQuantizedTied {+ EmbeddingConfig::ScaleBiasQuantizedTied {common: _,group_size,embedding_quantization_mode,@@ -285,7 +285,7 @@ impl<B: Backend> Embedding<B> {},}},- EmbeddingConfig::MLXQuantizedUntied {+ EmbeddingConfig::ScaleBiasQuantizedUntied {common: _,group_size,embedding_quantization_mode,@@ -364,7 +364,7 @@ impl<B: Backend> Embedding<B> {},}},- EmbeddingConfig::MLXSemiQuantizedUntied {+ EmbeddingConfig::ScaleBiasSemiQuantizedUntied {common: _,group_size,embedding_quantization_mode,diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex 33a88f2ba..57eb8995a 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -48,7 +48,7 @@ impl<B: Backend> dyn Linear<B> {) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {let output_dimension_sum: usize = output_dimensions.iter().sum();match config {- LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {+ LinearConfig::Quantized(quantization_config) | LinearConfig::ScaleBiasQuantized(quantization_config) => {let block = QuantizedLinear::new(context,quantization_config,@@ -111,14 +111,9 @@ impl<B: Backend> dyn Linear<B> {output_dim: usize,) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {match config {- LinearConfig::Quantized(config) | LinearConfig::MLXQuantized(config) => Ok(Box::new(QuantizedLinear::new(- context,- config,- input_dim,- output_dim,- parameter_tree,- Some(output_factors),- )?)),+ LinearConfig::Quantized(config) | LinearConfig::ScaleBiasQuantized(config) => Ok(Box::new(+ QuantizedLinear::new(context, config, input_dim, output_dim, parameter_tree, Some(output_factors))?,+ )),LinearConfig::QLoRA {quantization,lora_rank,diff --git a/crates/backend-uzu/src/encodable_block/linear/quantized.rs b/crates/backend-uzu/src/encodable_block/linear/quantized.rsindex dee4df5d5..510605d86 100644--- a/crates/backend-uzu/src/encodable_block/linear/quantized.rs+++ b/crates/backend-uzu/src/encodable_block/linear/quantized.rs@@ -40,9 +40,9 @@ pub enum QuantizedLinearError<B: Backend> {got: DataType,},#[error(- "Unexpected MLX shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"+ "Unexpected scale-bias shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]")]- InvalidMlxShapes {+ InvalidScaleBiasShapes {weights: Box<[usize]>,scales: Box<[usize]>,deq_biases: Box<[usize]>,@@ -54,9 +54,9 @@ pub enum QuantizedLinearError<B: Backend> {got: DataType,},#[error(- "Unexpected AWQ shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"+ "Unexpected scale-zero-point shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]")]- InvalidAwqShapes {+ InvalidScaleZeroPointShapes {weights: Box<[usize]>,scales: Box<[usize]>,zero_points: Box<[usize]>,@@ -134,7 +134,7 @@ impl<B: Backend> QuantizedLinear<B> {&& scales_shape == [output_dim, k_g]&& deq_biases_shape == [output_dim, k_g]){- return Err(QuantizedLinearError::InvalidMlxShapes {+ return Err(QuantizedLinearError::InvalidScaleBiasShapes {weights: weights_shape.into_boxed_slice(),scales: scales_shape.into_boxed_slice(),deq_biases: deq_biases_shape.into_boxed_slice(),@@ -163,7 +163,7 @@ impl<B: Backend> QuantizedLinear<B> {&& scales_shape == [output_dim, k_g]&& zero_points_shape == [output_dim, expected_zero_points_entries]){- return Err(QuantizedLinearError::InvalidAwqShapes {+ return Err(QuantizedLinearError::InvalidScaleZeroPointShapes {weights: weights_shape.into_boxed_slice(),scales: scales_shape.into_boxed_slice(),zero_points: zero_points_shape.into_boxed_slice(),diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rsindex afc80e7d5..3b39a0746 100644--- a/crates/backend-uzu/src/lib.rs+++ b/crates/backend-uzu/src/lib.rs@@ -18,7 +18,6 @@ pub use array::{Array, ArrayContextExt};#[cfg(metal_backend)]pub use audio::{NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};pub use backends::common::{AllocationAccessError, allocation_copy_from_slice, allocation_to_vec};-pub use config::ConfigDataType;pub use data_type::{ArrayElement, DataType};pub use language_model::gumbel::{gumbel_float, revidx};pub use parameters::{ParameterLoader, read_safetensors_metadata};diff --git a/crates/backend-uzu/src/utils/mod.rs b/crates/backend-uzu/src/utils/mod.rsindex 74b765864..f002e61cb 100644--- a/crates/backend-uzu/src/utils/mod.rs+++ b/crates/backend-uzu/src/utils/mod.rs@@ -6,5 +6,4 @@ pub mod model_size;pub mod pointers;pub mod strict_serde;pub mod version;-pub use version::{TOOLCHAIN_VERSION, VERSION};diff --git a/crates/backend-uzu/tests/BENCHMARKS.md b/crates/backend-uzu/tests/BENCHMARKS.mdindex 72d338ff4..4f1ece3b8 100644--- a/crates/backend-uzu/tests/BENCHMARKS.md+++ b/crates/backend-uzu/tests/BENCHMARKS.md@@ -24,16 +24,21 @@ baselines side by side.## Available benchmark groups-| Group id | Filter |-| ------------------------------ | ------------------------------ |-| `Metal/Kernel/Matmul/GEMM_MPP` | `Metal/Kernel/Matmul/GEMM_MPP` |-| `Metal/Kernel/RMSNorm` | `Metal/Kernel/RMSNorm` |-| `Metal/Kernel/Sampling/Argmax` | `Metal/Kernel/Sampling/Argmax` |+| Group id | Filter |+| --------------------------------- | --------------------------------- |+| `Metal/Kernel/Matmul/GEMM` | `Metal/Kernel/Matmul/GEMM` |+| `Metal/Kernel/Matmul/GEMM_MXU` | `Metal/Kernel/Matmul/GEMM_MXU` |+| `Metal/Kernel/QmmTransposed/...` | `Metal/Kernel/QmmTransposed` |+| `Metal/Kernel/QmvFast/...` | `Metal/Kernel/QmvFast` |+| `Metal/Kernel/RMSNorm` | `Metal/Kernel/RMSNorm` |+| `Metal/Kernel/Sampling/Argmax` | `Metal/Kernel/Sampling/Argmax` |++The prefix `Metal/Kernel/Matmul` runs both `GEMM` and `GEMM_MXU` in one pass.## Output layoutEvery run writes into `target/criterion/<label>/…`, where `<label>` is a-free-form name you choose (e.g. `m2_max`, `a18`). The Criterion baseline+free-form name you choose (e.g. `m2_max`, `a19`). The Criterion baselineyou saved lives at `target/criterion/<label>/<benchmark-path>/<baseline-name>/`.## Running on macOS@@ -43,9 +48,9 @@ resolve relative to the package dir:```bashCRITERION_HOME="$PWD/target/criterion/m2_max" cargo bench \- -p uzu \- --bench kernel -- "Metal/Kernel/RMSNorm" \- --save-baseline rms_norm_baseline_m2_max+ -p backend-uzu \+ --bench kernel -- "Metal/Kernel/Matmul" \+ --save-baseline matmul_baseline_m2_max```## Running on iPhone (via `cargo-dinghy`)@@ -55,9 +60,9 @@ app.Key flags:-- `-e CRITERION_HOME=target/criterion/a18` — on-device env var. Path is+- `-e CRITERION_HOME=target/criterion/a19` — on-device env var. Path isrelative to the app's cwd (`Documents/`), so this becomes- `Documents/target/criterion/a18/` on device.+ `Documents/target/criterion/a19/` on device.- `--copy-back "Documents/target=$(pwd)/target"` — after the run,`cargo-dinghy` pulls `Documents/target` from the device into yourrepo's `target/`. `$(pwd)` is required (absolute DST) because the@@ -69,15 +74,15 @@ DEVICE=<DEVICE_ID>cargo dinghy \-d "$DEVICE" \- -e CRITERION_HOME=target/criterion/a18 \+ -e CRITERION_HOME=target/criterion/a19 \--copy-back "Documents/target=$(pwd)/target" \- bench -p uzu --bench kernel -- \- "Metal/Kernel/Matmul/GEMM_MPP" \- --save-baseline matmul_gemm_mpp_baseline_a18+ bench -p backend-uzu --bench kernel -- \+ "Metal/Kernel/Matmul" \+ --save-baseline matmul_baseline_a19```After the run completes you'll have-`target/criterion/a18/Metal/Kernel/Matmul/GEMM_MPP/…/matmul_gemm_mpp_baseline_a18/`+`target/criterion/a19/Metal/Kernel/Matmul/<GEMM|GEMM_MXU>/…/matmul_baseline_a19/`on the host, next to any `m2_max/` baselines.## Viewing reports@@ -92,5 +97,5 @@ To inspect a specific label only:```bashopen target/criterion/m2_max/report/index.html-open target/criterion/a18/report/index.html+open target/criterion/a19/report/index.html```diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rsnew file mode 100644index 000000000..3a52529cf--- /dev/null+++ b/crates/backend-uzu/tests/common/matmul/harness.rs@@ -0,0 +1,161 @@+#[cfg(metal_backend)]+use backend_uzu::backends::metal::{Metal, MetalContext};+use backend_uzu::{+ ArrayContextExt, ArrayElement,+ backends::{+ common::{+ AllocationType, Backend, Context, Encoder,+ kernel::{+ ManualKernels,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ },+ },+ cpu::Cpu,+ },+};+use num_traits::Float;++use super::{+ super::helpers::{alloc_allocation_with_data, allocation_to_vec},+ Shape, Variant,+};++#[cfg(metal_backend)]+pub type MetalMatmulKernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel;++#[derive(Debug, Clone, Copy)]+pub struct Case {+ pub shape: Shape,+ pub ab_scale: f32,+ pub accumulate: bool,+ pub b_transpose: bool,+}++impl Case {+ pub const fn new(shape: Shape) -> Self {+ Self {+ shape,+ ab_scale: 1.0,+ accumulate: false,+ b_transpose: true,+ }+ }++ pub const fn with_ab_scale(+ mut self,+ ab_scale: f32,+ ) -> Self {+ self.ab_scale = ab_scale;+ self+ }++ pub const fn with_accumulate(+ mut self,+ accumulate: bool,+ ) -> Self {+ self.accumulate = accumulate;+ self+ }+}++pub struct Input<T: ArrayElement + Float> {+ pub a: Box<[T]>,+ pub b: Box<[T]>,+ pub d_prefill: Option<Box<[T]>>,+ pub case: Case,+}++pub fn deterministic_input<T: ArrayElement + Float>(case: Case) -> Input<T> {+ let Shape {+ m,+ k,+ n,+ } = case.shape;+ let a: Vec<T> = (0..m * k).map(|i| T::from(((i % 13) as f32) * 0.1 - 0.6).unwrap()).collect();+ let b: Vec<T> = (0..n * k).map(|i| T::from(((i % 17) as f32) * 0.1 - 0.8).unwrap()).collect();+ let d_prefill = case.accumulate.then(|| {+ (0..m * n).map(|i| T::from(((i % 7) as f32) * 0.03 - 0.09).unwrap()).collect::<Vec<_>>().into_boxed_slice()+ });+ Input {+ a: a.into_boxed_slice(),+ b: b.into_boxed_slice(),+ d_prefill,+ case,+ }+}++fn run<B: Backend, T: ArrayElement + Float>(+ context: &B::Context,+ kernel: &mut <B::Kernels as ManualKernels>::MatmulKernel,+ input: &Input<T>,+ encode: impl FnOnce(&mut <B::Kernels as ManualKernels>::MatmulKernel, MatmulArguments<B>, &mut Encoder<B>),+) -> Vec<T> {+ let Shape {+ m,+ k,+ n,+ } = input.case.shape;+ let b_array = if input.case.b_transpose {+ context.create_array_from(&[n, k], &input.b)+ } else {+ context.create_array_from(&[k, n], &input.b)+ };+ let a_allocation = alloc_allocation_with_data::<B, T>(context, &input.a);+ let mut d_allocation = if let Some(ref prefill) = input.d_prefill {+ alloc_allocation_with_data::<B, T>(context, prefill)+ } else {+ context+ .create_allocation(m * n * std::mem::size_of::<T>(), AllocationType::Global)+ .expect("create d allocation")+ };++ let c_arg = if input.case.accumulate {+ MatmulArgumentC::Accumulate+ } else {+ MatmulArgumentC::None+ };++ let mut encoder = Encoder::new(context).expect("encoder");+ encode(+ kernel,+ MatmulArguments {+ a: &a_allocation,+ a_offset: 0,+ b: b_array.allocation(),+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: input.case.b_transpose,+ ab_scale: input.case.ab_scale,+ c: c_arg,+ d: &mut d_allocation,+ batch_dim: m as u32,+ input_dim: k as u32,+ output_dim: n as u32,+ },+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ allocation_to_vec::<B, T>(&d_allocation)+}++pub fn cpu_reference<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = <Cpu as Backend>::Context::new().expect("CPU context");+ let mut kernel = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())+ .expect("CPU MatmulKernel");+ run::<Cpu, T>(&context, &mut kernel, input, |kernel, args, encoder| {+ kernel.encode(args, encoder);+ })+}++#[cfg(metal_backend)]+pub fn run_metal<T: ArrayElement + Float>(+ context: &MetalContext,+ kernel: &mut MetalMatmulKernel,+ input: &Input<T>,+ variant: Variant,+) -> Vec<T> {+ let path = variant.dispatch_path();+ run::<Metal, T>(context, kernel, input, |kernel, args, encoder| {+ kernel.encode_with_path(args, encoder, path);+ })+}diff --git a/crates/backend-uzu/tests/common/matmul/mod.rs b/crates/backend-uzu/tests/common/matmul/mod.rsnew file mode 100644index 000000000..da729946d--- /dev/null+++ b/crates/backend-uzu/tests/common/matmul/mod.rs@@ -0,0 +1,9 @@+pub mod harness;+pub mod shape;+pub mod variant;++#[cfg(metal_backend)]+pub use harness::run_metal;+pub use harness::{Case, cpu_reference, deterministic_input};+pub use shape::{Shape, all_correctness_shapes};+pub use variant::Variant;diff --git a/crates/backend-uzu/tests/common/matmul/shape.rs b/crates/backend-uzu/tests/common/matmul/shape.rsnew file mode 100644index 000000000..3165f2031--- /dev/null+++ b/crates/backend-uzu/tests/common/matmul/shape.rs@@ -0,0 +1,39 @@+use derive_more::Display;++#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]+#[display("M[{m}]K[{k}]N[{n}]")]+pub struct Shape {+ pub m: usize,+ pub k: usize,+ pub n: usize,+}++impl Shape {+ pub const fn new(+ m: usize,+ k: usize,+ n: usize,+ ) -> Self {+ Self {+ m,+ k,+ n,+ }+ }+}++const SHAPES_TINY: &[Shape] = &[Shape::new(64, 64, 64), Shape::new(16, 128, 256), Shape::new(128, 256, 128)];++const SHAPES_UNALIGNED: &[Shape] = &[+ Shape::new(7, 33, 11),+ Shape::new(33, 2048, 2048),+ Shape::new(64, 2048, 33),+ Shape::new(200, 2048, 2048),+ Shape::new(128, 2048, 200),+];++const SHAPES_MEDIUM: &[Shape] = &[Shape::new(128, 2048, 2048), Shape::new(256, 4096, 4096)];++pub fn all_correctness_shapes() -> impl Iterator<Item = Shape> {+ SHAPES_TINY.iter().chain(SHAPES_UNALIGNED.iter()).chain(SHAPES_MEDIUM.iter()).copied()+}diff --git a/crates/backend-uzu/tests/common/matmul/variant.rs b/crates/backend-uzu/tests/common/matmul/variant.rsnew file mode 100644index 000000000..34ce795ea--- /dev/null+++ b/crates/backend-uzu/tests/common/matmul/variant.rs@@ -0,0 +1,33 @@+#[cfg(metal_backend)]+use backend_uzu::backends::metal::{DeviceExt, MatmulDispatchPath, MetalContext};+use derive_more::Display;++#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]+pub enum Variant {+ #[display("GEMM")]+ Gemm,+ #[display("GEMM_MXU")]+ GemmMxu,+}++impl Variant {+ pub const fn requires_mxu(self) -> bool {+ matches!(self, Variant::GemmMxu)+ }++ #[cfg(metal_backend)]+ pub fn supported(+ self,+ context: &MetalContext,+ ) -> bool {+ !self.requires_mxu() || context.device.supports_mxu()+ }++ #[cfg(metal_backend)]+ pub const fn dispatch_path(self) -> MatmulDispatchPath {+ match self {+ Variant::Gemm => MatmulDispatchPath::Gemm,+ Variant::GemmMxu => MatmulDispatchPath::GemmMxu,+ }+ }+}diff --git a/crates/backend-uzu/tests/common/mod.rs b/crates/backend-uzu/tests/common/mod.rsindex 1d1e67811..ada88165e 100644--- a/crates/backend-uzu/tests/common/mod.rs+++ b/crates/backend-uzu/tests/common/mod.rs@@ -3,6 +3,7 @@pub mod assert;pub mod audio;pub mod helpers;+pub mod matmul;pub mod path;pub mod perf;pub mod proptest;diff --git a/crates/backend-uzu/tests/unit/kernel/bench_entry.rs b/crates/backend-uzu/tests/performance/kernel/bench_entry.rssimilarity index 88%rename from crates/backend-uzu/tests/unit/kernel/bench_entry.rsrename to crates/backend-uzu/tests/performance/kernel/bench_entry.rsindex dff5cf777..d3fd71b67 100644--- a/crates/backend-uzu/tests/unit/kernel/bench_entry.rs+++ b/crates/backend-uzu/tests/performance/kernel/bench_entry.rs@@ -17,6 +17,6 @@ fn bench_runner(benches: &[&dyn Fn()]) {criterion::runner(benches);}-use proc_macros::{__internal_uzu_bench as uzu_bench, __internal_uzu_ignored as uzu_test};+use proc_macros::__internal_uzu_bench as uzu_bench;include!("mod.rs");diff --git a/crates/backend-uzu/tests/performance/kernel/mod.rs b/crates/backend-uzu/tests/performance/kernel/mod.rsnew file mode 100644index 000000000..0223d0039--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/mod.rs@@ -0,0 +1,7 @@+#[macro_use]+#[path = "../../common/mod.rs"]+mod common;++mod quant_matmul;+mod rms_norm_bench;+mod sampling;diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rsnew file mode 100644index 000000000..857ee0c92--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs@@ -0,0 +1,2 @@+mod qmm_transposed_bench;+mod qmv_fast_bench;diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rsnew file mode 100644index 000000000..f2ba0d5d8--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs@@ -0,0 +1,132 @@+use backend_uzu::{+ ArrayElement,+ backends::common::{+ Allocation, Backend, Context, Encoder, Kernels,+ gpu_types::QuantizationMethod,+ kernel::QuantizedMatmulQmmTransposedKernel,+ },+};+use criterion::{BenchmarkId, Criterion, Throughput};+use half::{bf16, f16};+use itertools::iproduct;+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::SmallRng};++use crate::{+ common::{+ helpers::{alloc_allocation, alloc_allocation_with_data},+ type_short_name,+ },+ uzu_bench,+};++fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(+ c: &mut Criterion,+ context: &B::Context,+ label: &str,+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+) {+ let block_size: usize = if bits == 4 { 512 } else { 256 };++ for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336]) {+ if n % 32 != 0 || k % block_size != 0 {+ continue;+ }++ let (bm, bk, bn, wm, wn) = if m < 48 {+ (8u32, 32u32, 32u32, 1u32, 1u32)+ } else {+ (32u32, 32u32, 32u32, 2u32, 2u32)+ };++ let num_groups = k.div_ceil(group_size as usize);+ let mut rng = SmallRng::seed_from_u64(42);++ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(+ context,+ T::data_type(),+ group_size,+ bits,+ bm,+ bk,+ bn,+ wm,+ wn,+ quant_method,+ false,+ n % bn as usize == 0,+ )+ .unwrap();++ let w_buf = alloc_allocation_with_data::<B, u32>(+ context,+ &(0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect::<Vec<_>>(),+ );+ let scales_buf = alloc_allocation_with_data::<B, T>(+ context,+ &(0..n * num_groups).map(|_| T::from(rng.random_range(0.01f32..1.0f32)).unwrap()).collect::<Vec<_>>(),+ );+ let x_buf = alloc_allocation_with_data::<B, T>(+ context,+ &(0..m * k).map(|_| T::from(rng.random_range(-1.0f32..1.0f32)).unwrap()).collect::<Vec<_>>(),+ );+ let mut y_buf = alloc_allocation::<B, T>(context, m * n);++ let zp_stride = if bits == 4 {+ (num_groups + 1) / 2+ } else {+ num_groups+ };+ let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {+ alloc_allocation_with_data::<B, u8>(+ context,+ &(0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect::<Vec<_>>(),+ )+ });+ let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {+ alloc_allocation_with_data::<B, T>(+ context,+ &(0..n * num_groups).map(|_| T::from(rng.random_range(-0.5f32..0.5f32)).unwrap()).collect::<Vec<_>>(),+ )+ });++ let mut group = c.benchmark_group(format!("{}/Kernel/QmmTransposed/{}", type_short_name::<B>(), label));+ group.throughput(Throughput::Elements((m * n * k) as u64));+ group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(context).unwrap();+ for _ in 0..n_iters {+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ None::<&Allocation<B>>,+ k as u32,+ n as u32,+ m as u32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()+ })+ });+ drop(group);+ }+}++#[uzu_bench]+fn bench_qmm_transposed(c: &mut Criterion) {+ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();+ bench_qmm_transposed_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);+ bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmm_transposed_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);+ bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmm_transposed_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ });+}diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rsnew file mode 100644index 000000000..16d352560--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs@@ -0,0 +1,141 @@+use backend_uzu::{+ ArrayElement,+ backends::common::{+ Allocation, Backend, Context, Encoder, Kernels,+ gpu_types::QuantizationMethod,+ kernel::QuantizedMatmulQmvFastKernel,+ },+};+use criterion::{BenchmarkId, Criterion, Throughput};+use half::{bf16, f16};+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::SmallRng};++use crate::{+ common::{+ helpers::{alloc_allocation, alloc_allocation_with_data},+ type_short_name,+ },+ uzu_bench,+};++fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(+ rng: &mut R,+ range: std::ops::Range<T>,+ len: usize,+) -> Box<[T]> {+ (0..len).map(|_| rng.random_range(range.clone())).collect()+}++fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(+ c: &mut Criterion,+ context: &B::Context,+ label: &str,+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+) {+ let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));+ let block_size: usize = if bits == 4 {+ 512+ } else {+ 256+ };++ let nk_pairs: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];+ for &(n, k) in nk_pairs {+ if n % 8 != 0 || k % block_size != 0 {+ continue;+ }+ for m in [1usize, 2, 4, 8] {+ let num_groups = k.div_ceil(group_size as usize);+ let mut rng = SmallRng::seed_from_u64(42);++ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ context,+ T::data_type(),+ group_size,+ bits,+ quant_method,+ false,+ )+ .unwrap();++ let w_buf = alloc_allocation_with_data::<B, u32>(+ context,+ &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),+ );+ let scales_buf = alloc_allocation_with_data::<B, T>(+ context,+ &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)+ .iter()+ .map(|&v| T::from(v).unwrap())+ .collect::<Vec<_>>(),+ );+ let x_buf = alloc_allocation_with_data::<B, T>(+ context,+ &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k)+ .iter()+ .map(|&v| T::from(v).unwrap())+ .collect::<Vec<_>>(),+ );+ let mut y_buf = alloc_allocation::<B, T>(context, m * n);++ let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {+ let zp_stride = if bits == 4 {+ (num_groups + 1) / 2+ } else {+ num_groups+ };+ alloc_allocation_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))+ });+ let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {+ alloc_allocation_with_data::<B, T>(+ context,+ &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)+ .iter()+ .map(|&v| T::from(v).unwrap())+ .collect::<Vec<_>>(),+ )+ });++ group.throughput(Throughput::Elements((m * n * k) as u64));+ group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(context).unwrap();+ for _ in 0..n_iters {+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ None::<&Allocation<B>>,+ k as u32,+ n as u32,+ m as u32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()+ })+ });+ }+ }+}++#[uzu_bench]+fn bench_qmv_fast(c: &mut Criterion) {+ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);+ });+}diff --git a/crates/backend-uzu/tests/performance/kernel/rms_norm_bench.rs b/crates/backend-uzu/tests/performance/kernel/rms_norm_bench.rsnew file mode 100644index 000000000..b1614b707--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/rms_norm_bench.rs@@ -0,0 +1,102 @@+use backend_uzu::{+ ArrayContextExt, ArrayElement,+ backends::common::{Allocation, Backend, Context, Encoder, Kernels, kernel::RMSNormKernel},+};+use criterion::{BenchmarkId, Criterion, Throughput};+use itertools::iproduct;+use rand::{RngExt, SeedableRng, rngs::SmallRng};++use crate::{common::type_short_name, uzu_bench};++fn get_rms_norm_data(+ seed: u64,+ batch_size: usize,+ model_dim: usize,+) -> (Box<[f32]>, Box<[f32]>) {+ let mut rng = SmallRng::seed_from_u64(seed);+ let input_size = batch_size * model_dim;+ let mut input_data = vec![0.0f32; input_size];+ for x in input_data.iter_mut() {+ *x = rng.random_range(-2.0f32..2.0f32);+ }+ let mut scale_data = vec![0.0f32; model_dim];+ for x in scale_data.iter_mut() {+ *x = rng.random_range(0.1f32..3.0f32);+ }+ (input_data.into_boxed_slice(), scale_data.into_boxed_slice())+}++#[uzu_bench]+fn bench_rms_norm(c: &mut Criterion) {+ type T = f32;+ let epsilon = 1e-6f32;+ let scale_offset = 0.0f32;++ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();++ let kernel = <<B as Backend>::Kernels as Kernels>::RMSNormKernel::new(+ &context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ T::data_type(),+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ )+ .unwrap();++ let mut group = c.benchmark_group(format!("{}/Kernel/RMSNorm", type_short_name::<B>()));++ for (batch_size, model_dim) in iproduct!(+ [1, 4, 32, 128, 1024], // Batch sizes+ [+ 1024, // LFM2-350M+ 1152, // Gemma-3-1b+ 2048, // Llama-3.2-1B+ 2560, // Qwen3-4B+ 3072, // Llama-3.2-3B+ 4096, // Qwen3-8B+ 5120, // Qwen2.5-32B+ ]+ ) {+ let (input_data, scale_data) = get_rms_norm_data(1337, batch_size, model_dim);+ let input_size = batch_size * model_dim;++ let input_buffer = context.create_array_from(&[input_size], input_data.as_ref()).into_allocation();+ let scales_buffer = context.create_array_from(&[model_dim], scale_data.as_ref()).into_allocation();+ let mut output_buffer = context.create_array_uninitialized(&[input_size], T::data_type()).into_allocation();++ group.throughput(Throughput::Elements((batch_size * model_dim) as u64));++ group.bench_function(BenchmarkId::from_parameter(format!("Batch[{batch_size}]Dim[{model_dim}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(&context).unwrap();++ for _ in 0..n_iters {+ kernel.encode(+ Some(&input_buffer),+ &scales_buffer,+ &mut output_buffer,+ None::<&mut Allocation<B>>,+ None::<&Allocation<B>>,+ batch_size as u32,+ model_dim as u32,+ epsilon,+ scale_offset,+ 1.0,+ &mut encoder,+ );+ }++ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()+ })+ });+ }+ });+}diff --git a/crates/backend-uzu/tests/performance/kernel/sampling/argmax_bench.rs b/crates/backend-uzu/tests/performance/kernel/sampling/argmax_bench.rsnew file mode 100644index 000000000..f6d9783fc--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/sampling/argmax_bench.rs@@ -0,0 +1,111 @@+use std::mem::{MaybeUninit, size_of};++use backend_uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::common::{+ AllocationType, Backend, Context, Encoder, Kernels,+ gpu_types::ArgmaxPair,+ kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel},+ },+};+use criterion::{BenchmarkId, Criterion, Throughput};+use num_traits::{Float, NumCast};+use rand::{RngExt, SeedableRng, rngs::SmallRng};+use rand_distr::Normal;++use crate::{common::type_short_name, uzu_bench};++fn get_argmax_data<T: ArrayElement + Float>(+ seed: u64,+ batch_size: usize,+ vocab_size: usize,+) -> Box<[T]> {+ let mut rng = SmallRng::seed_from_u64(seed);+ let distribution = Normal::new(0.0, 32.0).unwrap();+ let mut logits: Box<[MaybeUninit<T>]> = Box::new_uninit_slice(batch_size * vocab_size);+ for logit in logits.iter_mut() {+ logit.write(<T as NumCast>::from(rng.sample(distribution)).unwrap());+ }+ unsafe { logits.assume_init() }+}++#[uzu_bench]+fn bench_argmax(c: &mut Criterion) {+ type T = f32;+ let batch_size = 1;++ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();++ let single_kernel =+ <<B as Backend>::Kernels as Kernels>::ArgmaxSingleKernel::new(&context, T::data_type()).unwrap();+ let twopass_kernel_main =+ <<B as Backend>::Kernels as Kernels>::ArgmaxMainKernel::new(&context, T::data_type()).unwrap();+ let twopass_kernel_final = <<B as Backend>::Kernels as Kernels>::ArgmaxFinalKernel::new(&context).unwrap();++ let mut output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();++ let mut group = c.benchmark_group(format!("{}/Kernel/Sampling/Argmax", type_short_name::<B>()));++ for vocab_size in [+ 65536, // LFM2-350M+ 128256, // Llama-3.2-1B+ 151936, // Qwen3-1.7B+ 262144, // Gemma-3-1b+ ] {+ let logits_data = get_argmax_data::<T>(1337, batch_size, vocab_size);+ let logits_buffer = context.create_array_from(&[logits_data.len()], logits_data.as_ref()).into_allocation();+ let mut twopass_partial_results_buffer = context+ .create_allocation(+ batch_size * vocab_size.div_ceil(4096) * size_of::<ArgmaxPair>(),+ AllocationType::Global,+ )+ .unwrap();++ group.throughput(Throughput::Elements(vocab_size as u64));++ group.bench_function(BenchmarkId::new("SinglePass", format!("Vocab[{vocab_size}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(&context).unwrap();++ for _ in 0..n_iters {+ single_kernel.encode(+ &logits_buffer,+ &mut output_buffer,+ batch_size as u32,+ vocab_size as u32,+ &mut encoder,+ );+ }++ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()+ })+ });++ group.bench_function(BenchmarkId::new("TwoPass", format!("Vocab[{vocab_size}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(&context).unwrap();++ for _ in 0..n_iters {+ twopass_kernel_main.encode(+ &logits_buffer,+ &mut twopass_partial_results_buffer,+ batch_size as u32,+ vocab_size as u32,+ &mut encoder,+ );+ twopass_kernel_final.encode(+ &twopass_partial_results_buffer,+ &mut output_buffer,+ batch_size as u32,+ vocab_size as u32,+ &mut encoder,+ );+ }++ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()+ })+ });+ }+ });+}diff --git a/crates/backend-uzu/tests/performance/kernel/sampling/mod.rs b/crates/backend-uzu/tests/performance/kernel/sampling/mod.rsnew file mode 100644index 000000000..ed5b250a0--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/sampling/mod.rs@@ -0,0 +1 @@+mod argmax_bench;diff --git a/crates/backend-uzu/tests/performance/matmul/bench.rs b/crates/backend-uzu/tests/performance/matmul/bench.rsdeleted file mode 100644index 980660d38..000000000--- a/crates/backend-uzu/tests/performance/matmul/bench.rs+++ /dev/null@@ -1,148 +0,0 @@-#![cfg(metal_backend)]--use backend_uzu::{- DataType,- backends::{- common::{- Allocation, AllocationType, AsBufferRangeMut, Backend, Context, DenseBuffer, Encoder,- kernel::{- ManualKernels,- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},- },- },- metal::Metal,- },-};--use super::{error::BenchError, output::PerfResult, shapes::TestShape};--type Ctx = <Metal as Backend>::Context;--const WARMUP_ITERATIONS: usize = 3;-const BENCHMARK_ITERATIONS: usize = 10;--fn fill_buffer_random(- allocation: &mut Allocation<Metal>,- byte_count: usize,-) {- let buffer_range = allocation.as_buffer_range_mut();- let range = buffer_range.range();- let slice = unsafe {- std::slice::from_raw_parts_mut(- (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),- byte_count,- )- };- for (i, byte) in slice.iter_mut().enumerate() {- *byte = (i % 251) as u8;- }-}--fn encode_and_run(- context: &Ctx,- kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,- shape: &TestShape,- a_allocation: &backend_uzu::backends::common::Allocation<Metal>,- b_allocation: &backend_uzu::backends::common::Allocation<Metal>,- d_allocation: &mut backend_uzu::backends::common::Allocation<Metal>,-) -> Result<f64, BenchError> {- let mut encoder = Encoder::new(context).map_err(|_| BenchError::CommandBuffer)?;-- kernel.encode(- MatmulArguments {- a: a_allocation,- a_offset: 0,- b: b_allocation,- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: 1.0,- c: MatmulArgumentC::None,- d: d_allocation,- batch_dim: shape.batch as u32,- input_dim: shape.input_dim as u32,- output_dim: shape.output_dim as u32,- },- &mut encoder,- );-- let completed = encoder.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;-- Ok(completed.gpu_execution_time().as_secs_f64() * 1000.0)-}--fn run_benchmark(- context: &Ctx,- data_type: DataType,- shape: &TestShape,-) -> Result<f64, BenchError> {- let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)- .map_err(|e| BenchError::Kernel(e.to_string()))?;-- let elem_size = data_type.size_in_bytes();- let a_byte_count = shape.batch * shape.input_dim * elem_size;- let b_byte_count = shape.output_dim * shape.input_dim * elem_size;- let d_byte_count = shape.batch * shape.output_dim * elem_size;-- let mut a_allocation =- context.create_allocation(a_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;- let mut b_allocation =- context.create_allocation(b_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;- let mut d_allocation =- context.create_allocation(d_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;-- fill_buffer_random(&mut a_allocation, a_byte_count);- fill_buffer_random(&mut b_allocation, b_byte_count);-- for iteration in 0..WARMUP_ITERATIONS {- encode_and_run(context, &mut kernel, shape, &a_allocation, &b_allocation, &mut d_allocation).map_err(- |source| BenchError::Warmup {- iteration,- source: Box::new(source),- },- )?;- }-- let mut gpu_time_total_ms = 0.0;- for iteration in 0..BENCHMARK_ITERATIONS {- let gpu_ms = encode_and_run(context, &mut kernel, shape, &a_allocation, &b_allocation, &mut d_allocation)- .map_err(|source| BenchError::Benchmark {- iteration,- source: Box::new(source),- })?;- gpu_time_total_ms += gpu_ms;- }-- Ok(gpu_time_total_ms / BENCHMARK_ITERATIONS as f64)-}--pub fn benchmark_single(- context: &Ctx,- data_type: DataType,- shape: &TestShape,-) -> PerfResult {- match run_benchmark(context, data_type, shape) {- Ok(duration_ms) => {- let flops = 2.0 * shape.batch as f64 * shape.input_dim as f64 * shape.output_dim as f64;- let gflops = flops / (duration_ms / 1000.0) / 1e9;- PerfResult {- combo: format!("{data_type:?}"),- shape: format!("{shape}"),- dispatch_path: "auto".to_owned(),- duration_ms,- gflops,- status: "ok".into(),- error: None,- }- },- Err(error) => PerfResult {- combo: format!("{data_type:?}"),- shape: format!("{shape}"),- dispatch_path: "auto".to_owned(),- duration_ms: 0.0,- gflops: 0.0,- status: "error".into(),- error: Some(error.to_string()),- },- }-}diff --git a/crates/backend-uzu/tests/performance/matmul/error.rs b/crates/backend-uzu/tests/performance/matmul/error.rsdeleted file mode 100644index b06aa8fd7..000000000--- a/crates/backend-uzu/tests/performance/matmul/error.rs+++ /dev/null@@ -1,21 +0,0 @@-use thiserror::Error;--#[derive(Debug, Error)]-pub enum BenchError {- #[error("failed to create command buffer")]- CommandBuffer,- #[error("kernel creation failed: {0}")]- Kernel(String),- #[error("buffer allocation failed")]- BufferAllocation,- #[error("warmup iteration {iteration}: {source}")]- Warmup {- iteration: usize,- source: Box<BenchError>,- },- #[error("benchmark iteration {iteration}: {source}")]- Benchmark {- iteration: usize,- source: Box<BenchError>,- },-}diff --git a/crates/backend-uzu/tests/performance/matmul/main.rs b/crates/backend-uzu/tests/performance/matmul/main.rsdeleted file mode 100644index 22db6b189..000000000--- a/crates/backend-uzu/tests/performance/matmul/main.rs+++ /dev/null@@ -1,67 +0,0 @@-#![cfg(metal_backend)]--use backend_uzu::{- DataType,- backends::{- common::{Backend, Context},- metal::Metal,- },-};-use indicatif::{ProgressBar, ProgressStyle};-use metal::MTLDeviceExt;--use crate::matmul::{bench, output::print_results_table, shapes::test_shapes};--type Ctx = <Metal as Backend>::Context;--fn write_json_results<T: serde::Serialize>(- test_name: &str,- device: &str,- results: &[T],-) {- if let Ok(dir) = std::env::var("UZU_TEST_RESULTS_DIR") {- let path = std::path::Path::new(&dir);- std::fs::create_dir_all(path).expect("create results dir");- let file = path.join(format!("{test_name}.json"));- let wrapper = serde_json::json!({ "device": device, "results": results });- let json = serde_json::to_string_pretty(&wrapper).expect("serialize");- std::fs::write(&file, json).expect("write results");- eprintln!("Results written to {}", file.display());- }-}--#[test]-#[ignore]-fn matmul_perf() {- let context = Ctx::new().expect("Metal context required");-- let data_types = [DataType::BF16, DataType::F16];- let test_shapes = test_shapes();-- eprintln!("Matmul perf: {} dtypes x {} shapes", data_types.len(), test_shapes.len(),);-- let progress_bar = ProgressBar::new_spinner();- progress_bar.set_style(- ProgressStyle::with_template("{spinner} {pos} benchmarks [{elapsed_precise}] {msg}").expect("progress style"),- );-- let mut results = Vec::new();-- for &data_type in &data_types {- for shape in &test_shapes {- progress_bar.set_message(format!("{data_type:?} {shape}"));- let result = bench::benchmark_single(&context, data_type, shape);- results.push(result);- progress_bar.inc(1);- }- }-- progress_bar.finish_with_message("done");- print_results_table(&results);- write_json_results("matmul_perf", &context.device.name(), &results);-- let error_results: Vec<_> = results.iter().filter(|r| r.status != "ok").collect();- if !error_results.is_empty() {- eprintln!("{} / {} cases had errors", error_results.len(), results.len());- }-}diff --git a/crates/backend-uzu/tests/performance/matmul/mod.rs b/crates/backend-uzu/tests/performance/matmul/mod.rsdeleted file mode 100644index f79669fac..000000000--- a/crates/backend-uzu/tests/performance/matmul/mod.rs+++ /dev/null@@ -1,6 +0,0 @@-mod main;--mod bench;-mod error;-mod output;-mod shapes;diff --git a/crates/backend-uzu/tests/performance/matmul/output.rs b/crates/backend-uzu/tests/performance/matmul/output.rsdeleted file mode 100644index cf2611518..000000000--- a/crates/backend-uzu/tests/performance/matmul/output.rs+++ /dev/null@@ -1,40 +0,0 @@-use comfy_table::{CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL};-use serde::Serialize;--#[derive(Serialize)]-pub struct PerfResult {- pub combo: String,- pub shape: String,- pub dispatch_path: String,- pub duration_ms: f64,- pub gflops: f64,- pub status: String,- #[serde(skip_serializing_if = "Option::is_none")]- pub error: Option<String>,-}--pub fn print_results_table(results: &[PerfResult]) {- let mut table = Table::new();- table- .load_preset(UTF8_FULL)- .apply_modifier(UTF8_ROUND_CORNERS)- .set_content_arrangement(ContentArrangement::Dynamic)- .set_header(vec!["Dtype", "Shape (MxKxN)", "Dispatch", "GFLOPS", "ms/iter", "Status"]);-- for r in results {- let (gflops_str, ms_str, status_str) = if r.status == "ok" {- (format!("{:.1}", r.gflops), format!("{:.3}", r.duration_ms), "ok".into())- } else {- ("-".into(), "-".into(), format!("ERR: {}", r.error.as_deref().unwrap_or("?")))- };- table.add_row(vec![r.combo.clone(), r.shape.clone(), r.dispatch_path.clone(), gflops_str, ms_str, status_str]);- }-- for col in [3, 4] {- if let Some(column) = table.column_mut(col) {- column.set_cell_alignment(CellAlignment::Right);- }- }-- println!("{table}");-}diff --git a/crates/backend-uzu/tests/performance/matmul/shapes.rs b/crates/backend-uzu/tests/performance/matmul/shapes.rsdeleted file mode 100644index d26cfebea..000000000--- a/crates/backend-uzu/tests/performance/matmul/shapes.rs+++ /dev/null@@ -1,41 +0,0 @@-pub struct TestShape {- pub batch: usize,- pub input_dim: usize,- pub output_dim: usize,-}--impl std::fmt::Display for TestShape {- fn fmt(- &self,- f: &mut std::fmt::Formatter<'_>,- ) -> std::fmt::Result {- write!(f, "{}x{}x{}", self.batch, self.input_dim, self.output_dim)- }-}--const BATCH_SIZES: &[usize] = &[1, 4, 16, 64, 128, 256, 512];--const MODEL_DIMS: &[(usize, usize)] = &[- (896, 896),- (896, 4864),- (4864, 896),- (2048, 2048),- (2048, 8192),- (8192, 2048),- (4096, 4096),- (4096, 14336),- (14336, 4096),-];--pub fn test_shapes() -> Vec<TestShape> {- MODEL_DIMS- .iter()- .flat_map(|&(input_dim, output_dim)| {- BATCH_SIZES.iter().map(move |&batch| TestShape {- batch,- input_dim,- output_dim,- })- })- .collect()-}diff --git a/crates/backend-uzu/tests/performance/mod.rs b/crates/backend-uzu/tests/performance/mod.rsindex e41fdb139..a980f17d8 100644--- a/crates/backend-uzu/tests/performance/mod.rs+++ b/crates/backend-uzu/tests/performance/mod.rs@@ -4,6 +4,4 @@ mod common;mod audio_kernel_perf_test;mod encodable_block;-#[cfg(metal_backend)]-mod matmul;mod model_loading_perf_test;diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rsindex f8ad17b9c..d20d3ae98 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs@@ -346,8 +346,8 @@ fn execute_quantized_matmul<B: Backend>(input_dim,output_dim,mode: match bits {- 4 => QuantizationMode::UINT4,- 8 => QuantizationMode::INT8,+ 4 => QuantizationMode::U4,+ 8 => QuantizationMode::I8,_ => panic!("Unsupported bits: {}", bits),},quantization_method,@@ -609,14 +609,14 @@ fn test_quant_matmul_perf_internal<B: Backend>() {];let configs = vec![- // 4-bit Mlx BF16+ // 4-bit scale-bias BF16TestConfig {quant_method: QuantizationMethod::ScaleBias,bits: 4,data_type: DataType::BF16,group_size: 128,},- // 8-bit Mlx BF16+ // 8-bit scale-bias BF16TestConfig {quant_method: QuantizationMethod::ScaleBias,bits: 8,diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rsindex 56ccc1434..38fd72538 100644--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs@@ -97,7 +97,7 @@ fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {model_dim,input_scale,group_size,- quant_mode: QuantizationMode::UINT8,+ quant_mode: QuantizationMode::U8,};let expected = get_output::<T, Cpu>(&input);@@ -111,7 +111,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {&context,T::data_type(),input.group_size,- input.quant_mode.to_u32(),+ input.quant_mode,).expect("Failed to create QuantizedEmbeddingLookupKernel");@@ -184,49 +184,49 @@ fn test_oob<T: ArrayElement + Float + Debug + Display>() {// UINT4 tests#[uzu_test]fn test_uint4_f32() {- test_quant_mode::<f32>(QuantizationMode::UINT4);+ test_quant_mode::<f32>(QuantizationMode::U4);}#[uzu_test]fn test_uint4_f16() {- test_quant_mode::<f16>(QuantizationMode::UINT4);+ test_quant_mode::<f16>(QuantizationMode::U4);}#[uzu_test]fn test_uint4_bf16() {- test_quant_mode::<bf16>(QuantizationMode::UINT4);+ test_quant_mode::<bf16>(QuantizationMode::U4);}// INT8 tests#[uzu_test]fn test_int8_f32() {- test_quant_mode::<f32>(QuantizationMode::INT8);+ test_quant_mode::<f32>(QuantizationMode::I8);}#[uzu_test]fn test_int8_f16() {- test_quant_mode::<f16>(QuantizationMode::INT8);+ test_quant_mode::<f16>(QuantizationMode::I8);}#[uzu_test]fn test_int8_bf16() {- test_quant_mode::<bf16>(QuantizationMode::INT8);+ test_quant_mode::<bf16>(QuantizationMode::I8);}// UINT8 tests#[uzu_test]fn test_uint8_f32() {- test_quant_mode::<f32>(QuantizationMode::UINT8);+ test_quant_mode::<f32>(QuantizationMode::U8);}#[uzu_test]fn test_uint8_f16() {- test_quant_mode::<f16>(QuantizationMode::UINT8);+ test_quant_mode::<f16>(QuantizationMode::U8);}#[uzu_test]fn test_uint8_bf16() {- test_quant_mode::<bf16>(QuantizationMode::UINT8);+ test_quant_mode::<bf16>(QuantizationMode::U8);}// OOB testsdiff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rsnew file mode 100644index 000000000..51a0e9085--- /dev/null+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs@@ -0,0 +1,131 @@+#![cfg(metal_backend)]++use std::fmt::{Debug, Display};++use backend_uzu::{+ ArrayElement,+ backends::{+ common::{Backend, Context, kernel::ManualKernels, kernel::matmul::MatmulKernel},+ metal::{Metal, MetalContext},+ },+};+use proc_macros::__internal_uzu_test as uzu_test;+use half::{bf16, f16};+use num_traits::Float;++use crate::common::{+ assert::assert_eq_float,+ matmul::{Case, Variant, all_correctness_shapes, cpu_reference, deterministic_input, run_metal},+};++fn check_case<T: ArrayElement + Float + Debug + Display>(+ context: &MetalContext,+ kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,+ variant: Variant,+ case: Case,+ tolerance: f32,+) {+ let input = deterministic_input::<T>(case);+ let expected = cpu_reference::<T>(&input);+ let actual = run_metal::<T>(context, kernel, &input, variant);+ assert_eq_float(+ &expected,+ &actual,+ tolerance,+ &format!("{:?} dtype={} {:?}", variant, std::any::type_name::<T>(), case),+ );+}++fn check_all_shapes<T: ArrayElement + Float + Debug + Display>(+ variant: Variant,+ ab_scale: f32,+ accumulate: bool,+ tolerance: f32,+) {+ let context = MetalContext::new().expect("Metal context");+ if !variant.supported(&context) {+ eprintln!("Skipping {variant:?}: device does not support MXU");+ return;+ }+ let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())+ .expect("MatmulKernel");+ for shape in all_correctness_shapes() {+ let case = Case::new(shape).with_ab_scale(ab_scale).with_accumulate(accumulate);+ check_case::<T>(&context, &mut kernel, variant, case, tolerance);+ }+}++#[uzu_test]+fn matches_cpu_reference_bf16_gemm() {+ check_all_shapes::<bf16>(Variant::Gemm, 1.0, false, 1.0);+}++#[uzu_test]+fn matches_cpu_reference_bf16_gemm_mxu() {+ check_all_shapes::<bf16>(Variant::GemmMxu, 1.0, false, 1.0);+}++#[uzu_test]+fn matches_cpu_reference_f16_gemm() {+ check_all_shapes::<f16>(Variant::Gemm, 1.0, false, 0.5);+}++#[uzu_test]+fn matches_cpu_reference_f16_gemm_mxu() {+ check_all_shapes::<f16>(Variant::GemmMxu, 1.0, false, 0.5);+}++#[uzu_test]+fn matches_cpu_reference_f32_gemm() {+ check_all_shapes::<f32>(Variant::Gemm, 1.0, false, 0.05);+}++#[uzu_test]+fn ab_scale_bf16_gemm() {+ check_all_shapes::<bf16>(Variant::Gemm, 0.5, false, 1.0);+}++#[uzu_test]+fn ab_scale_bf16_gemm_mxu() {+ check_all_shapes::<bf16>(Variant::GemmMxu, 0.5, false, 1.0);+}++#[uzu_test]+fn accumulate_bf16_gemm_mxu() {+ check_all_shapes::<bf16>(Variant::GemmMxu, 1.0, true, 1.0);+}++#[uzu_test]+fn scale_and_accumulate_bf16_gemm_mxu() {+ check_all_shapes::<bf16>(Variant::GemmMxu, 0.5, true, 1.0);+}++fn check_b_transpose_false<T: ArrayElement + Float + Debug + Display>(+ variant: Variant,+ tolerance: f32,+) {+ let context = MetalContext::new().expect("Metal context");+ if !variant.supported(&context) {+ eprintln!("Skipping {variant:?}: device does not support MXU");+ return;+ }+ let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())+ .expect("MatmulKernel");+ for shape in all_correctness_shapes() {+ let case = Case {+ b_transpose: false,+ ..Case::new(shape)+ };+ check_case::<T>(&context, &mut kernel, variant, case, tolerance);+ }+}++#[uzu_test]+fn b_transpose_false_bf16_gemm() {+ check_b_transpose_false::<bf16>(Variant::Gemm, 1.0);+}++#[uzu_test]+fn b_transpose_false_bf16_gemm_mxu() {+ check_b_transpose_false::<bf16>(Variant::GemmMxu, 1.0);+}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rsdeleted file mode 100644index 00204a6bd..000000000--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs+++ /dev/null@@ -1,336 +0,0 @@-#![cfg(metal_backend)]--use std::{- rc::Rc,-};--use criterion::{BenchmarkId, Criterion, Throughput};-use half::bf16;-use backend_uzu::{- ArrayContextExt, ArrayElement,- backends::{- common::{- AllocationType, Backend, Context, Encoder,- kernel::{- ManualKernels,- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},- },- },- cpu::Cpu,- metal::{DeviceExt, MatmulDispatchPath, Metal, MetalContext},- },-};--use crate::{- common::{- assert::assert_eq_float,- helpers::{alloc_allocation_with_data, allocation_to_vec},- type_short_name,- },- uzu_bench, uzu_test,-};--struct Input {- left: Box<[bf16]>,- right: Box<[bf16]>,- destination_prefill: Option<Box<[bf16]>>,- batch_dim: usize,- input_dim: usize,- output_dim: usize,- ab_scale: f32,-}--fn get_test_data(- batch_dim: usize,- input_dim: usize,- output_dim: usize,- ab_scale: f32,- accumulate: bool,-) -> Input {- let left: Vec<bf16> = (0..batch_dim * input_dim).map(|i| bf16::from_f32(((i % 13) as f32) * 0.1 - 0.6)).collect();- let right: Vec<bf16> = (0..output_dim * input_dim).map(|i| bf16::from_f32(((i % 17) as f32) * 0.1 - 0.8)).collect();- let destination_prefill = if accumulate {- Some((0..batch_dim * output_dim).map(|i| bf16::from_f32(((i % 7) as f32) * 0.03 - 0.09)).collect())- } else {- None- };-- Input {- left: left.into_boxed_slice(),- right: right.into_boxed_slice(),- destination_prefill,- batch_dim,- input_dim,- output_dim,- ab_scale,- }-}--fn get_cpu_output(input: &Input) -> Vec<bf16> {- let cpu_context = <Cpu as Backend>::Context::new().expect("CPU context");-- let right_array = cpu_context.create_array_from(&[input.output_dim, input.input_dim], &input.right);- let left_allocation = alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &input.left);- let mut destination_allocation = if let Some(ref prefill) = input.destination_prefill {- alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, prefill)- } else {- cpu_context- .create_allocation(- input.batch_dim * input.output_dim * std::mem::size_of::<bf16>(),- AllocationType::Global,- )- .expect("Failed to create allocation")- };-- let argument_c = if input.destination_prefill.is_some() {- MatmulArgumentC::Accumulate- } else {- MatmulArgumentC::None- };-- let mut matmul_kernel =- <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&cpu_context, bf16::data_type())- .expect("CPU MatmulKernel");-- let mut encoder = Encoder::new(cpu_context.as_ref()).expect("encoder");- matmul_kernel.encode(- MatmulArguments {- a: &left_allocation,- a_offset: 0,- b: right_array.allocation(),- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: input.ab_scale,- c: argument_c,- d: &mut destination_allocation,- batch_dim: input.batch_dim as u32,- input_dim: input.input_dim as u32,- output_dim: input.output_dim as u32,- },- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();- allocation_to_vec(&destination_allocation)-}--fn get_mpp_output(- metal_context: &MetalContext,- matmul_kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,- input: &Input,-) -> Vec<bf16> {- let right_array = metal_context.create_array_from(&[input.output_dim, input.input_dim], &input.right);- let left_allocation = alloc_allocation_with_data::<Metal, bf16>(metal_context, &input.left);- let mut destination_allocation = if let Some(ref prefill) = input.destination_prefill {- alloc_allocation_with_data::<Metal, bf16>(metal_context, prefill)- } else {- metal_context- .create_allocation(- input.batch_dim * input.output_dim * std::mem::size_of::<bf16>(),- AllocationType::Global,- )- .expect("Failed to create allocation")- };-- let argument_c = if input.destination_prefill.is_some() {- MatmulArgumentC::Accumulate- } else {- MatmulArgumentC::None- };-- let mut encoder = Encoder::new(metal_context).expect("encoder");- matmul_kernel.encode_with_path(- MatmulArguments {- a: &left_allocation,- a_offset: 0,- b: right_array.allocation(),- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: input.ab_scale,- c: argument_c,- d: &mut destination_allocation,- batch_dim: input.batch_dim as u32,- input_dim: input.input_dim as u32,- output_dim: input.output_dim as u32,- },- &mut encoder,- MatmulDispatchPath::GemmMpp,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();- allocation_to_vec(&destination_allocation)-}--fn create_metal_context_and_matmul_kernel()--> (Rc<MetalContext>, <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel) {- let metal_context = MetalContext::new().expect("Metal context");- let matmul_kernel =- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())- .expect("MatmulKernel");- (metal_context, matmul_kernel)-}--fn test_mpp_correctness(- batch_dim: usize,- input_dim: usize,- output_dim: usize,- ab_scale: f32,- accumulate: bool,- tolerance: f32,-) {- let (metal_context, mut matmul_kernel) = create_metal_context_and_matmul_kernel();- if !metal_context.device.supports_mxu() {- eprintln!("Skipping MPP test: device does not support MXU");- return;- }-- let input = get_test_data(batch_dim, input_dim, output_dim, ab_scale, accumulate);- let expected = get_cpu_output(&input);- let actual = get_mpp_output(&metal_context, &mut matmul_kernel, &input);- assert_eq_float(&expected, &actual, tolerance, "GEMM_MPP vs CPU reference");-}--// Basic correctness — explicitly dispatched to MPP kernel--#[uzu_test]-fn test_bf16_aligned() {- test_mpp_correctness(64, 2048, 2048, 1.0, false, 1.0);-}--#[uzu_test]-fn test_bf16_unaligned_m() {- test_mpp_correctness(33, 2048, 2048, 1.0, false, 1.0);-}--#[uzu_test]-fn test_bf16_unaligned_n() {- test_mpp_correctness(64, 2048, 33, 1.0, false, 1.0);-}--#[uzu_test]-fn test_bf16_large() {- test_mpp_correctness(128, 4096, 14336, 1.0, false, 2.0);-}--#[uzu_test]-fn test_bf16_small_batch() {- test_mpp_correctness(1, 2048, 2048, 1.0, false, 1.0);-}--// ab_scale--#[uzu_test]-fn test_bf16_ab_scale() {- test_mpp_correctness(64, 2048, 2048, 0.5, false, 1.0);-}--// accumulate--#[uzu_test]-fn test_bf16_accumulate() {- test_mpp_correctness(64, 2048, 2048, 1.0, true, 1.0);-}--// scale + accumulate combined--#[uzu_test]-fn test_bf16_scale_and_accumulate() {- test_mpp_correctness(64, 2048, 2048, 0.5, true, 1.0);-}--// 128x128 tile (M >= 128, N >= 128)--#[uzu_test]-fn test_bf16_128x128_aligned() {- test_mpp_correctness(128, 2048, 2048, 1.0, false, 1.0);-}--#[uzu_test]-fn test_bf16_128x128_unaligned_m() {- test_mpp_correctness(200, 2048, 2048, 1.0, false, 1.0);-}--#[uzu_test]-fn test_bf16_128x128_unaligned_n() {- test_mpp_correctness(128, 2048, 200, 1.0, false, 1.0);-}--#[uzu_test]-fn test_bf16_128x128_large() {- test_mpp_correctness(256, 4096, 4096, 1.0, false, 2.0);-}--#[uzu_test]-fn test_bf16_128x128_accumulate() {- test_mpp_correctness(128, 2048, 2048, 1.0, true, 1.0);-}--#[uzu_test]-fn test_bf16_128x128_scale_and_accumulate() {- test_mpp_correctness(256, 2048, 2048, 0.5, true, 1.0);-}--// Benchmarks--const BENCHMARK_SHAPES: &[(usize, usize, usize)] =- &[(128, 2048, 8192), (128, 4096, 14336), (256, 4096, 4096), (512, 8192, 2048)];--#[uzu_bench]-fn bench_gemm_mpp(criterion: &mut Criterion) {- let metal_context = MetalContext::new().unwrap();- if !metal_context.device.supports_mxu() {- return;- }-- let mut matmul_kernel =- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())- .expect("MatmulKernel");-- let mut benchmark_group =- criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()));-- for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {- let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type());- let left_allocation = metal_context- .create_allocation(batch_dim * input_dim * std::mem::size_of::<bf16>(), AllocationType::Global)- .expect("Failed to create allocation");- let mut destination_allocation = metal_context- .create_allocation(batch_dim * output_dim * std::mem::size_of::<bf16>(), AllocationType::Global)- .expect("Failed to create allocation");-- let floating_point_operations = 2 * batch_dim * input_dim * output_dim;- benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));-- benchmark_group.bench_function(- BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),- |bencher| {- bencher.iter_custom(|iteration_count| {- let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();-- for _ in 0..iteration_count {- matmul_kernel.encode_with_path(- MatmulArguments {- a: &left_allocation,- a_offset: 0,- b: right_array.allocation(),- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: 1.0,- c: MatmulArgumentC::None,- d: &mut destination_allocation,- batch_dim: batch_dim as u32,- input_dim: input_dim as u32,- output_dim: output_dim as u32,- },- &mut encoder,- MatmulDispatchPath::GemmMpp,- );- }-- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()- })- },- );- }-}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rsdeleted file mode 100644index 3bd5359b6..000000000--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs+++ /dev/null@@ -1,248 +0,0 @@-use std::{- fmt::{Debug, Display},-};--use backend_uzu::{- ArrayContextExt, ArrayElement,- backends::{- common::{- AllocationType, Backend, Context, Encoder,- kernel::{- ManualKernels,- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},- },- },- cpu::Cpu,- },-};-#[cfg(metal_backend)]-use backend_uzu::backends::metal::{MatmulDispatchPath, Metal, MetalContext};-#[cfg(metal_backend)]-use criterion::{BenchmarkId, Criterion, Throughput};-use half::{bf16, f16};-use num_traits::Float;--#[cfg(metal_backend)]-use crate::{common::type_short_name, uzu_bench};-use crate::{- common::{- assert::assert_eq_float,- helpers::{alloc_allocation_with_data, allocation_to_vec},- },- uzu_test,-};--struct Input<T: ArrayElement + Float> {- a: Box<[T]>,- b: Box<[T]>,- m: usize,- k: usize,- n: usize,-}--fn get_test_data<T: ArrayElement + Float>(- m: usize,- k: usize,- n: usize,-) -> (Input<T>, Vec<T>) {- let a: Vec<T> = (0..m * k).map(|i| T::from(((i % 13) as f32) * 0.1 - 0.6).unwrap()).collect();- let b: Vec<T> = (0..n * k).map(|i| T::from(((i % 17) as f32) * 0.1 - 0.8).unwrap()).collect();-- let input = Input {- a: a.into_boxed_slice(),- b: b.into_boxed_slice(),- m,- k,- n,- };-- let expected = get_output::<T, Cpu>(&input, 1.0);- (input, expected)-}--fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>, ab_scale: f32) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");-- let m = input.m as u32;- let k = input.k as u32;- let n = input.n as u32;-- let b_array = context.create_array_from(&[input.n, input.k], &input.b);- let a_allocation = alloc_allocation_with_data::<B, T>(&context, &input.a);- let mut d_allocation = context- .create_allocation(input.m * input.n * std::mem::size_of::<T>(), AllocationType::Global)- .expect("Failed to create allocation");-- let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())- .expect("Failed to create MatmulKernel");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- MatmulArguments {- a: &a_allocation,- a_offset: 0,- b: b_array.allocation(),- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale,- c: MatmulArgumentC::None,- d: &mut d_allocation,- batch_dim: m,- input_dim: k,- output_dim: n,- },- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();- allocation_to_vec::<B, T>(&d_allocation)-}--fn test<T: ArrayElement + Float + Debug + Display>(- m: usize,- k: usize,- n: usize,- eps: f32,-) {- let (input, expected) = get_test_data::<T>(m, k, n);- for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input, 1.0);- assert_eq_float(&expected, &output, eps, &format!("backend {}", std::any::type_name::<B>()));- });-}--fn test_with_scale<T: ArrayElement + Float + Debug + Display>(- m: usize,- k: usize,- n: usize,- ab_scale: f32,- eps: f32,-) {- let (input, _) = get_test_data::<T>(m, k, n);- let expected = get_output::<T, Cpu>(&input, ab_scale);- for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input, ab_scale);- assert_eq_float(&expected, &output, eps, &format!("backend {} ab_scale={ab_scale}", std::any::type_name::<B>()));- });-}--#[uzu_test]-fn test_f32_aligned() {- test::<f32>(64, 64, 64, 0.01);-}--#[uzu_test]-fn test_f16_aligned() {- test::<f16>(64, 64, 64, 0.01);-}--#[uzu_test]-fn test_bf16_aligned() {- test::<bf16>(64, 64, 64, 0.1);-}--#[uzu_test]-fn test_f32_unaligned() {- test::<f32>(7, 33, 11, 0.01);-}--#[uzu_test]-fn test_f16_unaligned() {- test::<f16>(7, 33, 11, 0.01);-}--#[uzu_test]-fn test_bf16_unaligned() {- test::<bf16>(7, 33, 11, 0.1);-}--#[uzu_test]-fn test_f32_large() {- test::<f32>(16, 128, 256, 0.01);-}--#[uzu_test]-fn test_f16_large() {- test::<f16>(16, 128, 256, 0.01);-}--#[uzu_test]-fn test_bf16_large() {- test::<bf16>(16, 128, 256, 0.1);-}--// ab_scale tests--#[uzu_test]-fn test_f32_ab_scale() {- test_with_scale::<f32>(16, 128, 256, 0.5, 0.01);-}--#[uzu_test]-fn test_bf16_ab_scale() {- test_with_scale::<bf16>(16, 128, 256, 0.5, 0.1);-}--// Benchmarks (classic ALU GEMM path, Metal only)--#[cfg(metal_backend)]-const BENCHMARK_SHAPES: &[(usize, usize, usize)] =- &[(128, 2048, 8192), (128, 4096, 14336), (256, 4096, 4096), (512, 8192, 2048)];--#[cfg(metal_backend)]-#[uzu_bench]-fn bench_gemm(criterion: &mut Criterion) {- let metal_context = MetalContext::new().unwrap();-- let mut matmul_kernel =- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())- .expect("MatmulKernel");-- let mut benchmark_group =- criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));-- for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {- let left_allocation = metal_context- .create_allocation(batch_dim * input_dim * std::mem::size_of::<bf16>(), AllocationType::Global)- .expect("Failed to create allocation");- let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type());- let mut destination_allocation = metal_context- .create_allocation(batch_dim * output_dim * std::mem::size_of::<bf16>(), AllocationType::Global)- .expect("Failed to create allocation");-- let floating_point_operations = 2 * batch_dim * input_dim * output_dim;- benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));-- benchmark_group.bench_function(- BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),- |bencher| {- bencher.iter_custom(|iteration_count| {- let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();-- for _ in 0..iteration_count {- matmul_kernel.encode_with_path(- MatmulArguments {- a: &left_allocation,- a_offset: 0,- b: right_array.allocation(),- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: 1.0,- c: MatmulArgumentC::None,- d: &mut destination_allocation,- batch_dim: batch_dim as u32,- input_dim: input_dim as u32,- output_dim: output_dim as u32,- },- &mut encoder,- MatmulDispatchPath::Gemm,- );- }-- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()- })- },- );- }-}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs b/crates/backend-uzu/tests/unit/kernel/matmul/mod.rsindex e7089955d..c84f3ea46 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs@@ -1,3 +1,3 @@-mod gemm_mpp_test;-mod gemm_test;+#[cfg(metal_backend)]+mod dispatch_paths_test;mod gemv_test;diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rsindex c11f4257e..e42308dc0 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs@@ -4,26 +4,20 @@ use backend_uzu::{ArrayElement, DataType,backends::{common::{- Allocation, Backend, Context, Encoder, Kernels,+ Backend, Context, Encoder, Kernels,gpu_types::QuantizationMethod,kernel::QuantizedMatmulQmmTransposedKernel,},cpu::Cpu,},};-use criterion::{BenchmarkId, Criterion, Throughput};use half::{bf16, f16};-use itertools::iproduct;use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::SmallRng};use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};use crate::{- common::{- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},- type_short_name,- },- uzu_bench, uzu_test,+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ uzu_test,};fn make_input_basic<T: ArrayElement + Float>(@@ -645,116 +639,3 @@ mod hadamard {);}-fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(- c: &mut Criterion,- context: &B::Context,- label: &str,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- // Use the dispatcher-selected tile configs: BM=8 small tile for M<48, big tile for M>=48.- // We always use the BM=8 small tile kernel for M in 4..=47 to match the dispatch threshold.- let block_size: usize = if bits == 4 { 512 } else { 256 };-- for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336]) {- if n % 32 != 0 || k % block_size != 0 {- continue;- }-- // Pick tile based on M: small (BM=8) for M<48, big (BM=32) for M>=48- let (bm, bk, bn, wm, wn) = if m < 48 {- (8u32, 32u32, 32u32, 1u32, 1u32)- } else {- (32u32, 32u32, 32u32, 2u32, 2u32)- };-- let num_groups = k.div_ceil(group_size as usize);- let mut rng = SmallRng::seed_from_u64(42);-- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- context,- T::data_type(),- group_size,- bits,- bm,- bk,- bn,- wm,- wn,- quant_method,- false,- n % bn as usize == 0,- )- .unwrap();-- let w_buf = alloc_allocation_with_data::<B, u32>(- context,- &(0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect::<Vec<_>>(),- );- let scales_buf = alloc_allocation_with_data::<B, T>(- context,- &(0..n * num_groups).map(|_| T::from(rng.random_range(0.01f32..1.0f32)).unwrap()).collect::<Vec<_>>(),- );- let x_buf = alloc_allocation_with_data::<B, T>(- context,- &(0..m * k).map(|_| T::from(rng.random_range(-1.0f32..1.0f32)).unwrap()).collect::<Vec<_>>(),- );- let mut y_buf = alloc_allocation::<B, T>(context, m * n);-- let zp_stride = if bits == 4 {- (num_groups + 1) / 2- } else {- num_groups- };- let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {- alloc_allocation_with_data::<B, u8>(- context,- &(0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect::<Vec<_>>(),- )- });- let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {- alloc_allocation_with_data::<B, T>(- context,- &(0..n * num_groups).map(|_| T::from(rng.random_range(-0.5f32..0.5f32)).unwrap()).collect::<Vec<_>>(),- )- });-- let mut group = c.benchmark_group(format!("{}/Kernel/QmmTransposed/{}", type_short_name::<B>(), label));- group.throughput(Throughput::Elements((m * n * k) as u64));- group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {- b.iter_custom(|n_iters| {- let mut encoder = Encoder::<B>::new(context).unwrap();- for _ in 0..n_iters {- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- None::<&Allocation<B>>,- k as u32,- n as u32,- m as u32,- &mut encoder,- );- }- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()- })- });- drop(group);- }-}--#[uzu_bench]-fn bench_qmm_transposed(c: &mut Criterion) {- for_each_backend!(|B| {- let context = <B as Backend>::Context::new().unwrap();- bench_qmm_transposed_typed::<B, bf16>(c, &context, "Mlx_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmm_transposed_typed::<B, bf16>(c, &context, "Mlx_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmm_transposed_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- });-}diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsindex e3015950b..24e497835 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -11,18 +11,12 @@ use backend_uzu::{cpu::Cpu,},};-use criterion::{BenchmarkId, Criterion, Throughput};-use half::{bf16, f16};use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::SmallRng};use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};use crate::{- common::{- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},- type_short_name,- },- uzu_bench, uzu_test,+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ uzu_test,};fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {@@ -295,124 +289,3 @@ qmv_fast_test!(test_gs128_4bit_mlx, gs = 128, bits = 4, format = QuantizationMetqmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, format = QuantizationMethod::ScaleBias);qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, format = QuantizationMethod::ScaleBias);qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, format = QuantizationMethod::ScaleBias);--fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(- rng: &mut R,- range: std::ops::Range<T>,- len: usize,-) -> Box<[T]> {- (0..len).map(|_| rng.random_range(range.clone())).collect()-}--fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(- c: &mut Criterion,- context: &B::Context,- label: &str,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));- let block_size: usize = if bits == 4 {- 512- } else {- 256- };-- let nk_pairs: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];- for &(n, k) in nk_pairs {- if n % 8 != 0 || k % block_size != 0 {- continue;- }- for m in [1usize, 2, 4, 8] {- let num_groups = k.div_ceil(group_size as usize);- let mut rng = SmallRng::seed_from_u64(42);-- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(- context,- T::data_type(),- group_size,- bits,- quant_method,- false,- )- .unwrap();-- let w_buf = alloc_allocation_with_data::<B, u32>(- context,- &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),- );- let scales_buf = alloc_allocation_with_data::<B, T>(- context,- &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)- .iter()- .map(|&v| T::from(v).unwrap())- .collect::<Vec<_>>(),- );- let x_buf = alloc_allocation_with_data::<B, T>(- context,- &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k)- .iter()- .map(|&v| T::from(v).unwrap())- .collect::<Vec<_>>(),- );- let mut y_buf = alloc_allocation::<B, T>(context, m * n);-- let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {- let zp_stride = if bits == 4 {- (num_groups + 1) / 2- } else {- num_groups- };- alloc_allocation_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))- });- let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {- alloc_allocation_with_data::<B, T>(- context,- &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)- .iter()- .map(|&v| T::from(v).unwrap())- .collect::<Vec<_>>(),- )- });-- group.throughput(Throughput::Elements((m * n * k) as u64));- group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {- b.iter_custom(|n_iters| {- let mut encoder = Encoder::<B>::new(context).unwrap();- for _ in 0..n_iters {- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- None::<&Allocation<B>>,- k as u32,- n as u32,- m as u32,- &mut encoder,- );- }- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()- })- });- }- }-}--#[uzu_bench]-fn bench_qmv_fast(c: &mut Criterion) {- for_each_backend!(|B| {- let context = <B as Backend>::Context::new().unwrap();- bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);- });-}diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rsindex 07700d020..e74b751d4 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs@@ -1,6 +1,5 @@use std::mem::{MaybeUninit, size_of};-use criterion::{BenchmarkId, Criterion, Throughput};use num_traits::{Float, NumCast};use proptest::prelude::*;use rand::{RngExt, SeedableRng, rngs::SmallRng};@@ -18,9 +17,8 @@ use crate::{common::{dispatch_dtype, for_each_context,proptest::{ComparableTestResults, TestContextes, kernel_data_type},- type_short_name,},- uzu_bench, uzu_test,+ uzu_test,};struct ArgmaxTestResults(Box<[u32]>);@@ -116,83 +114,3 @@ fn test_argmax_prop() {});}-#[uzu_bench]-fn bench_argmax(c: &mut Criterion) {- type T = f32;- let batch_size = 1;-- for_each_backend!(|B| {- let context = <B as Backend>::Context::new().unwrap();-- let single_kernel =- <<B as Backend>::Kernels as Kernels>::ArgmaxSingleKernel::new(&context, T::data_type()).unwrap();- let twopass_kernel_main =- <<B as Backend>::Kernels as Kernels>::ArgmaxMainKernel::new(&context, T::data_type()).unwrap();- let twopass_kernel_final = <<B as Backend>::Kernels as Kernels>::ArgmaxFinalKernel::new(&context).unwrap();-- let mut output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();-- let mut group = c.benchmark_group(format!("{}/Kernel/Sampling/Argmax", type_short_name::<B>()));-- for vocab_size in [- 65536, // LFM2-350M- 128256, // Llama-3.2-1B- 151936, // Qwen3-1.7B- 262144, // Gemma-3-1b- ] {- let logits_data = get_argmax_data::<T>(1337, batch_size, vocab_size);- let logits_buffer = context.create_array_from(&[logits_data.len()], logits_data.as_ref()).into_allocation();- let mut twopass_partial_results_buffer = context- .create_allocation(- batch_size * vocab_size.div_ceil(4096) * size_of::<ArgmaxPair>(),- AllocationType::Global,- )- .unwrap();-- group.throughput(Throughput::Elements(vocab_size as u64));-- group.bench_function(BenchmarkId::new("SinglePass", format!("Vocab[{vocab_size}]")), |b| {- b.iter_custom(|n_iters| {- let mut encoder = Encoder::<B>::new(&context).unwrap();-- for _ in 0..n_iters {- single_kernel.encode(- &logits_buffer,- &mut output_buffer,- batch_size as u32,- vocab_size as u32,- &mut encoder,- );- }-- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()- })- });-- group.bench_function(BenchmarkId::new("TwoPass", format!("Vocab[{vocab_size}]")), |b| {- b.iter_custom(|n_iters| {- let mut encoder = Encoder::<B>::new(&context).unwrap();-- for _ in 0..n_iters {- twopass_kernel_main.encode(- &logits_buffer,- &mut twopass_partial_results_buffer,- batch_size as u32,- vocab_size as u32,- &mut encoder,- );- twopass_kernel_final.encode(- &twopass_partial_results_buffer,- &mut output_buffer,- batch_size as u32,- vocab_size as u32,- &mut encoder,- );- }-- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()- })- });- }- });-}diff --git a/crates/benchmarks/src/runner/types/result.rs b/crates/benchmarks/src/runner/types/result.rsindex 417702c7c..7b3980ff2 100644--- a/crates/benchmarks/src/runner/types/result.rs+++ b/crates/benchmarks/src/runner/types/result.rs@@ -1,4 +1,4 @@-use backend_uzu::ConfigDataType;+use backend_uzu::DataType;use serde::{Deserialize, Serialize};use crate::runner::types::{Device, Task};@@ -9,7 +9,7 @@ pub struct Result {pub device: Device,pub engine_version: String,pub timestamp: u64,- pub precision: ConfigDataType,+ pub precision: DataType,pub memory_used: Option<u64>,pub tokens_count_input: u64,pub tokens_count_output: u64,diff --git a/crates/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs b/crates/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rsindex 0ff2320ff..1232d9ee1 100644--- a/crates/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs+++ b/crates/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs@@ -56,9 +56,11 @@ async fn test_storage_mock_registry_model_download_lifecycle(tracing::info!("pausing storage item");item.pause().await?;item.reconcile().await?;- let paused_state = item.state().await;+ let paused_state = wait_for_item_state(&item, &mut progress, "pause transitions to Paused", |state| {+ matches!(state.phase, DownloadPhase::Paused {})+ })+ .await;tracing::info!(?paused_state, "observed paused state");- assert!(matches!(paused_state.phase, DownloadPhase::Paused {}), "pause must transition to Paused; got {:?}", paused_state.phase);tracing::info!("resuming storage item");item.download().await?;