A high-performance inference engine for AI models
Unified Quantized Gemm (#432)
eugene7ade24dfd1f0
diff --git a/Cargo.lock b/Cargo.lockindex 91095f9e7..ef6103615 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -595,6 +595,7 @@ dependencies = ["rangemap","regex","rhai",+ "rstest","rustix","schemars 1.2.1","serde",diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.tomlindex 7134ac07e..1c340800d 100644--- a/crates/backend-uzu/Cargo.toml+++ b/crates/backend-uzu/Cargo.toml@@ -60,6 +60,7 @@ is_close.workspace = trueschemars.workspace = truecomfy-table.workspace = trueindicatif.workspace = true+rstest.workspace = true[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]criterion = { workspace = true, default-features = true }diff --git a/crates/backend-uzu/build/common/constraints.rs b/crates/backend-uzu/build/common/constraints.rsindex bc9c83298..c341f015f 100644--- a/crates/backend-uzu/build/common/constraints.rs+++ b/crates/backend-uzu/build/common/constraints.rs@@ -1,26 +1,60 @@-use rhai::{Dynamic, Engine, Scope};+use std::collections::{BTreeMap, BTreeSet};-/// Returns true if all constraints pass for the given name/value bindings.-/// Panics on eval failure (programming error in kernel definition).-pub fn satisfied<N: AsRef<str>, V: AsRef<str>>(- engine: &Engine,- bindings: &[(N, V)],- constraints: &[impl AsRef<str>],-) -> bool {- if constraints.is_empty() {- return true;+use rhai::{Dynamic, Engine, Module, Scope};++#[cfg(all(feature = "metal", target_os = "macos"))]+use crate::common::mangling::unqualify_variant;++#[cfg(not(all(feature = "metal", target_os = "macos")))]+fn unqualify_variant(value: &str) -> &str {+ value.rsplit("::").next().unwrap_or(value)+}++pub struct Evaluator {+ engine: Engine,+}++impl Evaluator {+ pub fn new<'a>(variant_values: impl IntoIterator<Item = &'a str>) -> Self {+ let mut engine = Engine::new();+ let mut namespaces: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();+ for value in variant_values {+ if let Some((namespace, name)) = value.rsplit_once("::") {+ namespaces.entry(namespace).or_default().insert(name);+ }+ }+ for (namespace, names) in namespaces {+ let mut module = Module::new();+ for name in names {+ module.set_var(name, name.to_string());+ }+ engine.register_static_module(namespace, module.into());+ }+ Self {+ engine,+ }}- let mut scope = Scope::with_capacity(bindings.len());- for (name, val) in bindings {- let val = val.as_ref();- scope.push(- name.as_ref().to_owned(),- engine.eval_expression::<Dynamic>(val).unwrap_or_else(|_| val.to_owned().into()),- );++ pub fn satisfied<N: AsRef<str>, V: AsRef<str>>(+ &self,+ bindings: &[(N, V)],+ constraints: &[impl AsRef<str>],+ ) -> bool {+ if constraints.is_empty() {+ return true;+ }+ let mut scope = Scope::with_capacity(bindings.len());+ for (name, val) in bindings {+ let val = unqualify_variant(val.as_ref());+ scope.push(+ name.as_ref().to_owned(),+ self.engine.eval_expression::<Dynamic>(val).unwrap_or_else(|_| val.to_owned().into()),+ );+ }+ constraints.iter().all(|c| {+ self.engine+ .eval_expression_with_scope::<bool>(&mut scope, c.as_ref())+ .unwrap_or_else(|e| panic!("constraint `{}` failed to evaluate: {e}", c.as_ref()))+ })}- constraints.iter().all(|c| {- engine- .eval_expression_with_scope::<bool>(&mut scope, c.as_ref())- .unwrap_or_else(|e| panic!("constraint `{}` failed to evaluate: {e}", c.as_ref()))- })}diff --git a/crates/backend-uzu/build/common/mangling.rs b/crates/backend-uzu/build/common/mangling.rsindex 2b256af80..8fe99edc6 100644--- a/crates/backend-uzu/build/common/mangling.rs+++ b/crates/backend-uzu/build/common/mangling.rs@@ -6,6 +6,10 @@ use itertools::Itertools;use proc_macro2::TokenStream;use quote::quote;+pub fn unqualify_variant(value: &str) -> &str {+ value.rsplit("::").next().unwrap_or(value)+}+pub fn static_mangle(function_name: impl AsRef<str>,variant: impl IntoIterator<Item = impl AsRef<str>>,@@ -14,7 +18,13 @@ pub fn static_mangle("_D{}{}{}",&function_name.as_ref().len().to_string(),function_name.as_ref(),- variant.into_iter().map(|v| format!("S{}V{}", v.as_ref().len(), v.as_ref().replace('-', "n"))).join("")+ variant+ .into_iter()+ .map(|v| {+ let v = unqualify_variant(v.as_ref()).replace('-', "n");+ format!("S{}V{}", v.len(), v)+ })+ .join(""))}diff --git a/crates/backend-uzu/build/cpu/compiler.rs b/crates/backend-uzu/build/cpu/compiler.rsindex 5e7ffd5e9..903dca53a 100644--- a/crates/backend-uzu/build/cpu/compiler.rs+++ b/crates/backend-uzu/build/cpu/compiler.rs@@ -503,7 +503,12 @@ impl CpuCompiler {parameter_idents = quote! { (#parameter_idents) };}- let rhai_engine = (!function_constraints.is_empty()).then(rhai::Engine::new);+ let variant_value_strs: Vec<String> = function_variants+ .iter()+ .flat_map(|(_, variants)| variants.iter().map(|v| v.to_token_stream().to_string()))+ .collect();+ let evaluator = (!function_constraints.is_empty())+ .then(|| crate::common::constraints::Evaluator::new(variant_value_strs.iter().map(|s| s.as_str())));let constraint_strs: Vec<String> =function_constraints.iter().map(|c| c.to_token_stream().to_string()).collect();@@ -527,7 +532,7 @@ impl CpuCompiler {}).multi_cartesian_product().filter(|variants| {- let Some(engine) = &rhai_engine else {+ let Some(evaluator) = &evaluator else {return true;};let bindings: Vec<(String, String)> = function_parameters@@ -535,7 +540,7 @@ impl CpuCompiler {.enumerate().map(|(i, p)| (p.name.to_string(), variants[i].1.to_string())).collect();- crate::common::constraints::satisfied(engine, &bindings, &constraint_strs)+ evaluator.satisfied(&bindings, &constraint_strs)}).map(|variants| {let (match_variants, generic_variants): (Vec<TokenStream>, Vec<TokenStream>) =diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rsindex 86903d05c..a44f01e78 100644--- a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs@@ -39,6 +39,11 @@ pub fn gpu_type_gen_option_set(option_set: &GpuTypeOptionSet) -> anyhow::Result<\x20 constexpr bool contains({underlying_c} flag) const thread {{\n\\x20\x20\x20 return (raw_value & flag) != 0;\n\\x20 }}\n\+ \x20 constexpr bool contains({underlying_c} flag) const constant {{\n\+ \x20\x20\x20 return (raw_value & flag) != 0;\n\+ \x20 }}\n\+ \x20 constexpr {underlying_c} bits() const thread {{ return raw_value; }}\n\+ \x20 constexpr {underlying_c} bits() const constant {{ return raw_value; }}\n\}};"))}diff --git a/crates/backend-uzu/build/metal/wrapper.rs b/crates/backend-uzu/build/metal/wrapper.rsindex 6ee119cbf..12cda6a27 100644--- a/crates/backend-uzu/build/metal/wrapper.rs+++ b/crates/backend-uzu/build/metal/wrapper.rs@@ -137,7 +137,9 @@ fn kernel_wrappers(kernel_wrappers.push(kernel_header(&bindings, base).into_boxed_str());}- let engine = rhai::Engine::new();+ let evaluator = crate::common::constraints::Evaluator::new(+ kernel.variants.as_deref().into_iter().flatten().flat_map(|tp| tp.variants.iter().map(|v| v.as_ref())),+ );for type_variant in if let Some(variants) = &kernel.variants {variants.iter()@@ -157,7 +159,7 @@ fn kernel_wrappers(vec![None]} {if let Some(ref tv) = type_variant {- if !crate::common::constraints::satisfied(&engine, tv, &kernel.constraints) {+ if !evaluator.satisfied(tv, &kernel.constraints) {continue;}}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rsindex a12f8652f..c77613df7 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs@@ -3,40 +3,114 @@ use derive_more::Display;#[repr(C)]#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]-pub enum GemmInputPrologueKind {+pub enum GemmWeightPrologueKind {FullPrecision,- ExternalRht,+ ScaleBiasDequant,+ ScaleZeroPointDequant,}-#[repr(C)]-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]-pub enum GemmOutputTransformKind {- Store,- Scale,- Accumulate,- Bias,- Rht,- ScaleAccumulate,- ScaleAccumulateBias,- ScaleAccumulateBiasRht,+bitflags! {+ #[repr(transparent)]+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+ pub struct GemmDTransform: u32 {+ const SCALE = 1 << 0;+ const ACCUMULATE = 1 << 1;+ const BIAS = 1 << 2;+ const RHT = 1 << 3;+ }}#[repr(C)]#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]-pub enum GemmWeightPrologueKind {- FullPrecision,- ScaleBiasDequant,- ScaleZeroPointDequant,+pub enum GemmTiling {+ T8x32x32_1x1,+ T64x32x32_2x2,+ T64x64x16_2x2,+ T64x64x32_2x2,+ T64x64x64_2x2,+ T32x32x32_2x2,+ T32x64x32_2x2,+ T64x32x32_4x1,+ T128x128x32_4x4,}-#[repr(C)]-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct GemmTilingConfig {- pub threadgroup_m: u32,- pub threadgroup_n: u32,- pub threadgroup_k: u32,- pub simdgroups_m: u32,- pub simdgroups_n: u32,+impl GemmTiling {+ pub const fn block_m(self) -> u32 {+ match self {+ Self::T8x32x32_1x1 => 8,+ Self::T64x32x32_2x2 => 64,+ Self::T64x64x16_2x2 => 64,+ Self::T64x64x32_2x2 => 64,+ Self::T64x64x64_2x2 => 64,+ Self::T32x32x32_2x2 => 32,+ Self::T32x64x32_2x2 => 32,+ Self::T64x32x32_4x1 => 64,+ Self::T128x128x32_4x4 => 128,+ }+ }++ pub const fn block_n(self) -> u32 {+ match self {+ Self::T8x32x32_1x1 => 32,+ Self::T64x32x32_2x2 => 32,+ Self::T64x64x16_2x2 => 64,+ Self::T64x64x32_2x2 => 64,+ Self::T64x64x64_2x2 => 64,+ Self::T32x32x32_2x2 => 32,+ Self::T32x64x32_2x2 => 64,+ Self::T64x32x32_4x1 => 32,+ Self::T128x128x32_4x4 => 128,+ }+ }++ pub const fn block_k(self) -> u32 {+ match self {+ Self::T8x32x32_1x1 => 32,+ Self::T64x32x32_2x2 => 32,+ Self::T64x64x16_2x2 => 16,+ Self::T64x64x32_2x2 => 32,+ Self::T64x64x64_2x2 => 64,+ Self::T32x32x32_2x2 => 32,+ Self::T32x64x32_2x2 => 32,+ Self::T64x32x32_4x1 => 32,+ Self::T128x128x32_4x4 => 32,+ }+ }++ pub const fn simdgroups_m(self) -> u32 {+ match self {+ Self::T8x32x32_1x1 => 1,+ Self::T64x32x32_2x2 => 2,+ Self::T64x64x16_2x2 => 2,+ Self::T64x64x32_2x2 => 2,+ Self::T64x64x64_2x2 => 2,+ Self::T32x32x32_2x2 => 2,+ Self::T32x64x32_2x2 => 2,+ Self::T64x32x32_4x1 => 4,+ Self::T128x128x32_4x4 => 4,+ }+ }++ pub const fn simdgroups_n(self) -> u32 {+ match self {+ Self::T8x32x32_1x1 => 1,+ Self::T64x32x32_2x2 => 2,+ Self::T64x64x16_2x2 => 2,+ Self::T64x64x32_2x2 => 2,+ Self::T64x64x64_2x2 => 2,+ Self::T32x32x32_2x2 => 2,+ Self::T32x64x32_2x2 => 2,+ Self::T64x32x32_4x1 => 1,+ Self::T128x128x32_4x4 => 4,+ }+ }+}++pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {+ t.simdgroups_m()+}+pub const fn gemm_tiling_simdgroups_per_column(t: GemmTiling) -> u32 {+ t.simdgroups_n()}bitflags! {@@ -48,3 +122,17 @@ bitflags! {const K = 1 << 2;}}++impl GemmAlignment {+ pub fn new(+ m: bool,+ n: bool,+ k: bool,+ ) -> Self {+ let mut bits = Self::empty();+ bits.set(Self::M, m);+ bits.set(Self::N, n);+ bits.set(Self::K, k);+ bits+ }+}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul.rs b/crates/backend-uzu/src/backends/common/kernel/matmul.rsdeleted file mode 100644index 3578ac4d5..000000000--- a/crates/backend-uzu/src/backends/common/kernel/matmul.rs+++ /dev/null@@ -1,61 +0,0 @@-use thiserror::Error;--use crate::{- DataType,- backends::common::{Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, kernel::ManualKernels},-};--#[derive(Debug, Error)]-pub enum MatmulError<B: Backend> {- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),-}--pub enum MatmulArgumentC<'a, B: Backend> {- None,- /// Accumulate: [M, N]- Accumulate,- /// Bias: [N] (broadcasted across M/batch)- Bias(&'a Allocation<B>),-}--// D = ab_scale * (A @ op(B)) + C, where op(B) = B^T when b_transpose else B.-pub struct MatmulArguments<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {- /// A: [M, K]- pub a: &'a Allocation<B>,- pub a_offset: usize,- /// B: [N, K] when b_transpose, else [K, N]- pub b: &'a TB,- pub b_offset: usize,- pub b_leading_dimension: Option<u32>,- pub b_transpose: bool,- /// AB scale: also known as alpha- pub ab_scale: f32,- /// C: behavior depends on enum variant- pub c: MatmulArgumentC<'a, B>,- /// D: [M, N]- pub d: &'a mut Allocation<B>,- /// M dimension: usually batch/number of tokens (rows of A, rows of C)- pub batch_dim: u32,- /// K dimension: usually input_dim/reduction dimension (cols of A, rows of B)- pub input_dim: u32,- /// N dimension: usually output_dim (cols of B, cols of C)- pub output_dim: u32,-}--pub trait MatmulKernel: Sized {- type Backend: Backend<Kernels: ManualKernels<MatmulKernel = Self>>;-- fn new(- context: &<Self::Backend as Backend>::Context,- data_type: DataType,- ) -> Result<Self, MatmulError<Self::Backend>>;-- fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Self::Backend>>>(- &mut self,- arguments: MatmulArguments<Self::Backend, TB>,- encoder: &mut Encoder<Self::Backend>,- );-}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rsnew file mode 100644index 000000000..8592fd179--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs@@ -0,0 +1,16 @@+use super::{d_ops::MatmulDOps, matmul_b::MatmulB};+use crate::backends::common::{Allocation, AsBufferRangeRef, Backend};++pub struct MatmulArguments<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {+ pub a: &'a Allocation<B>,+ pub a_offset: usize,+ pub b: MatmulB<'a, B, TB>,+ pub b_offset: usize,+ pub b_leading_dimension: Option<u32>,+ pub b_transpose: bool,+ pub d: &'a mut Allocation<B>,+ pub d_transform: MatmulDOps<'a, B>,+ pub m: u32,+ pub n: u32,+ pub k: u32,+}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/d_ops.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/d_ops.rsnew file mode 100644index 000000000..5ea89c7bd--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/d_ops.rs@@ -0,0 +1,64 @@+use crate::backends::common::{Allocation, Backend, gpu_types::gemm::GemmDTransform};++pub struct MatmulDOps<'a, B: Backend> {+ pub ab_scale: f32,+ pub accumulate: bool,+ pub bias: Option<&'a Allocation<B>>,+ pub rht_factors: Option<&'a Allocation<B>>,+}++impl<'a, B: Backend> MatmulDOps<'a, B> {+ pub fn none() -> Self {+ Self {+ ab_scale: 1.0,+ accumulate: false,+ bias: None,+ rht_factors: None,+ }+ }++ pub fn mask(&self) -> GemmDTransform {+ let mut m = GemmDTransform::empty();+ if self.ab_scale != 1.0 {+ m |= GemmDTransform::SCALE;+ }+ if self.accumulate {+ m |= GemmDTransform::ACCUMULATE;+ }+ if self.bias.is_some() {+ m |= GemmDTransform::BIAS;+ }+ if self.rht_factors.is_some() {+ m |= GemmDTransform::RHT;+ }+ m+ }++ pub fn without(+ self,+ bits: GemmDTransform,+ ) -> Self {+ Self {+ ab_scale: if bits.contains(GemmDTransform::SCALE) {+ 1.0+ } else {+ self.ab_scale+ },+ accumulate: if bits.contains(GemmDTransform::ACCUMULATE) {+ false+ } else {+ self.accumulate+ },+ bias: if bits.contains(GemmDTransform::BIAS) {+ None+ } else {+ self.bias+ },+ rht_factors: if bits.contains(GemmDTransform::RHT) {+ None+ } else {+ self.rht_factors+ },+ }+ }+}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/error.rsnew file mode 100644index 000000000..0c6fd8733--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs@@ -0,0 +1,25 @@+use thiserror::Error;++use crate::{+ DataType,+ backends::common::{Backend, gpu_types::gemm::GemmDTransform},+};++#[derive(Debug, Error)]+pub enum MatmulError<B: Backend> {+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+ #[error("Unsupported group size: {0}")]+ UnsupportedGroupSize(usize),+ #[error("Unsupported D-transform op {bit:?} on path {path}")]+ UnsupportedDOp {+ bit: GemmDTransform,+ path: &'static str,+ },+ #[error("Unsupported B layout on path {path}")]+ UnsupportedLayout {+ path: &'static str,+ },+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rsnew file mode 100644index 000000000..152799751--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs@@ -0,0 +1,28 @@+use super::{arguments::MatmulArguments, quant_combo::MatmulQuantCombo};+use crate::{+ DataType,+ backends::common::{AsBufferRangeRef, Backend, Buffer, Encoder, kernel::ManualKernels},+};++pub trait MatmulKernel: Sized {+ type Backend: Backend<Kernels: ManualKernels<MatmulKernel = Self>>;++ fn new(+ context: &<Self::Backend as Backend>::Context,+ data_type: DataType,+ ) -> Result<Self, <Self::Backend as Backend>::Error>;++ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Self::Backend>>>(+ &mut self,+ arguments: MatmulArguments<Self::Backend, TB>,+ encoder: &mut Encoder<Self::Backend>,+ ) -> Result<(), <Self::Backend as Backend>::Error>;++ fn preheat_quant_combo(+ &mut self,+ _context: &<Self::Backend as Backend>::Context,+ _combo: MatmulQuantCombo,+ ) -> Result<(), <Self::Backend as Backend>::Error> {+ Ok(())+ }+}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rsnew file mode 100644index 000000000..55774863a--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs@@ -0,0 +1,75 @@+use crate::{+ DataType,+ backends::common::{+ Allocation, AsBufferRangeRef, Backend,+ gpu_types::{QuantizationMode, gemm::GemmWeightPrologueKind},+ },+};++pub enum MatmulB<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {+ FullPrecision {+ b: &'a TB,+ },+ ScaleBiasDequant {+ b: &'a Allocation<B>,+ scales: &'a Allocation<B>,+ biases: &'a Allocation<B>,+ mode: QuantizationMode,+ group_size: u32,+ },+ ScaleZeroPointDequant {+ b: &'a Allocation<B>,+ scales: &'a Allocation<B>,+ zero_points: &'a Allocation<B>,+ mode: QuantizationMode,+ group_size: u32,+ },+}++impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {+ pub fn weight_prologue(&self) -> GemmWeightPrologueKind {+ match self {+ Self::FullPrecision {+ ..+ } => GemmWeightPrologueKind::FullPrecision,+ Self::ScaleBiasDequant {+ ..+ } => GemmWeightPrologueKind::ScaleBiasDequant,+ Self::ScaleZeroPointDequant {+ ..+ } => GemmWeightPrologueKind::ScaleZeroPointDequant,+ }+ }++ pub fn bits_per_weight(&self) -> Option<u32> {+ match self {+ Self::FullPrecision {+ ..+ } => None,+ Self::ScaleBiasDequant {+ mode,+ ..+ }+ | Self::ScaleZeroPointDequant {+ mode,+ ..+ } => Some(DataType::from(*mode).size_in_bits() as u32),+ }+ }++ pub fn group_size(&self) -> Option<u32> {+ match self {+ Self::FullPrecision {+ ..+ } => None,+ Self::ScaleBiasDequant {+ group_size,+ ..+ }+ | Self::ScaleZeroPointDequant {+ group_size,+ ..+ } => Some(*group_size),+ }+ }+}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rsnew file mode 100644index 000000000..c7ebfca8c--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs@@ -0,0 +1,13 @@+mod arguments;+mod d_ops;+mod error;+mod kernel;+mod matmul_b;+mod quant_combo;++pub use arguments::MatmulArguments;+pub use d_ops::MatmulDOps;+pub use error::MatmulError;+pub use kernel::MatmulKernel;+pub use matmul_b::MatmulB;+pub use quant_combo::MatmulQuantCombo;diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rsnew file mode 100644index 000000000..471933b14--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs@@ -0,0 +1,8 @@+use crate::backends::common::gpu_types::{QuantizationMethod, QuantizationMode};++#[derive(Debug, Clone, Copy)]+pub struct MatmulQuantCombo {+ pub method: QuantizationMethod,+ pub mode: QuantizationMode,+ pub group_size: u32,+}diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rsindex a2659f323..7df26e067 100644--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs@@ -11,6 +11,5 @@ pub mod kv_cache_update;pub mod matmul;pub mod mlp_gate_act_mul;pub mod moe;-pub mod quant_matmul;pub mod sampling;pub mod ssd_prefill;diff --git a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rsdeleted file mode 100644index 75c1acdd0..000000000--- a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs+++ /dev/null@@ -1,251 +0,0 @@-use crate::{- DataType,- backends::common::{- Allocation, Backend, Encoder, Kernels,- gpu_types::{QuantizationMethod, QuantizationMode},- kernel::{QuantizedMatmulQmmTransposedKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},- },-};--#[derive(Debug, thiserror::Error)]-pub enum QuantizedMatmulError<B: Backend> {- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),- #[error("Unsupported group size: {0}")]- UnsupportedGroupSize(usize),- #[error("Hadamard not supported for this kernel configuration")]- UnsupportedHadamard,-}--#[derive(Debug, Clone, Copy)]-pub struct QuantizedMatmulConfiguration {- pub data_type: DataType,- pub group_size: usize,- pub input_dim: usize,- pub output_dim: usize,- pub mode: QuantizationMode,- pub quantization_method: QuantizationMethod,- pub use_hadamard: bool,-}--pub struct QuantizedMatmulArguments<'a, B: Backend> {- pub a: &'a Allocation<B>,- pub a_offset: usize,- pub b: &'a Allocation<B>,- pub scales: &'a Allocation<B>,- pub zero_points_or_biases: &'a Allocation<B>,- pub output: &'a mut Allocation<B>,- pub hadamard_factors: Option<&'a Allocation<B>>,- pub batch_dim: usize,-}--pub struct QuantizedMatmulKernelEncodable<B: Backend> {- matrix_vector: MatrixVectorKernel<B>,- matrix_matrix: MatrixMatrixKernel<B>,- input_dim: usize,- output_dim: usize,- quantization_method: QuantizationMethod,-}--enum MatrixVectorKernel<B: Backend> {- Qmv(<B::Kernels as Kernels>::QuantizedMatmulQmvKernel),- QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel),-}--type QmmKernel<B> = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel;--enum MatrixMatrixKernel<B: Backend> {- Big(QmmKernel<B>),- BigAndSmall {- big: QmmKernel<B>,- small: QmmKernel<B>,- },-}--impl<B: Backend> MatrixMatrixKernel<B> {- fn pick(- &self,- batch_dim: usize,- ) -> Option<&QmmKernel<B>> {- match self {- Self::Big(big) => (batch_dim >= 32).then_some(big),- Self::BigAndSmall {- big,- small,- } => Some(if batch_dim < 48 {- small- } else {- big- }),- }- }-}--impl<B: Backend> QuantizedMatmulKernelEncodable<B> {- pub fn new(- context: &B::Context,- configuration: QuantizedMatmulConfiguration,- ) -> Result<Self, QuantizedMatmulError<B>> {- if !matches!(configuration.data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(QuantizedMatmulError::UnsupportedDataType(configuration.data_type));- }-- if !matches!(configuration.group_size, 32 | 64 | 128) {- return Err(QuantizedMatmulError::UnsupportedGroupSize(configuration.group_size));- }-- let bits = match configuration.mode {- QuantizationMode::U4 => 4,- QuantizationMode::I8 | QuantizationMode::U8 => 8,- };- let group_size = configuration.group_size as u32;-- let matrix_vector = if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {- MatrixVectorKernel::QmvFast(- <B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(- context,- configuration.data_type,- group_size,- bits,- configuration.quantization_method,- configuration.use_hadamard,- )- .map_err(QuantizedMatmulError::BackendError)?,- )- } else {- if configuration.use_hadamard {- return Err(QuantizedMatmulError::UnsupportedHadamard);- }-- MatrixVectorKernel::Qmv(- <B::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(- context,- configuration.data_type,- group_size,- bits,- configuration.quantization_method,- )- .map_err(QuantizedMatmulError::BackendError)?,- )- };-- let aligned_n_64 = configuration.output_dim % 64 == 0;- let aligned_n_32 = configuration.output_dim % 32 == 0;- let is_bf16 = configuration.data_type == DataType::BF16;- let can_use_64_tile = aligned_n_64 && is_bf16;-- let (bm, bk, bn, aligned_n) = if can_use_64_tile && matches!(configuration.group_size, 64 | 128) {- (64u32, 64u32, 64u32, true)- } else if can_use_64_tile {- (64u32, 32u32, 64u32, true)- } else {- (32u32, 32u32, 32u32, configuration.output_dim % 32 == 0)- };-- let big = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- context,- configuration.data_type,- group_size,- bits,- bm,- bk,- bn,- 2u32, // WM- 2u32, // WN- configuration.quantization_method,- configuration.use_hadamard,- aligned_n,- )- .map_err(QuantizedMatmulError::BackendError)?;-- let matrix_matrix = if aligned_n_32 && !configuration.use_hadamard {- let small = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- context,- configuration.data_type,- group_size,- bits,- 8u32, // BM- 32u32, // BK- 32u32, // BN- 1u32, // WM- 1u32, // WN- configuration.quantization_method,- false, // use_hadamard: BM=8 tile does not support hadamard- true, // aligned_n: BN=32, always aligned for the small tile- )- .map_err(QuantizedMatmulError::BackendError)?;- MatrixMatrixKernel::BigAndSmall {- big,- small,- }- } else {- MatrixMatrixKernel::Big(big)- };-- Ok(Self {- matrix_vector,- matrix_matrix,- input_dim: configuration.input_dim,- output_dim: configuration.output_dim,- quantization_method: configuration.quantization_method,- })- }-- pub fn encode(- &self,- encoder: &mut Encoder<B>,- arguments: QuantizedMatmulArguments<B>,- ) {- let QuantizedMatmulArguments {- a,- a_offset,- b,- scales,- zero_points_or_biases,- output,- hadamard_factors,- batch_dim,- } = arguments;-- let (zero_points, biases) = match self.quantization_method {- QuantizationMethod::ScaleZeroPoint => (Some(zero_points_or_biases), None),- QuantizationMethod::ScaleBias => (None, Some(zero_points_or_biases)),- };-- macro_rules! encode_kernel {- ($kernel:expr $(, $hadamard:expr)?) => {- $kernel.encode(- b,- scales,- zero_points,- biases,- (a, a_offset),- output,- $($hadamard,)?- self.input_dim as u32,- self.output_dim as u32,- batch_dim as u32,- encoder,- )- };- }-- if batch_dim >= 5 && self.output_dim > 1 {- if let Some(kernel) = self.matrix_matrix.pick(batch_dim) {- encode_kernel!(kernel, hadamard_factors);- return;- }- }-- match &self.matrix_vector {- MatrixVectorKernel::Qmv(k) => encode_kernel!(k),- MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, hadamard_factors),- }- }-}--#[cfg(test)]-#[path = "../../../../tests/unit/backends/common/kernel/quant_matmul_test.rs"]-mod tests;diff --git a/crates/backend-uzu/src/backends/cpu/error.rs b/crates/backend-uzu/src/backends/cpu/error.rsindex 57e1ba306..5b5ca0821 100644--- a/crates/backend-uzu/src/backends/cpu/error.rs+++ b/crates/backend-uzu/src/backends/cpu/error.rs@@ -1,11 +1,24 @@-use std::sync::mpsc::RecvError;+use std::{error::Error as StdError, sync::mpsc::RecvError};use thiserror::Error;+use crate::backends::{common::kernel::matmul::MatmulError, cpu::Cpu};+#[derive(Debug, Error)]pub enum CpuError {#[error("Not supported")]NotSupported,#[error("Command buffer execution failed")]CommandBufferExecutionFailed(RecvError),+ #[error("Kernel dispatch failed: {0}")]+ KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),+}++impl From<MatmulError<Cpu>> for CpuError {+ fn from(value: MatmulError<Cpu>) -> Self {+ match value {+ MatmulError::BackendError(e) => e,+ other => CpuError::KernelDispatchFailed(Box::new(other)),+ }+ }}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rsindex b68fab06e..a20050452 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs@@ -1 +1,2 @@+pub mod qmm_transposed;pub mod shaders;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rssimilarity index 56%rename from crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rsrename to crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rsindex 1c7e730cb..56b2afffc 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs@@ -1,6 +1,4 @@-use half::{bf16, f16};use num_traits::Float;-use proc_macros::kernel;use crate::{ArrayElement, backends::common::gpu_types::QuantizationMethod};@@ -33,7 +31,6 @@ pub fn qmm_transposed<T: ArrayElement + Float>(for l in 0..in_vec_size {let group_idx = l / group_size;- // Transposed: weight at row j, col llet weight_linear_idx = j * in_vec_size + l;let val_q = if bits == 4 {@@ -49,10 +46,6 @@ pub fn qmm_transposed<T: ArrayElement + Float>(let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();let scale_t = *scales.add(j * num_groups_k + group_idx);- // Dequantize in f32 to serve as a precision-neutral reference.- // Metal dequantizes in bf16/f16 which rounds differently across- // GPU hardware; computing in f32 avoids biasing toward any- // particular rounding pattern.let w_dequant_f32 = match quant_method {QuantizationMethod::ScaleZeroPoint => {let zp = zero_points.unwrap();@@ -83,64 +76,3 @@ pub fn qmm_transposed<T: ArrayElement + Float>(}}}--#[kernel(QuantizedMatmulQmmTransposed)]-#[variants(T, f32, f16, bf16)]-#[variants(GROUP_SIZE, 32, 64, 128)]-#[variants(BITS, 4, 8)]-#[variants(BM, 8, 32, 64)]-#[variants(BK, 32, 64)]-#[variants(BN, 32, 64)]-#[variants(WM, 1, 2)]-#[variants(WN, 1, 2)]-#[constraint(- (BM == 8 && BK == 32 && BN == 32 && WM == 1 && WN == 1) ||- (BM == 32 && BK == 32 && BN == 32 && WM == 2 && WN == 2) ||- (BM == 32 && BK == 64 && BN == 32 && WM == 2 && WN == 2) ||- (BM == 64 && BK == 32 && BN == 64 && WM == 2 && WN == 2) ||- (BM == 64 && BK == 64 && BN == 64 && WM == 2 && WN == 2))]-#[constraint(BK <= GROUP_SIZE)]-#[constraint(T != "f32" || BK < 64)]-pub fn quantized_matmul_qmm_transposed<- T: ArrayElement + Float,- const GROUP_SIZE: u32,- const BITS: u32,- const BM: u32,- const BK: u32,- const BN: u32,- const WM: u32,- const WN: u32,->(- weights: *const u32,- scales: *const T,- #[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,- input: *const T,- output: *mut T,- #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,- in_vec_size: u32,- out_vec_size: u32,- batch_size: u32,- #[specialize] quant_method: QuantizationMethod,- #[specialize] use_hadamard: bool,- #[specialize] aligned_n: bool,-) {- let _ = (BM, BK, BN, WM, WN); // tile params unused in tile-agnostic CPU reference- if use_hadamard {- unimplemented!("not supported yet");- }- qmm_transposed::<T>(- weights,- scales,- zero_points,- biases,- input,- output,- in_vec_size as usize,- out_vec_size as usize,- batch_size as usize,- quant_method,- GROUP_SIZE as usize,- BITS as usize,- );-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rssimilarity index 56%rename from crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/mod.rsrename to crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rsindex c8b61dac4..e4e00c1e2 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs@@ -1,3 +1,2 @@-pub mod qmm_transposed;pub mod qmv;pub mod qmv_fast;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv.rsrename to crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rssimilarity index 97%rename from crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rsrename to crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rsindex da39ebe32..0a42788ff 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs@@ -4,7 +4,7 @@ use proc_macros::kernel;use crate::{ArrayElement,- backends::{common::gpu_types::QuantizationMethod, cpu::kernel::quant_matmul::qmv::qmv},+ backends::{common::gpu_types::QuantizationMethod, cpu::kernel::matmul::gemv::qmv::qmv},};#[kernel(QuantizedMatmulQmvFast)]diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 9cac8dce3..58094e700 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -4,30 +4,44 @@ use crate::{DataType,backends::{common::{- AsBufferRangeMut, AsBufferRangeRef, Buffer, Encoder,- kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},+ Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,+ gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},+ kernel::{+ HadamardTransformKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, TensorAddBiasKernel,+ matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel},+ },+ },+ cpu::{+ BufferDowncastExt, Cpu, context::CpuContext, error::CpuError, kernel::matmul::quant::encode_quantized_gemm,},- cpu::{BufferDowncastExt, Cpu, context::CpuContext},},utils::pointers::{SendPtr, SendPtrMut},};pub struct MatmulCpuKernel {data_type: DataType,+ hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,+ bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,}impl MatmulKernel for MatmulCpuKernel {type Backend = Cpu;fn new(- _context: &CpuContext,+ context: &CpuContext,data_type: DataType,- ) -> Result<Self, MatmulError<Cpu>> {+ ) -> Result<Self, CpuError> {if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));+ return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());}+ let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)+ .map_err(CpuError::from)?;+ let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, true)+ .map_err(CpuError::from)?;Ok(Self {data_type,+ hadamard,+ bias_add,})}@@ -35,7 +49,38 @@ impl MatmulKernel for MatmulCpuKernel {&mut self,arguments: MatmulArguments<Cpu, TB>,encoder: &mut Encoder<Cpu>,- ) {+ ) -> Result<(), CpuError> {+ match arguments.b {+ MatmulB::FullPrecision {+ ..+ } => self.encode_fp(arguments, encoder).map_err(CpuError::from),+ MatmulB::ScaleBiasDequant {+ ..+ }+ | MatmulB::ScaleZeroPointDequant {+ ..+ } => self.encode_quant(arguments, encoder).map_err(CpuError::from),+ }+ }+}++impl MatmulCpuKernel {+ fn encode_fp<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(+ &mut self,+ arguments: MatmulArguments<Cpu, TB>,+ encoder: &mut Encoder<Cpu>,+ ) -> Result<(), MatmulError<Cpu>> {+ if arguments.d_transform.mask().contains(GemmDTransform::RHT) {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::RHT,+ path: "MatmulCpuKernel",+ });+ }++ let ab_scale = arguments.d_transform.ab_scale;+ let bias_alloc = arguments.d_transform.bias;+ let is_accumulate = arguments.d_transform.accumulate;+let MatmulArguments {a,a_offset,@@ -43,51 +88,49 @@ impl MatmulKernel for MatmulCpuKernel {b_offset,b_leading_dimension,b_transpose,- ab_scale,- c,d,- batch_dim,- input_dim,- output_dim,+ m,+ n,+ k,+ ..} = arguments;+ let weights = match b {+ MatmulB::FullPrecision {+ b: w,+ } => w,+ _ => unreachable!(),+ };+let command_buffer = encoder.as_command_buffer_mut();- let m = batch_dim as usize;- let n = output_dim as usize;- let k = input_dim as usize;- let lda = k;+ let m_u = m as usize;+ let n_u = n as usize;+ let k_u = k as usize;+ let lda = k_u;let ldb = b_leading_dimension.map(|n| n as usize).unwrap_or(if b_transpose {- k+ k_u} else {- n+ n_u});- let ldd = n;+ let ldd = n_u;let data_type = self.data_type;let a_buffer_range = a.as_buffer_range_ref();- let b_buffer_range = b.as_buffer_range_ref();+ let b_buffer_range = weights.as_buffer_range_ref();let d_buffer_range = d.as_buffer_range_mut();- let a_offset = a_buffer_range.range().start + a_offset;- let b_offset = b_buffer_range.range().start + b_offset;- let d_offset = d_buffer_range.range().start;+ let a_byte_off = a_buffer_range.range().start + a_offset;+ let b_byte_off = b_buffer_range.range().start + b_offset;+ let d_byte_off = d_buffer_range.range().start;- let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_offset));- let b_ptr = SendPtr(unsafe { &*b_buffer_range.buffer().downcast().get() }.as_ptr().wrapping_byte_add(b_offset));+ let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));+ let b_ptr =+ SendPtr(unsafe { &*b_buffer_range.buffer().downcast().get() }.as_ptr().wrapping_byte_add(b_byte_off));let d_ptr =- SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_offset) as *mut u8 });-- let (is_accumulate, bias_ptr) = match c {- MatmulArgumentC::Accumulate => (true, None),- MatmulArgumentC::Bias(bias) => {- let bias_buffer_range = bias.as_buffer_range_ref();- let bias_range = bias_buffer_range.range();- (- false,- Some(SendPtr(- unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start),- )),- )- },- MatmulArgumentC::None => (false, None),- };+ SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });++ let bias_ptr = bias_alloc.map(|bias| {+ let bias_buffer_range = bias.as_buffer_range_ref();+ let bias_range = bias_buffer_range.range();+ SendPtr(unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start))+ });command_buffer.push_command(move || match data_type {DataType::F32 => {@@ -96,9 +139,9 @@ impl MatmulKernel for MatmulCpuKernel {b_ptr.as_ptr() as *const f32,ab_scale,d_ptr.as_ptr() as *mut f32,- m,- n,- k,+ m_u,+ n_u,+ k_u,lda,ldb,ldd,@@ -106,7 +149,7 @@ impl MatmulKernel for MatmulCpuKernel {is_accumulate,);if let Some(bias) = bias_ptr {- apply_bias::<f32>(d_ptr.as_ptr() as *mut f32, bias.as_ptr() as *const f32, m, n);+ apply_bias::<f32>(d_ptr.as_ptr() as *mut f32, bias.as_ptr() as *const f32, m_u, n_u);}},DataType::F16 => {@@ -115,9 +158,9 @@ impl MatmulKernel for MatmulCpuKernel {b_ptr.as_ptr() as *const f16,ab_scale,d_ptr.as_ptr() as *mut f16,- m,- n,- k,+ m_u,+ n_u,+ k_u,lda,ldb,ldd,@@ -125,7 +168,7 @@ impl MatmulKernel for MatmulCpuKernel {is_accumulate,);if let Some(bias) = bias_ptr {- apply_bias::<f16>(d_ptr.as_ptr() as *mut f16, bias.as_ptr() as *const f16, m, n);+ apply_bias::<f16>(d_ptr.as_ptr() as *mut f16, bias.as_ptr() as *const f16, m_u, n_u);}},DataType::BF16 => {@@ -134,9 +177,9 @@ impl MatmulKernel for MatmulCpuKernel {b_ptr.as_ptr() as *const bf16,ab_scale,d_ptr.as_ptr() as *mut bf16,- m,- n,- k,+ m_u,+ n_u,+ k_u,lda,ldb,ldd,@@ -144,11 +187,189 @@ impl MatmulKernel for MatmulCpuKernel {is_accumulate,);if let Some(bias) = bias_ptr {- apply_bias::<bf16>(d_ptr.as_ptr() as *mut bf16, bias.as_ptr() as *const bf16, m, n);+ apply_bias::<bf16>(d_ptr.as_ptr() as *mut bf16, bias.as_ptr() as *const bf16, m_u, n_u);}},_ => unreachable!(),});++ Ok(())+ }++ fn encode_quant<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(+ &mut self,+ arguments: MatmulArguments<Cpu, TB>,+ encoder: &mut Encoder<Cpu>,+ ) -> Result<(), MatmulError<Cpu>> {+ let d_mask = arguments.d_transform.mask();+ if d_mask.contains(GemmDTransform::SCALE) {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::SCALE,+ path: "MatmulCpuKernel/Quant",+ });+ }+ if d_mask.contains(GemmDTransform::ACCUMULATE) {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::ACCUMULATE,+ path: "MatmulCpuKernel/Quant",+ });+ }+ if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {+ return Err(MatmulError::UnsupportedLayout {+ path: "MatmulCpuKernel/Quant",+ });+ }++ let group_size = match arguments.b {+ MatmulB::ScaleBiasDequant {+ group_size,+ ..+ }+ | MatmulB::ScaleZeroPointDequant {+ group_size,+ ..+ } => group_size,+ MatmulB::FullPrecision {+ ..+ } => unreachable!(),+ };+ if !matches!(group_size, 32 | 64 | 128) {+ return Err(MatmulError::UnsupportedGroupSize(group_size as usize));+ }++ let post_rht = arguments.d_transform.rht_factors;+ let post_bias = arguments.d_transform.bias;++ if arguments.m >= 5 && arguments.n > 1 {+ let MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ d,+ d_transform,+ m,+ n,+ k,+ } = arguments;+ let d_transform = d_transform.without(GemmDTransform::RHT | GemmDTransform::BIAS);+ encode_quantized_gemm(+ encoder,+ MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ d: &mut *d,+ d_transform,+ m,+ n,+ k,+ },+ self.data_type,+ );+ if let Some(bias) = post_bias {+ self.bias_add.encode(None::<&Allocation<Cpu>>, bias, &mut *d, n, m * n, encoder);+ }+ if let Some(factors) = post_rht {+ self.hadamard.encode(d, factors, n, m, encoder);+ }+ return Ok(());+ }++ let hadamard_factors = post_rht;++ let MatmulArguments {+ a,+ a_offset,+ b,+ d,+ m,+ n,+ k,+ ..+ } = arguments;+ let (weights, scales, zp_or_bias, method, mode, group_size) = match b {+ MatmulB::ScaleBiasDequant {+ b: w,+ scales,+ biases,+ mode,+ group_size,+ } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),+ MatmulB::ScaleZeroPointDequant {+ b: w,+ scales,+ zero_points,+ mode,+ group_size,+ } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),+ MatmulB::FullPrecision {+ ..+ } => unreachable!(),+ };++ let bits = match mode {+ QuantizationMode::U4 => 4u32,+ QuantizationMode::I8 | QuantizationMode::U8 => 8u32,+ };+ let use_fast = n % 8 == 0 && k % 512 == 0;+ let (zero_points, biases) = match method {+ QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias), None),+ QuantizationMethod::ScaleBias => (None, Some(zp_or_bias)),+ };++ let context = encoder.context();+ if use_fast {+ let kernel =+ <<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ context,+ self.data_type,+ group_size,+ bits,+ method,+ hadamard_factors.is_some(),+ )+ .map_err(MatmulError::BackendError)?;+ kernel.encode(+ weights,+ scales,+ zero_points,+ biases,+ (a, a_offset),+ &mut *d,+ hadamard_factors,+ k,+ n,+ m,+ encoder,+ );+ } else {+ if hadamard_factors.is_some() {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::RHT,+ path: "MatmulCpuKernel/Quant",+ });+ }+ let kernel =+ <<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(+ context,+ self.data_type,+ group_size,+ bits,+ method,+ )+ .map_err(MatmulError::BackendError)?;+ kernel.encode(weights, scales, zero_points, biases, (a, a_offset), &mut *d, k, n, m, encoder);+ }+ if let Some(bias) = post_bias {+ self.bias_add.encode(None::<&Allocation<Cpu>>, bias, d, n, m * n, encoder);+ }+ Ok(())}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rsindex b6ed115d4..ca23d47de 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs@@ -1,5 +1,7 @@pub mod gemm;+pub mod gemv;mod matmul;+pub(crate) mod quant;pub use matmul::MatmulCpuKernel;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rsnew file mode 100644index 000000000..3b08eb8c7--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs@@ -0,0 +1,155 @@+use half::{bf16, f16};+use num_traits::Float;++use crate::{+ ArrayElement, DataType,+ backends::{+ common::{+ AsBufferRangeMut, AsBufferRangeRef, Buffer, Encoder,+ gpu_types::{QuantizationMethod, QuantizationMode},+ kernel::matmul::{MatmulArguments, MatmulB},+ },+ cpu::{Cpu, kernel::matmul::gemm::qmm_transposed::qmm_transposed},+ },+ utils::pointers::{SendPtr, SendPtrMut},+};++pub(crate) fn encode_quantized_gemm<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(+ encoder: &mut Encoder<Cpu>,+ arguments: MatmulArguments<Cpu, TB>,+ data_type: DataType,+) {+ let MatmulArguments {+ a,+ a_offset,+ b,+ d,+ m,+ n,+ k,+ ..+ } = arguments;+ let (weights, scales, zp_or_bias, method, mode, group_size) = match b {+ MatmulB::ScaleBiasDequant {+ b: w,+ scales,+ biases,+ mode,+ group_size,+ } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),+ MatmulB::ScaleZeroPointDequant {+ b: w,+ scales,+ zero_points,+ mode,+ group_size,+ } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),+ MatmulB::FullPrecision {+ ..+ } => unreachable!(),+ };++ let bits: usize = match mode {+ QuantizationMode::U4 => 4,+ QuantizationMode::I8 | QuantizationMode::U8 => 8,+ };+ let in_vec_size = k as usize;+ let out_vec_size = n as usize;+ let batch_size = m as usize;+ let group_size = group_size as usize;++ let a_buf = a.as_buffer_range_ref();+ let b_buf = weights.as_buffer_range_ref();+ let scales_buf = scales.as_buffer_range_ref();+ let zp_buf = zp_or_bias.as_buffer_range_ref();+ let d_buf = d.as_buffer_range_mut();++ let a_byte_off = a_buf.range().start + a_offset * data_type.size_in_bytes();+ let b_byte_off = b_buf.range().start;+ let scales_byte_off = scales_buf.range().start;+ let zp_byte_off = zp_buf.range().start;+ let d_byte_off = d_buf.range().start;++ let a_ptr = SendPtr(unsafe { &*a_buf.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));+ let b_ptr = SendPtr(unsafe { &*b_buf.buffer().get() }.as_ptr().wrapping_byte_add(b_byte_off));+ let scales_ptr = SendPtr(unsafe { &*scales_buf.buffer().get() }.as_ptr().wrapping_byte_add(scales_byte_off));+ let zp_ptr = SendPtr(unsafe { &*zp_buf.buffer().get() }.as_ptr().wrapping_byte_add(zp_byte_off));+ let d_ptr = SendPtrMut(unsafe { (&*d_buf.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });++ let command_buffer = encoder.as_command_buffer_mut();+ command_buffer.push_command(move || match data_type {+ DataType::F32 => run::<f32>(+ a_ptr,+ b_ptr,+ scales_ptr,+ zp_ptr,+ d_ptr,+ in_vec_size,+ out_vec_size,+ batch_size,+ method,+ group_size,+ bits,+ ),+ DataType::F16 => run::<f16>(+ a_ptr,+ b_ptr,+ scales_ptr,+ zp_ptr,+ d_ptr,+ in_vec_size,+ out_vec_size,+ batch_size,+ method,+ group_size,+ bits,+ ),+ DataType::BF16 => run::<bf16>(+ a_ptr,+ b_ptr,+ scales_ptr,+ zp_ptr,+ d_ptr,+ in_vec_size,+ out_vec_size,+ batch_size,+ method,+ group_size,+ bits,+ ),+ _ => unreachable!(),+ });+}++fn run<T: ArrayElement + Float>(+ a_ptr: SendPtr<u8>,+ b_ptr: SendPtr<u8>,+ scales_ptr: SendPtr<u8>,+ zp_or_bias_ptr: SendPtr<u8>,+ d_ptr: SendPtrMut<u8>,+ in_vec_size: usize,+ out_vec_size: usize,+ batch_size: usize,+ quant_method: QuantizationMethod,+ group_size: usize,+ bits: usize,+) {+ let (zp, bias) = match quant_method {+ QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias_ptr.as_ptr() as *const u8), None),+ QuantizationMethod::ScaleBias => (None, Some(zp_or_bias_ptr.as_ptr() as *const T)),+ };+ qmm_transposed::<T>(+ b_ptr.as_ptr() as *const u32,+ scales_ptr.as_ptr() as *const T,+ zp,+ bias,+ a_ptr.as_ptr() as *const T,+ d_ptr.as_ptr() as *mut T,+ in_vec_size,+ out_vec_size,+ batch_size,+ quant_method,+ group_size,+ bits,+ );+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rsindex 421ec7c18..0658c20c0 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs@@ -12,7 +12,6 @@ mod matmul;mod mlp;mod moe;mod pooling;-mod quant_matmul;mod rms_norm;mod rope;mod sampling;diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rsindex 250432ce7..2d3c1e4ed 100644--- a/crates/backend-uzu/src/backends/metal/error.rs+++ b/crates/backend-uzu/src/backends/metal/error.rs@@ -1,6 +1,11 @@+use std::error::Error as StdError;+use thiserror::Error;-use crate::backends::metal::kernel::matmul::gemm::GemmSpecializationError;+use crate::backends::{+ common::kernel::matmul::MatmulError,+ metal::{Metal, kernel::matmul::gemm::GemmSpecializationError},+};#[derive(Debug, Error)]pub enum MetalError {@@ -26,10 +31,25 @@ pub enum MetalError {CannotCreateFunction,#[error("Cannot create pipeline state: {0}")]CannotCreatePipelineState(String),- #[error("Invalid GEMM specialization: {0:?}")]- InvalidGemmSpecialization(GemmSpecializationError),#[error("Can not allocate buffer with size={0}")]SparseBufferAlloc(usize),#[error("Can not allocate heap with size={0} and page size={1}")]SparseHeapAlloc(usize, usize),+ #[error("Kernel dispatch failed: {0}")]+ KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),+}++impl From<MatmulError<Metal>> for MetalError {+ fn from(value: MatmulError<Metal>) -> Self {+ match value {+ MatmulError::BackendError(e) => e,+ other => MetalError::KernelDispatchFailed(Box::new(other)),+ }+ }+}++impl From<GemmSpecializationError> for MetalError {+ fn from(value: GemmSpecializationError) -> Self {+ MetalError::KernelDispatchFailed(Box::new(value))+ }}diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.hindex b21071fa8..841f4cd72 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h@@ -5,35 +5,41 @@using namespace metal;namespace uzu::gemm {-enum class GemmInputPrologueKind : uint32_t {- FullPrecision = 0,- ExternalRht = 1,-};--enum class GemmOutputTransformKind : uint32_t {- Store = 0,- Scale = 1,- Accumulate = 2,- Bias = 3,- Rht = 4,- ScaleAccumulate = 5,- ScaleAccumulateBias = 6,- ScaleAccumulateBiasRht = 7,-};-enum class GemmWeightPrologueKind : uint32_t {FullPrecision = 0,ScaleBiasDequant = 1,ScaleZeroPointDequant = 2,};-typedef struct {- uint32_t threadgroup_m;- uint32_t threadgroup_n;- uint32_t threadgroup_k;- uint32_t simdgroups_m;- uint32_t simdgroups_n;-} GemmTilingConfig;+struct GemmDTransform {+ uint raw_value;+ constexpr GemmDTransform() thread : raw_value(0) {}+ constexpr GemmDTransform(uint __dsl_v) thread : raw_value(__dsl_v) {}+ static constant constexpr uint SCALE = 1 << 0;+ static constant constexpr uint ACCUMULATE = 1 << 1;+ static constant constexpr uint BIAS = 1 << 2;+ static constant constexpr uint RHT = 1 << 3;+ constexpr bool contains(uint flag) const thread {+ return (raw_value & flag) != 0;+ }+ constexpr bool contains(uint flag) const constant {+ return (raw_value & flag) != 0;+ }+ constexpr uint bits() const thread { return raw_value; }+ constexpr uint bits() const constant { return raw_value; }+};++enum class GemmTiling : uint32_t {+ T8x32x32_1x1 = 0,+ T64x32x32_2x2 = 1,+ T64x64x16_2x2 = 2,+ T64x64x32_2x2 = 3,+ T64x64x64_2x2 = 4,+ T32x32x32_2x2 = 5,+ T32x64x32_2x2 = 6,+ T64x32x32_4x1 = 7,+ T128x128x32_4x4 = 8,+};struct GemmAlignment {uint raw_value;@@ -45,5 +51,10 @@ struct GemmAlignment {constexpr bool contains(uint flag) const thread {return (raw_value & flag) != 0;}+ constexpr bool contains(uint flag) const constant {+ return (raw_value & flag) != 0;+ }+ constexpr uint bits() const thread { return raw_value; }+ constexpr uint bits() const constant { return raw_value; }};} // namespace uzu::gemmdiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.hindex c56a6f0bb..a1a2f1a9b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h@@ -14,25 +14,26 @@ namespace matmul {template <typename T,- ushort BLOCK_ROWS,- ushort BLOCK_COLS,+ ushort THREADGROUP_TILE_ROWS,+ ushort THREADGROUP_TILE_COLS,ushort DESTINATION_LEADING_DIMENSION,ushort REDUCTION_DIMENSION,ushort THREADGROUP_SIZE,ushort ALIGNMENT = 1,- ushort READS_PER_THREAD = (BLOCK_COLS * BLOCK_ROWS) / (THREADGROUP_SIZE),- ushort THREAD_COLS = BLOCK_COLS / READS_PER_THREAD,+ ushort READS_PER_THREAD =+ (THREADGROUP_TILE_COLS * THREADGROUP_TILE_ROWS) / (THREADGROUP_SIZE),+ ushort THREAD_COLS = THREADGROUP_TILE_COLS / READS_PER_THREAD,ushort THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>struct ThreadgroupLoader {METAL_CONST ushort ROW_ITERATIONS =- (BLOCK_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;+ (THREADGROUP_TILE_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;const int source_leading_dimension;const int tile_stride;const ushort thread_index;- const ushort block_row_index;- const ushort block_col_index;+ const ushort tile_row_index;+ const ushort tile_col_index;threadgroup T* destination;const device T* source;@@ -49,26 +50,27 @@ struct ThreadgroupLoader {): source_leading_dimension(source_leading_dim),tile_stride(- REDUCTION_DIMENSION ? BLOCK_COLS : BLOCK_ROWS * source_leading_dim+ REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS+ : THREADGROUP_TILE_ROWS * source_leading_dim),thread_index(thread_context.simdgroup_index * METAL_SIMD_SIZE +thread_context.simd_lane_id),- block_row_index(thread_index / THREAD_COLS),- block_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),+ tile_row_index(thread_index / THREAD_COLS),+ tile_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),destination(- destination_ptr + block_row_index * DESTINATION_LEADING_DIMENSION +- block_col_index+ destination_ptr + tile_row_index * DESTINATION_LEADING_DIMENSION ++ tile_col_index),source(- source_ptr + block_row_index * source_leading_dim + block_col_index+ source_ptr + tile_row_index * source_leading_dim + tile_col_index) {}template <typename UnaryOp>METAL_FUNC void apply_inplace_op(thread const UnaryOp& operation) const {METAL_PRAGMA_UNROLL- for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {METAL_PRAGMA_UNROLLfor (ushort j = 0; j < READS_PER_THREAD; j++) {destination[i * DESTINATION_LEADING_DIMENSION + j] =@@ -79,20 +81,23 @@ struct ThreadgroupLoader {METAL_FUNC void load_unsafe() const {METAL_PRAGMA_UNROLL- for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {- *((threadgroup ReadVector*)(&destination- [i * DESTINATION_LEADING_DIMENSION])) =- *((const device ReadVector*)(&source[i * source_leading_dimension]));+ for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {+ *reinterpret_cast<threadgroup ReadVector*>(+ &destination[i * DESTINATION_LEADING_DIMENSION]+ ) =+ *reinterpret_cast<const device ReadVector*>(+ &source[i * source_leading_dimension]+ );}}METAL_FUNC void load_safe(short2 source_tile_dimensions) const {source_tile_dimensions =- source_tile_dimensions - short2(block_col_index, block_row_index);+ source_tile_dimensions - short2(tile_col_index, tile_row_index);if (source_tile_dimensions.x <= 0 || source_tile_dimensions.y <= 0) {METAL_PRAGMA_UNROLL- for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {METAL_PRAGMA_UNROLLfor (ushort j = 0; j < READS_PER_THREAD; j++) {destination[i * DESTINATION_LEADING_DIMENSION + j] = T(0);@@ -105,7 +110,7 @@ struct ThreadgroupLoader {T loaded_values[READS_PER_THREAD];METAL_PRAGMA_UNROLL- for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {METAL_PRAGMA_UNROLLfor (ushort j = 0; j < READS_PER_THREAD; j++) {valid_mask[j] =diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex bbb06f838..c07e65a5b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -256,6 +256,56 @@ struct ThreadgroupTile {}}}++ METAL_FUNC void apply_bias(const device U* bias) {+ const device U* bias_pointer = bias + simdgroup_col_offset;++ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < TILE_ROWS; i++) {+ METAL_PRAGMA_UNROLL+ for (ushort j = 0; j < TILE_COLS; j++) {+ thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);+ const ushort col_offset =+ j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;+ METAL_PRAGMA_UNROLL+ for (ushort k = 0;+ k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;+ k++) {+ block_data[k] +=+ static_cast<AccumulatorType>(bias_pointer[col_offset + k]);+ }+ }+ }+ }++ METAL_FUNC void apply_bias_safe(+ const device U* bias,+ short2 tile_dimensions+ ) {+ const device U* bias_pointer = bias + simdgroup_col_offset;+ tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);++ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < TILE_ROWS; i++) {+ METAL_PRAGMA_UNROLL+ for (ushort j = 0; j < TILE_COLS; j++) {+ thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);+ const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;+ const ushort col_offset =+ j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;+ METAL_PRAGMA_UNROLL+ for (ushort k = 0;+ k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;+ k++) {+ if (row_offset < tile_dimensions.y &&+ col_offset + k < tile_dimensions.x) {+ block_data[k] +=+ static_cast<AccumulatorType>(bias_pointer[col_offset + k]);+ }+ }+ }+ }+ }};} // namespace matmuldiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_alignment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_alignment.hnew file mode 100644index 000000000..968ec5041--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_alignment.h@@ -0,0 +1,42 @@+#pragma once++#include "../../../common/integral_constant.h"+#include "../../common/defines.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <typename F>+METAL_FUNC void dispatch_gemm_alignment(uint mask, F f) {+ switch (mask & 0b111u) {+ case 0u:+ f(integral_constant<uint, 0u>{});+ break;+ case 1u:+ f(integral_constant<uint, 1u>{});+ break;+ case 2u:+ f(integral_constant<uint, 2u>{});+ break;+ case 3u:+ f(integral_constant<uint, 3u>{});+ break;+ case 4u:+ f(integral_constant<uint, 4u>{});+ break;+ case 5u:+ f(integral_constant<uint, 5u>{});+ break;+ case 6u:+ f(integral_constant<uint, 6u>{});+ break;+ case 7u:+ f(integral_constant<uint, 7u>{});+ break;+ }+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.hnew file mode 100644index 000000000..923f41c96--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h@@ -0,0 +1,74 @@+#pragma once++#include "../../../generated/gemm.h"++namespace uzu {+namespace gemm {++constexpr uint gemm_tiling_block_m(GemmTiling t) {+ return t == GemmTiling::T8x32x32_1x1 ? 8+ : t == GemmTiling::T64x32x32_2x2 ? 64+ : t == GemmTiling::T64x64x16_2x2 ? 64+ : t == GemmTiling::T64x64x32_2x2 ? 64+ : t == GemmTiling::T64x64x64_2x2 ? 64+ : t == GemmTiling::T32x32x32_2x2 ? 32+ : t == GemmTiling::T32x64x32_2x2 ? 32+ : t == GemmTiling::T64x32x32_4x1 ? 64+ : t == GemmTiling::T128x128x32_4x4 ? 128+ : 0;+}++constexpr uint gemm_tiling_block_n(GemmTiling t) {+ return t == GemmTiling::T8x32x32_1x1 ? 32+ : t == GemmTiling::T64x32x32_2x2 ? 32+ : t == GemmTiling::T64x64x16_2x2 ? 64+ : t == GemmTiling::T64x64x32_2x2 ? 64+ : t == GemmTiling::T64x64x64_2x2 ? 64+ : t == GemmTiling::T32x32x32_2x2 ? 32+ : t == GemmTiling::T32x64x32_2x2 ? 64+ : t == GemmTiling::T64x32x32_4x1 ? 32+ : t == GemmTiling::T128x128x32_4x4 ? 128+ : 0;+}++constexpr uint gemm_tiling_block_k(GemmTiling t) {+ return t == GemmTiling::T8x32x32_1x1 ? 32+ : t == GemmTiling::T64x32x32_2x2 ? 32+ : t == GemmTiling::T64x64x16_2x2 ? 16+ : t == GemmTiling::T64x64x32_2x2 ? 32+ : t == GemmTiling::T64x64x64_2x2 ? 64+ : t == GemmTiling::T32x32x32_2x2 ? 32+ : t == GemmTiling::T32x64x32_2x2 ? 32+ : t == GemmTiling::T64x32x32_4x1 ? 32+ : t == GemmTiling::T128x128x32_4x4 ? 32+ : 0;+}++constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {+ return t == GemmTiling::T8x32x32_1x1 ? 1+ : t == GemmTiling::T64x32x32_2x2 ? 2+ : t == GemmTiling::T64x64x16_2x2 ? 2+ : t == GemmTiling::T64x64x32_2x2 ? 2+ : t == GemmTiling::T64x64x64_2x2 ? 2+ : t == GemmTiling::T32x32x32_2x2 ? 2+ : t == GemmTiling::T32x64x32_2x2 ? 2+ : t == GemmTiling::T64x32x32_4x1 ? 4+ : t == GemmTiling::T128x128x32_4x4 ? 4+ : 0;+}++constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {+ return t == GemmTiling::T8x32x32_1x1 ? 1+ : t == GemmTiling::T64x32x32_2x2 ? 2+ : t == GemmTiling::T64x64x16_2x2 ? 2+ : t == GemmTiling::T64x64x32_2x2 ? 2+ : t == GemmTiling::T64x64x64_2x2 ? 2+ : t == GemmTiling::T32x32x32_2x2 ? 2+ : t == GemmTiling::T32x64x32_2x2 ? 2+ : t == GemmTiling::T64x32x32_4x1 ? 1+ : t == GemmTiling::T128x128x32_4x4 ? 4+ : 0;+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 3e62895df..b3dcb051c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -8,22 +8,21 @@#include "../../../generated/matmul.h"#include "../generated/gemm.h"#include "block_geometry.h"+#include "gemm_tiling.h"using namespace metal;namespace uzu {namespace gemm {-// MPP `matmul2d_descriptor.transpose_right` drives `TRANSPOSE_B` (MSL-// Spec 6.18).-template <- typename T,- ushort THREADGROUP_BLOCK_M,- ushort THREADGROUP_BLOCK_N,- ushort SIMDGROUPS_PER_ROW,- ushort SIMDGROUPS_PER_COLUMN,- bool TRANSPOSE_B>+template <typename T, GemmTiling GEMM_TILING, bool TRANSPOSE_B>struct MxuMmaCore {+ METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);+ METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);+ METAL_CONST ushort SIMDGROUPS_PER_ROW =+ gemm_tiling_simdgroups_per_row(GEMM_TILING);+ METAL_CONST ushort SIMDGROUPS_PER_COLUMN =+ gemm_tiling_simdgroups_per_column(GEMM_TILING);METAL_CONST ushort SIMDGROUP_BLOCK_M =THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;METAL_CONST ushort SIMDGROUP_BLOCK_N =@@ -43,9 +42,11 @@ struct MxuMmaCore {device T* d,const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,- GemmOutputTransformKind output_transform,+ GemmDTransform output_transform,+ const device T* output_bias,const thread ThreadContext& thread_context) {+ (void)output_bias;const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);const auto geometry =ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::@@ -98,12 +99,9 @@ struct MxuMmaCore {const int aligned_k_iterations = int(params->K) / int(THREADGROUP_BLOCK_K);- const bool apply_scale =- output_transform == GemmOutputTransformKind::Scale ||- output_transform == GemmOutputTransformKind::ScaleAccumulate;+ const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);const bool apply_accumulate =- output_transform == GemmOutputTransformKind::Accumulate ||- output_transform == GemmOutputTransformKind::ScaleAccumulate;+ output_transform.contains(GemmDTransform::ACCUMULATE);dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {dispatch_bool(diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_pack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_pack.hnew file mode 100644index 000000000..a9209512b--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_pack.h@@ -0,0 +1,20 @@+#pragma once++#include <metal_stdlib>++namespace uzu {+namespace gemm {++template <int bits, int word_size_bits = 8>+inline constexpr short get_pack_factor() {+ return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : word_size_bits / bits);+}++template <int bits, int word_size_bits = 8>+inline constexpr short get_bytes_per_pack() {+ constexpr int is_power_of_two_bits = (bits & (bits - 1)) == 0;+ return is_power_of_two_bits ? (word_size_bits / 8) : (bits == 5 ? 5 : 3);+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.hnew file mode 100644index 000000000..6e34d9caf--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h@@ -0,0 +1,170 @@+#pragma once++#include <metal_simdgroup>+#include <metal_stdlib>++#include "../../common/defines.h"+#include "quant_pack.h"+#include "quant_unpack.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <+ typename T,+ short THREADGROUP_TILE_ROWS,+ short THREADGROUP_TILE_COLS,+ short DESTINATION_LEADING_DIMENSION,+ short REDUCTION_DIMENSION,+ short THREADGROUP_SIZE,+ short GROUP_SIZE,+ short BITS>+struct QuantizedBlockLoaderScaleBias {+ static_assert(+ THREADGROUP_TILE_COLS <= GROUP_SIZE,+ "Group size should be larger than columns"+ );+ static_assert(+ GROUP_SIZE % THREADGROUP_TILE_COLS == 0,+ "Group size should be divisible by columns"+ );+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");++ METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();+ METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();+ METAL_CONST short THREADGROUP_TILE_COLS_PACKED =+ THREADGROUP_TILE_COLS / pack_factor;+ METAL_CONST short READS_PER_THREAD =+ (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)+ ? 1+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) /+ THREADGROUP_SIZE;+ METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;+ METAL_CONST bool TILE_HAS_IDLE_THREADS =+ THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;++ const int src_leading_dim;+ const int tile_stride;+ short group_step_counter;+ const int group_stride;++ const short thread_index;+ const short tile_row_index;+ const short tile_col_index;++ threadgroup T* dst;+ const device uint8_t* src;+ const device T* scales;+ const device T* biases;++ QuantizedBlockLoaderScaleBias(+ const device uint8_t* src_,+ const device T* scales_,+ const device T* biases_,+ const int src_leading_dim_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ )+ : src_leading_dim(src_leading_dim_),+ tile_stride(+ REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS_PACKED * bytes_per_pack+ : THREADGROUP_TILE_ROWS * src_leading_dim_ *+ bytes_per_pack / pack_factor+ ),+ group_step_counter(0),+ group_stride(THREADGROUP_TILE_ROWS * src_leading_dim_ / GROUP_SIZE),+ thread_index(simd_group_id * 32 + simd_lane_id),+ tile_row_index(+ READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED+ ),+ tile_col_index(+ (READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED+ ),+ dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION ++ tile_col_index * pack_factor),+ src(src_ ++ tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor ++ tile_col_index * bytes_per_pack),+ scales(scales_ + tile_row_index * src_leading_dim_ / GROUP_SIZE),+ biases(biases_ + tile_row_index * src_leading_dim_ / GROUP_SIZE) {}++ void load_unsafe() const {+ if constexpr (TILE_HAS_IDLE_THREADS) {+ if (tile_row_index >= THREADGROUP_TILE_ROWS) {+ return;+ }+ }++ T scale = *scales;+ T bias = *biases;+ for (int i = 0; i < READS_PER_THREAD; i++) {+ dequantize<T, pack_factor, BITS>(+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );+ }+ }++ void load_safe(short2 src_tile_dim) const {+ if constexpr (TILE_HAS_IDLE_THREADS) {+ if (tile_row_index >= THREADGROUP_TILE_ROWS) {+ return;+ }+ }++ if constexpr (REDUCTION_DIMENSION == 1) {+ if (tile_row_index >= src_tile_dim.x) {+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {+ dst[i] = T(0);+ }+ return;+ }+ } else {+ if (tile_row_index >= src_tile_dim.y) {+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {+ dst[i] = T(0);+ }+ return;+ }+ }++ T scale = *scales;+ T bias = *biases;+ for (int i = 0; i < READS_PER_THREAD; i++) {+ dequantize<T, pack_factor, BITS>(+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );+ }+ }++ void next() {+ src += tile_stride;+ if constexpr (REDUCTION_DIMENSION == 1) {+ if constexpr (GROUP_STEPS_PER_BLOCK > 1) {+ group_step_counter++;+ if (group_step_counter == GROUP_STEPS_PER_BLOCK) {+ group_step_counter = 0;+ scales++;+ biases++;+ }+ } else {+ scales++;+ biases++;+ }+ } else {+ scales += group_stride;+ biases += group_stride;+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hnew file mode 100644index 000000000..48cc1f462--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -0,0 +1,247 @@+#pragma once++#include <metal_simdgroup>+#include <metal_stdlib>++#include "../../common/defines.h"+#include "quant_pack.h"+#include "quant_unpack.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <+ typename T,+ short THREADGROUP_TILE_ROWS,+ short THREADGROUP_TILE_COLS,+ short DESTINATION_LEADING_DIMENSION,+ short REDUCTION_DIMENSION,+ short THREADGROUP_SIZE,+ short GROUP_SIZE,+ short BITS>+struct QuantizedBlockLoaderScaleZeroPoint {+ static_assert(+ THREADGROUP_TILE_COLS <= GROUP_SIZE,+ "Group size should be larger than columns"+ );+ static_assert(+ GROUP_SIZE % THREADGROUP_TILE_COLS == 0,+ "Group size should be divisible by columns"+ );+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");++ METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();+ METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();+ METAL_CONST short THREADGROUP_TILE_COLS_PACKED =+ THREADGROUP_TILE_COLS / pack_factor;+ METAL_CONST short READS_PER_THREAD =+ (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)+ ? 1+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) /+ THREADGROUP_SIZE;+ METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;+ METAL_CONST bool TILE_HAS_IDLE_THREADS =+ THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;++ const int src_leading_dim;+ const int groups_per_row;+ const int tile_stride;+ short group_step_counter;+ int k_base;+ const int group_stride;++ const short thread_index;+ const short tile_row_index;+ const short tile_col_index;++ threadgroup T* dst;+ const device uint8_t* src;+ const device T* scales;+ const device T* scales_row_start;+ const device uint8_t* zero_points_row_start;++ QuantizedBlockLoaderScaleZeroPoint(+ const device uint8_t* src_,+ const device T* scales_,+ const device uint8_t* zero_points_row_start_,+ const int src_leading_dim_,+ const int groups_per_row_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ )+ : src_leading_dim(src_leading_dim_), groups_per_row(groups_per_row_),+ tile_stride(+ REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS_PACKED * bytes_per_pack+ : THREADGROUP_TILE_ROWS * src_leading_dim_ *+ bytes_per_pack / pack_factor+ ),+ group_step_counter(0), k_base(0),+ group_stride(THREADGROUP_TILE_ROWS * groups_per_row_),+ thread_index(simd_group_id * 32 + simd_lane_id),+ tile_row_index(+ READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED+ ),+ tile_col_index(+ (READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED+ ),+ dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION ++ tile_col_index * pack_factor),+ src(src_ ++ tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor ++ tile_col_index * bytes_per_pack),+ scales(+ REDUCTION_DIMENSION == 1+ ? (scales_ + tile_row_index * groups_per_row_)+ : scales_+ ),+ scales_row_start(+ REDUCTION_DIMENSION == 1+ ? (scales_ + tile_row_index * groups_per_row_)+ : scales_+ ),+ zero_points_row_start(+ REDUCTION_DIMENSION == 1+ ? (zero_points_row_start_ ++ tile_row_index * (BITS == 4 ? ((groups_per_row_ + 1) / 2)+ : groups_per_row_))+ : zero_points_row_start_+ ) {}++ inline void current_scale_bias(+ thread T& out_scale,+ thread T& out_bias+ ) const {+ uint zero_point_value;+ T scale_value;+ int group_index;+ if constexpr (REDUCTION_DIMENSION == 0) {+ group_index = k_base / GROUP_SIZE;+ scale_value = scales_row_start[group_index];+ } else {+ group_index = static_cast<int>(scales - scales_row_start);+ scale_value = *scales;+ }+ if constexpr (BITS == 4) {+ const device uint8_t* zero_point_ptr =+ zero_points_row_start + (group_index >> 1);+ uint8_t zero_point_byte = *zero_point_ptr;+ zero_point_value =+ (uint(zero_point_byte) >> (uint(group_index & 1) * 4u)) & 0x0Fu;+ } else {+ zero_point_value = zero_points_row_start[group_index];+ }+ out_scale = scale_value;+ out_bias = static_cast<T>(-scale_value * static_cast<T>(zero_point_value));+ }++ void load_unsafe() const {+ if constexpr (TILE_HAS_IDLE_THREADS) {+ if (tile_row_index >= THREADGROUP_TILE_ROWS) {+ return;+ }+ }++ T scale;+ T bias;+ current_scale_bias(scale, bias);+ for (int i = 0; i < READS_PER_THREAD; i++) {+ dequantize<T, pack_factor, BITS>(+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );+ }+ }++ void load_safe(short2 src_tile_dim) const {+ if constexpr (TILE_HAS_IDLE_THREADS) {+ if (tile_row_index >= THREADGROUP_TILE_ROWS) {+ return;+ }+ }++ if constexpr (REDUCTION_DIMENSION == 1) {+ if (tile_row_index >= src_tile_dim.x) {+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {+ dst[i] = T(0);+ }+ return;+ }++ int valid_cols = src_tile_dim.y;+ int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;++ T scale;+ T bias;+ current_scale_bias(scale, bias);+ for (int i = 0; i < READS_PER_THREAD; i++) {+ int pack_index = tile_col_index + i;+ if (pack_index < valid_packs) {+ dequantize<T, pack_factor, BITS>(+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );++ if (pack_index == valid_packs - 1) {+ int remaining = valid_cols - pack_index * pack_factor;+ if (remaining < pack_factor) {+ for (int r = remaining; r < pack_factor; ++r) {+ dst[i * pack_factor + r] = T(0);+ }+ }+ }+ } else {+ for (int j = 0; j < pack_factor; ++j) {+ dst[i * pack_factor + j] = T(0);+ }+ }+ }+ return;+ } else {+ if (tile_row_index >= src_tile_dim.y) {+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {+ dst[i] = T(0);+ }+ return;+ }++ T scale;+ T bias;+ current_scale_bias(scale, bias);+ for (int i = 0; i < READS_PER_THREAD; i++) {+ dequantize<T, pack_factor, BITS>(+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );+ }+ }+ }++ void next() {+ src += tile_stride;+ if constexpr (REDUCTION_DIMENSION == 1) {+ if constexpr (GROUP_STEPS_PER_BLOCK > 1) {+ group_step_counter++;+ if (group_step_counter == GROUP_STEPS_PER_BLOCK) {+ group_step_counter = 0;+ scales++;+ }+ } else {+ scales++;+ }+ } else {+ k_base += THREADGROUP_TILE_ROWS;+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.hnew file mode 100644index 000000000..bdf8308be--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h@@ -0,0 +1,118 @@+#pragma once++#include <metal_stdlib>++#include "../../common/defines.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <typename U>+METAL_FUNC U uint_to_fp(uint32_t x) {+ return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);+}++template <>+METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {+ return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);+}++template <int BITS>+METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {+ static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");+ constexpr uint mask = (1u << BITS) - 1u;+ n &= uint4(mask);+ return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);+}++template <typename U, int BITS>+METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);++template <>+METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {+ return _uint4_to_fp4_float<4>(n);+}++template <>+METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {+ return _uint4_to_fp4_float<8>(n);+}++template <>+METAL_FUNC half4 uint4_to_fp4<half, 4>(uint4 n) {+ return half4(_uint4_to_fp4_float<4>(n));+}++template <>+METAL_FUNC half4 uint4_to_fp4<half, 8>(uint4 n) {+ return half4(_uint4_to_fp4_float<8>(n));+}++template <>+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {+ return bfloat4(_uint4_to_fp4_float<4>(n));+}++template <>+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {+ return bfloat4(_uint4_to_fp4_float<8>(n));+}++template <typename U, int N, int bits>+inline void dequantize(+ const device uint8_t* w,+ U scale,+ U bias,+ threadgroup U* w_local+) {+ static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");++ if (bits == 4) {+ U s0 = scale;+ U s1 = scale / static_cast<U>(16.0f);+ for (int i = 0; i < (N / 2); i++) {+ w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;+ w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;+ }+ } else if (bits == 8) {+ for (int i = 0; i < N; i++) {+ w_local[i] = scale * w[i] + bias;+ }+ }+}++template <>+inline void dequantize<bfloat, 8, 4>(+ const device uint8_t* w,+ bfloat scale,+ bfloat bias,+ threadgroup bfloat* w_local+) {+ const device uint32_t* w_ptr = reinterpret_cast<const device uint32_t*>(w);+ uint32_t packed = *w_ptr;++ bfloat4 v0, v1;++ v0.x = static_cast<bfloat>(packed & 0xF);+ v0.y = static_cast<bfloat>((packed >> 4) & 0xF);+ v0.z = static_cast<bfloat>((packed >> 8) & 0xF);+ v0.w = static_cast<bfloat>((packed >> 12) & 0xF);++ v1.x = static_cast<bfloat>((packed >> 16) & 0xF);+ v1.y = static_cast<bfloat>((packed >> 20) & 0xF);+ v1.z = static_cast<bfloat>((packed >> 24) & 0xF);+ v1.w = static_cast<bfloat>((packed >> 28) & 0xF);++ v0 = v0 * scale + bias;+ v1 = v1 * scale + bias;++ threadgroup bfloat4* out_ptr =+ reinterpret_cast<threadgroup bfloat4*>(w_local);+ out_ptr[0] = v0;+ out_ptr[1] = v1;+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 10fd83532..c860bc745 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -8,6 +8,10 @@#include "../../../generated/matmul.h"#include "../generated/gemm.h"#include "block_geometry.h"+#include "gemm_alignment.h"+#include "gemm_tiling.h"+#include "quant_scale_bias.h"+#include "quant_scale_zero_point.h"using namespace metal;@@ -16,13 +20,20 @@ namespace gemm {template <typename T,- int THREADGROUP_BLOCK_M,- int THREADGROUP_BLOCK_N,- int THREADGROUP_BLOCK_K,- int SIMDGROUPS_PER_ROW,- int SIMDGROUPS_PER_COLUMN,- bool TRANSPOSE_B>+ GemmTiling GEMM_TILING,+ bool TRANSPOSE_B,+ GemmWeightPrologueKind WEIGHT_PROLOGUE =+ GemmWeightPrologueKind::FullPrecision,+ int BITS = 0,+ int GROUP_SIZE = 0>struct SimdgroupMmaCore {+ METAL_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);+ METAL_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);+ METAL_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);+ METAL_CONST int SIMDGROUPS_PER_ROW =+ gemm_tiling_simdgroups_per_row(GEMM_TILING);+ METAL_CONST int SIMDGROUPS_PER_COLUMN =+ gemm_tiling_simdgroups_per_column(GEMM_TILING);METAL_CONST ushort PADDING_A = 16 / sizeof(T);METAL_CONST ushort PADDING_B = 16 / sizeof(T);METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;@@ -38,13 +49,31 @@ struct SimdgroupMmaCore {SHARED_STRIDE_A,true,THREADGROUP_THREADS>;- using BLoader = uzu::matmul::ThreadgroupLoader<+ using BLoaderFp = uzu::matmul::ThreadgroupLoader<T,TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N,SHARED_STRIDE_B,TRANSPOSE_B,THREADGROUP_THREADS>;+ using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<+ T,+ THREADGROUP_BLOCK_N,+ THREADGROUP_BLOCK_K,+ SHARED_STRIDE_B,+ 1,+ THREADGROUP_THREADS,+ GROUP_SIZE,+ BITS>;+ using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<+ T,+ THREADGROUP_BLOCK_N,+ THREADGROUP_BLOCK_K,+ SHARED_STRIDE_B,+ 1,+ THREADGROUP_THREADS,+ GROUP_SIZE,+ BITS>;using TileAccumulator = uzu::matmul::ThreadgroupTile<T,T,@@ -60,7 +89,7 @@ struct SimdgroupMmaCore {float,uzu::matmul::TransformNone<T, float>>;- template <bool M_aligned, bool N_aligned, bool K_aligned>+ template <uint GEMM_ALIGNMENT_RAW, typename BLoader>static METAL_FUNC void k_loop(threadgroup T* a_shared,threadgroup T* b_shared,@@ -72,6 +101,7 @@ struct SimdgroupMmaCore {thread const ushort& tile_block_cols,thread const ushort& leftover_block_depth) {+ constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};short2 tile_dimensions_a = short2(THREADGROUP_BLOCK_K, tile_block_rows);short2 tile_dimensions_b =TRANSPOSE_B ? short2(THREADGROUP_BLOCK_K, tile_block_cols)@@ -79,12 +109,12 @@ struct SimdgroupMmaCore {for (int k = 0; k < aligned_k_iterations; k++) {threadgroup_barrier(mem_flags::mem_threadgroup);- if constexpr (M_aligned) {+ if constexpr (gemm_alignment.contains(GemmAlignment::M)) {loader_a.load_unsafe();} else {loader_a.load_safe(tile_dimensions_a);}- if constexpr (N_aligned) {+ if constexpr (gemm_alignment.contains(GemmAlignment::N)) {loader_b.load_unsafe();} else {loader_b.load_safe(tile_dimensions_b);@@ -97,7 +127,7 @@ struct SimdgroupMmaCore {loader_b.next();}- if constexpr (!K_aligned) {+ if constexpr (!gemm_alignment.contains(GemmAlignment::K)) {threadgroup_barrier(mem_flags::mem_threadgroup);short2 last_tile_dimensions_a =@@ -114,13 +144,66 @@ struct SimdgroupMmaCore {}}+ template <uint GEMM_ALIGNMENT_RAW>+ static METAL_FUNC void finalize(+ thread TileAccumulator& accumulator,+ device T* d,+ const constant uzu::matmul::GemmParams* params,+ const thread ushort& tile_block_rows,+ const thread ushort& tile_block_cols,+ const bool needs_epilogue,+ const thread uzu::matmul::TransformScaleAccumulate<float, float>&+ epilogue,+ const device T* bias_block,+ const bool needs_bias+ ) {+ constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};+ if constexpr (+ gemm_alignment.contains(GemmAlignment::M) &&+ gemm_alignment.contains(GemmAlignment::N)+ ) {+ if (needs_epilogue) {+ accumulator.apply_epilogue(d, params->leading_dimension_d, 1, epilogue);+ }+ if (needs_bias) {+ accumulator.apply_bias(bias_block);+ }+ accumulator.store_result(d, params->leading_dimension_d);+ } else {+ if (needs_epilogue) {+ accumulator.apply_epilogue_safe(+ d,+ params->leading_dimension_d,+ 1,+ short2(tile_block_cols, tile_block_rows),+ epilogue+ );+ }+ if (needs_bias) {+ accumulator.apply_bias_safe(+ bias_block,+ short2(tile_block_cols, tile_block_rows)+ );+ }+ accumulator.store_result_safe(+ d,+ params->leading_dimension_d,+ short2(tile_block_cols, tile_block_rows)+ );+ }+ }+static METAL_FUNC void run(const device T* a,- const device T* b,+ const device uint8_t* b_packed,device T* d,const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,- GemmOutputTransformKind output_transform,+ GemmDTransform output_transform,+ const device T* scales,+ const device T* biases,+ const device uint8_t* zero_points,+ const device T* output_bias,threadgroup T* a_shared,threadgroup T* b_shared,const thread ThreadContext& thread_context@@ -139,89 +222,193 @@ struct SimdgroupMmaCore {const size_t block_col = size_t(geometry.block_col_start);a += block_row * params->leading_dimension_a;- b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;d += block_row * params->leading_dimension_d + block_col;thread ALoaderloader_a(a, params->leading_dimension_a, a_shared, thread_context);- thread BLoader- loader_b(b, params->leading_dimension_b, b_shared, thread_context);thread TileAccumulator accumulator(thread_context);const ushort tile_block_rows =min(THREADGROUP_BLOCK_M,- ((int)params->M) - int(geometry.block_row_start));+ static_cast<int>(params->M) -+ static_cast<int>(geometry.block_row_start));const ushort tile_block_cols =min(THREADGROUP_BLOCK_N,- ((int)params->N) - int(geometry.block_col_start));+ static_cast<int>(params->N) -+ static_cast<int>(geometry.block_col_start));const ushort leftover_block_depth =params->K - params->aligned_inner_iterations * THREADGROUP_BLOCK_K;- const bool needs_epilogue =- output_transform != GemmOutputTransformKind::Store;- const float alpha =- (output_transform == GemmOutputTransformKind::Scale ||- output_transform == GemmOutputTransformKind::ScaleAccumulate)- ? params->ab_scale- : 1.0f;- const float beta =- (output_transform == GemmOutputTransformKind::Accumulate ||- output_transform == GemmOutputTransformKind::ScaleAccumulate)- ? 1.0f- : 0.0f;+ const bool needs_scale = output_transform.contains(GemmDTransform::SCALE);+ const bool needs_accumulate =+ output_transform.contains(GemmDTransform::ACCUMULATE);+ const bool needs_bias = output_transform.contains(GemmDTransform::BIAS);+ const bool needs_epilogue = needs_scale || needs_accumulate;+ const float alpha = needs_scale ? params->ab_scale : 1.0f;+ const float beta = needs_accumulate ? 1.0f : 0.0f;uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);+ const device T* bias_block = output_bias + block_col;++ const bool all_aligned = ((alignment.contains(GemmAlignment::M)) ||+ (tile_block_rows == THREADGROUP_BLOCK_M)) &&+ ((alignment.contains(GemmAlignment::N)) ||+ (tile_block_cols == THREADGROUP_BLOCK_N)) &&+ alignment.contains(GemmAlignment::K);++ constexpr uint MASK_ALL = static_cast<uint>(GemmAlignment::M) |+ static_cast<uint>(GemmAlignment::N) |+ static_cast<uint>(GemmAlignment::K);++ const uint dynamic_alignment_mask =+ alignment.raw_value |+ ((tile_block_rows == THREADGROUP_BLOCK_M)+ ? static_cast<uint>(GemmAlignment::M)+ : 0u) |+ ((tile_block_cols == THREADGROUP_BLOCK_N)+ ? static_cast<uint>(GemmAlignment::N)+ : 0u);++ auto dispatch_kernel = [&](auto kernel_invoke) {+ if (all_aligned) {+ kernel_invoke(integral_constant<uint, MASK_ALL>{});+ } else {+ dispatch_gemm_alignment(dynamic_alignment_mask, kernel_invoke);+ }+ };- dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {- dispatch_bool(- alignment.contains(GemmAlignment::M) ||- (tile_block_rows == THREADGROUP_BLOCK_M),- [&](auto aligned_m) {- dispatch_bool(- alignment.contains(GemmAlignment::N) ||- (tile_block_cols == THREADGROUP_BLOCK_N),- [&](auto aligned_n) {- k_loop<aligned_m.value, aligned_n.value, aligned_k.value>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- accumulator,- tile_block_rows,- tile_block_cols,- leftover_block_depth- );- if constexpr (aligned_m.value && aligned_n.value) {- if (needs_epilogue) {- accumulator.apply_epilogue(- d,- params->leading_dimension_d,- 1,- epilogue- );- }- accumulator.store_result(d, params->leading_dimension_d);- } else {- if (needs_epilogue) {- accumulator.apply_epilogue_safe(- d,- params->leading_dimension_d,- 1,- short2(tile_block_cols, tile_block_rows),- epilogue- );- }- accumulator.store_result_safe(- d,- params->leading_dimension_d,- short2(tile_block_cols, tile_block_rows)- );- }- }- );- }+ if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision) {+ const device T* b = reinterpret_cast<const device T*>(b_packed);+ b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;+ thread BLoaderFp+ loader_b(b, params->leading_dimension_b, b_shared, thread_context);+ dispatch_kernel([&](auto gemm_alignment_mask) {+ constexpr uint mask = gemm_alignment_mask.value;+ k_loop<mask, BLoaderFp>(+ a_shared,+ b_shared,+ params->aligned_inner_iterations,+ loader_a,+ loader_b,+ accumulator,+ tile_block_rows,+ tile_block_cols,+ leftover_block_depth+ );+ finalize<mask>(+ accumulator,+ d,+ params,+ tile_block_rows,+ tile_block_cols,+ needs_epilogue,+ epilogue,+ bias_block,+ needs_bias+ );+ });+ return;+ }++ if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant) {+ constexpr int pack_factor = get_pack_factor<BITS, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();+ const int k_elements = int(params->K);+ const int weights_row_stride_bytes =+ k_elements * bytes_per_pack / pack_factor;+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;+ const device uint8_t* weights_block =+ b_packed + block_col * weights_row_stride_bytes;+ const device T* scales_offset = scales + block_col * groups_per_row;+ const device T* biases_offset = biases + block_col * groups_per_row;+ thread BLoaderScaleBias loader_b(+ weights_block,+ scales_offset,+ biases_offset,+ k_elements,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ dispatch_kernel([&](auto gemm_alignment_mask) {+ constexpr uint mask = gemm_alignment_mask.value;+ k_loop<mask, BLoaderScaleBias>(+ a_shared,+ b_shared,+ params->aligned_inner_iterations,+ loader_a,+ loader_b,+ accumulator,+ tile_block_rows,+ tile_block_cols,+ leftover_block_depth+ );+ finalize<mask>(+ accumulator,+ d,+ params,+ tile_block_rows,+ tile_block_cols,+ needs_epilogue,+ epilogue,+ bias_block,+ needs_bias+ );+ });+ return;+ }++ if constexpr (+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant+ ) {+ constexpr int pack_factor = get_pack_factor<BITS, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();+ const int k_elements = int(params->K);+ const int weights_row_stride_bytes =+ k_elements * bytes_per_pack / pack_factor;+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;+ const device uint8_t* weights_block =+ b_packed + block_col * weights_row_stride_bytes;+ const device T* scales_offset = scales + block_col * groups_per_row;+ const int zero_point_stride_per_row =+ (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;+ const device uint8_t* zero_points_row_start =+ zero_points + block_col * zero_point_stride_per_row;+ thread BLoaderScaleZeroPoint loader_b(+ weights_block,+ scales_offset,+ zero_points_row_start,+ k_elements,+ groups_per_row,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id);- });+ dispatch_kernel([&](auto gemm_alignment_mask) {+ constexpr uint mask = gemm_alignment_mask.value;+ k_loop<mask, BLoaderScaleZeroPoint>(+ a_shared,+ b_shared,+ params->aligned_inner_iterations,+ loader_a,+ loader_b,+ accumulator,+ tile_block_rows,+ tile_block_cols,+ leftover_block_depth+ );+ finalize<mask>(+ accumulator,+ d,+ params,+ tile_block_rows,+ tile_block_cols,+ needs_epilogue,+ epilogue,+ bias_block,+ needs_bias+ );+ });+ }}};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rsdeleted file mode 100644index 7b24036a7..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs+++ /dev/null@@ -1,251 +0,0 @@-use crate::{- DataType,- backends::{- common::{- Allocation, AsBufferRangeRef, Backend, Buffer,- gpu_types::{- GemmParams, QuantizationMode,- gemm::{- GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig,- GemmWeightPrologueKind,- },- },- },- metal::Metal,- },-};--pub struct GemmAlignmentAxes {- pub m: bool,- pub n: bool,- pub k: bool,-}--impl GemmAlignment {- pub fn from_axes(axes: GemmAlignmentAxes) -> Self {- let mut bits = Self::empty();- bits.set(Self::M, axes.m);- bits.set(Self::N, axes.n);- bits.set(Self::K, axes.k);- bits- }-}--pub enum GemmWeights<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>> = Allocation<B>> {- FullPrecision {- weights: &'a T,- },- ScaleBias {- weights: &'a T,- scales: &'a Allocation<B>,- biases: &'a Allocation<B>,- mode: QuantizationMode,- group_size: u32,- },- ScaleZeroPoint {- weights: &'a T,- scales: &'a Allocation<B>,- zero_points: &'a Allocation<B>,- mode: QuantizationMode,- group_size: u32,- },-}--impl<B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> GemmWeights<'_, B, T> {- pub fn weight_prologue(&self) -> GemmWeightPrologueKind {- match self {- Self::FullPrecision {- ..- } => GemmWeightPrologueKind::FullPrecision,- Self::ScaleBias {- ..- } => GemmWeightPrologueKind::ScaleBiasDequant,- Self::ScaleZeroPoint {- ..- } => GemmWeightPrologueKind::ScaleZeroPointDequant,- }- }-- pub fn bits_per_weight(&self) -> u32 {- match self {- Self::FullPrecision {- ..- } => 0,- Self::ScaleBias {- mode,- ..- }- | Self::ScaleZeroPoint {- mode,- ..- } => DataType::from(*mode).size_in_bits() as u32,- }- }-- pub fn group_size(&self) -> u32 {- match self {- Self::FullPrecision {- ..- } => 0,- Self::ScaleBias {- group_size,- ..- }- | Self::ScaleZeroPoint {- group_size,- ..- } => *group_size,- }- }-}--pub struct GemmDispatch<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> {- pub tiling_config: GemmTilingConfig,- pub input_prologue: GemmInputPrologueKind,- pub use_mxu: bool,- pub output_transform: GemmOutputTransformKind,- pub alignment: GemmAlignment,- pub transpose_b: bool,- pub a: &'a Allocation<B>,- pub a_offset: usize,- pub b: GemmWeights<'a, B, T>,- pub b_offset: usize,- pub d: &'a mut Allocation<B>,- pub params: GemmParams,- pub group_count_x: u32,- pub group_count_y: u32,-}--impl<B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> GemmDispatch<'_, B, T> {- pub(crate) fn specialization(&self) -> GemmSpecialization {- GemmSpecialization {- tiling_config: self.tiling_config,- input_prologue: self.input_prologue,- use_mxu: self.use_mxu,- output_transform: self.output_transform,- alignment: self.alignment,- transpose_b: self.transpose_b,- weight_prologue: self.b.weight_prologue(),- bits_per_weight: self.b.bits_per_weight(),- group_size: self.b.group_size(),- }- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub(crate) struct GemmSpecialization {- pub(crate) tiling_config: GemmTilingConfig,- pub(crate) input_prologue: GemmInputPrologueKind,- pub(crate) use_mxu: bool,- pub(crate) output_transform: GemmOutputTransformKind,- pub(crate) alignment: GemmAlignment,- pub(crate) transpose_b: bool,- pub(crate) weight_prologue: GemmWeightPrologueKind,- pub(crate) bits_per_weight: u32,- pub(crate) group_size: u32,-}--impl GemmSpecialization {- pub(crate) fn validate(&self) -> Result<(), GemmSpecializationError> {- if self.group_size != 0 && self.tiling_config.threadgroup_k > self.group_size {- return Err(GemmSpecializationError::ThreadgroupKExceedsGroupSize {- threadgroup_k: self.tiling_config.threadgroup_k,- group_size: self.group_size,- });- }- Ok(())- }-- pub(crate) fn precompile_configs(data_type: DataType) -> Vec<Self> {- let mut out = Vec::new();- for &(threadgroup_m, threadgroup_n, threadgroup_k) in simdgroup_tile_set(data_type) {- for align_mn in [true, false] {- let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {- m: align_mn,- n: align_mn,- k: true,- });- out.push(Self {- tiling_config: GemmTilingConfig {- threadgroup_m,- threadgroup_n,- threadgroup_k,- simdgroups_m: 2,- simdgroups_n: 2,- },- input_prologue: GemmInputPrologueKind::FullPrecision,- use_mxu: false,- output_transform: GemmOutputTransformKind::Store,- alignment,- transpose_b: true,- weight_prologue: GemmWeightPrologueKind::FullPrecision,- bits_per_weight: 0,- group_size: 0,- });- }- }-- for &(threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) in mxu_tile_set(data_type) {- for align_m in [true, false] {- for align_n in [true, false] {- for align_k in [true, false] {- let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {- m: align_m,- n: align_n,- k: align_k,- });- for output_transform in [- GemmOutputTransformKind::Store,- GemmOutputTransformKind::Scale,- GemmOutputTransformKind::Accumulate,- GemmOutputTransformKind::ScaleAccumulate,- ] {- out.push(Self {- tiling_config: GemmTilingConfig {- threadgroup_m,- threadgroup_n,- threadgroup_k: 32,- simdgroups_m,- simdgroups_n,- },- input_prologue: GemmInputPrologueKind::FullPrecision,- use_mxu: true,- output_transform,- alignment,- transpose_b: true,- weight_prologue: GemmWeightPrologueKind::FullPrecision,- bits_per_weight: 0,- group_size: 0,- });- }- }- }- }- }- out- }-}--fn simdgroup_tile_set(data_type: DataType) -> &'static [(u32, u32, u32)] {- match data_type {- DataType::BF16 => &[(64, 32, 32), (64, 64, 16)],- DataType::F16 => &[(64, 64, 16), (64, 32, 32)],- DataType::F32 => &[(32, 64, 16)],- _ => &[],- }-}--fn mxu_tile_set(data_type: DataType) -> &'static [(u32, u32, u32, u32)] {- if !matches!(data_type, DataType::F16 | DataType::BF16) {- return &[];- }- &[(64, 64, 2, 2), (32, 64, 2, 2), (64, 32, 4, 1), (128, 128, 4, 4)]-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum GemmSpecializationError {- ThreadgroupKExceedsGroupSize {- threadgroup_k: u32,- group_size: u32,- },-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rsnew file mode 100644index 000000000..5663f7268--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs@@ -0,0 +1,14 @@+use thiserror::Error;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]+pub enum GemmSpecializationError {+ #[error("threadgroup K={threadgroup_k} exceeds group size {group_size}")]+ ThreadgroupKExceedsGroupSize {+ threadgroup_k: u32,+ group_size: u32,+ },+ #[error("quantized B requires the simdgroup path")]+ QuantizedRequiresSimdgroup,+ #[error("quantized B requires transposed layout")]+ QuantizedRequiresTransposedB,+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rsdeleted file mode 100644index 7ea4fb243..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs+++ /dev/null@@ -1,180 +0,0 @@-use super::{GemmAlignmentAxes, GemmDispatch, GemmKernel, GemmWeights, MXU_THREADGROUP_BLOCK_K};-use crate::{- DataType,- backends::{- common::{- AsBufferRangeRef, Buffer, Encoder,- gpu_types::{- GemmParams,- gemm::{GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig},- },- kernel::{- TensorAddBiasKernel,- matmul::{MatmulArgumentC, MatmulArguments, MatmulError},- },- },- metal::{Metal, context::MetalContext, kernel::TensorAddBiasMetalKernel, metal_extensions::DeviceExt},- },-};--pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(- gemm: &mut GemmKernel,- bias_add: &mut TensorAddBiasMetalKernel,- data_type: DataType,- context: &MetalContext,- encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal, TB>,- use_mxu: bool,-) -> Result<(), MatmulError<Metal>> {- if use_mxu && !context.device.supports_mxu() {- return Err(MatmulError::UnsupportedDataType(data_type));- }-- let tile = if use_mxu {- select_mxu_tile(&arguments)- } else {- select_simdgroup_tile(data_type, &arguments)- };- let k_block = if use_mxu {- MXU_THREADGROUP_BLOCK_K- } else {- tile.threadgroup_k- };-- let threadgroups_per_row = arguments.output_dim.div_ceil(tile.threadgroup_n);- let threadgroups_per_column = arguments.batch_dim.div_ceil(tile.threadgroup_m);-- let (use_morton, group_count_x, group_count_y) = if use_mxu {- let max_dim = threadgroups_per_row.max(threadgroups_per_column);- let min_dim = threadgroups_per_row.min(threadgroups_per_column);- let morton_dim = max_dim.next_power_of_two();- let morton_total = morton_dim.saturating_mul(morton_dim);- let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);- let use_morton = min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);- if use_morton {- (true, morton_total, 1)- } else {- (false, threadgroups_per_row, threadgroups_per_column)- }- } else {- (false, threadgroups_per_row, threadgroups_per_column)- };-- let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {- m: arguments.batch_dim % tile.threadgroup_m == 0,- n: arguments.output_dim % tile.threadgroup_n == 0,- k: arguments.input_dim % k_block == 0,- });- let output_transform = output_transform_from(&arguments);-- let default_ldb = if arguments.b_transpose {- arguments.input_dim- } else {- arguments.output_dim- };- let params = GemmParams {- M: arguments.batch_dim,- N: arguments.output_dim,- K: arguments.input_dim,- leading_dimension_a: arguments.input_dim,- leading_dimension_b: arguments.b_leading_dimension.unwrap_or(default_ldb),- leading_dimension_d: arguments.output_dim,- threadgroups_per_row,- threadgroups_per_column,- aligned_inner_iterations: arguments.input_dim / k_block,- use_morton,- ab_scale: arguments.ab_scale,- };-- let dispatch = GemmDispatch {- tiling_config: tile,- input_prologue: GemmInputPrologueKind::FullPrecision,- use_mxu,- output_transform,- alignment,- transpose_b: arguments.b_transpose,- a: arguments.a,- a_offset: arguments.a_offset,- b: GemmWeights::FullPrecision {- weights: arguments.b,- },- b_offset: arguments.b_offset,- d: arguments.d,- params,- group_count_x,- group_count_y,- };-- gemm.encode(context, dispatch, encoder).map_err(MatmulError::BackendError)?;-- if let MatmulArgumentC::Bias(bias) = arguments.c {- bias_add.encode(- None::<&<Metal as crate::backends::common::Backend>::DenseBuffer>,- bias,- arguments.d,- arguments.output_dim,- arguments.batch_dim * arguments.output_dim,- encoder,- );- }-- Ok(())-}--fn select_simdgroup_tile<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(- data_type: DataType,- arguments: &MatmulArguments<Metal, TB>,-) -> GemmTilingConfig {- let (threadgroup_m, threadgroup_n, threadgroup_k) = match data_type {- DataType::F32 => (32u32, 64u32, 16u32),- _ => {- if 2 * arguments.batch_dim.max(arguments.output_dim) > arguments.input_dim {- (64, 64, 16)- } else {- (64, 32, 32)- }- },- };- GemmTilingConfig {- threadgroup_m,- threadgroup_n,- threadgroup_k,- simdgroups_m: 2,- simdgroups_n: 2,- }-}--fn select_mxu_tile<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(- arguments: &MatmulArguments<Metal, TB>-) -> GemmTilingConfig {- let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) =- if arguments.batch_dim >= 256 && arguments.output_dim >= 128 {- (128u32, 128u32, 4u32, 4u32)- } else if arguments.output_dim < 64 {- (64u32, 32u32, 4u32, 1u32)- } else if arguments.batch_dim < 64 {- (32u32, 64u32, 2u32, 2u32)- } else {- (64u32, 64u32, 2u32, 2u32)- };- GemmTilingConfig {- threadgroup_m,- threadgroup_n,- threadgroup_k: 32,- simdgroups_m,- simdgroups_n,- }-}--fn output_transform_from<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(- arguments: &MatmulArguments<Metal, TB>-) -> GemmOutputTransformKind {- let scale = arguments.ab_scale != 1.0;- let accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);- match (scale, accumulate) {- (false, false) => GemmOutputTransformKind::Store,- (true, false) => GemmOutputTransformKind::Scale,- (false, true) => GemmOutputTransformKind::Accumulate,- (true, true) => GemmOutputTransformKind::ScaleAccumulate,- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 00ad9e2d7..563af413d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -3,102 +3,138 @@#include "../../common/thread_context.h"#include "../generated/gemm.h"+#include "common/gemm_tiling.h"#include "common/mxu_mma_core.h"#include "common/simdgroup_mma_core.h"using namespace metal;using namespace uzu::gemm;-#define GEMM_MAX_THREADGROUP_A 2560-#define GEMM_MAX_THREADGROUP_B 1536+#define GEMM_TGA_ELEMENTS \+ (USE_MXU ? 1 \+ : (gemm_tiling_block_m(GEMM_TILING) * \+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(T)))))+#define GEMM_TGB_ELEMENTS \+ (USE_MXU ? 1 \+ : (gemm_tiling_block_n(GEMM_TILING) * \+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(T)))))template <typename T,- uint THREADGROUP_BLOCK_M,- uint THREADGROUP_BLOCK_N,- uint THREADGROUP_BLOCK_K,- uint SIMDGROUPS_M,- uint SIMDGROUPS_N,+ GemmTiling GEMM_TILING,bool TRANSPOSE_B,- bool USE_MXU>-VARIANTS(T, float, half, bfloat)-VARIANTS(THREADGROUP_BLOCK_M, 32, 64, 128)-VARIANTS(THREADGROUP_BLOCK_N, 32, 64, 128)-VARIANTS(THREADGROUP_BLOCK_K, 16, 32)-VARIANTS(SIMDGROUPS_M, 1, 2, 4)-VARIANTS(SIMDGROUPS_N, 1, 2, 4)+ bool USE_MXU,+ GemmWeightPrologueKind WEIGHT_PROLOGUE,+ uint BITS,+ uint GROUP_SIZE>+VARIANTS(T, half, bfloat)+VARIANTS(+ GEMM_TILING,+ GemmTiling::T8x32x32_1x1,+ GemmTiling::T64x32x32_2x2,+ GemmTiling::T64x64x16_2x2,+ GemmTiling::T64x64x32_2x2,+ GemmTiling::T64x64x64_2x2,+ GemmTiling::T32x32x32_2x2,+ GemmTiling::T32x64x32_2x2,+ GemmTiling::T64x32x32_4x1,+ GemmTiling::T128x128x32_4x4)VARIANTS(TRANSPOSE_B, false, true)VARIANTS(USE_MXU, false, true)-CONSTRAINT(!USE_MXU || THREADGROUP_BLOCK_M % SIMDGROUPS_M == 0)-CONSTRAINT(!USE_MXU || THREADGROUP_BLOCK_N % SIMDGROUPS_N == 0)-CONSTRAINT(!USE_MXU || (THREADGROUP_BLOCK_M / SIMDGROUPS_M) % 16 == 0)-CONSTRAINT(!USE_MXU || (THREADGROUP_BLOCK_N / SIMDGROUPS_N) % 16 == 0)-CONSTRAINT(USE_MXU || max(THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N) <= 32 * SIMDGROUPS_M * SIMDGROUPS_N)+VARIANTS(+ WEIGHT_PROLOGUE,+ GemmWeightPrologueKind::FullPrecision,+ GemmWeightPrologueKind::ScaleBiasDequant,+ GemmWeightPrologueKind::ScaleZeroPointDequant)+VARIANTS(BITS, 0, 4, 8)+VARIANTS(GROUP_SIZE, 0, 32, 64, 128)+CONSTRAINT(+ !USE_MXU || GEMM_TILING == GemmTiling::T64x64x32_2x2 ||+ GEMM_TILING == GemmTiling::T32x64x32_2x2 ||+ GEMM_TILING == GemmTiling::T64x32x32_4x1 ||+ GEMM_TILING == GemmTiling::T128x128x32_4x4)+CONSTRAINT(+ WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision ||+ (BITS == 0 && GROUP_SIZE == 0))+CONSTRAINT(+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||+ (BITS != 0 && GROUP_SIZE != 0))+CONSTRAINT(+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||+ (!USE_MXU && TRANSPOSE_B &&+ (GEMM_TILING == GemmTiling::T8x32x32_1x1 ||+ GEMM_TILING == GemmTiling::T32x32x32_2x2 ||+ GEMM_TILING == GemmTiling::T64x32x32_2x2 ||+ GEMM_TILING == GemmTiling::T64x64x32_2x2 ||+ GEMM_TILING == GemmTiling::T64x64x64_2x2)))+CONSTRAINT(+ GEMM_TILING != GemmTiling::T64x64x64_2x2 ||+ GROUP_SIZE == 64 || GROUP_SIZE == 128)KERNEL(Gemm)(const device T* a,const device uint8_t* b_packed,device T* d,const device T* scales- OPTIONAL(weight_prologue != GemmWeightPrologueKind::FullPrecision),+ OPTIONAL(WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision),const device T* biases- OPTIONAL(weight_prologue == GemmWeightPrologueKind::ScaleBiasDequant),+ OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant),const device uint8_t* zero_points- OPTIONAL(weight_prologue == GemmWeightPrologueKind::ScaleZeroPointDequant),+ OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant),+ const device T* output_bias+ OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),const constant uzu::matmul::GemmParams* params,const constant uint& group_count_x,const constant uint& group_count_y,- const GemmInputPrologueKind input_prologue SPECIALIZE,- const GemmWeightPrologueKind weight_prologue SPECIALIZE,- const GemmOutputTransformKind output_transform SPECIALIZE,+ const GemmDTransform output_transform SPECIALIZE,const GemmAlignment alignment SPECIALIZE,- const uint bits_per_weight SPECIALIZE,- const uint group_size SPECIALIZE,- threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],- threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],+ threadgroup T a_shared[GEMM_TGA_ELEMENTS],+ threadgroup T b_shared[GEMM_TGB_ELEMENTS],const uint group_x GROUPS(group_count_x),const uint group_y GROUPS(group_count_y),const uint thread_x THREADS(METAL_SIMD_SIZE),- const uint thread_y THREADS(SIMDGROUPS_N),- const uint thread_z THREADS(SIMDGROUPS_M),+ const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),+ const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),const ThreadContext thread_context) {- (void)scales;- (void)biases;- (void)zero_points;- (void)bits_per_weight;- (void)group_size;(void)group_x;(void)group_y;(void)thread_x;(void)thread_y;(void)thread_z;- (void)input_prologue;- (void)weight_prologue;- const device T* b = reinterpret_cast<const device T*>(b_packed);+if constexpr (USE_MXU) {- MxuMmaCore<- T,- THREADGROUP_BLOCK_M,- THREADGROUP_BLOCK_N,- SIMDGROUPS_M,- SIMDGROUPS_N,- TRANSPOSE_B>::- run(a, b, d, params, alignment, output_transform, thread_context);+ (void)scales;+ (void)biases;+ (void)zero_points;+ const device T* b = reinterpret_cast<const device T*>(b_packed);+ MxuMmaCore<T, GEMM_TILING, TRANSPOSE_B>::run(+ a,+ b,+ d,+ params,+ alignment,+ output_transform,+ output_bias,+ thread_context+ );} else {SimdgroupMmaCore<T,- THREADGROUP_BLOCK_M,- THREADGROUP_BLOCK_N,- THREADGROUP_BLOCK_K,- SIMDGROUPS_M,- SIMDGROUPS_N,- TRANSPOSE_B>::+ GEMM_TILING,+ TRANSPOSE_B,+ WEIGHT_PROLOGUE,+ BITS,+ GROUP_SIZE>::run(a,- b,+ b_packed,d,params,alignment,output_transform,+ scales,+ biases,+ zero_points,+ output_bias,a_shared,b_shared,thread_context);diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 5f00d0280..cc5650f78 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -1,20 +1,41 @@use std::collections::{HashMap, hash_map::Entry};-use super::dispatch::{GemmDispatch, GemmSpecialization, GemmWeights};+use super::{MXU_THREADGROUP_BLOCK_K, specialization::GemmSpecialization};use crate::{DataType,backends::{common::{- AsBufferRangeRef, Buffer, Encoder,- gpu_types::gemm::{GemmInputPrologueKind, GemmWeightPrologueKind},+ Allocation, AsBufferRangeRef, Backend, Buffer, Encoder,+ gpu_types::{+ GemmParams,+ gemm::{GemmAlignment, GemmDTransform, GemmTiling},+ },+ kernel::{+ HadamardTransformKernel, Kernels, TensorAddBiasKernel,+ matmul::{MatmulArguments, MatmulB, MatmulError, MatmulQuantCombo},+ },+ },+ metal::{+ Metal,+ context::MetalContext,+ error::MetalError,+ kernel::{GemmMetalKernel, TensorAddBiasMetalKernel},+ metal_extensions::DeviceExt,},- metal::{Metal, context::MetalContext, error::MetalError, kernel::GemmMetalKernel},},};-pub(crate) struct GemmKernel {+#[derive(Debug, Clone, Copy)]+pub enum GemmDispatchPath {+ Simdgroup,+ Mxu,+}++pub struct GemmKernel {data_type: DataType,kernels: HashMap<GemmSpecialization, GemmMetalKernel>,+ pub bias_add: TensorAddBiasMetalKernel,+ pub hadamard: <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel,}impl GemmKernel {@@ -22,9 +43,13 @@ impl GemmKernel {context: &MetalContext,data_type: DataType,) -> Result<Self, MetalError> {+ let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true)?;+ let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)?;let mut kernel = Self {data_type,kernels: HashMap::new(),+ bias_add,+ hadamard,};for specialization in GemmSpecialization::precompile_configs(data_type) {kernel.get_or_create(context, specialization)?;@@ -32,6 +57,17 @@ impl GemmKernel {Ok(kernel)}+ pub fn preheat_quant_combo(+ &mut self,+ context: &MetalContext,+ combo: MatmulQuantCombo,+ ) -> Result<(), MetalError> {+ for specialization in GemmSpecialization::quant_combo_specs(self.data_type, combo) {+ self.get_or_create(context, specialization)?;+ }+ Ok(())+ }+fn get_or_create(&mut self,context: &MetalContext,@@ -42,74 +78,327 @@ impl GemmKernel {Entry::Vacant(entry) => {let kernel = GemmMetalKernel::new(context,- self.data_type,- specialization.tiling_config.threadgroup_m,- specialization.tiling_config.threadgroup_n,- specialization.tiling_config.threadgroup_k,- specialization.tiling_config.simdgroups_m,- specialization.tiling_config.simdgroups_n,+ specialization.data_type,+ specialization.tiling,specialization.transpose_b,specialization.use_mxu,- specialization.input_prologue,specialization.weight_prologue,+ specialization.bits_per_weight.unwrap_or(0),+ specialization.group_size.unwrap_or(0),specialization.output_transform,specialization.alignment,- specialization.bits_per_weight,- specialization.group_size,)?;Ok(entry.insert(kernel))},}}- pub(crate) fn encode<T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+ pub fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(&mut self,- context: &MetalContext,- dispatch: GemmDispatch<'_, Metal, T>,+ arguments: MatmulArguments<'a, Metal, TB>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {- let specialization = dispatch.specialization();- assert_eq!(- specialization.input_prologue,- GemmInputPrologueKind::FullPrecision,- "unified GEMM only implements FullPrecision input prologue today",- );- assert_eq!(- specialization.weight_prologue,- GemmWeightPrologueKind::FullPrecision,- "unified GEMM only implements FullPrecision weight prologue today",- );- specialization.validate().map_err(MetalError::InvalidGemmSpecialization)?;- let kernel = self.get_or_create(context, specialization)?;- let (b, scales, biases, zero_points) = match &dispatch.b {- GemmWeights::FullPrecision {- weights,- } => (*weights, None, None, None),- GemmWeights::ScaleBias {- weights,- scales,- biases,+ let path = if encoder.context().device.supports_mxu()+ && matches!(self.data_type, DataType::F16 | DataType::BF16)+ && matches!(arguments.b, MatmulB::FullPrecision { .. })+ {+ GemmDispatchPath::Mxu+ } else {+ GemmDispatchPath::Simdgroup+ };+ self.encode_dispatch_path(arguments, path, encoder)+ }++ pub fn encode_dispatch_path<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+ &mut self,+ arguments: MatmulArguments<'a, Metal, TB>,+ path: GemmDispatchPath,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), MetalError> {+ let use_mxu = match path {+ GemmDispatchPath::Mxu => {+ assert!(+ encoder.context().device.supports_mxu(),+ "GemmDispatchPath::Mxu requested on hardware without MXU support",+ );+ assert!(+ matches!(self.data_type, DataType::F16 | DataType::BF16),+ "GemmDispatchPath::Mxu requires F16 or BF16 data type, got {:?}",+ self.data_type,+ );+ assert!(+ matches!(arguments.b, MatmulB::FullPrecision { .. }),+ "GemmDispatchPath::Mxu requires FullPrecision B",+ );+ true+ },+ GemmDispatchPath::Simdgroup => false,+ };++ let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });+ if is_quant {+ if arguments.d_transform.mask().contains(GemmDTransform::ACCUMULATE) {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::ACCUMULATE,+ path: "QuantGemm",+ }+ .into());+ }+ if !arguments.b_transpose || arguments.b_leading_dimension.is_some() || arguments.b_offset != 0 {+ return Err(MatmulError::UnsupportedLayout {+ path: "QuantGemm",+ }+ .into());+ }+ }++ let ab_scale = arguments.d_transform.ab_scale;+ let post_bias = if use_mxu {+ arguments.d_transform.bias+ } else {+ None+ };+ let output_bias = if use_mxu {+ None+ } else {+ arguments.d_transform.bias+ };+ let post_rht = arguments.d_transform.rht_factors;+ let mut output_transform = arguments.d_transform.mask();+ output_transform.remove(GemmDTransform::RHT);+ if use_mxu {+ output_transform.remove(GemmDTransform::BIAS);+ }++ let weight_prologue = arguments.b.weight_prologue();+ let bits_per_weight = arguments.b.bits_per_weight();+ let group_size = arguments.b.group_size();++ let MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ d,+ m,+ n,+ k,+ ..+ } = arguments;++ match b {+ MatmulB::FullPrecision {+ b: weights,+ } => {+ let tiling = if use_mxu {+ select_mxu_tiling(m, n)+ } else {+ select_simdgroup_tiling(m, n, k)+ };+ let k_block = if use_mxu {+ MXU_THREADGROUP_BLOCK_K+ } else {+ tiling.block_k()+ };++ let threadgroups_per_row = n.div_ceil(tiling.block_n());+ let threadgroups_per_column = m.div_ceil(tiling.block_m());++ let (use_morton, group_count_x, group_count_y) = if use_mxu {+ let max_dim = threadgroups_per_row.max(threadgroups_per_column);+ let min_dim = threadgroups_per_row.min(threadgroups_per_column);+ let morton_dim = max_dim.next_power_of_two();+ let morton_total = morton_dim.saturating_mul(morton_dim);+ let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);+ let use_morton = min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);+ if use_morton {+ (true, morton_total, 1)+ } else {+ (false, threadgroups_per_row, threadgroups_per_column)+ }+ } else {+ (false, threadgroups_per_row, threadgroups_per_column)+ };++ let alignment =+ GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % k_block == 0);++ let default_ldb = if b_transpose {+ k+ } else {+ n+ };+ let params = GemmParams {+ M: m,+ N: n,+ K: k,+ leading_dimension_a: k,+ leading_dimension_b: b_leading_dimension.unwrap_or(default_ldb),+ leading_dimension_d: n,+ threadgroups_per_row,+ threadgroups_per_column,+ aligned_inner_iterations: k / k_block,+ use_morton,+ ab_scale,+ };++ let specialization = GemmSpecialization {+ data_type: self.data_type,+ tiling,+ use_mxu,+ output_transform,+ alignment,+ transpose_b: b_transpose,+ weight_prologue,+ bits_per_weight,+ group_size,+ };+ specialization.validate().map_err(MetalError::from)?;+ let kernel = self.get_or_create(encoder.context(), specialization)?;+ kernel.encode(+ (a, a_offset),+ (weights, b_offset),+ &mut *d,+ None::<&Allocation<Metal>>,+ None::<&Allocation<Metal>>,+ None::<&Allocation<Metal>>,+ output_bias,+ std::slice::from_ref(¶ms),+ group_count_x,+ group_count_y,+ encoder,+ );+ },+ quant_b @ (MatmulB::ScaleBiasDequant {..- } => (*weights, Some(*scales), Some(*biases), None),- GemmWeights::ScaleZeroPoint {- weights,- scales,- zero_points,+ }+ | MatmulB::ScaleZeroPointDequant {..- } => (*weights, Some(*scales), None, Some(*zero_points)),- };- kernel.encode(- (dispatch.a, dispatch.a_offset),- (b, dispatch.b_offset),- dispatch.d,- scales,- biases,- zero_points,- std::slice::from_ref(&dispatch.params),- dispatch.group_count_x,- dispatch.group_count_y,- encoder,- );+ }) => {+ let (weights, scales, biases, zero_points) = match quant_b {+ MatmulB::ScaleBiasDequant {+ b: w,+ scales,+ biases,+ ..+ } => (w, Some(scales), Some(biases), None),+ MatmulB::ScaleZeroPointDequant {+ b: w,+ scales,+ zero_points,+ ..+ } => (w, Some(scales), None, Some(zero_points)),+ _ => unreachable!(),+ };++ let tiling = select_quant_tiling(m, n);+ let alignment =+ GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);+ let params = quant_params(m, n, k, tiling, ab_scale);+ let group_count_x = n.div_ceil(tiling.block_n());+ let group_count_y = m.div_ceil(tiling.block_m());++ let specialization = GemmSpecialization {+ data_type: self.data_type,+ tiling,+ use_mxu,+ output_transform,+ alignment,+ transpose_b: true,+ weight_prologue,+ bits_per_weight,+ group_size,+ };+ specialization.validate().map_err(MetalError::from)?;+ let kernel = self.get_or_create(encoder.context(), specialization)?;+ kernel.encode(+ (a, a_offset),+ (weights, b_offset),+ &mut *d,+ scales,+ biases,+ zero_points,+ output_bias,+ std::slice::from_ref(¶ms),+ group_count_x,+ group_count_y,+ encoder,+ );+ },+ }++ if let Some(bias) = post_bias {+ self.bias_add.encode(None::<&<Metal as Backend>::DenseBuffer>, bias, &mut *d, n, m * n, encoder);+ }+ if let Some(factors) = post_rht {+ self.hadamard.encode(d, factors, n, m, encoder);+ }Ok(())}}++fn quant_params(+ m: u32,+ n: u32,+ k: u32,+ tiling: GemmTiling,+ ab_scale: f32,+) -> GemmParams {+ GemmParams {+ M: m,+ N: n,+ K: k,+ leading_dimension_a: k,+ leading_dimension_b: k,+ leading_dimension_d: n,+ threadgroups_per_row: n.div_ceil(tiling.block_n()),+ threadgroups_per_column: m.div_ceil(tiling.block_m()),+ aligned_inner_iterations: k / tiling.block_k(),+ use_morton: false,+ ab_scale,+ }+}++fn select_simdgroup_tiling(+ m: u32,+ n: u32,+ k: u32,+) -> GemmTiling {+ if 2 * m.max(n) > k {+ GemmTiling::T64x64x16_2x2+ } else {+ GemmTiling::T64x32x32_2x2+ }+}++fn select_mxu_tiling(+ m: u32,+ n: u32,+) -> GemmTiling {+ if m >= 256 && n >= 128 {+ GemmTiling::T128x128x32_4x4+ } else if n < 64 {+ GemmTiling::T64x32x32_4x1+ } else if m < 64 {+ GemmTiling::T32x64x32_2x2+ } else {+ GemmTiling::T64x64x32_2x2+ }+}++fn select_quant_tiling(+ m: u32,+ n: u32,+) -> GemmTiling {+ if m < 32 {+ GemmTiling::T8x32x32_1x1+ } else if m >= 64 && n <= 2048 {+ GemmTiling::T64x32x32_2x2+ } else if m >= 64 && n >= 6144 && n % 64 == 0 {+ GemmTiling::T64x64x32_2x2+ } else {+ GemmTiling::T32x32x32_2x2+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rsindex 3dc90617f..05ecef5a5 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -1,13 +1,9 @@-mod dispatch;-pub(crate) mod fp;+mod error;mod kernel;-pub(crate) mod quant;+mod specialization;-pub use dispatch::GemmSpecializationError;-pub(crate) use dispatch::{GemmAlignmentAxes, GemmDispatch, GemmWeights};-pub(crate) use kernel::GemmKernel;--pub(crate) use crate::backends::common::gpu_types::gemm::GemmInputPrologueKind;+pub use error::GemmSpecializationError;+pub use kernel::{GemmDispatchPath, GemmKernel};// Matches `MxuMmaCore::THREADGROUP_BLOCK_K` in `common/mxu_mma_core.h`.pub(crate) const MXU_THREADGROUP_BLOCK_K: u32 = 256;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rsdeleted file mode 100644index 54617b5c1..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs+++ /dev/null@@ -1,96 +0,0 @@-use super::{GemmAlignmentAxes, GemmDispatch, GemmInputPrologueKind, GemmKernel, GemmWeights};-use crate::backends::{- common::{- Encoder,- gpu_types::{- GemmParams, QuantizationMethod,- gemm::{GemmAlignment, GemmOutputTransformKind, GemmTilingConfig},- },- kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError},- },- metal::{Metal, context::MetalContext},-};--pub(crate) fn encode(- gemm: &mut GemmKernel,- context: &MetalContext,- configuration: &QuantizedMatmulConfiguration,- arguments: QuantizedMatmulArguments<Metal>,- encoder: &mut Encoder<Metal>,-) -> Result<(), QuantizedMatmulError<Metal>> {- let tile = select_tile(configuration, arguments.batch_dim as u32);- let group_size = configuration.group_size as u32;- let batch_dim = arguments.batch_dim as u32;- let input_dim = configuration.input_dim as u32;- let output_dim = configuration.output_dim as u32;- let threadgroups_per_row = output_dim.div_ceil(tile.threadgroup_n);- let threadgroups_per_column = batch_dim.div_ceil(tile.threadgroup_m);- let dispatch = GemmDispatch {- tiling_config: tile,- input_prologue: GemmInputPrologueKind::FullPrecision,- use_mxu: false,- output_transform: GemmOutputTransformKind::Store,- alignment: GemmAlignment::from_axes(GemmAlignmentAxes {- m: batch_dim % tile.threadgroup_m == 0,- n: output_dim % tile.threadgroup_n == 0,- k: input_dim % tile.threadgroup_k == 0,- }),- transpose_b: true,- a: arguments.a,- a_offset: arguments.a_offset,- b: match configuration.quantization_method {- QuantizationMethod::ScaleBias => GemmWeights::ScaleBias {- weights: arguments.b,- scales: arguments.scales,- biases: arguments.zero_points_or_biases,- mode: configuration.mode,- group_size,- },- QuantizationMethod::ScaleZeroPoint => GemmWeights::ScaleZeroPoint {- weights: arguments.b,- scales: arguments.scales,- zero_points: arguments.zero_points_or_biases,- mode: configuration.mode,- group_size,- },- },- b_offset: 0,- d: arguments.output,- params: GemmParams {- M: batch_dim,- N: output_dim,- K: input_dim,- leading_dimension_a: input_dim,- leading_dimension_b: input_dim,- leading_dimension_d: output_dim,- threadgroups_per_row,- threadgroups_per_column,- aligned_inner_iterations: input_dim / tile.threadgroup_k,- use_morton: false,- ab_scale: 1.0,- },- group_count_x: threadgroups_per_row,- group_count_y: threadgroups_per_column,- };- gemm.encode(context, dispatch, encoder).map_err(QuantizedMatmulError::BackendError)-}--fn select_tile(- configuration: &QuantizedMatmulConfiguration,- batch_dim: u32,-) -> GemmTilingConfig {- let group_size = configuration.group_size as u32;- let threadgroup_k = group_size.min(32);- let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) = if batch_dim < 32 {- (32u32, 32u32, 1u32, 1u32)- } else {- (64u32, 64u32, 2u32, 2u32)- };- GemmTilingConfig {- threadgroup_m,- threadgroup_n,- threadgroup_k,- simdgroups_m,- simdgroups_n,- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsnew file mode 100644index 000000000..0c4fd79b2--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -0,0 +1,158 @@+use super::error::GemmSpecializationError;+use crate::{+ DataType,+ backends::common::{+ gpu_types::{+ QuantizationMethod,+ gemm::{GemmAlignment, GemmDTransform, GemmTiling, GemmWeightPrologueKind},+ },+ kernel::matmul::MatmulQuantCombo,+ },+};++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub(crate) struct GemmSpecialization {+ pub(crate) data_type: DataType,+ pub(crate) tiling: GemmTiling,+ pub(crate) use_mxu: bool,+ pub(crate) output_transform: GemmDTransform,+ pub(crate) alignment: GemmAlignment,+ pub(crate) transpose_b: bool,+ pub(crate) weight_prologue: GemmWeightPrologueKind,+ pub(crate) bits_per_weight: Option<u32>,+ pub(crate) group_size: Option<u32>,+}++impl GemmSpecialization {+ pub(crate) fn validate(&self) -> Result<(), GemmSpecializationError> {+ if let Some(group_size) = self.group_size {+ if self.tiling.block_k() > group_size {+ return Err(GemmSpecializationError::ThreadgroupKExceedsGroupSize {+ threadgroup_k: self.tiling.block_k(),+ group_size,+ });+ }+ }+ if self.weight_prologue != GemmWeightPrologueKind::FullPrecision {+ if self.use_mxu {+ return Err(GemmSpecializationError::QuantizedRequiresSimdgroup);+ }+ if !self.transpose_b {+ return Err(GemmSpecializationError::QuantizedRequiresTransposedB);+ }+ }+ Ok(())+ }++ pub(crate) fn precompile_configs(data_type: DataType) -> Vec<Self> {+ let mut out = Vec::new();+ for &tiling in simdgroup_tiling_set(data_type) {+ for align_mn in [true, false] {+ for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {+ out.push(Self {+ data_type,+ tiling,+ use_mxu: false,+ output_transform,+ alignment: GemmAlignment::new(align_mn, align_mn, true),+ transpose_b: true,+ weight_prologue: GemmWeightPrologueKind::FullPrecision,+ bits_per_weight: None,+ group_size: None,+ });+ }+ }+ }++ for &tiling in mxu_tiling_set(data_type) {+ for align_m in [true, false] {+ for align_n in [true, false] {+ for align_k in [true, false] {+ for output_transform in [+ GemmDTransform::empty(),+ GemmDTransform::SCALE,+ GemmDTransform::ACCUMULATE,+ GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,+ ] {+ out.push(Self {+ data_type,+ tiling,+ use_mxu: true,+ output_transform,+ alignment: GemmAlignment::new(align_m, align_n, align_k),+ transpose_b: true,+ weight_prologue: GemmWeightPrologueKind::FullPrecision,+ bits_per_weight: None,+ group_size: None,+ });+ }+ }+ }+ }+ }+ out+ }++ pub(crate) fn quant_combo_specs(+ data_type: DataType,+ combo: MatmulQuantCombo,+ ) -> Vec<Self> {+ let bits = DataType::from(combo.mode).size_in_bits() as u32;+ let group_size = combo.group_size;+ let weight_prologue = match combo.method {+ QuantizationMethod::ScaleBias => GemmWeightPrologueKind::ScaleBiasDequant,+ QuantizationMethod::ScaleZeroPoint => GemmWeightPrologueKind::ScaleZeroPointDequant,+ };+ let mut out = Vec::new();+ for &tiling in quant_tiling_set(data_type) {+ if tiling.block_k() > group_size {+ continue;+ }+ for align_n in [true, false] {+ for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {+ out.push(Self {+ data_type,+ tiling,+ use_mxu: false,+ output_transform,+ alignment: GemmAlignment::new(true, align_n, true),+ transpose_b: true,+ weight_prologue,+ bits_per_weight: Some(bits),+ group_size: Some(group_size),+ });+ }+ }+ }+ out+ }+}++fn simdgroup_tiling_set(data_type: DataType) -> &'static [GemmTiling] {+ match data_type {+ DataType::BF16 => &[GemmTiling::T64x32x32_2x2, GemmTiling::T64x64x16_2x2],+ DataType::F16 => &[GemmTiling::T64x64x16_2x2, GemmTiling::T64x32x32_2x2],+ _ => &[],+ }+}++fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {+ if !matches!(data_type, DataType::F16 | DataType::BF16) {+ return &[];+ }+ &[GemmTiling::T64x64x32_2x2, GemmTiling::T32x64x32_2x2, GemmTiling::T64x32x32_4x1, GemmTiling::T128x128x32_4x4]+}++pub(crate) fn quant_tiling_set(data_type: DataType) -> &'static [GemmTiling] {+ match data_type {+ DataType::BF16 => &[+ GemmTiling::T8x32x32_1x1,+ GemmTiling::T32x32x32_2x2,+ GemmTiling::T64x32x32_2x2,+ GemmTiling::T64x64x32_2x2,+ GemmTiling::T64x64x64_2x2,+ ],+ DataType::F16 => &[GemmTiling::T8x32x32_1x1, GemmTiling::T32x32x32_2x2],+ _ => &[],+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.hnew file mode 100644index 000000000..b76b4a5eb--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h@@ -0,0 +1,163 @@+#pragma once++#include <metal_stdlib>++#include "../../common/defines.h"+#include "../../gemm/common/quant_unpack.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <typename T, typename U, int VALUES_PER_THREAD, int BITS>+inline U load_vector(const device T* x, thread U* x_thread) {+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");++ using U4 = vec<U, 4>;+ U sum = 0;+ thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);+ for (int i = 0; i < VALUES_PER_THREAD / 4; i++) {+ U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);+ sum += v[0] + v[1] + v[2] + v[3];+ x_vec4[i] = v;+ }+ return sum;+}++template <typename T, typename U, int VALUES_PER_THREAD, int BITS>+inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");++ U sum = 0;+ for (int i = 0; i < VALUES_PER_THREAD; ++i) {+ x_thread[i] = 0;+ }+ for (int i = 0; i < N; ++i) {+ U v = x[i];+ sum += v;+ x_thread[i] = v;+ }+ return sum;+}++template <typename U, int VALUES_PER_THREAD, int BITS>+inline void qouter(+ const thread uint8_t* w,+ U x,+ U scale,+ U bias,+ thread U* result+) {+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");++ if (BITS == 4) {+ for (int i = 0; i < (VALUES_PER_THREAD / 2); i++) {+ result[2 * i] += x * (scale * uint_to_fp<U>(w[i] & 0x0fu) + bias);+ result[2 * i + 1] +=+ x * (scale * uint_to_fp<U>((w[i] >> 4) & 0x0fu) + bias);+ }+ } else if (BITS == 8) {+ for (int i = 0; i < VALUES_PER_THREAD; i++) {+ result[i] += x * (scale * w[i] + bias);+ }+ }+}++template <typename U, int VALUES_PER_THREAD, int BITS>+inline U qdot(+ const device uint8_t* w,+ const thread U* x_thread,+ U scale,+ U bias,+ U sum+) {+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");++ U accumulator = 0;+ if (BITS == 4) {+ using U4 = vec<U, 4>;+ const device ushort* weight_words =+ reinterpret_cast<const device ushort*>(w);+ const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);+ for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {+ uint weight_word = weight_words[i];+ U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(+ weight_word,+ weight_word >> 4,+ weight_word >> 8,+ weight_word >> 12+ ));+ accumulator += dot(x_vec4[i], weight_vec4);+ }+ } else if (BITS == 8) {+ using U4 = vec<U, 4>;+ const device uint* weight_words = reinterpret_cast<const device uint*>(w);+ const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);+ for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {+ uint weight_word = weight_words[i];+ U4 weight_vec4 = uint4_to_fp4<U, 8>(uint4(+ weight_word,+ weight_word >> 8,+ weight_word >> 16,+ weight_word >> 24+ ));+ accumulator += dot(x_vec4[i], weight_vec4);+ }+ }+ return scale * accumulator + sum * bias;+}++template <typename U, int VALUES_PER_THREAD, int BITS>+inline U qdot_safe(+ const device uint8_t* w,+ const thread U* x_thread,+ U scale,+ U bias,+ U sum,+ int N+) {+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");++ U accumulator = 0;+ if (BITS == 4) {+ using U4 = vec<U, 4>;+ const device uint16_t* weight_words =+ reinterpret_cast<const device uint16_t*>(w);+ const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);++ int full_chunks = N / 4;+ for (int i = 0; i < full_chunks; i++) {+ uint16_t weight_word = weight_words[i];+ U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(+ weight_word,+ weight_word >> 4,+ weight_word >> 8,+ weight_word >> 12+ ));+ accumulator += dot(x_vec4[i], weight_vec4);+ }++ int remainder = N & 3;+ if (remainder > 0) {+ uint16_t weight_word = weight_words[full_chunks];+ int base_index = 4 * full_chunks;+ accumulator += x_thread[base_index] * uint_to_fp<U>(weight_word & 0xf);+ if (remainder > 1)+ accumulator +=+ x_thread[base_index + 1] * uint_to_fp<U>((weight_word >> 4) & 0xf);+ if (remainder > 2)+ accumulator +=+ x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);+ }+ } else if (BITS == 8) {+ for (int i = 0; i < N; i++) {+ accumulator += x_thread[i] * w[i];+ }+ }++ return scale * accumulator + sum * bias;+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rsindex 8d3baf734..ea0f28a20 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs@@ -2,16 +2,20 @@ use super::{kernel::GemvKernel, spec::GemvSpecialization};use crate::backends::{common::{AsBufferRangeRef, Buffer, Encoder,- kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError},+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError},},metal::Metal,};-pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(kernel: &mut GemvKernel,encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal, TB>,+ arguments: MatmulArguments<'a, Metal, TB>,) -> Result<(), MatmulError<Metal>> {+ let ab_scale = arguments.d_transform.ab_scale;+ let output_bias = arguments.d_transform.bias;+ let is_accumulate = arguments.d_transform.accumulate;+let MatmulArguments {a,a_offset,@@ -19,38 +23,34 @@ pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(b_offset,b_leading_dimension,b_transpose,- ab_scale,- c,d,- batch_dim,- input_dim,- output_dim,+ m,+ n,+ k,+ ..} = arguments;+ let weights = match b {+ MatmulB::FullPrecision {+ b: w,+ } => w,+ _ => panic!("FP gemv requires FullPrecision B"),+ };assert!(b_transpose, "encode_gemv does not support b_transpose=false");assert!(b_offset == 0, "encode_gemv does not support nonzero b_offset");- assert!(- b_leading_dimension.is_none_or(|ld| ld == input_dim),- "encode_gemv does not support custom b_leading_dimension"- );+ assert!(b_leading_dimension.is_none_or(|ld| ld == k), "encode_gemv does not support custom b_leading_dimension");- let (is_accumulate, output_bias) = match c {- MatmulArgumentC::Accumulate => (true, None),- MatmulArgumentC::Bias(bias) => (false, Some(bias)),- MatmulArgumentC::None => (false, None),- };-- let specialization = GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());+ let specialization = GemvSpecialization::select(k, n, is_accumulate, output_bias.is_some());kernel.get_or_create(encoder.context(), specialization)?.encode(- b,+ weights,(a, a_offset),output_bias,d,- input_dim,- output_dim,- input_dim,+ k,+ n,+ k,ab_scale,- batch_dim,+ m,specialization.output_rows_per_threadgroup(),encoder,);diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalindex c601f6fff..660b21222 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -14,7 +14,7 @@ using namespace metal;#define GEMV_MAX_THREADGROUP_MEMORY 320template <typename T>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, half, bfloat)KERNEL(MatmulGemv)(const device T* matrix,const device T* input_vector,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rsindex 48df5aef2..ca75e02b7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -1,5 +1,7 @@pub(crate) mod fp;mod kernel;+mod quant_kernel;mod spec;pub(crate) use kernel::GemvKernel;+pub(crate) use quant_kernel::QuantGemvKernel;diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metalsimilarity index 98%rename from crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv.metalrename to crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metalindex c65ebc01b..e376972a9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal@@ -1,9 +1,11 @@#include <metal_stdlib>-#include "../common/dsl.h"-#include "../generated/quantization_method.h"-#include "quant_matmul.h"+#include "../../common/dsl.h"+#include "../../generated/quantization_method.h"+#include "../gemm/common/quant_pack.h"+#include "common/qmv_ops.h"using namespace uzu::quantization_method;+using namespace uzu::gemm;template <typename T, uint GROUP_SIZE, uint BITS>VARIANTS(T, float, half, bfloat)diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metalsimilarity index 96%rename from crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalrename to crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metalindex fe1a5e818..a27f1591c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal@@ -1,10 +1,12 @@#include <metal_stdlib>-#include "../common/dsl.h"-#include "../hadamard_transform/hadamard_transform.h"-#include "../generated/quantization_method.h"-#include "quant_matmul.h"+#include "../../common/dsl.h"+#include "../../hadamard_transform/hadamard_transform.h"+#include "../../generated/quantization_method.h"+#include "../gemm/common/quant_pack.h"+#include "common/qmv_ops.h"using namespace uzu::quantization_method;+using namespace uzu::gemm;template <typename T, uint GROUP_SIZE, uint BITS>VARIANTS(T, float, half, bfloat)diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rsnew file mode 100644index 000000000..322c016b4--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs@@ -0,0 +1,173 @@+use std::collections::{HashMap, hash_map::Entry};++use crate::{+ DataType,+ backends::{+ common::{+ AsBufferRangeRef, Buffer, Encoder,+ gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},+ kernel::{+ Kernels, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,+ matmul::{MatmulArguments, MatmulB, MatmulError},+ },+ },+ metal::{Metal, context::MetalContext},+ },+};++const PATH: &str = "QuantGemv";++#[derive(PartialEq, Eq, Hash, Clone, Copy)]+struct QmvKey {+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+ use_hadamard: bool,+}++pub(crate) struct QuantGemvKernel {+ data_type: DataType,+ qmv: HashMap<QmvKey, <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel>,+ qmv_fast: HashMap<+ QmvKey,+ <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel,+ >,+}++impl QuantGemvKernel {+ pub(crate) fn new(+ _context: &MetalContext,+ data_type: DataType,+ ) -> Self {+ Self {+ data_type,+ qmv: HashMap::new(),+ qmv_fast: HashMap::new(),+ }+ }++ pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+ &mut self,+ encoder: &mut Encoder<Metal>,+ arguments: MatmulArguments<'a, Metal, TB>,+ ) -> Result<(), MatmulError<Metal>> {+ let mask = arguments.d_transform.mask();+ if mask.contains(GemmDTransform::SCALE) {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::SCALE,+ path: PATH,+ });+ }+ if mask.contains(GemmDTransform::ACCUMULATE) {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::ACCUMULATE,+ path: PATH,+ });+ }+ if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {+ return Err(MatmulError::UnsupportedLayout {+ path: PATH,+ });+ }++ let hadamard_factors = arguments.d_transform.rht_factors;++ let MatmulArguments {+ a,+ a_offset,+ b,+ d,+ m,+ n,+ k,+ ..+ } = arguments;+ let (weights, scales, zp_or_bias, method, mode, group_size) = match b {+ MatmulB::ScaleBiasDequant {+ b: w,+ scales,+ biases,+ mode,+ group_size,+ } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),+ MatmulB::ScaleZeroPointDequant {+ b: w,+ scales,+ zero_points,+ mode,+ group_size,+ } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),+ MatmulB::FullPrecision {+ ..+ } => panic!("QuantGemvKernel requires quantized B"),+ };++ let bits = match mode {+ QuantizationMode::U4 => 4u32,+ QuantizationMode::I8 | QuantizationMode::U8 => 8u32,+ };+ let use_fast = n % 8 == 0 && k % 512 == 0;++ let (zero_points, biases) = match method {+ QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias), None),+ QuantizationMethod::ScaleBias => (None, Some(zp_or_bias)),+ };++ if use_fast {+ let key = QmvKey {+ group_size,+ bits,+ quant_method: method,+ use_hadamard: hadamard_factors.is_some(),+ };+ let context = encoder.context();+ let kernel = match self.qmv_fast.entry(key) {+ Entry::Occupied(entry) => entry.into_mut(),+ Entry::Vacant(entry) => {+ let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ context,+ self.data_type,+ group_size,+ bits,+ method,+ hadamard_factors.is_some(),+ )+ .map_err(MatmulError::BackendError)?;+ entry.insert(kernel)+ },+ };+ kernel.encode(weights, scales, zero_points, biases, (a, a_offset), d, hadamard_factors, k, n, m, encoder);+ } else {+ if hadamard_factors.is_some() {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::RHT,+ path: PATH,+ });+ }+ let key = QmvKey {+ group_size,+ bits,+ quant_method: method,+ use_hadamard: false,+ };+ let context = encoder.context();+ let kernel = match self.qmv.entry(key) {+ Entry::Occupied(entry) => entry.into_mut(),+ Entry::Vacant(entry) => {+ let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(+ context,+ self.data_type,+ group_size,+ bits,+ method,+ )+ .map_err(MatmulError::BackendError)?;+ entry.insert(kernel)+ },+ };+ kernel.encode(weights, scales, zero_points, biases, (a, a_offset), d, k, n, m, encoder);+ }++ Ok(())+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rsindex 4e83e73cd..343189ea4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs@@ -66,16 +66,6 @@ impl GemvSpecialization {is_accumulate: false,is_bias: false,}],- DataType::F32 => &[Self {- threadgroup_rows: 8,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- is_accumulate: false,- is_bias: false,- }],_ => &[],}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 8f803c24a..7756cc248 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,29 +1,29 @@pub mod gemm;pub mod gemv;-pub mod quant;use std::sync::OnceLock;-use self::{gemm::GemmKernel, gemv::GemvKernel};+pub use self::gemm::{GemmDispatchPath, GemmKernel};+use self::gemv::{GemvKernel, QuantGemvKernel};use crate::{DataType,backends::{common::{- AsBufferRangeRef, Buffer, Encoder,+ AsBufferRangeRef, Backend, Buffer, Encoder,+ gpu_types::gemm::GemmDTransform,kernel::{- TensorAddBiasKernel,- matmul::{MatmulArguments, MatmulError, MatmulKernel},+ HadamardTransformKernel, TensorAddBiasKernel,+ matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel, MatmulQuantCombo},},},- metal::{Metal, context::MetalContext, kernel::TensorAddBiasMetalKernel, metal_extensions::DeviceExt},+ metal::{Metal, context::MetalContext, error::MetalError},},};pub struct MatmulMetalKernel {- data_type: DataType,gemv: GemvKernel,- pub(crate) gemm: GemmKernel,- pub(crate) bias_add: TensorAddBiasMetalKernel,+ quant_gemv: QuantGemvKernel,+ pub gemm: GemmKernel,}const DEFAULT_GEMV_MAX_BATCH: u32 = 8;@@ -36,45 +36,102 @@ fn max_gemv_batch_threshold() -> u32 {}impl MatmulMetalKernel {- fn is_mxu_eligible(- &self,- context: &MetalContext,- ) -> bool {- context.device.supports_mxu() && matches!(self.data_type, DataType::F16 | DataType::BF16)- }-}+ fn dispatch_gemv<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+ &mut self,+ arguments: MatmulArguments<'a, Metal, TB>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), MatmulError<Metal>> {+ let post_rht = arguments.d_transform.rht_factors;-/// Explicit dispatch paths for testing individual kernels independent of production routing.-#[derive(Debug, Clone, Copy)]-pub enum MatmulDispatchPath {- Auto,- Gemv,- Gemm,- GemmMxu,-}+ let MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ d,+ d_transform,+ m,+ n,+ k,+ } = arguments;-impl MatmulMetalKernel {- pub fn encode_with_path(+ // A temopary hack, will port the post ops to the gemv later.+ let d_transform = d_transform.without(GemmDTransform::RHT);++ gemv::fp::encode(+ &mut self.gemv,+ encoder,+ MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ d: &mut *d,+ d_transform,+ m,+ n,+ k,+ },+ )?;++ if let Some(factors) = post_rht {+ self.gemm.hadamard.encode(d, factors, n, m, encoder);+ }+ Ok(())+ }++ fn dispatch_quant_gemv<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(&mut self,- arguments: MatmulArguments<Metal>,+ arguments: MatmulArguments<'a, Metal, TB>,encoder: &mut Encoder<Metal>,- path: MatmulDispatchPath,- ) {- let context = encoder.context();- match path {- MatmulDispatchPath::Auto => self.encode(arguments, encoder),- MatmulDispatchPath::Gemv => {- gemv::fp::encode(&mut self.gemv, encoder, arguments).expect("Failed to encode GEMV")- },- MatmulDispatchPath::Gemm => {- gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, false)- .expect("Failed to encode Gemm")- },- MatmulDispatchPath::GemmMxu => {- gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, true)- .expect("Failed to encode GemmMxu")+ ) -> Result<(), MatmulError<Metal>> {+ if arguments.b_offset != 0 {+ return Err(MatmulError::UnsupportedLayout {+ path: "QuantGemv",+ });+ }+ let post_bias = arguments.d_transform.bias;++ let MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ d,+ d_transform,+ m,+ n,+ k,+ } = arguments;+ let d_transform = d_transform.without(GemmDTransform::BIAS);++ self.quant_gemv.encode(+ encoder,+ MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ d: &mut *d,+ d_transform,+ m,+ n,+ k,},+ )?;++ if let Some(bias) = post_bias {+ self.gemm.bias_add.encode(None::<&<Metal as Backend>::DenseBuffer>, bias, d, n, m * n, encoder);}+ Ok(())}}@@ -84,20 +141,19 @@ impl MatmulKernel for MatmulMetalKernel {fn new(context: &MetalContext,data_type: DataType,- ) -> Result<Self, MatmulError<Metal>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));+ ) -> Result<Self, MetalError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16) {+ return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());}- let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true).map_err(MatmulError::BackendError)?;- let gemm = GemmKernel::new(context, data_type).map_err(MatmulError::BackendError)?;- let gemv = GemvKernel::new(context, data_type)?;+ let gemm = GemmKernel::new(context, data_type)?;+ let gemv = GemvKernel::new(context, data_type).map_err(MetalError::from)?;+ let quant_gemv = QuantGemvKernel::new(context, data_type);Ok(Self {- data_type,gemv,+ quant_gemv,gemm,- bias_add,})}@@ -105,21 +161,33 @@ impl MatmulKernel for MatmulMetalKernel {&mut self,arguments: MatmulArguments<Metal, TB>,encoder: &mut Encoder<Metal>,- ) {- // gemv only handles the canonical B layout; non-canonical inputs go to GEMM.- let context = encoder.context();- let gemv_eligible = arguments.b_transpose- && arguments.b_offset == 0- && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.input_dim)- && arguments.batch_dim <= max_gemv_batch_threshold();+ ) -> Result<(), MetalError> {+ let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });+ let gemv_eligible = if is_quant {+ arguments.m < 5 || arguments.n == 1+ } else {+ arguments.b_transpose+ && arguments.b_offset == 0+ && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.k)+ && arguments.m <= max_gemv_batch_threshold()+ };if gemv_eligible {- gemv::fp::encode(&mut self.gemv, encoder, arguments).expect("Failed to encode GEMV kernel");- return;+ if is_quant {+ self.dispatch_quant_gemv(arguments, encoder).map_err(MetalError::from)+ } else {+ self.dispatch_gemv(arguments, encoder).map_err(MetalError::from)+ }+ } else {+ self.gemm.encode(arguments, encoder)}+ }- let use_mxu = self.is_mxu_eligible(context);- gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, use_mxu)- .expect("Failed to encode GEMM");+ fn preheat_quant_combo(+ &mut self,+ context: &MetalContext,+ combo: MatmulQuantCombo,+ ) -> Result<(), MetalError> {+ self.gemm.preheat_quant_combo(context, combo)}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rsdeleted file mode 100644index 6ccbbcd74..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs+++ /dev/null@@ -1,37 +0,0 @@-use super::{MatmulMetalKernel, gemm};-use crate::backends::{- common::{- Encoder,- kernel::quant_matmul::{- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,- QuantizedMatmulKernelEncodable,- },- },- metal::{Metal, context::MetalContext},-};--#[derive(Debug, Clone, Copy)]-pub enum QuantizedMatmulDispatchPath {- Auto,- Gemm,-}--pub fn encode_quantized_matmul_with_path(- context: &MetalContext,- matmul: &mut MatmulMetalKernel,- encodable: &QuantizedMatmulKernelEncodable<Metal>,- configuration: &QuantizedMatmulConfiguration,- arguments: QuantizedMatmulArguments<Metal>,- encoder: &mut Encoder<Metal>,- path: QuantizedMatmulDispatchPath,-) -> Result<(), QuantizedMatmulError<Metal>> {- match path {- QuantizedMatmulDispatchPath::Auto => {- encodable.encode(encoder, arguments);- Ok(())- },- QuantizedMatmulDispatchPath::Gemm => {- gemm::quant::encode(&mut matmul.gemm, context, configuration, arguments, encoder)- },- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rsindex 916078500..e03c9e43c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs@@ -1,6 +1,9 @@pub mod matmul;-use crate::backends::common::kernel::ManualKernels;+use crate::backends::common::{+ gpu_types::gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_row},+ kernel::ManualKernels,+};include!(concat!(env!("OUT_DIR"), "/dsl.rs"));diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex c6b8cffae..bccc6aab4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -1,8 +1,8 @@#include <metal_stdlib>#include "../activation/activations.h"+#include "../common/defines.h"#include "../common/dsl.h"#include "../common/thread_context.h"-#include "../quant_matmul/mma.h"// ------------------------ helpers ------------------------static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.hdeleted file mode 100644index f86a61e25..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h+++ /dev/null@@ -1,536 +0,0 @@-#pragma once--#include "../common/defines.h"-#include <metal_simdgroup>-#include <metal_simdgroup_matrix>-#include <metal_stdlib>--using namespace metal;--namespace matmul_utils {--template <typename T>-struct pointer_element_t_impl {- using type = T;-};-template <typename T>-struct pointer_element_t_impl<device T*> {- using type = T;-};-template <typename T>-struct pointer_element_t_impl<threadgroup T*> {- using type = T;-};-template <typename T>--using pointer_element_t = typename pointer_element_t_impl<T>::type;--template <typename T, typename U = T>-struct TransformNone {- METAL_FUNC static U apply(U val) { return val; }-};--template <typename T, int FRAG_ROWS_, int FRAG_COLS_>-struct BaseMMAFrag {- static_assert(- FRAG_ROWS_ == 8,- "Only 8 x 8 fragment matrices are currently supported"- );- static_assert(- FRAG_COLS_ == 8,- "Only 8 x 8 fragment matrices are currently supported"- );-};--template <typename T>-struct BaseMMAFrag<T, 8, 8> {- METAL_CONST int FRAG_ROWS = 8;- METAL_CONST int FRAG_COLS = 8;- METAL_CONST int ELEMS_PER_FRAG = (FRAG_ROWS * FRAG_COLS) / 32;- METAL_CONST int ELEM_ROWS = 1;- METAL_CONST int ELEM_COLS = 2;-- static_assert(- ELEM_ROWS * ELEM_COLS == ELEMS_PER_FRAG,- "MMAFrag shape is not consistent with MMAFrag size"- );-- typedef metal::simdgroup_matrix<T, FRAG_ROWS, FRAG_COLS> mat_type;- typedef metal::vec<T, ELEMS_PER_FRAG> frag_type;-- METAL_FUNC static constexpr short2 get_coord(- ushort simd_lane_id [[thread_index_in_simdgroup]]- ) {- const short qid = simd_lane_id / 4;- const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);- const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;- return short2{fn, fm};- }-- template <typename SrcPtrType, typename StrX, typename StrY>- METAL_FUNC static constexpr void load(- thread frag_type& dst,- SrcPtrType src,- StrX str_x,- StrY str_y- ) {- METAL_PRAGMA_UNROLL- for (short i = 0; i < ELEM_ROWS; i++) {- METAL_PRAGMA_UNROLL- for (short j = 0; j < ELEM_COLS; j++) {- dst[i * ELEM_COLS + j] = static_cast<T>(src[i * str_x + j * str_y]);- }- }- }-- template <typename DstPtrType, typename StrX, typename StrY>- METAL_FUNC static constexpr void store(- const thread frag_type& src,- DstPtrType dst,- StrX str_x,- StrY str_y- ) {- using U = pointer_element_t<DstPtrType>;-- METAL_PRAGMA_UNROLL- for (short i = 0; i < ELEM_ROWS; i++) {- METAL_PRAGMA_UNROLL- for (short j = 0; j < ELEM_COLS; j++) {- dst[i * str_x + j * str_y] = static_cast<U>(src[i * ELEM_COLS + j]);- }- }- }-- template <- typename DstPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX,- typename OffY>- METAL_FUNC static constexpr void store_safe(- const thread frag_type& src,- DstPtrType dst,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = 0,- OffY off_y = 0- ) {- using U = pointer_element_t<DstPtrType>;-- METAL_PRAGMA_UNROLL- for (short i = 0; i < ELEM_ROWS; i++) {- METAL_PRAGMA_UNROLL- for (short j = 0; j < ELEM_COLS; j++) {- if ((off_x + i) < lim_x && (off_y + j) < lim_y) {- dst[(off_x + i) * str_x + (off_y + j) * str_y] =- static_cast<U>(src[i * ELEM_COLS + j]);- }- }- }- }-- METAL_FUNC static constexpr void mma(- thread frag_type& D,- thread frag_type& A,- thread frag_type& B,- thread frag_type& C- ) {- mat_type D_mat;- mat_type A_mat;- mat_type B_mat;- mat_type C_mat;-- reinterpret_cast<thread frag_type&>(A_mat.thread_elements()) = A;- reinterpret_cast<thread frag_type&>(B_mat.thread_elements()) = B;- reinterpret_cast<thread frag_type&>(C_mat.thread_elements()) = C;-- mma(D_mat, A_mat, B_mat, C_mat);-- D = reinterpret_cast<thread frag_type&>(D_mat.thread_elements());- }-- METAL_FUNC static constexpr void mma(- thread mat_type& D,- thread mat_type& A,- thread mat_type& B,- thread mat_type& C- ) {- simdgroup_multiply_accumulate(D, A, B, C);- }-};--template <- typename T,- int TILE_ROWS_,- int TILE_COLS_,- class MMAFrag_ = BaseMMAFrag<T, 8, 8>>-struct MMATile {- using MMAFrag_t = MMAFrag_;- using elem_type = T;- METAL_CONST int FRAG_ROWS = MMAFrag_t::FRAG_ROWS;- METAL_CONST int FRAG_COLS = MMAFrag_t::FRAG_COLS;- METAL_CONST int ELEMS_PER_FRAG = MMAFrag_t::ELEMS_PER_FRAG;-- METAL_CONST int TILE_ROWS = TILE_ROWS_;- METAL_CONST int TILE_COLS = TILE_COLS_;-- METAL_CONST int ROWS = TILE_ROWS * FRAG_ROWS;- METAL_CONST int COLS = TILE_COLS * FRAG_COLS;-- METAL_CONST int NUM_FRAGS = TILE_ROWS * TILE_COLS;- METAL_CONST int ELEMS_PER_TILE = NUM_FRAGS * ELEMS_PER_FRAG;-- typedef typename MMAFrag_t::mat_type mat_type;- typedef typename MMAFrag_t::frag_type frag_type;-- frag_type val_frags[NUM_FRAGS] = {frag_type(0)};-- METAL_FUNC MMATile() thread {}-- METAL_FUNC constexpr void clear() {- METAL_PRAGMA_UNROLL- for (short i = 0; i < NUM_FRAGS; ++i) {- val_frags[i] = frag_type(0);- }- }-- METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {- return val_frags[i * TILE_COLS + j];- }-- METAL_FUNC constexpr const thread frag_type& frag_at(- const short i,- const short j- ) const {- return val_frags[i * TILE_COLS + j];- }-- METAL_FUNC thread elem_type* elems() {- return reinterpret_cast<thread elem_type*>(val_frags);- }-- METAL_FUNC const thread elem_type* elems() const {- return reinterpret_cast<const thread elem_type*>(val_frags);- }-- template <typename U, int w_x, int w_y, int str_x, int str_y>- METAL_FUNC void load(const threadgroup U* src) {- METAL_PRAGMA_UNROLL- for (short i = 0; i < TILE_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (short j = 0; j < TILE_COLS; ++j) {- MMAFrag_t::load(- frag_at(i, j),- &(- src[(i * FRAG_ROWS) * w_x * str_x +- (j * FRAG_COLS) * w_y * str_y]- ),- str_x,- str_y- );- }- }- }-- template <typename U, int w_x, int w_y>- METAL_FUNC void store(device U* dst, const int ld) const {- METAL_PRAGMA_UNROLL- for (short i = 0; i < TILE_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (short j = 0; j < TILE_COLS; ++j) {- MMAFrag_t::store(- frag_at(i, j),- &(dst[(i * FRAG_ROWS) * w_x * ld + (j * FRAG_COLS) * w_y]),- ld,- 1- );- }- }- }-- // Store results into threadgroup memory (same layout as device variant)- template <typename U, int w_x, int w_y>- METAL_FUNC void store_tg(threadgroup U* dst, const int ld) const {- METAL_PRAGMA_UNROLL- for (short i = 0; i < TILE_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (short j = 0; j < TILE_COLS; ++j) {- MMAFrag_t::store(- frag_at(i, j),- &(dst[(i * FRAG_ROWS) * w_x * ld + (j * FRAG_COLS) * w_y]),- ld,- 1- );- }- }- }-- template <typename U, int w_x, int w_y>- METAL_FUNC void store_safe(- device U* dst,- const int ld,- const short2 dst_tile_dims- ) const {- METAL_PRAGMA_UNROLL- for (int i = 0; i < TILE_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (int j = 0; j < TILE_COLS; ++j) {- MMAFrag_t::store_safe(- frag_at(i, j),- dst,- ld,- 1,- dst_tile_dims.y,- dst_tile_dims.x,- (i * FRAG_ROWS) * w_x,- (j * FRAG_COLS) * w_y- );- }- }- }-};--template <typename T, typename U, int M, int N, int K>-METAL_FUNC void tile_matmad(- thread MMATile<T, M, N>& D,- thread MMATile<U, M, K>& A,- thread MMATile<U, K, N>& B,- thread MMATile<T, M, N>& C-) {- METAL_PRAGMA_UNROLL- for (short m = 0; m < M; ++m) {- METAL_PRAGMA_UNROLL- for (short n = 0; n < N; ++n) {- short n_serp = (m % 2) ? (N - 1 - n) : n;- METAL_PRAGMA_UNROLL- for (short k = 0; k < K; ++k) {- MMATile<T, M, N>::MMAFrag_t::mma(- D.frag_at(m, n_serp),- A.frag_at(m, k),- B.frag_at(k, n_serp),- C.frag_at(m, n_serp)- );- }- }- }-}--template <- typename T,- typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- short lda_tgp,- short ldb_tgp,- typename AccumType = float,- typename Epilogue = TransformNone<U, AccumType>>-struct BlockMMA {- // MMAFrag size- METAL_CONST short FRAG_SIZE = 8;- using MMAFrag_acc_t = BaseMMAFrag<AccumType, FRAG_SIZE, FRAG_SIZE>;-- // Warp tile simdgroup matrix strides along M- METAL_CONST short TM_stride = FRAG_SIZE * WM;- // Warp tile simdgroup matrix strides along M- METAL_CONST short TN_stride = FRAG_SIZE * WN;-- // Warp tile size along M- METAL_CONST short TM = BM / (FRAG_SIZE * WM);- // Warp tile size along N- METAL_CONST short TN = BN / (FRAG_SIZE * WN);-- // Threadgroup A strides- METAL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M- METAL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K-- // Threadgroup B strides- METAL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K- METAL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N-- // Threadgroup strides along K- METAL_CONST short tile_stride_a = FRAG_SIZE * A_str_k;- METAL_CONST short tile_stride_b = FRAG_SIZE * B_str_k;-- // Simdgroup matrices- MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;- MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;-- // Offsets within threadgroup- short sm;- short sn;-- short As_offset;- short Bs_offset;-- /* Constructor */- METAL_FUNC BlockMMA(- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]- ) {- // Determine thread position in simdgroup matrix- short tm = FRAG_SIZE * (simd_group_id / WN);- short tn = FRAG_SIZE * (simd_group_id % WN);-- short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);- sm = simd_coord.y;- sn = simd_coord.x;-- // Determine thread and simdgroup offset- As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K- Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N-- sm += tm;- sn += tn;- }-- /* (BM, BK) X (BK, BN) multiply accumulate function */- METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {- // Adjust for simdgroup and thread location- As += As_offset;- Bs += Bs_offset;-- // Iterate over BK in blocks of FRAG_SIZE- METAL_PRAGMA_UNROLL- for (short kk = 0; kk < BK; kk += FRAG_SIZE) {- simdgroup_barrier(mem_flags::mem_none);-- Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);-- simdgroup_barrier(mem_flags::mem_none);-- Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);-- simdgroup_barrier(mem_flags::mem_none);-- tile_matmad(Ctile, Atile, Btile, Ctile);-- // Progress to next simdgroup tile- As += tile_stride_a;- Bs += tile_stride_b;- }- }-- /* Store results from simdgroup_matrix results into device memory */- METAL_FUNC void store_result(device U* D, const int ldd) {- // Apply epilogue- METAL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);- }-- // Adjust for simdgroup and thread location- D += sm * ldd + sn;-- Ctile.template store<U, WM, WN>(D, ldd);- }-- /* Store results into threadgroup memory */- METAL_FUNC void store_result_tg(threadgroup U* D, const int ldd) {- // Apply epilogue- METAL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);- }-- // Adjust for simdgroup and thread location- D += sm * ldd + sn;-- Ctile.template store_tg<U, WM, WN>(D, ldd);- }-- METAL_FUNC void store_result_safe(- device U* D,- const int ldd,- short2 dst_tile_dims- ) {- // Apply epilogue- METAL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);- }-- // Adjust for simdgroup and thread location- D += sm * ldd + sn;- dst_tile_dims -= short2(sn, sm);-- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)- return;-- Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);- }-};--template <- typename T,- int BM,- int BK,- int lda_tgp,- int reduction_dim,- int tgp_size,- int align_M = 1,- int align_K = 4>-struct BlockLoader {- METAL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;-- const int src_ld;- const short thread_idx;- const short bi;- const short bj;-- threadgroup T* dst;- const device T* src;-- BlockLoader(- const device T* src_,- const int src_ld_,- threadgroup T* dst_,- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]- )- : src_ld(src_ld_), thread_idx(simd_group_id * 32 + simd_lane_id),- bi(n_reads * thread_idx / BK), bj((n_reads * thread_idx) % BK),- dst(dst_ + bi * lda_tgp + bj), src(src_ + bi * src_ld + bj) {}-- void load_unsafe() const {- if (BM * BK < tgp_size && bi >= BM) {- return;- }-- for (int i = 0; i < n_reads; i++) {- dst[i] = src[i];- }- }-- void load_safe(short2 src_tile_dim) const {- if (BM * BK < tgp_size && bi >= BM) {- return;- }-- if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {- for (int i = 0; i < n_reads; i++) {- dst[i] = T(0);- }- return;- }-- for (int i = 0; i < n_reads; i++) {- dst[i] = src[i];- }- }-- void next() {- src += BK; // Advance by tile size- }-};--} // namespace matmul_utils\ No newline at end of filediff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metaldeleted file mode 100644index 322ac3708..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ /dev/null@@ -1,194 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"-#include "../generated/quantization_method.h"-#include "../hadamard_transform/hadamard_transform.h"-#include "quant_matmul.h"--using namespace uzu::quantization_method;--// Transposed QMM kernel. BM = batch tile rows, BK = K-axis tile, BN = output-// tile cols. WM, WN = simdgroup tile counts along M and N axes.-template <typename T, uint GROUP_SIZE, uint BITS, uint BM, uint BK, uint BN, uint WM, uint WN>-VARIANTS(T, float, half, bfloat)-VARIANTS(GROUP_SIZE, 32, 64, 128)-VARIANTS(BITS, 4, 8)-VARIANTS(BM, 8, 32, 64)-VARIANTS(BK, 32, 64)-VARIANTS(BN, 32, 64)-VARIANTS(WM, 1, 2)-VARIANTS(WN, 1, 2)-CONSTRAINT(- (BM == 8 && BK == 32 && BN == 32 && WM == 1 && WN == 1) ||- (BM == 32 && BK == 32 && BN == 32 && WM == 2 && WN == 2) ||- (BM == 32 && BK == 64 && BN == 32 && WM == 2 && WN == 2) ||- (BM == 64 && BK == 32 && BN == 64 && WM == 2 && WN == 2) ||- (BM == 64 && BK == 64 && BN == 64 && WM == 2 && WN == 2))-CONSTRAINT(BK <= GROUP_SIZE)-CONSTRAINT(T != "float" || BK < 64)-PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(- const device uint32_t* weights,- const device T* scales,- const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),- const device T* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),- const device T* input,- device T* output,- const device int32_t* hadamard_factors OPTIONAL(use_hadamard),- const constant uint& in_vec_size,- const constant uint& out_vec_size,- const constant uint& batch_size,- threadgroup T Xs[BM * (BK + 16 / sizeof(T))],- threadgroup T Ws[BN * (BK + 16 / sizeof(T))],- const QuantizationMethod quant_method SPECIALIZE,- const bool use_hadamard SPECIALIZE,- const bool aligned_n SPECIALIZE,- const uint out_block_idx GROUPS(out_vec_size.div_ceil(BN)),- const uint batch_block_idx GROUPS(batch_size.div_ceil(BM)),- const uint simd_lane THREADS(32),- const uint simd_group THREADS(WM * WN)-) {- if (quant_method == QuantizationMethod::ScaleBias) {- if (aligned_n) {- qmm_transposed_impl<- T,- GROUP_SIZE,- BITS,- true,- BM,- BK,- BN,- QuantizationMethod::ScaleBias,- WM,- WN>(- weights,- scales,- zero_points,- biases,- input,- output,- Xs,- Ws,- in_vec_size,- out_vec_size,- batch_size,- out_block_idx,- batch_block_idx,- simd_group,- simd_lane- );- } else {- qmm_transposed_impl<- T,- GROUP_SIZE,- BITS,- false,- BM,- BK,- BN,- QuantizationMethod::ScaleBias,- WM,- WN>(- weights,- scales,- zero_points,- biases,- input,- output,- Xs,- Ws,- in_vec_size,- out_vec_size,- batch_size,- out_block_idx,- batch_block_idx,- simd_group,- simd_lane- );- }- } else {- if (aligned_n) {- qmm_transposed_impl<- T,- GROUP_SIZE,- BITS,- true,- BM,- BK,- BN,- QuantizationMethod::ScaleZeroPoint,- WM,- WN>(- weights,- scales,- zero_points,- biases,- input,- output,- Xs,- Ws,- in_vec_size,- out_vec_size,- batch_size,- out_block_idx,- batch_block_idx,- simd_group,- simd_lane- );- } else {- qmm_transposed_impl<- T,- GROUP_SIZE,- BITS,- false,- BM,- BK,- BN,- QuantizationMethod::ScaleZeroPoint,- WM,- WN>(- weights,- scales,- zero_points,- biases,- input,- output,- Xs,- Ws,- in_vec_size,- out_vec_size,- batch_size,- out_block_idx,- batch_block_idx,- simd_group,- simd_lane- );- }- }-- if (use_hadamard) {- threadgroup_barrier(mem_flags::mem_device);-- // Each lane covers BN/32 output columns (1 at BN=32, 2 at BN=64).- constexpr uint NUM_SIMD_GROUPS = WM * WN;- constexpr uint ROWS_PER_SIMD_GROUP = BM / NUM_SIMD_GROUPS;- constexpr uint COLS_PER_LANE = BN / 32;-- const uint row_start =- batch_block_idx * BM + simd_group * ROWS_PER_SIMD_GROUP;- const uint row_end = min(row_start + ROWS_PER_SIMD_GROUP, batch_size);-- for (uint c = 0; c < COLS_PER_LANE; c++) {- const uint out_col = out_block_idx * BN + c * 32 + simd_lane;- const int32_t factor = hadamard_factors[out_col];-- for (uint row = row_start; row < row_end; row++) {- uint output_index = row * out_vec_size + out_col;-- output[output_index] = simdgroup_random_hadamard_transform(- static_cast<ushort>(simd_lane),- output[output_index],- factor- );- }- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.hdeleted file mode 100644index c518d59b4..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h+++ /dev/null@@ -1,842 +0,0 @@-#pragma once--#include <metal_simdgroup>-#include <metal_stdlib>--using namespace metal;--#include "../generated/quantization_method.h"-#include "mma.h"--using QuantizationMethod = uzu::quantization_method::QuantizationMethod;--// Avoids air.convert (SFU) for int → float which is slower on Apple GPU.-template <typename U>-METAL_FUNC U uint_to_fp(uint32_t x) {- return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);-}--template <>-METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {- return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);-}--// Unpack 4 lanes of `BITS`-wide packed uints (low bits of each uint) into-// vec<U, 4> of floats. `BITS` in [1, 23] to fit the float23 mantissa trick.-template <int BITS>-METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {- static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");- constexpr uint mask = (1u << BITS) - 1u;- n &= uint4(mask);- return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);-}--template <typename U, int BITS>-METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);--template <>-METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {- return _uint4_to_fp4_float<4>(n);-}--template <>-METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {- return _uint4_to_fp4_float<8>(n);-}--template <>-METAL_FUNC half4 uint4_to_fp4<half, 4>(uint4 n) {- return half4(_uint4_to_fp4_float<4>(n));-}--template <>-METAL_FUNC half4 uint4_to_fp4<half, 8>(uint4 n) {- return half4(_uint4_to_fp4_float<8>(n));-}--template <>-METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {- return bfloat4(_uint4_to_fp4_float<4>(n));-}--template <>-METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {- return bfloat4(_uint4_to_fp4_float<8>(n));-}--template <int bits, int wsize = 8>-inline constexpr short get_pack_factor() {- return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : wsize / bits);-}--template <int bits, int wsize = 8>-inline constexpr short get_bytes_per_pack() {- constexpr int power_of_2_bits = (bits & (bits - 1)) == 0;- return power_of_2_bits ? (wsize / 8) : (bits == 5 ? 5 : 3);-}--template <typename T, typename U, int values_per_thread, int bits>-inline U load_vector(const device T* x, thread U* x_thread) {- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- using U4 = vec<U, 4>;- U sum = 0;- thread U4* x4 = (thread U4*)x_thread;- for (int i = 0; i < values_per_thread / 4; i++) {- U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);- sum += v[0] + v[1] + v[2] + v[3];- x4[i] = v;- }- return sum;-}--template <typename T, typename U, int values_per_thread, int bits>-inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- U sum = 0;- for (int i = 0; i < values_per_thread; ++i) {- x_thread[i] = 0;- }- for (int i = 0; i < N; ++i) {- U v = x[i];- sum += v;- x_thread[i] = v;- }- return sum;-}--template <typename U, int values_per_thread, int bits>-inline void qouter(- const thread uint8_t* w,- U x,- U scale,- U bias,- thread U* result-) {- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- if (bits == 4) {- for (int i = 0; i < (values_per_thread / 2); i++) {- result[2 * i] += x * (scale * uint_to_fp<U>(w[i] & 0x0fu) + bias);- result[2 * i + 1] +=- x * (scale * uint_to_fp<U>((w[i] >> 4) & 0x0fu) + bias);- }- } else if (bits == 8) {- for (int i = 0; i < values_per_thread; i++) {- result[i] += x * (scale * w[i] + bias);- }- }-}--template <typename U, int values_per_thread, int bits>-inline U qdot(- const device uint8_t* w,- const thread U* x_thread,- U scale,- U bias,- U sum-) {- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- U accum = 0;- if (bits == 4) {- using U4 = vec<U, 4>;- const device ushort* ws = (const device ushort*)w;- const thread U4* x4 = (const thread U4*)x_thread;- for (int i = 0; i < (values_per_thread / 4); i++) {- uint wi = ws[i];- U4 w_vec = uint4_to_fp4<U, 4>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));- accum += dot(x4[i], w_vec);- }- } else if (bits == 8) {- using U4 = vec<U, 4>;- const device uint* ws = (const device uint*)w;- const thread U4* x4 = (const thread U4*)x_thread;- for (int i = 0; i < (values_per_thread / 4); i++) {- uint wi = ws[i];- U4 w_vec = uint4_to_fp4<U, 8>(uint4(wi, wi >> 8, wi >> 16, wi >> 24));- accum += dot(x4[i], w_vec);- }- }- return scale * accum + sum * bias;-}--template <typename U, int values_per_thread, int bits>-inline U qdot_safe(- const device uint8_t* w,- const thread U* x_thread,- U scale,- U bias,- U sum,- int N-) {- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- U accum = 0;- if (bits == 4) {- using U4 = vec<U, 4>;- const device uint16_t* ws = (const device uint16_t*)w;- const thread U4* x4 = (const thread U4*)x_thread;-- int full = N / 4;- for (int i = 0; i < full; i++) {- uint16_t wi = ws[i];- U4 w_vec = uint4_to_fp4<U, 4>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));- accum += dot(x4[i], w_vec);- }-- int rem = N & 3;- if (rem > 0) {- uint16_t wv = ws[full];- int base = 4 * full;- if (rem > 0)- accum += x_thread[base] * uint_to_fp<U>(wv & 0xf);- if (rem > 1)- accum += x_thread[base + 1] * uint_to_fp<U>((wv >> 4) & 0xf);- if (rem > 2)- accum += x_thread[base + 2] * uint_to_fp<U>((wv >> 8) & 0xf);- }- } else if (bits == 8) {- for (int i = 0; i < N; i++) {- accum += x_thread[i] * w[i];- }- }-- return scale * accum + sum * bias;-}--template <typename U, int N, int bits>-inline void dequantize(- const device uint8_t* w,- U scale,- U bias,- threadgroup U* w_local-) {- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- if (bits == 4) {- U s0 = scale;- U s1 = scale / static_cast<U>(16.0f);- for (int i = 0; i < (N / 2); i++) {- w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;- w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;- }- } else if (bits == 8) {- for (int i = 0; i < N; i++) {- w_local[i] = scale * w[i] + bias;- }- }-}--template <>-inline void dequantize<bfloat, 8, 4>(- const device uint8_t* w,- bfloat scale,- bfloat bias,- threadgroup bfloat* w_local-) {- const device uint32_t* w_ptr = (const device uint32_t*)w;- uint32_t packed = *w_ptr;-- bfloat4 v0, v1;-- // Low 4 nibbles- v0.x = static_cast<bfloat>(packed & 0xF);- v0.y = static_cast<bfloat>((packed >> 4) & 0xF);- v0.z = static_cast<bfloat>((packed >> 8) & 0xF);- v0.w = static_cast<bfloat>((packed >> 12) & 0xF);-- // High 4 nibbles- v1.x = static_cast<bfloat>((packed >> 16) & 0xF);- v1.y = static_cast<bfloat>((packed >> 20) & 0xF);- v1.z = static_cast<bfloat>((packed >> 24) & 0xF);- v1.w = static_cast<bfloat>((packed >> 28) & 0xF);-- v0 = v0 * scale + bias;- v1 = v1 * scale + bias;-- threadgroup bfloat4* out_ptr = (threadgroup bfloat4*)w_local;- out_ptr[0] = v0;- out_ptr[1] = v1;-}--template <- typename T,- short BROWS,- short BCOLS,- short dst_ld,- short reduction_dim,- short tgp_size,- short group_size,- short bits>-struct QuantizedBlockLoaderMlx {- static_assert(- BCOLS <= group_size,- "Group size should be larger than columns"- );- static_assert(- group_size % BCOLS == 0,- "Group size should be divisible by columns"- );- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- METAL_CONST short pack_factor = get_pack_factor<bits, 8>();- METAL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();- METAL_CONST short BCOLS_PACKED = BCOLS / pack_factor;- METAL_CONST short n_reads =- (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;- METAL_CONST short group_steps = group_size / BCOLS;-- const int src_ld;- const int tile_stride;- short group_step_cnt;- const int group_stride;-- const short thread_idx;- const short bi;- const short bj;-- threadgroup T* dst;- const device uint8_t* src;- const device T* scales;- const device T* biases;-- QuantizedBlockLoaderMlx(- const device uint8_t* src_,- const device T* scales_,- const device T* biases_,- const int src_ld_,- threadgroup T* dst_,- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]- )- : src_ld(src_ld_),- tile_stride(- reduction_dim ? BCOLS_PACKED * bytes_per_pack- : BROWS * src_ld * bytes_per_pack / pack_factor- ),- group_step_cnt(0), group_stride(BROWS * src_ld / group_size),- thread_idx(simd_group_id * 32 + simd_lane_id),- bi(n_reads * thread_idx / BCOLS_PACKED),- bj((n_reads * thread_idx) % BCOLS_PACKED),- dst(dst_ + bi * dst_ld + bj * pack_factor),- src(src_ + bi * src_ld * bytes_per_pack / pack_factor +- bj * bytes_per_pack),- scales(scales_ + bi * src_ld / group_size),- biases(biases_ + bi * src_ld / group_size) {}-- void load_unsafe() const {- if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {- return;- }-- T scale = *scales;- T bias = *biases;- for (int i = 0; i < n_reads; i++) {- dequantize<T, pack_factor, bits>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );- }- }-- void load_safe(short2 src_tile_dim) const {- if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {- return;- }-- if (reduction_dim == 1 && bi >= src_tile_dim.x) {- for (int i = 0; i < n_reads * pack_factor; i++) {- dst[i] = T(0);- }- return;- }-- if (reduction_dim == 0 && bi >= src_tile_dim.y) {- for (int i = 0; i < n_reads * pack_factor; i++) {- dst[i] = T(0);- }- return;- }-- T scale = *scales;- T bias = *biases;- for (int i = 0; i < n_reads; i++) {- dequantize<T, pack_factor, bits>(- (device uint8_t*)(src + i * bytes_per_pack),- scale,- bias,- dst + i * pack_factor- );- }- }-- void next() {- src += tile_stride;- if (reduction_dim == 1) {- if (group_steps > 1) {- group_step_cnt++;- if (group_step_cnt == group_steps) {- group_step_cnt = 0;- scales++;- biases++;- }- } else {- scales++;- biases++;- }- } else {- scales += group_stride;- biases += group_stride;- }- }-};--template <- typename T,- short BROWS,- short BCOLS,- short dst_ld,- short reduction_dim,- short tgp_size,- short group_size,- short bits,- bool per_output_layout = false>-struct QuantizedBlockLoaderZp {- static_assert(- BCOLS <= group_size,- "Group size should be larger than columns"- );- static_assert(- group_size % BCOLS == 0,- "Group size should be divisible by columns"- );- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- METAL_CONST short pack_factor = get_pack_factor<bits, 8>();- METAL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();- METAL_CONST short BCOLS_PACKED = BCOLS / pack_factor;- METAL_CONST short n_reads =- (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;- METAL_CONST short group_steps = group_size / BCOLS;-- const int src_ld;- const int groups_per_row;- const int tile_stride;- short group_step_cnt;- int k_base;- const int group_stride;-- const short thread_idx;- const short bi;- const short bj;-- threadgroup T* dst;- const device uint8_t* src;- const device T* scales;- const device T* scales_row_start;- const device uint8_t* zps_row_start;- const int out_group_base;- const int out_groups_total;- const int zp_stride_total;-- QuantizedBlockLoaderZp(- const device uint8_t* src_,- const device T* scales_,- const device uint8_t* zero_points_row_start_,- const int src_ld_,- const int groups_per_row_,- threadgroup T* dst_,- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]],- const int out_group_base_ = 0,- const int out_groups_total_ = 0,- const int zp_stride_total_ = 0- )- : src_ld(src_ld_), groups_per_row(groups_per_row_),- tile_stride(- reduction_dim ? BCOLS_PACKED * bytes_per_pack- : BROWS * src_ld * bytes_per_pack / pack_factor- ),- group_step_cnt(0), k_base(0), group_stride(BROWS * groups_per_row_),- thread_idx(simd_group_id * 32 + simd_lane_id),- bi(n_reads * thread_idx / BCOLS_PACKED),- bj((n_reads * thread_idx) % BCOLS_PACKED),- dst(dst_ + bi * dst_ld + bj * pack_factor),- src(src_ + bi * src_ld * bytes_per_pack / pack_factor +- bj * bytes_per_pack),- scales(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),- scales_row_start(- reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_- ),- zps_row_start(- reduction_dim == 1 ? (zero_points_row_start_ +- bi * (bits == 4 ? ((groups_per_row_ + 1) / 2)- : groups_per_row_))- : zero_points_row_start_- ),- out_group_base(per_output_layout ? out_group_base_ : 0),- out_groups_total(per_output_layout ? out_groups_total_ : 0),- zp_stride_total(per_output_layout ? zp_stride_total_ : 0) {}-- inline void current_scale_bias(- thread T& out_scale,- thread T& out_bias- ) const {- uint zp_n;- T scale_val;- if (per_output_layout) {- const int row_idx = k_base + bi;- const int scale_index = row_idx * groups_per_row + out_group_base;- scale_val = scales_row_start[scale_index];- if (bits == 4) {- const int byte_index =- row_idx * zp_stride_total + (out_group_base >> 1);- uint8_t zp_b = zps_row_start[byte_index];- zp_n = (uint(zp_b) >> (uint(out_group_base & 1) * 4u)) & 0x0Fu;- } else {- const int zp_index = row_idx * zp_stride_total + out_group_base;- zp_n = zps_row_start[zp_index];- }- } else {- int g = reduction_dim == 0 ? (k_base / group_size)- : (int)(scales - scales_row_start);- scale_val = reduction_dim == 0 ? scales_row_start[g] : *scales;- if (bits == 4) {- const device uint8_t* zp_ptr = zps_row_start + (g >> 1);- uint8_t zp_b = *zp_ptr;- zp_n = (uint(zp_b) >> (uint(g & 1) * 4u)) & 0x0Fu;- } else {- zp_n = zps_row_start[g];- }- }- out_scale = scale_val;- out_bias = static_cast<T>(-scale_val * static_cast<T>(zp_n));- }-- void load_unsafe() const {- if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {- return;- }-- T scale;- T bias;- current_scale_bias(scale, bias);- for (int i = 0; i < n_reads; i++) {- dequantize<T, pack_factor, bits>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );- }- }-- void load_safe(short2 src_tile_dim) const {- if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {- return;- }-- if (reduction_dim == 1) {- // N-tail: zero out rows beyond valid outputs- if (bi >= src_tile_dim.x) {- for (int i = 0; i < n_reads * pack_factor; i++) {- dst[i] = T(0);- }- return;- }-- int valid_cols = src_tile_dim.y; // 0..BK- int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;-- T scale;- T bias;- current_scale_bias(scale, bias);- for (int i = 0; i < n_reads; i++) {- int pack_idx = bj + i; // global pack index across the BK packs- if (pack_idx < valid_packs) {- dequantize<T, pack_factor, bits>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );-- // Mask the last pack if needed- if (pack_idx == valid_packs - 1) {- int rem = valid_cols - pack_idx * pack_factor;- if (rem < pack_factor) {- for (int r = rem; r < pack_factor; ++r) {- dst[i * pack_factor + r] = T(0);- }- }- }- } else {- for (int j = 0; j < pack_factor; ++j) {- dst[i * pack_factor + j] = T(0);- }- }- }- return;- }-- if (reduction_dim == 0 && bi >= src_tile_dim.y) {- for (int i = 0; i < n_reads * pack_factor; i++) {- dst[i] = T(0);- }- return;- }-- T scale;- T bias;- current_scale_bias(scale, bias);- for (int i = 0; i < n_reads; i++) {- dequantize<T, pack_factor, bits>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );- }- }-- void next() {- src += tile_stride;- if (reduction_dim == 1) {- if (group_steps > 1) {- group_step_cnt++;- if (group_step_cnt == group_steps) {- group_step_cnt = 0;- scales++;- }- } else {- scales++;- }- } else {- k_base += BROWS;- }- }-};--template <- typename LoaderW,- typename LoaderX,- typename Mma,- typename T,- const bool aligned_N,- const int BM,- const int BK,- const int BN>-inline void qmm_transposed_core(- thread LoaderX& loader_x,- thread LoaderW& loader_w,- thread Mma& mma_op,- const short num_els,- const short num_outs,- const int in_vec_size,- device T* output,- const int out_vec_size,- threadgroup T* Xs,- threadgroup T* Ws-) {- if (num_els < BM) {- if (!aligned_N && num_outs < BN) {- for (int k = 0; k < in_vec_size; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_safe(short2(BK, num_els));- loader_w.load_safe(short2(BK, num_outs));- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- loader_x.next();- loader_w.next();- }- } else {- for (int k = 0; k < in_vec_size; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_safe(short2(BK, num_els));- loader_w.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- loader_x.next();- loader_w.next();- }- }- } else {- if (!aligned_N && num_outs < BN) {- for (int k = 0; k < in_vec_size; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_unsafe();- loader_w.load_safe(short2(BK, num_outs));- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- loader_x.next();- loader_w.next();- }- } else {- for (int k = 0; k < in_vec_size; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_unsafe();- loader_w.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_op.mma(Xs, Ws);- loader_x.next();- loader_w.next();- }- }- }-- // Store results to device memory- threadgroup_barrier(mem_flags::mem_threadgroup);- if (num_els < BM || num_outs < BN) {- mma_op.store_result_safe(output, out_vec_size, short2(num_outs, num_els));- } else {- mma_op.store_result(output, out_vec_size);- }-}--template <- typename T,- const uint group_size,- const uint bits,- const bool aligned_N,- const int BM = 32,- const int BK = 32,- const int BN = 32,- const QuantizationMethod quant_method = QuantizationMethod::ScaleZeroPoint,- const int WM = 2,- const int WN = 2>-void qmm_transposed_impl(- const device uint32_t* weights,- const device T* scales,- const device uint8_t* zero_points,- const device T* biases,- const device T* input,- device T* output,- threadgroup T* Xs,- threadgroup T* Ws,- const int in_vec_size,- const int out_vec_size,- const int batch_size,- uint out_block_idx,- uint batch_block_idx,- uint simd_group,- uint simd_lane-) {- static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");- static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");- constexpr int pack_factor = get_pack_factor<bits, 8>();- constexpr int bytes_per_pack = get_bytes_per_pack<bits>();- constexpr int BK_padded = (BK + 16 / sizeof(T));-- using mma_t = matmul_utils::- BlockMMA<T, T, BM, BN, BK, WM, WN, false, true, BK_padded, BK_padded>;- using loader_x_t =- matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;-- const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;- const int in_vec_size_g = (in_vec_size + group_size - 1) / group_size;- const int out_row = batch_block_idx * BM;- const int out_col = out_block_idx * BN;-- auto wl = (const device uint8_t*)weights;-- const device T* x_block = input + out_row * static_cast<int64_t>(in_vec_size);- const device uint8_t* w_block = wl + out_col * in_vec_size_w;- scales += out_col * in_vec_size_g;- biases += out_col * in_vec_size_g;- device T* y_block =- output + out_row * static_cast<int64_t>(out_vec_size) + out_col;-- const short num_els = min(BM, batch_size - out_row);- const short num_outs = min(BN, out_vec_size - out_col);- loader_x_t loader_x(x_block, in_vec_size, Xs, simd_group, simd_lane);- mma_t mma_op(simd_group, simd_lane);-- if (quant_method == QuantizationMethod::ScaleBias) {- using loader_w_t = QuantizedBlockLoaderMlx<- T,- BN,- BK,- BK_padded,- 1,- WM * WN * 32,- group_size,- bits>;- loader_w_t loader_w(- w_block,- scales,- biases,- in_vec_size,- Ws,- simd_group,- simd_lane- );- qmm_transposed_core<- loader_w_t,- loader_x_t,- mma_t,- T,- aligned_N,- BM,- BK,- BN>(- loader_x,- loader_w,- mma_op,- num_els,- num_outs,- in_vec_size,- y_block,- out_vec_size,- Xs,- Ws- );- } else {- using loader_w_t = QuantizedBlockLoaderZp<- T,- BN,- BK,- BK_padded,- 1,- WM * WN * 32,- group_size,- bits>;- const device uint8_t* zero_points_row =- zero_points +- out_col * (bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g);- loader_w_t loader_w(- w_block,- scales,- zero_points_row,- in_vec_size,- in_vec_size_g,- Ws,- simd_group,- simd_lane- );- qmm_transposed_core<- loader_w_t,- loader_x_t,- mma_t,- T,- aligned_N,- BM,- BK,- BN>(- loader_x,- loader_w,- mma_op,- num_els,- num_outs,- in_vec_size,- y_block,- out_vec_size,- Xs,- Ws- );- }-}diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rsindex d00c639d3..955ae6cc1 100644--- a/crates/backend-uzu/src/backends/metal/mod.rs+++ b/crates/backend-uzu/src/backends/metal/mod.rs@@ -13,8 +13,5 @@ pub use backend::Metal;pub use buffer::BufferDowncastExt;pub use context::MetalContext;pub use device_capabilities::MetalDeviceCapabilities;-pub use kernel::matmul::{- MatmulDispatchPath,- quant::{QuantizedMatmulDispatchPath, encode_quantized_matmul_with_path},-};+pub use kernel::matmul::{GemmDispatchPath, GemmKernel};pub use metal_extensions::{DeviceExt, DeviceGeneration};diff --git a/crates/backend-uzu/src/config/embedding.rs b/crates/backend-uzu/src/config/embedding.rsindex 80f43d8e4..69e89a3ff 100644--- a/crates/backend-uzu/src/config/embedding.rs+++ b/crates/backend-uzu/src/config/embedding.rs@@ -23,7 +23,7 @@ pub enum EmbeddingConfig {common: EmbeddingConfigCommon,precision: DataType,},- #[serde(rename = "ScaleBiasQuantizedTiedEmbeddingConfig")]+ #[serde(rename = "MLXQuantizedTiedEmbeddingConfig")]ScaleBiasQuantizedTied {#[serde(flatten)]common: EmbeddingConfigCommon,@@ -32,7 +32,7 @@ pub enum EmbeddingConfig {activation_quantization_mode: Option<QuantizationMode>,activation_precision: DataType,},- #[serde(rename = "ScaleBiasQuantizedUntiedEmbeddingConfig")]+ #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]ScaleBiasQuantizedUntied {#[serde(flatten)]common: EmbeddingConfigCommon,@@ -41,7 +41,7 @@ pub enum EmbeddingConfig {activation_quantization_mode: Option<QuantizationMode>,activation_precision: DataType,},- #[serde(rename = "ScaleBiasSemiQuantizedUntiedEmbeddingConfig")]+ #[serde(rename = "MLXSemiQuantizedUntiedEmbeddingConfig")]ScaleBiasSemiQuantizedUntied {#[serde(flatten)]common: EmbeddingConfigCommon,diff --git a/crates/backend-uzu/src/data_type.rs b/crates/backend-uzu/src/data_type.rsindex 5af111f1a..ccb18332b 100644--- a/crates/backend-uzu/src/data_type.rs+++ b/crates/backend-uzu/src/data_type.rs@@ -3,7 +3,7 @@ use half::{bf16, f16};use num_traits::NumCast;use proc_macros::uzu_config;-#[derive(Eq, PartialOrd, Ord, Copy)]+#[derive(Eq, PartialOrd, Ord, Copy, Hash)]#[uzu_config]pub enum DataType {// Floating pointdiff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rsindex 7d9744439..63d530590 100644--- a/crates/backend-uzu/src/encodable_block/attention.rs+++ b/crates/backend-uzu/src/encodable_block/attention.rs@@ -15,7 +15,7 @@ use crate::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel, ManualKernels,SigmoidGateKernel, SoftmaxKernel,attention::{AttentionGemmArguments, AttentionGemmBlock},- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},config::AttentionConfig,@@ -287,7 +287,7 @@ impl<B: Backend> Attention<B> {max_sequence_length as u32,encoder,);- Ok(values)+ Ok::<_, B::Error>(values)}).transpose()?;@@ -595,19 +595,25 @@ impl<B: Backend> Attention<B> {MatmulArguments {a: queries,a_offset: group_index * gqa_factor * suffix_length * head_dim * dt_bytes,- b: keys,+ b: MatmulB::FullPrecision {+ b: keys,+ },b_offset: group_index * k_head_stride as usize * dt_bytes,b_leading_dimension: Some(k_seq_stride),b_transpose: true,- ab_scale: scale,- c: MatmulArgumentC::None,d: &mut group_scores,- batch_dim: (gqa_factor * suffix_length) as u32,- input_dim: head_dim as u32,- output_dim: sequence_length as u32,+ d_transform: MatmulDOps {+ ab_scale: scale,+ accumulate: false,+ bias: None,+ rht_factors: None,+ },+ m: (gqa_factor * suffix_length) as u32,+ n: sequence_length as u32,+ k: head_dim as u32,},encoder,- );+ )?;scatter_scores.encode(&group_scores,&mut scores,@@ -645,19 +651,20 @@ impl<B: Backend> Attention<B> {MatmulArguments {a: &scores,a_offset: group_index * gqa_factor * suffix_length * sequence_length * dt_bytes,- b: values,+ b: MatmulB::FullPrecision {+ b: values,+ },b_offset: group_index * v_head_stride as usize * dt_bytes,b_leading_dimension: Some(v_seq_stride),b_transpose: false,- ab_scale: 1.0,- c: MatmulArgumentC::None,d: &mut group_output,- batch_dim: (gqa_factor * suffix_length) as u32,- input_dim: sequence_length as u32,- output_dim: head_dim as u32,+ d_transform: MatmulDOps::none(),+ m: (gqa_factor * suffix_length) as u32,+ n: head_dim as u32,+ k: sequence_length as u32,},encoder,- );+ )?;self.fallback_scatter_values_kernel.encode(&group_output,&mut *attention_output,diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 1b0897e02..365eb3ff4 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -6,14 +6,10 @@ use crate::{DataType,backends::common::{Allocation, Backend, Encoder, Kernels,- gpu_types::QuantizationMethod,+ gpu_types::{QuantizationMethod, QuantizationMode},kernel::{FullPrecisionEmbeddingLookupKernel, ManualKernels, QuantizedEmbeddingLookupKernel,- matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},- quant_matmul::{- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,- QuantizedMatmulKernelEncodable,- },+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},},},config::EmbeddingConfig,@@ -24,16 +20,17 @@ use crate::{pub enum EmbeddingError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),- #[error("Matmul error: {0}")]- MatmulError(#[from] MatmulError<B>),- #[error("QuantizedMatmul error: {0}")]- QuantizedMatmulError(#[from] QuantizedMatmulError<B>),#[error("Parameter loading error: {0}")]ParameterError(#[from] ParameterLoaderError<B>),#[error("Unsupported configuration: {0}")]UnsupportedConfiguration(String),}+struct ReadoutQuantConfig {+ mode: QuantizationMode,+ group_size: u32,+}+enum TiedEmbeddingType<B: Backend> {FullPrecision {weights: Allocation<B>,@@ -45,7 +42,8 @@ enum TiedEmbeddingType<B: Backend> {scales: Allocation<B>,biases: Allocation<B>,lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,- readout: QuantizedMatmulKernelEncodable<B>,+ readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,+ readout_config: ReadoutQuantConfig,},}@@ -71,7 +69,8 @@ enum UntiedEmbeddingReadoutType<B: Backend> {weights: Allocation<B>,scales: Allocation<B>,biases: Allocation<B>,- readout: QuantizedMatmulKernelEncodable<B>,+ readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,+ readout_config: ReadoutQuantConfig,},}@@ -184,7 +183,10 @@ impl<B: Backend> Embedding<B> {let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(<B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)?);+ let readout = RefCell::new(+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)+ .map_err(EmbeddingError::BackendError)?,+ );EmbeddingTying::Tied {ty: TiedEmbeddingType::FullPrecision {@@ -210,7 +212,10 @@ impl<B: Backend> Embedding<B> {let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(<B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)?);+ let readout = RefCell::new(+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)+ .map_err(EmbeddingError::BackendError)?,+ );EmbeddingTying::Untied {input_ty: UntiedEmbeddingLookupType::FullPrecision {@@ -256,18 +261,23 @@ impl<B: Backend> Embedding<B> {(*embedding_quantization_mode).into(),).map_err(EmbeddingError::BackendError)?;- let readout = QuantizedMatmulKernelEncodable::new(- context,- QuantizedMatmulConfiguration {- data_type,- group_size: *group_size,- input_dim: model_dim as usize,- output_dim: vocab_size as usize,- mode: *embedding_quantization_mode,- quantization_method: QuantizationMethod::ScaleBias,- use_hadamard: false,- },- )?;+ let readout_config = ReadoutQuantConfig {+ mode: *embedding_quantization_mode,+ group_size: *group_size as u32,+ };+ let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)+ .map_err(EmbeddingError::BackendError)?;+ readout_kernel+ .preheat_quant_combo(+ context,+ MatmulQuantCombo {+ method: QuantizationMethod::ScaleBias,+ mode: *embedding_quantization_mode,+ group_size: *group_size as u32,+ },+ )+ .map_err(EmbeddingError::BackendError)?;+ let readout = RefCell::new(readout_kernel);if let Some(activation_quantization_mode) = activation_quantization_mode {return Err(EmbeddingError::UnsupportedConfiguration(format!(@@ -282,6 +292,7 @@ impl<B: Backend> Embedding<B> {biases,lookup,readout,+ readout_config,},}},@@ -330,18 +341,23 @@ impl<B: Backend> Embedding<B> {(*embedding_quantization_mode).into(),).map_err(EmbeddingError::BackendError)?;- let readout = QuantizedMatmulKernelEncodable::new(- context,- QuantizedMatmulConfiguration {- data_type,- group_size: *group_size,- input_dim: model_dim as usize,- output_dim: vocab_size as usize,- mode: *embedding_quantization_mode,- quantization_method: QuantizationMethod::ScaleBias,- use_hadamard: false,- },- )?;+ let readout_config = ReadoutQuantConfig {+ mode: *embedding_quantization_mode,+ group_size: *group_size as u32,+ };+ let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)+ .map_err(EmbeddingError::BackendError)?;+ readout_kernel+ .preheat_quant_combo(+ context,+ MatmulQuantCombo {+ method: QuantizationMethod::ScaleBias,+ mode: *embedding_quantization_mode,+ group_size: *group_size as u32,+ },+ )+ .map_err(EmbeddingError::BackendError)?;+ let readout = RefCell::new(readout_kernel);if let Some(activation_quantization_mode) = activation_quantization_mode {return Err(EmbeddingError::UnsupportedConfiguration(format!(@@ -361,6 +377,7 @@ impl<B: Backend> Embedding<B> {scales: output_scales,biases: output_biases,readout,+ readout_config,},}},@@ -397,18 +414,23 @@ impl<B: Backend> Embedding<B> {let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;- let readout = QuantizedMatmulKernelEncodable::new(- context,- QuantizedMatmulConfiguration {- data_type,- group_size: *group_size,- input_dim: model_dim as usize,- output_dim: vocab_size as usize,- mode: *embedding_quantization_mode,- quantization_method: QuantizationMethod::ScaleBias,- use_hadamard: false,- },- )?;+ let readout_config = ReadoutQuantConfig {+ mode: *embedding_quantization_mode,+ group_size: *group_size as u32,+ };+ let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)+ .map_err(EmbeddingError::BackendError)?;+ readout_kernel+ .preheat_quant_combo(+ context,+ MatmulQuantCombo {+ method: QuantizationMethod::ScaleBias,+ mode: *embedding_quantization_mode,+ group_size: *group_size as u32,+ },+ )+ .map_err(EmbeddingError::BackendError)?;+ let readout = RefCell::new(readout_kernel);if let Some(activation_quantization_mode) = activation_quantization_mode {return Err(EmbeddingError::UnsupportedConfiguration(format!(@@ -426,6 +448,7 @@ impl<B: Backend> Embedding<B> {scales: output_scales,biases: output_biases,readout,+ readout_config,},}},@@ -495,6 +518,7 @@ impl<B: Backend> Embedding<B> {biases,lookup,readout: _,+ readout_config: _,},}| EmbeddingTying::Untied {@@ -558,23 +582,27 @@ impl<B: Backend> Embedding<B> {} => {let input_dim = self.model_dim as usize;let output_dim = self.vocab_size as usize;- readout.borrow_mut().encode(- MatmulArguments {- a: input_allocation,- a_offset: 0,- b: weights,- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: 1.0,- c: MatmulArgumentC::None,- d: &mut output_allocation,- batch_dim: batch_dim as u32,- input_dim: input_dim as u32,- output_dim: output_dim as u32,- },- encoder,- );+ readout+ .borrow_mut()+ .encode(+ MatmulArguments {+ a: input_allocation,+ a_offset: 0,+ b: MatmulB::FullPrecision {+ b: weights,+ },+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut output_allocation,+ d_transform: MatmulDOps::none(),+ m: batch_dim as u32,+ n: output_dim as u32,+ k: input_dim as u32,+ },+ encoder,+ )+ .map_err(EmbeddingError::BackendError)?;},EmbeddingTying::Tied {ty:@@ -584,6 +612,7 @@ impl<B: Backend> Embedding<B> {biases,lookup: _,readout,+ readout_config,},}| EmbeddingTying::Untied {@@ -594,21 +623,35 @@ impl<B: Backend> Embedding<B> {scales,biases,readout,+ readout_config,},} => {- readout.encode(- encoder,- QuantizedMatmulArguments {- a: input_allocation,- a_offset: 0,- b: weights,- scales,- zero_points_or_biases: biases,- output: &mut output_allocation,- hadamard_factors: None,- batch_dim,- },- );+ let b_variant: MatmulB<'_, B> = MatmulB::ScaleBiasDequant {+ b: weights,+ scales,+ biases,+ mode: readout_config.mode,+ group_size: readout_config.group_size,+ };+ readout+ .borrow_mut()+ .encode(+ MatmulArguments {+ a: input_allocation,+ a_offset: 0,+ b: b_variant,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut output_allocation,+ d_transform: MatmulDOps::none(),+ m: batch_dim as u32,+ n: self.vocab_size,+ k: self.model_dim,+ },+ encoder,+ )+ .map_err(EmbeddingError::BackendError)?;},};diff --git a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs b/crates/backend-uzu/src/encodable_block/linear/full_precision.rsdeleted file mode 100644index 7c5e1063e..000000000--- a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs+++ /dev/null@@ -1,153 +0,0 @@-use std::cell::RefCell;--use thiserror::Error;--use super::Linear;-use crate::{- DataType,- array::size_for_shape,- backends::common::{- Allocation, Backend, Encoder,- kernel::{- ManualKernels,- matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},- },- },- parameters::{ParameterLoaderError, ParameterTree},-};--#[derive(Debug, Error)]-pub enum FullPrecisionLinearError<B: Backend> {- #[error("Matmul error: {0}")]- MatmulError(#[from] MatmulError<B>),- #[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError<B>),- #[error("Unsupported data type for full precision linear kernel: {0:?}")]- UnsupportedDataType(DataType),- #[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]- InvalidWeightsShape {- got: Box<[usize]>,- expected_output_dim: usize,- expected_input_dim: usize,- },- #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidWeightsDataType {- expected: DataType,- got: DataType,- },- #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]- InvalidBiasShape {- got: Box<[usize]>,- expected_output_dim: usize,- },- #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidBiasDataType {- expected: DataType,- got: DataType,- },-}--pub struct FullPrecisionLinear<B: Backend> {- kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,- bias: Option<Allocation<B>>,- weights: Allocation<B>,- input_dim: usize,- output_dim: usize,- precision: DataType,-}--impl<B: Backend> FullPrecisionLinear<B> {- pub fn new(- context: &B::Context,- precision: DataType,- input_dim: usize,- output_dim: usize,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, FullPrecisionLinearError<B>> {- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(FullPrecisionLinearError::UnsupportedDataType(precision));- }-- let weights_leaf = parameter_tree.leaf("weights").map_err(FullPrecisionLinearError::ParameterError)?;- let weights_shape = weights_leaf.shape().to_vec();- if weights_shape != [output_dim, input_dim] {- return Err(FullPrecisionLinearError::InvalidWeightsShape {- got: weights_shape.into_boxed_slice(),- expected_output_dim: output_dim,- expected_input_dim: input_dim,- });- }-- if weights_leaf.data_type() != precision {- return Err(FullPrecisionLinearError::InvalidWeightsDataType {- expected: precision,- got: weights_leaf.data_type(),- });- }-- let bias = match parameter_tree.leaf("biases") {- Ok(biases_leaf) => {- let bias_shape = biases_leaf.shape().to_vec();- if bias_shape != [output_dim] {- return Err(FullPrecisionLinearError::InvalidBiasShape {- got: bias_shape.into_boxed_slice(),- expected_output_dim: output_dim,- });- }-- if biases_leaf.data_type() != precision {- return Err(FullPrecisionLinearError::InvalidBiasDataType {- expected: precision,- got: biases_leaf.data_type(),- });- }-- Some(biases_leaf.read_allocation().map_err(FullPrecisionLinearError::ParameterError)?)- },- Err(_) => None,- };-- let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, precision)?;-- Ok(Self {- kernel: RefCell::new(kernel),- bias,- weights: weights_leaf.read_allocation().map_err(FullPrecisionLinearError::ParameterError)?,- input_dim,- output_dim,- precision,- })- }-}--impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {- fn encode(- &self,- input: Allocation<B>,- batch_dim: usize,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {- let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.precision))?;- self.kernel.borrow_mut().encode(- MatmulArguments {- a: &input,- a_offset: 0,- b: &self.weights,- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: 1.0,- c: match self.bias.as_ref() {- Some(bias) => MatmulArgumentC::Bias(bias),- None => MatmulArgumentC::None,- },- d: &mut output,- batch_dim: batch_dim as u32,- input_dim: self.input_dim as u32,- output_dim: self.output_dim as u32,- },- encoder,- );- Ok(output)- }-}diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsnew file mode 100644index 000000000..8524b3e7f--- /dev/null+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -0,0 +1,368 @@+use std::cell::RefCell;++use thiserror::Error;++use super::Linear;+use crate::{+ DataType,+ array::size_for_shape,+ backends::common::{+ Allocation, Backend, Encoder,+ gpu_types::{QuantizationMethod, QuantizationMode},+ kernel::{+ ManualKernels,+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},+ },+ },+ config::QuantizationConfig,+ parameters::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum LinearMatmulError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loading error: {0}")]+ ParameterError(#[from] ParameterLoaderError<B>),+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+ #[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]+ InvalidFpWeightsShape {+ got: Box<[usize]>,+ expected_output_dim: usize,+ expected_input_dim: usize,+ },+ #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidWeightsDataType {+ expected: DataType,+ got: DataType,+ },+ #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidScalesDataType {+ expected: DataType,+ got: DataType,+ },+ #[error(+ "Unexpected scale-bias shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"+ )]+ InvalidScaleBiasShapes {+ weights: Box<[usize]>,+ scales: Box<[usize]>,+ deq_biases: Box<[usize]>,+ packing_divisor: usize,+ },+ #[error("deq_biases dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidDeqBiasesDataType {+ expected: DataType,+ got: DataType,+ },+ #[error(+ "Unexpected scale-zero-point shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"+ )]+ InvalidScaleZeroPointShapes {+ weights: Box<[usize]>,+ scales: Box<[usize]>,+ zero_points: Box<[usize]>,+ packing_divisor: usize,+ packing_minus_one: usize,+ },+ #[error("Zero-points dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidZeroPointsDataType {+ expected: DataType,+ got: DataType,+ },+ #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]+ InvalidBiasShape {+ got: Box<[usize]>,+ expected_output_dim: usize,+ },+ #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidBiasDataType {+ expected: DataType,+ got: DataType,+ },+}++enum Mode<B: Backend> {+ FullPrecision,+ Quantized {+ method: QuantizationMethod,+ mode: QuantizationMode,+ group_size: u32,+ scales: Allocation<B>,+ zero_points_or_biases: Allocation<B>,+ output_hadamard_factors: Option<Allocation<B>>,+ },+}++pub struct LinearMatmul<B: Backend> {+ kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,+ weights: Allocation<B>,+ biases: Option<Allocation<B>>,+ input_dim: usize,+ output_dim: usize,+ data_type: DataType,+ mode: Mode<B>,+}++impl<B: Backend> LinearMatmul<B> {+ pub fn full_precision(+ context: &B::Context,+ precision: DataType,+ input_dim: usize,+ output_dim: usize,+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, LinearMatmulError<B>> {+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(LinearMatmulError::UnsupportedDataType(precision));+ }++ let weights_leaf = parameter_tree.leaf("weights")?;+ let weights_shape = weights_leaf.shape().to_vec();+ if weights_shape != [output_dim, input_dim] {+ return Err(LinearMatmulError::InvalidFpWeightsShape {+ got: weights_shape.into_boxed_slice(),+ expected_output_dim: output_dim,+ expected_input_dim: input_dim,+ });+ }+ if weights_leaf.data_type() != precision {+ return Err(LinearMatmulError::InvalidWeightsDataType {+ expected: precision,+ got: weights_leaf.data_type(),+ });+ }++ let biases = load_biases(precision, output_dim, parameter_tree)?;++ let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, precision)+ .map_err(LinearMatmulError::BackendError)?;+ let weights = weights_leaf.read_allocation()?;++ Ok(Self {+ kernel: RefCell::new(kernel),+ weights,+ biases,+ input_dim,+ output_dim,+ data_type: precision,+ mode: Mode::FullPrecision,+ })+ }++ pub fn quantized(+ context: &B::Context,+ config: &QuantizationConfig,+ input_dim: usize,+ output_dim: usize,+ parameter_tree: &ParameterTree<B::Context>,+ output_hadamard_factors: Option<Allocation<B>>,+ ) -> Result<Self, LinearMatmulError<B>> {+ let data_type: DataType = config.activation_precision.into();+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(LinearMatmulError::UnsupportedDataType(data_type));+ }++ let packing_divisor = config.weight_quantization_mode.packing_divisor();+ let storage_type = config.weight_quantization_mode.storage_type();++ let weights_leaf = parameter_tree.leaf("weights")?;+ if weights_leaf.data_type() != storage_type {+ return Err(LinearMatmulError::InvalidWeightsDataType {+ expected: storage_type,+ got: weights_leaf.data_type(),+ });+ }++ let scales_leaf = parameter_tree.leaf("scales")?;+ if scales_leaf.data_type() != data_type {+ return Err(LinearMatmulError::InvalidScalesDataType {+ expected: data_type,+ got: scales_leaf.data_type(),+ });+ }++ let k_g = input_dim.div_ceil(config.group_size);+ let weights_shape = weights_leaf.shape().to_vec();+ let scales_shape = scales_leaf.shape().to_vec();++ let (quantization_method, zero_points_or_biases) = match parameter_tree.leaf("deq_biases") {+ Ok(deq_biases) => {+ let deq_biases_shape = deq_biases.shape().to_vec();+ if !(weights_shape == [output_dim, input_dim / packing_divisor]+ && scales_shape == [output_dim, k_g]+ && deq_biases_shape == [output_dim, k_g])+ {+ return Err(LinearMatmulError::InvalidScaleBiasShapes {+ weights: weights_shape.into_boxed_slice(),+ scales: scales_shape.into_boxed_slice(),+ deq_biases: deq_biases_shape.into_boxed_slice(),+ packing_divisor,+ });+ }+ if deq_biases.data_type() != data_type {+ return Err(LinearMatmulError::InvalidDeqBiasesDataType {+ expected: data_type,+ got: deq_biases.data_type(),+ });+ }+ (QuantizationMethod::ScaleBias, deq_biases.read_allocation()?)+ },+ Err(_) => {+ let zero_points_leaf = parameter_tree.leaf("zero_points")?;+ let zero_points_shape = zero_points_leaf.shape().to_vec();+ let expected_zero_points_entries = k_g.div_ceil(packing_divisor);+ if !(weights_shape == [output_dim, input_dim / packing_divisor]+ && scales_shape == [output_dim, k_g]+ && zero_points_shape == [output_dim, expected_zero_points_entries])+ {+ return Err(LinearMatmulError::InvalidScaleZeroPointShapes {+ weights: weights_shape.into_boxed_slice(),+ scales: scales_shape.into_boxed_slice(),+ zero_points: zero_points_shape.into_boxed_slice(),+ packing_divisor,+ packing_minus_one: packing_divisor - 1,+ });+ }+ if zero_points_leaf.data_type() != storage_type {+ return Err(LinearMatmulError::InvalidZeroPointsDataType {+ expected: storage_type,+ got: zero_points_leaf.data_type(),+ });+ }+ (QuantizationMethod::ScaleZeroPoint, zero_points_leaf.read_allocation()?)+ },+ };++ let biases = load_biases(data_type, output_dim, parameter_tree)?;++ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)+ .map_err(LinearMatmulError::BackendError)?;+ kernel+ .preheat_quant_combo(+ context,+ MatmulQuantCombo {+ method: quantization_method,+ mode: config.weight_quantization_mode,+ group_size: config.group_size as u32,+ },+ )+ .map_err(LinearMatmulError::BackendError)?;++ Ok(Self {+ kernel: RefCell::new(kernel),+ weights: weights_leaf.read_allocation()?,+ biases,+ input_dim,+ output_dim,+ data_type,+ mode: Mode::Quantized {+ method: quantization_method,+ mode: config.weight_quantization_mode,+ group_size: config.group_size as u32,+ scales: scales_leaf.read_allocation()?,+ zero_points_or_biases,+ output_hadamard_factors,+ },+ })+ }+}++fn load_biases<B: Backend>(+ data_type: DataType,+ output_dim: usize,+ parameter_tree: &ParameterTree<B::Context>,+) -> Result<Option<Allocation<B>>, LinearMatmulError<B>> {+ match parameter_tree.leaf("biases") {+ Ok(biases_leaf) => {+ let bias_shape = biases_leaf.shape().to_vec();+ if bias_shape != [output_dim] {+ return Err(LinearMatmulError::InvalidBiasShape {+ got: bias_shape.into_boxed_slice(),+ expected_output_dim: output_dim,+ });+ }+ if biases_leaf.data_type() != data_type {+ return Err(LinearMatmulError::InvalidBiasDataType {+ expected: data_type,+ got: biases_leaf.data_type(),+ });+ }+ Ok(Some(biases_leaf.read_allocation()?))+ },+ Err(_) => Ok(None),+ }+}++impl<B: Backend> Linear<B> for LinearMatmul<B> {+ fn encode(+ &self,+ input: Allocation<B>,+ batch_dim: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.data_type))?;++ let b = match &self.mode {+ Mode::FullPrecision => MatmulB::FullPrecision {+ b: &self.weights,+ },+ Mode::Quantized {+ method,+ mode,+ group_size,+ scales,+ zero_points_or_biases,+ ..+ } => match method {+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {+ b: &self.weights,+ scales,+ biases: zero_points_or_biases,+ mode: *mode,+ group_size: *group_size,+ },+ QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {+ b: &self.weights,+ scales,+ zero_points: zero_points_or_biases,+ mode: *mode,+ group_size: *group_size,+ },+ },+ };++ let rht_factors = match &self.mode {+ Mode::Quantized {+ output_hadamard_factors: Some(factors),+ ..+ } => Some(factors),+ _ => None,+ };+ let d_transform = MatmulDOps {+ ab_scale: 1.0,+ accumulate: false,+ bias: self.biases.as_ref(),+ rht_factors,+ };++ self.kernel.borrow_mut().encode(+ MatmulArguments {+ a: &input,+ a_offset: 0,+ b,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut output,+ d_transform,+ m: batch_dim as u32,+ n: self.output_dim as u32,+ k: self.input_dim as u32,+ },+ encoder,+ )?;++ Ok(output)+ }+}diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex 57eb8995a..a7056b36c 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -1,11 +1,9 @@-mod full_precision;+mod matmul;mod qlora_wrapper;-mod quantized;mod rht_wrapper;-pub use full_precision::{FullPrecisionLinear, FullPrecisionLinearError};+pub use matmul::{LinearMatmul, LinearMatmulError};pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};-pub use quantized::{QuantizedLinear, QuantizedLinearError};pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};use thiserror::Error;@@ -26,10 +24,8 @@ pub trait Linear<B: Backend> {#[derive(Debug, Error)]pub enum LinearBlockError<B: Backend> {- #[error("QuantizedLinear error: {0}")]- QuantizedLinearError(#[from] QuantizedLinearError<B>),- #[error("FullPrecisionLinear error: {0}")]- FullPrecisionLinearError(#[from] FullPrecisionLinearError<B>),+ #[error("LinearMatmul error: {0}")]+ LinearMatmulError(#[from] LinearMatmulError<B>),#[error("QLoRALinearWrapper error: {0}")]QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),#[error("RHTLinearWrapper error: {0}")]@@ -49,7 +45,7 @@ impl<B: Backend> dyn Linear<B> {let output_dimension_sum: usize = output_dimensions.iter().sum();match config {LinearConfig::Quantized(quantization_config) | LinearConfig::ScaleBiasQuantized(quantization_config) => {- let block = QuantizedLinear::new(+ let block = LinearMatmul::quantized(context,quantization_config,input_dimension,@@ -62,7 +58,7 @@ impl<B: Backend> dyn Linear<B> {LinearConfig::FullPrecision {precision,} => {- let block = FullPrecisionLinear::new(+ let block = LinearMatmul::full_precision(context,(*precision).into(),input_dimension,@@ -112,7 +108,7 @@ impl<B: Backend> dyn Linear<B> {) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {match config {LinearConfig::Quantized(config) | LinearConfig::ScaleBiasQuantized(config) => Ok(Box::new(- QuantizedLinear::new(context, config, input_dim, output_dim, parameter_tree, Some(output_factors))?,+ LinearMatmul::quantized(context, config, input_dim, output_dim, parameter_tree, Some(output_factors))?,)),LinearConfig::QLoRA {quantization,diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex 08b5516f7..c3fae59ec 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -9,13 +9,13 @@ use crate::{Allocation, Backend, Encoder,kernel::{ManualKernels,- matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},config::QuantizationConfig,encodable_block::{Linear,- linear::{LinearBlockError, QuantizedLinear, QuantizedLinearError},+ linear::{LinearBlockError, LinearMatmul, LinearMatmulError},},prelude::{ParameterLoaderError, ParameterTree},};@@ -24,16 +24,16 @@ use crate::{pub enum QLoRALinearWrapperError<B: Backend> {#[error("Inner linear error: {0}")]InnerLinearError(#[from] Box<LinearBlockError<B>>),- #[error("Quantized linear error: {0}")]- QuantizedLinearError(#[from] QuantizedLinearError<B>),+ #[error("Linear matmul error: {0}")]+ LinearMatmulError(#[from] LinearMatmulError<B>),#[error("Parameter loader error: {0}")]ParameterLoaderError(#[from] ParameterLoaderError<B>),- #[error("Matmul error: {0}")]- MatmulError(#[from] MatmulError<B>),+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),}pub struct QLoRALinearWrapper<B: Backend> {- base_linear: QuantizedLinear<B>,+ base_linear: LinearMatmul<B>,adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,adapter_down: Allocation<B>,adapter_up: Allocation<B>,@@ -57,7 +57,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {) -> Result<Self, QLoRALinearWrapperError<B>> {let data_type = quantization.activation_precision.into();- let base_linear = QuantizedLinear::new(+ let base_linear = LinearMatmul::quantized(context,quantization,input_dim,@@ -65,8 +65,10 @@ impl<B: Backend> QLoRALinearWrapper<B> {parameter_tree,output_quantized_hadamard_factors,)?;- let adapter_kernel =- RefCell::new(<<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)?);+ let adapter_kernel = RefCell::new(+ <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)+ .map_err(QLoRALinearWrapperError::BackendError)?,+ );let adapter_down_leaf = parameter_tree.leaf("down_weights")?;adapter_down_leaf.validate_shape(&[lora_rank, input_dim], data_type)?;@@ -105,19 +107,20 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {MatmulArguments {a: &input,a_offset: 0,- b: &self.adapter_down,+ b: MatmulB::FullPrecision {+ b: &self.adapter_down,+ },b_offset: 0,b_leading_dimension: None,b_transpose: true,- ab_scale: 1.0,- c: MatmulArgumentC::None,d: &mut intermediate,- batch_dim: batch_dim as u32,- input_dim: self.input_dim as u32,- output_dim: self.lora_rank as u32,+ d_transform: MatmulDOps::none(),+ m: batch_dim as u32,+ n: self.lora_rank as u32,+ k: self.input_dim as u32,},encoder,- );+ )?;let mut output = self.base_linear.encode(input, batch_dim, encoder)?;@@ -125,19 +128,25 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {MatmulArguments {a: &intermediate,a_offset: 0,- b: &self.adapter_up,+ b: MatmulB::FullPrecision {+ b: &self.adapter_up,+ },b_offset: 0,b_leading_dimension: None,b_transpose: true,- ab_scale: self.lora_scale,- c: MatmulArgumentC::Accumulate,d: &mut output,- batch_dim: batch_dim as u32,- input_dim: self.lora_rank as u32,- output_dim: self.output_dim as u32,+ d_transform: MatmulDOps {+ ab_scale: self.lora_scale,+ accumulate: true,+ bias: None,+ rht_factors: None,+ },+ m: batch_dim as u32,+ n: self.output_dim as u32,+ k: self.lora_rank as u32,},encoder,- );+ )?;Ok(output)}diff --git a/crates/backend-uzu/src/encodable_block/linear/quantized.rs b/crates/backend-uzu/src/encodable_block/linear/quantized.rsdeleted file mode 100644index 510605d86..000000000--- a/crates/backend-uzu/src/encodable_block/linear/quantized.rs+++ /dev/null@@ -1,283 +0,0 @@-use thiserror::Error;--use super::Linear;-use crate::{- DataType,- array::size_for_shape,- backends::common::{- Allocation, Backend, Encoder,- gpu_types::QuantizationMethod,- kernel::{- Kernels, TensorAddBiasKernel,- quant_matmul::{- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,- QuantizedMatmulKernelEncodable,- },- },- },- config::QuantizationConfig,- parameters::{ParameterLoaderError, ParameterTree},-};--#[derive(Debug, Error)]-pub enum QuantizedLinearError<B: Backend> {- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("QuantizedMatmul error: {0}")]- QuantizedMatmulError(#[source] QuantizedMatmulError<B>),- #[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError<B>),- #[error("Unsupported data type for quantized kernel: {0:?}")]- UnsupportedDataType(DataType),- #[error("Expected weights of type {expected:?}, got {got:?}")]- InvalidWeightsDataType {- expected: DataType,- got: DataType,- },- #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidScalesDataType {- expected: DataType,- got: DataType,- },- #[error(- "Unexpected scale-bias shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"- )]- InvalidScaleBiasShapes {- weights: Box<[usize]>,- scales: Box<[usize]>,- deq_biases: Box<[usize]>,- packing_divisor: usize,- },- #[error("deq_biases dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidDeqBiasesDataType {- expected: DataType,- got: DataType,- },- #[error(- "Unexpected scale-zero-point shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"- )]- InvalidScaleZeroPointShapes {- weights: Box<[usize]>,- scales: Box<[usize]>,- zero_points: Box<[usize]>,- packing_divisor: usize,- packing_minus_one: usize,- },- #[error("Zero-points dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidZeroPointsDataType {- expected: DataType,- got: DataType,- },- #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]- InvalidBiasShape {- got: Box<[usize]>,- expected_output_dim: usize,- },- #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidBiasDataType {- expected: DataType,- got: DataType,- },-}--pub struct QuantizedLinear<B: Backend> {- kernel: QuantizedMatmulKernelEncodable<B>,- bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,- biases: Option<Allocation<B>>,- weights: Allocation<B>,- scales: Allocation<B>,- zero_points_or_biases: Allocation<B>,- output_hadamard_factors: Option<Allocation<B>>,- output_dim: usize,- output_data_type: DataType,-}--impl<B: Backend> QuantizedLinear<B> {- pub fn new(- context: &B::Context,- config: &QuantizationConfig,- input_dim: usize,- output_dim: usize,- parameter_tree: &ParameterTree<B::Context>,- output_hadamard_factors: Option<Allocation<B>>,- ) -> Result<Self, QuantizedLinearError<B>> {- let kernel_data_type: DataType = config.activation_precision.into();- if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(QuantizedLinearError::UnsupportedDataType(kernel_data_type));- }-- let weights_leaf = parameter_tree.leaf("weights").map_err(QuantizedLinearError::ParameterError)?;- let packing_divisor = config.weight_quantization_mode.packing_divisor();- let storage_type = config.weight_quantization_mode.storage_type();- if weights_leaf.data_type() != storage_type {- return Err(QuantizedLinearError::InvalidWeightsDataType {- expected: storage_type,- got: weights_leaf.data_type(),- });- }-- let scales_leaf = parameter_tree.leaf("scales").map_err(QuantizedLinearError::ParameterError)?;- if scales_leaf.data_type() != kernel_data_type {- return Err(QuantizedLinearError::InvalidScalesDataType {- expected: kernel_data_type,- got: scales_leaf.data_type(),- });- }-- let k_g = (input_dim + config.group_size - 1) / config.group_size;- let weights_shape = weights_leaf.shape().to_vec();- let scales_shape = scales_leaf.shape().to_vec();- let (quantization_method, zero_points_or_biases) = match parameter_tree.leaf("deq_biases") {- Ok(deq_biases) => {- let deq_biases_shape = deq_biases.shape().to_vec();- if !(weights_shape == [output_dim, input_dim / packing_divisor]- && scales_shape == [output_dim, k_g]- && deq_biases_shape == [output_dim, k_g])- {- return Err(QuantizedLinearError::InvalidScaleBiasShapes {- weights: weights_shape.into_boxed_slice(),- scales: scales_shape.into_boxed_slice(),- deq_biases: deq_biases_shape.into_boxed_slice(),- packing_divisor,- });- }-- if deq_biases.data_type() != kernel_data_type {- return Err(QuantizedLinearError::InvalidDeqBiasesDataType {- expected: kernel_data_type,- got: deq_biases.data_type(),- });- }-- (- QuantizationMethod::ScaleBias,- deq_biases.read_allocation().map_err(QuantizedLinearError::ParameterError)?,- )- },- Err(_) => {- let zero_points_leaf =- parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;- let zero_points_shape = zero_points_leaf.shape().to_vec();- let expected_zero_points_entries = (k_g + packing_divisor - 1) / packing_divisor;- if !(weights_shape == [output_dim, input_dim / packing_divisor]- && scales_shape == [output_dim, k_g]- && zero_points_shape == [output_dim, expected_zero_points_entries])- {- return Err(QuantizedLinearError::InvalidScaleZeroPointShapes {- weights: weights_shape.into_boxed_slice(),- scales: scales_shape.into_boxed_slice(),- zero_points: zero_points_shape.into_boxed_slice(),- packing_divisor,- packing_minus_one: packing_divisor - 1,- });- }-- if zero_points_leaf.data_type() != storage_type {- return Err(QuantizedLinearError::InvalidZeroPointsDataType {- expected: storage_type,- got: zero_points_leaf.data_type(),- });- }-- (- QuantizationMethod::ScaleZeroPoint,- zero_points_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,- )- },- };-- let (bias_add_kernel, biases) = match parameter_tree.leaf("biases") {- Ok(biases_leaf) => {- let bias_shape = biases_leaf.shape().to_vec();- if bias_shape != [output_dim] {- return Err(QuantizedLinearError::InvalidBiasShape {- got: bias_shape.into_boxed_slice(),- expected_output_dim: output_dim,- });- }-- if biases_leaf.data_type() != kernel_data_type {- return Err(QuantizedLinearError::InvalidBiasDataType {- expected: kernel_data_type,- got: biases_leaf.data_type(),- });- }-- let bias_add_kernel =- <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, kernel_data_type, true)- .map_err(QuantizedLinearError::BackendError)?;- (- Some(bias_add_kernel),- Some(biases_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?),- )- },- Err(_) => (None, None),- };-- let kernel = QuantizedMatmulKernelEncodable::new(- context,- QuantizedMatmulConfiguration {- data_type: kernel_data_type,- group_size: config.group_size,- input_dim,- output_dim,- mode: config.weight_quantization_mode,- quantization_method,- use_hadamard: output_hadamard_factors.is_some(),- },- )- .map_err(QuantizedLinearError::QuantizedMatmulError)?;-- Ok(Self {- kernel,- bias_add_kernel,- biases,- weights: weights_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,- scales: scales_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,- zero_points_or_biases,- output_hadamard_factors,- output_dim,- output_data_type: kernel_data_type,- })- }-}--impl<B: Backend> Linear<B> for QuantizedLinear<B> {- fn encode(- &self,- input: Allocation<B>,- batch_dim: usize,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {- let mut output =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;-- self.kernel.encode(- encoder,- QuantizedMatmulArguments {- a: &input,- a_offset: 0,- b: &self.weights,- scales: &self.scales,- zero_points_or_biases: &self.zero_points_or_biases,- output: &mut output,- hadamard_factors: self.output_hadamard_factors.as_ref(),- batch_dim,- },- );-- if let (Some(bias_add_kernel), Some(biases)) = (&self.bias_add_kernel, &self.biases) {- let total_length = batch_dim * self.output_dim;- bias_add_kernel.encode(- None::<&Allocation<B>>,- biases,- &mut output,- self.output_dim as u32,- total_length as u32,- encoder,- );- }-- Ok(output)- }-}diff --git a/crates/backend-uzu/src/session/tts_session.rs b/crates/backend-uzu/src/session/tts_session.rsindex 6a4198e3a..7f3246ece 100644--- a/crates/backend-uzu/src/session/tts_session.rs+++ b/crates/backend-uzu/src/session/tts_session.rs@@ -34,7 +34,7 @@ use crate::{kernel::{EmbeddingRowsSumKernel, ManualKernels, TensorAddScaleKernel, TokenCopySampledKernel,kv_cache_update::KVCacheUpdate,- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},config::{ModelMetadata, TtsMessageProcessorConfig, TtsModelConfig},diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rsindex eb87e5091..d4371fbe7 100644--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs@@ -483,23 +483,27 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {if semantic_bridge.fast_model_dim != fast_model_dim || semantic_bridge.slow_model_dim != slow_model_dim {return Err(Error::GenerateFailed);}- semantic_bridge.projection.encode(- MatmulArguments {- a: slow_hidden_capture,- a_offset: 0,- b: weights,- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: 1.0,- c: MatmulArgumentC::None,- d: output_embedding,- batch_dim: 1,- input_dim: slow_model_dim as u32,- output_dim: fast_model_dim as u32,- },- encoder,- );+ semantic_bridge+ .projection+ .encode(+ MatmulArguments {+ a: slow_hidden_capture,+ a_offset: 0,+ b: MatmulB::FullPrecision {+ b: weights,+ },+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: output_embedding,+ d_transform: MatmulDOps::none(),+ m: 1,+ n: fast_model_dim as u32,+ k: slow_model_dim as u32,+ },+ encoder,+ )+ .expect("encode failed");return Ok(());}diff --git a/crates/backend-uzu/tests/common/matmul/bench.rs b/crates/backend-uzu/tests/common/matmul/bench.rsnew file mode 100644index 000000000..5626f8523--- /dev/null+++ b/crates/backend-uzu/tests/common/matmul/bench.rs@@ -0,0 +1,18 @@+use backend_uzu::backends::common::{Backend, Encoder};+use criterion::Bencher;++pub fn iter_encode_loop<B: Backend, F>(+ context: &B::Context,+ bencher: &mut Bencher,+ mut encode: F,+) where+ F: FnMut(&mut Encoder<B>),+{+ bencher.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(context).unwrap();+ for _ in 0..n_iters {+ encode(&mut encoder);+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()+ });+}diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rsindex 4aeb83d5d..105f169db 100644--- a/crates/backend-uzu/tests/common/matmul/harness.rs+++ b/crates/backend-uzu/tests/common/matmul/harness.rs@@ -1,5 +1,5 @@#[cfg(metal_backend)]-use backend_uzu::backends::metal::{Metal, MetalContext};+use backend_uzu::backends::metal::{GemmDispatchPath, Metal, MetalContext};use backend_uzu::{ArrayContextExt, ArrayElement,backends::{@@ -7,7 +7,7 @@ use backend_uzu::{Allocation, AllocationType, AsBufferRangeRef, Backend, Buffer, Context, Encoder,kernel::{ManualKernels,- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},cpu::Cpu,@@ -17,7 +17,7 @@ use num_traits::Float;use super::{super::helpers::{alloc_allocation_with_data, allocation_to_vec},- Shape, Variant,+ Shape,};#[cfg(metal_backend)]@@ -109,10 +109,11 @@ fn run<B: Backend, T: ArrayElement + Float>(.expect("create d allocation")};- let c_arg = if input.case.accumulate {- MatmulArgumentC::Accumulate- } else {- MatmulArgumentC::None+ let d_transform = MatmulDOps::<'_, B> {+ ab_scale: input.case.ab_scale,+ accumulate: input.case.accumulate,+ bias: None,+ rht_factors: None,};let mut encoder = Encoder::new(context).expect("encoder");@@ -121,16 +122,17 @@ fn run<B: Backend, T: ArrayElement + Float>(MatmulArguments::<'_, B> {a: &a_allocation,a_offset: 0,- b: b_array.allocation(),+ b: MatmulB::FullPrecision {+ b: b_array.allocation(),+ },b_offset: 0,b_leading_dimension: None,b_transpose: input.case.b_transpose,- ab_scale: input.case.ab_scale,- c: c_arg,d: &mut d_allocation,- batch_dim: m as u32,- input_dim: k as u32,- output_dim: n as u32,+ d_transform,+ m: m as u32,+ n: n as u32,+ k: k as u32,},&mut encoder,);@@ -143,7 +145,7 @@ pub fn cpu_reference<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let mut kernel = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type()).expect("CPU MatmulKernel");run::<Cpu, T>(&context, &mut kernel, input, |kernel, args, encoder| {- kernel.encode(args, encoder);+ kernel.encode(args, encoder).expect("encode failed");})}@@ -152,10 +154,12 @@ pub fn run_metal<T: ArrayElement + Float>(context: &MetalContext,kernel: &mut MetalMatmulKernel,input: &Input<T>,- variant: Variant,+ path: Option<GemmDispatchPath>,) -> Vec<T> {- let path = variant.dispatch_path();- run::<Metal, T>(context, kernel, input, |kernel, args, encoder| {- kernel.encode_with_path(args, encoder, path);+ run::<Metal, T>(context, kernel, input, |kernel, args, encoder| match path {+ None => kernel.encode(args, encoder).expect("matmul encode failed"),+ Some(gemm_path) => {+ kernel.gemm.encode_dispatch_path(args, gemm_path, encoder).expect("gemm encode_dispatch_path failed")+ },})}diff --git a/crates/backend-uzu/tests/common/matmul/mod.rs b/crates/backend-uzu/tests/common/matmul/mod.rsindex da729946d..2647d5809 100644--- a/crates/backend-uzu/tests/common/matmul/mod.rs+++ b/crates/backend-uzu/tests/common/matmul/mod.rs@@ -1,9 +1,16 @@+pub mod bench;pub mod harness;+pub mod quant;pub mod shape;-pub mod variant;+pub use bench::iter_encode_loop;#[cfg(metal_backend)]pub use harness::run_metal;pub use harness::{Case, cpu_reference, deterministic_input};-pub use shape::{Shape, all_correctness_shapes};-pub use variant::Variant;+#[cfg(metal_backend)]+pub use quant::run_quant_metal;+pub use quant::{QuantBuffers, QuantInput, quant_arguments, run_quant_cpu};+pub use shape::{+ Shape, all_correctness_shapes, bench_fp_gemm_shapes, bench_quant_gemm_shapes, bench_quant_gemv_shapes,+ qwen3_layer_shapes,+};diff --git a/crates/backend-uzu/tests/common/matmul/quant.rs b/crates/backend-uzu/tests/common/matmul/quant.rsnew file mode 100644index 000000000..c6cc2bdfa--- /dev/null+++ b/crates/backend-uzu/tests/common/matmul/quant.rs@@ -0,0 +1,185 @@+#[cfg(metal_backend)]+use backend_uzu::backends::metal::{GemmDispatchPath, Metal, MetalContext};+use backend_uzu::{+ ArrayElement,+ backends::{+ common::{+ Allocation, Backend, Context, Encoder,+ gpu_types::{QuantizationMethod, QuantizationMode},+ kernel::{+ ManualKernels,+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ },+ },+ cpu::Cpu,+ },+};+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::SmallRng};++use super::super::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec};++pub struct QuantInput<T: ArrayElement + Float> {+ pub w_packed: Vec<u32>,+ pub scales: Vec<T>,+ pub zero_points: Option<Vec<u8>>,+ pub biases: Option<Vec<T>>,+ pub x: Vec<T>,+ pub k: u32,+ pub n: u32,+ pub m: u32,+ pub group_size: u32,+ pub quant_method: QuantizationMethod,+ pub mode: QuantizationMode,+}++fn mode_for_bits(bits: u32) -> QuantizationMode {+ match bits {+ 4 => QuantizationMode::U4,+ 8 => QuantizationMode::I8,+ _ => unreachable!("unsupported bits: {bits}"),+ }+}++impl<T: ArrayElement + Float> QuantInput<T> {+ pub fn new(+ m: usize,+ k: usize,+ n: usize,+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+ seed: u64,+ ) -> Self {+ let num_groups_k = k.div_ceil(group_size as usize);+ let mut rng = SmallRng::seed_from_u64(seed);++ let w_packed: Vec<u32> = (0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect();+ let scales: Vec<T> =+ (0..n * num_groups_k).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();+ let x: Vec<T> = (0..m * k).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();++ let zp_stride = if bits == 4 {+ num_groups_k.div_ceil(2)+ } else {+ num_groups_k+ };+ let (zero_points, biases) = match quant_method {+ QuantizationMethod::ScaleZeroPoint => {+ (Some((0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)+ },+ QuantizationMethod::ScaleBias => (+ None,+ Some((0..n * num_groups_k).map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap()).collect()),+ ),+ };++ Self {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ quant_method,+ mode: mode_for_bits(bits),+ }+ }+}++pub struct QuantBuffers<B: Backend, T: ArrayElement + Float> {+ pub w: Allocation<B>,+ pub scales: Allocation<B>,+ pub zp: Option<Allocation<B>>,+ pub bias: Option<Allocation<B>>,+ pub x: Allocation<B>,+ pub y: Allocation<B>,+ _t: std::marker::PhantomData<T>,+}++impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {+ pub fn allocate(+ context: &B::Context,+ input: &QuantInput<T>,+ ) -> Self {+ Self {+ w: alloc_allocation_with_data::<B, u32>(context, &input.w_packed),+ scales: alloc_allocation_with_data::<B, T>(context, &input.scales),+ zp: input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(context, zp)),+ bias: input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(context, b)),+ x: alloc_allocation_with_data::<B, T>(context, &input.x),+ y: alloc_allocation::<B, T>(context, (input.m as usize) * (input.n as usize)),+ _t: std::marker::PhantomData,+ }+ }+}++pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(+ buffers: &'a mut QuantBuffers<B, T>,+ input: &QuantInput<T>,+) -> MatmulArguments<'a, B> {+ let b_variant = match input.quant_method {+ QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {+ b: &buffers.w,+ scales: &buffers.scales,+ zero_points: buffers.zp.as_ref().expect("zp buffer"),+ mode: input.mode,+ group_size: input.group_size,+ },+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {+ b: &buffers.w,+ scales: &buffers.scales,+ biases: buffers.bias.as_ref().expect("bias buffer"),+ mode: input.mode,+ group_size: input.group_size,+ },+ };+ MatmulArguments {+ a: &buffers.x,+ a_offset: 0,+ b: b_variant,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut buffers.y,+ d_transform: MatmulDOps::none(),+ m: input.m,+ n: input.n,+ k: input.k,+ }+}++pub fn run_quant_cpu<T: ArrayElement + Float>(input: &QuantInput<T>) -> Vec<T> {+ let context = <Cpu as Backend>::Context::new().expect("Cpu context");+ let mut buffers = QuantBuffers::<Cpu, T>::allocate(&context, input);+ let mut matmul = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())+ .expect("MatmulCpuKernel");+ let mut encoder = Encoder::<Cpu>::new(&context).expect("encoder");+ matmul.encode(quant_arguments(&mut buffers, input), &mut encoder).expect("encode cpu quant");+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ allocation_to_vec::<Cpu, T>(&buffers.y)+}++#[cfg(metal_backend)]+pub fn run_quant_metal<T: ArrayElement + Float>(+ context: &MetalContext,+ input: &QuantInput<T>,+ path: Option<GemmDispatchPath>,+) -> Vec<T> {+ let mut buffers = QuantBuffers::<Metal, T>::allocate(context, input);+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type())+ .expect("MatmulMetalKernel");+ let mut encoder = Encoder::<Metal>::new(context).expect("encoder");+ let args = quant_arguments(&mut buffers, input);+ match path {+ None => matmul.encode(args, &mut encoder).expect("matmul encode failed"),+ Some(gemm_path) => {+ matmul.gemm.encode_dispatch_path(args, gemm_path, &mut encoder).expect("gemm encode_dispatch_path failed")+ },+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ allocation_to_vec::<Metal, T>(&buffers.y)+}diff --git a/crates/backend-uzu/tests/common/matmul/shape.rs b/crates/backend-uzu/tests/common/matmul/shape.rsindex 3165f2031..33e7d58ef 100644--- a/crates/backend-uzu/tests/common/matmul/shape.rs+++ b/crates/backend-uzu/tests/common/matmul/shape.rs@@ -37,3 +37,79 @@ const SHAPES_MEDIUM: &[Shape] = &[Shape::new(128, 2048, 2048), Shape::new(256, 4pub fn all_correctness_shapes() -> impl Iterator<Item = Shape> {SHAPES_TINY.iter().chain(SHAPES_UNALIGNED.iter()).chain(SHAPES_MEDIUM.iter()).copied()}++const BENCH_FP_GEMM: &[Shape] = &[+ Shape::new(128, 2048, 8192),+ Shape::new(128, 4096, 14336),+ Shape::new(256, 4096, 4096),+ Shape::new(512, 8192, 2048),+];++pub fn bench_fp_gemm_shapes() -> impl Iterator<Item = Shape> {+ BENCH_FP_GEMM.iter().copied()+}++const BENCH_NK: &[(usize, usize)] =+ &[(2048, 2048), (2048, 4096), (4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];++pub fn bench_quant_gemm_shapes(bits: u32) -> impl Iterator<Item = Shape> {+ let block_size: usize = if bits == 4 {+ 512+ } else {+ 256+ };+ let ms = &[4usize, 5, 6, 7, 8, 16, 32, 48, 64];+ BENCH_NK+ .iter()+ .filter(move |&&(n, k)| n % 32 == 0 && k % block_size == 0)+ .flat_map(move |&(n, k)| ms.iter().map(move |&m| Shape::new(m, k, n)))+}++pub fn bench_quant_gemv_shapes(bits: u32) -> impl Iterator<Item = Shape> {+ let block_size: usize = if bits == 4 {+ 512+ } else {+ 256+ };+ let nk: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];+ let ms = &[1usize, 2, 4, 8];+ nk.iter()+ .filter(move |&&(n, k)| n % 8 == 0 && k % block_size == 0)+ .flat_map(move |&(n, k)| ms.iter().map(move |&m| Shape::new(m, k, n)))+}++const QWEN3_LAYERS: &[(&str, usize, usize)] = &[+ ("0.6b_qkv", 1024, 4096),+ ("0.6b_o", 2048, 1024),+ ("0.6b_gateup", 1024, 6144),+ ("0.6b_down", 3072, 1024),+ ("1.7b_qkv", 2048, 4096),+ ("1.7b_o", 2048, 2048),+ ("1.7b_gateup", 2048, 12288),+ ("1.7b_down", 6144, 2048),+ ("4b_qkv", 2560, 6144),+ ("4b_o", 4096, 2560),+ ("4b_gateup", 2560, 19456),+ ("4b_down", 9728, 2560),+ ("8b_qkv", 4096, 6144),+ ("8b_o", 4096, 4096),+ ("8b_gateup", 4096, 24576),+ ("8b_down", 12288, 4096),+];++pub fn qwen3_layer_shapes(bits: u32) -> impl Iterator<Item = (&'static str, Shape)> {+ let block_size: usize = if bits == 4 {+ 512+ } else {+ 256+ };+ let decode_ms = &[1usize, 4, 8];+ let prefill_ms = &[32usize, 64];+ QWEN3_LAYERS.iter().flat_map(move |&(label, k, n)| {+ decode_ms+ .iter()+ .chain(prefill_ms.iter())+ .filter(move |_| k % block_size == 0)+ .map(move |&m| (label, Shape::new(m, k, n)))+ })+}diff --git a/crates/backend-uzu/tests/common/matmul/variant.rs b/crates/backend-uzu/tests/common/matmul/variant.rsdeleted file mode 100644index 34ce795ea..000000000--- a/crates/backend-uzu/tests/common/matmul/variant.rs+++ /dev/null@@ -1,33 +0,0 @@-#[cfg(metal_backend)]-use backend_uzu::backends::metal::{DeviceExt, MatmulDispatchPath, MetalContext};-use derive_more::Display;--#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]-pub enum Variant {- #[display("GEMM")]- Gemm,- #[display("GEMM_MXU")]- GemmMxu,-}--impl Variant {- pub const fn requires_mxu(self) -> bool {- matches!(self, Variant::GemmMxu)- }-- #[cfg(metal_backend)]- pub fn supported(- self,- context: &MetalContext,- ) -> bool {- !self.requires_mxu() || context.device.supports_mxu()- }-- #[cfg(metal_backend)]- pub const fn dispatch_path(self) -> MatmulDispatchPath {- match self {- Variant::Gemm => MatmulDispatchPath::Gemm,- Variant::GemmMxu => MatmulDispatchPath::GemmMxu,- }- }-}diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rsnew file mode 100644index 000000000..6c091c87e--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs@@ -0,0 +1,74 @@+#![cfg(metal_backend)]++use backend_uzu::{+ ArrayContextExt, ArrayElement,+ backends::{+ common::{+ AllocationType, Backend, Context,+ kernel::{+ ManualKernels,+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ },+ },+ metal::{GemmDispatchPath, Metal, MetalContext},+ },+};+use criterion::{BenchmarkId, Criterion, Throughput};+use half::bf16;++use crate::{+ common::{+ matmul::{bench_fp_gemm_shapes, iter_encode_loop},+ type_short_name,+ },+ uzu_bench,+};++#[uzu_bench]+fn bench_gemm(c: &mut Criterion) {+ let context = MetalContext::new().expect("Metal context");+ let mut kernel =+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())+ .expect("MatmulKernel");++ let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));++ for shape in bench_fp_gemm_shapes() {+ let (m, k, n) = (shape.m, shape.k, shape.n);+ let a = context+ .create_allocation(m * k * std::mem::size_of::<bf16>(), AllocationType::Global)+ .expect("a allocation");+ let b_array = context.create_array_uninitialized(&[n, k], bf16::data_type());+ let mut d = context+ .create_allocation(m * n * std::mem::size_of::<bf16>(), AllocationType::Global)+ .expect("d allocation");++ group.throughput(Throughput::Elements((2 * m * k * n) as u64));+ group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {+ iter_encode_loop::<Metal, _>(&context, b, |encoder| {+ kernel+ .gemm+ .encode_dispatch_path(+ MatmulArguments {+ a: &a,+ a_offset: 0,+ b: MatmulB::FullPrecision {+ b: b_array.allocation(),+ },+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut d,+ d_transform: MatmulDOps::none(),+ m: m as u32,+ n: n as u32,+ k: k as u32,+ },+ GemmDispatchPath::Mxu,+ encoder,+ )+ .expect("encode_dispatch_path failed");+ });+ });+ }+}diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/mod.rs b/crates/backend-uzu/tests/performance/kernel/matmul/mod.rsnew file mode 100644index 000000000..8ba502560--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/matmul/mod.rs@@ -0,0 +1,4 @@+mod gemm_bench;+mod quant_gemm_bench;+mod quant_gemv_bench;+mod qwen3_bench;diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rsnew file mode 100644index 000000000..55920ac09--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs@@ -0,0 +1,68 @@+#![cfg(metal_backend)]++use backend_uzu::{+ ArrayElement,+ backends::{+ common::{+ Backend, Context,+ gpu_types::QuantizationMethod,+ kernel::{ManualKernels, matmul::MatmulKernel},+ },+ metal::{GemmDispatchPath, Metal, MetalContext},+ },+};+use criterion::{BenchmarkId, Criterion, Throughput};+use half::{bf16, f16};+use num_traits::Float;++use crate::{+ common::{+ matmul::{QuantBuffers, QuantInput, bench_quant_gemm_shapes, iter_encode_loop, quant_arguments},+ type_short_name,+ },+ uzu_bench,+};++fn bench_unified_quant_typed<T: ArrayElement + Float>(+ c: &mut Criterion,+ context: &MetalContext,+ label: &str,+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+) {+ for shape in bench_quant_gemm_shapes(bits) {+ let (m, k, n) = (shape.m, shape.k, shape.n);+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);+ let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);+ let mut matmul =+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type()).unwrap();++ let mut group =+ c.benchmark_group(format!("{}/Kernel/UnifiedQuantizedGemm/{}", type_short_name::<Metal>(), label));+ group.throughput(Throughput::Elements((m * n * k) as u64));+ group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {+ iter_encode_loop::<Metal, _>(context, b, |encoder| {+ matmul+ .gemm+ .encode_dispatch_path(+ quant_arguments(&mut buffers, &input),+ GemmDispatchPath::Simdgroup,+ encoder,+ )+ .expect("encode unified quant matmul");+ });+ });+ drop(group);+ }+}++#[uzu_bench]+fn bench_unified_quantized_gemm(c: &mut Criterion) {+ let context = MetalContext::new().expect("Metal context");+ bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);+ bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);+ bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);+ bench_unified_quant_typed::<f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+}diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rsnew file mode 100644index 000000000..41e15db09--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs@@ -0,0 +1,78 @@+use backend_uzu::{+ ArrayElement,+ backends::common::{+ Allocation, Backend, Context, Kernels, gpu_types::QuantizationMethod, kernel::QuantizedMatmulQmvFastKernel,+ },+};+use criterion::{BenchmarkId, Criterion, Throughput};+use half::{bf16, f16};+use num_traits::Float;++use crate::{+ common::{+ matmul::{QuantBuffers, QuantInput, bench_quant_gemv_shapes, iter_encode_loop},+ type_short_name,+ },+ uzu_bench,+};++fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(+ c: &mut Criterion,+ context: &B::Context,+ label: &str,+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+) {+ let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));++ for shape in bench_quant_gemv_shapes(bits) {+ let (m, k, n) = (shape.m, shape.k, shape.n);+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);+ let mut buffers = QuantBuffers::<B, T>::allocate(context, &input);++ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ context,+ T::data_type(),+ group_size,+ bits,+ quant_method,+ false,+ )+ .unwrap();++ group.throughput(Throughput::Elements((m * n * k) as u64));+ group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {+ iter_encode_loop::<B, _>(context, b, |encoder| {+ kernel.encode(+ &buffers.w,+ &buffers.scales,+ buffers.zp.as_ref(),+ buffers.bias.as_ref(),+ &buffers.x,+ &mut buffers.y,+ None::<&Allocation<B>>,+ k as u32,+ n as u32,+ m as u32,+ encoder,+ );+ });+ });+ }+}++#[uzu_bench]+fn bench_qmv_fast(c: &mut Criterion) {+ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);+ });+}diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rsnew file mode 100644index 000000000..5fe879f7f--- /dev/null+++ b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs@@ -0,0 +1,57 @@+#![cfg(metal_backend)]++use backend_uzu::{+ ArrayElement,+ backends::{+ common::{+ Backend, Context,+ gpu_types::QuantizationMethod,+ kernel::{ManualKernels, matmul::MatmulKernel},+ },+ metal::{Metal, MetalContext},+ },+};+use criterion::{BenchmarkId, Criterion, Throughput};+use half::bf16;+use num_traits::Float;++use crate::{+ common::{+ matmul::{QuantBuffers, QuantInput, iter_encode_loop, quant_arguments, qwen3_layer_shapes},+ type_short_name,+ },+ uzu_bench,+};++fn bench_qwen3_layers_typed<T: ArrayElement + Float>(+ c: &mut Criterion,+ context: &MetalContext,+ label: &str,+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+) {+ let mut group =+ c.benchmark_group(format!("{}/Kernel/Qwen3Layers/{}", type_short_name::<Metal>(), label));+ for (layer, shape) in qwen3_layer_shapes(bits) {+ let (m, k, n) = (shape.m, shape.k, shape.n);+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);+ let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);+ let mut matmul =+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type()).unwrap();++ group.throughput(Throughput::Elements((m * n * k) as u64));+ group.bench_function(BenchmarkId::from_parameter(format!("{layer}_{shape}")), |b| {+ iter_encode_loop::<Metal, _>(context, b, |encoder| {+ matmul.encode(quant_arguments(&mut buffers, &input), encoder).expect("encode qwen3 layer");+ });+ });+ }+}++#[uzu_bench]+fn bench_qwen3_layers(c: &mut Criterion) {+ let context = MetalContext::new().expect("Metal context");+ bench_qwen3_layers_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleBias);+ bench_qwen3_layers_typed::<bf16>(c, &context, "ZP_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleZeroPoint);+}diff --git a/crates/backend-uzu/tests/performance/kernel/mod.rs b/crates/backend-uzu/tests/performance/kernel/mod.rsindex 0223d0039..58b255470 100644--- a/crates/backend-uzu/tests/performance/kernel/mod.rs+++ b/crates/backend-uzu/tests/performance/kernel/mod.rs@@ -2,6 +2,6 @@#[path = "../../common/mod.rs"]mod common;-mod quant_matmul;+mod matmul;mod rms_norm_bench;mod sampling;diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rsdeleted file mode 100644index 857ee0c92..000000000--- a/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs+++ /dev/null@@ -1,2 +0,0 @@-mod qmm_transposed_bench;-mod qmv_fast_bench;diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rsdeleted file mode 100644index f2ba0d5d8..000000000--- a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs+++ /dev/null@@ -1,132 +0,0 @@-use backend_uzu::{- ArrayElement,- backends::common::{- Allocation, Backend, Context, Encoder, Kernels,- gpu_types::QuantizationMethod,- kernel::QuantizedMatmulQmmTransposedKernel,- },-};-use criterion::{BenchmarkId, Criterion, Throughput};-use half::{bf16, f16};-use itertools::iproduct;-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::SmallRng};--use crate::{- common::{- helpers::{alloc_allocation, alloc_allocation_with_data},- type_short_name,- },- uzu_bench,-};--fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(- c: &mut Criterion,- context: &B::Context,- label: &str,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- let block_size: usize = if bits == 4 { 512 } else { 256 };-- for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336]) {- if n % 32 != 0 || k % block_size != 0 {- continue;- }-- let (bm, bk, bn, wm, wn) = if m < 48 {- (8u32, 32u32, 32u32, 1u32, 1u32)- } else {- (32u32, 32u32, 32u32, 2u32, 2u32)- };-- let num_groups = k.div_ceil(group_size as usize);- let mut rng = SmallRng::seed_from_u64(42);-- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- context,- T::data_type(),- group_size,- bits,- bm,- bk,- bn,- wm,- wn,- quant_method,- false,- n % bn as usize == 0,- )- .unwrap();-- let w_buf = alloc_allocation_with_data::<B, u32>(- context,- &(0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect::<Vec<_>>(),- );- let scales_buf = alloc_allocation_with_data::<B, T>(- context,- &(0..n * num_groups).map(|_| T::from(rng.random_range(0.01f32..1.0f32)).unwrap()).collect::<Vec<_>>(),- );- let x_buf = alloc_allocation_with_data::<B, T>(- context,- &(0..m * k).map(|_| T::from(rng.random_range(-1.0f32..1.0f32)).unwrap()).collect::<Vec<_>>(),- );- let mut y_buf = alloc_allocation::<B, T>(context, m * n);-- let zp_stride = if bits == 4 {- (num_groups + 1) / 2- } else {- num_groups- };- let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {- alloc_allocation_with_data::<B, u8>(- context,- &(0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect::<Vec<_>>(),- )- });- let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {- alloc_allocation_with_data::<B, T>(- context,- &(0..n * num_groups).map(|_| T::from(rng.random_range(-0.5f32..0.5f32)).unwrap()).collect::<Vec<_>>(),- )- });-- let mut group = c.benchmark_group(format!("{}/Kernel/QmmTransposed/{}", type_short_name::<B>(), label));- group.throughput(Throughput::Elements((m * n * k) as u64));- group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {- b.iter_custom(|n_iters| {- let mut encoder = Encoder::<B>::new(context).unwrap();- for _ in 0..n_iters {- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- None::<&Allocation<B>>,- k as u32,- n as u32,- m as u32,- &mut encoder,- );- }- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()- })- });- drop(group);- }-}--#[uzu_bench]-fn bench_qmm_transposed(c: &mut Criterion) {- for_each_backend!(|B| {- let context = <B as Backend>::Context::new().unwrap();- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmm_transposed_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- });-}diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rsdeleted file mode 100644index 16d352560..000000000--- a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs+++ /dev/null@@ -1,141 +0,0 @@-use backend_uzu::{- ArrayElement,- backends::common::{- Allocation, Backend, Context, Encoder, Kernels,- gpu_types::QuantizationMethod,- kernel::QuantizedMatmulQmvFastKernel,- },-};-use criterion::{BenchmarkId, Criterion, Throughput};-use half::{bf16, f16};-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::SmallRng};--use crate::{- common::{- helpers::{alloc_allocation, alloc_allocation_with_data},- type_short_name,- },- uzu_bench,-};--fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(- rng: &mut R,- range: std::ops::Range<T>,- len: usize,-) -> Box<[T]> {- (0..len).map(|_| rng.random_range(range.clone())).collect()-}--fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(- c: &mut Criterion,- context: &B::Context,- label: &str,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));- let block_size: usize = if bits == 4 {- 512- } else {- 256- };-- let nk_pairs: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];- for &(n, k) in nk_pairs {- if n % 8 != 0 || k % block_size != 0 {- continue;- }- for m in [1usize, 2, 4, 8] {- let num_groups = k.div_ceil(group_size as usize);- let mut rng = SmallRng::seed_from_u64(42);-- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(- context,- T::data_type(),- group_size,- bits,- quant_method,- false,- )- .unwrap();-- let w_buf = alloc_allocation_with_data::<B, u32>(- context,- &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),- );- let scales_buf = alloc_allocation_with_data::<B, T>(- context,- &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)- .iter()- .map(|&v| T::from(v).unwrap())- .collect::<Vec<_>>(),- );- let x_buf = alloc_allocation_with_data::<B, T>(- context,- &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k)- .iter()- .map(|&v| T::from(v).unwrap())- .collect::<Vec<_>>(),- );- let mut y_buf = alloc_allocation::<B, T>(context, m * n);-- let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {- let zp_stride = if bits == 4 {- (num_groups + 1) / 2- } else {- num_groups- };- alloc_allocation_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))- });- let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {- alloc_allocation_with_data::<B, T>(- context,- &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)- .iter()- .map(|&v| T::from(v).unwrap())- .collect::<Vec<_>>(),- )- });-- group.throughput(Throughput::Elements((m * n * k) as u64));- group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {- b.iter_custom(|n_iters| {- let mut encoder = Encoder::<B>::new(context).unwrap();- for _ in 0..n_iters {- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- None::<&Allocation<B>>,- k as u32,- n as u32,- m as u32,- &mut encoder,- );- }- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()- })- });- }- }-}--#[uzu_bench]-fn bench_qmv_fast(c: &mut Criterion) {- for_each_backend!(|B| {- let context = <B as Backend>::Context::new().unwrap();- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);- });-}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rsdeleted file mode 100644index d20d3ae98..000000000--- a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs+++ /dev/null@@ -1,687 +0,0 @@-#[macro_use]-#[path = "../../../../common/mod.rs"]-mod common;--use std::time::Instant;--use half::{bf16, f16};--use crate::{- DataType,- backends::common::{- Allocation, AllocationType, Backend, Context, Encoder, allocation_as_bytes,- gpu_types::{QuantizationMethod, QuantizationMode},- kernel::quant_matmul::{- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,- tests::common::helpers::alloc_allocation_with_data,- },- },-};--fn pack_u4_weights(values: &[u8]) -> Vec<u8> {- let mut out = Vec::with_capacity((values.len() + 3) / 4 * 2);- for chunk in values.chunks(4) {- let w0 = (*chunk.get(0).unwrap_or(&0) as u16) & 0x0F;- let w1 = ((*chunk.get(1).unwrap_or(&0) as u16) & 0x0F) * 16;- let w2 = ((*chunk.get(2).unwrap_or(&0) as u16) & 0x0F) * 256;- let w3 = ((*chunk.get(3).unwrap_or(&0) as u16) & 0x0F) * 4096;-- let word: u16 = w0 | w1 | w2 | w3;- out.push(word as u8);- out.push((word >> 8) as u8);- }- out-}--fn quantize_value(- value: f32,- dtype: DataType,-) -> f32 {- match dtype {- DataType::F16 => f16::from_f32(value).to_f32(),- DataType::BF16 => bf16::from_f32(value).to_f32(),- _ => value,- }-}--fn quantize_slice(- values: &[f32],- dtype: DataType,-) -> Vec<f32> {- values.iter().map(|&v| quantize_value(v, dtype)).collect()-}--struct ExecutionResult {- elapsed: f64,-}--fn create_test_weights(- output_dim: usize,- input_dim: usize,- bits: usize,-) -> Vec<u8> {- let mut weights_quant: Vec<u8> = Vec::with_capacity(output_dim * input_dim);-- // Weights stored as [output_dim × input_dim]- for row in 0..output_dim {- for _col in 0..input_dim {- let v = if bits == 4 {- ((row + 1) & 0x0F) as u8- } else {- ((row + 1) & 0xFF) as u8- };- weights_quant.push(v);- }- }-- weights_quant-}--fn check_tolerance(- exp: f32,- got: f32,-) -> bool {- let diff = (exp - got).abs() as f64;- // For F16 at boundaries (like 65504), machine epsilon is 32 (approx 0.05% relative error)- // We use a conservative relative tolerance of 0.2%- let rel_tol = 0.002;- let abs_tol = 0.1f64;- let tol = abs_tol.max(exp.abs() as f64 * rel_tol);- diff > tol-}--fn get_4bit_value(- data: &[u8],- index: usize,-) -> f32 {- let word_idx = index / 4;- let word_offset = index % 4;- let byte_idx = word_idx * 2;-- let word = if byte_idx + 1 < data.len() {- data[byte_idx] as u16 | ((data[byte_idx + 1] as u16) * 256)- } else {- 0- };-- ((word >> (word_offset * 4)) & 0x0F) as f32-}--fn get_zp_value(- zero_points: &[u8],- stride: usize,- row_idx: usize,- group_idx: usize,- bits: usize,-) -> f32 {- if bits == 4 {- let byte_index = row_idx * stride + (group_idx >> 1);- let byte = zero_points.get(byte_index).copied().unwrap_or(0);- if (group_idx & 1) == 0 {- (byte & 0x0F) as f32- } else {- ((byte >> 4) & 0x0F) as f32- }- } else {- zero_points.get(row_idx * stride + group_idx).copied().unwrap_or(0) as f32- }-}--fn cpu_reference(- batch: usize,- output_dim: usize,- input_dim: usize,- a: &[f32],- b_quant: &[u8],- scales: &[f32],- biases: &[f32],- group_size: usize,- dtype: DataType,- bits: usize,- quantization_method: QuantizationMethod,- zero_points: &[u8],- zero_points_stride: usize,-) -> Vec<f32> {- let num_groups = (input_dim + group_size - 1) / group_size;-- let mut y = vec![0.0f32; batch * output_dim];- for i in 0..batch {- for j in 0..output_dim {- let mut acc = 0.0f32;- for g in 0..num_groups {- let scale = scales[j * num_groups + g];- let bias = match quantization_method {- QuantizationMethod::ScaleZeroPoint => {- let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);- -scale * zp- },- QuantizationMethod::ScaleBias => biases[j * num_groups + g],- };- let l_start = g * group_size;- let l_end = (l_start + group_size).min(input_dim);- let mut group_acc = 0.0f32;- let mut group_sum = 0.0f32;- for l in l_start..l_end {- let weight_linear_idx = j * input_dim + l;-- let val_q = if bits == 4 {- get_4bit_value(b_quant, weight_linear_idx)- } else {- b_quant[weight_linear_idx] as f32- };-- let val_a = a[i * input_dim + l];- group_acc += val_a * val_q;- group_sum += val_a;- }- acc += scale * group_acc + bias * group_sum;- }- y[i * output_dim + j] = quantize_value(acc, dtype);- }- }- y-}--struct TestQuantParams {- scales: Vec<f32>,- scales_quant: Vec<f32>,- biases: Vec<f32>,- zero_points: Vec<u8>,- zero_points_stride: usize,-}--fn generate_test_quant_params(- output_dim: usize,- input_dim: usize,- group_size: usize,- data_type: DataType,- bits: usize,- quantization_method: QuantizationMethod,- randomize_zp: bool,-) -> TestQuantParams {- let num_groups = (input_dim + group_size - 1) / group_size;-- let len = output_dim * num_groups;- let scales = vec![1.0; len];- let scales_quant = quantize_slice(&scales, data_type);- let mut biases = vec![0.0; len];-- let zero_points_stride = if bits == 4 {- ((num_groups + 1) / 2).max(1)- } else {- num_groups- };- let mut zero_points = vec![0u8; output_dim * zero_points_stride];-- if quantization_method == QuantizationMethod::ScaleZeroPoint && randomize_zp {- for j in 0..output_dim {- for g in 0..num_groups {- let base_val = j + 3 * g;- let zp_val: u8 = if bits == 4 {- (base_val as u8) & 0x0F- } else {- base_val as u8- };-- if bits == 4 {- let byte_index = j * zero_points_stride + (g >> 1);- if (g & 1) == 0 {- zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val & 0x0F);- } else {- zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);- }- } else {- let byte_index = j * zero_points_stride + g;- zero_points[byte_index] = zp_val;- }-- let s = scales_quant[j * num_groups + g];- let b = quantize_value(-s * (zp_val as f32), data_type);- biases[j * num_groups + g] = b;- }- }- }-- if quantization_method == QuantizationMethod::ScaleBias {- for j in 0..output_dim {- for g in 0..num_groups {- let base_val = j * 7 + g * 3;- let bias_val = (base_val % 19) as f32 * 0.125;- biases[j * num_groups + g] = quantize_value(bias_val, data_type);- }- }- }-- TestQuantParams {- scales,- scales_quant,- biases,- zero_points,- zero_points_stride,- }-}--fn allocation_from_f32_slice<B: Backend>(- ctx: &B::Context,- dtype: DataType,- values: &[f32],-) -> Allocation<B> {- match dtype {- DataType::F16 => {- let data: Vec<f16> = values.iter().map(|&v| f16::from_f32(v)).collect();- alloc_allocation_with_data::<B, f16>(ctx, data.as_slice())- },- DataType::BF16 => {- let data: Vec<bf16> = values.iter().map(|&v| bf16::from_f32(v)).collect();- alloc_allocation_with_data::<B, bf16>(ctx, data.as_slice())- },- DataType::F32 => alloc_allocation_with_data::<B, f32>(ctx, values),- other => {- panic!("Unsupported dtype for allocation_from_f32_slice: {:?}", other)- },- }-}--fn execute_quantized_matmul<B: Backend>(- ctx: &B::Context,- batch: usize,- input_dim: usize,- output_dim: usize,- iterations: usize,- validate: bool,- quantization_method: QuantizationMethod,- randomize_zp: bool,- group_size: usize,- data_type: DataType,- bits: usize,-) -> ExecutionResult {- let weights_quant = create_test_weights(output_dim, input_dim, bits);- let weights_packed = if bits == 4 {- pack_u4_weights(&weights_quant)- } else {- weights_quant.clone()- };-- let params = generate_test_quant_params(- output_dim,- input_dim,- group_size,- data_type,- bits,- quantization_method,- randomize_zp,- );- // X is batch × input_dim- let x_f32: Vec<f32> = if batch == 1 {- // Vector case: 1 × input_dim- (1..=input_dim).map(|i| i as f32 / input_dim as f32).collect()- } else {- let mut x_vals: Vec<f32> = Vec::with_capacity(batch * input_dim);- for _row in 0..batch {- for i in 0..input_dim {- x_vals.push((i + 1) as f32 / input_dim as f32);- }- }- x_vals- };- let x_quant = quantize_slice(&x_f32, data_type);-- let w_buf = alloc_allocation_with_data::<B, u8>(ctx, &weights_packed);- let s_buf = allocation_from_f32_slice::<B>(ctx, data_type, ¶ms.scales);-- let b_buf = match quantization_method {- QuantizationMethod::ScaleZeroPoint => alloc_allocation_with_data::<B, u8>(ctx, ¶ms.zero_points),- QuantizationMethod::ScaleBias => allocation_from_f32_slice::<B>(ctx, data_type, ¶ms.biases),- };- let x_buf = allocation_from_f32_slice::<B>(ctx, data_type, &x_f32);- let mut y_buf = ctx- .create_allocation(batch * output_dim * data_type.size_in_bytes(), AllocationType::Global)- .expect("Failed to create allocation");-- let kernel = QuantizedMatmulKernelEncodable::<B>::new(- &ctx,- QuantizedMatmulConfiguration {- data_type,- group_size,- input_dim,- output_dim,- mode: match bits {- 4 => QuantizationMode::U4,- 8 => QuantizationMode::I8,- _ => panic!("Unsupported bits: {}", bits),- },- quantization_method,- use_hadamard: false,- },- )- .unwrap();-- if iterations > 1 {- for _ in 0..3 {- let args = QuantizedMatmulArguments {- a: &x_buf,- a_offset: 0,- b: &w_buf,- scales: &s_buf,- zero_points_or_biases: &b_buf,- output: &mut y_buf,- hadamard_factors: None,- batch_dim: batch,- };- let mut encoder = Encoder::new(ctx).unwrap();- kernel.encode(&mut encoder, args);- encoder.end_encoding().submit().wait_until_completed().unwrap();- }- }-- let start = Instant::now();- for _ in 0..iterations {- let args = QuantizedMatmulArguments {- a: &x_buf,- a_offset: 0,- b: &w_buf,- scales: &s_buf,- zero_points_or_biases: &b_buf,- output: &mut y_buf,- hadamard_factors: None,- batch_dim: batch,- };- let mut encoder = Encoder::new(ctx).unwrap();- kernel.encode(&mut encoder, args);- encoder.end_encoding().submit().wait_until_completed().unwrap();- }- let elapsed = start.elapsed();-- if validate {- let y_expected = cpu_reference(- batch,- output_dim,- input_dim,- &x_quant,- &weights_packed,- ¶ms.scales_quant,- ¶ms.biases,- group_size,- data_type,- bits,- quantization_method,- ¶ms.zero_points,- params.zero_points_stride,- );-- let y_out_f32: Vec<f32> = match data_type {- DataType::F16 => {- let y_out = bytemuck::cast_slice::<u8, f16>(allocation_as_bytes(&y_buf));- y_out.iter().map(|&v| v.to_f32()).collect()- },- DataType::BF16 => {- let y_out = bytemuck::cast_slice::<u8, bf16>(allocation_as_bytes(&y_buf));- y_out.iter().map(|&v| v.to_f32()).collect()- },- DataType::F32 => bytemuck::cast_slice::<u8, f32>(allocation_as_bytes(&y_buf)).to_vec(),- other => panic!("Unsupported dtype for validation: {:?}", other),- };-- let mut debug_prints = 0;-- for (i, (&exp, &got)) in y_expected.iter().zip(y_out_f32.iter()).enumerate() {- let diff = (exp - got).abs();-- if check_tolerance(exp, got) {- if debug_prints < 16 {- println!("\n detail idx {} diff {} exp {} got {}", i, diff, exp, got);- }- debug_prints += 1;- }- }-- if debug_prints > 0 {- let first_error = y_expected- .iter()- .zip(y_out_f32.iter())- .enumerate()- .find(|&(_, (&e, &g))| check_tolerance(e, g))- .map(|(i, _)| i)- .unwrap_or(0);- let last_error = y_expected- .iter()- .zip(y_out_f32.iter())- .enumerate()- .filter(|&(_, (&e, &g))| check_tolerance(e, g))- .last()- .map(|(i, _)| i)- .unwrap_or(0);- println!("\nTotal errors: {} out of {} outputs", debug_prints, batch * output_dim);- println!("Error range: indices {}-{}", first_error, last_error);- println!("Dims batch={} output_dim={} input_dim={}", batch, output_dim, input_dim);- panic!("Validation failed with {} mismatches", debug_prints);- }-- ExecutionResult {- elapsed: elapsed.as_secs_f64(),- }- } else {- ExecutionResult {- elapsed: elapsed.as_secs_f64(),- }- }-}--struct TestConfig {- quant_method: QuantizationMethod,- bits: usize,- data_type: DataType,- group_size: usize,-}--const QMV_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];-const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];--fn run_kernel_test<B: Backend>(- ctx: &B::Context,- batch: usize,- output_dim: usize,- input_dim: usize,- config: &TestConfig,- validate: bool,- iterations: usize,-) -> ExecutionResult {- let randomize_zp = config.quant_method == QuantizationMethod::ScaleZeroPoint;-- execute_quantized_matmul::<B>(- ctx,- batch,- input_dim,- output_dim,- iterations,- validate,- config.quant_method,- randomize_zp,- config.group_size,- config.data_type,- config.bits,- )-}--fn test_quant_gmv_internal<B: Backend>() {- let ctx = match B::Context::new() {- Ok(c) => c,- Err(_) => {- println!("{} backend not available — skipping QMV test", std::any::type_name::<B>());- return;- },- };-- let configs = vec![- TestConfig {- quant_method: QuantizationMethod::ScaleZeroPoint,- bits: 4,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_method: QuantizationMethod::ScaleZeroPoint,- bits: 8,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_method: QuantizationMethod::ScaleBias,- bits: 4,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_method: QuantizationMethod::ScaleBias,- bits: 8,- data_type: DataType::F32,- group_size: 64,- },- ];-- for config in &configs {- for &(output_dim, input_dim) in QMV_DIMS {- run_kernel_test::<B>(&ctx, 1, output_dim, input_dim, config, true, 1);- }- }-}--fn test_quant_gmm_transposed_internal<B: Backend>() {- let ctx = match B::Context::new() {- Ok(c) => c,- Err(_) => {- println!("{} not available — skipping QMM transposed test", std::any::type_name::<B>());- return;- },- };-- let configs = vec![- TestConfig {- quant_method: QuantizationMethod::ScaleZeroPoint,- bits: 4,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_method: QuantizationMethod::ScaleZeroPoint,- bits: 8,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_method: QuantizationMethod::ScaleBias,- bits: 4,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_method: QuantizationMethod::ScaleBias,- bits: 8,- data_type: DataType::F32,- group_size: 64,- },- ];-- for config in &configs {- for &(batch, output_dim, input_dim) in QMM_DIMS {- run_kernel_test::<B>(&ctx, batch, output_dim, input_dim, config, true, 1);- }- }-}--fn test_quant_matmul_perf_internal<B: Backend>() {- let ctx = match B::Context::new() {- Ok(c) => c,- Err(_) => {- println!("{} backend not available — skipping Perf test", std::any::type_name::<B>());- return;- },- };-- // Llama 3 8B approximate shapes- let shapes = vec![- // Decoding (M=1)- (1, 4096, 4096),- (1, 14336, 4096),- (1, 4096, 14336),- // Prefill (Small batch M=128)- (128, 4096, 4096),- ];-- let configs = vec![- // 4-bit scale-bias BF16- TestConfig {- quant_method: QuantizationMethod::ScaleBias,- bits: 4,- data_type: DataType::BF16,- group_size: 128,- },- // 8-bit scale-bias BF16- TestConfig {- quant_method: QuantizationMethod::ScaleBias,- bits: 8,- data_type: DataType::BF16,- group_size: 128,- },- // 4-bit ZP F16- TestConfig {- quant_method: QuantizationMethod::ScaleZeroPoint,- bits: 4,- data_type: DataType::F16,- group_size: 64,- },- // 8-bit ZP F16- TestConfig {- quant_method: QuantizationMethod::ScaleZeroPoint,- bits: 8,- data_type: DataType::F16,- group_size: 64,- },- ];-- println!(- "{:<20} | {:<10} | {:<6} | {:<5} | {:<5} | {:<15}",- "Kernel Config", "M x N x K", "Type", "Bits", "Group", "Avg Duration"- );- println!("{}", "-".repeat(85));-- let iterations = 20;- for config in &configs {- for &(batch, output_dim, input_dim) in &shapes {- let result = run_kernel_test::<B>(&ctx, batch, output_dim, input_dim, config, false, iterations);- let avg = result.elapsed / (iterations as f64) * 1000.0; // ms-- println!(- "{:<20} | {:<10} | {:<6} | {:<5} | {:<5} | {:.4} ms",- format!("{:?}", config.quant_method),- format!("{}x{}x{}", batch, output_dim, input_dim),- format!("{:?}", config.data_type),- config.bits,- config.group_size,- avg- );- }- }-}--#[test]-fn test_quant_gmv() {- for_each_non_cpu_backend!(|B| {- test_quant_gmv_internal::<B>();- })-}--#[test]-fn test_quant_gmm_transposed() {- for_each_non_cpu_backend!(|B| {- test_quant_gmm_transposed_internal::<B>();- })-}--#[test]-#[ignore]-fn test_quant_matmul_perf() {- for_each_non_cpu_backend!(|B| {- test_quant_matmul_perf_internal::<B>();- })-}diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rsindex 5e6969ae5..1bc168176 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs@@ -1,3 +1,4 @@+use backend_uzu::{DataType,backends::{@@ -6,7 +7,7 @@ use backend_uzu::{kernel::{AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,ManualKernels, SoftmaxKernel,- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},cpu::Cpu,@@ -91,16 +92,31 @@ fn pipeline_output<B: Backend>(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -let mut enc = Encoder::new(ctx.as_ref()).unwrap();for g in 0..NUM_GROUPS {- matmul.encode(- MatmulArguments {- a: &qa, a_offset: g as usize * GQA as usize * SUFFIX as usize * q_row,- b: &ka, b_offset: g as usize * head_stride * dt,- b_leading_dimension: Some(HEAD_DIM), b_transpose: true,- ab_scale: scale, c: MatmulArgumentC::None, d: &mut grp_s,- batch_dim: GQA * SUFFIX, input_dim: HEAD_DIM, output_dim: SEQ,- },- &mut enc,- );+ matmul+ .encode(+ MatmulArguments {+ a: &qa,+ a_offset: g as usize * GQA as usize * SUFFIX as usize * q_row,+ b: MatmulB::FullPrecision {+ b: &ka,+ },+ b_offset: g as usize * head_stride * dt,+ b_leading_dimension: Some(HEAD_DIM),+ b_transpose: true,+ d: &mut grp_s,+ d_transform: MatmulDOps {+ ab_scale: scale,+ accumulate: false,+ bias: None,+ rht_factors: None,+ },+ m: GQA * SUFFIX,+ n: SEQ,+ k: HEAD_DIM,+ },+ &mut enc,+ )+ .expect("encode failed");scatter_scores.encode(&grp_s, &mut scores, None, None::<&Allocation<B>>, None,g, GQA, SEQ, SUFFIX, GQA * SUFFIX * SEQ, &mut enc,@@ -108,16 +124,26 @@ fn pipeline_output<B: Backend>(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -}softmax.encode(&mut scores, None::<&Allocation<B>>, SEQ, NUM_HEADS, SUFFIX, &mut enc);for g in 0..NUM_GROUPS {- matmul.encode(- MatmulArguments {- a: &scores, a_offset: g as usize * GQA as usize * SUFFIX as usize * s_row,- b: &va, b_offset: g as usize * head_stride * dt,- b_leading_dimension: Some(HEAD_DIM), b_transpose: false,- ab_scale: 1.0, c: MatmulArgumentC::None, d: &mut grp_o,- batch_dim: GQA * SUFFIX, input_dim: SEQ, output_dim: HEAD_DIM,- },- &mut enc,- );+ matmul+ .encode(+ MatmulArguments {+ a: &scores,+ a_offset: g as usize * GQA as usize * SUFFIX as usize * s_row,+ b: MatmulB::FullPrecision {+ b: &va,+ },+ b_offset: g as usize * head_stride * dt,+ b_leading_dimension: Some(HEAD_DIM),+ b_transpose: false,+ d: &mut grp_o,+ d_transform: MatmulDOps::none(),+ m: GQA * SUFFIX,+ n: HEAD_DIM,+ k: SEQ,+ },+ &mut enc,+ )+ .expect("encode failed");scatter_values.encode(&grp_o, &mut out, g, GQA, SUFFIX, NUM_HEADS, HEAD_DIM,GQA * SUFFIX * HEAD_DIM, &mut enc,diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rsindex 51a0e9085..97ece1a8b 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs@@ -6,126 +6,86 @@ use backend_uzu::{ArrayElement,backends::{common::{Backend, Context, kernel::ManualKernels, kernel::matmul::MatmulKernel},- metal::{Metal, MetalContext},+ metal::{DeviceExt, GemmDispatchPath, Metal, MetalContext},},};-use proc_macros::__internal_uzu_test as uzu_test;use half::{bf16, f16};use num_traits::Float;+use rstest::rstest;use crate::common::{assert::assert_eq_float,- matmul::{Case, Variant, all_correctness_shapes, cpu_reference, deterministic_input, run_metal},+ matmul::{Case, all_correctness_shapes, cpu_reference, deterministic_input, run_metal},};+fn gemm_paths_for_hw(context: &MetalContext) -> Vec<GemmDispatchPath> {+ let mut paths = vec![GemmDispatchPath::Simdgroup];+ if context.device.supports_mxu() {+ paths.push(GemmDispatchPath::Mxu);+ }+ paths+}+fn check_case<T: ArrayElement + Float + Debug + Display>(context: &MetalContext,kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,- variant: Variant,+ path: Option<GemmDispatchPath>,case: Case,tolerance: f32,) {let input = deterministic_input::<T>(case);let expected = cpu_reference::<T>(&input);- let actual = run_metal::<T>(context, kernel, &input, variant);+ let actual = run_metal::<T>(context, kernel, &input, path);assert_eq_float(&expected,&actual,tolerance,- &format!("{:?} dtype={} {:?}", variant, std::any::type_name::<T>(), case),+ &format!("{path:?} dtype={} {case:?}", std::any::type_name::<T>()),);}-fn check_all_shapes<T: ArrayElement + Float + Debug + Display>(- variant: Variant,- ab_scale: f32,- accumulate: bool,+fn run_matrix<T: ArrayElement + Float + Debug + Display>(+ case_for_shape: impl Fn(crate::common::matmul::Shape) -> Case,tolerance: f32,) {let context = MetalContext::new().expect("Metal context");- if !variant.supported(&context) {- eprintln!("Skipping {variant:?}: device does not support MXU");- return;- }let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type()).expect("MatmulKernel");- for shape in all_correctness_shapes() {- let case = Case::new(shape).with_ab_scale(ab_scale).with_accumulate(accumulate);- check_case::<T>(&context, &mut kernel, variant, case, tolerance);+ for path in gemm_paths_for_hw(&context) {+ for shape in all_correctness_shapes() {+ let case = case_for_shape(shape);+ check_case::<T>(&context, &mut kernel, Some(path), case, tolerance);+ }}}-#[uzu_test]-fn matches_cpu_reference_bf16_gemm() {- check_all_shapes::<bf16>(Variant::Gemm, 1.0, false, 1.0);-}--#[uzu_test]-fn matches_cpu_reference_bf16_gemm_mxu() {- check_all_shapes::<bf16>(Variant::GemmMxu, 1.0, false, 1.0);-}--#[uzu_test]-fn matches_cpu_reference_f16_gemm() {- check_all_shapes::<f16>(Variant::Gemm, 1.0, false, 0.5);-}--#[uzu_test]-fn matches_cpu_reference_f16_gemm_mxu() {- check_all_shapes::<f16>(Variant::GemmMxu, 1.0, false, 0.5);-}--#[uzu_test]-fn matches_cpu_reference_f32_gemm() {- check_all_shapes::<f32>(Variant::Gemm, 1.0, false, 0.05);-}--#[uzu_test]-fn ab_scale_bf16_gemm() {- check_all_shapes::<bf16>(Variant::Gemm, 0.5, false, 1.0);-}--#[uzu_test]-fn ab_scale_bf16_gemm_mxu() {- check_all_shapes::<bf16>(Variant::GemmMxu, 0.5, false, 1.0);-}--#[uzu_test]-fn accumulate_bf16_gemm_mxu() {- check_all_shapes::<bf16>(Variant::GemmMxu, 1.0, true, 1.0);+#[rstest]+#[case::base(1.0, false)]+#[case::ab_scale(0.5, false)]+#[case::accumulate(1.0, true)]+#[case::scale_and_accumulate(0.5, true)]+fn matches_cpu_reference_bf16(+ #[case] ab_scale: f32,+ #[case] accumulate: bool,+) {+ run_matrix::<bf16>(+ |shape| Case::new(shape).with_ab_scale(ab_scale).with_accumulate(accumulate),+ 1.0,+ );}-#[uzu_test]-fn scale_and_accumulate_bf16_gemm_mxu() {- check_all_shapes::<bf16>(Variant::GemmMxu, 0.5, true, 1.0);+#[test]+fn matches_cpu_reference_f16() {+ run_matrix::<f16>(|shape| Case::new(shape), 0.5);}-fn check_b_transpose_false<T: ArrayElement + Float + Debug + Display>(- variant: Variant,- tolerance: f32,-) {- let context = MetalContext::new().expect("Metal context");- if !variant.supported(&context) {- eprintln!("Skipping {variant:?}: device does not support MXU");- return;- }- let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())- .expect("MatmulKernel");- for shape in all_correctness_shapes() {- let case = Case {+#[test]+fn b_transpose_false_bf16() {+ run_matrix::<bf16>(+ |shape| Case {b_transpose: false,..Case::new(shape)- };- check_case::<T>(&context, &mut kernel, variant, case, tolerance);- }-}--#[uzu_test]-fn b_transpose_false_bf16_gemm() {- check_b_transpose_false::<bf16>(Variant::Gemm, 1.0);-}--#[uzu_test]-fn b_transpose_false_bf16_gemm_mxu() {- check_b_transpose_false::<bf16>(Variant::GemmMxu, 1.0);+ },+ 1.0,+ );}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rsindex d99850a4b..d71526524 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs@@ -1,9 +1,5 @@-use std::{- fmt::{Debug, Display},-};+use std::fmt::{Debug, Display};-use half::{bf16, f16};-use num_traits::Float;use backend_uzu::{ArrayContextExt, ArrayElement,backends::{@@ -11,16 +7,19 @@ use backend_uzu::{AllocationType, Backend, Context, Encoder,kernel::{ManualKernels,- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},cpu::Cpu,},};+use half::{bf16, f16};+use num_traits::Float;+use rstest::rstest;-use crate::{- common::{assert::assert_eq_float, helpers::{alloc_allocation_with_data, allocation_to_vec}},- uzu_test,+use crate::common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation_with_data, allocation_to_vec},};struct Input<T: ArrayElement + Float> {@@ -68,23 +67,26 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.expect("Failed to create MatmulKernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- MatmulArguments {- a: &a_allocation,- a_offset: 0,- b: b_array.allocation(),- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- ab_scale: 1.0,- c: MatmulArgumentC::None,- d: &mut d_allocation,- batch_dim: m,- input_dim: k,- output_dim: n,- },- &mut encoder,- );+ kernel+ .encode(+ MatmulArguments {+ a: &a_allocation,+ a_offset: 0,+ b: MatmulB::FullPrecision {+ b: b_array.allocation(),+ },+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut d_allocation,+ d_transform: MatmulDOps::none(),+ m,+ n,+ k,+ },+ &mut encoder,+ )+ .expect("encode failed");encoder.end_encoding().submit().wait_until_completed().unwrap();allocation_to_vec::<B, T>(&d_allocation)}@@ -102,107 +104,34 @@ fn test<T: ArrayElement + Float + Debug + Display>(});}-#[uzu_test]-fn test_f32_m1() {- test::<f32>(1, 128, 64, 0.01);-}--#[uzu_test]-fn test_f16_m1() {- test::<f16>(1, 128, 64, 0.01);-}--#[uzu_test]-fn test_bf16_m1() {- test::<bf16>(1, 128, 64, 0.1);-}--#[uzu_test]-fn test_f32_batched() {- test::<f32>(4, 128, 64, 0.01);-}--#[uzu_test]-fn test_f16_batched() {- test::<f16>(4, 128, 64, 0.01);-}--#[uzu_test]-fn test_bf16_batched() {- test::<bf16>(4, 128, 64, 0.1);-}--#[uzu_test]-fn test_f32_max_batch() {- test::<f32>(8, 128, 64, 0.01);-}--#[uzu_test]-fn test_f16_max_batch() {- test::<f16>(8, 128, 64, 0.01);-}--#[uzu_test]-fn test_bf16_max_batch() {- test::<bf16>(8, 128, 64, 0.1);-}--#[uzu_test]-fn test_f32_unaligned_k() {- test::<f32>(1, 33, 64, 0.01);-}--#[uzu_test]-fn test_f16_unaligned_k() {- test::<f16>(1, 33, 64, 0.01);-}--#[uzu_test]-fn test_bf16_unaligned_k() {- test::<bf16>(1, 33, 64, 0.1);-}--#[uzu_test]-fn test_f32_unaligned_n() {- test::<f32>(1, 128, 11, 0.01);-}--#[uzu_test]-fn test_f16_unaligned_n() {- test::<f16>(1, 128, 11, 0.01);-}--#[uzu_test]-fn test_bf16_unaligned_n() {- test::<bf16>(1, 128, 11, 0.1);-}--#[uzu_test]-fn test_f32_large() {- test::<f32>(1, 4096, 2048, 0.05);-}--#[uzu_test]-fn test_f16_large() {- test::<f16>(1, 4096, 2048, 0.5);-}--#[uzu_test]-fn test_bf16_large() {- test::<bf16>(1, 4096, 2048, 1.0);-}--#[uzu_test]-fn test_f32_small_n() {- test::<f32>(1, 128, 3, 0.01);-}--#[uzu_test]-fn test_f16_small_n() {- test::<f16>(1, 128, 3, 0.01);-}--#[uzu_test]-fn test_bf16_small_n() {- test::<bf16>(1, 128, 3, 0.1);+#[rstest]+#[case::m1(1, 128, 64)]+#[case::batched(4, 128, 64)]+#[case::max_batch(8, 128, 64)]+#[case::unaligned_k(1, 33, 64)]+#[case::unaligned_n(1, 128, 11)]+#[case::large(1, 4096, 2048)]+#[case::small_n(1, 128, 3)]+fn gemv_f16(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+) {+ test::<f16>(m, k, n, 0.01);+}++#[rstest]+#[case::m1(1, 128, 64)]+#[case::batched(4, 128, 64)]+#[case::max_batch(8, 128, 64)]+#[case::unaligned_k(1, 33, 64)]+#[case::unaligned_n(1, 128, 11)]+#[case::large(1, 4096, 2048)]+#[case::small_n(1, 128, 3)]+fn gemv_bf16(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+) {+ test::<bf16>(m, k, n, 0.1);}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs b/crates/backend-uzu/tests/unit/kernel/matmul/mod.rsindex c84f3ea46..f9615e95c 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs@@ -1,3 +1,5 @@#[cfg(metal_backend)]mod dispatch_paths_test;mod gemv_test;+#[cfg(metal_backend)]+mod quant_dispatch_test;diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rsnew file mode 100644index 000000000..aa37cb960--- /dev/null+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs@@ -0,0 +1,223 @@+#![cfg(metal_backend)]++use std::{+ error::Error as StdError,+ fmt::{Debug, Display},+};++use backend_uzu::{+ ArrayElement,+ backends::{+ common::{+ Backend, Context, Encoder,+ gpu_types::{QuantizationMethod, gemm::GemmDTransform},+ kernel::{+ ManualKernels,+ matmul::{MatmulDOps, MatmulError, MatmulKernel},+ },+ },+ metal::{GemmDispatchPath, Metal, MetalContext},+ },+};+use half::bf16;+use num_traits::Float;+use proc_macros::__internal_uzu_test as uzu_test;+use rstest::rstest;++use crate::common::{+ helpers::allocation_to_vec,+ matmul::{QuantBuffers, QuantInput, quant_arguments, run_quant_cpu, run_quant_metal},+};++fn check_tolerance(+ expected: f32,+ actual: f32,+ rel_tol: f64,+ abs_tol: f64,+) -> bool {+ let diff = (expected - actual).abs() as f64;+ let tol = abs_tol.max(expected.abs() as f64 * rel_tol);+ diff <= tol+}++fn assert_parity<T: ArrayElement + Float + Debug + Display>(+ label: &str,+ reference: &[T],+ actual: &[T],+ rel_tol: f64,+ abs_tol: f64,+) {+ assert_eq!(reference.len(), actual.len(), "{label}: length mismatch");+ let mut errors = 0usize;+ for (i, (&exp, &got)) in reference.iter().zip(actual.iter()).enumerate() {+ let exp_f32 = exp.to_f32().unwrap();+ let got_f32 = got.to_f32().unwrap();+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {+ if errors < 5 {+ eprintln!(" {label} idx={i} expected={exp_f32} got={got_f32} diff={}", (exp_f32 - got_f32).abs());+ }+ errors += 1;+ }+ }+ assert_eq!(errors, 0, "{label}: {errors} mismatches");+}++fn run_parity<T: ArrayElement + Float + Debug + Display>(+ m: usize,+ k: usize,+ n: usize,+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+ rel_tol: f64,+ abs_tol: f64,+) {+ let context = MetalContext::new().expect("Metal context");+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 0);+ let unified = run_quant_metal::<T>(&context, &input, Some(GemmDispatchPath::Simdgroup));+ let reference = run_quant_cpu::<T>(&input);++ assert_parity::<T>(+ &format!(+ "m={m} k={k} n={n} gs={group_size} bits={bits} method={quant_method:?} dtype={}",+ std::any::type_name::<T>()+ ),+ &reference,+ &unified,+ rel_tol,+ abs_tol,+ );+}++#[rstest]+#[case::gs32_4bit_mlx_prefill ( 64, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]+#[case::gs64_4bit_mlx_prefill ( 64, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]+#[case::gs128_4bit_mlx_prefill( 64, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]+#[case::gs32_8bit_mlx_prefill ( 64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]+#[case::gs64_8bit_zp_prefill ( 64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::gs128_8bit_zp_prefill ( 64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::gs32_4bit_mlx_decode ( 8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]+#[case::gs64_4bit_zp_decode ( 8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::gs32_unaligned_n ( 64, 256, 96, 32, 4, QuantizationMethod::ScaleBias)]+fn parity_bf16(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+ #[case] gs: u32,+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+) {+ run_parity::<bf16>(m, k, n, gs, bits, method, 0.05, 0.4);+}++#[rstest]+#[case::gs64_4bit_mlx (32, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]+#[case::gs128_8bit_zp (32, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]+fn parity_f16(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+ #[case] gs: u32,+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+) {+ run_parity::<half::f16>(m, k, n, gs, bits, method, 0.01, 0.05);+}++#[uzu_test]+fn parity_bf16_gs32_4bit_mlx_with_bias() {+ let context = MetalContext::new().expect("Metal context");+ let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);++ let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.5 + 0.1 * (j % 5) as f32).collect();+ let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();++ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let bias_pp_buf =+ crate::common::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())+ .expect("MatmulMetalKernel");++ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");+ let mut args = quant_arguments(&mut buffers, &input);+ args.d_transform = MatmulDOps {+ ab_scale: 1.0,+ accumulate: false,+ bias: Some(&bias_pp_buf),+ rht_factors: None,+ };+ matmul+ .gemm+ .encode_dispatch_path(args, GemmDispatchPath::Simdgroup, &mut encoder)+ .expect("encode quant with bias");+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);++ let mut reference = run_quant_cpu::<bf16>(&input);+ for i in 0..(input.m as usize) {+ for j in 0..(input.n as usize) {+ let idx = i * input.n as usize + j;+ reference[idx] = bf16::from_f32(reference[idx].to_f32() + bias_f32[j]);+ }+ }+ assert_parity::<bf16>("with_bias", &reference, &actual, 0.05, 0.4);+}++#[uzu_test]+fn quant_gemm_nonzero_b_offset_returns_unsupported_layout() {+ let context = MetalContext::new().expect("Metal context");+ let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())+ .expect("MatmulMetalKernel");++ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");+ let mut args = quant_arguments(&mut buffers, &input);+ args.b_offset = 16;+ let result = matmul.encode(args, &mut encoder);++ let err = result.expect_err("expected error");+ let matmul: &MatmulError<Metal> = (&err as &dyn StdError)+ .source()+ .and_then(|s| s.downcast_ref::<MatmulError<Metal>>())+ .expect("expected MatmulError source");+ assert!(+ matches!(matmul, MatmulError::UnsupportedLayout { path: "QuantGemm" }),+ "got {matmul:?}"+ );+}++#[uzu_test]+fn quant_gemm_accumulate_returns_unsupported_dop() {+ let context = MetalContext::new().expect("Metal context");+ let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())+ .expect("MatmulMetalKernel");++ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");+ let mut args = quant_arguments(&mut buffers, &input);+ args.d_transform = MatmulDOps {+ ab_scale: 1.0,+ accumulate: true,+ bias: None,+ rht_factors: None,+ };+ let result = matmul.encode(args, &mut encoder);++ let err = result.expect_err("expected error");+ let matmul: &MatmulError<Metal> = (&err as &dyn StdError)+ .source()+ .and_then(|s| s.downcast_ref::<MatmulError<Metal>>())+ .expect("expected MatmulError source");+ assert!(+ matches!(+ matmul,+ MatmulError::UnsupportedDOp {+ bit: GemmDTransform::ACCUMULATE,+ ..+ }+ ),+ "got {matmul:?}"+ );+}diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rsindex 81b002146..06b62afc1 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs@@ -2,7 +2,6 @@#[path = "../../../common/mod.rs"]mod common;-mod qmm_transposed_test;mod qmv_fast_test;mod qmv_test;diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rsdeleted file mode 100644index e42308dc0..000000000--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs+++ /dev/null@@ -1,641 +0,0 @@-use std::fmt::{Debug, Display};--use backend_uzu::{- ArrayElement, DataType,- backends::{- common::{- Backend, Context, Encoder, Kernels,- gpu_types::QuantizationMethod,- kernel::QuantizedMatmulQmmTransposedKernel,- },- cpu::Cpu,- },-};-use half::{bf16, f16};-use num_traits::Float;--use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};-use crate::{- common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},- uzu_test,-};--fn make_input_basic<T: ArrayElement + Float>(- m: usize,- k: usize,- n: usize,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) -> Input<T> {- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;-- let max_val = if bits == 4 {- 15u8- } else {- 255u8- };- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);- for j in 0..n {- for l in 0..k {- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);- }- }- let w_packed = pack_weights_u32(&weights_raw, bits);-- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);- }- }- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = match quant_method {- QuantizationMethod::ScaleZeroPoint => {- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;- zp_raw.push(zp_val);- }- }- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);- for j in 0..n {- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];- let packed_row = pack_zero_points(row, bits);- let mut padded = packed_row;- padded.resize(zp_stride, 0);- zp_packed.extend_from_slice(&padded);- }- (Some(zp_packed), None)- },- QuantizationMethod::ScaleBias => {- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);- }- }- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();- (None, Some(biases))- },- };-- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);- for i in 0..m {- for l in 0..k {- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);- }- }- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- quant_method,- }-}--fn make_input_edge<T: ArrayElement + Float>(- n: usize,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) -> Input<T> {- let m = 2usize;- let k = group_size as usize;- let num_groups_k = 1;-- let weights_raw: Vec<u8> = vec![1u8; n * k];- let w_packed = pack_weights_u32(&weights_raw, bits);-- let scales: Vec<T> = vec![T::one(); n * num_groups_k];-- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = match quant_method {- QuantizationMethod::ScaleZeroPoint => {- let zp_packed = vec![0u8; n * zp_stride];- (Some(zp_packed), None)- },- QuantizationMethod::ScaleBias => {- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];- (None, Some(biases))- },- };-- let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- quant_method,- }-}--fn get_output<B: Backend, T: ArrayElement + Float>(- input: &Input<T>,- bm: u32,- bk: u32,- bn: u32,- wm: u32,- wn: u32,- aligned_n: bool,- hadamard_factors: Option<&[i32]>,-) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");- let use_hadamard = hadamard_factors.is_some();- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- &context,- T::data_type(),- input.group_size,- input.bits,- bm,- bk,- bn,- wm,- wn,- input.quant_method,- use_hadamard,- aligned_n,- )- .expect("Failed to create QuantizedMatmulQmmTransposedKernel");-- let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);-- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));- let had_buf = hadamard_factors.map(|f| alloc_allocation_with_data::<B, i32>(&context, f));-- let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);- let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- had_buf.as_ref(),- input.k,- input.n,- input.m,- &mut encoder,- );-- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- allocation_to_vec(&y_buf)-}--fn check_against_expected<T: ArrayElement + Float + Debug + Display>(- input: &Input<T>,- expected: &[T],- bm: u32,- bk: u32,- bn: u32,- wm: u32,- wn: u32,- aligned_n: bool,-) {- // GPU tiled accumulation rounds differently from CPU scalar reference.- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {- DataType::BF16 => (0.05, 0.5),- DataType::F16 => (0.02, 0.5),- _ => (0.01, 0.1),- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input, bm, bk, bn, wm, wn, aligned_n, None);- assert_eq!(expected.len(), output.len(), "Output length mismatch");-- let mut errors = 0;- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {- let exp_f32 = exp.to_f32().unwrap();- let got_f32 = got.to_f32().unwrap();- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {- if errors < 5 {- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());- }- errors += 1;- }- }- assert_eq!(- errors,- 0,- "QMM transposed kernel: backend={}, shape=(BM={},BK={},BN={}), m={}, k={}, n={}, gs={}, bits={}, format={:?}: {} mismatches",- std::any::type_name::<B>(),- bm,- bk,- bn,- input.m,- input.k,- input.n,- input.group_size,- input.bits,- input.quant_method,- errors,- );- });-}--struct TileCfg {- bm: u32,- bk: u32,- bn: u32,- wm: u32,- wn: u32,-}--impl TileCfg {- const fn aligned_n_default(&self) -> bool {- // BN=64 tiles require output_dim % 64 == 0; BN=32 checks actual N at test time.- self.bn == 64- }-}--const CFG_BM8: TileCfg = TileCfg {- bm: 8,- bk: 32,- bn: 32,- wm: 1,- wn: 1,-};-const CFG_32X32: TileCfg = TileCfg {- bm: 32,- bk: 32,- bn: 32,- wm: 2,- wn: 2,-};-const CFG_WIDE: TileCfg = TileCfg {- bm: 64,- bk: 32,- bn: 64,- wm: 2,- wn: 2,-};-const CFG_64X64: TileCfg = TileCfg {- bm: 64,- bk: 64,- bn: 64,- wm: 2,- wn: 2,-};--fn pick_aligned_n(- cfg: &TileCfg,- n: usize,-) -> bool {- if cfg.bn == 32 {- (n % 32) == 0- } else {- cfg.aligned_n_default()- }-}--fn test_basic<T: ArrayElement + Float + Debug + Display>(- cfg: &TileCfg,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);- let gs = group_size as usize;- let bn = cfg.bn as usize;- // half+8bit: smaller dims to cap accumulated rounding error.- let dims: Vec<(usize, usize, usize)> = if is_half && bits == 8 {- vec![(2, gs, bn), (4, gs, bn)]- } else {- vec![(2, gs * 2, bn), (4, gs * 2, bn * 2), (8, gs, bn * 2)]- };-- for (m, k, n) in dims {- let input = make_input_basic::<T>(m, k, n, group_size, bits, quant_method);- let aligned_n = pick_aligned_n(cfg, n);- let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n, None);- check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n);- }-}--fn test_edge<T: ArrayElement + Float + Debug + Display>(- cfg: &TileCfg,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- // Edge N is one BN tile wide — smallest legal N for the shape.- let n = cfg.bn as usize;- let input = make_input_edge::<T>(n, group_size, bits, quant_method);- let aligned_n = pick_aligned_n(cfg, n);- let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n, None);- check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n);-}--// No CPU Hadamard reference — verify all three tile shapes agree instead.-fn test_hadamard_consistency<T: ArrayElement + Float + Debug + Display>(- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- // m=n=64 satisfies both BN=32 and BN=64 alignment; group_size >= 64 for BK=64.- let m = 64usize;- let k = group_size as usize * 2;- let n = 64usize;-- let input = make_input_basic::<T>(m, k, n, group_size, bits, quant_method);-- // ±1 factors: non-trivial Hadamard, bounded output across all dtypes.- let factors: Vec<i32> = (0..n)- .map(|i| {- if (i * 13 + 7) % 2 == 0 {- 1- } else {- -1- }- })- .collect();-- // Hadamard amplifies by sqrt(32) ≈ 5.65; tolerance widened accordingly.- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {- DataType::BF16 => (0.1, 1.0),- DataType::F16 => (0.05, 1.0),- _ => (0.02, 0.5),- };-- for_each_non_cpu_backend!(|B| {- let shapes: [(u32, u32, u32, u32, u32, &str); 3] =- [(32, 32, 32, 2, 2, "32x32"), (64, 32, 64, 2, 2, "wide"), (64, 64, 64, 2, 2, "64x64")];-- let outputs: Vec<(&str, Vec<T>)> = shapes- .iter()- .map(|&(bm, bk, bn, wm, wn, name)| {- let out = get_output::<B, T>(&input, bm, bk, bn, wm, wn, true, Some(&factors));- (name, out)- })- .collect();-- // 32x32 is the reference — its Hadamard path is byte-identical to the original.- let (_, ref reference) = outputs[0];-- for (name, output) in outputs.iter().skip(1) {- let mut errors = 0;- for (i, (&r, &o)) in reference.iter().zip(output.iter()).enumerate() {- let r_f32 = r.to_f32().unwrap();- let o_f32 = o.to_f32().unwrap();- if !check_tolerance(r_f32, o_f32, rel_tol, abs_tol) {- if errors < 5 {- eprintln!(" idx={} 32x32={} {}={} diff={}", i, r_f32, name, o_f32, (r_f32 - o_f32).abs());- }- errors += 1;- }- }- assert_eq!(- errors,- 0,- "Hadamard cross-shape mismatch: backend={}, {} vs 32x32 reference, gs={}, bits={}, format={:?}: {} mismatches",- std::any::type_name::<B>(),- name,- group_size,- bits,- quant_method,- errors,- );- }- });-}--// ---- Test-function macros -------------------------------------------------macro_rules! basic_tests {- ($cfg:expr; $(- $(#[$attr:meta])*- fn $name:ident : $t:ty, $gs:literal, $bits:literal, $format:expr;- )*) => {- $(- $(#[$attr])*- #[uzu_test]- fn $name() {- test_basic::<$t>($cfg, $gs, $bits, $format);- }- )*- };-}--macro_rules! edge_tests {- ($cfg:expr; $(- $(#[$attr:meta])*- fn $name:ident : $t:ty, $gs:literal, $bits:literal, $format:expr;- )*) => {- $(- $(#[$attr])*- #[uzu_test]- fn $name() {- test_edge::<$t>($cfg, $gs, $bits, $format);- }- )*- };-}--macro_rules! hadamard_tests {- ($(- $(#[$attr:meta])*- fn $name:ident : $t:ty, $gs:literal, $bits:literal, $format:expr;- )*) => {- $(- $(#[$attr])*- #[uzu_test]- fn $name() {- test_hadamard_consistency::<$t>($gs, $bits, $format);- }- )*- };-}--mod tile_bm8 {- use super::*;-- basic_tests!(&CFG_BM8;- // bf16 4-bit ZP has known precision limits- #[ignore] fn test_bf16_gs32_4bit_zp: bf16, 32, 4, QuantizationMethod::ScaleZeroPoint;- #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs32_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs64_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;- );-- edge_tests!(&CFG_BM8;- fn test_edge_bf16_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;- fn test_edge_bf16_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;- fn test_edge_bf16_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;- );-}--// ---- tile_32x32 — the original base shape, full dtype matrix ---------------mod tile_32x32 {- use super::*;-- basic_tests!(&CFG_32X32;- // 4-bit, zero points- fn test_f32_gs32_4bit_zp: f32, 32, 4, QuantizationMethod::ScaleZeroPoint;- fn test_f32_gs64_4bit_zp: f32, 64, 4, QuantizationMethod::ScaleZeroPoint;- fn test_f32_gs128_4bit_zp: f32, 128, 4, QuantizationMethod::ScaleZeroPoint;- fn test_f16_gs32_4bit_zp: f16, 32, 4, QuantizationMethod::ScaleZeroPoint;- fn test_f16_gs64_4bit_zp: f16, 64, 4, QuantizationMethod::ScaleZeroPoint;- fn test_f16_gs128_4bit_zp: f16, 128, 4, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs32_4bit_zp: bf16, 32, 4, QuantizationMethod::ScaleZeroPoint;- #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;- #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, QuantizationMethod::ScaleZeroPoint;- // 8-bit, zero points- fn test_f32_gs32_8bit_zp: f32, 32, 8, QuantizationMethod::ScaleZeroPoint;- fn test_f32_gs64_8bit_zp: f32, 64, 8, QuantizationMethod::ScaleZeroPoint;- fn test_f32_gs128_8bit_zp: f32, 128, 8, QuantizationMethod::ScaleZeroPoint;- fn test_f16_gs32_8bit_zp: f16, 32, 8, QuantizationMethod::ScaleZeroPoint;- fn test_f16_gs64_8bit_zp: f16, 64, 8, QuantizationMethod::ScaleZeroPoint;- fn test_f16_gs128_8bit_zp: f16, 128, 8, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs32_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs64_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs128_8bit_zp: bf16,128, 8, QuantizationMethod::ScaleZeroPoint;- // 4-bit, mlx quant- fn test_f32_gs32_4bit_mlx: f32, 32, 4, QuantizationMethod::ScaleBias;- fn test_f32_gs64_4bit_mlx: f32, 64, 4, QuantizationMethod::ScaleBias;- fn test_f32_gs128_4bit_mlx: f32, 128, 4, QuantizationMethod::ScaleBias;- fn test_f16_gs32_4bit_mlx: f16, 32, 4, QuantizationMethod::ScaleBias;- fn test_f16_gs64_4bit_mlx: f16, 64, 4, QuantizationMethod::ScaleBias;- fn test_f16_gs128_4bit_mlx: f16, 128, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs128_4bit_mlx: bf16,128, 4, QuantizationMethod::ScaleBias;- // 8-bit, mlx quant- fn test_f32_gs32_8bit_mlx: f32, 32, 8, QuantizationMethod::ScaleBias;- fn test_f32_gs64_8bit_mlx: f32, 64, 8, QuantizationMethod::ScaleBias;- fn test_f32_gs128_8bit_mlx: f32, 128, 8, QuantizationMethod::ScaleBias;- fn test_f16_gs32_8bit_mlx: f16, 32, 8, QuantizationMethod::ScaleBias;- fn test_f16_gs64_8bit_mlx: f16, 64, 8, QuantizationMethod::ScaleBias;- fn test_f16_gs128_8bit_mlx: f16, 128, 8, QuantizationMethod::ScaleBias;- fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;- fn test_bf16_gs128_8bit_mlx: bf16,128, 8, QuantizationMethod::ScaleBias;- );-- edge_tests!(&CFG_32X32;- fn test_edge_f32_4bit_zp: f32, 32, 4, QuantizationMethod::ScaleZeroPoint;- fn test_edge_f32_8bit_zp: f32, 32, 8, QuantizationMethod::ScaleZeroPoint;- fn test_edge_f16_4bit_zp: f16, 32, 4, QuantizationMethod::ScaleZeroPoint;- fn test_edge_f32_4bit_mlx: f32, 32, 4, QuantizationMethod::ScaleBias;- fn test_edge_f32_8bit_mlx: f32, 32, 8, QuantizationMethod::ScaleBias;- fn test_edge_f16_4bit_mlx: f16, 32, 4, QuantizationMethod::ScaleBias;- );-}--// ---- tile_wide — BM=64, BK=32, BN=64, bf16 only ----------------------------mod tile_wide {- use super::*;-- basic_tests!(&CFG_WIDE;- // 4-bit, zero points- fn test_bf16_gs32_4bit_zp: bf16, 32, 4, QuantizationMethod::ScaleZeroPoint;- #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;- #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, QuantizationMethod::ScaleZeroPoint;- // 8-bit, zero points- fn test_bf16_gs32_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs64_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs128_8bit_zp: bf16, 128, 8, QuantizationMethod::ScaleZeroPoint;- // 4-bit, mlx quant- fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, QuantizationMethod::ScaleBias;- // 8-bit, mlx quant- fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;- fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, QuantizationMethod::ScaleBias;- );-- edge_tests!(&CFG_WIDE;- fn test_edge_bf16_4bit_zp: bf16, 32, 4, QuantizationMethod::ScaleZeroPoint;- fn test_edge_bf16_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;- fn test_edge_bf16_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;- fn test_edge_bf16_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;- );-}--// ---- tile_64x64 — BM=BK=BN=64, bf16 only, group_size >= 64 ----------------mod tile_64x64 {- use super::*;-- basic_tests!(&CFG_64X64;- // 4-bit, zero points- #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;- #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, QuantizationMethod::ScaleZeroPoint;- // 8-bit, zero points- fn test_bf16_gs64_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;- fn test_bf16_gs128_8bit_zp: bf16, 128, 8, QuantizationMethod::ScaleZeroPoint;- // 4-bit, mlx quant- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, QuantizationMethod::ScaleBias;- // 8-bit, mlx quant- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;- fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, QuantizationMethod::ScaleBias;- );-- edge_tests!(&CFG_64X64;- fn test_edge_bf16_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;- fn test_edge_bf16_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;- fn test_edge_bf16_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;- fn test_edge_bf16_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;- );-}--// ---- hadamard — cross-shape consistency of the generalized HT loop ---------mod hadamard {- use super::*;-- // f32 at (64,64,64) overflows threadgroup memory (34816 > 32768 bytes).- hadamard_tests!(- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, QuantizationMethod::ScaleBias;- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;- fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, QuantizationMethod::ScaleBias;- fn test_f16_gs64_4bit_mlx: f16, 64, 4, QuantizationMethod::ScaleBias;- fn test_f16_gs128_4bit_mlx: f16, 128, 4, QuantizationMethod::ScaleBias;- );-}-diff --git a/crates/download-manager/src/backends/universal/context.rs b/crates/download-manager/src/backends/universal/context.rsindex 1c810f97b..bd9e43b84 100644--- a/crates/download-manager/src/backends/universal/context.rs+++ b/crates/download-manager/src/backends/universal/context.rs@@ -184,6 +184,18 @@ async fn download_once(resume_from_bytes},StatusCode::OK => 0,+ StatusCode::RANGE_NOT_SATISFIABLE => {+ let advertised_total =+ content_range.as_ref().and_then(|header| header.to_str().ok()).and_then(parse_content_range_total);+ if advertised_total == Some(resume_from_bytes) {+ backend_event_sender.send_progress(generation, resume_from_bytes, Some(resume_from_bytes)).await;+ return tokio::fs::rename(resume_artifact_path, &config.destination)+ .await+ .map_err(|error| error.to_string());+ }+ let _ = tokio::fs::remove_file(resume_artifact_path).await;+ return Err(format!("server did not honor range request: status {status}"));+ },_ => return Err(format!("server did not honor range request: status {status}")),}} else {@@ -238,3 +250,12 @@ fn parse_content_range_start(header_value: &str) -> Option<u64> {let (start, _) = range.split_once('-')?;start.parse::<u64>().ok()}++fn parse_content_range_total(header_value: &str) -> Option<u64> {+ let value = header_value.strip_prefix("bytes ")?.trim_start();+ let (_, total) = value.split_once('/')?;+ if total == "*" {+ return None;+ }+ total.parse::<u64>().ok()+}