Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Unified Quantized Gemm (#432)

eugene7ade24dfd1f0
diff --git a/Cargo.lock b/Cargo.lock
index 91095f9e7..ef6103615 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -595,6 +595,7 @@ dependencies = [
"rangemap",
"regex",
"rhai",
+ "rstest",
"rustix",
"schemars 1.2.1",
"serde",
diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.toml
index 7134ac07e..1c340800d 100644
--- a/crates/backend-uzu/Cargo.toml
+++ b/crates/backend-uzu/Cargo.toml
@@ -60,6 +60,7 @@ is_close.workspace = true
schemars.workspace = true
comfy-table.workspace = true
indicatif.workspace = true
+rstest.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
criterion = { workspace = true, default-features = true }
diff --git a/crates/backend-uzu/build/common/constraints.rs b/crates/backend-uzu/build/common/constraints.rs
index bc9c83298..c341f015f 100644
--- a/crates/backend-uzu/build/common/constraints.rs
+++ b/crates/backend-uzu/build/common/constraints.rs
@@ -1,26 +1,60 @@
-use rhai::{Dynamic, Engine, Scope};
+use std::collections::{BTreeMap, BTreeSet};
-/// Returns true if all constraints pass for the given name/value bindings.
-/// Panics on eval failure (programming error in kernel definition).
-pub fn satisfied<N: AsRef<str>, V: AsRef<str>>(
- engine: &Engine,
- bindings: &[(N, V)],
- constraints: &[impl AsRef<str>],
-) -> bool {
- if constraints.is_empty() {
- return true;
+use rhai::{Dynamic, Engine, Module, Scope};
+
+#[cfg(all(feature = "metal", target_os = "macos"))]
+use crate::common::mangling::unqualify_variant;
+
+#[cfg(not(all(feature = "metal", target_os = "macos")))]
+fn unqualify_variant(value: &str) -> &str {
+ value.rsplit("::").next().unwrap_or(value)
+}
+
+pub struct Evaluator {
+ engine: Engine,
+}
+
+impl Evaluator {
+ pub fn new<'a>(variant_values: impl IntoIterator<Item = &'a str>) -> Self {
+ let mut engine = Engine::new();
+ let mut namespaces: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
+ for value in variant_values {
+ if let Some((namespace, name)) = value.rsplit_once("::") {
+ namespaces.entry(namespace).or_default().insert(name);
+ }
+ }
+ for (namespace, names) in namespaces {
+ let mut module = Module::new();
+ for name in names {
+ module.set_var(name, name.to_string());
+ }
+ engine.register_static_module(namespace, module.into());
+ }
+ Self {
+ engine,
+ }
}
- let mut scope = Scope::with_capacity(bindings.len());
- for (name, val) in bindings {
- let val = val.as_ref();
- scope.push(
- name.as_ref().to_owned(),
- engine.eval_expression::<Dynamic>(val).unwrap_or_else(|_| val.to_owned().into()),
- );
+
+ pub fn satisfied<N: AsRef<str>, V: AsRef<str>>(
+ &self,
+ bindings: &[(N, V)],
+ constraints: &[impl AsRef<str>],
+ ) -> bool {
+ if constraints.is_empty() {
+ return true;
+ }
+ let mut scope = Scope::with_capacity(bindings.len());
+ for (name, val) in bindings {
+ let val = unqualify_variant(val.as_ref());
+ scope.push(
+ name.as_ref().to_owned(),
+ self.engine.eval_expression::<Dynamic>(val).unwrap_or_else(|_| val.to_owned().into()),
+ );
+ }
+ constraints.iter().all(|c| {
+ self.engine
+ .eval_expression_with_scope::<bool>(&mut scope, c.as_ref())
+ .unwrap_or_else(|e| panic!("constraint `{}` failed to evaluate: {e}", c.as_ref()))
+ })
}
- constraints.iter().all(|c| {
- engine
- .eval_expression_with_scope::<bool>(&mut scope, c.as_ref())
- .unwrap_or_else(|e| panic!("constraint `{}` failed to evaluate: {e}", c.as_ref()))
- })
}
diff --git a/crates/backend-uzu/build/common/mangling.rs b/crates/backend-uzu/build/common/mangling.rs
index 2b256af80..8fe99edc6 100644
--- a/crates/backend-uzu/build/common/mangling.rs
+++ b/crates/backend-uzu/build/common/mangling.rs
@@ -6,6 +6,10 @@ use itertools::Itertools;
use proc_macro2::TokenStream;
use quote::quote;
+pub fn unqualify_variant(value: &str) -> &str {
+ value.rsplit("::").next().unwrap_or(value)
+}
+
pub fn static_mangle(
function_name: impl AsRef<str>,
variant: impl IntoIterator<Item = impl AsRef<str>>,
@@ -14,7 +18,13 @@ pub fn static_mangle(
"_D{}{}{}",
&function_name.as_ref().len().to_string(),
function_name.as_ref(),
- variant.into_iter().map(|v| format!("S{}V{}", v.as_ref().len(), v.as_ref().replace('-', "n"))).join("")
+ variant
+ .into_iter()
+ .map(|v| {
+ let v = unqualify_variant(v.as_ref()).replace('-', "n");
+ format!("S{}V{}", v.len(), v)
+ })
+ .join("")
)
}
diff --git a/crates/backend-uzu/build/cpu/compiler.rs b/crates/backend-uzu/build/cpu/compiler.rs
index 5e7ffd5e9..903dca53a 100644
--- a/crates/backend-uzu/build/cpu/compiler.rs
+++ b/crates/backend-uzu/build/cpu/compiler.rs
@@ -503,7 +503,12 @@ impl CpuCompiler {
parameter_idents = quote! { (#parameter_idents) };
}
- let rhai_engine = (!function_constraints.is_empty()).then(rhai::Engine::new);
+ let variant_value_strs: Vec<String> = function_variants
+ .iter()
+ .flat_map(|(_, variants)| variants.iter().map(|v| v.to_token_stream().to_string()))
+ .collect();
+ let evaluator = (!function_constraints.is_empty())
+ .then(|| crate::common::constraints::Evaluator::new(variant_value_strs.iter().map(|s| s.as_str())));
let constraint_strs: Vec<String> =
function_constraints.iter().map(|c| c.to_token_stream().to_string()).collect();
@@ -527,7 +532,7 @@ impl CpuCompiler {
})
.multi_cartesian_product()
.filter(|variants| {
- let Some(engine) = &rhai_engine else {
+ let Some(evaluator) = &evaluator else {
return true;
};
let bindings: Vec<(String, String)> = function_parameters
@@ -535,7 +540,7 @@ impl CpuCompiler {
.enumerate()
.map(|(i, p)| (p.name.to_string(), variants[i].1.to_string()))
.collect();
- crate::common::constraints::satisfied(engine, &bindings, &constraint_strs)
+ evaluator.satisfied(&bindings, &constraint_strs)
})
.map(|variants| {
let (match_variants, generic_variants): (Vec<TokenStream>, Vec<TokenStream>) =
diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs
index 86903d05c..a44f01e78 100644
--- a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs
+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs
@@ -39,6 +39,11 @@ pub fn gpu_type_gen_option_set(option_set: &GpuTypeOptionSet) -> anyhow::Result<
\x20 constexpr bool contains({underlying_c} flag) const thread {{\n\
\x20\x20\x20 return (raw_value & flag) != 0;\n\
\x20 }}\n\
+ \x20 constexpr bool contains({underlying_c} flag) const constant {{\n\
+ \x20\x20\x20 return (raw_value & flag) != 0;\n\
+ \x20 }}\n\
+ \x20 constexpr {underlying_c} bits() const thread {{ return raw_value; }}\n\
+ \x20 constexpr {underlying_c} bits() const constant {{ return raw_value; }}\n\
}};"
))
}
diff --git a/crates/backend-uzu/build/metal/wrapper.rs b/crates/backend-uzu/build/metal/wrapper.rs
index 6ee119cbf..12cda6a27 100644
--- a/crates/backend-uzu/build/metal/wrapper.rs
+++ b/crates/backend-uzu/build/metal/wrapper.rs
@@ -137,7 +137,9 @@ fn kernel_wrappers(
kernel_wrappers.push(kernel_header(&bindings, base).into_boxed_str());
}
- let engine = rhai::Engine::new();
+ let evaluator = crate::common::constraints::Evaluator::new(
+ kernel.variants.as_deref().into_iter().flatten().flat_map(|tp| tp.variants.iter().map(|v| v.as_ref())),
+ );
for type_variant in if let Some(variants) = &kernel.variants {
variants
.iter()
@@ -157,7 +159,7 @@ fn kernel_wrappers(
vec![None]
} {
if let Some(ref tv) = type_variant {
- if !crate::common::constraints::satisfied(&engine, tv, &kernel.constraints) {
+ if !evaluator.satisfied(tv, &kernel.constraints) {
continue;
}
}
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
index a12f8652f..c77613df7 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
@@ -3,40 +3,114 @@ use derive_more::Display;
#[repr(C)]
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum GemmInputPrologueKind {
+pub enum GemmWeightPrologueKind {
FullPrecision,
- ExternalRht,
+ ScaleBiasDequant,
+ ScaleZeroPointDequant,
}
-#[repr(C)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum GemmOutputTransformKind {
- Store,
- Scale,
- Accumulate,
- Bias,
- Rht,
- ScaleAccumulate,
- ScaleAccumulateBias,
- ScaleAccumulateBiasRht,
+bitflags! {
+ #[repr(transparent)]
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+ pub struct GemmDTransform: u32 {
+ const SCALE = 1 << 0;
+ const ACCUMULATE = 1 << 1;
+ const BIAS = 1 << 2;
+ const RHT = 1 << 3;
+ }
}
#[repr(C)]
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum GemmWeightPrologueKind {
- FullPrecision,
- ScaleBiasDequant,
- ScaleZeroPointDequant,
+pub enum GemmTiling {
+ T8x32x32_1x1,
+ T64x32x32_2x2,
+ T64x64x16_2x2,
+ T64x64x32_2x2,
+ T64x64x64_2x2,
+ T32x32x32_2x2,
+ T32x64x32_2x2,
+ T64x32x32_4x1,
+ T128x128x32_4x4,
}
-#[repr(C)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct GemmTilingConfig {
- pub threadgroup_m: u32,
- pub threadgroup_n: u32,
- pub threadgroup_k: u32,
- pub simdgroups_m: u32,
- pub simdgroups_n: u32,
+impl GemmTiling {
+ pub const fn block_m(self) -> u32 {
+ match self {
+ Self::T8x32x32_1x1 => 8,
+ Self::T64x32x32_2x2 => 64,
+ Self::T64x64x16_2x2 => 64,
+ Self::T64x64x32_2x2 => 64,
+ Self::T64x64x64_2x2 => 64,
+ Self::T32x32x32_2x2 => 32,
+ Self::T32x64x32_2x2 => 32,
+ Self::T64x32x32_4x1 => 64,
+ Self::T128x128x32_4x4 => 128,
+ }
+ }
+
+ pub const fn block_n(self) -> u32 {
+ match self {
+ Self::T8x32x32_1x1 => 32,
+ Self::T64x32x32_2x2 => 32,
+ Self::T64x64x16_2x2 => 64,
+ Self::T64x64x32_2x2 => 64,
+ Self::T64x64x64_2x2 => 64,
+ Self::T32x32x32_2x2 => 32,
+ Self::T32x64x32_2x2 => 64,
+ Self::T64x32x32_4x1 => 32,
+ Self::T128x128x32_4x4 => 128,
+ }
+ }
+
+ pub const fn block_k(self) -> u32 {
+ match self {
+ Self::T8x32x32_1x1 => 32,
+ Self::T64x32x32_2x2 => 32,
+ Self::T64x64x16_2x2 => 16,
+ Self::T64x64x32_2x2 => 32,
+ Self::T64x64x64_2x2 => 64,
+ Self::T32x32x32_2x2 => 32,
+ Self::T32x64x32_2x2 => 32,
+ Self::T64x32x32_4x1 => 32,
+ Self::T128x128x32_4x4 => 32,
+ }
+ }
+
+ pub const fn simdgroups_m(self) -> u32 {
+ match self {
+ Self::T8x32x32_1x1 => 1,
+ Self::T64x32x32_2x2 => 2,
+ Self::T64x64x16_2x2 => 2,
+ Self::T64x64x32_2x2 => 2,
+ Self::T64x64x64_2x2 => 2,
+ Self::T32x32x32_2x2 => 2,
+ Self::T32x64x32_2x2 => 2,
+ Self::T64x32x32_4x1 => 4,
+ Self::T128x128x32_4x4 => 4,
+ }
+ }
+
+ pub const fn simdgroups_n(self) -> u32 {
+ match self {
+ Self::T8x32x32_1x1 => 1,
+ Self::T64x32x32_2x2 => 2,
+ Self::T64x64x16_2x2 => 2,
+ Self::T64x64x32_2x2 => 2,
+ Self::T64x64x64_2x2 => 2,
+ Self::T32x32x32_2x2 => 2,
+ Self::T32x64x32_2x2 => 2,
+ Self::T64x32x32_4x1 => 1,
+ Self::T128x128x32_4x4 => 4,
+ }
+ }
+}
+
+pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {
+ t.simdgroups_m()
+}
+pub const fn gemm_tiling_simdgroups_per_column(t: GemmTiling) -> u32 {
+ t.simdgroups_n()
}
bitflags! {
@@ -48,3 +122,17 @@ bitflags! {
const K = 1 << 2;
}
}
+
+impl GemmAlignment {
+ pub fn new(
+ m: bool,
+ n: bool,
+ k: bool,
+ ) -> Self {
+ let mut bits = Self::empty();
+ bits.set(Self::M, m);
+ bits.set(Self::N, n);
+ bits.set(Self::K, k);
+ bits
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul.rs b/crates/backend-uzu/src/backends/common/kernel/matmul.rs
deleted file mode 100644
index 3578ac4d5..000000000
--- a/crates/backend-uzu/src/backends/common/kernel/matmul.rs
+++ /dev/null
@@ -1,61 +0,0 @@
-use thiserror::Error;
-
-use crate::{
- DataType,
- backends::common::{Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, kernel::ManualKernels},
-};
-
-#[derive(Debug, Error)]
-pub enum MatmulError<B: Backend> {
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
-}
-
-pub enum MatmulArgumentC<'a, B: Backend> {
- None,
- /// Accumulate: [M, N]
- Accumulate,
- /// Bias: [N] (broadcasted across M/batch)
- Bias(&'a Allocation<B>),
-}
-
-// D = ab_scale * (A @ op(B)) + C, where op(B) = B^T when b_transpose else B.
-pub struct MatmulArguments<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {
- /// A: [M, K]
- pub a: &'a Allocation<B>,
- pub a_offset: usize,
- /// B: [N, K] when b_transpose, else [K, N]
- pub b: &'a TB,
- pub b_offset: usize,
- pub b_leading_dimension: Option<u32>,
- pub b_transpose: bool,
- /// AB scale: also known as alpha
- pub ab_scale: f32,
- /// C: behavior depends on enum variant
- pub c: MatmulArgumentC<'a, B>,
- /// D: [M, N]
- pub d: &'a mut Allocation<B>,
- /// M dimension: usually batch/number of tokens (rows of A, rows of C)
- pub batch_dim: u32,
- /// K dimension: usually input_dim/reduction dimension (cols of A, rows of B)
- pub input_dim: u32,
- /// N dimension: usually output_dim (cols of B, cols of C)
- pub output_dim: u32,
-}
-
-pub trait MatmulKernel: Sized {
- type Backend: Backend<Kernels: ManualKernels<MatmulKernel = Self>>;
-
- fn new(
- context: &<Self::Backend as Backend>::Context,
- data_type: DataType,
- ) -> Result<Self, MatmulError<Self::Backend>>;
-
- fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Self::Backend>>>(
- &mut self,
- arguments: MatmulArguments<Self::Backend, TB>,
- encoder: &mut Encoder<Self::Backend>,
- );
-}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs
new file mode 100644
index 000000000..8592fd179
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs
@@ -0,0 +1,16 @@
+use super::{d_ops::MatmulDOps, matmul_b::MatmulB};
+use crate::backends::common::{Allocation, AsBufferRangeRef, Backend};
+
+pub struct MatmulArguments<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {
+ pub a: &'a Allocation<B>,
+ pub a_offset: usize,
+ pub b: MatmulB<'a, B, TB>,
+ pub b_offset: usize,
+ pub b_leading_dimension: Option<u32>,
+ pub b_transpose: bool,
+ pub d: &'a mut Allocation<B>,
+ pub d_transform: MatmulDOps<'a, B>,
+ pub m: u32,
+ pub n: u32,
+ pub k: u32,
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/d_ops.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/d_ops.rs
new file mode 100644
index 000000000..5ea89c7bd
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/d_ops.rs
@@ -0,0 +1,64 @@
+use crate::backends::common::{Allocation, Backend, gpu_types::gemm::GemmDTransform};
+
+pub struct MatmulDOps<'a, B: Backend> {
+ pub ab_scale: f32,
+ pub accumulate: bool,
+ pub bias: Option<&'a Allocation<B>>,
+ pub rht_factors: Option<&'a Allocation<B>>,
+}
+
+impl<'a, B: Backend> MatmulDOps<'a, B> {
+ pub fn none() -> Self {
+ Self {
+ ab_scale: 1.0,
+ accumulate: false,
+ bias: None,
+ rht_factors: None,
+ }
+ }
+
+ pub fn mask(&self) -> GemmDTransform {
+ let mut m = GemmDTransform::empty();
+ if self.ab_scale != 1.0 {
+ m |= GemmDTransform::SCALE;
+ }
+ if self.accumulate {
+ m |= GemmDTransform::ACCUMULATE;
+ }
+ if self.bias.is_some() {
+ m |= GemmDTransform::BIAS;
+ }
+ if self.rht_factors.is_some() {
+ m |= GemmDTransform::RHT;
+ }
+ m
+ }
+
+ pub fn without(
+ self,
+ bits: GemmDTransform,
+ ) -> Self {
+ Self {
+ ab_scale: if bits.contains(GemmDTransform::SCALE) {
+ 1.0
+ } else {
+ self.ab_scale
+ },
+ accumulate: if bits.contains(GemmDTransform::ACCUMULATE) {
+ false
+ } else {
+ self.accumulate
+ },
+ bias: if bits.contains(GemmDTransform::BIAS) {
+ None
+ } else {
+ self.bias
+ },
+ rht_factors: if bits.contains(GemmDTransform::RHT) {
+ None
+ } else {
+ self.rht_factors
+ },
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs
new file mode 100644
index 000000000..0c6fd8733
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs
@@ -0,0 +1,25 @@
+use thiserror::Error;
+
+use crate::{
+ DataType,
+ backends::common::{Backend, gpu_types::gemm::GemmDTransform},
+};
+
+#[derive(Debug, Error)]
+pub enum MatmulError<B: Backend> {
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error("Unsupported group size: {0}")]
+ UnsupportedGroupSize(usize),
+ #[error("Unsupported D-transform op {bit:?} on path {path}")]
+ UnsupportedDOp {
+ bit: GemmDTransform,
+ path: &'static str,
+ },
+ #[error("Unsupported B layout on path {path}")]
+ UnsupportedLayout {
+ path: &'static str,
+ },
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
new file mode 100644
index 000000000..152799751
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -0,0 +1,28 @@
+use super::{arguments::MatmulArguments, quant_combo::MatmulQuantCombo};
+use crate::{
+ DataType,
+ backends::common::{AsBufferRangeRef, Backend, Buffer, Encoder, kernel::ManualKernels},
+};
+
+pub trait MatmulKernel: Sized {
+ type Backend: Backend<Kernels: ManualKernels<MatmulKernel = Self>>;
+
+ fn new(
+ context: &<Self::Backend as Backend>::Context,
+ data_type: DataType,
+ ) -> Result<Self, <Self::Backend as Backend>::Error>;
+
+ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Self::Backend>>>(
+ &mut self,
+ arguments: MatmulArguments<Self::Backend, TB>,
+ encoder: &mut Encoder<Self::Backend>,
+ ) -> Result<(), <Self::Backend as Backend>::Error>;
+
+ fn preheat_quant_combo(
+ &mut self,
+ _context: &<Self::Backend as Backend>::Context,
+ _combo: MatmulQuantCombo,
+ ) -> Result<(), <Self::Backend as Backend>::Error> {
+ Ok(())
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
new file mode 100644
index 000000000..55774863a
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
@@ -0,0 +1,75 @@
+use crate::{
+ DataType,
+ backends::common::{
+ Allocation, AsBufferRangeRef, Backend,
+ gpu_types::{QuantizationMode, gemm::GemmWeightPrologueKind},
+ },
+};
+
+pub enum MatmulB<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {
+ FullPrecision {
+ b: &'a TB,
+ },
+ ScaleBiasDequant {
+ b: &'a Allocation<B>,
+ scales: &'a Allocation<B>,
+ biases: &'a Allocation<B>,
+ mode: QuantizationMode,
+ group_size: u32,
+ },
+ ScaleZeroPointDequant {
+ b: &'a Allocation<B>,
+ scales: &'a Allocation<B>,
+ zero_points: &'a Allocation<B>,
+ mode: QuantizationMode,
+ group_size: u32,
+ },
+}
+
+impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {
+ pub fn weight_prologue(&self) -> GemmWeightPrologueKind {
+ match self {
+ Self::FullPrecision {
+ ..
+ } => GemmWeightPrologueKind::FullPrecision,
+ Self::ScaleBiasDequant {
+ ..
+ } => GemmWeightPrologueKind::ScaleBiasDequant,
+ Self::ScaleZeroPointDequant {
+ ..
+ } => GemmWeightPrologueKind::ScaleZeroPointDequant,
+ }
+ }
+
+ pub fn bits_per_weight(&self) -> Option<u32> {
+ match self {
+ Self::FullPrecision {
+ ..
+ } => None,
+ Self::ScaleBiasDequant {
+ mode,
+ ..
+ }
+ | Self::ScaleZeroPointDequant {
+ mode,
+ ..
+ } => Some(DataType::from(*mode).size_in_bits() as u32),
+ }
+ }
+
+ pub fn group_size(&self) -> Option<u32> {
+ match self {
+ Self::FullPrecision {
+ ..
+ } => None,
+ Self::ScaleBiasDequant {
+ group_size,
+ ..
+ }
+ | Self::ScaleZeroPointDequant {
+ group_size,
+ ..
+ } => Some(*group_size),
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs
new file mode 100644
index 000000000..c7ebfca8c
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs
@@ -0,0 +1,13 @@
+mod arguments;
+mod d_ops;
+mod error;
+mod kernel;
+mod matmul_b;
+mod quant_combo;
+
+pub use arguments::MatmulArguments;
+pub use d_ops::MatmulDOps;
+pub use error::MatmulError;
+pub use kernel::MatmulKernel;
+pub use matmul_b::MatmulB;
+pub use quant_combo::MatmulQuantCombo;
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs
new file mode 100644
index 000000000..471933b14
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs
@@ -0,0 +1,8 @@
+use crate::backends::common::gpu_types::{QuantizationMethod, QuantizationMode};
+
+#[derive(Debug, Clone, Copy)]
+pub struct MatmulQuantCombo {
+ pub method: QuantizationMethod,
+ pub mode: QuantizationMode,
+ pub group_size: u32,
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rs
index a2659f323..7df26e067 100644
--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs
@@ -11,6 +11,5 @@ pub mod kv_cache_update;
pub mod matmul;
pub mod mlp_gate_act_mul;
pub mod moe;
-pub mod quant_matmul;
pub mod sampling;
pub mod ssd_prefill;
diff --git a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs
deleted file mode 100644
index 75c1acdd0..000000000
--- a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs
+++ /dev/null
@@ -1,251 +0,0 @@
-use crate::{
- DataType,
- backends::common::{
- Allocation, Backend, Encoder, Kernels,
- gpu_types::{QuantizationMethod, QuantizationMode},
- kernel::{QuantizedMatmulQmmTransposedKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},
- },
-};
-
-#[derive(Debug, thiserror::Error)]
-pub enum QuantizedMatmulError<B: Backend> {
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
- #[error("Unsupported group size: {0}")]
- UnsupportedGroupSize(usize),
- #[error("Hadamard not supported for this kernel configuration")]
- UnsupportedHadamard,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct QuantizedMatmulConfiguration {
- pub data_type: DataType,
- pub group_size: usize,
- pub input_dim: usize,
- pub output_dim: usize,
- pub mode: QuantizationMode,
- pub quantization_method: QuantizationMethod,
- pub use_hadamard: bool,
-}
-
-pub struct QuantizedMatmulArguments<'a, B: Backend> {
- pub a: &'a Allocation<B>,
- pub a_offset: usize,
- pub b: &'a Allocation<B>,
- pub scales: &'a Allocation<B>,
- pub zero_points_or_biases: &'a Allocation<B>,
- pub output: &'a mut Allocation<B>,
- pub hadamard_factors: Option<&'a Allocation<B>>,
- pub batch_dim: usize,
-}
-
-pub struct QuantizedMatmulKernelEncodable<B: Backend> {
- matrix_vector: MatrixVectorKernel<B>,
- matrix_matrix: MatrixMatrixKernel<B>,
- input_dim: usize,
- output_dim: usize,
- quantization_method: QuantizationMethod,
-}
-
-enum MatrixVectorKernel<B: Backend> {
- Qmv(<B::Kernels as Kernels>::QuantizedMatmulQmvKernel),
- QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel),
-}
-
-type QmmKernel<B> = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel;
-
-enum MatrixMatrixKernel<B: Backend> {
- Big(QmmKernel<B>),
- BigAndSmall {
- big: QmmKernel<B>,
- small: QmmKernel<B>,
- },
-}
-
-impl<B: Backend> MatrixMatrixKernel<B> {
- fn pick(
- &self,
- batch_dim: usize,
- ) -> Option<&QmmKernel<B>> {
- match self {
- Self::Big(big) => (batch_dim >= 32).then_some(big),
- Self::BigAndSmall {
- big,
- small,
- } => Some(if batch_dim < 48 {
- small
- } else {
- big
- }),
- }
- }
-}
-
-impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
- pub fn new(
- context: &B::Context,
- configuration: QuantizedMatmulConfiguration,
- ) -> Result<Self, QuantizedMatmulError<B>> {
- if !matches!(configuration.data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(QuantizedMatmulError::UnsupportedDataType(configuration.data_type));
- }
-
- if !matches!(configuration.group_size, 32 | 64 | 128) {
- return Err(QuantizedMatmulError::UnsupportedGroupSize(configuration.group_size));
- }
-
- let bits = match configuration.mode {
- QuantizationMode::U4 => 4,
- QuantizationMode::I8 | QuantizationMode::U8 => 8,
- };
- let group_size = configuration.group_size as u32;
-
- let matrix_vector = if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {
- MatrixVectorKernel::QmvFast(
- <B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- configuration.quantization_method,
- configuration.use_hadamard,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- )
- } else {
- if configuration.use_hadamard {
- return Err(QuantizedMatmulError::UnsupportedHadamard);
- }
-
- MatrixVectorKernel::Qmv(
- <B::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- configuration.quantization_method,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- )
- };
-
- let aligned_n_64 = configuration.output_dim % 64 == 0;
- let aligned_n_32 = configuration.output_dim % 32 == 0;
- let is_bf16 = configuration.data_type == DataType::BF16;
- let can_use_64_tile = aligned_n_64 && is_bf16;
-
- let (bm, bk, bn, aligned_n) = if can_use_64_tile && matches!(configuration.group_size, 64 | 128) {
- (64u32, 64u32, 64u32, true)
- } else if can_use_64_tile {
- (64u32, 32u32, 64u32, true)
- } else {
- (32u32, 32u32, 32u32, configuration.output_dim % 32 == 0)
- };
-
- let big = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- bm,
- bk,
- bn,
- 2u32, // WM
- 2u32, // WN
- configuration.quantization_method,
- configuration.use_hadamard,
- aligned_n,
- )
- .map_err(QuantizedMatmulError::BackendError)?;
-
- let matrix_matrix = if aligned_n_32 && !configuration.use_hadamard {
- let small = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- 8u32, // BM
- 32u32, // BK
- 32u32, // BN
- 1u32, // WM
- 1u32, // WN
- configuration.quantization_method,
- false, // use_hadamard: BM=8 tile does not support hadamard
- true, // aligned_n: BN=32, always aligned for the small tile
- )
- .map_err(QuantizedMatmulError::BackendError)?;
- MatrixMatrixKernel::BigAndSmall {
- big,
- small,
- }
- } else {
- MatrixMatrixKernel::Big(big)
- };
-
- Ok(Self {
- matrix_vector,
- matrix_matrix,
- input_dim: configuration.input_dim,
- output_dim: configuration.output_dim,
- quantization_method: configuration.quantization_method,
- })
- }
-
- pub fn encode(
- &self,
- encoder: &mut Encoder<B>,
- arguments: QuantizedMatmulArguments<B>,
- ) {
- let QuantizedMatmulArguments {
- a,
- a_offset,
- b,
- scales,
- zero_points_or_biases,
- output,
- hadamard_factors,
- batch_dim,
- } = arguments;
-
- let (zero_points, biases) = match self.quantization_method {
- QuantizationMethod::ScaleZeroPoint => (Some(zero_points_or_biases), None),
- QuantizationMethod::ScaleBias => (None, Some(zero_points_or_biases)),
- };
-
- macro_rules! encode_kernel {
- ($kernel:expr $(, $hadamard:expr)?) => {
- $kernel.encode(
- b,
- scales,
- zero_points,
- biases,
- (a, a_offset),
- output,
- $($hadamard,)?
- self.input_dim as u32,
- self.output_dim as u32,
- batch_dim as u32,
- encoder,
- )
- };
- }
-
- if batch_dim >= 5 && self.output_dim > 1 {
- if let Some(kernel) = self.matrix_matrix.pick(batch_dim) {
- encode_kernel!(kernel, hadamard_factors);
- return;
- }
- }
-
- match &self.matrix_vector {
- MatrixVectorKernel::Qmv(k) => encode_kernel!(k),
- MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, hadamard_factors),
- }
- }
-}
-
-#[cfg(test)]
-#[path = "../../../../tests/unit/backends/common/kernel/quant_matmul_test.rs"]
-mod tests;
diff --git a/crates/backend-uzu/src/backends/cpu/error.rs b/crates/backend-uzu/src/backends/cpu/error.rs
index 57e1ba306..5b5ca0821 100644
--- a/crates/backend-uzu/src/backends/cpu/error.rs
+++ b/crates/backend-uzu/src/backends/cpu/error.rs
@@ -1,11 +1,24 @@
-use std::sync::mpsc::RecvError;
+use std::{error::Error as StdError, sync::mpsc::RecvError};
use thiserror::Error;
+use crate::backends::{common::kernel::matmul::MatmulError, cpu::Cpu};
+
#[derive(Debug, Error)]
pub enum CpuError {
#[error("Not supported")]
NotSupported,
#[error("Command buffer execution failed")]
CommandBufferExecutionFailed(RecvError),
+ #[error("Kernel dispatch failed: {0}")]
+ KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),
+}
+
+impl From<MatmulError<Cpu>> for CpuError {
+ fn from(value: MatmulError<Cpu>) -> Self {
+ match value {
+ MatmulError::BackendError(e) => e,
+ other => CpuError::KernelDispatchFailed(Box::new(other)),
+ }
+ }
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs
index b68fab06e..a20050452 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs
@@ -1 +1,2 @@
+pub mod qmm_transposed;
pub mod shaders;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs
similarity index 56%
rename from crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs
index 1c7e730cb..56b2afffc 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs
@@ -1,6 +1,4 @@
-use half::{bf16, f16};
use num_traits::Float;
-use proc_macros::kernel;
use crate::{ArrayElement, backends::common::gpu_types::QuantizationMethod};
@@ -33,7 +31,6 @@ pub fn qmm_transposed<T: ArrayElement + Float>(
for l in 0..in_vec_size {
let group_idx = l / group_size;
- // Transposed: weight at row j, col l
let weight_linear_idx = j * in_vec_size + l;
let val_q = if bits == 4 {
@@ -49,10 +46,6 @@ pub fn qmm_transposed<T: ArrayElement + Float>(
let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();
let scale_t = *scales.add(j * num_groups_k + group_idx);
- // Dequantize in f32 to serve as a precision-neutral reference.
- // Metal dequantizes in bf16/f16 which rounds differently across
- // GPU hardware; computing in f32 avoids biasing toward any
- // particular rounding pattern.
let w_dequant_f32 = match quant_method {
QuantizationMethod::ScaleZeroPoint => {
let zp = zero_points.unwrap();
@@ -83,64 +76,3 @@ pub fn qmm_transposed<T: ArrayElement + Float>(
}
}
}
-
-#[kernel(QuantizedMatmulQmmTransposed)]
-#[variants(T, f32, f16, bf16)]
-#[variants(GROUP_SIZE, 32, 64, 128)]
-#[variants(BITS, 4, 8)]
-#[variants(BM, 8, 32, 64)]
-#[variants(BK, 32, 64)]
-#[variants(BN, 32, 64)]
-#[variants(WM, 1, 2)]
-#[variants(WN, 1, 2)]
-#[constraint(
- (BM == 8 && BK == 32 && BN == 32 && WM == 1 && WN == 1) ||
- (BM == 32 && BK == 32 && BN == 32 && WM == 2 && WN == 2) ||
- (BM == 32 && BK == 64 && BN == 32 && WM == 2 && WN == 2) ||
- (BM == 64 && BK == 32 && BN == 64 && WM == 2 && WN == 2) ||
- (BM == 64 && BK == 64 && BN == 64 && WM == 2 && WN == 2))]
-#[constraint(BK <= GROUP_SIZE)]
-#[constraint(T != "f32" || BK < 64)]
-pub fn quantized_matmul_qmm_transposed<
- T: ArrayElement + Float,
- const GROUP_SIZE: u32,
- const BITS: u32,
- const BM: u32,
- const BK: u32,
- const BN: u32,
- const WM: u32,
- const WN: u32,
->(
- weights: *const u32,
- scales: *const T,
- #[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,
- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,
- input: *const T,
- output: *mut T,
- #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
- in_vec_size: u32,
- out_vec_size: u32,
- batch_size: u32,
- #[specialize] quant_method: QuantizationMethod,
- #[specialize] use_hadamard: bool,
- #[specialize] aligned_n: bool,
-) {
- let _ = (BM, BK, BN, WM, WN); // tile params unused in tile-agnostic CPU reference
- if use_hadamard {
- unimplemented!("not supported yet");
- }
- qmm_transposed::<T>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- in_vec_size as usize,
- out_vec_size as usize,
- batch_size as usize,
- quant_method,
- GROUP_SIZE as usize,
- BITS as usize,
- );
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs
similarity index 56%
rename from crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs
index c8b61dac4..e4e00c1e2 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs
@@ -1,3 +1,2 @@
-pub mod qmm_transposed;
pub mod qmv;
pub mod qmv_fast;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
similarity index 97%
rename from crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
index da39ebe32..0a42788ff 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
@@ -4,7 +4,7 @@ use proc_macros::kernel;
use crate::{
ArrayElement,
- backends::{common::gpu_types::QuantizationMethod, cpu::kernel::quant_matmul::qmv::qmv},
+ backends::{common::gpu_types::QuantizationMethod, cpu::kernel::matmul::gemv::qmv::qmv},
};
#[kernel(QuantizedMatmulQmvFast)]
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 9cac8dce3..58094e700 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -4,30 +4,44 @@ use crate::{
DataType,
backends::{
common::{
- AsBufferRangeMut, AsBufferRangeRef, Buffer, Encoder,
- kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
+ Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,
+ gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},
+ kernel::{
+ HadamardTransformKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, TensorAddBiasKernel,
+ matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel},
+ },
+ },
+ cpu::{
+ BufferDowncastExt, Cpu, context::CpuContext, error::CpuError, kernel::matmul::quant::encode_quantized_gemm,
},
- cpu::{BufferDowncastExt, Cpu, context::CpuContext},
},
utils::pointers::{SendPtr, SendPtrMut},
};
pub struct MatmulCpuKernel {
data_type: DataType,
+ hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,
+ bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,
}
impl MatmulKernel for MatmulCpuKernel {
type Backend = Cpu;
fn new(
- _context: &CpuContext,
+ context: &CpuContext,
data_type: DataType,
- ) -> Result<Self, MatmulError<Cpu>> {
+ ) -> Result<Self, CpuError> {
if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
+ return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());
}
+ let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)
+ .map_err(CpuError::from)?;
+ let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, true)
+ .map_err(CpuError::from)?;
Ok(Self {
data_type,
+ hadamard,
+ bias_add,
})
}
@@ -35,7 +49,38 @@ impl MatmulKernel for MatmulCpuKernel {
&mut self,
arguments: MatmulArguments<Cpu, TB>,
encoder: &mut Encoder<Cpu>,
- ) {
+ ) -> Result<(), CpuError> {
+ match arguments.b {
+ MatmulB::FullPrecision {
+ ..
+ } => self.encode_fp(arguments, encoder).map_err(CpuError::from),
+ MatmulB::ScaleBiasDequant {
+ ..
+ }
+ | MatmulB::ScaleZeroPointDequant {
+ ..
+ } => self.encode_quant(arguments, encoder).map_err(CpuError::from),
+ }
+ }
+}
+
+impl MatmulCpuKernel {
+ fn encode_fp<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
+ &mut self,
+ arguments: MatmulArguments<Cpu, TB>,
+ encoder: &mut Encoder<Cpu>,
+ ) -> Result<(), MatmulError<Cpu>> {
+ if arguments.d_transform.mask().contains(GemmDTransform::RHT) {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::RHT,
+ path: "MatmulCpuKernel",
+ });
+ }
+
+ let ab_scale = arguments.d_transform.ab_scale;
+ let bias_alloc = arguments.d_transform.bias;
+ let is_accumulate = arguments.d_transform.accumulate;
+
let MatmulArguments {
a,
a_offset,
@@ -43,51 +88,49 @@ impl MatmulKernel for MatmulCpuKernel {
b_offset,
b_leading_dimension,
b_transpose,
- ab_scale,
- c,
d,
- batch_dim,
- input_dim,
- output_dim,
+ m,
+ n,
+ k,
+ ..
} = arguments;
+ let weights = match b {
+ MatmulB::FullPrecision {
+ b: w,
+ } => w,
+ _ => unreachable!(),
+ };
+
let command_buffer = encoder.as_command_buffer_mut();
- let m = batch_dim as usize;
- let n = output_dim as usize;
- let k = input_dim as usize;
- let lda = k;
+ let m_u = m as usize;
+ let n_u = n as usize;
+ let k_u = k as usize;
+ let lda = k_u;
let ldb = b_leading_dimension.map(|n| n as usize).unwrap_or(if b_transpose {
- k
+ k_u
} else {
- n
+ n_u
});
- let ldd = n;
+ let ldd = n_u;
let data_type = self.data_type;
let a_buffer_range = a.as_buffer_range_ref();
- let b_buffer_range = b.as_buffer_range_ref();
+ let b_buffer_range = weights.as_buffer_range_ref();
let d_buffer_range = d.as_buffer_range_mut();
- let a_offset = a_buffer_range.range().start + a_offset;
- let b_offset = b_buffer_range.range().start + b_offset;
- let d_offset = d_buffer_range.range().start;
+ let a_byte_off = a_buffer_range.range().start + a_offset;
+ let b_byte_off = b_buffer_range.range().start + b_offset;
+ let d_byte_off = d_buffer_range.range().start;
- let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_offset));
- let b_ptr = SendPtr(unsafe { &*b_buffer_range.buffer().downcast().get() }.as_ptr().wrapping_byte_add(b_offset));
+ let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));
+ let b_ptr =
+ SendPtr(unsafe { &*b_buffer_range.buffer().downcast().get() }.as_ptr().wrapping_byte_add(b_byte_off));
let d_ptr =
- SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_offset) as *mut u8 });
-
- let (is_accumulate, bias_ptr) = match c {
- MatmulArgumentC::Accumulate => (true, None),
- MatmulArgumentC::Bias(bias) => {
- let bias_buffer_range = bias.as_buffer_range_ref();
- let bias_range = bias_buffer_range.range();
- (
- false,
- Some(SendPtr(
- unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start),
- )),
- )
- },
- MatmulArgumentC::None => (false, None),
- };
+ SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });
+
+ let bias_ptr = bias_alloc.map(|bias| {
+ let bias_buffer_range = bias.as_buffer_range_ref();
+ let bias_range = bias_buffer_range.range();
+ SendPtr(unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start))
+ });
command_buffer.push_command(move || match data_type {
DataType::F32 => {
@@ -96,9 +139,9 @@ impl MatmulKernel for MatmulCpuKernel {
b_ptr.as_ptr() as *const f32,
ab_scale,
d_ptr.as_ptr() as *mut f32,
- m,
- n,
- k,
+ m_u,
+ n_u,
+ k_u,
lda,
ldb,
ldd,
@@ -106,7 +149,7 @@ impl MatmulKernel for MatmulCpuKernel {
is_accumulate,
);
if let Some(bias) = bias_ptr {
- apply_bias::<f32>(d_ptr.as_ptr() as *mut f32, bias.as_ptr() as *const f32, m, n);
+ apply_bias::<f32>(d_ptr.as_ptr() as *mut f32, bias.as_ptr() as *const f32, m_u, n_u);
}
},
DataType::F16 => {
@@ -115,9 +158,9 @@ impl MatmulKernel for MatmulCpuKernel {
b_ptr.as_ptr() as *const f16,
ab_scale,
d_ptr.as_ptr() as *mut f16,
- m,
- n,
- k,
+ m_u,
+ n_u,
+ k_u,
lda,
ldb,
ldd,
@@ -125,7 +168,7 @@ impl MatmulKernel for MatmulCpuKernel {
is_accumulate,
);
if let Some(bias) = bias_ptr {
- apply_bias::<f16>(d_ptr.as_ptr() as *mut f16, bias.as_ptr() as *const f16, m, n);
+ apply_bias::<f16>(d_ptr.as_ptr() as *mut f16, bias.as_ptr() as *const f16, m_u, n_u);
}
},
DataType::BF16 => {
@@ -134,9 +177,9 @@ impl MatmulKernel for MatmulCpuKernel {
b_ptr.as_ptr() as *const bf16,
ab_scale,
d_ptr.as_ptr() as *mut bf16,
- m,
- n,
- k,
+ m_u,
+ n_u,
+ k_u,
lda,
ldb,
ldd,
@@ -144,11 +187,189 @@ impl MatmulKernel for MatmulCpuKernel {
is_accumulate,
);
if let Some(bias) = bias_ptr {
- apply_bias::<bf16>(d_ptr.as_ptr() as *mut bf16, bias.as_ptr() as *const bf16, m, n);
+ apply_bias::<bf16>(d_ptr.as_ptr() as *mut bf16, bias.as_ptr() as *const bf16, m_u, n_u);
}
},
_ => unreachable!(),
});
+
+ Ok(())
+ }
+
+ fn encode_quant<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
+ &mut self,
+ arguments: MatmulArguments<Cpu, TB>,
+ encoder: &mut Encoder<Cpu>,
+ ) -> Result<(), MatmulError<Cpu>> {
+ let d_mask = arguments.d_transform.mask();
+ if d_mask.contains(GemmDTransform::SCALE) {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::SCALE,
+ path: "MatmulCpuKernel/Quant",
+ });
+ }
+ if d_mask.contains(GemmDTransform::ACCUMULATE) {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::ACCUMULATE,
+ path: "MatmulCpuKernel/Quant",
+ });
+ }
+ if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {
+ return Err(MatmulError::UnsupportedLayout {
+ path: "MatmulCpuKernel/Quant",
+ });
+ }
+
+ let group_size = match arguments.b {
+ MatmulB::ScaleBiasDequant {
+ group_size,
+ ..
+ }
+ | MatmulB::ScaleZeroPointDequant {
+ group_size,
+ ..
+ } => group_size,
+ MatmulB::FullPrecision {
+ ..
+ } => unreachable!(),
+ };
+ if !matches!(group_size, 32 | 64 | 128) {
+ return Err(MatmulError::UnsupportedGroupSize(group_size as usize));
+ }
+
+ let post_rht = arguments.d_transform.rht_factors;
+ let post_bias = arguments.d_transform.bias;
+
+ if arguments.m >= 5 && arguments.n > 1 {
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ d,
+ d_transform,
+ m,
+ n,
+ k,
+ } = arguments;
+ let d_transform = d_transform.without(GemmDTransform::RHT | GemmDTransform::BIAS);
+ encode_quantized_gemm(
+ encoder,
+ MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ d: &mut *d,
+ d_transform,
+ m,
+ n,
+ k,
+ },
+ self.data_type,
+ );
+ if let Some(bias) = post_bias {
+ self.bias_add.encode(None::<&Allocation<Cpu>>, bias, &mut *d, n, m * n, encoder);
+ }
+ if let Some(factors) = post_rht {
+ self.hadamard.encode(d, factors, n, m, encoder);
+ }
+ return Ok(());
+ }
+
+ let hadamard_factors = post_rht;
+
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ d,
+ m,
+ n,
+ k,
+ ..
+ } = arguments;
+ let (weights, scales, zp_or_bias, method, mode, group_size) = match b {
+ MatmulB::ScaleBiasDequant {
+ b: w,
+ scales,
+ biases,
+ mode,
+ group_size,
+ } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),
+ MatmulB::ScaleZeroPointDequant {
+ b: w,
+ scales,
+ zero_points,
+ mode,
+ group_size,
+ } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ MatmulB::FullPrecision {
+ ..
+ } => unreachable!(),
+ };
+
+ let bits = match mode {
+ QuantizationMode::U4 => 4u32,
+ QuantizationMode::I8 | QuantizationMode::U8 => 8u32,
+ };
+ let use_fast = n % 8 == 0 && k % 512 == 0;
+ let (zero_points, biases) = match method {
+ QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias), None),
+ QuantizationMethod::ScaleBias => (None, Some(zp_or_bias)),
+ };
+
+ let context = encoder.context();
+ if use_fast {
+ let kernel =
+ <<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ context,
+ self.data_type,
+ group_size,
+ bits,
+ method,
+ hadamard_factors.is_some(),
+ )
+ .map_err(MatmulError::BackendError)?;
+ kernel.encode(
+ weights,
+ scales,
+ zero_points,
+ biases,
+ (a, a_offset),
+ &mut *d,
+ hadamard_factors,
+ k,
+ n,
+ m,
+ encoder,
+ );
+ } else {
+ if hadamard_factors.is_some() {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::RHT,
+ path: "MatmulCpuKernel/Quant",
+ });
+ }
+ let kernel =
+ <<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
+ context,
+ self.data_type,
+ group_size,
+ bits,
+ method,
+ )
+ .map_err(MatmulError::BackendError)?;
+ kernel.encode(weights, scales, zero_points, biases, (a, a_offset), &mut *d, k, n, m, encoder);
+ }
+ if let Some(bias) = post_bias {
+ self.bias_add.encode(None::<&Allocation<Cpu>>, bias, d, n, m * n, encoder);
+ }
+ Ok(())
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
index b6ed115d4..ca23d47de 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
@@ -1,5 +1,7 @@
pub mod gemm;
+pub mod gemv;
mod matmul;
+pub(crate) mod quant;
pub use matmul::MatmulCpuKernel;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs
new file mode 100644
index 000000000..3b08eb8c7
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs
@@ -0,0 +1,155 @@
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::{
+ ArrayElement, DataType,
+ backends::{
+ common::{
+ AsBufferRangeMut, AsBufferRangeRef, Buffer, Encoder,
+ gpu_types::{QuantizationMethod, QuantizationMode},
+ kernel::matmul::{MatmulArguments, MatmulB},
+ },
+ cpu::{Cpu, kernel::matmul::gemm::qmm_transposed::qmm_transposed},
+ },
+ utils::pointers::{SendPtr, SendPtrMut},
+};
+
+pub(crate) fn encode_quantized_gemm<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
+ encoder: &mut Encoder<Cpu>,
+ arguments: MatmulArguments<Cpu, TB>,
+ data_type: DataType,
+) {
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ d,
+ m,
+ n,
+ k,
+ ..
+ } = arguments;
+ let (weights, scales, zp_or_bias, method, mode, group_size) = match b {
+ MatmulB::ScaleBiasDequant {
+ b: w,
+ scales,
+ biases,
+ mode,
+ group_size,
+ } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),
+ MatmulB::ScaleZeroPointDequant {
+ b: w,
+ scales,
+ zero_points,
+ mode,
+ group_size,
+ } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ MatmulB::FullPrecision {
+ ..
+ } => unreachable!(),
+ };
+
+ let bits: usize = match mode {
+ QuantizationMode::U4 => 4,
+ QuantizationMode::I8 | QuantizationMode::U8 => 8,
+ };
+ let in_vec_size = k as usize;
+ let out_vec_size = n as usize;
+ let batch_size = m as usize;
+ let group_size = group_size as usize;
+
+ let a_buf = a.as_buffer_range_ref();
+ let b_buf = weights.as_buffer_range_ref();
+ let scales_buf = scales.as_buffer_range_ref();
+ let zp_buf = zp_or_bias.as_buffer_range_ref();
+ let d_buf = d.as_buffer_range_mut();
+
+ let a_byte_off = a_buf.range().start + a_offset * data_type.size_in_bytes();
+ let b_byte_off = b_buf.range().start;
+ let scales_byte_off = scales_buf.range().start;
+ let zp_byte_off = zp_buf.range().start;
+ let d_byte_off = d_buf.range().start;
+
+ let a_ptr = SendPtr(unsafe { &*a_buf.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));
+ let b_ptr = SendPtr(unsafe { &*b_buf.buffer().get() }.as_ptr().wrapping_byte_add(b_byte_off));
+ let scales_ptr = SendPtr(unsafe { &*scales_buf.buffer().get() }.as_ptr().wrapping_byte_add(scales_byte_off));
+ let zp_ptr = SendPtr(unsafe { &*zp_buf.buffer().get() }.as_ptr().wrapping_byte_add(zp_byte_off));
+ let d_ptr = SendPtrMut(unsafe { (&*d_buf.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });
+
+ let command_buffer = encoder.as_command_buffer_mut();
+ command_buffer.push_command(move || match data_type {
+ DataType::F32 => run::<f32>(
+ a_ptr,
+ b_ptr,
+ scales_ptr,
+ zp_ptr,
+ d_ptr,
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ method,
+ group_size,
+ bits,
+ ),
+ DataType::F16 => run::<f16>(
+ a_ptr,
+ b_ptr,
+ scales_ptr,
+ zp_ptr,
+ d_ptr,
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ method,
+ group_size,
+ bits,
+ ),
+ DataType::BF16 => run::<bf16>(
+ a_ptr,
+ b_ptr,
+ scales_ptr,
+ zp_ptr,
+ d_ptr,
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ method,
+ group_size,
+ bits,
+ ),
+ _ => unreachable!(),
+ });
+}
+
+fn run<T: ArrayElement + Float>(
+ a_ptr: SendPtr<u8>,
+ b_ptr: SendPtr<u8>,
+ scales_ptr: SendPtr<u8>,
+ zp_or_bias_ptr: SendPtr<u8>,
+ d_ptr: SendPtrMut<u8>,
+ in_vec_size: usize,
+ out_vec_size: usize,
+ batch_size: usize,
+ quant_method: QuantizationMethod,
+ group_size: usize,
+ bits: usize,
+) {
+ let (zp, bias) = match quant_method {
+ QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias_ptr.as_ptr() as *const u8), None),
+ QuantizationMethod::ScaleBias => (None, Some(zp_or_bias_ptr.as_ptr() as *const T)),
+ };
+ qmm_transposed::<T>(
+ b_ptr.as_ptr() as *const u32,
+ scales_ptr.as_ptr() as *const T,
+ zp,
+ bias,
+ a_ptr.as_ptr() as *const T,
+ d_ptr.as_ptr() as *mut T,
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ quant_method,
+ group_size,
+ bits,
+ );
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
index 421ec7c18..0658c20c0 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
@@ -12,7 +12,6 @@ mod matmul;
mod mlp;
mod moe;
mod pooling;
-mod quant_matmul;
mod rms_norm;
mod rope;
mod sampling;
diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rs
index 250432ce7..2d3c1e4ed 100644
--- a/crates/backend-uzu/src/backends/metal/error.rs
+++ b/crates/backend-uzu/src/backends/metal/error.rs
@@ -1,6 +1,11 @@
+use std::error::Error as StdError;
+
use thiserror::Error;
-use crate::backends::metal::kernel::matmul::gemm::GemmSpecializationError;
+use crate::backends::{
+ common::kernel::matmul::MatmulError,
+ metal::{Metal, kernel::matmul::gemm::GemmSpecializationError},
+};
#[derive(Debug, Error)]
pub enum MetalError {
@@ -26,10 +31,25 @@ pub enum MetalError {
CannotCreateFunction,
#[error("Cannot create pipeline state: {0}")]
CannotCreatePipelineState(String),
- #[error("Invalid GEMM specialization: {0:?}")]
- InvalidGemmSpecialization(GemmSpecializationError),
#[error("Can not allocate buffer with size={0}")]
SparseBufferAlloc(usize),
#[error("Can not allocate heap with size={0} and page size={1}")]
SparseHeapAlloc(usize, usize),
+ #[error("Kernel dispatch failed: {0}")]
+ KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),
+}
+
+impl From<MatmulError<Metal>> for MetalError {
+ fn from(value: MatmulError<Metal>) -> Self {
+ match value {
+ MatmulError::BackendError(e) => e,
+ other => MetalError::KernelDispatchFailed(Box::new(other)),
+ }
+ }
+}
+
+impl From<GemmSpecializationError> for MetalError {
+ fn from(value: GemmSpecializationError) -> Self {
+ MetalError::KernelDispatchFailed(Box::new(value))
+ }
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
index b21071fa8..841f4cd72 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
@@ -5,35 +5,41 @@
using namespace metal;
namespace uzu::gemm {
-enum class GemmInputPrologueKind : uint32_t {
- FullPrecision = 0,
- ExternalRht = 1,
-};
-
-enum class GemmOutputTransformKind : uint32_t {
- Store = 0,
- Scale = 1,
- Accumulate = 2,
- Bias = 3,
- Rht = 4,
- ScaleAccumulate = 5,
- ScaleAccumulateBias = 6,
- ScaleAccumulateBiasRht = 7,
-};
-
enum class GemmWeightPrologueKind : uint32_t {
FullPrecision = 0,
ScaleBiasDequant = 1,
ScaleZeroPointDequant = 2,
};
-typedef struct {
- uint32_t threadgroup_m;
- uint32_t threadgroup_n;
- uint32_t threadgroup_k;
- uint32_t simdgroups_m;
- uint32_t simdgroups_n;
-} GemmTilingConfig;
+struct GemmDTransform {
+ uint raw_value;
+ constexpr GemmDTransform() thread : raw_value(0) {}
+ constexpr GemmDTransform(uint __dsl_v) thread : raw_value(__dsl_v) {}
+ static constant constexpr uint SCALE = 1 << 0;
+ static constant constexpr uint ACCUMULATE = 1 << 1;
+ static constant constexpr uint BIAS = 1 << 2;
+ static constant constexpr uint RHT = 1 << 3;
+ constexpr bool contains(uint flag) const thread {
+ return (raw_value & flag) != 0;
+ }
+ constexpr bool contains(uint flag) const constant {
+ return (raw_value & flag) != 0;
+ }
+ constexpr uint bits() const thread { return raw_value; }
+ constexpr uint bits() const constant { return raw_value; }
+};
+
+enum class GemmTiling : uint32_t {
+ T8x32x32_1x1 = 0,
+ T64x32x32_2x2 = 1,
+ T64x64x16_2x2 = 2,
+ T64x64x32_2x2 = 3,
+ T64x64x64_2x2 = 4,
+ T32x32x32_2x2 = 5,
+ T32x64x32_2x2 = 6,
+ T64x32x32_4x1 = 7,
+ T128x128x32_4x4 = 8,
+};
struct GemmAlignment {
uint raw_value;
@@ -45,5 +51,10 @@ struct GemmAlignment {
constexpr bool contains(uint flag) const thread {
return (raw_value & flag) != 0;
}
+ constexpr bool contains(uint flag) const constant {
+ return (raw_value & flag) != 0;
+ }
+ constexpr uint bits() const thread { return raw_value; }
+ constexpr uint bits() const constant { return raw_value; }
};
} // namespace uzu::gemm
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
index c56a6f0bb..a1a2f1a9b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
@@ -14,25 +14,26 @@ namespace matmul {
template <
typename T,
- ushort BLOCK_ROWS,
- ushort BLOCK_COLS,
+ ushort THREADGROUP_TILE_ROWS,
+ ushort THREADGROUP_TILE_COLS,
ushort DESTINATION_LEADING_DIMENSION,
ushort REDUCTION_DIMENSION,
ushort THREADGROUP_SIZE,
ushort ALIGNMENT = 1,
- ushort READS_PER_THREAD = (BLOCK_COLS * BLOCK_ROWS) / (THREADGROUP_SIZE),
- ushort THREAD_COLS = BLOCK_COLS / READS_PER_THREAD,
+ ushort READS_PER_THREAD =
+ (THREADGROUP_TILE_COLS * THREADGROUP_TILE_ROWS) / (THREADGROUP_SIZE),
+ ushort THREAD_COLS = THREADGROUP_TILE_COLS / READS_PER_THREAD,
ushort THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>
struct ThreadgroupLoader {
METAL_CONST ushort ROW_ITERATIONS =
- (BLOCK_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;
+ (THREADGROUP_TILE_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;
const int source_leading_dimension;
const int tile_stride;
const ushort thread_index;
- const ushort block_row_index;
- const ushort block_col_index;
+ const ushort tile_row_index;
+ const ushort tile_col_index;
threadgroup T* destination;
const device T* source;
@@ -49,26 +50,27 @@ struct ThreadgroupLoader {
)
: source_leading_dimension(source_leading_dim),
tile_stride(
- REDUCTION_DIMENSION ? BLOCK_COLS : BLOCK_ROWS * source_leading_dim
+ REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS
+ : THREADGROUP_TILE_ROWS * source_leading_dim
),
thread_index(
thread_context.simdgroup_index * METAL_SIMD_SIZE +
thread_context.simd_lane_id
),
- block_row_index(thread_index / THREAD_COLS),
- block_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),
+ tile_row_index(thread_index / THREAD_COLS),
+ tile_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),
destination(
- destination_ptr + block_row_index * DESTINATION_LEADING_DIMENSION +
- block_col_index
+ destination_ptr + tile_row_index * DESTINATION_LEADING_DIMENSION +
+ tile_col_index
),
source(
- source_ptr + block_row_index * source_leading_dim + block_col_index
+ source_ptr + tile_row_index * source_leading_dim + tile_col_index
) {}
template <typename UnaryOp>
METAL_FUNC void apply_inplace_op(thread const UnaryOp& operation) const {
METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < READS_PER_THREAD; j++) {
destination[i * DESTINATION_LEADING_DIMENSION + j] =
@@ -79,20 +81,23 @@ struct ThreadgroupLoader {
METAL_FUNC void load_unsafe() const {
METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
- *((threadgroup ReadVector*)(&destination
- [i * DESTINATION_LEADING_DIMENSION])) =
- *((const device ReadVector*)(&source[i * source_leading_dimension]));
+ for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {
+ *reinterpret_cast<threadgroup ReadVector*>(
+ &destination[i * DESTINATION_LEADING_DIMENSION]
+ ) =
+ *reinterpret_cast<const device ReadVector*>(
+ &source[i * source_leading_dimension]
+ );
}
}
METAL_FUNC void load_safe(short2 source_tile_dimensions) const {
source_tile_dimensions =
- source_tile_dimensions - short2(block_col_index, block_row_index);
+ source_tile_dimensions - short2(tile_col_index, tile_row_index);
if (source_tile_dimensions.x <= 0 || source_tile_dimensions.y <= 0) {
METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < READS_PER_THREAD; j++) {
destination[i * DESTINATION_LEADING_DIMENSION + j] = T(0);
@@ -105,7 +110,7 @@ struct ThreadgroupLoader {
T loaded_values[READS_PER_THREAD];
METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < READS_PER_THREAD; j++) {
valid_mask[j] =
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index bbb06f838..c07e65a5b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -256,6 +256,56 @@ struct ThreadgroupTile {
}
}
}
+
+ METAL_FUNC void apply_bias(const device U* bias) {
+ const device U* bias_pointer = bias + simdgroup_col_offset;
+
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < TILE_ROWS; i++) {
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0; j < TILE_COLS; j++) {
+ thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
+ const ushort col_offset =
+ j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
+ METAL_PRAGMA_UNROLL
+ for (ushort k = 0;
+ k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
+ k++) {
+ block_data[k] +=
+ static_cast<AccumulatorType>(bias_pointer[col_offset + k]);
+ }
+ }
+ }
+ }
+
+ METAL_FUNC void apply_bias_safe(
+ const device U* bias,
+ short2 tile_dimensions
+ ) {
+ const device U* bias_pointer = bias + simdgroup_col_offset;
+ tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
+
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < TILE_ROWS; i++) {
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0; j < TILE_COLS; j++) {
+ thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
+ const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
+ const ushort col_offset =
+ j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
+ METAL_PRAGMA_UNROLL
+ for (ushort k = 0;
+ k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
+ k++) {
+ if (row_offset < tile_dimensions.y &&
+ col_offset + k < tile_dimensions.x) {
+ block_data[k] +=
+ static_cast<AccumulatorType>(bias_pointer[col_offset + k]);
+ }
+ }
+ }
+ }
+ }
};
} // namespace matmul
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_alignment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_alignment.h
new file mode 100644
index 000000000..968ec5041
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_alignment.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#include "../../../common/integral_constant.h"
+#include "../../common/defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <typename F>
+METAL_FUNC void dispatch_gemm_alignment(uint mask, F f) {
+ switch (mask & 0b111u) {
+ case 0u:
+ f(integral_constant<uint, 0u>{});
+ break;
+ case 1u:
+ f(integral_constant<uint, 1u>{});
+ break;
+ case 2u:
+ f(integral_constant<uint, 2u>{});
+ break;
+ case 3u:
+ f(integral_constant<uint, 3u>{});
+ break;
+ case 4u:
+ f(integral_constant<uint, 4u>{});
+ break;
+ case 5u:
+ f(integral_constant<uint, 5u>{});
+ break;
+ case 6u:
+ f(integral_constant<uint, 6u>{});
+ break;
+ case 7u:
+ f(integral_constant<uint, 7u>{});
+ break;
+ }
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
new file mode 100644
index 000000000..923f41c96
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
@@ -0,0 +1,74 @@
+#pragma once
+
+#include "../../../generated/gemm.h"
+
+namespace uzu {
+namespace gemm {
+
+constexpr uint gemm_tiling_block_m(GemmTiling t) {
+ return t == GemmTiling::T8x32x32_1x1 ? 8
+ : t == GemmTiling::T64x32x32_2x2 ? 64
+ : t == GemmTiling::T64x64x16_2x2 ? 64
+ : t == GemmTiling::T64x64x32_2x2 ? 64
+ : t == GemmTiling::T64x64x64_2x2 ? 64
+ : t == GemmTiling::T32x32x32_2x2 ? 32
+ : t == GemmTiling::T32x64x32_2x2 ? 32
+ : t == GemmTiling::T64x32x32_4x1 ? 64
+ : t == GemmTiling::T128x128x32_4x4 ? 128
+ : 0;
+}
+
+constexpr uint gemm_tiling_block_n(GemmTiling t) {
+ return t == GemmTiling::T8x32x32_1x1 ? 32
+ : t == GemmTiling::T64x32x32_2x2 ? 32
+ : t == GemmTiling::T64x64x16_2x2 ? 64
+ : t == GemmTiling::T64x64x32_2x2 ? 64
+ : t == GemmTiling::T64x64x64_2x2 ? 64
+ : t == GemmTiling::T32x32x32_2x2 ? 32
+ : t == GemmTiling::T32x64x32_2x2 ? 64
+ : t == GemmTiling::T64x32x32_4x1 ? 32
+ : t == GemmTiling::T128x128x32_4x4 ? 128
+ : 0;
+}
+
+constexpr uint gemm_tiling_block_k(GemmTiling t) {
+ return t == GemmTiling::T8x32x32_1x1 ? 32
+ : t == GemmTiling::T64x32x32_2x2 ? 32
+ : t == GemmTiling::T64x64x16_2x2 ? 16
+ : t == GemmTiling::T64x64x32_2x2 ? 32
+ : t == GemmTiling::T64x64x64_2x2 ? 64
+ : t == GemmTiling::T32x32x32_2x2 ? 32
+ : t == GemmTiling::T32x64x32_2x2 ? 32
+ : t == GemmTiling::T64x32x32_4x1 ? 32
+ : t == GemmTiling::T128x128x32_4x4 ? 32
+ : 0;
+}
+
+constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {
+ return t == GemmTiling::T8x32x32_1x1 ? 1
+ : t == GemmTiling::T64x32x32_2x2 ? 2
+ : t == GemmTiling::T64x64x16_2x2 ? 2
+ : t == GemmTiling::T64x64x32_2x2 ? 2
+ : t == GemmTiling::T64x64x64_2x2 ? 2
+ : t == GemmTiling::T32x32x32_2x2 ? 2
+ : t == GemmTiling::T32x64x32_2x2 ? 2
+ : t == GemmTiling::T64x32x32_4x1 ? 4
+ : t == GemmTiling::T128x128x32_4x4 ? 4
+ : 0;
+}
+
+constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {
+ return t == GemmTiling::T8x32x32_1x1 ? 1
+ : t == GemmTiling::T64x32x32_2x2 ? 2
+ : t == GemmTiling::T64x64x16_2x2 ? 2
+ : t == GemmTiling::T64x64x32_2x2 ? 2
+ : t == GemmTiling::T64x64x64_2x2 ? 2
+ : t == GemmTiling::T32x32x32_2x2 ? 2
+ : t == GemmTiling::T32x64x32_2x2 ? 2
+ : t == GemmTiling::T64x32x32_4x1 ? 1
+ : t == GemmTiling::T128x128x32_4x4 ? 4
+ : 0;
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 3e62895df..b3dcb051c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -8,22 +8,21 @@
#include "../../../generated/matmul.h"
#include "../generated/gemm.h"
#include "block_geometry.h"
+#include "gemm_tiling.h"
using namespace metal;
namespace uzu {
namespace gemm {
-// MPP `matmul2d_descriptor.transpose_right` drives `TRANSPOSE_B` (MSL
-// Spec 6.18).
-template <
- typename T,
- ushort THREADGROUP_BLOCK_M,
- ushort THREADGROUP_BLOCK_N,
- ushort SIMDGROUPS_PER_ROW,
- ushort SIMDGROUPS_PER_COLUMN,
- bool TRANSPOSE_B>
+template <typename T, GemmTiling GEMM_TILING, bool TRANSPOSE_B>
struct MxuMmaCore {
+ METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
+ METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
+ METAL_CONST ushort SIMDGROUPS_PER_ROW =
+ gemm_tiling_simdgroups_per_row(GEMM_TILING);
+ METAL_CONST ushort SIMDGROUPS_PER_COLUMN =
+ gemm_tiling_simdgroups_per_column(GEMM_TILING);
METAL_CONST ushort SIMDGROUP_BLOCK_M =
THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
METAL_CONST ushort SIMDGROUP_BLOCK_N =
@@ -43,9 +42,11 @@ struct MxuMmaCore {
device T* d,
const constant uzu::matmul::GemmParams* params,
GemmAlignment alignment,
- GemmOutputTransformKind output_transform,
+ GemmDTransform output_transform,
+ const device T* output_bias,
const thread ThreadContext& thread_context
) {
+ (void)output_bias;
const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);
const auto geometry =
ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::
@@ -98,12 +99,9 @@ struct MxuMmaCore {
const int aligned_k_iterations = int(params->K) / int(THREADGROUP_BLOCK_K);
- const bool apply_scale =
- output_transform == GemmOutputTransformKind::Scale ||
- output_transform == GemmOutputTransformKind::ScaleAccumulate;
+ const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);
const bool apply_accumulate =
- output_transform == GemmOutputTransformKind::Accumulate ||
- output_transform == GemmOutputTransformKind::ScaleAccumulate;
+ output_transform.contains(GemmDTransform::ACCUMULATE);
dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {
dispatch_bool(
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_pack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_pack.h
new file mode 100644
index 000000000..a9209512b
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_pack.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <metal_stdlib>
+
+namespace uzu {
+namespace gemm {
+
+template <int bits, int word_size_bits = 8>
+inline constexpr short get_pack_factor() {
+ return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : word_size_bits / bits);
+}
+
+template <int bits, int word_size_bits = 8>
+inline constexpr short get_bytes_per_pack() {
+ constexpr int is_power_of_two_bits = (bits & (bits - 1)) == 0;
+ return is_power_of_two_bits ? (word_size_bits / 8) : (bits == 5 ? 5 : 3);
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
new file mode 100644
index 000000000..6e34d9caf
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
@@ -0,0 +1,170 @@
+#pragma once
+
+#include <metal_simdgroup>
+#include <metal_stdlib>
+
+#include "../../common/defines.h"
+#include "quant_pack.h"
+#include "quant_unpack.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <
+ typename T,
+ short THREADGROUP_TILE_ROWS,
+ short THREADGROUP_TILE_COLS,
+ short DESTINATION_LEADING_DIMENSION,
+ short REDUCTION_DIMENSION,
+ short THREADGROUP_SIZE,
+ short GROUP_SIZE,
+ short BITS>
+struct QuantizedBlockLoaderScaleBias {
+ static_assert(
+ THREADGROUP_TILE_COLS <= GROUP_SIZE,
+ "Group size should be larger than columns"
+ );
+ static_assert(
+ GROUP_SIZE % THREADGROUP_TILE_COLS == 0,
+ "Group size should be divisible by columns"
+ );
+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
+
+ METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();
+ METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();
+ METAL_CONST short THREADGROUP_TILE_COLS_PACKED =
+ THREADGROUP_TILE_COLS / pack_factor;
+ METAL_CONST short READS_PER_THREAD =
+ (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
+ ? 1
+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) /
+ THREADGROUP_SIZE;
+ METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
+ METAL_CONST bool TILE_HAS_IDLE_THREADS =
+ THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
+
+ const int src_leading_dim;
+ const int tile_stride;
+ short group_step_counter;
+ const int group_stride;
+
+ const short thread_index;
+ const short tile_row_index;
+ const short tile_col_index;
+
+ threadgroup T* dst;
+ const device uint8_t* src;
+ const device T* scales;
+ const device T* biases;
+
+ QuantizedBlockLoaderScaleBias(
+ const device uint8_t* src_,
+ const device T* scales_,
+ const device T* biases_,
+ const int src_leading_dim_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ )
+ : src_leading_dim(src_leading_dim_),
+ tile_stride(
+ REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS_PACKED * bytes_per_pack
+ : THREADGROUP_TILE_ROWS * src_leading_dim_ *
+ bytes_per_pack / pack_factor
+ ),
+ group_step_counter(0),
+ group_stride(THREADGROUP_TILE_ROWS * src_leading_dim_ / GROUP_SIZE),
+ thread_index(simd_group_id * 32 + simd_lane_id),
+ tile_row_index(
+ READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED
+ ),
+ tile_col_index(
+ (READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED
+ ),
+ dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION +
+ tile_col_index * pack_factor),
+ src(src_ +
+ tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor +
+ tile_col_index * bytes_per_pack),
+ scales(scales_ + tile_row_index * src_leading_dim_ / GROUP_SIZE),
+ biases(biases_ + tile_row_index * src_leading_dim_ / GROUP_SIZE) {}
+
+ void load_unsafe() const {
+ if constexpr (TILE_HAS_IDLE_THREADS) {
+ if (tile_row_index >= THREADGROUP_TILE_ROWS) {
+ return;
+ }
+ }
+
+ T scale = *scales;
+ T bias = *biases;
+ for (int i = 0; i < READS_PER_THREAD; i++) {
+ dequantize<T, pack_factor, BITS>(
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
+ }
+ }
+
+ void load_safe(short2 src_tile_dim) const {
+ if constexpr (TILE_HAS_IDLE_THREADS) {
+ if (tile_row_index >= THREADGROUP_TILE_ROWS) {
+ return;
+ }
+ }
+
+ if constexpr (REDUCTION_DIMENSION == 1) {
+ if (tile_row_index >= src_tile_dim.x) {
+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+ } else {
+ if (tile_row_index >= src_tile_dim.y) {
+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+ }
+
+ T scale = *scales;
+ T bias = *biases;
+ for (int i = 0; i < READS_PER_THREAD; i++) {
+ dequantize<T, pack_factor, BITS>(
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
+ }
+ }
+
+ void next() {
+ src += tile_stride;
+ if constexpr (REDUCTION_DIMENSION == 1) {
+ if constexpr (GROUP_STEPS_PER_BLOCK > 1) {
+ group_step_counter++;
+ if (group_step_counter == GROUP_STEPS_PER_BLOCK) {
+ group_step_counter = 0;
+ scales++;
+ biases++;
+ }
+ } else {
+ scales++;
+ biases++;
+ }
+ } else {
+ scales += group_stride;
+ biases += group_stride;
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
new file mode 100644
index 000000000..48cc1f462
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -0,0 +1,247 @@
+#pragma once
+
+#include <metal_simdgroup>
+#include <metal_stdlib>
+
+#include "../../common/defines.h"
+#include "quant_pack.h"
+#include "quant_unpack.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <
+ typename T,
+ short THREADGROUP_TILE_ROWS,
+ short THREADGROUP_TILE_COLS,
+ short DESTINATION_LEADING_DIMENSION,
+ short REDUCTION_DIMENSION,
+ short THREADGROUP_SIZE,
+ short GROUP_SIZE,
+ short BITS>
+struct QuantizedBlockLoaderScaleZeroPoint {
+ static_assert(
+ THREADGROUP_TILE_COLS <= GROUP_SIZE,
+ "Group size should be larger than columns"
+ );
+ static_assert(
+ GROUP_SIZE % THREADGROUP_TILE_COLS == 0,
+ "Group size should be divisible by columns"
+ );
+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
+
+ METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();
+ METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();
+ METAL_CONST short THREADGROUP_TILE_COLS_PACKED =
+ THREADGROUP_TILE_COLS / pack_factor;
+ METAL_CONST short READS_PER_THREAD =
+ (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
+ ? 1
+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) /
+ THREADGROUP_SIZE;
+ METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
+ METAL_CONST bool TILE_HAS_IDLE_THREADS =
+ THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
+
+ const int src_leading_dim;
+ const int groups_per_row;
+ const int tile_stride;
+ short group_step_counter;
+ int k_base;
+ const int group_stride;
+
+ const short thread_index;
+ const short tile_row_index;
+ const short tile_col_index;
+
+ threadgroup T* dst;
+ const device uint8_t* src;
+ const device T* scales;
+ const device T* scales_row_start;
+ const device uint8_t* zero_points_row_start;
+
+ QuantizedBlockLoaderScaleZeroPoint(
+ const device uint8_t* src_,
+ const device T* scales_,
+ const device uint8_t* zero_points_row_start_,
+ const int src_leading_dim_,
+ const int groups_per_row_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ )
+ : src_leading_dim(src_leading_dim_), groups_per_row(groups_per_row_),
+ tile_stride(
+ REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS_PACKED * bytes_per_pack
+ : THREADGROUP_TILE_ROWS * src_leading_dim_ *
+ bytes_per_pack / pack_factor
+ ),
+ group_step_counter(0), k_base(0),
+ group_stride(THREADGROUP_TILE_ROWS * groups_per_row_),
+ thread_index(simd_group_id * 32 + simd_lane_id),
+ tile_row_index(
+ READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED
+ ),
+ tile_col_index(
+ (READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED
+ ),
+ dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION +
+ tile_col_index * pack_factor),
+ src(src_ +
+ tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor +
+ tile_col_index * bytes_per_pack),
+ scales(
+ REDUCTION_DIMENSION == 1
+ ? (scales_ + tile_row_index * groups_per_row_)
+ : scales_
+ ),
+ scales_row_start(
+ REDUCTION_DIMENSION == 1
+ ? (scales_ + tile_row_index * groups_per_row_)
+ : scales_
+ ),
+ zero_points_row_start(
+ REDUCTION_DIMENSION == 1
+ ? (zero_points_row_start_ +
+ tile_row_index * (BITS == 4 ? ((groups_per_row_ + 1) / 2)
+ : groups_per_row_))
+ : zero_points_row_start_
+ ) {}
+
+ inline void current_scale_bias(
+ thread T& out_scale,
+ thread T& out_bias
+ ) const {
+ uint zero_point_value;
+ T scale_value;
+ int group_index;
+ if constexpr (REDUCTION_DIMENSION == 0) {
+ group_index = k_base / GROUP_SIZE;
+ scale_value = scales_row_start[group_index];
+ } else {
+ group_index = static_cast<int>(scales - scales_row_start);
+ scale_value = *scales;
+ }
+ if constexpr (BITS == 4) {
+ const device uint8_t* zero_point_ptr =
+ zero_points_row_start + (group_index >> 1);
+ uint8_t zero_point_byte = *zero_point_ptr;
+ zero_point_value =
+ (uint(zero_point_byte) >> (uint(group_index & 1) * 4u)) & 0x0Fu;
+ } else {
+ zero_point_value = zero_points_row_start[group_index];
+ }
+ out_scale = scale_value;
+ out_bias = static_cast<T>(-scale_value * static_cast<T>(zero_point_value));
+ }
+
+ void load_unsafe() const {
+ if constexpr (TILE_HAS_IDLE_THREADS) {
+ if (tile_row_index >= THREADGROUP_TILE_ROWS) {
+ return;
+ }
+ }
+
+ T scale;
+ T bias;
+ current_scale_bias(scale, bias);
+ for (int i = 0; i < READS_PER_THREAD; i++) {
+ dequantize<T, pack_factor, BITS>(
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
+ }
+ }
+
+ void load_safe(short2 src_tile_dim) const {
+ if constexpr (TILE_HAS_IDLE_THREADS) {
+ if (tile_row_index >= THREADGROUP_TILE_ROWS) {
+ return;
+ }
+ }
+
+ if constexpr (REDUCTION_DIMENSION == 1) {
+ if (tile_row_index >= src_tile_dim.x) {
+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+
+ int valid_cols = src_tile_dim.y;
+ int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;
+
+ T scale;
+ T bias;
+ current_scale_bias(scale, bias);
+ for (int i = 0; i < READS_PER_THREAD; i++) {
+ int pack_index = tile_col_index + i;
+ if (pack_index < valid_packs) {
+ dequantize<T, pack_factor, BITS>(
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
+
+ if (pack_index == valid_packs - 1) {
+ int remaining = valid_cols - pack_index * pack_factor;
+ if (remaining < pack_factor) {
+ for (int r = remaining; r < pack_factor; ++r) {
+ dst[i * pack_factor + r] = T(0);
+ }
+ }
+ }
+ } else {
+ for (int j = 0; j < pack_factor; ++j) {
+ dst[i * pack_factor + j] = T(0);
+ }
+ }
+ }
+ return;
+ } else {
+ if (tile_row_index >= src_tile_dim.y) {
+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+
+ T scale;
+ T bias;
+ current_scale_bias(scale, bias);
+ for (int i = 0; i < READS_PER_THREAD; i++) {
+ dequantize<T, pack_factor, BITS>(
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
+ }
+ }
+ }
+
+ void next() {
+ src += tile_stride;
+ if constexpr (REDUCTION_DIMENSION == 1) {
+ if constexpr (GROUP_STEPS_PER_BLOCK > 1) {
+ group_step_counter++;
+ if (group_step_counter == GROUP_STEPS_PER_BLOCK) {
+ group_step_counter = 0;
+ scales++;
+ }
+ } else {
+ scales++;
+ }
+ } else {
+ k_base += THREADGROUP_TILE_ROWS;
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h
new file mode 100644
index 000000000..bdf8308be
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h
@@ -0,0 +1,118 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../common/defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <typename U>
+METAL_FUNC U uint_to_fp(uint32_t x) {
+ return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);
+}
+
+template <>
+METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {
+ return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);
+}
+
+template <int BITS>
+METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {
+ static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");
+ constexpr uint mask = (1u << BITS) - 1u;
+ n &= uint4(mask);
+ return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);
+}
+
+template <typename U, int BITS>
+METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);
+
+template <>
+METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {
+ return _uint4_to_fp4_float<4>(n);
+}
+
+template <>
+METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {
+ return _uint4_to_fp4_float<8>(n);
+}
+
+template <>
+METAL_FUNC half4 uint4_to_fp4<half, 4>(uint4 n) {
+ return half4(_uint4_to_fp4_float<4>(n));
+}
+
+template <>
+METAL_FUNC half4 uint4_to_fp4<half, 8>(uint4 n) {
+ return half4(_uint4_to_fp4_float<8>(n));
+}
+
+template <>
+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {
+ return bfloat4(_uint4_to_fp4_float<4>(n));
+}
+
+template <>
+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {
+ return bfloat4(_uint4_to_fp4_float<8>(n));
+}
+
+template <typename U, int N, int bits>
+inline void dequantize(
+ const device uint8_t* w,
+ U scale,
+ U bias,
+ threadgroup U* w_local
+) {
+ static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
+
+ if (bits == 4) {
+ U s0 = scale;
+ U s1 = scale / static_cast<U>(16.0f);
+ for (int i = 0; i < (N / 2); i++) {
+ w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;
+ w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;
+ }
+ } else if (bits == 8) {
+ for (int i = 0; i < N; i++) {
+ w_local[i] = scale * w[i] + bias;
+ }
+ }
+}
+
+template <>
+inline void dequantize<bfloat, 8, 4>(
+ const device uint8_t* w,
+ bfloat scale,
+ bfloat bias,
+ threadgroup bfloat* w_local
+) {
+ const device uint32_t* w_ptr = reinterpret_cast<const device uint32_t*>(w);
+ uint32_t packed = *w_ptr;
+
+ bfloat4 v0, v1;
+
+ v0.x = static_cast<bfloat>(packed & 0xF);
+ v0.y = static_cast<bfloat>((packed >> 4) & 0xF);
+ v0.z = static_cast<bfloat>((packed >> 8) & 0xF);
+ v0.w = static_cast<bfloat>((packed >> 12) & 0xF);
+
+ v1.x = static_cast<bfloat>((packed >> 16) & 0xF);
+ v1.y = static_cast<bfloat>((packed >> 20) & 0xF);
+ v1.z = static_cast<bfloat>((packed >> 24) & 0xF);
+ v1.w = static_cast<bfloat>((packed >> 28) & 0xF);
+
+ v0 = v0 * scale + bias;
+ v1 = v1 * scale + bias;
+
+ threadgroup bfloat4* out_ptr =
+ reinterpret_cast<threadgroup bfloat4*>(w_local);
+ out_ptr[0] = v0;
+ out_ptr[1] = v1;
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 10fd83532..c860bc745 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -8,6 +8,10 @@
#include "../../../generated/matmul.h"
#include "../generated/gemm.h"
#include "block_geometry.h"
+#include "gemm_alignment.h"
+#include "gemm_tiling.h"
+#include "quant_scale_bias.h"
+#include "quant_scale_zero_point.h"
using namespace metal;
@@ -16,13 +20,20 @@ namespace gemm {
template <
typename T,
- int THREADGROUP_BLOCK_M,
- int THREADGROUP_BLOCK_N,
- int THREADGROUP_BLOCK_K,
- int SIMDGROUPS_PER_ROW,
- int SIMDGROUPS_PER_COLUMN,
- bool TRANSPOSE_B>
+ GemmTiling GEMM_TILING,
+ bool TRANSPOSE_B,
+ GemmWeightPrologueKind WEIGHT_PROLOGUE =
+ GemmWeightPrologueKind::FullPrecision,
+ int BITS = 0,
+ int GROUP_SIZE = 0>
struct SimdgroupMmaCore {
+ METAL_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
+ METAL_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
+ METAL_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);
+ METAL_CONST int SIMDGROUPS_PER_ROW =
+ gemm_tiling_simdgroups_per_row(GEMM_TILING);
+ METAL_CONST int SIMDGROUPS_PER_COLUMN =
+ gemm_tiling_simdgroups_per_column(GEMM_TILING);
METAL_CONST ushort PADDING_A = 16 / sizeof(T);
METAL_CONST ushort PADDING_B = 16 / sizeof(T);
METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;
@@ -38,13 +49,31 @@ struct SimdgroupMmaCore {
SHARED_STRIDE_A,
true,
THREADGROUP_THREADS>;
- using BLoader = uzu::matmul::ThreadgroupLoader<
+ using BLoaderFp = uzu::matmul::ThreadgroupLoader<
T,
TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,
TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N,
SHARED_STRIDE_B,
TRANSPOSE_B,
THREADGROUP_THREADS>;
+ using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<
+ T,
+ THREADGROUP_BLOCK_N,
+ THREADGROUP_BLOCK_K,
+ SHARED_STRIDE_B,
+ 1,
+ THREADGROUP_THREADS,
+ GROUP_SIZE,
+ BITS>;
+ using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<
+ T,
+ THREADGROUP_BLOCK_N,
+ THREADGROUP_BLOCK_K,
+ SHARED_STRIDE_B,
+ 1,
+ THREADGROUP_THREADS,
+ GROUP_SIZE,
+ BITS>;
using TileAccumulator = uzu::matmul::ThreadgroupTile<
T,
T,
@@ -60,7 +89,7 @@ struct SimdgroupMmaCore {
float,
uzu::matmul::TransformNone<T, float>>;
- template <bool M_aligned, bool N_aligned, bool K_aligned>
+ template <uint GEMM_ALIGNMENT_RAW, typename BLoader>
static METAL_FUNC void k_loop(
threadgroup T* a_shared,
threadgroup T* b_shared,
@@ -72,6 +101,7 @@ struct SimdgroupMmaCore {
thread const ushort& tile_block_cols,
thread const ushort& leftover_block_depth
) {
+ constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};
short2 tile_dimensions_a = short2(THREADGROUP_BLOCK_K, tile_block_rows);
short2 tile_dimensions_b =
TRANSPOSE_B ? short2(THREADGROUP_BLOCK_K, tile_block_cols)
@@ -79,12 +109,12 @@ struct SimdgroupMmaCore {
for (int k = 0; k < aligned_k_iterations; k++) {
threadgroup_barrier(mem_flags::mem_threadgroup);
- if constexpr (M_aligned) {
+ if constexpr (gemm_alignment.contains(GemmAlignment::M)) {
loader_a.load_unsafe();
} else {
loader_a.load_safe(tile_dimensions_a);
}
- if constexpr (N_aligned) {
+ if constexpr (gemm_alignment.contains(GemmAlignment::N)) {
loader_b.load_unsafe();
} else {
loader_b.load_safe(tile_dimensions_b);
@@ -97,7 +127,7 @@ struct SimdgroupMmaCore {
loader_b.next();
}
- if constexpr (!K_aligned) {
+ if constexpr (!gemm_alignment.contains(GemmAlignment::K)) {
threadgroup_barrier(mem_flags::mem_threadgroup);
short2 last_tile_dimensions_a =
@@ -114,13 +144,66 @@ struct SimdgroupMmaCore {
}
}
+ template <uint GEMM_ALIGNMENT_RAW>
+ static METAL_FUNC void finalize(
+ thread TileAccumulator& accumulator,
+ device T* d,
+ const constant uzu::matmul::GemmParams* params,
+ const thread ushort& tile_block_rows,
+ const thread ushort& tile_block_cols,
+ const bool needs_epilogue,
+ const thread uzu::matmul::TransformScaleAccumulate<float, float>&
+ epilogue,
+ const device T* bias_block,
+ const bool needs_bias
+ ) {
+ constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};
+ if constexpr (
+ gemm_alignment.contains(GemmAlignment::M) &&
+ gemm_alignment.contains(GemmAlignment::N)
+ ) {
+ if (needs_epilogue) {
+ accumulator.apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
+ }
+ if (needs_bias) {
+ accumulator.apply_bias(bias_block);
+ }
+ accumulator.store_result(d, params->leading_dimension_d);
+ } else {
+ if (needs_epilogue) {
+ accumulator.apply_epilogue_safe(
+ d,
+ params->leading_dimension_d,
+ 1,
+ short2(tile_block_cols, tile_block_rows),
+ epilogue
+ );
+ }
+ if (needs_bias) {
+ accumulator.apply_bias_safe(
+ bias_block,
+ short2(tile_block_cols, tile_block_rows)
+ );
+ }
+ accumulator.store_result_safe(
+ d,
+ params->leading_dimension_d,
+ short2(tile_block_cols, tile_block_rows)
+ );
+ }
+ }
+
static METAL_FUNC void run(
const device T* a,
- const device T* b,
+ const device uint8_t* b_packed,
device T* d,
const constant uzu::matmul::GemmParams* params,
GemmAlignment alignment,
- GemmOutputTransformKind output_transform,
+ GemmDTransform output_transform,
+ const device T* scales,
+ const device T* biases,
+ const device uint8_t* zero_points,
+ const device T* output_bias,
threadgroup T* a_shared,
threadgroup T* b_shared,
const thread ThreadContext& thread_context
@@ -139,89 +222,193 @@ struct SimdgroupMmaCore {
const size_t block_col = size_t(geometry.block_col_start);
a += block_row * params->leading_dimension_a;
- b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;
d += block_row * params->leading_dimension_d + block_col;
thread ALoader
loader_a(a, params->leading_dimension_a, a_shared, thread_context);
- thread BLoader
- loader_b(b, params->leading_dimension_b, b_shared, thread_context);
thread TileAccumulator accumulator(thread_context);
const ushort tile_block_rows =
min(THREADGROUP_BLOCK_M,
- ((int)params->M) - int(geometry.block_row_start));
+ static_cast<int>(params->M) -
+ static_cast<int>(geometry.block_row_start));
const ushort tile_block_cols =
min(THREADGROUP_BLOCK_N,
- ((int)params->N) - int(geometry.block_col_start));
+ static_cast<int>(params->N) -
+ static_cast<int>(geometry.block_col_start));
const ushort leftover_block_depth =
params->K - params->aligned_inner_iterations * THREADGROUP_BLOCK_K;
- const bool needs_epilogue =
- output_transform != GemmOutputTransformKind::Store;
- const float alpha =
- (output_transform == GemmOutputTransformKind::Scale ||
- output_transform == GemmOutputTransformKind::ScaleAccumulate)
- ? params->ab_scale
- : 1.0f;
- const float beta =
- (output_transform == GemmOutputTransformKind::Accumulate ||
- output_transform == GemmOutputTransformKind::ScaleAccumulate)
- ? 1.0f
- : 0.0f;
+ const bool needs_scale = output_transform.contains(GemmDTransform::SCALE);
+ const bool needs_accumulate =
+ output_transform.contains(GemmDTransform::ACCUMULATE);
+ const bool needs_bias = output_transform.contains(GemmDTransform::BIAS);
+ const bool needs_epilogue = needs_scale || needs_accumulate;
+ const float alpha = needs_scale ? params->ab_scale : 1.0f;
+ const float beta = needs_accumulate ? 1.0f : 0.0f;
uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);
+ const device T* bias_block = output_bias + block_col;
+
+ const bool all_aligned = ((alignment.contains(GemmAlignment::M)) ||
+ (tile_block_rows == THREADGROUP_BLOCK_M)) &&
+ ((alignment.contains(GemmAlignment::N)) ||
+ (tile_block_cols == THREADGROUP_BLOCK_N)) &&
+ alignment.contains(GemmAlignment::K);
+
+ constexpr uint MASK_ALL = static_cast<uint>(GemmAlignment::M) |
+ static_cast<uint>(GemmAlignment::N) |
+ static_cast<uint>(GemmAlignment::K);
+
+ const uint dynamic_alignment_mask =
+ alignment.raw_value |
+ ((tile_block_rows == THREADGROUP_BLOCK_M)
+ ? static_cast<uint>(GemmAlignment::M)
+ : 0u) |
+ ((tile_block_cols == THREADGROUP_BLOCK_N)
+ ? static_cast<uint>(GemmAlignment::N)
+ : 0u);
+
+ auto dispatch_kernel = [&](auto kernel_invoke) {
+ if (all_aligned) {
+ kernel_invoke(integral_constant<uint, MASK_ALL>{});
+ } else {
+ dispatch_gemm_alignment(dynamic_alignment_mask, kernel_invoke);
+ }
+ };
- dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {
- dispatch_bool(
- alignment.contains(GemmAlignment::M) ||
- (tile_block_rows == THREADGROUP_BLOCK_M),
- [&](auto aligned_m) {
- dispatch_bool(
- alignment.contains(GemmAlignment::N) ||
- (tile_block_cols == THREADGROUP_BLOCK_N),
- [&](auto aligned_n) {
- k_loop<aligned_m.value, aligned_n.value, aligned_k.value>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- accumulator,
- tile_block_rows,
- tile_block_cols,
- leftover_block_depth
- );
- if constexpr (aligned_m.value && aligned_n.value) {
- if (needs_epilogue) {
- accumulator.apply_epilogue(
- d,
- params->leading_dimension_d,
- 1,
- epilogue
- );
- }
- accumulator.store_result(d, params->leading_dimension_d);
- } else {
- if (needs_epilogue) {
- accumulator.apply_epilogue_safe(
- d,
- params->leading_dimension_d,
- 1,
- short2(tile_block_cols, tile_block_rows),
- epilogue
- );
- }
- accumulator.store_result_safe(
- d,
- params->leading_dimension_d,
- short2(tile_block_cols, tile_block_rows)
- );
- }
- }
- );
- }
+ if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision) {
+ const device T* b = reinterpret_cast<const device T*>(b_packed);
+ b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;
+ thread BLoaderFp
+ loader_b(b, params->leading_dimension_b, b_shared, thread_context);
+ dispatch_kernel([&](auto gemm_alignment_mask) {
+ constexpr uint mask = gemm_alignment_mask.value;
+ k_loop<mask, BLoaderFp>(
+ a_shared,
+ b_shared,
+ params->aligned_inner_iterations,
+ loader_a,
+ loader_b,
+ accumulator,
+ tile_block_rows,
+ tile_block_cols,
+ leftover_block_depth
+ );
+ finalize<mask>(
+ accumulator,
+ d,
+ params,
+ tile_block_rows,
+ tile_block_cols,
+ needs_epilogue,
+ epilogue,
+ bias_block,
+ needs_bias
+ );
+ });
+ return;
+ }
+
+ if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant) {
+ constexpr int pack_factor = get_pack_factor<BITS, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
+ const int k_elements = int(params->K);
+ const int weights_row_stride_bytes =
+ k_elements * bytes_per_pack / pack_factor;
+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
+ const device uint8_t* weights_block =
+ b_packed + block_col * weights_row_stride_bytes;
+ const device T* scales_offset = scales + block_col * groups_per_row;
+ const device T* biases_offset = biases + block_col * groups_per_row;
+ thread BLoaderScaleBias loader_b(
+ weights_block,
+ scales_offset,
+ biases_offset,
+ k_elements,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ dispatch_kernel([&](auto gemm_alignment_mask) {
+ constexpr uint mask = gemm_alignment_mask.value;
+ k_loop<mask, BLoaderScaleBias>(
+ a_shared,
+ b_shared,
+ params->aligned_inner_iterations,
+ loader_a,
+ loader_b,
+ accumulator,
+ tile_block_rows,
+ tile_block_cols,
+ leftover_block_depth
+ );
+ finalize<mask>(
+ accumulator,
+ d,
+ params,
+ tile_block_rows,
+ tile_block_cols,
+ needs_epilogue,
+ epilogue,
+ bias_block,
+ needs_bias
+ );
+ });
+ return;
+ }
+
+ if constexpr (
+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant
+ ) {
+ constexpr int pack_factor = get_pack_factor<BITS, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
+ const int k_elements = int(params->K);
+ const int weights_row_stride_bytes =
+ k_elements * bytes_per_pack / pack_factor;
+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
+ const device uint8_t* weights_block =
+ b_packed + block_col * weights_row_stride_bytes;
+ const device T* scales_offset = scales + block_col * groups_per_row;
+ const int zero_point_stride_per_row =
+ (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
+ const device uint8_t* zero_points_row_start =
+ zero_points + block_col * zero_point_stride_per_row;
+ thread BLoaderScaleZeroPoint loader_b(
+ weights_block,
+ scales_offset,
+ zero_points_row_start,
+ k_elements,
+ groups_per_row,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
);
- });
+ dispatch_kernel([&](auto gemm_alignment_mask) {
+ constexpr uint mask = gemm_alignment_mask.value;
+ k_loop<mask, BLoaderScaleZeroPoint>(
+ a_shared,
+ b_shared,
+ params->aligned_inner_iterations,
+ loader_a,
+ loader_b,
+ accumulator,
+ tile_block_rows,
+ tile_block_cols,
+ leftover_block_depth
+ );
+ finalize<mask>(
+ accumulator,
+ d,
+ params,
+ tile_block_rows,
+ tile_block_cols,
+ needs_epilogue,
+ epilogue,
+ bias_block,
+ needs_bias
+ );
+ });
+ }
}
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs
deleted file mode 100644
index 7b24036a7..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs
+++ /dev/null
@@ -1,251 +0,0 @@
-use crate::{
- DataType,
- backends::{
- common::{
- Allocation, AsBufferRangeRef, Backend, Buffer,
- gpu_types::{
- GemmParams, QuantizationMode,
- gemm::{
- GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig,
- GemmWeightPrologueKind,
- },
- },
- },
- metal::Metal,
- },
-};
-
-pub struct GemmAlignmentAxes {
- pub m: bool,
- pub n: bool,
- pub k: bool,
-}
-
-impl GemmAlignment {
- pub fn from_axes(axes: GemmAlignmentAxes) -> Self {
- let mut bits = Self::empty();
- bits.set(Self::M, axes.m);
- bits.set(Self::N, axes.n);
- bits.set(Self::K, axes.k);
- bits
- }
-}
-
-pub enum GemmWeights<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>> = Allocation<B>> {
- FullPrecision {
- weights: &'a T,
- },
- ScaleBias {
- weights: &'a T,
- scales: &'a Allocation<B>,
- biases: &'a Allocation<B>,
- mode: QuantizationMode,
- group_size: u32,
- },
- ScaleZeroPoint {
- weights: &'a T,
- scales: &'a Allocation<B>,
- zero_points: &'a Allocation<B>,
- mode: QuantizationMode,
- group_size: u32,
- },
-}
-
-impl<B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> GemmWeights<'_, B, T> {
- pub fn weight_prologue(&self) -> GemmWeightPrologueKind {
- match self {
- Self::FullPrecision {
- ..
- } => GemmWeightPrologueKind::FullPrecision,
- Self::ScaleBias {
- ..
- } => GemmWeightPrologueKind::ScaleBiasDequant,
- Self::ScaleZeroPoint {
- ..
- } => GemmWeightPrologueKind::ScaleZeroPointDequant,
- }
- }
-
- pub fn bits_per_weight(&self) -> u32 {
- match self {
- Self::FullPrecision {
- ..
- } => 0,
- Self::ScaleBias {
- mode,
- ..
- }
- | Self::ScaleZeroPoint {
- mode,
- ..
- } => DataType::from(*mode).size_in_bits() as u32,
- }
- }
-
- pub fn group_size(&self) -> u32 {
- match self {
- Self::FullPrecision {
- ..
- } => 0,
- Self::ScaleBias {
- group_size,
- ..
- }
- | Self::ScaleZeroPoint {
- group_size,
- ..
- } => *group_size,
- }
- }
-}
-
-pub struct GemmDispatch<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> {
- pub tiling_config: GemmTilingConfig,
- pub input_prologue: GemmInputPrologueKind,
- pub use_mxu: bool,
- pub output_transform: GemmOutputTransformKind,
- pub alignment: GemmAlignment,
- pub transpose_b: bool,
- pub a: &'a Allocation<B>,
- pub a_offset: usize,
- pub b: GemmWeights<'a, B, T>,
- pub b_offset: usize,
- pub d: &'a mut Allocation<B>,
- pub params: GemmParams,
- pub group_count_x: u32,
- pub group_count_y: u32,
-}
-
-impl<B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> GemmDispatch<'_, B, T> {
- pub(crate) fn specialization(&self) -> GemmSpecialization {
- GemmSpecialization {
- tiling_config: self.tiling_config,
- input_prologue: self.input_prologue,
- use_mxu: self.use_mxu,
- output_transform: self.output_transform,
- alignment: self.alignment,
- transpose_b: self.transpose_b,
- weight_prologue: self.b.weight_prologue(),
- bits_per_weight: self.b.bits_per_weight(),
- group_size: self.b.group_size(),
- }
- }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub(crate) struct GemmSpecialization {
- pub(crate) tiling_config: GemmTilingConfig,
- pub(crate) input_prologue: GemmInputPrologueKind,
- pub(crate) use_mxu: bool,
- pub(crate) output_transform: GemmOutputTransformKind,
- pub(crate) alignment: GemmAlignment,
- pub(crate) transpose_b: bool,
- pub(crate) weight_prologue: GemmWeightPrologueKind,
- pub(crate) bits_per_weight: u32,
- pub(crate) group_size: u32,
-}
-
-impl GemmSpecialization {
- pub(crate) fn validate(&self) -> Result<(), GemmSpecializationError> {
- if self.group_size != 0 && self.tiling_config.threadgroup_k > self.group_size {
- return Err(GemmSpecializationError::ThreadgroupKExceedsGroupSize {
- threadgroup_k: self.tiling_config.threadgroup_k,
- group_size: self.group_size,
- });
- }
- Ok(())
- }
-
- pub(crate) fn precompile_configs(data_type: DataType) -> Vec<Self> {
- let mut out = Vec::new();
- for &(threadgroup_m, threadgroup_n, threadgroup_k) in simdgroup_tile_set(data_type) {
- for align_mn in [true, false] {
- let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {
- m: align_mn,
- n: align_mn,
- k: true,
- });
- out.push(Self {
- tiling_config: GemmTilingConfig {
- threadgroup_m,
- threadgroup_n,
- threadgroup_k,
- simdgroups_m: 2,
- simdgroups_n: 2,
- },
- input_prologue: GemmInputPrologueKind::FullPrecision,
- use_mxu: false,
- output_transform: GemmOutputTransformKind::Store,
- alignment,
- transpose_b: true,
- weight_prologue: GemmWeightPrologueKind::FullPrecision,
- bits_per_weight: 0,
- group_size: 0,
- });
- }
- }
-
- for &(threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) in mxu_tile_set(data_type) {
- for align_m in [true, false] {
- for align_n in [true, false] {
- for align_k in [true, false] {
- let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {
- m: align_m,
- n: align_n,
- k: align_k,
- });
- for output_transform in [
- GemmOutputTransformKind::Store,
- GemmOutputTransformKind::Scale,
- GemmOutputTransformKind::Accumulate,
- GemmOutputTransformKind::ScaleAccumulate,
- ] {
- out.push(Self {
- tiling_config: GemmTilingConfig {
- threadgroup_m,
- threadgroup_n,
- threadgroup_k: 32,
- simdgroups_m,
- simdgroups_n,
- },
- input_prologue: GemmInputPrologueKind::FullPrecision,
- use_mxu: true,
- output_transform,
- alignment,
- transpose_b: true,
- weight_prologue: GemmWeightPrologueKind::FullPrecision,
- bits_per_weight: 0,
- group_size: 0,
- });
- }
- }
- }
- }
- }
- out
- }
-}
-
-fn simdgroup_tile_set(data_type: DataType) -> &'static [(u32, u32, u32)] {
- match data_type {
- DataType::BF16 => &[(64, 32, 32), (64, 64, 16)],
- DataType::F16 => &[(64, 64, 16), (64, 32, 32)],
- DataType::F32 => &[(32, 64, 16)],
- _ => &[],
- }
-}
-
-fn mxu_tile_set(data_type: DataType) -> &'static [(u32, u32, u32, u32)] {
- if !matches!(data_type, DataType::F16 | DataType::BF16) {
- return &[];
- }
- &[(64, 64, 2, 2), (32, 64, 2, 2), (64, 32, 4, 1), (128, 128, 4, 4)]
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum GemmSpecializationError {
- ThreadgroupKExceedsGroupSize {
- threadgroup_k: u32,
- group_size: u32,
- },
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs
new file mode 100644
index 000000000..5663f7268
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs
@@ -0,0 +1,14 @@
+use thiserror::Error;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
+pub enum GemmSpecializationError {
+ #[error("threadgroup K={threadgroup_k} exceeds group size {group_size}")]
+ ThreadgroupKExceedsGroupSize {
+ threadgroup_k: u32,
+ group_size: u32,
+ },
+ #[error("quantized B requires the simdgroup path")]
+ QuantizedRequiresSimdgroup,
+ #[error("quantized B requires transposed layout")]
+ QuantizedRequiresTransposedB,
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs
deleted file mode 100644
index 7ea4fb243..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs
+++ /dev/null
@@ -1,180 +0,0 @@
-use super::{GemmAlignmentAxes, GemmDispatch, GemmKernel, GemmWeights, MXU_THREADGROUP_BLOCK_K};
-use crate::{
- DataType,
- backends::{
- common::{
- AsBufferRangeRef, Buffer, Encoder,
- gpu_types::{
- GemmParams,
- gemm::{GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig},
- },
- kernel::{
- TensorAddBiasKernel,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulError},
- },
- },
- metal::{Metal, context::MetalContext, kernel::TensorAddBiasMetalKernel, metal_extensions::DeviceExt},
- },
-};
-
-pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
- gemm: &mut GemmKernel,
- bias_add: &mut TensorAddBiasMetalKernel,
- data_type: DataType,
- context: &MetalContext,
- encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal, TB>,
- use_mxu: bool,
-) -> Result<(), MatmulError<Metal>> {
- if use_mxu && !context.device.supports_mxu() {
- return Err(MatmulError::UnsupportedDataType(data_type));
- }
-
- let tile = if use_mxu {
- select_mxu_tile(&arguments)
- } else {
- select_simdgroup_tile(data_type, &arguments)
- };
- let k_block = if use_mxu {
- MXU_THREADGROUP_BLOCK_K
- } else {
- tile.threadgroup_k
- };
-
- let threadgroups_per_row = arguments.output_dim.div_ceil(tile.threadgroup_n);
- let threadgroups_per_column = arguments.batch_dim.div_ceil(tile.threadgroup_m);
-
- let (use_morton, group_count_x, group_count_y) = if use_mxu {
- let max_dim = threadgroups_per_row.max(threadgroups_per_column);
- let min_dim = threadgroups_per_row.min(threadgroups_per_column);
- let morton_dim = max_dim.next_power_of_two();
- let morton_total = morton_dim.saturating_mul(morton_dim);
- let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);
- let use_morton = min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);
- if use_morton {
- (true, morton_total, 1)
- } else {
- (false, threadgroups_per_row, threadgroups_per_column)
- }
- } else {
- (false, threadgroups_per_row, threadgroups_per_column)
- };
-
- let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {
- m: arguments.batch_dim % tile.threadgroup_m == 0,
- n: arguments.output_dim % tile.threadgroup_n == 0,
- k: arguments.input_dim % k_block == 0,
- });
- let output_transform = output_transform_from(&arguments);
-
- let default_ldb = if arguments.b_transpose {
- arguments.input_dim
- } else {
- arguments.output_dim
- };
- let params = GemmParams {
- M: arguments.batch_dim,
- N: arguments.output_dim,
- K: arguments.input_dim,
- leading_dimension_a: arguments.input_dim,
- leading_dimension_b: arguments.b_leading_dimension.unwrap_or(default_ldb),
- leading_dimension_d: arguments.output_dim,
- threadgroups_per_row,
- threadgroups_per_column,
- aligned_inner_iterations: arguments.input_dim / k_block,
- use_morton,
- ab_scale: arguments.ab_scale,
- };
-
- let dispatch = GemmDispatch {
- tiling_config: tile,
- input_prologue: GemmInputPrologueKind::FullPrecision,
- use_mxu,
- output_transform,
- alignment,
- transpose_b: arguments.b_transpose,
- a: arguments.a,
- a_offset: arguments.a_offset,
- b: GemmWeights::FullPrecision {
- weights: arguments.b,
- },
- b_offset: arguments.b_offset,
- d: arguments.d,
- params,
- group_count_x,
- group_count_y,
- };
-
- gemm.encode(context, dispatch, encoder).map_err(MatmulError::BackendError)?;
-
- if let MatmulArgumentC::Bias(bias) = arguments.c {
- bias_add.encode(
- None::<&<Metal as crate::backends::common::Backend>::DenseBuffer>,
- bias,
- arguments.d,
- arguments.output_dim,
- arguments.batch_dim * arguments.output_dim,
- encoder,
- );
- }
-
- Ok(())
-}
-
-fn select_simdgroup_tile<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
- data_type: DataType,
- arguments: &MatmulArguments<Metal, TB>,
-) -> GemmTilingConfig {
- let (threadgroup_m, threadgroup_n, threadgroup_k) = match data_type {
- DataType::F32 => (32u32, 64u32, 16u32),
- _ => {
- if 2 * arguments.batch_dim.max(arguments.output_dim) > arguments.input_dim {
- (64, 64, 16)
- } else {
- (64, 32, 32)
- }
- },
- };
- GemmTilingConfig {
- threadgroup_m,
- threadgroup_n,
- threadgroup_k,
- simdgroups_m: 2,
- simdgroups_n: 2,
- }
-}
-
-fn select_mxu_tile<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
- arguments: &MatmulArguments<Metal, TB>
-) -> GemmTilingConfig {
- let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) =
- if arguments.batch_dim >= 256 && arguments.output_dim >= 128 {
- (128u32, 128u32, 4u32, 4u32)
- } else if arguments.output_dim < 64 {
- (64u32, 32u32, 4u32, 1u32)
- } else if arguments.batch_dim < 64 {
- (32u32, 64u32, 2u32, 2u32)
- } else {
- (64u32, 64u32, 2u32, 2u32)
- };
- GemmTilingConfig {
- threadgroup_m,
- threadgroup_n,
- threadgroup_k: 32,
- simdgroups_m,
- simdgroups_n,
- }
-}
-
-fn output_transform_from<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
- arguments: &MatmulArguments<Metal, TB>
-) -> GemmOutputTransformKind {
- let scale = arguments.ab_scale != 1.0;
- let accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);
- match (scale, accumulate) {
- (false, false) => GemmOutputTransformKind::Store,
- (true, false) => GemmOutputTransformKind::Scale,
- (false, true) => GemmOutputTransformKind::Accumulate,
- (true, true) => GemmOutputTransformKind::ScaleAccumulate,
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 00ad9e2d7..563af413d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -3,102 +3,138 @@
#include "../../common/thread_context.h"
#include "../generated/gemm.h"
+#include "common/gemm_tiling.h"
#include "common/mxu_mma_core.h"
#include "common/simdgroup_mma_core.h"
using namespace metal;
using namespace uzu::gemm;
-#define GEMM_MAX_THREADGROUP_A 2560
-#define GEMM_MAX_THREADGROUP_B 1536
+#define GEMM_TGA_ELEMENTS \
+ (USE_MXU ? 1 \
+ : (gemm_tiling_block_m(GEMM_TILING) * \
+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(T)))))
+#define GEMM_TGB_ELEMENTS \
+ (USE_MXU ? 1 \
+ : (gemm_tiling_block_n(GEMM_TILING) * \
+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(T)))))
template <
typename T,
- uint THREADGROUP_BLOCK_M,
- uint THREADGROUP_BLOCK_N,
- uint THREADGROUP_BLOCK_K,
- uint SIMDGROUPS_M,
- uint SIMDGROUPS_N,
+ GemmTiling GEMM_TILING,
bool TRANSPOSE_B,
- bool USE_MXU>
-VARIANTS(T, float, half, bfloat)
-VARIANTS(THREADGROUP_BLOCK_M, 32, 64, 128)
-VARIANTS(THREADGROUP_BLOCK_N, 32, 64, 128)
-VARIANTS(THREADGROUP_BLOCK_K, 16, 32)
-VARIANTS(SIMDGROUPS_M, 1, 2, 4)
-VARIANTS(SIMDGROUPS_N, 1, 2, 4)
+ bool USE_MXU,
+ GemmWeightPrologueKind WEIGHT_PROLOGUE,
+ uint BITS,
+ uint GROUP_SIZE>
+VARIANTS(T, half, bfloat)
+VARIANTS(
+ GEMM_TILING,
+ GemmTiling::T8x32x32_1x1,
+ GemmTiling::T64x32x32_2x2,
+ GemmTiling::T64x64x16_2x2,
+ GemmTiling::T64x64x32_2x2,
+ GemmTiling::T64x64x64_2x2,
+ GemmTiling::T32x32x32_2x2,
+ GemmTiling::T32x64x32_2x2,
+ GemmTiling::T64x32x32_4x1,
+ GemmTiling::T128x128x32_4x4)
VARIANTS(TRANSPOSE_B, false, true)
VARIANTS(USE_MXU, false, true)
-CONSTRAINT(!USE_MXU || THREADGROUP_BLOCK_M % SIMDGROUPS_M == 0)
-CONSTRAINT(!USE_MXU || THREADGROUP_BLOCK_N % SIMDGROUPS_N == 0)
-CONSTRAINT(!USE_MXU || (THREADGROUP_BLOCK_M / SIMDGROUPS_M) % 16 == 0)
-CONSTRAINT(!USE_MXU || (THREADGROUP_BLOCK_N / SIMDGROUPS_N) % 16 == 0)
-CONSTRAINT(USE_MXU || max(THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N) <= 32 * SIMDGROUPS_M * SIMDGROUPS_N)
+VARIANTS(
+ WEIGHT_PROLOGUE,
+ GemmWeightPrologueKind::FullPrecision,
+ GemmWeightPrologueKind::ScaleBiasDequant,
+ GemmWeightPrologueKind::ScaleZeroPointDequant)
+VARIANTS(BITS, 0, 4, 8)
+VARIANTS(GROUP_SIZE, 0, 32, 64, 128)
+CONSTRAINT(
+ !USE_MXU || GEMM_TILING == GemmTiling::T64x64x32_2x2 ||
+ GEMM_TILING == GemmTiling::T32x64x32_2x2 ||
+ GEMM_TILING == GemmTiling::T64x32x32_4x1 ||
+ GEMM_TILING == GemmTiling::T128x128x32_4x4)
+CONSTRAINT(
+ WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision ||
+ (BITS == 0 && GROUP_SIZE == 0))
+CONSTRAINT(
+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||
+ (BITS != 0 && GROUP_SIZE != 0))
+CONSTRAINT(
+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||
+ (!USE_MXU && TRANSPOSE_B &&
+ (GEMM_TILING == GemmTiling::T8x32x32_1x1 ||
+ GEMM_TILING == GemmTiling::T32x32x32_2x2 ||
+ GEMM_TILING == GemmTiling::T64x32x32_2x2 ||
+ GEMM_TILING == GemmTiling::T64x64x32_2x2 ||
+ GEMM_TILING == GemmTiling::T64x64x64_2x2)))
+CONSTRAINT(
+ GEMM_TILING != GemmTiling::T64x64x64_2x2 ||
+ GROUP_SIZE == 64 || GROUP_SIZE == 128)
KERNEL(Gemm)(
const device T* a,
const device uint8_t* b_packed,
device T* d,
const device T* scales
- OPTIONAL(weight_prologue != GemmWeightPrologueKind::FullPrecision),
+ OPTIONAL(WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision),
const device T* biases
- OPTIONAL(weight_prologue == GemmWeightPrologueKind::ScaleBiasDequant),
+ OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant),
const device uint8_t* zero_points
- OPTIONAL(weight_prologue == GemmWeightPrologueKind::ScaleZeroPointDequant),
+ OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant),
+ const device T* output_bias
+ OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),
const constant uzu::matmul::GemmParams* params,
const constant uint& group_count_x,
const constant uint& group_count_y,
- const GemmInputPrologueKind input_prologue SPECIALIZE,
- const GemmWeightPrologueKind weight_prologue SPECIALIZE,
- const GemmOutputTransformKind output_transform SPECIALIZE,
+ const GemmDTransform output_transform SPECIALIZE,
const GemmAlignment alignment SPECIALIZE,
- const uint bits_per_weight SPECIALIZE,
- const uint group_size SPECIALIZE,
- threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],
- threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],
+ threadgroup T a_shared[GEMM_TGA_ELEMENTS],
+ threadgroup T b_shared[GEMM_TGB_ELEMENTS],
const uint group_x GROUPS(group_count_x),
const uint group_y GROUPS(group_count_y),
const uint thread_x THREADS(METAL_SIMD_SIZE),
- const uint thread_y THREADS(SIMDGROUPS_N),
- const uint thread_z THREADS(SIMDGROUPS_M),
+ const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),
+ const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),
const ThreadContext thread_context
) {
- (void)scales;
- (void)biases;
- (void)zero_points;
- (void)bits_per_weight;
- (void)group_size;
(void)group_x;
(void)group_y;
(void)thread_x;
(void)thread_y;
(void)thread_z;
- (void)input_prologue;
- (void)weight_prologue;
- const device T* b = reinterpret_cast<const device T*>(b_packed);
+
if constexpr (USE_MXU) {
- MxuMmaCore<
- T,
- THREADGROUP_BLOCK_M,
- THREADGROUP_BLOCK_N,
- SIMDGROUPS_M,
- SIMDGROUPS_N,
- TRANSPOSE_B>::
- run(a, b, d, params, alignment, output_transform, thread_context);
+ (void)scales;
+ (void)biases;
+ (void)zero_points;
+ const device T* b = reinterpret_cast<const device T*>(b_packed);
+ MxuMmaCore<T, GEMM_TILING, TRANSPOSE_B>::run(
+ a,
+ b,
+ d,
+ params,
+ alignment,
+ output_transform,
+ output_bias,
+ thread_context
+ );
} else {
SimdgroupMmaCore<
T,
- THREADGROUP_BLOCK_M,
- THREADGROUP_BLOCK_N,
- THREADGROUP_BLOCK_K,
- SIMDGROUPS_M,
- SIMDGROUPS_N,
- TRANSPOSE_B>::
+ GEMM_TILING,
+ TRANSPOSE_B,
+ WEIGHT_PROLOGUE,
+ BITS,
+ GROUP_SIZE>::
run(a,
- b,
+ b_packed,
d,
params,
alignment,
output_transform,
+ scales,
+ biases,
+ zero_points,
+ output_bias,
a_shared,
b_shared,
thread_context);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 5f00d0280..cc5650f78 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -1,20 +1,41 @@
use std::collections::{HashMap, hash_map::Entry};
-use super::dispatch::{GemmDispatch, GemmSpecialization, GemmWeights};
+use super::{MXU_THREADGROUP_BLOCK_K, specialization::GemmSpecialization};
use crate::{
DataType,
backends::{
common::{
- AsBufferRangeRef, Buffer, Encoder,
- gpu_types::gemm::{GemmInputPrologueKind, GemmWeightPrologueKind},
+ Allocation, AsBufferRangeRef, Backend, Buffer, Encoder,
+ gpu_types::{
+ GemmParams,
+ gemm::{GemmAlignment, GemmDTransform, GemmTiling},
+ },
+ kernel::{
+ HadamardTransformKernel, Kernels, TensorAddBiasKernel,
+ matmul::{MatmulArguments, MatmulB, MatmulError, MatmulQuantCombo},
+ },
+ },
+ metal::{
+ Metal,
+ context::MetalContext,
+ error::MetalError,
+ kernel::{GemmMetalKernel, TensorAddBiasMetalKernel},
+ metal_extensions::DeviceExt,
},
- metal::{Metal, context::MetalContext, error::MetalError, kernel::GemmMetalKernel},
},
};
-pub(crate) struct GemmKernel {
+#[derive(Debug, Clone, Copy)]
+pub enum GemmDispatchPath {
+ Simdgroup,
+ Mxu,
+}
+
+pub struct GemmKernel {
data_type: DataType,
kernels: HashMap<GemmSpecialization, GemmMetalKernel>,
+ pub bias_add: TensorAddBiasMetalKernel,
+ pub hadamard: <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel,
}
impl GemmKernel {
@@ -22,9 +43,13 @@ impl GemmKernel {
context: &MetalContext,
data_type: DataType,
) -> Result<Self, MetalError> {
+ let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true)?;
+ let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)?;
let mut kernel = Self {
data_type,
kernels: HashMap::new(),
+ bias_add,
+ hadamard,
};
for specialization in GemmSpecialization::precompile_configs(data_type) {
kernel.get_or_create(context, specialization)?;
@@ -32,6 +57,17 @@ impl GemmKernel {
Ok(kernel)
}
+ pub fn preheat_quant_combo(
+ &mut self,
+ context: &MetalContext,
+ combo: MatmulQuantCombo,
+ ) -> Result<(), MetalError> {
+ for specialization in GemmSpecialization::quant_combo_specs(self.data_type, combo) {
+ self.get_or_create(context, specialization)?;
+ }
+ Ok(())
+ }
+
fn get_or_create(
&mut self,
context: &MetalContext,
@@ -42,74 +78,327 @@ impl GemmKernel {
Entry::Vacant(entry) => {
let kernel = GemmMetalKernel::new(
context,
- self.data_type,
- specialization.tiling_config.threadgroup_m,
- specialization.tiling_config.threadgroup_n,
- specialization.tiling_config.threadgroup_k,
- specialization.tiling_config.simdgroups_m,
- specialization.tiling_config.simdgroups_n,
+ specialization.data_type,
+ specialization.tiling,
specialization.transpose_b,
specialization.use_mxu,
- specialization.input_prologue,
specialization.weight_prologue,
+ specialization.bits_per_weight.unwrap_or(0),
+ specialization.group_size.unwrap_or(0),
specialization.output_transform,
specialization.alignment,
- specialization.bits_per_weight,
- specialization.group_size,
)?;
Ok(entry.insert(kernel))
},
}
}
- pub(crate) fn encode<T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+ pub fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
&mut self,
- context: &MetalContext,
- dispatch: GemmDispatch<'_, Metal, T>,
+ arguments: MatmulArguments<'a, Metal, TB>,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
- let specialization = dispatch.specialization();
- assert_eq!(
- specialization.input_prologue,
- GemmInputPrologueKind::FullPrecision,
- "unified GEMM only implements FullPrecision input prologue today",
- );
- assert_eq!(
- specialization.weight_prologue,
- GemmWeightPrologueKind::FullPrecision,
- "unified GEMM only implements FullPrecision weight prologue today",
- );
- specialization.validate().map_err(MetalError::InvalidGemmSpecialization)?;
- let kernel = self.get_or_create(context, specialization)?;
- let (b, scales, biases, zero_points) = match &dispatch.b {
- GemmWeights::FullPrecision {
- weights,
- } => (*weights, None, None, None),
- GemmWeights::ScaleBias {
- weights,
- scales,
- biases,
+ let path = if encoder.context().device.supports_mxu()
+ && matches!(self.data_type, DataType::F16 | DataType::BF16)
+ && matches!(arguments.b, MatmulB::FullPrecision { .. })
+ {
+ GemmDispatchPath::Mxu
+ } else {
+ GemmDispatchPath::Simdgroup
+ };
+ self.encode_dispatch_path(arguments, path, encoder)
+ }
+
+ pub fn encode_dispatch_path<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+ &mut self,
+ arguments: MatmulArguments<'a, Metal, TB>,
+ path: GemmDispatchPath,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), MetalError> {
+ let use_mxu = match path {
+ GemmDispatchPath::Mxu => {
+ assert!(
+ encoder.context().device.supports_mxu(),
+ "GemmDispatchPath::Mxu requested on hardware without MXU support",
+ );
+ assert!(
+ matches!(self.data_type, DataType::F16 | DataType::BF16),
+ "GemmDispatchPath::Mxu requires F16 or BF16 data type, got {:?}",
+ self.data_type,
+ );
+ assert!(
+ matches!(arguments.b, MatmulB::FullPrecision { .. }),
+ "GemmDispatchPath::Mxu requires FullPrecision B",
+ );
+ true
+ },
+ GemmDispatchPath::Simdgroup => false,
+ };
+
+ let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });
+ if is_quant {
+ if arguments.d_transform.mask().contains(GemmDTransform::ACCUMULATE) {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::ACCUMULATE,
+ path: "QuantGemm",
+ }
+ .into());
+ }
+ if !arguments.b_transpose || arguments.b_leading_dimension.is_some() || arguments.b_offset != 0 {
+ return Err(MatmulError::UnsupportedLayout {
+ path: "QuantGemm",
+ }
+ .into());
+ }
+ }
+
+ let ab_scale = arguments.d_transform.ab_scale;
+ let post_bias = if use_mxu {
+ arguments.d_transform.bias
+ } else {
+ None
+ };
+ let output_bias = if use_mxu {
+ None
+ } else {
+ arguments.d_transform.bias
+ };
+ let post_rht = arguments.d_transform.rht_factors;
+ let mut output_transform = arguments.d_transform.mask();
+ output_transform.remove(GemmDTransform::RHT);
+ if use_mxu {
+ output_transform.remove(GemmDTransform::BIAS);
+ }
+
+ let weight_prologue = arguments.b.weight_prologue();
+ let bits_per_weight = arguments.b.bits_per_weight();
+ let group_size = arguments.b.group_size();
+
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ d,
+ m,
+ n,
+ k,
+ ..
+ } = arguments;
+
+ match b {
+ MatmulB::FullPrecision {
+ b: weights,
+ } => {
+ let tiling = if use_mxu {
+ select_mxu_tiling(m, n)
+ } else {
+ select_simdgroup_tiling(m, n, k)
+ };
+ let k_block = if use_mxu {
+ MXU_THREADGROUP_BLOCK_K
+ } else {
+ tiling.block_k()
+ };
+
+ let threadgroups_per_row = n.div_ceil(tiling.block_n());
+ let threadgroups_per_column = m.div_ceil(tiling.block_m());
+
+ let (use_morton, group_count_x, group_count_y) = if use_mxu {
+ let max_dim = threadgroups_per_row.max(threadgroups_per_column);
+ let min_dim = threadgroups_per_row.min(threadgroups_per_column);
+ let morton_dim = max_dim.next_power_of_two();
+ let morton_total = morton_dim.saturating_mul(morton_dim);
+ let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);
+ let use_morton = min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);
+ if use_morton {
+ (true, morton_total, 1)
+ } else {
+ (false, threadgroups_per_row, threadgroups_per_column)
+ }
+ } else {
+ (false, threadgroups_per_row, threadgroups_per_column)
+ };
+
+ let alignment =
+ GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % k_block == 0);
+
+ let default_ldb = if b_transpose {
+ k
+ } else {
+ n
+ };
+ let params = GemmParams {
+ M: m,
+ N: n,
+ K: k,
+ leading_dimension_a: k,
+ leading_dimension_b: b_leading_dimension.unwrap_or(default_ldb),
+ leading_dimension_d: n,
+ threadgroups_per_row,
+ threadgroups_per_column,
+ aligned_inner_iterations: k / k_block,
+ use_morton,
+ ab_scale,
+ };
+
+ let specialization = GemmSpecialization {
+ data_type: self.data_type,
+ tiling,
+ use_mxu,
+ output_transform,
+ alignment,
+ transpose_b: b_transpose,
+ weight_prologue,
+ bits_per_weight,
+ group_size,
+ };
+ specialization.validate().map_err(MetalError::from)?;
+ let kernel = self.get_or_create(encoder.context(), specialization)?;
+ kernel.encode(
+ (a, a_offset),
+ (weights, b_offset),
+ &mut *d,
+ None::<&Allocation<Metal>>,
+ None::<&Allocation<Metal>>,
+ None::<&Allocation<Metal>>,
+ output_bias,
+ std::slice::from_ref(&params),
+ group_count_x,
+ group_count_y,
+ encoder,
+ );
+ },
+ quant_b @ (MatmulB::ScaleBiasDequant {
..
- } => (*weights, Some(*scales), Some(*biases), None),
- GemmWeights::ScaleZeroPoint {
- weights,
- scales,
- zero_points,
+ }
+ | MatmulB::ScaleZeroPointDequant {
..
- } => (*weights, Some(*scales), None, Some(*zero_points)),
- };
- kernel.encode(
- (dispatch.a, dispatch.a_offset),
- (b, dispatch.b_offset),
- dispatch.d,
- scales,
- biases,
- zero_points,
- std::slice::from_ref(&dispatch.params),
- dispatch.group_count_x,
- dispatch.group_count_y,
- encoder,
- );
+ }) => {
+ let (weights, scales, biases, zero_points) = match quant_b {
+ MatmulB::ScaleBiasDequant {
+ b: w,
+ scales,
+ biases,
+ ..
+ } => (w, Some(scales), Some(biases), None),
+ MatmulB::ScaleZeroPointDequant {
+ b: w,
+ scales,
+ zero_points,
+ ..
+ } => (w, Some(scales), None, Some(zero_points)),
+ _ => unreachable!(),
+ };
+
+ let tiling = select_quant_tiling(m, n);
+ let alignment =
+ GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
+ let params = quant_params(m, n, k, tiling, ab_scale);
+ let group_count_x = n.div_ceil(tiling.block_n());
+ let group_count_y = m.div_ceil(tiling.block_m());
+
+ let specialization = GemmSpecialization {
+ data_type: self.data_type,
+ tiling,
+ use_mxu,
+ output_transform,
+ alignment,
+ transpose_b: true,
+ weight_prologue,
+ bits_per_weight,
+ group_size,
+ };
+ specialization.validate().map_err(MetalError::from)?;
+ let kernel = self.get_or_create(encoder.context(), specialization)?;
+ kernel.encode(
+ (a, a_offset),
+ (weights, b_offset),
+ &mut *d,
+ scales,
+ biases,
+ zero_points,
+ output_bias,
+ std::slice::from_ref(&params),
+ group_count_x,
+ group_count_y,
+ encoder,
+ );
+ },
+ }
+
+ if let Some(bias) = post_bias {
+ self.bias_add.encode(None::<&<Metal as Backend>::DenseBuffer>, bias, &mut *d, n, m * n, encoder);
+ }
+ if let Some(factors) = post_rht {
+ self.hadamard.encode(d, factors, n, m, encoder);
+ }
Ok(())
}
}
+
+fn quant_params(
+ m: u32,
+ n: u32,
+ k: u32,
+ tiling: GemmTiling,
+ ab_scale: f32,
+) -> GemmParams {
+ GemmParams {
+ M: m,
+ N: n,
+ K: k,
+ leading_dimension_a: k,
+ leading_dimension_b: k,
+ leading_dimension_d: n,
+ threadgroups_per_row: n.div_ceil(tiling.block_n()),
+ threadgroups_per_column: m.div_ceil(tiling.block_m()),
+ aligned_inner_iterations: k / tiling.block_k(),
+ use_morton: false,
+ ab_scale,
+ }
+}
+
+fn select_simdgroup_tiling(
+ m: u32,
+ n: u32,
+ k: u32,
+) -> GemmTiling {
+ if 2 * m.max(n) > k {
+ GemmTiling::T64x64x16_2x2
+ } else {
+ GemmTiling::T64x32x32_2x2
+ }
+}
+
+fn select_mxu_tiling(
+ m: u32,
+ n: u32,
+) -> GemmTiling {
+ if m >= 256 && n >= 128 {
+ GemmTiling::T128x128x32_4x4
+ } else if n < 64 {
+ GemmTiling::T64x32x32_4x1
+ } else if m < 64 {
+ GemmTiling::T32x64x32_2x2
+ } else {
+ GemmTiling::T64x64x32_2x2
+ }
+}
+
+fn select_quant_tiling(
+ m: u32,
+ n: u32,
+) -> GemmTiling {
+ if m < 32 {
+ GemmTiling::T8x32x32_1x1
+ } else if m >= 64 && n <= 2048 {
+ GemmTiling::T64x32x32_2x2
+ } else if m >= 64 && n >= 6144 && n % 64 == 0 {
+ GemmTiling::T64x64x32_2x2
+ } else {
+ GemmTiling::T32x32x32_2x2
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
index 3dc90617f..05ecef5a5 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -1,13 +1,9 @@
-mod dispatch;
-pub(crate) mod fp;
+mod error;
mod kernel;
-pub(crate) mod quant;
+mod specialization;
-pub use dispatch::GemmSpecializationError;
-pub(crate) use dispatch::{GemmAlignmentAxes, GemmDispatch, GemmWeights};
-pub(crate) use kernel::GemmKernel;
-
-pub(crate) use crate::backends::common::gpu_types::gemm::GemmInputPrologueKind;
+pub use error::GemmSpecializationError;
+pub use kernel::{GemmDispatchPath, GemmKernel};
// Matches `MxuMmaCore::THREADGROUP_BLOCK_K` in `common/mxu_mma_core.h`.
pub(crate) const MXU_THREADGROUP_BLOCK_K: u32 = 256;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs
deleted file mode 100644
index 54617b5c1..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs
+++ /dev/null
@@ -1,96 +0,0 @@
-use super::{GemmAlignmentAxes, GemmDispatch, GemmInputPrologueKind, GemmKernel, GemmWeights};
-use crate::backends::{
- common::{
- Encoder,
- gpu_types::{
- GemmParams, QuantizationMethod,
- gemm::{GemmAlignment, GemmOutputTransformKind, GemmTilingConfig},
- },
- kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError},
- },
- metal::{Metal, context::MetalContext},
-};
-
-pub(crate) fn encode(
- gemm: &mut GemmKernel,
- context: &MetalContext,
- configuration: &QuantizedMatmulConfiguration,
- arguments: QuantizedMatmulArguments<Metal>,
- encoder: &mut Encoder<Metal>,
-) -> Result<(), QuantizedMatmulError<Metal>> {
- let tile = select_tile(configuration, arguments.batch_dim as u32);
- let group_size = configuration.group_size as u32;
- let batch_dim = arguments.batch_dim as u32;
- let input_dim = configuration.input_dim as u32;
- let output_dim = configuration.output_dim as u32;
- let threadgroups_per_row = output_dim.div_ceil(tile.threadgroup_n);
- let threadgroups_per_column = batch_dim.div_ceil(tile.threadgroup_m);
- let dispatch = GemmDispatch {
- tiling_config: tile,
- input_prologue: GemmInputPrologueKind::FullPrecision,
- use_mxu: false,
- output_transform: GemmOutputTransformKind::Store,
- alignment: GemmAlignment::from_axes(GemmAlignmentAxes {
- m: batch_dim % tile.threadgroup_m == 0,
- n: output_dim % tile.threadgroup_n == 0,
- k: input_dim % tile.threadgroup_k == 0,
- }),
- transpose_b: true,
- a: arguments.a,
- a_offset: arguments.a_offset,
- b: match configuration.quantization_method {
- QuantizationMethod::ScaleBias => GemmWeights::ScaleBias {
- weights: arguments.b,
- scales: arguments.scales,
- biases: arguments.zero_points_or_biases,
- mode: configuration.mode,
- group_size,
- },
- QuantizationMethod::ScaleZeroPoint => GemmWeights::ScaleZeroPoint {
- weights: arguments.b,
- scales: arguments.scales,
- zero_points: arguments.zero_points_or_biases,
- mode: configuration.mode,
- group_size,
- },
- },
- b_offset: 0,
- d: arguments.output,
- params: GemmParams {
- M: batch_dim,
- N: output_dim,
- K: input_dim,
- leading_dimension_a: input_dim,
- leading_dimension_b: input_dim,
- leading_dimension_d: output_dim,
- threadgroups_per_row,
- threadgroups_per_column,
- aligned_inner_iterations: input_dim / tile.threadgroup_k,
- use_morton: false,
- ab_scale: 1.0,
- },
- group_count_x: threadgroups_per_row,
- group_count_y: threadgroups_per_column,
- };
- gemm.encode(context, dispatch, encoder).map_err(QuantizedMatmulError::BackendError)
-}
-
-fn select_tile(
- configuration: &QuantizedMatmulConfiguration,
- batch_dim: u32,
-) -> GemmTilingConfig {
- let group_size = configuration.group_size as u32;
- let threadgroup_k = group_size.min(32);
- let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) = if batch_dim < 32 {
- (32u32, 32u32, 1u32, 1u32)
- } else {
- (64u32, 64u32, 2u32, 2u32)
- };
- GemmTilingConfig {
- threadgroup_m,
- threadgroup_n,
- threadgroup_k,
- simdgroups_m,
- simdgroups_n,
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
new file mode 100644
index 000000000..0c4fd79b2
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -0,0 +1,158 @@
+use super::error::GemmSpecializationError;
+use crate::{
+ DataType,
+ backends::common::{
+ gpu_types::{
+ QuantizationMethod,
+ gemm::{GemmAlignment, GemmDTransform, GemmTiling, GemmWeightPrologueKind},
+ },
+ kernel::matmul::MatmulQuantCombo,
+ },
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) struct GemmSpecialization {
+ pub(crate) data_type: DataType,
+ pub(crate) tiling: GemmTiling,
+ pub(crate) use_mxu: bool,
+ pub(crate) output_transform: GemmDTransform,
+ pub(crate) alignment: GemmAlignment,
+ pub(crate) transpose_b: bool,
+ pub(crate) weight_prologue: GemmWeightPrologueKind,
+ pub(crate) bits_per_weight: Option<u32>,
+ pub(crate) group_size: Option<u32>,
+}
+
+impl GemmSpecialization {
+ pub(crate) fn validate(&self) -> Result<(), GemmSpecializationError> {
+ if let Some(group_size) = self.group_size {
+ if self.tiling.block_k() > group_size {
+ return Err(GemmSpecializationError::ThreadgroupKExceedsGroupSize {
+ threadgroup_k: self.tiling.block_k(),
+ group_size,
+ });
+ }
+ }
+ if self.weight_prologue != GemmWeightPrologueKind::FullPrecision {
+ if self.use_mxu {
+ return Err(GemmSpecializationError::QuantizedRequiresSimdgroup);
+ }
+ if !self.transpose_b {
+ return Err(GemmSpecializationError::QuantizedRequiresTransposedB);
+ }
+ }
+ Ok(())
+ }
+
+ pub(crate) fn precompile_configs(data_type: DataType) -> Vec<Self> {
+ let mut out = Vec::new();
+ for &tiling in simdgroup_tiling_set(data_type) {
+ for align_mn in [true, false] {
+ for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {
+ out.push(Self {
+ data_type,
+ tiling,
+ use_mxu: false,
+ output_transform,
+ alignment: GemmAlignment::new(align_mn, align_mn, true),
+ transpose_b: true,
+ weight_prologue: GemmWeightPrologueKind::FullPrecision,
+ bits_per_weight: None,
+ group_size: None,
+ });
+ }
+ }
+ }
+
+ for &tiling in mxu_tiling_set(data_type) {
+ for align_m in [true, false] {
+ for align_n in [true, false] {
+ for align_k in [true, false] {
+ for output_transform in [
+ GemmDTransform::empty(),
+ GemmDTransform::SCALE,
+ GemmDTransform::ACCUMULATE,
+ GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,
+ ] {
+ out.push(Self {
+ data_type,
+ tiling,
+ use_mxu: true,
+ output_transform,
+ alignment: GemmAlignment::new(align_m, align_n, align_k),
+ transpose_b: true,
+ weight_prologue: GemmWeightPrologueKind::FullPrecision,
+ bits_per_weight: None,
+ group_size: None,
+ });
+ }
+ }
+ }
+ }
+ }
+ out
+ }
+
+ pub(crate) fn quant_combo_specs(
+ data_type: DataType,
+ combo: MatmulQuantCombo,
+ ) -> Vec<Self> {
+ let bits = DataType::from(combo.mode).size_in_bits() as u32;
+ let group_size = combo.group_size;
+ let weight_prologue = match combo.method {
+ QuantizationMethod::ScaleBias => GemmWeightPrologueKind::ScaleBiasDequant,
+ QuantizationMethod::ScaleZeroPoint => GemmWeightPrologueKind::ScaleZeroPointDequant,
+ };
+ let mut out = Vec::new();
+ for &tiling in quant_tiling_set(data_type) {
+ if tiling.block_k() > group_size {
+ continue;
+ }
+ for align_n in [true, false] {
+ for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {
+ out.push(Self {
+ data_type,
+ tiling,
+ use_mxu: false,
+ output_transform,
+ alignment: GemmAlignment::new(true, align_n, true),
+ transpose_b: true,
+ weight_prologue,
+ bits_per_weight: Some(bits),
+ group_size: Some(group_size),
+ });
+ }
+ }
+ }
+ out
+ }
+}
+
+fn simdgroup_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
+ match data_type {
+ DataType::BF16 => &[GemmTiling::T64x32x32_2x2, GemmTiling::T64x64x16_2x2],
+ DataType::F16 => &[GemmTiling::T64x64x16_2x2, GemmTiling::T64x32x32_2x2],
+ _ => &[],
+ }
+}
+
+fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
+ if !matches!(data_type, DataType::F16 | DataType::BF16) {
+ return &[];
+ }
+ &[GemmTiling::T64x64x32_2x2, GemmTiling::T32x64x32_2x2, GemmTiling::T64x32x32_4x1, GemmTiling::T128x128x32_4x4]
+}
+
+pub(crate) fn quant_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
+ match data_type {
+ DataType::BF16 => &[
+ GemmTiling::T8x32x32_1x1,
+ GemmTiling::T32x32x32_2x2,
+ GemmTiling::T64x32x32_2x2,
+ GemmTiling::T64x64x32_2x2,
+ GemmTiling::T64x64x64_2x2,
+ ],
+ DataType::F16 => &[GemmTiling::T8x32x32_1x1, GemmTiling::T32x32x32_2x2],
+ _ => &[],
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h
new file mode 100644
index 000000000..b76b4a5eb
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h
@@ -0,0 +1,163 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../common/defines.h"
+#include "../../gemm/common/quant_unpack.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <typename T, typename U, int VALUES_PER_THREAD, int BITS>
+inline U load_vector(const device T* x, thread U* x_thread) {
+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
+
+ using U4 = vec<U, 4>;
+ U sum = 0;
+ thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);
+ for (int i = 0; i < VALUES_PER_THREAD / 4; i++) {
+ U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);
+ sum += v[0] + v[1] + v[2] + v[3];
+ x_vec4[i] = v;
+ }
+ return sum;
+}
+
+template <typename T, typename U, int VALUES_PER_THREAD, int BITS>
+inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {
+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
+
+ U sum = 0;
+ for (int i = 0; i < VALUES_PER_THREAD; ++i) {
+ x_thread[i] = 0;
+ }
+ for (int i = 0; i < N; ++i) {
+ U v = x[i];
+ sum += v;
+ x_thread[i] = v;
+ }
+ return sum;
+}
+
+template <typename U, int VALUES_PER_THREAD, int BITS>
+inline void qouter(
+ const thread uint8_t* w,
+ U x,
+ U scale,
+ U bias,
+ thread U* result
+) {
+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
+
+ if (BITS == 4) {
+ for (int i = 0; i < (VALUES_PER_THREAD / 2); i++) {
+ result[2 * i] += x * (scale * uint_to_fp<U>(w[i] & 0x0fu) + bias);
+ result[2 * i + 1] +=
+ x * (scale * uint_to_fp<U>((w[i] >> 4) & 0x0fu) + bias);
+ }
+ } else if (BITS == 8) {
+ for (int i = 0; i < VALUES_PER_THREAD; i++) {
+ result[i] += x * (scale * w[i] + bias);
+ }
+ }
+}
+
+template <typename U, int VALUES_PER_THREAD, int BITS>
+inline U qdot(
+ const device uint8_t* w,
+ const thread U* x_thread,
+ U scale,
+ U bias,
+ U sum
+) {
+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
+
+ U accumulator = 0;
+ if (BITS == 4) {
+ using U4 = vec<U, 4>;
+ const device ushort* weight_words =
+ reinterpret_cast<const device ushort*>(w);
+ const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
+ for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {
+ uint weight_word = weight_words[i];
+ U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(
+ weight_word,
+ weight_word >> 4,
+ weight_word >> 8,
+ weight_word >> 12
+ ));
+ accumulator += dot(x_vec4[i], weight_vec4);
+ }
+ } else if (BITS == 8) {
+ using U4 = vec<U, 4>;
+ const device uint* weight_words = reinterpret_cast<const device uint*>(w);
+ const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
+ for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {
+ uint weight_word = weight_words[i];
+ U4 weight_vec4 = uint4_to_fp4<U, 8>(uint4(
+ weight_word,
+ weight_word >> 8,
+ weight_word >> 16,
+ weight_word >> 24
+ ));
+ accumulator += dot(x_vec4[i], weight_vec4);
+ }
+ }
+ return scale * accumulator + sum * bias;
+}
+
+template <typename U, int VALUES_PER_THREAD, int BITS>
+inline U qdot_safe(
+ const device uint8_t* w,
+ const thread U* x_thread,
+ U scale,
+ U bias,
+ U sum,
+ int N
+) {
+ static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
+
+ U accumulator = 0;
+ if (BITS == 4) {
+ using U4 = vec<U, 4>;
+ const device uint16_t* weight_words =
+ reinterpret_cast<const device uint16_t*>(w);
+ const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
+
+ int full_chunks = N / 4;
+ for (int i = 0; i < full_chunks; i++) {
+ uint16_t weight_word = weight_words[i];
+ U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(
+ weight_word,
+ weight_word >> 4,
+ weight_word >> 8,
+ weight_word >> 12
+ ));
+ accumulator += dot(x_vec4[i], weight_vec4);
+ }
+
+ int remainder = N & 3;
+ if (remainder > 0) {
+ uint16_t weight_word = weight_words[full_chunks];
+ int base_index = 4 * full_chunks;
+ accumulator += x_thread[base_index] * uint_to_fp<U>(weight_word & 0xf);
+ if (remainder > 1)
+ accumulator +=
+ x_thread[base_index + 1] * uint_to_fp<U>((weight_word >> 4) & 0xf);
+ if (remainder > 2)
+ accumulator +=
+ x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);
+ }
+ } else if (BITS == 8) {
+ for (int i = 0; i < N; i++) {
+ accumulator += x_thread[i] * w[i];
+ }
+ }
+
+ return scale * accumulator + sum * bias;
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
index 8d3baf734..ea0f28a20 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
@@ -2,16 +2,20 @@ use super::{kernel::GemvKernel, spec::GemvSpecialization};
use crate::backends::{
common::{
AsBufferRangeRef, Buffer, Encoder,
- kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError},
+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError},
},
metal::Metal,
};
-pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
kernel: &mut GemvKernel,
encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal, TB>,
+ arguments: MatmulArguments<'a, Metal, TB>,
) -> Result<(), MatmulError<Metal>> {
+ let ab_scale = arguments.d_transform.ab_scale;
+ let output_bias = arguments.d_transform.bias;
+ let is_accumulate = arguments.d_transform.accumulate;
+
let MatmulArguments {
a,
a_offset,
@@ -19,38 +23,34 @@ pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
b_offset,
b_leading_dimension,
b_transpose,
- ab_scale,
- c,
d,
- batch_dim,
- input_dim,
- output_dim,
+ m,
+ n,
+ k,
+ ..
} = arguments;
+ let weights = match b {
+ MatmulB::FullPrecision {
+ b: w,
+ } => w,
+ _ => panic!("FP gemv requires FullPrecision B"),
+ };
assert!(b_transpose, "encode_gemv does not support b_transpose=false");
assert!(b_offset == 0, "encode_gemv does not support nonzero b_offset");
- assert!(
- b_leading_dimension.is_none_or(|ld| ld == input_dim),
- "encode_gemv does not support custom b_leading_dimension"
- );
+ assert!(b_leading_dimension.is_none_or(|ld| ld == k), "encode_gemv does not support custom b_leading_dimension");
- let (is_accumulate, output_bias) = match c {
- MatmulArgumentC::Accumulate => (true, None),
- MatmulArgumentC::Bias(bias) => (false, Some(bias)),
- MatmulArgumentC::None => (false, None),
- };
-
- let specialization = GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());
+ let specialization = GemvSpecialization::select(k, n, is_accumulate, output_bias.is_some());
kernel.get_or_create(encoder.context(), specialization)?.encode(
- b,
+ weights,
(a, a_offset),
output_bias,
d,
- input_dim,
- output_dim,
- input_dim,
+ k,
+ n,
+ k,
ab_scale,
- batch_dim,
+ m,
specialization.output_rows_per_threadgroup(),
encoder,
);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
index c601f6fff..660b21222 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -14,7 +14,7 @@ using namespace metal;
#define GEMV_MAX_THREADGROUP_MEMORY 320
template <typename T>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, half, bfloat)
KERNEL(MatmulGemv)(
const device T* matrix,
const device T* input_vector,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
index 48df5aef2..ca75e02b7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -1,5 +1,7 @@
pub(crate) mod fp;
mod kernel;
+mod quant_kernel;
mod spec;
pub(crate) use kernel::GemvKernel;
+pub(crate) use quant_kernel::QuantGemvKernel;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
similarity index 98%
rename from crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
rename to crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
index c65ebc01b..e376972a9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
@@ -1,9 +1,11 @@
#include <metal_stdlib>
-#include "../common/dsl.h"
-#include "../generated/quantization_method.h"
-#include "quant_matmul.h"
+#include "../../common/dsl.h"
+#include "../../generated/quantization_method.h"
+#include "../gemm/common/quant_pack.h"
+#include "common/qmv_ops.h"
using namespace uzu::quantization_method;
+using namespace uzu::gemm;
template <typename T, uint GROUP_SIZE, uint BITS>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
similarity index 96%
rename from crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
rename to crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
index fe1a5e818..a27f1591c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
@@ -1,10 +1,12 @@
#include <metal_stdlib>
-#include "../common/dsl.h"
-#include "../hadamard_transform/hadamard_transform.h"
-#include "../generated/quantization_method.h"
-#include "quant_matmul.h"
+#include "../../common/dsl.h"
+#include "../../hadamard_transform/hadamard_transform.h"
+#include "../../generated/quantization_method.h"
+#include "../gemm/common/quant_pack.h"
+#include "common/qmv_ops.h"
using namespace uzu::quantization_method;
+using namespace uzu::gemm;
template <typename T, uint GROUP_SIZE, uint BITS>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
new file mode 100644
index 000000000..322c016b4
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
@@ -0,0 +1,173 @@
+use std::collections::{HashMap, hash_map::Entry};
+
+use crate::{
+ DataType,
+ backends::{
+ common::{
+ AsBufferRangeRef, Buffer, Encoder,
+ gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},
+ kernel::{
+ Kernels, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,
+ matmul::{MatmulArguments, MatmulB, MatmulError},
+ },
+ },
+ metal::{Metal, context::MetalContext},
+ },
+};
+
+const PATH: &str = "QuantGemv";
+
+#[derive(PartialEq, Eq, Hash, Clone, Copy)]
+struct QmvKey {
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+ use_hadamard: bool,
+}
+
+pub(crate) struct QuantGemvKernel {
+ data_type: DataType,
+ qmv: HashMap<QmvKey, <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel>,
+ qmv_fast: HashMap<
+ QmvKey,
+ <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel,
+ >,
+}
+
+impl QuantGemvKernel {
+ pub(crate) fn new(
+ _context: &MetalContext,
+ data_type: DataType,
+ ) -> Self {
+ Self {
+ data_type,
+ qmv: HashMap::new(),
+ qmv_fast: HashMap::new(),
+ }
+ }
+
+ pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+ &mut self,
+ encoder: &mut Encoder<Metal>,
+ arguments: MatmulArguments<'a, Metal, TB>,
+ ) -> Result<(), MatmulError<Metal>> {
+ let mask = arguments.d_transform.mask();
+ if mask.contains(GemmDTransform::SCALE) {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::SCALE,
+ path: PATH,
+ });
+ }
+ if mask.contains(GemmDTransform::ACCUMULATE) {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::ACCUMULATE,
+ path: PATH,
+ });
+ }
+ if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {
+ return Err(MatmulError::UnsupportedLayout {
+ path: PATH,
+ });
+ }
+
+ let hadamard_factors = arguments.d_transform.rht_factors;
+
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ d,
+ m,
+ n,
+ k,
+ ..
+ } = arguments;
+ let (weights, scales, zp_or_bias, method, mode, group_size) = match b {
+ MatmulB::ScaleBiasDequant {
+ b: w,
+ scales,
+ biases,
+ mode,
+ group_size,
+ } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),
+ MatmulB::ScaleZeroPointDequant {
+ b: w,
+ scales,
+ zero_points,
+ mode,
+ group_size,
+ } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ MatmulB::FullPrecision {
+ ..
+ } => panic!("QuantGemvKernel requires quantized B"),
+ };
+
+ let bits = match mode {
+ QuantizationMode::U4 => 4u32,
+ QuantizationMode::I8 | QuantizationMode::U8 => 8u32,
+ };
+ let use_fast = n % 8 == 0 && k % 512 == 0;
+
+ let (zero_points, biases) = match method {
+ QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias), None),
+ QuantizationMethod::ScaleBias => (None, Some(zp_or_bias)),
+ };
+
+ if use_fast {
+ let key = QmvKey {
+ group_size,
+ bits,
+ quant_method: method,
+ use_hadamard: hadamard_factors.is_some(),
+ };
+ let context = encoder.context();
+ let kernel = match self.qmv_fast.entry(key) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => {
+ let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ context,
+ self.data_type,
+ group_size,
+ bits,
+ method,
+ hadamard_factors.is_some(),
+ )
+ .map_err(MatmulError::BackendError)?;
+ entry.insert(kernel)
+ },
+ };
+ kernel.encode(weights, scales, zero_points, biases, (a, a_offset), d, hadamard_factors, k, n, m, encoder);
+ } else {
+ if hadamard_factors.is_some() {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::RHT,
+ path: PATH,
+ });
+ }
+ let key = QmvKey {
+ group_size,
+ bits,
+ quant_method: method,
+ use_hadamard: false,
+ };
+ let context = encoder.context();
+ let kernel = match self.qmv.entry(key) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => {
+ let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
+ context,
+ self.data_type,
+ group_size,
+ bits,
+ method,
+ )
+ .map_err(MatmulError::BackendError)?;
+ entry.insert(kernel)
+ },
+ };
+ kernel.encode(weights, scales, zero_points, biases, (a, a_offset), d, k, n, m, encoder);
+ }
+
+ Ok(())
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
index 4e83e73cd..343189ea4 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
@@ -66,16 +66,6 @@ impl GemvSpecialization {
is_accumulate: false,
is_bias: false,
}],
- DataType::F32 => &[Self {
- threadgroup_rows: 8,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- is_accumulate: false,
- is_bias: false,
- }],
_ => &[],
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 8f803c24a..7756cc248 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,29 +1,29 @@
pub mod gemm;
pub mod gemv;
-pub mod quant;
use std::sync::OnceLock;
-use self::{gemm::GemmKernel, gemv::GemvKernel};
+pub use self::gemm::{GemmDispatchPath, GemmKernel};
+use self::gemv::{GemvKernel, QuantGemvKernel};
use crate::{
DataType,
backends::{
common::{
- AsBufferRangeRef, Buffer, Encoder,
+ AsBufferRangeRef, Backend, Buffer, Encoder,
+ gpu_types::gemm::GemmDTransform,
kernel::{
- TensorAddBiasKernel,
- matmul::{MatmulArguments, MatmulError, MatmulKernel},
+ HadamardTransformKernel, TensorAddBiasKernel,
+ matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel, MatmulQuantCombo},
},
},
- metal::{Metal, context::MetalContext, kernel::TensorAddBiasMetalKernel, metal_extensions::DeviceExt},
+ metal::{Metal, context::MetalContext, error::MetalError},
},
};
pub struct MatmulMetalKernel {
- data_type: DataType,
gemv: GemvKernel,
- pub(crate) gemm: GemmKernel,
- pub(crate) bias_add: TensorAddBiasMetalKernel,
+ quant_gemv: QuantGemvKernel,
+ pub gemm: GemmKernel,
}
const DEFAULT_GEMV_MAX_BATCH: u32 = 8;
@@ -36,45 +36,102 @@ fn max_gemv_batch_threshold() -> u32 {
}
impl MatmulMetalKernel {
- fn is_mxu_eligible(
- &self,
- context: &MetalContext,
- ) -> bool {
- context.device.supports_mxu() && matches!(self.data_type, DataType::F16 | DataType::BF16)
- }
-}
+ fn dispatch_gemv<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+ &mut self,
+ arguments: MatmulArguments<'a, Metal, TB>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), MatmulError<Metal>> {
+ let post_rht = arguments.d_transform.rht_factors;
-/// Explicit dispatch paths for testing individual kernels independent of production routing.
-#[derive(Debug, Clone, Copy)]
-pub enum MatmulDispatchPath {
- Auto,
- Gemv,
- Gemm,
- GemmMxu,
-}
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ d,
+ d_transform,
+ m,
+ n,
+ k,
+ } = arguments;
-impl MatmulMetalKernel {
- pub fn encode_with_path(
+ // A temopary hack, will port the post ops to the gemv later.
+ let d_transform = d_transform.without(GemmDTransform::RHT);
+
+ gemv::fp::encode(
+ &mut self.gemv,
+ encoder,
+ MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ d: &mut *d,
+ d_transform,
+ m,
+ n,
+ k,
+ },
+ )?;
+
+ if let Some(factors) = post_rht {
+ self.gemm.hadamard.encode(d, factors, n, m, encoder);
+ }
+ Ok(())
+ }
+
+ fn dispatch_quant_gemv<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
&mut self,
- arguments: MatmulArguments<Metal>,
+ arguments: MatmulArguments<'a, Metal, TB>,
encoder: &mut Encoder<Metal>,
- path: MatmulDispatchPath,
- ) {
- let context = encoder.context();
- match path {
- MatmulDispatchPath::Auto => self.encode(arguments, encoder),
- MatmulDispatchPath::Gemv => {
- gemv::fp::encode(&mut self.gemv, encoder, arguments).expect("Failed to encode GEMV")
- },
- MatmulDispatchPath::Gemm => {
- gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, false)
- .expect("Failed to encode Gemm")
- },
- MatmulDispatchPath::GemmMxu => {
- gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, true)
- .expect("Failed to encode GemmMxu")
+ ) -> Result<(), MatmulError<Metal>> {
+ if arguments.b_offset != 0 {
+ return Err(MatmulError::UnsupportedLayout {
+ path: "QuantGemv",
+ });
+ }
+ let post_bias = arguments.d_transform.bias;
+
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ d,
+ d_transform,
+ m,
+ n,
+ k,
+ } = arguments;
+ let d_transform = d_transform.without(GemmDTransform::BIAS);
+
+ self.quant_gemv.encode(
+ encoder,
+ MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ d: &mut *d,
+ d_transform,
+ m,
+ n,
+ k,
},
+ )?;
+
+ if let Some(bias) = post_bias {
+ self.gemm.bias_add.encode(None::<&<Metal as Backend>::DenseBuffer>, bias, d, n, m * n, encoder);
}
+ Ok(())
}
}
@@ -84,20 +141,19 @@ impl MatmulKernel for MatmulMetalKernel {
fn new(
context: &MetalContext,
data_type: DataType,
- ) -> Result<Self, MatmulError<Metal>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
+ ) -> Result<Self, MetalError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16) {
+ return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());
}
- let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true).map_err(MatmulError::BackendError)?;
- let gemm = GemmKernel::new(context, data_type).map_err(MatmulError::BackendError)?;
- let gemv = GemvKernel::new(context, data_type)?;
+ let gemm = GemmKernel::new(context, data_type)?;
+ let gemv = GemvKernel::new(context, data_type).map_err(MetalError::from)?;
+ let quant_gemv = QuantGemvKernel::new(context, data_type);
Ok(Self {
- data_type,
gemv,
+ quant_gemv,
gemm,
- bias_add,
})
}
@@ -105,21 +161,33 @@ impl MatmulKernel for MatmulMetalKernel {
&mut self,
arguments: MatmulArguments<Metal, TB>,
encoder: &mut Encoder<Metal>,
- ) {
- // gemv only handles the canonical B layout; non-canonical inputs go to GEMM.
- let context = encoder.context();
- let gemv_eligible = arguments.b_transpose
- && arguments.b_offset == 0
- && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.input_dim)
- && arguments.batch_dim <= max_gemv_batch_threshold();
+ ) -> Result<(), MetalError> {
+ let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });
+ let gemv_eligible = if is_quant {
+ arguments.m < 5 || arguments.n == 1
+ } else {
+ arguments.b_transpose
+ && arguments.b_offset == 0
+ && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.k)
+ && arguments.m <= max_gemv_batch_threshold()
+ };
if gemv_eligible {
- gemv::fp::encode(&mut self.gemv, encoder, arguments).expect("Failed to encode GEMV kernel");
- return;
+ if is_quant {
+ self.dispatch_quant_gemv(arguments, encoder).map_err(MetalError::from)
+ } else {
+ self.dispatch_gemv(arguments, encoder).map_err(MetalError::from)
+ }
+ } else {
+ self.gemm.encode(arguments, encoder)
}
+ }
- let use_mxu = self.is_mxu_eligible(context);
- gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, use_mxu)
- .expect("Failed to encode GEMM");
+ fn preheat_quant_combo(
+ &mut self,
+ context: &MetalContext,
+ combo: MatmulQuantCombo,
+ ) -> Result<(), MetalError> {
+ self.gemm.preheat_quant_combo(context, combo)
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs
deleted file mode 100644
index 6ccbbcd74..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-use super::{MatmulMetalKernel, gemm};
-use crate::backends::{
- common::{
- Encoder,
- kernel::quant_matmul::{
- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
- QuantizedMatmulKernelEncodable,
- },
- },
- metal::{Metal, context::MetalContext},
-};
-
-#[derive(Debug, Clone, Copy)]
-pub enum QuantizedMatmulDispatchPath {
- Auto,
- Gemm,
-}
-
-pub fn encode_quantized_matmul_with_path(
- context: &MetalContext,
- matmul: &mut MatmulMetalKernel,
- encodable: &QuantizedMatmulKernelEncodable<Metal>,
- configuration: &QuantizedMatmulConfiguration,
- arguments: QuantizedMatmulArguments<Metal>,
- encoder: &mut Encoder<Metal>,
- path: QuantizedMatmulDispatchPath,
-) -> Result<(), QuantizedMatmulError<Metal>> {
- match path {
- QuantizedMatmulDispatchPath::Auto => {
- encodable.encode(encoder, arguments);
- Ok(())
- },
- QuantizedMatmulDispatchPath::Gemm => {
- gemm::quant::encode(&mut matmul.gemm, context, configuration, arguments, encoder)
- },
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rs
index 916078500..e03c9e43c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs
@@ -1,6 +1,9 @@
pub mod matmul;
-use crate::backends::common::kernel::ManualKernels;
+use crate::backends::common::{
+ gpu_types::gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_row},
+ kernel::ManualKernels,
+};
include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index c6b8cffae..bccc6aab4 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -1,8 +1,8 @@
#include <metal_stdlib>
#include "../activation/activations.h"
+#include "../common/defines.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
-#include "../quant_matmul/mma.h"
// ------------------------ helpers ------------------------
static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }
diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h
deleted file mode 100644
index f86a61e25..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h
+++ /dev/null
@@ -1,536 +0,0 @@
-#pragma once
-
-#include "../common/defines.h"
-#include <metal_simdgroup>
-#include <metal_simdgroup_matrix>
-#include <metal_stdlib>
-
-using namespace metal;
-
-namespace matmul_utils {
-
-template <typename T>
-struct pointer_element_t_impl {
- using type = T;
-};
-template <typename T>
-struct pointer_element_t_impl<device T*> {
- using type = T;
-};
-template <typename T>
-struct pointer_element_t_impl<threadgroup T*> {
- using type = T;
-};
-template <typename T>
-
-using pointer_element_t = typename pointer_element_t_impl<T>::type;
-
-template <typename T, typename U = T>
-struct TransformNone {
- METAL_FUNC static U apply(U val) { return val; }
-};
-
-template <typename T, int FRAG_ROWS_, int FRAG_COLS_>
-struct BaseMMAFrag {
- static_assert(
- FRAG_ROWS_ == 8,
- "Only 8 x 8 fragment matrices are currently supported"
- );
- static_assert(
- FRAG_COLS_ == 8,
- "Only 8 x 8 fragment matrices are currently supported"
- );
-};
-
-template <typename T>
-struct BaseMMAFrag<T, 8, 8> {
- METAL_CONST int FRAG_ROWS = 8;
- METAL_CONST int FRAG_COLS = 8;
- METAL_CONST int ELEMS_PER_FRAG = (FRAG_ROWS * FRAG_COLS) / 32;
- METAL_CONST int ELEM_ROWS = 1;
- METAL_CONST int ELEM_COLS = 2;
-
- static_assert(
- ELEM_ROWS * ELEM_COLS == ELEMS_PER_FRAG,
- "MMAFrag shape is not consistent with MMAFrag size"
- );
-
- typedef metal::simdgroup_matrix<T, FRAG_ROWS, FRAG_COLS> mat_type;
- typedef metal::vec<T, ELEMS_PER_FRAG> frag_type;
-
- METAL_FUNC static constexpr short2 get_coord(
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- ) {
- const short qid = simd_lane_id / 4;
- const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);
- const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;
- return short2{fn, fm};
- }
-
- template <typename SrcPtrType, typename StrX, typename StrY>
- METAL_FUNC static constexpr void load(
- thread frag_type& dst,
- SrcPtrType src,
- StrX str_x,
- StrY str_y
- ) {
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < ELEM_ROWS; i++) {
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < ELEM_COLS; j++) {
- dst[i * ELEM_COLS + j] = static_cast<T>(src[i * str_x + j * str_y]);
- }
- }
- }
-
- template <typename DstPtrType, typename StrX, typename StrY>
- METAL_FUNC static constexpr void store(
- const thread frag_type& src,
- DstPtrType dst,
- StrX str_x,
- StrY str_y
- ) {
- using U = pointer_element_t<DstPtrType>;
-
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < ELEM_ROWS; i++) {
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < ELEM_COLS; j++) {
- dst[i * str_x + j * str_y] = static_cast<U>(src[i * ELEM_COLS + j]);
- }
- }
- }
-
- template <
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX,
- typename OffY>
- METAL_FUNC static constexpr void store_safe(
- const thread frag_type& src,
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = 0,
- OffY off_y = 0
- ) {
- using U = pointer_element_t<DstPtrType>;
-
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < ELEM_ROWS; i++) {
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < ELEM_COLS; j++) {
- if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
- dst[(off_x + i) * str_x + (off_y + j) * str_y] =
- static_cast<U>(src[i * ELEM_COLS + j]);
- }
- }
- }
- }
-
- METAL_FUNC static constexpr void mma(
- thread frag_type& D,
- thread frag_type& A,
- thread frag_type& B,
- thread frag_type& C
- ) {
- mat_type D_mat;
- mat_type A_mat;
- mat_type B_mat;
- mat_type C_mat;
-
- reinterpret_cast<thread frag_type&>(A_mat.thread_elements()) = A;
- reinterpret_cast<thread frag_type&>(B_mat.thread_elements()) = B;
- reinterpret_cast<thread frag_type&>(C_mat.thread_elements()) = C;
-
- mma(D_mat, A_mat, B_mat, C_mat);
-
- D = reinterpret_cast<thread frag_type&>(D_mat.thread_elements());
- }
-
- METAL_FUNC static constexpr void mma(
- thread mat_type& D,
- thread mat_type& A,
- thread mat_type& B,
- thread mat_type& C
- ) {
- simdgroup_multiply_accumulate(D, A, B, C);
- }
-};
-
-template <
- typename T,
- int TILE_ROWS_,
- int TILE_COLS_,
- class MMAFrag_ = BaseMMAFrag<T, 8, 8>>
-struct MMATile {
- using MMAFrag_t = MMAFrag_;
- using elem_type = T;
- METAL_CONST int FRAG_ROWS = MMAFrag_t::FRAG_ROWS;
- METAL_CONST int FRAG_COLS = MMAFrag_t::FRAG_COLS;
- METAL_CONST int ELEMS_PER_FRAG = MMAFrag_t::ELEMS_PER_FRAG;
-
- METAL_CONST int TILE_ROWS = TILE_ROWS_;
- METAL_CONST int TILE_COLS = TILE_COLS_;
-
- METAL_CONST int ROWS = TILE_ROWS * FRAG_ROWS;
- METAL_CONST int COLS = TILE_COLS * FRAG_COLS;
-
- METAL_CONST int NUM_FRAGS = TILE_ROWS * TILE_COLS;
- METAL_CONST int ELEMS_PER_TILE = NUM_FRAGS * ELEMS_PER_FRAG;
-
- typedef typename MMAFrag_t::mat_type mat_type;
- typedef typename MMAFrag_t::frag_type frag_type;
-
- frag_type val_frags[NUM_FRAGS] = {frag_type(0)};
-
- METAL_FUNC MMATile() thread {}
-
- METAL_FUNC constexpr void clear() {
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < NUM_FRAGS; ++i) {
- val_frags[i] = frag_type(0);
- }
- }
-
- METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {
- return val_frags[i * TILE_COLS + j];
- }
-
- METAL_FUNC constexpr const thread frag_type& frag_at(
- const short i,
- const short j
- ) const {
- return val_frags[i * TILE_COLS + j];
- }
-
- METAL_FUNC thread elem_type* elems() {
- return reinterpret_cast<thread elem_type*>(val_frags);
- }
-
- METAL_FUNC const thread elem_type* elems() const {
- return reinterpret_cast<const thread elem_type*>(val_frags);
- }
-
- template <typename U, int w_x, int w_y, int str_x, int str_y>
- METAL_FUNC void load(const threadgroup U* src) {
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < TILE_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < TILE_COLS; ++j) {
- MMAFrag_t::load(
- frag_at(i, j),
- &(
- src[(i * FRAG_ROWS) * w_x * str_x +
- (j * FRAG_COLS) * w_y * str_y]
- ),
- str_x,
- str_y
- );
- }
- }
- }
-
- template <typename U, int w_x, int w_y>
- METAL_FUNC void store(device U* dst, const int ld) const {
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < TILE_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < TILE_COLS; ++j) {
- MMAFrag_t::store(
- frag_at(i, j),
- &(dst[(i * FRAG_ROWS) * w_x * ld + (j * FRAG_COLS) * w_y]),
- ld,
- 1
- );
- }
- }
- }
-
- // Store results into threadgroup memory (same layout as device variant)
- template <typename U, int w_x, int w_y>
- METAL_FUNC void store_tg(threadgroup U* dst, const int ld) const {
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < TILE_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < TILE_COLS; ++j) {
- MMAFrag_t::store(
- frag_at(i, j),
- &(dst[(i * FRAG_ROWS) * w_x * ld + (j * FRAG_COLS) * w_y]),
- ld,
- 1
- );
- }
- }
- }
-
- template <typename U, int w_x, int w_y>
- METAL_FUNC void store_safe(
- device U* dst,
- const int ld,
- const short2 dst_tile_dims
- ) const {
- METAL_PRAGMA_UNROLL
- for (int i = 0; i < TILE_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (int j = 0; j < TILE_COLS; ++j) {
- MMAFrag_t::store_safe(
- frag_at(i, j),
- dst,
- ld,
- 1,
- dst_tile_dims.y,
- dst_tile_dims.x,
- (i * FRAG_ROWS) * w_x,
- (j * FRAG_COLS) * w_y
- );
- }
- }
- }
-};
-
-template <typename T, typename U, int M, int N, int K>
-METAL_FUNC void tile_matmad(
- thread MMATile<T, M, N>& D,
- thread MMATile<U, M, K>& A,
- thread MMATile<U, K, N>& B,
- thread MMATile<T, M, N>& C
-) {
- METAL_PRAGMA_UNROLL
- for (short m = 0; m < M; ++m) {
- METAL_PRAGMA_UNROLL
- for (short n = 0; n < N; ++n) {
- short n_serp = (m % 2) ? (N - 1 - n) : n;
- METAL_PRAGMA_UNROLL
- for (short k = 0; k < K; ++k) {
- MMATile<T, M, N>::MMAFrag_t::mma(
- D.frag_at(m, n_serp),
- A.frag_at(m, k),
- B.frag_at(k, n_serp),
- C.frag_at(m, n_serp)
- );
- }
- }
- }
-}
-
-template <
- typename T,
- typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- short lda_tgp,
- short ldb_tgp,
- typename AccumType = float,
- typename Epilogue = TransformNone<U, AccumType>>
-struct BlockMMA {
- // MMAFrag size
- METAL_CONST short FRAG_SIZE = 8;
- using MMAFrag_acc_t = BaseMMAFrag<AccumType, FRAG_SIZE, FRAG_SIZE>;
-
- // Warp tile simdgroup matrix strides along M
- METAL_CONST short TM_stride = FRAG_SIZE * WM;
- // Warp tile simdgroup matrix strides along M
- METAL_CONST short TN_stride = FRAG_SIZE * WN;
-
- // Warp tile size along M
- METAL_CONST short TM = BM / (FRAG_SIZE * WM);
- // Warp tile size along N
- METAL_CONST short TN = BN / (FRAG_SIZE * WN);
-
- // Threadgroup A strides
- METAL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
- METAL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K
-
- // Threadgroup B strides
- METAL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K
- METAL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
-
- // Threadgroup strides along K
- METAL_CONST short tile_stride_a = FRAG_SIZE * A_str_k;
- METAL_CONST short tile_stride_b = FRAG_SIZE * B_str_k;
-
- // Simdgroup matrices
- MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;
- MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;
- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;
-
- // Offsets within threadgroup
- short sm;
- short sn;
-
- short As_offset;
- short Bs_offset;
-
- /* Constructor */
- METAL_FUNC BlockMMA(
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- ) {
- // Determine thread position in simdgroup matrix
- short tm = FRAG_SIZE * (simd_group_id / WN);
- short tn = FRAG_SIZE * (simd_group_id % WN);
-
- short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);
- sm = simd_coord.y;
- sn = simd_coord.x;
-
- // Determine thread and simdgroup offset
- As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K
- Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N
-
- sm += tm;
- sn += tn;
- }
-
- /* (BM, BK) X (BK, BN) multiply accumulate function */
- METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {
- // Adjust for simdgroup and thread location
- As += As_offset;
- Bs += Bs_offset;
-
- // Iterate over BK in blocks of FRAG_SIZE
- METAL_PRAGMA_UNROLL
- for (short kk = 0; kk < BK; kk += FRAG_SIZE) {
- simdgroup_barrier(mem_flags::mem_none);
-
- Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);
-
- simdgroup_barrier(mem_flags::mem_none);
-
- Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);
-
- simdgroup_barrier(mem_flags::mem_none);
-
- tile_matmad(Ctile, Atile, Btile, Ctile);
-
- // Progress to next simdgroup tile
- As += tile_stride_a;
- Bs += tile_stride_b;
- }
- }
-
- /* Store results from simdgroup_matrix results into device memory */
- METAL_FUNC void store_result(device U* D, const int ldd) {
- // Apply epilogue
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {
- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
- }
-
- // Adjust for simdgroup and thread location
- D += sm * ldd + sn;
-
- Ctile.template store<U, WM, WN>(D, ldd);
- }
-
- /* Store results into threadgroup memory */
- METAL_FUNC void store_result_tg(threadgroup U* D, const int ldd) {
- // Apply epilogue
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {
- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
- }
-
- // Adjust for simdgroup and thread location
- D += sm * ldd + sn;
-
- Ctile.template store_tg<U, WM, WN>(D, ldd);
- }
-
- METAL_FUNC void store_result_safe(
- device U* D,
- const int ldd,
- short2 dst_tile_dims
- ) {
- // Apply epilogue
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {
- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
- }
-
- // Adjust for simdgroup and thread location
- D += sm * ldd + sn;
- dst_tile_dims -= short2(sn, sm);
-
- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
- return;
-
- Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);
- }
-};
-
-template <
- typename T,
- int BM,
- int BK,
- int lda_tgp,
- int reduction_dim,
- int tgp_size,
- int align_M = 1,
- int align_K = 4>
-struct BlockLoader {
- METAL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;
-
- const int src_ld;
- const short thread_idx;
- const short bi;
- const short bj;
-
- threadgroup T* dst;
- const device T* src;
-
- BlockLoader(
- const device T* src_,
- const int src_ld_,
- threadgroup T* dst_,
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- )
- : src_ld(src_ld_), thread_idx(simd_group_id * 32 + simd_lane_id),
- bi(n_reads * thread_idx / BK), bj((n_reads * thread_idx) % BK),
- dst(dst_ + bi * lda_tgp + bj), src(src_ + bi * src_ld + bj) {}
-
- void load_unsafe() const {
- if (BM * BK < tgp_size && bi >= BM) {
- return;
- }
-
- for (int i = 0; i < n_reads; i++) {
- dst[i] = src[i];
- }
- }
-
- void load_safe(short2 src_tile_dim) const {
- if (BM * BK < tgp_size && bi >= BM) {
- return;
- }
-
- if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {
- for (int i = 0; i < n_reads; i++) {
- dst[i] = T(0);
- }
- return;
- }
-
- for (int i = 0; i < n_reads; i++) {
- dst[i] = src[i];
- }
- }
-
- void next() {
- src += BK; // Advance by tile size
- }
-};
-
-} // namespace matmul_utils
\ No newline at end of file
diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
deleted file mode 100644
index 322ac3708..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ /dev/null
@@ -1,194 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-#include "../generated/quantization_method.h"
-#include "../hadamard_transform/hadamard_transform.h"
-#include "quant_matmul.h"
-
-using namespace uzu::quantization_method;
-
-// Transposed QMM kernel. BM = batch tile rows, BK = K-axis tile, BN = output
-// tile cols. WM, WN = simdgroup tile counts along M and N axes.
-template <typename T, uint GROUP_SIZE, uint BITS, uint BM, uint BK, uint BN, uint WM, uint WN>
-VARIANTS(T, float, half, bfloat)
-VARIANTS(GROUP_SIZE, 32, 64, 128)
-VARIANTS(BITS, 4, 8)
-VARIANTS(BM, 8, 32, 64)
-VARIANTS(BK, 32, 64)
-VARIANTS(BN, 32, 64)
-VARIANTS(WM, 1, 2)
-VARIANTS(WN, 1, 2)
-CONSTRAINT(
- (BM == 8 && BK == 32 && BN == 32 && WM == 1 && WN == 1) ||
- (BM == 32 && BK == 32 && BN == 32 && WM == 2 && WN == 2) ||
- (BM == 32 && BK == 64 && BN == 32 && WM == 2 && WN == 2) ||
- (BM == 64 && BK == 32 && BN == 64 && WM == 2 && WN == 2) ||
- (BM == 64 && BK == 64 && BN == 64 && WM == 2 && WN == 2))
-CONSTRAINT(BK <= GROUP_SIZE)
-CONSTRAINT(T != "float" || BK < 64)
-PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
- const device uint32_t* weights,
- const device T* scales,
- const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),
- const device T* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),
- const device T* input,
- device T* output,
- const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
- const constant uint& in_vec_size,
- const constant uint& out_vec_size,
- const constant uint& batch_size,
- threadgroup T Xs[BM * (BK + 16 / sizeof(T))],
- threadgroup T Ws[BN * (BK + 16 / sizeof(T))],
- const QuantizationMethod quant_method SPECIALIZE,
- const bool use_hadamard SPECIALIZE,
- const bool aligned_n SPECIALIZE,
- const uint out_block_idx GROUPS(out_vec_size.div_ceil(BN)),
- const uint batch_block_idx GROUPS(batch_size.div_ceil(BM)),
- const uint simd_lane THREADS(32),
- const uint simd_group THREADS(WM * WN)
-) {
- if (quant_method == QuantizationMethod::ScaleBias) {
- if (aligned_n) {
- qmm_transposed_impl<
- T,
- GROUP_SIZE,
- BITS,
- true,
- BM,
- BK,
- BN,
- QuantizationMethod::ScaleBias,
- WM,
- WN>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- Xs,
- Ws,
- in_vec_size,
- out_vec_size,
- batch_size,
- out_block_idx,
- batch_block_idx,
- simd_group,
- simd_lane
- );
- } else {
- qmm_transposed_impl<
- T,
- GROUP_SIZE,
- BITS,
- false,
- BM,
- BK,
- BN,
- QuantizationMethod::ScaleBias,
- WM,
- WN>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- Xs,
- Ws,
- in_vec_size,
- out_vec_size,
- batch_size,
- out_block_idx,
- batch_block_idx,
- simd_group,
- simd_lane
- );
- }
- } else {
- if (aligned_n) {
- qmm_transposed_impl<
- T,
- GROUP_SIZE,
- BITS,
- true,
- BM,
- BK,
- BN,
- QuantizationMethod::ScaleZeroPoint,
- WM,
- WN>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- Xs,
- Ws,
- in_vec_size,
- out_vec_size,
- batch_size,
- out_block_idx,
- batch_block_idx,
- simd_group,
- simd_lane
- );
- } else {
- qmm_transposed_impl<
- T,
- GROUP_SIZE,
- BITS,
- false,
- BM,
- BK,
- BN,
- QuantizationMethod::ScaleZeroPoint,
- WM,
- WN>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- Xs,
- Ws,
- in_vec_size,
- out_vec_size,
- batch_size,
- out_block_idx,
- batch_block_idx,
- simd_group,
- simd_lane
- );
- }
- }
-
- if (use_hadamard) {
- threadgroup_barrier(mem_flags::mem_device);
-
- // Each lane covers BN/32 output columns (1 at BN=32, 2 at BN=64).
- constexpr uint NUM_SIMD_GROUPS = WM * WN;
- constexpr uint ROWS_PER_SIMD_GROUP = BM / NUM_SIMD_GROUPS;
- constexpr uint COLS_PER_LANE = BN / 32;
-
- const uint row_start =
- batch_block_idx * BM + simd_group * ROWS_PER_SIMD_GROUP;
- const uint row_end = min(row_start + ROWS_PER_SIMD_GROUP, batch_size);
-
- for (uint c = 0; c < COLS_PER_LANE; c++) {
- const uint out_col = out_block_idx * BN + c * 32 + simd_lane;
- const int32_t factor = hadamard_factors[out_col];
-
- for (uint row = row_start; row < row_end; row++) {
- uint output_index = row * out_vec_size + out_col;
-
- output[output_index] = simdgroup_random_hadamard_transform(
- static_cast<ushort>(simd_lane),
- output[output_index],
- factor
- );
- }
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
deleted file mode 100644
index c518d59b4..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
+++ /dev/null
@@ -1,842 +0,0 @@
-#pragma once
-
-#include <metal_simdgroup>
-#include <metal_stdlib>
-
-using namespace metal;
-
-#include "../generated/quantization_method.h"
-#include "mma.h"
-
-using QuantizationMethod = uzu::quantization_method::QuantizationMethod;
-
-// Avoids air.convert (SFU) for int → float which is slower on Apple GPU.
-template <typename U>
-METAL_FUNC U uint_to_fp(uint32_t x) {
- return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);
-}
-
-template <>
-METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {
- return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);
-}
-
-// Unpack 4 lanes of `BITS`-wide packed uints (low bits of each uint) into
-// vec<U, 4> of floats. `BITS` in [1, 23] to fit the float23 mantissa trick.
-template <int BITS>
-METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {
- static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");
- constexpr uint mask = (1u << BITS) - 1u;
- n &= uint4(mask);
- return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);
-}
-
-template <typename U, int BITS>
-METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);
-
-template <>
-METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {
- return _uint4_to_fp4_float<4>(n);
-}
-
-template <>
-METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {
- return _uint4_to_fp4_float<8>(n);
-}
-
-template <>
-METAL_FUNC half4 uint4_to_fp4<half, 4>(uint4 n) {
- return half4(_uint4_to_fp4_float<4>(n));
-}
-
-template <>
-METAL_FUNC half4 uint4_to_fp4<half, 8>(uint4 n) {
- return half4(_uint4_to_fp4_float<8>(n));
-}
-
-template <>
-METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {
- return bfloat4(_uint4_to_fp4_float<4>(n));
-}
-
-template <>
-METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {
- return bfloat4(_uint4_to_fp4_float<8>(n));
-}
-
-template <int bits, int wsize = 8>
-inline constexpr short get_pack_factor() {
- return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : wsize / bits);
-}
-
-template <int bits, int wsize = 8>
-inline constexpr short get_bytes_per_pack() {
- constexpr int power_of_2_bits = (bits & (bits - 1)) == 0;
- return power_of_2_bits ? (wsize / 8) : (bits == 5 ? 5 : 3);
-}
-
-template <typename T, typename U, int values_per_thread, int bits>
-inline U load_vector(const device T* x, thread U* x_thread) {
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- using U4 = vec<U, 4>;
- U sum = 0;
- thread U4* x4 = (thread U4*)x_thread;
- for (int i = 0; i < values_per_thread / 4; i++) {
- U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);
- sum += v[0] + v[1] + v[2] + v[3];
- x4[i] = v;
- }
- return sum;
-}
-
-template <typename T, typename U, int values_per_thread, int bits>
-inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- U sum = 0;
- for (int i = 0; i < values_per_thread; ++i) {
- x_thread[i] = 0;
- }
- for (int i = 0; i < N; ++i) {
- U v = x[i];
- sum += v;
- x_thread[i] = v;
- }
- return sum;
-}
-
-template <typename U, int values_per_thread, int bits>
-inline void qouter(
- const thread uint8_t* w,
- U x,
- U scale,
- U bias,
- thread U* result
-) {
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- if (bits == 4) {
- for (int i = 0; i < (values_per_thread / 2); i++) {
- result[2 * i] += x * (scale * uint_to_fp<U>(w[i] & 0x0fu) + bias);
- result[2 * i + 1] +=
- x * (scale * uint_to_fp<U>((w[i] >> 4) & 0x0fu) + bias);
- }
- } else if (bits == 8) {
- for (int i = 0; i < values_per_thread; i++) {
- result[i] += x * (scale * w[i] + bias);
- }
- }
-}
-
-template <typename U, int values_per_thread, int bits>
-inline U qdot(
- const device uint8_t* w,
- const thread U* x_thread,
- U scale,
- U bias,
- U sum
-) {
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- U accum = 0;
- if (bits == 4) {
- using U4 = vec<U, 4>;
- const device ushort* ws = (const device ushort*)w;
- const thread U4* x4 = (const thread U4*)x_thread;
- for (int i = 0; i < (values_per_thread / 4); i++) {
- uint wi = ws[i];
- U4 w_vec = uint4_to_fp4<U, 4>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));
- accum += dot(x4[i], w_vec);
- }
- } else if (bits == 8) {
- using U4 = vec<U, 4>;
- const device uint* ws = (const device uint*)w;
- const thread U4* x4 = (const thread U4*)x_thread;
- for (int i = 0; i < (values_per_thread / 4); i++) {
- uint wi = ws[i];
- U4 w_vec = uint4_to_fp4<U, 8>(uint4(wi, wi >> 8, wi >> 16, wi >> 24));
- accum += dot(x4[i], w_vec);
- }
- }
- return scale * accum + sum * bias;
-}
-
-template <typename U, int values_per_thread, int bits>
-inline U qdot_safe(
- const device uint8_t* w,
- const thread U* x_thread,
- U scale,
- U bias,
- U sum,
- int N
-) {
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- U accum = 0;
- if (bits == 4) {
- using U4 = vec<U, 4>;
- const device uint16_t* ws = (const device uint16_t*)w;
- const thread U4* x4 = (const thread U4*)x_thread;
-
- int full = N / 4;
- for (int i = 0; i < full; i++) {
- uint16_t wi = ws[i];
- U4 w_vec = uint4_to_fp4<U, 4>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));
- accum += dot(x4[i], w_vec);
- }
-
- int rem = N & 3;
- if (rem > 0) {
- uint16_t wv = ws[full];
- int base = 4 * full;
- if (rem > 0)
- accum += x_thread[base] * uint_to_fp<U>(wv & 0xf);
- if (rem > 1)
- accum += x_thread[base + 1] * uint_to_fp<U>((wv >> 4) & 0xf);
- if (rem > 2)
- accum += x_thread[base + 2] * uint_to_fp<U>((wv >> 8) & 0xf);
- }
- } else if (bits == 8) {
- for (int i = 0; i < N; i++) {
- accum += x_thread[i] * w[i];
- }
- }
-
- return scale * accum + sum * bias;
-}
-
-template <typename U, int N, int bits>
-inline void dequantize(
- const device uint8_t* w,
- U scale,
- U bias,
- threadgroup U* w_local
-) {
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- if (bits == 4) {
- U s0 = scale;
- U s1 = scale / static_cast<U>(16.0f);
- for (int i = 0; i < (N / 2); i++) {
- w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;
- w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;
- }
- } else if (bits == 8) {
- for (int i = 0; i < N; i++) {
- w_local[i] = scale * w[i] + bias;
- }
- }
-}
-
-template <>
-inline void dequantize<bfloat, 8, 4>(
- const device uint8_t* w,
- bfloat scale,
- bfloat bias,
- threadgroup bfloat* w_local
-) {
- const device uint32_t* w_ptr = (const device uint32_t*)w;
- uint32_t packed = *w_ptr;
-
- bfloat4 v0, v1;
-
- // Low 4 nibbles
- v0.x = static_cast<bfloat>(packed & 0xF);
- v0.y = static_cast<bfloat>((packed >> 4) & 0xF);
- v0.z = static_cast<bfloat>((packed >> 8) & 0xF);
- v0.w = static_cast<bfloat>((packed >> 12) & 0xF);
-
- // High 4 nibbles
- v1.x = static_cast<bfloat>((packed >> 16) & 0xF);
- v1.y = static_cast<bfloat>((packed >> 20) & 0xF);
- v1.z = static_cast<bfloat>((packed >> 24) & 0xF);
- v1.w = static_cast<bfloat>((packed >> 28) & 0xF);
-
- v0 = v0 * scale + bias;
- v1 = v1 * scale + bias;
-
- threadgroup bfloat4* out_ptr = (threadgroup bfloat4*)w_local;
- out_ptr[0] = v0;
- out_ptr[1] = v1;
-}
-
-template <
- typename T,
- short BROWS,
- short BCOLS,
- short dst_ld,
- short reduction_dim,
- short tgp_size,
- short group_size,
- short bits>
-struct QuantizedBlockLoaderMlx {
- static_assert(
- BCOLS <= group_size,
- "Group size should be larger than columns"
- );
- static_assert(
- group_size % BCOLS == 0,
- "Group size should be divisible by columns"
- );
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- METAL_CONST short pack_factor = get_pack_factor<bits, 8>();
- METAL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
- METAL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
- METAL_CONST short n_reads =
- (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
- METAL_CONST short group_steps = group_size / BCOLS;
-
- const int src_ld;
- const int tile_stride;
- short group_step_cnt;
- const int group_stride;
-
- const short thread_idx;
- const short bi;
- const short bj;
-
- threadgroup T* dst;
- const device uint8_t* src;
- const device T* scales;
- const device T* biases;
-
- QuantizedBlockLoaderMlx(
- const device uint8_t* src_,
- const device T* scales_,
- const device T* biases_,
- const int src_ld_,
- threadgroup T* dst_,
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- )
- : src_ld(src_ld_),
- tile_stride(
- reduction_dim ? BCOLS_PACKED * bytes_per_pack
- : BROWS * src_ld * bytes_per_pack / pack_factor
- ),
- group_step_cnt(0), group_stride(BROWS * src_ld / group_size),
- thread_idx(simd_group_id * 32 + simd_lane_id),
- bi(n_reads * thread_idx / BCOLS_PACKED),
- bj((n_reads * thread_idx) % BCOLS_PACKED),
- dst(dst_ + bi * dst_ld + bj * pack_factor),
- src(src_ + bi * src_ld * bytes_per_pack / pack_factor +
- bj * bytes_per_pack),
- scales(scales_ + bi * src_ld / group_size),
- biases(biases_ + bi * src_ld / group_size) {}
-
- void load_unsafe() const {
- if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {
- return;
- }
-
- T scale = *scales;
- T bias = *biases;
- for (int i = 0; i < n_reads; i++) {
- dequantize<T, pack_factor, bits>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
- }
- }
-
- void load_safe(short2 src_tile_dim) const {
- if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {
- return;
- }
-
- if (reduction_dim == 1 && bi >= src_tile_dim.x) {
- for (int i = 0; i < n_reads * pack_factor; i++) {
- dst[i] = T(0);
- }
- return;
- }
-
- if (reduction_dim == 0 && bi >= src_tile_dim.y) {
- for (int i = 0; i < n_reads * pack_factor; i++) {
- dst[i] = T(0);
- }
- return;
- }
-
- T scale = *scales;
- T bias = *biases;
- for (int i = 0; i < n_reads; i++) {
- dequantize<T, pack_factor, bits>(
- (device uint8_t*)(src + i * bytes_per_pack),
- scale,
- bias,
- dst + i * pack_factor
- );
- }
- }
-
- void next() {
- src += tile_stride;
- if (reduction_dim == 1) {
- if (group_steps > 1) {
- group_step_cnt++;
- if (group_step_cnt == group_steps) {
- group_step_cnt = 0;
- scales++;
- biases++;
- }
- } else {
- scales++;
- biases++;
- }
- } else {
- scales += group_stride;
- biases += group_stride;
- }
- }
-};
-
-template <
- typename T,
- short BROWS,
- short BCOLS,
- short dst_ld,
- short reduction_dim,
- short tgp_size,
- short group_size,
- short bits,
- bool per_output_layout = false>
-struct QuantizedBlockLoaderZp {
- static_assert(
- BCOLS <= group_size,
- "Group size should be larger than columns"
- );
- static_assert(
- group_size % BCOLS == 0,
- "Group size should be divisible by columns"
- );
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- METAL_CONST short pack_factor = get_pack_factor<bits, 8>();
- METAL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
- METAL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
- METAL_CONST short n_reads =
- (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
- METAL_CONST short group_steps = group_size / BCOLS;
-
- const int src_ld;
- const int groups_per_row;
- const int tile_stride;
- short group_step_cnt;
- int k_base;
- const int group_stride;
-
- const short thread_idx;
- const short bi;
- const short bj;
-
- threadgroup T* dst;
- const device uint8_t* src;
- const device T* scales;
- const device T* scales_row_start;
- const device uint8_t* zps_row_start;
- const int out_group_base;
- const int out_groups_total;
- const int zp_stride_total;
-
- QuantizedBlockLoaderZp(
- const device uint8_t* src_,
- const device T* scales_,
- const device uint8_t* zero_points_row_start_,
- const int src_ld_,
- const int groups_per_row_,
- threadgroup T* dst_,
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]],
- const int out_group_base_ = 0,
- const int out_groups_total_ = 0,
- const int zp_stride_total_ = 0
- )
- : src_ld(src_ld_), groups_per_row(groups_per_row_),
- tile_stride(
- reduction_dim ? BCOLS_PACKED * bytes_per_pack
- : BROWS * src_ld * bytes_per_pack / pack_factor
- ),
- group_step_cnt(0), k_base(0), group_stride(BROWS * groups_per_row_),
- thread_idx(simd_group_id * 32 + simd_lane_id),
- bi(n_reads * thread_idx / BCOLS_PACKED),
- bj((n_reads * thread_idx) % BCOLS_PACKED),
- dst(dst_ + bi * dst_ld + bj * pack_factor),
- src(src_ + bi * src_ld * bytes_per_pack / pack_factor +
- bj * bytes_per_pack),
- scales(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),
- scales_row_start(
- reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_
- ),
- zps_row_start(
- reduction_dim == 1 ? (zero_points_row_start_ +
- bi * (bits == 4 ? ((groups_per_row_ + 1) / 2)
- : groups_per_row_))
- : zero_points_row_start_
- ),
- out_group_base(per_output_layout ? out_group_base_ : 0),
- out_groups_total(per_output_layout ? out_groups_total_ : 0),
- zp_stride_total(per_output_layout ? zp_stride_total_ : 0) {}
-
- inline void current_scale_bias(
- thread T& out_scale,
- thread T& out_bias
- ) const {
- uint zp_n;
- T scale_val;
- if (per_output_layout) {
- const int row_idx = k_base + bi;
- const int scale_index = row_idx * groups_per_row + out_group_base;
- scale_val = scales_row_start[scale_index];
- if (bits == 4) {
- const int byte_index =
- row_idx * zp_stride_total + (out_group_base >> 1);
- uint8_t zp_b = zps_row_start[byte_index];
- zp_n = (uint(zp_b) >> (uint(out_group_base & 1) * 4u)) & 0x0Fu;
- } else {
- const int zp_index = row_idx * zp_stride_total + out_group_base;
- zp_n = zps_row_start[zp_index];
- }
- } else {
- int g = reduction_dim == 0 ? (k_base / group_size)
- : (int)(scales - scales_row_start);
- scale_val = reduction_dim == 0 ? scales_row_start[g] : *scales;
- if (bits == 4) {
- const device uint8_t* zp_ptr = zps_row_start + (g >> 1);
- uint8_t zp_b = *zp_ptr;
- zp_n = (uint(zp_b) >> (uint(g & 1) * 4u)) & 0x0Fu;
- } else {
- zp_n = zps_row_start[g];
- }
- }
- out_scale = scale_val;
- out_bias = static_cast<T>(-scale_val * static_cast<T>(zp_n));
- }
-
- void load_unsafe() const {
- if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {
- return;
- }
-
- T scale;
- T bias;
- current_scale_bias(scale, bias);
- for (int i = 0; i < n_reads; i++) {
- dequantize<T, pack_factor, bits>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
- }
- }
-
- void load_safe(short2 src_tile_dim) const {
- if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {
- return;
- }
-
- if (reduction_dim == 1) {
- // N-tail: zero out rows beyond valid outputs
- if (bi >= src_tile_dim.x) {
- for (int i = 0; i < n_reads * pack_factor; i++) {
- dst[i] = T(0);
- }
- return;
- }
-
- int valid_cols = src_tile_dim.y; // 0..BK
- int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;
-
- T scale;
- T bias;
- current_scale_bias(scale, bias);
- for (int i = 0; i < n_reads; i++) {
- int pack_idx = bj + i; // global pack index across the BK packs
- if (pack_idx < valid_packs) {
- dequantize<T, pack_factor, bits>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
-
- // Mask the last pack if needed
- if (pack_idx == valid_packs - 1) {
- int rem = valid_cols - pack_idx * pack_factor;
- if (rem < pack_factor) {
- for (int r = rem; r < pack_factor; ++r) {
- dst[i * pack_factor + r] = T(0);
- }
- }
- }
- } else {
- for (int j = 0; j < pack_factor; ++j) {
- dst[i * pack_factor + j] = T(0);
- }
- }
- }
- return;
- }
-
- if (reduction_dim == 0 && bi >= src_tile_dim.y) {
- for (int i = 0; i < n_reads * pack_factor; i++) {
- dst[i] = T(0);
- }
- return;
- }
-
- T scale;
- T bias;
- current_scale_bias(scale, bias);
- for (int i = 0; i < n_reads; i++) {
- dequantize<T, pack_factor, bits>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
- }
- }
-
- void next() {
- src += tile_stride;
- if (reduction_dim == 1) {
- if (group_steps > 1) {
- group_step_cnt++;
- if (group_step_cnt == group_steps) {
- group_step_cnt = 0;
- scales++;
- }
- } else {
- scales++;
- }
- } else {
- k_base += BROWS;
- }
- }
-};
-
-template <
- typename LoaderW,
- typename LoaderX,
- typename Mma,
- typename T,
- const bool aligned_N,
- const int BM,
- const int BK,
- const int BN>
-inline void qmm_transposed_core(
- thread LoaderX& loader_x,
- thread LoaderW& loader_w,
- thread Mma& mma_op,
- const short num_els,
- const short num_outs,
- const int in_vec_size,
- device T* output,
- const int out_vec_size,
- threadgroup T* Xs,
- threadgroup T* Ws
-) {
- if (num_els < BM) {
- if (!aligned_N && num_outs < BN) {
- for (int k = 0; k < in_vec_size; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_safe(short2(BK, num_els));
- loader_w.load_safe(short2(BK, num_outs));
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- loader_x.next();
- loader_w.next();
- }
- } else {
- for (int k = 0; k < in_vec_size; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_safe(short2(BK, num_els));
- loader_w.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- loader_x.next();
- loader_w.next();
- }
- }
- } else {
- if (!aligned_N && num_outs < BN) {
- for (int k = 0; k < in_vec_size; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_unsafe();
- loader_w.load_safe(short2(BK, num_outs));
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- loader_x.next();
- loader_w.next();
- }
- } else {
- for (int k = 0; k < in_vec_size; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_unsafe();
- loader_w.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_op.mma(Xs, Ws);
- loader_x.next();
- loader_w.next();
- }
- }
- }
-
- // Store results to device memory
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (num_els < BM || num_outs < BN) {
- mma_op.store_result_safe(output, out_vec_size, short2(num_outs, num_els));
- } else {
- mma_op.store_result(output, out_vec_size);
- }
-}
-
-template <
- typename T,
- const uint group_size,
- const uint bits,
- const bool aligned_N,
- const int BM = 32,
- const int BK = 32,
- const int BN = 32,
- const QuantizationMethod quant_method = QuantizationMethod::ScaleZeroPoint,
- const int WM = 2,
- const int WN = 2>
-void qmm_transposed_impl(
- const device uint32_t* weights,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* biases,
- const device T* input,
- device T* output,
- threadgroup T* Xs,
- threadgroup T* Ws,
- const int in_vec_size,
- const int out_vec_size,
- const int batch_size,
- uint out_block_idx,
- uint batch_block_idx,
- uint simd_group,
- uint simd_lane
-) {
- static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");
- static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");
- constexpr int pack_factor = get_pack_factor<bits, 8>();
- constexpr int bytes_per_pack = get_bytes_per_pack<bits>();
- constexpr int BK_padded = (BK + 16 / sizeof(T));
-
- using mma_t = matmul_utils::
- BlockMMA<T, T, BM, BN, BK, WM, WN, false, true, BK_padded, BK_padded>;
- using loader_x_t =
- matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;
-
- const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
- const int in_vec_size_g = (in_vec_size + group_size - 1) / group_size;
- const int out_row = batch_block_idx * BM;
- const int out_col = out_block_idx * BN;
-
- auto wl = (const device uint8_t*)weights;
-
- const device T* x_block = input + out_row * static_cast<int64_t>(in_vec_size);
- const device uint8_t* w_block = wl + out_col * in_vec_size_w;
- scales += out_col * in_vec_size_g;
- biases += out_col * in_vec_size_g;
- device T* y_block =
- output + out_row * static_cast<int64_t>(out_vec_size) + out_col;
-
- const short num_els = min(BM, batch_size - out_row);
- const short num_outs = min(BN, out_vec_size - out_col);
- loader_x_t loader_x(x_block, in_vec_size, Xs, simd_group, simd_lane);
- mma_t mma_op(simd_group, simd_lane);
-
- if (quant_method == QuantizationMethod::ScaleBias) {
- using loader_w_t = QuantizedBlockLoaderMlx<
- T,
- BN,
- BK,
- BK_padded,
- 1,
- WM * WN * 32,
- group_size,
- bits>;
- loader_w_t loader_w(
- w_block,
- scales,
- biases,
- in_vec_size,
- Ws,
- simd_group,
- simd_lane
- );
- qmm_transposed_core<
- loader_w_t,
- loader_x_t,
- mma_t,
- T,
- aligned_N,
- BM,
- BK,
- BN>(
- loader_x,
- loader_w,
- mma_op,
- num_els,
- num_outs,
- in_vec_size,
- y_block,
- out_vec_size,
- Xs,
- Ws
- );
- } else {
- using loader_w_t = QuantizedBlockLoaderZp<
- T,
- BN,
- BK,
- BK_padded,
- 1,
- WM * WN * 32,
- group_size,
- bits>;
- const device uint8_t* zero_points_row =
- zero_points +
- out_col * (bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g);
- loader_w_t loader_w(
- w_block,
- scales,
- zero_points_row,
- in_vec_size,
- in_vec_size_g,
- Ws,
- simd_group,
- simd_lane
- );
- qmm_transposed_core<
- loader_w_t,
- loader_x_t,
- mma_t,
- T,
- aligned_N,
- BM,
- BK,
- BN>(
- loader_x,
- loader_w,
- mma_op,
- num_els,
- num_outs,
- in_vec_size,
- y_block,
- out_vec_size,
- Xs,
- Ws
- );
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rs
index d00c639d3..955ae6cc1 100644
--- a/crates/backend-uzu/src/backends/metal/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/mod.rs
@@ -13,8 +13,5 @@ pub use backend::Metal;
pub use buffer::BufferDowncastExt;
pub use context::MetalContext;
pub use device_capabilities::MetalDeviceCapabilities;
-pub use kernel::matmul::{
- MatmulDispatchPath,
- quant::{QuantizedMatmulDispatchPath, encode_quantized_matmul_with_path},
-};
+pub use kernel::matmul::{GemmDispatchPath, GemmKernel};
pub use metal_extensions::{DeviceExt, DeviceGeneration};
diff --git a/crates/backend-uzu/src/config/embedding.rs b/crates/backend-uzu/src/config/embedding.rs
index 80f43d8e4..69e89a3ff 100644
--- a/crates/backend-uzu/src/config/embedding.rs
+++ b/crates/backend-uzu/src/config/embedding.rs
@@ -23,7 +23,7 @@ pub enum EmbeddingConfig {
common: EmbeddingConfigCommon,
precision: DataType,
},
- #[serde(rename = "ScaleBiasQuantizedTiedEmbeddingConfig")]
+ #[serde(rename = "MLXQuantizedTiedEmbeddingConfig")]
ScaleBiasQuantizedTied {
#[serde(flatten)]
common: EmbeddingConfigCommon,
@@ -32,7 +32,7 @@ pub enum EmbeddingConfig {
activation_quantization_mode: Option<QuantizationMode>,
activation_precision: DataType,
},
- #[serde(rename = "ScaleBiasQuantizedUntiedEmbeddingConfig")]
+ #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]
ScaleBiasQuantizedUntied {
#[serde(flatten)]
common: EmbeddingConfigCommon,
@@ -41,7 +41,7 @@ pub enum EmbeddingConfig {
activation_quantization_mode: Option<QuantizationMode>,
activation_precision: DataType,
},
- #[serde(rename = "ScaleBiasSemiQuantizedUntiedEmbeddingConfig")]
+ #[serde(rename = "MLXSemiQuantizedUntiedEmbeddingConfig")]
ScaleBiasSemiQuantizedUntied {
#[serde(flatten)]
common: EmbeddingConfigCommon,
diff --git a/crates/backend-uzu/src/data_type.rs b/crates/backend-uzu/src/data_type.rs
index 5af111f1a..ccb18332b 100644
--- a/crates/backend-uzu/src/data_type.rs
+++ b/crates/backend-uzu/src/data_type.rs
@@ -3,7 +3,7 @@ use half::{bf16, f16};
use num_traits::NumCast;
use proc_macros::uzu_config;
-#[derive(Eq, PartialOrd, Ord, Copy)]
+#[derive(Eq, PartialOrd, Ord, Copy, Hash)]
#[uzu_config]
pub enum DataType {
// Floating point
diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rs
index 7d9744439..63d530590 100644
--- a/crates/backend-uzu/src/encodable_block/attention.rs
+++ b/crates/backend-uzu/src/encodable_block/attention.rs
@@ -15,7 +15,7 @@ use crate::{
AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel, ManualKernels,
SigmoidGateKernel, SoftmaxKernel,
attention::{AttentionGemmArguments, AttentionGemmBlock},
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
config::AttentionConfig,
@@ -287,7 +287,7 @@ impl<B: Backend> Attention<B> {
max_sequence_length as u32,
encoder,
);
- Ok(values)
+ Ok::<_, B::Error>(values)
})
.transpose()?;
@@ -595,19 +595,25 @@ impl<B: Backend> Attention<B> {
MatmulArguments {
a: queries,
a_offset: group_index * gqa_factor * suffix_length * head_dim * dt_bytes,
- b: keys,
+ b: MatmulB::FullPrecision {
+ b: keys,
+ },
b_offset: group_index * k_head_stride as usize * dt_bytes,
b_leading_dimension: Some(k_seq_stride),
b_transpose: true,
- ab_scale: scale,
- c: MatmulArgumentC::None,
d: &mut group_scores,
- batch_dim: (gqa_factor * suffix_length) as u32,
- input_dim: head_dim as u32,
- output_dim: sequence_length as u32,
+ d_transform: MatmulDOps {
+ ab_scale: scale,
+ accumulate: false,
+ bias: None,
+ rht_factors: None,
+ },
+ m: (gqa_factor * suffix_length) as u32,
+ n: sequence_length as u32,
+ k: head_dim as u32,
},
encoder,
- );
+ )?;
scatter_scores.encode(
&group_scores,
&mut scores,
@@ -645,19 +651,20 @@ impl<B: Backend> Attention<B> {
MatmulArguments {
a: &scores,
a_offset: group_index * gqa_factor * suffix_length * sequence_length * dt_bytes,
- b: values,
+ b: MatmulB::FullPrecision {
+ b: values,
+ },
b_offset: group_index * v_head_stride as usize * dt_bytes,
b_leading_dimension: Some(v_seq_stride),
b_transpose: false,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
d: &mut group_output,
- batch_dim: (gqa_factor * suffix_length) as u32,
- input_dim: sequence_length as u32,
- output_dim: head_dim as u32,
+ d_transform: MatmulDOps::none(),
+ m: (gqa_factor * suffix_length) as u32,
+ n: head_dim as u32,
+ k: sequence_length as u32,
},
encoder,
- );
+ )?;
self.fallback_scatter_values_kernel.encode(
&group_output,
&mut *attention_output,
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 1b0897e02..365eb3ff4 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -6,14 +6,10 @@ use crate::{
DataType,
backends::common::{
Allocation, Backend, Encoder, Kernels,
- gpu_types::QuantizationMethod,
+ gpu_types::{QuantizationMethod, QuantizationMode},
kernel::{
FullPrecisionEmbeddingLookupKernel, ManualKernels, QuantizedEmbeddingLookupKernel,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
- quant_matmul::{
- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
- QuantizedMatmulKernelEncodable,
- },
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},
},
},
config::EmbeddingConfig,
@@ -24,16 +20,17 @@ use crate::{
pub enum EmbeddingError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
- #[error("Matmul error: {0}")]
- MatmulError(#[from] MatmulError<B>),
- #[error("QuantizedMatmul error: {0}")]
- QuantizedMatmulError(#[from] QuantizedMatmulError<B>),
#[error("Parameter loading error: {0}")]
ParameterError(#[from] ParameterLoaderError<B>),
#[error("Unsupported configuration: {0}")]
UnsupportedConfiguration(String),
}
+struct ReadoutQuantConfig {
+ mode: QuantizationMode,
+ group_size: u32,
+}
+
enum TiedEmbeddingType<B: Backend> {
FullPrecision {
weights: Allocation<B>,
@@ -45,7 +42,8 @@ enum TiedEmbeddingType<B: Backend> {
scales: Allocation<B>,
biases: Allocation<B>,
lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
- readout: QuantizedMatmulKernelEncodable<B>,
+ readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
+ readout_config: ReadoutQuantConfig,
},
}
@@ -71,7 +69,8 @@ enum UntiedEmbeddingReadoutType<B: Backend> {
weights: Allocation<B>,
scales: Allocation<B>,
biases: Allocation<B>,
- readout: QuantizedMatmulKernelEncodable<B>,
+ readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
+ readout_config: ReadoutQuantConfig,
},
}
@@ -184,7 +183,10 @@ impl<B: Backend> Embedding<B> {
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(<B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)?);
+ let readout = RefCell::new(
+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
+ .map_err(EmbeddingError::BackendError)?,
+ );
EmbeddingTying::Tied {
ty: TiedEmbeddingType::FullPrecision {
@@ -210,7 +212,10 @@ impl<B: Backend> Embedding<B> {
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(<B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)?);
+ let readout = RefCell::new(
+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
+ .map_err(EmbeddingError::BackendError)?,
+ );
EmbeddingTying::Untied {
input_ty: UntiedEmbeddingLookupType::FullPrecision {
@@ -256,18 +261,23 @@ impl<B: Backend> Embedding<B> {
(*embedding_quantization_mode).into(),
)
.map_err(EmbeddingError::BackendError)?;
- let readout = QuantizedMatmulKernelEncodable::new(
- context,
- QuantizedMatmulConfiguration {
- data_type,
- group_size: *group_size,
- input_dim: model_dim as usize,
- output_dim: vocab_size as usize,
- mode: *embedding_quantization_mode,
- quantization_method: QuantizationMethod::ScaleBias,
- use_hadamard: false,
- },
- )?;
+ let readout_config = ReadoutQuantConfig {
+ mode: *embedding_quantization_mode,
+ group_size: *group_size as u32,
+ };
+ let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
+ .map_err(EmbeddingError::BackendError)?;
+ readout_kernel
+ .preheat_quant_combo(
+ context,
+ MatmulQuantCombo {
+ method: QuantizationMethod::ScaleBias,
+ mode: *embedding_quantization_mode,
+ group_size: *group_size as u32,
+ },
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let readout = RefCell::new(readout_kernel);
if let Some(activation_quantization_mode) = activation_quantization_mode {
return Err(EmbeddingError::UnsupportedConfiguration(format!(
@@ -282,6 +292,7 @@ impl<B: Backend> Embedding<B> {
biases,
lookup,
readout,
+ readout_config,
},
}
},
@@ -330,18 +341,23 @@ impl<B: Backend> Embedding<B> {
(*embedding_quantization_mode).into(),
)
.map_err(EmbeddingError::BackendError)?;
- let readout = QuantizedMatmulKernelEncodable::new(
- context,
- QuantizedMatmulConfiguration {
- data_type,
- group_size: *group_size,
- input_dim: model_dim as usize,
- output_dim: vocab_size as usize,
- mode: *embedding_quantization_mode,
- quantization_method: QuantizationMethod::ScaleBias,
- use_hadamard: false,
- },
- )?;
+ let readout_config = ReadoutQuantConfig {
+ mode: *embedding_quantization_mode,
+ group_size: *group_size as u32,
+ };
+ let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
+ .map_err(EmbeddingError::BackendError)?;
+ readout_kernel
+ .preheat_quant_combo(
+ context,
+ MatmulQuantCombo {
+ method: QuantizationMethod::ScaleBias,
+ mode: *embedding_quantization_mode,
+ group_size: *group_size as u32,
+ },
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let readout = RefCell::new(readout_kernel);
if let Some(activation_quantization_mode) = activation_quantization_mode {
return Err(EmbeddingError::UnsupportedConfiguration(format!(
@@ -361,6 +377,7 @@ impl<B: Backend> Embedding<B> {
scales: output_scales,
biases: output_biases,
readout,
+ readout_config,
},
}
},
@@ -397,18 +414,23 @@ impl<B: Backend> Embedding<B> {
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
- let readout = QuantizedMatmulKernelEncodable::new(
- context,
- QuantizedMatmulConfiguration {
- data_type,
- group_size: *group_size,
- input_dim: model_dim as usize,
- output_dim: vocab_size as usize,
- mode: *embedding_quantization_mode,
- quantization_method: QuantizationMethod::ScaleBias,
- use_hadamard: false,
- },
- )?;
+ let readout_config = ReadoutQuantConfig {
+ mode: *embedding_quantization_mode,
+ group_size: *group_size as u32,
+ };
+ let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
+ .map_err(EmbeddingError::BackendError)?;
+ readout_kernel
+ .preheat_quant_combo(
+ context,
+ MatmulQuantCombo {
+ method: QuantizationMethod::ScaleBias,
+ mode: *embedding_quantization_mode,
+ group_size: *group_size as u32,
+ },
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let readout = RefCell::new(readout_kernel);
if let Some(activation_quantization_mode) = activation_quantization_mode {
return Err(EmbeddingError::UnsupportedConfiguration(format!(
@@ -426,6 +448,7 @@ impl<B: Backend> Embedding<B> {
scales: output_scales,
biases: output_biases,
readout,
+ readout_config,
},
}
},
@@ -495,6 +518,7 @@ impl<B: Backend> Embedding<B> {
biases,
lookup,
readout: _,
+ readout_config: _,
},
}
| EmbeddingTying::Untied {
@@ -558,23 +582,27 @@ impl<B: Backend> Embedding<B> {
} => {
let input_dim = self.model_dim as usize;
let output_dim = self.vocab_size as usize;
- readout.borrow_mut().encode(
- MatmulArguments {
- a: input_allocation,
- a_offset: 0,
- b: weights,
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
- d: &mut output_allocation,
- batch_dim: batch_dim as u32,
- input_dim: input_dim as u32,
- output_dim: output_dim as u32,
- },
- encoder,
- );
+ readout
+ .borrow_mut()
+ .encode(
+ MatmulArguments {
+ a: input_allocation,
+ a_offset: 0,
+ b: MatmulB::FullPrecision {
+ b: weights,
+ },
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut output_allocation,
+ d_transform: MatmulDOps::none(),
+ m: batch_dim as u32,
+ n: output_dim as u32,
+ k: input_dim as u32,
+ },
+ encoder,
+ )
+ .map_err(EmbeddingError::BackendError)?;
},
EmbeddingTying::Tied {
ty:
@@ -584,6 +612,7 @@ impl<B: Backend> Embedding<B> {
biases,
lookup: _,
readout,
+ readout_config,
},
}
| EmbeddingTying::Untied {
@@ -594,21 +623,35 @@ impl<B: Backend> Embedding<B> {
scales,
biases,
readout,
+ readout_config,
},
} => {
- readout.encode(
- encoder,
- QuantizedMatmulArguments {
- a: input_allocation,
- a_offset: 0,
- b: weights,
- scales,
- zero_points_or_biases: biases,
- output: &mut output_allocation,
- hadamard_factors: None,
- batch_dim,
- },
- );
+ let b_variant: MatmulB<'_, B> = MatmulB::ScaleBiasDequant {
+ b: weights,
+ scales,
+ biases,
+ mode: readout_config.mode,
+ group_size: readout_config.group_size,
+ };
+ readout
+ .borrow_mut()
+ .encode(
+ MatmulArguments {
+ a: input_allocation,
+ a_offset: 0,
+ b: b_variant,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut output_allocation,
+ d_transform: MatmulDOps::none(),
+ m: batch_dim as u32,
+ n: self.vocab_size,
+ k: self.model_dim,
+ },
+ encoder,
+ )
+ .map_err(EmbeddingError::BackendError)?;
},
};
diff --git a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs b/crates/backend-uzu/src/encodable_block/linear/full_precision.rs
deleted file mode 100644
index 7c5e1063e..000000000
--- a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs
+++ /dev/null
@@ -1,153 +0,0 @@
-use std::cell::RefCell;
-
-use thiserror::Error;
-
-use super::Linear;
-use crate::{
- DataType,
- array::size_for_shape,
- backends::common::{
- Allocation, Backend, Encoder,
- kernel::{
- ManualKernels,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
- },
- },
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-#[derive(Debug, Error)]
-pub enum FullPrecisionLinearError<B: Backend> {
- #[error("Matmul error: {0}")]
- MatmulError(#[from] MatmulError<B>),
- #[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError<B>),
- #[error("Unsupported data type for full precision linear kernel: {0:?}")]
- UnsupportedDataType(DataType),
- #[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]
- InvalidWeightsShape {
- got: Box<[usize]>,
- expected_output_dim: usize,
- expected_input_dim: usize,
- },
- #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidWeightsDataType {
- expected: DataType,
- got: DataType,
- },
- #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]
- InvalidBiasShape {
- got: Box<[usize]>,
- expected_output_dim: usize,
- },
- #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidBiasDataType {
- expected: DataType,
- got: DataType,
- },
-}
-
-pub struct FullPrecisionLinear<B: Backend> {
- kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
- bias: Option<Allocation<B>>,
- weights: Allocation<B>,
- input_dim: usize,
- output_dim: usize,
- precision: DataType,
-}
-
-impl<B: Backend> FullPrecisionLinear<B> {
- pub fn new(
- context: &B::Context,
- precision: DataType,
- input_dim: usize,
- output_dim: usize,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, FullPrecisionLinearError<B>> {
- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(FullPrecisionLinearError::UnsupportedDataType(precision));
- }
-
- let weights_leaf = parameter_tree.leaf("weights").map_err(FullPrecisionLinearError::ParameterError)?;
- let weights_shape = weights_leaf.shape().to_vec();
- if weights_shape != [output_dim, input_dim] {
- return Err(FullPrecisionLinearError::InvalidWeightsShape {
- got: weights_shape.into_boxed_slice(),
- expected_output_dim: output_dim,
- expected_input_dim: input_dim,
- });
- }
-
- if weights_leaf.data_type() != precision {
- return Err(FullPrecisionLinearError::InvalidWeightsDataType {
- expected: precision,
- got: weights_leaf.data_type(),
- });
- }
-
- let bias = match parameter_tree.leaf("biases") {
- Ok(biases_leaf) => {
- let bias_shape = biases_leaf.shape().to_vec();
- if bias_shape != [output_dim] {
- return Err(FullPrecisionLinearError::InvalidBiasShape {
- got: bias_shape.into_boxed_slice(),
- expected_output_dim: output_dim,
- });
- }
-
- if biases_leaf.data_type() != precision {
- return Err(FullPrecisionLinearError::InvalidBiasDataType {
- expected: precision,
- got: biases_leaf.data_type(),
- });
- }
-
- Some(biases_leaf.read_allocation().map_err(FullPrecisionLinearError::ParameterError)?)
- },
- Err(_) => None,
- };
-
- let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, precision)?;
-
- Ok(Self {
- kernel: RefCell::new(kernel),
- bias,
- weights: weights_leaf.read_allocation().map_err(FullPrecisionLinearError::ParameterError)?,
- input_dim,
- output_dim,
- precision,
- })
- }
-}
-
-impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
- fn encode(
- &self,
- input: Allocation<B>,
- batch_dim: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
- let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.precision))?;
- self.kernel.borrow_mut().encode(
- MatmulArguments {
- a: &input,
- a_offset: 0,
- b: &self.weights,
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: 1.0,
- c: match self.bias.as_ref() {
- Some(bias) => MatmulArgumentC::Bias(bias),
- None => MatmulArgumentC::None,
- },
- d: &mut output,
- batch_dim: batch_dim as u32,
- input_dim: self.input_dim as u32,
- output_dim: self.output_dim as u32,
- },
- encoder,
- );
- Ok(output)
- }
-}
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
new file mode 100644
index 000000000..8524b3e7f
--- /dev/null
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -0,0 +1,368 @@
+use std::cell::RefCell;
+
+use thiserror::Error;
+
+use super::Linear;
+use crate::{
+ DataType,
+ array::size_for_shape,
+ backends::common::{
+ Allocation, Backend, Encoder,
+ gpu_types::{QuantizationMethod, QuantizationMode},
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},
+ },
+ },
+ config::QuantizationConfig,
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum LinearMatmulError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(#[from] ParameterLoaderError<B>),
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]
+ InvalidFpWeightsShape {
+ got: Box<[usize]>,
+ expected_output_dim: usize,
+ expected_input_dim: usize,
+ },
+ #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidWeightsDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidScalesDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error(
+ "Unexpected scale-bias shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"
+ )]
+ InvalidScaleBiasShapes {
+ weights: Box<[usize]>,
+ scales: Box<[usize]>,
+ deq_biases: Box<[usize]>,
+ packing_divisor: usize,
+ },
+ #[error("deq_biases dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidDeqBiasesDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error(
+ "Unexpected scale-zero-point shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"
+ )]
+ InvalidScaleZeroPointShapes {
+ weights: Box<[usize]>,
+ scales: Box<[usize]>,
+ zero_points: Box<[usize]>,
+ packing_divisor: usize,
+ packing_minus_one: usize,
+ },
+ #[error("Zero-points dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidZeroPointsDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]
+ InvalidBiasShape {
+ got: Box<[usize]>,
+ expected_output_dim: usize,
+ },
+ #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidBiasDataType {
+ expected: DataType,
+ got: DataType,
+ },
+}
+
+enum Mode<B: Backend> {
+ FullPrecision,
+ Quantized {
+ method: QuantizationMethod,
+ mode: QuantizationMode,
+ group_size: u32,
+ scales: Allocation<B>,
+ zero_points_or_biases: Allocation<B>,
+ output_hadamard_factors: Option<Allocation<B>>,
+ },
+}
+
+pub struct LinearMatmul<B: Backend> {
+ kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
+ weights: Allocation<B>,
+ biases: Option<Allocation<B>>,
+ input_dim: usize,
+ output_dim: usize,
+ data_type: DataType,
+ mode: Mode<B>,
+}
+
+impl<B: Backend> LinearMatmul<B> {
+ pub fn full_precision(
+ context: &B::Context,
+ precision: DataType,
+ input_dim: usize,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, LinearMatmulError<B>> {
+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(LinearMatmulError::UnsupportedDataType(precision));
+ }
+
+ let weights_leaf = parameter_tree.leaf("weights")?;
+ let weights_shape = weights_leaf.shape().to_vec();
+ if weights_shape != [output_dim, input_dim] {
+ return Err(LinearMatmulError::InvalidFpWeightsShape {
+ got: weights_shape.into_boxed_slice(),
+ expected_output_dim: output_dim,
+ expected_input_dim: input_dim,
+ });
+ }
+ if weights_leaf.data_type() != precision {
+ return Err(LinearMatmulError::InvalidWeightsDataType {
+ expected: precision,
+ got: weights_leaf.data_type(),
+ });
+ }
+
+ let biases = load_biases(precision, output_dim, parameter_tree)?;
+
+ let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, precision)
+ .map_err(LinearMatmulError::BackendError)?;
+ let weights = weights_leaf.read_allocation()?;
+
+ Ok(Self {
+ kernel: RefCell::new(kernel),
+ weights,
+ biases,
+ input_dim,
+ output_dim,
+ data_type: precision,
+ mode: Mode::FullPrecision,
+ })
+ }
+
+ pub fn quantized(
+ context: &B::Context,
+ config: &QuantizationConfig,
+ input_dim: usize,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<B::Context>,
+ output_hadamard_factors: Option<Allocation<B>>,
+ ) -> Result<Self, LinearMatmulError<B>> {
+ let data_type: DataType = config.activation_precision.into();
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(LinearMatmulError::UnsupportedDataType(data_type));
+ }
+
+ let packing_divisor = config.weight_quantization_mode.packing_divisor();
+ let storage_type = config.weight_quantization_mode.storage_type();
+
+ let weights_leaf = parameter_tree.leaf("weights")?;
+ if weights_leaf.data_type() != storage_type {
+ return Err(LinearMatmulError::InvalidWeightsDataType {
+ expected: storage_type,
+ got: weights_leaf.data_type(),
+ });
+ }
+
+ let scales_leaf = parameter_tree.leaf("scales")?;
+ if scales_leaf.data_type() != data_type {
+ return Err(LinearMatmulError::InvalidScalesDataType {
+ expected: data_type,
+ got: scales_leaf.data_type(),
+ });
+ }
+
+ let k_g = input_dim.div_ceil(config.group_size);
+ let weights_shape = weights_leaf.shape().to_vec();
+ let scales_shape = scales_leaf.shape().to_vec();
+
+ let (quantization_method, zero_points_or_biases) = match parameter_tree.leaf("deq_biases") {
+ Ok(deq_biases) => {
+ let deq_biases_shape = deq_biases.shape().to_vec();
+ if !(weights_shape == [output_dim, input_dim / packing_divisor]
+ && scales_shape == [output_dim, k_g]
+ && deq_biases_shape == [output_dim, k_g])
+ {
+ return Err(LinearMatmulError::InvalidScaleBiasShapes {
+ weights: weights_shape.into_boxed_slice(),
+ scales: scales_shape.into_boxed_slice(),
+ deq_biases: deq_biases_shape.into_boxed_slice(),
+ packing_divisor,
+ });
+ }
+ if deq_biases.data_type() != data_type {
+ return Err(LinearMatmulError::InvalidDeqBiasesDataType {
+ expected: data_type,
+ got: deq_biases.data_type(),
+ });
+ }
+ (QuantizationMethod::ScaleBias, deq_biases.read_allocation()?)
+ },
+ Err(_) => {
+ let zero_points_leaf = parameter_tree.leaf("zero_points")?;
+ let zero_points_shape = zero_points_leaf.shape().to_vec();
+ let expected_zero_points_entries = k_g.div_ceil(packing_divisor);
+ if !(weights_shape == [output_dim, input_dim / packing_divisor]
+ && scales_shape == [output_dim, k_g]
+ && zero_points_shape == [output_dim, expected_zero_points_entries])
+ {
+ return Err(LinearMatmulError::InvalidScaleZeroPointShapes {
+ weights: weights_shape.into_boxed_slice(),
+ scales: scales_shape.into_boxed_slice(),
+ zero_points: zero_points_shape.into_boxed_slice(),
+ packing_divisor,
+ packing_minus_one: packing_divisor - 1,
+ });
+ }
+ if zero_points_leaf.data_type() != storage_type {
+ return Err(LinearMatmulError::InvalidZeroPointsDataType {
+ expected: storage_type,
+ got: zero_points_leaf.data_type(),
+ });
+ }
+ (QuantizationMethod::ScaleZeroPoint, zero_points_leaf.read_allocation()?)
+ },
+ };
+
+ let biases = load_biases(data_type, output_dim, parameter_tree)?;
+
+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
+ .map_err(LinearMatmulError::BackendError)?;
+ kernel
+ .preheat_quant_combo(
+ context,
+ MatmulQuantCombo {
+ method: quantization_method,
+ mode: config.weight_quantization_mode,
+ group_size: config.group_size as u32,
+ },
+ )
+ .map_err(LinearMatmulError::BackendError)?;
+
+ Ok(Self {
+ kernel: RefCell::new(kernel),
+ weights: weights_leaf.read_allocation()?,
+ biases,
+ input_dim,
+ output_dim,
+ data_type,
+ mode: Mode::Quantized {
+ method: quantization_method,
+ mode: config.weight_quantization_mode,
+ group_size: config.group_size as u32,
+ scales: scales_leaf.read_allocation()?,
+ zero_points_or_biases,
+ output_hadamard_factors,
+ },
+ })
+ }
+}
+
+fn load_biases<B: Backend>(
+ data_type: DataType,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<B::Context>,
+) -> Result<Option<Allocation<B>>, LinearMatmulError<B>> {
+ match parameter_tree.leaf("biases") {
+ Ok(biases_leaf) => {
+ let bias_shape = biases_leaf.shape().to_vec();
+ if bias_shape != [output_dim] {
+ return Err(LinearMatmulError::InvalidBiasShape {
+ got: bias_shape.into_boxed_slice(),
+ expected_output_dim: output_dim,
+ });
+ }
+ if biases_leaf.data_type() != data_type {
+ return Err(LinearMatmulError::InvalidBiasDataType {
+ expected: data_type,
+ got: biases_leaf.data_type(),
+ });
+ }
+ Ok(Some(biases_leaf.read_allocation()?))
+ },
+ Err(_) => Ok(None),
+ }
+}
+
+impl<B: Backend> Linear<B> for LinearMatmul<B> {
+ fn encode(
+ &self,
+ input: Allocation<B>,
+ batch_dim: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.data_type))?;
+
+ let b = match &self.mode {
+ Mode::FullPrecision => MatmulB::FullPrecision {
+ b: &self.weights,
+ },
+ Mode::Quantized {
+ method,
+ mode,
+ group_size,
+ scales,
+ zero_points_or_biases,
+ ..
+ } => match method {
+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
+ b: &self.weights,
+ scales,
+ biases: zero_points_or_biases,
+ mode: *mode,
+ group_size: *group_size,
+ },
+ QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
+ b: &self.weights,
+ scales,
+ zero_points: zero_points_or_biases,
+ mode: *mode,
+ group_size: *group_size,
+ },
+ },
+ };
+
+ let rht_factors = match &self.mode {
+ Mode::Quantized {
+ output_hadamard_factors: Some(factors),
+ ..
+ } => Some(factors),
+ _ => None,
+ };
+ let d_transform = MatmulDOps {
+ ab_scale: 1.0,
+ accumulate: false,
+ bias: self.biases.as_ref(),
+ rht_factors,
+ };
+
+ self.kernel.borrow_mut().encode(
+ MatmulArguments {
+ a: &input,
+ a_offset: 0,
+ b,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut output,
+ d_transform,
+ m: batch_dim as u32,
+ n: self.output_dim as u32,
+ k: self.input_dim as u32,
+ },
+ encoder,
+ )?;
+
+ Ok(output)
+ }
+}
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index 57eb8995a..a7056b36c 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -1,11 +1,9 @@
-mod full_precision;
+mod matmul;
mod qlora_wrapper;
-mod quantized;
mod rht_wrapper;
-pub use full_precision::{FullPrecisionLinear, FullPrecisionLinearError};
+pub use matmul::{LinearMatmul, LinearMatmulError};
pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};
-pub use quantized::{QuantizedLinear, QuantizedLinearError};
pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};
use thiserror::Error;
@@ -26,10 +24,8 @@ pub trait Linear<B: Backend> {
#[derive(Debug, Error)]
pub enum LinearBlockError<B: Backend> {
- #[error("QuantizedLinear error: {0}")]
- QuantizedLinearError(#[from] QuantizedLinearError<B>),
- #[error("FullPrecisionLinear error: {0}")]
- FullPrecisionLinearError(#[from] FullPrecisionLinearError<B>),
+ #[error("LinearMatmul error: {0}")]
+ LinearMatmulError(#[from] LinearMatmulError<B>),
#[error("QLoRALinearWrapper error: {0}")]
QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),
#[error("RHTLinearWrapper error: {0}")]
@@ -49,7 +45,7 @@ impl<B: Backend> dyn Linear<B> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
match config {
LinearConfig::Quantized(quantization_config) | LinearConfig::ScaleBiasQuantized(quantization_config) => {
- let block = QuantizedLinear::new(
+ let block = LinearMatmul::quantized(
context,
quantization_config,
input_dimension,
@@ -62,7 +58,7 @@ impl<B: Backend> dyn Linear<B> {
LinearConfig::FullPrecision {
precision,
} => {
- let block = FullPrecisionLinear::new(
+ let block = LinearMatmul::full_precision(
context,
(*precision).into(),
input_dimension,
@@ -112,7 +108,7 @@ impl<B: Backend> dyn Linear<B> {
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
match config {
LinearConfig::Quantized(config) | LinearConfig::ScaleBiasQuantized(config) => Ok(Box::new(
- QuantizedLinear::new(context, config, input_dim, output_dim, parameter_tree, Some(output_factors))?,
+ LinearMatmul::quantized(context, config, input_dim, output_dim, parameter_tree, Some(output_factors))?,
)),
LinearConfig::QLoRA {
quantization,
diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
index 08b5516f7..c3fae59ec 100644
--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -9,13 +9,13 @@ use crate::{
Allocation, Backend, Encoder,
kernel::{
ManualKernels,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
config::QuantizationConfig,
encodable_block::{
Linear,
- linear::{LinearBlockError, QuantizedLinear, QuantizedLinearError},
+ linear::{LinearBlockError, LinearMatmul, LinearMatmulError},
},
prelude::{ParameterLoaderError, ParameterTree},
};
@@ -24,16 +24,16 @@ use crate::{
pub enum QLoRALinearWrapperError<B: Backend> {
#[error("Inner linear error: {0}")]
InnerLinearError(#[from] Box<LinearBlockError<B>>),
- #[error("Quantized linear error: {0}")]
- QuantizedLinearError(#[from] QuantizedLinearError<B>),
+ #[error("Linear matmul error: {0}")]
+ LinearMatmulError(#[from] LinearMatmulError<B>),
#[error("Parameter loader error: {0}")]
ParameterLoaderError(#[from] ParameterLoaderError<B>),
- #[error("Matmul error: {0}")]
- MatmulError(#[from] MatmulError<B>),
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
}
pub struct QLoRALinearWrapper<B: Backend> {
- base_linear: QuantizedLinear<B>,
+ base_linear: LinearMatmul<B>,
adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
adapter_down: Allocation<B>,
adapter_up: Allocation<B>,
@@ -57,7 +57,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
) -> Result<Self, QLoRALinearWrapperError<B>> {
let data_type = quantization.activation_precision.into();
- let base_linear = QuantizedLinear::new(
+ let base_linear = LinearMatmul::quantized(
context,
quantization,
input_dim,
@@ -65,8 +65,10 @@ impl<B: Backend> QLoRALinearWrapper<B> {
parameter_tree,
output_quantized_hadamard_factors,
)?;
- let adapter_kernel =
- RefCell::new(<<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)?);
+ let adapter_kernel = RefCell::new(
+ <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)
+ .map_err(QLoRALinearWrapperError::BackendError)?,
+ );
let adapter_down_leaf = parameter_tree.leaf("down_weights")?;
adapter_down_leaf.validate_shape(&[lora_rank, input_dim], data_type)?;
@@ -105,19 +107,20 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
MatmulArguments {
a: &input,
a_offset: 0,
- b: &self.adapter_down,
+ b: MatmulB::FullPrecision {
+ b: &self.adapter_down,
+ },
b_offset: 0,
b_leading_dimension: None,
b_transpose: true,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
d: &mut intermediate,
- batch_dim: batch_dim as u32,
- input_dim: self.input_dim as u32,
- output_dim: self.lora_rank as u32,
+ d_transform: MatmulDOps::none(),
+ m: batch_dim as u32,
+ n: self.lora_rank as u32,
+ k: self.input_dim as u32,
},
encoder,
- );
+ )?;
let mut output = self.base_linear.encode(input, batch_dim, encoder)?;
@@ -125,19 +128,25 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
MatmulArguments {
a: &intermediate,
a_offset: 0,
- b: &self.adapter_up,
+ b: MatmulB::FullPrecision {
+ b: &self.adapter_up,
+ },
b_offset: 0,
b_leading_dimension: None,
b_transpose: true,
- ab_scale: self.lora_scale,
- c: MatmulArgumentC::Accumulate,
d: &mut output,
- batch_dim: batch_dim as u32,
- input_dim: self.lora_rank as u32,
- output_dim: self.output_dim as u32,
+ d_transform: MatmulDOps {
+ ab_scale: self.lora_scale,
+ accumulate: true,
+ bias: None,
+ rht_factors: None,
+ },
+ m: batch_dim as u32,
+ n: self.output_dim as u32,
+ k: self.lora_rank as u32,
},
encoder,
- );
+ )?;
Ok(output)
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/quantized.rs b/crates/backend-uzu/src/encodable_block/linear/quantized.rs
deleted file mode 100644
index 510605d86..000000000
--- a/crates/backend-uzu/src/encodable_block/linear/quantized.rs
+++ /dev/null
@@ -1,283 +0,0 @@
-use thiserror::Error;
-
-use super::Linear;
-use crate::{
- DataType,
- array::size_for_shape,
- backends::common::{
- Allocation, Backend, Encoder,
- gpu_types::QuantizationMethod,
- kernel::{
- Kernels, TensorAddBiasKernel,
- quant_matmul::{
- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
- QuantizedMatmulKernelEncodable,
- },
- },
- },
- config::QuantizationConfig,
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-#[derive(Debug, Error)]
-pub enum QuantizedLinearError<B: Backend> {
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("QuantizedMatmul error: {0}")]
- QuantizedMatmulError(#[source] QuantizedMatmulError<B>),
- #[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError<B>),
- #[error("Unsupported data type for quantized kernel: {0:?}")]
- UnsupportedDataType(DataType),
- #[error("Expected weights of type {expected:?}, got {got:?}")]
- InvalidWeightsDataType {
- expected: DataType,
- got: DataType,
- },
- #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidScalesDataType {
- expected: DataType,
- got: DataType,
- },
- #[error(
- "Unexpected scale-bias shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"
- )]
- InvalidScaleBiasShapes {
- weights: Box<[usize]>,
- scales: Box<[usize]>,
- deq_biases: Box<[usize]>,
- packing_divisor: usize,
- },
- #[error("deq_biases dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidDeqBiasesDataType {
- expected: DataType,
- got: DataType,
- },
- #[error(
- "Unexpected scale-zero-point shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"
- )]
- InvalidScaleZeroPointShapes {
- weights: Box<[usize]>,
- scales: Box<[usize]>,
- zero_points: Box<[usize]>,
- packing_divisor: usize,
- packing_minus_one: usize,
- },
- #[error("Zero-points dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidZeroPointsDataType {
- expected: DataType,
- got: DataType,
- },
- #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]
- InvalidBiasShape {
- got: Box<[usize]>,
- expected_output_dim: usize,
- },
- #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidBiasDataType {
- expected: DataType,
- got: DataType,
- },
-}
-
-pub struct QuantizedLinear<B: Backend> {
- kernel: QuantizedMatmulKernelEncodable<B>,
- bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
- biases: Option<Allocation<B>>,
- weights: Allocation<B>,
- scales: Allocation<B>,
- zero_points_or_biases: Allocation<B>,
- output_hadamard_factors: Option<Allocation<B>>,
- output_dim: usize,
- output_data_type: DataType,
-}
-
-impl<B: Backend> QuantizedLinear<B> {
- pub fn new(
- context: &B::Context,
- config: &QuantizationConfig,
- input_dim: usize,
- output_dim: usize,
- parameter_tree: &ParameterTree<B::Context>,
- output_hadamard_factors: Option<Allocation<B>>,
- ) -> Result<Self, QuantizedLinearError<B>> {
- let kernel_data_type: DataType = config.activation_precision.into();
- if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(QuantizedLinearError::UnsupportedDataType(kernel_data_type));
- }
-
- let weights_leaf = parameter_tree.leaf("weights").map_err(QuantizedLinearError::ParameterError)?;
- let packing_divisor = config.weight_quantization_mode.packing_divisor();
- let storage_type = config.weight_quantization_mode.storage_type();
- if weights_leaf.data_type() != storage_type {
- return Err(QuantizedLinearError::InvalidWeightsDataType {
- expected: storage_type,
- got: weights_leaf.data_type(),
- });
- }
-
- let scales_leaf = parameter_tree.leaf("scales").map_err(QuantizedLinearError::ParameterError)?;
- if scales_leaf.data_type() != kernel_data_type {
- return Err(QuantizedLinearError::InvalidScalesDataType {
- expected: kernel_data_type,
- got: scales_leaf.data_type(),
- });
- }
-
- let k_g = (input_dim + config.group_size - 1) / config.group_size;
- let weights_shape = weights_leaf.shape().to_vec();
- let scales_shape = scales_leaf.shape().to_vec();
- let (quantization_method, zero_points_or_biases) = match parameter_tree.leaf("deq_biases") {
- Ok(deq_biases) => {
- let deq_biases_shape = deq_biases.shape().to_vec();
- if !(weights_shape == [output_dim, input_dim / packing_divisor]
- && scales_shape == [output_dim, k_g]
- && deq_biases_shape == [output_dim, k_g])
- {
- return Err(QuantizedLinearError::InvalidScaleBiasShapes {
- weights: weights_shape.into_boxed_slice(),
- scales: scales_shape.into_boxed_slice(),
- deq_biases: deq_biases_shape.into_boxed_slice(),
- packing_divisor,
- });
- }
-
- if deq_biases.data_type() != kernel_data_type {
- return Err(QuantizedLinearError::InvalidDeqBiasesDataType {
- expected: kernel_data_type,
- got: deq_biases.data_type(),
- });
- }
-
- (
- QuantizationMethod::ScaleBias,
- deq_biases.read_allocation().map_err(QuantizedLinearError::ParameterError)?,
- )
- },
- Err(_) => {
- let zero_points_leaf =
- parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;
- let zero_points_shape = zero_points_leaf.shape().to_vec();
- let expected_zero_points_entries = (k_g + packing_divisor - 1) / packing_divisor;
- if !(weights_shape == [output_dim, input_dim / packing_divisor]
- && scales_shape == [output_dim, k_g]
- && zero_points_shape == [output_dim, expected_zero_points_entries])
- {
- return Err(QuantizedLinearError::InvalidScaleZeroPointShapes {
- weights: weights_shape.into_boxed_slice(),
- scales: scales_shape.into_boxed_slice(),
- zero_points: zero_points_shape.into_boxed_slice(),
- packing_divisor,
- packing_minus_one: packing_divisor - 1,
- });
- }
-
- if zero_points_leaf.data_type() != storage_type {
- return Err(QuantizedLinearError::InvalidZeroPointsDataType {
- expected: storage_type,
- got: zero_points_leaf.data_type(),
- });
- }
-
- (
- QuantizationMethod::ScaleZeroPoint,
- zero_points_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,
- )
- },
- };
-
- let (bias_add_kernel, biases) = match parameter_tree.leaf("biases") {
- Ok(biases_leaf) => {
- let bias_shape = biases_leaf.shape().to_vec();
- if bias_shape != [output_dim] {
- return Err(QuantizedLinearError::InvalidBiasShape {
- got: bias_shape.into_boxed_slice(),
- expected_output_dim: output_dim,
- });
- }
-
- if biases_leaf.data_type() != kernel_data_type {
- return Err(QuantizedLinearError::InvalidBiasDataType {
- expected: kernel_data_type,
- got: biases_leaf.data_type(),
- });
- }
-
- let bias_add_kernel =
- <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, kernel_data_type, true)
- .map_err(QuantizedLinearError::BackendError)?;
- (
- Some(bias_add_kernel),
- Some(biases_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?),
- )
- },
- Err(_) => (None, None),
- };
-
- let kernel = QuantizedMatmulKernelEncodable::new(
- context,
- QuantizedMatmulConfiguration {
- data_type: kernel_data_type,
- group_size: config.group_size,
- input_dim,
- output_dim,
- mode: config.weight_quantization_mode,
- quantization_method,
- use_hadamard: output_hadamard_factors.is_some(),
- },
- )
- .map_err(QuantizedLinearError::QuantizedMatmulError)?;
-
- Ok(Self {
- kernel,
- bias_add_kernel,
- biases,
- weights: weights_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,
- scales: scales_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,
- zero_points_or_biases,
- output_hadamard_factors,
- output_dim,
- output_data_type: kernel_data_type,
- })
- }
-}
-
-impl<B: Backend> Linear<B> for QuantizedLinear<B> {
- fn encode(
- &self,
- input: Allocation<B>,
- batch_dim: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
- let mut output =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;
-
- self.kernel.encode(
- encoder,
- QuantizedMatmulArguments {
- a: &input,
- a_offset: 0,
- b: &self.weights,
- scales: &self.scales,
- zero_points_or_biases: &self.zero_points_or_biases,
- output: &mut output,
- hadamard_factors: self.output_hadamard_factors.as_ref(),
- batch_dim,
- },
- );
-
- if let (Some(bias_add_kernel), Some(biases)) = (&self.bias_add_kernel, &self.biases) {
- let total_length = batch_dim * self.output_dim;
- bias_add_kernel.encode(
- None::<&Allocation<B>>,
- biases,
- &mut output,
- self.output_dim as u32,
- total_length as u32,
- encoder,
- );
- }
-
- Ok(output)
- }
-}
diff --git a/crates/backend-uzu/src/session/tts_session.rs b/crates/backend-uzu/src/session/tts_session.rs
index 6a4198e3a..7f3246ece 100644
--- a/crates/backend-uzu/src/session/tts_session.rs
+++ b/crates/backend-uzu/src/session/tts_session.rs
@@ -34,7 +34,7 @@ use crate::{
kernel::{
EmbeddingRowsSumKernel, ManualKernels, TensorAddScaleKernel, TokenCopySampledKernel,
kv_cache_update::KVCacheUpdate,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
config::{ModelMetadata, TtsMessageProcessorConfig, TtsModelConfig},
diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
index eb87e5091..d4371fbe7 100644
--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
@@ -483,23 +483,27 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
if semantic_bridge.fast_model_dim != fast_model_dim || semantic_bridge.slow_model_dim != slow_model_dim {
return Err(Error::GenerateFailed);
}
- semantic_bridge.projection.encode(
- MatmulArguments {
- a: slow_hidden_capture,
- a_offset: 0,
- b: weights,
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
- d: output_embedding,
- batch_dim: 1,
- input_dim: slow_model_dim as u32,
- output_dim: fast_model_dim as u32,
- },
- encoder,
- );
+ semantic_bridge
+ .projection
+ .encode(
+ MatmulArguments {
+ a: slow_hidden_capture,
+ a_offset: 0,
+ b: MatmulB::FullPrecision {
+ b: weights,
+ },
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: output_embedding,
+ d_transform: MatmulDOps::none(),
+ m: 1,
+ n: fast_model_dim as u32,
+ k: slow_model_dim as u32,
+ },
+ encoder,
+ )
+ .expect("encode failed");
return Ok(());
}
diff --git a/crates/backend-uzu/tests/common/matmul/bench.rs b/crates/backend-uzu/tests/common/matmul/bench.rs
new file mode 100644
index 000000000..5626f8523
--- /dev/null
+++ b/crates/backend-uzu/tests/common/matmul/bench.rs
@@ -0,0 +1,18 @@
+use backend_uzu::backends::common::{Backend, Encoder};
+use criterion::Bencher;
+
+pub fn iter_encode_loop<B: Backend, F>(
+ context: &B::Context,
+ bencher: &mut Bencher,
+ mut encode: F,
+) where
+ F: FnMut(&mut Encoder<B>),
+{
+ bencher.iter_custom(|n_iters| {
+ let mut encoder = Encoder::<B>::new(context).unwrap();
+ for _ in 0..n_iters {
+ encode(&mut encoder);
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
+ });
+}
diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rs
index 4aeb83d5d..105f169db 100644
--- a/crates/backend-uzu/tests/common/matmul/harness.rs
+++ b/crates/backend-uzu/tests/common/matmul/harness.rs
@@ -1,5 +1,5 @@
#[cfg(metal_backend)]
-use backend_uzu::backends::metal::{Metal, MetalContext};
+use backend_uzu::backends::metal::{GemmDispatchPath, Metal, MetalContext};
use backend_uzu::{
ArrayContextExt, ArrayElement,
backends::{
@@ -7,7 +7,7 @@ use backend_uzu::{
Allocation, AllocationType, AsBufferRangeRef, Backend, Buffer, Context, Encoder,
kernel::{
ManualKernels,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
cpu::Cpu,
@@ -17,7 +17,7 @@ use num_traits::Float;
use super::{
super::helpers::{alloc_allocation_with_data, allocation_to_vec},
- Shape, Variant,
+ Shape,
};
#[cfg(metal_backend)]
@@ -109,10 +109,11 @@ fn run<B: Backend, T: ArrayElement + Float>(
.expect("create d allocation")
};
- let c_arg = if input.case.accumulate {
- MatmulArgumentC::Accumulate
- } else {
- MatmulArgumentC::None
+ let d_transform = MatmulDOps::<'_, B> {
+ ab_scale: input.case.ab_scale,
+ accumulate: input.case.accumulate,
+ bias: None,
+ rht_factors: None,
};
let mut encoder = Encoder::new(context).expect("encoder");
@@ -121,16 +122,17 @@ fn run<B: Backend, T: ArrayElement + Float>(
MatmulArguments::<'_, B> {
a: &a_allocation,
a_offset: 0,
- b: b_array.allocation(),
+ b: MatmulB::FullPrecision {
+ b: b_array.allocation(),
+ },
b_offset: 0,
b_leading_dimension: None,
b_transpose: input.case.b_transpose,
- ab_scale: input.case.ab_scale,
- c: c_arg,
d: &mut d_allocation,
- batch_dim: m as u32,
- input_dim: k as u32,
- output_dim: n as u32,
+ d_transform,
+ m: m as u32,
+ n: n as u32,
+ k: k as u32,
},
&mut encoder,
);
@@ -143,7 +145,7 @@ pub fn cpu_reference<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let mut kernel = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
.expect("CPU MatmulKernel");
run::<Cpu, T>(&context, &mut kernel, input, |kernel, args, encoder| {
- kernel.encode(args, encoder);
+ kernel.encode(args, encoder).expect("encode failed");
})
}
@@ -152,10 +154,12 @@ pub fn run_metal<T: ArrayElement + Float>(
context: &MetalContext,
kernel: &mut MetalMatmulKernel,
input: &Input<T>,
- variant: Variant,
+ path: Option<GemmDispatchPath>,
) -> Vec<T> {
- let path = variant.dispatch_path();
- run::<Metal, T>(context, kernel, input, |kernel, args, encoder| {
- kernel.encode_with_path(args, encoder, path);
+ run::<Metal, T>(context, kernel, input, |kernel, args, encoder| match path {
+ None => kernel.encode(args, encoder).expect("matmul encode failed"),
+ Some(gemm_path) => {
+ kernel.gemm.encode_dispatch_path(args, gemm_path, encoder).expect("gemm encode_dispatch_path failed")
+ },
})
}
diff --git a/crates/backend-uzu/tests/common/matmul/mod.rs b/crates/backend-uzu/tests/common/matmul/mod.rs
index da729946d..2647d5809 100644
--- a/crates/backend-uzu/tests/common/matmul/mod.rs
+++ b/crates/backend-uzu/tests/common/matmul/mod.rs
@@ -1,9 +1,16 @@
+pub mod bench;
pub mod harness;
+pub mod quant;
pub mod shape;
-pub mod variant;
+pub use bench::iter_encode_loop;
#[cfg(metal_backend)]
pub use harness::run_metal;
pub use harness::{Case, cpu_reference, deterministic_input};
-pub use shape::{Shape, all_correctness_shapes};
-pub use variant::Variant;
+#[cfg(metal_backend)]
+pub use quant::run_quant_metal;
+pub use quant::{QuantBuffers, QuantInput, quant_arguments, run_quant_cpu};
+pub use shape::{
+ Shape, all_correctness_shapes, bench_fp_gemm_shapes, bench_quant_gemm_shapes, bench_quant_gemv_shapes,
+ qwen3_layer_shapes,
+};
diff --git a/crates/backend-uzu/tests/common/matmul/quant.rs b/crates/backend-uzu/tests/common/matmul/quant.rs
new file mode 100644
index 000000000..c6cc2bdfa
--- /dev/null
+++ b/crates/backend-uzu/tests/common/matmul/quant.rs
@@ -0,0 +1,185 @@
+#[cfg(metal_backend)]
+use backend_uzu::backends::metal::{GemmDispatchPath, Metal, MetalContext};
+use backend_uzu::{
+ ArrayElement,
+ backends::{
+ common::{
+ Allocation, Backend, Context, Encoder,
+ gpu_types::{QuantizationMethod, QuantizationMode},
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ },
+ },
+ cpu::Cpu,
+ },
+};
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+
+use super::super::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec};
+
+pub struct QuantInput<T: ArrayElement + Float> {
+ pub w_packed: Vec<u32>,
+ pub scales: Vec<T>,
+ pub zero_points: Option<Vec<u8>>,
+ pub biases: Option<Vec<T>>,
+ pub x: Vec<T>,
+ pub k: u32,
+ pub n: u32,
+ pub m: u32,
+ pub group_size: u32,
+ pub quant_method: QuantizationMethod,
+ pub mode: QuantizationMode,
+}
+
+fn mode_for_bits(bits: u32) -> QuantizationMode {
+ match bits {
+ 4 => QuantizationMode::U4,
+ 8 => QuantizationMode::I8,
+ _ => unreachable!("unsupported bits: {bits}"),
+ }
+}
+
+impl<T: ArrayElement + Float> QuantInput<T> {
+ pub fn new(
+ m: usize,
+ k: usize,
+ n: usize,
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+ seed: u64,
+ ) -> Self {
+ let num_groups_k = k.div_ceil(group_size as usize);
+ let mut rng = SmallRng::seed_from_u64(seed);
+
+ let w_packed: Vec<u32> = (0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect();
+ let scales: Vec<T> =
+ (0..n * num_groups_k).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();
+ let x: Vec<T> = (0..m * k).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();
+
+ let zp_stride = if bits == 4 {
+ num_groups_k.div_ceil(2)
+ } else {
+ num_groups_k
+ };
+ let (zero_points, biases) = match quant_method {
+ QuantizationMethod::ScaleZeroPoint => {
+ (Some((0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)
+ },
+ QuantizationMethod::ScaleBias => (
+ None,
+ Some((0..n * num_groups_k).map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap()).collect()),
+ ),
+ };
+
+ Self {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ quant_method,
+ mode: mode_for_bits(bits),
+ }
+ }
+}
+
+pub struct QuantBuffers<B: Backend, T: ArrayElement + Float> {
+ pub w: Allocation<B>,
+ pub scales: Allocation<B>,
+ pub zp: Option<Allocation<B>>,
+ pub bias: Option<Allocation<B>>,
+ pub x: Allocation<B>,
+ pub y: Allocation<B>,
+ _t: std::marker::PhantomData<T>,
+}
+
+impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {
+ pub fn allocate(
+ context: &B::Context,
+ input: &QuantInput<T>,
+ ) -> Self {
+ Self {
+ w: alloc_allocation_with_data::<B, u32>(context, &input.w_packed),
+ scales: alloc_allocation_with_data::<B, T>(context, &input.scales),
+ zp: input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(context, zp)),
+ bias: input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(context, b)),
+ x: alloc_allocation_with_data::<B, T>(context, &input.x),
+ y: alloc_allocation::<B, T>(context, (input.m as usize) * (input.n as usize)),
+ _t: std::marker::PhantomData,
+ }
+ }
+}
+
+pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(
+ buffers: &'a mut QuantBuffers<B, T>,
+ input: &QuantInput<T>,
+) -> MatmulArguments<'a, B> {
+ let b_variant = match input.quant_method {
+ QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
+ b: &buffers.w,
+ scales: &buffers.scales,
+ zero_points: buffers.zp.as_ref().expect("zp buffer"),
+ mode: input.mode,
+ group_size: input.group_size,
+ },
+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
+ b: &buffers.w,
+ scales: &buffers.scales,
+ biases: buffers.bias.as_ref().expect("bias buffer"),
+ mode: input.mode,
+ group_size: input.group_size,
+ },
+ };
+ MatmulArguments {
+ a: &buffers.x,
+ a_offset: 0,
+ b: b_variant,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut buffers.y,
+ d_transform: MatmulDOps::none(),
+ m: input.m,
+ n: input.n,
+ k: input.k,
+ }
+}
+
+pub fn run_quant_cpu<T: ArrayElement + Float>(input: &QuantInput<T>) -> Vec<T> {
+ let context = <Cpu as Backend>::Context::new().expect("Cpu context");
+ let mut buffers = QuantBuffers::<Cpu, T>::allocate(&context, input);
+ let mut matmul = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
+ .expect("MatmulCpuKernel");
+ let mut encoder = Encoder::<Cpu>::new(&context).expect("encoder");
+ matmul.encode(quant_arguments(&mut buffers, input), &mut encoder).expect("encode cpu quant");
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ allocation_to_vec::<Cpu, T>(&buffers.y)
+}
+
+#[cfg(metal_backend)]
+pub fn run_quant_metal<T: ArrayElement + Float>(
+ context: &MetalContext,
+ input: &QuantInput<T>,
+ path: Option<GemmDispatchPath>,
+) -> Vec<T> {
+ let mut buffers = QuantBuffers::<Metal, T>::allocate(context, input);
+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type())
+ .expect("MatmulMetalKernel");
+ let mut encoder = Encoder::<Metal>::new(context).expect("encoder");
+ let args = quant_arguments(&mut buffers, input);
+ match path {
+ None => matmul.encode(args, &mut encoder).expect("matmul encode failed"),
+ Some(gemm_path) => {
+ matmul.gemm.encode_dispatch_path(args, gemm_path, &mut encoder).expect("gemm encode_dispatch_path failed")
+ },
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ allocation_to_vec::<Metal, T>(&buffers.y)
+}
diff --git a/crates/backend-uzu/tests/common/matmul/shape.rs b/crates/backend-uzu/tests/common/matmul/shape.rs
index 3165f2031..33e7d58ef 100644
--- a/crates/backend-uzu/tests/common/matmul/shape.rs
+++ b/crates/backend-uzu/tests/common/matmul/shape.rs
@@ -37,3 +37,79 @@ const SHAPES_MEDIUM: &[Shape] = &[Shape::new(128, 2048, 2048), Shape::new(256, 4
pub fn all_correctness_shapes() -> impl Iterator<Item = Shape> {
SHAPES_TINY.iter().chain(SHAPES_UNALIGNED.iter()).chain(SHAPES_MEDIUM.iter()).copied()
}
+
+const BENCH_FP_GEMM: &[Shape] = &[
+ Shape::new(128, 2048, 8192),
+ Shape::new(128, 4096, 14336),
+ Shape::new(256, 4096, 4096),
+ Shape::new(512, 8192, 2048),
+];
+
+pub fn bench_fp_gemm_shapes() -> impl Iterator<Item = Shape> {
+ BENCH_FP_GEMM.iter().copied()
+}
+
+const BENCH_NK: &[(usize, usize)] =
+ &[(2048, 2048), (2048, 4096), (4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
+
+pub fn bench_quant_gemm_shapes(bits: u32) -> impl Iterator<Item = Shape> {
+ let block_size: usize = if bits == 4 {
+ 512
+ } else {
+ 256
+ };
+ let ms = &[4usize, 5, 6, 7, 8, 16, 32, 48, 64];
+ BENCH_NK
+ .iter()
+ .filter(move |&&(n, k)| n % 32 == 0 && k % block_size == 0)
+ .flat_map(move |&(n, k)| ms.iter().map(move |&m| Shape::new(m, k, n)))
+}
+
+pub fn bench_quant_gemv_shapes(bits: u32) -> impl Iterator<Item = Shape> {
+ let block_size: usize = if bits == 4 {
+ 512
+ } else {
+ 256
+ };
+ let nk: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
+ let ms = &[1usize, 2, 4, 8];
+ nk.iter()
+ .filter(move |&&(n, k)| n % 8 == 0 && k % block_size == 0)
+ .flat_map(move |&(n, k)| ms.iter().map(move |&m| Shape::new(m, k, n)))
+}
+
+const QWEN3_LAYERS: &[(&str, usize, usize)] = &[
+ ("0.6b_qkv", 1024, 4096),
+ ("0.6b_o", 2048, 1024),
+ ("0.6b_gateup", 1024, 6144),
+ ("0.6b_down", 3072, 1024),
+ ("1.7b_qkv", 2048, 4096),
+ ("1.7b_o", 2048, 2048),
+ ("1.7b_gateup", 2048, 12288),
+ ("1.7b_down", 6144, 2048),
+ ("4b_qkv", 2560, 6144),
+ ("4b_o", 4096, 2560),
+ ("4b_gateup", 2560, 19456),
+ ("4b_down", 9728, 2560),
+ ("8b_qkv", 4096, 6144),
+ ("8b_o", 4096, 4096),
+ ("8b_gateup", 4096, 24576),
+ ("8b_down", 12288, 4096),
+];
+
+pub fn qwen3_layer_shapes(bits: u32) -> impl Iterator<Item = (&'static str, Shape)> {
+ let block_size: usize = if bits == 4 {
+ 512
+ } else {
+ 256
+ };
+ let decode_ms = &[1usize, 4, 8];
+ let prefill_ms = &[32usize, 64];
+ QWEN3_LAYERS.iter().flat_map(move |&(label, k, n)| {
+ decode_ms
+ .iter()
+ .chain(prefill_ms.iter())
+ .filter(move |_| k % block_size == 0)
+ .map(move |&m| (label, Shape::new(m, k, n)))
+ })
+}
diff --git a/crates/backend-uzu/tests/common/matmul/variant.rs b/crates/backend-uzu/tests/common/matmul/variant.rs
deleted file mode 100644
index 34ce795ea..000000000
--- a/crates/backend-uzu/tests/common/matmul/variant.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-#[cfg(metal_backend)]
-use backend_uzu::backends::metal::{DeviceExt, MatmulDispatchPath, MetalContext};
-use derive_more::Display;
-
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
-pub enum Variant {
- #[display("GEMM")]
- Gemm,
- #[display("GEMM_MXU")]
- GemmMxu,
-}
-
-impl Variant {
- pub const fn requires_mxu(self) -> bool {
- matches!(self, Variant::GemmMxu)
- }
-
- #[cfg(metal_backend)]
- pub fn supported(
- self,
- context: &MetalContext,
- ) -> bool {
- !self.requires_mxu() || context.device.supports_mxu()
- }
-
- #[cfg(metal_backend)]
- pub const fn dispatch_path(self) -> MatmulDispatchPath {
- match self {
- Variant::Gemm => MatmulDispatchPath::Gemm,
- Variant::GemmMxu => MatmulDispatchPath::GemmMxu,
- }
- }
-}
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs
new file mode 100644
index 000000000..6c091c87e
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs
@@ -0,0 +1,74 @@
+#![cfg(metal_backend)]
+
+use backend_uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::{
+ common::{
+ AllocationType, Backend, Context,
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ },
+ },
+ metal::{GemmDispatchPath, Metal, MetalContext},
+ },
+};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::bf16;
+
+use crate::{
+ common::{
+ matmul::{bench_fp_gemm_shapes, iter_encode_loop},
+ type_short_name,
+ },
+ uzu_bench,
+};
+
+#[uzu_bench]
+fn bench_gemm(c: &mut Criterion) {
+ let context = MetalContext::new().expect("Metal context");
+ let mut kernel =
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())
+ .expect("MatmulKernel");
+
+ let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));
+
+ for shape in bench_fp_gemm_shapes() {
+ let (m, k, n) = (shape.m, shape.k, shape.n);
+ let a = context
+ .create_allocation(m * k * std::mem::size_of::<bf16>(), AllocationType::Global)
+ .expect("a allocation");
+ let b_array = context.create_array_uninitialized(&[n, k], bf16::data_type());
+ let mut d = context
+ .create_allocation(m * n * std::mem::size_of::<bf16>(), AllocationType::Global)
+ .expect("d allocation");
+
+ group.throughput(Throughput::Elements((2 * m * k * n) as u64));
+ group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {
+ iter_encode_loop::<Metal, _>(&context, b, |encoder| {
+ kernel
+ .gemm
+ .encode_dispatch_path(
+ MatmulArguments {
+ a: &a,
+ a_offset: 0,
+ b: MatmulB::FullPrecision {
+ b: b_array.allocation(),
+ },
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut d,
+ d_transform: MatmulDOps::none(),
+ m: m as u32,
+ n: n as u32,
+ k: k as u32,
+ },
+ GemmDispatchPath::Mxu,
+ encoder,
+ )
+ .expect("encode_dispatch_path failed");
+ });
+ });
+ }
+}
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/mod.rs b/crates/backend-uzu/tests/performance/kernel/matmul/mod.rs
new file mode 100644
index 000000000..8ba502560
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/mod.rs
@@ -0,0 +1,4 @@
+mod gemm_bench;
+mod quant_gemm_bench;
+mod quant_gemv_bench;
+mod qwen3_bench;
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs
new file mode 100644
index 000000000..55920ac09
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs
@@ -0,0 +1,68 @@
+#![cfg(metal_backend)]
+
+use backend_uzu::{
+ ArrayElement,
+ backends::{
+ common::{
+ Backend, Context,
+ gpu_types::QuantizationMethod,
+ kernel::{ManualKernels, matmul::MatmulKernel},
+ },
+ metal::{GemmDispatchPath, Metal, MetalContext},
+ },
+};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::{
+ common::{
+ matmul::{QuantBuffers, QuantInput, bench_quant_gemm_shapes, iter_encode_loop, quant_arguments},
+ type_short_name,
+ },
+ uzu_bench,
+};
+
+fn bench_unified_quant_typed<T: ArrayElement + Float>(
+ c: &mut Criterion,
+ context: &MetalContext,
+ label: &str,
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+) {
+ for shape in bench_quant_gemm_shapes(bits) {
+ let (m, k, n) = (shape.m, shape.k, shape.n);
+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);
+ let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);
+ let mut matmul =
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type()).unwrap();
+
+ let mut group =
+ c.benchmark_group(format!("{}/Kernel/UnifiedQuantizedGemm/{}", type_short_name::<Metal>(), label));
+ group.throughput(Throughput::Elements((m * n * k) as u64));
+ group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {
+ iter_encode_loop::<Metal, _>(context, b, |encoder| {
+ matmul
+ .gemm
+ .encode_dispatch_path(
+ quant_arguments(&mut buffers, &input),
+ GemmDispatchPath::Simdgroup,
+ encoder,
+ )
+ .expect("encode unified quant matmul");
+ });
+ });
+ drop(group);
+ }
+}
+
+#[uzu_bench]
+fn bench_unified_quantized_gemm(c: &mut Criterion) {
+ let context = MetalContext::new().expect("Metal context");
+ bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
+ bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
+ bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_unified_quant_typed::<f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+}
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
new file mode 100644
index 000000000..41e15db09
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
@@ -0,0 +1,78 @@
+use backend_uzu::{
+ ArrayElement,
+ backends::common::{
+ Allocation, Backend, Context, Kernels, gpu_types::QuantizationMethod, kernel::QuantizedMatmulQmvFastKernel,
+ },
+};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::{
+ common::{
+ matmul::{QuantBuffers, QuantInput, bench_quant_gemv_shapes, iter_encode_loop},
+ type_short_name,
+ },
+ uzu_bench,
+};
+
+fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
+ c: &mut Criterion,
+ context: &B::Context,
+ label: &str,
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+) {
+ let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));
+
+ for shape in bench_quant_gemv_shapes(bits) {
+ let (m, k, n) = (shape.m, shape.k, shape.n);
+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);
+ let mut buffers = QuantBuffers::<B, T>::allocate(context, &input);
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ context,
+ T::data_type(),
+ group_size,
+ bits,
+ quant_method,
+ false,
+ )
+ .unwrap();
+
+ group.throughput(Throughput::Elements((m * n * k) as u64));
+ group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {
+ iter_encode_loop::<B, _>(context, b, |encoder| {
+ kernel.encode(
+ &buffers.w,
+ &buffers.scales,
+ buffers.zp.as_ref(),
+ buffers.bias.as_ref(),
+ &buffers.x,
+ &mut buffers.y,
+ None::<&Allocation<B>>,
+ k as u32,
+ n as u32,
+ m as u32,
+ encoder,
+ );
+ });
+ });
+ }
+}
+
+#[uzu_bench]
+fn bench_qmv_fast(c: &mut Criterion) {
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);
+ });
+}
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs
new file mode 100644
index 000000000..5fe879f7f
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs
@@ -0,0 +1,57 @@
+#![cfg(metal_backend)]
+
+use backend_uzu::{
+ ArrayElement,
+ backends::{
+ common::{
+ Backend, Context,
+ gpu_types::QuantizationMethod,
+ kernel::{ManualKernels, matmul::MatmulKernel},
+ },
+ metal::{Metal, MetalContext},
+ },
+};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::bf16;
+use num_traits::Float;
+
+use crate::{
+ common::{
+ matmul::{QuantBuffers, QuantInput, iter_encode_loop, quant_arguments, qwen3_layer_shapes},
+ type_short_name,
+ },
+ uzu_bench,
+};
+
+fn bench_qwen3_layers_typed<T: ArrayElement + Float>(
+ c: &mut Criterion,
+ context: &MetalContext,
+ label: &str,
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+) {
+ let mut group =
+ c.benchmark_group(format!("{}/Kernel/Qwen3Layers/{}", type_short_name::<Metal>(), label));
+ for (layer, shape) in qwen3_layer_shapes(bits) {
+ let (m, k, n) = (shape.m, shape.k, shape.n);
+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);
+ let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);
+ let mut matmul =
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type()).unwrap();
+
+ group.throughput(Throughput::Elements((m * n * k) as u64));
+ group.bench_function(BenchmarkId::from_parameter(format!("{layer}_{shape}")), |b| {
+ iter_encode_loop::<Metal, _>(context, b, |encoder| {
+ matmul.encode(quant_arguments(&mut buffers, &input), encoder).expect("encode qwen3 layer");
+ });
+ });
+ }
+}
+
+#[uzu_bench]
+fn bench_qwen3_layers(c: &mut Criterion) {
+ let context = MetalContext::new().expect("Metal context");
+ bench_qwen3_layers_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleBias);
+ bench_qwen3_layers_typed::<bf16>(c, &context, "ZP_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleZeroPoint);
+}
diff --git a/crates/backend-uzu/tests/performance/kernel/mod.rs b/crates/backend-uzu/tests/performance/kernel/mod.rs
index 0223d0039..58b255470 100644
--- a/crates/backend-uzu/tests/performance/kernel/mod.rs
+++ b/crates/backend-uzu/tests/performance/kernel/mod.rs
@@ -2,6 +2,6 @@
#[path = "../../common/mod.rs"]
mod common;
-mod quant_matmul;
+mod matmul;
mod rms_norm_bench;
mod sampling;
diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs
deleted file mode 100644
index 857ee0c92..000000000
--- a/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs
+++ /dev/null
@@ -1,2 +0,0 @@
-mod qmm_transposed_bench;
-mod qmv_fast_bench;
diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs
deleted file mode 100644
index f2ba0d5d8..000000000
--- a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs
+++ /dev/null
@@ -1,132 +0,0 @@
-use backend_uzu::{
- ArrayElement,
- backends::common::{
- Allocation, Backend, Context, Encoder, Kernels,
- gpu_types::QuantizationMethod,
- kernel::QuantizedMatmulQmmTransposedKernel,
- },
-};
-use criterion::{BenchmarkId, Criterion, Throughput};
-use half::{bf16, f16};
-use itertools::iproduct;
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::SmallRng};
-
-use crate::{
- common::{
- helpers::{alloc_allocation, alloc_allocation_with_data},
- type_short_name,
- },
- uzu_bench,
-};
-
-fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(
- c: &mut Criterion,
- context: &B::Context,
- label: &str,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- let block_size: usize = if bits == 4 { 512 } else { 256 };
-
- for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336]) {
- if n % 32 != 0 || k % block_size != 0 {
- continue;
- }
-
- let (bm, bk, bn, wm, wn) = if m < 48 {
- (8u32, 32u32, 32u32, 1u32, 1u32)
- } else {
- (32u32, 32u32, 32u32, 2u32, 2u32)
- };
-
- let num_groups = k.div_ceil(group_size as usize);
- let mut rng = SmallRng::seed_from_u64(42);
-
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- context,
- T::data_type(),
- group_size,
- bits,
- bm,
- bk,
- bn,
- wm,
- wn,
- quant_method,
- false,
- n % bn as usize == 0,
- )
- .unwrap();
-
- let w_buf = alloc_allocation_with_data::<B, u32>(
- context,
- &(0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect::<Vec<_>>(),
- );
- let scales_buf = alloc_allocation_with_data::<B, T>(
- context,
- &(0..n * num_groups).map(|_| T::from(rng.random_range(0.01f32..1.0f32)).unwrap()).collect::<Vec<_>>(),
- );
- let x_buf = alloc_allocation_with_data::<B, T>(
- context,
- &(0..m * k).map(|_| T::from(rng.random_range(-1.0f32..1.0f32)).unwrap()).collect::<Vec<_>>(),
- );
- let mut y_buf = alloc_allocation::<B, T>(context, m * n);
-
- let zp_stride = if bits == 4 {
- (num_groups + 1) / 2
- } else {
- num_groups
- };
- let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {
- alloc_allocation_with_data::<B, u8>(
- context,
- &(0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect::<Vec<_>>(),
- )
- });
- let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {
- alloc_allocation_with_data::<B, T>(
- context,
- &(0..n * num_groups).map(|_| T::from(rng.random_range(-0.5f32..0.5f32)).unwrap()).collect::<Vec<_>>(),
- )
- });
-
- let mut group = c.benchmark_group(format!("{}/Kernel/QmmTransposed/{}", type_short_name::<B>(), label));
- group.throughput(Throughput::Elements((m * n * k) as u64));
- group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {
- b.iter_custom(|n_iters| {
- let mut encoder = Encoder::<B>::new(context).unwrap();
- for _ in 0..n_iters {
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- None::<&Allocation<B>>,
- k as u32,
- n as u32,
- m as u32,
- &mut encoder,
- );
- }
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
- })
- });
- drop(group);
- }
-}
-
-#[uzu_bench]
-fn bench_qmm_transposed(c: &mut Criterion) {
- for_each_backend!(|B| {
- let context = <B as Backend>::Context::new().unwrap();
- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmm_transposed_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- });
-}
diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs
deleted file mode 100644
index 16d352560..000000000
--- a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs
+++ /dev/null
@@ -1,141 +0,0 @@
-use backend_uzu::{
- ArrayElement,
- backends::common::{
- Allocation, Backend, Context, Encoder, Kernels,
- gpu_types::QuantizationMethod,
- kernel::QuantizedMatmulQmvFastKernel,
- },
-};
-use criterion::{BenchmarkId, Criterion, Throughput};
-use half::{bf16, f16};
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::SmallRng};
-
-use crate::{
- common::{
- helpers::{alloc_allocation, alloc_allocation_with_data},
- type_short_name,
- },
- uzu_bench,
-};
-
-fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(
- rng: &mut R,
- range: std::ops::Range<T>,
- len: usize,
-) -> Box<[T]> {
- (0..len).map(|_| rng.random_range(range.clone())).collect()
-}
-
-fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
- c: &mut Criterion,
- context: &B::Context,
- label: &str,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));
- let block_size: usize = if bits == 4 {
- 512
- } else {
- 256
- };
-
- let nk_pairs: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
- for &(n, k) in nk_pairs {
- if n % 8 != 0 || k % block_size != 0 {
- continue;
- }
- for m in [1usize, 2, 4, 8] {
- let num_groups = k.div_ceil(group_size as usize);
- let mut rng = SmallRng::seed_from_u64(42);
-
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
- context,
- T::data_type(),
- group_size,
- bits,
- quant_method,
- false,
- )
- .unwrap();
-
- let w_buf = alloc_allocation_with_data::<B, u32>(
- context,
- &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),
- );
- let scales_buf = alloc_allocation_with_data::<B, T>(
- context,
- &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)
- .iter()
- .map(|&v| T::from(v).unwrap())
- .collect::<Vec<_>>(),
- );
- let x_buf = alloc_allocation_with_data::<B, T>(
- context,
- &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k)
- .iter()
- .map(|&v| T::from(v).unwrap())
- .collect::<Vec<_>>(),
- );
- let mut y_buf = alloc_allocation::<B, T>(context, m * n);
-
- let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {
- let zp_stride = if bits == 4 {
- (num_groups + 1) / 2
- } else {
- num_groups
- };
- alloc_allocation_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))
- });
- let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {
- alloc_allocation_with_data::<B, T>(
- context,
- &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)
- .iter()
- .map(|&v| T::from(v).unwrap())
- .collect::<Vec<_>>(),
- )
- });
-
- group.throughput(Throughput::Elements((m * n * k) as u64));
- group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {
- b.iter_custom(|n_iters| {
- let mut encoder = Encoder::<B>::new(context).unwrap();
- for _ in 0..n_iters {
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- None::<&Allocation<B>>,
- k as u32,
- n as u32,
- m as u32,
- &mut encoder,
- );
- }
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
- })
- });
- }
- }
-}
-
-#[uzu_bench]
-fn bench_qmv_fast(c: &mut Criterion) {
- for_each_backend!(|B| {
- let context = <B as Backend>::Context::new().unwrap();
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);
- });
-}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
deleted file mode 100644
index d20d3ae98..000000000
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
+++ /dev/null
@@ -1,687 +0,0 @@
-#[macro_use]
-#[path = "../../../../common/mod.rs"]
-mod common;
-
-use std::time::Instant;
-
-use half::{bf16, f16};
-
-use crate::{
- DataType,
- backends::common::{
- Allocation, AllocationType, Backend, Context, Encoder, allocation_as_bytes,
- gpu_types::{QuantizationMethod, QuantizationMode},
- kernel::quant_matmul::{
- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
- tests::common::helpers::alloc_allocation_with_data,
- },
- },
-};
-
-fn pack_u4_weights(values: &[u8]) -> Vec<u8> {
- let mut out = Vec::with_capacity((values.len() + 3) / 4 * 2);
- for chunk in values.chunks(4) {
- let w0 = (*chunk.get(0).unwrap_or(&0) as u16) & 0x0F;
- let w1 = ((*chunk.get(1).unwrap_or(&0) as u16) & 0x0F) * 16;
- let w2 = ((*chunk.get(2).unwrap_or(&0) as u16) & 0x0F) * 256;
- let w3 = ((*chunk.get(3).unwrap_or(&0) as u16) & 0x0F) * 4096;
-
- let word: u16 = w0 | w1 | w2 | w3;
- out.push(word as u8);
- out.push((word >> 8) as u8);
- }
- out
-}
-
-fn quantize_value(
- value: f32,
- dtype: DataType,
-) -> f32 {
- match dtype {
- DataType::F16 => f16::from_f32(value).to_f32(),
- DataType::BF16 => bf16::from_f32(value).to_f32(),
- _ => value,
- }
-}
-
-fn quantize_slice(
- values: &[f32],
- dtype: DataType,
-) -> Vec<f32> {
- values.iter().map(|&v| quantize_value(v, dtype)).collect()
-}
-
-struct ExecutionResult {
- elapsed: f64,
-}
-
-fn create_test_weights(
- output_dim: usize,
- input_dim: usize,
- bits: usize,
-) -> Vec<u8> {
- let mut weights_quant: Vec<u8> = Vec::with_capacity(output_dim * input_dim);
-
- // Weights stored as [output_dim × input_dim]
- for row in 0..output_dim {
- for _col in 0..input_dim {
- let v = if bits == 4 {
- ((row + 1) & 0x0F) as u8
- } else {
- ((row + 1) & 0xFF) as u8
- };
- weights_quant.push(v);
- }
- }
-
- weights_quant
-}
-
-fn check_tolerance(
- exp: f32,
- got: f32,
-) -> bool {
- let diff = (exp - got).abs() as f64;
- // For F16 at boundaries (like 65504), machine epsilon is 32 (approx 0.05% relative error)
- // We use a conservative relative tolerance of 0.2%
- let rel_tol = 0.002;
- let abs_tol = 0.1f64;
- let tol = abs_tol.max(exp.abs() as f64 * rel_tol);
- diff > tol
-}
-
-fn get_4bit_value(
- data: &[u8],
- index: usize,
-) -> f32 {
- let word_idx = index / 4;
- let word_offset = index % 4;
- let byte_idx = word_idx * 2;
-
- let word = if byte_idx + 1 < data.len() {
- data[byte_idx] as u16 | ((data[byte_idx + 1] as u16) * 256)
- } else {
- 0
- };
-
- ((word >> (word_offset * 4)) & 0x0F) as f32
-}
-
-fn get_zp_value(
- zero_points: &[u8],
- stride: usize,
- row_idx: usize,
- group_idx: usize,
- bits: usize,
-) -> f32 {
- if bits == 4 {
- let byte_index = row_idx * stride + (group_idx >> 1);
- let byte = zero_points.get(byte_index).copied().unwrap_or(0);
- if (group_idx & 1) == 0 {
- (byte & 0x0F) as f32
- } else {
- ((byte >> 4) & 0x0F) as f32
- }
- } else {
- zero_points.get(row_idx * stride + group_idx).copied().unwrap_or(0) as f32
- }
-}
-
-fn cpu_reference(
- batch: usize,
- output_dim: usize,
- input_dim: usize,
- a: &[f32],
- b_quant: &[u8],
- scales: &[f32],
- biases: &[f32],
- group_size: usize,
- dtype: DataType,
- bits: usize,
- quantization_method: QuantizationMethod,
- zero_points: &[u8],
- zero_points_stride: usize,
-) -> Vec<f32> {
- let num_groups = (input_dim + group_size - 1) / group_size;
-
- let mut y = vec![0.0f32; batch * output_dim];
- for i in 0..batch {
- for j in 0..output_dim {
- let mut acc = 0.0f32;
- for g in 0..num_groups {
- let scale = scales[j * num_groups + g];
- let bias = match quantization_method {
- QuantizationMethod::ScaleZeroPoint => {
- let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);
- -scale * zp
- },
- QuantizationMethod::ScaleBias => biases[j * num_groups + g],
- };
- let l_start = g * group_size;
- let l_end = (l_start + group_size).min(input_dim);
- let mut group_acc = 0.0f32;
- let mut group_sum = 0.0f32;
- for l in l_start..l_end {
- let weight_linear_idx = j * input_dim + l;
-
- let val_q = if bits == 4 {
- get_4bit_value(b_quant, weight_linear_idx)
- } else {
- b_quant[weight_linear_idx] as f32
- };
-
- let val_a = a[i * input_dim + l];
- group_acc += val_a * val_q;
- group_sum += val_a;
- }
- acc += scale * group_acc + bias * group_sum;
- }
- y[i * output_dim + j] = quantize_value(acc, dtype);
- }
- }
- y
-}
-
-struct TestQuantParams {
- scales: Vec<f32>,
- scales_quant: Vec<f32>,
- biases: Vec<f32>,
- zero_points: Vec<u8>,
- zero_points_stride: usize,
-}
-
-fn generate_test_quant_params(
- output_dim: usize,
- input_dim: usize,
- group_size: usize,
- data_type: DataType,
- bits: usize,
- quantization_method: QuantizationMethod,
- randomize_zp: bool,
-) -> TestQuantParams {
- let num_groups = (input_dim + group_size - 1) / group_size;
-
- let len = output_dim * num_groups;
- let scales = vec![1.0; len];
- let scales_quant = quantize_slice(&scales, data_type);
- let mut biases = vec![0.0; len];
-
- let zero_points_stride = if bits == 4 {
- ((num_groups + 1) / 2).max(1)
- } else {
- num_groups
- };
- let mut zero_points = vec![0u8; output_dim * zero_points_stride];
-
- if quantization_method == QuantizationMethod::ScaleZeroPoint && randomize_zp {
- for j in 0..output_dim {
- for g in 0..num_groups {
- let base_val = j + 3 * g;
- let zp_val: u8 = if bits == 4 {
- (base_val as u8) & 0x0F
- } else {
- base_val as u8
- };
-
- if bits == 4 {
- let byte_index = j * zero_points_stride + (g >> 1);
- if (g & 1) == 0 {
- zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val & 0x0F);
- } else {
- zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);
- }
- } else {
- let byte_index = j * zero_points_stride + g;
- zero_points[byte_index] = zp_val;
- }
-
- let s = scales_quant[j * num_groups + g];
- let b = quantize_value(-s * (zp_val as f32), data_type);
- biases[j * num_groups + g] = b;
- }
- }
- }
-
- if quantization_method == QuantizationMethod::ScaleBias {
- for j in 0..output_dim {
- for g in 0..num_groups {
- let base_val = j * 7 + g * 3;
- let bias_val = (base_val % 19) as f32 * 0.125;
- biases[j * num_groups + g] = quantize_value(bias_val, data_type);
- }
- }
- }
-
- TestQuantParams {
- scales,
- scales_quant,
- biases,
- zero_points,
- zero_points_stride,
- }
-}
-
-fn allocation_from_f32_slice<B: Backend>(
- ctx: &B::Context,
- dtype: DataType,
- values: &[f32],
-) -> Allocation<B> {
- match dtype {
- DataType::F16 => {
- let data: Vec<f16> = values.iter().map(|&v| f16::from_f32(v)).collect();
- alloc_allocation_with_data::<B, f16>(ctx, data.as_slice())
- },
- DataType::BF16 => {
- let data: Vec<bf16> = values.iter().map(|&v| bf16::from_f32(v)).collect();
- alloc_allocation_with_data::<B, bf16>(ctx, data.as_slice())
- },
- DataType::F32 => alloc_allocation_with_data::<B, f32>(ctx, values),
- other => {
- panic!("Unsupported dtype for allocation_from_f32_slice: {:?}", other)
- },
- }
-}
-
-fn execute_quantized_matmul<B: Backend>(
- ctx: &B::Context,
- batch: usize,
- input_dim: usize,
- output_dim: usize,
- iterations: usize,
- validate: bool,
- quantization_method: QuantizationMethod,
- randomize_zp: bool,
- group_size: usize,
- data_type: DataType,
- bits: usize,
-) -> ExecutionResult {
- let weights_quant = create_test_weights(output_dim, input_dim, bits);
- let weights_packed = if bits == 4 {
- pack_u4_weights(&weights_quant)
- } else {
- weights_quant.clone()
- };
-
- let params = generate_test_quant_params(
- output_dim,
- input_dim,
- group_size,
- data_type,
- bits,
- quantization_method,
- randomize_zp,
- );
- // X is batch × input_dim
- let x_f32: Vec<f32> = if batch == 1 {
- // Vector case: 1 × input_dim
- (1..=input_dim).map(|i| i as f32 / input_dim as f32).collect()
- } else {
- let mut x_vals: Vec<f32> = Vec::with_capacity(batch * input_dim);
- for _row in 0..batch {
- for i in 0..input_dim {
- x_vals.push((i + 1) as f32 / input_dim as f32);
- }
- }
- x_vals
- };
- let x_quant = quantize_slice(&x_f32, data_type);
-
- let w_buf = alloc_allocation_with_data::<B, u8>(ctx, &weights_packed);
- let s_buf = allocation_from_f32_slice::<B>(ctx, data_type, &params.scales);
-
- let b_buf = match quantization_method {
- QuantizationMethod::ScaleZeroPoint => alloc_allocation_with_data::<B, u8>(ctx, &params.zero_points),
- QuantizationMethod::ScaleBias => allocation_from_f32_slice::<B>(ctx, data_type, &params.biases),
- };
- let x_buf = allocation_from_f32_slice::<B>(ctx, data_type, &x_f32);
- let mut y_buf = ctx
- .create_allocation(batch * output_dim * data_type.size_in_bytes(), AllocationType::Global)
- .expect("Failed to create allocation");
-
- let kernel = QuantizedMatmulKernelEncodable::<B>::new(
- &ctx,
- QuantizedMatmulConfiguration {
- data_type,
- group_size,
- input_dim,
- output_dim,
- mode: match bits {
- 4 => QuantizationMode::U4,
- 8 => QuantizationMode::I8,
- _ => panic!("Unsupported bits: {}", bits),
- },
- quantization_method,
- use_hadamard: false,
- },
- )
- .unwrap();
-
- if iterations > 1 {
- for _ in 0..3 {
- let args = QuantizedMatmulArguments {
- a: &x_buf,
- a_offset: 0,
- b: &w_buf,
- scales: &s_buf,
- zero_points_or_biases: &b_buf,
- output: &mut y_buf,
- hadamard_factors: None,
- batch_dim: batch,
- };
- let mut encoder = Encoder::new(ctx).unwrap();
- kernel.encode(&mut encoder, args);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
- }
- }
-
- let start = Instant::now();
- for _ in 0..iterations {
- let args = QuantizedMatmulArguments {
- a: &x_buf,
- a_offset: 0,
- b: &w_buf,
- scales: &s_buf,
- zero_points_or_biases: &b_buf,
- output: &mut y_buf,
- hadamard_factors: None,
- batch_dim: batch,
- };
- let mut encoder = Encoder::new(ctx).unwrap();
- kernel.encode(&mut encoder, args);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
- }
- let elapsed = start.elapsed();
-
- if validate {
- let y_expected = cpu_reference(
- batch,
- output_dim,
- input_dim,
- &x_quant,
- &weights_packed,
- &params.scales_quant,
- &params.biases,
- group_size,
- data_type,
- bits,
- quantization_method,
- &params.zero_points,
- params.zero_points_stride,
- );
-
- let y_out_f32: Vec<f32> = match data_type {
- DataType::F16 => {
- let y_out = bytemuck::cast_slice::<u8, f16>(allocation_as_bytes(&y_buf));
- y_out.iter().map(|&v| v.to_f32()).collect()
- },
- DataType::BF16 => {
- let y_out = bytemuck::cast_slice::<u8, bf16>(allocation_as_bytes(&y_buf));
- y_out.iter().map(|&v| v.to_f32()).collect()
- },
- DataType::F32 => bytemuck::cast_slice::<u8, f32>(allocation_as_bytes(&y_buf)).to_vec(),
- other => panic!("Unsupported dtype for validation: {:?}", other),
- };
-
- let mut debug_prints = 0;
-
- for (i, (&exp, &got)) in y_expected.iter().zip(y_out_f32.iter()).enumerate() {
- let diff = (exp - got).abs();
-
- if check_tolerance(exp, got) {
- if debug_prints < 16 {
- println!("\n detail idx {} diff {} exp {} got {}", i, diff, exp, got);
- }
- debug_prints += 1;
- }
- }
-
- if debug_prints > 0 {
- let first_error = y_expected
- .iter()
- .zip(y_out_f32.iter())
- .enumerate()
- .find(|&(_, (&e, &g))| check_tolerance(e, g))
- .map(|(i, _)| i)
- .unwrap_or(0);
- let last_error = y_expected
- .iter()
- .zip(y_out_f32.iter())
- .enumerate()
- .filter(|&(_, (&e, &g))| check_tolerance(e, g))
- .last()
- .map(|(i, _)| i)
- .unwrap_or(0);
- println!("\nTotal errors: {} out of {} outputs", debug_prints, batch * output_dim);
- println!("Error range: indices {}-{}", first_error, last_error);
- println!("Dims batch={} output_dim={} input_dim={}", batch, output_dim, input_dim);
- panic!("Validation failed with {} mismatches", debug_prints);
- }
-
- ExecutionResult {
- elapsed: elapsed.as_secs_f64(),
- }
- } else {
- ExecutionResult {
- elapsed: elapsed.as_secs_f64(),
- }
- }
-}
-
-struct TestConfig {
- quant_method: QuantizationMethod,
- bits: usize,
- data_type: DataType,
- group_size: usize,
-}
-
-const QMV_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];
-const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];
-
-fn run_kernel_test<B: Backend>(
- ctx: &B::Context,
- batch: usize,
- output_dim: usize,
- input_dim: usize,
- config: &TestConfig,
- validate: bool,
- iterations: usize,
-) -> ExecutionResult {
- let randomize_zp = config.quant_method == QuantizationMethod::ScaleZeroPoint;
-
- execute_quantized_matmul::<B>(
- ctx,
- batch,
- input_dim,
- output_dim,
- iterations,
- validate,
- config.quant_method,
- randomize_zp,
- config.group_size,
- config.data_type,
- config.bits,
- )
-}
-
-fn test_quant_gmv_internal<B: Backend>() {
- let ctx = match B::Context::new() {
- Ok(c) => c,
- Err(_) => {
- println!("{} backend not available — skipping QMV test", std::any::type_name::<B>());
- return;
- },
- };
-
- let configs = vec![
- TestConfig {
- quant_method: QuantizationMethod::ScaleZeroPoint,
- bits: 4,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_method: QuantizationMethod::ScaleZeroPoint,
- bits: 8,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_method: QuantizationMethod::ScaleBias,
- bits: 4,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_method: QuantizationMethod::ScaleBias,
- bits: 8,
- data_type: DataType::F32,
- group_size: 64,
- },
- ];
-
- for config in &configs {
- for &(output_dim, input_dim) in QMV_DIMS {
- run_kernel_test::<B>(&ctx, 1, output_dim, input_dim, config, true, 1);
- }
- }
-}
-
-fn test_quant_gmm_transposed_internal<B: Backend>() {
- let ctx = match B::Context::new() {
- Ok(c) => c,
- Err(_) => {
- println!("{} not available — skipping QMM transposed test", std::any::type_name::<B>());
- return;
- },
- };
-
- let configs = vec![
- TestConfig {
- quant_method: QuantizationMethod::ScaleZeroPoint,
- bits: 4,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_method: QuantizationMethod::ScaleZeroPoint,
- bits: 8,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_method: QuantizationMethod::ScaleBias,
- bits: 4,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_method: QuantizationMethod::ScaleBias,
- bits: 8,
- data_type: DataType::F32,
- group_size: 64,
- },
- ];
-
- for config in &configs {
- for &(batch, output_dim, input_dim) in QMM_DIMS {
- run_kernel_test::<B>(&ctx, batch, output_dim, input_dim, config, true, 1);
- }
- }
-}
-
-fn test_quant_matmul_perf_internal<B: Backend>() {
- let ctx = match B::Context::new() {
- Ok(c) => c,
- Err(_) => {
- println!("{} backend not available — skipping Perf test", std::any::type_name::<B>());
- return;
- },
- };
-
- // Llama 3 8B approximate shapes
- let shapes = vec![
- // Decoding (M=1)
- (1, 4096, 4096),
- (1, 14336, 4096),
- (1, 4096, 14336),
- // Prefill (Small batch M=128)
- (128, 4096, 4096),
- ];
-
- let configs = vec![
- // 4-bit scale-bias BF16
- TestConfig {
- quant_method: QuantizationMethod::ScaleBias,
- bits: 4,
- data_type: DataType::BF16,
- group_size: 128,
- },
- // 8-bit scale-bias BF16
- TestConfig {
- quant_method: QuantizationMethod::ScaleBias,
- bits: 8,
- data_type: DataType::BF16,
- group_size: 128,
- },
- // 4-bit ZP F16
- TestConfig {
- quant_method: QuantizationMethod::ScaleZeroPoint,
- bits: 4,
- data_type: DataType::F16,
- group_size: 64,
- },
- // 8-bit ZP F16
- TestConfig {
- quant_method: QuantizationMethod::ScaleZeroPoint,
- bits: 8,
- data_type: DataType::F16,
- group_size: 64,
- },
- ];
-
- println!(
- "{:<20} | {:<10} | {:<6} | {:<5} | {:<5} | {:<15}",
- "Kernel Config", "M x N x K", "Type", "Bits", "Group", "Avg Duration"
- );
- println!("{}", "-".repeat(85));
-
- let iterations = 20;
- for config in &configs {
- for &(batch, output_dim, input_dim) in &shapes {
- let result = run_kernel_test::<B>(&ctx, batch, output_dim, input_dim, config, false, iterations);
- let avg = result.elapsed / (iterations as f64) * 1000.0; // ms
-
- println!(
- "{:<20} | {:<10} | {:<6} | {:<5} | {:<5} | {:.4} ms",
- format!("{:?}", config.quant_method),
- format!("{}x{}x{}", batch, output_dim, input_dim),
- format!("{:?}", config.data_type),
- config.bits,
- config.group_size,
- avg
- );
- }
- }
-}
-
-#[test]
-fn test_quant_gmv() {
- for_each_non_cpu_backend!(|B| {
- test_quant_gmv_internal::<B>();
- })
-}
-
-#[test]
-fn test_quant_gmm_transposed() {
- for_each_non_cpu_backend!(|B| {
- test_quant_gmm_transposed_internal::<B>();
- })
-}
-
-#[test]
-#[ignore]
-fn test_quant_matmul_perf() {
- for_each_non_cpu_backend!(|B| {
- test_quant_matmul_perf_internal::<B>();
- })
-}
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs
index 5e6969ae5..1bc168176 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs
@@ -1,3 +1,4 @@
+
use backend_uzu::{
DataType,
backends::{
@@ -6,7 +7,7 @@ use backend_uzu::{
kernel::{
AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,
ManualKernels, SoftmaxKernel,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
cpu::Cpu,
@@ -91,16 +92,31 @@ fn pipeline_output<B: Backend>(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -
let mut enc = Encoder::new(ctx.as_ref()).unwrap();
for g in 0..NUM_GROUPS {
- matmul.encode(
- MatmulArguments {
- a: &qa, a_offset: g as usize * GQA as usize * SUFFIX as usize * q_row,
- b: &ka, b_offset: g as usize * head_stride * dt,
- b_leading_dimension: Some(HEAD_DIM), b_transpose: true,
- ab_scale: scale, c: MatmulArgumentC::None, d: &mut grp_s,
- batch_dim: GQA * SUFFIX, input_dim: HEAD_DIM, output_dim: SEQ,
- },
- &mut enc,
- );
+ matmul
+ .encode(
+ MatmulArguments {
+ a: &qa,
+ a_offset: g as usize * GQA as usize * SUFFIX as usize * q_row,
+ b: MatmulB::FullPrecision {
+ b: &ka,
+ },
+ b_offset: g as usize * head_stride * dt,
+ b_leading_dimension: Some(HEAD_DIM),
+ b_transpose: true,
+ d: &mut grp_s,
+ d_transform: MatmulDOps {
+ ab_scale: scale,
+ accumulate: false,
+ bias: None,
+ rht_factors: None,
+ },
+ m: GQA * SUFFIX,
+ n: SEQ,
+ k: HEAD_DIM,
+ },
+ &mut enc,
+ )
+ .expect("encode failed");
scatter_scores.encode(
&grp_s, &mut scores, None, None::<&Allocation<B>>, None,
g, GQA, SEQ, SUFFIX, GQA * SUFFIX * SEQ, &mut enc,
@@ -108,16 +124,26 @@ fn pipeline_output<B: Backend>(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -
}
softmax.encode(&mut scores, None::<&Allocation<B>>, SEQ, NUM_HEADS, SUFFIX, &mut enc);
for g in 0..NUM_GROUPS {
- matmul.encode(
- MatmulArguments {
- a: &scores, a_offset: g as usize * GQA as usize * SUFFIX as usize * s_row,
- b: &va, b_offset: g as usize * head_stride * dt,
- b_leading_dimension: Some(HEAD_DIM), b_transpose: false,
- ab_scale: 1.0, c: MatmulArgumentC::None, d: &mut grp_o,
- batch_dim: GQA * SUFFIX, input_dim: SEQ, output_dim: HEAD_DIM,
- },
- &mut enc,
- );
+ matmul
+ .encode(
+ MatmulArguments {
+ a: &scores,
+ a_offset: g as usize * GQA as usize * SUFFIX as usize * s_row,
+ b: MatmulB::FullPrecision {
+ b: &va,
+ },
+ b_offset: g as usize * head_stride * dt,
+ b_leading_dimension: Some(HEAD_DIM),
+ b_transpose: false,
+ d: &mut grp_o,
+ d_transform: MatmulDOps::none(),
+ m: GQA * SUFFIX,
+ n: HEAD_DIM,
+ k: SEQ,
+ },
+ &mut enc,
+ )
+ .expect("encode failed");
scatter_values.encode(
&grp_o, &mut out, g, GQA, SUFFIX, NUM_HEADS, HEAD_DIM,
GQA * SUFFIX * HEAD_DIM, &mut enc,
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
index 51a0e9085..97ece1a8b 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
@@ -6,126 +6,86 @@ use backend_uzu::{
ArrayElement,
backends::{
common::{Backend, Context, kernel::ManualKernels, kernel::matmul::MatmulKernel},
- metal::{Metal, MetalContext},
+ metal::{DeviceExt, GemmDispatchPath, Metal, MetalContext},
},
};
-use proc_macros::__internal_uzu_test as uzu_test;
use half::{bf16, f16};
use num_traits::Float;
+use rstest::rstest;
use crate::common::{
assert::assert_eq_float,
- matmul::{Case, Variant, all_correctness_shapes, cpu_reference, deterministic_input, run_metal},
+ matmul::{Case, all_correctness_shapes, cpu_reference, deterministic_input, run_metal},
};
+fn gemm_paths_for_hw(context: &MetalContext) -> Vec<GemmDispatchPath> {
+ let mut paths = vec![GemmDispatchPath::Simdgroup];
+ if context.device.supports_mxu() {
+ paths.push(GemmDispatchPath::Mxu);
+ }
+ paths
+}
+
fn check_case<T: ArrayElement + Float + Debug + Display>(
context: &MetalContext,
kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,
- variant: Variant,
+ path: Option<GemmDispatchPath>,
case: Case,
tolerance: f32,
) {
let input = deterministic_input::<T>(case);
let expected = cpu_reference::<T>(&input);
- let actual = run_metal::<T>(context, kernel, &input, variant);
+ let actual = run_metal::<T>(context, kernel, &input, path);
assert_eq_float(
&expected,
&actual,
tolerance,
- &format!("{:?} dtype={} {:?}", variant, std::any::type_name::<T>(), case),
+ &format!("{path:?} dtype={} {case:?}", std::any::type_name::<T>()),
);
}
-fn check_all_shapes<T: ArrayElement + Float + Debug + Display>(
- variant: Variant,
- ab_scale: f32,
- accumulate: bool,
+fn run_matrix<T: ArrayElement + Float + Debug + Display>(
+ case_for_shape: impl Fn(crate::common::matmul::Shape) -> Case,
tolerance: f32,
) {
let context = MetalContext::new().expect("Metal context");
- if !variant.supported(&context) {
- eprintln!("Skipping {variant:?}: device does not support MXU");
- return;
- }
let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
.expect("MatmulKernel");
- for shape in all_correctness_shapes() {
- let case = Case::new(shape).with_ab_scale(ab_scale).with_accumulate(accumulate);
- check_case::<T>(&context, &mut kernel, variant, case, tolerance);
+ for path in gemm_paths_for_hw(&context) {
+ for shape in all_correctness_shapes() {
+ let case = case_for_shape(shape);
+ check_case::<T>(&context, &mut kernel, Some(path), case, tolerance);
+ }
}
}
-#[uzu_test]
-fn matches_cpu_reference_bf16_gemm() {
- check_all_shapes::<bf16>(Variant::Gemm, 1.0, false, 1.0);
-}
-
-#[uzu_test]
-fn matches_cpu_reference_bf16_gemm_mxu() {
- check_all_shapes::<bf16>(Variant::GemmMxu, 1.0, false, 1.0);
-}
-
-#[uzu_test]
-fn matches_cpu_reference_f16_gemm() {
- check_all_shapes::<f16>(Variant::Gemm, 1.0, false, 0.5);
-}
-
-#[uzu_test]
-fn matches_cpu_reference_f16_gemm_mxu() {
- check_all_shapes::<f16>(Variant::GemmMxu, 1.0, false, 0.5);
-}
-
-#[uzu_test]
-fn matches_cpu_reference_f32_gemm() {
- check_all_shapes::<f32>(Variant::Gemm, 1.0, false, 0.05);
-}
-
-#[uzu_test]
-fn ab_scale_bf16_gemm() {
- check_all_shapes::<bf16>(Variant::Gemm, 0.5, false, 1.0);
-}
-
-#[uzu_test]
-fn ab_scale_bf16_gemm_mxu() {
- check_all_shapes::<bf16>(Variant::GemmMxu, 0.5, false, 1.0);
-}
-
-#[uzu_test]
-fn accumulate_bf16_gemm_mxu() {
- check_all_shapes::<bf16>(Variant::GemmMxu, 1.0, true, 1.0);
+#[rstest]
+#[case::base(1.0, false)]
+#[case::ab_scale(0.5, false)]
+#[case::accumulate(1.0, true)]
+#[case::scale_and_accumulate(0.5, true)]
+fn matches_cpu_reference_bf16(
+ #[case] ab_scale: f32,
+ #[case] accumulate: bool,
+) {
+ run_matrix::<bf16>(
+ |shape| Case::new(shape).with_ab_scale(ab_scale).with_accumulate(accumulate),
+ 1.0,
+ );
}
-#[uzu_test]
-fn scale_and_accumulate_bf16_gemm_mxu() {
- check_all_shapes::<bf16>(Variant::GemmMxu, 0.5, true, 1.0);
+#[test]
+fn matches_cpu_reference_f16() {
+ run_matrix::<f16>(|shape| Case::new(shape), 0.5);
}
-fn check_b_transpose_false<T: ArrayElement + Float + Debug + Display>(
- variant: Variant,
- tolerance: f32,
-) {
- let context = MetalContext::new().expect("Metal context");
- if !variant.supported(&context) {
- eprintln!("Skipping {variant:?}: device does not support MXU");
- return;
- }
- let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
- .expect("MatmulKernel");
- for shape in all_correctness_shapes() {
- let case = Case {
+#[test]
+fn b_transpose_false_bf16() {
+ run_matrix::<bf16>(
+ |shape| Case {
b_transpose: false,
..Case::new(shape)
- };
- check_case::<T>(&context, &mut kernel, variant, case, tolerance);
- }
-}
-
-#[uzu_test]
-fn b_transpose_false_bf16_gemm() {
- check_b_transpose_false::<bf16>(Variant::Gemm, 1.0);
-}
-
-#[uzu_test]
-fn b_transpose_false_bf16_gemm_mxu() {
- check_b_transpose_false::<bf16>(Variant::GemmMxu, 1.0);
+ },
+ 1.0,
+ );
}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
index d99850a4b..d71526524 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
@@ -1,9 +1,5 @@
-use std::{
- fmt::{Debug, Display},
-};
+use std::fmt::{Debug, Display};
-use half::{bf16, f16};
-use num_traits::Float;
use backend_uzu::{
ArrayContextExt, ArrayElement,
backends::{
@@ -11,16 +7,19 @@ use backend_uzu::{
AllocationType, Backend, Context, Encoder,
kernel::{
ManualKernels,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
cpu::Cpu,
},
};
+use half::{bf16, f16};
+use num_traits::Float;
+use rstest::rstest;
-use crate::{
- common::{assert::assert_eq_float, helpers::{alloc_allocation_with_data, allocation_to_vec}},
- uzu_test,
+use crate::common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation_with_data, allocation_to_vec},
};
struct Input<T: ArrayElement + Float> {
@@ -68,23 +67,26 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.expect("Failed to create MatmulKernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- MatmulArguments {
- a: &a_allocation,
- a_offset: 0,
- b: b_array.allocation(),
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
- d: &mut d_allocation,
- batch_dim: m,
- input_dim: k,
- output_dim: n,
- },
- &mut encoder,
- );
+ kernel
+ .encode(
+ MatmulArguments {
+ a: &a_allocation,
+ a_offset: 0,
+ b: MatmulB::FullPrecision {
+ b: b_array.allocation(),
+ },
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut d_allocation,
+ d_transform: MatmulDOps::none(),
+ m,
+ n,
+ k,
+ },
+ &mut encoder,
+ )
+ .expect("encode failed");
encoder.end_encoding().submit().wait_until_completed().unwrap();
allocation_to_vec::<B, T>(&d_allocation)
}
@@ -102,107 +104,34 @@ fn test<T: ArrayElement + Float + Debug + Display>(
});
}
-#[uzu_test]
-fn test_f32_m1() {
- test::<f32>(1, 128, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_m1() {
- test::<f16>(1, 128, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_m1() {
- test::<bf16>(1, 128, 64, 0.1);
-}
-
-#[uzu_test]
-fn test_f32_batched() {
- test::<f32>(4, 128, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_batched() {
- test::<f16>(4, 128, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_batched() {
- test::<bf16>(4, 128, 64, 0.1);
-}
-
-#[uzu_test]
-fn test_f32_max_batch() {
- test::<f32>(8, 128, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_max_batch() {
- test::<f16>(8, 128, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_max_batch() {
- test::<bf16>(8, 128, 64, 0.1);
-}
-
-#[uzu_test]
-fn test_f32_unaligned_k() {
- test::<f32>(1, 33, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_unaligned_k() {
- test::<f16>(1, 33, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_unaligned_k() {
- test::<bf16>(1, 33, 64, 0.1);
-}
-
-#[uzu_test]
-fn test_f32_unaligned_n() {
- test::<f32>(1, 128, 11, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_unaligned_n() {
- test::<f16>(1, 128, 11, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_unaligned_n() {
- test::<bf16>(1, 128, 11, 0.1);
-}
-
-#[uzu_test]
-fn test_f32_large() {
- test::<f32>(1, 4096, 2048, 0.05);
-}
-
-#[uzu_test]
-fn test_f16_large() {
- test::<f16>(1, 4096, 2048, 0.5);
-}
-
-#[uzu_test]
-fn test_bf16_large() {
- test::<bf16>(1, 4096, 2048, 1.0);
-}
-
-#[uzu_test]
-fn test_f32_small_n() {
- test::<f32>(1, 128, 3, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_small_n() {
- test::<f16>(1, 128, 3, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_small_n() {
- test::<bf16>(1, 128, 3, 0.1);
+#[rstest]
+#[case::m1(1, 128, 64)]
+#[case::batched(4, 128, 64)]
+#[case::max_batch(8, 128, 64)]
+#[case::unaligned_k(1, 33, 64)]
+#[case::unaligned_n(1, 128, 11)]
+#[case::large(1, 4096, 2048)]
+#[case::small_n(1, 128, 3)]
+fn gemv_f16(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+) {
+ test::<f16>(m, k, n, 0.01);
+}
+
+#[rstest]
+#[case::m1(1, 128, 64)]
+#[case::batched(4, 128, 64)]
+#[case::max_batch(8, 128, 64)]
+#[case::unaligned_k(1, 33, 64)]
+#[case::unaligned_n(1, 128, 11)]
+#[case::large(1, 4096, 2048)]
+#[case::small_n(1, 128, 3)]
+fn gemv_bf16(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+) {
+ test::<bf16>(m, k, n, 0.1);
}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs b/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs
index c84f3ea46..f9615e95c 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs
@@ -1,3 +1,5 @@
#[cfg(metal_backend)]
mod dispatch_paths_test;
mod gemv_test;
+#[cfg(metal_backend)]
+mod quant_dispatch_test;
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
new file mode 100644
index 000000000..aa37cb960
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
@@ -0,0 +1,223 @@
+#![cfg(metal_backend)]
+
+use std::{
+ error::Error as StdError,
+ fmt::{Debug, Display},
+};
+
+use backend_uzu::{
+ ArrayElement,
+ backends::{
+ common::{
+ Backend, Context, Encoder,
+ gpu_types::{QuantizationMethod, gemm::GemmDTransform},
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulDOps, MatmulError, MatmulKernel},
+ },
+ },
+ metal::{GemmDispatchPath, Metal, MetalContext},
+ },
+};
+use half::bf16;
+use num_traits::Float;
+use proc_macros::__internal_uzu_test as uzu_test;
+use rstest::rstest;
+
+use crate::common::{
+ helpers::allocation_to_vec,
+ matmul::{QuantBuffers, QuantInput, quant_arguments, run_quant_cpu, run_quant_metal},
+};
+
+fn check_tolerance(
+ expected: f32,
+ actual: f32,
+ rel_tol: f64,
+ abs_tol: f64,
+) -> bool {
+ let diff = (expected - actual).abs() as f64;
+ let tol = abs_tol.max(expected.abs() as f64 * rel_tol);
+ diff <= tol
+}
+
+fn assert_parity<T: ArrayElement + Float + Debug + Display>(
+ label: &str,
+ reference: &[T],
+ actual: &[T],
+ rel_tol: f64,
+ abs_tol: f64,
+) {
+ assert_eq!(reference.len(), actual.len(), "{label}: length mismatch");
+ let mut errors = 0usize;
+ for (i, (&exp, &got)) in reference.iter().zip(actual.iter()).enumerate() {
+ let exp_f32 = exp.to_f32().unwrap();
+ let got_f32 = got.to_f32().unwrap();
+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
+ if errors < 5 {
+ eprintln!(" {label} idx={i} expected={exp_f32} got={got_f32} diff={}", (exp_f32 - got_f32).abs());
+ }
+ errors += 1;
+ }
+ }
+ assert_eq!(errors, 0, "{label}: {errors} mismatches");
+}
+
+fn run_parity<T: ArrayElement + Float + Debug + Display>(
+ m: usize,
+ k: usize,
+ n: usize,
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+ rel_tol: f64,
+ abs_tol: f64,
+) {
+ let context = MetalContext::new().expect("Metal context");
+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 0);
+ let unified = run_quant_metal::<T>(&context, &input, Some(GemmDispatchPath::Simdgroup));
+ let reference = run_quant_cpu::<T>(&input);
+
+ assert_parity::<T>(
+ &format!(
+ "m={m} k={k} n={n} gs={group_size} bits={bits} method={quant_method:?} dtype={}",
+ std::any::type_name::<T>()
+ ),
+ &reference,
+ &unified,
+ rel_tol,
+ abs_tol,
+ );
+}
+
+#[rstest]
+#[case::gs32_4bit_mlx_prefill ( 64, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::gs64_4bit_mlx_prefill ( 64, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
+#[case::gs128_4bit_mlx_prefill( 64, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]
+#[case::gs32_8bit_mlx_prefill ( 64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]
+#[case::gs64_8bit_zp_prefill ( 64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs128_8bit_zp_prefill ( 64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs32_4bit_mlx_decode ( 8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::gs64_4bit_zp_decode ( 8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs32_unaligned_n ( 64, 256, 96, 32, 4, QuantizationMethod::ScaleBias)]
+fn parity_bf16(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+ #[case] gs: u32,
+ #[case] bits: u32,
+ #[case] method: QuantizationMethod,
+) {
+ run_parity::<bf16>(m, k, n, gs, bits, method, 0.05, 0.4);
+}
+
+#[rstest]
+#[case::gs64_4bit_mlx (32, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
+#[case::gs128_8bit_zp (32, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
+fn parity_f16(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+ #[case] gs: u32,
+ #[case] bits: u32,
+ #[case] method: QuantizationMethod,
+) {
+ run_parity::<half::f16>(m, k, n, gs, bits, method, 0.01, 0.05);
+}
+
+#[uzu_test]
+fn parity_bf16_gs32_4bit_mlx_with_bias() {
+ let context = MetalContext::new().expect("Metal context");
+ let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
+
+ let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.5 + 0.1 * (j % 5) as f32).collect();
+ let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let bias_pp_buf =
+ crate::common::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);
+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())
+ .expect("MatmulMetalKernel");
+
+ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
+ let mut args = quant_arguments(&mut buffers, &input);
+ args.d_transform = MatmulDOps {
+ ab_scale: 1.0,
+ accumulate: false,
+ bias: Some(&bias_pp_buf),
+ rht_factors: None,
+ };
+ matmul
+ .gemm
+ .encode_dispatch_path(args, GemmDispatchPath::Simdgroup, &mut encoder)
+ .expect("encode quant with bias");
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);
+
+ let mut reference = run_quant_cpu::<bf16>(&input);
+ for i in 0..(input.m as usize) {
+ for j in 0..(input.n as usize) {
+ let idx = i * input.n as usize + j;
+ reference[idx] = bf16::from_f32(reference[idx].to_f32() + bias_f32[j]);
+ }
+ }
+ assert_parity::<bf16>("with_bias", &reference, &actual, 0.05, 0.4);
+}
+
+#[uzu_test]
+fn quant_gemm_nonzero_b_offset_returns_unsupported_layout() {
+ let context = MetalContext::new().expect("Metal context");
+ let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())
+ .expect("MatmulMetalKernel");
+
+ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
+ let mut args = quant_arguments(&mut buffers, &input);
+ args.b_offset = 16;
+ let result = matmul.encode(args, &mut encoder);
+
+ let err = result.expect_err("expected error");
+ let matmul: &MatmulError<Metal> = (&err as &dyn StdError)
+ .source()
+ .and_then(|s| s.downcast_ref::<MatmulError<Metal>>())
+ .expect("expected MatmulError source");
+ assert!(
+ matches!(matmul, MatmulError::UnsupportedLayout { path: "QuantGemm" }),
+ "got {matmul:?}"
+ );
+}
+
+#[uzu_test]
+fn quant_gemm_accumulate_returns_unsupported_dop() {
+ let context = MetalContext::new().expect("Metal context");
+ let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())
+ .expect("MatmulMetalKernel");
+
+ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
+ let mut args = quant_arguments(&mut buffers, &input);
+ args.d_transform = MatmulDOps {
+ ab_scale: 1.0,
+ accumulate: true,
+ bias: None,
+ rht_factors: None,
+ };
+ let result = matmul.encode(args, &mut encoder);
+
+ let err = result.expect_err("expected error");
+ let matmul: &MatmulError<Metal> = (&err as &dyn StdError)
+ .source()
+ .and_then(|s| s.downcast_ref::<MatmulError<Metal>>())
+ .expect("expected MatmulError source");
+ assert!(
+ matches!(
+ matmul,
+ MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::ACCUMULATE,
+ ..
+ }
+ ),
+ "got {matmul:?}"
+ );
+}
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs
index 81b002146..06b62afc1 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs
@@ -2,7 +2,6 @@
#[path = "../../../common/mod.rs"]
mod common;
-mod qmm_transposed_test;
mod qmv_fast_test;
mod qmv_test;
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
deleted file mode 100644
index e42308dc0..000000000
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
+++ /dev/null
@@ -1,641 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use backend_uzu::{
- ArrayElement, DataType,
- backends::{
- common::{
- Backend, Context, Encoder, Kernels,
- gpu_types::QuantizationMethod,
- kernel::QuantizedMatmulQmmTransposedKernel,
- },
- cpu::Cpu,
- },
-};
-use half::{bf16, f16};
-use num_traits::Float;
-
-use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
-use crate::{
- common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
- uzu_test,
-};
-
-fn make_input_basic<T: ArrayElement + Float>(
- m: usize,
- k: usize,
- n: usize,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) -> Input<T> {
- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
-
- let max_val = if bits == 4 {
- 15u8
- } else {
- 255u8
- };
- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
- for j in 0..n {
- for l in 0..k {
- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
- }
- }
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
- }
- }
- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = match quant_method {
- QuantizationMethod::ScaleZeroPoint => {
- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
- zp_raw.push(zp_val);
- }
- }
- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
- for j in 0..n {
- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
- let packed_row = pack_zero_points(row, bits);
- let mut padded = packed_row;
- padded.resize(zp_stride, 0);
- zp_packed.extend_from_slice(&padded);
- }
- (Some(zp_packed), None)
- },
- QuantizationMethod::ScaleBias => {
- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
- }
- }
- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- (None, Some(biases))
- },
- };
-
- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
- for i in 0..m {
- for l in 0..k {
- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
- }
- }
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- quant_method,
- }
-}
-
-fn make_input_edge<T: ArrayElement + Float>(
- n: usize,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) -> Input<T> {
- let m = 2usize;
- let k = group_size as usize;
- let num_groups_k = 1;
-
- let weights_raw: Vec<u8> = vec![1u8; n * k];
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- let scales: Vec<T> = vec![T::one(); n * num_groups_k];
-
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = match quant_method {
- QuantizationMethod::ScaleZeroPoint => {
- let zp_packed = vec![0u8; n * zp_stride];
- (Some(zp_packed), None)
- },
- QuantizationMethod::ScaleBias => {
- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
- (None, Some(biases))
- },
- };
-
- let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- quant_method,
- }
-}
-
-fn get_output<B: Backend, T: ArrayElement + Float>(
- input: &Input<T>,
- bm: u32,
- bk: u32,
- bn: u32,
- wm: u32,
- wn: u32,
- aligned_n: bool,
- hadamard_factors: Option<&[i32]>,
-) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let use_hadamard = hadamard_factors.is_some();
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- &context,
- T::data_type(),
- input.group_size,
- input.bits,
- bm,
- bk,
- bn,
- wm,
- wn,
- input.quant_method,
- use_hadamard,
- aligned_n,
- )
- .expect("Failed to create QuantizedMatmulQmmTransposedKernel");
-
- let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);
-
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));
- let had_buf = hadamard_factors.map(|f| alloc_allocation_with_data::<B, i32>(&context, f));
-
- let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);
- let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- had_buf.as_ref(),
- input.k,
- input.n,
- input.m,
- &mut encoder,
- );
-
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- allocation_to_vec(&y_buf)
-}
-
-fn check_against_expected<T: ArrayElement + Float + Debug + Display>(
- input: &Input<T>,
- expected: &[T],
- bm: u32,
- bk: u32,
- bn: u32,
- wm: u32,
- wn: u32,
- aligned_n: bool,
-) {
- // GPU tiled accumulation rounds differently from CPU scalar reference.
- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
- DataType::BF16 => (0.05, 0.5),
- DataType::F16 => (0.02, 0.5),
- _ => (0.01, 0.1),
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input, bm, bk, bn, wm, wn, aligned_n, None);
- assert_eq!(expected.len(), output.len(), "Output length mismatch");
-
- let mut errors = 0;
- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
- let exp_f32 = exp.to_f32().unwrap();
- let got_f32 = got.to_f32().unwrap();
- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
- if errors < 5 {
- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
- }
- errors += 1;
- }
- }
- assert_eq!(
- errors,
- 0,
- "QMM transposed kernel: backend={}, shape=(BM={},BK={},BN={}), m={}, k={}, n={}, gs={}, bits={}, format={:?}: {} mismatches",
- std::any::type_name::<B>(),
- bm,
- bk,
- bn,
- input.m,
- input.k,
- input.n,
- input.group_size,
- input.bits,
- input.quant_method,
- errors,
- );
- });
-}
-
-struct TileCfg {
- bm: u32,
- bk: u32,
- bn: u32,
- wm: u32,
- wn: u32,
-}
-
-impl TileCfg {
- const fn aligned_n_default(&self) -> bool {
- // BN=64 tiles require output_dim % 64 == 0; BN=32 checks actual N at test time.
- self.bn == 64
- }
-}
-
-const CFG_BM8: TileCfg = TileCfg {
- bm: 8,
- bk: 32,
- bn: 32,
- wm: 1,
- wn: 1,
-};
-const CFG_32X32: TileCfg = TileCfg {
- bm: 32,
- bk: 32,
- bn: 32,
- wm: 2,
- wn: 2,
-};
-const CFG_WIDE: TileCfg = TileCfg {
- bm: 64,
- bk: 32,
- bn: 64,
- wm: 2,
- wn: 2,
-};
-const CFG_64X64: TileCfg = TileCfg {
- bm: 64,
- bk: 64,
- bn: 64,
- wm: 2,
- wn: 2,
-};
-
-fn pick_aligned_n(
- cfg: &TileCfg,
- n: usize,
-) -> bool {
- if cfg.bn == 32 {
- (n % 32) == 0
- } else {
- cfg.aligned_n_default()
- }
-}
-
-fn test_basic<T: ArrayElement + Float + Debug + Display>(
- cfg: &TileCfg,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);
- let gs = group_size as usize;
- let bn = cfg.bn as usize;
- // half+8bit: smaller dims to cap accumulated rounding error.
- let dims: Vec<(usize, usize, usize)> = if is_half && bits == 8 {
- vec![(2, gs, bn), (4, gs, bn)]
- } else {
- vec![(2, gs * 2, bn), (4, gs * 2, bn * 2), (8, gs, bn * 2)]
- };
-
- for (m, k, n) in dims {
- let input = make_input_basic::<T>(m, k, n, group_size, bits, quant_method);
- let aligned_n = pick_aligned_n(cfg, n);
- let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n, None);
- check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n);
- }
-}
-
-fn test_edge<T: ArrayElement + Float + Debug + Display>(
- cfg: &TileCfg,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- // Edge N is one BN tile wide — smallest legal N for the shape.
- let n = cfg.bn as usize;
- let input = make_input_edge::<T>(n, group_size, bits, quant_method);
- let aligned_n = pick_aligned_n(cfg, n);
- let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n, None);
- check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n);
-}
-
-// No CPU Hadamard reference — verify all three tile shapes agree instead.
-fn test_hadamard_consistency<T: ArrayElement + Float + Debug + Display>(
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- // m=n=64 satisfies both BN=32 and BN=64 alignment; group_size >= 64 for BK=64.
- let m = 64usize;
- let k = group_size as usize * 2;
- let n = 64usize;
-
- let input = make_input_basic::<T>(m, k, n, group_size, bits, quant_method);
-
- // ±1 factors: non-trivial Hadamard, bounded output across all dtypes.
- let factors: Vec<i32> = (0..n)
- .map(|i| {
- if (i * 13 + 7) % 2 == 0 {
- 1
- } else {
- -1
- }
- })
- .collect();
-
- // Hadamard amplifies by sqrt(32) ≈ 5.65; tolerance widened accordingly.
- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
- DataType::BF16 => (0.1, 1.0),
- DataType::F16 => (0.05, 1.0),
- _ => (0.02, 0.5),
- };
-
- for_each_non_cpu_backend!(|B| {
- let shapes: [(u32, u32, u32, u32, u32, &str); 3] =
- [(32, 32, 32, 2, 2, "32x32"), (64, 32, 64, 2, 2, "wide"), (64, 64, 64, 2, 2, "64x64")];
-
- let outputs: Vec<(&str, Vec<T>)> = shapes
- .iter()
- .map(|&(bm, bk, bn, wm, wn, name)| {
- let out = get_output::<B, T>(&input, bm, bk, bn, wm, wn, true, Some(&factors));
- (name, out)
- })
- .collect();
-
- // 32x32 is the reference — its Hadamard path is byte-identical to the original.
- let (_, ref reference) = outputs[0];
-
- for (name, output) in outputs.iter().skip(1) {
- let mut errors = 0;
- for (i, (&r, &o)) in reference.iter().zip(output.iter()).enumerate() {
- let r_f32 = r.to_f32().unwrap();
- let o_f32 = o.to_f32().unwrap();
- if !check_tolerance(r_f32, o_f32, rel_tol, abs_tol) {
- if errors < 5 {
- eprintln!(" idx={} 32x32={} {}={} diff={}", i, r_f32, name, o_f32, (r_f32 - o_f32).abs());
- }
- errors += 1;
- }
- }
- assert_eq!(
- errors,
- 0,
- "Hadamard cross-shape mismatch: backend={}, {} vs 32x32 reference, gs={}, bits={}, format={:?}: {} mismatches",
- std::any::type_name::<B>(),
- name,
- group_size,
- bits,
- quant_method,
- errors,
- );
- }
- });
-}
-
-// ---- Test-function macros -----------------------------------------------
-
-macro_rules! basic_tests {
- ($cfg:expr; $(
- $(#[$attr:meta])*
- fn $name:ident : $t:ty, $gs:literal, $bits:literal, $format:expr;
- )*) => {
- $(
- $(#[$attr])*
- #[uzu_test]
- fn $name() {
- test_basic::<$t>($cfg, $gs, $bits, $format);
- }
- )*
- };
-}
-
-macro_rules! edge_tests {
- ($cfg:expr; $(
- $(#[$attr:meta])*
- fn $name:ident : $t:ty, $gs:literal, $bits:literal, $format:expr;
- )*) => {
- $(
- $(#[$attr])*
- #[uzu_test]
- fn $name() {
- test_edge::<$t>($cfg, $gs, $bits, $format);
- }
- )*
- };
-}
-
-macro_rules! hadamard_tests {
- ($(
- $(#[$attr:meta])*
- fn $name:ident : $t:ty, $gs:literal, $bits:literal, $format:expr;
- )*) => {
- $(
- $(#[$attr])*
- #[uzu_test]
- fn $name() {
- test_hadamard_consistency::<$t>($gs, $bits, $format);
- }
- )*
- };
-}
-
-mod tile_bm8 {
- use super::*;
-
- basic_tests!(&CFG_BM8;
- // bf16 4-bit ZP has known precision limits
- #[ignore] fn test_bf16_gs32_4bit_zp: bf16, 32, 4, QuantizationMethod::ScaleZeroPoint;
- #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs32_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs64_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;
- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;
- );
-
- edge_tests!(&CFG_BM8;
- fn test_edge_bf16_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;
- fn test_edge_bf16_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;
- fn test_edge_bf16_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;
- );
-}
-
-// ---- tile_32x32 — the original base shape, full dtype matrix -------------
-
-mod tile_32x32 {
- use super::*;
-
- basic_tests!(&CFG_32X32;
- // 4-bit, zero points
- fn test_f32_gs32_4bit_zp: f32, 32, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_f32_gs64_4bit_zp: f32, 64, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_f32_gs128_4bit_zp: f32, 128, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_f16_gs32_4bit_zp: f16, 32, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_f16_gs64_4bit_zp: f16, 64, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_f16_gs128_4bit_zp: f16, 128, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs32_4bit_zp: bf16, 32, 4, QuantizationMethod::ScaleZeroPoint;
- #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;
- #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, QuantizationMethod::ScaleZeroPoint;
- // 8-bit, zero points
- fn test_f32_gs32_8bit_zp: f32, 32, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_f32_gs64_8bit_zp: f32, 64, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_f32_gs128_8bit_zp: f32, 128, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_f16_gs32_8bit_zp: f16, 32, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_f16_gs64_8bit_zp: f16, 64, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_f16_gs128_8bit_zp: f16, 128, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs32_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs64_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs128_8bit_zp: bf16,128, 8, QuantizationMethod::ScaleZeroPoint;
- // 4-bit, mlx quant
- fn test_f32_gs32_4bit_mlx: f32, 32, 4, QuantizationMethod::ScaleBias;
- fn test_f32_gs64_4bit_mlx: f32, 64, 4, QuantizationMethod::ScaleBias;
- fn test_f32_gs128_4bit_mlx: f32, 128, 4, QuantizationMethod::ScaleBias;
- fn test_f16_gs32_4bit_mlx: f16, 32, 4, QuantizationMethod::ScaleBias;
- fn test_f16_gs64_4bit_mlx: f16, 64, 4, QuantizationMethod::ScaleBias;
- fn test_f16_gs128_4bit_mlx: f16, 128, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs128_4bit_mlx: bf16,128, 4, QuantizationMethod::ScaleBias;
- // 8-bit, mlx quant
- fn test_f32_gs32_8bit_mlx: f32, 32, 8, QuantizationMethod::ScaleBias;
- fn test_f32_gs64_8bit_mlx: f32, 64, 8, QuantizationMethod::ScaleBias;
- fn test_f32_gs128_8bit_mlx: f32, 128, 8, QuantizationMethod::ScaleBias;
- fn test_f16_gs32_8bit_mlx: f16, 32, 8, QuantizationMethod::ScaleBias;
- fn test_f16_gs64_8bit_mlx: f16, 64, 8, QuantizationMethod::ScaleBias;
- fn test_f16_gs128_8bit_mlx: f16, 128, 8, QuantizationMethod::ScaleBias;
- fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;
- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;
- fn test_bf16_gs128_8bit_mlx: bf16,128, 8, QuantizationMethod::ScaleBias;
- );
-
- edge_tests!(&CFG_32X32;
- fn test_edge_f32_4bit_zp: f32, 32, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_edge_f32_8bit_zp: f32, 32, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_edge_f16_4bit_zp: f16, 32, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_edge_f32_4bit_mlx: f32, 32, 4, QuantizationMethod::ScaleBias;
- fn test_edge_f32_8bit_mlx: f32, 32, 8, QuantizationMethod::ScaleBias;
- fn test_edge_f16_4bit_mlx: f16, 32, 4, QuantizationMethod::ScaleBias;
- );
-}
-
-// ---- tile_wide — BM=64, BK=32, BN=64, bf16 only --------------------------
-
-mod tile_wide {
- use super::*;
-
- basic_tests!(&CFG_WIDE;
- // 4-bit, zero points
- fn test_bf16_gs32_4bit_zp: bf16, 32, 4, QuantizationMethod::ScaleZeroPoint;
- #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;
- #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, QuantizationMethod::ScaleZeroPoint;
- // 8-bit, zero points
- fn test_bf16_gs32_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs64_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs128_8bit_zp: bf16, 128, 8, QuantizationMethod::ScaleZeroPoint;
- // 4-bit, mlx quant
- fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, QuantizationMethod::ScaleBias;
- // 8-bit, mlx quant
- fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;
- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;
- fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, QuantizationMethod::ScaleBias;
- );
-
- edge_tests!(&CFG_WIDE;
- fn test_edge_bf16_4bit_zp: bf16, 32, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_edge_bf16_8bit_zp: bf16, 32, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_edge_bf16_4bit_mlx: bf16, 32, 4, QuantizationMethod::ScaleBias;
- fn test_edge_bf16_8bit_mlx: bf16, 32, 8, QuantizationMethod::ScaleBias;
- );
-}
-
-// ---- tile_64x64 — BM=BK=BN=64, bf16 only, group_size >= 64 --------------
-
-mod tile_64x64 {
- use super::*;
-
- basic_tests!(&CFG_64X64;
- // 4-bit, zero points
- #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;
- #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, QuantizationMethod::ScaleZeroPoint;
- // 8-bit, zero points
- fn test_bf16_gs64_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_bf16_gs128_8bit_zp: bf16, 128, 8, QuantizationMethod::ScaleZeroPoint;
- // 4-bit, mlx quant
- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, QuantizationMethod::ScaleBias;
- // 8-bit, mlx quant
- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;
- fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, QuantizationMethod::ScaleBias;
- );
-
- edge_tests!(&CFG_64X64;
- fn test_edge_bf16_4bit_zp: bf16, 64, 4, QuantizationMethod::ScaleZeroPoint;
- fn test_edge_bf16_8bit_zp: bf16, 64, 8, QuantizationMethod::ScaleZeroPoint;
- fn test_edge_bf16_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;
- fn test_edge_bf16_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;
- );
-}
-
-// ---- hadamard — cross-shape consistency of the generalized HT loop -------
-
-mod hadamard {
- use super::*;
-
- // f32 at (64,64,64) overflows threadgroup memory (34816 > 32768 bytes).
- hadamard_tests!(
- fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, QuantizationMethod::ScaleBias;
- fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, QuantizationMethod::ScaleBias;
- fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, QuantizationMethod::ScaleBias;
- fn test_f16_gs64_4bit_mlx: f16, 64, 4, QuantizationMethod::ScaleBias;
- fn test_f16_gs128_4bit_mlx: f16, 128, 4, QuantizationMethod::ScaleBias;
- );
-}
-
diff --git a/crates/download-manager/src/backends/universal/context.rs b/crates/download-manager/src/backends/universal/context.rs
index 1c810f97b..bd9e43b84 100644
--- a/crates/download-manager/src/backends/universal/context.rs
+++ b/crates/download-manager/src/backends/universal/context.rs
@@ -184,6 +184,18 @@ async fn download_once(
resume_from_bytes
},
StatusCode::OK => 0,
+ StatusCode::RANGE_NOT_SATISFIABLE => {
+ let advertised_total =
+ content_range.as_ref().and_then(|header| header.to_str().ok()).and_then(parse_content_range_total);
+ if advertised_total == Some(resume_from_bytes) {
+ backend_event_sender.send_progress(generation, resume_from_bytes, Some(resume_from_bytes)).await;
+ return tokio::fs::rename(resume_artifact_path, &config.destination)
+ .await
+ .map_err(|error| error.to_string());
+ }
+ let _ = tokio::fs::remove_file(resume_artifact_path).await;
+ return Err(format!("server did not honor range request: status {status}"));
+ },
_ => return Err(format!("server did not honor range request: status {status}")),
}
} else {
@@ -238,3 +250,12 @@ fn parse_content_range_start(header_value: &str) -> Option<u64> {
let (start, _) = range.split_once('-')?;
start.parse::<u64>().ok()
}
+
+fn parse_content_range_total(header_value: &str) -> Option<u64> {
+ let value = header_value.strip_prefix("bytes ")?.trim_start();
+ let (_, total) = value.split_once('/')?;
+ if total == "*" {
+ return None;
+ }
+ total.parse::<u64>().ok()
+}