Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Unify Gemm (#411)

eugene5c3dc00df293
diff --git a/Cargo.lock b/Cargo.lock
index 9e471853a..3fd5ea8d9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -559,6 +559,7 @@ dependencies = [
"anyhow",
"async-trait",
"bindgen",
+ "bitflags 2.11.1",
"blake3",
"bytemuck",
"bytesize",
diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.toml
index f33baca5f..7134ac07e 100644
--- a/crates/backend-uzu/Cargo.toml
+++ b/crates/backend-uzu/Cargo.toml
@@ -11,6 +11,8 @@ build = "build/main.rs"
[dependencies]
proc-macros.workspace = true
+bitflags.workspace = true
+derive_more.workspace = true
shoji.workspace = true
half.workspace = true
ndarray.workspace = true
@@ -111,7 +113,7 @@ path = "tests/unit/kernel/test_entry.rs"
[[bench]]
name = "kernel"
-path = "tests/unit/kernel/bench_entry.rs"
+path = "tests/performance/kernel/bench_entry.rs"
[[test]]
name = "performance"
diff --git a/crates/backend-uzu/build/common/enum_paths.rs b/crates/backend-uzu/build/common/enum_paths.rs
index 90880ba3a..991991808 100644
--- a/crates/backend-uzu/build/common/enum_paths.rs
+++ b/crates/backend-uzu/build/common/enum_paths.rs
@@ -5,35 +5,52 @@ use syn::{Type, TypePath, visit_mut::VisitMut};
use super::gpu_types::{GpuType, GpuTypeName, GpuTypePath, GpuTypes};
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum GpuTypeKind {
+ Enum,
+ OptionSet,
+}
+
+#[derive(Clone)]
+struct GpuTypeEntry {
+ path: GpuTypePath,
+ #[allow(dead_code)]
+ kind: GpuTypeKind,
+}
+
#[derive(Clone)]
pub struct EnumPaths {
- short_name_to_full_path: HashMap<GpuTypeName, GpuTypePath>,
+ short_name_to_entry: HashMap<GpuTypeName, GpuTypeEntry>,
}
impl EnumPaths {
pub fn from_gpu_types(gpu_types: &GpuTypes) -> anyhow::Result<Self> {
- let mut short_name_to_full_path = HashMap::new();
+ let mut short_name_to_entry = HashMap::new();
for file in gpu_types.files.iter() {
for ty in file.types.iter() {
- if let GpuType::Enum(enum_type) = ty {
- let name = GpuTypeName::from(enum_type.name.as_ref());
- let path = GpuTypePath::from(format!(
- "crate::backends::common::gpu_types::{}::{}",
- file.name, enum_type.name
- ));
- match short_name_to_full_path.entry(name) {
- Entry::Occupied(occupied) => {
- bail!("gpu type `{}` is duplicated", occupied.key())
- },
- Entry::Vacant(vacant) => {
- vacant.insert(path);
- },
- }
+ let (name_str, kind) = match ty {
+ GpuType::Enum(enum_type) => (enum_type.name.as_ref(), GpuTypeKind::Enum),
+ GpuType::OptionSet(option_set) => (option_set.name.as_ref(), GpuTypeKind::OptionSet),
+ GpuType::Struct(_) => continue,
+ };
+ let name = GpuTypeName::from(name_str);
+ let path =
+ GpuTypePath::from(format!("crate::backends::common::gpu_types::{}::{}", file.name, name_str));
+ match short_name_to_entry.entry(name) {
+ Entry::Occupied(occupied) => {
+ bail!("gpu type `{}` is duplicated", occupied.key())
+ },
+ Entry::Vacant(vacant) => {
+ vacant.insert(GpuTypeEntry {
+ path,
+ kind,
+ });
+ },
}
}
}
Ok(Self {
- short_name_to_full_path,
+ short_name_to_entry,
})
}
@@ -41,7 +58,15 @@ impl EnumPaths {
&self,
short_name: &str,
) -> Option<&str> {
- self.short_name_to_full_path.get(short_name).map(|path| &**path)
+ self.short_name_to_entry.get(short_name).map(|entry| &*entry.path)
+ }
+
+ #[allow(dead_code)]
+ pub fn kind_for(
+ &self,
+ short_name: &str,
+ ) -> Option<GpuTypeKind> {
+ self.short_name_to_entry.get(short_name).map(|entry| entry.kind)
}
pub fn canonicalize_type(
diff --git a/crates/backend-uzu/build/common/gpu_types/item_option_set.rs b/crates/backend-uzu/build/common/gpu_types/item_option_set.rs
new file mode 100644
index 000000000..2bce9e2ea
--- /dev/null
+++ b/crates/backend-uzu/build/common/gpu_types/item_option_set.rs
@@ -0,0 +1,76 @@
+use anyhow::Context;
+use proc_macro2::TokenStream;
+use quote::ToTokens;
+use syn::{
+ Attribute, Expr, Ident, Token, Visibility, braced,
+ parse::{Parse, ParseStream},
+};
+
+#[derive(Debug)]
+pub struct GpuTypeOptionSetVariant {
+ pub name: String,
+ /// Raw expression (e.g. `1 << 0`) emitted verbatim into C++.
+ pub value_expression: String,
+}
+
+#[derive(Debug)]
+pub struct GpuTypeOptionSet {
+ pub name: String,
+ pub underlying_type: String,
+ pub variants: Vec<GpuTypeOptionSetVariant>,
+}
+
+impl GpuTypeOptionSet {
+ /// Parse the token-stream inside a `bitflags! { … }` macro invocation.
+ pub fn parse(tokens: TokenStream) -> anyhow::Result<Self> {
+ let block: BitflagsBlock = syn::parse2(tokens).context("Cannot parse bitflags! contents")?;
+ Ok(Self {
+ name: block.name.to_string(),
+ underlying_type: block.underlying.to_string(),
+ variants: block
+ .variants
+ .into_iter()
+ .map(|(name, expression)| GpuTypeOptionSetVariant {
+ name: name.to_string(),
+ value_expression: expression.into_token_stream().to_string(),
+ })
+ .collect(),
+ })
+ }
+}
+
+struct BitflagsBlock {
+ name: Ident,
+ underlying: Ident,
+ variants: Vec<(Ident, Expr)>,
+}
+
+impl Parse for BitflagsBlock {
+ fn parse(input: ParseStream) -> syn::Result<Self> {
+ let _attrs = input.call(Attribute::parse_outer)?;
+ let _vis: Visibility = input.parse()?;
+ let _struct_kw: Token![struct] = input.parse()?;
+ let name: Ident = input.parse()?;
+ let _colon: Token![:] = input.parse()?;
+ let underlying: Ident = input.parse()?;
+
+ let body;
+ braced!(body in input);
+
+ let mut variants = Vec::new();
+ while !body.is_empty() {
+ let _const_kw: Token![const] = body.parse()?;
+ let variant_name: Ident = body.parse()?;
+ let _equals: Token![=] = body.parse()?;
+ let expression: Expr = body.parse()?;
+ let _semi: Token![;] = body.parse()?;
+ variants.push((variant_name, expression));
+ }
+
+ Ok(Self {
+ name,
+ underlying,
+ variants,
+ })
+ }
+}
diff --git a/crates/backend-uzu/build/common/gpu_types/mod.rs b/crates/backend-uzu/build/common/gpu_types/mod.rs
index b1b5cdbff..c03b4b98b 100644
--- a/crates/backend-uzu/build/common/gpu_types/mod.rs
+++ b/crates/backend-uzu/build/common/gpu_types/mod.rs
@@ -11,9 +11,11 @@ use syn::{Attribute, Ident, Item};
use walkdir::WalkDir;
mod item_enum;
+mod item_option_set;
mod item_struct;
pub use item_enum::GpuTypeEnum;
+pub use item_option_set::{GpuTypeOptionSet, GpuTypeOptionSetVariant};
pub use item_struct::{GpuTypeStruct, GpuTypeStructFieldType};
#[derive(Clone, Debug, PartialEq, Eq, Hash, From, AsRef, Deref, Display)]
@@ -47,6 +49,7 @@ fn ensure_repr_c(attrs: &[Attribute]) -> anyhow::Result<()> {
pub enum GpuType {
Enum(GpuTypeEnum),
Struct(GpuTypeStruct),
+ OptionSet(GpuTypeOptionSet),
}
#[derive(Debug)]
@@ -68,6 +71,9 @@ impl GpuTypeFile {
.filter_map(|item| match item {
Item::Enum(item) => Some(GpuTypeEnum::parse(item).map(GpuType::Enum)),
Item::Struct(item) => Some(GpuTypeStruct::parse(item).map(GpuType::Struct)),
+ Item::Macro(item) if item.mac.path.is_ident("bitflags") => {
+ Some(GpuTypeOptionSet::parse(item.mac.tokens).map(GpuType::OptionSet))
+ },
_ => None,
})
.collect::<anyhow::Result<Box<[GpuType]>>>()?;
diff --git a/crates/backend-uzu/build/metal/bindgen/specialize.rs b/crates/backend-uzu/build/metal/bindgen/specialize.rs
index e4e1fdb13..4f8715d3c 100644
--- a/crates/backend-uzu/build/metal/bindgen/specialize.rs
+++ b/crates/backend-uzu/build/metal/bindgen/specialize.rs
@@ -1,17 +1,18 @@
use anyhow::{Context, Result};
use proc_macro2::TokenStream;
use quote::{format_ident, quote};
-use syn::{Ident, Type, parse_quote};
+use syn::{Ident, Type};
use super::super::{
ast::{MetalArgumentType, MetalKernelInfo},
- enum_path_rewrite::is_enum_c_type,
+ enum_path_rewrite::gpu_type_kind_for_c_type,
};
-use crate::common::enum_paths::EnumPaths;
+use crate::common::enum_paths::{EnumPaths, GpuTypeKind};
enum SpecializeLowering {
Direct,
- CastTo(Type),
+ Enum,
+ OptionSet,
}
struct SpecializeArgument {
@@ -45,10 +46,10 @@ pub fn parse(
format!("specialize type `{}` in kernel `{}` cannot be parsed", rust_type_text, kernel_name)
})?;
let function_constant_index = base_function_constant_index.unwrap_or(0) + offset;
- let lowering = if is_enum_c_type(enum_paths, &argument.c_type) {
- SpecializeLowering::CastTo(parse_quote!(u32))
- } else {
- SpecializeLowering::Direct
+ let lowering = match gpu_type_kind_for_c_type(enum_paths, &argument.c_type) {
+ Some(GpuTypeKind::Enum) => SpecializeLowering::Enum,
+ Some(GpuTypeKind::OptionSet) => SpecializeLowering::OptionSet,
+ None => SpecializeLowering::Direct,
};
Ok(SpecializeArgument {
name,
@@ -90,8 +91,11 @@ impl SpecializeEmission {
SpecializeLowering::Direct => {
quote! { function_constants.set_value(&#name, #index); }
},
- SpecializeLowering::CastTo(wire_type) => {
- quote! { function_constants.set_value(&(#name as #wire_type), #index); }
+ SpecializeLowering::Enum => {
+ quote! { function_constants.set_value(&(#name as u32), #index); }
+ },
+ SpecializeLowering::OptionSet => {
+ quote! { function_constants.set_value(&#name.bits(), #index); }
},
}
})
@@ -115,7 +119,7 @@ impl SpecializeEmission {
return quote! { &entry_name };
}
let argument_names: Vec<&Ident> = self.arguments.iter().map(|argument| &argument.name).collect();
- let format_string = format!("{{}}{}", "_{}".repeat(argument_names.len()));
+ let format_string = format!("{{}}{}", "_{:?}".repeat(argument_names.len()));
quote! { &format!(#format_string, &entry_name #(, #argument_names)*) }
}
}
diff --git a/crates/backend-uzu/build/metal/enum_path_rewrite.rs b/crates/backend-uzu/build/metal/enum_path_rewrite.rs
index 50a1e689f..27e17dc17 100644
--- a/crates/backend-uzu/build/metal/enum_path_rewrite.rs
+++ b/crates/backend-uzu/build/metal/enum_path_rewrite.rs
@@ -3,21 +3,30 @@ use proc_macro2::TokenStream;
use quote::ToTokens;
use syn::{Expr, Path, parse_quote};
-use crate::common::{enum_paths::EnumPaths, expr_rewrite::rewrite_paths_with};
+use crate::common::{
+ enum_paths::{EnumPaths, GpuTypeKind},
+ expr_rewrite::rewrite_paths_with,
+};
-pub fn is_enum_c_type(
+pub fn gpu_type_kind_for_c_type(
enum_paths: &EnumPaths,
c_type: &str,
-) -> bool {
+) -> Option<GpuTypeKind> {
let trimmed = c_type.trim_start_matches("const ").trim();
- let Some(uzu_path) = trimmed.strip_prefix("uzu::") else {
- return false;
- };
- let Some(short_name) = uzu_path.rsplit("::").next() else {
- return false;
- };
+ let uzu_path = trimmed.strip_prefix("uzu::")?;
+ let short_name = uzu_path.rsplit("::").next()?;
let expected_rust_path = format!("crate::backends::common::gpu_types::{uzu_path}");
- enum_paths.full_path_for(short_name) == Some(expected_rust_path.as_str())
+ if enum_paths.full_path_for(short_name) != Some(expected_rust_path.as_str()) {
+ return None;
+ }
+ enum_paths.kind_for(short_name)
+}
+
+pub fn is_enum_c_type(
+ enum_paths: &EnumPaths,
+ c_type: &str,
+) -> bool {
+ gpu_type_kind_for_c_type(enum_paths, c_type).is_some()
}
pub fn rewrite_for_rust(
diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs
new file mode 100644
index 000000000..86903d05c
--- /dev/null
+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs
@@ -0,0 +1,44 @@
+use crate::common::gpu_types::GpuTypeOptionSet;
+
+fn r2c(ty: &str) -> anyhow::Result<&'static str> {
+ Ok(match ty {
+ "u8" => "uint8_t",
+ "u16" => "uint16_t",
+ "u32" => "uint",
+ "u64" => "uint64_t",
+ "i8" => "int8_t",
+ "i16" => "int16_t",
+ "i32" => "int",
+ "i64" => "int64_t",
+ unknown => anyhow::bail!("Unsupported option-set underlying type: {unknown}"),
+ })
+}
+
+/// Emit a Metal-side option-set as a struct wrapping the underlying primitive,
+/// with named static constants for each flag and a `contains()` helper.
+/// Implicit conversions from the underlying type allow a `uint` function
+/// constant to be passed where the struct is expected.
+pub fn gpu_type_gen_option_set(option_set: &GpuTypeOptionSet) -> anyhow::Result<String> {
+ let name = &option_set.name;
+ let underlying_c = r2c(&option_set.underlying_type)?;
+
+ let variants = option_set
+ .variants
+ .iter()
+ .map(|variant| {
+ format!(" static constant constexpr {underlying_c} {} = {};\n", variant.name, variant.value_expression)
+ })
+ .collect::<String>();
+
+ Ok(format!(
+ "struct {name} {{\n\
+ \x20 {underlying_c} raw_value;\n\
+ \x20 constexpr {name}() thread : raw_value(0) {{}}\n\
+ \x20 constexpr {name}({underlying_c} __dsl_v) thread : raw_value(__dsl_v) {{}}\n\
+ {variants}\
+ \x20 constexpr bool contains({underlying_c} flag) const thread {{\n\
+ \x20\x20\x20 return (raw_value & flag) != 0;\n\
+ \x20 }}\n\
+ }};"
+ ))
+}
diff --git a/crates/backend-uzu/build/metal/gpu_types/mod.rs b/crates/backend-uzu/build/metal/gpu_types/mod.rs
index 14e3b25ab..f98c7ebea 100644
--- a/crates/backend-uzu/build/metal/gpu_types/mod.rs
+++ b/crates/backend-uzu/build/metal/gpu_types/mod.rs
@@ -6,9 +6,11 @@ use itertools::Itertools;
use crate::common::gpu_types::{GpuType, GpuTypeFile, GpuTypes};
mod gpu_type_enum;
+mod gpu_type_option_set;
mod gpu_type_struct;
use gpu_type_enum::gpu_type_gen_enum;
+use gpu_type_option_set::gpu_type_gen_option_set;
use gpu_type_struct::gpu_type_gen_struct;
pub async fn gpu_type_gen(
@@ -38,6 +40,8 @@ async fn gpu_type_gen_file(
.with_context(|| format!("Failed to generate bindings for {gpu_type_enum:?}")),
GpuType::Struct(gpu_type_struct) => gpu_type_gen_struct(gpu_type_struct)
.with_context(|| format!("Failed to generate bindings for {gpu_type_struct:?}")),
+ GpuType::OptionSet(gpu_type_option_set) => gpu_type_gen_option_set(gpu_type_option_set)
+ .with_context(|| format!("Failed to generate bindings for {gpu_type_option_set:?}")),
})
.process_results(|mut it| it.join("\n\n"))?;
diff --git a/crates/backend-uzu/build/metal/gpu_types/template.ht b/crates/backend-uzu/build/metal/gpu_types/template.ht
index 8696754a5..ed7f31935 100644
--- a/crates/backend-uzu/build/metal/gpu_types/template.ht
+++ b/crates/backend-uzu/build/metal/gpu_types/template.ht
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/{module_name}.rs - do not edit manually
+// Auto-generated from gpu_types/{module_name} - do not edit manually
#pragma once
#include <metal_stdlib>
diff --git a/crates/backend-uzu/build/metal/wrapper.rs b/crates/backend-uzu/build/metal/wrapper.rs
index cc2280daf..6ee119cbf 100644
--- a/crates/backend-uzu/build/metal/wrapper.rs
+++ b/crates/backend-uzu/build/metal/wrapper.rs
@@ -211,7 +211,7 @@ fn kernel_wrappers(
wrapper_arguments.push("uint3 __dsl_axis_idx [[thread_position_in_grid]]".into());
}
- if kernel.has_groups() {
+ if kernel.has_groups() || kernel.has_thread_context() {
wrapper_arguments.push("uint3 __dsl_group_idx [[threadgroup_position_in_grid]]".into());
}
@@ -257,7 +257,7 @@ fn kernel_wrappers(
MetalArgumentType::Threads(_) => {
format!("__dsl_thread_idx.{}", thread_axis_letters.next().unwrap())
},
- MetalArgumentType::ThreadContext => "ThreadContext { .simdgroup_index = __dsl_simd_lane_idx, .threadgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),
+ MetalArgumentType::ThreadContext => "ThreadContext { .simd_lane_id = __dsl_simd_lane_idx, .simdgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup, .threadgroup_position = __dsl_group_idx }".into(),
})
.collect::<Vec<_>>()
.join(", ")
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime.rs b/crates/backend-uzu/src/audio/nanocodec/runtime.rs
index 65304cde7..8ad4a33c1 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime.rs
@@ -22,8 +22,8 @@ use crate::{
},
},
config::{
- ConfigDataType, DecoderConfig, DescriptAudioCodecConfig, EmbeddingConfig, EmbeddingConfigCommon,
- NormalizationConfig, TtsAudioDecoderConfig, TtsConfig,
+ DecoderConfig, DescriptAudioCodecConfig, EmbeddingConfig, EmbeddingConfigCommon, NormalizationConfig,
+ TtsAudioDecoderConfig, TtsConfig,
},
encodable_block::{Decoder, LayerExecutables, RMSNorm},
forward_pass::{model_shape::ModelShape, state::SharedBuffers},
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
index 6f75dde5a..e066650fe 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
@@ -1,7 +1,7 @@
use super::*;
pub(super) fn resolve_descript_audio_codec_vocoder_data_type(
- top_level_precision: ConfigDataType,
+ top_level_precision: DataType,
config: &DescriptAudioCodecConfig,
) -> AudioResult<DataType> {
let resolved_precision = top_level_precision;
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
new file mode 100644
index 000000000..a12f8652f
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
@@ -0,0 +1,50 @@
+use bitflags::bitflags;
+use derive_more::Display;
+
+#[repr(C)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum GemmInputPrologueKind {
+ FullPrecision,
+ ExternalRht,
+}
+
+#[repr(C)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum GemmOutputTransformKind {
+ Store,
+ Scale,
+ Accumulate,
+ Bias,
+ Rht,
+ ScaleAccumulate,
+ ScaleAccumulateBias,
+ ScaleAccumulateBiasRht,
+}
+
+#[repr(C)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum GemmWeightPrologueKind {
+ FullPrecision,
+ ScaleBiasDequant,
+ ScaleZeroPointDequant,
+}
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct GemmTilingConfig {
+ pub threadgroup_m: u32,
+ pub threadgroup_n: u32,
+ pub threadgroup_k: u32,
+ pub simdgroups_m: u32,
+ pub simdgroups_n: u32,
+}
+
+bitflags! {
+ #[repr(transparent)]
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+ pub struct GemmAlignment: u32 {
+ const M = 1 << 0;
+ const N = 1 << 1;
+ const K = 1 << 2;
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/matmul.rs b/crates/backend-uzu/src/backends/common/gpu_types/matmul.rs
index 26089436f..ebb3a2442 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/matmul.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/matmul.rs
@@ -11,7 +11,7 @@ pub struct GemmParams {
pub leading_dimension_d: u32,
pub threadgroups_per_column: u32,
pub threadgroups_per_row: u32,
- pub swizzle_log: u32,
pub aligned_inner_iterations: u32,
pub use_morton: bool,
+ pub ab_scale: f32,
}
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
index cb4a46b70..8d39a391d 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
@@ -6,6 +6,7 @@
pub mod activation_type;
pub mod argmax;
pub mod attention;
+pub mod gemm;
pub mod kv_cache_update;
pub mod matmul;
pub mod quantization;
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs
index 38445caf1..ea72aba6e 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs
@@ -1,66 +1,42 @@
+use derive_more::Display;
use proc_macros::uzu_config;
use crate::DataType;
#[repr(C)]
-#[derive(Copy, Eq)]
+#[derive(Display, Copy, Eq, Hash)]
#[uzu_config]
-#[serde(rename_all = "lowercase")]
pub enum QuantizationMode {
- UINT4,
- INT8,
- UINT8,
+ #[serde(rename = "uint4")]
+ U4,
+ #[serde(rename = "int8")]
+ I8,
+ #[serde(rename = "uint8")]
+ U8,
}
impl QuantizationMode {
pub fn packing_divisor(&self) -> usize {
- match self {
- QuantizationMode::UINT4 => 2,
- QuantizationMode::INT8 => 1,
- QuantizationMode::UINT8 => 1,
- }
+ let bits = DataType::from(*self).size_in_bits();
+ assert_eq!(8 % bits, 0, "QuantizationMode bit width ({bits}) must divide 8 evenly");
+ 8 / bits
}
pub fn storage_type(&self) -> DataType {
match self {
- QuantizationMode::UINT4 => DataType::U8,
- QuantizationMode::INT8 => DataType::I8,
- QuantizationMode::UINT8 => DataType::U8,
+ QuantizationMode::U4 => DataType::U8,
+ QuantizationMode::I8 => DataType::I8,
+ QuantizationMode::U8 => DataType::U8,
}
}
-
- pub fn to_u32(&self) -> u32 {
- (*self).into()
- }
}
impl From<QuantizationMode> for DataType {
fn from(val: QuantizationMode) -> Self {
match val {
- QuantizationMode::UINT4 => DataType::U4,
- QuantizationMode::INT8 => DataType::I8,
- QuantizationMode::UINT8 => DataType::U8,
- }
- }
-}
-
-impl From<u32> for QuantizationMode {
- fn from(val: u32) -> Self {
- match val {
- 0 => QuantizationMode::UINT4,
- 1 => QuantizationMode::INT8,
- 2 => QuantizationMode::UINT8,
- _ => panic!("Invalid QuantizationMode value: {val}"),
- }
- }
-}
-
-impl Into<u32> for QuantizationMode {
- fn into(self) -> u32 {
- match self {
- QuantizationMode::UINT4 => 0,
- QuantizationMode::INT8 => 1,
- QuantizationMode::UINT8 => 2,
+ QuantizationMode::U4 => DataType::U4,
+ QuantizationMode::I8 => DataType::I8,
+ QuantizationMode::U8 => DataType::U8,
}
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs
index 39c345e26..75c1acdd0 100644
--- a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs
@@ -97,8 +97,8 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
}
let bits = match configuration.mode {
- QuantizationMode::UINT4 => 4,
- QuantizationMode::INT8 | QuantizationMode::UINT8 => 8,
+ QuantizationMode::U4 => 4,
+ QuantizationMode::I8 | QuantizationMode::U8 => 8,
};
let group_size = configuration.group_size as u32;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index 44be4fa5e..f63a9dae9 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -21,15 +21,9 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
group_size: u32,
#[allow(unused)]
#[specialize]
- quant_mode: u32,
+ quantization_mode: QuantizationMode,
) {
- let quant_mode = QuantizationMode::from(quant_mode);
-
- let packing_divisor: u32 = if quant_mode == QuantizationMode::UINT4 {
- 2
- } else {
- 1
- };
+ let packing_divisor = quantization_mode.packing_divisor() as u32;
let weights_stride = model_dim / packing_divisor;
let num_groups = (model_dim + group_size - 1) / group_size;
@@ -49,8 +43,8 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
let scale = *scales.add((token_id as u32 * num_groups + group_idx) as usize);
let bias = *biases.add((token_id as u32 * num_groups + group_idx) as usize);
- let quantized_value: i32 = match quant_mode {
- QuantizationMode::UINT4 => {
+ let quantized_value: i32 = match quantization_mode {
+ QuantizationMode::U4 => {
let byte_idx = (token_id as u32 * weights_stride + dim_idx / 2) as usize;
let packed = *weights.add(byte_idx);
if (dim_idx & 1) == 0 {
@@ -59,12 +53,12 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
((packed >> 4) & 0x0F) as i32
}
},
- QuantizationMode::INT8 => {
+ QuantizationMode::I8 => {
let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;
let weights_i8 = weights as *const i8;
*weights_i8.add(elem_idx) as i32
},
- QuantizationMode::UINT8 => {
+ QuantizationMode::U8 => {
let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;
*weights.add(elem_idx) as i32
},
diff --git a/crates/backend-uzu/src/backends/metal/dense_buffer.rs b/crates/backend-uzu/src/backends/metal/dense_buffer.rs
index bba78219e..761821331 100644
--- a/crates/backend-uzu/src/backends/metal/dense_buffer.rs
+++ b/crates/backend-uzu/src/backends/metal/dense_buffer.rs
@@ -14,7 +14,7 @@ impl Buffer for Retained<ProtocolObject<dyn MTLBuffer>> {
}
fn size(&self) -> usize {
- (**self).length()
+ self.length()
}
}
diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rs
index a14e3459a..250432ce7 100644
--- a/crates/backend-uzu/src/backends/metal/error.rs
+++ b/crates/backend-uzu/src/backends/metal/error.rs
@@ -1,5 +1,7 @@
use thiserror::Error;
+use crate::backends::metal::kernel::matmul::gemm::GemmSpecializationError;
+
#[derive(Debug, Error)]
pub enum MetalError {
#[error("Cannot open device")]
@@ -24,6 +26,8 @@ pub enum MetalError {
CannotCreateFunction,
#[error("Cannot create pipeline state: {0}")]
CannotCreatePipelineState(String),
+ #[error("Invalid GEMM specialization: {0:?}")]
+ InvalidGemmSpecialization(GemmSpecializationError),
#[error("Can not allocate buffer with size={0}")]
SparseBufferAlloc(usize),
#[error("Can not allocate heap with size={0} and page size={1}")]
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index a4ac61a72..fd40c931f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -139,27 +139,11 @@ PUBLIC KERNEL(AttentionGemm)(
const int key_source_stride = int(params.k_strides[2]);
const int value_source_stride = int(params.v_strides[2]);
- thread QueryLoader query_loader(
- q,
- query_source_stride,
- query_shared,
- thread_context.threadgroup_index,
- thread_context.simdgroup_index
- );
- thread KeyLoader key_loader(
- k,
- key_source_stride,
- key_shared,
- thread_context.threadgroup_index,
- thread_context.simdgroup_index
- );
- thread ValueLoader value_loader(
- v,
- value_source_stride,
- value_shared,
- thread_context.threadgroup_index,
- thread_context.simdgroup_index
- );
+ thread QueryLoader
+ query_loader(q, query_source_stride, query_shared, thread_context);
+ thread KeyLoader key_loader(k, key_source_stride, key_shared, thread_context);
+ thread ValueLoader
+ value_loader(v, value_source_stride, value_shared, thread_context);
TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));
@@ -224,13 +208,13 @@ PUBLIC KERNEL(AttentionGemm)(
// Lane coordinates and pointer offsets
const short2 lane_coordinates =
SimdgroupMultiplyAccumulateType::get_lane_coordinates(
- thread_context.simdgroup_index
+ thread_context.simd_lane_id
);
const short lane_row = lane_coordinates.y;
const short lane_col = lane_coordinates.x;
const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS *
- short(thread_context.threadgroup_index);
+ short(thread_context.simdgroup_index);
const short query_shared_offset =
(simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index 368cc31e6..e9c89c2f7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -74,16 +74,16 @@ PUBLIC KERNEL(AttentionSinglePass)(
: suffix_position + q_seq_idx;
queries += q_offset * HEAD_DIM +
- thread_context.simdgroup_index * qk_elements_per_thread;
+ thread_context.simd_lane_id * qk_elements_per_thread;
keys += kv_head_idx * k_head_stride +
- thread_context.threadgroup_index * k_seq_stride +
- thread_context.simdgroup_index * qk_elements_per_thread;
+ thread_context.simdgroup_index * k_seq_stride +
+ thread_context.simd_lane_id * qk_elements_per_thread;
values += kv_head_idx * v_head_stride +
- thread_context.threadgroup_index * v_seq_stride +
- thread_context.simdgroup_index * value_elements_per_thread;
+ thread_context.simdgroup_index * v_seq_stride +
+ thread_context.simd_lane_id * value_elements_per_thread;
out += o_offset * value_dim +
- thread_context.threadgroup_index * value_elements_per_thread;
+ thread_context.simdgroup_index * value_elements_per_thread;
// Read the query and 0 the output accumulator
for (int i = 0; i < qk_elements_per_thread; i++) {
@@ -95,7 +95,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
U max_score = -INFINITY;
U sum_exp_score = 0;
- if (has_sinks && thread_context.threadgroup_index == 0) {
+ if (has_sinks && thread_context.simdgroup_index == 0) {
const int num_q_heads = static_cast<int>(num_heads);
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
@@ -103,7 +103,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// For each key
- for (uint i = thread_context.threadgroup_index; i < sequence_length;
+ for (uint i = thread_context.simdgroup_index; i < sequence_length;
i += SEQUENCE_BLOCK_SIZE) {
if (should_use_key(
ring_params,
@@ -151,27 +151,27 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// Each thread has a partial part of the output so we need to combine them.
- if (thread_context.simdgroup_index == 0) {
- shared_max_scores[thread_context.threadgroup_index] = max_score;
- shared_sum_exp_scores[thread_context.threadgroup_index] = sum_exp_score;
+ if (thread_context.simd_lane_id == 0) {
+ shared_max_scores[thread_context.simdgroup_index] = max_score;
+ shared_sum_exp_scores[thread_context.simdgroup_index] = sum_exp_score;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = shared_max_scores[thread_context.simdgroup_index];
+ max_score = shared_max_scores[thread_context.simd_lane_id];
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
sum_exp_score =
- simd_sum(shared_sum_exp_scores[thread_context.simdgroup_index] * factor);
+ simd_sum(shared_sum_exp_scores[thread_context.simd_lane_id] * factor);
// Now we need to aggregate all the outputs
for (int i = 0; i < value_elements_per_thread; i++) {
shared_outputs
- [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +
- thread_context.threadgroup_index] = o[i];
+ [thread_context.simd_lane_id * HEAD_BLOCK_SIZE +
+ thread_context.simdgroup_index] = o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
o[i] = simd_sum(
shared_outputs
- [thread_context.threadgroup_index * HEAD_BLOCK_SIZE +
- thread_context.simdgroup_index] *
+ [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +
+ thread_context.simd_lane_id] *
factor
) /
sum_exp_score;
@@ -179,7 +179,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// And write the output
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
for (int i = 0; i < value_elements_per_thread; i++) {
out[i] = static_cast<T>(o[i]);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index 3c36971e8..713629426 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -57,7 +57,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
// Flat query head mapping — fixed hardware shape, dynamic logical mapping
const uint head_idx = head_group_idx * SIMDGROUPS_PER_THREADGROUP +
- thread_context.threadgroup_index;
+ thread_context.simdgroup_index;
if (head_idx >= num_heads)
return;
@@ -74,14 +74,14 @@ PUBLIC KERNEL(AttentionTwoPass1)(
thread U q[elements_per_thread];
thread U o[elements_per_thread] = {0};
- queries += q_offset * HEAD_DIM +
- thread_context.simdgroup_index * elements_per_thread;
+ queries +=
+ q_offset * HEAD_DIM + thread_context.simd_lane_id * elements_per_thread;
keys += kv_head_idx * k_head_stride + block_idx * k_seq_stride +
- thread_context.simdgroup_index * elements_per_thread;
+ thread_context.simd_lane_id * elements_per_thread;
values += kv_head_idx * v_head_stride + block_idx * v_seq_stride +
- thread_context.simdgroup_index * elements_per_thread;
+ thread_context.simd_lane_id * elements_per_thread;
out += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM +
- thread_context.simdgroup_index * elements_per_thread;
+ thread_context.simd_lane_id * elements_per_thread;
sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
@@ -135,7 +135,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
}
// Write partial output
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
sums[0] = sum_exp_score;
maxs[0] = max_score;
}
@@ -170,7 +170,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(
// Find global max across all blocks (each lane handles strided blocks)
U max_score = -1e9;
- for (uint b = thread_context.simdgroup_index; b < TOTAL_BLOCKS_COUNT;
+ for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT;
b += METAL_SIMD_SIZE) {
max_score = max(max_score, maxs[b]);
}
@@ -178,7 +178,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(
// Find global sum across all blocks
U sum_exp = 0;
- for (uint b = thread_context.simdgroup_index; b < TOTAL_BLOCKS_COUNT;
+ for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT;
b += METAL_SIMD_SIZE) {
sum_exp += sums[b] * fast::exp(maxs[b] - new_max);
}
@@ -189,7 +189,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(
thread U o[elements_per_thread] = {0};
for (uint chunk = 0; chunk < TOTAL_BLOCKS_COUNT; chunk += METAL_SIMD_SIZE) {
- uint block_idx = chunk + thread_context.simdgroup_index;
+ uint block_idx = chunk + thread_context.simd_lane_id;
U factor = (block_idx < TOTAL_BLOCKS_COUNT)
? fast::exp(maxs[block_idx] - new_max)
: 0;
@@ -197,7 +197,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(
const device float* block_partials =
partials + o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +
block_idx * HEAD_DIM +
- thread_context.threadgroup_index * elements_per_thread;
+ thread_context.simdgroup_index * elements_per_thread;
for (uint i = 0; i < elements_per_thread; i++) {
U val = (block_idx < TOTAL_BLOCKS_COUNT) ? block_partials[i] * factor : 0;
@@ -206,9 +206,9 @@ PUBLIC KERNEL(AttentionTwoPass2)(
}
// Write the output
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
out += o_offset * HEAD_DIM +
- thread_context.threadgroup_index * elements_per_thread;
+ thread_context.simdgroup_index * elements_per_thread;
for (uint i = 0; i < elements_per_thread; i++) {
out[i] = static_cast<T>(o[i] / sum_exp_score);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h b/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h
index f0fa45ae7..4c0691786 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h
@@ -4,8 +4,13 @@
using namespace metal;
struct ThreadContext {
- uint simdgroup_index;
- uint threadgroup_index;
- uint simdgroup_size;
- uint simdgroups_per_threadgroup;
+ uint simd_lane_id; // [[thread_index_in_simdgroup]] — 0..simdgroup_size
+ uint simdgroup_index; // [[simdgroup_index_in_threadgroup]] —
+ // 0..simdgroups_per_threadgroup
+ uint simdgroup_size; // [[threads_per_simdgroup]] — typically 32
+ uint simdgroups_per_threadgroup; // [[simdgroups_per_threadgroup]]
+ uint3 threadgroup_position; // [[threadgroup_position_in_grid]]
+ uint3 threadgroup_size;
+ uint3 threadgroup_count;
+ uint3 grid_size;
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
index 34a0bf229..df37462bc 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
@@ -38,23 +38,23 @@ static typename Op::value_type threadgroup_cooperative_reduce(
) {
// Phase 1: reduce within each simdgroup
typename Op::value_type local = Op::simd_reduce(value);
- if (thread_context.simdgroup_index == 0) {
- shared[thread_context.threadgroup_index] = local;
+ if (thread_context.simd_lane_id == 0) {
+ shared[thread_context.simdgroup_index] = local;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// Phase 2: first simdgroup reduces across simdgropus
- if (thread_context.threadgroup_index == 0) {
+ if (thread_context.simdgroup_index == 0) {
typename Op::value_type total =
- thread_context.simdgroup_index <
+ thread_context.simd_lane_id <
(BLOCK_SIZE + thread_context.simdgroup_size - 1) /
thread_context.simdgroup_size
- ? shared[thread_context.simdgroup_index]
+ ? shared[thread_context.simd_lane_id]
: Op::identity;
total = Op::simd_reduce(total);
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
shared[0] = total;
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index f06629847..d0852fb1d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -17,13 +17,10 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
constant uint32_t& model_dim,
constant float& input_scale,
const uint32_t group_size SPECIALIZE,
- const uint32_t quant_mode SPECIALIZE,
+ const QuantizationMode quantization_mode SPECIALIZE,
const uint dim_idx AXIS(model_dim, 16),
const uint batch_idx AXIS(batch_size, 16)
) {
- const QuantizationMode quantization_mode = QuantizationMode(
- quant_mode
- ); // TODO: should be accepted as a kernel argument
const uint thread_position_in_grid = batch_idx * model_dim + dim_idx;
const uint64_t token_id = token_ids[batch_idx];
if (token_id >= vocab_size) {
@@ -38,12 +35,12 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const T bias = biases[token_id * num_groups + group_idx];
const uint packing_divisor =
- quantization_mode == QuantizationMode::UINT4 ? 2 : 1;
+ quantization_mode == QuantizationMode::U4 ? 2 : 1;
const uint weights_stride = model_dim / packing_divisor;
int quantized_value = 0;
switch (quantization_mode) {
- case QuantizationMode::UINT4: {
+ case QuantizationMode::U4: {
const uint byte_idx = token_id * weights_stride + (dim_idx / 2);
const uint8_t packed = weights[byte_idx];
if ((dim_idx & 1) == 0) {
@@ -53,14 +50,14 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
}
break;
};
- case QuantizationMode::INT8: {
+ case QuantizationMode::I8: {
const uint elem_idx = token_id * weights_stride + dim_idx;
const device int8_t* weights_i8 =
reinterpret_cast<const device int8_t*>(weights);
quantized_value = int(weights_i8[elem_idx]);
break;
};
- case QuantizationMode::UINT8: {
+ case QuantizationMode::U8: {
const uint elem_idx = token_id * weights_stride + dim_idx;
quantized_value = int(weights[elem_idx]);
break;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
index c5f89ed64..eda99e0a2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/activation_type.rs - do not edit manually
+// Auto-generated from gpu_types/activation_type - do not edit manually
#pragma once
#include <metal_stdlib>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/argmax.h b/crates/backend-uzu/src/backends/metal/kernel/generated/argmax.h
index 3b818d559..a67743173 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/argmax.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/argmax.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/argmax.rs - do not edit manually
+// Auto-generated from gpu_types/argmax - do not edit manually
#pragma once
#include <metal_stdlib>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h b/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h
index 30680f516..55951bc81 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/attention.rs - do not edit manually
+// Auto-generated from gpu_types/attention - do not edit manually
#pragma once
#include <metal_stdlib>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
new file mode 100644
index 000000000..b21071fa8
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
@@ -0,0 +1,49 @@
+// Auto-generated from gpu_types/gemm - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::gemm {
+enum class GemmInputPrologueKind : uint32_t {
+ FullPrecision = 0,
+ ExternalRht = 1,
+};
+
+enum class GemmOutputTransformKind : uint32_t {
+ Store = 0,
+ Scale = 1,
+ Accumulate = 2,
+ Bias = 3,
+ Rht = 4,
+ ScaleAccumulate = 5,
+ ScaleAccumulateBias = 6,
+ ScaleAccumulateBiasRht = 7,
+};
+
+enum class GemmWeightPrologueKind : uint32_t {
+ FullPrecision = 0,
+ ScaleBiasDequant = 1,
+ ScaleZeroPointDequant = 2,
+};
+
+typedef struct {
+ uint32_t threadgroup_m;
+ uint32_t threadgroup_n;
+ uint32_t threadgroup_k;
+ uint32_t simdgroups_m;
+ uint32_t simdgroups_n;
+} GemmTilingConfig;
+
+struct GemmAlignment {
+ uint raw_value;
+ constexpr GemmAlignment() thread : raw_value(0) {}
+ constexpr GemmAlignment(uint __dsl_v) thread : raw_value(__dsl_v) {}
+ static constant constexpr uint M = 1 << 0;
+ static constant constexpr uint N = 1 << 1;
+ static constant constexpr uint K = 1 << 2;
+ constexpr bool contains(uint flag) const thread {
+ return (raw_value & flag) != 0;
+ }
+};
+} // namespace uzu::gemm
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/kv_cache_update.h b/crates/backend-uzu/src/backends/metal/kernel/generated/kv_cache_update.h
index c42228faa..b4a175e60 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/kv_cache_update.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/kv_cache_update.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/kv_cache_update.rs - do not edit manually
+// Auto-generated from gpu_types/kv_cache_update - do not edit manually
#pragma once
#include <metal_stdlib>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/matmul.h b/crates/backend-uzu/src/backends/metal/kernel/generated/matmul.h
index 400808477..072833b93 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/matmul.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/matmul.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/matmul.rs - do not edit manually
+// Auto-generated from gpu_types/matmul - do not edit manually
#pragma once
#include <metal_stdlib>
@@ -14,8 +14,8 @@ typedef struct {
uint32_t leading_dimension_d;
uint32_t threadgroups_per_column;
uint32_t threadgroups_per_row;
- uint32_t swizzle_log;
uint32_t aligned_inner_iterations;
bool use_morton;
+ float ab_scale;
} GemmParams;
} // namespace uzu::matmul
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization.h b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization.h
index 0315c17d3..01c1ffba7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/quantization.rs - do not edit manually
+// Auto-generated from gpu_types/quantization - do not edit manually
#pragma once
#include <metal_stdlib>
@@ -6,8 +6,8 @@ using namespace metal;
namespace uzu::quantization {
enum class QuantizationMode : uint32_t {
- UINT4 = 0,
- INT8 = 1,
- UINT8 = 2,
+ U4 = 0,
+ I8 = 1,
+ U8 = 2,
};
} // namespace uzu::quantization
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h
index 98bd012aa..3a265d899 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/quantization_method.rs - do not edit manually
+// Auto-generated from gpu_types/quantization_method - do not edit manually
#pragma once
#include <metal_stdlib>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/ring.h b/crates/backend-uzu/src/backends/metal/kernel/generated/ring.h
index a10f4c220..003e6f971 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/ring.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/ring.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/ring.rs - do not edit manually
+// Auto-generated from gpu_types/ring - do not edit manually
#pragma once
#include <metal_stdlib>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/trie.h b/crates/backend-uzu/src/backends/metal/kernel/generated/trie.h
index 0ccfaf5d1..80b04ee4b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/trie.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/trie.h
@@ -1,4 +1,4 @@
-// Auto-generated from gpu_types/trie.rs - do not edit manually
+// Auto-generated from gpu_types/trie - do not edit manually
#pragma once
#include <metal_stdlib>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h
deleted file mode 100644
index 278600759..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h
+++ /dev/null
@@ -1,100 +0,0 @@
-#pragma once
-
-#include "../../common/integral_constant.h"
-#include "../../common/thread_context.h"
-#include "defines.h"
-
-using namespace metal;
-
-namespace uzu {
-namespace matmul {
-
-///////////////////////////////////////////////////////////////////////////////
-// AluFragmentOps - stateless traits/ops for 8x8 ALU matmul (simdgroup_matrix)
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename T>
-struct AluFragmentOps {
- METAL_CONST ushort FRAGMENT_ROWS = 8;
- METAL_CONST ushort FRAGMENT_COLS = 8;
- METAL_CONST ushort ELEMENTS_PER_THREAD =
- (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
- METAL_CONST ushort THREAD_ELEMENT_ROWS = 1;
- METAL_CONST ushort THREAD_ELEMENT_COLS = 2;
- METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = 1;
-
- static_assert(
- THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,
- "AluFragmentOps shape is not consistent with element count"
- );
-
- typedef metal::simdgroup_matrix<T, FRAGMENT_ROWS, FRAGMENT_COLS>
- SimdgroupMatrixType;
-
- template <typename U>
- using ThreadVector = metal::vec<U, ELEMENTS_PER_THREAD>;
-
- METAL_FUNC static constexpr void multiply_accumulate(
- thread ThreadVector<T>& D,
- thread ThreadVector<T>& A,
- thread ThreadVector<T>& B,
- thread ThreadVector<T>& C
- ) {
- SimdgroupMatrixType D_mat;
- SimdgroupMatrixType A_mat;
- SimdgroupMatrixType B_mat;
- SimdgroupMatrixType C_mat;
-
- reinterpret_cast<thread ThreadVector<T>&>(A_mat.thread_elements()) = A;
- reinterpret_cast<thread ThreadVector<T>&>(B_mat.thread_elements()) = B;
- reinterpret_cast<thread ThreadVector<T>&>(C_mat.thread_elements()) = C;
-
- simdgroup_multiply_accumulate(D_mat, A_mat, B_mat, C_mat);
-
- D = reinterpret_cast<thread ThreadVector<T>&>(D_mat.thread_elements());
- }
-
- template <class OutputTile, class LeftTile, class RightTile>
- METAL_FUNC static void tile_matmul(
- thread OutputTile& output,
- thread LeftTile& left,
- thread RightTile& right
- ) {
- constexpr ushort tile_m = OutputTile::TILE_ROWS;
- constexpr ushort tile_n = OutputTile::TILE_COLS;
- constexpr ushort tile_k = LeftTile::TILE_COLS;
-
- static_assert(
- tile_m == LeftTile::TILE_ROWS,
- "tile matmul: M dimensions do not match"
- );
- static_assert(
- tile_n == RightTile::TILE_COLS,
- "tile matmul: N dimensions do not match"
- );
- static_assert(
- tile_k == RightTile::TILE_ROWS,
- "tile matmul: K dimensions do not match"
- );
-
- METAL_PRAGMA_UNROLL
- for (ushort row = 0; row < tile_m; ++row) {
- METAL_PRAGMA_UNROLL
- for (ushort col = 0; col < tile_n; ++col) {
- const ushort column_serpentine = (row % 2) ? (tile_n - 1 - col) : col;
- METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < tile_k; ++k) {
- multiply_accumulate(
- output.fragment_at(row, column_serpentine),
- left.fragment_at(row, k),
- right.fragment_at(k, column_serpentine),
- output.fragment_at(row, column_serpentine)
- );
- }
- }
- }
- }
-};
-
-} // namespace matmul
-} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
index b67370a2a..3f2973d06 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
@@ -60,11 +60,11 @@ struct Fragment {
METAL_FUNC static constexpr short2 get_position(
const thread ThreadContext& thread_context
) {
- const ushort simdgroup_index = ushort(thread_context.simdgroup_index);
- const short quad = simdgroup_index / 4;
- const short row = (quad & 4) + (simdgroup_index / 2) % 4;
+ const ushort simd_lane_id = ushort(thread_context.simd_lane_id);
+ const short quad = simd_lane_id / 4;
+ const short row = (quad & 4) + (simd_lane_id / 2) % 4;
const short col =
- ((quad & 2) + simdgroup_index % 2) * Ops::THREAD_ELEMENT_COLS;
+ ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;
return short2{col, row};
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h
deleted file mode 100644
index 6b1251c0a..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h
+++ /dev/null
@@ -1,355 +0,0 @@
-#pragma once
-
-#include "loader.h"
-#include "threadgroup_tile.h"
-#include "../../generated/matmul.h"
-
-using namespace metal;
-
-namespace uzu {
-namespace matmul {
-
-///////////////////////////////////////////////////////////////////////////////
-// ThreadgroupGemm Kernel
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- typename U,
- int BLOCK_ROWS,
- int BLOCK_COLS,
- int BLOCK_DEPTH,
- int SIMDGROUPS_PER_ROW,
- int SIMDGROUPS_PER_COLUMN,
- bool transpose_a,
- bool transpose_b,
- bool MN_aligned,
- bool K_aligned,
- typename AccumulatorType = float,
- typename Epilogue = TransformNone<U, AccumulatorType>>
-struct ThreadgroupGemm {
- METAL_CONST ushort THREADGROUP_PADDING_A = 16 / sizeof(T);
- METAL_CONST ushort THREADGROUP_PADDING_B = 16 / sizeof(T);
- METAL_CONST ushort THREADGROUP_MEMORY_SIZE_A =
- transpose_a ? BLOCK_DEPTH * (BLOCK_ROWS + THREADGROUP_PADDING_A)
- : BLOCK_ROWS * (BLOCK_DEPTH + THREADGROUP_PADDING_A);
- METAL_CONST ushort THREADGROUP_MEMORY_SIZE_B =
- transpose_b ? BLOCK_COLS * (BLOCK_DEPTH + THREADGROUP_PADDING_B)
- : BLOCK_DEPTH * (BLOCK_COLS + THREADGROUP_PADDING_B);
- METAL_CONST ushort THREADGROUP_MEMORY_SIZE =
- THREADGROUP_MEMORY_SIZE_A + THREADGROUP_MEMORY_SIZE_B;
-
- METAL_CONST ushort THREADGROUP_SIZE =
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
-
- using LoaderAType = ThreadgroupLoader<
- T,
- transpose_a ? BLOCK_DEPTH : BLOCK_ROWS,
- transpose_a ? BLOCK_ROWS : BLOCK_DEPTH,
- transpose_a ? BLOCK_ROWS + THREADGROUP_PADDING_A
- : BLOCK_DEPTH + THREADGROUP_PADDING_A,
- !transpose_a,
- THREADGROUP_SIZE>;
- using LoaderBType = ThreadgroupLoader<
- T,
- transpose_b ? BLOCK_COLS : BLOCK_DEPTH,
- transpose_b ? BLOCK_DEPTH : BLOCK_COLS,
- transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B
- : BLOCK_COLS + THREADGROUP_PADDING_B,
- transpose_b,
- THREADGROUP_SIZE>;
- using ThreadgroupTileType = ThreadgroupTile<
- T,
- U,
- BLOCK_ROWS,
- BLOCK_COLS,
- BLOCK_DEPTH,
- SIMDGROUPS_PER_ROW,
- SIMDGROUPS_PER_COLUMN,
- transpose_a,
- transpose_b,
- transpose_a ? BLOCK_ROWS + THREADGROUP_PADDING_A
- : BLOCK_DEPTH + THREADGROUP_PADDING_A,
- transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B
- : BLOCK_COLS + THREADGROUP_PADDING_B,
- AccumulatorType,
- Epilogue>;
-
- template <bool M_aligned, bool N_aligned, bool K_aligned_>
- static METAL_FUNC void gemm_loop(
- threadgroup T* a_shared,
- threadgroup T* b_shared,
- const int aligned_k_iterations,
- thread LoaderAType& loader_a,
- thread LoaderBType& loader_b,
- thread ThreadgroupTileType& threadgroup_tile,
- thread const ushort& threadgroup_block_rows,
- thread const ushort& threadgroup_block_cols,
- thread const ushort& leftover_block_depth
- ) {
- short2 tile_dimensions_a =
- transpose_a ? short2(threadgroup_block_rows, BLOCK_DEPTH)
- : short2(BLOCK_DEPTH, threadgroup_block_rows);
- short2 tile_dimensions_b =
- transpose_b ? short2(BLOCK_DEPTH, threadgroup_block_cols)
- : short2(threadgroup_block_cols, BLOCK_DEPTH);
-
- for (int k = 0; k < aligned_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (M_aligned) {
- loader_a.load_unsafe();
- } else {
- loader_a.load_safe(tile_dimensions_a);
- }
-
- if (N_aligned) {
- loader_b.load_unsafe();
- } else {
- loader_b.load_safe(tile_dimensions_b);
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- threadgroup_tile.multiply_accumulate(a_shared, b_shared);
-
- loader_a.next();
- loader_b.next();
- }
-
- if (!K_aligned_) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- short2 last_tile_dimensions_a =
- transpose_a ? short2(threadgroup_block_rows, leftover_block_depth)
- : short2(leftover_block_depth, threadgroup_block_rows);
- short2 last_tile_dimensions_b =
- transpose_b ? short2(leftover_block_depth, threadgroup_block_cols)
- : short2(threadgroup_block_cols, leftover_block_depth);
-
- loader_a.load_safe(last_tile_dimensions_a);
- loader_b.load_safe(last_tile_dimensions_b);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- threadgroup_tile.multiply_accumulate(a_shared, b_shared);
- }
- }
-
- static METAL_FUNC void run(
- const device T* a,
- const device T* b,
- device U* d,
- const constant GemmParams* params,
- const constant float& ab_scale,
- const bool is_accumulate,
- threadgroup T* a_shared,
- threadgroup T* b_shared,
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint2 threadgroup_position [[threadgroup_position_in_grid]],
- uint3 thread_position [[thread_position_in_threadgroup]]
- ) {
- (void)thread_position;
-
- const int swizzle_stride = pow2(params->swizzle_log);
- const int swizzled_row_id = threadgroup_position.y * swizzle_stride +
- (threadgroup_position.x % swizzle_stride);
- const int swizzled_col_id = threadgroup_position.x / swizzle_stride;
-
- if (params->threadgroups_per_row <= swizzled_col_id ||
- params->threadgroups_per_column <= swizzled_row_id) {
- return;
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- const int output_row = swizzled_row_id * BLOCK_ROWS;
- const int output_col = swizzled_col_id * BLOCK_COLS;
- const size_t output_row_long = size_t(output_row);
- const size_t output_col_long = size_t(output_col);
-
- a += transpose_a ? output_row_long
- : output_row_long * params->leading_dimension_a;
- b += transpose_b ? output_col_long * params->leading_dimension_b
- : output_col_long;
- d += output_row_long * params->leading_dimension_d + output_col_long;
-
- thread LoaderAType loader_a(
- a,
- params->leading_dimension_a,
- a_shared,
- simd_group_id,
- simd_lane_id
- );
- thread LoaderBType loader_b(
- b,
- params->leading_dimension_b,
- b_shared,
- simd_group_id,
- simd_lane_id
- );
-
- thread ThreadgroupTileType threadgroup_tile(simd_group_id, simd_lane_id);
-
- TransformScaleAccumulate<AccumulatorType, AccumulatorType> epilogue(
- ab_scale,
- is_accumulate ? 1.0f : 0.0f
- );
-
- ///////////////////////////////////////////////////////////////////////////
- // MNK aligned loop
- if (MN_aligned) {
- for (int k = 0; k < params->aligned_inner_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_a.load_unsafe();
- loader_b.load_unsafe();
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- threadgroup_tile.multiply_accumulate(a_shared, b_shared);
-
- loader_a.next();
- loader_b.next();
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- if (!K_aligned) {
- int leftover_block_depth =
- params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
- short2 tile_dimensions_a =
- transpose_a ? short2(BLOCK_ROWS, leftover_block_depth)
- : short2(leftover_block_depth, BLOCK_ROWS);
- short2 tile_dimensions_b =
- transpose_b ? short2(leftover_block_depth, BLOCK_COLS)
- : short2(BLOCK_COLS, leftover_block_depth);
-
- loader_a.load_safe(tile_dimensions_a);
- loader_b.load_safe(tile_dimensions_b);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- threadgroup_tile.multiply_accumulate(a_shared, b_shared);
- }
-
- threadgroup_tile
- .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
- threadgroup_tile.store_result(d, params->leading_dimension_d);
- return;
- }
- ///////////////////////////////////////////////////////////////////////////
- // MN unaligned loop
- else {
- ushort threadgroup_block_rows =
- min(BLOCK_ROWS, ((int)params->M) - output_row);
- ushort threadgroup_block_cols =
- min(BLOCK_COLS, ((int)params->N) - output_col);
- ushort leftover_block_depth =
- params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
-
- if (threadgroup_block_rows == BLOCK_ROWS &&
- threadgroup_block_cols == BLOCK_COLS) {
- gemm_loop<true, true, K_aligned>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- threadgroup_tile,
- threadgroup_block_rows,
- threadgroup_block_cols,
- leftover_block_depth
- );
-
- threadgroup_tile
- .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
- threadgroup_tile.store_result(d, params->leading_dimension_d);
- return;
-
- } else if (threadgroup_block_cols == BLOCK_COLS) {
- gemm_loop<false, true, K_aligned>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- threadgroup_tile,
- threadgroup_block_rows,
- threadgroup_block_cols,
- leftover_block_depth
- );
-
- threadgroup_tile.apply_epilogue_safe(
- d,
- params->leading_dimension_d,
- 1,
- short2(threadgroup_block_cols, threadgroup_block_rows),
- epilogue
- );
- threadgroup_tile.store_result_safe(
- d,
- params->leading_dimension_d,
- short2(threadgroup_block_cols, threadgroup_block_rows)
- );
- return;
-
- } else if (threadgroup_block_rows == BLOCK_ROWS) {
- gemm_loop<true, false, K_aligned>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- threadgroup_tile,
- threadgroup_block_rows,
- threadgroup_block_cols,
- leftover_block_depth
- );
-
- threadgroup_tile.apply_epilogue_safe(
- d,
- params->leading_dimension_d,
- 1,
- short2(threadgroup_block_cols, threadgroup_block_rows),
- epilogue
- );
- threadgroup_tile.store_result_safe(
- d,
- params->leading_dimension_d,
- short2(threadgroup_block_cols, threadgroup_block_rows)
- );
- return;
-
- } else {
- gemm_loop<false, false, K_aligned>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- threadgroup_tile,
- threadgroup_block_rows,
- threadgroup_block_cols,
- leftover_block_depth
- );
-
- threadgroup_tile.apply_epilogue_safe(
- d,
- params->leading_dimension_d,
- 1,
- short2(threadgroup_block_cols, threadgroup_block_rows),
- epilogue
- );
- threadgroup_tile.store_result_safe(
- d,
- params->leading_dimension_d,
- short2(threadgroup_block_cols, threadgroup_block_rows)
- );
- return;
- }
- }
- }
-};
-
-} // namespace matmul
-} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
deleted file mode 100644
index 77cc9092d..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
+++ /dev/null
@@ -1,204 +0,0 @@
-#pragma once
-
-#include "../../common/defines.h"
-#include "../../common/integral_constant.h"
-using namespace uzu;
-#include "fragment.h"
-#include "mxu_fragment_ops.h"
-#include "mxu_gemm_loop.h"
-#include "../../generated/matmul.h"
-
-using namespace metal;
-
-namespace uzu {
-namespace matmul {
-
-template <
- typename T,
- ushort BLOCK_ROWS,
- ushort BLOCK_COLS,
- ushort SIMDGROUPS_PER_ROW,
- ushort SIMDGROUPS_PER_COLUMN,
- bool APPLY_AB_SCALE,
- bool IS_ACCUMULATE>
-struct GemmMppCore {
- METAL_CONST ushort SIMDGROUP_BLOCK_M = BLOCK_ROWS / SIMDGROUPS_PER_ROW;
- METAL_CONST ushort SIMDGROUP_BLOCK_N = BLOCK_COLS / SIMDGROUPS_PER_COLUMN;
-
- METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;
- METAL_CONST ushort BLOCK_K = 256;
-
- METAL_CONST ushort TILES_M =
- SIMDGROUP_BLOCK_M / MxuFragmentOps::FRAGMENT_ROWS;
- METAL_CONST ushort TILES_N =
- SIMDGROUP_BLOCK_N / MxuFragmentOps::FRAGMENT_COLS;
-
- using AccumulatorType = float;
-
- static METAL_FUNC void run(
- const device T* left_matrix,
- const device T* right_matrix,
- device T* output_matrix,
- const constant GemmParams* params,
- const bool align_m,
- const bool align_n,
- const bool align_k,
- const float ab_scale,
- uint simd_group_id,
- uint2 threadgroup_position,
- const thread ThreadContext& thread_context
- ) {
- uint tile_id_x, tile_id_y;
- if (params->use_morton) {
- uint linear_id = threadgroup_position.x;
- uint morton_x = linear_id;
- uint morton_y = linear_id >> 1;
- morton_x &= 0x55555555u;
- morton_x = (morton_x | (morton_x >> 1)) & 0x33333333u;
- morton_x = (morton_x | (morton_x >> 2)) & 0x0F0F0F0Fu;
- morton_x = (morton_x | (morton_x >> 4)) & 0x00FF00FFu;
- morton_x = (morton_x | (morton_x >> 8)) & 0x0000FFFFu;
- morton_y &= 0x55555555u;
- morton_y = (morton_y | (morton_y >> 1)) & 0x33333333u;
- morton_y = (morton_y | (morton_y >> 2)) & 0x0F0F0F0Fu;
- morton_y = (morton_y | (morton_y >> 4)) & 0x00FF00FFu;
- morton_y = (morton_y | (morton_y >> 8)) & 0x0000FFFFu;
- tile_id_x = morton_x;
- tile_id_y = morton_y;
- } else {
- tile_id_x = threadgroup_position.x;
- tile_id_y = threadgroup_position.y;
- }
-
- if (tile_id_x >= params->threadgroups_per_row ||
- tile_id_y >= params->threadgroups_per_column) {
- return;
- }
-
- const uint block_row_start = tile_id_y * BLOCK_ROWS;
- const uint block_col_start = tile_id_x * BLOCK_COLS;
- const size_t block_row_start_long = size_t(block_row_start);
- const size_t block_col_start_long = size_t(block_col_start);
-
- const device T* left_block_ptr =
- left_matrix + block_row_start_long * params->leading_dimension_a;
- const device T* right_block_ptr =
- right_matrix + block_col_start_long * params->leading_dimension_b;
-
- const ushort tile_row_offset =
- SIMDGROUP_BLOCK_M * (simd_group_id / SIMDGROUPS_PER_COLUMN);
- const ushort tile_col_offset =
- SIMDGROUP_BLOCK_N * (simd_group_id % SIMDGROUPS_PER_COLUMN);
-
- device T* output_ptr =
- output_matrix + block_row_start_long * params->leading_dimension_d +
- block_col_start_long + tile_row_offset * params->leading_dimension_d +
- tile_col_offset;
-
- const short simdgroup_limit_m =
- align_m
- ? SIMDGROUP_BLOCK_M
- : short(
- min(int(SIMDGROUP_BLOCK_M),
- int(params->M) - int(block_row_start + tile_row_offset))
- );
- const short simdgroup_limit_n =
- align_n
- ? SIMDGROUP_BLOCK_N
- : short(
- min(int(SIMDGROUP_BLOCK_N),
- int(params->N) - int(block_col_start + tile_col_offset))
- );
-
- const device T* left_simdgroup_ptr =
- left_block_ptr + size_t(tile_row_offset) * params->leading_dimension_a;
- const device T* right_simdgroup_ptr =
- right_block_ptr +
- size_t(tile_col_offset) * int(params->leading_dimension_b);
-
- const int aligned_k_iterations = int(params->K) / int(BLOCK_K);
-
- dispatch_bool(align_k, [&](auto aligned_k) {
- dispatch_bool(
- align_m || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),
- [&](auto aligned_m) {
- dispatch_bool(
- align_n || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),
- [&](auto aligned_n) {
- auto accumulator_tile = gemm_loop<
- T,
- SIMDGROUP_BLOCK_M,
- SIMDGROUP_BLOCK_N,
- SIMDGROUP_BLOCK_K,
- BLOCK_K,
- false,
- true,
- aligned_m.value,
- aligned_n.value,
- aligned_k.value,
- AccumulatorType>(
- left_simdgroup_ptr,
- right_simdgroup_ptr,
- int(params->leading_dimension_a),
- int(params->leading_dimension_b),
- int(params->K),
- aligned_k_iterations,
- simdgroup_limit_m,
- simdgroup_limit_n,
- thread_context
- );
-
- if constexpr (APPLY_AB_SCALE) {
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;
- i++) {
- accumulator_tile.elements()[i] *=
- AccumulatorType(ab_scale);
- }
- }
-
- if constexpr (IS_ACCUMULATE) {
- Fragment<T, TILES_M, TILES_N, MxuFragmentOps>
- existing_output(thread_context);
- if constexpr (aligned_m.value && aligned_n.value) {
- existing_output.load(
- output_ptr,
- int(params->leading_dimension_d)
- );
- } else {
- existing_output.load_safe(
- output_ptr,
- int(params->leading_dimension_d),
- short2(simdgroup_limit_n, simdgroup_limit_m)
- );
- }
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;
- i++) {
- accumulator_tile.elements()[i] +=
- AccumulatorType(existing_output.elements()[i]);
- }
- }
-
- if constexpr (aligned_m.value && aligned_n.value) {
- accumulator_tile.store(
- output_ptr,
- int(params->leading_dimension_d)
- );
- } else {
- accumulator_tile.store_safe(
- output_ptr,
- int(params->leading_dimension_d),
- short2(simdgroup_limit_n, simdgroup_limit_m)
- );
- }
- }
- );
- }
- );
- });
- }
-};
-
-} // namespace matmul
-} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
index 9fa6d3048..c56a6f0bb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
@@ -1,6 +1,7 @@
#pragma once
#include "../../common/defines.h"
+#include "../../common/thread_context.h"
using namespace metal;
@@ -44,14 +45,16 @@ struct ThreadgroupLoader {
const device T* source_ptr,
const int source_leading_dim,
threadgroup T* destination_ptr,
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
+ const thread ThreadContext& thread_context
)
: source_leading_dimension(source_leading_dim),
tile_stride(
REDUCTION_DIMENSION ? BLOCK_COLS : BLOCK_ROWS * source_leading_dim
),
- thread_index(simd_group_id * METAL_SIMD_SIZE + simd_lane_id),
+ thread_index(
+ thread_context.simdgroup_index * METAL_SIMD_SIZE +
+ thread_context.simd_lane_id
+ ),
block_row_index(thread_index / THREAD_COLS),
block_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),
destination(
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
index a00c9b4ec..609f3bb40 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
@@ -60,6 +60,8 @@ METAL_FUNC auto gemm_loop(
thread_context
);
+ volatile int mxu_iteration_fence;
+
const int left_offset =
transpose_a ? inner_k * leading_dimension_a : inner_k;
const int right_offset =
@@ -98,6 +100,8 @@ METAL_FUNC auto gemm_loop(
left_tile,
right_tile
);
+
+ (void)mxu_iteration_fence;
}
left_ptr += transpose_a ? (BLOCK_K * leading_dimension_a) : BLOCK_K;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index 082cd7284..bbb06f838 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -1,5 +1,6 @@
#pragma once
+#include "../../common/thread_context.h"
#include "simdgroup_fragment.h"
using namespace metal;
@@ -80,17 +81,18 @@ struct ThreadgroupTile {
ushort a_shared_offset;
ushort b_shared_offset;
- METAL_FUNC ThreadgroupTile(
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- ) {
+ METAL_FUNC ThreadgroupTile(const thread ThreadContext& thread_context) {
ushort tile_row_base =
- SIMDGROUP_BLOCK_SIZE * (simd_group_id / SIMDGROUPS_PER_COLUMN);
+ SIMDGROUP_BLOCK_SIZE *
+ (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
ushort tile_col_base =
- SIMDGROUP_BLOCK_SIZE * (simd_group_id % SIMDGROUPS_PER_COLUMN);
+ SIMDGROUP_BLOCK_SIZE *
+ (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
const ushort2 simdgroup_coordinates = ushort2(
- SimdgroupMultiplyAccumulateType::get_lane_coordinates(simd_lane_id)
+ SimdgroupMultiplyAccumulateType::get_lane_coordinates(
+ thread_context.simd_lane_id
+ )
);
simdgroup_row_offset = simdgroup_coordinates.y;
simdgroup_col_offset = simdgroup_coordinates.x;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
deleted file mode 100644
index 02ea47e3a..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
+++ /dev/null
@@ -1,65 +0,0 @@
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-
-#include "common/gemm.h"
-
-using namespace uzu::matmul;
-
-#define GEMM_MAX_THREADGROUP_A 2560
-#define GEMM_MAX_THREADGROUP_B 1536
-
-template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint BLOCK_DEPTH, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN, bool MN_ALIGNED, bool K_ALIGNED, bool TRANSPOSE_B>
-VARIANTS(T, float, half, bfloat)
-VARIANTS(BLOCK_ROWS, 32, 64)
-VARIANTS(BLOCK_COLS, 32, 64)
-VARIANTS(BLOCK_DEPTH, 8, 16, 32)
-VARIANTS(SIMDGROUPS_PER_ROW, 1, 2, 4)
-VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)
-VARIANTS(MN_ALIGNED, false, true)
-VARIANTS(K_ALIGNED, false, true)
-VARIANTS(TRANSPOSE_B, false, true)
-CONSTRAINT(max(BLOCK_ROWS, BLOCK_COLS) <= 32 * SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN)
-KERNEL(MatmulGemm)(
- const device T* a,
- const device T* b,
- device T* d,
- const constant uzu::matmul::GemmParams* params,
- const constant uint& group_count_x,
- const constant uint& group_count_y,
- const constant float& ab_scale,
- const bool is_accumulate SPECIALIZE,
- threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],
- threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],
- const uint group_x GROUPS(group_count_x),
- const uint group_y GROUPS(group_count_y),
- const uint thread_x THREADS(32),
- const uint thread_y THREADS(2),
- const uint thread_z THREADS(2),
- const ThreadContext thread_context
-) {
- ThreadgroupGemm<
- T,
- T,
- BLOCK_ROWS,
- BLOCK_COLS,
- BLOCK_DEPTH,
- SIMDGROUPS_PER_ROW,
- SIMDGROUPS_PER_COLUMN,
- false, // transpose_a
- TRANSPOSE_B,
- MN_ALIGNED,
- K_ALIGNED,
- float>::
- run(a,
- b,
- d,
- params,
- ab_scale,
- is_accumulate,
- a_shared,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.threadgroup_index,
- uint2(group_x, group_y),
- uint3(thread_x, thread_y, thread_z));
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs
deleted file mode 100644
index 96a346750..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs
+++ /dev/null
@@ -1,176 +0,0 @@
-use crate::{
- DataType,
- backends::{
- common::{Context, kernel::matmul::MatmulArgumentC},
- metal::{Metal, context::MetalContext},
- },
-};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct GemmSpecialization {
- pub block_rows: u32,
- pub block_cols: u32,
- pub block_depth: u32,
- pub simdgroups_per_row: u32,
- pub simdgroups_per_column: u32,
- pub align_mn: bool,
- pub align_k: bool,
- pub transpose_b: bool,
- pub is_accumulate: bool,
-}
-
-impl GemmSpecialization {
- pub fn precompile_configs(data_type: DataType) -> &'static [Self] {
- match data_type {
- DataType::BF16 => &[
- Self {
- block_rows: 64,
- block_cols: 32,
- block_depth: 32,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_mn: false,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 32,
- block_depth: 32,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_mn: true,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- block_depth: 16,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_mn: false,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- block_depth: 16,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_mn: true,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- block_depth: 16,
- simdgroups_per_row: 1,
- simdgroups_per_column: 2,
- align_mn: true,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- ],
- DataType::F16 => &[
- Self {
- block_rows: 64,
- block_cols: 64,
- block_depth: 16,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_mn: true,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- block_depth: 16,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_mn: false,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- ],
- DataType::F32 => &[
- Self {
- block_rows: 32,
- block_cols: 64,
- block_depth: 16,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_mn: false,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 32,
- block_cols: 64,
- block_depth: 16,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_mn: true,
- align_k: true,
- transpose_b: true,
- is_accumulate: false,
- },
- ],
- _ => &[],
- }
- }
-
- pub fn select(
- context: &MetalContext,
- data_type: DataType,
- batch_dim: u32,
- input_dim: u32,
- output_dim: u32,
- transpose_b: bool,
- c: &MatmulArgumentC<Metal>,
- ) -> Self {
- let overall_work_elements = batch_dim * output_dim;
- let is_float32 = matches!(data_type, DataType::F32);
- let prefer_half_or_tf32 = !is_float32 || MetalContext::tf32_enabled();
-
- let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column) =
- if context.is_high_performance() && overall_work_elements >= (1 << 20) {
- if prefer_half_or_tf32 {
- if 2 * std::cmp::max(batch_dim, output_dim) > input_dim {
- (64, 64, 16, 2, 2)
- } else {
- (64, 32, 32, 2, 2)
- }
- } else {
- (32, 64, 16, 2, 2)
- }
- } else if prefer_half_or_tf32 {
- (64, 32, 32, 2, 2)
- } else {
- (32, 64, 16, 2, 2)
- };
-
- Self {
- block_rows,
- block_cols,
- block_depth,
- simdgroups_per_row,
- simdgroups_per_column,
- align_mn: (batch_dim % block_rows) == 0 && (output_dim % block_cols) == 0,
- align_k: (input_dim % block_depth) == 0,
- transpose_b,
- is_accumulate: matches!(c, MatmulArgumentC::Accumulate),
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h
new file mode 100644
index 000000000..84b37753a
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h
@@ -0,0 +1,60 @@
+#pragma once
+
+#include "../../common/defines.h"
+#include "../../../generated/matmul.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+static METAL_FUNC uint morton_expand_bits(uint x) {
+ x &= 0x55555555u;
+ x = (x | (x >> 1)) & 0x33333333u;
+ x = (x | (x >> 2)) & 0x0F0F0F0Fu;
+ x = (x | (x >> 4)) & 0x00FF00FFu;
+ x = (x | (x >> 8)) & 0x0000FFFFu;
+ return x;
+}
+
+// Morton-unpacks a 1D dispatch index (`fp.rs` sets `group_count_y = 1`) into
+// 2D.
+static METAL_FUNC uint2 morton_tile_id(uint2 threadgroup_position) {
+ return uint2(
+ morton_expand_bits(threadgroup_position.x),
+ morton_expand_bits(threadgroup_position.x >> 1)
+ );
+}
+
+static METAL_FUNC uint2 tile_id(
+ uint2 threadgroup_position,
+ const constant uzu::matmul::GemmParams* params
+) {
+ if (params->use_morton) {
+ return morton_tile_id(threadgroup_position);
+ }
+ return threadgroup_position;
+}
+
+// `out_of_bounds` accounts for morton-padded tile ids past the actual grid.
+template <uint THREADGROUP_BLOCK_M, uint THREADGROUP_BLOCK_N>
+struct ThreadgroupTileGeometry {
+ uint2 tile_id;
+ uint block_row_start;
+ uint block_col_start;
+ bool out_of_bounds;
+
+ static METAL_FUNC ThreadgroupTileGeometry
+ compute(uint2 tile_id, const constant uzu::matmul::GemmParams* params) {
+ ThreadgroupTileGeometry geometry;
+ geometry.tile_id = tile_id;
+ geometry.out_of_bounds = (tile_id.x >= params->threadgroups_per_row) ||
+ (tile_id.y >= params->threadgroups_per_column);
+ geometry.block_row_start = tile_id.y * THREADGROUP_BLOCK_M;
+ geometry.block_col_start = tile_id.x * THREADGROUP_BLOCK_N;
+ return geometry;
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
new file mode 100644
index 000000000..3e62895df
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -0,0 +1,198 @@
+#pragma once
+
+#include "../../../common/integral_constant.h"
+#include "../../../common/thread_context.h"
+#include "../../common/fragment.h"
+#include "../../common/mxu_fragment_ops.h"
+#include "../../common/mxu_gemm_loop.h"
+#include "../../../generated/matmul.h"
+#include "../generated/gemm.h"
+#include "block_geometry.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+// MPP `matmul2d_descriptor.transpose_right` drives `TRANSPOSE_B` (MSL
+// Spec 6.18).
+template <
+ typename T,
+ ushort THREADGROUP_BLOCK_M,
+ ushort THREADGROUP_BLOCK_N,
+ ushort SIMDGROUPS_PER_ROW,
+ ushort SIMDGROUPS_PER_COLUMN,
+ bool TRANSPOSE_B>
+struct MxuMmaCore {
+ METAL_CONST ushort SIMDGROUP_BLOCK_M =
+ THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
+ METAL_CONST ushort SIMDGROUP_BLOCK_N =
+ THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
+ METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;
+ METAL_CONST ushort THREADGROUP_BLOCK_K = 256;
+ METAL_CONST ushort TILES_M =
+ SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
+ METAL_CONST ushort TILES_N =
+ SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;
+
+ using AccumulatorType = float;
+
+ static METAL_FUNC void run(
+ const device T* a,
+ const device T* b,
+ device T* d,
+ const constant uzu::matmul::GemmParams* params,
+ GemmAlignment alignment,
+ GemmOutputTransformKind output_transform,
+ const thread ThreadContext& thread_context
+ ) {
+ const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);
+ const auto geometry =
+ ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::
+ compute(tile, params);
+ if (geometry.out_of_bounds) {
+ return;
+ }
+
+ const size_t block_row = size_t(geometry.block_row_start);
+ const size_t block_col = size_t(geometry.block_col_start);
+
+ const device T* a_block = a + block_row * params->leading_dimension_a;
+ const device T* b_block =
+ b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);
+
+ const ushort tile_row_offset =
+ SIMDGROUP_BLOCK_M *
+ (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
+ const ushort tile_col_offset =
+ SIMDGROUP_BLOCK_N *
+ (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
+
+ device T* d_simdgroup =
+ d + block_row * params->leading_dimension_d + block_col +
+ tile_row_offset * params->leading_dimension_d + tile_col_offset;
+
+ const short simdgroup_limit_m =
+ alignment.contains(GemmAlignment::M)
+ ? SIMDGROUP_BLOCK_M
+ : short(
+ min(int(SIMDGROUP_BLOCK_M),
+ int(params->M) -
+ int(geometry.block_row_start + tile_row_offset))
+ );
+ const short simdgroup_limit_n =
+ alignment.contains(GemmAlignment::N)
+ ? SIMDGROUP_BLOCK_N
+ : short(
+ min(int(SIMDGROUP_BLOCK_N),
+ int(params->N) -
+ int(geometry.block_col_start + tile_col_offset))
+ );
+
+ const device T* a_simdgroup =
+ a_block + size_t(tile_row_offset) * params->leading_dimension_a;
+ const device T* b_simdgroup =
+ b_block + (TRANSPOSE_B ? size_t(tile_col_offset) *
+ int(params->leading_dimension_b)
+ : size_t(tile_col_offset));
+
+ const int aligned_k_iterations = int(params->K) / int(THREADGROUP_BLOCK_K);
+
+ const bool apply_scale =
+ output_transform == GemmOutputTransformKind::Scale ||
+ output_transform == GemmOutputTransformKind::ScaleAccumulate;
+ const bool apply_accumulate =
+ output_transform == GemmOutputTransformKind::Accumulate ||
+ output_transform == GemmOutputTransformKind::ScaleAccumulate;
+
+ dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {
+ dispatch_bool(
+ alignment.contains(GemmAlignment::M) ||
+ (simdgroup_limit_m == SIMDGROUP_BLOCK_M),
+ [&](auto aligned_m) {
+ dispatch_bool(
+ alignment.contains(GemmAlignment::N) ||
+ (simdgroup_limit_n == SIMDGROUP_BLOCK_N),
+ [&](auto aligned_n) {
+ auto accumulator_tile = uzu::matmul::gemm_loop<
+ T,
+ SIMDGROUP_BLOCK_M,
+ SIMDGROUP_BLOCK_N,
+ SIMDGROUP_BLOCK_K,
+ THREADGROUP_BLOCK_K,
+ false,
+ TRANSPOSE_B,
+ aligned_m.value,
+ aligned_n.value,
+ aligned_k.value,
+ AccumulatorType>(
+ a_simdgroup,
+ b_simdgroup,
+ int(params->leading_dimension_a),
+ int(params->leading_dimension_b),
+ int(params->K),
+ aligned_k_iterations,
+ simdgroup_limit_m,
+ simdgroup_limit_n,
+ thread_context
+ );
+
+ if (apply_scale) {
+ const AccumulatorType scale =
+ AccumulatorType(params->ab_scale);
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;
+ i++) {
+ accumulator_tile.elements()[i] *= scale;
+ }
+ }
+
+ if (apply_accumulate) {
+ uzu::matmul::Fragment<
+ T,
+ TILES_M,
+ TILES_N,
+ uzu::matmul::MxuFragmentOps>
+ existing_output(thread_context);
+ if constexpr (aligned_m.value && aligned_n.value) {
+ existing_output.load(
+ d_simdgroup,
+ int(params->leading_dimension_d)
+ );
+ } else {
+ existing_output.load_safe(
+ d_simdgroup,
+ int(params->leading_dimension_d),
+ short2(simdgroup_limit_n, simdgroup_limit_m)
+ );
+ }
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;
+ i++) {
+ accumulator_tile.elements()[i] +=
+ AccumulatorType(existing_output.elements()[i]);
+ }
+ }
+
+ if constexpr (aligned_m.value && aligned_n.value) {
+ accumulator_tile.store(
+ d_simdgroup,
+ int(params->leading_dimension_d)
+ );
+ } else {
+ accumulator_tile.store_safe(
+ d_simdgroup,
+ int(params->leading_dimension_d),
+ short2(simdgroup_limit_n, simdgroup_limit_m)
+ );
+ }
+ }
+ );
+ }
+ );
+ });
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
new file mode 100644
index 000000000..10fd83532
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -0,0 +1,229 @@
+#pragma once
+
+#include "../../../common/integral_constant.h"
+#include "../../../common/thread_context.h"
+#include "../../common/defines.h"
+#include "../../common/loader.h"
+#include "../../common/threadgroup_tile.h"
+#include "../../../generated/matmul.h"
+#include "../generated/gemm.h"
+#include "block_geometry.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <
+ typename T,
+ int THREADGROUP_BLOCK_M,
+ int THREADGROUP_BLOCK_N,
+ int THREADGROUP_BLOCK_K,
+ int SIMDGROUPS_PER_ROW,
+ int SIMDGROUPS_PER_COLUMN,
+ bool TRANSPOSE_B>
+struct SimdgroupMmaCore {
+ METAL_CONST ushort PADDING_A = 16 / sizeof(T);
+ METAL_CONST ushort PADDING_B = 16 / sizeof(T);
+ METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;
+ METAL_CONST ushort SHARED_STRIDE_B =
+ (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;
+ METAL_CONST ushort THREADGROUP_THREADS =
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
+
+ using ALoader = uzu::matmul::ThreadgroupLoader<
+ T,
+ THREADGROUP_BLOCK_M,
+ THREADGROUP_BLOCK_K,
+ SHARED_STRIDE_A,
+ true,
+ THREADGROUP_THREADS>;
+ using BLoader = uzu::matmul::ThreadgroupLoader<
+ T,
+ TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,
+ TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N,
+ SHARED_STRIDE_B,
+ TRANSPOSE_B,
+ THREADGROUP_THREADS>;
+ using TileAccumulator = uzu::matmul::ThreadgroupTile<
+ T,
+ T,
+ THREADGROUP_BLOCK_M,
+ THREADGROUP_BLOCK_N,
+ THREADGROUP_BLOCK_K,
+ SIMDGROUPS_PER_ROW,
+ SIMDGROUPS_PER_COLUMN,
+ false,
+ TRANSPOSE_B,
+ SHARED_STRIDE_A,
+ SHARED_STRIDE_B,
+ float,
+ uzu::matmul::TransformNone<T, float>>;
+
+ template <bool M_aligned, bool N_aligned, bool K_aligned>
+ static METAL_FUNC void k_loop(
+ threadgroup T* a_shared,
+ threadgroup T* b_shared,
+ const int aligned_k_iterations,
+ thread ALoader& loader_a,
+ thread BLoader& loader_b,
+ thread TileAccumulator& accumulator,
+ thread const ushort& tile_block_rows,
+ thread const ushort& tile_block_cols,
+ thread const ushort& leftover_block_depth
+ ) {
+ short2 tile_dimensions_a = short2(THREADGROUP_BLOCK_K, tile_block_rows);
+ short2 tile_dimensions_b =
+ TRANSPOSE_B ? short2(THREADGROUP_BLOCK_K, tile_block_cols)
+ : short2(tile_block_cols, THREADGROUP_BLOCK_K);
+
+ for (int k = 0; k < aligned_k_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if constexpr (M_aligned) {
+ loader_a.load_unsafe();
+ } else {
+ loader_a.load_safe(tile_dimensions_a);
+ }
+ if constexpr (N_aligned) {
+ loader_b.load_unsafe();
+ } else {
+ loader_b.load_safe(tile_dimensions_b);
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ accumulator.multiply_accumulate(a_shared, b_shared);
+
+ loader_a.next();
+ loader_b.next();
+ }
+
+ if constexpr (!K_aligned) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ short2 last_tile_dimensions_a =
+ short2(leftover_block_depth, tile_block_rows);
+ short2 last_tile_dimensions_b =
+ TRANSPOSE_B ? short2(leftover_block_depth, tile_block_cols)
+ : short2(tile_block_cols, leftover_block_depth);
+
+ loader_a.load_safe(last_tile_dimensions_a);
+ loader_b.load_safe(last_tile_dimensions_b);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ accumulator.multiply_accumulate(a_shared, b_shared);
+ }
+ }
+
+ static METAL_FUNC void run(
+ const device T* a,
+ const device T* b,
+ device T* d,
+ const constant uzu::matmul::GemmParams* params,
+ GemmAlignment alignment,
+ GemmOutputTransformKind output_transform,
+ threadgroup T* a_shared,
+ threadgroup T* b_shared,
+ const thread ThreadContext& thread_context
+ ) {
+ const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);
+ const auto geometry =
+ ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::
+ compute(tile, params);
+ if (geometry.out_of_bounds) {
+ return;
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ const size_t block_row = size_t(geometry.block_row_start);
+ const size_t block_col = size_t(geometry.block_col_start);
+
+ a += block_row * params->leading_dimension_a;
+ b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;
+ d += block_row * params->leading_dimension_d + block_col;
+
+ thread ALoader
+ loader_a(a, params->leading_dimension_a, a_shared, thread_context);
+ thread BLoader
+ loader_b(b, params->leading_dimension_b, b_shared, thread_context);
+ thread TileAccumulator accumulator(thread_context);
+
+ const ushort tile_block_rows =
+ min(THREADGROUP_BLOCK_M,
+ ((int)params->M) - int(geometry.block_row_start));
+ const ushort tile_block_cols =
+ min(THREADGROUP_BLOCK_N,
+ ((int)params->N) - int(geometry.block_col_start));
+ const ushort leftover_block_depth =
+ params->K - params->aligned_inner_iterations * THREADGROUP_BLOCK_K;
+
+ const bool needs_epilogue =
+ output_transform != GemmOutputTransformKind::Store;
+ const float alpha =
+ (output_transform == GemmOutputTransformKind::Scale ||
+ output_transform == GemmOutputTransformKind::ScaleAccumulate)
+ ? params->ab_scale
+ : 1.0f;
+ const float beta =
+ (output_transform == GemmOutputTransformKind::Accumulate ||
+ output_transform == GemmOutputTransformKind::ScaleAccumulate)
+ ? 1.0f
+ : 0.0f;
+ uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);
+
+ dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {
+ dispatch_bool(
+ alignment.contains(GemmAlignment::M) ||
+ (tile_block_rows == THREADGROUP_BLOCK_M),
+ [&](auto aligned_m) {
+ dispatch_bool(
+ alignment.contains(GemmAlignment::N) ||
+ (tile_block_cols == THREADGROUP_BLOCK_N),
+ [&](auto aligned_n) {
+ k_loop<aligned_m.value, aligned_n.value, aligned_k.value>(
+ a_shared,
+ b_shared,
+ params->aligned_inner_iterations,
+ loader_a,
+ loader_b,
+ accumulator,
+ tile_block_rows,
+ tile_block_cols,
+ leftover_block_depth
+ );
+ if constexpr (aligned_m.value && aligned_n.value) {
+ if (needs_epilogue) {
+ accumulator.apply_epilogue(
+ d,
+ params->leading_dimension_d,
+ 1,
+ epilogue
+ );
+ }
+ accumulator.store_result(d, params->leading_dimension_d);
+ } else {
+ if (needs_epilogue) {
+ accumulator.apply_epilogue_safe(
+ d,
+ params->leading_dimension_d,
+ 1,
+ short2(tile_block_cols, tile_block_rows),
+ epilogue
+ );
+ }
+ accumulator.store_result_safe(
+ d,
+ params->leading_dimension_d,
+ short2(tile_block_cols, tile_block_rows)
+ );
+ }
+ }
+ );
+ }
+ );
+ });
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs
new file mode 100644
index 000000000..57e38cbbb
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs
@@ -0,0 +1,247 @@
+use crate::{
+ DataType,
+ backends::common::{
+ Allocation, Backend,
+ gpu_types::{
+ GemmParams, QuantizationMode,
+ gemm::{
+ GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig, GemmWeightPrologueKind,
+ },
+ },
+ },
+};
+
+pub struct GemmAlignmentAxes {
+ pub m: bool,
+ pub n: bool,
+ pub k: bool,
+}
+
+impl GemmAlignment {
+ pub fn from_axes(axes: GemmAlignmentAxes) -> Self {
+ let mut bits = Self::empty();
+ bits.set(Self::M, axes.m);
+ bits.set(Self::N, axes.n);
+ bits.set(Self::K, axes.k);
+ bits
+ }
+}
+
+pub enum GemmWeights<'a, B: Backend> {
+ FullPrecision {
+ weights: &'a Allocation<B>,
+ },
+ ScaleBias {
+ weights: &'a Allocation<B>,
+ scales: &'a Allocation<B>,
+ biases: &'a Allocation<B>,
+ mode: QuantizationMode,
+ group_size: u32,
+ },
+ ScaleZeroPoint {
+ weights: &'a Allocation<B>,
+ scales: &'a Allocation<B>,
+ zero_points: &'a Allocation<B>,
+ mode: QuantizationMode,
+ group_size: u32,
+ },
+}
+
+impl<B: Backend> GemmWeights<'_, B> {
+ pub fn weight_prologue(&self) -> GemmWeightPrologueKind {
+ match self {
+ Self::FullPrecision {
+ ..
+ } => GemmWeightPrologueKind::FullPrecision,
+ Self::ScaleBias {
+ ..
+ } => GemmWeightPrologueKind::ScaleBiasDequant,
+ Self::ScaleZeroPoint {
+ ..
+ } => GemmWeightPrologueKind::ScaleZeroPointDequant,
+ }
+ }
+
+ pub fn bits_per_weight(&self) -> u32 {
+ match self {
+ Self::FullPrecision {
+ ..
+ } => 0,
+ Self::ScaleBias {
+ mode,
+ ..
+ }
+ | Self::ScaleZeroPoint {
+ mode,
+ ..
+ } => DataType::from(*mode).size_in_bits() as u32,
+ }
+ }
+
+ pub fn group_size(&self) -> u32 {
+ match self {
+ Self::FullPrecision {
+ ..
+ } => 0,
+ Self::ScaleBias {
+ group_size,
+ ..
+ }
+ | Self::ScaleZeroPoint {
+ group_size,
+ ..
+ } => *group_size,
+ }
+ }
+}
+
+pub struct GemmDispatch<'a, B: Backend> {
+ pub tiling_config: GemmTilingConfig,
+ pub input_prologue: GemmInputPrologueKind,
+ pub use_mxu: bool,
+ pub output_transform: GemmOutputTransformKind,
+ pub alignment: GemmAlignment,
+ pub transpose_b: bool,
+ pub a: &'a Allocation<B>,
+ pub a_offset: usize,
+ pub b: GemmWeights<'a, B>,
+ pub b_offset: usize,
+ pub d: &'a mut Allocation<B>,
+ pub params: GemmParams,
+ pub group_count_x: u32,
+ pub group_count_y: u32,
+}
+
+impl<B: Backend> GemmDispatch<'_, B> {
+ pub(crate) fn specialization(&self) -> GemmSpecialization {
+ GemmSpecialization {
+ tiling_config: self.tiling_config,
+ input_prologue: self.input_prologue,
+ use_mxu: self.use_mxu,
+ output_transform: self.output_transform,
+ alignment: self.alignment,
+ transpose_b: self.transpose_b,
+ weight_prologue: self.b.weight_prologue(),
+ bits_per_weight: self.b.bits_per_weight(),
+ group_size: self.b.group_size(),
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) struct GemmSpecialization {
+ pub(crate) tiling_config: GemmTilingConfig,
+ pub(crate) input_prologue: GemmInputPrologueKind,
+ pub(crate) use_mxu: bool,
+ pub(crate) output_transform: GemmOutputTransformKind,
+ pub(crate) alignment: GemmAlignment,
+ pub(crate) transpose_b: bool,
+ pub(crate) weight_prologue: GemmWeightPrologueKind,
+ pub(crate) bits_per_weight: u32,
+ pub(crate) group_size: u32,
+}
+
+impl GemmSpecialization {
+ pub(crate) fn validate(&self) -> Result<(), GemmSpecializationError> {
+ if self.group_size != 0 && self.tiling_config.threadgroup_k > self.group_size {
+ return Err(GemmSpecializationError::ThreadgroupKExceedsGroupSize {
+ threadgroup_k: self.tiling_config.threadgroup_k,
+ group_size: self.group_size,
+ });
+ }
+ Ok(())
+ }
+
+ pub(crate) fn precompile_configs(data_type: DataType) -> Vec<Self> {
+ let mut out = Vec::new();
+ for &(threadgroup_m, threadgroup_n, threadgroup_k) in simdgroup_tile_set(data_type) {
+ for align_mn in [true, false] {
+ let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {
+ m: align_mn,
+ n: align_mn,
+ k: true,
+ });
+ out.push(Self {
+ tiling_config: GemmTilingConfig {
+ threadgroup_m,
+ threadgroup_n,
+ threadgroup_k,
+ simdgroups_m: 2,
+ simdgroups_n: 2,
+ },
+ input_prologue: GemmInputPrologueKind::FullPrecision,
+ use_mxu: false,
+ output_transform: GemmOutputTransformKind::Store,
+ alignment,
+ transpose_b: true,
+ weight_prologue: GemmWeightPrologueKind::FullPrecision,
+ bits_per_weight: 0,
+ group_size: 0,
+ });
+ }
+ }
+
+ for &(threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) in mxu_tile_set(data_type) {
+ for align_m in [true, false] {
+ for align_n in [true, false] {
+ for align_k in [true, false] {
+ let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {
+ m: align_m,
+ n: align_n,
+ k: align_k,
+ });
+ for output_transform in [
+ GemmOutputTransformKind::Store,
+ GemmOutputTransformKind::Scale,
+ GemmOutputTransformKind::Accumulate,
+ GemmOutputTransformKind::ScaleAccumulate,
+ ] {
+ out.push(Self {
+ tiling_config: GemmTilingConfig {
+ threadgroup_m,
+ threadgroup_n,
+ threadgroup_k: 32,
+ simdgroups_m,
+ simdgroups_n,
+ },
+ input_prologue: GemmInputPrologueKind::FullPrecision,
+ use_mxu: true,
+ output_transform,
+ alignment,
+ transpose_b: true,
+ weight_prologue: GemmWeightPrologueKind::FullPrecision,
+ bits_per_weight: 0,
+ group_size: 0,
+ });
+ }
+ }
+ }
+ }
+ }
+ out
+ }
+}
+
+fn simdgroup_tile_set(data_type: DataType) -> &'static [(u32, u32, u32)] {
+ match data_type {
+ DataType::BF16 => &[(64, 32, 32), (64, 64, 16)],
+ DataType::F16 => &[(64, 64, 16), (64, 32, 32)],
+ DataType::F32 => &[(32, 64, 16)],
+ _ => &[],
+ }
+}
+
+fn mxu_tile_set(data_type: DataType) -> &'static [(u32, u32, u32, u32)] {
+ if !matches!(data_type, DataType::F16 | DataType::BF16) {
+ return &[];
+ }
+ &[(64, 64, 2, 2), (32, 64, 2, 2), (64, 32, 4, 1), (128, 128, 4, 4)]
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GemmSpecializationError {
+ ThreadgroupKExceedsGroupSize {
+ threadgroup_k: u32,
+ group_size: u32,
+ },
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs
new file mode 100644
index 000000000..4f39ce56b
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs
@@ -0,0 +1,176 @@
+use super::{GemmAlignmentAxes, GemmDispatch, GemmKernel, GemmWeights, MXU_THREADGROUP_BLOCK_K};
+use crate::{
+ DataType,
+ backends::{
+ common::{
+ Encoder,
+ gpu_types::{
+ GemmParams,
+ gemm::{GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig},
+ },
+ kernel::{
+ TensorAddBiasKernel,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError},
+ },
+ },
+ metal::{Metal, context::MetalContext, kernel::TensorAddBiasMetalKernel, metal_extensions::DeviceExt},
+ },
+};
+
+pub(crate) fn encode(
+ gemm: &mut GemmKernel,
+ bias_add: &mut TensorAddBiasMetalKernel,
+ data_type: DataType,
+ context: &MetalContext,
+ encoder: &mut Encoder<Metal>,
+ arguments: MatmulArguments<Metal>,
+ use_mxu: bool,
+) -> Result<(), MatmulError<Metal>> {
+ if use_mxu && !context.device.supports_mxu() {
+ return Err(MatmulError::UnsupportedDataType(data_type));
+ }
+
+ let tile = if use_mxu {
+ select_mxu_tile(&arguments)
+ } else {
+ select_simdgroup_tile(data_type, &arguments)
+ };
+ let k_block = if use_mxu {
+ MXU_THREADGROUP_BLOCK_K
+ } else {
+ tile.threadgroup_k
+ };
+
+ let threadgroups_per_row = arguments.output_dim.div_ceil(tile.threadgroup_n);
+ let threadgroups_per_column = arguments.batch_dim.div_ceil(tile.threadgroup_m);
+
+ let (use_morton, group_count_x, group_count_y) = if use_mxu {
+ let max_dim = threadgroups_per_row.max(threadgroups_per_column);
+ let min_dim = threadgroups_per_row.min(threadgroups_per_column);
+ let morton_dim = max_dim.next_power_of_two();
+ let morton_total = morton_dim.saturating_mul(morton_dim);
+ let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);
+ let use_morton = min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);
+ if use_morton {
+ (true, morton_total, 1)
+ } else {
+ (false, threadgroups_per_row, threadgroups_per_column)
+ }
+ } else {
+ (false, threadgroups_per_row, threadgroups_per_column)
+ };
+
+ let alignment = GemmAlignment::from_axes(GemmAlignmentAxes {
+ m: arguments.batch_dim % tile.threadgroup_m == 0,
+ n: arguments.output_dim % tile.threadgroup_n == 0,
+ k: arguments.input_dim % k_block == 0,
+ });
+ let output_transform = output_transform_from(&arguments);
+
+ let default_ldb = if arguments.b_transpose {
+ arguments.input_dim
+ } else {
+ arguments.output_dim
+ };
+ let params = GemmParams {
+ M: arguments.batch_dim,
+ N: arguments.output_dim,
+ K: arguments.input_dim,
+ leading_dimension_a: arguments.input_dim,
+ leading_dimension_b: arguments.b_leading_dimension.unwrap_or(default_ldb),
+ leading_dimension_d: arguments.output_dim,
+ threadgroups_per_row,
+ threadgroups_per_column,
+ aligned_inner_iterations: arguments.input_dim / k_block,
+ use_morton,
+ ab_scale: arguments.ab_scale,
+ };
+
+ let dispatch = GemmDispatch {
+ tiling_config: tile,
+ input_prologue: GemmInputPrologueKind::FullPrecision,
+ use_mxu,
+ output_transform,
+ alignment,
+ transpose_b: arguments.b_transpose,
+ a: arguments.a,
+ a_offset: arguments.a_offset,
+ b: GemmWeights::FullPrecision {
+ weights: arguments.b,
+ },
+ b_offset: arguments.b_offset,
+ d: arguments.d,
+ params,
+ group_count_x,
+ group_count_y,
+ };
+
+ gemm.encode(context, dispatch, encoder).map_err(MatmulError::BackendError)?;
+
+ if let MatmulArgumentC::Bias(bias) = arguments.c {
+ bias_add.encode(
+ None::<&<Metal as crate::backends::common::Backend>::DenseBuffer>,
+ bias,
+ arguments.d,
+ arguments.output_dim,
+ arguments.batch_dim * arguments.output_dim,
+ encoder,
+ );
+ }
+
+ Ok(())
+}
+
+fn select_simdgroup_tile(
+ data_type: DataType,
+ arguments: &MatmulArguments<Metal>,
+) -> GemmTilingConfig {
+ let (threadgroup_m, threadgroup_n, threadgroup_k) = match data_type {
+ DataType::F32 => (32u32, 64u32, 16u32),
+ _ => {
+ if 2 * arguments.batch_dim.max(arguments.output_dim) > arguments.input_dim {
+ (64, 64, 16)
+ } else {
+ (64, 32, 32)
+ }
+ },
+ };
+ GemmTilingConfig {
+ threadgroup_m,
+ threadgroup_n,
+ threadgroup_k,
+ simdgroups_m: 2,
+ simdgroups_n: 2,
+ }
+}
+
+fn select_mxu_tile(arguments: &MatmulArguments<Metal>) -> GemmTilingConfig {
+ let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) =
+ if arguments.batch_dim >= 256 && arguments.output_dim >= 128 {
+ (128u32, 128u32, 4u32, 4u32)
+ } else if arguments.output_dim < 64 {
+ (64u32, 32u32, 4u32, 1u32)
+ } else if arguments.batch_dim < 64 {
+ (32u32, 64u32, 2u32, 2u32)
+ } else {
+ (64u32, 64u32, 2u32, 2u32)
+ };
+ GemmTilingConfig {
+ threadgroup_m,
+ threadgroup_n,
+ threadgroup_k: 32,
+ simdgroups_m,
+ simdgroups_n,
+ }
+}
+
+fn output_transform_from(arguments: &MatmulArguments<Metal>) -> GemmOutputTransformKind {
+ let scale = arguments.ab_scale != 1.0;
+ let accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);
+ match (scale, accumulate) {
+ (false, false) => GemmOutputTransformKind::Store,
+ (true, false) => GemmOutputTransformKind::Scale,
+ (false, true) => GemmOutputTransformKind::Accumulate,
+ (true, true) => GemmOutputTransformKind::ScaleAccumulate,
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
new file mode 100644
index 000000000..00ad9e2d7
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -0,0 +1,106 @@
+#include "../../common/dsl.h"
+#include "../../common/defines.h"
+#include "../../common/thread_context.h"
+#include "../generated/gemm.h"
+
+#include "common/mxu_mma_core.h"
+#include "common/simdgroup_mma_core.h"
+
+using namespace metal;
+using namespace uzu::gemm;
+
+#define GEMM_MAX_THREADGROUP_A 2560
+#define GEMM_MAX_THREADGROUP_B 1536
+
+template <
+ typename T,
+ uint THREADGROUP_BLOCK_M,
+ uint THREADGROUP_BLOCK_N,
+ uint THREADGROUP_BLOCK_K,
+ uint SIMDGROUPS_M,
+ uint SIMDGROUPS_N,
+ bool TRANSPOSE_B,
+ bool USE_MXU>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(THREADGROUP_BLOCK_M, 32, 64, 128)
+VARIANTS(THREADGROUP_BLOCK_N, 32, 64, 128)
+VARIANTS(THREADGROUP_BLOCK_K, 16, 32)
+VARIANTS(SIMDGROUPS_M, 1, 2, 4)
+VARIANTS(SIMDGROUPS_N, 1, 2, 4)
+VARIANTS(TRANSPOSE_B, false, true)
+VARIANTS(USE_MXU, false, true)
+CONSTRAINT(!USE_MXU || THREADGROUP_BLOCK_M % SIMDGROUPS_M == 0)
+CONSTRAINT(!USE_MXU || THREADGROUP_BLOCK_N % SIMDGROUPS_N == 0)
+CONSTRAINT(!USE_MXU || (THREADGROUP_BLOCK_M / SIMDGROUPS_M) % 16 == 0)
+CONSTRAINT(!USE_MXU || (THREADGROUP_BLOCK_N / SIMDGROUPS_N) % 16 == 0)
+CONSTRAINT(USE_MXU || max(THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N) <= 32 * SIMDGROUPS_M * SIMDGROUPS_N)
+KERNEL(Gemm)(
+ const device T* a,
+ const device uint8_t* b_packed,
+ device T* d,
+ const device T* scales
+ OPTIONAL(weight_prologue != GemmWeightPrologueKind::FullPrecision),
+ const device T* biases
+ OPTIONAL(weight_prologue == GemmWeightPrologueKind::ScaleBiasDequant),
+ const device uint8_t* zero_points
+ OPTIONAL(weight_prologue == GemmWeightPrologueKind::ScaleZeroPointDequant),
+ const constant uzu::matmul::GemmParams* params,
+ const constant uint& group_count_x,
+ const constant uint& group_count_y,
+ const GemmInputPrologueKind input_prologue SPECIALIZE,
+ const GemmWeightPrologueKind weight_prologue SPECIALIZE,
+ const GemmOutputTransformKind output_transform SPECIALIZE,
+ const GemmAlignment alignment SPECIALIZE,
+ const uint bits_per_weight SPECIALIZE,
+ const uint group_size SPECIALIZE,
+ threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],
+ threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],
+ const uint group_x GROUPS(group_count_x),
+ const uint group_y GROUPS(group_count_y),
+ const uint thread_x THREADS(METAL_SIMD_SIZE),
+ const uint thread_y THREADS(SIMDGROUPS_N),
+ const uint thread_z THREADS(SIMDGROUPS_M),
+ const ThreadContext thread_context
+) {
+ (void)scales;
+ (void)biases;
+ (void)zero_points;
+ (void)bits_per_weight;
+ (void)group_size;
+ (void)group_x;
+ (void)group_y;
+ (void)thread_x;
+ (void)thread_y;
+ (void)thread_z;
+ (void)input_prologue;
+ (void)weight_prologue;
+ const device T* b = reinterpret_cast<const device T*>(b_packed);
+ if constexpr (USE_MXU) {
+ MxuMmaCore<
+ T,
+ THREADGROUP_BLOCK_M,
+ THREADGROUP_BLOCK_N,
+ SIMDGROUPS_M,
+ SIMDGROUPS_N,
+ TRANSPOSE_B>::
+ run(a, b, d, params, alignment, output_transform, thread_context);
+ } else {
+ SimdgroupMmaCore<
+ T,
+ THREADGROUP_BLOCK_M,
+ THREADGROUP_BLOCK_N,
+ THREADGROUP_BLOCK_K,
+ SIMDGROUPS_M,
+ SIMDGROUPS_N,
+ TRANSPOSE_B>::
+ run(a,
+ b,
+ d,
+ params,
+ alignment,
+ output_transform,
+ a_shared,
+ b_shared,
+ thread_context);
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
new file mode 100644
index 000000000..44a925987
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -0,0 +1,115 @@
+use std::collections::{HashMap, hash_map::Entry};
+
+use super::dispatch::{GemmDispatch, GemmSpecialization, GemmWeights};
+use crate::{
+ DataType,
+ backends::{
+ common::{
+ Encoder,
+ gpu_types::gemm::{GemmInputPrologueKind, GemmWeightPrologueKind},
+ },
+ metal::{Metal, context::MetalContext, error::MetalError, kernel::GemmMetalKernel},
+ },
+};
+
+pub(crate) struct GemmKernel {
+ data_type: DataType,
+ kernels: HashMap<GemmSpecialization, GemmMetalKernel>,
+}
+
+impl GemmKernel {
+ pub(crate) fn new(
+ context: &MetalContext,
+ data_type: DataType,
+ ) -> Result<Self, MetalError> {
+ let mut kernel = Self {
+ data_type,
+ kernels: HashMap::new(),
+ };
+ for specialization in GemmSpecialization::precompile_configs(data_type) {
+ kernel.get_or_create(context, specialization)?;
+ }
+ Ok(kernel)
+ }
+
+ fn get_or_create(
+ &mut self,
+ context: &MetalContext,
+ specialization: GemmSpecialization,
+ ) -> Result<&GemmMetalKernel, MetalError> {
+ match self.kernels.entry(specialization) {
+ Entry::Occupied(entry) => Ok(entry.into_mut()),
+ Entry::Vacant(entry) => {
+ let kernel = GemmMetalKernel::new(
+ context,
+ self.data_type,
+ specialization.tiling_config.threadgroup_m,
+ specialization.tiling_config.threadgroup_n,
+ specialization.tiling_config.threadgroup_k,
+ specialization.tiling_config.simdgroups_m,
+ specialization.tiling_config.simdgroups_n,
+ specialization.transpose_b,
+ specialization.use_mxu,
+ specialization.input_prologue,
+ specialization.weight_prologue,
+ specialization.output_transform,
+ specialization.alignment,
+ specialization.bits_per_weight,
+ specialization.group_size,
+ )?;
+ Ok(entry.insert(kernel))
+ },
+ }
+ }
+
+ pub(crate) fn encode(
+ &mut self,
+ context: &MetalContext,
+ dispatch: GemmDispatch<'_, Metal>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), MetalError> {
+ let specialization = dispatch.specialization();
+ assert_eq!(
+ specialization.input_prologue,
+ GemmInputPrologueKind::FullPrecision,
+ "unified GEMM only implements FullPrecision input prologue today",
+ );
+ assert_eq!(
+ specialization.weight_prologue,
+ GemmWeightPrologueKind::FullPrecision,
+ "unified GEMM only implements FullPrecision weight prologue today",
+ );
+ specialization.validate().map_err(MetalError::InvalidGemmSpecialization)?;
+ let kernel = self.get_or_create(context, specialization)?;
+ let (b, scales, biases, zero_points) = match &dispatch.b {
+ GemmWeights::FullPrecision {
+ weights,
+ } => (*weights, None, None, None),
+ GemmWeights::ScaleBias {
+ weights,
+ scales,
+ biases,
+ ..
+ } => (*weights, Some(*scales), Some(*biases), None),
+ GemmWeights::ScaleZeroPoint {
+ weights,
+ scales,
+ zero_points,
+ ..
+ } => (*weights, Some(*scales), None, Some(*zero_points)),
+ };
+ kernel.encode(
+ (dispatch.a, dispatch.a_offset),
+ (b, dispatch.b_offset),
+ dispatch.d,
+ scales,
+ biases,
+ zero_points,
+ std::slice::from_ref(&dispatch.params),
+ dispatch.group_count_x,
+ dispatch.group_count_y,
+ encoder,
+ );
+ Ok(())
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
new file mode 100644
index 000000000..3dc90617f
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -0,0 +1,13 @@
+mod dispatch;
+pub(crate) mod fp;
+mod kernel;
+pub(crate) mod quant;
+
+pub use dispatch::GemmSpecializationError;
+pub(crate) use dispatch::{GemmAlignmentAxes, GemmDispatch, GemmWeights};
+pub(crate) use kernel::GemmKernel;
+
+pub(crate) use crate::backends::common::gpu_types::gemm::GemmInputPrologueKind;
+
+// Matches `MxuMmaCore::THREADGROUP_BLOCK_K` in `common/mxu_mma_core.h`.
+pub(crate) const MXU_THREADGROUP_BLOCK_K: u32 = 256;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs
new file mode 100644
index 000000000..54617b5c1
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/quant.rs
@@ -0,0 +1,96 @@
+use super::{GemmAlignmentAxes, GemmDispatch, GemmInputPrologueKind, GemmKernel, GemmWeights};
+use crate::backends::{
+ common::{
+ Encoder,
+ gpu_types::{
+ GemmParams, QuantizationMethod,
+ gemm::{GemmAlignment, GemmOutputTransformKind, GemmTilingConfig},
+ },
+ kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError},
+ },
+ metal::{Metal, context::MetalContext},
+};
+
+pub(crate) fn encode(
+ gemm: &mut GemmKernel,
+ context: &MetalContext,
+ configuration: &QuantizedMatmulConfiguration,
+ arguments: QuantizedMatmulArguments<Metal>,
+ encoder: &mut Encoder<Metal>,
+) -> Result<(), QuantizedMatmulError<Metal>> {
+ let tile = select_tile(configuration, arguments.batch_dim as u32);
+ let group_size = configuration.group_size as u32;
+ let batch_dim = arguments.batch_dim as u32;
+ let input_dim = configuration.input_dim as u32;
+ let output_dim = configuration.output_dim as u32;
+ let threadgroups_per_row = output_dim.div_ceil(tile.threadgroup_n);
+ let threadgroups_per_column = batch_dim.div_ceil(tile.threadgroup_m);
+ let dispatch = GemmDispatch {
+ tiling_config: tile,
+ input_prologue: GemmInputPrologueKind::FullPrecision,
+ use_mxu: false,
+ output_transform: GemmOutputTransformKind::Store,
+ alignment: GemmAlignment::from_axes(GemmAlignmentAxes {
+ m: batch_dim % tile.threadgroup_m == 0,
+ n: output_dim % tile.threadgroup_n == 0,
+ k: input_dim % tile.threadgroup_k == 0,
+ }),
+ transpose_b: true,
+ a: arguments.a,
+ a_offset: arguments.a_offset,
+ b: match configuration.quantization_method {
+ QuantizationMethod::ScaleBias => GemmWeights::ScaleBias {
+ weights: arguments.b,
+ scales: arguments.scales,
+ biases: arguments.zero_points_or_biases,
+ mode: configuration.mode,
+ group_size,
+ },
+ QuantizationMethod::ScaleZeroPoint => GemmWeights::ScaleZeroPoint {
+ weights: arguments.b,
+ scales: arguments.scales,
+ zero_points: arguments.zero_points_or_biases,
+ mode: configuration.mode,
+ group_size,
+ },
+ },
+ b_offset: 0,
+ d: arguments.output,
+ params: GemmParams {
+ M: batch_dim,
+ N: output_dim,
+ K: input_dim,
+ leading_dimension_a: input_dim,
+ leading_dimension_b: input_dim,
+ leading_dimension_d: output_dim,
+ threadgroups_per_row,
+ threadgroups_per_column,
+ aligned_inner_iterations: input_dim / tile.threadgroup_k,
+ use_morton: false,
+ ab_scale: 1.0,
+ },
+ group_count_x: threadgroups_per_row,
+ group_count_y: threadgroups_per_column,
+ };
+ gemm.encode(context, dispatch, encoder).map_err(QuantizedMatmulError::BackendError)
+}
+
+fn select_tile(
+ configuration: &QuantizedMatmulConfiguration,
+ batch_dim: u32,
+) -> GemmTilingConfig {
+ let group_size = configuration.group_size as u32;
+ let threadgroup_k = group_size.min(32);
+ let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) = if batch_dim < 32 {
+ (32u32, 32u32, 1u32, 1u32)
+ } else {
+ (64u32, 64u32, 2u32, 2u32)
+ };
+ GemmTilingConfig {
+ threadgroup_m,
+ threadgroup_n,
+ threadgroup_k,
+ simdgroups_m,
+ simdgroups_n,
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
deleted file mode 100644
index 6fc7399d7..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
+++ /dev/null
@@ -1,62 +0,0 @@
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-
-#include "common/gemm_mpp_core.h"
-
-using namespace uzu::matmul;
-
-template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN>
-VARIANTS(T, float, half, bfloat)
-VARIANTS(BLOCK_ROWS, 32, 64, 128)
-VARIANTS(BLOCK_COLS, 32, 64, 128)
-VARIANTS(SIMDGROUPS_PER_ROW, 2, 4)
-VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2, 4)
-CONSTRAINT(
- (BLOCK_ROWS == 32 && BLOCK_COLS == 64 && SIMDGROUPS_PER_ROW == 2 && SIMDGROUPS_PER_COLUMN == 2) ||
- (BLOCK_ROWS == 64 && BLOCK_COLS == 32 && SIMDGROUPS_PER_ROW == 4 && SIMDGROUPS_PER_COLUMN == 1) ||
- (BLOCK_ROWS == 64 && BLOCK_COLS == 64 && SIMDGROUPS_PER_ROW == 2 && SIMDGROUPS_PER_COLUMN == 2) ||
- (BLOCK_ROWS == 128 && BLOCK_COLS == 128 && SIMDGROUPS_PER_ROW == 4 && SIMDGROUPS_PER_COLUMN == 4))
-KERNEL(MatmulGemmMpp)(
- const device T* left_matrix,
- const device T* right_matrix,
- device T* output_matrix,
- const constant uzu::matmul::GemmParams* params,
- const constant uint& group_count_x,
- const constant uint& group_count_y,
- const constant float& ab_scale,
- const bool align_m SPECIALIZE,
- const bool align_n SPECIALIZE,
- const bool align_k SPECIALIZE,
- const bool apply_ab_scale SPECIALIZE,
- const bool is_accumulate SPECIALIZE,
- const uint group_x GROUPS(group_count_x),
- const uint group_y GROUPS(group_count_y),
- const uint thread_x THREADS(32),
- const uint thread_y THREADS(SIMDGROUPS_PER_ROW),
- const uint thread_z THREADS(SIMDGROUPS_PER_COLUMN),
- const ThreadContext thread_context
-) {
- dispatch_bool(apply_ab_scale, [&](auto scale_enabled) {
- dispatch_bool(is_accumulate, [&](auto accumulate_enabled) {
- GemmMppCore<
- T,
- BLOCK_ROWS,
- BLOCK_COLS,
- SIMDGROUPS_PER_ROW,
- SIMDGROUPS_PER_COLUMN,
- scale_enabled.value,
- accumulate_enabled.value>::
- run(left_matrix,
- right_matrix,
- output_matrix,
- params,
- align_m,
- align_n,
- align_k,
- ab_scale,
- thread_context.threadgroup_index,
- uint2(group_x, group_y),
- thread_context);
- });
- });
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs
deleted file mode 100644
index a957795d0..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs
+++ /dev/null
@@ -1,130 +0,0 @@
-use std::sync::OnceLock;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct GemmMppSpecialization {
- pub block_rows: u32,
- pub block_cols: u32,
- pub simdgroups_per_row: u32,
- pub simdgroups_per_column: u32,
- pub align_m: bool,
- pub align_n: bool,
- pub align_k: bool,
- pub apply_ab_scale: bool,
- pub is_accumulate: bool,
-}
-
-const fn config(
- block_rows: u32,
- block_cols: u32,
- simdgroups_per_row: u32,
- simdgroups_per_column: u32,
- align_m: bool,
- align_n: bool,
- is_accumulate: bool,
-) -> GemmMppSpecialization {
- GemmMppSpecialization {
- block_rows,
- block_cols,
- simdgroups_per_row,
- simdgroups_per_column,
- align_m,
- align_n,
- align_k: false,
- apply_ab_scale: false,
- is_accumulate,
- }
-}
-
-static GEMM_MPP_PRECOMPILE: OnceLock<&'static [GemmMppSpecialization]> = OnceLock::new();
-
-impl GemmMppSpecialization {
- pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
- if !matches!(data_type, crate::DataType::F16 | crate::DataType::BF16) {
- return &[];
- }
- GEMM_MPP_PRECOMPILE.get_or_init(|| {
- const BASE: &[GemmMppSpecialization] = &[
- config(64, 64, 2, 2, true, true, false),
- config(64, 64, 2, 2, false, true, false),
- config(64, 64, 2, 2, true, false, false),
- config(64, 64, 2, 2, false, false, false),
- config(32, 64, 2, 2, true, true, false),
- config(32, 64, 2, 2, false, true, false),
- config(64, 32, 4, 1, true, true, false),
- config(64, 32, 4, 1, true, false, false),
- config(128, 128, 4, 4, true, true, false),
- config(128, 128, 4, 4, false, true, false),
- config(128, 128, 4, 4, true, false, false),
- config(128, 128, 4, 4, false, false, false),
- config(64, 64, 2, 2, true, true, true),
- config(64, 64, 2, 2, false, true, true),
- config(64, 64, 2, 2, true, false, true),
- config(64, 64, 2, 2, false, false, true),
- config(32, 64, 2, 2, true, true, true),
- config(32, 64, 2, 2, false, true, true),
- config(64, 32, 4, 1, true, true, true),
- config(64, 32, 4, 1, true, false, true),
- config(128, 128, 4, 4, true, true, true),
- config(128, 128, 4, 4, false, true, true),
- config(128, 128, 4, 4, true, false, true),
- config(128, 128, 4, 4, false, false, true),
- ];
-
- let mut expanded = Vec::with_capacity(BASE.len() * 4);
- for &base in BASE {
- expanded.push(Self {
- align_k: true,
- apply_ab_scale: true,
- ..base
- });
- expanded.push(Self {
- align_k: true,
- apply_ab_scale: false,
- ..base
- });
- expanded.push(Self {
- align_k: false,
- apply_ab_scale: true,
- ..base
- });
- expanded.push(Self {
- align_k: false,
- apply_ab_scale: false,
- ..base
- });
- }
-
- Box::leak(expanded.into_boxed_slice())
- })
- }
-
- pub fn select(
- m: u32,
- n: u32,
- k: u32,
- is_accumulate: bool,
- apply_ab_scale: bool,
- ) -> Self {
- let (block_rows, block_cols, simdgroups_per_row, simdgroups_per_column) = if m >= 128 && n >= 128 {
- (128, 128, 4u32, 4u32)
- } else if n < 64 {
- (64, 32, 4u32, 1u32)
- } else if m < 64 {
- (32, 64, 2u32, 2u32)
- } else {
- (64, 64, 2u32, 2u32)
- };
-
- Self {
- block_rows,
- block_cols,
- simdgroups_per_row,
- simdgroups_per_column,
- align_m: (m % block_rows) == 0,
- align_n: (n % block_cols) == 0,
- align_k: (k % 256) == 0,
- apply_ab_scale,
- is_accumulate,
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
new file mode 100644
index 000000000..90441c3db
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
@@ -0,0 +1,59 @@
+use super::{kernel::GemvKernel, spec::GemvSpecialization};
+use crate::backends::{
+ common::{
+ Encoder,
+ kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError},
+ },
+ metal::Metal,
+};
+
+pub(crate) fn encode(
+ kernel: &mut GemvKernel,
+ encoder: &mut Encoder<Metal>,
+ arguments: MatmulArguments<Metal>,
+) -> Result<(), MatmulError<Metal>> {
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ ab_scale,
+ c,
+ d,
+ batch_dim,
+ input_dim,
+ output_dim,
+ } = arguments;
+ assert!(b_transpose, "encode_gemv does not support b_transpose=false");
+ assert!(b_offset == 0, "encode_gemv does not support nonzero b_offset");
+ assert!(
+ b_leading_dimension.is_none_or(|ld| ld == input_dim),
+ "encode_gemv does not support custom b_leading_dimension"
+ );
+
+ let (is_accumulate, output_bias) = match c {
+ MatmulArgumentC::Accumulate => (true, None),
+ MatmulArgumentC::Bias(bias) => (false, Some(bias)),
+ MatmulArgumentC::None => (false, None),
+ };
+
+ let specialization = GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());
+
+ kernel.get_or_create(encoder.context(), specialization)?.encode(
+ b,
+ (a, a_offset),
+ output_bias,
+ d,
+ input_dim,
+ output_dim,
+ input_dim,
+ ab_scale,
+ batch_dim,
+ specialization.output_rows_per_threadgroup(),
+ encoder,
+ );
+
+ Ok(())
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
similarity index 94%
rename from crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.metal
rename to crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
index b08eb3e73..c601f6fff 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -1,8 +1,8 @@
#include <metal_stdlib>
-#include "../common/defines.h"
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../common/thread_context.h"
#include <metal_simdgroup>
@@ -47,7 +47,7 @@ KERNEL(MatmulGemv)(
}
// Simdgroup guard — kill excess simdgroups beyond what this tile uses
- if (thread_context.threadgroup_index >= tg_simd_rows * tg_simd_cols) {
+ if (thread_context.simdgroup_index >= tg_simd_rows * tg_simd_cols) {
return;
}
@@ -68,26 +68,26 @@ KERNEL(MatmulGemv)(
const uint thread_row_in_simdgroup =
sg_thread_cols != 32
- ? uint(thread_context.simdgroup_index) / uint(sg_thread_cols)
+ ? uint(thread_context.simd_lane_id) / uint(sg_thread_cols)
: 0;
const int thread_col_in_simdgroup =
sg_thread_cols != 32
- ? uint(thread_context.simdgroup_index) % uint(sg_thread_cols)
- : uint(thread_context.simdgroup_index);
+ ? uint(thread_context.simd_lane_id) % uint(sg_thread_cols)
+ : uint(thread_context.simd_lane_id);
const int simdgroup_column_index =
- tg_simd_cols != 1 ? uint(thread_context.threadgroup_index % tg_simd_cols)
+ tg_simd_cols != 1 ? uint(thread_context.simdgroup_index % tg_simd_cols)
: 0;
const uint simdgroup_row_thread_base =
tg_simd_cols != 1
? uint(sg_thread_rows) *
- uint(thread_context.threadgroup_index / tg_simd_cols)
- : uint(sg_thread_rows) * uint(thread_context.threadgroup_index);
+ uint(thread_context.simdgroup_index / tg_simd_cols)
+ : uint(sg_thread_rows) * uint(thread_context.simdgroup_index);
const uint simdgroup_col_thread_base =
tg_simd_cols != 1
? uint(sg_thread_cols) *
- uint(thread_context.threadgroup_index % tg_simd_cols)
+ uint(thread_context.simdgroup_index % tg_simd_cols)
: 0;
uint output_block_row_offset =
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
new file mode 100644
index 000000000..03d67e584
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -0,0 +1,57 @@
+use std::collections::{HashMap, hash_map::Entry};
+
+use super::spec::GemvSpecialization;
+use crate::{
+ DataType,
+ backends::{
+ common::kernel::matmul::MatmulError,
+ metal::{Metal, context::MetalContext, kernel::MatmulGemvMetalKernel},
+ },
+};
+
+pub(crate) struct GemvKernel {
+ data_type: DataType,
+ pipelines: HashMap<GemvSpecialization, MatmulGemvMetalKernel>,
+}
+
+impl GemvKernel {
+ pub(crate) fn new(
+ context: &MetalContext,
+ data_type: DataType,
+ ) -> Result<Self, MatmulError<Metal>> {
+ let mut kernel = Self {
+ data_type,
+ pipelines: HashMap::new(),
+ };
+ for &config in GemvSpecialization::precompile_configs(data_type) {
+ kernel.get_or_create(context, config)?;
+ }
+ Ok(kernel)
+ }
+
+ pub(crate) fn get_or_create(
+ &mut self,
+ context: &MetalContext,
+ specialization: GemvSpecialization,
+ ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {
+ match self.pipelines.entry(specialization) {
+ Entry::Occupied(entry) => Ok(entry.into_mut()),
+ Entry::Vacant(entry) => {
+ let kernel = MatmulGemvMetalKernel::new(
+ context,
+ self.data_type,
+ specialization.threadgroup_rows,
+ specialization.threadgroup_cols,
+ specialization.threads_per_simdgroup_row,
+ specialization.threads_per_simdgroup_col,
+ specialization.elements_per_thread_row,
+ specialization.elements_per_thread_col,
+ specialization.is_accumulate,
+ specialization.is_bias,
+ )
+ .map_err(MatmulError::BackendError)?;
+ Ok(entry.insert(kernel))
+ },
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
new file mode 100644
index 000000000..48df5aef2
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -0,0 +1,5 @@
+pub(crate) mod fp;
+mod kernel;
+mod spec;
+
+pub(crate) use kernel::GemvKernel;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/metal/kernel/matmul/gemv.rs
rename to crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 1939c7c37..464bbd0d8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,44 +1,32 @@
-use std::{
- collections::{HashMap, hash_map::Entry},
- sync::OnceLock,
-};
+pub mod gemm;
+pub mod gemv;
+pub mod quant;
+
+use std::sync::OnceLock;
+use self::{gemm::GemmKernel, gemv::GemvKernel};
use crate::{
DataType,
backends::{
common::{
- Allocation, Encoder,
- gpu_types::GemmParams,
+ Encoder,
kernel::{
TensorAddBiasKernel,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
- },
- },
- metal::{
- Metal,
- context::MetalContext,
- kernel::{
- MatmulGemmMetalKernel, MatmulGemmMppMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel,
+ matmul::{MatmulArguments, MatmulError, MatmulKernel},
},
- metal_extensions::DeviceExt,
},
+ metal::{Metal, context::MetalContext, kernel::TensorAddBiasMetalKernel, metal_extensions::DeviceExt},
},
};
-mod gemm;
-mod gemm_mpp;
-mod gemv;
-
pub struct MatmulMetalKernel {
data_type: DataType,
- gemv_kernels: HashMap<gemv::GemvSpecialization, MatmulGemvMetalKernel>,
- gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,
- gemm_mpp_kernels: HashMap<gemm_mpp::GemmMppSpecialization, MatmulGemmMppMetalKernel>,
- bias_add: TensorAddBiasMetalKernel,
+ gemv: GemvKernel,
+ pub(crate) gemm: GemmKernel,
+ pub(crate) bias_add: TensorAddBiasMetalKernel,
}
const DEFAULT_GEMV_MAX_BATCH: u32 = 8;
-
static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();
fn max_gemv_batch_threshold() -> u32 {
@@ -48,286 +36,12 @@ fn max_gemv_batch_threshold() -> u32 {
}
impl MatmulMetalKernel {
- fn is_mpp_eligible(
+ fn is_mxu_eligible(
&self,
context: &MetalContext,
) -> bool {
context.device.supports_mxu() && matches!(self.data_type, DataType::F16 | DataType::BF16)
}
-
- fn get_or_create_gemv(
- &mut self,
- context: &MetalContext,
- specialization: gemv::GemvSpecialization,
- ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {
- match self.gemv_kernels.entry(specialization) {
- Entry::Occupied(entry) => Ok(entry.into_mut()),
- Entry::Vacant(entry) => {
- let kernel = MatmulGemvMetalKernel::new(
- context,
- self.data_type,
- specialization.threadgroup_rows,
- specialization.threadgroup_cols,
- specialization.threads_per_simdgroup_row,
- specialization.threads_per_simdgroup_col,
- specialization.elements_per_thread_row,
- specialization.elements_per_thread_col,
- specialization.is_accumulate,
- specialization.is_bias,
- )
- .map_err(MatmulError::BackendError)?;
- Ok(entry.insert(kernel))
- },
- }
- }
-
- fn get_or_create_gemm(
- &mut self,
- context: &MetalContext,
- specialization: gemm::GemmSpecialization,
- ) -> Result<&MatmulGemmMetalKernel, MatmulError<Metal>> {
- match self.gemm_kernels.entry(specialization) {
- Entry::Occupied(entry) => Ok(entry.into_mut()),
- Entry::Vacant(entry) => {
- let kernel = MatmulGemmMetalKernel::new(
- context,
- self.data_type,
- specialization.block_rows as u32,
- specialization.block_cols as u32,
- specialization.block_depth as u32,
- specialization.simdgroups_per_row as u32,
- specialization.simdgroups_per_column as u32,
- specialization.align_mn,
- specialization.align_k,
- specialization.transpose_b,
- specialization.is_accumulate,
- )
- .map_err(MatmulError::BackendError)?;
- Ok(entry.insert(kernel))
- },
- }
- }
-
- fn get_or_create_gemm_mpp(
- &mut self,
- context: &MetalContext,
- specialization: gemm_mpp::GemmMppSpecialization,
- ) -> Result<&MatmulGemmMppMetalKernel, MatmulError<Metal>> {
- match self.gemm_mpp_kernels.entry(specialization) {
- Entry::Occupied(entry) => Ok(entry.into_mut()),
- Entry::Vacant(entry) => {
- let kernel = MatmulGemmMppMetalKernel::new(
- context,
- self.data_type,
- specialization.block_rows,
- specialization.block_cols,
- specialization.simdgroups_per_row,
- specialization.simdgroups_per_column,
- specialization.align_m,
- specialization.align_n,
- specialization.align_k,
- specialization.apply_ab_scale,
- specialization.is_accumulate,
- )
- .map_err(MatmulError::BackendError)?;
- Ok(entry.insert(kernel))
- },
- }
- }
-
- fn encode_gemv(
- &mut self,
- encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal>,
- ) -> Result<(), MatmulError<Metal>> {
- let MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- ab_scale,
- c,
- d,
- batch_dim,
- input_dim,
- output_dim,
- } = arguments;
- assert!(b_transpose, "encode_gemv does not yet support b_transpose=false");
-
- let (is_accumulate, output_bias) = match c {
- MatmulArgumentC::Accumulate => (true, None),
- MatmulArgumentC::Bias(bias) => (false, Some(bias)),
- MatmulArgumentC::None => (false, None),
- };
-
- let specialization =
- gemv::GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());
-
- self.get_or_create_gemv(encoder.context(), specialization)?.encode(
- (b, b_offset),
- (a, a_offset),
- output_bias,
- d,
- input_dim,
- output_dim,
- b_leading_dimension.unwrap_or(input_dim),
- ab_scale,
- batch_dim,
- specialization.output_rows_per_threadgroup(),
- encoder,
- );
-
- Ok(())
- }
-
- fn encode_gemm(
- &mut self,
- encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal>,
- ) -> Result<(), MatmulError<Metal>> {
- let MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- ab_scale,
- c,
- d,
- batch_dim,
- input_dim,
- output_dim,
- } = arguments;
-
- let specialization = gemm::GemmSpecialization::select(
- encoder.context(),
- self.data_type,
- batch_dim,
- input_dim,
- output_dim,
- b_transpose,
- &c,
- );
-
- let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);
- let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);
-
- let default_ldb = if b_transpose {
- input_dim
- } else {
- output_dim
- };
- let params = GemmParams {
- M: batch_dim,
- N: output_dim,
- K: input_dim,
- leading_dimension_a: input_dim,
- leading_dimension_b: b_leading_dimension.unwrap_or(default_ldb),
- leading_dimension_d: output_dim,
- threadgroups_per_row,
- threadgroups_per_column,
- swizzle_log: 0,
- aligned_inner_iterations: input_dim / specialization.block_depth,
- use_morton: false,
- };
-
- let kernel = self.get_or_create_gemm(encoder.context(), specialization)?;
-
- kernel.encode(
- (a, a_offset),
- (b, b_offset),
- &mut *d,
- std::slice::from_ref(&params),
- threadgroups_per_row,
- threadgroups_per_column,
- ab_scale,
- encoder,
- );
-
- if let MatmulArgumentC::Bias(bias) = c {
- self.bias_add.encode(None::<&Allocation<Metal>>, bias, d, output_dim, batch_dim * output_dim, encoder);
- }
-
- Ok(())
- }
-
- fn encode_gemm_mpp(
- &mut self,
- encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal>,
- ) -> Result<(), MatmulError<Metal>> {
- let MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- ab_scale,
- c,
- d,
- batch_dim,
- input_dim,
- output_dim,
- } = arguments;
- assert!(b_transpose, "encode_gemm_mpp does not yet support b_transpose=false");
- let is_accumulate = matches!(c, MatmulArgumentC::Accumulate);
- let specialization =
- gemm_mpp::GemmMppSpecialization::select(batch_dim, output_dim, input_dim, is_accumulate, ab_scale != 1.0);
-
- let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);
- let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);
-
- let max_threadgroups_per_dim = threadgroups_per_row.max(threadgroups_per_column);
- let min_threadgroups_per_dim = threadgroups_per_row.min(threadgroups_per_column);
- let morton_dim = max_threadgroups_per_dim.next_power_of_two();
- let morton_total = morton_dim.saturating_mul(morton_dim);
- let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);
- // Morton ordering improves cache locality but requires padding to a square power-of-two grid.
- // Allow up to 4x overhead (idle threadgroups) before the padding cost outweighs the benefit.
- let use_morton = min_threadgroups_per_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);
-
- let params = GemmParams {
- M: batch_dim,
- N: output_dim,
- K: input_dim,
- leading_dimension_a: input_dim,
- leading_dimension_b: b_leading_dimension.unwrap_or(input_dim),
- leading_dimension_d: output_dim,
- threadgroups_per_row,
- threadgroups_per_column,
- swizzle_log: 0,
- aligned_inner_iterations: 0,
- use_morton,
- };
-
- let (group_count_x, group_count_y) = if use_morton {
- (morton_total, 1_u32)
- } else {
- (threadgroups_per_row, threadgroups_per_column)
- };
-
- let kernel = self.get_or_create_gemm_mpp(encoder.context(), specialization)?;
- kernel.encode(
- (a, a_offset),
- (b, b_offset),
- &mut *d,
- std::slice::from_ref(&params),
- group_count_x,
- group_count_y,
- ab_scale,
- encoder,
- );
-
- if let MatmulArgumentC::Bias(bias) = c {
- self.bias_add.encode(None::<&Allocation<Metal>>, bias, d, output_dim, batch_dim * output_dim, encoder);
- }
-
- Ok(())
- }
}
/// Explicit dispatch paths for testing individual kernels independent of production routing.
@@ -336,7 +50,7 @@ pub enum MatmulDispatchPath {
Auto,
Gemv,
Gemm,
- GemmMpp,
+ GemmMxu,
}
impl MatmulMetalKernel {
@@ -346,11 +60,20 @@ impl MatmulMetalKernel {
encoder: &mut Encoder<Metal>,
path: MatmulDispatchPath,
) {
+ let context = encoder.context();
match path {
MatmulDispatchPath::Auto => self.encode(arguments, encoder),
- MatmulDispatchPath::Gemv => self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV"),
- MatmulDispatchPath::Gemm => self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM"),
- MatmulDispatchPath::GemmMpp => self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP"),
+ MatmulDispatchPath::Gemv => {
+ gemv::fp::encode(&mut self.gemv, encoder, arguments).expect("Failed to encode GEMV")
+ },
+ MatmulDispatchPath::Gemm => {
+ gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, false)
+ .expect("Failed to encode Gemm")
+ },
+ MatmulDispatchPath::GemmMxu => {
+ gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, true)
+ .expect("Failed to encode GemmMxu")
+ },
}
}
}
@@ -367,28 +90,15 @@ impl MatmulKernel for MatmulMetalKernel {
}
let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true).map_err(MatmulError::BackendError)?;
+ let gemm = GemmKernel::new(context, data_type).map_err(MatmulError::BackendError)?;
+ let gemv = GemvKernel::new(context, data_type)?;
- let mut kernel = Self {
+ Ok(Self {
data_type,
- gemv_kernels: HashMap::new(),
- gemm_kernels: HashMap::new(),
- gemm_mpp_kernels: HashMap::new(),
+ gemv,
+ gemm,
bias_add,
- };
-
- for &config in gemv::GemvSpecialization::precompile_configs(data_type) {
- kernel.get_or_create_gemv(context, config)?;
- }
- for &config in gemm::GemmSpecialization::precompile_configs(data_type) {
- kernel.get_or_create_gemm(context, config)?;
- }
- if context.device.supports_mxu() {
- for &config in gemm_mpp::GemmMppSpecialization::precompile_configs(data_type) {
- kernel.get_or_create_gemm_mpp(context, config)?;
- }
- }
-
- Ok(kernel)
+ })
}
fn encode(
@@ -396,14 +106,20 @@ impl MatmulKernel for MatmulMetalKernel {
arguments: MatmulArguments<Metal>,
encoder: &mut Encoder<Metal>,
) {
- if !arguments.b_transpose {
- self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM kernel");
- } else if arguments.batch_dim <= max_gemv_batch_threshold() {
- self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV kernel");
- } else if self.is_mpp_eligible(encoder.context()) {
- self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP kernel");
- } else {
- self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM kernel");
+ // gemv only handles the canonical B layout; non-canonical inputs go to GEMM.
+ let context = encoder.context();
+ let gemv_eligible = arguments.b_transpose
+ && arguments.b_offset == 0
+ && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.input_dim)
+ && arguments.batch_dim <= max_gemv_batch_threshold();
+
+ if gemv_eligible {
+ gemv::fp::encode(&mut self.gemv, encoder, arguments).expect("Failed to encode GEMV kernel");
+ return;
}
+
+ let use_mxu = self.is_mxu_eligible(context);
+ gemm::fp::encode(&mut self.gemm, &mut self.bias_add, self.data_type, context, encoder, arguments, use_mxu)
+ .expect("Failed to encode GEMM");
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs
new file mode 100644
index 000000000..6ccbbcd74
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/quant.rs
@@ -0,0 +1,37 @@
+use super::{MatmulMetalKernel, gemm};
+use crate::backends::{
+ common::{
+ Encoder,
+ kernel::quant_matmul::{
+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
+ QuantizedMatmulKernelEncodable,
+ },
+ },
+ metal::{Metal, context::MetalContext},
+};
+
+#[derive(Debug, Clone, Copy)]
+pub enum QuantizedMatmulDispatchPath {
+ Auto,
+ Gemm,
+}
+
+pub fn encode_quantized_matmul_with_path(
+ context: &MetalContext,
+ matmul: &mut MatmulMetalKernel,
+ encodable: &QuantizedMatmulKernelEncodable<Metal>,
+ configuration: &QuantizedMatmulConfiguration,
+ arguments: QuantizedMatmulArguments<Metal>,
+ encoder: &mut Encoder<Metal>,
+ path: QuantizedMatmulDispatchPath,
+) -> Result<(), QuantizedMatmulError<Metal>> {
+ match path {
+ QuantizedMatmulDispatchPath::Auto => {
+ encodable.encode(encoder, arguments);
+ Ok(())
+ },
+ QuantizedMatmulDispatchPath::Gemm => {
+ gemm::quant::encode(&mut matmul.gemm, context, configuration, arguments, encoder)
+ },
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal
index 429651362..75e029e9b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -37,7 +37,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
return;
const uint expert_u = uint(expert_id);
- const uint h_idx = h_block_idx * 4 + thread_context.threadgroup_index;
+ const uint h_idx = h_block_idx * 4 + thread_context.simdgroup_index;
if (h_idx >= d_ff)
return;
@@ -55,7 +55,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
const uint vec_iters = d_model / 128;
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + thread_context.simdgroup_index * 4;
+ uint base_idx = i * 128 + thread_context.simd_lane_id * 4;
device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);
device const T* w_up_vec2 =
@@ -75,7 +75,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
}
}
- uint leftover_start = vec_iters * 128 + thread_context.simdgroup_index;
+ uint leftover_start = vec_iters * 128 + thread_context.simd_lane_id;
for (uint idx = leftover_start; idx < d_model; idx += 32) {
float xv = float(x[idx]);
acc_up += xv * float(w_up_row[idx]);
@@ -89,7 +89,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
acc_gate = simd_sum(acc_gate);
}
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
float up_val = clamp(
acc_up + float(biases[bias_base + h_idx]),
up_clip_min,
@@ -138,7 +138,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
const uint d_block GROUPS(d_model.div_ceil(8)),
const uint tid THREADS(256)
) {
- const uint my_col = d_block * 8 + thread_context.threadgroup_index;
+ const uint my_col = d_block * 8 + thread_context.simdgroup_index;
if (my_col >= d_model)
return;
@@ -160,7 +160,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
// Vectorized reduction
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + thread_context.simdgroup_index * 4;
+ uint base_idx = i * 128 + thread_context.simd_lane_id * 4;
float4 h_vec =
*reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
@@ -173,21 +173,20 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
}
// Remainder
- for (uint idx = vec_iters * 128 + thread_context.simdgroup_index;
- idx < d_ff;
+ for (uint idx = vec_iters * 128 + thread_context.simd_lane_id; idx < d_ff;
idx += 32) {
acc += hidden_ptr[idx] * float(w2_ptr[idx]);
}
// Simdgroup reduction and accumulate
float result = simd_sum(acc);
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);
final_acc += prob * result;
}
}
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
y[my_col] = T(final_acc);
}
}
\ No newline at end of file
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index c573e94ba..4a46fe2a0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -44,7 +44,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
return;
// Each simdgroup outputs one hidden element
- const uint h_idx = h_block_idx * 4 + thread_context.threadgroup_index;
+ const uint h_idx = h_block_idx * 4 + thread_context.simdgroup_index;
if (h_idx >= d_ff)
return;
@@ -66,7 +66,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
const uint vec_iters = d_model / 128;
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + thread_context.simdgroup_index * 4;
+ uint base_idx = i * 128 + thread_context.simd_lane_id * 4;
device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);
device const T* w_up_vec =
@@ -87,7 +87,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
}
// Handle leftover elements
- uint leftover_start = vec_iters * 128 + thread_context.simdgroup_index;
+ uint leftover_start = vec_iters * 128 + thread_context.simd_lane_id;
for (uint idx = leftover_start; idx < d_model; idx += 32) {
float xv = float(x_ptr[idx]);
acc_up += xv * float(w_up_row[idx]);
@@ -103,7 +103,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
}
// Lane 0 applies activation and writes result
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
float up_val = acc_up + float(up_biases[bias_base + h_idx]);
up_val = clamp(up_val, up_clip_min, up_clip_max);
@@ -152,7 +152,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Each simdgroup computes one output column
const uint my_col =
- tgpig_x * SIMDGROUPS_PER_TG + thread_context.threadgroup_index;
+ tgpig_x * SIMDGROUPS_PER_TG + thread_context.simdgroup_index;
if (my_col >= d_model)
return;
@@ -174,7 +174,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
for (uint iter = 0; iter < k_vec_iters; ++iter) {
const uint k_base =
- iter * (8 * THREADS_PER_SIMD) + thread_context.simdgroup_index;
+ iter * (8 * THREADS_PER_SIMD) + thread_context.simd_lane_id;
// hidden: stride-32 per lane, already f32 from Pass A
const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
@@ -220,7 +220,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle remaining full iterations
for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
- const uint k = iter * THREADS_PER_SIMD + thread_context.simdgroup_index;
+ const uint k = iter * THREADS_PER_SIMD + thread_context.simd_lane_id;
acc =
fma(AccumT(hidden[hidden_base + k]),
AccumT(w2_all[w2_col_base + k]),
@@ -229,8 +229,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle leftover elements (d_ff % 32)
const uint leftover_start = k_iters * THREADS_PER_SIMD;
- if (leftover_start + thread_context.simdgroup_index < d_ff) {
- const uint k = leftover_start + thread_context.simdgroup_index;
+ if (leftover_start + thread_context.simd_lane_id < d_ff) {
+ const uint k = leftover_start + thread_context.simd_lane_id;
acc =
fma(AccumT(hidden[hidden_base + k]),
AccumT(w2_all[w2_col_base + k]),
@@ -241,7 +241,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
AccumT result = simd_sum(acc);
// Lane 0 writes result
- if (thread_context.simdgroup_index == 0) {
+ if (thread_context.simd_lane_id == 0) {
const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
result += AccumT(down_biases[bias_idx]);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index 2c46971e0..c6b8cffae 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -106,15 +106,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
// simdgroup tile mapping
const uint sg_col_count = Bn / SgBn;
- const uint row_sg = thread_context.threadgroup_index / sg_col_count;
- const uint col_sg = thread_context.threadgroup_index % sg_col_count;
+ const uint row_sg = thread_context.simdgroup_index / sg_col_count;
+ const uint col_sg = thread_context.simdgroup_index % sg_col_count;
const uint row_sg_off = row_sg * SgBm;
const uint col_sg_off = col_sg * SgBn;
// Guard against misconfigured TG sizes
constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);
- if (thread_context.threadgroup_index >= SG_EXPECTED)
+ if (thread_context.simdgroup_index >= SG_EXPECTED)
return;
// partial accumulators (8x8 tiles per simdgroup)
@@ -295,7 +295,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Epilogue: activation/gating → hidden_out (f32) ----
- const uint sg_lane_start = thread_context.threadgroup_index * 32;
+ const uint sg_lane_start = thread_context.simdgroup_index * 32;
const uint sg_lane_end = sg_lane_start + 32;
if (lin < sg_lane_start || lin >= sg_lane_end)
return;
@@ -448,14 +448,14 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
// 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)
- const uint row_sg = thread_context.threadgroup_index / 2; // 0-1 → rows
- const uint col_sg = thread_context.threadgroup_index % 2; // 0-1 → cols
+ const uint row_sg = thread_context.simdgroup_index / 2; // 0-1 → rows
+ const uint col_sg = thread_context.simdgroup_index % 2; // 0-1 → cols
const uint row_sg_off = row_sg * SgBm;
const uint col_sg_off = col_sg * SgBn;
// Guard against excessive simdgroups
constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);
- if (thread_context.threadgroup_index >= SG_EXPECTED)
+ if (thread_context.simdgroup_index >= SG_EXPECTED)
return;
// 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles
@@ -563,11 +563,11 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Writeout with fragment extraction ----
- const uint lane_qid = thread_context.simdgroup_index >> 2;
+ const uint lane_qid = thread_context.simd_lane_id >> 2;
const uint lane_row =
- (lane_qid & 4u) + ((thread_context.simdgroup_index >> 1) & 3u);
+ (lane_qid & 4u) + ((thread_context.simd_lane_id >> 1) & 3u);
const uint lane_col_base =
- ((lane_qid & 2u) * 2u) + ((thread_context.simdgroup_index & 1u) * 2u);
+ ((lane_qid & 2u) * 2u) + ((thread_context.simd_lane_id & 1u) * 2u);
const uint local_row = row_sg_off + lane_row;
if (local_row >= m_rows)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal
index cbb1a161e..78eca3add 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -52,12 +52,12 @@ PUBLIC KERNEL(MoeRouterTopK)(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint row = thread_context.threadgroup_index; row < e;
+ for (uint row = thread_context.simdgroup_index; row < e;
row += thread_context.simdgroups_per_threadgroup) {
const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;
float4 accum4 = float4(0.0f);
- for (uint c = thread_context.simdgroup_index; c < vecs; c += 32u) {
+ for (uint c = thread_context.simd_lane_id; c < vecs; c += 32u) {
const float4 wv = float4(
w_vec[c * 4 + 0],
w_vec[c * 4 + 1],
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index 3eade2617..860fb1332 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -65,7 +65,7 @@ inline void moe_scatter_buckets_impl(
if (ue >= e0 && ue < e0 + tile_e) {
const uint te = ue - e0;
atomic_fetch_add_explicit(
- &sg_counts[thread_context.threadgroup_index * TILE_E + te],
+ &sg_counts[thread_context.simdgroup_index * TILE_E + te],
1u,
memory_order_relaxed
);
@@ -104,7 +104,7 @@ inline void moe_scatter_buckets_impl(
// Phase 3: deterministic write with 32-lane sequencing, no device atomics
for (uint step = 0; step < thread_context.simdgroup_size; ++step) {
- if (thread_context.simdgroup_index == step) {
+ if (thread_context.simd_lane_id == step) {
const uint t = t_start + lid;
if (t < t_end) {
const uint base = t * K;
@@ -118,7 +118,7 @@ inline void moe_scatter_buckets_impl(
const uint base_idx_block =
(block_id * num_tiles + tile_id) * TILE_E + te;
const uint idx_sg_te =
- thread_context.threadgroup_index * TILE_E + te;
+ thread_context.simdgroup_index * TILE_E + te;
const uint local = atomic_fetch_add_explicit(
&sg_counts[idx_sg_te],
1u,
diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rs
index 478f2add5..d00c639d3 100644
--- a/crates/backend-uzu/src/backends/metal/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/mod.rs
@@ -13,5 +13,8 @@ pub use backend::Metal;
pub use buffer::BufferDowncastExt;
pub use context::MetalContext;
pub use device_capabilities::MetalDeviceCapabilities;
-pub use kernel::matmul::MatmulDispatchPath;
+pub use kernel::matmul::{
+ MatmulDispatchPath,
+ quant::{QuantizedMatmulDispatchPath, encode_quantized_matmul_with_path},
+};
pub use metal_extensions::{DeviceExt, DeviceGeneration};
diff --git a/crates/backend-uzu/src/config/attention.rs b/crates/backend-uzu/src/config/attention.rs
index cf665ee2a..1326dc828 100644
--- a/crates/backend-uzu/src/config/attention.rs
+++ b/crates/backend-uzu/src/config/attention.rs
@@ -1,6 +1,7 @@
use proc_macros::uzu_config;
-use super::{ConfigDataType, LinearConfig, NormalizationConfig, UpcastMode};
+use super::{LinearConfig, NormalizationConfig, UpcastMode};
+use crate::DataType;
pub const VALUE_NORM_EPSILON: f32 = 1e-6;
@@ -34,7 +35,7 @@ impl AttentionConfig {
}
Some(NormalizationConfig {
scale_precision: self.qkv_projection_config.activation_precision(),
- accumulation_precision: ConfigDataType::Float32,
+ accumulation_precision: DataType::F32,
epsilon: VALUE_NORM_EPSILON,
scale_offset: None,
upcast_mode: UpcastMode::FullLayer,
diff --git a/crates/backend-uzu/src/config/common.rs b/crates/backend-uzu/src/config/common.rs
deleted file mode 100644
index 4d9429729..000000000
--- a/crates/backend-uzu/src/config/common.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-use proc_macros::uzu_config;
-
-use crate::DataType;
-
-#[derive(Eq, Copy)]
-#[uzu_config]
-#[serde(rename = "DataType")]
-#[serde(rename_all = "lowercase")]
-pub enum ConfigDataType {
- BFloat16,
- Float16,
- Float32,
- Int4,
- Int8,
-}
-
-impl From<ConfigDataType> for DataType {
- fn from(val: ConfigDataType) -> Self {
- match val {
- ConfigDataType::BFloat16 => DataType::BF16,
- ConfigDataType::Float16 => DataType::F16,
- ConfigDataType::Float32 => DataType::F32,
- ConfigDataType::Int4 => DataType::I4,
- ConfigDataType::Int8 => DataType::I8,
- }
- }
-}
-
-impl From<DataType> for ConfigDataType {
- fn from(dtype: DataType) -> Self {
- match dtype {
- DataType::BF16 => ConfigDataType::BFloat16,
- DataType::F16 => ConfigDataType::Float16,
- DataType::F32 => ConfigDataType::Float32,
- DataType::I4 => ConfigDataType::Int4,
- DataType::I8 => ConfigDataType::Int8,
- _ => panic!("Unsupported data type: {0:?}", dtype),
- }
- }
-}
diff --git a/crates/backend-uzu/src/config/embedding.rs b/crates/backend-uzu/src/config/embedding.rs
index 61ba7cd40..80f43d8e4 100644
--- a/crates/backend-uzu/src/config/embedding.rs
+++ b/crates/backend-uzu/src/config/embedding.rs
@@ -1,6 +1,6 @@
use proc_macros::uzu_config;
-use crate::{ConfigDataType, backends::common::gpu_types::QuantizationMode};
+use crate::{DataType, backends::common::gpu_types::QuantizationMode};
#[uzu_config]
pub struct EmbeddingConfigCommon {
@@ -15,40 +15,40 @@ pub enum EmbeddingConfig {
Tied {
#[serde(flatten)]
common: EmbeddingConfigCommon,
- precision: ConfigDataType,
+ precision: DataType,
},
#[serde(rename = "UntiedEmbeddingConfig")]
Untied {
#[serde(flatten)]
common: EmbeddingConfigCommon,
- precision: ConfigDataType,
+ precision: DataType,
},
- #[serde(rename = "MLXQuantizedTiedEmbeddingConfig")]
- MLXQuantizedTied {
+ #[serde(rename = "ScaleBiasQuantizedTiedEmbeddingConfig")]
+ ScaleBiasQuantizedTied {
#[serde(flatten)]
common: EmbeddingConfigCommon,
group_size: usize,
embedding_quantization_mode: QuantizationMode,
activation_quantization_mode: Option<QuantizationMode>,
- activation_precision: ConfigDataType,
+ activation_precision: DataType,
},
- #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]
- MLXQuantizedUntied {
+ #[serde(rename = "ScaleBiasQuantizedUntiedEmbeddingConfig")]
+ ScaleBiasQuantizedUntied {
#[serde(flatten)]
common: EmbeddingConfigCommon,
group_size: usize,
embedding_quantization_mode: QuantizationMode,
activation_quantization_mode: Option<QuantizationMode>,
- activation_precision: ConfigDataType,
+ activation_precision: DataType,
},
- #[serde(rename = "MLXSemiQuantizedUntiedEmbeddingConfig")]
- MLXSemiQuantizedUntied {
+ #[serde(rename = "ScaleBiasSemiQuantizedUntiedEmbeddingConfig")]
+ ScaleBiasSemiQuantizedUntied {
#[serde(flatten)]
common: EmbeddingConfigCommon,
group_size: usize,
embedding_quantization_mode: QuantizationMode,
activation_quantization_mode: Option<QuantizationMode>,
- activation_precision: ConfigDataType,
+ activation_precision: DataType,
},
}
@@ -63,22 +63,22 @@ impl EmbeddingConfig {
common,
..
}
- | EmbeddingConfig::MLXQuantizedTied {
+ | EmbeddingConfig::ScaleBiasQuantizedTied {
common,
..
}
- | EmbeddingConfig::MLXQuantizedUntied {
+ | EmbeddingConfig::ScaleBiasQuantizedUntied {
common,
..
}
- | EmbeddingConfig::MLXSemiQuantizedUntied {
+ | EmbeddingConfig::ScaleBiasSemiQuantizedUntied {
common,
..
} => common,
}
}
- pub fn activation_precision(&self) -> ConfigDataType {
+ pub fn activation_precision(&self) -> DataType {
match self {
EmbeddingConfig::Tied {
precision,
@@ -88,15 +88,15 @@ impl EmbeddingConfig {
precision,
..
} => *precision,
- EmbeddingConfig::MLXQuantizedTied {
+ EmbeddingConfig::ScaleBiasQuantizedTied {
activation_precision,
..
}
- | EmbeddingConfig::MLXQuantizedUntied {
+ | EmbeddingConfig::ScaleBiasQuantizedUntied {
activation_precision,
..
}
- | EmbeddingConfig::MLXSemiQuantizedUntied {
+ | EmbeddingConfig::ScaleBiasSemiQuantizedUntied {
activation_precision,
..
} => *activation_precision,
diff --git a/crates/backend-uzu/src/config/linear.rs b/crates/backend-uzu/src/config/linear.rs
index 29acbe71c..4c2e4948e 100644
--- a/crates/backend-uzu/src/config/linear.rs
+++ b/crates/backend-uzu/src/config/linear.rs
@@ -1,13 +1,13 @@
use proc_macros::uzu_config;
-use crate::{ConfigDataType, backends::common::gpu_types::QuantizationMode};
+use crate::{DataType, backends::common::gpu_types::QuantizationMode};
#[uzu_config]
pub struct QuantizationConfig {
pub group_size: usize,
pub weight_quantization_mode: QuantizationMode,
pub activation_quantization_mode: Option<QuantizationMode>,
- pub activation_precision: ConfigDataType,
+ pub activation_precision: DataType,
}
#[uzu_config]
@@ -15,12 +15,12 @@ pub struct QuantizationConfig {
pub enum LinearConfig {
#[serde(rename = "FullPrecisionLinearConfig")]
FullPrecision {
- precision: ConfigDataType,
+ precision: DataType,
},
#[serde(rename = "GroupQuantizedLinearConfig")]
Quantized(QuantizationConfig),
#[serde(rename = "MLXQuantizedLinearConfig")]
- MLXQuantized(QuantizationConfig),
+ ScaleBiasQuantized(QuantizationConfig),
#[serde(rename = "QLoRALinearConfig")]
QLoRA {
#[serde(flatten)]
@@ -36,13 +36,13 @@ pub enum LinearConfig {
}
impl LinearConfig {
- pub fn activation_precision(&self) -> ConfigDataType {
+ pub fn activation_precision(&self) -> DataType {
match self {
LinearConfig::FullPrecision {
precision,
} => *precision,
LinearConfig::Quantized(quantization) => quantization.activation_precision,
- LinearConfig::MLXQuantized(quantization) => quantization.activation_precision,
+ LinearConfig::ScaleBiasQuantized(quantization) => quantization.activation_precision,
LinearConfig::QLoRA {
quantization,
..
diff --git a/crates/backend-uzu/src/config/mixer.rs b/crates/backend-uzu/src/config/mixer.rs
index a720378b8..56fc990f1 100644
--- a/crates/backend-uzu/src/config/mixer.rs
+++ b/crates/backend-uzu/src/config/mixer.rs
@@ -1,6 +1,7 @@
use proc_macros::uzu_config;
-use super::{AttentionConfig, ConfigDataType, DeltaNetAttentionConfig, Mamba2Config, ShortConvConfig};
+use super::{AttentionConfig, DeltaNetAttentionConfig, Mamba2Config, ShortConvConfig};
+use crate::DataType;
#[uzu_config]
#[serde(tag = "type")]
@@ -44,7 +45,7 @@ impl MixerConfig {
}
}
- pub fn activation_precision(&self) -> ConfigDataType {
+ pub fn activation_precision(&self) -> DataType {
match self {
MixerConfig::Attention(c) => c.qkv_projection_config.activation_precision(),
MixerConfig::Mamba(c) => c.in_projection_config.activation_precision(),
diff --git a/crates/backend-uzu/src/config/mod.rs b/crates/backend-uzu/src/config/mod.rs
index 399b09cd3..65b81eb82 100644
--- a/crates/backend-uzu/src/config/mod.rs
+++ b/crates/backend-uzu/src/config/mod.rs
@@ -1,7 +1,6 @@
mod attention;
mod classifier;
mod classifier_model;
-mod common;
mod decoder;
mod delta_net;
mod embedding;
@@ -25,7 +24,6 @@ mod tts_model;
pub use attention::*;
pub use classifier::*;
pub use classifier_model::*;
-pub use common::*;
pub use decoder::*;
pub use delta_net::*;
pub use embedding::*;
diff --git a/crates/backend-uzu/src/config/normalization/normalization_config.rs b/crates/backend-uzu/src/config/normalization/normalization_config.rs
index 8937bf290..212e5a1a9 100644
--- a/crates/backend-uzu/src/config/normalization/normalization_config.rs
+++ b/crates/backend-uzu/src/config/normalization/normalization_config.rs
@@ -1,12 +1,12 @@
use proc_macros::uzu_config;
use super::UpcastMode;
-use crate::ConfigDataType;
+use crate::DataType;
#[uzu_config]
pub struct NormalizationConfig {
- pub scale_precision: ConfigDataType,
- pub accumulation_precision: ConfigDataType,
+ pub scale_precision: DataType,
+ pub accumulation_precision: DataType,
pub epsilon: f32,
pub scale_offset: Option<f32>,
pub upcast_mode: UpcastMode,
diff --git a/crates/backend-uzu/src/config/rope.rs b/crates/backend-uzu/src/config/rope.rs
index ff96eab49..31a1a4375 100644
--- a/crates/backend-uzu/src/config/rope.rs
+++ b/crates/backend-uzu/src/config/rope.rs
@@ -1,10 +1,10 @@
use proc_macros::uzu_config;
-use crate::ConfigDataType;
+use crate::DataType;
#[uzu_config]
pub struct RopeConfigCommon {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub base: f32,
pub max_sequence_length: usize,
pub head_dim: usize,
diff --git a/crates/backend-uzu/src/config/short_conv.rs b/crates/backend-uzu/src/config/short_conv.rs
index cf196f65b..d12b53555 100644
--- a/crates/backend-uzu/src/config/short_conv.rs
+++ b/crates/backend-uzu/src/config/short_conv.rs
@@ -1,10 +1,11 @@
use proc_macros::uzu_config;
-use super::{common::ConfigDataType, linear::LinearConfig};
+use super::linear::LinearConfig;
+use crate::DataType;
#[uzu_config]
pub struct SeparableCausalConvConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub has_biases: bool,
}
diff --git a/crates/backend-uzu/src/config/tts_model.rs b/crates/backend-uzu/src/config/tts_model.rs
index b34f9ebfb..4851cb7b3 100644
--- a/crates/backend-uzu/src/config/tts_model.rs
+++ b/crates/backend-uzu/src/config/tts_model.rs
@@ -4,7 +4,7 @@ use std::path::Path;
use proc_macros::uzu_config;
use super::{EmbeddingConfig, EmbeddingConfigCommon, NormalizationConfig, TransformerConfig};
-use crate::{ConfigDataType, backends::common::ActivationConfig};
+use crate::{DataType, backends::common::ActivationConfig};
#[uzu_config]
pub struct TtsMessageProcessorConfig {
@@ -17,7 +17,7 @@ pub struct TtsConfig {
pub text_decoder_config: TtsTextDecoderConfig,
pub audio_decoder_config: TtsAudioDecoderConfig,
pub vocoder_config: NoopVocoderConfig,
- pub activation_precision: ConfigDataType,
+ pub activation_precision: DataType,
}
#[uzu_config]
@@ -50,21 +50,21 @@ pub struct FishAudioTextDecoderConfig {
pub num_codebooks: usize,
pub max_seq_len: usize,
pub scale_codebook_embeddings: bool,
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub short_logits_size: usize,
pub repeat_window_size: usize,
}
#[uzu_config]
pub struct FullPrecisionLinearConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
}
#[uzu_config]
pub struct TiedEmbeddingConfig {
#[serde(flatten)]
pub common: EmbeddingConfigCommon,
- pub precision: ConfigDataType,
+ pub precision: DataType,
}
impl TiedEmbeddingConfig {
@@ -86,7 +86,7 @@ pub enum TtsAudioDecoderConfig {
#[uzu_config]
pub struct DescriptAudioCodecConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub quantizer_config: DescriptAudioQuantizerConfig,
pub decoder_config: DescriptAudioDacDecoderConfig,
pub samplerate: u32,
@@ -104,7 +104,7 @@ pub struct DescriptAudioCodecConfig {
#[uzu_config]
pub struct DescriptAudioDacDecoderConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub conv_config: DescriptAudioCausalConv1dConfig,
pub snake_config: DescriptAudioSnake1dConfig,
pub decoder_block_config: DescriptAudioDacDecoderBlockConfig,
@@ -113,7 +113,7 @@ pub struct DescriptAudioDacDecoderConfig {
#[uzu_config]
pub struct DescriptAudioDacDecoderBlockConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub snake_config: DescriptAudioSnake1dConfig,
pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,
pub res_unit_config: DescriptAudioResidualUnitConfig,
@@ -122,7 +122,7 @@ pub struct DescriptAudioDacDecoderBlockConfig {
#[uzu_config]
pub struct DescriptAudioResidualUnitConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub snake_config: DescriptAudioSnake1dConfig,
pub conv_config: DescriptAudioCausalConv1dConfig,
pub causal: bool,
@@ -130,24 +130,24 @@ pub struct DescriptAudioResidualUnitConfig {
#[uzu_config]
pub struct DescriptAudioCausalConv1dConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub has_biases: bool,
}
#[uzu_config]
pub struct DescriptAudioCausalTransposeConv1dConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub has_biases: bool,
}
#[uzu_config]
pub struct DescriptAudioSnake1dConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
}
#[uzu_config]
pub struct DescriptAudioQuantizerConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub semantic_quantizer_config: DescriptAudioResidualVectorQuantizeConfig,
pub quantizer_config: DescriptAudioResidualVectorQuantizeConfig,
pub post_module_config: TransformerConfig,
@@ -156,13 +156,13 @@ pub struct DescriptAudioQuantizerConfig {
#[uzu_config]
pub struct DescriptAudioResidualVectorQuantizeConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub vq_config: DescriptAudioVectorQuantizeConfig,
}
#[uzu_config]
pub struct DescriptAudioVectorQuantizeConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub codebook_config: TiedEmbeddingConfig,
pub out_proj_config: FullPrecisionLinearConfig,
}
@@ -174,14 +174,14 @@ pub struct DescriptAudioUpsamplerConfig {
#[uzu_config]
pub struct DescriptAudioUpsamplingBlockConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,
pub convnext_config: DescriptAudioConvNeXtConfig,
}
#[uzu_config]
pub struct DescriptAudioConvNeXtConfig {
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub activation: ActivationConfig,
pub dwconv_config: DescriptAudioCausalConv1dConfig,
pub norm_config: NormalizationConfig,
diff --git a/crates/backend-uzu/src/data_type.rs b/crates/backend-uzu/src/data_type.rs
index c4621c4e9..5af111f1a 100644
--- a/crates/backend-uzu/src/data_type.rs
+++ b/crates/backend-uzu/src/data_type.rs
@@ -1,25 +1,41 @@
use bytemuck::Pod;
use half::{bf16, f16};
use num_traits::NumCast;
+use proc_macros::uzu_config;
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Copy, Clone)]
+#[derive(Eq, PartialOrd, Ord, Copy)]
+#[uzu_config]
pub enum DataType {
// Floating point
+ #[serde(rename = "bfloat16")]
BF16,
+ #[serde(rename = "float16")]
F16,
+ #[serde(rename = "float32")]
F32,
+ #[serde(rename = "float64")]
F64,
// Sub-byte integers
+ #[serde(rename = "int4")]
I4,
+ #[serde(rename = "uint4")]
U4,
// Normal integers
+ #[serde(rename = "int8")]
I8,
+ #[serde(rename = "uint8")]
U8,
+ #[serde(rename = "int16")]
I16,
+ #[serde(rename = "uint16")]
U16,
+ #[serde(rename = "int32")]
I32,
+ #[serde(rename = "uint32")]
U32,
+ #[serde(rename = "int64")]
I64,
+ #[serde(rename = "uint64")]
U64,
}
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 0a7047244..1b0897e02 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -223,7 +223,7 @@ impl<B: Backend> Embedding<B> {
},
}
},
- EmbeddingConfig::MLXQuantizedTied {
+ EmbeddingConfig::ScaleBiasQuantizedTied {
common: _,
group_size,
embedding_quantization_mode,
@@ -285,7 +285,7 @@ impl<B: Backend> Embedding<B> {
},
}
},
- EmbeddingConfig::MLXQuantizedUntied {
+ EmbeddingConfig::ScaleBiasQuantizedUntied {
common: _,
group_size,
embedding_quantization_mode,
@@ -364,7 +364,7 @@ impl<B: Backend> Embedding<B> {
},
}
},
- EmbeddingConfig::MLXSemiQuantizedUntied {
+ EmbeddingConfig::ScaleBiasSemiQuantizedUntied {
common: _,
group_size,
embedding_quantization_mode,
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index 33a88f2ba..57eb8995a 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -48,7 +48,7 @@ impl<B: Backend> dyn Linear<B> {
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
match config {
- LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {
+ LinearConfig::Quantized(quantization_config) | LinearConfig::ScaleBiasQuantized(quantization_config) => {
let block = QuantizedLinear::new(
context,
quantization_config,
@@ -111,14 +111,9 @@ impl<B: Backend> dyn Linear<B> {
output_dim: usize,
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
match config {
- LinearConfig::Quantized(config) | LinearConfig::MLXQuantized(config) => Ok(Box::new(QuantizedLinear::new(
- context,
- config,
- input_dim,
- output_dim,
- parameter_tree,
- Some(output_factors),
- )?)),
+ LinearConfig::Quantized(config) | LinearConfig::ScaleBiasQuantized(config) => Ok(Box::new(
+ QuantizedLinear::new(context, config, input_dim, output_dim, parameter_tree, Some(output_factors))?,
+ )),
LinearConfig::QLoRA {
quantization,
lora_rank,
diff --git a/crates/backend-uzu/src/encodable_block/linear/quantized.rs b/crates/backend-uzu/src/encodable_block/linear/quantized.rs
index dee4df5d5..510605d86 100644
--- a/crates/backend-uzu/src/encodable_block/linear/quantized.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/quantized.rs
@@ -40,9 +40,9 @@ pub enum QuantizedLinearError<B: Backend> {
got: DataType,
},
#[error(
- "Unexpected MLX shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"
+ "Unexpected scale-bias shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"
)]
- InvalidMlxShapes {
+ InvalidScaleBiasShapes {
weights: Box<[usize]>,
scales: Box<[usize]>,
deq_biases: Box<[usize]>,
@@ -54,9 +54,9 @@ pub enum QuantizedLinearError<B: Backend> {
got: DataType,
},
#[error(
- "Unexpected AWQ shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"
+ "Unexpected scale-zero-point shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"
)]
- InvalidAwqShapes {
+ InvalidScaleZeroPointShapes {
weights: Box<[usize]>,
scales: Box<[usize]>,
zero_points: Box<[usize]>,
@@ -134,7 +134,7 @@ impl<B: Backend> QuantizedLinear<B> {
&& scales_shape == [output_dim, k_g]
&& deq_biases_shape == [output_dim, k_g])
{
- return Err(QuantizedLinearError::InvalidMlxShapes {
+ return Err(QuantizedLinearError::InvalidScaleBiasShapes {
weights: weights_shape.into_boxed_slice(),
scales: scales_shape.into_boxed_slice(),
deq_biases: deq_biases_shape.into_boxed_slice(),
@@ -163,7 +163,7 @@ impl<B: Backend> QuantizedLinear<B> {
&& scales_shape == [output_dim, k_g]
&& zero_points_shape == [output_dim, expected_zero_points_entries])
{
- return Err(QuantizedLinearError::InvalidAwqShapes {
+ return Err(QuantizedLinearError::InvalidScaleZeroPointShapes {
weights: weights_shape.into_boxed_slice(),
scales: scales_shape.into_boxed_slice(),
zero_points: zero_points_shape.into_boxed_slice(),
diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rs
index afc80e7d5..3b39a0746 100644
--- a/crates/backend-uzu/src/lib.rs
+++ b/crates/backend-uzu/src/lib.rs
@@ -18,7 +18,6 @@ pub use array::{Array, ArrayContextExt};
#[cfg(metal_backend)]
pub use audio::{NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};
pub use backends::common::{AllocationAccessError, allocation_copy_from_slice, allocation_to_vec};
-pub use config::ConfigDataType;
pub use data_type::{ArrayElement, DataType};
pub use language_model::gumbel::{gumbel_float, revidx};
pub use parameters::{ParameterLoader, read_safetensors_metadata};
diff --git a/crates/backend-uzu/src/utils/mod.rs b/crates/backend-uzu/src/utils/mod.rs
index 74b765864..f002e61cb 100644
--- a/crates/backend-uzu/src/utils/mod.rs
+++ b/crates/backend-uzu/src/utils/mod.rs
@@ -6,5 +6,4 @@ pub mod model_size;
pub mod pointers;
pub mod strict_serde;
pub mod version;
-
pub use version::{TOOLCHAIN_VERSION, VERSION};
diff --git a/crates/backend-uzu/tests/BENCHMARKS.md b/crates/backend-uzu/tests/BENCHMARKS.md
index 72d338ff4..4f1ece3b8 100644
--- a/crates/backend-uzu/tests/BENCHMARKS.md
+++ b/crates/backend-uzu/tests/BENCHMARKS.md
@@ -24,16 +24,21 @@ baselines side by side.
## Available benchmark groups
-| Group id | Filter |
-| ------------------------------ | ------------------------------ |
-| `Metal/Kernel/Matmul/GEMM_MPP` | `Metal/Kernel/Matmul/GEMM_MPP` |
-| `Metal/Kernel/RMSNorm` | `Metal/Kernel/RMSNorm` |
-| `Metal/Kernel/Sampling/Argmax` | `Metal/Kernel/Sampling/Argmax` |
+| Group id | Filter |
+| --------------------------------- | --------------------------------- |
+| `Metal/Kernel/Matmul/GEMM` | `Metal/Kernel/Matmul/GEMM` |
+| `Metal/Kernel/Matmul/GEMM_MXU` | `Metal/Kernel/Matmul/GEMM_MXU` |
+| `Metal/Kernel/QmmTransposed/...` | `Metal/Kernel/QmmTransposed` |
+| `Metal/Kernel/QmvFast/...` | `Metal/Kernel/QmvFast` |
+| `Metal/Kernel/RMSNorm` | `Metal/Kernel/RMSNorm` |
+| `Metal/Kernel/Sampling/Argmax` | `Metal/Kernel/Sampling/Argmax` |
+
+The prefix `Metal/Kernel/Matmul` runs both `GEMM` and `GEMM_MXU` in one pass.
## Output layout
Every run writes into `target/criterion/<label>/…`, where `<label>` is a
-free-form name you choose (e.g. `m2_max`, `a18`). The Criterion baseline
+free-form name you choose (e.g. `m2_max`, `a19`). The Criterion baseline
you saved lives at `target/criterion/<label>/<benchmark-path>/<baseline-name>/`.
## Running on macOS
@@ -43,9 +48,9 @@ resolve relative to the package dir:
```bash
CRITERION_HOME="$PWD/target/criterion/m2_max" cargo bench \
- -p uzu \
- --bench kernel -- "Metal/Kernel/RMSNorm" \
- --save-baseline rms_norm_baseline_m2_max
+ -p backend-uzu \
+ --bench kernel -- "Metal/Kernel/Matmul" \
+ --save-baseline matmul_baseline_m2_max
```
## Running on iPhone (via `cargo-dinghy`)
@@ -55,9 +60,9 @@ app.
Key flags:
-- `-e CRITERION_HOME=target/criterion/a18` — on-device env var. Path is
+- `-e CRITERION_HOME=target/criterion/a19` — on-device env var. Path is
relative to the app's cwd (`Documents/`), so this becomes
- `Documents/target/criterion/a18/` on device.
+ `Documents/target/criterion/a19/` on device.
- `--copy-back "Documents/target=$(pwd)/target"` — after the run,
`cargo-dinghy` pulls `Documents/target` from the device into your
repo's `target/`. `$(pwd)` is required (absolute DST) because the
@@ -69,15 +74,15 @@ DEVICE=<DEVICE_ID>
cargo dinghy \
-d "$DEVICE" \
- -e CRITERION_HOME=target/criterion/a18 \
+ -e CRITERION_HOME=target/criterion/a19 \
--copy-back "Documents/target=$(pwd)/target" \
- bench -p uzu --bench kernel -- \
- "Metal/Kernel/Matmul/GEMM_MPP" \
- --save-baseline matmul_gemm_mpp_baseline_a18
+ bench -p backend-uzu --bench kernel -- \
+ "Metal/Kernel/Matmul" \
+ --save-baseline matmul_baseline_a19
```
After the run completes you'll have
-`target/criterion/a18/Metal/Kernel/Matmul/GEMM_MPP/…/matmul_gemm_mpp_baseline_a18/`
+`target/criterion/a19/Metal/Kernel/Matmul/<GEMM|GEMM_MXU>/…/matmul_baseline_a19/`
on the host, next to any `m2_max/` baselines.
## Viewing reports
@@ -92,5 +97,5 @@ To inspect a specific label only:
```bash
open target/criterion/m2_max/report/index.html
-open target/criterion/a18/report/index.html
+open target/criterion/a19/report/index.html
```
diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rs
new file mode 100644
index 000000000..3a52529cf
--- /dev/null
+++ b/crates/backend-uzu/tests/common/matmul/harness.rs
@@ -0,0 +1,161 @@
+#[cfg(metal_backend)]
+use backend_uzu::backends::metal::{Metal, MetalContext};
+use backend_uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::{
+ common::{
+ AllocationType, Backend, Context, Encoder,
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ },
+ },
+ cpu::Cpu,
+ },
+};
+use num_traits::Float;
+
+use super::{
+ super::helpers::{alloc_allocation_with_data, allocation_to_vec},
+ Shape, Variant,
+};
+
+#[cfg(metal_backend)]
+pub type MetalMatmulKernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel;
+
+#[derive(Debug, Clone, Copy)]
+pub struct Case {
+ pub shape: Shape,
+ pub ab_scale: f32,
+ pub accumulate: bool,
+ pub b_transpose: bool,
+}
+
+impl Case {
+ pub const fn new(shape: Shape) -> Self {
+ Self {
+ shape,
+ ab_scale: 1.0,
+ accumulate: false,
+ b_transpose: true,
+ }
+ }
+
+ pub const fn with_ab_scale(
+ mut self,
+ ab_scale: f32,
+ ) -> Self {
+ self.ab_scale = ab_scale;
+ self
+ }
+
+ pub const fn with_accumulate(
+ mut self,
+ accumulate: bool,
+ ) -> Self {
+ self.accumulate = accumulate;
+ self
+ }
+}
+
+pub struct Input<T: ArrayElement + Float> {
+ pub a: Box<[T]>,
+ pub b: Box<[T]>,
+ pub d_prefill: Option<Box<[T]>>,
+ pub case: Case,
+}
+
+pub fn deterministic_input<T: ArrayElement + Float>(case: Case) -> Input<T> {
+ let Shape {
+ m,
+ k,
+ n,
+ } = case.shape;
+ let a: Vec<T> = (0..m * k).map(|i| T::from(((i % 13) as f32) * 0.1 - 0.6).unwrap()).collect();
+ let b: Vec<T> = (0..n * k).map(|i| T::from(((i % 17) as f32) * 0.1 - 0.8).unwrap()).collect();
+ let d_prefill = case.accumulate.then(|| {
+ (0..m * n).map(|i| T::from(((i % 7) as f32) * 0.03 - 0.09).unwrap()).collect::<Vec<_>>().into_boxed_slice()
+ });
+ Input {
+ a: a.into_boxed_slice(),
+ b: b.into_boxed_slice(),
+ d_prefill,
+ case,
+ }
+}
+
+fn run<B: Backend, T: ArrayElement + Float>(
+ context: &B::Context,
+ kernel: &mut <B::Kernels as ManualKernels>::MatmulKernel,
+ input: &Input<T>,
+ encode: impl FnOnce(&mut <B::Kernels as ManualKernels>::MatmulKernel, MatmulArguments<B>, &mut Encoder<B>),
+) -> Vec<T> {
+ let Shape {
+ m,
+ k,
+ n,
+ } = input.case.shape;
+ let b_array = if input.case.b_transpose {
+ context.create_array_from(&[n, k], &input.b)
+ } else {
+ context.create_array_from(&[k, n], &input.b)
+ };
+ let a_allocation = alloc_allocation_with_data::<B, T>(context, &input.a);
+ let mut d_allocation = if let Some(ref prefill) = input.d_prefill {
+ alloc_allocation_with_data::<B, T>(context, prefill)
+ } else {
+ context
+ .create_allocation(m * n * std::mem::size_of::<T>(), AllocationType::Global)
+ .expect("create d allocation")
+ };
+
+ let c_arg = if input.case.accumulate {
+ MatmulArgumentC::Accumulate
+ } else {
+ MatmulArgumentC::None
+ };
+
+ let mut encoder = Encoder::new(context).expect("encoder");
+ encode(
+ kernel,
+ MatmulArguments {
+ a: &a_allocation,
+ a_offset: 0,
+ b: b_array.allocation(),
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: input.case.b_transpose,
+ ab_scale: input.case.ab_scale,
+ c: c_arg,
+ d: &mut d_allocation,
+ batch_dim: m as u32,
+ input_dim: k as u32,
+ output_dim: n as u32,
+ },
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ allocation_to_vec::<B, T>(&d_allocation)
+}
+
+pub fn cpu_reference<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = <Cpu as Backend>::Context::new().expect("CPU context");
+ let mut kernel = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
+ .expect("CPU MatmulKernel");
+ run::<Cpu, T>(&context, &mut kernel, input, |kernel, args, encoder| {
+ kernel.encode(args, encoder);
+ })
+}
+
+#[cfg(metal_backend)]
+pub fn run_metal<T: ArrayElement + Float>(
+ context: &MetalContext,
+ kernel: &mut MetalMatmulKernel,
+ input: &Input<T>,
+ variant: Variant,
+) -> Vec<T> {
+ let path = variant.dispatch_path();
+ run::<Metal, T>(context, kernel, input, |kernel, args, encoder| {
+ kernel.encode_with_path(args, encoder, path);
+ })
+}
diff --git a/crates/backend-uzu/tests/common/matmul/mod.rs b/crates/backend-uzu/tests/common/matmul/mod.rs
new file mode 100644
index 000000000..da729946d
--- /dev/null
+++ b/crates/backend-uzu/tests/common/matmul/mod.rs
@@ -0,0 +1,9 @@
+pub mod harness;
+pub mod shape;
+pub mod variant;
+
+#[cfg(metal_backend)]
+pub use harness::run_metal;
+pub use harness::{Case, cpu_reference, deterministic_input};
+pub use shape::{Shape, all_correctness_shapes};
+pub use variant::Variant;
diff --git a/crates/backend-uzu/tests/common/matmul/shape.rs b/crates/backend-uzu/tests/common/matmul/shape.rs
new file mode 100644
index 000000000..3165f2031
--- /dev/null
+++ b/crates/backend-uzu/tests/common/matmul/shape.rs
@@ -0,0 +1,39 @@
+use derive_more::Display;
+
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
+#[display("M[{m}]K[{k}]N[{n}]")]
+pub struct Shape {
+ pub m: usize,
+ pub k: usize,
+ pub n: usize,
+}
+
+impl Shape {
+ pub const fn new(
+ m: usize,
+ k: usize,
+ n: usize,
+ ) -> Self {
+ Self {
+ m,
+ k,
+ n,
+ }
+ }
+}
+
+const SHAPES_TINY: &[Shape] = &[Shape::new(64, 64, 64), Shape::new(16, 128, 256), Shape::new(128, 256, 128)];
+
+const SHAPES_UNALIGNED: &[Shape] = &[
+ Shape::new(7, 33, 11),
+ Shape::new(33, 2048, 2048),
+ Shape::new(64, 2048, 33),
+ Shape::new(200, 2048, 2048),
+ Shape::new(128, 2048, 200),
+];
+
+const SHAPES_MEDIUM: &[Shape] = &[Shape::new(128, 2048, 2048), Shape::new(256, 4096, 4096)];
+
+pub fn all_correctness_shapes() -> impl Iterator<Item = Shape> {
+ SHAPES_TINY.iter().chain(SHAPES_UNALIGNED.iter()).chain(SHAPES_MEDIUM.iter()).copied()
+}
diff --git a/crates/backend-uzu/tests/common/matmul/variant.rs b/crates/backend-uzu/tests/common/matmul/variant.rs
new file mode 100644
index 000000000..34ce795ea
--- /dev/null
+++ b/crates/backend-uzu/tests/common/matmul/variant.rs
@@ -0,0 +1,33 @@
+#[cfg(metal_backend)]
+use backend_uzu::backends::metal::{DeviceExt, MatmulDispatchPath, MetalContext};
+use derive_more::Display;
+
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
+pub enum Variant {
+ #[display("GEMM")]
+ Gemm,
+ #[display("GEMM_MXU")]
+ GemmMxu,
+}
+
+impl Variant {
+ pub const fn requires_mxu(self) -> bool {
+ matches!(self, Variant::GemmMxu)
+ }
+
+ #[cfg(metal_backend)]
+ pub fn supported(
+ self,
+ context: &MetalContext,
+ ) -> bool {
+ !self.requires_mxu() || context.device.supports_mxu()
+ }
+
+ #[cfg(metal_backend)]
+ pub const fn dispatch_path(self) -> MatmulDispatchPath {
+ match self {
+ Variant::Gemm => MatmulDispatchPath::Gemm,
+ Variant::GemmMxu => MatmulDispatchPath::GemmMxu,
+ }
+ }
+}
diff --git a/crates/backend-uzu/tests/common/mod.rs b/crates/backend-uzu/tests/common/mod.rs
index 1d1e67811..ada88165e 100644
--- a/crates/backend-uzu/tests/common/mod.rs
+++ b/crates/backend-uzu/tests/common/mod.rs
@@ -3,6 +3,7 @@
pub mod assert;
pub mod audio;
pub mod helpers;
+pub mod matmul;
pub mod path;
pub mod perf;
pub mod proptest;
diff --git a/crates/backend-uzu/tests/unit/kernel/bench_entry.rs b/crates/backend-uzu/tests/performance/kernel/bench_entry.rs
similarity index 88%
rename from crates/backend-uzu/tests/unit/kernel/bench_entry.rs
rename to crates/backend-uzu/tests/performance/kernel/bench_entry.rs
index dff5cf777..d3fd71b67 100644
--- a/crates/backend-uzu/tests/unit/kernel/bench_entry.rs
+++ b/crates/backend-uzu/tests/performance/kernel/bench_entry.rs
@@ -17,6 +17,6 @@ fn bench_runner(benches: &[&dyn Fn()]) {
criterion::runner(benches);
}
-use proc_macros::{__internal_uzu_bench as uzu_bench, __internal_uzu_ignored as uzu_test};
+use proc_macros::__internal_uzu_bench as uzu_bench;
include!("mod.rs");
diff --git a/crates/backend-uzu/tests/performance/kernel/mod.rs b/crates/backend-uzu/tests/performance/kernel/mod.rs
new file mode 100644
index 000000000..0223d0039
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/mod.rs
@@ -0,0 +1,7 @@
+#[macro_use]
+#[path = "../../common/mod.rs"]
+mod common;
+
+mod quant_matmul;
+mod rms_norm_bench;
+mod sampling;
diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs
new file mode 100644
index 000000000..857ee0c92
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/quant_matmul/mod.rs
@@ -0,0 +1,2 @@
+mod qmm_transposed_bench;
+mod qmv_fast_bench;
diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs
new file mode 100644
index 000000000..f2ba0d5d8
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmm_transposed_bench.rs
@@ -0,0 +1,132 @@
+use backend_uzu::{
+ ArrayElement,
+ backends::common::{
+ Allocation, Backend, Context, Encoder, Kernels,
+ gpu_types::QuantizationMethod,
+ kernel::QuantizedMatmulQmmTransposedKernel,
+ },
+};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::{bf16, f16};
+use itertools::iproduct;
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+
+use crate::{
+ common::{
+ helpers::{alloc_allocation, alloc_allocation_with_data},
+ type_short_name,
+ },
+ uzu_bench,
+};
+
+fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(
+ c: &mut Criterion,
+ context: &B::Context,
+ label: &str,
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+) {
+ let block_size: usize = if bits == 4 { 512 } else { 256 };
+
+ for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336]) {
+ if n % 32 != 0 || k % block_size != 0 {
+ continue;
+ }
+
+ let (bm, bk, bn, wm, wn) = if m < 48 {
+ (8u32, 32u32, 32u32, 1u32, 1u32)
+ } else {
+ (32u32, 32u32, 32u32, 2u32, 2u32)
+ };
+
+ let num_groups = k.div_ceil(group_size as usize);
+ let mut rng = SmallRng::seed_from_u64(42);
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
+ context,
+ T::data_type(),
+ group_size,
+ bits,
+ bm,
+ bk,
+ bn,
+ wm,
+ wn,
+ quant_method,
+ false,
+ n % bn as usize == 0,
+ )
+ .unwrap();
+
+ let w_buf = alloc_allocation_with_data::<B, u32>(
+ context,
+ &(0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect::<Vec<_>>(),
+ );
+ let scales_buf = alloc_allocation_with_data::<B, T>(
+ context,
+ &(0..n * num_groups).map(|_| T::from(rng.random_range(0.01f32..1.0f32)).unwrap()).collect::<Vec<_>>(),
+ );
+ let x_buf = alloc_allocation_with_data::<B, T>(
+ context,
+ &(0..m * k).map(|_| T::from(rng.random_range(-1.0f32..1.0f32)).unwrap()).collect::<Vec<_>>(),
+ );
+ let mut y_buf = alloc_allocation::<B, T>(context, m * n);
+
+ let zp_stride = if bits == 4 {
+ (num_groups + 1) / 2
+ } else {
+ num_groups
+ };
+ let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {
+ alloc_allocation_with_data::<B, u8>(
+ context,
+ &(0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect::<Vec<_>>(),
+ )
+ });
+ let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {
+ alloc_allocation_with_data::<B, T>(
+ context,
+ &(0..n * num_groups).map(|_| T::from(rng.random_range(-0.5f32..0.5f32)).unwrap()).collect::<Vec<_>>(),
+ )
+ });
+
+ let mut group = c.benchmark_group(format!("{}/Kernel/QmmTransposed/{}", type_short_name::<B>(), label));
+ group.throughput(Throughput::Elements((m * n * k) as u64));
+ group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {
+ b.iter_custom(|n_iters| {
+ let mut encoder = Encoder::<B>::new(context).unwrap();
+ for _ in 0..n_iters {
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ None::<&Allocation<B>>,
+ k as u32,
+ n as u32,
+ m as u32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
+ })
+ });
+ drop(group);
+ }
+}
+
+#[uzu_bench]
+fn bench_qmm_transposed(c: &mut Criterion) {
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+ bench_qmm_transposed_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
+ bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmm_transposed_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
+ bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmm_transposed_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ });
+}
diff --git a/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs
new file mode 100644
index 000000000..16d352560
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/quant_matmul/qmv_fast_bench.rs
@@ -0,0 +1,141 @@
+use backend_uzu::{
+ ArrayElement,
+ backends::common::{
+ Allocation, Backend, Context, Encoder, Kernels,
+ gpu_types::QuantizationMethod,
+ kernel::QuantizedMatmulQmvFastKernel,
+ },
+};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::{bf16, f16};
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+
+use crate::{
+ common::{
+ helpers::{alloc_allocation, alloc_allocation_with_data},
+ type_short_name,
+ },
+ uzu_bench,
+};
+
+fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(
+ rng: &mut R,
+ range: std::ops::Range<T>,
+ len: usize,
+) -> Box<[T]> {
+ (0..len).map(|_| rng.random_range(range.clone())).collect()
+}
+
+fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
+ c: &mut Criterion,
+ context: &B::Context,
+ label: &str,
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+) {
+ let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));
+ let block_size: usize = if bits == 4 {
+ 512
+ } else {
+ 256
+ };
+
+ let nk_pairs: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
+ for &(n, k) in nk_pairs {
+ if n % 8 != 0 || k % block_size != 0 {
+ continue;
+ }
+ for m in [1usize, 2, 4, 8] {
+ let num_groups = k.div_ceil(group_size as usize);
+ let mut rng = SmallRng::seed_from_u64(42);
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ context,
+ T::data_type(),
+ group_size,
+ bits,
+ quant_method,
+ false,
+ )
+ .unwrap();
+
+ let w_buf = alloc_allocation_with_data::<B, u32>(
+ context,
+ &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),
+ );
+ let scales_buf = alloc_allocation_with_data::<B, T>(
+ context,
+ &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)
+ .iter()
+ .map(|&v| T::from(v).unwrap())
+ .collect::<Vec<_>>(),
+ );
+ let x_buf = alloc_allocation_with_data::<B, T>(
+ context,
+ &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k)
+ .iter()
+ .map(|&v| T::from(v).unwrap())
+ .collect::<Vec<_>>(),
+ );
+ let mut y_buf = alloc_allocation::<B, T>(context, m * n);
+
+ let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {
+ let zp_stride = if bits == 4 {
+ (num_groups + 1) / 2
+ } else {
+ num_groups
+ };
+ alloc_allocation_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))
+ });
+ let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {
+ alloc_allocation_with_data::<B, T>(
+ context,
+ &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)
+ .iter()
+ .map(|&v| T::from(v).unwrap())
+ .collect::<Vec<_>>(),
+ )
+ });
+
+ group.throughput(Throughput::Elements((m * n * k) as u64));
+ group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {
+ b.iter_custom(|n_iters| {
+ let mut encoder = Encoder::<B>::new(context).unwrap();
+ for _ in 0..n_iters {
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ None::<&Allocation<B>>,
+ k as u32,
+ n as u32,
+ m as u32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
+ })
+ });
+ }
+ }
+}
+
+#[uzu_bench]
+fn bench_qmv_fast(c: &mut Criterion) {
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);
+ });
+}
diff --git a/crates/backend-uzu/tests/performance/kernel/rms_norm_bench.rs b/crates/backend-uzu/tests/performance/kernel/rms_norm_bench.rs
new file mode 100644
index 000000000..b1614b707
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/rms_norm_bench.rs
@@ -0,0 +1,102 @@
+use backend_uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::common::{Allocation, Backend, Context, Encoder, Kernels, kernel::RMSNormKernel},
+};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use itertools::iproduct;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+
+use crate::{common::type_short_name, uzu_bench};
+
+fn get_rms_norm_data(
+ seed: u64,
+ batch_size: usize,
+ model_dim: usize,
+) -> (Box<[f32]>, Box<[f32]>) {
+ let mut rng = SmallRng::seed_from_u64(seed);
+ let input_size = batch_size * model_dim;
+ let mut input_data = vec![0.0f32; input_size];
+ for x in input_data.iter_mut() {
+ *x = rng.random_range(-2.0f32..2.0f32);
+ }
+ let mut scale_data = vec![0.0f32; model_dim];
+ for x in scale_data.iter_mut() {
+ *x = rng.random_range(0.1f32..3.0f32);
+ }
+ (input_data.into_boxed_slice(), scale_data.into_boxed_slice())
+}
+
+#[uzu_bench]
+fn bench_rms_norm(c: &mut Criterion) {
+ type T = f32;
+ let epsilon = 1e-6f32;
+ let scale_offset = 0.0f32;
+
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::RMSNormKernel::new(
+ &context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ )
+ .unwrap();
+
+ let mut group = c.benchmark_group(format!("{}/Kernel/RMSNorm", type_short_name::<B>()));
+
+ for (batch_size, model_dim) in iproduct!(
+ [1, 4, 32, 128, 1024], // Batch sizes
+ [
+ 1024, // LFM2-350M
+ 1152, // Gemma-3-1b
+ 2048, // Llama-3.2-1B
+ 2560, // Qwen3-4B
+ 3072, // Llama-3.2-3B
+ 4096, // Qwen3-8B
+ 5120, // Qwen2.5-32B
+ ]
+ ) {
+ let (input_data, scale_data) = get_rms_norm_data(1337, batch_size, model_dim);
+ let input_size = batch_size * model_dim;
+
+ let input_buffer = context.create_array_from(&[input_size], input_data.as_ref()).into_allocation();
+ let scales_buffer = context.create_array_from(&[model_dim], scale_data.as_ref()).into_allocation();
+ let mut output_buffer = context.create_array_uninitialized(&[input_size], T::data_type()).into_allocation();
+
+ group.throughput(Throughput::Elements((batch_size * model_dim) as u64));
+
+ group.bench_function(BenchmarkId::from_parameter(format!("Batch[{batch_size}]Dim[{model_dim}]")), |b| {
+ b.iter_custom(|n_iters| {
+ let mut encoder = Encoder::<B>::new(&context).unwrap();
+
+ for _ in 0..n_iters {
+ kernel.encode(
+ Some(&input_buffer),
+ &scales_buffer,
+ &mut output_buffer,
+ None::<&mut Allocation<B>>,
+ None::<&Allocation<B>>,
+ batch_size as u32,
+ model_dim as u32,
+ epsilon,
+ scale_offset,
+ 1.0,
+ &mut encoder,
+ );
+ }
+
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
+ })
+ });
+ }
+ });
+}
diff --git a/crates/backend-uzu/tests/performance/kernel/sampling/argmax_bench.rs b/crates/backend-uzu/tests/performance/kernel/sampling/argmax_bench.rs
new file mode 100644
index 000000000..f6d9783fc
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/sampling/argmax_bench.rs
@@ -0,0 +1,111 @@
+use std::mem::{MaybeUninit, size_of};
+
+use backend_uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::common::{
+ AllocationType, Backend, Context, Encoder, Kernels,
+ gpu_types::ArgmaxPair,
+ kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel},
+ },
+};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use num_traits::{Float, NumCast};
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+use rand_distr::Normal;
+
+use crate::{common::type_short_name, uzu_bench};
+
+fn get_argmax_data<T: ArrayElement + Float>(
+ seed: u64,
+ batch_size: usize,
+ vocab_size: usize,
+) -> Box<[T]> {
+ let mut rng = SmallRng::seed_from_u64(seed);
+ let distribution = Normal::new(0.0, 32.0).unwrap();
+ let mut logits: Box<[MaybeUninit<T>]> = Box::new_uninit_slice(batch_size * vocab_size);
+ for logit in logits.iter_mut() {
+ logit.write(<T as NumCast>::from(rng.sample(distribution)).unwrap());
+ }
+ unsafe { logits.assume_init() }
+}
+
+#[uzu_bench]
+fn bench_argmax(c: &mut Criterion) {
+ type T = f32;
+ let batch_size = 1;
+
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+
+ let single_kernel =
+ <<B as Backend>::Kernels as Kernels>::ArgmaxSingleKernel::new(&context, T::data_type()).unwrap();
+ let twopass_kernel_main =
+ <<B as Backend>::Kernels as Kernels>::ArgmaxMainKernel::new(&context, T::data_type()).unwrap();
+ let twopass_kernel_final = <<B as Backend>::Kernels as Kernels>::ArgmaxFinalKernel::new(&context).unwrap();
+
+ let mut output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();
+
+ let mut group = c.benchmark_group(format!("{}/Kernel/Sampling/Argmax", type_short_name::<B>()));
+
+ for vocab_size in [
+ 65536, // LFM2-350M
+ 128256, // Llama-3.2-1B
+ 151936, // Qwen3-1.7B
+ 262144, // Gemma-3-1b
+ ] {
+ let logits_data = get_argmax_data::<T>(1337, batch_size, vocab_size);
+ let logits_buffer = context.create_array_from(&[logits_data.len()], logits_data.as_ref()).into_allocation();
+ let mut twopass_partial_results_buffer = context
+ .create_allocation(
+ batch_size * vocab_size.div_ceil(4096) * size_of::<ArgmaxPair>(),
+ AllocationType::Global,
+ )
+ .unwrap();
+
+ group.throughput(Throughput::Elements(vocab_size as u64));
+
+ group.bench_function(BenchmarkId::new("SinglePass", format!("Vocab[{vocab_size}]")), |b| {
+ b.iter_custom(|n_iters| {
+ let mut encoder = Encoder::<B>::new(&context).unwrap();
+
+ for _ in 0..n_iters {
+ single_kernel.encode(
+ &logits_buffer,
+ &mut output_buffer,
+ batch_size as u32,
+ vocab_size as u32,
+ &mut encoder,
+ );
+ }
+
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
+ })
+ });
+
+ group.bench_function(BenchmarkId::new("TwoPass", format!("Vocab[{vocab_size}]")), |b| {
+ b.iter_custom(|n_iters| {
+ let mut encoder = Encoder::<B>::new(&context).unwrap();
+
+ for _ in 0..n_iters {
+ twopass_kernel_main.encode(
+ &logits_buffer,
+ &mut twopass_partial_results_buffer,
+ batch_size as u32,
+ vocab_size as u32,
+ &mut encoder,
+ );
+ twopass_kernel_final.encode(
+ &twopass_partial_results_buffer,
+ &mut output_buffer,
+ batch_size as u32,
+ vocab_size as u32,
+ &mut encoder,
+ );
+ }
+
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
+ })
+ });
+ }
+ });
+}
diff --git a/crates/backend-uzu/tests/performance/kernel/sampling/mod.rs b/crates/backend-uzu/tests/performance/kernel/sampling/mod.rs
new file mode 100644
index 000000000..ed5b250a0
--- /dev/null
+++ b/crates/backend-uzu/tests/performance/kernel/sampling/mod.rs
@@ -0,0 +1 @@
+mod argmax_bench;
diff --git a/crates/backend-uzu/tests/performance/matmul/bench.rs b/crates/backend-uzu/tests/performance/matmul/bench.rs
deleted file mode 100644
index 980660d38..000000000
--- a/crates/backend-uzu/tests/performance/matmul/bench.rs
+++ /dev/null
@@ -1,148 +0,0 @@
-#![cfg(metal_backend)]
-
-use backend_uzu::{
- DataType,
- backends::{
- common::{
- Allocation, AllocationType, AsBufferRangeMut, Backend, Context, DenseBuffer, Encoder,
- kernel::{
- ManualKernels,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
- },
- },
- metal::Metal,
- },
-};
-
-use super::{error::BenchError, output::PerfResult, shapes::TestShape};
-
-type Ctx = <Metal as Backend>::Context;
-
-const WARMUP_ITERATIONS: usize = 3;
-const BENCHMARK_ITERATIONS: usize = 10;
-
-fn fill_buffer_random(
- allocation: &mut Allocation<Metal>,
- byte_count: usize,
-) {
- let buffer_range = allocation.as_buffer_range_mut();
- let range = buffer_range.range();
- let slice = unsafe {
- std::slice::from_raw_parts_mut(
- (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),
- byte_count,
- )
- };
- for (i, byte) in slice.iter_mut().enumerate() {
- *byte = (i % 251) as u8;
- }
-}
-
-fn encode_and_run(
- context: &Ctx,
- kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,
- shape: &TestShape,
- a_allocation: &backend_uzu::backends::common::Allocation<Metal>,
- b_allocation: &backend_uzu::backends::common::Allocation<Metal>,
- d_allocation: &mut backend_uzu::backends::common::Allocation<Metal>,
-) -> Result<f64, BenchError> {
- let mut encoder = Encoder::new(context).map_err(|_| BenchError::CommandBuffer)?;
-
- kernel.encode(
- MatmulArguments {
- a: a_allocation,
- a_offset: 0,
- b: b_allocation,
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
- d: d_allocation,
- batch_dim: shape.batch as u32,
- input_dim: shape.input_dim as u32,
- output_dim: shape.output_dim as u32,
- },
- &mut encoder,
- );
-
- let completed = encoder.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;
-
- Ok(completed.gpu_execution_time().as_secs_f64() * 1000.0)
-}
-
-fn run_benchmark(
- context: &Ctx,
- data_type: DataType,
- shape: &TestShape,
-) -> Result<f64, BenchError> {
- let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
- .map_err(|e| BenchError::Kernel(e.to_string()))?;
-
- let elem_size = data_type.size_in_bytes();
- let a_byte_count = shape.batch * shape.input_dim * elem_size;
- let b_byte_count = shape.output_dim * shape.input_dim * elem_size;
- let d_byte_count = shape.batch * shape.output_dim * elem_size;
-
- let mut a_allocation =
- context.create_allocation(a_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;
- let mut b_allocation =
- context.create_allocation(b_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;
- let mut d_allocation =
- context.create_allocation(d_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;
-
- fill_buffer_random(&mut a_allocation, a_byte_count);
- fill_buffer_random(&mut b_allocation, b_byte_count);
-
- for iteration in 0..WARMUP_ITERATIONS {
- encode_and_run(context, &mut kernel, shape, &a_allocation, &b_allocation, &mut d_allocation).map_err(
- |source| BenchError::Warmup {
- iteration,
- source: Box::new(source),
- },
- )?;
- }
-
- let mut gpu_time_total_ms = 0.0;
- for iteration in 0..BENCHMARK_ITERATIONS {
- let gpu_ms = encode_and_run(context, &mut kernel, shape, &a_allocation, &b_allocation, &mut d_allocation)
- .map_err(|source| BenchError::Benchmark {
- iteration,
- source: Box::new(source),
- })?;
- gpu_time_total_ms += gpu_ms;
- }
-
- Ok(gpu_time_total_ms / BENCHMARK_ITERATIONS as f64)
-}
-
-pub fn benchmark_single(
- context: &Ctx,
- data_type: DataType,
- shape: &TestShape,
-) -> PerfResult {
- match run_benchmark(context, data_type, shape) {
- Ok(duration_ms) => {
- let flops = 2.0 * shape.batch as f64 * shape.input_dim as f64 * shape.output_dim as f64;
- let gflops = flops / (duration_ms / 1000.0) / 1e9;
- PerfResult {
- combo: format!("{data_type:?}"),
- shape: format!("{shape}"),
- dispatch_path: "auto".to_owned(),
- duration_ms,
- gflops,
- status: "ok".into(),
- error: None,
- }
- },
- Err(error) => PerfResult {
- combo: format!("{data_type:?}"),
- shape: format!("{shape}"),
- dispatch_path: "auto".to_owned(),
- duration_ms: 0.0,
- gflops: 0.0,
- status: "error".into(),
- error: Some(error.to_string()),
- },
- }
-}
diff --git a/crates/backend-uzu/tests/performance/matmul/error.rs b/crates/backend-uzu/tests/performance/matmul/error.rs
deleted file mode 100644
index b06aa8fd7..000000000
--- a/crates/backend-uzu/tests/performance/matmul/error.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-use thiserror::Error;
-
-#[derive(Debug, Error)]
-pub enum BenchError {
- #[error("failed to create command buffer")]
- CommandBuffer,
- #[error("kernel creation failed: {0}")]
- Kernel(String),
- #[error("buffer allocation failed")]
- BufferAllocation,
- #[error("warmup iteration {iteration}: {source}")]
- Warmup {
- iteration: usize,
- source: Box<BenchError>,
- },
- #[error("benchmark iteration {iteration}: {source}")]
- Benchmark {
- iteration: usize,
- source: Box<BenchError>,
- },
-}
diff --git a/crates/backend-uzu/tests/performance/matmul/main.rs b/crates/backend-uzu/tests/performance/matmul/main.rs
deleted file mode 100644
index 22db6b189..000000000
--- a/crates/backend-uzu/tests/performance/matmul/main.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-#![cfg(metal_backend)]
-
-use backend_uzu::{
- DataType,
- backends::{
- common::{Backend, Context},
- metal::Metal,
- },
-};
-use indicatif::{ProgressBar, ProgressStyle};
-use metal::MTLDeviceExt;
-
-use crate::matmul::{bench, output::print_results_table, shapes::test_shapes};
-
-type Ctx = <Metal as Backend>::Context;
-
-fn write_json_results<T: serde::Serialize>(
- test_name: &str,
- device: &str,
- results: &[T],
-) {
- if let Ok(dir) = std::env::var("UZU_TEST_RESULTS_DIR") {
- let path = std::path::Path::new(&dir);
- std::fs::create_dir_all(path).expect("create results dir");
- let file = path.join(format!("{test_name}.json"));
- let wrapper = serde_json::json!({ "device": device, "results": results });
- let json = serde_json::to_string_pretty(&wrapper).expect("serialize");
- std::fs::write(&file, json).expect("write results");
- eprintln!("Results written to {}", file.display());
- }
-}
-
-#[test]
-#[ignore]
-fn matmul_perf() {
- let context = Ctx::new().expect("Metal context required");
-
- let data_types = [DataType::BF16, DataType::F16];
- let test_shapes = test_shapes();
-
- eprintln!("Matmul perf: {} dtypes x {} shapes", data_types.len(), test_shapes.len(),);
-
- let progress_bar = ProgressBar::new_spinner();
- progress_bar.set_style(
- ProgressStyle::with_template("{spinner} {pos} benchmarks [{elapsed_precise}] {msg}").expect("progress style"),
- );
-
- let mut results = Vec::new();
-
- for &data_type in &data_types {
- for shape in &test_shapes {
- progress_bar.set_message(format!("{data_type:?} {shape}"));
- let result = bench::benchmark_single(&context, data_type, shape);
- results.push(result);
- progress_bar.inc(1);
- }
- }
-
- progress_bar.finish_with_message("done");
- print_results_table(&results);
- write_json_results("matmul_perf", &context.device.name(), &results);
-
- let error_results: Vec<_> = results.iter().filter(|r| r.status != "ok").collect();
- if !error_results.is_empty() {
- eprintln!("{} / {} cases had errors", error_results.len(), results.len());
- }
-}
diff --git a/crates/backend-uzu/tests/performance/matmul/mod.rs b/crates/backend-uzu/tests/performance/matmul/mod.rs
deleted file mode 100644
index f79669fac..000000000
--- a/crates/backend-uzu/tests/performance/matmul/mod.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-mod main;
-
-mod bench;
-mod error;
-mod output;
-mod shapes;
diff --git a/crates/backend-uzu/tests/performance/matmul/output.rs b/crates/backend-uzu/tests/performance/matmul/output.rs
deleted file mode 100644
index cf2611518..000000000
--- a/crates/backend-uzu/tests/performance/matmul/output.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-use comfy_table::{CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL};
-use serde::Serialize;
-
-#[derive(Serialize)]
-pub struct PerfResult {
- pub combo: String,
- pub shape: String,
- pub dispatch_path: String,
- pub duration_ms: f64,
- pub gflops: f64,
- pub status: String,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error: Option<String>,
-}
-
-pub fn print_results_table(results: &[PerfResult]) {
- let mut table = Table::new();
- table
- .load_preset(UTF8_FULL)
- .apply_modifier(UTF8_ROUND_CORNERS)
- .set_content_arrangement(ContentArrangement::Dynamic)
- .set_header(vec!["Dtype", "Shape (MxKxN)", "Dispatch", "GFLOPS", "ms/iter", "Status"]);
-
- for r in results {
- let (gflops_str, ms_str, status_str) = if r.status == "ok" {
- (format!("{:.1}", r.gflops), format!("{:.3}", r.duration_ms), "ok".into())
- } else {
- ("-".into(), "-".into(), format!("ERR: {}", r.error.as_deref().unwrap_or("?")))
- };
- table.add_row(vec![r.combo.clone(), r.shape.clone(), r.dispatch_path.clone(), gflops_str, ms_str, status_str]);
- }
-
- for col in [3, 4] {
- if let Some(column) = table.column_mut(col) {
- column.set_cell_alignment(CellAlignment::Right);
- }
- }
-
- println!("{table}");
-}
diff --git a/crates/backend-uzu/tests/performance/matmul/shapes.rs b/crates/backend-uzu/tests/performance/matmul/shapes.rs
deleted file mode 100644
index d26cfebea..000000000
--- a/crates/backend-uzu/tests/performance/matmul/shapes.rs
+++ /dev/null
@@ -1,41 +0,0 @@
-pub struct TestShape {
- pub batch: usize,
- pub input_dim: usize,
- pub output_dim: usize,
-}
-
-impl std::fmt::Display for TestShape {
- fn fmt(
- &self,
- f: &mut std::fmt::Formatter<'_>,
- ) -> std::fmt::Result {
- write!(f, "{}x{}x{}", self.batch, self.input_dim, self.output_dim)
- }
-}
-
-const BATCH_SIZES: &[usize] = &[1, 4, 16, 64, 128, 256, 512];
-
-const MODEL_DIMS: &[(usize, usize)] = &[
- (896, 896),
- (896, 4864),
- (4864, 896),
- (2048, 2048),
- (2048, 8192),
- (8192, 2048),
- (4096, 4096),
- (4096, 14336),
- (14336, 4096),
-];
-
-pub fn test_shapes() -> Vec<TestShape> {
- MODEL_DIMS
- .iter()
- .flat_map(|&(input_dim, output_dim)| {
- BATCH_SIZES.iter().map(move |&batch| TestShape {
- batch,
- input_dim,
- output_dim,
- })
- })
- .collect()
-}
diff --git a/crates/backend-uzu/tests/performance/mod.rs b/crates/backend-uzu/tests/performance/mod.rs
index e41fdb139..a980f17d8 100644
--- a/crates/backend-uzu/tests/performance/mod.rs
+++ b/crates/backend-uzu/tests/performance/mod.rs
@@ -4,6 +4,4 @@ mod common;
mod audio_kernel_perf_test;
mod encodable_block;
-#[cfg(metal_backend)]
-mod matmul;
mod model_loading_perf_test;
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
index f8ad17b9c..d20d3ae98 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
@@ -346,8 +346,8 @@ fn execute_quantized_matmul<B: Backend>(
input_dim,
output_dim,
mode: match bits {
- 4 => QuantizationMode::UINT4,
- 8 => QuantizationMode::INT8,
+ 4 => QuantizationMode::U4,
+ 8 => QuantizationMode::I8,
_ => panic!("Unsupported bits: {}", bits),
},
quantization_method,
@@ -609,14 +609,14 @@ fn test_quant_matmul_perf_internal<B: Backend>() {
];
let configs = vec![
- // 4-bit Mlx BF16
+ // 4-bit scale-bias BF16
TestConfig {
quant_method: QuantizationMethod::ScaleBias,
bits: 4,
data_type: DataType::BF16,
group_size: 128,
},
- // 8-bit Mlx BF16
+ // 8-bit scale-bias BF16
TestConfig {
quant_method: QuantizationMethod::ScaleBias,
bits: 8,
diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
index 56ccc1434..38fd72538 100644
--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
@@ -97,7 +97,7 @@ fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
model_dim,
input_scale,
group_size,
- quant_mode: QuantizationMode::UINT8,
+ quant_mode: QuantizationMode::U8,
};
let expected = get_output::<T, Cpu>(&input);
@@ -111,7 +111,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
&context,
T::data_type(),
input.group_size,
- input.quant_mode.to_u32(),
+ input.quant_mode,
)
.expect("Failed to create QuantizedEmbeddingLookupKernel");
@@ -184,49 +184,49 @@ fn test_oob<T: ArrayElement + Float + Debug + Display>() {
// UINT4 tests
#[uzu_test]
fn test_uint4_f32() {
- test_quant_mode::<f32>(QuantizationMode::UINT4);
+ test_quant_mode::<f32>(QuantizationMode::U4);
}
#[uzu_test]
fn test_uint4_f16() {
- test_quant_mode::<f16>(QuantizationMode::UINT4);
+ test_quant_mode::<f16>(QuantizationMode::U4);
}
#[uzu_test]
fn test_uint4_bf16() {
- test_quant_mode::<bf16>(QuantizationMode::UINT4);
+ test_quant_mode::<bf16>(QuantizationMode::U4);
}
// INT8 tests
#[uzu_test]
fn test_int8_f32() {
- test_quant_mode::<f32>(QuantizationMode::INT8);
+ test_quant_mode::<f32>(QuantizationMode::I8);
}
#[uzu_test]
fn test_int8_f16() {
- test_quant_mode::<f16>(QuantizationMode::INT8);
+ test_quant_mode::<f16>(QuantizationMode::I8);
}
#[uzu_test]
fn test_int8_bf16() {
- test_quant_mode::<bf16>(QuantizationMode::INT8);
+ test_quant_mode::<bf16>(QuantizationMode::I8);
}
// UINT8 tests
#[uzu_test]
fn test_uint8_f32() {
- test_quant_mode::<f32>(QuantizationMode::UINT8);
+ test_quant_mode::<f32>(QuantizationMode::U8);
}
#[uzu_test]
fn test_uint8_f16() {
- test_quant_mode::<f16>(QuantizationMode::UINT8);
+ test_quant_mode::<f16>(QuantizationMode::U8);
}
#[uzu_test]
fn test_uint8_bf16() {
- test_quant_mode::<bf16>(QuantizationMode::UINT8);
+ test_quant_mode::<bf16>(QuantizationMode::U8);
}
// OOB tests
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
new file mode 100644
index 000000000..51a0e9085
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
@@ -0,0 +1,131 @@
+#![cfg(metal_backend)]
+
+use std::fmt::{Debug, Display};
+
+use backend_uzu::{
+ ArrayElement,
+ backends::{
+ common::{Backend, Context, kernel::ManualKernels, kernel::matmul::MatmulKernel},
+ metal::{Metal, MetalContext},
+ },
+};
+use proc_macros::__internal_uzu_test as uzu_test;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::common::{
+ assert::assert_eq_float,
+ matmul::{Case, Variant, all_correctness_shapes, cpu_reference, deterministic_input, run_metal},
+};
+
+fn check_case<T: ArrayElement + Float + Debug + Display>(
+ context: &MetalContext,
+ kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,
+ variant: Variant,
+ case: Case,
+ tolerance: f32,
+) {
+ let input = deterministic_input::<T>(case);
+ let expected = cpu_reference::<T>(&input);
+ let actual = run_metal::<T>(context, kernel, &input, variant);
+ assert_eq_float(
+ &expected,
+ &actual,
+ tolerance,
+ &format!("{:?} dtype={} {:?}", variant, std::any::type_name::<T>(), case),
+ );
+}
+
+fn check_all_shapes<T: ArrayElement + Float + Debug + Display>(
+ variant: Variant,
+ ab_scale: f32,
+ accumulate: bool,
+ tolerance: f32,
+) {
+ let context = MetalContext::new().expect("Metal context");
+ if !variant.supported(&context) {
+ eprintln!("Skipping {variant:?}: device does not support MXU");
+ return;
+ }
+ let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
+ .expect("MatmulKernel");
+ for shape in all_correctness_shapes() {
+ let case = Case::new(shape).with_ab_scale(ab_scale).with_accumulate(accumulate);
+ check_case::<T>(&context, &mut kernel, variant, case, tolerance);
+ }
+}
+
+#[uzu_test]
+fn matches_cpu_reference_bf16_gemm() {
+ check_all_shapes::<bf16>(Variant::Gemm, 1.0, false, 1.0);
+}
+
+#[uzu_test]
+fn matches_cpu_reference_bf16_gemm_mxu() {
+ check_all_shapes::<bf16>(Variant::GemmMxu, 1.0, false, 1.0);
+}
+
+#[uzu_test]
+fn matches_cpu_reference_f16_gemm() {
+ check_all_shapes::<f16>(Variant::Gemm, 1.0, false, 0.5);
+}
+
+#[uzu_test]
+fn matches_cpu_reference_f16_gemm_mxu() {
+ check_all_shapes::<f16>(Variant::GemmMxu, 1.0, false, 0.5);
+}
+
+#[uzu_test]
+fn matches_cpu_reference_f32_gemm() {
+ check_all_shapes::<f32>(Variant::Gemm, 1.0, false, 0.05);
+}
+
+#[uzu_test]
+fn ab_scale_bf16_gemm() {
+ check_all_shapes::<bf16>(Variant::Gemm, 0.5, false, 1.0);
+}
+
+#[uzu_test]
+fn ab_scale_bf16_gemm_mxu() {
+ check_all_shapes::<bf16>(Variant::GemmMxu, 0.5, false, 1.0);
+}
+
+#[uzu_test]
+fn accumulate_bf16_gemm_mxu() {
+ check_all_shapes::<bf16>(Variant::GemmMxu, 1.0, true, 1.0);
+}
+
+#[uzu_test]
+fn scale_and_accumulate_bf16_gemm_mxu() {
+ check_all_shapes::<bf16>(Variant::GemmMxu, 0.5, true, 1.0);
+}
+
+fn check_b_transpose_false<T: ArrayElement + Float + Debug + Display>(
+ variant: Variant,
+ tolerance: f32,
+) {
+ let context = MetalContext::new().expect("Metal context");
+ if !variant.supported(&context) {
+ eprintln!("Skipping {variant:?}: device does not support MXU");
+ return;
+ }
+ let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
+ .expect("MatmulKernel");
+ for shape in all_correctness_shapes() {
+ let case = Case {
+ b_transpose: false,
+ ..Case::new(shape)
+ };
+ check_case::<T>(&context, &mut kernel, variant, case, tolerance);
+ }
+}
+
+#[uzu_test]
+fn b_transpose_false_bf16_gemm() {
+ check_b_transpose_false::<bf16>(Variant::Gemm, 1.0);
+}
+
+#[uzu_test]
+fn b_transpose_false_bf16_gemm_mxu() {
+ check_b_transpose_false::<bf16>(Variant::GemmMxu, 1.0);
+}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
deleted file mode 100644
index 00204a6bd..000000000
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
+++ /dev/null
@@ -1,336 +0,0 @@
-#![cfg(metal_backend)]
-
-use std::{
- rc::Rc,
-};
-
-use criterion::{BenchmarkId, Criterion, Throughput};
-use half::bf16;
-use backend_uzu::{
- ArrayContextExt, ArrayElement,
- backends::{
- common::{
- AllocationType, Backend, Context, Encoder,
- kernel::{
- ManualKernels,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
- },
- },
- cpu::Cpu,
- metal::{DeviceExt, MatmulDispatchPath, Metal, MetalContext},
- },
-};
-
-use crate::{
- common::{
- assert::assert_eq_float,
- helpers::{alloc_allocation_with_data, allocation_to_vec},
- type_short_name,
- },
- uzu_bench, uzu_test,
-};
-
-struct Input {
- left: Box<[bf16]>,
- right: Box<[bf16]>,
- destination_prefill: Option<Box<[bf16]>>,
- batch_dim: usize,
- input_dim: usize,
- output_dim: usize,
- ab_scale: f32,
-}
-
-fn get_test_data(
- batch_dim: usize,
- input_dim: usize,
- output_dim: usize,
- ab_scale: f32,
- accumulate: bool,
-) -> Input {
- let left: Vec<bf16> = (0..batch_dim * input_dim).map(|i| bf16::from_f32(((i % 13) as f32) * 0.1 - 0.6)).collect();
- let right: Vec<bf16> = (0..output_dim * input_dim).map(|i| bf16::from_f32(((i % 17) as f32) * 0.1 - 0.8)).collect();
- let destination_prefill = if accumulate {
- Some((0..batch_dim * output_dim).map(|i| bf16::from_f32(((i % 7) as f32) * 0.03 - 0.09)).collect())
- } else {
- None
- };
-
- Input {
- left: left.into_boxed_slice(),
- right: right.into_boxed_slice(),
- destination_prefill,
- batch_dim,
- input_dim,
- output_dim,
- ab_scale,
- }
-}
-
-fn get_cpu_output(input: &Input) -> Vec<bf16> {
- let cpu_context = <Cpu as Backend>::Context::new().expect("CPU context");
-
- let right_array = cpu_context.create_array_from(&[input.output_dim, input.input_dim], &input.right);
- let left_allocation = alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &input.left);
- let mut destination_allocation = if let Some(ref prefill) = input.destination_prefill {
- alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, prefill)
- } else {
- cpu_context
- .create_allocation(
- input.batch_dim * input.output_dim * std::mem::size_of::<bf16>(),
- AllocationType::Global,
- )
- .expect("Failed to create allocation")
- };
-
- let argument_c = if input.destination_prefill.is_some() {
- MatmulArgumentC::Accumulate
- } else {
- MatmulArgumentC::None
- };
-
- let mut matmul_kernel =
- <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&cpu_context, bf16::data_type())
- .expect("CPU MatmulKernel");
-
- let mut encoder = Encoder::new(cpu_context.as_ref()).expect("encoder");
- matmul_kernel.encode(
- MatmulArguments {
- a: &left_allocation,
- a_offset: 0,
- b: right_array.allocation(),
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: input.ab_scale,
- c: argument_c,
- d: &mut destination_allocation,
- batch_dim: input.batch_dim as u32,
- input_dim: input.input_dim as u32,
- output_dim: input.output_dim as u32,
- },
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
- allocation_to_vec(&destination_allocation)
-}
-
-fn get_mpp_output(
- metal_context: &MetalContext,
- matmul_kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,
- input: &Input,
-) -> Vec<bf16> {
- let right_array = metal_context.create_array_from(&[input.output_dim, input.input_dim], &input.right);
- let left_allocation = alloc_allocation_with_data::<Metal, bf16>(metal_context, &input.left);
- let mut destination_allocation = if let Some(ref prefill) = input.destination_prefill {
- alloc_allocation_with_data::<Metal, bf16>(metal_context, prefill)
- } else {
- metal_context
- .create_allocation(
- input.batch_dim * input.output_dim * std::mem::size_of::<bf16>(),
- AllocationType::Global,
- )
- .expect("Failed to create allocation")
- };
-
- let argument_c = if input.destination_prefill.is_some() {
- MatmulArgumentC::Accumulate
- } else {
- MatmulArgumentC::None
- };
-
- let mut encoder = Encoder::new(metal_context).expect("encoder");
- matmul_kernel.encode_with_path(
- MatmulArguments {
- a: &left_allocation,
- a_offset: 0,
- b: right_array.allocation(),
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: input.ab_scale,
- c: argument_c,
- d: &mut destination_allocation,
- batch_dim: input.batch_dim as u32,
- input_dim: input.input_dim as u32,
- output_dim: input.output_dim as u32,
- },
- &mut encoder,
- MatmulDispatchPath::GemmMpp,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
- allocation_to_vec(&destination_allocation)
-}
-
-fn create_metal_context_and_matmul_kernel()
--> (Rc<MetalContext>, <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel) {
- let metal_context = MetalContext::new().expect("Metal context");
- let matmul_kernel =
- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())
- .expect("MatmulKernel");
- (metal_context, matmul_kernel)
-}
-
-fn test_mpp_correctness(
- batch_dim: usize,
- input_dim: usize,
- output_dim: usize,
- ab_scale: f32,
- accumulate: bool,
- tolerance: f32,
-) {
- let (metal_context, mut matmul_kernel) = create_metal_context_and_matmul_kernel();
- if !metal_context.device.supports_mxu() {
- eprintln!("Skipping MPP test: device does not support MXU");
- return;
- }
-
- let input = get_test_data(batch_dim, input_dim, output_dim, ab_scale, accumulate);
- let expected = get_cpu_output(&input);
- let actual = get_mpp_output(&metal_context, &mut matmul_kernel, &input);
- assert_eq_float(&expected, &actual, tolerance, "GEMM_MPP vs CPU reference");
-}
-
-// Basic correctness — explicitly dispatched to MPP kernel
-
-#[uzu_test]
-fn test_bf16_aligned() {
- test_mpp_correctness(64, 2048, 2048, 1.0, false, 1.0);
-}
-
-#[uzu_test]
-fn test_bf16_unaligned_m() {
- test_mpp_correctness(33, 2048, 2048, 1.0, false, 1.0);
-}
-
-#[uzu_test]
-fn test_bf16_unaligned_n() {
- test_mpp_correctness(64, 2048, 33, 1.0, false, 1.0);
-}
-
-#[uzu_test]
-fn test_bf16_large() {
- test_mpp_correctness(128, 4096, 14336, 1.0, false, 2.0);
-}
-
-#[uzu_test]
-fn test_bf16_small_batch() {
- test_mpp_correctness(1, 2048, 2048, 1.0, false, 1.0);
-}
-
-// ab_scale
-
-#[uzu_test]
-fn test_bf16_ab_scale() {
- test_mpp_correctness(64, 2048, 2048, 0.5, false, 1.0);
-}
-
-// accumulate
-
-#[uzu_test]
-fn test_bf16_accumulate() {
- test_mpp_correctness(64, 2048, 2048, 1.0, true, 1.0);
-}
-
-// scale + accumulate combined
-
-#[uzu_test]
-fn test_bf16_scale_and_accumulate() {
- test_mpp_correctness(64, 2048, 2048, 0.5, true, 1.0);
-}
-
-// 128x128 tile (M >= 128, N >= 128)
-
-#[uzu_test]
-fn test_bf16_128x128_aligned() {
- test_mpp_correctness(128, 2048, 2048, 1.0, false, 1.0);
-}
-
-#[uzu_test]
-fn test_bf16_128x128_unaligned_m() {
- test_mpp_correctness(200, 2048, 2048, 1.0, false, 1.0);
-}
-
-#[uzu_test]
-fn test_bf16_128x128_unaligned_n() {
- test_mpp_correctness(128, 2048, 200, 1.0, false, 1.0);
-}
-
-#[uzu_test]
-fn test_bf16_128x128_large() {
- test_mpp_correctness(256, 4096, 4096, 1.0, false, 2.0);
-}
-
-#[uzu_test]
-fn test_bf16_128x128_accumulate() {
- test_mpp_correctness(128, 2048, 2048, 1.0, true, 1.0);
-}
-
-#[uzu_test]
-fn test_bf16_128x128_scale_and_accumulate() {
- test_mpp_correctness(256, 2048, 2048, 0.5, true, 1.0);
-}
-
-// Benchmarks
-
-const BENCHMARK_SHAPES: &[(usize, usize, usize)] =
- &[(128, 2048, 8192), (128, 4096, 14336), (256, 4096, 4096), (512, 8192, 2048)];
-
-#[uzu_bench]
-fn bench_gemm_mpp(criterion: &mut Criterion) {
- let metal_context = MetalContext::new().unwrap();
- if !metal_context.device.supports_mxu() {
- return;
- }
-
- let mut matmul_kernel =
- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())
- .expect("MatmulKernel");
-
- let mut benchmark_group =
- criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()));
-
- for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {
- let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type());
- let left_allocation = metal_context
- .create_allocation(batch_dim * input_dim * std::mem::size_of::<bf16>(), AllocationType::Global)
- .expect("Failed to create allocation");
- let mut destination_allocation = metal_context
- .create_allocation(batch_dim * output_dim * std::mem::size_of::<bf16>(), AllocationType::Global)
- .expect("Failed to create allocation");
-
- let floating_point_operations = 2 * batch_dim * input_dim * output_dim;
- benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));
-
- benchmark_group.bench_function(
- BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),
- |bencher| {
- bencher.iter_custom(|iteration_count| {
- let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();
-
- for _ in 0..iteration_count {
- matmul_kernel.encode_with_path(
- MatmulArguments {
- a: &left_allocation,
- a_offset: 0,
- b: right_array.allocation(),
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
- d: &mut destination_allocation,
- batch_dim: batch_dim as u32,
- input_dim: input_dim as u32,
- output_dim: output_dim as u32,
- },
- &mut encoder,
- MatmulDispatchPath::GemmMpp,
- );
- }
-
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
- })
- },
- );
- }
-}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
deleted file mode 100644
index 3bd5359b6..000000000
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
+++ /dev/null
@@ -1,248 +0,0 @@
-use std::{
- fmt::{Debug, Display},
-};
-
-use backend_uzu::{
- ArrayContextExt, ArrayElement,
- backends::{
- common::{
- AllocationType, Backend, Context, Encoder,
- kernel::{
- ManualKernels,
- matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
- },
- },
- cpu::Cpu,
- },
-};
-#[cfg(metal_backend)]
-use backend_uzu::backends::metal::{MatmulDispatchPath, Metal, MetalContext};
-#[cfg(metal_backend)]
-use criterion::{BenchmarkId, Criterion, Throughput};
-use half::{bf16, f16};
-use num_traits::Float;
-
-#[cfg(metal_backend)]
-use crate::{common::type_short_name, uzu_bench};
-use crate::{
- common::{
- assert::assert_eq_float,
- helpers::{alloc_allocation_with_data, allocation_to_vec},
- },
- uzu_test,
-};
-
-struct Input<T: ArrayElement + Float> {
- a: Box<[T]>,
- b: Box<[T]>,
- m: usize,
- k: usize,
- n: usize,
-}
-
-fn get_test_data<T: ArrayElement + Float>(
- m: usize,
- k: usize,
- n: usize,
-) -> (Input<T>, Vec<T>) {
- let a: Vec<T> = (0..m * k).map(|i| T::from(((i % 13) as f32) * 0.1 - 0.6).unwrap()).collect();
- let b: Vec<T> = (0..n * k).map(|i| T::from(((i % 17) as f32) * 0.1 - 0.8).unwrap()).collect();
-
- let input = Input {
- a: a.into_boxed_slice(),
- b: b.into_boxed_slice(),
- m,
- k,
- n,
- };
-
- let expected = get_output::<T, Cpu>(&input, 1.0);
- (input, expected)
-}
-
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>, ab_scale: f32) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
-
- let m = input.m as u32;
- let k = input.k as u32;
- let n = input.n as u32;
-
- let b_array = context.create_array_from(&[input.n, input.k], &input.b);
- let a_allocation = alloc_allocation_with_data::<B, T>(&context, &input.a);
- let mut d_allocation = context
- .create_allocation(input.m * input.n * std::mem::size_of::<T>(), AllocationType::Global)
- .expect("Failed to create allocation");
-
- let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
- .expect("Failed to create MatmulKernel");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- MatmulArguments {
- a: &a_allocation,
- a_offset: 0,
- b: b_array.allocation(),
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale,
- c: MatmulArgumentC::None,
- d: &mut d_allocation,
- batch_dim: m,
- input_dim: k,
- output_dim: n,
- },
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
- allocation_to_vec::<B, T>(&d_allocation)
-}
-
-fn test<T: ArrayElement + Float + Debug + Display>(
- m: usize,
- k: usize,
- n: usize,
- eps: f32,
-) {
- let (input, expected) = get_test_data::<T>(m, k, n);
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input, 1.0);
- assert_eq_float(&expected, &output, eps, &format!("backend {}", std::any::type_name::<B>()));
- });
-}
-
-fn test_with_scale<T: ArrayElement + Float + Debug + Display>(
- m: usize,
- k: usize,
- n: usize,
- ab_scale: f32,
- eps: f32,
-) {
- let (input, _) = get_test_data::<T>(m, k, n);
- let expected = get_output::<T, Cpu>(&input, ab_scale);
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input, ab_scale);
- assert_eq_float(&expected, &output, eps, &format!("backend {} ab_scale={ab_scale}", std::any::type_name::<B>()));
- });
-}
-
-#[uzu_test]
-fn test_f32_aligned() {
- test::<f32>(64, 64, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_aligned() {
- test::<f16>(64, 64, 64, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_aligned() {
- test::<bf16>(64, 64, 64, 0.1);
-}
-
-#[uzu_test]
-fn test_f32_unaligned() {
- test::<f32>(7, 33, 11, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_unaligned() {
- test::<f16>(7, 33, 11, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_unaligned() {
- test::<bf16>(7, 33, 11, 0.1);
-}
-
-#[uzu_test]
-fn test_f32_large() {
- test::<f32>(16, 128, 256, 0.01);
-}
-
-#[uzu_test]
-fn test_f16_large() {
- test::<f16>(16, 128, 256, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_large() {
- test::<bf16>(16, 128, 256, 0.1);
-}
-
-// ab_scale tests
-
-#[uzu_test]
-fn test_f32_ab_scale() {
- test_with_scale::<f32>(16, 128, 256, 0.5, 0.01);
-}
-
-#[uzu_test]
-fn test_bf16_ab_scale() {
- test_with_scale::<bf16>(16, 128, 256, 0.5, 0.1);
-}
-
-// Benchmarks (classic ALU GEMM path, Metal only)
-
-#[cfg(metal_backend)]
-const BENCHMARK_SHAPES: &[(usize, usize, usize)] =
- &[(128, 2048, 8192), (128, 4096, 14336), (256, 4096, 4096), (512, 8192, 2048)];
-
-#[cfg(metal_backend)]
-#[uzu_bench]
-fn bench_gemm(criterion: &mut Criterion) {
- let metal_context = MetalContext::new().unwrap();
-
- let mut matmul_kernel =
- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())
- .expect("MatmulKernel");
-
- let mut benchmark_group =
- criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));
-
- for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {
- let left_allocation = metal_context
- .create_allocation(batch_dim * input_dim * std::mem::size_of::<bf16>(), AllocationType::Global)
- .expect("Failed to create allocation");
- let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type());
- let mut destination_allocation = metal_context
- .create_allocation(batch_dim * output_dim * std::mem::size_of::<bf16>(), AllocationType::Global)
- .expect("Failed to create allocation");
-
- let floating_point_operations = 2 * batch_dim * input_dim * output_dim;
- benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));
-
- benchmark_group.bench_function(
- BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),
- |bencher| {
- bencher.iter_custom(|iteration_count| {
- let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();
-
- for _ in 0..iteration_count {
- matmul_kernel.encode_with_path(
- MatmulArguments {
- a: &left_allocation,
- a_offset: 0,
- b: right_array.allocation(),
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- ab_scale: 1.0,
- c: MatmulArgumentC::None,
- d: &mut destination_allocation,
- batch_dim: batch_dim as u32,
- input_dim: input_dim as u32,
- output_dim: output_dim as u32,
- },
- &mut encoder,
- MatmulDispatchPath::Gemm,
- );
- }
-
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
- })
- },
- );
- }
-}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs b/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs
index e7089955d..c84f3ea46 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/mod.rs
@@ -1,3 +1,3 @@
-mod gemm_mpp_test;
-mod gemm_test;
+#[cfg(metal_backend)]
+mod dispatch_paths_test;
mod gemv_test;
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
index c11f4257e..e42308dc0 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
@@ -4,26 +4,20 @@ use backend_uzu::{
ArrayElement, DataType,
backends::{
common::{
- Allocation, Backend, Context, Encoder, Kernels,
+ Backend, Context, Encoder, Kernels,
gpu_types::QuantizationMethod,
kernel::QuantizedMatmulQmmTransposedKernel,
},
cpu::Cpu,
},
};
-use criterion::{BenchmarkId, Criterion, Throughput};
use half::{bf16, f16};
-use itertools::iproduct;
use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::SmallRng};
use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
use crate::{
- common::{
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
- type_short_name,
- },
- uzu_bench, uzu_test,
+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ uzu_test,
};
fn make_input_basic<T: ArrayElement + Float>(
@@ -645,116 +639,3 @@ mod hadamard {
);
}
-fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(
- c: &mut Criterion,
- context: &B::Context,
- label: &str,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- // Use the dispatcher-selected tile configs: BM=8 small tile for M<48, big tile for M>=48.
- // We always use the BM=8 small tile kernel for M in 4..=47 to match the dispatch threshold.
- let block_size: usize = if bits == 4 { 512 } else { 256 };
-
- for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336]) {
- if n % 32 != 0 || k % block_size != 0 {
- continue;
- }
-
- // Pick tile based on M: small (BM=8) for M<48, big (BM=32) for M>=48
- let (bm, bk, bn, wm, wn) = if m < 48 {
- (8u32, 32u32, 32u32, 1u32, 1u32)
- } else {
- (32u32, 32u32, 32u32, 2u32, 2u32)
- };
-
- let num_groups = k.div_ceil(group_size as usize);
- let mut rng = SmallRng::seed_from_u64(42);
-
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- context,
- T::data_type(),
- group_size,
- bits,
- bm,
- bk,
- bn,
- wm,
- wn,
- quant_method,
- false,
- n % bn as usize == 0,
- )
- .unwrap();
-
- let w_buf = alloc_allocation_with_data::<B, u32>(
- context,
- &(0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect::<Vec<_>>(),
- );
- let scales_buf = alloc_allocation_with_data::<B, T>(
- context,
- &(0..n * num_groups).map(|_| T::from(rng.random_range(0.01f32..1.0f32)).unwrap()).collect::<Vec<_>>(),
- );
- let x_buf = alloc_allocation_with_data::<B, T>(
- context,
- &(0..m * k).map(|_| T::from(rng.random_range(-1.0f32..1.0f32)).unwrap()).collect::<Vec<_>>(),
- );
- let mut y_buf = alloc_allocation::<B, T>(context, m * n);
-
- let zp_stride = if bits == 4 {
- (num_groups + 1) / 2
- } else {
- num_groups
- };
- let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {
- alloc_allocation_with_data::<B, u8>(
- context,
- &(0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect::<Vec<_>>(),
- )
- });
- let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {
- alloc_allocation_with_data::<B, T>(
- context,
- &(0..n * num_groups).map(|_| T::from(rng.random_range(-0.5f32..0.5f32)).unwrap()).collect::<Vec<_>>(),
- )
- });
-
- let mut group = c.benchmark_group(format!("{}/Kernel/QmmTransposed/{}", type_short_name::<B>(), label));
- group.throughput(Throughput::Elements((m * n * k) as u64));
- group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {
- b.iter_custom(|n_iters| {
- let mut encoder = Encoder::<B>::new(context).unwrap();
- for _ in 0..n_iters {
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- None::<&Allocation<B>>,
- k as u32,
- n as u32,
- m as u32,
- &mut encoder,
- );
- }
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
- })
- });
- drop(group);
- }
-}
-
-#[uzu_bench]
-fn bench_qmm_transposed(c: &mut Criterion) {
- for_each_backend!(|B| {
- let context = <B as Backend>::Context::new().unwrap();
- bench_qmm_transposed_typed::<B, bf16>(c, &context, "Mlx_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmm_transposed_typed::<B, bf16>(c, &context, "Mlx_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
- bench_qmm_transposed_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmm_transposed_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- });
-}
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index e3015950b..24e497835 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -11,18 +11,12 @@ use backend_uzu::{
cpu::Cpu,
},
};
-use criterion::{BenchmarkId, Criterion, Throughput};
-use half::{bf16, f16};
use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::SmallRng};
use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
use crate::{
- common::{
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
- type_short_name,
- },
- uzu_bench, uzu_test,
+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ uzu_test,
};
fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
@@ -295,124 +289,3 @@ qmv_fast_test!(test_gs128_4bit_mlx, gs = 128, bits = 4, format = QuantizationMet
qmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, format = QuantizationMethod::ScaleBias);
-
-fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(
- rng: &mut R,
- range: std::ops::Range<T>,
- len: usize,
-) -> Box<[T]> {
- (0..len).map(|_| rng.random_range(range.clone())).collect()
-}
-
-fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
- c: &mut Criterion,
- context: &B::Context,
- label: &str,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));
- let block_size: usize = if bits == 4 {
- 512
- } else {
- 256
- };
-
- let nk_pairs: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
- for &(n, k) in nk_pairs {
- if n % 8 != 0 || k % block_size != 0 {
- continue;
- }
- for m in [1usize, 2, 4, 8] {
- let num_groups = k.div_ceil(group_size as usize);
- let mut rng = SmallRng::seed_from_u64(42);
-
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
- context,
- T::data_type(),
- group_size,
- bits,
- quant_method,
- false,
- )
- .unwrap();
-
- let w_buf = alloc_allocation_with_data::<B, u32>(
- context,
- &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),
- );
- let scales_buf = alloc_allocation_with_data::<B, T>(
- context,
- &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)
- .iter()
- .map(|&v| T::from(v).unwrap())
- .collect::<Vec<_>>(),
- );
- let x_buf = alloc_allocation_with_data::<B, T>(
- context,
- &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k)
- .iter()
- .map(|&v| T::from(v).unwrap())
- .collect::<Vec<_>>(),
- );
- let mut y_buf = alloc_allocation::<B, T>(context, m * n);
-
- let zp_buf = (quant_method == QuantizationMethod::ScaleZeroPoint).then(|| {
- let zp_stride = if bits == 4 {
- (num_groups + 1) / 2
- } else {
- num_groups
- };
- alloc_allocation_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))
- });
- let bias_buf = (quant_method == QuantizationMethod::ScaleBias).then(|| {
- alloc_allocation_with_data::<B, T>(
- context,
- &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)
- .iter()
- .map(|&v| T::from(v).unwrap())
- .collect::<Vec<_>>(),
- )
- });
-
- group.throughput(Throughput::Elements((m * n * k) as u64));
- group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {
- b.iter_custom(|n_iters| {
- let mut encoder = Encoder::<B>::new(context).unwrap();
- for _ in 0..n_iters {
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- None::<&Allocation<B>>,
- k as u32,
- n as u32,
- m as u32,
- &mut encoder,
- );
- }
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
- })
- });
- }
- }
-}
-
-#[uzu_bench]
-fn bench_qmv_fast(c: &mut Criterion) {
- for_each_backend!(|B| {
- let context = <B as Backend>::Context::new().unwrap();
- bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);
- });
-}
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
index 07700d020..e74b751d4 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
@@ -1,6 +1,5 @@
use std::mem::{MaybeUninit, size_of};
-use criterion::{BenchmarkId, Criterion, Throughput};
use num_traits::{Float, NumCast};
use proptest::prelude::*;
use rand::{RngExt, SeedableRng, rngs::SmallRng};
@@ -18,9 +17,8 @@ use crate::{
common::{
dispatch_dtype, for_each_context,
proptest::{ComparableTestResults, TestContextes, kernel_data_type},
- type_short_name,
},
- uzu_bench, uzu_test,
+ uzu_test,
};
struct ArgmaxTestResults(Box<[u32]>);
@@ -116,83 +114,3 @@ fn test_argmax_prop() {
});
}
-#[uzu_bench]
-fn bench_argmax(c: &mut Criterion) {
- type T = f32;
- let batch_size = 1;
-
- for_each_backend!(|B| {
- let context = <B as Backend>::Context::new().unwrap();
-
- let single_kernel =
- <<B as Backend>::Kernels as Kernels>::ArgmaxSingleKernel::new(&context, T::data_type()).unwrap();
- let twopass_kernel_main =
- <<B as Backend>::Kernels as Kernels>::ArgmaxMainKernel::new(&context, T::data_type()).unwrap();
- let twopass_kernel_final = <<B as Backend>::Kernels as Kernels>::ArgmaxFinalKernel::new(&context).unwrap();
-
- let mut output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();
-
- let mut group = c.benchmark_group(format!("{}/Kernel/Sampling/Argmax", type_short_name::<B>()));
-
- for vocab_size in [
- 65536, // LFM2-350M
- 128256, // Llama-3.2-1B
- 151936, // Qwen3-1.7B
- 262144, // Gemma-3-1b
- ] {
- let logits_data = get_argmax_data::<T>(1337, batch_size, vocab_size);
- let logits_buffer = context.create_array_from(&[logits_data.len()], logits_data.as_ref()).into_allocation();
- let mut twopass_partial_results_buffer = context
- .create_allocation(
- batch_size * vocab_size.div_ceil(4096) * size_of::<ArgmaxPair>(),
- AllocationType::Global,
- )
- .unwrap();
-
- group.throughput(Throughput::Elements(vocab_size as u64));
-
- group.bench_function(BenchmarkId::new("SinglePass", format!("Vocab[{vocab_size}]")), |b| {
- b.iter_custom(|n_iters| {
- let mut encoder = Encoder::<B>::new(&context).unwrap();
-
- for _ in 0..n_iters {
- single_kernel.encode(
- &logits_buffer,
- &mut output_buffer,
- batch_size as u32,
- vocab_size as u32,
- &mut encoder,
- );
- }
-
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
- })
- });
-
- group.bench_function(BenchmarkId::new("TwoPass", format!("Vocab[{vocab_size}]")), |b| {
- b.iter_custom(|n_iters| {
- let mut encoder = Encoder::<B>::new(&context).unwrap();
-
- for _ in 0..n_iters {
- twopass_kernel_main.encode(
- &logits_buffer,
- &mut twopass_partial_results_buffer,
- batch_size as u32,
- vocab_size as u32,
- &mut encoder,
- );
- twopass_kernel_final.encode(
- &twopass_partial_results_buffer,
- &mut output_buffer,
- batch_size as u32,
- vocab_size as u32,
- &mut encoder,
- );
- }
-
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
- })
- });
- }
- });
-}
diff --git a/crates/benchmarks/src/runner/types/result.rs b/crates/benchmarks/src/runner/types/result.rs
index 417702c7c..7b3980ff2 100644
--- a/crates/benchmarks/src/runner/types/result.rs
+++ b/crates/benchmarks/src/runner/types/result.rs
@@ -1,4 +1,4 @@
-use backend_uzu::ConfigDataType;
+use backend_uzu::DataType;
use serde::{Deserialize, Serialize};
use crate::runner::types::{Device, Task};
@@ -9,7 +9,7 @@ pub struct Result {
pub device: Device,
pub engine_version: String,
pub timestamp: u64,
- pub precision: ConfigDataType,
+ pub precision: DataType,
pub memory_used: Option<u64>,
pub tokens_count_input: u64,
pub tokens_count_output: u64,
diff --git a/crates/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs b/crates/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs
index 0ff2320ff..1232d9ee1 100644
--- a/crates/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs
+++ b/crates/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs
@@ -56,9 +56,11 @@ async fn test_storage_mock_registry_model_download_lifecycle(
tracing::info!("pausing storage item");
item.pause().await?;
item.reconcile().await?;
- let paused_state = item.state().await;
+ let paused_state = wait_for_item_state(&item, &mut progress, "pause transitions to Paused", |state| {
+ matches!(state.phase, DownloadPhase::Paused {})
+ })
+ .await;
tracing::info!(?paused_state, "observed paused state");
- assert!(matches!(paused_state.phase, DownloadPhase::Paused {}), "pause must transition to Paused; got {:?}", paused_state.phase);
tracing::info!("resuming storage item");
item.download().await?;