Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Constants (#629)

eugene8a9986ebd654
diff --git a/crates/backend-uzu/build/common/enum_paths.rs b/crates/backend-uzu/build/common/enum_paths.rs
index 991991808..4dc54a9f2 100644
--- a/crates/backend-uzu/build/common/enum_paths.rs
+++ b/crates/backend-uzu/build/common/enum_paths.rs
@@ -31,7 +31,7 @@ impl EnumPaths {
let (name_str, kind) = match ty {
GpuType::Enum(enum_type) => (enum_type.name.as_ref(), GpuTypeKind::Enum),
GpuType::OptionSet(option_set) => (option_set.name.as_ref(), GpuTypeKind::OptionSet),
- GpuType::Struct(_) => continue,
+ GpuType::Constant(_) | GpuType::Struct(_) => continue,
};
let name = GpuTypeName::from(name_str);
let path =
diff --git a/crates/backend-uzu/build/common/gpu_types/item_constant.rs b/crates/backend-uzu/build/common/gpu_types/item_constant.rs
new file mode 100644
index 000000000..10a1731ce
--- /dev/null
+++ b/crates/backend-uzu/build/common/gpu_types/item_constant.rs
@@ -0,0 +1,120 @@
+use anyhow::bail;
+use quote::ToTokens;
+use syn::{
+ BinOp, Expr, ExprBinary, ExprGroup, ExprLit, ExprParen, ExprPath, ExprUnary, ItemConst, Lit, Type, TypePath, UnOp,
+};
+
+#[derive(Debug)]
+pub struct GpuTypeConstant {
+ pub name: Box<str>,
+ pub ty: Box<str>,
+ pub value_expression: Box<str>,
+}
+
+impl GpuTypeConstant {
+ pub fn parse(item: ItemConst) -> anyhow::Result<Self> {
+ let Type::Path(TypePath {
+ qself: None,
+ path,
+ }) = *item.ty
+ else {
+ bail!("GPU constant type must be a path, found {:?}", item.ty);
+ };
+
+ let is_bool = path.is_ident("bool");
+ let value_expression = render_const_expr(&item.expr, is_bool)?.into();
+
+ Ok(Self {
+ name: item.ident.to_string().into(),
+ ty: path.into_token_stream().to_string().into(),
+ value_expression,
+ })
+ }
+}
+
+fn render_const_expr(
+ expr: &Expr,
+ is_bool: bool,
+) -> anyhow::Result<String> {
+ match expr {
+ Expr::Lit(ExprLit {
+ lit: Lit::Int(value),
+ ..
+ }) => Ok(value.base10_digits().to_string()),
+ Expr::Lit(ExprLit {
+ lit: Lit::Float(value),
+ ..
+ }) => Ok(value.base10_digits().to_string()),
+ Expr::Lit(ExprLit {
+ lit: Lit::Bool(value),
+ ..
+ }) => Ok(value.value.to_string()),
+ Expr::Path(ExprPath {
+ qself: None,
+ path,
+ ..
+ }) => {
+ let Some(ident) = path.get_ident() else {
+ bail!(
+ "GPU constant expression may only reference a bare constant name, found `{}`",
+ path.to_token_stream()
+ );
+ };
+ Ok(ident.to_string())
+ },
+ Expr::Unary(ExprUnary {
+ op,
+ expr,
+ ..
+ }) => {
+ let operand = render_const_expr(expr, is_bool)?;
+ let op = match op {
+ UnOp::Neg(_) => "-",
+ UnOp::Not(_) => {
+ if is_bool {
+ "!"
+ } else {
+ "~"
+ }
+ },
+ other => bail!("Unsupported unary operator in GPU constant expression: `{}`", other.to_token_stream()),
+ };
+ Ok(format!("{op}{operand}"))
+ },
+ Expr::Binary(ExprBinary {
+ left,
+ op,
+ right,
+ ..
+ }) => {
+ let left = render_const_expr(left, is_bool)?;
+ let right = render_const_expr(right, is_bool)?;
+ Ok(format!("{left} {} {right}", binary_op(op)?))
+ },
+ Expr::Paren(ExprParen {
+ expr,
+ ..
+ }) => Ok(format!("({})", render_const_expr(expr, is_bool)?)),
+ Expr::Group(ExprGroup {
+ expr,
+ ..
+ }) => render_const_expr(expr, is_bool),
+ other => bail!("Unsupported GPU constant expression: `{}`", other.to_token_stream()),
+ }
+}
+
+fn binary_op(op: &BinOp) -> anyhow::Result<&'static str> {
+ Ok(match op {
+ BinOp::Add(_) => "+",
+ BinOp::Sub(_) => "-",
+ BinOp::Mul(_) => "*",
+ BinOp::Div(_) => "/",
+ BinOp::Rem(_) => "%",
+ BinOp::BitAnd(_) => "&",
+ BinOp::BitOr(_) => "|",
+ BinOp::BitXor(_) => "^",
+ BinOp::Shl(_) => "<<",
+ BinOp::Shr(_) => ">>",
+ other => bail!("Unsupported binary operator in GPU constant expression: `{}`", other.to_token_stream()),
+ })
+}
diff --git a/crates/backend-uzu/build/common/gpu_types/mod.rs b/crates/backend-uzu/build/common/gpu_types/mod.rs
index c03b4b98b..88c7b08b5 100644
--- a/crates/backend-uzu/build/common/gpu_types/mod.rs
+++ b/crates/backend-uzu/build/common/gpu_types/mod.rs
@@ -10,10 +10,12 @@ use derive_more::{AsRef, Deref, Display, From};
use syn::{Attribute, Ident, Item};
use walkdir::WalkDir;
+mod item_constant;
mod item_enum;
mod item_option_set;
mod item_struct;
+pub use item_constant::GpuTypeConstant;
pub use item_enum::GpuTypeEnum;
pub use item_option_set::{GpuTypeOptionSet, GpuTypeOptionSetVariant};
pub use item_struct::{GpuTypeStruct, GpuTypeStructFieldType};
@@ -47,6 +49,7 @@ fn ensure_repr_c(attrs: &[Attribute]) -> anyhow::Result<()> {
#[derive(Debug)]
pub enum GpuType {
+ Constant(GpuTypeConstant),
Enum(GpuTypeEnum),
Struct(GpuTypeStruct),
OptionSet(GpuTypeOptionSet),
@@ -69,6 +72,7 @@ impl GpuTypeFile {
.items
.into_iter()
.filter_map(|item| match item {
+ Item::Const(item) => Some(GpuTypeConstant::parse(item).map(GpuType::Constant)),
Item::Enum(item) => Some(GpuTypeEnum::parse(item).map(GpuType::Enum)),
Item::Struct(item) => Some(GpuTypeStruct::parse(item).map(GpuType::Struct)),
Item::Macro(item) if item.mac.path.is_ident("bitflags") => {
diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_constant.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_constant.rs
new file mode 100644
index 000000000..114b94e76
--- /dev/null
+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_constant.rs
@@ -0,0 +1,7 @@
+use super::rust_to_metal;
+use crate::common::gpu_types::GpuTypeConstant;
+
+pub fn gpu_type_gen_constant(constant: &GpuTypeConstant) -> anyhow::Result<String> {
+ let ty = rust_to_metal(&constant.ty)?;
+ Ok(format!("static constant constexpr {ty} {} = {};", constant.name, constant.value_expression))
+}
diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs
index 68f166abc..384f935b3 100644
--- a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs
+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs
@@ -1,26 +1,13 @@
+use super::rust_to_metal;
use crate::common::gpu_types::GpuTypeOptionSet;
-fn r2c(ty: &str) -> anyhow::Result<&'static str> {
- Ok(match ty {
- "u8" => "uint8_t",
- "u16" => "uint16_t",
- "u32" => "uint",
- "u64" => "uint64_t",
- "i8" => "int8_t",
- "i16" => "int16_t",
- "i32" => "int",
- "i64" => "int64_t",
- unknown => anyhow::bail!("Unsupported option-set underlying type: {unknown}"),
- })
-}
-
/// Emit a Metal-side option-set as a struct wrapping the underlying primitive,
/// with named static constants for each flag and a `contains()` helper.
-/// Implicit conversions from the underlying type allow a `uint` function
+/// Implicit conversions from the underlying type allow a `uint32_t` function
/// constant to be passed where the struct is expected.
pub fn gpu_type_gen_option_set(option_set: &GpuTypeOptionSet) -> anyhow::Result<String> {
let name = &option_set.name;
- let underlying_c = r2c(&option_set.underlying_type)?;
+ let underlying_c = rust_to_metal(&option_set.underlying_type)?;
let variants = option_set
.variants
diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_struct.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_struct.rs
index 111513edb..3124c9e6c 100644
--- a/crates/backend-uzu/build/metal/gpu_types/gpu_type_struct.rs
+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_struct.rs
@@ -1,36 +1,18 @@
use std::iter::once;
-use anyhow::anyhow;
use itertools::Itertools;
+use super::rust_to_metal;
use crate::common::gpu_types::{GpuTypeStruct, GpuTypeStructFieldType};
-fn r2c(ty: &str) -> anyhow::Result<&'static str> {
- match ty {
- "i8" => Ok("int8_t"),
- "i16" => Ok("int16_t"),
- "i32" => Ok("int32_t"),
- "i64" => Ok("int64_t"),
- "u8" => Ok("uint8_t"),
- "u16" => Ok("uint16_t"),
- "u32" => Ok("uint32_t"),
- "u64" => Ok("uint64_t"),
- "f32" => Ok("float"),
- "bool" => Ok("bool"),
- "usize" => Ok("size_t"),
- "isize" => Ok("ptrdiff_t"),
- _ => Err(anyhow!("Only primitive types are supported, found {ty}")),
- }
-}
-
pub fn gpu_type_gen_struct(gpu_type_struct: &GpuTypeStruct) -> anyhow::Result<String> {
once(Ok("typedef struct {".into()))
.chain(gpu_type_struct.fields.iter().map(|field| match &field.ty {
- GpuTypeStructFieldType::Scalar(ty) => Ok(format!(" {} {};", r2c(ty.as_ref())?, field.name)),
+ GpuTypeStructFieldType::Scalar(ty) => Ok(format!(" {} {};", rust_to_metal(ty.as_ref())?, field.name)),
GpuTypeStructFieldType::Array {
element,
length,
- } => Ok(format!(" {} {}[{}];", r2c(element.as_ref())?, field.name, *length)),
+ } => Ok(format!(" {} {}[{}];", rust_to_metal(element.as_ref())?, field.name, *length)),
}))
.chain(once(Ok(format!("}} {};", gpu_type_struct.name.as_ref()))))
.process_results(|mut it| it.join("\n"))
diff --git a/crates/backend-uzu/build/metal/gpu_types/mod.rs b/crates/backend-uzu/build/metal/gpu_types/mod.rs
index 5dcda01aa..b5968d062 100644
--- a/crates/backend-uzu/build/metal/gpu_types/mod.rs
+++ b/crates/backend-uzu/build/metal/gpu_types/mod.rs
@@ -5,14 +5,34 @@ use itertools::Itertools;
use crate::common::gpu_types::{GpuType, GpuTypeFile, GpuTypes};
+mod gpu_type_constant;
mod gpu_type_enum;
mod gpu_type_option_set;
mod gpu_type_struct;
+use gpu_type_constant::gpu_type_gen_constant;
use gpu_type_enum::gpu_type_gen_enum;
use gpu_type_option_set::gpu_type_gen_option_set;
use gpu_type_struct::gpu_type_gen_struct;
+fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {
+ match ty {
+ "i8" => Ok("int8_t"),
+ "i16" => Ok("int16_t"),
+ "i32" => Ok("int32_t"),
+ "i64" => Ok("int64_t"),
+ "u8" => Ok("uint8_t"),
+ "u16" => Ok("uint16_t"),
+ "u32" => Ok("uint32_t"),
+ "u64" => Ok("uint64_t"),
+ "f32" => Ok("float"),
+ "bool" => Ok("bool"),
+ "usize" => Ok("size_t"),
+ "isize" => Ok("ptrdiff_t"),
+ unknown => anyhow::bail!("Unsupported GPU type: {unknown}"),
+ }
+}
+
pub async fn gpu_type_gen(
gpu_types_dir: &Path,
gpu_types: &GpuTypes,
@@ -36,6 +56,8 @@ async fn gpu_type_gen_file(
.types
.iter()
.map(|gpu_type| match gpu_type {
+ GpuType::Constant(gpu_type_constant) => gpu_type_gen_constant(gpu_type_constant)
+ .with_context(|| format!("Failed to generate bindings for {gpu_type_constant:?}")),
GpuType::Enum(gpu_type_enum) => gpu_type_gen_enum(gpu_type_enum)
.with_context(|| format!("Failed to generate bindings for {gpu_type_enum:?}")),
GpuType::Struct(gpu_type_struct) => gpu_type_gen_struct(gpu_type_struct)
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs
index d29a63a54..0da9fa36f 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs
@@ -1,3 +1,5 @@
+pub const HADAMARD_TRANSFORM_BLOCK_SIZE: usize = 32;
+
#[repr(C)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum HadamardTransformOrder {
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
index 87213c086..820eec4b0 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
@@ -2,14 +2,15 @@ use half::bf16;
use num_traits::{Float, NumCast};
use proc_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::HadamardTransformOrder};
+use crate::{
+ array::ArrayElement,
+ backends::common::gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},
+};
-const SIMD_SIZE: usize = 32;
-
-fn hadamard_transform(values: &mut [f32; SIMD_SIZE]) {
+fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE]) {
let mut stride = 1;
- while stride < SIMD_SIZE {
- for lane in 0..SIMD_SIZE {
+ while stride < HADAMARD_TRANSFORM_BLOCK_SIZE {
+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {
if lane & stride == 0 {
let a = values[lane];
let b = values[lane | stride];
@@ -19,7 +20,7 @@ fn hadamard_transform(values: &mut [f32; SIMD_SIZE]) {
}
stride <<= 1;
}
- let scale = 1.0 / (SIMD_SIZE as f32).sqrt();
+ let scale = 1.0 / (HADAMARD_TRANSFORM_BLOCK_SIZE as f32).sqrt();
for v in values.iter_mut() {
*v *= scale;
}
@@ -36,13 +37,16 @@ pub fn hadamard_transform_mul<T: ArrayElement + Float>(
) {
let hidden_dim = hidden_dim as usize;
let batch_size = batch_size as usize;
- assert_eq!(hidden_dim % SIMD_SIZE, 0, "hidden_dim must be a multiple of {SIMD_SIZE}");
+ assert!(
+ hidden_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ "hidden_dim must be a multiple of {HADAMARD_TRANSFORM_BLOCK_SIZE}"
+ );
for batch in 0..batch_size {
let row_offset = batch * hidden_dim;
- for stripe_start in (0..hidden_dim).step_by(SIMD_SIZE) {
- let mut stripe = [0.0f32; SIMD_SIZE];
- for lane in 0..SIMD_SIZE {
+ for stripe_start in (0..hidden_dim).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE) {
+ let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE];
+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {
let v: f32 = NumCast::from(unsafe { *data.add(row_offset + stripe_start + lane) }).unwrap();
let f = unsafe { *factors.add(stripe_start + lane) } as f32;
stripe[lane] = match transform_order {
@@ -53,7 +57,7 @@ pub fn hadamard_transform_mul<T: ArrayElement + Float>(
hadamard_transform(&mut stripe);
- for lane in 0..SIMD_SIZE {
+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {
let f = unsafe { *factors.add(stripe_start + lane) } as f32;
let result = match transform_order {
HadamardTransformOrder::Input => stripe[lane],
diff --git a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
index ae2221399..5502c4a1d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
@@ -20,10 +20,8 @@ inline T activate_silu(T x) {
template <typename T>
inline T activate_gelu(T x) {
- constexpr float k0 = 0.044715f;
- constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
float xf = float(x);
- float yf = 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));
+ float yf = 0.5f * xf * (1.0f + metal::precise::tanh(GELU_K1 * (xf + GELU_K0 * xf * xf * xf)));
return T(yf);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h b/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h
index d61fe789d..a0375fe9c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h
@@ -14,7 +14,7 @@ using namespace metal;
#define DSL_XSTR(X) DSL_STR(X)
#define VARIANTS(TYPENAME, ...) DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
-#define CONSTRAINT(C) DSL_META("dsl.constraint", #C)
+#define CONSTRAINT(C) DSL_META("dsl.constraint", DSL_XSTR(C))
#define PUBLIC DSL_META("dsl.public")
#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
index e149ae8a9..8c9c33170 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
@@ -5,6 +5,10 @@
using namespace metal;
namespace uzu::activation_type {
+static constant constexpr float GELU_K0 = 0.044715;
+
+static constant constexpr float GELU_K1 = 0.7978846;
+
enum class ActivationType : uint32_t {
SILU = 0,
GELUApprox = 1,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
index cdf2e6208..80daae838 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
@@ -13,18 +13,18 @@ enum class GemmBPrologueKind : uint32_t {
};
struct GemmDTransform {
- uint raw_value;
+ uint32_t raw_value;
constexpr GemmDTransform() thread : raw_value(0) {}
- constexpr GemmDTransform(uint __dsl_v) thread : raw_value(__dsl_v) {}
- static constant constexpr uint SCALE = 1 << 0;
- static constant constexpr uint ACCUMULATE = 1 << 1;
- static constant constexpr uint BIAS = 1 << 2;
- static constant constexpr uint RHT = 1 << 3;
- static constant constexpr uint SOFT_CAP = 1 << 4;
- constexpr bool contains(uint flag) const thread { return (raw_value & flag) != 0; }
- constexpr bool contains(uint flag) const constant { return (raw_value & flag) != 0; }
- constexpr uint bits() const thread { return raw_value; }
- constexpr uint bits() const constant { return raw_value; }
+ constexpr GemmDTransform(uint32_t __dsl_v) thread : raw_value(__dsl_v) {}
+ static constant constexpr uint32_t SCALE = 1 << 0;
+ static constant constexpr uint32_t ACCUMULATE = 1 << 1;
+ static constant constexpr uint32_t BIAS = 1 << 2;
+ static constant constexpr uint32_t RHT = 1 << 3;
+ static constant constexpr uint32_t SOFT_CAP = 1 << 4;
+ constexpr bool contains(uint32_t flag) const thread { return (raw_value & flag) != 0; }
+ constexpr bool contains(uint32_t flag) const constant { return (raw_value & flag) != 0; }
+ constexpr uint32_t bits() const thread { return raw_value; }
+ constexpr uint32_t bits() const constant { return raw_value; }
};
enum class GemmTiling : uint32_t {
@@ -41,16 +41,18 @@ enum class GemmTiling : uint32_t {
Tile128x128x256_Simdgroups4x4 = 10,
};
+static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;
+
struct GemmAlignment {
- uint raw_value;
+ uint32_t raw_value;
constexpr GemmAlignment() thread : raw_value(0) {}
- constexpr GemmAlignment(uint __dsl_v) thread : raw_value(__dsl_v) {}
- static constant constexpr uint M = 1 << 0;
- static constant constexpr uint N = 1 << 1;
- static constant constexpr uint K = 1 << 2;
- constexpr bool contains(uint flag) const thread { return (raw_value & flag) != 0; }
- constexpr bool contains(uint flag) const constant { return (raw_value & flag) != 0; }
- constexpr uint bits() const thread { return raw_value; }
- constexpr uint bits() const constant { return raw_value; }
+ constexpr GemmAlignment(uint32_t __dsl_v) thread : raw_value(__dsl_v) {}
+ static constant constexpr uint32_t M = 1 << 0;
+ static constant constexpr uint32_t N = 1 << 1;
+ static constant constexpr uint32_t K = 1 << 2;
+ constexpr bool contains(uint32_t flag) const thread { return (raw_value & flag) != 0; }
+ constexpr bool contains(uint32_t flag) const constant { return (raw_value & flag) != 0; }
+ constexpr uint32_t bits() const thread { return raw_value; }
+ constexpr uint32_t bits() const constant { return raw_value; }
};
} // namespace uzu::gemm
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h
index e9f7bd50c..d94188451 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h
@@ -5,6 +5,8 @@
using namespace metal;
namespace uzu::hadamard_order {
+static constant constexpr size_t HADAMARD_TRANSFORM_BLOCK_SIZE = 32;
+
enum class HadamardTransformOrder : uint32_t {
Input = 0,
Output = 1,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
index 810c91953..655772529 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
@@ -3,16 +3,23 @@
#include <metal_stdlib>
#include "../common/defines.h"
#include "../common/thread_context.h"
+#include "../generated/hadamard_order.h"
using namespace metal;
+using uzu::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE;
+
+static_assert(
+ HADAMARD_TRANSFORM_BLOCK_SIZE == METAL_SIMD_SIZE,
+ "Hadamard transform block size must match the Metal SIMD width"
+);
static METAL_FUNC float simdgroup_hadamard_transform(ushort lane_index, float lane_value) {
- for (ushort stride = 1; stride < METAL_SIMD_SIZE; stride <<= 1) {
+ for (ushort stride = 1; stride < HADAMARD_TRANSFORM_BLOCK_SIZE; stride <<= 1) {
float partner_lane_value = simd_shuffle_xor(lane_value, stride);
lane_value = (lane_index & stride) ? (partner_lane_value - lane_value) : (partner_lane_value + lane_value);
}
- return lane_value / sqrt((float)METAL_SIMD_SIZE);
+ return lane_value / sqrt(static_cast<float>(HADAMARD_TRANSFORM_BLOCK_SIZE));
}
template <typename T>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
index f09eb4b60..992a04095 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
@@ -19,7 +19,7 @@ PUBLIC KERNEL(HadamardTransform)(
uint batch_index GROUPS(batch_size),
uint lane_index THREADS(METAL_SIMD_SIZE)
) {
- uint factor_index = block_index * METAL_SIMD_SIZE + lane_index;
+ uint factor_index = block_index * HADAMARD_TRANSFORM_BLOCK_SIZE + lane_index;
uint element_index = batch_index * hidden_dim + factor_index;
if (transform_order == HadamardTransformOrder::Input) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
index 35ba6d7c7..86c770a1b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
@@ -15,13 +15,13 @@ static METAL_FUNC void apply_output_random_hadamard_transform(
ushort simdgroup_count,
const thread ThreadContext& thread_context
) {
- const ushort stripes_per_row = tile_block_cols / METAL_SIMD_SIZE;
+ const ushort stripes_per_row = tile_block_cols / HADAMARD_TRANSFORM_BLOCK_SIZE;
const ushort simd_lane = thread_context.simd_lane_id;
const uint total_work = uint(tile_block_rows) * uint(stripes_per_row);
for (uint cell = thread_context.simdgroup_index; cell < total_work; cell += simdgroup_count) {
const ushort row_local = ushort(cell / stripes_per_row);
const ushort stripe = ushort(cell % stripes_per_row);
- const ushort col_local = stripe * ushort(METAL_SIMD_SIZE) + simd_lane;
+ const ushort col_local = stripe * ushort(HADAMARD_TRANSFORM_BLOCK_SIZE) + simd_lane;
const size_t output_index = size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);
output_block[output_index] =
simdgroup_output_random_hadamard_transform(simd_lane, output_block[output_index], rht_factors_block[col_local]);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 0185c2141..c0fb0fb24 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -36,7 +36,7 @@ struct MxuMmaCore {
UZU_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
- UZU_CONST ushort SIMDGROUP_BLOCK_K = 32;
+ UZU_CONST ushort SIMDGROUP_BLOCK_K = static_cast<ushort>(MXU_SIMDGROUP_BLOCK_K);
UZU_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);
static_assert(
THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 6b3b6a85e..611e438a1 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -8,7 +8,10 @@ use crate::{
backends::{
common::{
Allocation, BufferArg, Encoder,
- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ gpu_types::{
+ HADAMARD_TRANSFORM_BLOCK_SIZE,
+ gemm::{GemmBPrologueKind, GemmDTransform},
+ },
kernel::matmul::{MatmulArguments, MatmulB, MatmulError},
},
metal::{Metal, context::MetalContext, device_tier::DeviceTier, kernel::GemvMetalKernel},
@@ -62,7 +65,7 @@ impl GemvSpecialization {
if args.d_transform.accumulate && !args.n.is_multiple_of(32) {
return None;
}
- if args.d_transform.rht_factors.is_some() && !args.n.is_multiple_of(32) {
+ if args.d_transform.rht_factors.is_some() && !args.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE as u32) {
return None;
}
if is_quant {
diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rs
index bfa4c1c5f..22edc84b3 100644
--- a/crates/backend-uzu/src/backends/metal/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/mod.rs
@@ -9,6 +9,14 @@ mod kernel;
mod metal_extensions;
mod sparse;
+use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;
+
+pub(crate) const METAL_SIMD_SIZE: u32 = 32;
+
+const _: () = {
+ assert!(HADAMARD_TRANSFORM_BLOCK_SIZE == METAL_SIMD_SIZE as usize);
+};
+
pub use backend::Metal;
pub use context::MetalContext;
pub use kernel::matmul::GemmDispatchPath;
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index f11f4b62a..6ae003af2 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -5,7 +5,7 @@ use crate::{
array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
- gpu_types::{QuantizationMethod, QuantizationMode},
+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},
kernel::{
FullPrecisionEmbeddingLookupKernel, LogitSoftCapKernel, QuantizedEmbeddingLookupKernel,
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
@@ -212,7 +212,7 @@ impl<B: Backend> Embedding<B> {
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
quantization_spec,
adapter_spec: None,
- incoherence_block_size: Some(32),
+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
incoherence_processing_mode: IncoherenceProcessingMode::Output,
..
}) => {
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index 009999b64..dec2968d4 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -8,7 +8,7 @@ pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};
use thiserror::Error;
use crate::{
- backends::common::{Allocation, Backend, Encoder},
+ backends::common::{Allocation, Backend, Encoder, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE},
config::weight_matrix::{
AnyWeightMatrixSpec, Layout,
full_precision_spec::FullPrecisionSpec,
@@ -86,7 +86,7 @@ impl<B: Backend> dyn Linear<B> {
},
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
adapter_spec: None,
- incoherence_block_size: Some(32),
+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
..
}) => Ok(Box::new(RHTLinearWrapper::new(
@@ -213,7 +213,7 @@ impl<B: Backend> dyn Linear<B> {
let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
if let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
adapter_spec: None,
- incoherence_block_size: Some(32),
+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
..
}) = spec
diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
index 5f49743de..7a1b67f95 100644
--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -5,7 +5,7 @@ use crate::{
array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
- gpu_types::HadamardTransformOrder,
+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},
kernel::{
HadamardTransformKernel, Kernels,
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
@@ -60,7 +60,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
) -> Result<Self, QLoRALinearWrapperError<B>> {
let use_incoherence_signs = match (incoherence_block_size, incoherence_processing_mode) {
(None, _) => false,
- (Some(32), IncoherenceProcessingMode::InputOutput) => true,
+ (Some(HADAMARD_TRANSFORM_BLOCK_SIZE), IncoherenceProcessingMode::InputOutput) => true,
(incoherence_block_size, incoherence_processing_mode) => {
return Err(QLoRALinearWrapperError::UnsupportedConfiguration(format!(
"incoherence block_size={incoherence_block_size:?}, processing_mode={incoherence_processing_mode:?}"
diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
index 5348feadf..1e18afaef 100644
--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
@@ -3,7 +3,7 @@ use thiserror::Error;
use crate::{
backends::common::{
Allocation, Backend, Encoder,
- gpu_types::HadamardTransformOrder,
+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},
kernel::{HadamardTransformKernel, Kernels},
},
config::weight_matrix::{
@@ -49,7 +49,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
adapter_spec: None,
- incoherence_block_size: Some(32),
+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
..
}) = &spec