A high-performance inference engine for AI models
Constants (#629)
eugene8a9986ebd654
diff --git a/crates/backend-uzu/build/common/enum_paths.rs b/crates/backend-uzu/build/common/enum_paths.rsindex 991991808..4dc54a9f2 100644--- a/crates/backend-uzu/build/common/enum_paths.rs+++ b/crates/backend-uzu/build/common/enum_paths.rs@@ -31,7 +31,7 @@ impl EnumPaths {let (name_str, kind) = match ty {GpuType::Enum(enum_type) => (enum_type.name.as_ref(), GpuTypeKind::Enum),GpuType::OptionSet(option_set) => (option_set.name.as_ref(), GpuTypeKind::OptionSet),- GpuType::Struct(_) => continue,+ GpuType::Constant(_) | GpuType::Struct(_) => continue,};let name = GpuTypeName::from(name_str);let path =diff --git a/crates/backend-uzu/build/common/gpu_types/item_constant.rs b/crates/backend-uzu/build/common/gpu_types/item_constant.rsnew file mode 100644index 000000000..10a1731ce--- /dev/null+++ b/crates/backend-uzu/build/common/gpu_types/item_constant.rs@@ -0,0 +1,120 @@+use anyhow::bail;+use quote::ToTokens;+use syn::{+ BinOp, Expr, ExprBinary, ExprGroup, ExprLit, ExprParen, ExprPath, ExprUnary, ItemConst, Lit, Type, TypePath, UnOp,+};++#[derive(Debug)]+pub struct GpuTypeConstant {+ pub name: Box<str>,+ pub ty: Box<str>,+ pub value_expression: Box<str>,+}++impl GpuTypeConstant {+ pub fn parse(item: ItemConst) -> anyhow::Result<Self> {+ let Type::Path(TypePath {+ qself: None,+ path,+ }) = *item.ty+ else {+ bail!("GPU constant type must be a path, found {:?}", item.ty);+ };++ let is_bool = path.is_ident("bool");+ let value_expression = render_const_expr(&item.expr, is_bool)?.into();++ Ok(Self {+ name: item.ident.to_string().into(),+ ty: path.into_token_stream().to_string().into(),+ value_expression,+ })+ }+}++fn render_const_expr(+ expr: &Expr,+ is_bool: bool,+) -> anyhow::Result<String> {+ match expr {+ Expr::Lit(ExprLit {+ lit: Lit::Int(value),+ ..+ }) => Ok(value.base10_digits().to_string()),+ Expr::Lit(ExprLit {+ lit: Lit::Float(value),+ ..+ }) => Ok(value.base10_digits().to_string()),+ Expr::Lit(ExprLit {+ lit: Lit::Bool(value),+ ..+ }) => Ok(value.value.to_string()),+ Expr::Path(ExprPath {+ qself: None,+ path,+ ..+ }) => {+ let Some(ident) = path.get_ident() else {+ bail!(+ "GPU constant expression may only reference a bare constant name, found `{}`",+ path.to_token_stream()+ );+ };+ Ok(ident.to_string())+ },+ Expr::Unary(ExprUnary {+ op,+ expr,+ ..+ }) => {+ let operand = render_const_expr(expr, is_bool)?;+ let op = match op {+ UnOp::Neg(_) => "-",+ UnOp::Not(_) => {+ if is_bool {+ "!"+ } else {+ "~"+ }+ },+ other => bail!("Unsupported unary operator in GPU constant expression: `{}`", other.to_token_stream()),+ };+ Ok(format!("{op}{operand}"))+ },+ Expr::Binary(ExprBinary {+ left,+ op,+ right,+ ..+ }) => {+ let left = render_const_expr(left, is_bool)?;+ let right = render_const_expr(right, is_bool)?;+ Ok(format!("{left} {} {right}", binary_op(op)?))+ },+ Expr::Paren(ExprParen {+ expr,+ ..+ }) => Ok(format!("({})", render_const_expr(expr, is_bool)?)),+ Expr::Group(ExprGroup {+ expr,+ ..+ }) => render_const_expr(expr, is_bool),+ other => bail!("Unsupported GPU constant expression: `{}`", other.to_token_stream()),+ }+}++fn binary_op(op: &BinOp) -> anyhow::Result<&'static str> {+ Ok(match op {+ BinOp::Add(_) => "+",+ BinOp::Sub(_) => "-",+ BinOp::Mul(_) => "*",+ BinOp::Div(_) => "/",+ BinOp::Rem(_) => "%",+ BinOp::BitAnd(_) => "&",+ BinOp::BitOr(_) => "|",+ BinOp::BitXor(_) => "^",+ BinOp::Shl(_) => "<<",+ BinOp::Shr(_) => ">>",+ other => bail!("Unsupported binary operator in GPU constant expression: `{}`", other.to_token_stream()),+ })+}diff --git a/crates/backend-uzu/build/common/gpu_types/mod.rs b/crates/backend-uzu/build/common/gpu_types/mod.rsindex c03b4b98b..88c7b08b5 100644--- a/crates/backend-uzu/build/common/gpu_types/mod.rs+++ b/crates/backend-uzu/build/common/gpu_types/mod.rs@@ -10,10 +10,12 @@ use derive_more::{AsRef, Deref, Display, From};use syn::{Attribute, Ident, Item};use walkdir::WalkDir;+mod item_constant;mod item_enum;mod item_option_set;mod item_struct;+pub use item_constant::GpuTypeConstant;pub use item_enum::GpuTypeEnum;pub use item_option_set::{GpuTypeOptionSet, GpuTypeOptionSetVariant};pub use item_struct::{GpuTypeStruct, GpuTypeStructFieldType};@@ -47,6 +49,7 @@ fn ensure_repr_c(attrs: &[Attribute]) -> anyhow::Result<()> {#[derive(Debug)]pub enum GpuType {+ Constant(GpuTypeConstant),Enum(GpuTypeEnum),Struct(GpuTypeStruct),OptionSet(GpuTypeOptionSet),@@ -69,6 +72,7 @@ impl GpuTypeFile {.items.into_iter().filter_map(|item| match item {+ Item::Const(item) => Some(GpuTypeConstant::parse(item).map(GpuType::Constant)),Item::Enum(item) => Some(GpuTypeEnum::parse(item).map(GpuType::Enum)),Item::Struct(item) => Some(GpuTypeStruct::parse(item).map(GpuType::Struct)),Item::Macro(item) if item.mac.path.is_ident("bitflags") => {diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_constant.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_constant.rsnew file mode 100644index 000000000..114b94e76--- /dev/null+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_constant.rs@@ -0,0 +1,7 @@+use super::rust_to_metal;+use crate::common::gpu_types::GpuTypeConstant;++pub fn gpu_type_gen_constant(constant: &GpuTypeConstant) -> anyhow::Result<String> {+ let ty = rust_to_metal(&constant.ty)?;+ Ok(format!("static constant constexpr {ty} {} = {};", constant.name, constant.value_expression))+}diff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rsindex 68f166abc..384f935b3 100644--- a/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_option_set.rs@@ -1,26 +1,13 @@+use super::rust_to_metal;use crate::common::gpu_types::GpuTypeOptionSet;-fn r2c(ty: &str) -> anyhow::Result<&'static str> {- Ok(match ty {- "u8" => "uint8_t",- "u16" => "uint16_t",- "u32" => "uint",- "u64" => "uint64_t",- "i8" => "int8_t",- "i16" => "int16_t",- "i32" => "int",- "i64" => "int64_t",- unknown => anyhow::bail!("Unsupported option-set underlying type: {unknown}"),- })-}-/// Emit a Metal-side option-set as a struct wrapping the underlying primitive,/// with named static constants for each flag and a `contains()` helper.-/// Implicit conversions from the underlying type allow a `uint` function+/// Implicit conversions from the underlying type allow a `uint32_t` function/// constant to be passed where the struct is expected.pub fn gpu_type_gen_option_set(option_set: &GpuTypeOptionSet) -> anyhow::Result<String> {let name = &option_set.name;- let underlying_c = r2c(&option_set.underlying_type)?;+ let underlying_c = rust_to_metal(&option_set.underlying_type)?;let variants = option_set.variantsdiff --git a/crates/backend-uzu/build/metal/gpu_types/gpu_type_struct.rs b/crates/backend-uzu/build/metal/gpu_types/gpu_type_struct.rsindex 111513edb..3124c9e6c 100644--- a/crates/backend-uzu/build/metal/gpu_types/gpu_type_struct.rs+++ b/crates/backend-uzu/build/metal/gpu_types/gpu_type_struct.rs@@ -1,36 +1,18 @@use std::iter::once;-use anyhow::anyhow;use itertools::Itertools;+use super::rust_to_metal;use crate::common::gpu_types::{GpuTypeStruct, GpuTypeStructFieldType};-fn r2c(ty: &str) -> anyhow::Result<&'static str> {- match ty {- "i8" => Ok("int8_t"),- "i16" => Ok("int16_t"),- "i32" => Ok("int32_t"),- "i64" => Ok("int64_t"),- "u8" => Ok("uint8_t"),- "u16" => Ok("uint16_t"),- "u32" => Ok("uint32_t"),- "u64" => Ok("uint64_t"),- "f32" => Ok("float"),- "bool" => Ok("bool"),- "usize" => Ok("size_t"),- "isize" => Ok("ptrdiff_t"),- _ => Err(anyhow!("Only primitive types are supported, found {ty}")),- }-}-pub fn gpu_type_gen_struct(gpu_type_struct: &GpuTypeStruct) -> anyhow::Result<String> {once(Ok("typedef struct {".into())).chain(gpu_type_struct.fields.iter().map(|field| match &field.ty {- GpuTypeStructFieldType::Scalar(ty) => Ok(format!(" {} {};", r2c(ty.as_ref())?, field.name)),+ GpuTypeStructFieldType::Scalar(ty) => Ok(format!(" {} {};", rust_to_metal(ty.as_ref())?, field.name)),GpuTypeStructFieldType::Array {element,length,- } => Ok(format!(" {} {}[{}];", r2c(element.as_ref())?, field.name, *length)),+ } => Ok(format!(" {} {}[{}];", rust_to_metal(element.as_ref())?, field.name, *length)),})).chain(once(Ok(format!("}} {};", gpu_type_struct.name.as_ref())))).process_results(|mut it| it.join("\n"))diff --git a/crates/backend-uzu/build/metal/gpu_types/mod.rs b/crates/backend-uzu/build/metal/gpu_types/mod.rsindex 5dcda01aa..b5968d062 100644--- a/crates/backend-uzu/build/metal/gpu_types/mod.rs+++ b/crates/backend-uzu/build/metal/gpu_types/mod.rs@@ -5,14 +5,34 @@ use itertools::Itertools;use crate::common::gpu_types::{GpuType, GpuTypeFile, GpuTypes};+mod gpu_type_constant;mod gpu_type_enum;mod gpu_type_option_set;mod gpu_type_struct;+use gpu_type_constant::gpu_type_gen_constant;use gpu_type_enum::gpu_type_gen_enum;use gpu_type_option_set::gpu_type_gen_option_set;use gpu_type_struct::gpu_type_gen_struct;+fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {+ match ty {+ "i8" => Ok("int8_t"),+ "i16" => Ok("int16_t"),+ "i32" => Ok("int32_t"),+ "i64" => Ok("int64_t"),+ "u8" => Ok("uint8_t"),+ "u16" => Ok("uint16_t"),+ "u32" => Ok("uint32_t"),+ "u64" => Ok("uint64_t"),+ "f32" => Ok("float"),+ "bool" => Ok("bool"),+ "usize" => Ok("size_t"),+ "isize" => Ok("ptrdiff_t"),+ unknown => anyhow::bail!("Unsupported GPU type: {unknown}"),+ }+}+pub async fn gpu_type_gen(gpu_types_dir: &Path,gpu_types: &GpuTypes,@@ -36,6 +56,8 @@ async fn gpu_type_gen_file(.types.iter().map(|gpu_type| match gpu_type {+ GpuType::Constant(gpu_type_constant) => gpu_type_gen_constant(gpu_type_constant)+ .with_context(|| format!("Failed to generate bindings for {gpu_type_constant:?}")),GpuType::Enum(gpu_type_enum) => gpu_type_gen_enum(gpu_type_enum).with_context(|| format!("Failed to generate bindings for {gpu_type_enum:?}")),GpuType::Struct(gpu_type_struct) => gpu_type_gen_struct(gpu_type_struct)diff --git a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rsindex d29a63a54..0da9fa36f 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs@@ -1,3 +1,5 @@+pub const HADAMARD_TRANSFORM_BLOCK_SIZE: usize = 32;+#[repr(C)]#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum HadamardTransformOrder {diff --git a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rsindex 87213c086..820eec4b0 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs@@ -2,14 +2,15 @@ use half::bf16;use num_traits::{Float, NumCast};use proc_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::HadamardTransformOrder};+use crate::{+ array::ArrayElement,+ backends::common::gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},+};-const SIMD_SIZE: usize = 32;--fn hadamard_transform(values: &mut [f32; SIMD_SIZE]) {+fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE]) {let mut stride = 1;- while stride < SIMD_SIZE {- for lane in 0..SIMD_SIZE {+ while stride < HADAMARD_TRANSFORM_BLOCK_SIZE {+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {if lane & stride == 0 {let a = values[lane];let b = values[lane | stride];@@ -19,7 +20,7 @@ fn hadamard_transform(values: &mut [f32; SIMD_SIZE]) {}stride <<= 1;}- let scale = 1.0 / (SIMD_SIZE as f32).sqrt();+ let scale = 1.0 / (HADAMARD_TRANSFORM_BLOCK_SIZE as f32).sqrt();for v in values.iter_mut() {*v *= scale;}@@ -36,13 +37,16 @@ pub fn hadamard_transform_mul<T: ArrayElement + Float>() {let hidden_dim = hidden_dim as usize;let batch_size = batch_size as usize;- assert_eq!(hidden_dim % SIMD_SIZE, 0, "hidden_dim must be a multiple of {SIMD_SIZE}");+ assert!(+ hidden_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),+ "hidden_dim must be a multiple of {HADAMARD_TRANSFORM_BLOCK_SIZE}"+ );for batch in 0..batch_size {let row_offset = batch * hidden_dim;- for stripe_start in (0..hidden_dim).step_by(SIMD_SIZE) {- let mut stripe = [0.0f32; SIMD_SIZE];- for lane in 0..SIMD_SIZE {+ for stripe_start in (0..hidden_dim).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE) {+ let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE];+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {let v: f32 = NumCast::from(unsafe { *data.add(row_offset + stripe_start + lane) }).unwrap();let f = unsafe { *factors.add(stripe_start + lane) } as f32;stripe[lane] = match transform_order {@@ -53,7 +57,7 @@ pub fn hadamard_transform_mul<T: ArrayElement + Float>(hadamard_transform(&mut stripe);- for lane in 0..SIMD_SIZE {+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {let f = unsafe { *factors.add(stripe_start + lane) } as f32;let result = match transform_order {HadamardTransformOrder::Input => stripe[lane],diff --git a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.hindex ae2221399..5502c4a1d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h+++ b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h@@ -20,10 +20,8 @@ inline T activate_silu(T x) {template <typename T>inline T activate_gelu(T x) {- constexpr float k0 = 0.044715f;- constexpr float k1 = 0.7978845608f; // sqrt(2/pi)float xf = float(x);- float yf = 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));+ float yf = 0.5f * xf * (1.0f + metal::precise::tanh(GELU_K1 * (xf + GELU_K0 * xf * xf * xf)));return T(yf);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h b/crates/backend-uzu/src/backends/metal/kernel/common/dsl.hindex d61fe789d..a0375fe9c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h@@ -14,7 +14,7 @@ using namespace metal;#define DSL_XSTR(X) DSL_STR(X)#define VARIANTS(TYPENAME, ...) DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)-#define CONSTRAINT(C) DSL_META("dsl.constraint", #C)+#define CONSTRAINT(C) DSL_META("dsl.constraint", DSL_XSTR(C))#define PUBLIC DSL_META("dsl.public")#define KERNEL(NAME) DSL_META("dsl.kernel") void NAMEdiff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.hindex e149ae8a9..8c9c33170 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h@@ -5,6 +5,10 @@using namespace metal;namespace uzu::activation_type {+static constant constexpr float GELU_K0 = 0.044715;++static constant constexpr float GELU_K1 = 0.7978846;+enum class ActivationType : uint32_t {SILU = 0,GELUApprox = 1,diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.hindex cdf2e6208..80daae838 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h@@ -13,18 +13,18 @@ enum class GemmBPrologueKind : uint32_t {};struct GemmDTransform {- uint raw_value;+ uint32_t raw_value;constexpr GemmDTransform() thread : raw_value(0) {}- constexpr GemmDTransform(uint __dsl_v) thread : raw_value(__dsl_v) {}- static constant constexpr uint SCALE = 1 << 0;- static constant constexpr uint ACCUMULATE = 1 << 1;- static constant constexpr uint BIAS = 1 << 2;- static constant constexpr uint RHT = 1 << 3;- static constant constexpr uint SOFT_CAP = 1 << 4;- constexpr bool contains(uint flag) const thread { return (raw_value & flag) != 0; }- constexpr bool contains(uint flag) const constant { return (raw_value & flag) != 0; }- constexpr uint bits() const thread { return raw_value; }- constexpr uint bits() const constant { return raw_value; }+ constexpr GemmDTransform(uint32_t __dsl_v) thread : raw_value(__dsl_v) {}+ static constant constexpr uint32_t SCALE = 1 << 0;+ static constant constexpr uint32_t ACCUMULATE = 1 << 1;+ static constant constexpr uint32_t BIAS = 1 << 2;+ static constant constexpr uint32_t RHT = 1 << 3;+ static constant constexpr uint32_t SOFT_CAP = 1 << 4;+ constexpr bool contains(uint32_t flag) const thread { return (raw_value & flag) != 0; }+ constexpr bool contains(uint32_t flag) const constant { return (raw_value & flag) != 0; }+ constexpr uint32_t bits() const thread { return raw_value; }+ constexpr uint32_t bits() const constant { return raw_value; }};enum class GemmTiling : uint32_t {@@ -41,16 +41,18 @@ enum class GemmTiling : uint32_t {Tile128x128x256_Simdgroups4x4 = 10,};+static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;+struct GemmAlignment {- uint raw_value;+ uint32_t raw_value;constexpr GemmAlignment() thread : raw_value(0) {}- constexpr GemmAlignment(uint __dsl_v) thread : raw_value(__dsl_v) {}- static constant constexpr uint M = 1 << 0;- static constant constexpr uint N = 1 << 1;- static constant constexpr uint K = 1 << 2;- constexpr bool contains(uint flag) const thread { return (raw_value & flag) != 0; }- constexpr bool contains(uint flag) const constant { return (raw_value & flag) != 0; }- constexpr uint bits() const thread { return raw_value; }- constexpr uint bits() const constant { return raw_value; }+ constexpr GemmAlignment(uint32_t __dsl_v) thread : raw_value(__dsl_v) {}+ static constant constexpr uint32_t M = 1 << 0;+ static constant constexpr uint32_t N = 1 << 1;+ static constant constexpr uint32_t K = 1 << 2;+ constexpr bool contains(uint32_t flag) const thread { return (raw_value & flag) != 0; }+ constexpr bool contains(uint32_t flag) const constant { return (raw_value & flag) != 0; }+ constexpr uint32_t bits() const thread { return raw_value; }+ constexpr uint32_t bits() const constant { return raw_value; }};} // namespace uzu::gemmdiff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.hindex e9f7bd50c..d94188451 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h@@ -5,6 +5,8 @@using namespace metal;namespace uzu::hadamard_order {+static constant constexpr size_t HADAMARD_TRANSFORM_BLOCK_SIZE = 32;+enum class HadamardTransformOrder : uint32_t {Input = 0,Output = 1,diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.hindex 810c91953..655772529 100644--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h@@ -3,16 +3,23 @@#include <metal_stdlib>#include "../common/defines.h"#include "../common/thread_context.h"+#include "../generated/hadamard_order.h"using namespace metal;+using uzu::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE;++static_assert(+ HADAMARD_TRANSFORM_BLOCK_SIZE == METAL_SIMD_SIZE,+ "Hadamard transform block size must match the Metal SIMD width"+);static METAL_FUNC float simdgroup_hadamard_transform(ushort lane_index, float lane_value) {- for (ushort stride = 1; stride < METAL_SIMD_SIZE; stride <<= 1) {+ for (ushort stride = 1; stride < HADAMARD_TRANSFORM_BLOCK_SIZE; stride <<= 1) {float partner_lane_value = simd_shuffle_xor(lane_value, stride);lane_value = (lane_index & stride) ? (partner_lane_value - lane_value) : (partner_lane_value + lane_value);}- return lane_value / sqrt((float)METAL_SIMD_SIZE);+ return lane_value / sqrt(static_cast<float>(HADAMARD_TRANSFORM_BLOCK_SIZE));}template <typename T>diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metalindex f09eb4b60..992a04095 100644--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal@@ -19,7 +19,7 @@ PUBLIC KERNEL(HadamardTransform)(uint batch_index GROUPS(batch_size),uint lane_index THREADS(METAL_SIMD_SIZE)) {- uint factor_index = block_index * METAL_SIMD_SIZE + lane_index;+ uint factor_index = block_index * HADAMARD_TRANSFORM_BLOCK_SIZE + lane_index;uint element_index = batch_index * hidden_dim + factor_index;if (transform_order == HadamardTransformOrder::Input) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.hindex 35ba6d7c7..86c770a1b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h@@ -15,13 +15,13 @@ static METAL_FUNC void apply_output_random_hadamard_transform(ushort simdgroup_count,const thread ThreadContext& thread_context) {- const ushort stripes_per_row = tile_block_cols / METAL_SIMD_SIZE;+ const ushort stripes_per_row = tile_block_cols / HADAMARD_TRANSFORM_BLOCK_SIZE;const ushort simd_lane = thread_context.simd_lane_id;const uint total_work = uint(tile_block_rows) * uint(stripes_per_row);for (uint cell = thread_context.simdgroup_index; cell < total_work; cell += simdgroup_count) {const ushort row_local = ushort(cell / stripes_per_row);const ushort stripe = ushort(cell % stripes_per_row);- const ushort col_local = stripe * ushort(METAL_SIMD_SIZE) + simd_lane;+ const ushort col_local = stripe * ushort(HADAMARD_TRANSFORM_BLOCK_SIZE) + simd_lane;const size_t output_index = size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);output_block[output_index] =simdgroup_output_random_hadamard_transform(simd_lane, output_block[output_index], rht_factors_block[col_local]);diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 0185c2141..c0fb0fb24 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -36,7 +36,7 @@ struct MxuMmaCore {UZU_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;- UZU_CONST ushort SIMDGROUP_BLOCK_K = 32;+ UZU_CONST ushort SIMDGROUP_BLOCK_K = static_cast<ushort>(MXU_SIMDGROUP_BLOCK_K);UZU_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);static_assert(THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 6b3b6a85e..611e438a1 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -8,7 +8,10 @@ use crate::{backends::{common::{Allocation, BufferArg, Encoder,- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ gpu_types::{+ HADAMARD_TRANSFORM_BLOCK_SIZE,+ gemm::{GemmBPrologueKind, GemmDTransform},+ },kernel::matmul::{MatmulArguments, MatmulB, MatmulError},},metal::{Metal, context::MetalContext, device_tier::DeviceTier, kernel::GemvMetalKernel},@@ -62,7 +65,7 @@ impl GemvSpecialization {if args.d_transform.accumulate && !args.n.is_multiple_of(32) {return None;}- if args.d_transform.rht_factors.is_some() && !args.n.is_multiple_of(32) {+ if args.d_transform.rht_factors.is_some() && !args.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE as u32) {return None;}if is_quant {diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rsindex bfa4c1c5f..22edc84b3 100644--- a/crates/backend-uzu/src/backends/metal/mod.rs+++ b/crates/backend-uzu/src/backends/metal/mod.rs@@ -9,6 +9,14 @@ mod kernel;mod metal_extensions;mod sparse;+use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;++pub(crate) const METAL_SIMD_SIZE: u32 = 32;++const _: () = {+ assert!(HADAMARD_TRANSFORM_BLOCK_SIZE == METAL_SIMD_SIZE as usize);+};+pub use backend::Metal;pub use context::MetalContext;pub use kernel::matmul::GemmDispatchPath;diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex f11f4b62a..6ae003af2 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -5,7 +5,7 @@ use crate::{array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,- gpu_types::{QuantizationMethod, QuantizationMode},+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},kernel::{FullPrecisionEmbeddingLookupKernel, LogitSoftCapKernel, QuantizedEmbeddingLookupKernel,matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},@@ -212,7 +212,7 @@ impl<B: Backend> Embedding<B> {AnyWeightMatrixSpec::HybridSpec(HybridSpec {quantization_spec,adapter_spec: None,- incoherence_block_size: Some(32),+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),incoherence_processing_mode: IncoherenceProcessingMode::Output,..}) => {diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex 009999b64..dec2968d4 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -8,7 +8,7 @@ pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};use thiserror::Error;use crate::{- backends::common::{Allocation, Backend, Encoder},+ backends::common::{Allocation, Backend, Encoder, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE},config::weight_matrix::{AnyWeightMatrixSpec, Layout,full_precision_spec::FullPrecisionSpec,@@ -86,7 +86,7 @@ impl<B: Backend> dyn Linear<B> {},AnyWeightMatrixSpec::HybridSpec(HybridSpec {adapter_spec: None,- incoherence_block_size: Some(32),+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,..}) => Ok(Box::new(RHTLinearWrapper::new(@@ -213,7 +213,7 @@ impl<B: Backend> dyn Linear<B> {let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;if let AnyWeightMatrixSpec::HybridSpec(HybridSpec {adapter_spec: None,- incoherence_block_size: Some(32),+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,..}) = specdiff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex 5f49743de..7a1b67f95 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -5,7 +5,7 @@ use crate::{array::size_for_shape,backends::common::{Allocation, Backend, Encoder,- gpu_types::HadamardTransformOrder,+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},kernel::{HadamardTransformKernel, Kernels,matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},@@ -60,7 +60,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {) -> Result<Self, QLoRALinearWrapperError<B>> {let use_incoherence_signs = match (incoherence_block_size, incoherence_processing_mode) {(None, _) => false,- (Some(32), IncoherenceProcessingMode::InputOutput) => true,+ (Some(HADAMARD_TRANSFORM_BLOCK_SIZE), IncoherenceProcessingMode::InputOutput) => true,(incoherence_block_size, incoherence_processing_mode) => {return Err(QLoRALinearWrapperError::UnsupportedConfiguration(format!("incoherence block_size={incoherence_block_size:?}, processing_mode={incoherence_processing_mode:?}"diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex 5348feadf..1e18afaef 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -3,7 +3,7 @@ use thiserror::Error;use crate::{backends::common::{Allocation, Backend, Encoder,- gpu_types::HadamardTransformOrder,+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},kernel::{HadamardTransformKernel, Kernels},},config::weight_matrix::{@@ -49,7 +49,7 @@ impl<B: Backend> RHTLinearWrapper<B> {let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;let AnyWeightMatrixSpec::HybridSpec(HybridSpec {adapter_spec: None,- incoherence_block_size: Some(32),+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,..}) = &spec