A high-performance inference engine for AI models
RHT (#277)
eugene32655c2db262
diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rsindex 0561ac39f..6e34368ff 100644--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs@@ -9,6 +9,7 @@ pub struct MlpGateActMulEncodable<B: Backend> {kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,activation: ActivationConfig,hidden_dim: usize,+ hadamard_factors: Option<B::Buffer>,}impl<B: Backend> MlpGateActMulEncodable<B> {@@ -17,12 +18,14 @@ impl<B: Backend> MlpGateActMulEncodable<B> {data_type: DataType,activation: ActivationConfig,hidden_dim: usize,+ hadamard_factors: Option<B::Buffer>,) -> Result<Self, B::Error> {- let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type)?;+ let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type, hadamard_factors.is_some())?;Ok(Self {kernel,activation,hidden_dim,+ hadamard_factors,})}@@ -36,7 +39,15 @@ impl<B: Backend> MlpGateActMulEncodable<B> {if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")}- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, self.activation.act_type(), encoder);+ self.kernel.encode(+ fused_up,+ hidden,+ self.hadamard_factors.as_ref(),+ self.hidden_dim as i32,+ m,+ self.activation.act_type(),+ encoder,+ );Ok(())}}diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex 63f31070e..56c7c6895 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -18,6 +18,8 @@ pub enum QuantizedMatmulError<B: Backend> {UnsupportedDataType(DataType),#[error("Unsupported group size: {0}")]UnsupportedGroupSize(usize),+ #[error("Hadamard not supported for this kernel configuration")]+ UnsupportedHadamard,}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -34,6 +36,7 @@ pub struct QuantizedMatmulConfiguration {pub output_dim: usize,pub mode: QuantizationMode,pub quantization_type: QuantizedMatmulType,+ pub use_hadamard: bool,}pub struct QuantizedMatmulArguments<'a, B: Backend> {@@ -43,6 +46,7 @@ pub struct QuantizedMatmulArguments<'a, B: Backend> {pub scales_buffer: &'a B::Buffer,pub zero_points_or_biases_buffer: &'a B::Buffer,pub output_buffer: &'a mut B::Buffer,+ pub hadamard_factors: Option<&'a B::Buffer>,pub batch_dim: usize,}@@ -96,10 +100,15 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {bits,use_zero_points,use_mlx_quant,+ configuration.use_hadamard,).map_err(QuantizedMatmulError::BackendError)?,)} else {+ if configuration.use_hadamard {+ return Err(QuantizedMatmulError::UnsupportedHadamard);+ }+MatrixVectorKernel::Qmv(<B::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(context,@@ -117,45 +126,47 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {let aligned_n_64 = configuration.output_dim % 64 == 0;let is_bf16 = configuration.data_type == DataType::BF16;- let matrix_matrix = if aligned_n_64 && is_bf16 && matches!(configuration.group_size, 64 | 128) {- MatrixMatrixKernel::QmmTransposed64x64(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(- context,- configuration.data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,+ let matrix_matrix =+ if !configuration.use_hadamard && aligned_n_64 && is_bf16 && matches!(configuration.group_size, 64 | 128) {+ MatrixMatrixKernel::QmmTransposed64x64(+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )+ .map_err(QuantizedMatmulError::BackendError)?,)- .map_err(QuantizedMatmulError::BackendError)?,- )- } else if aligned_n_64 && is_bf16 {- MatrixMatrixKernel::QmmTransposedWide(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(- context,- configuration.data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,+ } else if !configuration.use_hadamard && aligned_n_64 && is_bf16 {+ MatrixMatrixKernel::QmmTransposedWide(+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )+ .map_err(QuantizedMatmulError::BackendError)?,)- .map_err(QuantizedMatmulError::BackendError)?,- )- } else {- let aligned_n = configuration.output_dim % 32 == 0;- MatrixMatrixKernel::QmmTransposed(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- context,- configuration.data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- aligned_n,+ } else {+ let aligned_n = configuration.output_dim % 32 == 0;+ MatrixMatrixKernel::QmmTransposed(+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ configuration.use_hadamard,+ aligned_n,+ )+ .map_err(QuantizedMatmulError::BackendError)?,)- .map_err(QuantizedMatmulError::BackendError)?,- )- };+ };Ok(Self {matrix_vector,@@ -177,7 +188,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {};macro_rules! encode_kernel {- ($kernel:expr) => {+ ($kernel:expr, $($hadamard:expr)?) => {$kernel.encode(arguments.b_buffer,arguments.scales_buffer,@@ -185,6 +196,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {biases,(arguments.a_buffer, arguments.a_offset),arguments.output_buffer,+ $($hadamard,)?self.input_dim as u32,self.output_dim as u32,arguments.batch_dim as u32,@@ -195,14 +207,14 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {if arguments.batch_dim < 32 || self.output_dim == 1 {match &self.matrix_vector {- MatrixVectorKernel::Qmv(k) => encode_kernel!(k),- MatrixVectorKernel::QmvFast(k) => encode_kernel!(k),+ MatrixVectorKernel::Qmv(k) => encode_kernel!(k,),+ MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),}} else {match &self.matrix_matrix {- MatrixMatrixKernel::QmmTransposed(k) => encode_kernel!(k),- MatrixMatrixKernel::QmmTransposed64x64(k) => encode_kernel!(k),- MatrixMatrixKernel::QmmTransposedWide(k) => encode_kernel!(k),+ MatrixMatrixKernel::QmmTransposed(k) => encode_kernel!(k, arguments.hadamard_factors),+ MatrixMatrixKernel::QmmTransposed64x64(k) => encode_kernel!(k,),+ MatrixMatrixKernel::QmmTransposedWide(k) => encode_kernel!(k,),}}diff --git a/crates/uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rsnew file mode 100644index 000000000..3343c848a--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs@@ -0,0 +1,16 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(HadamardTransform)]+#[variants(T, f32, f16, bf16)]+pub fn hadamard_transform_mul<T: ArrayElement + Float>(+ #[allow(unused)] data: *mut T,+ #[allow(unused)] factors: *const i32,+ #[allow(unused)] hidden_dim: u32,+ #[allow(unused)] batch_size: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/hadamard_transform/mod.rs b/crates/uzu/src/backends/cpu/kernel/hadamard_transform/mod.rsnew file mode 100644index 000000000..1259c9457--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/hadamard_transform/mod.rs@@ -0,0 +1 @@+pub mod hadamard_transform;diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rsindex 477711f5c..51c8609bf 100644--- a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs+++ b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs@@ -9,10 +9,15 @@ use crate::ArrayElement;pub fn mlp_gate_act_mul<T: ArrayElement + Float>(fused_up: *const T,hidden: *mut T,+ #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,h: i32,m: i32,act_type: crate::backends::common::gpu_types::activation_type::ActivationType,+ #[specialize] use_hadamard: bool,) {+ if use_hadamard {+ unimplemented!("not supported yet");+ }for j in 0..h {for row in 0..m {let base = row * 2 * h;diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rsindex 91bd75de1..1e14e6cd4 100644--- a/crates/uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs@@ -4,6 +4,7 @@ mod attention;mod audio;mod delta_net;mod embedding;+mod hadamard_transform;mod kv_cache_update;mod layer_norm;diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rsindex 5a0374c78..c04bcb55f 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs@@ -95,13 +95,18 @@ pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE#[optional(use_mlx_quant)] biases: Option<*const T>,input: *const T,output: *mut T,+ #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,in_vec_size: u32,out_vec_size: u32,batch_size: u32,#[specialize] use_zero_points: bool,#[specialize] use_mlx_quant: bool,+ #[specialize] use_hadamard: bool,#[specialize] aligned_n: bool,) {+ if use_hadamard {+ unimplemented!("not supported yet");+ }qmm_transposed::<T>(weights,scales,diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rsindex aa05b7ac0..789ee0529 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs@@ -15,12 +15,17 @@ pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32,#[optional(use_mlx_quant)] biases: Option<*const T>,input: *const T,output: *mut T,+ #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,in_vec_size: u32,out_vec_size: u32,batch_size: u32,#[specialize] use_zero_points: bool,#[specialize] use_mlx_quant: bool,+ #[specialize] use_hadamard: bool,) {+ if use_hadamard {+ unimplemented!("not supported yet");+ }qmv::<T>(weights,scales,diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rsindex 05d147995..db26073fa 100644--- a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs@@ -19,6 +19,7 @@ pub fn rms_norm<scales: *const ScaleT,output: *mut OutputT,#[optional(copy_to_shortcut)] shortcut: Option<*mut InputT>,+ #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,batch_size: u32,element_count: u32,epsilon: f32,@@ -27,7 +28,12 @@ pub fn rms_norm<#[specialize] full_layer: bool,#[specialize] copy_to_shortcut: bool,#[specialize] residual_add: bool,+ #[specialize] use_hadamard: bool,) {+ if use_hadamard {+ unimplemented!("not supported yet");+ }+let input = match in_place {true => output as *const InputT,false => input.unwrap(),diff --git a/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.hnew file mode 100644index 000000000..d5f62b642--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h@@ -0,0 +1,30 @@+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/thread_context.h"++using namespace metal;++static METAL_FUNC float simdgroup_hadamard_transform(+ ushort lane_index,+ float lane_value+) {+ for (ushort stride = 1; stride < METAL_SIMD_SIZE; stride <<= 1) {+ float partner_lane_value = simd_shuffle_xor(lane_value, stride);+ lane_value = (lane_index & stride) ? (partner_lane_value - lane_value)+ : (partner_lane_value + lane_value);+ }++ return lane_value / sqrt((float)METAL_SIMD_SIZE);+}++template <typename T>+static METAL_FUNC T simdgroup_random_hadamard_transform(+ ushort lane_index,+ T lane_value,+ int32_t lane_factor+) {+ return T(simdgroup_hadamard_transform(+ lane_index,+ float(lane_value) * float(lane_factor)+ ));+}diff --git a/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metalnew file mode 100644index 000000000..4056b73b5--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal@@ -0,0 +1,27 @@+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/dsl.h"+#include "hadamard_transform.h"++using namespace metal;++template <typename T>+VARIANTS(T, half, bfloat, float)+PUBLIC KERNEL(HadamardTransform)(+ device T* data,+ const device int32_t* factors,+ constant uint& hidden_dim,+ constant uint& batch_size,+ uint block_index GROUPS(hidden_dim.div_ceil(METAL_SIMD_SIZE)),+ uint batch_index GROUPS(batch_size),+ uint lane_index THREADS(METAL_SIMD_SIZE)+) {+ uint factor_index = block_index * METAL_SIMD_SIZE + lane_index;+ uint element_index = batch_index * hidden_dim + factor_index;++ data[element_index] = simdgroup_random_hadamard_transform(+ lane_index,+ data[element_index],+ factors[factor_index]+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex c17c6b07e..157527261 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include "../activation/activations.h"#include "../common/dsl.h"+#include "../hadamard_transform/hadamard_transform.h"using namespace uzu::activation_type;@@ -9,9 +10,11 @@ VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(MlpGateActMul) (const device T* fused_up,device T* hidden,+ const device int32_t* hadamard_factors OPTIONAL(use_hadamard),const constant int& h,const constant int& m,const constant ActivationType& act_type,+ const bool use_hadamard SPECIALIZE,uint j AXIS(h, 64),uint row AXIS(m, 1)) {@@ -19,6 +22,15 @@ PUBLIC KERNEL(MlpGateActMul) (T up = fused_up[base + j];T gate = fused_up[base + h + j];T g = activate(gate, act_type);- float out_f = float(up) * float(g);- hidden[row * h + j] = T(out_f);+ T result = T(float(up) * float(g));++ if (use_hadamard) {+ result = simdgroup_random_hadamard_transform(+ static_cast<ushort>(j % METAL_SIMD_SIZE),+ result,+ hadamard_factors[j]+ );+ }++ hidden[row * h + j] = result;}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalindex 9db3a1769..ab1ee36c0 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -1,5 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"+#include "../hadamard_transform/hadamard_transform.h"#include "quant_matmul.h"template <typename T, uint GROUP_SIZE, uint BITS>@@ -13,6 +14,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(const device T* biases OPTIONAL(use_mlx_quant),const device T* input,device T* output,+ const device int32_t* hadamard_factors OPTIONAL(use_hadamard),const constant uint& in_vec_size,const constant uint& out_vec_size,const constant uint& batch_size,@@ -20,6 +22,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(threadgroup T Ws[32 * (32 + 16 / sizeof(T))],const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,+ const bool use_hadamard SPECIALIZE,const bool aligned_n SPECIALIZE,const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),const uint batch_block_idx GROUPS(batch_size.div_ceil(32)),@@ -103,4 +106,30 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)();}}++ if (use_hadamard) {+ threadgroup_barrier(mem_flags::mem_device);++ constexpr uint BLOCK_SIZE = 32;+ constexpr uint NUM_SIMD_GROUPS = 4;++ constexpr uint ROWS_PER_SIMD_GROUP = BLOCK_SIZE / NUM_SIMD_GROUPS;++ const uint out_col = out_block_idx * BLOCK_SIZE + simd_lane;+ const int32_t factor = hadamard_factors[out_col];++ const uint row_start =+ batch_block_idx * BLOCK_SIZE + simd_group * ROWS_PER_SIMD_GROUP;+ const uint row_end = min(row_start + ROWS_PER_SIMD_GROUP, batch_size);++ for (uint row = row_start; row < row_end; row++) {+ uint output_index = row * out_vec_size + out_col;++ output[output_index] = simdgroup_random_hadamard_transform(+ static_cast<ushort>(simd_lane),+ output[output_index],+ factor+ );+ }+ }}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalindex 9277fc434..4353e23ec 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -1,5 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"+#include "../hadamard_transform/hadamard_transform.h"#include "quant_matmul.h"template <typename T, uint GROUP_SIZE, uint BITS>@@ -13,11 +14,14 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(const device T* biases OPTIONAL(use_mlx_quant),const device T* input,device T* output,+ const device int32_t* hadamard_factors OPTIONAL(use_hadamard),const constant uint& in_vec_size,const constant uint& out_vec_size,const constant uint& batch_size,const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,+ const bool use_hadamard SPECIALIZE,+ threadgroup float shared_results[METAL_SIMD_SIZE],const uint batch_idx GROUPS(batch_size),const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),const uint simd_lane THREADS(32),@@ -144,8 +148,32 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(for (uint row = 0; row < results_per_simdgroup; row++) {result[row] = simd_sum(result[row]);+ }++ if (use_hadamard) {+ if (simd_lane == 0) {+ for (uint row = 0; row < results_per_simdgroup; row++) {+ shared_results[simd_group * results_per_simdgroup + row] = result[row];+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (simd_group == 0) {+ uint global_out_idx = out_block_idx * 32 + simd_lane;+ if (global_out_idx < out_vec_size) {+ output[simd_lane] = simdgroup_random_hadamard_transform(+ static_cast<ushort>(simd_lane),+ T(shared_results[simd_lane]),+ hadamard_factors[global_out_idx]+ );+ }+ }+ } else {if (simd_lane == 0) {- output[row] = static_cast<T>(result[row]);+ for (uint row = 0; row < results_per_simdgroup; row++) {+ output[row] = static_cast<T>(result[row]);+ }}}}diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 553f0c52c..61b9757b2 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -3,6 +3,7 @@#include "../common/dsl.h"#include "../common/thread_context.h"#include "../common/threadgroup_reduce.h"+#include "../hadamard_transform/hadamard_transform.h"using namespace metal;@@ -18,6 +19,7 @@ PUBLIC KERNEL(RMSNorm)(const device ScaleT* scales,device OutputT* output,device InputT* shortcut OPTIONAL(copy_to_shortcut),+ const device int32_t* hadamard_factors OPTIONAL(use_hadamard),constant uint& batch_size,constant uint& element_count,constant float& epsilon,@@ -26,6 +28,7 @@ PUBLIC KERNEL(RMSNorm)(const bool full_layer SPECIALIZE,const bool copy_to_shortcut SPECIALIZE,const bool residual_add SPECIALIZE,+ const bool use_hadamard SPECIALIZE,threadgroup AccumT shared_sum[METAL_SIMD_SIZE],const ThreadContext thread_context,const uint batch_idx GROUPS(batch_size),@@ -91,11 +94,21 @@ PUBLIC KERNEL(RMSNorm)(// If full_layer, normalize and scale in AccumT, cast to OutputT at the end// If not, cast to OutputT after normalize, scale in OutputT+ OutputT val;if (full_layer) {- output[i] = static_cast<OutputT>(x * rms_inv * scale);+ val = static_cast<OutputT>(x * rms_inv * scale);} else {- output[i] =- static_cast<OutputT>(x * rms_inv) * static_cast<OutputT>(scale);+ val = static_cast<OutputT>(x * rms_inv) * static_cast<OutputT>(scale);}++ if (use_hadamard) {+ val = static_cast<OutputT>(simdgroup_random_hadamard_transform(+ static_cast<ushort>(thread_in_row % METAL_SIMD_SIZE),+ val,+ hadamard_factors[i]+ ));+ }++ output[i] = val;}}diff --git a/crates/uzu/src/config/linear.rs b/crates/uzu/src/config/linear.rsindex 287367936..0456dfa34 100644--- a/crates/uzu/src/config/linear.rs+++ b/crates/uzu/src/config/linear.rs@@ -28,6 +28,11 @@ pub enum LinearConfig {lora_rank: usize,lora_scale: f32,},+ #[serde(rename = "RHTLinearWrapperConfig")]+ RHTLinearWrapper {+ block_size: usize,+ inner_config: Box<LinearConfig>,+ },}impl LinearConfig {@@ -42,6 +47,10 @@ impl LinearConfig {quantization,..} => quantization.activation_precision,+ LinearConfig::RHTLinearWrapper {+ inner_config,+ ..+ } => inner_config.activation_precision(),}}}diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex 313a603e6..dd15aaf9d 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -146,7 +146,7 @@ impl<B: Backend> ClassifierLayer<B> {).expect("Failed to create pre-MLP norm kernel");- let mlp = <dyn Mlp<B>>::new(+ let (mlp, mlp_hadamard_factors) = <dyn Mlp<B>>::new(&layer_config.mlp_config,model_dim,hidden_dim,@@ -155,6 +155,8 @@ impl<B: Backend> ClassifierLayer<B> {).expect("Failed to create mlp block");+ assert!(mlp_hadamard_factors.is_none(), "classifier doesn't support hadamard");+let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(Normalization::new(diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex bc5335c07..a7565bc4c 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -187,6 +187,7 @@ impl<B: Backend> Decoder<B> {ArrayId::Main,ArrayId::Main,&decoder_weight_loader.subtree("output_norm").unwrap(),+ None,Some(ArrayId::Shortcut),true,)diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex 35daf60e7..64d119fa3 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -297,6 +297,7 @@ impl<B: Backend> Embedding<B> {output_dim: vocab_size as usize,mode: *embedding_quantization_mode,quantization_type: QuantizedMatmulType::Mlx,+ use_hadamard: false,},)?;@@ -376,6 +377,7 @@ impl<B: Backend> Embedding<B> {output_dim: vocab_size as usize,mode: *embedding_quantization_mode,quantization_type: QuantizedMatmulType::Mlx,+ use_hadamard: false,},)?;@@ -445,6 +447,7 @@ impl<B: Backend> Embedding<B> {output_dim: vocab_size as usize,mode: *embedding_quantization_mode,quantization_type: QuantizedMatmulType::Mlx,+ use_hadamard: false,},)?;@@ -651,6 +654,7 @@ impl<B: Backend> Embedding<B> {scales_buffer: scales,zero_points_or_biases_buffer: biases,output_buffer: output,+ hadamard_factors: None,batch_dim,},)?;diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex 8f58dda5c..1a0c0fa45 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -59,19 +59,7 @@ impl<B: Backend> LayerExecutables<B> {let tensor_add = TensorAddBiasKernel::new(context, intermediate_data_type, false).expect("Failed to create TensorAddBiasKernel kernel"); // TODO: this function return Result- let pre_attention_norm = RMSNorm::new(- context,- intermediate_data_type,- layer_config.pre_attention_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),- Some(ArrayId::Shortcut),- layer_index > 0,- )- .expect("Failed to create RMS norm kernel");-- let mixer = match &layer_config.mixer_config {+ let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {MixerConfig::Attention(attention_config) => {let rope_block = rope.expect("RoPE encoder missing for attention layer");@@ -82,7 +70,7 @@ impl<B: Backend> LayerExecutables<B> {let q_dim = layer_num_heads * layer_head_dim;let kv_dim = layer_num_groups * layer_head_dim;- let qkv_projection = <dyn Linear<B>>::new(+ let (qkv_projection, input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&attention_config.qkv_projection_config,attention_config.has_qkv_biases,model_dim,@@ -161,14 +149,17 @@ impl<B: Backend> LayerExecutables<B> {).expect("Failed to create AttentionWrapper kernel");- MixerExecutables::Attention {- qkv_projection,- gate_projection,- qk_norm,- rope: rope_block,- attention,- out_projection,- }+ (+ MixerExecutables::Attention {+ qkv_projection,+ gate_projection,+ qk_norm,+ rope: rope_block,+ attention,+ out_projection,+ },+ input_hadamard_factors,+ )},MixerConfig::Mamba(mamba_config) => {let mixer = MambaMixer::new(@@ -182,12 +173,15 @@ impl<B: Backend> LayerExecutables<B> {num_groups,decoder_layer_loader,);- MixerExecutables::StateSpace {- mixer,- }+ (+ MixerExecutables::StateSpace {+ mixer,+ },+ None,+ )},MixerConfig::ShortConv(short_conv_config) => {- let mixer = ShortConvMixer::new(+ let (mixer, input_hadamard_factors) = ShortConvMixer::new(context,layer_type.clone(),short_conv_config.clone(),@@ -195,21 +189,39 @@ impl<B: Backend> LayerExecutables<B> {model_dim,decoder_layer_loader,);- MixerExecutables::ShortConv {- mixer,- }+ (+ MixerExecutables::ShortConv {+ mixer,+ },+ input_hadamard_factors,+ )},MixerConfig::DeltaNet(delta_net_config) => {let mixer =DeltaNetMixer::new(context, delta_net_config.clone(), layer_index, model_dim, decoder_layer_loader).expect("Failed to create DeltaNet mixer");-- MixerExecutables::DeltaNet {- mixer,- }+ (+ MixerExecutables::DeltaNet {+ mixer,+ },+ None,+ )},};+ let pre_attention_norm = RMSNorm::new(+ context,+ intermediate_data_type,+ layer_config.pre_attention_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),+ mixer_hadamard_factors,+ Some(ArrayId::Shortcut),+ layer_index > 0,+ )+ .expect("Failed to create RMS norm kernel");+let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {Some(RMSNorm::new(@@ -220,6 +232,7 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,&decoder_layer_loader.subtree("post_mixer_norm").unwrap(),None,+ None,false,).expect("Failed to create RMS norm kernel"),@@ -228,6 +241,15 @@ impl<B: Backend> LayerExecutables<B> {None};+ let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(+ &layer_config.mlp_config,+ model_dim,+ hidden_dim,+ context,+ &decoder_layer_loader.subtree("mlp").unwrap(),+ )+ .expect("Failed to create mlp block");+let pre_mlp_norm = RMSNorm::new(context,intermediate_data_type,@@ -235,20 +257,12 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,ArrayId::Main,&decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),+ mlp_input_hadamard_factors,Some(ArrayId::Shortcut),true,).expect("Failed to create RMS norm kernel");- let mlp = <dyn Mlp<B>>::new(- &layer_config.mlp_config,- model_dim,- hidden_dim,- context,- &decoder_layer_loader.subtree("mlp").unwrap(),- )- .expect("Failed to create mlp block");-let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(RMSNorm::new(@@ -259,6 +273,7 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),None,+ None,false,).expect("Failed to create RMS norm kernel"),diff --git a/crates/uzu/src/encodable_block/linear/mod.rs b/crates/uzu/src/encodable_block/linear/mod.rsindex de530a9e4..b014a1f56 100644--- a/crates/uzu/src/encodable_block/linear/mod.rs+++ b/crates/uzu/src/encodable_block/linear/mod.rs@@ -1,17 +1,19 @@mod full_precision;mod qlora_wrapper;mod quantized;+mod rht_wrapper;pub use full_precision::{FullPrecisionLinear, FullPrecisionLinearError};pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};pub use quantized::{QuantizedLinear, QuantizedLinearError};+pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};use thiserror::Error;use crate::{backends::common::{Backend, Encoder},config::LinearConfig,forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,+ parameters::{ParameterLoaderError, ParameterTree},};pub trait Linear<B: Backend> {@@ -28,8 +30,12 @@ pub enum LinearBlockError<B: Backend> {QuantizedLinearError(#[from] QuantizedLinearError<B>),#[error("FullPrecisionLinear error: {0}")]FullPrecisionLinearError(#[from] FullPrecisionLinearError<B>),+ #[error("RHTLinearWrapper error: {0}")]+ RHTLinearWrapperError(#[from] RHTLinearWrapperError<B>),#[error("QLoRALinearWrapper error: {0}")]QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),+ #[error("Parameter loading error: {0}")]+ ParameterError(#[from] ParameterLoaderError<B>),}impl<B: Backend> dyn Linear<B> {@@ -54,6 +60,7 @@ impl<B: Backend> dyn Linear<B> {parameter_tree,input_array_id,output_array_id,+ None,)?;Ok(Box::new(block))},@@ -84,11 +91,116 @@ impl<B: Backend> dyn Linear<B> {input_dimension,output_dimension_sum,parameter_tree,+ None,input_array_id,output_array_id,)?;Ok(Box::new(block))},+ LinearConfig::RHTLinearWrapper {+ block_size,+ inner_config,+ } => {+ let block = RHTLinearWrapper::new(+ context,+ *block_size,+ inner_config,+ input_dimension,+ output_dimension_sum,+ parameter_tree,+ input_array_id,+ output_array_id,+ )?;+ Ok(Box::new(block))+ },+ }+ }++ pub fn new_with_output_hadamard(+ context: &B::Context,+ config: &LinearConfig,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ parameter_tree: &ParameterTree<B::Context>,+ output_factors: B::Buffer,+ input_dim: usize,+ output_dim: usize,+ ) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {+ match config {+ LinearConfig::Quantized(config) | LinearConfig::MLXQuantized(config) => Ok(Box::new(QuantizedLinear::new(+ context,+ config,+ input_dim,+ output_dim,+ parameter_tree,+ input_array_id,+ output_array_id,+ Some(output_factors),+ )?)),+ LinearConfig::QLoRA {+ quantization,+ lora_rank,+ lora_scale,+ } => Ok(Box::new(QLoRALinearWrapper::new(+ context,+ quantization,+ *lora_rank,+ *lora_scale,+ input_dim,+ output_dim,+ parameter_tree,+ Some(output_factors),+ input_array_id,+ output_array_id,+ )?)),+ inner_config => unimplemented!("{inner_config:?} doesn't support fused output hadamard"),+ }+ }++ pub fn new_extracting_input_hadamard<const N: usize>(+ config: &LinearConfig,+ _has_biases: bool,+ input_dimension: usize,+ output_dimensions: [usize; N],+ context: &B::Context,+ parameter_tree: &ParameterTree<B::Context>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<(Box<dyn Linear<B>>, Option<B::Buffer>), LinearBlockError<B>> {+ let output_dimension_sum: usize = output_dimensions.iter().sum();+ match config {+ LinearConfig::RHTLinearWrapper {+ inner_config,+ ..+ } => {+ let input_factors = parameter_tree.leaf("input_factors")?.read_buffer()?;+ let output_factors = parameter_tree.leaf("output_factors")?.read_buffer()?;+ let inner_tree = parameter_tree.subtree("inner_linear")?;+ let inner_linear = Self::new_with_output_hadamard(+ context,+ inner_config,+ input_array_id,+ output_array_id,+ &inner_tree,+ output_factors,+ input_dimension,+ output_dimension_sum,+ )?;+ Ok((inner_linear, Some(input_factors)))+ },+ other => {+ let linear = Self::new(+ other,+ _has_biases,+ input_dimension,+ output_dimensions,+ context,+ parameter_tree,+ input_array_id,+ output_array_id,+ )?;+ Ok((linear, None))+ },}}}diff --git a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rsindex 37d7eecea..2c4508825 100644--- a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -85,6 +85,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {input_dim: usize,output_dim: usize,parameter_tree: &ParameterTree<B::Context>,+ output_quantized_hadamard_factors: Option<B::Buffer>,input_array_id: ArrayId,output_array_id: ArrayId,) -> Result<Self, QLoRALinearWrapperError<B>> {@@ -98,6 +99,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {parameter_tree,input_array_id,output_array_id,+ output_quantized_hadamard_factors,)?;let adapter_kernel =RefCell::new(<<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)?);diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rsindex 0d235415e..b2501e9b7 100644--- a/crates/uzu/src/encodable_block/linear/quantized.rs+++ b/crates/uzu/src/encodable_block/linear/quantized.rs@@ -92,6 +92,7 @@ pub struct QuantizedLinear<B: Backend> {weights_buffer: Rc<RefCell<B::Buffer>>,scales_buffer: Rc<RefCell<B::Buffer>>,zero_points_or_biases_buffer: Rc<RefCell<B::Buffer>>,+ output_hadamard_factors: Option<B::Buffer>,output_dim: usize,input_array_id: ArrayId,output_array_id: ArrayId,@@ -106,6 +107,7 @@ impl<B: Backend> QuantizedLinear<B> {parameter_tree: &ParameterTree<B::Context>,input_array_id: ArrayId,output_array_id: ArrayId,+ output_hadamard_factors: Option<B::Buffer>,) -> Result<Self, QuantizedLinearError<B>> {let kernel_data_type: DataType = config.activation_precision.into();if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {@@ -220,6 +222,7 @@ impl<B: Backend> QuantizedLinear<B> {output_dim,mode: config.weight_quantization_mode,quantization_type,+ use_hadamard: output_hadamard_factors.is_some(),},).map_err(QuantizedLinearError::QuantizedMatmulError)?;@@ -231,6 +234,7 @@ impl<B: Backend> QuantizedLinear<B> {weights_buffer: weights.buffer(),scales_buffer: scales.buffer(),zero_points_or_biases_buffer,+ output_hadamard_factors,output_dim,input_array_id,output_array_id,@@ -261,6 +265,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {scales_buffer: self.scales_buffer.borrow().deref(),zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),output_buffer: output_buf_borrow.deref_mut(),+ hadamard_factors: self.output_hadamard_factors.as_ref(),batch_dim,},)diff --git a/crates/uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu/src/encodable_block/linear/rht_wrapper.rsnew file mode 100644index 000000000..7151bb0c7--- /dev/null+++ b/crates/uzu/src/encodable_block/linear/rht_wrapper.rs@@ -0,0 +1,101 @@+use std::ops::DerefMut;++use thiserror::Error;++use super::{Linear, LinearBlockError};+use crate::{+ backends::common::{+ Backend, Encoder,+ kernel::{HadamardTransformKernel, Kernels},+ },+ config::LinearConfig,+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum RHTLinearWrapperError<B: Backend> {+ #[error("Inner linear error: {0}")]+ InnerLinearError(#[from] Box<LinearBlockError<B>>),+ #[error("Parameter loading error: {0}")]+ ParameterError(#[from] ParameterLoaderError<B>),+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+}++pub struct RHTLinearWrapper<B: Backend> {+ input_hadamard_kernel: <B::Kernels as Kernels>::HadamardTransformKernel,+ input_factors: B::Buffer,+ inner_linear: Box<dyn Linear<B>>,+ input_dimension: usize,+ input_array_id: ArrayId,+}++impl<B: Backend> RHTLinearWrapper<B> {+ pub fn new(+ context: &B::Context,+ block_size: usize,+ inner_config: &LinearConfig,+ input_dimension: usize,+ output_dimension: usize,+ parameter_tree: &ParameterTree<B::Context>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Self, RHTLinearWrapperError<B>> {+ assert!(block_size == 32, "only block size 32 hadamard is supported");++ let data_type = inner_config.activation_precision().into();++ let input_factors_leaf = parameter_tree.leaf("input_factors")?;+ let output_factors_leaf = parameter_tree.leaf("output_factors")?;++ let input_factors = input_factors_leaf.read_buffer()?;+ let output_factors = output_factors_leaf.read_buffer()?;++ let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)+ .map_err(RHTLinearWrapperError::BackendError)?;++ let inner_linear_tree = parameter_tree.subtree("inner_linear")?;++ let inner_linear = <dyn Linear<B>>::new_with_output_hadamard(+ context,+ inner_config,+ input_array_id,+ output_array_id,+ &inner_linear_tree,+ output_factors,+ input_dimension,+ output_dimension,+ )+ .map_err(|e| RHTLinearWrapperError::InnerLinearError(Box::new(e)))?;++ Ok(Self {+ input_hadamard_kernel,+ input_factors,+ inner_linear,+ input_dimension,+ input_array_id,+ })+ }+}++impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {+ fn encode(+ &self,+ state: &mut ForwardPassState<B>,+ encoder: &mut Encoder<B>,+ ) -> Result<(), B::Error> {+ // Input Hadamard+ let input_array = state.array(self.input_array_id);+ self.input_hadamard_kernel.encode(+ input_array.buffer().borrow_mut().deref_mut(),+ &self.input_factors,+ self.input_dimension as u32,+ state.active_row_count() as u32,+ encoder,+ );++ // Matmul with fused output Hadamard+ self.inner_linear.encode(state, encoder)+ }+}diff --git a/crates/uzu/src/encodable_block/mlp/mod.rs b/crates/uzu/src/encodable_block/mlp/mod.rsindex 23cb57405..f2e4046d4 100644--- a/crates/uzu/src/encodable_block/mlp/mod.rs+++ b/crates/uzu/src/encodable_block/mlp/mod.rs@@ -41,11 +41,11 @@ impl<B: Backend> dyn Mlp<B> {hidden_dimension: usize,context: &B::Context,parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Box<dyn Mlp<B>>, MlpBlockError<B>> {+ ) -> Result<(Box<dyn Mlp<B>>, Option<B::Buffer>), MlpBlockError<B>> {if let MLPConfig::Dense(dense_config) = config {let data_type: DataType = dense_config.linear_config.activation_precision().into();- let up_projection = <dyn Linear<B>>::new(+ let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&dense_config.linear_config,false,model_dimension,@@ -56,11 +56,7 @@ impl<B: Backend> dyn Mlp<B> {ArrayId::MlpFusedUp,)?;- let gate_activation =- MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)- .map_err(MlpBlockError::BackendError)?;-- let down_projection = <dyn Linear<B>>::new(+ let (down_projection, down_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&dense_config.linear_config,false,hidden_dimension,@@ -71,13 +67,22 @@ impl<B: Backend> dyn Mlp<B> {ArrayId::Main,)?;- return Ok(Box::new(DenseMlp::new(up_projection, gate_activation, down_projection)));+ let gate = MlpGateActMulEncodable::new(+ context,+ data_type,+ dense_config.activation.clone(),+ hidden_dimension,+ down_input_hadamard_factors,+ )+ .map_err(MlpBlockError::BackendError)?;++ return Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors));}if let MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {let mixture_of_experts_block =MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;- return Ok(Box::new(mixture_of_experts_block));+ return Ok((Box::new(mixture_of_experts_block), None));}unreachable!("Unknown MLP config")diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rsindex a14a00f7b..af06a3fbd 100644--- a/crates/uzu/src/encodable_block/mlp/moe.rs+++ b/crates/uzu/src/encodable_block/mlp/moe.rs@@ -105,6 +105,11 @@ impl<B: Backend> MoeBlock<B> {} => {unimplemented!("QLoRA router not yet supported for MoE");},+ LinearConfig::RHTLinearWrapper {+ ..+ } => {+ unimplemented!("RHTLinearWrapper router not yet supported for MoE");+ },};let router_topk_kernel = <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, router_data_type)diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rsindex 1c6fcf34b..d2349e401 100644--- a/crates/uzu/src/encodable_block/normalization.rs+++ b/crates/uzu/src/encodable_block/normalization.rs@@ -54,6 +54,7 @@ impl<B: Backend> Normalization<B> {output_array_id,parameter_tree,None,+ None,false,)?))}diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex 7e06ae5de..71f5462e5 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -34,6 +34,7 @@ pub struct RMSNorm<B: Backend> {output_array_id: ArrayId,shortcut_array_id: Option<ArrayId>,scales_buffer: Rc<RefCell<B::Buffer>>,+ hadamard_factors_buffer: Option<B::Buffer>,use_sampling_range: bool,}@@ -45,6 +46,7 @@ impl<B: Backend> RMSNorm<B> {input_array_id: ArrayId,output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,+ hadamard_factors_buffer: Option<B::Buffer>,shortcut_array_id: Option<ArrayId>,residual_add: bool,) -> Result<Self, RMSNormError<B>> {@@ -68,6 +70,7 @@ impl<B: Backend> RMSNorm<B> {config.upcast_mode == UpcastMode::FullLayer,shortcut_array_id.is_some(),residual_add,+ hadamard_factors_buffer.is_some(),).map_err(RMSNormError::BackendError)?;@@ -78,6 +81,7 @@ impl<B: Backend> RMSNorm<B> {output_array_id,shortcut_array_id,scales_buffer: scales.buffer(),+ hadamard_factors_buffer,use_sampling_range: false,})}@@ -131,6 +135,7 @@ impl<B: Backend> RMSNorm<B> {self.scales_buffer.borrow().deref(),(output_array.buffer().borrow_mut().deref_mut(), output_offset),shortcut_borrow.as_deref_mut().map(|b| (b, input_offset)),+ self.hadamard_factors_buffer.as_ref(),batch_len as u32,element_count as u32,self.config.epsilon,diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex 2623867f6..c429fcc8f 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -35,7 +35,7 @@ impl<B: Backend> ShortConvMixer<B> {layer_index: usize,model_dim: usize,decoder_layer_loader: &ParameterTree<B::Context>,- ) -> Self {+ ) -> (Self, Option<B::Buffer>) {if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);}@@ -45,7 +45,7 @@ impl<B: Backend> ShortConvMixer<B> {let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();- let in_projection = <dyn Linear<B>>::new(+ let (in_projection, in_proj_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&short_conv_config.in_projection_config,false,model_dim,@@ -86,19 +86,22 @@ impl<B: Backend> ShortConvMixer<B> {let short_conv_trie = <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias).expect("Failed to create short conv trie kernel");- Self {- layer_index,- config: short_conv_config,- model_dim,- in_projection,- out_projection,- short_conv_pack,- short_conv_prefill,- short_conv_decode,- short_conv_trie,- conv_weight,- conv_bias,- }+ (+ Self {+ layer_index,+ config: short_conv_config,+ model_dim,+ in_projection,+ out_projection,+ short_conv_pack,+ short_conv_prefill,+ short_conv_decode,+ short_conv_trie,+ conv_weight,+ conv_bias,+ },+ in_proj_input_hadamard_factors,+ )}fn clear_suffix_state_valid_range(diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rsindex fae070856..70e7de14c 100644--- a/crates/uzu/src/forward_pass/model_shape.rs+++ b/crates/uzu/src/forward_pass/model_shape.rs@@ -147,6 +147,12 @@ impl ModelShape {} => {max_lora_intermediate = Some(max_lora_intermediate.unwrap_or(0).max(*lora_rank));},+ LinearConfig::RHTLinearWrapper {+ inner_config,+ ..+ } => {+ linear_configs.push(inner_config.as_ref());+ },_ => (),};}diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rsindex 3121a5f82..b64afcee1 100644--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs@@ -342,6 +342,7 @@ fn execute_quantized_matmul<B: Backend>(_ => panic!("Unsupported bits: {}", bits),},quantization_type,+ use_hadamard: false,},).unwrap();@@ -355,6 +356,7 @@ fn execute_quantized_matmul<B: Backend>(scales_buffer: &s_buf,zero_points_or_biases_buffer: &b_buf,output_buffer: &mut y_buf,+ hadamard_factors: None,batch_dim: batch,};let mut encoder = Encoder::new(ctx).unwrap();@@ -372,6 +374,7 @@ fn execute_quantized_matmul<B: Backend>(scales_buffer: &s_buf,zero_points_or_biases_buffer: &b_buf,output_buffer: &mut y_buf,+ hadamard_factors: None,batch_dim: batch,};let mut encoder = Encoder::new(ctx).unwrap();diff --git a/crates/uzu/tests/unit/config/linear_test.rs b/crates/uzu/tests/unit/config/linear_test.rsindex 844af1a4e..9c3fc37a1 100644--- a/crates/uzu/tests/unit/config/linear_test.rs+++ b/crates/uzu/tests/unit/config/linear_test.rs@@ -30,3 +30,33 @@ fn test_linear_config() {let deserialized_config: LinearConfig = from_str(config_str).unwrap();assert_eq!(deserialized_config, ground_truth_config);}++#[test]+fn test_rht_linear_wrapper_config() {+ let config_str = r#"+ {+ "type": "RHTLinearWrapperConfig",+ "block_size": 32,+ "inner_config": {+ "type": "GroupQuantizedLinearConfig",+ "group_size": 32,+ "weight_quantization_mode": "uint4",+ "activation_quantization_mode": null,+ "activation_precision": "bfloat16"+ }+ }+ "#;++ let ground_truth_config = LinearConfig::RHTLinearWrapper {+ block_size: 32,+ inner_config: Box::new(LinearConfig::Quantized(QuantizationConfig {+ group_size: 32,+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: None,+ activation_precision: ConfigDataType::BFloat16,+ })),+ };++ let deserialized_config: LinearConfig = from_str(config_str).unwrap();+ assert_eq!(deserialized_config, ground_truth_config);+}diff --git a/crates/uzu/tests/unit/kernel/activation_mul_test.rs b/crates/uzu/tests/unit/kernel/activation_mul_test.rsindex e00b64974..5fdfbd68a 100644--- a/crates/uzu/tests/unit/kernel/activation_mul_test.rs+++ b/crates/uzu/tests/unit/kernel/activation_mul_test.rs@@ -47,7 +47,7 @@ fn get_test_data<T: ArrayElement + Float>(act_type: ActivationType) -> (Input<T>fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::MlpGateActMulKernel::new(&context, T::data_type())+ let kernel = <<B as Backend>::Kernels as Kernels>::MlpGateActMulKernel::new(&context, T::data_type(), false).expect("Failed to create MlpGateActMulKernel");let fused_len = (input.m * 2 * input.h) as usize;@@ -59,6 +59,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {kernel.encode(fused_up_array.buffer().borrow().deref(),hidden_array.buffer().borrow_mut().deref_mut(),+ None::<&B::Buffer>,input.h,input.m,input.act_type,diff --git a/crates/uzu/tests/unit/kernel/hadamard_transform_test.rs b/crates/uzu/tests/unit/kernel/hadamard_transform_test.rsnew file mode 100644index 000000000..be60407c1--- /dev/null+++ b/crates/uzu/tests/unit/kernel/hadamard_transform_test.rs@@ -0,0 +1,162 @@+use std::{+ fmt::Debug,+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement,+ backends::common::{Backend, Context, Encoder, Kernels, kernel::HadamardTransformKernel},+};++use crate::uzu_test;++const BLOCK_SIZE: usize = 32;++fn reference_hadamard_transform_mul(+ data: &[f64],+ factors: &[f64],+ channel_count: usize,+) -> Vec<f64> {+ let batch_count = data.len() / channel_count;+ let normalization_factor = 1.0 / (BLOCK_SIZE as f64).sqrt();+ let mut result = data.to_vec();++ for batch_index in 0..batch_count {+ let batch_offset = batch_index * channel_count;++ for block_start in (0..channel_count).step_by(BLOCK_SIZE) {+ for lane in 0..BLOCK_SIZE {+ let index = batch_offset + block_start + lane;+ let factor_index = block_start + lane;+ result[index] *= factors[factor_index];+ }++ let mut stride = 1;+ while stride < BLOCK_SIZE {+ for pair_start in (0..BLOCK_SIZE).step_by(stride * 2) {+ for offset in 0..stride {+ let index_a = batch_offset + block_start + pair_start + offset;+ let index_b = index_a + stride;+ let sum = result[index_a] + result[index_b];+ let difference = result[index_a] - result[index_b];+ result[index_a] = sum;+ result[index_b] = difference;+ }+ }+ stride *= 2;+ }++ for lane in 0..BLOCK_SIZE {+ let index = batch_offset + block_start + lane;+ result[index] *= normalization_factor;+ }+ }+ }++ result+}++struct TestInput<T: ArrayElement + Float> {+ data: Box<[T]>,+ factors: Box<[i32]>,+ channel_count: usize,+ batch_count: usize,+}++fn generate_test_input<T: ArrayElement + Float>(+ batch_count: usize,+ channel_count: usize,+) -> (TestInput<T>, Vec<f64>) {+ let total_elements = batch_count * channel_count;++ let data_f64: Vec<f64> = (0..total_elements).map(|index| ((index as f64) * 0.1).sin() * 2.0).collect();++ let factors_i32: Vec<i32> = (0..channel_count)+ .map(|index| {+ if index % 3 == 0 {+ -1+ } else {+ 1+ }+ })+ .collect();++ let factors_f64: Vec<f64> = factors_i32.iter().map(|&v| v as f64).collect();+ let expected = reference_hadamard_transform_mul(&data_f64, &factors_f64, channel_count);++ let data: Vec<T> = data_f64.iter().map(|&value| T::from(value).unwrap()).collect();++ let input = TestInput {+ data: data.into_boxed_slice(),+ factors: factors_i32.into_boxed_slice(),+ channel_count,+ batch_count,+ };++ (input, expected)+}++fn run_kernel<T: ArrayElement + Float, B: Backend>(input: &TestInput<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create context");++ let kernel = <<B as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(&context, T::data_type())+ .expect("Failed to create HadamardTransformKernel");++ let total_elements = input.batch_count * input.channel_count;+ let data_array = context.create_array_from(&[total_elements], &input.data, "data");+ let factors_array = context.create_array_from::<i32>(&[input.channel_count], &input.factors, "factors");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ data_array.buffer().borrow_mut().deref_mut(),+ factors_array.buffer().borrow().deref(),+ input.channel_count as u32,+ input.batch_count as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ data_array.as_slice().to_vec()+}++fn test_hadamard_transform<T: ArrayElement + Float + Debug>(tolerance: f64) {+ let test_cases = [(1, 32), (1, 64), (1, 128), (4, 32), (4, 256), (2, 2048)];++ for (batch_count, channel_count) in test_cases {+ let (input, expected) = generate_test_input::<T>(batch_count, channel_count);++ for_each_non_cpu_backend!(|B| {+ let actual = run_kernel::<T, B>(&input);+ assert_eq!(actual.len(), expected.len());++ for (index, (actual_value, &expected_value)) in actual.iter().zip(expected.iter()).enumerate() {+ let actual_f64 = actual_value.to_f64().unwrap();+ let error = (actual_f64 - expected_value).abs();+ let relative_bound = expected_value.abs() * tolerance;+ let absolute_bound = tolerance;+ assert!(+ error <= relative_bound.max(absolute_bound),+ "Mismatch at index {index} for batch_count={batch_count}, channel_count={channel_count}: \+ actual={actual_f64}, expected={expected_value}, error={error}"+ );+ }+ });+ }+}++#[uzu_test]+fn test_f32() {+ test_hadamard_transform::<f32>(1e-4);+}++#[uzu_test]+fn test_f16() {+ test_hadamard_transform::<f16>(0.05);+}++#[uzu_test]+fn test_bf16() {+ test_hadamard_transform::<bf16>(0.1);+}diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rsindex dc6e2b888..e8c5db0e4 100644--- a/crates/uzu/tests/unit/kernel/mod.rs+++ b/crates/uzu/tests/unit/kernel/mod.rs@@ -8,6 +8,7 @@ mod attention;mod audio;mod delta_net_test;mod embedding;+mod hadamard_transform_test;mod kv_cache_update_test;mod layer_norm_test;mod matmul;diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rsindex db6185dd3..fcbb00756 100644--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs@@ -23,6 +23,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {input.bits,input.use_zero_points,input.use_mlx_quant,+ false,aligned_n,).expect("Failed to create QuantizedMatmulQmmTransposedKernel");@@ -45,6 +46,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {bias_buf.as_ref(),&x_buf,&mut y_buf,+ None::<&B::Buffer>,input.k,input.n,input.m,diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsindex 34ab9a9db..e9583aff4 100644--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -84,6 +84,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {input.bits,input.use_zero_points,input.use_mlx_quant,+ false,).expect("Failed to create QuantizedMatmulQmvFastKernel");kernel.encode(@@ -93,6 +94,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {bias_buf.as_ref(),&x_buf,&mut y_buf,+ None::<&B::Buffer>,input.k,input.n,input.m,@@ -341,6 +343,7 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(bits,use_zero_points,use_mlx_quant,+ false,).unwrap();@@ -391,6 +394,7 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(bias_buf.as_ref(),&x_buf,&mut y_buf,+ None::<&B::Buffer>,k as u32,n as u32,m as u32,diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rsindex 11a648738..459ed5689 100644--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs@@ -133,6 +133,7 @@ fn get_output<input.full_layer,false,false,+ false,).expect("Failed to create RMSNormKernel");@@ -155,6 +156,7 @@ fn get_output<scales_array.buffer().borrow().deref(),output_array.buffer().borrow_mut().deref_mut(),None::<&mut B::Buffer>,+ None::<&B::Buffer>,input.batch_size,input.element_count,input.epsilon,@@ -407,6 +409,7 @@ fn bench_rms_norm(c: &mut Criterion) {false,false,false,+ false,).unwrap();@@ -453,6 +456,7 @@ fn bench_rms_norm(c: &mut Criterion) {&scales_buffer,&mut output_buffer,None::<&mut <B as Backend>::Buffer>,+ None::<&<B as Backend>::Buffer>,batch_size as u32,model_dim as u32,epsilon,