Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

RHT (#277)

eugene32655c2db262
diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
index 0561ac39f..6e34368ff 100644
--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
@@ -9,6 +9,7 @@ pub struct MlpGateActMulEncodable<B: Backend> {
kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,
activation: ActivationConfig,
hidden_dim: usize,
+ hadamard_factors: Option<B::Buffer>,
}
impl<B: Backend> MlpGateActMulEncodable<B> {
@@ -17,12 +18,14 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
data_type: DataType,
activation: ActivationConfig,
hidden_dim: usize,
+ hadamard_factors: Option<B::Buffer>,
) -> Result<Self, B::Error> {
- let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type)?;
+ let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type, hadamard_factors.is_some())?;
Ok(Self {
kernel,
activation,
hidden_dim,
+ hadamard_factors,
})
}
@@ -36,7 +39,15 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
}
- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, self.activation.act_type(), encoder);
+ self.kernel.encode(
+ fused_up,
+ hidden,
+ self.hadamard_factors.as_ref(),
+ self.hidden_dim as i32,
+ m,
+ self.activation.act_type(),
+ encoder,
+ );
Ok(())
}
}
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index 63f31070e..56c7c6895 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -18,6 +18,8 @@ pub enum QuantizedMatmulError<B: Backend> {
UnsupportedDataType(DataType),
#[error("Unsupported group size: {0}")]
UnsupportedGroupSize(usize),
+ #[error("Hadamard not supported for this kernel configuration")]
+ UnsupportedHadamard,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -34,6 +36,7 @@ pub struct QuantizedMatmulConfiguration {
pub output_dim: usize,
pub mode: QuantizationMode,
pub quantization_type: QuantizedMatmulType,
+ pub use_hadamard: bool,
}
pub struct QuantizedMatmulArguments<'a, B: Backend> {
@@ -43,6 +46,7 @@ pub struct QuantizedMatmulArguments<'a, B: Backend> {
pub scales_buffer: &'a B::Buffer,
pub zero_points_or_biases_buffer: &'a B::Buffer,
pub output_buffer: &'a mut B::Buffer,
+ pub hadamard_factors: Option<&'a B::Buffer>,
pub batch_dim: usize,
}
@@ -96,10 +100,15 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
bits,
use_zero_points,
use_mlx_quant,
+ configuration.use_hadamard,
)
.map_err(QuantizedMatmulError::BackendError)?,
)
} else {
+ if configuration.use_hadamard {
+ return Err(QuantizedMatmulError::UnsupportedHadamard);
+ }
+
MatrixVectorKernel::Qmv(
<B::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
context,
@@ -117,45 +126,47 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
let aligned_n_64 = configuration.output_dim % 64 == 0;
let is_bf16 = configuration.data_type == DataType::BF16;
- let matrix_matrix = if aligned_n_64 && is_bf16 && matches!(configuration.group_size, 64 | 128) {
- MatrixMatrixKernel::QmmTransposed64x64(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
+ let matrix_matrix =
+ if !configuration.use_hadamard && aligned_n_64 && is_bf16 && matches!(configuration.group_size, 64 | 128) {
+ MatrixMatrixKernel::QmmTransposed64x64(
+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
)
- .map_err(QuantizedMatmulError::BackendError)?,
- )
- } else if aligned_n_64 && is_bf16 {
- MatrixMatrixKernel::QmmTransposedWide(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
+ } else if !configuration.use_hadamard && aligned_n_64 && is_bf16 {
+ MatrixMatrixKernel::QmmTransposedWide(
+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
)
- .map_err(QuantizedMatmulError::BackendError)?,
- )
- } else {
- let aligned_n = configuration.output_dim % 32 == 0;
- MatrixMatrixKernel::QmmTransposed(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- aligned_n,
+ } else {
+ let aligned_n = configuration.output_dim % 32 == 0;
+ MatrixMatrixKernel::QmmTransposed(
+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ configuration.use_hadamard,
+ aligned_n,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
)
- .map_err(QuantizedMatmulError::BackendError)?,
- )
- };
+ };
Ok(Self {
matrix_vector,
@@ -177,7 +188,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
};
macro_rules! encode_kernel {
- ($kernel:expr) => {
+ ($kernel:expr, $($hadamard:expr)?) => {
$kernel.encode(
arguments.b_buffer,
arguments.scales_buffer,
@@ -185,6 +196,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
biases,
(arguments.a_buffer, arguments.a_offset),
arguments.output_buffer,
+ $($hadamard,)?
self.input_dim as u32,
self.output_dim as u32,
arguments.batch_dim as u32,
@@ -195,14 +207,14 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
if arguments.batch_dim < 32 || self.output_dim == 1 {
match &self.matrix_vector {
- MatrixVectorKernel::Qmv(k) => encode_kernel!(k),
- MatrixVectorKernel::QmvFast(k) => encode_kernel!(k),
+ MatrixVectorKernel::Qmv(k) => encode_kernel!(k,),
+ MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),
}
} else {
match &self.matrix_matrix {
- MatrixMatrixKernel::QmmTransposed(k) => encode_kernel!(k),
- MatrixMatrixKernel::QmmTransposed64x64(k) => encode_kernel!(k),
- MatrixMatrixKernel::QmmTransposedWide(k) => encode_kernel!(k),
+ MatrixMatrixKernel::QmmTransposed(k) => encode_kernel!(k, arguments.hadamard_factors),
+ MatrixMatrixKernel::QmmTransposed64x64(k) => encode_kernel!(k,),
+ MatrixMatrixKernel::QmmTransposedWide(k) => encode_kernel!(k,),
}
}
diff --git a/crates/uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
new file mode 100644
index 000000000..3343c848a
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
@@ -0,0 +1,16 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(HadamardTransform)]
+#[variants(T, f32, f16, bf16)]
+pub fn hadamard_transform_mul<T: ArrayElement + Float>(
+ #[allow(unused)] data: *mut T,
+ #[allow(unused)] factors: *const i32,
+ #[allow(unused)] hidden_dim: u32,
+ #[allow(unused)] batch_size: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/hadamard_transform/mod.rs b/crates/uzu/src/backends/cpu/kernel/hadamard_transform/mod.rs
new file mode 100644
index 000000000..1259c9457
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/hadamard_transform/mod.rs
@@ -0,0 +1 @@
+pub mod hadamard_transform;
diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
index 477711f5c..51c8609bf 100644
--- a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
@@ -9,10 +9,15 @@ use crate::ArrayElement;
pub fn mlp_gate_act_mul<T: ArrayElement + Float>(
fused_up: *const T,
hidden: *mut T,
+ #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
h: i32,
m: i32,
act_type: crate::backends::common::gpu_types::activation_type::ActivationType,
+ #[specialize] use_hadamard: bool,
) {
+ if use_hadamard {
+ unimplemented!("not supported yet");
+ }
for j in 0..h {
for row in 0..m {
let base = row * 2 * h;
diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rs
index 91bd75de1..1e14e6cd4 100644
--- a/crates/uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs
@@ -4,6 +4,7 @@ mod attention;
mod audio;
mod delta_net;
mod embedding;
+mod hadamard_transform;
mod kv_cache_update;
mod layer_norm;
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
index 5a0374c78..c04bcb55f 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
@@ -95,13 +95,18 @@ pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE
#[optional(use_mlx_quant)] biases: Option<*const T>,
input: *const T,
output: *mut T,
+ #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
in_vec_size: u32,
out_vec_size: u32,
batch_size: u32,
#[specialize] use_zero_points: bool,
#[specialize] use_mlx_quant: bool,
+ #[specialize] use_hadamard: bool,
#[specialize] aligned_n: bool,
) {
+ if use_hadamard {
+ unimplemented!("not supported yet");
+ }
qmm_transposed::<T>(
weights,
scales,
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
index aa05b7ac0..789ee0529 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
@@ -15,12 +15,17 @@ pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32,
#[optional(use_mlx_quant)] biases: Option<*const T>,
input: *const T,
output: *mut T,
+ #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
in_vec_size: u32,
out_vec_size: u32,
batch_size: u32,
#[specialize] use_zero_points: bool,
#[specialize] use_mlx_quant: bool,
+ #[specialize] use_hadamard: bool,
) {
+ if use_hadamard {
+ unimplemented!("not supported yet");
+ }
qmv::<T>(
weights,
scales,
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
index 05d147995..db26073fa 100644
--- a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
@@ -19,6 +19,7 @@ pub fn rms_norm<
scales: *const ScaleT,
output: *mut OutputT,
#[optional(copy_to_shortcut)] shortcut: Option<*mut InputT>,
+ #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
batch_size: u32,
element_count: u32,
epsilon: f32,
@@ -27,7 +28,12 @@ pub fn rms_norm<
#[specialize] full_layer: bool,
#[specialize] copy_to_shortcut: bool,
#[specialize] residual_add: bool,
+ #[specialize] use_hadamard: bool,
) {
+ if use_hadamard {
+ unimplemented!("not supported yet");
+ }
+
let input = match in_place {
true => output as *const InputT,
false => input.unwrap(),
diff --git a/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
new file mode 100644
index 000000000..d5f62b642
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
@@ -0,0 +1,30 @@
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../common/thread_context.h"
+
+using namespace metal;
+
+static METAL_FUNC float simdgroup_hadamard_transform(
+ ushort lane_index,
+ float lane_value
+) {
+ for (ushort stride = 1; stride < METAL_SIMD_SIZE; stride <<= 1) {
+ float partner_lane_value = simd_shuffle_xor(lane_value, stride);
+ lane_value = (lane_index & stride) ? (partner_lane_value - lane_value)
+ : (partner_lane_value + lane_value);
+ }
+
+ return lane_value / sqrt((float)METAL_SIMD_SIZE);
+}
+
+template <typename T>
+static METAL_FUNC T simdgroup_random_hadamard_transform(
+ ushort lane_index,
+ T lane_value,
+ int32_t lane_factor
+) {
+ return T(simdgroup_hadamard_transform(
+ lane_index,
+ float(lane_value) * float(lane_factor)
+ ));
+}
diff --git a/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
new file mode 100644
index 000000000..4056b73b5
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
@@ -0,0 +1,27 @@
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "hadamard_transform.h"
+
+using namespace metal;
+
+template <typename T>
+VARIANTS(T, half, bfloat, float)
+PUBLIC KERNEL(HadamardTransform)(
+ device T* data,
+ const device int32_t* factors,
+ constant uint& hidden_dim,
+ constant uint& batch_size,
+ uint block_index GROUPS(hidden_dim.div_ceil(METAL_SIMD_SIZE)),
+ uint batch_index GROUPS(batch_size),
+ uint lane_index THREADS(METAL_SIMD_SIZE)
+) {
+ uint factor_index = block_index * METAL_SIMD_SIZE + lane_index;
+ uint element_index = batch_index * hidden_dim + factor_index;
+
+ data[element_index] = simdgroup_random_hadamard_transform(
+ lane_index,
+ data[element_index],
+ factors[factor_index]
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index c17c6b07e..157527261 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include "../activation/activations.h"
#include "../common/dsl.h"
+#include "../hadamard_transform/hadamard_transform.h"
using namespace uzu::activation_type;
@@ -9,9 +10,11 @@ VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(MlpGateActMul) (
const device T* fused_up,
device T* hidden,
+ const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
const constant int& h,
const constant int& m,
const constant ActivationType& act_type,
+ const bool use_hadamard SPECIALIZE,
uint j AXIS(h, 64),
uint row AXIS(m, 1)
) {
@@ -19,6 +22,15 @@ PUBLIC KERNEL(MlpGateActMul) (
T up = fused_up[base + j];
T gate = fused_up[base + h + j];
T g = activate(gate, act_type);
- float out_f = float(up) * float(g);
- hidden[row * h + j] = T(out_f);
+ T result = T(float(up) * float(g));
+
+ if (use_hadamard) {
+ result = simdgroup_random_hadamard_transform(
+ static_cast<ushort>(j % METAL_SIMD_SIZE),
+ result,
+ hadamard_factors[j]
+ );
+ }
+
+ hidden[row * h + j] = result;
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
index 9db3a1769..ab1ee36c0 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -1,5 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
+#include "../hadamard_transform/hadamard_transform.h"
#include "quant_matmul.h"
template <typename T, uint GROUP_SIZE, uint BITS>
@@ -13,6 +14,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
const device T* biases OPTIONAL(use_mlx_quant),
const device T* input,
device T* output,
+ const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
const constant uint& in_vec_size,
const constant uint& out_vec_size,
const constant uint& batch_size,
@@ -20,6 +22,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
threadgroup T Ws[32 * (32 + 16 / sizeof(T))],
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
+ const bool use_hadamard SPECIALIZE,
const bool aligned_n SPECIALIZE,
const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),
const uint batch_block_idx GROUPS(batch_size.div_ceil(32)),
@@ -103,4 +106,30 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
);
}
}
+
+ if (use_hadamard) {
+ threadgroup_barrier(mem_flags::mem_device);
+
+ constexpr uint BLOCK_SIZE = 32;
+ constexpr uint NUM_SIMD_GROUPS = 4;
+
+ constexpr uint ROWS_PER_SIMD_GROUP = BLOCK_SIZE / NUM_SIMD_GROUPS;
+
+ const uint out_col = out_block_idx * BLOCK_SIZE + simd_lane;
+ const int32_t factor = hadamard_factors[out_col];
+
+ const uint row_start =
+ batch_block_idx * BLOCK_SIZE + simd_group * ROWS_PER_SIMD_GROUP;
+ const uint row_end = min(row_start + ROWS_PER_SIMD_GROUP, batch_size);
+
+ for (uint row = row_start; row < row_end; row++) {
+ uint output_index = row * out_vec_size + out_col;
+
+ output[output_index] = simdgroup_random_hadamard_transform(
+ static_cast<ushort>(simd_lane),
+ output[output_index],
+ factor
+ );
+ }
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
index 9277fc434..4353e23ec 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -1,5 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
+#include "../hadamard_transform/hadamard_transform.h"
#include "quant_matmul.h"
template <typename T, uint GROUP_SIZE, uint BITS>
@@ -13,11 +14,14 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
const device T* biases OPTIONAL(use_mlx_quant),
const device T* input,
device T* output,
+ const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
const constant uint& in_vec_size,
const constant uint& out_vec_size,
const constant uint& batch_size,
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
+ const bool use_hadamard SPECIALIZE,
+ threadgroup float shared_results[METAL_SIMD_SIZE],
const uint batch_idx GROUPS(batch_size),
const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),
const uint simd_lane THREADS(32),
@@ -144,8 +148,32 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
for (uint row = 0; row < results_per_simdgroup; row++) {
result[row] = simd_sum(result[row]);
+ }
+
+ if (use_hadamard) {
+ if (simd_lane == 0) {
+ for (uint row = 0; row < results_per_simdgroup; row++) {
+ shared_results[simd_group * results_per_simdgroup + row] = result[row];
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (simd_group == 0) {
+ uint global_out_idx = out_block_idx * 32 + simd_lane;
+ if (global_out_idx < out_vec_size) {
+ output[simd_lane] = simdgroup_random_hadamard_transform(
+ static_cast<ushort>(simd_lane),
+ T(shared_results[simd_lane]),
+ hadamard_factors[global_out_idx]
+ );
+ }
+ }
+ } else {
if (simd_lane == 0) {
- output[row] = static_cast<T>(result[row]);
+ for (uint row = 0; row < results_per_simdgroup; row++) {
+ output[row] = static_cast<T>(result[row]);
+ }
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 553f0c52c..61b9757b2 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -3,6 +3,7 @@
#include "../common/dsl.h"
#include "../common/thread_context.h"
#include "../common/threadgroup_reduce.h"
+#include "../hadamard_transform/hadamard_transform.h"
using namespace metal;
@@ -18,6 +19,7 @@ PUBLIC KERNEL(RMSNorm)(
const device ScaleT* scales,
device OutputT* output,
device InputT* shortcut OPTIONAL(copy_to_shortcut),
+ const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
constant uint& batch_size,
constant uint& element_count,
constant float& epsilon,
@@ -26,6 +28,7 @@ PUBLIC KERNEL(RMSNorm)(
const bool full_layer SPECIALIZE,
const bool copy_to_shortcut SPECIALIZE,
const bool residual_add SPECIALIZE,
+ const bool use_hadamard SPECIALIZE,
threadgroup AccumT shared_sum[METAL_SIMD_SIZE],
const ThreadContext thread_context,
const uint batch_idx GROUPS(batch_size),
@@ -91,11 +94,21 @@ PUBLIC KERNEL(RMSNorm)(
// If full_layer, normalize and scale in AccumT, cast to OutputT at the end
// If not, cast to OutputT after normalize, scale in OutputT
+ OutputT val;
if (full_layer) {
- output[i] = static_cast<OutputT>(x * rms_inv * scale);
+ val = static_cast<OutputT>(x * rms_inv * scale);
} else {
- output[i] =
- static_cast<OutputT>(x * rms_inv) * static_cast<OutputT>(scale);
+ val = static_cast<OutputT>(x * rms_inv) * static_cast<OutputT>(scale);
}
+
+ if (use_hadamard) {
+ val = static_cast<OutputT>(simdgroup_random_hadamard_transform(
+ static_cast<ushort>(thread_in_row % METAL_SIMD_SIZE),
+ val,
+ hadamard_factors[i]
+ ));
+ }
+
+ output[i] = val;
}
}
diff --git a/crates/uzu/src/config/linear.rs b/crates/uzu/src/config/linear.rs
index 287367936..0456dfa34 100644
--- a/crates/uzu/src/config/linear.rs
+++ b/crates/uzu/src/config/linear.rs
@@ -28,6 +28,11 @@ pub enum LinearConfig {
lora_rank: usize,
lora_scale: f32,
},
+ #[serde(rename = "RHTLinearWrapperConfig")]
+ RHTLinearWrapper {
+ block_size: usize,
+ inner_config: Box<LinearConfig>,
+ },
}
impl LinearConfig {
@@ -42,6 +47,10 @@ impl LinearConfig {
quantization,
..
} => quantization.activation_precision,
+ LinearConfig::RHTLinearWrapper {
+ inner_config,
+ ..
+ } => inner_config.activation_precision(),
}
}
}
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index 313a603e6..dd15aaf9d 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -146,7 +146,7 @@ impl<B: Backend> ClassifierLayer<B> {
)
.expect("Failed to create pre-MLP norm kernel");
- let mlp = <dyn Mlp<B>>::new(
+ let (mlp, mlp_hadamard_factors) = <dyn Mlp<B>>::new(
&layer_config.mlp_config,
model_dim,
hidden_dim,
@@ -155,6 +155,8 @@ impl<B: Backend> ClassifierLayer<B> {
)
.expect("Failed to create mlp block");
+ assert!(mlp_hadamard_factors.is_none(), "classifier doesn't support hadamard");
+
let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(
Normalization::new(
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index bc5335c07..a7565bc4c 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -187,6 +187,7 @@ impl<B: Backend> Decoder<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_weight_loader.subtree("output_norm").unwrap(),
+ None,
Some(ArrayId::Shortcut),
true,
)
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index 35daf60e7..64d119fa3 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -297,6 +297,7 @@ impl<B: Backend> Embedding<B> {
output_dim: vocab_size as usize,
mode: *embedding_quantization_mode,
quantization_type: QuantizedMatmulType::Mlx,
+ use_hadamard: false,
},
)?;
@@ -376,6 +377,7 @@ impl<B: Backend> Embedding<B> {
output_dim: vocab_size as usize,
mode: *embedding_quantization_mode,
quantization_type: QuantizedMatmulType::Mlx,
+ use_hadamard: false,
},
)?;
@@ -445,6 +447,7 @@ impl<B: Backend> Embedding<B> {
output_dim: vocab_size as usize,
mode: *embedding_quantization_mode,
quantization_type: QuantizedMatmulType::Mlx,
+ use_hadamard: false,
},
)?;
@@ -651,6 +654,7 @@ impl<B: Backend> Embedding<B> {
scales_buffer: scales,
zero_points_or_biases_buffer: biases,
output_buffer: output,
+ hadamard_factors: None,
batch_dim,
},
)?;
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index 8f58dda5c..1a0c0fa45 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -59,19 +59,7 @@ impl<B: Backend> LayerExecutables<B> {
let tensor_add = TensorAddBiasKernel::new(context, intermediate_data_type, false)
.expect("Failed to create TensorAddBiasKernel kernel"); // TODO: this function return Result
- let pre_attention_norm = RMSNorm::new(
- context,
- intermediate_data_type,
- layer_config.pre_attention_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
- Some(ArrayId::Shortcut),
- layer_index > 0,
- )
- .expect("Failed to create RMS norm kernel");
-
- let mixer = match &layer_config.mixer_config {
+ let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {
MixerConfig::Attention(attention_config) => {
let rope_block = rope.expect("RoPE encoder missing for attention layer");
@@ -82,7 +70,7 @@ impl<B: Backend> LayerExecutables<B> {
let q_dim = layer_num_heads * layer_head_dim;
let kv_dim = layer_num_groups * layer_head_dim;
- let qkv_projection = <dyn Linear<B>>::new(
+ let (qkv_projection, input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&attention_config.qkv_projection_config,
attention_config.has_qkv_biases,
model_dim,
@@ -161,14 +149,17 @@ impl<B: Backend> LayerExecutables<B> {
)
.expect("Failed to create AttentionWrapper kernel");
- MixerExecutables::Attention {
- qkv_projection,
- gate_projection,
- qk_norm,
- rope: rope_block,
- attention,
- out_projection,
- }
+ (
+ MixerExecutables::Attention {
+ qkv_projection,
+ gate_projection,
+ qk_norm,
+ rope: rope_block,
+ attention,
+ out_projection,
+ },
+ input_hadamard_factors,
+ )
},
MixerConfig::Mamba(mamba_config) => {
let mixer = MambaMixer::new(
@@ -182,12 +173,15 @@ impl<B: Backend> LayerExecutables<B> {
num_groups,
decoder_layer_loader,
);
- MixerExecutables::StateSpace {
- mixer,
- }
+ (
+ MixerExecutables::StateSpace {
+ mixer,
+ },
+ None,
+ )
},
MixerConfig::ShortConv(short_conv_config) => {
- let mixer = ShortConvMixer::new(
+ let (mixer, input_hadamard_factors) = ShortConvMixer::new(
context,
layer_type.clone(),
short_conv_config.clone(),
@@ -195,21 +189,39 @@ impl<B: Backend> LayerExecutables<B> {
model_dim,
decoder_layer_loader,
);
- MixerExecutables::ShortConv {
- mixer,
- }
+ (
+ MixerExecutables::ShortConv {
+ mixer,
+ },
+ input_hadamard_factors,
+ )
},
MixerConfig::DeltaNet(delta_net_config) => {
let mixer =
DeltaNetMixer::new(context, delta_net_config.clone(), layer_index, model_dim, decoder_layer_loader)
.expect("Failed to create DeltaNet mixer");
-
- MixerExecutables::DeltaNet {
- mixer,
- }
+ (
+ MixerExecutables::DeltaNet {
+ mixer,
+ },
+ None,
+ )
},
};
+ let pre_attention_norm = RMSNorm::new(
+ context,
+ intermediate_data_type,
+ layer_config.pre_attention_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
+ mixer_hadamard_factors,
+ Some(ArrayId::Shortcut),
+ layer_index > 0,
+ )
+ .expect("Failed to create RMS norm kernel");
+
let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
Some(
RMSNorm::new(
@@ -220,6 +232,7 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
&decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
None,
+ None,
false,
)
.expect("Failed to create RMS norm kernel"),
@@ -228,6 +241,15 @@ impl<B: Backend> LayerExecutables<B> {
None
};
+ let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(
+ &layer_config.mlp_config,
+ model_dim,
+ hidden_dim,
+ context,
+ &decoder_layer_loader.subtree("mlp").unwrap(),
+ )
+ .expect("Failed to create mlp block");
+
let pre_mlp_norm = RMSNorm::new(
context,
intermediate_data_type,
@@ -235,20 +257,12 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
+ mlp_input_hadamard_factors,
Some(ArrayId::Shortcut),
true,
)
.expect("Failed to create RMS norm kernel");
- let mlp = <dyn Mlp<B>>::new(
- &layer_config.mlp_config,
- model_dim,
- hidden_dim,
- context,
- &decoder_layer_loader.subtree("mlp").unwrap(),
- )
- .expect("Failed to create mlp block");
-
let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(
RMSNorm::new(
@@ -259,6 +273,7 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
None,
+ None,
false,
)
.expect("Failed to create RMS norm kernel"),
diff --git a/crates/uzu/src/encodable_block/linear/mod.rs b/crates/uzu/src/encodable_block/linear/mod.rs
index de530a9e4..b014a1f56 100644
--- a/crates/uzu/src/encodable_block/linear/mod.rs
+++ b/crates/uzu/src/encodable_block/linear/mod.rs
@@ -1,17 +1,19 @@
mod full_precision;
mod qlora_wrapper;
mod quantized;
+mod rht_wrapper;
pub use full_precision::{FullPrecisionLinear, FullPrecisionLinearError};
pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};
pub use quantized::{QuantizedLinear, QuantizedLinearError};
+pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};
use thiserror::Error;
use crate::{
backends::common::{Backend, Encoder},
config::LinearConfig,
forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
+ parameters::{ParameterLoaderError, ParameterTree},
};
pub trait Linear<B: Backend> {
@@ -28,8 +30,12 @@ pub enum LinearBlockError<B: Backend> {
QuantizedLinearError(#[from] QuantizedLinearError<B>),
#[error("FullPrecisionLinear error: {0}")]
FullPrecisionLinearError(#[from] FullPrecisionLinearError<B>),
+ #[error("RHTLinearWrapper error: {0}")]
+ RHTLinearWrapperError(#[from] RHTLinearWrapperError<B>),
#[error("QLoRALinearWrapper error: {0}")]
QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(#[from] ParameterLoaderError<B>),
}
impl<B: Backend> dyn Linear<B> {
@@ -54,6 +60,7 @@ impl<B: Backend> dyn Linear<B> {
parameter_tree,
input_array_id,
output_array_id,
+ None,
)?;
Ok(Box::new(block))
},
@@ -84,11 +91,116 @@ impl<B: Backend> dyn Linear<B> {
input_dimension,
output_dimension_sum,
parameter_tree,
+ None,
input_array_id,
output_array_id,
)?;
Ok(Box::new(block))
},
+ LinearConfig::RHTLinearWrapper {
+ block_size,
+ inner_config,
+ } => {
+ let block = RHTLinearWrapper::new(
+ context,
+ *block_size,
+ inner_config,
+ input_dimension,
+ output_dimension_sum,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )?;
+ Ok(Box::new(block))
+ },
+ }
+ }
+
+ pub fn new_with_output_hadamard(
+ context: &B::Context,
+ config: &LinearConfig,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ parameter_tree: &ParameterTree<B::Context>,
+ output_factors: B::Buffer,
+ input_dim: usize,
+ output_dim: usize,
+ ) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
+ match config {
+ LinearConfig::Quantized(config) | LinearConfig::MLXQuantized(config) => Ok(Box::new(QuantizedLinear::new(
+ context,
+ config,
+ input_dim,
+ output_dim,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ Some(output_factors),
+ )?)),
+ LinearConfig::QLoRA {
+ quantization,
+ lora_rank,
+ lora_scale,
+ } => Ok(Box::new(QLoRALinearWrapper::new(
+ context,
+ quantization,
+ *lora_rank,
+ *lora_scale,
+ input_dim,
+ output_dim,
+ parameter_tree,
+ Some(output_factors),
+ input_array_id,
+ output_array_id,
+ )?)),
+ inner_config => unimplemented!("{inner_config:?} doesn't support fused output hadamard"),
+ }
+ }
+
+ pub fn new_extracting_input_hadamard<const N: usize>(
+ config: &LinearConfig,
+ _has_biases: bool,
+ input_dimension: usize,
+ output_dimensions: [usize; N],
+ context: &B::Context,
+ parameter_tree: &ParameterTree<B::Context>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<(Box<dyn Linear<B>>, Option<B::Buffer>), LinearBlockError<B>> {
+ let output_dimension_sum: usize = output_dimensions.iter().sum();
+ match config {
+ LinearConfig::RHTLinearWrapper {
+ inner_config,
+ ..
+ } => {
+ let input_factors = parameter_tree.leaf("input_factors")?.read_buffer()?;
+ let output_factors = parameter_tree.leaf("output_factors")?.read_buffer()?;
+ let inner_tree = parameter_tree.subtree("inner_linear")?;
+ let inner_linear = Self::new_with_output_hadamard(
+ context,
+ inner_config,
+ input_array_id,
+ output_array_id,
+ &inner_tree,
+ output_factors,
+ input_dimension,
+ output_dimension_sum,
+ )?;
+ Ok((inner_linear, Some(input_factors)))
+ },
+ other => {
+ let linear = Self::new(
+ other,
+ _has_biases,
+ input_dimension,
+ output_dimensions,
+ context,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )?;
+ Ok((linear, None))
+ },
}
}
}
diff --git a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs
index 37d7eecea..2c4508825 100644
--- a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -85,6 +85,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
input_dim: usize,
output_dim: usize,
parameter_tree: &ParameterTree<B::Context>,
+ output_quantized_hadamard_factors: Option<B::Buffer>,
input_array_id: ArrayId,
output_array_id: ArrayId,
) -> Result<Self, QLoRALinearWrapperError<B>> {
@@ -98,6 +99,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
parameter_tree,
input_array_id,
output_array_id,
+ output_quantized_hadamard_factors,
)?;
let adapter_kernel =
RefCell::new(<<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)?);
diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rs
index 0d235415e..b2501e9b7 100644
--- a/crates/uzu/src/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/encodable_block/linear/quantized.rs
@@ -92,6 +92,7 @@ pub struct QuantizedLinear<B: Backend> {
weights_buffer: Rc<RefCell<B::Buffer>>,
scales_buffer: Rc<RefCell<B::Buffer>>,
zero_points_or_biases_buffer: Rc<RefCell<B::Buffer>>,
+ output_hadamard_factors: Option<B::Buffer>,
output_dim: usize,
input_array_id: ArrayId,
output_array_id: ArrayId,
@@ -106,6 +107,7 @@ impl<B: Backend> QuantizedLinear<B> {
parameter_tree: &ParameterTree<B::Context>,
input_array_id: ArrayId,
output_array_id: ArrayId,
+ output_hadamard_factors: Option<B::Buffer>,
) -> Result<Self, QuantizedLinearError<B>> {
let kernel_data_type: DataType = config.activation_precision.into();
if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
@@ -220,6 +222,7 @@ impl<B: Backend> QuantizedLinear<B> {
output_dim,
mode: config.weight_quantization_mode,
quantization_type,
+ use_hadamard: output_hadamard_factors.is_some(),
},
)
.map_err(QuantizedLinearError::QuantizedMatmulError)?;
@@ -231,6 +234,7 @@ impl<B: Backend> QuantizedLinear<B> {
weights_buffer: weights.buffer(),
scales_buffer: scales.buffer(),
zero_points_or_biases_buffer,
+ output_hadamard_factors,
output_dim,
input_array_id,
output_array_id,
@@ -261,6 +265,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
scales_buffer: self.scales_buffer.borrow().deref(),
zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),
output_buffer: output_buf_borrow.deref_mut(),
+ hadamard_factors: self.output_hadamard_factors.as_ref(),
batch_dim,
},
)
diff --git a/crates/uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu/src/encodable_block/linear/rht_wrapper.rs
new file mode 100644
index 000000000..7151bb0c7
--- /dev/null
+++ b/crates/uzu/src/encodable_block/linear/rht_wrapper.rs
@@ -0,0 +1,101 @@
+use std::ops::DerefMut;
+
+use thiserror::Error;
+
+use super::{Linear, LinearBlockError};
+use crate::{
+ backends::common::{
+ Backend, Encoder,
+ kernel::{HadamardTransformKernel, Kernels},
+ },
+ config::LinearConfig,
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum RHTLinearWrapperError<B: Backend> {
+ #[error("Inner linear error: {0}")]
+ InnerLinearError(#[from] Box<LinearBlockError<B>>),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(#[from] ParameterLoaderError<B>),
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+}
+
+pub struct RHTLinearWrapper<B: Backend> {
+ input_hadamard_kernel: <B::Kernels as Kernels>::HadamardTransformKernel,
+ input_factors: B::Buffer,
+ inner_linear: Box<dyn Linear<B>>,
+ input_dimension: usize,
+ input_array_id: ArrayId,
+}
+
+impl<B: Backend> RHTLinearWrapper<B> {
+ pub fn new(
+ context: &B::Context,
+ block_size: usize,
+ inner_config: &LinearConfig,
+ input_dimension: usize,
+ output_dimension: usize,
+ parameter_tree: &ParameterTree<B::Context>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Self, RHTLinearWrapperError<B>> {
+ assert!(block_size == 32, "only block size 32 hadamard is supported");
+
+ let data_type = inner_config.activation_precision().into();
+
+ let input_factors_leaf = parameter_tree.leaf("input_factors")?;
+ let output_factors_leaf = parameter_tree.leaf("output_factors")?;
+
+ let input_factors = input_factors_leaf.read_buffer()?;
+ let output_factors = output_factors_leaf.read_buffer()?;
+
+ let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)
+ .map_err(RHTLinearWrapperError::BackendError)?;
+
+ let inner_linear_tree = parameter_tree.subtree("inner_linear")?;
+
+ let inner_linear = <dyn Linear<B>>::new_with_output_hadamard(
+ context,
+ inner_config,
+ input_array_id,
+ output_array_id,
+ &inner_linear_tree,
+ output_factors,
+ input_dimension,
+ output_dimension,
+ )
+ .map_err(|e| RHTLinearWrapperError::InnerLinearError(Box::new(e)))?;
+
+ Ok(Self {
+ input_hadamard_kernel,
+ input_factors,
+ inner_linear,
+ input_dimension,
+ input_array_id,
+ })
+ }
+}
+
+impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState<B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), B::Error> {
+ // Input Hadamard
+ let input_array = state.array(self.input_array_id);
+ self.input_hadamard_kernel.encode(
+ input_array.buffer().borrow_mut().deref_mut(),
+ &self.input_factors,
+ self.input_dimension as u32,
+ state.active_row_count() as u32,
+ encoder,
+ );
+
+ // Matmul with fused output Hadamard
+ self.inner_linear.encode(state, encoder)
+ }
+}
diff --git a/crates/uzu/src/encodable_block/mlp/mod.rs b/crates/uzu/src/encodable_block/mlp/mod.rs
index 23cb57405..f2e4046d4 100644
--- a/crates/uzu/src/encodable_block/mlp/mod.rs
+++ b/crates/uzu/src/encodable_block/mlp/mod.rs
@@ -41,11 +41,11 @@ impl<B: Backend> dyn Mlp<B> {
hidden_dimension: usize,
context: &B::Context,
parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Box<dyn Mlp<B>>, MlpBlockError<B>> {
+ ) -> Result<(Box<dyn Mlp<B>>, Option<B::Buffer>), MlpBlockError<B>> {
if let MLPConfig::Dense(dense_config) = config {
let data_type: DataType = dense_config.linear_config.activation_precision().into();
- let up_projection = <dyn Linear<B>>::new(
+ let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&dense_config.linear_config,
false,
model_dimension,
@@ -56,11 +56,7 @@ impl<B: Backend> dyn Mlp<B> {
ArrayId::MlpFusedUp,
)?;
- let gate_activation =
- MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)
- .map_err(MlpBlockError::BackendError)?;
-
- let down_projection = <dyn Linear<B>>::new(
+ let (down_projection, down_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&dense_config.linear_config,
false,
hidden_dimension,
@@ -71,13 +67,22 @@ impl<B: Backend> dyn Mlp<B> {
ArrayId::Main,
)?;
- return Ok(Box::new(DenseMlp::new(up_projection, gate_activation, down_projection)));
+ let gate = MlpGateActMulEncodable::new(
+ context,
+ data_type,
+ dense_config.activation.clone(),
+ hidden_dimension,
+ down_input_hadamard_factors,
+ )
+ .map_err(MlpBlockError::BackendError)?;
+
+ return Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors));
}
if let MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
let mixture_of_experts_block =
MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;
- return Ok(Box::new(mixture_of_experts_block));
+ return Ok((Box::new(mixture_of_experts_block), None));
}
unreachable!("Unknown MLP config")
diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rs
index a14a00f7b..af06a3fbd 100644
--- a/crates/uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/uzu/src/encodable_block/mlp/moe.rs
@@ -105,6 +105,11 @@ impl<B: Backend> MoeBlock<B> {
} => {
unimplemented!("QLoRA router not yet supported for MoE");
},
+ LinearConfig::RHTLinearWrapper {
+ ..
+ } => {
+ unimplemented!("RHTLinearWrapper router not yet supported for MoE");
+ },
};
let router_topk_kernel = <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, router_data_type)
diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rs
index 1c6fcf34b..d2349e401 100644
--- a/crates/uzu/src/encodable_block/normalization.rs
+++ b/crates/uzu/src/encodable_block/normalization.rs
@@ -54,6 +54,7 @@ impl<B: Backend> Normalization<B> {
output_array_id,
parameter_tree,
None,
+ None,
false,
)?))
}
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index 7e06ae5de..71f5462e5 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -34,6 +34,7 @@ pub struct RMSNorm<B: Backend> {
output_array_id: ArrayId,
shortcut_array_id: Option<ArrayId>,
scales_buffer: Rc<RefCell<B::Buffer>>,
+ hadamard_factors_buffer: Option<B::Buffer>,
use_sampling_range: bool,
}
@@ -45,6 +46,7 @@ impl<B: Backend> RMSNorm<B> {
input_array_id: ArrayId,
output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
+ hadamard_factors_buffer: Option<B::Buffer>,
shortcut_array_id: Option<ArrayId>,
residual_add: bool,
) -> Result<Self, RMSNormError<B>> {
@@ -68,6 +70,7 @@ impl<B: Backend> RMSNorm<B> {
config.upcast_mode == UpcastMode::FullLayer,
shortcut_array_id.is_some(),
residual_add,
+ hadamard_factors_buffer.is_some(),
)
.map_err(RMSNormError::BackendError)?;
@@ -78,6 +81,7 @@ impl<B: Backend> RMSNorm<B> {
output_array_id,
shortcut_array_id,
scales_buffer: scales.buffer(),
+ hadamard_factors_buffer,
use_sampling_range: false,
})
}
@@ -131,6 +135,7 @@ impl<B: Backend> RMSNorm<B> {
self.scales_buffer.borrow().deref(),
(output_array.buffer().borrow_mut().deref_mut(), output_offset),
shortcut_borrow.as_deref_mut().map(|b| (b, input_offset)),
+ self.hadamard_factors_buffer.as_ref(),
batch_len as u32,
element_count as u32,
self.config.epsilon,
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index 2623867f6..c429fcc8f 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -35,7 +35,7 @@ impl<B: Backend> ShortConvMixer<B> {
layer_index: usize,
model_dim: usize,
decoder_layer_loader: &ParameterTree<B::Context>,
- ) -> Self {
+ ) -> (Self, Option<B::Buffer>) {
if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {
panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);
}
@@ -45,7 +45,7 @@ impl<B: Backend> ShortConvMixer<B> {
let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();
- let in_projection = <dyn Linear<B>>::new(
+ let (in_projection, in_proj_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&short_conv_config.in_projection_config,
false,
model_dim,
@@ -86,19 +86,22 @@ impl<B: Backend> ShortConvMixer<B> {
let short_conv_trie = <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)
.expect("Failed to create short conv trie kernel");
- Self {
- layer_index,
- config: short_conv_config,
- model_dim,
- in_projection,
- out_projection,
- short_conv_pack,
- short_conv_prefill,
- short_conv_decode,
- short_conv_trie,
- conv_weight,
- conv_bias,
- }
+ (
+ Self {
+ layer_index,
+ config: short_conv_config,
+ model_dim,
+ in_projection,
+ out_projection,
+ short_conv_pack,
+ short_conv_prefill,
+ short_conv_decode,
+ short_conv_trie,
+ conv_weight,
+ conv_bias,
+ },
+ in_proj_input_hadamard_factors,
+ )
}
fn clear_suffix_state_valid_range(
diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rs
index fae070856..70e7de14c 100644
--- a/crates/uzu/src/forward_pass/model_shape.rs
+++ b/crates/uzu/src/forward_pass/model_shape.rs
@@ -147,6 +147,12 @@ impl ModelShape {
} => {
max_lora_intermediate = Some(max_lora_intermediate.unwrap_or(0).max(*lora_rank));
},
+ LinearConfig::RHTLinearWrapper {
+ inner_config,
+ ..
+ } => {
+ linear_configs.push(inner_config.as_ref());
+ },
_ => (),
};
}
diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
index 3121a5f82..b64afcee1 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
@@ -342,6 +342,7 @@ fn execute_quantized_matmul<B: Backend>(
_ => panic!("Unsupported bits: {}", bits),
},
quantization_type,
+ use_hadamard: false,
},
)
.unwrap();
@@ -355,6 +356,7 @@ fn execute_quantized_matmul<B: Backend>(
scales_buffer: &s_buf,
zero_points_or_biases_buffer: &b_buf,
output_buffer: &mut y_buf,
+ hadamard_factors: None,
batch_dim: batch,
};
let mut encoder = Encoder::new(ctx).unwrap();
@@ -372,6 +374,7 @@ fn execute_quantized_matmul<B: Backend>(
scales_buffer: &s_buf,
zero_points_or_biases_buffer: &b_buf,
output_buffer: &mut y_buf,
+ hadamard_factors: None,
batch_dim: batch,
};
let mut encoder = Encoder::new(ctx).unwrap();
diff --git a/crates/uzu/tests/unit/config/linear_test.rs b/crates/uzu/tests/unit/config/linear_test.rs
index 844af1a4e..9c3fc37a1 100644
--- a/crates/uzu/tests/unit/config/linear_test.rs
+++ b/crates/uzu/tests/unit/config/linear_test.rs
@@ -30,3 +30,33 @@ fn test_linear_config() {
let deserialized_config: LinearConfig = from_str(config_str).unwrap();
assert_eq!(deserialized_config, ground_truth_config);
}
+
+#[test]
+fn test_rht_linear_wrapper_config() {
+ let config_str = r#"
+ {
+ "type": "RHTLinearWrapperConfig",
+ "block_size": 32,
+ "inner_config": {
+ "type": "GroupQuantizedLinearConfig",
+ "group_size": 32,
+ "weight_quantization_mode": "uint4",
+ "activation_quantization_mode": null,
+ "activation_precision": "bfloat16"
+ }
+ }
+ "#;
+
+ let ground_truth_config = LinearConfig::RHTLinearWrapper {
+ block_size: 32,
+ inner_config: Box::new(LinearConfig::Quantized(QuantizationConfig {
+ group_size: 32,
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: None,
+ activation_precision: ConfigDataType::BFloat16,
+ })),
+ };
+
+ let deserialized_config: LinearConfig = from_str(config_str).unwrap();
+ assert_eq!(deserialized_config, ground_truth_config);
+}
diff --git a/crates/uzu/tests/unit/kernel/activation_mul_test.rs b/crates/uzu/tests/unit/kernel/activation_mul_test.rs
index e00b64974..5fdfbd68a 100644
--- a/crates/uzu/tests/unit/kernel/activation_mul_test.rs
+++ b/crates/uzu/tests/unit/kernel/activation_mul_test.rs
@@ -47,7 +47,7 @@ fn get_test_data<T: ArrayElement + Float>(act_type: ActivationType) -> (Input<T>
fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::MlpGateActMulKernel::new(&context, T::data_type())
+ let kernel = <<B as Backend>::Kernels as Kernels>::MlpGateActMulKernel::new(&context, T::data_type(), false)
.expect("Failed to create MlpGateActMulKernel");
let fused_len = (input.m * 2 * input.h) as usize;
@@ -59,6 +59,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
kernel.encode(
fused_up_array.buffer().borrow().deref(),
hidden_array.buffer().borrow_mut().deref_mut(),
+ None::<&B::Buffer>,
input.h,
input.m,
input.act_type,
diff --git a/crates/uzu/tests/unit/kernel/hadamard_transform_test.rs b/crates/uzu/tests/unit/kernel/hadamard_transform_test.rs
new file mode 100644
index 000000000..be60407c1
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/hadamard_transform_test.rs
@@ -0,0 +1,162 @@
+use std::{
+ fmt::Debug,
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::common::{Backend, Context, Encoder, Kernels, kernel::HadamardTransformKernel},
+};
+
+use crate::uzu_test;
+
+const BLOCK_SIZE: usize = 32;
+
+fn reference_hadamard_transform_mul(
+ data: &[f64],
+ factors: &[f64],
+ channel_count: usize,
+) -> Vec<f64> {
+ let batch_count = data.len() / channel_count;
+ let normalization_factor = 1.0 / (BLOCK_SIZE as f64).sqrt();
+ let mut result = data.to_vec();
+
+ for batch_index in 0..batch_count {
+ let batch_offset = batch_index * channel_count;
+
+ for block_start in (0..channel_count).step_by(BLOCK_SIZE) {
+ for lane in 0..BLOCK_SIZE {
+ let index = batch_offset + block_start + lane;
+ let factor_index = block_start + lane;
+ result[index] *= factors[factor_index];
+ }
+
+ let mut stride = 1;
+ while stride < BLOCK_SIZE {
+ for pair_start in (0..BLOCK_SIZE).step_by(stride * 2) {
+ for offset in 0..stride {
+ let index_a = batch_offset + block_start + pair_start + offset;
+ let index_b = index_a + stride;
+ let sum = result[index_a] + result[index_b];
+ let difference = result[index_a] - result[index_b];
+ result[index_a] = sum;
+ result[index_b] = difference;
+ }
+ }
+ stride *= 2;
+ }
+
+ for lane in 0..BLOCK_SIZE {
+ let index = batch_offset + block_start + lane;
+ result[index] *= normalization_factor;
+ }
+ }
+ }
+
+ result
+}
+
+struct TestInput<T: ArrayElement + Float> {
+ data: Box<[T]>,
+ factors: Box<[i32]>,
+ channel_count: usize,
+ batch_count: usize,
+}
+
+fn generate_test_input<T: ArrayElement + Float>(
+ batch_count: usize,
+ channel_count: usize,
+) -> (TestInput<T>, Vec<f64>) {
+ let total_elements = batch_count * channel_count;
+
+ let data_f64: Vec<f64> = (0..total_elements).map(|index| ((index as f64) * 0.1).sin() * 2.0).collect();
+
+ let factors_i32: Vec<i32> = (0..channel_count)
+ .map(|index| {
+ if index % 3 == 0 {
+ -1
+ } else {
+ 1
+ }
+ })
+ .collect();
+
+ let factors_f64: Vec<f64> = factors_i32.iter().map(|&v| v as f64).collect();
+ let expected = reference_hadamard_transform_mul(&data_f64, &factors_f64, channel_count);
+
+ let data: Vec<T> = data_f64.iter().map(|&value| T::from(value).unwrap()).collect();
+
+ let input = TestInput {
+ data: data.into_boxed_slice(),
+ factors: factors_i32.into_boxed_slice(),
+ channel_count,
+ batch_count,
+ };
+
+ (input, expected)
+}
+
+fn run_kernel<T: ArrayElement + Float, B: Backend>(input: &TestInput<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(&context, T::data_type())
+ .expect("Failed to create HadamardTransformKernel");
+
+ let total_elements = input.batch_count * input.channel_count;
+ let data_array = context.create_array_from(&[total_elements], &input.data, "data");
+ let factors_array = context.create_array_from::<i32>(&[input.channel_count], &input.factors, "factors");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ data_array.buffer().borrow_mut().deref_mut(),
+ factors_array.buffer().borrow().deref(),
+ input.channel_count as u32,
+ input.batch_count as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ data_array.as_slice().to_vec()
+}
+
+fn test_hadamard_transform<T: ArrayElement + Float + Debug>(tolerance: f64) {
+ let test_cases = [(1, 32), (1, 64), (1, 128), (4, 32), (4, 256), (2, 2048)];
+
+ for (batch_count, channel_count) in test_cases {
+ let (input, expected) = generate_test_input::<T>(batch_count, channel_count);
+
+ for_each_non_cpu_backend!(|B| {
+ let actual = run_kernel::<T, B>(&input);
+ assert_eq!(actual.len(), expected.len());
+
+ for (index, (actual_value, &expected_value)) in actual.iter().zip(expected.iter()).enumerate() {
+ let actual_f64 = actual_value.to_f64().unwrap();
+ let error = (actual_f64 - expected_value).abs();
+ let relative_bound = expected_value.abs() * tolerance;
+ let absolute_bound = tolerance;
+ assert!(
+ error <= relative_bound.max(absolute_bound),
+ "Mismatch at index {index} for batch_count={batch_count}, channel_count={channel_count}: \
+ actual={actual_f64}, expected={expected_value}, error={error}"
+ );
+ }
+ });
+ }
+}
+
+#[uzu_test]
+fn test_f32() {
+ test_hadamard_transform::<f32>(1e-4);
+}
+
+#[uzu_test]
+fn test_f16() {
+ test_hadamard_transform::<f16>(0.05);
+}
+
+#[uzu_test]
+fn test_bf16() {
+ test_hadamard_transform::<bf16>(0.1);
+}
diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rs
index dc6e2b888..e8c5db0e4 100644
--- a/crates/uzu/tests/unit/kernel/mod.rs
+++ b/crates/uzu/tests/unit/kernel/mod.rs
@@ -8,6 +8,7 @@ mod attention;
mod audio;
mod delta_net_test;
mod embedding;
+mod hadamard_transform_test;
mod kv_cache_update_test;
mod layer_norm_test;
mod matmul;
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
index db6185dd3..fcbb00756 100644
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
@@ -23,6 +23,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
input.bits,
input.use_zero_points,
input.use_mlx_quant,
+ false,
aligned_n,
)
.expect("Failed to create QuantizedMatmulQmmTransposedKernel");
@@ -45,6 +46,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
bias_buf.as_ref(),
&x_buf,
&mut y_buf,
+ None::<&B::Buffer>,
input.k,
input.n,
input.m,
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index 34ab9a9db..e9583aff4 100644
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -84,6 +84,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
input.bits,
input.use_zero_points,
input.use_mlx_quant,
+ false,
)
.expect("Failed to create QuantizedMatmulQmvFastKernel");
kernel.encode(
@@ -93,6 +94,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
bias_buf.as_ref(),
&x_buf,
&mut y_buf,
+ None::<&B::Buffer>,
input.k,
input.n,
input.m,
@@ -341,6 +343,7 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
bits,
use_zero_points,
use_mlx_quant,
+ false,
)
.unwrap();
@@ -391,6 +394,7 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
bias_buf.as_ref(),
&x_buf,
&mut y_buf,
+ None::<&B::Buffer>,
k as u32,
n as u32,
m as u32,
diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
index 11a648738..459ed5689 100644
--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
@@ -133,6 +133,7 @@ fn get_output<
input.full_layer,
false,
false,
+ false,
)
.expect("Failed to create RMSNormKernel");
@@ -155,6 +156,7 @@ fn get_output<
scales_array.buffer().borrow().deref(),
output_array.buffer().borrow_mut().deref_mut(),
None::<&mut B::Buffer>,
+ None::<&B::Buffer>,
input.batch_size,
input.element_count,
input.epsilon,
@@ -407,6 +409,7 @@ fn bench_rms_norm(c: &mut Criterion) {
false,
false,
false,
+ false,
)
.unwrap();
@@ -453,6 +456,7 @@ fn bench_rms_norm(c: &mut Criterion) {
&scales_buffer,
&mut output_buffer,
None::<&mut <B as Backend>::Buffer>,
+ None::<&<B as Backend>::Buffer>,
batch_size as u32,
model_dim as u32,
epsilon,