A high-performance inference engine for AI models
A8W8 bias/zp perf (#647)
CCYeh6ad005d6c868
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_a.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_a.rsindex 49565bf8c..78a643f66 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_a.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_a.rs@@ -8,6 +8,7 @@ pub enum MatmulA<'a, B: Backend> {Int8Symmetric {values: &'a Allocation<B>,scales: &'a Allocation<B>,+ group_sums: Option<&'a Allocation<B>>,},}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rsindex 0d46008b1..486d537d1 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs@@ -110,6 +110,7 @@ impl MatmulKernel for MatmulCpuKernel {MatmulA::Int8Symmetric {values,scales,+ group_sums: _,} => {let weight_gs_ok = matches!(b.group_size(), Some(32 | 64 | 128));let weights_ok = matches!(diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs b/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rsindex 87e6e4fca..2899848af 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs@@ -13,10 +13,14 @@ pub fn activations_prepare<InputT: ArrayElement + Float>(input: *const InputT,q_out: *mut i8,scales_out: *mut f32,+ #[optional(emit_group_sums)] group_sums_out: Option<*mut i32>,rht_factors: *const i32,batch_size: u32,element_count: u32,group_size: u32,+ #[allow(unused)]+ #[specialize]+ emit_group_sums: bool,) {let rows = batch_size as usize;let columns = element_count as usize;@@ -45,9 +49,14 @@ pub fn activations_prepare<InputT: ArrayElement + Float>(let slice = &prepared[start..end];let divisor = min_max_symmetric_divisor(slice);unsafe { *scales_out.add(row * groups + group) = divisor };+ let mut group_sum = 0i32;for index in start..end {let q = quantize_symmetric_i8(prepared[index], divisor);unsafe { *q_out.add(row * columns + index) = q };+ group_sum += q as i32;+ }+ if let Some(group_sums_out) = group_sums_out {+ unsafe { *group_sums_out.add(row * groups + group) = group_sum };}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 5cd74dc90..20c89be17 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -253,6 +253,13 @@ struct MxuMmaCore {return values[tile * SLOTS_PER_TILE + index];}+ METAL_FUNC void clear() thread {+ METAL_PRAGMA_UNROLL+ for (ushort index = 0; index < TILES * SLOTS_PER_TILE; ++index) {+ values[index] = 0.0f;+ }+ }+METAL_FUNC float at(const short offset) const thread {const ushort tile = ushort(offset) / FRAGMENT_EXTENT;const ushort index = (ushort(offset) % FRAGMENT_EXTENT) / SLOT_STRIDE;@@ -274,7 +281,8 @@ struct MxuMmaCore {for (ushort index = 0; index < SLOTS_PER_TILE; ++index) {const short coord = origin + tile * FRAGMENT_EXTENT + index * SLOT_STRIDE;if (ALIGNED || coord < limit) {- sink(tile, index, (abs_base + uint(coord)) * groups_per_row + group_index);+ const uint line_index = abs_base + uint(coord);+ sink(tile, index, line_index * groups_per_row + group_index, line_index);}}}@@ -300,7 +308,7 @@ struct MxuMmaCore {static METAL_FUNC float int8_weight_dequant_bias(const float scale,const uint scale_index,- const uint col,+ const uint weight_column,const uint weight_group,const uint weight_groups_per_row,const device BT* biases,@@ -309,19 +317,24 @@ struct MxuMmaCore {if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {return static_cast<float>(biases[scale_index]);} else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {- const device uint8_t* zero_points_row =- zero_points + col * zero_point_row_stride<ushort(BITS)>(weight_groups_per_row);- return -scale * float(decode_zero_point<ushort(BITS)>(zero_points_row, weight_group));+ if constexpr (BITS == 8) {+ return -scale * float(zero_points[scale_index]);+ } else {+ const device uint8_t* zero_points_row =+ zero_points + weight_column * zero_point_row_stride<ushort(BITS)>(weight_groups_per_row);+ return -scale * float(decode_zero_point<ushort(BITS)>(zero_points_row, weight_group));+ }} else {return 0.0f;}}template <bool ALIGNED_M, bool ALIGNED_N>- static METAL_FUNC AccumFragment symmetric_int8_activation_k_loop(+ static METAL_FUNC AccumFragment int8_activation_k_loop(const device int8_t* a_int8_simdgroup,const device uint8_t* b_packed_simdgroup,const device float* a_scales,+ const device int32_t* a_group_sums,const device BT* b_scales,const device BT* biases,const device uint8_t* zero_points,@@ -346,15 +359,6 @@ struct MxuMmaCore {constexpr int k_bytes_per_weight_group = (BITS == 4) ? (int(GROUP_SIZE) / 2) : int(GROUP_SIZE);constexpr int act_chunks_per_weight_group = int(GROUP_SIZE) / int(SIMDGROUP_BLOCK_K);- uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, Ops> ones_tile;- if constexpr (int8_activation_needs_weight_correction) {- thread int8_t* ones = ones_tile.elements();- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ones_tile.ELEMENTS_PER_FRAGMENT; ++i) {- ones[i] = int8_t(1);- }- }-METAL_PRAGMA_NO_UNROLLfor (int weight_group = 0; weight_group < weight_group_iterations; ++weight_group) {const uint weight_group_index = k_offset_weight_groups + uint(weight_group);@@ -367,15 +371,14 @@ struct MxuMmaCore {abs_col_base,weight_groups_per_row,weight_group_index,- [&](ushort tile_col, ushort thread_col, uint scale_index) {+ [&](ushort tile_col, ushort thread_col, uint scale_index, uint weight_column) {const float scale = static_cast<float>(b_scales[scale_index]);weight_scales.slot(tile_col, thread_col) = scale;if constexpr (int8_activation_needs_weight_correction) {- const uint col = scale_index / weight_groups_per_row;weight_corrections.slot(tile_col, thread_col) = scale * int8_weight_midpoint + int8_weight_dequant_bias(scale,scale_index,- col,+ weight_column,weight_group_index,weight_groups_per_row,biases,@@ -385,6 +388,11 @@ struct MxuMmaCore {});+ ActivationLineCache activation_corrections;+ if constexpr (int8_activation_needs_weight_correction) {+ activation_corrections.clear();+ }+METAL_PRAGMA_NO_UNROLLfor (int act_chunk = 0; act_chunk < act_chunks_per_weight_group; ++act_chunk) {const int k_element_offset = act_chunk * int(SIMDGROUP_BLOCK_K);@@ -414,36 +422,45 @@ struct MxuMmaCore {abs_row_base,act_groups_per_row,act_group_index,- [&](ushort tile_row, ushort thread_row, uint scale_index) {- activation_scales.slot(tile_row, thread_row) = a_scales[scale_index];+ [&](ushort tile_row, ushort thread_row, uint scale_index, uint) {+ const float activation_scale = a_scales[scale_index];+ activation_scales.slot(tile_row, thread_row) = activation_scale;+ if constexpr (int8_activation_needs_weight_correction) {+ activation_corrections.slot(tile_row, thread_row) +=+ activation_scale * float(a_group_sums[scale_index]);+ }});- uzu::matmul::Fragment<int, TILES_M, TILES_N, Ops> activation_row_sums;- if constexpr (int8_activation_needs_weight_correction) {- Ops::template fragment_mm<false, true>(activation_row_sums, activation_tile, ones_tile);- }AccumFragment::zip_for_each_coord(thread_context.simd_lane_id,- [&](short row, short col, thread float& accumulated, thread int& products, thread int& row_sums) {+ [&](short row, short col, thread float& accumulated, thread int& products) {if (!ALIGNED_M && row >= simdgroup_limit_m) {return;}if (!ALIGNED_N && col >= simdgroup_limit_n) {return;}- float scaled_products = weight_scales.at(col - position.x) * float(products);- if constexpr (int8_activation_needs_weight_correction) {- scaled_products += weight_corrections.at(col - position.x) * float(row_sums);- }- accumulated += activation_scales.at(row - position.y) * scaled_products;+ accumulated +=+ activation_scales.at(row - position.y) * weight_scales.at(col - position.x) * float(products);},accumulator,- chunk_products,- activation_row_sums+ chunk_products);}+ if constexpr (int8_activation_needs_weight_correction) {+ accumulator.map_coords(thread_context.simd_lane_id, [&](short row, short col, float value) {+ if (!ALIGNED_M && row >= simdgroup_limit_m) {+ return value;+ }+ if (!ALIGNED_N && col >= simdgroup_limit_n) {+ return value;+ }+ return value + weight_corrections.at(col - position.x) * activation_corrections.at(row - position.y);+ });+ }+a_int8_simdgroup += GROUP_SIZE;b_packed_simdgroup += k_bytes_per_weight_group;}@@ -465,6 +482,7 @@ struct MxuMmaCore {const device int32_t* rht_factors,const device int8_t* a_int8,const device float* a_scales,+ const device int32_t* a_group_sums,threadgroup BT* b_shared,const thread ThreadContext& thread_context) {@@ -544,10 +562,11 @@ struct MxuMmaCore {size_t(tile_row_offset) * params->leading_dimension_a;const device uint8_t* b_packed_simdgroup =quantized_weights.block + size_t(tile_col_offset) * quantized_weights.row_stride_bytes;- return symmetric_int8_activation_k_loop<aligned_m.value, aligned_n.value>(+ return int8_activation_k_loop<aligned_m.value, aligned_n.value>(a_int8_simdgroup,b_packed_simdgroup,a_scales,+ a_group_sums,scales,biases,zero_points,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 56b43101d..d43e78977 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -11,6 +11,7 @@ using namespace metal;using namespace uzu::gemm;#define A_IS_INT8 (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)+#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant)#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision && !A_IS_INT8)#define GEMM_TGA_ELEMENTS \((USE_MXU) ? 1 : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))@@ -114,6 +115,7 @@ KERNEL(Gemm)(OPTIONAL(output_transform.contains(GemmDTransform::RHT)),const device int8_t* a_int8 OPTIONAL(A_IS_INT8),const device float* a_scales OPTIONAL(A_IS_INT8),+ const device int32_t* a_group_sums OPTIONAL(NEEDS_ASYMMETRIC_WEIGHT_CORRECTION),const constant uzu::matmul::GemmParams* params,const constant uint& group_count_x,const constant uint& group_count_y,@@ -152,6 +154,7 @@ KERNEL(Gemm)(rht_factors,a_int8,a_scales,+ a_group_sums,b_shared,thread_context);diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 19b38102f..6afe3efce 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -325,6 +325,7 @@ impl GemmKernel {Some((a, a_offset)),None,None,+ None,GemmAPrologueKind::FullPrecision,weights,None,@@ -393,6 +394,7 @@ impl GemmKernel {rht_factors,None::<&Allocation<Metal>>,None::<&Allocation<Metal>>,+ None::<&Allocation<Metal>>,std::slice::from_ref(¶ms),group_count_x,group_count_y,@@ -432,14 +434,15 @@ impl GemmKernel {let a_prologue = a.prologue_kind();let a_is_int8 = a_prologue == GemmAPrologueKind::Int8Symmetric;- let (a_full_precision, a_int8, a_scales) = match a {+ let (a_full_precision, a_int8, a_scales, a_group_sums) = match a {MatmulA::FullPrecision {values,offset,- } => (Some((values, offset)), None, None),+ } => (Some((values, offset)), None, None, None),MatmulA::Int8Symmetric {values,scales: activation_scales,+ group_sums: activation_group_sums,} => {validate_int8_activation_arguments(use_mxu, k, b_prologue, bits_per_b, group_size)?;if output_transform.contains(GemmDTransform::SOFT_CAP) {@@ -449,7 +452,7 @@ impl GemmKernel {}.into());}- (None, Some(values), Some(activation_scales))+ (None, Some(values), Some(activation_scales), activation_group_sums)},};@@ -495,6 +498,7 @@ impl GemmKernel {a_full_precision,a_int8,a_scales,+ a_group_sums,a_prologue,weights,scales,@@ -542,6 +546,7 @@ impl GemmKernel {rht_factors,a_int8,a_scales,+ a_group_sums,std::slice::from_ref(¶ms),group_count_x,group_count_y,@@ -574,6 +579,7 @@ impl GemmKernel {a_full_precision: Option<(&Allocation<Metal>, usize)>,a_int8: Option<&Allocation<Metal>>,a_scales: Option<&Allocation<Metal>>,+ a_group_sums: Option<&Allocation<Metal>>,a_prologue: GemmAPrologueKind,weights: WB,scales: Option<&Allocation<Metal>>,@@ -645,6 +651,7 @@ impl GemmKernel {None::<&Allocation<Metal>>,a_int8,a_scales,+ a_group_sums,std::slice::from_ref(¶ms),base_gx,base_gy,diff --git a/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal b/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metalindex 54db55b69..099f03488 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal@@ -13,10 +13,12 @@ PUBLIC KERNEL(RHTQuantizeActivations)(const device InputT* input,device int8_t* q_out,device float* scales_out,+ device int32_t* group_sums_out OPTIONAL(emit_group_sums),const device int32_t* rht_factors,constant uint& batch_size,constant uint& element_count,constant uint& group_size,+ const bool emit_group_sums SPECIALIZE,uint block_index GROUPS(element_count.div_ceil(METAL_SIMD_SIZE)),uint batch_index GROUPS(batch_size),uint lane_index THREADS(METAL_SIMD_SIZE)@@ -29,9 +31,19 @@ PUBLIC KERNEL(RHTQuantizeActivations)(const float magnitude = max(fabs(simd_min(value)), fabs(simd_max(value)));const float scale = isfinite(magnitude) && magnitude > 0.0f ? magnitude / SYM_QMAX : 1.0f;++ const int8_t code = static_cast<int8_t>(clamp(round(value / scale), -SYM_QMAX, SYM_QMAX));+ q_out[element_index] = code;++ const uint group_index = batch_index * (element_count / group_size) + block_index;if (lane_index == 0) {- scales_out[batch_index * (element_count / group_size) + block_index] = scale;+ scales_out[group_index] = scale;}- q_out[element_index] = static_cast<int8_t>(clamp(round(value / scale), -SYM_QMAX, SYM_QMAX));+ if (emit_group_sums) {+ const int group_sum = simd_sum(int(code));+ if (lane_index == 0) {+ group_sums_out[group_index] = group_sum;+ }+ }}diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex 4ccf94fb4..12e51b498 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -71,8 +71,19 @@ pub enum RHTLinearWrapperError<B: Backend> {UnsupportedConfiguration(String),}+fn weights_need_group_sums(quantization_spec: &AnyWeightMatrixSpec) -> bool {+ !matches!(+ quantization_spec,+ AnyWeightMatrixSpec::IntSpec(IntSpec {+ is_symmetric: true,+ ..+ })+ )+}+struct SymmetricInt8Preparation<B: Backend> {kernel: <B::Kernels as Kernels>::RHTQuantizeActivationsKernel,+ emit_group_sums: bool,}pub struct RHTLinearWrapper<B: Backend> {@@ -131,10 +142,12 @@ impl<B: Backend> RHTLinearWrapper<B> {input_data_type,output_data_type,) {+ let emit_group_sums = weights_need_group_sums(&quantization_spec);Some(- <B::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(context, input_data_type)+ <B::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(context, input_data_type, emit_group_sums).map(|kernel| SymmetricInt8Preparation {kernel,+ emit_group_sums,}).map_err(RHTLinearWrapperError::BackendError)?,)@@ -177,11 +190,16 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {let mut values =encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;let mut scales = encoder.allocate_scratch(size_for_shape(&[batch_dim, groups_per_row], DataType::F32))?;+ let mut group_sums = preparation+ .emit_group_sums+ .then(|| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups_per_row], DataType::I32)))+ .transpose()?;preparation.kernel.encode(&input,&mut values,&mut scales,+ group_sums.as_mut(),&self.input_factors,batch_dim as u32,self.input_dimension as u32,@@ -192,6 +210,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {MatmulA::Int8Symmetric {values: &values,scales: &scales,+ group_sums: group_sums.as_ref(),},batch_dim,encoder,diff --git a/crates/backend-uzu/src/tests/matmul/quant.rs b/crates/backend-uzu/src/tests/matmul/quant.rsindex c153438de..6675f7984 100644--- a/crates/backend-uzu/src/tests/matmul/quant.rs+++ b/crates/backend-uzu/src/tests/matmul/quant.rs@@ -27,6 +27,7 @@ use crate::{pub struct PreparedInt8A {pub values: Vec<i8>,pub scales: Vec<f32>,+ pub group_sums: Vec<i32>,}pub struct QuantInput<T: ArrayElement + Float> {@@ -109,6 +110,7 @@ impl<T: ArrayElement + Float> QuantInput<T> {let groups = columns.div_ceil(group_size);let mut values = vec![0i8; rows * columns];let mut scales = vec![0.0f32; rows * groups];+ let mut group_sums = vec![0i32; rows * groups];for row in 0..rows {for group in 0..groups {@@ -118,15 +120,20 @@ impl<T: ArrayElement + Float> QuantInput<T> {(start..end).map(|column| self.x[row * columns + column].to_f32().unwrap()).collect::<Vec<_>>();let divisor = min_max_symmetric_divisor(&prepared);scales[row * groups + group] = divisor;+ let mut group_sum = 0i32;for (column, value) in (start..end).zip(prepared) {- values[row * columns + column] = quantize_symmetric_i8(value, divisor);+ let code = quantize_symmetric_i8(value, divisor);+ values[row * columns + column] = code;+ group_sum += i32::from(code);}+ group_sums[row * groups + group] = group_sum;}}self.prepared_a = Some(PreparedInt8A {values,scales,+ group_sums,});self}@@ -151,6 +158,7 @@ pub struct QuantBuffers<B: Backend, T: ArrayElement + Float> {pub x: Allocation<B>,pub prepared_a: Option<Allocation<B>>,pub prepared_a_scales: Option<Allocation<B>>,+ pub prepared_a_group_sums: Option<Allocation<B>>,pub y: Allocation<B>,_t: std::marker::PhantomData<T>,}@@ -174,6 +182,10 @@ impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {.prepared_a.as_ref().map(|prepared| alloc_allocation_with_data::<B, f32>(context, &prepared.scales)),+ prepared_a_group_sums: input+ .prepared_a+ .as_ref()+ .map(|prepared| alloc_allocation_with_data::<B, i32>(context, &prepared.group_sums)),y: alloc_allocation::<B, T>(context, (input.m as usize) * (input.n as usize)),_t: std::marker::PhantomData,}@@ -210,6 +222,9 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(Some(_) => MatmulA::Int8Symmetric {values: buffers.prepared_a.as_ref().expect("prepared activation buffer"),scales: buffers.prepared_a_scales.as_ref().expect("prepared activation scales"),+ // Symmetric weights carry no correction term, so the GEMM never reads these.+ group_sums: (input.quant_method != QuantizationMethod::ScaleSymmetric)+ .then(|| buffers.prepared_a_group_sums.as_ref().expect("prepared activation row sums")),},None => MatmulA::FullPrecision {values: &buffers.x,diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rsindex c197ab72c..38b4ad6cd 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs@@ -154,6 +154,7 @@ fn encode_step(&data.activations,&mut data.a_int8,&mut data.a_scales,+ None::<&mut Allocation<Metal>>,&data.rht_factors,data.m,data.k,@@ -164,6 +165,7 @@ fn encode_step(a: MatmulA::Int8Symmetric {values: &data.a_int8,scales: &data.a_scales,+ group_sums: None,},b: MatmulB::ScaleSymmetricDequant {b: &data.weights_u8,@@ -269,7 +271,7 @@ fn bench_a8w(c: &mut Criterion) {let device_tier = context.device_tier();let prepare =- <MetalPrepare as RHTQuantizeActivationsKernel>::new(&context, DataType::BF16).expect("prepare kernel");+ <MetalPrepare as RHTQuantizeActivationsKernel>::new(&context, DataType::BF16, false).expect("prepare kernel");let hadamard =<MetalHadamard as HadamardTransformKernel>::new(&context, DataType::BF16, HadamardTransformOrder::Input).expect("hadamard kernel");diff --git a/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs b/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rsindex 9f6525c87..91f3d7251 100644--- a/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs+++ b/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs@@ -42,21 +42,25 @@ fn rht_quantize_matches_cpu() {let mut metal_scales = alloc_allocation::<Metal, f32>(&metal, rows * groups);let mut cpu_values = alloc_allocation::<Cpu, i8>(&cpu, rows * columns);let mut cpu_scales = alloc_allocation::<Cpu, f32>(&cpu, rows * groups);+ let mut metal_group_sums = alloc_allocation::<Metal, i32>(&metal, rows * groups);+ let mut cpu_group_sums = alloc_allocation::<Cpu, i32>(&cpu, rows * groups);let metal_input = alloc_allocation_with_data::<Metal, f32>(&metal, &input_data);let metal_factors = alloc_allocation_with_data::<Metal, i32>(&metal, &factors_data);let cpu_input = alloc_allocation_with_data::<Cpu, f32>(&cpu, &input_data);let cpu_factors = alloc_allocation_with_data::<Cpu, i32>(&cpu, &factors_data);- let metal_kernel = RHTQuantizeActivationsMetalKernel::new(&metal, DataType::F32).expect("metal prepare");- let cpu_kernel = <<Cpu as Backend>::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(&cpu, DataType::F32)- .expect("cpu prepare");+ let metal_kernel = RHTQuantizeActivationsMetalKernel::new(&metal, DataType::F32, true).expect("metal prepare");+ let cpu_kernel =+ <<Cpu as Backend>::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(&cpu, DataType::F32, true)+ .expect("cpu prepare");let mut metal_enc = Encoder::<Metal>::new(&metal).expect("metal encoder");metal_kernel.encode(&metal_input,&mut metal_values,&mut metal_scales,+ Some(&mut metal_group_sums),&metal_factors,rows as u32,columns as u32,@@ -70,6 +74,7 @@ fn rht_quantize_matches_cpu() {&cpu_input,&mut cpu_values,&mut cpu_scales,+ Some(&mut cpu_group_sums),&cpu_factors,rows as u32,columns as u32,@@ -84,5 +89,18 @@ fn rht_quantize_matches_cpu() {let rel = (a - e).abs() / e.abs().max(1e-6);assert!(rel < 1e-3, "scale {i}: {a} != {e}");}- assert!(mv.iter().zip(cv).all(|(a, e)| (*a as i32 - e as i32).abs() <= 1));+ assert!(mv.iter().zip(&cv).all(|(a, e)| (*a as i32 - *e as i32).abs() <= 1));++ let mrs = allocation_to_vec::<Metal, i32>(&metal_group_sums);+ let crs = allocation_to_vec::<Cpu, i32>(&cpu_group_sums);+ for (codes, sums, label) in [(&mv, &mrs, "metal"), (&cv, &crs, "cpu")] {+ for row in 0..rows {+ for group in 0..groups {+ let start = row * columns + group * group_size as usize;+ let expected: i32 = codes[start..start + group_size as usize].iter().map(|code| *code as i32).sum();+ assert_eq!(sums[row * groups + group], expected, "{label} group_sum r{row} g{group}");+ }+ }+ }+ assert!(mrs.iter().zip(&crs).all(|(a, e)| (a - e).abs() <= group_size as i32));}