Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

A8W8 bias/zp perf (#647)

CCYeh6ad005d6c868
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_a.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_a.rs
index 49565bf8c..78a643f66 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_a.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_a.rs
@@ -8,6 +8,7 @@ pub enum MatmulA<'a, B: Backend> {
Int8Symmetric {
values: &'a Allocation<B>,
scales: &'a Allocation<B>,
+ group_sums: Option<&'a Allocation<B>>,
},
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs
index 0d46008b1..486d537d1 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs
@@ -110,6 +110,7 @@ impl MatmulKernel for MatmulCpuKernel {
MatmulA::Int8Symmetric {
values,
scales,
+ group_sums: _,
} => {
let weight_gs_ok = matches!(b.group_size(), Some(32 | 64 | 128));
let weights_ok = matches!(
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs b/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs
index 87e6e4fca..2899848af 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs
@@ -13,10 +13,14 @@ pub fn activations_prepare<InputT: ArrayElement + Float>(
input: *const InputT,
q_out: *mut i8,
scales_out: *mut f32,
+ #[optional(emit_group_sums)] group_sums_out: Option<*mut i32>,
rht_factors: *const i32,
batch_size: u32,
element_count: u32,
group_size: u32,
+ #[allow(unused)]
+ #[specialize]
+ emit_group_sums: bool,
) {
let rows = batch_size as usize;
let columns = element_count as usize;
@@ -45,9 +49,14 @@ pub fn activations_prepare<InputT: ArrayElement + Float>(
let slice = &prepared[start..end];
let divisor = min_max_symmetric_divisor(slice);
unsafe { *scales_out.add(row * groups + group) = divisor };
+ let mut group_sum = 0i32;
for index in start..end {
let q = quantize_symmetric_i8(prepared[index], divisor);
unsafe { *q_out.add(row * columns + index) = q };
+ group_sum += q as i32;
+ }
+ if let Some(group_sums_out) = group_sums_out {
+ unsafe { *group_sums_out.add(row * groups + group) = group_sum };
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 5cd74dc90..20c89be17 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -253,6 +253,13 @@ struct MxuMmaCore {
return values[tile * SLOTS_PER_TILE + index];
}
+ METAL_FUNC void clear() thread {
+ METAL_PRAGMA_UNROLL
+ for (ushort index = 0; index < TILES * SLOTS_PER_TILE; ++index) {
+ values[index] = 0.0f;
+ }
+ }
+
METAL_FUNC float at(const short offset) const thread {
const ushort tile = ushort(offset) / FRAGMENT_EXTENT;
const ushort index = (ushort(offset) % FRAGMENT_EXTENT) / SLOT_STRIDE;
@@ -274,7 +281,8 @@ struct MxuMmaCore {
for (ushort index = 0; index < SLOTS_PER_TILE; ++index) {
const short coord = origin + tile * FRAGMENT_EXTENT + index * SLOT_STRIDE;
if (ALIGNED || coord < limit) {
- sink(tile, index, (abs_base + uint(coord)) * groups_per_row + group_index);
+ const uint line_index = abs_base + uint(coord);
+ sink(tile, index, line_index * groups_per_row + group_index, line_index);
}
}
}
@@ -300,7 +308,7 @@ struct MxuMmaCore {
static METAL_FUNC float int8_weight_dequant_bias(
const float scale,
const uint scale_index,
- const uint col,
+ const uint weight_column,
const uint weight_group,
const uint weight_groups_per_row,
const device BT* biases,
@@ -309,19 +317,24 @@ struct MxuMmaCore {
if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
return static_cast<float>(biases[scale_index]);
} else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
- const device uint8_t* zero_points_row =
- zero_points + col * zero_point_row_stride<ushort(BITS)>(weight_groups_per_row);
- return -scale * float(decode_zero_point<ushort(BITS)>(zero_points_row, weight_group));
+ if constexpr (BITS == 8) {
+ return -scale * float(zero_points[scale_index]);
+ } else {
+ const device uint8_t* zero_points_row =
+ zero_points + weight_column * zero_point_row_stride<ushort(BITS)>(weight_groups_per_row);
+ return -scale * float(decode_zero_point<ushort(BITS)>(zero_points_row, weight_group));
+ }
} else {
return 0.0f;
}
}
template <bool ALIGNED_M, bool ALIGNED_N>
- static METAL_FUNC AccumFragment symmetric_int8_activation_k_loop(
+ static METAL_FUNC AccumFragment int8_activation_k_loop(
const device int8_t* a_int8_simdgroup,
const device uint8_t* b_packed_simdgroup,
const device float* a_scales,
+ const device int32_t* a_group_sums,
const device BT* b_scales,
const device BT* biases,
const device uint8_t* zero_points,
@@ -346,15 +359,6 @@ struct MxuMmaCore {
constexpr int k_bytes_per_weight_group = (BITS == 4) ? (int(GROUP_SIZE) / 2) : int(GROUP_SIZE);
constexpr int act_chunks_per_weight_group = int(GROUP_SIZE) / int(SIMDGROUP_BLOCK_K);
- uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, Ops> ones_tile;
- if constexpr (int8_activation_needs_weight_correction) {
- thread int8_t* ones = ones_tile.elements();
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < ones_tile.ELEMENTS_PER_FRAGMENT; ++i) {
- ones[i] = int8_t(1);
- }
- }
-
METAL_PRAGMA_NO_UNROLL
for (int weight_group = 0; weight_group < weight_group_iterations; ++weight_group) {
const uint weight_group_index = k_offset_weight_groups + uint(weight_group);
@@ -367,15 +371,14 @@ struct MxuMmaCore {
abs_col_base,
weight_groups_per_row,
weight_group_index,
- [&](ushort tile_col, ushort thread_col, uint scale_index) {
+ [&](ushort tile_col, ushort thread_col, uint scale_index, uint weight_column) {
const float scale = static_cast<float>(b_scales[scale_index]);
weight_scales.slot(tile_col, thread_col) = scale;
if constexpr (int8_activation_needs_weight_correction) {
- const uint col = scale_index / weight_groups_per_row;
weight_corrections.slot(tile_col, thread_col) = scale * int8_weight_midpoint + int8_weight_dequant_bias(
scale,
scale_index,
- col,
+ weight_column,
weight_group_index,
weight_groups_per_row,
biases,
@@ -385,6 +388,11 @@ struct MxuMmaCore {
}
);
+ ActivationLineCache activation_corrections;
+ if constexpr (int8_activation_needs_weight_correction) {
+ activation_corrections.clear();
+ }
+
METAL_PRAGMA_NO_UNROLL
for (int act_chunk = 0; act_chunk < act_chunks_per_weight_group; ++act_chunk) {
const int k_element_offset = act_chunk * int(SIMDGROUP_BLOCK_K);
@@ -414,36 +422,45 @@ struct MxuMmaCore {
abs_row_base,
act_groups_per_row,
act_group_index,
- [&](ushort tile_row, ushort thread_row, uint scale_index) {
- activation_scales.slot(tile_row, thread_row) = a_scales[scale_index];
+ [&](ushort tile_row, ushort thread_row, uint scale_index, uint) {
+ const float activation_scale = a_scales[scale_index];
+ activation_scales.slot(tile_row, thread_row) = activation_scale;
+ if constexpr (int8_activation_needs_weight_correction) {
+ activation_corrections.slot(tile_row, thread_row) +=
+ activation_scale * float(a_group_sums[scale_index]);
+ }
}
);
- uzu::matmul::Fragment<int, TILES_M, TILES_N, Ops> activation_row_sums;
- if constexpr (int8_activation_needs_weight_correction) {
- Ops::template fragment_mm<false, true>(activation_row_sums, activation_tile, ones_tile);
- }
AccumFragment::zip_for_each_coord(
thread_context.simd_lane_id,
- [&](short row, short col, thread float& accumulated, thread int& products, thread int& row_sums) {
+ [&](short row, short col, thread float& accumulated, thread int& products) {
if (!ALIGNED_M && row >= simdgroup_limit_m) {
return;
}
if (!ALIGNED_N && col >= simdgroup_limit_n) {
return;
}
- float scaled_products = weight_scales.at(col - position.x) * float(products);
- if constexpr (int8_activation_needs_weight_correction) {
- scaled_products += weight_corrections.at(col - position.x) * float(row_sums);
- }
- accumulated += activation_scales.at(row - position.y) * scaled_products;
+ accumulated +=
+ activation_scales.at(row - position.y) * weight_scales.at(col - position.x) * float(products);
},
accumulator,
- chunk_products,
- activation_row_sums
+ chunk_products
);
}
+ if constexpr (int8_activation_needs_weight_correction) {
+ accumulator.map_coords(thread_context.simd_lane_id, [&](short row, short col, float value) {
+ if (!ALIGNED_M && row >= simdgroup_limit_m) {
+ return value;
+ }
+ if (!ALIGNED_N && col >= simdgroup_limit_n) {
+ return value;
+ }
+ return value + weight_corrections.at(col - position.x) * activation_corrections.at(row - position.y);
+ });
+ }
+
a_int8_simdgroup += GROUP_SIZE;
b_packed_simdgroup += k_bytes_per_weight_group;
}
@@ -465,6 +482,7 @@ struct MxuMmaCore {
const device int32_t* rht_factors,
const device int8_t* a_int8,
const device float* a_scales,
+ const device int32_t* a_group_sums,
threadgroup BT* b_shared,
const thread ThreadContext& thread_context
) {
@@ -544,10 +562,11 @@ struct MxuMmaCore {
size_t(tile_row_offset) * params->leading_dimension_a;
const device uint8_t* b_packed_simdgroup =
quantized_weights.block + size_t(tile_col_offset) * quantized_weights.row_stride_bytes;
- return symmetric_int8_activation_k_loop<aligned_m.value, aligned_n.value>(
+ return int8_activation_k_loop<aligned_m.value, aligned_n.value>(
a_int8_simdgroup,
b_packed_simdgroup,
a_scales,
+ a_group_sums,
scales,
biases,
zero_points,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 56b43101d..d43e78977 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -11,6 +11,7 @@ using namespace metal;
using namespace uzu::gemm;
#define A_IS_INT8 (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)
+#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant)
#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision && !A_IS_INT8)
#define GEMM_TGA_ELEMENTS \
((USE_MXU) ? 1 : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
@@ -114,6 +115,7 @@ KERNEL(Gemm)(
OPTIONAL(output_transform.contains(GemmDTransform::RHT)),
const device int8_t* a_int8 OPTIONAL(A_IS_INT8),
const device float* a_scales OPTIONAL(A_IS_INT8),
+ const device int32_t* a_group_sums OPTIONAL(NEEDS_ASYMMETRIC_WEIGHT_CORRECTION),
const constant uzu::matmul::GemmParams* params,
const constant uint& group_count_x,
const constant uint& group_count_y,
@@ -152,6 +154,7 @@ KERNEL(Gemm)(
rht_factors,
a_int8,
a_scales,
+ a_group_sums,
b_shared,
thread_context
);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 19b38102f..6afe3efce 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -325,6 +325,7 @@ impl GemmKernel {
Some((a, a_offset)),
None,
None,
+ None,
GemmAPrologueKind::FullPrecision,
weights,
None,
@@ -393,6 +394,7 @@ impl GemmKernel {
rht_factors,
None::<&Allocation<Metal>>,
None::<&Allocation<Metal>>,
+ None::<&Allocation<Metal>>,
std::slice::from_ref(&params),
group_count_x,
group_count_y,
@@ -432,14 +434,15 @@ impl GemmKernel {
let a_prologue = a.prologue_kind();
let a_is_int8 = a_prologue == GemmAPrologueKind::Int8Symmetric;
- let (a_full_precision, a_int8, a_scales) = match a {
+ let (a_full_precision, a_int8, a_scales, a_group_sums) = match a {
MatmulA::FullPrecision {
values,
offset,
- } => (Some((values, offset)), None, None),
+ } => (Some((values, offset)), None, None, None),
MatmulA::Int8Symmetric {
values,
scales: activation_scales,
+ group_sums: activation_group_sums,
} => {
validate_int8_activation_arguments(use_mxu, k, b_prologue, bits_per_b, group_size)?;
if output_transform.contains(GemmDTransform::SOFT_CAP) {
@@ -449,7 +452,7 @@ impl GemmKernel {
}
.into());
}
- (None, Some(values), Some(activation_scales))
+ (None, Some(values), Some(activation_scales), activation_group_sums)
},
};
@@ -495,6 +498,7 @@ impl GemmKernel {
a_full_precision,
a_int8,
a_scales,
+ a_group_sums,
a_prologue,
weights,
scales,
@@ -542,6 +546,7 @@ impl GemmKernel {
rht_factors,
a_int8,
a_scales,
+ a_group_sums,
std::slice::from_ref(&params),
group_count_x,
group_count_y,
@@ -574,6 +579,7 @@ impl GemmKernel {
a_full_precision: Option<(&Allocation<Metal>, usize)>,
a_int8: Option<&Allocation<Metal>>,
a_scales: Option<&Allocation<Metal>>,
+ a_group_sums: Option<&Allocation<Metal>>,
a_prologue: GemmAPrologueKind,
weights: WB,
scales: Option<&Allocation<Metal>>,
@@ -645,6 +651,7 @@ impl GemmKernel {
None::<&Allocation<Metal>>,
a_int8,
a_scales,
+ a_group_sums,
std::slice::from_ref(&params),
base_gx,
base_gy,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal b/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal
index 54db55b69..099f03488 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal
@@ -13,10 +13,12 @@ PUBLIC KERNEL(RHTQuantizeActivations)(
const device InputT* input,
device int8_t* q_out,
device float* scales_out,
+ device int32_t* group_sums_out OPTIONAL(emit_group_sums),
const device int32_t* rht_factors,
constant uint& batch_size,
constant uint& element_count,
constant uint& group_size,
+ const bool emit_group_sums SPECIALIZE,
uint block_index GROUPS(element_count.div_ceil(METAL_SIMD_SIZE)),
uint batch_index GROUPS(batch_size),
uint lane_index THREADS(METAL_SIMD_SIZE)
@@ -29,9 +31,19 @@ PUBLIC KERNEL(RHTQuantizeActivations)(
const float magnitude = max(fabs(simd_min(value)), fabs(simd_max(value)));
const float scale = isfinite(magnitude) && magnitude > 0.0f ? magnitude / SYM_QMAX : 1.0f;
+
+ const int8_t code = static_cast<int8_t>(clamp(round(value / scale), -SYM_QMAX, SYM_QMAX));
+ q_out[element_index] = code;
+
+ const uint group_index = batch_index * (element_count / group_size) + block_index;
if (lane_index == 0) {
- scales_out[batch_index * (element_count / group_size) + block_index] = scale;
+ scales_out[group_index] = scale;
}
- q_out[element_index] = static_cast<int8_t>(clamp(round(value / scale), -SYM_QMAX, SYM_QMAX));
+ if (emit_group_sums) {
+ const int group_sum = simd_sum(int(code));
+ if (lane_index == 0) {
+ group_sums_out[group_index] = group_sum;
+ }
+ }
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
index 4ccf94fb4..12e51b498 100644
--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
@@ -71,8 +71,19 @@ pub enum RHTLinearWrapperError<B: Backend> {
UnsupportedConfiguration(String),
}
+fn weights_need_group_sums(quantization_spec: &AnyWeightMatrixSpec) -> bool {
+ !matches!(
+ quantization_spec,
+ AnyWeightMatrixSpec::IntSpec(IntSpec {
+ is_symmetric: true,
+ ..
+ })
+ )
+}
+
struct SymmetricInt8Preparation<B: Backend> {
kernel: <B::Kernels as Kernels>::RHTQuantizeActivationsKernel,
+ emit_group_sums: bool,
}
pub struct RHTLinearWrapper<B: Backend> {
@@ -131,10 +142,12 @@ impl<B: Backend> RHTLinearWrapper<B> {
input_data_type,
output_data_type,
) {
+ let emit_group_sums = weights_need_group_sums(&quantization_spec);
Some(
- <B::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(context, input_data_type)
+ <B::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(context, input_data_type, emit_group_sums)
.map(|kernel| SymmetricInt8Preparation {
kernel,
+ emit_group_sums,
})
.map_err(RHTLinearWrapperError::BackendError)?,
)
@@ -177,11 +190,16 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
let mut values =
encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;
let mut scales = encoder.allocate_scratch(size_for_shape(&[batch_dim, groups_per_row], DataType::F32))?;
+ let mut group_sums = preparation
+ .emit_group_sums
+ .then(|| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups_per_row], DataType::I32)))
+ .transpose()?;
preparation.kernel.encode(
&input,
&mut values,
&mut scales,
+ group_sums.as_mut(),
&self.input_factors,
batch_dim as u32,
self.input_dimension as u32,
@@ -192,6 +210,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
MatmulA::Int8Symmetric {
values: &values,
scales: &scales,
+ group_sums: group_sums.as_ref(),
},
batch_dim,
encoder,
diff --git a/crates/backend-uzu/src/tests/matmul/quant.rs b/crates/backend-uzu/src/tests/matmul/quant.rs
index c153438de..6675f7984 100644
--- a/crates/backend-uzu/src/tests/matmul/quant.rs
+++ b/crates/backend-uzu/src/tests/matmul/quant.rs
@@ -27,6 +27,7 @@ use crate::{
pub struct PreparedInt8A {
pub values: Vec<i8>,
pub scales: Vec<f32>,
+ pub group_sums: Vec<i32>,
}
pub struct QuantInput<T: ArrayElement + Float> {
@@ -109,6 +110,7 @@ impl<T: ArrayElement + Float> QuantInput<T> {
let groups = columns.div_ceil(group_size);
let mut values = vec![0i8; rows * columns];
let mut scales = vec![0.0f32; rows * groups];
+ let mut group_sums = vec![0i32; rows * groups];
for row in 0..rows {
for group in 0..groups {
@@ -118,15 +120,20 @@ impl<T: ArrayElement + Float> QuantInput<T> {
(start..end).map(|column| self.x[row * columns + column].to_f32().unwrap()).collect::<Vec<_>>();
let divisor = min_max_symmetric_divisor(&prepared);
scales[row * groups + group] = divisor;
+ let mut group_sum = 0i32;
for (column, value) in (start..end).zip(prepared) {
- values[row * columns + column] = quantize_symmetric_i8(value, divisor);
+ let code = quantize_symmetric_i8(value, divisor);
+ values[row * columns + column] = code;
+ group_sum += i32::from(code);
}
+ group_sums[row * groups + group] = group_sum;
}
}
self.prepared_a = Some(PreparedInt8A {
values,
scales,
+ group_sums,
});
self
}
@@ -151,6 +158,7 @@ pub struct QuantBuffers<B: Backend, T: ArrayElement + Float> {
pub x: Allocation<B>,
pub prepared_a: Option<Allocation<B>>,
pub prepared_a_scales: Option<Allocation<B>>,
+ pub prepared_a_group_sums: Option<Allocation<B>>,
pub y: Allocation<B>,
_t: std::marker::PhantomData<T>,
}
@@ -174,6 +182,10 @@ impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {
.prepared_a
.as_ref()
.map(|prepared| alloc_allocation_with_data::<B, f32>(context, &prepared.scales)),
+ prepared_a_group_sums: input
+ .prepared_a
+ .as_ref()
+ .map(|prepared| alloc_allocation_with_data::<B, i32>(context, &prepared.group_sums)),
y: alloc_allocation::<B, T>(context, (input.m as usize) * (input.n as usize)),
_t: std::marker::PhantomData,
}
@@ -210,6 +222,9 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(
Some(_) => MatmulA::Int8Symmetric {
values: buffers.prepared_a.as_ref().expect("prepared activation buffer"),
scales: buffers.prepared_a_scales.as_ref().expect("prepared activation scales"),
+ // Symmetric weights carry no correction term, so the GEMM never reads these.
+ group_sums: (input.quant_method != QuantizationMethod::ScaleSymmetric)
+ .then(|| buffers.prepared_a_group_sums.as_ref().expect("prepared activation row sums")),
},
None => MatmulA::FullPrecision {
values: &buffers.x,
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs
index c197ab72c..38b4ad6cd 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs
@@ -154,6 +154,7 @@ fn encode_step(
&data.activations,
&mut data.a_int8,
&mut data.a_scales,
+ None::<&mut Allocation<Metal>>,
&data.rht_factors,
data.m,
data.k,
@@ -164,6 +165,7 @@ fn encode_step(
a: MatmulA::Int8Symmetric {
values: &data.a_int8,
scales: &data.a_scales,
+ group_sums: None,
},
b: MatmulB::ScaleSymmetricDequant {
b: &data.weights_u8,
@@ -269,7 +271,7 @@ fn bench_a8w(c: &mut Criterion) {
let device_tier = context.device_tier();
let prepare =
- <MetalPrepare as RHTQuantizeActivationsKernel>::new(&context, DataType::BF16).expect("prepare kernel");
+ <MetalPrepare as RHTQuantizeActivationsKernel>::new(&context, DataType::BF16, false).expect("prepare kernel");
let hadamard =
<MetalHadamard as HadamardTransformKernel>::new(&context, DataType::BF16, HadamardTransformOrder::Input)
.expect("hadamard kernel");
diff --git a/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs b/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs
index 9f6525c87..91f3d7251 100644
--- a/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs
@@ -42,21 +42,25 @@ fn rht_quantize_matches_cpu() {
let mut metal_scales = alloc_allocation::<Metal, f32>(&metal, rows * groups);
let mut cpu_values = alloc_allocation::<Cpu, i8>(&cpu, rows * columns);
let mut cpu_scales = alloc_allocation::<Cpu, f32>(&cpu, rows * groups);
+ let mut metal_group_sums = alloc_allocation::<Metal, i32>(&metal, rows * groups);
+ let mut cpu_group_sums = alloc_allocation::<Cpu, i32>(&cpu, rows * groups);
let metal_input = alloc_allocation_with_data::<Metal, f32>(&metal, &input_data);
let metal_factors = alloc_allocation_with_data::<Metal, i32>(&metal, &factors_data);
let cpu_input = alloc_allocation_with_data::<Cpu, f32>(&cpu, &input_data);
let cpu_factors = alloc_allocation_with_data::<Cpu, i32>(&cpu, &factors_data);
- let metal_kernel = RHTQuantizeActivationsMetalKernel::new(&metal, DataType::F32).expect("metal prepare");
- let cpu_kernel = <<Cpu as Backend>::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(&cpu, DataType::F32)
- .expect("cpu prepare");
+ let metal_kernel = RHTQuantizeActivationsMetalKernel::new(&metal, DataType::F32, true).expect("metal prepare");
+ let cpu_kernel =
+ <<Cpu as Backend>::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(&cpu, DataType::F32, true)
+ .expect("cpu prepare");
let mut metal_enc = Encoder::<Metal>::new(&metal).expect("metal encoder");
metal_kernel.encode(
&metal_input,
&mut metal_values,
&mut metal_scales,
+ Some(&mut metal_group_sums),
&metal_factors,
rows as u32,
columns as u32,
@@ -70,6 +74,7 @@ fn rht_quantize_matches_cpu() {
&cpu_input,
&mut cpu_values,
&mut cpu_scales,
+ Some(&mut cpu_group_sums),
&cpu_factors,
rows as u32,
columns as u32,
@@ -84,5 +89,18 @@ fn rht_quantize_matches_cpu() {
let rel = (a - e).abs() / e.abs().max(1e-6);
assert!(rel < 1e-3, "scale {i}: {a} != {e}");
}
- assert!(mv.iter().zip(cv).all(|(a, e)| (*a as i32 - e as i32).abs() <= 1));
+ assert!(mv.iter().zip(&cv).all(|(a, e)| (*a as i32 - *e as i32).abs() <= 1));
+
+ let mrs = allocation_to_vec::<Metal, i32>(&metal_group_sums);
+ let crs = allocation_to_vec::<Cpu, i32>(&cpu_group_sums);
+ for (codes, sums, label) in [(&mv, &mrs, "metal"), (&cv, &crs, "cpu")] {
+ for row in 0..rows {
+ for group in 0..groups {
+ let start = row * columns + group * group_size as usize;
+ let expected: i32 = codes[start..start + group_size as usize].iter().map(|code| *code as i32).sum();
+ assert_eq!(sums[row * groups + group], expected, "{label} group_sum r{row} g{group}");
+ }
+ }
+ }
+ assert!(mrs.iter().zip(&crs).all(|(a, e)| (a - e).abs() <= group_size as i32));
}