A high-performance inference engine for AI models
Symmetric quants (#456)
uuuvn2a6beca8b5e0
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rsindex c77613df7..d433ae913 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs@@ -7,6 +7,7 @@ pub enum GemmWeightPrologueKind {FullPrecision,ScaleBiasDequant,ScaleZeroPointDequant,+ ScaleSymmetricDequant,}bitflags! {diff --git a/crates/backend-uzu/src/backends/common/gpu_types/quantization_method.rs b/crates/backend-uzu/src/backends/common/gpu_types/quantization_method.rsindex 2127719e0..1e5965c12 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/quantization_method.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/quantization_method.rs@@ -1,14 +1,11 @@use std::fmt;-use serde::{Deserialize, Serialize};-#[repr(C)]-#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]pub enum QuantizationMethod {- #[serde(rename = "mlx")]ScaleBias,- #[serde(rename = "awq")]ScaleZeroPoint,+ ScaleSymmetric,}impl fmt::Display for QuantizationMethod {diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rsindex 55774863a..0045eb13e 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs@@ -24,6 +24,12 @@ pub enum MatmulB<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {mode: QuantizationMode,group_size: u32,},+ ScaleSymmetricDequant {+ b: &'a Allocation<B>,+ scales: &'a Allocation<B>,+ mode: QuantizationMode,+ group_size: u32,+ },}impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {@@ -38,6 +44,9 @@ impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {Self::ScaleZeroPointDequant {..} => GemmWeightPrologueKind::ScaleZeroPointDequant,+ Self::ScaleSymmetricDequant {+ ..+ } => GemmWeightPrologueKind::ScaleSymmetricDequant,}}@@ -53,6 +62,10 @@ impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {| Self::ScaleZeroPointDequant {mode,..+ }+ | Self::ScaleSymmetricDequant {+ mode,+ ..} => Some(DataType::from(*mode).size_in_bits() as u32),}}@@ -69,6 +82,10 @@ impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {| Self::ScaleZeroPointDequant {group_size,..+ }+ | Self::ScaleSymmetricDequant {+ group_size,+ ..} => Some(*group_size),}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex 97e257a8a..92abae641 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -3,7 +3,7 @@ use num_traits::Float;use proc_macros::kernel;use crate::{- ArrayElement,+ ArrayElement, DataType,backends::common::gpu_types::{QuantizationMethod, QuantizationMode},};@@ -89,6 +89,12 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(};let bias = match quantization_method {+ QuantizationMethod::ScaleBias => biases+ .expect("ScaleBias quantized embedding requires biases")+ .add((token_id as u32 * num_groups + group_idx) as usize)+ .read()+ .to_f32()+ .unwrap(),QuantizationMethod::ScaleZeroPoint => {let zero_points = zero_points.expect("ScaleZeroPoint quantized embedding requires zero_points");let zero_point = match quantization_mode {@@ -107,12 +113,10 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(};-scale.to_f32().unwrap() * zero_point as f32},- QuantizationMethod::ScaleBias => biases- .expect("ScaleBias quantized embedding requires biases")- .add((token_id as u32 * num_groups + group_idx) as usize)- .read()- .to_f32()- .unwrap(),+ QuantizationMethod::ScaleSymmetric => {+ let midpoint = 1 << (DataType::from(quantization_mode).size_in_bits() - 1);+ -scale.to_f32().unwrap() * midpoint as f32+ },};let out_f = scale.to_f32().unwrap() * quantized_value as f32 + bias;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rsindex 56b2afffc..7244c30e9 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs@@ -47,6 +47,10 @@ pub fn qmm_transposed<T: ArrayElement + Float>(let scale_t = *scales.add(j * num_groups_k + group_idx);let w_dequant_f32 = match quant_method {+ QuantizationMethod::ScaleBias => {+ let bias_t = *biases.unwrap().add(j * num_groups_k + group_idx);+ scale_t.to_f32().unwrap() * val_q + bias_t.to_f32().unwrap()+ },QuantizationMethod::ScaleZeroPoint => {let zp = zero_points.unwrap();let zp_val = if bits == 4 {@@ -62,9 +66,9 @@ pub fn qmm_transposed<T: ArrayElement + Float>(};scale_t.to_f32().unwrap() * (val_q - zp_val)},- QuantizationMethod::ScaleBias => {- let bias_t = *biases.unwrap().add(j * num_groups_k + group_idx);- scale_t.to_f32().unwrap() * val_q + bias_t.to_f32().unwrap()+ QuantizationMethod::ScaleSymmetric => {+ let midpoint = (1 << (bits - 1)) as f32;+ scale_t.to_f32().unwrap() * (val_q - midpoint)},};diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rsindex 17736db09..e55e07fb5 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs@@ -65,6 +65,10 @@ pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT:let scale = (*scales.add(j * num_groups_k + group_idx)).to_f32().unwrap();let bias = match quant_method {+ QuantizationMethod::ScaleBias => {+ // Biases are [N, num_groups_k]+ (*biases.unwrap().add(j * num_groups_k + group_idx)).to_f32().unwrap()+ },QuantizationMethod::ScaleZeroPoint => {let zp = zero_points.unwrap();// Zero points are [N, zp_stride_k]@@ -81,9 +85,9 @@ pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT:};-scale * zp_val},- QuantizationMethod::ScaleBias => {- // Biases are [N, num_groups_k]- (*biases.unwrap().add(j * num_groups_k + group_idx)).to_f32().unwrap()+ QuantizationMethod::ScaleSymmetric => {+ let midpoint = 1 << (bits - 1);+ -scale * midpoint as f32},};diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 24c408ea9..c668c17de 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -73,6 +73,9 @@ impl MatmulKernel for MatmulCpuKernel {}| MatmulB::ScaleZeroPointDequant {..+ }+ | MatmulB::ScaleSymmetricDequant {+ ..} => Ok(self.encode_quant(arguments, encoder)?),}}@@ -209,6 +212,10 @@ impl MatmulCpuKernel {| MatmulB::ScaleZeroPointDequant {group_size,..+ }+ | MatmulB::ScaleSymmetricDequant {+ group_size,+ ..} => group_size,MatmulB::FullPrecision {..@@ -293,14 +300,20 @@ impl MatmulCpuKernel {biases,mode,group_size,- } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),+ } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),MatmulB::ScaleZeroPointDequant {b: w,scales,zero_points,mode,group_size,- } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),+ } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),+ MatmulB::ScaleSymmetricDequant {+ b: w,+ scales,+ mode,+ group_size,+ } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),MatmulB::FullPrecision {..} => unreachable!(),@@ -312,8 +325,13 @@ impl MatmulCpuKernel {};let use_fast = n % 8 == 0 && k % 512 == 0;let (zero_points, biases) = match method {- QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias), None),- QuantizationMethod::ScaleBias => (None, Some(zp_or_bias)),+ QuantizationMethod::ScaleBias => {+ (None, Some(zp_or_bias.expect("ScaleBias quantized matmul requires biases")))+ },+ QuantizationMethod::ScaleZeroPoint => {+ (Some(zp_or_bias.expect("ScaleZeroPoint quantized matmul requires zero_points")), None)+ },+ QuantizationMethod::ScaleSymmetric => (None, None),};let context = encoder.context();diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rsindex 3b08eb8c7..1ada717e6 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs@@ -36,14 +36,20 @@ pub(crate) fn encode_quantized_gemm<TB: AsBufferRangeRef<Buffer: Buffer<Backendbiases,mode,group_size,- } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),+ } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),MatmulB::ScaleZeroPointDequant {b: w,scales,zero_points,mode,group_size,- } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),+ } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),+ MatmulB::ScaleSymmetricDequant {+ b: w,+ scales,+ mode,+ group_size,+ } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),MatmulB::FullPrecision {..} => unreachable!(),@@ -61,19 +67,20 @@ pub(crate) fn encode_quantized_gemm<TB: AsBufferRangeRef<Buffer: Buffer<Backendlet a_buf = a.as_buffer_range_ref();let b_buf = weights.as_buffer_range_ref();let scales_buf = scales.as_buffer_range_ref();- let zp_buf = zp_or_bias.as_buffer_range_ref();+ let zp_buf = zp_or_bias.map(|zp_or_bias| zp_or_bias.as_buffer_range_ref());let d_buf = d.as_buffer_range_mut();let a_byte_off = a_buf.range().start + a_offset * data_type.size_in_bytes();let b_byte_off = b_buf.range().start;let scales_byte_off = scales_buf.range().start;- let zp_byte_off = zp_buf.range().start;+ let zp_byte_off = zp_buf.as_ref().map(|zp_buf| zp_buf.range().start);let d_byte_off = d_buf.range().start;let a_ptr = SendPtr(unsafe { &*a_buf.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));let b_ptr = SendPtr(unsafe { &*b_buf.buffer().get() }.as_ptr().wrapping_byte_add(b_byte_off));let scales_ptr = SendPtr(unsafe { &*scales_buf.buffer().get() }.as_ptr().wrapping_byte_add(scales_byte_off));- let zp_ptr = SendPtr(unsafe { &*zp_buf.buffer().get() }.as_ptr().wrapping_byte_add(zp_byte_off));+ let zp_ptr = zp_buf+ .map(|zp_buf| SendPtr(unsafe { &*zp_buf.buffer().get() }.as_ptr().wrapping_byte_add(zp_byte_off.unwrap())));let d_ptr = SendPtrMut(unsafe { (&*d_buf.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });let command_buffer = encoder.as_command_buffer_mut();@@ -125,7 +132,7 @@ fn run<T: ArrayElement + Float>(a_ptr: SendPtr<u8>,b_ptr: SendPtr<u8>,scales_ptr: SendPtr<u8>,- zp_or_bias_ptr: SendPtr<u8>,+ zp_or_bias_ptr: Option<SendPtr<u8>>,d_ptr: SendPtrMut<u8>,in_vec_size: usize,out_vec_size: usize,@@ -135,8 +142,14 @@ fn run<T: ArrayElement + Float>(bits: usize,) {let (zp, bias) = match quant_method {- QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias_ptr.as_ptr() as *const u8), None),- QuantizationMethod::ScaleBias => (None, Some(zp_or_bias_ptr.as_ptr() as *const T)),+ QuantizationMethod::ScaleBias => {+ (None, Some(zp_or_bias_ptr.expect("ScaleBias quantized matmul requires biases").as_ptr() as *const T))+ },+ QuantizationMethod::ScaleZeroPoint => (+ Some(zp_or_bias_ptr.expect("ScaleZeroPoint quantized matmul requires zero_points").as_ptr() as *const u8),+ None,+ ),+ QuantizationMethod::ScaleSymmetric => (None, None),};qmm_transposed::<T>(b_ptr.as_ptr() as *const u32,diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 14a6b81fc..b7521d1a2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -76,7 +76,7 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (float bias = 0.0f;if (quantization_method == QuantizationMethod::ScaleBias) {bias = float(biases[token_id * num_groups + group_idx]);- } else {+ } else if (quantization_method == QuantizationMethod::ScaleZeroPoint) {uint zero_point = 0;if (quantization_mode == QuantizationMode::U4) {const uint8_t packed = zero_points[token_id * zp_stride + group_idx / 2];@@ -86,6 +86,9 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (zero_point = zero_points[token_id * zp_stride + group_idx];}bias = -float(scale) * float(zero_point);+ } else {+ const uint midpoint = quantization_mode == QuantizationMode::U4 ? 8 : 128;+ bias = -float(scale) * float(midpoint);}float out_f = float(scale) * float(quantized_value) + bias;diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.hindex 841f4cd72..ea1911fa2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h@@ -9,6 +9,7 @@ enum class GemmWeightPrologueKind : uint32_t {FullPrecision = 0,ScaleBiasDequant = 1,ScaleZeroPointDequant = 2,+ ScaleSymmetricDequant = 3,};struct GemmDTransform {diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.hindex 3a265d899..49eadce22 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h@@ -8,5 +8,6 @@ namespace uzu::quantization_method {enum class QuantizationMethod : uint32_t {ScaleBias = 0,ScaleZeroPoint = 1,+ ScaleSymmetric = 2,};} // namespace uzu::quantization_methoddiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex 48cc1f462..5f21844d6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -20,7 +20,8 @@ template <short REDUCTION_DIMENSION,short THREADGROUP_SIZE,short GROUP_SIZE,- short BITS>+ short BITS,+ bool SCALE_SYMMETRIC = false>struct QuantizedBlockLoaderScaleZeroPoint {static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE,@@ -103,11 +104,14 @@ struct QuantizedBlockLoaderScaleZeroPoint {: scales_),zero_points_row_start(- REDUCTION_DIMENSION == 1- ? (zero_points_row_start_ +- tile_row_index * (BITS == 4 ? ((groups_per_row_ + 1) / 2)- : groups_per_row_))- : zero_points_row_start_+ SCALE_SYMMETRIC+ ? nullptr+ : (REDUCTION_DIMENSION == 1+ ? (zero_points_row_start_ ++ tile_row_index * (BITS == 4+ ? ((groups_per_row_ + 1) / 2)+ : groups_per_row_))+ : zero_points_row_start_)) {}inline void current_scale_bias(@@ -124,7 +128,9 @@ struct QuantizedBlockLoaderScaleZeroPoint {group_index = static_cast<int>(scales - scales_row_start);scale_value = *scales;}- if constexpr (BITS == 4) {+ if constexpr (SCALE_SYMMETRIC) {+ zero_point_value = 1u << (BITS - 1);+ } else if constexpr (BITS == 4) {const device uint8_t* zero_point_ptr =zero_points_row_start + (group_index >> 1);uint8_t zero_point_byte = *zero_point_ptr;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 4f4626f2c..88c5dcbbb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -76,6 +76,16 @@ struct SimdgroupMmaCore {THREADGROUP_THREADS,GROUP_SIZE,BITS>;+ using BLoaderScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<+ BT,+ THREADGROUP_BLOCK_N,+ THREADGROUP_BLOCK_K,+ SHARED_STRIDE_B,+ 1,+ THREADGROUP_THREADS,+ GROUP_SIZE,+ BITS,+ true>;using TileAccumulator = uzu::matmul::ThreadgroupTile<AT,BT,@@ -411,6 +421,56 @@ struct SimdgroupMmaCore {needs_bias);});+ return;+ }++ if constexpr (+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleSymmetricDequant+ ) {+ constexpr int pack_factor = get_pack_factor<BITS, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();+ const int k_elements = int(params->K);+ const int weights_row_stride_bytes =+ k_elements * bytes_per_pack / pack_factor;+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;+ const device uint8_t* weights_block =+ b_packed + block_col * weights_row_stride_bytes;+ const device BT* scales_offset = scales + block_col * groups_per_row;+ thread BLoaderScaleSymmetric loader_b(+ weights_block,+ scales_offset,+ nullptr,+ k_elements,+ groups_per_row,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ dispatch_kernel([&](auto gemm_alignment_mask) {+ constexpr uint mask = gemm_alignment_mask.value;+ k_loop<mask, BLoaderScaleSymmetric>(+ a_shared,+ b_shared,+ params->aligned_inner_iterations,+ loader_a,+ loader_b,+ accumulator,+ tile_block_rows,+ tile_block_cols,+ leftover_block_depth+ );+ finalize<mask>(+ accumulator,+ d,+ params,+ tile_block_rows,+ tile_block_cols,+ needs_epilogue,+ epilogue,+ bias_block,+ needs_bias+ );+ });}}};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 84b978d29..66d4cb4f1 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -49,7 +49,8 @@ VARIANTS(WEIGHT_PROLOGUE,GemmWeightPrologueKind::FullPrecision,GemmWeightPrologueKind::ScaleBiasDequant,- GemmWeightPrologueKind::ScaleZeroPointDequant)+ GemmWeightPrologueKind::ScaleZeroPointDequant,+ GemmWeightPrologueKind::ScaleSymmetricDequant)VARIANTS(BITS, 0, 4, 8)VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)CONSTRAINT(diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 45d53f296..e75597cdd 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -153,7 +153,7 @@ impl GemmKernel {GemmDispatchPath::Simdgroup => false,};- let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });+ let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });if is_quant {let d_mask = arguments.d_transform.mask();if d_mask.contains(GemmDTransform::ACCUMULATE) {@@ -299,6 +299,9 @@ impl GemmKernel {}| MatmulB::ScaleZeroPointDequant {..+ }+ | MatmulB::ScaleSymmetricDequant {+ ..}) => {let (weights, scales, biases, zero_points) = match quant_b {MatmulB::ScaleBiasDequant {@@ -313,6 +316,11 @@ impl GemmKernel {zero_points,..} => (w, Some(scales), None, Some(zero_points)),+ MatmulB::ScaleSymmetricDequant {+ b: w,+ scales,+ ..+ } => (w, Some(scales), None, None),_ => unreachable!(),};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsindex c6c4568e7..a6fc03fc2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -102,6 +102,7 @@ impl GemmSpecialization {let weight_prologue = match combo.method {QuantizationMethod::ScaleBias => GemmWeightPrologueKind::ScaleBiasDequant,QuantizationMethod::ScaleZeroPoint => GemmWeightPrologueKind::ScaleZeroPointDequant,+ QuantizationMethod::ScaleSymmetric => GemmWeightPrologueKind::ScaleSymmetricDequant,};let mut out = Vec::new();for &tiling in quant_tiling_set(weights_data_type) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metalindex 025d39fd9..903436a33 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal@@ -72,7 +72,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(const device WeightT* biases_row_base = nullptr;if (quant_method == QuantizationMethod::ScaleBias) {biases_row_base = biases + out_row * in_vec_size_g;- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {zps_row_base = zero_points + out_row * zp_stride;}@@ -97,7 +97,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(U b = static_cast<U>(bl[g]);result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {const device uint8_t* zl = zps_row_base + row * zp_stride;U zp;if (BITS == 4) {@@ -108,6 +108,15 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(}result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);+ } else {+ constexpr U midpoint = U(1u << (BITS - 1));+ result[row] += qdot<U, values_per_thread, BITS>(+ wl,+ x_thread,+ s,+ -s * midpoint,+ sum+ );}}@@ -141,7 +150,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(U b = static_cast<U>(bl[g]);result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {const device uint8_t* zl = zps_row_base + row * zp_stride;U zp;if (BITS == 4) {@@ -152,6 +161,15 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(}result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);+ } else {+ constexpr U midpoint = U(1u << (BITS - 1));+ result[row] += qdot<U, values_per_thread, BITS>(+ wl,+ x_thread,+ s,+ -s * midpoint,+ sum+ );}}}@@ -175,7 +193,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(const device WeightT* biases_row_base = nullptr;if (quant_method == QuantizationMethod::ScaleBias) {biases_row_base = biases + used_out_row * in_vec_size_g;- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {zps_row_base = zero_points + used_out_row * zp_stride;}@@ -198,7 +216,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(U b = static_cast<U>(bl[g]);result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {const device uint8_t* zl = zps_row_base + row * zp_stride;U zp;if (BITS == 4) {@@ -209,6 +227,15 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(}result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);+ } else {+ constexpr U midpoint = U(1u << (BITS - 1));+ result[row] += qdot<U, values_per_thread, BITS>(+ wl,+ x_thread,+ s,+ -s * midpoint,+ sum+ );}}@@ -247,7 +274,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(sum,remaining);- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {const device uint8_t* zl = zps_row_base + row * zp_stride;U zp;if (BITS == 4) {@@ -258,6 +285,16 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(}result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);+ } else {+ constexpr U midpoint = U(1u << (BITS - 1));+ result[row] += qdot_safe<U, values_per_thread, BITS>(+ wl,+ x_thread,+ s,+ -s * midpoint,+ sum,+ remaining+ );}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metalindex c2ba438db..de733f048 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal@@ -65,7 +65,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(if (quant_method == QuantizationMethod::ScaleBias) {biases += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {if (BITS == 4) {zp_stride = (in_vec_size_g + 1) / 2;zps = zero_points + out_row * zp_stride;@@ -109,7 +109,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(qdot<U, values_per_thread, BITS>(wl2, x_thread, s2, b2, sum);result[3] +=qdot<U, values_per_thread, BITS>(wl3, x_thread, s3, b3, sum);- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {uchar4 zp_bytes = uchar4(zps[0],zps[zp_stride],@@ -151,6 +151,36 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(-s3 * static_cast<U>(zp_nibbles.w),sum);+ } else {+ constexpr U midpoint = U(1u << (BITS - 1));+ result[0] += qdot<U, values_per_thread, BITS>(+ wl0,+ x_thread,+ s0,+ -s0 * midpoint,+ sum+ );+ result[1] += qdot<U, values_per_thread, BITS>(+ wl1,+ x_thread,+ s1,+ -s1 * midpoint,+ sum+ );+ result[2] += qdot<U, values_per_thread, BITS>(+ wl2,+ x_thread,+ s2,+ -s2 * midpoint,+ sum+ );+ result[3] += qdot<U, values_per_thread, BITS>(+ wl3,+ x_thread,+ s3,+ -s3 * midpoint,+ sum+ );}}@@ -158,7 +188,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(scales += block_size / GROUP_SIZE;if (quant_method == QuantizationMethod::ScaleBias) {biases += block_size / GROUP_SIZE;- } else {+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {if (BITS == 4) {zps += (block_size / GROUP_SIZE) / 2;} else {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rsindex c2e4d6b33..fc44762c1 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs@@ -95,14 +95,20 @@ impl QuantGemvKernel {biases,mode,group_size,- } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),+ } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),MatmulB::ScaleZeroPointDequant {b: w,scales,zero_points,mode,group_size,- } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),+ } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),+ MatmulB::ScaleSymmetricDequant {+ b: w,+ scales,+ mode,+ group_size,+ } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),MatmulB::FullPrecision {..} => panic!("QuantGemvKernel requires quantized B"),@@ -115,8 +121,13 @@ impl QuantGemvKernel {let use_fast = n % 8 == 0 && k % 512 == 0;let (zero_points, biases) = match method {- QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias), None),- QuantizationMethod::ScaleBias => (None, Some(zp_or_bias)),+ QuantizationMethod::ScaleBias => {+ (None, Some(zp_or_bias.expect("ScaleBias quantized matmul requires biases")))+ },+ QuantizationMethod::ScaleZeroPoint => {+ (Some(zp_or_bias.expect("ScaleZeroPoint quantized matmul requires zero_points")), None)+ },+ QuantizationMethod::ScaleSymmetric => (None, None),};if use_fast {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 9c8015bf2..ceaacb941 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -166,7 +166,7 @@ impl MatmulKernel for MatmulMetalKernel {arguments: MatmulArguments<Metal, TB>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {- let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });+ let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });let gemv_eligible = if is_quant {arguments.m < 5 || arguments.n == 1} else {diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 3f846f377..8c4fe862c 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -52,7 +52,7 @@ enum TiedEmbeddingType<B: Backend> {Quantized {weights: Allocation<B>,scales: Allocation<B>,- zero_points_or_biases: Allocation<B>,+ zero_points_or_biases: Option<Allocation<B>>,quantization_method: QuantizationMethod,output_hadamard_factors: Option<Allocation<B>>,lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,@@ -69,7 +69,7 @@ enum UntiedEmbeddingLookupType<B: Backend> {Quantized {weights: Allocation<B>,scales: Allocation<B>,- zero_points_or_biases: Allocation<B>,+ zero_points_or_biases: Option<Allocation<B>>,quantization_method: QuantizationMethod,output_hadamard_factors: Option<Allocation<B>>,lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,@@ -84,7 +84,7 @@ enum UntiedEmbeddingReadoutType<B: Backend> {Quantized {weights: Allocation<B>,scales: Allocation<B>,- zero_points_or_biases: Allocation<B>,+ zero_points_or_biases: Option<Allocation<B>>,readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,readout_config: ReadoutQuantConfig,},@@ -502,8 +502,9 @@ impl<B: Backend> Embedding<B> {output_ty: _,} => {let (zero_points, biases) = match quantization_method {- QuantizationMethod::ScaleZeroPoint => (Some(zero_points_or_biases), None),- QuantizationMethod::ScaleBias => (None, Some(zero_points_or_biases)),+ QuantizationMethod::ScaleBias => (None, zero_points_or_biases.as_ref()),+ QuantizationMethod::ScaleZeroPoint => (zero_points_or_biases.as_ref(), None),+ QuantizationMethod::ScaleSymmetric => (None, None),};lookup.encode(token_ids,@@ -603,14 +604,22 @@ impl<B: Backend> Embedding<B> {QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {b: weights,scales,- biases: zero_points_or_biases,+ biases: zero_points_or_biases.as_ref().expect("ScaleBias quantization requires biases"),mode: readout_config.mode,group_size: readout_config.group_size,},QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {b: weights,scales,- zero_points: zero_points_or_biases,+ zero_points: zero_points_or_biases+ .as_ref()+ .expect("ScaleZeroPoint quantization requires zero_points"),+ mode: readout_config.mode,+ group_size: readout_config.group_size,+ },+ QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {+ b: weights,+ scales,mode: readout_config.mode,group_size: readout_config.group_size,},@@ -658,6 +667,13 @@ fn input_quantization_from_spec<B: Backend>(layout: Layout::InputOutput,..}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ bits,+ group_size,+ is_symmetric: true,+ layout: Layout::InputOutput,+ ..+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleSymmetric),spec => Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),}}@@ -679,6 +695,13 @@ fn output_quantization_from_spec<B: Backend>(layout: Layout::OutputInput,..}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ bits,+ group_size,+ is_symmetric: true,+ layout: Layout::OutputInput,+ ..+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleSymmetric),spec => Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),}}@@ -708,7 +731,7 @@ fn load_quantized_embedding_parts<B: Backend>(quantization_mode: QuantizationMode,quantization_method: QuantizationMethod,group_size: usize,-) -> Result<(Allocation<B>, Allocation<B>, Allocation<B>), EmbeddingError<B>> {+) -> Result<(Allocation<B>, Allocation<B>, Option<Allocation<B>>), EmbeddingError<B>> {let packing_divisor = quantization_mode.packing_divisor();let storage_data_type = quantization_mode.storage_type();let num_groups = model_dim.div_ceil(group_size);@@ -720,14 +743,17 @@ fn load_quantized_embedding_parts<B: Backend>(let scales = tree.leaf("scales")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?;let zero_points_or_biases = match quantization_method {QuantizationMethod::ScaleBias => {- tree.leaf("biases")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?+ Some(tree.leaf("biases")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?)},QuantizationMethod::ScaleZeroPoint => {let expected_zero_points_entries = num_groups.div_ceil(packing_divisor);- tree.leaf("zero_points")?- .validate(&[vocab_size, expected_zero_points_entries], storage_data_type)?- .read_allocation()?+ Some(+ tree.leaf("zero_points")?+ .validate(&[vocab_size, expected_zero_points_entries], storage_data_type)?+ .read_allocation()?,+ )},+ QuantizationMethod::ScaleSymmetric => None,};Ok((weights, scales, zero_points_or_biases))diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex da54a4872..d851fc9d7 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -37,7 +37,7 @@ enum Mode<B: Backend> {mode: QuantizationMode,group_size: u32,scales: Allocation<B>,- zero_points_or_biases: Allocation<B>,+ zero_points_or_biases: Option<Allocation<B>>,output_hadamard_factors: Option<Allocation<B>>,},}@@ -121,6 +121,13 @@ impl<B: Backend> LinearMatmul<B> {layout: Layout::OutputInput,..}) => (bits, group_size, QuantizationMethod::ScaleZeroPoint),+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ bits,+ group_size,+ is_symmetric: true,+ layout: Layout::OutputInput,+ ..+ }) => (bits, group_size, QuantizationMethod::ScaleSymmetric),spec => return Err(LinearMatmulError::UnsupportedConfiguration(format!("{spec:?}"))),};@@ -151,15 +158,18 @@ impl<B: Backend> LinearMatmul<B> {let scales = weights_tree.leaf("scales")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?;let zero_points_or_biases = match quantization_method {QuantizationMethod::ScaleBias => {- weights_tree.leaf("biases")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?+ Some(weights_tree.leaf("biases")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?)},QuantizationMethod::ScaleZeroPoint => {let expected_zero_points_entries = k_g.div_ceil(packing_divisor);- weights_tree- .leaf("zero_points")?- .validate(&[output_dim, expected_zero_points_entries], storage_type)?- .read_allocation()?+ Some(+ weights_tree+ .leaf("zero_points")?+ .validate(&[output_dim, expected_zero_points_entries], storage_type)?+ .read_allocation()?,+ )},+ QuantizationMethod::ScaleSymmetric => None,};let biases = load_biases(weights_data_type, output_data_type, output_dim, bias_tree)?;@@ -242,14 +252,22 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {b: &self.weights,scales,- biases: zero_points_or_biases,+ biases: zero_points_or_biases.as_ref().expect("ScaleBias quantization requires biases"),mode: *mode,group_size: *group_size,},QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {b: &self.weights,scales,- zero_points: zero_points_or_biases,+ zero_points: zero_points_or_biases+ .as_ref()+ .expect("ScaleZeroPoint quantization requires zero_points"),+ mode: *mode,+ group_size: *group_size,+ },+ QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {+ b: &self.weights,+ scales,mode: *mode,group_size: *group_size,},diff --git a/crates/backend-uzu/tests/common/matmul/quant.rs b/crates/backend-uzu/tests/common/matmul/quant.rsindex bad1def0d..3790826e4 100644--- a/crates/backend-uzu/tests/common/matmul/quant.rs+++ b/crates/backend-uzu/tests/common/matmul/quant.rs@@ -65,13 +65,14 @@ impl<T: ArrayElement + Float> QuantInput<T> {num_groups_k};let (zero_points, biases) = match quant_method {- QuantizationMethod::ScaleZeroPoint => {- (Some((0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)- },QuantizationMethod::ScaleBias => (None,Some((0..n * num_groups_k).map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap()).collect()),),+ QuantizationMethod::ScaleZeroPoint => {+ (Some((0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)+ },+ QuantizationMethod::ScaleSymmetric => (None, None),};Self {@@ -122,6 +123,13 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(input: &QuantInput<T>,) -> MatmulArguments<'a, B> {let b_variant = match input.quant_method {+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {+ b: &buffers.w,+ scales: &buffers.scales,+ biases: buffers.bias.as_ref().expect("bias buffer"),+ mode: input.mode,+ group_size: input.group_size,+ },QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {b: &buffers.w,scales: &buffers.scales,@@ -129,10 +137,9 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(mode: input.mode,group_size: input.group_size,},- QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {+ QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {b: &buffers.w,scales: &buffers.scales,- biases: buffers.bias.as_ref().expect("bias buffer"),mode: input.mode,group_size: input.group_size,},diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rsindex d007d1647..40a942036 100644--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs@@ -1,6 +1,4 @@-use std::{- fmt::{Debug, Display},-};+use std::fmt::{Debug, Display};use backend_uzu::{ArrayContextExt, ArrayElement, DataType,@@ -119,6 +117,43 @@ fn get_test_data_zero_point_group16<T: ArrayElement + Float>() -> (Input<T>, Vec(input, expected)}+fn get_test_data_symmetric_u8<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let batch_size = 3u32;+ let vocab_size = 8u32;+ let model_dim = 64u32;+ let group_size = 32u32;+ let input_scale = 1.5f32;+ let quant_mode = QuantizationMode::U8;++ let token_ids: Box<[u64]> = Box::new([2, 5, 0]);++ let weights: Vec<u8> =+ (0..vocab_size as usize * model_dim as usize).map(|i| ((i * 37 + 11) & 0xFF) as u8).collect();++ let num_groups = (model_dim + group_size - 1) / group_size;+ let scales: Vec<T> = (0..vocab_size as usize * num_groups as usize)+ .map(|i| T::from(0.5 + (i as f32 * 0.1).sin() * 0.3).unwrap())+ .collect();++ let input = Input {+ token_ids,+ weights: weights.into_boxed_slice(),+ scales: scales.into_boxed_slice(),+ zero_points: None,+ biases: None,+ batch_size,+ vocab_size,+ model_dim,+ input_scale,+ group_size,+ quant_mode,+ quant_method: QuantizationMethod::ScaleSymmetric,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}+fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {let batch_size = 2u32;let vocab_size = 4u32;@@ -184,10 +219,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.zero_points.as_ref().map(|zero_points| context.create_array_from(&[input.vocab_size as usize, zero_point_stride], zero_points));- let biases_array = input- .biases- .as_ref()- .map(|biases| context.create_array_from(&[input.vocab_size as usize, num_groups], biases));+ let biases_array =+ input.biases.as_ref().map(|biases| context.create_array_from(&[input.vocab_size as usize, num_groups], biases));let mut output = context.create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type()).into_allocation();@@ -233,6 +266,24 @@ fn test_zero_point_group16<T: ArrayElement + Float + Debug + Display>() {});}+fn test_symmetric_u8<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.5f32+ } else {+ 1e-4+ };+ let (input, expected) = get_test_data_symmetric_u8::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("QuantizedEmbeddingLookup ScaleSymmetric test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}+#[cfg(metal_backend)]fn test_zero_point_group16_hadamard_constructor<T: ArrayElement + Float + Debug + Display>() {use backend_uzu::backends::metal::Metal;@@ -344,6 +395,11 @@ fn test_uint8_bf16() {test_quant_mode::<bf16>(QuantizationMode::U8);}+#[uzu_test]+fn test_uint8_symmetric_bf16() {+ test_symmetric_u8::<bf16>();+}+// OOB tests#[uzu_test]fn test_oob_f32() {diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rsindex 5db7b81cc..9490c274d 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs@@ -97,6 +97,8 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(#[case::gs32_8bit_mlx_prefill ( 64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]#[case::gs64_8bit_zp_prefill ( 64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]#[case::gs128_8bit_zp_prefill ( 64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::gs32_4bit_symmetric_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]+#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]#[case::gs32_4bit_mlx_decode ( 8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]#[case::gs16_4bit_zp_decode ( 8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs64_4bit_zp_decode ( 8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsindex 41232cb36..77904bbf5 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -4,8 +4,8 @@ use backend_uzu::{ArrayElement, DataType,backends::{common::{- Allocation, Backend, Context, Encoder, Kernels, gpu_types::QuantizationMethod,-+ Allocation, Backend, Context, Encoder, Kernels,+ gpu_types::QuantizationMethod,kernel::{QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},},cpu::Cpu,@@ -139,6 +139,16 @@ fn get_test_data<T: ArrayElement + Float>(};let (zero_points, biases) = match quant_method {+ QuantizationMethod::ScaleBias => {+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);+ }+ }+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();+ (None, Some(biases))+ },QuantizationMethod::ScaleZeroPoint => {let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);for j in 0..n {@@ -157,16 +167,7 @@ fn get_test_data<T: ArrayElement + Float>(}(Some(zp_packed), None)},- QuantizationMethod::ScaleBias => {- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);- }- }- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();- (None, Some(biases))- },+ QuantizationMethod::ScaleSymmetric => (None, None),};let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);@@ -281,14 +282,6 @@ macro_rules! qmv_fast_test {};}-qmv_fast_test!(test_gs16_4bit_zp, gs = 16, bits = 4, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs16_8bit_zp, gs = 16, bits = 8, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, format = QuantizationMethod::ScaleZeroPoint);qmv_fast_test!(test_gs16_4bit_mlx, gs = 16, bits = 4, format = QuantizationMethod::ScaleBias);qmv_fast_test!(test_gs32_4bit_mlx, gs = 32, bits = 4, format = QuantizationMethod::ScaleBias);qmv_fast_test!(test_gs64_4bit_mlx, gs = 64, bits = 4, format = QuantizationMethod::ScaleBias);@@ -297,3 +290,12 @@ qmv_fast_test!(test_gs16_8bit_mlx, gs = 16, bits = 8, format = QuantizationMethoqmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, format = QuantizationMethod::ScaleBias);qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, format = QuantizationMethod::ScaleBias);qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, format = QuantizationMethod::ScaleBias);+qmv_fast_test!(test_gs16_4bit_zp, gs = 16, bits = 4, format = QuantizationMethod::ScaleZeroPoint);+qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, format = QuantizationMethod::ScaleZeroPoint);+qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, format = QuantizationMethod::ScaleZeroPoint);+qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, format = QuantizationMethod::ScaleZeroPoint);+qmv_fast_test!(test_gs16_8bit_zp, gs = 16, bits = 8, format = QuantizationMethod::ScaleZeroPoint);+qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, format = QuantizationMethod::ScaleZeroPoint);+qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, format = QuantizationMethod::ScaleZeroPoint);+qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, format = QuantizationMethod::ScaleZeroPoint);+qmv_fast_test!(test_gs32_8bit_symmetric, gs = 32, bits = 8, format = QuantizationMethod::ScaleSymmetric);diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rsindex d38d7fdcf..3e21443bc 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs@@ -3,10 +3,7 @@ use std::fmt::{Debug, Display};use backend_uzu::{ArrayElement, DataType,backends::{- common::{- Backend, Context, Encoder, Kernels, gpu_types::QuantizationMethod,- kernel::QuantizedMatmulQmvKernel,- },+ common::{Backend, Context, Encoder, Kernels, gpu_types::QuantizationMethod, kernel::QuantizedMatmulQmvKernel},cpu::Cpu,},};@@ -105,6 +102,16 @@ fn get_test_data_basic<T: ArrayElement + Float>(};let (zero_points, biases) = match quant_method {+ QuantizationMethod::ScaleBias => {+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);+ }+ }+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();+ (None, Some(biases))+ },QuantizationMethod::ScaleZeroPoint => {let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);for j in 0..n {@@ -124,16 +131,7 @@ fn get_test_data_basic<T: ArrayElement + Float>(}(Some(zp_packed), None)},- QuantizationMethod::ScaleBias => {- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);- }- }- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();- (None, Some(biases))- },+ QuantizationMethod::ScaleSymmetric => (None, None),};// X: [m × k]@@ -188,15 +186,16 @@ fn get_test_data_edge<T: ArrayElement + Float>(};let (zero_points, biases) = match quant_method {+ QuantizationMethod::ScaleBias => {+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];+ (None, Some(biases))+ },QuantizationMethod::ScaleZeroPoint => {// Zero points all 0 → bias = 0let zp_packed = vec![0u8; n * zp_stride];(Some(zp_packed), None)},- QuantizationMethod::ScaleBias => {- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];- (None, Some(biases))- },+ QuantizationMethod::ScaleSymmetric => (None, None),};// X: simple increasing values@@ -302,151 +301,158 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>(test_internal::<T>(&input, &expected);}-// ── 4-bit, zero points ─────────────────────────────────────────────────────+// ── 4-bit, mlx quant ───────────────────────────────────────────────────────#[uzu_test]-fn test_gs16_4bit_zp() {+fn test_gs16_4bit_mlx() {for_each_float_type!(|F| {- test_basic::<F>(16, 4, QuantizationMethod::ScaleZeroPoint);+ test_basic::<F>(16, 4, QuantizationMethod::ScaleBias);})}#[uzu_test]-fn test_gs32_4bit_zp() {+fn test_gs32_4bit_mlx() {for_each_float_type!(|F| {- test_basic::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);+ test_basic::<F>(32, 4, QuantizationMethod::ScaleBias);})}#[uzu_test]-fn test_gs64_4bit_zp() {+fn test_gs64_4bit_mlx() {for_each_float_type!(|F| {- test_basic::<F>(64, 4, QuantizationMethod::ScaleZeroPoint);+ test_basic::<F>(64, 4, QuantizationMethod::ScaleBias);})}#[uzu_test]-fn test_gs128_4bit_zp() {+fn test_gs128_4bit_mlx() {for_each_float_type!(|F| {- test_basic::<F>(128, 4, QuantizationMethod::ScaleZeroPoint);+ test_basic::<F>(128, 4, QuantizationMethod::ScaleBias);})}-// ── 8-bit, zero points ─────────────────────────────────────────────────────+// ── 8-bit, mlx quant ───────────────────────────────────────────────────────#[uzu_test]-fn test_gs16_8bit_zp() {+fn test_gs16_8bit_mlx() {for_each_float_type!(|F| {- test_basic::<F>(16, 8, QuantizationMethod::ScaleZeroPoint);+ test_basic::<F>(16, 8, QuantizationMethod::ScaleBias);})}#[uzu_test]-fn test_gs32_8bit_zp() {+fn test_gs32_8bit_mlx() {for_each_float_type!(|F| {- test_basic::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);+ test_basic::<F>(32, 8, QuantizationMethod::ScaleBias);})}#[uzu_test]-fn test_gs64_8bit_zp() {+fn test_gs64_8bit_mlx() {for_each_float_type!(|F| {- test_basic::<F>(64, 8, QuantizationMethod::ScaleZeroPoint);+ test_basic::<F>(64, 8, QuantizationMethod::ScaleBias);})}#[uzu_test]-fn test_gs128_8bit_zp() {+fn test_gs128_8bit_mlx() {for_each_float_type!(|F| {- test_basic::<F>(128, 8, QuantizationMethod::ScaleZeroPoint);+ test_basic::<F>(128, 8, QuantizationMethod::ScaleBias);})}-// ── 4-bit, mlx quant ───────────────────────────────────────────────────────+// ── 4-bit, zero points ─────────────────────────────────────────────────────#[uzu_test]-fn test_gs16_4bit_mlx() {+fn test_gs16_4bit_zp() {for_each_float_type!(|F| {- test_basic::<F>(16, 4, QuantizationMethod::ScaleBias);+ test_basic::<F>(16, 4, QuantizationMethod::ScaleZeroPoint);})}#[uzu_test]-fn test_gs32_4bit_mlx() {+fn test_gs32_4bit_zp() {for_each_float_type!(|F| {- test_basic::<F>(32, 4, QuantizationMethod::ScaleBias);+ test_basic::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);})}#[uzu_test]-fn test_gs64_4bit_mlx() {+fn test_gs64_4bit_zp() {for_each_float_type!(|F| {- test_basic::<F>(64, 4, QuantizationMethod::ScaleBias);+ test_basic::<F>(64, 4, QuantizationMethod::ScaleZeroPoint);})}#[uzu_test]-fn test_gs128_4bit_mlx() {+fn test_gs128_4bit_zp() {for_each_float_type!(|F| {- test_basic::<F>(128, 4, QuantizationMethod::ScaleBias);+ test_basic::<F>(128, 4, QuantizationMethod::ScaleZeroPoint);})}-// ── 8-bit, mlx quant ───────────────────────────────────────────────────────+// ── 8-bit, zero points ─────────────────────────────────────────────────────#[uzu_test]-fn test_gs16_8bit_mlx() {+fn test_gs16_8bit_zp() {for_each_float_type!(|F| {- test_basic::<F>(16, 8, QuantizationMethod::ScaleBias);+ test_basic::<F>(16, 8, QuantizationMethod::ScaleZeroPoint);})}#[uzu_test]-fn test_gs32_8bit_mlx() {+fn test_gs32_8bit_zp() {for_each_float_type!(|F| {- test_basic::<F>(32, 8, QuantizationMethod::ScaleBias);+ test_basic::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);})}#[uzu_test]-fn test_gs64_8bit_mlx() {+fn test_gs64_8bit_zp() {for_each_float_type!(|F| {- test_basic::<F>(64, 8, QuantizationMethod::ScaleBias);+ test_basic::<F>(64, 8, QuantizationMethod::ScaleZeroPoint);})}#[uzu_test]-fn test_gs128_8bit_mlx() {+fn test_gs128_8bit_zp() {for_each_float_type!(|F| {- test_basic::<F>(128, 8, QuantizationMethod::ScaleBias);+ test_basic::<F>(128, 8, QuantizationMethod::ScaleZeroPoint);+ })+}++#[uzu_test]+fn test_gs32_8bit_symmetric() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 8, QuantizationMethod::ScaleSymmetric);})}// ── Edge cases ──────────────────────────────────────────────────────────────#[uzu_test]-fn test_edge_4bit_zp() {+fn test_edge_4bit_mlx() {for_each_float_type!(|F| {- test_edge::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);+ test_edge::<F>(32, 4, QuantizationMethod::ScaleBias);})}#[uzu_test]-fn test_edge_8bit_zp() {+fn test_edge_8bit_mlx() {for_each_float_type!(|F| {- test_edge::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);+ test_edge::<F>(32, 8, QuantizationMethod::ScaleBias);})}#[uzu_test]-fn test_edge_4bit_mlx() {+fn test_edge_4bit_zp() {for_each_float_type!(|F| {- test_edge::<F>(32, 4, QuantizationMethod::ScaleBias);+ test_edge::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);})}#[uzu_test]-fn test_edge_8bit_mlx() {+fn test_edge_8bit_zp() {for_each_float_type!(|F| {- test_edge::<F>(32, 8, QuantizationMethod::ScaleBias);+ test_edge::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);})}