Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Symmetric quants (#456)

uuuvn2a6beca8b5e0
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
index c77613df7..d433ae913 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
@@ -7,6 +7,7 @@ pub enum GemmWeightPrologueKind {
FullPrecision,
ScaleBiasDequant,
ScaleZeroPointDequant,
+ ScaleSymmetricDequant,
}
bitflags! {
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/quantization_method.rs b/crates/backend-uzu/src/backends/common/gpu_types/quantization_method.rs
index 2127719e0..1e5965c12 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/quantization_method.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/quantization_method.rs
@@ -1,14 +1,11 @@
use std::fmt;
-use serde::{Deserialize, Serialize};
-
#[repr(C)]
-#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub enum QuantizationMethod {
- #[serde(rename = "mlx")]
ScaleBias,
- #[serde(rename = "awq")]
ScaleZeroPoint,
+ ScaleSymmetric,
}
impl fmt::Display for QuantizationMethod {
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
index 55774863a..0045eb13e 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
@@ -24,6 +24,12 @@ pub enum MatmulB<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {
mode: QuantizationMode,
group_size: u32,
},
+ ScaleSymmetricDequant {
+ b: &'a Allocation<B>,
+ scales: &'a Allocation<B>,
+ mode: QuantizationMode,
+ group_size: u32,
+ },
}
impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {
@@ -38,6 +44,9 @@ impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {
Self::ScaleZeroPointDequant {
..
} => GemmWeightPrologueKind::ScaleZeroPointDequant,
+ Self::ScaleSymmetricDequant {
+ ..
+ } => GemmWeightPrologueKind::ScaleSymmetricDequant,
}
}
@@ -53,6 +62,10 @@ impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {
| Self::ScaleZeroPointDequant {
mode,
..
+ }
+ | Self::ScaleSymmetricDequant {
+ mode,
+ ..
} => Some(DataType::from(*mode).size_in_bits() as u32),
}
}
@@ -69,6 +82,10 @@ impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {
| Self::ScaleZeroPointDequant {
group_size,
..
+ }
+ | Self::ScaleSymmetricDequant {
+ group_size,
+ ..
} => Some(*group_size),
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index 97e257a8a..92abae641 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -3,7 +3,7 @@ use num_traits::Float;
use proc_macros::kernel;
use crate::{
- ArrayElement,
+ ArrayElement, DataType,
backends::common::gpu_types::{QuantizationMethod, QuantizationMode},
};
@@ -89,6 +89,12 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
};
let bias = match quantization_method {
+ QuantizationMethod::ScaleBias => biases
+ .expect("ScaleBias quantized embedding requires biases")
+ .add((token_id as u32 * num_groups + group_idx) as usize)
+ .read()
+ .to_f32()
+ .unwrap(),
QuantizationMethod::ScaleZeroPoint => {
let zero_points = zero_points.expect("ScaleZeroPoint quantized embedding requires zero_points");
let zero_point = match quantization_mode {
@@ -107,12 +113,10 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
};
-scale.to_f32().unwrap() * zero_point as f32
},
- QuantizationMethod::ScaleBias => biases
- .expect("ScaleBias quantized embedding requires biases")
- .add((token_id as u32 * num_groups + group_idx) as usize)
- .read()
- .to_f32()
- .unwrap(),
+ QuantizationMethod::ScaleSymmetric => {
+ let midpoint = 1 << (DataType::from(quantization_mode).size_in_bits() - 1);
+ -scale.to_f32().unwrap() * midpoint as f32
+ },
};
let out_f = scale.to_f32().unwrap() * quantized_value as f32 + bias;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs
index 56b2afffc..7244c30e9 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs
@@ -47,6 +47,10 @@ pub fn qmm_transposed<T: ArrayElement + Float>(
let scale_t = *scales.add(j * num_groups_k + group_idx);
let w_dequant_f32 = match quant_method {
+ QuantizationMethod::ScaleBias => {
+ let bias_t = *biases.unwrap().add(j * num_groups_k + group_idx);
+ scale_t.to_f32().unwrap() * val_q + bias_t.to_f32().unwrap()
+ },
QuantizationMethod::ScaleZeroPoint => {
let zp = zero_points.unwrap();
let zp_val = if bits == 4 {
@@ -62,9 +66,9 @@ pub fn qmm_transposed<T: ArrayElement + Float>(
};
scale_t.to_f32().unwrap() * (val_q - zp_val)
},
- QuantizationMethod::ScaleBias => {
- let bias_t = *biases.unwrap().add(j * num_groups_k + group_idx);
- scale_t.to_f32().unwrap() * val_q + bias_t.to_f32().unwrap()
+ QuantizationMethod::ScaleSymmetric => {
+ let midpoint = (1 << (bits - 1)) as f32;
+ scale_t.to_f32().unwrap() * (val_q - midpoint)
},
};
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
index 17736db09..e55e07fb5 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
@@ -65,6 +65,10 @@ pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT:
let scale = (*scales.add(j * num_groups_k + group_idx)).to_f32().unwrap();
let bias = match quant_method {
+ QuantizationMethod::ScaleBias => {
+ // Biases are [N, num_groups_k]
+ (*biases.unwrap().add(j * num_groups_k + group_idx)).to_f32().unwrap()
+ },
QuantizationMethod::ScaleZeroPoint => {
let zp = zero_points.unwrap();
// Zero points are [N, zp_stride_k]
@@ -81,9 +85,9 @@ pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT:
};
-scale * zp_val
},
- QuantizationMethod::ScaleBias => {
- // Biases are [N, num_groups_k]
- (*biases.unwrap().add(j * num_groups_k + group_idx)).to_f32().unwrap()
+ QuantizationMethod::ScaleSymmetric => {
+ let midpoint = 1 << (bits - 1);
+ -scale * midpoint as f32
},
};
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 24c408ea9..c668c17de 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -73,6 +73,9 @@ impl MatmulKernel for MatmulCpuKernel {
}
| MatmulB::ScaleZeroPointDequant {
..
+ }
+ | MatmulB::ScaleSymmetricDequant {
+ ..
} => Ok(self.encode_quant(arguments, encoder)?),
}
}
@@ -209,6 +212,10 @@ impl MatmulCpuKernel {
| MatmulB::ScaleZeroPointDequant {
group_size,
..
+ }
+ | MatmulB::ScaleSymmetricDequant {
+ group_size,
+ ..
} => group_size,
MatmulB::FullPrecision {
..
@@ -293,14 +300,20 @@ impl MatmulCpuKernel {
biases,
mode,
group_size,
- } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),
+ } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),
MatmulB::ScaleZeroPointDequant {
b: w,
scales,
zero_points,
mode,
group_size,
- } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ MatmulB::ScaleSymmetricDequant {
+ b: w,
+ scales,
+ mode,
+ group_size,
+ } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),
MatmulB::FullPrecision {
..
} => unreachable!(),
@@ -312,8 +325,13 @@ impl MatmulCpuKernel {
};
let use_fast = n % 8 == 0 && k % 512 == 0;
let (zero_points, biases) = match method {
- QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias), None),
- QuantizationMethod::ScaleBias => (None, Some(zp_or_bias)),
+ QuantizationMethod::ScaleBias => {
+ (None, Some(zp_or_bias.expect("ScaleBias quantized matmul requires biases")))
+ },
+ QuantizationMethod::ScaleZeroPoint => {
+ (Some(zp_or_bias.expect("ScaleZeroPoint quantized matmul requires zero_points")), None)
+ },
+ QuantizationMethod::ScaleSymmetric => (None, None),
};
let context = encoder.context();
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs
index 3b08eb8c7..1ada717e6 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs
@@ -36,14 +36,20 @@ pub(crate) fn encode_quantized_gemm<TB: AsBufferRangeRef<Buffer: Buffer<Backend
biases,
mode,
group_size,
- } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),
+ } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),
MatmulB::ScaleZeroPointDequant {
b: w,
scales,
zero_points,
mode,
group_size,
- } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ MatmulB::ScaleSymmetricDequant {
+ b: w,
+ scales,
+ mode,
+ group_size,
+ } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),
MatmulB::FullPrecision {
..
} => unreachable!(),
@@ -61,19 +67,20 @@ pub(crate) fn encode_quantized_gemm<TB: AsBufferRangeRef<Buffer: Buffer<Backend
let a_buf = a.as_buffer_range_ref();
let b_buf = weights.as_buffer_range_ref();
let scales_buf = scales.as_buffer_range_ref();
- let zp_buf = zp_or_bias.as_buffer_range_ref();
+ let zp_buf = zp_or_bias.map(|zp_or_bias| zp_or_bias.as_buffer_range_ref());
let d_buf = d.as_buffer_range_mut();
let a_byte_off = a_buf.range().start + a_offset * data_type.size_in_bytes();
let b_byte_off = b_buf.range().start;
let scales_byte_off = scales_buf.range().start;
- let zp_byte_off = zp_buf.range().start;
+ let zp_byte_off = zp_buf.as_ref().map(|zp_buf| zp_buf.range().start);
let d_byte_off = d_buf.range().start;
let a_ptr = SendPtr(unsafe { &*a_buf.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));
let b_ptr = SendPtr(unsafe { &*b_buf.buffer().get() }.as_ptr().wrapping_byte_add(b_byte_off));
let scales_ptr = SendPtr(unsafe { &*scales_buf.buffer().get() }.as_ptr().wrapping_byte_add(scales_byte_off));
- let zp_ptr = SendPtr(unsafe { &*zp_buf.buffer().get() }.as_ptr().wrapping_byte_add(zp_byte_off));
+ let zp_ptr = zp_buf
+ .map(|zp_buf| SendPtr(unsafe { &*zp_buf.buffer().get() }.as_ptr().wrapping_byte_add(zp_byte_off.unwrap())));
let d_ptr = SendPtrMut(unsafe { (&*d_buf.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });
let command_buffer = encoder.as_command_buffer_mut();
@@ -125,7 +132,7 @@ fn run<T: ArrayElement + Float>(
a_ptr: SendPtr<u8>,
b_ptr: SendPtr<u8>,
scales_ptr: SendPtr<u8>,
- zp_or_bias_ptr: SendPtr<u8>,
+ zp_or_bias_ptr: Option<SendPtr<u8>>,
d_ptr: SendPtrMut<u8>,
in_vec_size: usize,
out_vec_size: usize,
@@ -135,8 +142,14 @@ fn run<T: ArrayElement + Float>(
bits: usize,
) {
let (zp, bias) = match quant_method {
- QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias_ptr.as_ptr() as *const u8), None),
- QuantizationMethod::ScaleBias => (None, Some(zp_or_bias_ptr.as_ptr() as *const T)),
+ QuantizationMethod::ScaleBias => {
+ (None, Some(zp_or_bias_ptr.expect("ScaleBias quantized matmul requires biases").as_ptr() as *const T))
+ },
+ QuantizationMethod::ScaleZeroPoint => (
+ Some(zp_or_bias_ptr.expect("ScaleZeroPoint quantized matmul requires zero_points").as_ptr() as *const u8),
+ None,
+ ),
+ QuantizationMethod::ScaleSymmetric => (None, None),
};
qmm_transposed::<T>(
b_ptr.as_ptr() as *const u32,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 14a6b81fc..b7521d1a2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -76,7 +76,7 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
float bias = 0.0f;
if (quantization_method == QuantizationMethod::ScaleBias) {
bias = float(biases[token_id * num_groups + group_idx]);
- } else {
+ } else if (quantization_method == QuantizationMethod::ScaleZeroPoint) {
uint zero_point = 0;
if (quantization_mode == QuantizationMode::U4) {
const uint8_t packed = zero_points[token_id * zp_stride + group_idx / 2];
@@ -86,6 +86,9 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
zero_point = zero_points[token_id * zp_stride + group_idx];
}
bias = -float(scale) * float(zero_point);
+ } else {
+ const uint midpoint = quantization_mode == QuantizationMode::U4 ? 8 : 128;
+ bias = -float(scale) * float(midpoint);
}
float out_f = float(scale) * float(quantized_value) + bias;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
index 841f4cd72..ea1911fa2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
@@ -9,6 +9,7 @@ enum class GemmWeightPrologueKind : uint32_t {
FullPrecision = 0,
ScaleBiasDequant = 1,
ScaleZeroPointDequant = 2,
+ ScaleSymmetricDequant = 3,
};
struct GemmDTransform {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h
index 3a265d899..49eadce22 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/quantization_method.h
@@ -8,5 +8,6 @@ namespace uzu::quantization_method {
enum class QuantizationMethod : uint32_t {
ScaleBias = 0,
ScaleZeroPoint = 1,
+ ScaleSymmetric = 2,
};
} // namespace uzu::quantization_method
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
index 48cc1f462..5f21844d6 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -20,7 +20,8 @@ template <
short REDUCTION_DIMENSION,
short THREADGROUP_SIZE,
short GROUP_SIZE,
- short BITS>
+ short BITS,
+ bool SCALE_SYMMETRIC = false>
struct QuantizedBlockLoaderScaleZeroPoint {
static_assert(
THREADGROUP_TILE_COLS <= GROUP_SIZE,
@@ -103,11 +104,14 @@ struct QuantizedBlockLoaderScaleZeroPoint {
: scales_
),
zero_points_row_start(
- REDUCTION_DIMENSION == 1
- ? (zero_points_row_start_ +
- tile_row_index * (BITS == 4 ? ((groups_per_row_ + 1) / 2)
- : groups_per_row_))
- : zero_points_row_start_
+ SCALE_SYMMETRIC
+ ? nullptr
+ : (REDUCTION_DIMENSION == 1
+ ? (zero_points_row_start_ +
+ tile_row_index * (BITS == 4
+ ? ((groups_per_row_ + 1) / 2)
+ : groups_per_row_))
+ : zero_points_row_start_)
) {}
inline void current_scale_bias(
@@ -124,7 +128,9 @@ struct QuantizedBlockLoaderScaleZeroPoint {
group_index = static_cast<int>(scales - scales_row_start);
scale_value = *scales;
}
- if constexpr (BITS == 4) {
+ if constexpr (SCALE_SYMMETRIC) {
+ zero_point_value = 1u << (BITS - 1);
+ } else if constexpr (BITS == 4) {
const device uint8_t* zero_point_ptr =
zero_points_row_start + (group_index >> 1);
uint8_t zero_point_byte = *zero_point_ptr;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 4f4626f2c..88c5dcbbb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -76,6 +76,16 @@ struct SimdgroupMmaCore {
THREADGROUP_THREADS,
GROUP_SIZE,
BITS>;
+ using BLoaderScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<
+ BT,
+ THREADGROUP_BLOCK_N,
+ THREADGROUP_BLOCK_K,
+ SHARED_STRIDE_B,
+ 1,
+ THREADGROUP_THREADS,
+ GROUP_SIZE,
+ BITS,
+ true>;
using TileAccumulator = uzu::matmul::ThreadgroupTile<
AT,
BT,
@@ -411,6 +421,56 @@ struct SimdgroupMmaCore {
needs_bias
);
});
+ return;
+ }
+
+ if constexpr (
+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleSymmetricDequant
+ ) {
+ constexpr int pack_factor = get_pack_factor<BITS, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
+ const int k_elements = int(params->K);
+ const int weights_row_stride_bytes =
+ k_elements * bytes_per_pack / pack_factor;
+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
+ const device uint8_t* weights_block =
+ b_packed + block_col * weights_row_stride_bytes;
+ const device BT* scales_offset = scales + block_col * groups_per_row;
+ thread BLoaderScaleSymmetric loader_b(
+ weights_block,
+ scales_offset,
+ nullptr,
+ k_elements,
+ groups_per_row,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ dispatch_kernel([&](auto gemm_alignment_mask) {
+ constexpr uint mask = gemm_alignment_mask.value;
+ k_loop<mask, BLoaderScaleSymmetric>(
+ a_shared,
+ b_shared,
+ params->aligned_inner_iterations,
+ loader_a,
+ loader_b,
+ accumulator,
+ tile_block_rows,
+ tile_block_cols,
+ leftover_block_depth
+ );
+ finalize<mask>(
+ accumulator,
+ d,
+ params,
+ tile_block_rows,
+ tile_block_cols,
+ needs_epilogue,
+ epilogue,
+ bias_block,
+ needs_bias
+ );
+ });
}
}
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 84b978d29..66d4cb4f1 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -49,7 +49,8 @@ VARIANTS(
WEIGHT_PROLOGUE,
GemmWeightPrologueKind::FullPrecision,
GemmWeightPrologueKind::ScaleBiasDequant,
- GemmWeightPrologueKind::ScaleZeroPointDequant)
+ GemmWeightPrologueKind::ScaleZeroPointDequant,
+ GemmWeightPrologueKind::ScaleSymmetricDequant)
VARIANTS(BITS, 0, 4, 8)
VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)
CONSTRAINT(
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 45d53f296..e75597cdd 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -153,7 +153,7 @@ impl GemmKernel {
GemmDispatchPath::Simdgroup => false,
};
- let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });
+ let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });
if is_quant {
let d_mask = arguments.d_transform.mask();
if d_mask.contains(GemmDTransform::ACCUMULATE) {
@@ -299,6 +299,9 @@ impl GemmKernel {
}
| MatmulB::ScaleZeroPointDequant {
..
+ }
+ | MatmulB::ScaleSymmetricDequant {
+ ..
}) => {
let (weights, scales, biases, zero_points) = match quant_b {
MatmulB::ScaleBiasDequant {
@@ -313,6 +316,11 @@ impl GemmKernel {
zero_points,
..
} => (w, Some(scales), None, Some(zero_points)),
+ MatmulB::ScaleSymmetricDequant {
+ b: w,
+ scales,
+ ..
+ } => (w, Some(scales), None, None),
_ => unreachable!(),
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
index c6c4568e7..a6fc03fc2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -102,6 +102,7 @@ impl GemmSpecialization {
let weight_prologue = match combo.method {
QuantizationMethod::ScaleBias => GemmWeightPrologueKind::ScaleBiasDequant,
QuantizationMethod::ScaleZeroPoint => GemmWeightPrologueKind::ScaleZeroPointDequant,
+ QuantizationMethod::ScaleSymmetric => GemmWeightPrologueKind::ScaleSymmetricDequant,
};
let mut out = Vec::new();
for &tiling in quant_tiling_set(weights_data_type) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
index 025d39fd9..903436a33 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
@@ -72,7 +72,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
const device WeightT* biases_row_base = nullptr;
if (quant_method == QuantizationMethod::ScaleBias) {
biases_row_base = biases + out_row * in_vec_size_g;
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
zps_row_base = zero_points + out_row * zp_stride;
}
@@ -97,7 +97,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
U b = static_cast<U>(bl[g]);
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
const device uint8_t* zl = zps_row_base + row * zp_stride;
U zp;
if (BITS == 4) {
@@ -108,6 +108,15 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
}
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
+ } else {
+ constexpr U midpoint = U(1u << (BITS - 1));
+ result[row] += qdot<U, values_per_thread, BITS>(
+ wl,
+ x_thread,
+ s,
+ -s * midpoint,
+ sum
+ );
}
}
@@ -141,7 +150,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
U b = static_cast<U>(bl[g]);
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
const device uint8_t* zl = zps_row_base + row * zp_stride;
U zp;
if (BITS == 4) {
@@ -152,6 +161,15 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
}
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
+ } else {
+ constexpr U midpoint = U(1u << (BITS - 1));
+ result[row] += qdot<U, values_per_thread, BITS>(
+ wl,
+ x_thread,
+ s,
+ -s * midpoint,
+ sum
+ );
}
}
}
@@ -175,7 +193,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
const device WeightT* biases_row_base = nullptr;
if (quant_method == QuantizationMethod::ScaleBias) {
biases_row_base = biases + used_out_row * in_vec_size_g;
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
zps_row_base = zero_points + used_out_row * zp_stride;
}
@@ -198,7 +216,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
U b = static_cast<U>(bl[g]);
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
const device uint8_t* zl = zps_row_base + row * zp_stride;
U zp;
if (BITS == 4) {
@@ -209,6 +227,15 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
}
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
+ } else {
+ constexpr U midpoint = U(1u << (BITS - 1));
+ result[row] += qdot<U, values_per_thread, BITS>(
+ wl,
+ x_thread,
+ s,
+ -s * midpoint,
+ sum
+ );
}
}
@@ -247,7 +274,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
sum,
remaining
);
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
const device uint8_t* zl = zps_row_base + row * zp_stride;
U zp;
if (BITS == 4) {
@@ -258,6 +285,16 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
}
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
+ } else {
+ constexpr U midpoint = U(1u << (BITS - 1));
+ result[row] += qdot_safe<U, values_per_thread, BITS>(
+ wl,
+ x_thread,
+ s,
+ -s * midpoint,
+ sum,
+ remaining
+ );
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
index c2ba438db..de733f048 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
@@ -65,7 +65,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
if (quant_method == QuantizationMethod::ScaleBias) {
biases += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
if (BITS == 4) {
zp_stride = (in_vec_size_g + 1) / 2;
zps = zero_points + out_row * zp_stride;
@@ -109,7 +109,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
qdot<U, values_per_thread, BITS>(wl2, x_thread, s2, b2, sum);
result[3] +=
qdot<U, values_per_thread, BITS>(wl3, x_thread, s3, b3, sum);
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
uchar4 zp_bytes = uchar4(
zps[0],
zps[zp_stride],
@@ -151,6 +151,36 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
-s3 * static_cast<U>(zp_nibbles.w),
sum
);
+ } else {
+ constexpr U midpoint = U(1u << (BITS - 1));
+ result[0] += qdot<U, values_per_thread, BITS>(
+ wl0,
+ x_thread,
+ s0,
+ -s0 * midpoint,
+ sum
+ );
+ result[1] += qdot<U, values_per_thread, BITS>(
+ wl1,
+ x_thread,
+ s1,
+ -s1 * midpoint,
+ sum
+ );
+ result[2] += qdot<U, values_per_thread, BITS>(
+ wl2,
+ x_thread,
+ s2,
+ -s2 * midpoint,
+ sum
+ );
+ result[3] += qdot<U, values_per_thread, BITS>(
+ wl3,
+ x_thread,
+ s3,
+ -s3 * midpoint,
+ sum
+ );
}
}
@@ -158,7 +188,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
scales += block_size / GROUP_SIZE;
if (quant_method == QuantizationMethod::ScaleBias) {
biases += block_size / GROUP_SIZE;
- } else {
+ } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
if (BITS == 4) {
zps += (block_size / GROUP_SIZE) / 2;
} else {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
index c2e4d6b33..fc44762c1 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
@@ -95,14 +95,20 @@ impl QuantGemvKernel {
biases,
mode,
group_size,
- } => (w, scales, biases, QuantizationMethod::ScaleBias, mode, group_size),
+ } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),
MatmulB::ScaleZeroPointDequant {
b: w,
scales,
zero_points,
mode,
group_size,
- } => (w, scales, zero_points, QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),
+ MatmulB::ScaleSymmetricDequant {
+ b: w,
+ scales,
+ mode,
+ group_size,
+ } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),
MatmulB::FullPrecision {
..
} => panic!("QuantGemvKernel requires quantized B"),
@@ -115,8 +121,13 @@ impl QuantGemvKernel {
let use_fast = n % 8 == 0 && k % 512 == 0;
let (zero_points, biases) = match method {
- QuantizationMethod::ScaleZeroPoint => (Some(zp_or_bias), None),
- QuantizationMethod::ScaleBias => (None, Some(zp_or_bias)),
+ QuantizationMethod::ScaleBias => {
+ (None, Some(zp_or_bias.expect("ScaleBias quantized matmul requires biases")))
+ },
+ QuantizationMethod::ScaleZeroPoint => {
+ (Some(zp_or_bias.expect("ScaleZeroPoint quantized matmul requires zero_points")), None)
+ },
+ QuantizationMethod::ScaleSymmetric => (None, None),
};
if use_fast {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 9c8015bf2..ceaacb941 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -166,7 +166,7 @@ impl MatmulKernel for MatmulMetalKernel {
arguments: MatmulArguments<Metal, TB>,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
- let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });
+ let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });
let gemv_eligible = if is_quant {
arguments.m < 5 || arguments.n == 1
} else {
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 3f846f377..8c4fe862c 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -52,7 +52,7 @@ enum TiedEmbeddingType<B: Backend> {
Quantized {
weights: Allocation<B>,
scales: Allocation<B>,
- zero_points_or_biases: Allocation<B>,
+ zero_points_or_biases: Option<Allocation<B>>,
quantization_method: QuantizationMethod,
output_hadamard_factors: Option<Allocation<B>>,
lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
@@ -69,7 +69,7 @@ enum UntiedEmbeddingLookupType<B: Backend> {
Quantized {
weights: Allocation<B>,
scales: Allocation<B>,
- zero_points_or_biases: Allocation<B>,
+ zero_points_or_biases: Option<Allocation<B>>,
quantization_method: QuantizationMethod,
output_hadamard_factors: Option<Allocation<B>>,
lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
@@ -84,7 +84,7 @@ enum UntiedEmbeddingReadoutType<B: Backend> {
Quantized {
weights: Allocation<B>,
scales: Allocation<B>,
- zero_points_or_biases: Allocation<B>,
+ zero_points_or_biases: Option<Allocation<B>>,
readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
readout_config: ReadoutQuantConfig,
},
@@ -502,8 +502,9 @@ impl<B: Backend> Embedding<B> {
output_ty: _,
} => {
let (zero_points, biases) = match quantization_method {
- QuantizationMethod::ScaleZeroPoint => (Some(zero_points_or_biases), None),
- QuantizationMethod::ScaleBias => (None, Some(zero_points_or_biases)),
+ QuantizationMethod::ScaleBias => (None, zero_points_or_biases.as_ref()),
+ QuantizationMethod::ScaleZeroPoint => (zero_points_or_biases.as_ref(), None),
+ QuantizationMethod::ScaleSymmetric => (None, None),
};
lookup.encode(
token_ids,
@@ -603,14 +604,22 @@ impl<B: Backend> Embedding<B> {
QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
b: weights,
scales,
- biases: zero_points_or_biases,
+ biases: zero_points_or_biases.as_ref().expect("ScaleBias quantization requires biases"),
mode: readout_config.mode,
group_size: readout_config.group_size,
},
QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
b: weights,
scales,
- zero_points: zero_points_or_biases,
+ zero_points: zero_points_or_biases
+ .as_ref()
+ .expect("ScaleZeroPoint quantization requires zero_points"),
+ mode: readout_config.mode,
+ group_size: readout_config.group_size,
+ },
+ QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {
+ b: weights,
+ scales,
mode: readout_config.mode,
group_size: readout_config.group_size,
},
@@ -658,6 +667,13 @@ fn input_quantization_from_spec<B: Backend>(
layout: Layout::InputOutput,
..
}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),
+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ bits,
+ group_size,
+ is_symmetric: true,
+ layout: Layout::InputOutput,
+ ..
+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleSymmetric),
spec => Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
}
}
@@ -679,6 +695,13 @@ fn output_quantization_from_spec<B: Backend>(
layout: Layout::OutputInput,
..
}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),
+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ bits,
+ group_size,
+ is_symmetric: true,
+ layout: Layout::OutputInput,
+ ..
+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleSymmetric),
spec => Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
}
}
@@ -708,7 +731,7 @@ fn load_quantized_embedding_parts<B: Backend>(
quantization_mode: QuantizationMode,
quantization_method: QuantizationMethod,
group_size: usize,
-) -> Result<(Allocation<B>, Allocation<B>, Allocation<B>), EmbeddingError<B>> {
+) -> Result<(Allocation<B>, Allocation<B>, Option<Allocation<B>>), EmbeddingError<B>> {
let packing_divisor = quantization_mode.packing_divisor();
let storage_data_type = quantization_mode.storage_type();
let num_groups = model_dim.div_ceil(group_size);
@@ -720,14 +743,17 @@ fn load_quantized_embedding_parts<B: Backend>(
let scales = tree.leaf("scales")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?;
let zero_points_or_biases = match quantization_method {
QuantizationMethod::ScaleBias => {
- tree.leaf("biases")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?
+ Some(tree.leaf("biases")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?)
},
QuantizationMethod::ScaleZeroPoint => {
let expected_zero_points_entries = num_groups.div_ceil(packing_divisor);
- tree.leaf("zero_points")?
- .validate(&[vocab_size, expected_zero_points_entries], storage_data_type)?
- .read_allocation()?
+ Some(
+ tree.leaf("zero_points")?
+ .validate(&[vocab_size, expected_zero_points_entries], storage_data_type)?
+ .read_allocation()?,
+ )
},
+ QuantizationMethod::ScaleSymmetric => None,
};
Ok((weights, scales, zero_points_or_biases))
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
index da54a4872..d851fc9d7 100644
--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -37,7 +37,7 @@ enum Mode<B: Backend> {
mode: QuantizationMode,
group_size: u32,
scales: Allocation<B>,
- zero_points_or_biases: Allocation<B>,
+ zero_points_or_biases: Option<Allocation<B>>,
output_hadamard_factors: Option<Allocation<B>>,
},
}
@@ -121,6 +121,13 @@ impl<B: Backend> LinearMatmul<B> {
layout: Layout::OutputInput,
..
}) => (bits, group_size, QuantizationMethod::ScaleZeroPoint),
+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ bits,
+ group_size,
+ is_symmetric: true,
+ layout: Layout::OutputInput,
+ ..
+ }) => (bits, group_size, QuantizationMethod::ScaleSymmetric),
spec => return Err(LinearMatmulError::UnsupportedConfiguration(format!("{spec:?}"))),
};
@@ -151,15 +158,18 @@ impl<B: Backend> LinearMatmul<B> {
let scales = weights_tree.leaf("scales")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?;
let zero_points_or_biases = match quantization_method {
QuantizationMethod::ScaleBias => {
- weights_tree.leaf("biases")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?
+ Some(weights_tree.leaf("biases")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?)
},
QuantizationMethod::ScaleZeroPoint => {
let expected_zero_points_entries = k_g.div_ceil(packing_divisor);
- weights_tree
- .leaf("zero_points")?
- .validate(&[output_dim, expected_zero_points_entries], storage_type)?
- .read_allocation()?
+ Some(
+ weights_tree
+ .leaf("zero_points")?
+ .validate(&[output_dim, expected_zero_points_entries], storage_type)?
+ .read_allocation()?,
+ )
},
+ QuantizationMethod::ScaleSymmetric => None,
};
let biases = load_biases(weights_data_type, output_data_type, output_dim, bias_tree)?;
@@ -242,14 +252,22 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
b: &self.weights,
scales,
- biases: zero_points_or_biases,
+ biases: zero_points_or_biases.as_ref().expect("ScaleBias quantization requires biases"),
mode: *mode,
group_size: *group_size,
},
QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
b: &self.weights,
scales,
- zero_points: zero_points_or_biases,
+ zero_points: zero_points_or_biases
+ .as_ref()
+ .expect("ScaleZeroPoint quantization requires zero_points"),
+ mode: *mode,
+ group_size: *group_size,
+ },
+ QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {
+ b: &self.weights,
+ scales,
mode: *mode,
group_size: *group_size,
},
diff --git a/crates/backend-uzu/tests/common/matmul/quant.rs b/crates/backend-uzu/tests/common/matmul/quant.rs
index bad1def0d..3790826e4 100644
--- a/crates/backend-uzu/tests/common/matmul/quant.rs
+++ b/crates/backend-uzu/tests/common/matmul/quant.rs
@@ -65,13 +65,14 @@ impl<T: ArrayElement + Float> QuantInput<T> {
num_groups_k
};
let (zero_points, biases) = match quant_method {
- QuantizationMethod::ScaleZeroPoint => {
- (Some((0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)
- },
QuantizationMethod::ScaleBias => (
None,
Some((0..n * num_groups_k).map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap()).collect()),
),
+ QuantizationMethod::ScaleZeroPoint => {
+ (Some((0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)
+ },
+ QuantizationMethod::ScaleSymmetric => (None, None),
};
Self {
@@ -122,6 +123,13 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(
input: &QuantInput<T>,
) -> MatmulArguments<'a, B> {
let b_variant = match input.quant_method {
+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
+ b: &buffers.w,
+ scales: &buffers.scales,
+ biases: buffers.bias.as_ref().expect("bias buffer"),
+ mode: input.mode,
+ group_size: input.group_size,
+ },
QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
b: &buffers.w,
scales: &buffers.scales,
@@ -129,10 +137,9 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(
mode: input.mode,
group_size: input.group_size,
},
- QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
+ QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {
b: &buffers.w,
scales: &buffers.scales,
- biases: buffers.bias.as_ref().expect("bias buffer"),
mode: input.mode,
group_size: input.group_size,
},
diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
index d007d1647..40a942036 100644
--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
@@ -1,6 +1,4 @@
-use std::{
- fmt::{Debug, Display},
-};
+use std::fmt::{Debug, Display};
use backend_uzu::{
ArrayContextExt, ArrayElement, DataType,
@@ -119,6 +117,43 @@ fn get_test_data_zero_point_group16<T: ArrayElement + Float>() -> (Input<T>, Vec
(input, expected)
}
+fn get_test_data_symmetric_u8<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let batch_size = 3u32;
+ let vocab_size = 8u32;
+ let model_dim = 64u32;
+ let group_size = 32u32;
+ let input_scale = 1.5f32;
+ let quant_mode = QuantizationMode::U8;
+
+ let token_ids: Box<[u64]> = Box::new([2, 5, 0]);
+
+ let weights: Vec<u8> =
+ (0..vocab_size as usize * model_dim as usize).map(|i| ((i * 37 + 11) & 0xFF) as u8).collect();
+
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ let scales: Vec<T> = (0..vocab_size as usize * num_groups as usize)
+ .map(|i| T::from(0.5 + (i as f32 * 0.1).sin() * 0.3).unwrap())
+ .collect();
+
+ let input = Input {
+ token_ids,
+ weights: weights.into_boxed_slice(),
+ scales: scales.into_boxed_slice(),
+ zero_points: None,
+ biases: None,
+ batch_size,
+ vocab_size,
+ model_dim,
+ input_scale,
+ group_size,
+ quant_mode,
+ quant_method: QuantizationMethod::ScaleSymmetric,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
let batch_size = 2u32;
let vocab_size = 4u32;
@@ -184,10 +219,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.zero_points
.as_ref()
.map(|zero_points| context.create_array_from(&[input.vocab_size as usize, zero_point_stride], zero_points));
- let biases_array = input
- .biases
- .as_ref()
- .map(|biases| context.create_array_from(&[input.vocab_size as usize, num_groups], biases));
+ let biases_array =
+ input.biases.as_ref().map(|biases| context.create_array_from(&[input.vocab_size as usize, num_groups], biases));
let mut output = context
.create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type())
.into_allocation();
@@ -233,6 +266,24 @@ fn test_zero_point_group16<T: ArrayElement + Float + Debug + Display>() {
});
}
+fn test_symmetric_u8<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.5f32
+ } else {
+ 1e-4
+ };
+ let (input, expected) = get_test_data_symmetric_u8::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("QuantizedEmbeddingLookup ScaleSymmetric test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
#[cfg(metal_backend)]
fn test_zero_point_group16_hadamard_constructor<T: ArrayElement + Float + Debug + Display>() {
use backend_uzu::backends::metal::Metal;
@@ -344,6 +395,11 @@ fn test_uint8_bf16() {
test_quant_mode::<bf16>(QuantizationMode::U8);
}
+#[uzu_test]
+fn test_uint8_symmetric_bf16() {
+ test_symmetric_u8::<bf16>();
+}
+
// OOB tests
#[uzu_test]
fn test_oob_f32() {
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
index 5db7b81cc..9490c274d 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
@@ -97,6 +97,8 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(
#[case::gs32_8bit_mlx_prefill ( 64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]
#[case::gs64_8bit_zp_prefill ( 64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
#[case::gs128_8bit_zp_prefill ( 64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs32_4bit_symmetric_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]
+#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
#[case::gs32_4bit_mlx_decode ( 8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
#[case::gs16_4bit_zp_decode ( 8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_4bit_zp_decode ( 8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index 41232cb36..77904bbf5 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -4,8 +4,8 @@ use backend_uzu::{
ArrayElement, DataType,
backends::{
common::{
- Allocation, Backend, Context, Encoder, Kernels, gpu_types::QuantizationMethod,
-
+ Allocation, Backend, Context, Encoder, Kernels,
+ gpu_types::QuantizationMethod,
kernel::{QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},
},
cpu::Cpu,
@@ -139,6 +139,16 @@ fn get_test_data<T: ArrayElement + Float>(
};
let (zero_points, biases) = match quant_method {
+ QuantizationMethod::ScaleBias => {
+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
+ }
+ }
+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+ (None, Some(biases))
+ },
QuantizationMethod::ScaleZeroPoint => {
let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
for j in 0..n {
@@ -157,16 +167,7 @@ fn get_test_data<T: ArrayElement + Float>(
}
(Some(zp_packed), None)
},
- QuantizationMethod::ScaleBias => {
- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
- }
- }
- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- (None, Some(biases))
- },
+ QuantizationMethod::ScaleSymmetric => (None, None),
};
let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
@@ -281,14 +282,6 @@ macro_rules! qmv_fast_test {
};
}
-qmv_fast_test!(test_gs16_4bit_zp, gs = 16, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs16_8bit_zp, gs = 16, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
qmv_fast_test!(test_gs16_4bit_mlx, gs = 16, bits = 4, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs32_4bit_mlx, gs = 32, bits = 4, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs64_4bit_mlx, gs = 64, bits = 4, format = QuantizationMethod::ScaleBias);
@@ -297,3 +290,12 @@ qmv_fast_test!(test_gs16_8bit_mlx, gs = 16, bits = 8, format = QuantizationMetho
qmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, format = QuantizationMethod::ScaleBias);
+qmv_fast_test!(test_gs16_4bit_zp, gs = 16, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs16_8bit_zp, gs = 16, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs32_8bit_symmetric, gs = 32, bits = 8, format = QuantizationMethod::ScaleSymmetric);
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
index d38d7fdcf..3e21443bc 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
@@ -3,10 +3,7 @@ use std::fmt::{Debug, Display};
use backend_uzu::{
ArrayElement, DataType,
backends::{
- common::{
- Backend, Context, Encoder, Kernels, gpu_types::QuantizationMethod,
- kernel::QuantizedMatmulQmvKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, gpu_types::QuantizationMethod, kernel::QuantizedMatmulQmvKernel},
cpu::Cpu,
},
};
@@ -105,6 +102,16 @@ fn get_test_data_basic<T: ArrayElement + Float>(
};
let (zero_points, biases) = match quant_method {
+ QuantizationMethod::ScaleBias => {
+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
+ }
+ }
+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+ (None, Some(biases))
+ },
QuantizationMethod::ScaleZeroPoint => {
let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
for j in 0..n {
@@ -124,16 +131,7 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
(Some(zp_packed), None)
},
- QuantizationMethod::ScaleBias => {
- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
- }
- }
- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- (None, Some(biases))
- },
+ QuantizationMethod::ScaleSymmetric => (None, None),
};
// X: [m × k]
@@ -188,15 +186,16 @@ fn get_test_data_edge<T: ArrayElement + Float>(
};
let (zero_points, biases) = match quant_method {
+ QuantizationMethod::ScaleBias => {
+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
+ (None, Some(biases))
+ },
QuantizationMethod::ScaleZeroPoint => {
// Zero points all 0 → bias = 0
let zp_packed = vec![0u8; n * zp_stride];
(Some(zp_packed), None)
},
- QuantizationMethod::ScaleBias => {
- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
- (None, Some(biases))
- },
+ QuantizationMethod::ScaleSymmetric => (None, None),
};
// X: simple increasing values
@@ -302,151 +301,158 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>(
test_internal::<T>(&input, &expected);
}
-// ── 4-bit, zero points ─────────────────────────────────────────────────────
+// ── 4-bit, mlx quant ───────────────────────────────────────────────────────
#[uzu_test]
-fn test_gs16_4bit_zp() {
+fn test_gs16_4bit_mlx() {
for_each_float_type!(|F| {
- test_basic::<F>(16, 4, QuantizationMethod::ScaleZeroPoint);
+ test_basic::<F>(16, 4, QuantizationMethod::ScaleBias);
})
}
#[uzu_test]
-fn test_gs32_4bit_zp() {
+fn test_gs32_4bit_mlx() {
for_each_float_type!(|F| {
- test_basic::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);
+ test_basic::<F>(32, 4, QuantizationMethod::ScaleBias);
})
}
#[uzu_test]
-fn test_gs64_4bit_zp() {
+fn test_gs64_4bit_mlx() {
for_each_float_type!(|F| {
- test_basic::<F>(64, 4, QuantizationMethod::ScaleZeroPoint);
+ test_basic::<F>(64, 4, QuantizationMethod::ScaleBias);
})
}
#[uzu_test]
-fn test_gs128_4bit_zp() {
+fn test_gs128_4bit_mlx() {
for_each_float_type!(|F| {
- test_basic::<F>(128, 4, QuantizationMethod::ScaleZeroPoint);
+ test_basic::<F>(128, 4, QuantizationMethod::ScaleBias);
})
}
-// ── 8-bit, zero points ─────────────────────────────────────────────────────
+// ── 8-bit, mlx quant ───────────────────────────────────────────────────────
#[uzu_test]
-fn test_gs16_8bit_zp() {
+fn test_gs16_8bit_mlx() {
for_each_float_type!(|F| {
- test_basic::<F>(16, 8, QuantizationMethod::ScaleZeroPoint);
+ test_basic::<F>(16, 8, QuantizationMethod::ScaleBias);
})
}
#[uzu_test]
-fn test_gs32_8bit_zp() {
+fn test_gs32_8bit_mlx() {
for_each_float_type!(|F| {
- test_basic::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);
+ test_basic::<F>(32, 8, QuantizationMethod::ScaleBias);
})
}
#[uzu_test]
-fn test_gs64_8bit_zp() {
+fn test_gs64_8bit_mlx() {
for_each_float_type!(|F| {
- test_basic::<F>(64, 8, QuantizationMethod::ScaleZeroPoint);
+ test_basic::<F>(64, 8, QuantizationMethod::ScaleBias);
})
}
#[uzu_test]
-fn test_gs128_8bit_zp() {
+fn test_gs128_8bit_mlx() {
for_each_float_type!(|F| {
- test_basic::<F>(128, 8, QuantizationMethod::ScaleZeroPoint);
+ test_basic::<F>(128, 8, QuantizationMethod::ScaleBias);
})
}
-// ── 4-bit, mlx quant ───────────────────────────────────────────────────────
+// ── 4-bit, zero points ─────────────────────────────────────────────────────
#[uzu_test]
-fn test_gs16_4bit_mlx() {
+fn test_gs16_4bit_zp() {
for_each_float_type!(|F| {
- test_basic::<F>(16, 4, QuantizationMethod::ScaleBias);
+ test_basic::<F>(16, 4, QuantizationMethod::ScaleZeroPoint);
})
}
#[uzu_test]
-fn test_gs32_4bit_mlx() {
+fn test_gs32_4bit_zp() {
for_each_float_type!(|F| {
- test_basic::<F>(32, 4, QuantizationMethod::ScaleBias);
+ test_basic::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);
})
}
#[uzu_test]
-fn test_gs64_4bit_mlx() {
+fn test_gs64_4bit_zp() {
for_each_float_type!(|F| {
- test_basic::<F>(64, 4, QuantizationMethod::ScaleBias);
+ test_basic::<F>(64, 4, QuantizationMethod::ScaleZeroPoint);
})
}
#[uzu_test]
-fn test_gs128_4bit_mlx() {
+fn test_gs128_4bit_zp() {
for_each_float_type!(|F| {
- test_basic::<F>(128, 4, QuantizationMethod::ScaleBias);
+ test_basic::<F>(128, 4, QuantizationMethod::ScaleZeroPoint);
})
}
-// ── 8-bit, mlx quant ───────────────────────────────────────────────────────
+// ── 8-bit, zero points ─────────────────────────────────────────────────────
#[uzu_test]
-fn test_gs16_8bit_mlx() {
+fn test_gs16_8bit_zp() {
for_each_float_type!(|F| {
- test_basic::<F>(16, 8, QuantizationMethod::ScaleBias);
+ test_basic::<F>(16, 8, QuantizationMethod::ScaleZeroPoint);
})
}
#[uzu_test]
-fn test_gs32_8bit_mlx() {
+fn test_gs32_8bit_zp() {
for_each_float_type!(|F| {
- test_basic::<F>(32, 8, QuantizationMethod::ScaleBias);
+ test_basic::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);
})
}
#[uzu_test]
-fn test_gs64_8bit_mlx() {
+fn test_gs64_8bit_zp() {
for_each_float_type!(|F| {
- test_basic::<F>(64, 8, QuantizationMethod::ScaleBias);
+ test_basic::<F>(64, 8, QuantizationMethod::ScaleZeroPoint);
})
}
#[uzu_test]
-fn test_gs128_8bit_mlx() {
+fn test_gs128_8bit_zp() {
for_each_float_type!(|F| {
- test_basic::<F>(128, 8, QuantizationMethod::ScaleBias);
+ test_basic::<F>(128, 8, QuantizationMethod::ScaleZeroPoint);
+ })
+}
+
+#[uzu_test]
+fn test_gs32_8bit_symmetric() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 8, QuantizationMethod::ScaleSymmetric);
})
}
// ── Edge cases ──────────────────────────────────────────────────────────────
#[uzu_test]
-fn test_edge_4bit_zp() {
+fn test_edge_4bit_mlx() {
for_each_float_type!(|F| {
- test_edge::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);
+ test_edge::<F>(32, 4, QuantizationMethod::ScaleBias);
})
}
#[uzu_test]
-fn test_edge_8bit_zp() {
+fn test_edge_8bit_mlx() {
for_each_float_type!(|F| {
- test_edge::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);
+ test_edge::<F>(32, 8, QuantizationMethod::ScaleBias);
})
}
#[uzu_test]
-fn test_edge_4bit_mlx() {
+fn test_edge_4bit_zp() {
for_each_float_type!(|F| {
- test_edge::<F>(32, 4, QuantizationMethod::ScaleBias);
+ test_edge::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);
})
}
#[uzu_test]
-fn test_edge_8bit_mlx() {
+fn test_edge_8bit_zp() {
for_each_float_type!(|F| {
- test_edge::<F>(32, 8, QuantizationMethod::ScaleBias);
+ test_edge::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);
})
}