Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Group size 16 quant (#455)

uuuvn43dcba2606cf
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
index f306865e0..17736db09 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
@@ -100,7 +100,7 @@ pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT:
#[variants(WeightT, f32, f16, bf16)]
#[variants(InputT, f32, f16, bf16)]
#[variants(OutputT, f32, f16, bf16)]
-#[variants(GROUP_SIZE, 32, 64, 128)]
+#[variants(GROUP_SIZE, 16, 32, 64, 128)]
#[variants(BITS, 4, 8)]
pub fn quantized_matmul_qmv<
WeightT: ArrayElement + Float,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
index dee1f9c6a..1956bc9e5 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
@@ -11,7 +11,7 @@ use crate::{
#[variants(WeightT, f32, f16, bf16)]
#[variants(InputT, f32, f16, bf16)]
#[variants(OutputT, f32, f16, bf16)]
-#[variants(GROUP_SIZE, 32, 64, 128)]
+#[variants(GROUP_SIZE, 16, 32, 64, 128)]
#[variants(BITS, 4, 8)]
pub fn quantized_matmul_qmv_fast<
WeightT: ArrayElement + Float,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 85916fa0e..24c408ea9 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -214,7 +214,7 @@ impl MatmulCpuKernel {
..
} => unreachable!(),
};
- if !matches!(group_size, 32 | 64 | 128) {
+ if !matches!(group_size, 16 | 32 | 64 | 128) {
return Err(MatmulError::UnsupportedGroupSize(group_size as usize));
}
diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rs
index 099eb48ef..5a59d16da 100644
--- a/crates/backend-uzu/src/backends/metal/error.rs
+++ b/crates/backend-uzu/src/backends/metal/error.rs
@@ -29,8 +29,8 @@ pub enum MetalError {
CommandBufferExecutionFailed(String),
#[error("Cannot create event")]
CannotCreateEvent,
- #[error("Cannot create function")]
- CannotCreateFunction,
+ #[error("Cannot create function: {0}")]
+ CannotCreateFunction(String),
#[error("Cannot create pipeline state: {0}")]
CannotCreatePipelineState(String),
#[error("Can not allocate buffer with size={0}")]
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index b8c22eab5..84b978d29 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -51,7 +51,7 @@ VARIANTS(
GemmWeightPrologueKind::ScaleBiasDequant,
GemmWeightPrologueKind::ScaleZeroPointDequant)
VARIANTS(BITS, 0, 4, 8)
-VARIANTS(GROUP_SIZE, 0, 32, 64, 128)
+VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)
CONSTRAINT(
!USE_MXU || GEMM_TILING == GemmTiling::T64x64x32_2x2 ||
GEMM_TILING == GemmTiling::T32x64x32_2x2 ||
@@ -67,10 +67,14 @@ CONSTRAINT(
WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||
(!USE_MXU && TRANSPOSE_B &&
(GEMM_TILING == GemmTiling::T8x32x32_1x1 ||
+ GEMM_TILING == GemmTiling::T64x64x16_2x2 ||
GEMM_TILING == GemmTiling::T32x32x32_2x2 ||
GEMM_TILING == GemmTiling::T64x32x32_2x2 ||
GEMM_TILING == GemmTiling::T64x64x32_2x2 ||
GEMM_TILING == GemmTiling::T64x64x64_2x2)))
+CONSTRAINT(
+ WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||
+ GROUP_SIZE != 16 || GEMM_TILING == GemmTiling::T64x64x16_2x2)
CONSTRAINT(
GEMM_TILING != GemmTiling::T64x64x64_2x2 ||
GROUP_SIZE == 64 || GROUP_SIZE == 128)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 6be34454c..45d53f296 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -316,7 +316,7 @@ impl GemmKernel {
_ => unreachable!(),
};
- let tiling = select_quant_tiling(m, n);
+ let tiling = select_quant_tiling(m, n, group_size.unwrap());
let alignment =
GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
let params = quant_params(m, n, k, tiling, ab_scale);
@@ -414,8 +414,11 @@ fn select_mxu_tiling(
fn select_quant_tiling(
m: u32,
n: u32,
+ group_size: u32,
) -> GemmTiling {
- if m < 32 {
+ if group_size < 32 {
+ GemmTiling::T64x64x16_2x2
+ } else if m < 32 {
GemmTiling::T8x32x32_1x1
} else if m >= 64 && n <= 2048 {
GemmTiling::T64x32x32_2x2
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 04985b10f..c6c4568e7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -147,13 +147,14 @@ fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
pub(crate) fn quant_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
match data_type {
DataType::BF16 => &[
+ GemmTiling::T64x64x16_2x2,
GemmTiling::T8x32x32_1x1,
GemmTiling::T32x32x32_2x2,
GemmTiling::T64x32x32_2x2,
GemmTiling::T64x64x32_2x2,
GemmTiling::T64x64x64_2x2,
],
- DataType::F16 => &[GemmTiling::T8x32x32_1x1, GemmTiling::T32x32x32_2x2],
+ DataType::F16 => &[GemmTiling::T64x64x16_2x2, GemmTiling::T8x32x32_1x1, GemmTiling::T32x32x32_2x2],
_ => &[],
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
index c1da8180f..025d39fd9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
@@ -16,7 +16,7 @@ template <
VARIANTS(WeightT, float, half, bfloat)
VARIANTS(InputT, float, half, bfloat)
VARIANTS(OutputT, float, half, bfloat)
-VARIANTS(GROUP_SIZE, 32, 64, 128)
+VARIANTS(GROUP_SIZE, 16, 32, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmv)(
const device uint32_t* weights,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
index 89859f704..c2ba438db 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
@@ -17,7 +17,7 @@ template <
VARIANTS(WeightT, float, half, bfloat)
VARIANTS(InputT, float, half, bfloat)
VARIANTS(OutputT, float, half, bfloat)
-VARIANTS(GROUP_SIZE, 32, 64, 128)
+VARIANTS(GROUP_SIZE, 16, 32, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmvFast)(
const device uint32_t* weights,
diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs
index a5508b9fa..980fa1ee8 100644
--- a/crates/backend-uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs
+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/library_extensions_pipeline.rs
@@ -26,7 +26,7 @@ impl LibraryPipelineExtensions for ProtocolObject<dyn MTLLibrary> {
},
None => self.new_function_with_name(function_name),
}
- .ok_or(MetalError::CannotCreateFunction)?;
+ .ok_or_else(|| MetalError::CannotCreateFunction(function_name.to_owned()))?;
let device = self.device();
diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
index 1d2507b18..d007d1647 100644
--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
@@ -22,13 +22,15 @@ struct Input<T: ArrayElement + Float> {
token_ids: Box<[u64]>,
weights: Box<[u8]>,
scales: Box<[T]>,
- biases: Box<[T]>,
+ zero_points: Option<Box<[u8]>>,
+ biases: Option<Box<[T]>>,
batch_size: u32,
vocab_size: u32,
model_dim: u32,
input_scale: f32,
group_size: u32,
quant_mode: QuantizationMode,
+ quant_method: QuantizationMethod,
}
fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Input<T>, Vec<T>) {
@@ -58,13 +60,59 @@ fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Inpu
token_ids,
weights: weights.into_boxed_slice(),
scales: scales.into_boxed_slice(),
- biases: biases.into_boxed_slice(),
+ zero_points: None,
+ biases: Some(biases.into_boxed_slice()),
batch_size,
vocab_size,
model_dim,
input_scale,
group_size,
quant_mode,
+ quant_method: QuantizationMethod::ScaleBias,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_zero_point_group16<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let batch_size = 3u32;
+ let vocab_size = 8u32;
+ let model_dim = 64u32;
+ let group_size = 16u32;
+ let input_scale = 1.5f32;
+ let quant_mode = QuantizationMode::U4;
+
+ let token_ids: Box<[u64]> = Box::new([2, 5, 0]);
+
+ let packing_divisor = quant_mode.packing_divisor() as u32;
+ let weights_stride = model_dim / packing_divisor;
+ let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize)
+ .map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) * 16))
+ .collect();
+
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ let scales: Vec<T> = (0..vocab_size as usize * num_groups as usize)
+ .map(|i| T::from(0.5 + (i as f32 * 0.1).sin() * 0.3).unwrap())
+ .collect();
+ let zero_point_stride = num_groups.div_ceil(packing_divisor);
+ let zero_points: Vec<u8> = (0..vocab_size as usize * zero_point_stride as usize)
+ .map(|i| ((i % 16) as u8) | ((((i + 5) % 16) as u8) * 16))
+ .collect();
+
+ let input = Input {
+ token_ids,
+ weights: weights.into_boxed_slice(),
+ scales: scales.into_boxed_slice(),
+ zero_points: Some(zero_points.into_boxed_slice()),
+ biases: None,
+ batch_size,
+ vocab_size,
+ model_dim,
+ input_scale,
+ group_size,
+ quant_mode,
+ quant_method: QuantizationMethod::ScaleZeroPoint,
};
let expected = get_output::<T, Cpu>(&input);
@@ -93,13 +141,15 @@ fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
token_ids,
weights: weights.into_boxed_slice(),
scales: scales.into_boxed_slice(),
- biases: biases.into_boxed_slice(),
+ zero_points: None,
+ biases: Some(biases.into_boxed_slice()),
batch_size,
vocab_size,
model_dim,
input_scale,
group_size,
quant_mode: QuantizationMode::U8,
+ quant_method: QuantizationMethod::ScaleBias,
};
let expected = get_output::<T, Cpu>(&input);
@@ -114,7 +164,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
T::data_type(),
input.group_size,
input.quant_mode,
- QuantizationMethod::ScaleBias,
+ input.quant_method,
false,
)
.expect("Failed to create QuantizedEmbeddingLookupKernel");
@@ -126,7 +176,18 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let token_ids_array = context.create_array_from(&[input.batch_size as usize], &input.token_ids);
let weights_array = context.create_array_from(&[input.vocab_size as usize, weights_stride], &input.weights);
let scales_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.scales);
- let biases_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.biases);
+ let zero_point_stride = match input.quant_mode {
+ QuantizationMode::U4 => num_groups.div_ceil(2),
+ QuantizationMode::I8 | QuantizationMode::U8 => num_groups,
+ };
+ let zero_points_array = input
+ .zero_points
+ .as_ref()
+ .map(|zero_points| context.create_array_from(&[input.vocab_size as usize, zero_point_stride], zero_points));
+ let biases_array = input
+ .biases
+ .as_ref()
+ .map(|biases| context.create_array_from(&[input.vocab_size as usize, num_groups], biases));
let mut output = context
.create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type())
.into_allocation();
@@ -136,8 +197,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
token_ids_array.allocation(),
weights_array.allocation(),
scales_array.allocation(),
- None::<&backend_uzu::backends::common::Allocation<B>>,
- Some(biases_array.allocation()),
+ zero_points_array.as_ref().map(|array| array.allocation()),
+ biases_array.as_ref().map(|array| array.allocation()),
&mut output,
None::<&backend_uzu::backends::common::Allocation<B>>,
input.batch_size,
@@ -151,6 +212,43 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
crate::common::helpers::allocation_to_vec(&output)
}
+fn test_zero_point_group16<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.5f32
+ } else {
+ 1e-4
+ };
+ let (input, expected) = get_test_data_zero_point_group16::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!(
+ "QuantizedEmbeddingLookup ScaleZeroPoint group16 test failed for backend {}",
+ std::any::type_name::<B>()
+ ),
+ );
+ });
+}
+
+#[cfg(metal_backend)]
+fn test_zero_point_group16_hadamard_constructor<T: ArrayElement + Float + Debug + Display>() {
+ use backend_uzu::backends::metal::Metal;
+
+ let context = <Metal as Backend>::Context::new().expect("Metal context");
+ <<Metal as Backend>::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ &context,
+ T::data_type(),
+ 16,
+ QuantizationMode::U4,
+ QuantizationMethod::ScaleZeroPoint,
+ true,
+ )
+ .expect("QuantizedEmbeddingLookupKernel group16 zero-point hadamard");
+}
+
fn test_quant_mode<T: ArrayElement + Float + Debug + Display>(quant_mode: QuantizationMode) {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.5f32
@@ -203,6 +301,17 @@ fn test_uint4_bf16() {
test_quant_mode::<bf16>(QuantizationMode::U4);
}
+#[uzu_test]
+fn test_uint4_zero_point_group16_bf16() {
+ test_zero_point_group16::<bf16>();
+}
+
+#[cfg(metal_backend)]
+#[uzu_test]
+fn test_uint4_zero_point_group16_hadamard_bf16_constructor() {
+ test_zero_point_group16_hadamard_constructor::<bf16>();
+}
+
// INT8 tests
#[uzu_test]
fn test_int8_f32() {
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
index 66b5f3a41..5db7b81cc 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
@@ -90,6 +90,7 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(
}
#[rstest]
+#[case::gs16_4bit_zp_prefill ( 64, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs32_4bit_mlx_prefill ( 64, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
#[case::gs64_4bit_mlx_prefill ( 64, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
#[case::gs128_4bit_mlx_prefill( 64, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]
@@ -97,6 +98,7 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(
#[case::gs64_8bit_zp_prefill ( 64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
#[case::gs128_8bit_zp_prefill ( 64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
#[case::gs32_4bit_mlx_decode ( 8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::gs16_4bit_zp_decode ( 8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_4bit_zp_decode ( 8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs32_unaligned_n ( 64, 256, 96, 32, 4, QuantizationMethod::ScaleBias)]
fn parity_bf16(
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index a5d7a53e3..41232cb36 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -281,15 +281,19 @@ macro_rules! qmv_fast_test {
};
}
+qmv_fast_test!(test_gs16_4bit_zp, gs = 16, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs16_8bit_zp, gs = 16, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
+qmv_fast_test!(test_gs16_4bit_mlx, gs = 16, bits = 4, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs32_4bit_mlx, gs = 32, bits = 4, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs64_4bit_mlx, gs = 64, bits = 4, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs128_4bit_mlx, gs = 128, bits = 4, format = QuantizationMethod::ScaleBias);
+qmv_fast_test!(test_gs16_8bit_mlx, gs = 16, bits = 8, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, format = QuantizationMethod::ScaleBias);
qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, format = QuantizationMethod::ScaleBias);
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
index f80e165d9..d38d7fdcf 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
@@ -303,6 +303,13 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>(
}
// ── 4-bit, zero points ─────────────────────────────────────────────────────
+#[uzu_test]
+fn test_gs16_4bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(16, 4, QuantizationMethod::ScaleZeroPoint);
+ })
+}
+
#[uzu_test]
fn test_gs32_4bit_zp() {
for_each_float_type!(|F| {
@@ -326,6 +333,13 @@ fn test_gs128_4bit_zp() {
// ── 8-bit, zero points ─────────────────────────────────────────────────────
+#[uzu_test]
+fn test_gs16_8bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(16, 8, QuantizationMethod::ScaleZeroPoint);
+ })
+}
+
#[uzu_test]
fn test_gs32_8bit_zp() {
for_each_float_type!(|F| {
@@ -349,6 +363,13 @@ fn test_gs128_8bit_zp() {
// ── 4-bit, mlx quant ───────────────────────────────────────────────────────
+#[uzu_test]
+fn test_gs16_4bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(16, 4, QuantizationMethod::ScaleBias);
+ })
+}
+
#[uzu_test]
fn test_gs32_4bit_mlx() {
for_each_float_type!(|F| {
@@ -372,6 +393,13 @@ fn test_gs128_4bit_mlx() {
// ── 8-bit, mlx quant ───────────────────────────────────────────────────────
+#[uzu_test]
+fn test_gs16_8bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(16, 8, QuantizationMethod::ScaleBias);
+ })
+}
+
#[uzu_test]
fn test_gs32_8bit_mlx() {
for_each_float_type!(|F| {