Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Compare gated activation quantization by dequantized error (#891)

CCYehf0f2bf8639b4
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
index 26e5a68d2..b131a1851 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
@@ -116,7 +116,7 @@ fn test_gated_act_mul_interleaved_silu_bf16() {
interleaved_test::<bf16>(ActivationType::SILU);
}
-fn run_nibble_grouped_quantized<B: Backend>(input: &InterleavedInput<bf16>) -> Vec<i8> {
+fn run_nibble_grouped_quantized<B: Backend>(input: &InterleavedInput<bf16>) -> (Vec<i8>, Vec<f32>) {
let context = B::Context::new().expect("create context");
let act_operand = create_buffer_with_data::<B, bf16>(&context, &input.fused_up);
let factors = create_buffer_with_data::<B, i32>(&context, &input.hadamard_factors);
@@ -143,15 +143,29 @@ fn run_nibble_grouped_quantized<B: Backend>(input: &InterleavedInput<bf16>) -> V
&mut command_buffer,
);
command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- buffer_to_vec(&values)
+ (buffer_to_vec(&values), buffer_to_vec(&scales))
}
#[uzu_test]
fn test_gated_act_mul_nibble_grouped_quantization_matches_cpu() {
let input = interleaved_input::<bf16>(ActivationType::SILU);
- let expected_values = run_nibble_grouped_quantized::<Cpu>(&input);
+ let (expected_values, expected_scales) = run_nibble_grouped_quantized::<Cpu>(&input);
for_each_non_cpu_backend!(|B| {
- assert_eq!(run_nibble_grouped_quantized::<B>(&input), expected_values, "gated activation bytes differ");
+ let (actual_values, actual_scales) = run_nibble_grouped_quantized::<B>(&input);
+ let group_size = input.gated_dim as usize;
+ // Boundary rounding can shift one code; normalize dequantized error by group peak.
+ for (index, (&actual, &expected)) in actual_values.iter().zip(&expected_values).enumerate() {
+ let group = index / group_size;
+ let (actual_scale, expected_scale) = (actual_scales[group], expected_scales[group]);
+ if index % group_size == 0 {
+ let scale_error = (actual_scale - expected_scale).abs() / expected_scale;
+ assert!(scale_error < 1e-3, "scale {group}: {actual_scale} != {expected_scale}");
+ }
+ let actual = actual as f32 * actual_scale;
+ let expected = expected as f32 * expected_scale;
+ let error = (actual - expected).abs() / (127.0 * actual_scale.max(expected_scale));
+ assert!(error <= 1e-2, "gated activation value {index}: {actual} != {expected} (error {error})");
+ }
});
}