Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: correct Q5_K GGUF packing (#2437)

Raashish Aggarwalb73d89d3b66f
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index 4186aa4a2..e0f520cb0 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -943,7 +943,7 @@ def q5_k_quant_block(
from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq
blocks.reshape(blocks.shape[0], -1)
- blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)
+ blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=5, scale_dtype=torch.float32, imatrix=imatrix)
scales, d_scale = scales["scale"], scales["d_scale"]
mins, d_wmin = mins["wmin"], mins["d_wmin"]
@@ -975,7 +975,7 @@ def q5_k_quant_block(
output_scale[:, :4] |= (q_scales[:, 4:] >> 4) << 6
output_scale[:, 4:8] |= (q_mins[:, 4:] >> 4) << 6
- output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)
+ output_qs = (all_L[:, ::2] & 0xF) | (all_L[:, 1::2] << 4)
output_qh = all_L >> 4 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)
output_qh = np.bitwise_or.reduce(output_qh.cpu().numpy(), axis=1, dtype=np.uint8).astype(
np.uint8
diff --git a/test/unit/test_cpu/export/test_gguf_format.py b/test/unit/test_cpu/export/test_gguf_format.py
index d0b9fb4d2..76cd84b24 100644
--- a/test/unit/test_cpu/export/test_gguf_format.py
+++ b/test/unit/test_cpu/export/test_gguf_format.py
@@ -562,6 +562,47 @@ class TestGGUF:
assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"
+class TestGGUFQ5K:
+ """Q5_K blocks must round-trip through gguf's reference dequantizer."""
+
+ def _dequantize(self, packed, qtype, shape):
+ gguf_quants = pytest.importorskip("gguf.quants")
+ from gguf import GGMLQuantizationType
+
+ return torch.from_numpy(gguf_quants.dequantize(packed, GGMLQuantizationType[qtype.upper()])).reshape(shape)
+
+ def test_q5_k_pack_matches_fake_quant(self):
+ from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq
+ from auto_round.export.export_to_gguf.packing import ggml_quant
+
+ torch.manual_seed(0)
+ tensor = torch.randn(4, 256)
+ qdq, scales, mins = quant_tensor_gguf_asym_dq(tensor, bits=5, scale_dtype=torch.float32)
+ packed = ggml_quant(
+ qdq.clone(),
+ "q5_k",
+ scale=scales["scale"],
+ wmin=mins["wmin"],
+ d_scale=scales["d_scale"],
+ d_wmin=mins["d_wmin"],
+ device="cpu",
+ )
+ dequantized = self._dequantize(packed, "q5_k", tensor.shape)
+ torch.testing.assert_close(dequantized, qdq, atol=1e-2, rtol=0)
+
+ def test_q5_k_pack_without_stored_params_uses_5_bits(self):
+ from auto_round.export.export_to_gguf.packing import ggml_quant
+
+ torch.manual_seed(0)
+ tensor = torch.randn(4, 256)
+ mse = {}
+ for qtype in ("q4_k", "q5_k"):
+ packed = ggml_quant(tensor.clone(), qtype, device="cpu")
+ mse[qtype] = ((self._dequantize(packed, qtype, tensor.shape) - tensor) ** 2).mean().item()
+ # One extra bit should roughly quarter the error.
+ assert mse["q5_k"] < mse["q4_k"] / 2, mse
+
+
class TestGGUFZeroBlock:
"""All-zero blocks (e.g. padded/unused vocab rows in an embedding tensor) must
quantize with scale d=0.0, not NaN. A single NaN fp16 `d` in an exported GGUF