Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: keep 0 in the range of GGUF asymmetric K-quant groups (#2438)

Raashish Aggarwal5a113d1cb416
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index f111ef430..badeeda03 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -722,7 +722,9 @@ def iterative_wls_quant_search_chunk(
quant_data = torch.empty_like(chunk)
diff = torch.empty_like(chunk)
- rmin = torch.min(chunk, dim=1, keepdim=True)[0]
+ # Like llama.cpp's make_qkx2_quants, keep 0 in the range so the min is never
+ # positive: the packed K-quant formats can only store non-negative mins (-rmin).
+ rmin = torch.min(chunk, dim=1, keepdim=True)[0].clamp_(max=0)
rmax = torch.max(chunk, dim=1, keepdim=True)[0]
sum_w = torch.sum(chunk_weights, dim=1, keepdim=True)
sum_x = torch.sum(chunk_weights * chunk, dim=1, keepdim=True)
diff --git a/test/unit/test_cpu/export/test_gguf_format.py b/test/unit/test_cpu/export/test_gguf_format.py
index 9929ba594..d0b9fb4d2 100644
--- a/test/unit/test_cpu/export/test_gguf_format.py
+++ b/test/unit/test_cpu/export/test_gguf_format.py
@@ -590,3 +590,29 @@ class TestGGUFZeroBlock:
d = np.ascontiguousarray(packed.reshape(2, 210)[:, -2:]).view(np.float16)
assert not np.isnan(d).any(), f"packed Q6_K d scales contain NaN: {d}"
assert d[0] == 0.0
+
+
+class TestGGUFPositiveGroup:
+ """K-quant groups whose values are all positive must keep 0 in the range (as llama.cpp
+ does), because Q2_K/Q4_K/Q5_K can only store non-negative mins."""
+
+ @pytest.mark.parametrize("qtype, bits", [("q2_k", 2), ("q4_k", 4)])
+ def test_all_positive_groups(self, qtype, bits):
+ gguf_quants = pytest.importorskip("gguf.quants")
+ from gguf import GGMLQuantizationType
+
+ from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq
+ from auto_round.export.export_to_gguf.packing import ggml_quant
+
+ torch.manual_seed(0)
+ tensor = torch.randn(2, 256)
+ tensor[1] = torch.rand(256) + 0.5
+
+ qdq, _, _ = quant_tensor_gguf_asym_dq(tensor.clone(), bits=bits, scale_dtype=torch.float32)
+ # With 0 kept in the range the MSE is about 0.014 (2-bit) or 0.0007 (4-bit);
+ # with a positive min that cannot be stored it is above 0.06.
+ assert ((qdq[1] - tensor[1]) ** 2).mean() < 0.03
+
+ packed = ggml_quant(tensor.clone(), qtype, device="cpu")
+ dequantized = gguf_quants.dequantize(packed, GGMLQuantizationType[qtype.upper()])
+ torch.testing.assert_close(torch.from_numpy(dequantized).reshape(tensor.shape), qdq, atol=1e-2, rtol=0)