PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: keep 0 in the range of GGUF asymmetric K-quant groups (#2438)
Raashish Aggarwal5a113d1cb416
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex f111ef430..badeeda03 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -722,7 +722,9 @@ def iterative_wls_quant_search_chunk(quant_data = torch.empty_like(chunk)diff = torch.empty_like(chunk)- rmin = torch.min(chunk, dim=1, keepdim=True)[0]+ # Like llama.cpp's make_qkx2_quants, keep 0 in the range so the min is never+ # positive: the packed K-quant formats can only store non-negative mins (-rmin).+ rmin = torch.min(chunk, dim=1, keepdim=True)[0].clamp_(max=0)rmax = torch.max(chunk, dim=1, keepdim=True)[0]sum_w = torch.sum(chunk_weights, dim=1, keepdim=True)sum_x = torch.sum(chunk_weights * chunk, dim=1, keepdim=True)diff --git a/test/unit/test_cpu/export/test_gguf_format.py b/test/unit/test_cpu/export/test_gguf_format.pyindex 9929ba594..d0b9fb4d2 100644--- a/test/unit/test_cpu/export/test_gguf_format.py+++ b/test/unit/test_cpu/export/test_gguf_format.py@@ -590,3 +590,29 @@ class TestGGUFZeroBlock:d = np.ascontiguousarray(packed.reshape(2, 210)[:, -2:]).view(np.float16)assert not np.isnan(d).any(), f"packed Q6_K d scales contain NaN: {d}"assert d[0] == 0.0+++class TestGGUFPositiveGroup:+ """K-quant groups whose values are all positive must keep 0 in the range (as llama.cpp+ does), because Q2_K/Q4_K/Q5_K can only store non-negative mins."""++ @pytest.mark.parametrize("qtype, bits", [("q2_k", 2), ("q4_k", 4)])+ def test_all_positive_groups(self, qtype, bits):+ gguf_quants = pytest.importorskip("gguf.quants")+ from gguf import GGMLQuantizationType++ from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+ from auto_round.export.export_to_gguf.packing import ggml_quant++ torch.manual_seed(0)+ tensor = torch.randn(2, 256)+ tensor[1] = torch.rand(256) + 0.5++ qdq, _, _ = quant_tensor_gguf_asym_dq(tensor.clone(), bits=bits, scale_dtype=torch.float32)+ # With 0 kept in the range the MSE is about 0.014 (2-bit) or 0.0007 (4-bit);+ # with a positive min that cannot be stored it is above 0.06.+ assert ((qdq[1] - tensor[1]) ** 2).mean() < 0.03++ packed = ggml_quant(tensor.clone(), qtype, device="cpu")+ dequantized = gguf_quants.dequantize(packed, GGMLQuantizationType[qtype.upper()])+ torch.testing.assert_close(torch.from_numpy(dequantized).reshape(tensor.shape), qdq, atol=1e-2, rtol=0)