PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: guard zero-division in GGUF quant kernels to avoid NaN block scales (#1909)
Entrpi4a60b6fe5044
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex b1b60dc6b..bff88da2f 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -194,7 +194,9 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):# Compute sumlx and suml2 using the pre-allocated L buffersumlx = (w * chunk * L).sum(dim=-1)suml2 = (w * L * L).sum(dim=-1)- scales = sumlx / suml2+ # Guard the all-zero group case (suml2 == 0): 0/0 would produce a NaN+ # scale that ends up as a NaN fp16 `d` in the exported GGUF block.+ scales = sumlx * get_reciprocal(suml2)if return_early:iscales_inv = (1 / iscales).reshape(iscales.shape[:2])@@ -488,7 +490,7 @@ def make_qp_quants(nmax, data, quant_weights):if n_changed == 0:break- return sumlx / suml2, L+ return sumlx * get_reciprocal(suml2), L@register_qtype("bf16")diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex 56daf87f9..bef7f7712 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -344,3 +344,33 @@ class TestGGUF:assert tensor_types["blk.0.attn_k.weight"] == "Q4_K"assert tensor_types["blk.0.ffn_down_exps.weight"] == "Q5_0"assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"+++class TestGGUFZeroBlock:+ """All-zero blocks (e.g. padded/unused vocab rows in an embedding tensor) must+ quantize with scale d=0.0, not NaN. A single NaN fp16 `d` in an exported GGUF+ makes llama.cpp return NaN logits for any batch touching those rows."""++ def test_q6_k_all_zero_block_no_nan(self):+ from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq++ tensor = torch.zeros(2, 256)+ tensor[1] = torch.randn(256)+ qdq, scales, _ = quant_tensor_gguf_sym_dq(tensor, bits=6, scale_dtype=torch.float32)+ assert not torch.isnan(scales["scale"]).any(), "Q6_K sub-scales contain NaN for all-zero block"+ assert not torch.isnan(scales["d_scale"]).any(), "Q6_K d_scale contains NaN for all-zero block"+ assert not torch.isnan(qdq).any()+ assert torch.equal(qdq[0], torch.zeros(256)), "all-zero block must reconstruct exactly to zeros"++ def test_q6_k_all_zero_block_packs_zero_d(self):+ import numpy as np++ from auto_round.export.export_to_gguf.packing import ggml_quant++ tensor = torch.zeros(2, 256)+ tensor[1] = torch.randn(256)+ packed = ggml_quant(tensor, "q6_k", device="cpu")+ # Q6_K block layout: [ql(128), qh(64), scales(16), d(fp16)]+ d = np.ascontiguousarray(packed.reshape(2, 210)[:, -2:]).view(np.float16)+ assert not np.isnan(d).any(), f"packed Q6_K d scales contain NaN: {d}"+ assert d[0] == 0.0