Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: guard zero-division in GGUF quant kernels to avoid NaN block scales (#1909)

Entrpi4a60b6fe5044
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index b1b60dc6b..bff88da2f 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -194,7 +194,9 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):
# Compute sumlx and suml2 using the pre-allocated L buffer
sumlx = (w * chunk * L).sum(dim=-1)
suml2 = (w * L * L).sum(dim=-1)
- scales = sumlx / suml2
+ # Guard the all-zero group case (suml2 == 0): 0/0 would produce a NaN
+ # scale that ends up as a NaN fp16 `d` in the exported GGUF block.
+ scales = sumlx * get_reciprocal(suml2)
if return_early:
iscales_inv = (1 / iscales).reshape(iscales.shape[:2])
@@ -488,7 +490,7 @@ def make_qp_quants(nmax, data, quant_weights):
if n_changed == 0:
break
- return sumlx / suml2, L
+ return sumlx * get_reciprocal(suml2), L
@register_qtype("bf16")
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index 56daf87f9..bef7f7712 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -344,3 +344,33 @@ class TestGGUF:
assert tensor_types["blk.0.attn_k.weight"] == "Q4_K"
assert tensor_types["blk.0.ffn_down_exps.weight"] == "Q5_0"
assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"
+
+
+class TestGGUFZeroBlock:
+ """All-zero blocks (e.g. padded/unused vocab rows in an embedding tensor) must
+ quantize with scale d=0.0, not NaN. A single NaN fp16 `d` in an exported GGUF
+ makes llama.cpp return NaN logits for any batch touching those rows."""
+
+ def test_q6_k_all_zero_block_no_nan(self):
+ from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq
+
+ tensor = torch.zeros(2, 256)
+ tensor[1] = torch.randn(256)
+ qdq, scales, _ = quant_tensor_gguf_sym_dq(tensor, bits=6, scale_dtype=torch.float32)
+ assert not torch.isnan(scales["scale"]).any(), "Q6_K sub-scales contain NaN for all-zero block"
+ assert not torch.isnan(scales["d_scale"]).any(), "Q6_K d_scale contains NaN for all-zero block"
+ assert not torch.isnan(qdq).any()
+ assert torch.equal(qdq[0], torch.zeros(256)), "all-zero block must reconstruct exactly to zeros"
+
+ def test_q6_k_all_zero_block_packs_zero_d(self):
+ import numpy as np
+
+ from auto_round.export.export_to_gguf.packing import ggml_quant
+
+ tensor = torch.zeros(2, 256)
+ tensor[1] = torch.randn(256)
+ packed = ggml_quant(tensor, "q6_k", device="cpu")
+ # Q6_K block layout: [ql(128), qh(64), scales(16), d(fp16)]
+ d = np.ascontiguousarray(packed.reshape(2, 210)[:, -2:]).view(np.float16)
+ assert not np.isnan(d).any(), f"packed Q6_K d scales contain NaN: {d}"
+ assert d[0] == 0.0