PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: keep make_q3_quants sums in sync with accepted levels (#2439)
Raashish Aggarwal7059a8b1e1a1
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex e0f520cb0..7a38e7262 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -375,14 +375,14 @@ def make_q3_quants(data, bits, do_rmse=False, v=0):sl2[tmp_replace_idx] += w_tmp[tmp_replace_idx] * new_L[tmp_replace_idx] * new_L[tmp_replace_idx]# Further check condition for improvement- replace_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)+ replace_idx &= tmp_replace_idx & (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)# Update L in-placeL_tmp[replace_idx] = new_L[replace_idx]- # Update global sums- sumlx = slx- suml2 = sl2+ # Update global sums only where the new value was accepted+ sumlx = torch.where(replace_idx, slx, sumlx)+ suml2 = torch.where(replace_idx, sl2, suml2)# Compute final scale and return quantized Lreturn sumlx * get_reciprocal(suml2), L.to(torch.uint8)diff --git a/test/unit/test_cpu/export/test_gguf_format.py b/test/unit/test_cpu/export/test_gguf_format.pyindex 76cd84b24..d0cd74423 100644--- a/test/unit/test_cpu/export/test_gguf_format.py+++ b/test/unit/test_cpu/export/test_gguf_format.py@@ -103,6 +103,20 @@ def test_update_module_applies_replacements_for_gguf(monkeypatch):assert calls == [(model, {"gguf_export": True})]+def test_make_q3_quants_rmse_scale_matches_levels():+ """The refined Q3_K sub-block scale must be the least-squares fit for the returned levels,+ sum(w*x*L) / sum(w*L*L) with w = x**2, as in llama.cpp's make_q3_quants."""+ from auto_round.export.export_to_gguf.packing import make_q3_quants++ torch.manual_seed(0)+ data = torch.randn(4, 16, 16)+ scales, levels = make_q3_quants(data.clone(), bits=3, do_rmse=True)+ levels = levels.to(torch.int8).float()+ weights = data**2+ expected = (weights * data * levels).sum(-1) / (weights * levels * levels).sum(-1)+ torch.testing.assert_close(scales, expected)++class TestGGUF:@classmethod