Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: keep make_q3_quants sums in sync with accepted levels (#2439)

Raashish Aggarwal7059a8b1e1a1
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index e0f520cb0..7a38e7262 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -375,14 +375,14 @@ def make_q3_quants(data, bits, do_rmse=False, v=0):
sl2[tmp_replace_idx] += w_tmp[tmp_replace_idx] * new_L[tmp_replace_idx] * new_L[tmp_replace_idx]
# Further check condition for improvement
- replace_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)
+ replace_idx &= tmp_replace_idx & (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)
# Update L in-place
L_tmp[replace_idx] = new_L[replace_idx]
- # Update global sums
- sumlx = slx
- suml2 = sl2
+ # Update global sums only where the new value was accepted
+ sumlx = torch.where(replace_idx, slx, sumlx)
+ suml2 = torch.where(replace_idx, sl2, suml2)
# Compute final scale and return quantized L
return sumlx * get_reciprocal(suml2), L.to(torch.uint8)
diff --git a/test/unit/test_cpu/export/test_gguf_format.py b/test/unit/test_cpu/export/test_gguf_format.py
index 76cd84b24..d0cd74423 100644
--- a/test/unit/test_cpu/export/test_gguf_format.py
+++ b/test/unit/test_cpu/export/test_gguf_format.py
@@ -103,6 +103,20 @@ def test_update_module_applies_replacements_for_gguf(monkeypatch):
assert calls == [(model, {"gguf_export": True})]
+def test_make_q3_quants_rmse_scale_matches_levels():
+ """The refined Q3_K sub-block scale must be the least-squares fit for the returned levels,
+ sum(w*x*L) / sum(w*L*L) with w = x**2, as in llama.cpp's make_q3_quants."""
+ from auto_round.export.export_to_gguf.packing import make_q3_quants
+
+ torch.manual_seed(0)
+ data = torch.randn(4, 16, 16)
+ scales, levels = make_q3_quants(data.clone(), bits=3, do_rmse=True)
+ levels = levels.to(torch.int8).float()
+ weights = data**2
+ expected = (weights * data * levels).sum(-1) / (weights * levels * levels).sum(-1)
+ torch.testing.assert_close(scales, expected)
+
+
class TestGGUF:
@classmethod