PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix auto_scheme gguf format wrong bits if use tiny model (#1953)
Heng Guob1a6baf9a319
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex a19a40115..dbee03e46 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -1197,21 +1197,22 @@ def _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_pa# lm_head option restriction for DP ## lm_head is critical — its quantization error goes directly into ## logits with no subsequent LayerNorm dampening. Instead of removing #- # it from DP, we restrict its candidate options so the DP can only #- # assign it >= 6 bit schemes, then let DP globally optimize. #+ # it from DP, we bias its candidate options toward higher precision #+ # or lower loss, then relax the restriction if it cannot fit budget. ## ## Rules (only if user hasn't already fixed it): #- # 1. No option has bits >= 6 → keep all options (DP picks best)#- # 2. Exactly one option bits >= 6 → restrict to only that option #+ # 1. No option has bits >= 6 → prefer lowest-loss available #+ # 2. Exactly one option bits >= 6 → prefer that high-bit option ## 3. Multiple options bits >= 6: ## - target_bits > 6 → restrict to only the highest-bit option ## - target_bits <= 6 → keep all >=6 options, let DP decide #+ # Any restriction above is relaxed if it makes the budget infeasible.## ------------------------------------------------------------------ #high_bit_indices = [i for i in range(len(schemes)) if _get_scheme_bits(schemes[i]) >= 6]if len(high_bit_indices) == 0:- # Rule 1: no option >= 6 bit → restrict to the lowest-loss scheme+ # Rule 1: no option >= 6 bit → keep the lowest-loss scheme if budget allows.allowed_indices = {sorted_indices[0]} if sorted_indices else Noneelif len(high_bit_indices) == 1:# Rule 2: exactly one >= 6 bit option → restrict to it@@ -1633,9 +1634,6 @@ def _gen_layer_config(layer_bits, _ = compute_layer_bits(m, auto_scheme.ignore_scale_zp_bits)target_params_cnt -= layer_bits- head_name = get_lm_head_name(model)- if head_name is not None and (head_name not in fixed_layer_scheme and head_name in quant_layer_names):- _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores)# As only a small amount of calibration data is used and embedding layers are inherently sparse,# we cannot obtain a reliable score.if not_fixed_embedding_layers_names:@@ -1650,6 +1648,10 @@ def _gen_layer_config(layer_bits, _ = compute_layer_bits(embedding_layer, auto_scheme.ignore_scale_zp_bits)target_params_cnt -= layer_bits+ head_name = get_lm_head_name(model)+ if head_name is not None and (head_name not in fixed_layer_scheme and head_name in quant_layer_names):+ _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores)+if target_params_cnt <= 0:raise ValueError("Avg bits is too small")diff --git a/test/test_cpu/schemes/test_auto_scheme_internal.py b/test/test_cpu/schemes/test_auto_scheme_internal.pynew file mode 100644index 000000000..35a6e69c5--- /dev/null+++ b/test/test_cpu/schemes/test_auto_scheme_internal.py@@ -0,0 +1,55 @@+from auto_round.auto_scheme.delta_loss import _apply_head_trick+from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+++def test_apply_head_trick_keeps_lowest_loss_candidate_when_budget_allows():+ schemes = AutoScheme(avg_bits=3, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M")).options+ total_scores = {+ "lm_head": [+ [0, 1244659712, 41.0, ["lm_head"]],+ [1, 2489319424, 11.9, ["lm_head"]],+ ],+ "model.layers.0.mlp.down_proj": [+ [0, 100, 10.0, ["model.layers.0.mlp.down_proj"]],+ [1, 200, 5.0, ["model.layers.0.mlp.down_proj"]],+ ],+ }++ _apply_head_trick(+ head_name="lm_head",+ schemes=schemes,+ sorted_indices=[1, 0],+ target_bits=3,+ target_params_cnt=4891670016,+ total_scores=total_scores,+ )++ assert total_scores["lm_head"] == [[1, 2489319424, 11.9, ["lm_head"]]]+++def test_apply_head_trick_relaxes_lowest_loss_candidate_when_budget_is_tight():+ schemes = AutoScheme(avg_bits=3, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M")).options+ total_scores = {+ "lm_head": [+ [0, 1244659712, 41.0, ["lm_head"]],+ [1, 2489319424, 11.9, ["lm_head"]],+ ],+ "model.layers.0.mlp.down_proj": [+ [0, 100, 10.0, ["model.layers.0.mlp.down_proj"]],+ [1, 200, 5.0, ["model.layers.0.mlp.down_proj"]],+ ],+ }++ _apply_head_trick(+ head_name="lm_head",+ schemes=schemes,+ sorted_indices=[1, 0],+ target_bits=3,+ target_params_cnt=1244659812,+ total_scores=total_scores,+ )++ assert total_scores["lm_head"] == [+ [0, 1244659712, 41.0, ["lm_head"]],+ [1, 2489319424, 11.9, ["lm_head"]],+ ]