PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: give attn_v of LLM_TYPE_70B models Q5_K like llama.cpp (#2443)
Raashish Aggarwalae21ef9bb58c
diff --git a/auto_round/export/export_to_gguf/gguf_dtype.py b/auto_round/export/export_to_gguf/gguf_dtype.pyindex 6c173d3de..0da290c76 100644--- a/auto_round/export/export_to_gguf/gguf_dtype.py+++ b/auto_round/export/export_to_gguf/gguf_dtype.py@@ -92,6 +92,28 @@ def _use_more_bits(i_layer: int, n_layer: int) -> bool:return i_layer < n_layer // 8 or i_layer >= 7 * n_layer // 8 or (i_layer - n_layer // 8) % 3 == 2+# Number of transformer blocks for which llama.cpp sets LLM_TYPE_70B, per architecture+# (see the per-architecture load_hparams in llama.cpp's src/models/*.cpp).+# Keyed by gguf.MODEL_ARCH member name, so that importing this module does not import gguf.+_LLM_TYPE_70B_BLOCK_COUNT = {"LLAMA": 80, "QWEN2": 80, "DECI": 80, "OLMO": 80, "JAIS2": 68}+++def _is_llm_type_70b(model_arch, hparams: dict) -> bool:+ """Whether llama.cpp would classify the model as LLM_TYPE_70B."""+ arch_name = getattr(model_arch, "name", None)+ n_layer = hparams.get("num_hidden_layers", hparams.get("n_layer", hparams.get("num_layers")))+ if arch_name not in _LLM_TYPE_70B_BLOCK_COUNT or n_layer is None:+ return False+ if int(n_layer) != _LLM_TYPE_70B_BLOCK_COUNT[arch_name]:+ return False+ if arch_name == "LLAMA":+ # 80-layer LLaMA without grouped-query attention is LLM_TYPE_65B+ n_head = hparams.get("num_attention_heads", hparams.get("n_head"))+ n_head_kv = hparams.get("num_key_value_heads", hparams.get("n_head_kv", n_head))+ return n_head != n_head_kv+ return True++def _get_layer_id(name: str, fallback: int) -> int:parts = name.split(".")if len(parts) > 1 and parts[0] == "blk" and parts[1].isdigit():@@ -252,6 +274,12 @@ class GGUFDTypeSelector:qtype = gguf.GGMLQuantizationType.Q6_Kelif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S and self.i_attention_wv < 4:qtype = gguf.GGMLQuantizationType.Q5_K+ if _is_llm_type_70b(self.model_arch, self.hparams) and qtype in (+ gguf.GGMLQuantizationType.Q3_K,+ gguf.GGMLQuantizationType.Q4_K,+ ):+ # llama.cpp: attn_v is 8x smaller than attn_q in these models, so more bits are cheap+ qtype = gguf.GGMLQuantizationType.Q5_Kif n_expert == 8:qtype = gguf.GGMLQuantizationType.Q8_0self.i_attention_wv += 1diff --git a/test/unit/common/export/test_gguf_dtype_helpers.py b/test/unit/common/export/test_gguf_dtype_helpers.pyindex f8606cb24..f2daa635e 100644--- a/test/unit/common/export/test_gguf_dtype_helpers.py+++ b/test/unit/common/export/test_gguf_dtype_helpers.py@@ -322,3 +322,44 @@ class TestGGUFDTypeSelector:)assert selector.select_qtype("blk.0.ffn_down.weight", n_dims=2) == gguf.GGMLQuantizationType.Q4_K+++class TestGGUFDTypeSelectorLLMType70B:+ """llama.cpp gives attn_v of LLM_TYPE_70B models Q5_K instead of Q3_K/Q4_K."""++ @staticmethod+ def _attn_v_qtypes(model_arch, hparams, gguf_format):+ from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector, gguf_format_to_ftype++ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype(gguf_format), model_arch)+ n_layer = hparams["num_hidden_layers"]+ return [selector.select_qtype(f"blk.{i}.attn_v.weight", n_dims=2) for i in range(n_layer)]++ def test_llama_70b_attn_v_uses_q5_k(self):+ hparams = {"num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8}+ qtypes = self._attn_v_qtypes(gguf.MODEL_ARCH.LLAMA, hparams, "gguf:q4_k_m")+ # use_more_bits layers stay Q6_K, every other layer is bumped from Q4_K to Q5_K+ assert set(qtypes) == {gguf.GGMLQuantizationType.Q5_K, gguf.GGMLQuantizationType.Q6_K}+ assert gguf.GGMLQuantizationType.Q5_K in qtypes++ qtypes = self._attn_v_qtypes(gguf.MODEL_ARCH.LLAMA, hparams, "gguf:q3_k_s")+ assert set(qtypes) == {gguf.GGMLQuantizationType.Q5_K}++ def test_qwen2_80_layers_attn_v_uses_q5_k(self):+ hparams = {"num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8}+ qtypes = self._attn_v_qtypes(gguf.MODEL_ARCH.QWEN2, hparams, "gguf:q4_k_s")+ assert set(qtypes) == {gguf.GGMLQuantizationType.Q5_K}++ @pytest.mark.parametrize(+ "model_arch, hparams",+ [+ # 80-layer LLaMA without GQA is LLM_TYPE_65B in llama.cpp+ (gguf.MODEL_ARCH.LLAMA, {"num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 64}),+ (gguf.MODEL_ARCH.LLAMA, {"num_hidden_layers": 32, "num_attention_heads": 32, "num_key_value_heads": 8}),+ (None, {"num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8}),+ ],+ )+ def test_other_models_keep_q4_k(self, model_arch, hparams):+ qtypes = self._attn_v_qtypes(model_arch, hparams, "gguf:q4_k_s")+ assert gguf.GGMLQuantizationType.Q4_K in qtypes+ assert gguf.GGMLQuantizationType.Q5_K not in qtypes[4:]