Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: give attn_v of LLM_TYPE_70B models Q5_K like llama.cpp (#2443)

Raashish Aggarwalae21ef9bb58c
diff --git a/auto_round/export/export_to_gguf/gguf_dtype.py b/auto_round/export/export_to_gguf/gguf_dtype.py
index 6c173d3de..0da290c76 100644
--- a/auto_round/export/export_to_gguf/gguf_dtype.py
+++ b/auto_round/export/export_to_gguf/gguf_dtype.py
@@ -92,6 +92,28 @@ def _use_more_bits(i_layer: int, n_layer: int) -> bool:
return i_layer < n_layer // 8 or i_layer >= 7 * n_layer // 8 or (i_layer - n_layer // 8) % 3 == 2
+# Number of transformer blocks for which llama.cpp sets LLM_TYPE_70B, per architecture
+# (see the per-architecture load_hparams in llama.cpp's src/models/*.cpp).
+# Keyed by gguf.MODEL_ARCH member name, so that importing this module does not import gguf.
+_LLM_TYPE_70B_BLOCK_COUNT = {"LLAMA": 80, "QWEN2": 80, "DECI": 80, "OLMO": 80, "JAIS2": 68}
+
+
+def _is_llm_type_70b(model_arch, hparams: dict) -> bool:
+ """Whether llama.cpp would classify the model as LLM_TYPE_70B."""
+ arch_name = getattr(model_arch, "name", None)
+ n_layer = hparams.get("num_hidden_layers", hparams.get("n_layer", hparams.get("num_layers")))
+ if arch_name not in _LLM_TYPE_70B_BLOCK_COUNT or n_layer is None:
+ return False
+ if int(n_layer) != _LLM_TYPE_70B_BLOCK_COUNT[arch_name]:
+ return False
+ if arch_name == "LLAMA":
+ # 80-layer LLaMA without grouped-query attention is LLM_TYPE_65B
+ n_head = hparams.get("num_attention_heads", hparams.get("n_head"))
+ n_head_kv = hparams.get("num_key_value_heads", hparams.get("n_head_kv", n_head))
+ return n_head != n_head_kv
+ return True
+
+
def _get_layer_id(name: str, fallback: int) -> int:
parts = name.split(".")
if len(parts) > 1 and parts[0] == "blk" and parts[1].isdigit():
@@ -252,6 +274,12 @@ class GGUFDTypeSelector:
qtype = gguf.GGMLQuantizationType.Q6_K
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S and self.i_attention_wv < 4:
qtype = gguf.GGMLQuantizationType.Q5_K
+ if _is_llm_type_70b(self.model_arch, self.hparams) and qtype in (
+ gguf.GGMLQuantizationType.Q3_K,
+ gguf.GGMLQuantizationType.Q4_K,
+ ):
+ # llama.cpp: attn_v is 8x smaller than attn_q in these models, so more bits are cheap
+ qtype = gguf.GGMLQuantizationType.Q5_K
if n_expert == 8:
qtype = gguf.GGMLQuantizationType.Q8_0
self.i_attention_wv += 1
diff --git a/test/unit/common/export/test_gguf_dtype_helpers.py b/test/unit/common/export/test_gguf_dtype_helpers.py
index f8606cb24..f2daa635e 100644
--- a/test/unit/common/export/test_gguf_dtype_helpers.py
+++ b/test/unit/common/export/test_gguf_dtype_helpers.py
@@ -322,3 +322,44 @@ class TestGGUFDTypeSelector:
)
assert selector.select_qtype("blk.0.ffn_down.weight", n_dims=2) == gguf.GGMLQuantizationType.Q4_K
+
+
+class TestGGUFDTypeSelectorLLMType70B:
+ """llama.cpp gives attn_v of LLM_TYPE_70B models Q5_K instead of Q3_K/Q4_K."""
+
+ @staticmethod
+ def _attn_v_qtypes(model_arch, hparams, gguf_format):
+ from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector, gguf_format_to_ftype
+
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype(gguf_format), model_arch)
+ n_layer = hparams["num_hidden_layers"]
+ return [selector.select_qtype(f"blk.{i}.attn_v.weight", n_dims=2) for i in range(n_layer)]
+
+ def test_llama_70b_attn_v_uses_q5_k(self):
+ hparams = {"num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8}
+ qtypes = self._attn_v_qtypes(gguf.MODEL_ARCH.LLAMA, hparams, "gguf:q4_k_m")
+ # use_more_bits layers stay Q6_K, every other layer is bumped from Q4_K to Q5_K
+ assert set(qtypes) == {gguf.GGMLQuantizationType.Q5_K, gguf.GGMLQuantizationType.Q6_K}
+ assert gguf.GGMLQuantizationType.Q5_K in qtypes
+
+ qtypes = self._attn_v_qtypes(gguf.MODEL_ARCH.LLAMA, hparams, "gguf:q3_k_s")
+ assert set(qtypes) == {gguf.GGMLQuantizationType.Q5_K}
+
+ def test_qwen2_80_layers_attn_v_uses_q5_k(self):
+ hparams = {"num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8}
+ qtypes = self._attn_v_qtypes(gguf.MODEL_ARCH.QWEN2, hparams, "gguf:q4_k_s")
+ assert set(qtypes) == {gguf.GGMLQuantizationType.Q5_K}
+
+ @pytest.mark.parametrize(
+ "model_arch, hparams",
+ [
+ # 80-layer LLaMA without GQA is LLM_TYPE_65B in llama.cpp
+ (gguf.MODEL_ARCH.LLAMA, {"num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 64}),
+ (gguf.MODEL_ARCH.LLAMA, {"num_hidden_layers": 32, "num_attention_heads": 32, "num_key_value_heads": 8}),
+ (None, {"num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8}),
+ ],
+ )
+ def test_other_models_keep_q4_k(self, model_arch, hparams):
+ qtypes = self._attn_v_qtypes(model_arch, hparams, "gguf:q4_k_s")
+ assert gguf.GGMLQuantizationType.Q4_K in qtypes
+ assert gguf.GGMLQuantizationType.Q5_K not in qtypes[4:]