PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf bug (#610)
Wenhua Cheng40d99f2ffba4
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 0ceef40c1..cc439ebd3 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -383,9 +383,8 @@ class AutoRound(object):def _dq_check(self):"""Reset the default value of super_bits and super_group_size"""- from auto_round.export.export_to_gguf.config import GGUF_CONFIGif self.data_type.endswith("_dq"):- gguf_config = GGUF_CONFIG[f"gguf:q{self.bits}_k_s"]+ gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bitsself.super_group_size = gguf_config["super_group_size"] \if self.super_group_size is None else self.super_group_size@@ -466,7 +465,7 @@ class AutoRound(object):logger.warning("We recommend setting `iters=0` when exporting to GGUF format,"" as we have optimized the RTN method for this case."- " We will release new algorithms for certain configurations in the future."+ " We are likely to release new algorithms for certain configurations in the future.")if self.seqlen is not None and hasattr(self.model, "config") and \diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex d227cfa9d..536f3f6c4 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -21,7 +21,7 @@ GGUF_INNER_CONFIG["gguf:q4_0"] = {"sym": True,"data_type": "int","embedding": "gguf:q4_0",- "lm_head": "gguf:q6_k_s",+ "lm_head": "gguf:q6_k","super_bits": None,"super_group_size": None,}@@ -33,7 +33,7 @@ GGUF_INNER_CONFIG["gguf:q4_1"] = {"sym": False,"data_type": "int_asym_float_zp","embedding": "gguf:q4_1",- "lm_head": "gguf:q6_k_s",+ "lm_head": "gguf:q6_k","super_bits": None,"super_group_size": None,}@@ -45,7 +45,7 @@ GGUF_INNER_CONFIG["gguf:q5_0"] = {"sym": True,"data_type": "int","embedding": "gguf:q5_0",- "lm_head": "gguf:q6_k_s",+ "lm_head": "gguf:q6_k","super_bits": None,"super_group_size": None,}@@ -57,7 +57,7 @@ GGUF_INNER_CONFIG["gguf:q5_1"] = {"sym": False,"data_type": "int_asym_float_zp","embedding": "gguf:q5_1",- "lm_head": "gguf:q6_k_s",+ "lm_head": "gguf:q6_k","super_bits": None,"super_group_size": None,}diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 621d974d1..c498fbb6a 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -454,7 +454,8 @@ def tune(args):to_quant_block_names=args.to_quant_block_names,enable_torch_compile=enable_torch_compile,device_map=args.device_map,- model_kwargs=model_kwargs+ model_kwargs=model_kwargs,+ data_type=args.data_type,)model, _ = autoround.quantize()diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex dbeb8a3bf..4d9c5164a 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -185,7 +185,7 @@ class TestGGUF(unittest.TestCase):super_bits=8)quantized_model_path = "./saved"- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k")gguf_file = os.listdir("saved")[0]model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"