Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf bug (#610)

Wenhua Cheng40d99f2ffba4
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 0ceef40c1..cc439ebd3 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -383,9 +383,8 @@ class AutoRound(object):
def _dq_check(self):
"""Reset the default value of super_bits and super_group_size"""
- from auto_round.export.export_to_gguf.config import GGUF_CONFIG
if self.data_type.endswith("_dq"):
- gguf_config = GGUF_CONFIG[f"gguf:q{self.bits}_k_s"]
+ gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]
self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bits
self.super_group_size = gguf_config["super_group_size"] \
if self.super_group_size is None else self.super_group_size
@@ -466,7 +465,7 @@ class AutoRound(object):
logger.warning(
"We recommend setting `iters=0` when exporting to GGUF format,"
" as we have optimized the RTN method for this case."
- " We will release new algorithms for certain configurations in the future."
+ " We are likely to release new algorithms for certain configurations in the future."
)
if self.seqlen is not None and hasattr(self.model, "config") and \
diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.py
index d227cfa9d..536f3f6c4 100644
--- a/auto_round/export/export_to_gguf/config.py
+++ b/auto_round/export/export_to_gguf/config.py
@@ -21,7 +21,7 @@ GGUF_INNER_CONFIG["gguf:q4_0"] = {
"sym": True,
"data_type": "int",
"embedding": "gguf:q4_0",
- "lm_head": "gguf:q6_k_s",
+ "lm_head": "gguf:q6_k",
"super_bits": None,
"super_group_size": None,
}
@@ -33,7 +33,7 @@ GGUF_INNER_CONFIG["gguf:q4_1"] = {
"sym": False,
"data_type": "int_asym_float_zp",
"embedding": "gguf:q4_1",
- "lm_head": "gguf:q6_k_s",
+ "lm_head": "gguf:q6_k",
"super_bits": None,
"super_group_size": None,
}
@@ -45,7 +45,7 @@ GGUF_INNER_CONFIG["gguf:q5_0"] = {
"sym": True,
"data_type": "int",
"embedding": "gguf:q5_0",
- "lm_head": "gguf:q6_k_s",
+ "lm_head": "gguf:q6_k",
"super_bits": None,
"super_group_size": None,
}
@@ -57,7 +57,7 @@ GGUF_INNER_CONFIG["gguf:q5_1"] = {
"sym": False,
"data_type": "int_asym_float_zp",
"embedding": "gguf:q5_1",
- "lm_head": "gguf:q6_k_s",
+ "lm_head": "gguf:q6_k",
"super_bits": None,
"super_group_size": None,
}
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 621d974d1..c498fbb6a 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -454,7 +454,8 @@ def tune(args):
to_quant_block_names=args.to_quant_block_names,
enable_torch_compile=enable_torch_compile,
device_map=args.device_map,
- model_kwargs=model_kwargs
+ model_kwargs=model_kwargs,
+ data_type=args.data_type,
)
model, _ = autoround.quantize()
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index dbeb8a3bf..4d9c5164a 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -185,7 +185,7 @@ class TestGGUF(unittest.TestCase):
super_bits=8
)
quantized_model_path = "./saved"
- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k")
gguf_file = os.listdir("saved")[0]
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"