PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
mv some checkers (#611)
Wenhua Chengc484bc81ae40
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex cc439ebd3..b78be16ea 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -461,13 +461,24 @@ class AutoRound(object):has_besides_gguf = Trueif has_gguf and has_besides_gguf:raise ValueError("gguf format is not compatible with other formats, please choose only one of them")- if has_gguf and self.iters!=0:+ if has_gguf and self.iters != 0:logger.warning("We recommend setting `iters=0` when exporting to GGUF format,"" as we have optimized the RTN method for this case."" We are likely to release new algorithms for certain configurations in the future.")+ ##check group_size 32 for auto_round+ if self.data_type == "int" and hasattr(self, "formats") and (+ "auto_round" in self.formats or "auto_gptq" in self.formats or "auto_awq" in self.formats):+ for n, m in self.model.named_modules():+ if isinstance(m, self.supported_types) :+ if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:+ self.layer_config[n] = {"bits": 16}+ logger.info(+ f"{n} will not be quantized due to its shape not being divisible by 32,"+ " resulting in an exporting issue to autogptq")+if self.seqlen is not None and hasattr(self.model, "config") and \hasattr(self.model.config, "max_position_embeddings"):if self.model.config.max_position_embeddings < self.seqlen:@@ -1319,7 +1330,7 @@ class AutoRound(object):logger.info("switch to cpu to cache block inputs")if (self.has_qlayer_outside_block orself.__class__.__name__ == "AutoRoundMLLM"):- logger.warning("We strongly recommend using more GPUs."+ logger.warning("We strongly recommend using more GPUs in calibration."" Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy.")self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex 536f3f6c4..484c5f033 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -157,8 +157,6 @@ GGUF_CONFIG["gguf:q5_0"] = GGUF_INNER_CONFIG["gguf:q5_0"]GGUF_CONFIG["gguf:q5_0"]["mostly"]= "gguf:q5_0"GGUF_CONFIG["gguf:q5_1"] = GGUF_INNER_CONFIG["gguf:q5_1"]GGUF_CONFIG["gguf:q5_1"]["mostly"] = "gguf:q5_1"-GGUF_CONFIG["gguf:q2_k"] = GGUF_INNER_CONFIG["gguf:q2_k"]-GGUF_CONFIG["gguf:q2_k"]["mostly"] = "gguf:q2_k"GGUF_CONFIG["gguf:q2_k_s"] = GGUF_INNER_CONFIG["gguf:q2_k"]GGUF_CONFIG["gguf:q2_k_s"]["mostly"]= "gguf:q2_k"# GGUF_CONFIG["gguf:q3_k"] = GGUF_INNER_CONFIG["gguf:q3_k"]@@ -175,8 +173,8 @@ GGUF_CONFIG["gguf:q4_k_s"] = GGUF_INNER_CONFIG["gguf:q4_k"]GGUF_CONFIG["gguf:q4_k_s"]["mostly"]= "gguf:q4_k"GGUF_CONFIG["gguf:q4_k_m"] = GGUF_INNER_CONFIG["gguf:q4_k"]GGUF_CONFIG["gguf:q4_k_m"]["mostly"] = "gguf:q4_k"-GGUF_CONFIG["gguf:q5_k"] = GGUF_INNER_CONFIG["gguf:q5_k"]-GGUF_CONFIG["gguf:q5_k"]["mostly"]= "gguf:q5_k"+# GGUF_CONFIG["gguf:q5_k"] = GGUF_INNER_CONFIG["gguf:q5_k"]+# GGUF_CONFIG["gguf:q5_k"]["mostly"]= "gguf:q5_k"GGUF_CONFIG["gguf:q5_k_s"] = GGUF_INNER_CONFIG["gguf:q5_k"]GGUF_CONFIG["gguf:q5_k_s"]["mostly"] = "gguf:q5_k"GGUF_CONFIG["gguf:q5_k_m"] = GGUF_INNER_CONFIG["gguf:q5_k"]diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex f7fc3af4e..7499b1630 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -403,14 +403,6 @@ def tune(args):round = AutoRoundAdamlayer_config = {}- for n, m in model.named_modules():- if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.pytorch_utils.Conv1D):- if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- layer_config[n] = {"bits": 16}- logger.info(- f"{n} will not be quantized due to its shape not being divisible by 32,"- " resulting in an exporting issue to autogptq")-not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)for name in not_quantize_layer_names:layer_config[name] = {"bits": 16}diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex c498fbb6a..5e3fdea8f 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -369,13 +369,6 @@ def tune(args):##TODO gptq, awq could support some mixed precision configlogger.warning(f"mixed precision exporting does not support {format} currently")- for n, m in model.named_modules():- if isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):- if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- layer_config[n] = {"bits": 32}- logger.info(- f"{n} will not be quantized due to its shape not being divisible by 32,"- " resulting in an exporting issue to autogptq")lm_head_layer_name = "lm_head"for n, _ in model.named_modules():lm_head_layer_name = ndiff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex 4d9c5164a..b58d3acc2 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -239,3 +239,4 @@ class TestGGUF(unittest.TestCase):if __name__ == "__main__":unittest.main()+