PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bug of exporting fp8 static (#1361)
Heng Guob04f69c4359b
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 7801d8009..500ddc115 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -70,7 +70,7 @@ def is_wfp8afp8(ar):def is_static_wfp8afp8(ar_or_format: Union[str, Callable]) -> bool:if isinstance(ar_or_format, str):- return "fp8_static" in ar_or_format+ return "fp8_static" in ar_or_format.lower()if ar_or_format.act_dynamic:return Falseif is_wfp8afp8(ar_or_format):diff --git a/auto_round/formats.py b/auto_round/formats.pyindex a0689fb82..5d53d0f9c 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -1076,6 +1076,10 @@ class AutoRoundFormat(OutputFormat):backend = "auto_round"export_func = save_quantized_as_autoround+ elif AutoRoundExportFormat.FP8_STATIC.value in backend:+ from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fp++ export_func = save_quantized_as_static_fpelse:from auto_round.export.export_to_autoround.export import save_quantized_as_autorounddiff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex f7702108f..dfe339590 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -214,13 +214,9 @@ class TestAutoRound:bits=8,group_size=-1,iters=0,- act_bits=8,+ scheme="fp8_static",nsamples=2,seqlen=2,- data_type="fp8",- act_data_type="fp8",- act_dynamic=False,- act_group_size=0,static_kv_dtype=static_kv_dtype,)quantized_model_path = "./saved"diff --git a/test/test_cpu/quantization/test_act_quantization.py b/test/test_cpu/quantization/test_act_quantization.pyindex 47cda3599..72739931e 100644--- a/test/test_cpu/quantization/test_act_quantization.py+++ b/test/test_cpu/quantization/test_act_quantization.py@@ -176,36 +176,38 @@ class TestAutoRoundAct:assert "bits" in kproj_config.keys() and kproj_config["bits"] == 8shutil.rmtree(quantized_model_path, ignore_errors=True)- def test_act_config_FP8_saving(self, tiny_opt_model_path, dataloader):- scheme = "FP8_STATIC"- layer_config = {- "lm_head": {"act_bits": 8, "bits": 8},- # check fp8 woq config- "k_proj": {- "bits": 8,- "group_size": 0,- "data_type": "fp",- "act_bits": 16,- "act_data_type": "fp",- },- }- autoround = AutoRound(- tiny_opt_model_path,- scheme=scheme,- iters=2,- seqlen=2,- dataset=dataloader,- layer_config=layer_config,- )- quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- from transformers import AutoConfig-- extra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]- assert "lm_head" not in extra_config-- # check inblock layer config values- kproj_config = extra_config["model.decoder.layers.0.self_attn.k_proj"]- assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16- assert "group_size" in kproj_config.keys() and kproj_config["group_size"] == 0- shutil.rmtree(quantized_model_path, ignore_errors=True)+ # def test_act_config_FP8_saving(self, tiny_opt_model_path, dataloader):+ # # fp8_static only support to export llm_compressor or auto_round (auto_round:llm_compressor in fact)+ # # auto_round is deprecated since only can load via specific script+ # scheme = "FP8_STATIC"+ # layer_config = {+ # "lm_head": {"act_bits": 8, "bits": 8},+ # # check fp8 woq config+ # "k_proj": {+ # "bits": 8,+ # "group_size": 0,+ # "data_type": "fp",+ # "act_bits": 16,+ # "act_data_type": "fp",+ # },+ # }+ # autoround = AutoRound(+ # tiny_opt_model_path,+ # scheme=scheme,+ # iters=2,+ # seqlen=2,+ # dataset=dataloader,+ # layer_config=layer_config,+ # )+ # quantized_model_path = self.save_dir+ # autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ # from transformers import AutoConfig++ # extra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]+ # assert "lm_head" not in extra_config++ # # check inblock layer config values+ # kproj_config = extra_config["model.decoder.layers.0.self_attn.k_proj"]+ # assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16+ # assert "group_size" in kproj_config.keys() and kproj_config["group_size"] == 0+ # shutil.rmtree(quantized_model_path, ignore_errors=True)