Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bug of exporting fp8 static (#1361)

Heng Guob04f69c4359b
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 7801d8009..500ddc115 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -70,7 +70,7 @@ def is_wfp8afp8(ar):
def is_static_wfp8afp8(ar_or_format: Union[str, Callable]) -> bool:
if isinstance(ar_or_format, str):
- return "fp8_static" in ar_or_format
+ return "fp8_static" in ar_or_format.lower()
if ar_or_format.act_dynamic:
return False
if is_wfp8afp8(ar_or_format):
diff --git a/auto_round/formats.py b/auto_round/formats.py
index a0689fb82..5d53d0f9c 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -1076,6 +1076,10 @@ class AutoRoundFormat(OutputFormat):
backend = "auto_round"
export_func = save_quantized_as_autoround
+ elif AutoRoundExportFormat.FP8_STATIC.value in backend:
+ from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fp
+
+ export_func = save_quantized_as_static_fp
else:
from auto_round.export.export_to_autoround.export import save_quantized_as_autoround
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index f7702108f..dfe339590 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -214,13 +214,9 @@ class TestAutoRound:
bits=8,
group_size=-1,
iters=0,
- act_bits=8,
+ scheme="fp8_static",
nsamples=2,
seqlen=2,
- data_type="fp8",
- act_data_type="fp8",
- act_dynamic=False,
- act_group_size=0,
static_kv_dtype=static_kv_dtype,
)
quantized_model_path = "./saved"
diff --git a/test/test_cpu/quantization/test_act_quantization.py b/test/test_cpu/quantization/test_act_quantization.py
index 47cda3599..72739931e 100644
--- a/test/test_cpu/quantization/test_act_quantization.py
+++ b/test/test_cpu/quantization/test_act_quantization.py
@@ -176,36 +176,38 @@ class TestAutoRoundAct:
assert "bits" in kproj_config.keys() and kproj_config["bits"] == 8
shutil.rmtree(quantized_model_path, ignore_errors=True)
- def test_act_config_FP8_saving(self, tiny_opt_model_path, dataloader):
- scheme = "FP8_STATIC"
- layer_config = {
- "lm_head": {"act_bits": 8, "bits": 8},
- # check fp8 woq config
- "k_proj": {
- "bits": 8,
- "group_size": 0,
- "data_type": "fp",
- "act_bits": 16,
- "act_data_type": "fp",
- },
- }
- autoround = AutoRound(
- tiny_opt_model_path,
- scheme=scheme,
- iters=2,
- seqlen=2,
- dataset=dataloader,
- layer_config=layer_config,
- )
- quantized_model_path = self.save_dir
- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
- from transformers import AutoConfig
-
- extra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]
- assert "lm_head" not in extra_config
-
- # check inblock layer config values
- kproj_config = extra_config["model.decoder.layers.0.self_attn.k_proj"]
- assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16
- assert "group_size" in kproj_config.keys() and kproj_config["group_size"] == 0
- shutil.rmtree(quantized_model_path, ignore_errors=True)
+ # def test_act_config_FP8_saving(self, tiny_opt_model_path, dataloader):
+ # # fp8_static only support to export llm_compressor or auto_round (auto_round:llm_compressor in fact)
+ # # auto_round is deprecated since only can load via specific script
+ # scheme = "FP8_STATIC"
+ # layer_config = {
+ # "lm_head": {"act_bits": 8, "bits": 8},
+ # # check fp8 woq config
+ # "k_proj": {
+ # "bits": 8,
+ # "group_size": 0,
+ # "data_type": "fp",
+ # "act_bits": 16,
+ # "act_data_type": "fp",
+ # },
+ # }
+ # autoround = AutoRound(
+ # tiny_opt_model_path,
+ # scheme=scheme,
+ # iters=2,
+ # seqlen=2,
+ # dataset=dataloader,
+ # layer_config=layer_config,
+ # )
+ # quantized_model_path = self.save_dir
+ # autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ # from transformers import AutoConfig
+
+ # extra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]
+ # assert "lm_head" not in extra_config
+
+ # # check inblock layer config values
+ # kproj_config = extra_config["model.decoder.layers.0.self_attn.k_proj"]
+ # assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16
+ # assert "group_size" in kproj_config.keys() and kproj_config["group_size"] == 0
+ # shutil.rmtree(quantized_model_path, ignore_errors=True)