PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Regression] fix FP8_STATIC loading (#1452)
Xin He9855b4ba20c2
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex 38bea79f3..82fcdcf20 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -155,6 +155,7 @@ def save_quantized_as_autoround(tokenizer: Callable = None,layer_config: dict = None,inplace: bool = True,+ backend: str = None,device: Union[str, torch.device] = "cpu",serialization_dict: dict = None,**kwargs,@@ -165,6 +166,8 @@ def save_quantized_as_autoround(quantization_config = serialization_dictquantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)quantization_config["quant_method"] = "auto-round"+ if backend:+ quantization_config["packing_format"] = backendif "e5m2" in serialization_dict.get("data_type", "fp8"):quantization_config["fmt"] = "e5m2"else:diff --git a/auto_round/formats.py b/auto_round/formats.pyindex 6fe957c61..e6fc7f56f 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -1086,7 +1086,7 @@ class AutoRoundFormat(OutputFormat):elif serialization_dict.get("data_type", "int") == "fp" and serialization_dict.get("bits", 16) == 8:from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround- backend = "auto_round"+ backend = "auto_round:fp8_static" if serialization_dict.get("act_bits", 16) == 8 else Noneexport_func = save_quantized_as_autoroundelse:from auto_round.export.export_to_autoround.export import save_quantized_as_autorounddiff --git a/test/test_cuda/schemes/test_scheme.py b/test/test_cuda/schemes/test_scheme.pyindex 255fdc80e..8ab78756b 100644--- a/test/test_cuda/schemes/test_scheme.py+++ b/test/test_cuda/schemes/test_scheme.py@@ -2,6 +2,7 @@ import osimport shutilimport pytest+import transformersfrom packaging import versionfrom auto_round import AutoRound@@ -36,12 +37,16 @@ class TestAutoRound:def test_w4a16(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="W4A16", nsamples=1, iters=1)assert ar.bits == 4- ar.quantize()+ ar.quantize_and_save()+ model = transformers.AutoModelForCausalLM.from_pretrained("tmp_autoround", trust_remote_code=True)+ assert model is not None, "Model loading failed after quantization with W4A16 scheme"def test_w2a16(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="W2A16", nsamples=1, iters=1)assert ar.bits == 2- ar.quantize()+ ar.quantize_and_save()+ model = transformers.AutoModelForCausalLM.from_pretrained("tmp_autoround", trust_remote_code=True)+ assert model is not None, "Model loading failed after quantization with W2A16 scheme"def test_mxfp4(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="MXFP4_RCEIL", nsamples=1, iters=1)@@ -49,7 +54,9 @@ class TestAutoRound:assert ar.act_bits == 4assert ar.data_type == "mx_fp"assert ar.act_data_type == "mx_fp_rceil"- ar.quantize()+ ar.quantize_and_save()+ model = transformers.AutoModelForCausalLM.from_pretrained("tmp_autoround", trust_remote_code=True)+ assert model is not None, "Model loading failed after quantization with MXFP4 scheme"def test_fp8_static(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="FP8_STATIC", nsamples=1, iters=1)@@ -59,13 +66,15 @@ class TestAutoRound:assert ar.act_data_type == "fp"assert ar.group_size == -1assert ar.act_dynamic is False- ar.quantize()+ ar.quantize_and_save()+ model = transformers.AutoModelForCausalLM.from_pretrained("tmp_autoround", trust_remote_code=True)+ assert model is not None, "Model loading failed after quantization with FP8_STATIC scheme"## RTN testsdef test_w2a16_rtn(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="W2A16", nsamples=1, iters=0)assert ar.bits == 2- ar.quantize()+ ar.quantize_and_save()def test_mxfp4_rtn(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="MXFP4", nsamples=1, iters=0)@@ -73,7 +82,7 @@ class TestAutoRound:assert ar.act_bits == 4assert ar.data_type == "mx_fp"assert ar.act_data_type == "mx_fp"- ar.quantize()+ ar.quantize_and_save()def test_fp8_static_rtn(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="FP8_STATIC", nsamples=1, iters=0)@@ -83,7 +92,7 @@ class TestAutoRound:assert ar.act_data_type == "fp"assert ar.group_size == -1assert ar.act_dynamic is False- ar.quantize()+ ar.quantize_and_save()def test_scheme_in_layer_config(self):model_path = get_model_path("facebook/opt-125m")@@ -94,7 +103,9 @@ class TestAutoRound:}ar = AutoRound(model_path, scheme="W3A16", nsamples=1, iters=1, layer_config=layer_config)- ar.quantize()+ ar.quantize_and_save()+ model = transformers.AutoModelForCausalLM.from_pretrained("tmp_autoround", trust_remote_code=True)+ assert model is not None, "Model loading failed after quantization with layer-specific schemes"for n, m in ar.model.named_modules():if n == "model.decoder.layers.2.self_attn.q_proj":assert m.bits == 2