PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix MXFP/NVFP + FP8 Attn/KV (#1086)
Yi Liucaa246f93f75
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 7abcbc511..cc12ee09a 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1417,11 +1417,14 @@ class BaseCompressor(object):enable_imatrix = Trueelif self.data_type == "int" and self.sym:enable_imatrix = True-if enable_imatrix:self._quant_rtn_with_imatrix(all_to_quantized_module_names)elif self.act_bits <= 8 and check_need_act_calibration(- self.act_dynamic, self.act_data_type, self.act_bits+ self.act_dynamic,+ self.act_data_type,+ self.act_bits,+ self.static_kv_dtype,+ self.static_attention_dtype,): # TODO, mixed datatype has bughook_handles = self._register_act_max_hook(self.model)try:@@ -3145,6 +3148,8 @@ class BaseCompressor(object):"super_bits","super_group_size","regex_config",+ "static_kv_dtype",+ "static_attention_dtype",]if isinstance(self.dataset, str):serialization_keys.append("dataset")diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 9c6c02877..1e5683623 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -146,8 +146,14 @@ def check_skippable_keywords(key):def check_need_act_calibration(- is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: Union[int, None] = 16+ is_act_dynamic: Union[bool, None],+ act_data_type: Union[str, None] = None,+ act_bits: Union[int, None] = 16,+ static_kv_dtype: Union[str, None] = None,+ static_attention_dtype: Union[str, None] = None,) -> bool:+ if static_kv_dtype is not None or static_attention_dtype is not None:+ return Trueif act_bits is None or act_bits > 8:return False# None is dynamicdiff --git a/test/test_cpu/test_mxfp_nvfp.py b/test/test_cpu/test_mxfp_nvfp.pyindex ce61a9435..ba9d3a1a8 100644--- a/test/test_cpu/test_mxfp_nvfp.py+++ b/test/test_cpu/test_mxfp_nvfp.py@@ -370,6 +370,70 @@ class TestAutoRoundFP(unittest.TestCase):self.assertGreater(result["results"]["piqa"]["acc,none"], 0.60)shutil.rmtree(quantized_model_path, ignore_errors=True)+ @parameterized.expand(+ [+ # scheme, static_kv_dtype, static_attention_dtype+ ("MXFP4", None, "fp8"),+ ("MXFP4", "fp8", None),+ ("MXFP8", None, "fp8"),+ ("MXFP8", "fp8", None),+ ("NVFP4", None, "fp8"),+ ("NVFP4", "fp8", None),+ ]+ )+ def test_fp8_kv_attn(self, scheme, static_kv_dtype, static_attention_dtype):+ model_name = self.model_name+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer+ from transformers.models.opt.modeling_opt import OPTForCausalLM++ config = AutoConfig.from_pretrained(model_name)+ config.num_hidden_layers = 1+ model = OPTForCausalLM(config)+ tokenizer = AutoTokenizer.from_pretrained(model_name)++ autoround = AutoRound(+ model,+ tokenizer,+ scheme=scheme,+ iters=0,+ seqlen=2,+ dataset=self.llm_dataloader,+ static_kv_dtype=static_kv_dtype,+ static_attention_dtype=static_attention_dtype,+ )++ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(+ output_dir=quantized_model_path,+ format="auto_round",+ )++ attn = compressed_model.model.decoder.layers[0].self_attn+ q_proj = attn.q_proj++ # weight_scale should exist for all quantized schemes+ assert hasattr(q_proj, "weight_scale"), f"Missing weight_scale in q_proj for scheme={scheme}"+ if static_kv_dtype == "fp8":+ assert (+ compressed_model.config.quantization_config["static_kv_dtype"] == "fp8"+ ), f"Invalid static_kv_dtype in config for scheme={scheme}, static_kv_dtype={static_kv_dtype}"++ # Only when static_kv_dtype / static_attention_dtype are fp8 do we expect FP8 KV scales+ if static_kv_dtype == "fp8" or static_attention_dtype == "fp8":+ assert attn.k_scale is not None and attn.v_scale is not None, (+ f"Missing k_scale/v_scale in attention for scheme={scheme}, "+ f"static_kv_dtype={static_kv_dtype}, static_attention_dtype={static_attention_dtype}"+ )++ if static_attention_dtype == "fp8":+ assert (+ compressed_model.config.quantization_config["static_attention_dtype"] == "fp8"+ ), f"Invalid static_attention_dtype in config for scheme={scheme}, static_attention_dtype={static_attention_dtype}"+ assert (+ getattr(attn, "q_scale", None) is not None+ ), f"Missing q_scale in attention for scheme={scheme}, static_attention_dtype={static_attention_dtype}"+ shutil.rmtree(quantized_model_path, ignore_errors=True)+if __name__ == "__main__":unittest.main()