Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix MXFP/NVFP + FP8 Attn/KV (#1086)

Yi Liucaa246f93f75
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 7abcbc511..cc12ee09a 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1417,11 +1417,14 @@ class BaseCompressor(object):
enable_imatrix = True
elif self.data_type == "int" and self.sym:
enable_imatrix = True
-
if enable_imatrix:
self._quant_rtn_with_imatrix(all_to_quantized_module_names)
elif self.act_bits <= 8 and check_need_act_calibration(
- self.act_dynamic, self.act_data_type, self.act_bits
+ self.act_dynamic,
+ self.act_data_type,
+ self.act_bits,
+ self.static_kv_dtype,
+ self.static_attention_dtype,
): # TODO, mixed datatype has bug
hook_handles = self._register_act_max_hook(self.model)
try:
@@ -3145,6 +3148,8 @@ class BaseCompressor(object):
"super_bits",
"super_group_size",
"regex_config",
+ "static_kv_dtype",
+ "static_attention_dtype",
]
if isinstance(self.dataset, str):
serialization_keys.append("dataset")
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 9c6c02877..1e5683623 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -146,8 +146,14 @@ def check_skippable_keywords(key):
def check_need_act_calibration(
- is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: Union[int, None] = 16
+ is_act_dynamic: Union[bool, None],
+ act_data_type: Union[str, None] = None,
+ act_bits: Union[int, None] = 16,
+ static_kv_dtype: Union[str, None] = None,
+ static_attention_dtype: Union[str, None] = None,
) -> bool:
+ if static_kv_dtype is not None or static_attention_dtype is not None:
+ return True
if act_bits is None or act_bits > 8:
return False
# None is dynamic
diff --git a/test/test_cpu/test_mxfp_nvfp.py b/test/test_cpu/test_mxfp_nvfp.py
index ce61a9435..ba9d3a1a8 100644
--- a/test/test_cpu/test_mxfp_nvfp.py
+++ b/test/test_cpu/test_mxfp_nvfp.py
@@ -370,6 +370,70 @@ class TestAutoRoundFP(unittest.TestCase):
self.assertGreater(result["results"]["piqa"]["acc,none"], 0.60)
shutil.rmtree(quantized_model_path, ignore_errors=True)
+ @parameterized.expand(
+ [
+ # scheme, static_kv_dtype, static_attention_dtype
+ ("MXFP4", None, "fp8"),
+ ("MXFP4", "fp8", None),
+ ("MXFP8", None, "fp8"),
+ ("MXFP8", "fp8", None),
+ ("NVFP4", None, "fp8"),
+ ("NVFP4", "fp8", None),
+ ]
+ )
+ def test_fp8_kv_attn(self, scheme, static_kv_dtype, static_attention_dtype):
+ model_name = self.model_name
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+ from transformers.models.opt.modeling_opt import OPTForCausalLM
+
+ config = AutoConfig.from_pretrained(model_name)
+ config.num_hidden_layers = 1
+ model = OPTForCausalLM(config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ scheme=scheme,
+ iters=0,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ static_kv_dtype=static_kv_dtype,
+ static_attention_dtype=static_attention_dtype,
+ )
+
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(
+ output_dir=quantized_model_path,
+ format="auto_round",
+ )
+
+ attn = compressed_model.model.decoder.layers[0].self_attn
+ q_proj = attn.q_proj
+
+ # weight_scale should exist for all quantized schemes
+ assert hasattr(q_proj, "weight_scale"), f"Missing weight_scale in q_proj for scheme={scheme}"
+ if static_kv_dtype == "fp8":
+ assert (
+ compressed_model.config.quantization_config["static_kv_dtype"] == "fp8"
+ ), f"Invalid static_kv_dtype in config for scheme={scheme}, static_kv_dtype={static_kv_dtype}"
+
+ # Only when static_kv_dtype / static_attention_dtype are fp8 do we expect FP8 KV scales
+ if static_kv_dtype == "fp8" or static_attention_dtype == "fp8":
+ assert attn.k_scale is not None and attn.v_scale is not None, (
+ f"Missing k_scale/v_scale in attention for scheme={scheme}, "
+ f"static_kv_dtype={static_kv_dtype}, static_attention_dtype={static_attention_dtype}"
+ )
+
+ if static_attention_dtype == "fp8":
+ assert (
+ compressed_model.config.quantization_config["static_attention_dtype"] == "fp8"
+ ), f"Invalid static_attention_dtype in config for scheme={scheme}, static_attention_dtype={static_attention_dtype}"
+ assert (
+ getattr(attn, "q_scale", None) is not None
+ ), f"Missing q_scale in attention for scheme={scheme}, static_attention_dtype={static_attention_dtype}"
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
if __name__ == "__main__":
unittest.main()