PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support one more mxfp4 variant (#2132)
Wenhua Chengf706da6736c9
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex 8f924acd9..7f453066a 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -342,11 +342,76 @@ def quant_mx_rceil(return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None+"""+Implementation based on:++MXAttention: Data-Free Optimal Scaling and Pre-Normalization Quantization+for MXFP4 Attention.++Jianlin Yu, Jing Lin, Linghui Kong, et al.+arXiv:2607.24377, 2026.+https://arxiv.org/abs/2607.24377+"""+++# Only for mxfp4+def quant_mx_rceil_v2(+ tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs+):+ """Quantize the given tensor using the specified parameters.++ This function performs quantization on the `tensor` tensor according to the+ given bit width (`bits`), data type (`data_type`), and additional parameters.+ The quantization process involves scaling the tensor values and adjusting+ the exponent and mantissa to fit within the specified format.++ Args:+ tensor (torch.Tensor): The tensor containing the tensors to be quantized.+ bits (int): The bit width to be used for quantization.+ group_size (int): The group size of sharing scale and exponent.+ data_type (str): The data type for quantization (e.g., 'mx_fp4').+ v (float): A value used for adjusting the tensors.+ max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.+ mantissa_rounding (str): rounding method for mantissa,currently support even,nearest,floor++ Returns:+ tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).++ Raises:+ KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ data_type = data_type if data_type in MXFP_FORMAT_CACHE else "mx_fp" + str(bits)+ ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]+ orig_dtype = tensor.dtype+ tensor = tensor.to(torch.float32)+ max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)+ if isinstance(max_scale, torch.Tensor):+ max_val *= (max_scale.unsqueeze(dim=-1)).to(tensor.device)+ else:+ max_val *= max_scale++ # shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))+ shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), ceil_ste(torch.log2(max_val / 7.25)))+ scale_emax = 2.0 ** float(8 - 1) - 1+ shared_exp = shared_exp.clamp(min=-scale_emax, max=scale_emax)++ scale = torch.pow(2.0, shared_exp.float())+ tensor = tensor / scale + v+ tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)+ tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)++ tensor = tensor * scale+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)+ return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None++for key in MXFP_FORMAT_CACHE.keys():QUANT_FUNC_WITH_DTYPE[key] = quant_mxQUANT_FUNC_WITH_DTYPE[key + "_rceil"] = quant_mx_rceilQUANT_FUNC_WITH_DTYPE["opt_rtn_" + key] = quant_mx_opt_rtnQUANT_FUNC_WITH_DTYPE["mx_fp_rceil"] = quant_mx_rceil+QUANT_FUNC_WITH_DTYPE["mx_fp4_rceil_v2"] = quant_mx_rceil_v2QUANT_FUNC_WITH_DTYPE["opt_rtn_mx_fp"] = quant_mx_opt_rtnif __name__ == "__main__":diff --git a/docs/mxnv_acc.md b/docs/mxnv_acc.mdindex 24ee865af..6081e3ed6 100644--- a/docs/mxnv_acc.md+++ b/docs/mxnv_acc.md@@ -1,4 +1,29 @@-Average accuracy of hellaswag,lambada_openai,mmlu,piqa,winogrande.+### MXFP4 variants+++#### Qwen3-8B++| Method | avg | MMLU | arc_challenge | arc_easy | boolq | gsm8k | hellaswag | lambada_openai | openbookqa | piqa | truthfulqa_mc1 | winogrande |+|--------|----:|------:|--------------:|---------:|-------:|------:|-----------:|----------------:|------------:|-----:|----------------:|------------:|+| ocp rtn | 0.607072727 | 0.6613 | 0.4932 | 0.7715 | 0.8593 | 0.8127 | 0.5223 | 0.5851 | 0.2800 | 0.7301 | 0.3293 | 0.6330 |+| rceil rtn | 0.614190909 | 0.6718 | 0.4974 | 0.7656 | 0.8450 | 0.8271 | 0.5299 | 0.5905 | 0.2880 | 0.7427 | 0.3525 | 0.6456 |+| rceil 7.25 RTN | 0.617018182 | 0.6780 | 0.5000 | 0.7778 | 0.8498 | 0.8317 | 0.5338 | 0.5991 | 0.2940 | 0.7329 | 0.3476 | 0.6425 |+| ocp iters 200 | 0.624609091 | 0.6771 | 0.5171 | 0.8119 | 0.8456 | 0.7961 | 0.5284 | 0.5979 | 0.3080 | 0.7546 | 0.3513 | 0.6827 |+| rceil iters 200 | 0.618427273 | 0.6794 | 0.5111 | 0.7984 | 0.8550 | 0.7680 | 0.5242 | 0.5993 | 0.3040 | 0.7530 | 0.3378 | 0.6725 |+| rceil 7.25 iters 200 | 0.627009091 | 0.6837 | 0.5307 | 0.8136 | 0.8569 | 0.8052 | 0.5317 | 0.6113 | 0.2960 | 0.7514 | 0.3378 | 0.6788 |++#### Llama3.1-8B-I++| Method | avg | MMLU | arc_challenge | arc_easy | boolq | gsm8k | hellaswag | lambada_openai | openbookqa | piqa | truthfulqa_mc1 | winogrande |+|--------|----:|------:|--------------:|---------:|-------:|------:|-----------:|----------------:|------------:|-----:|----------------:|------------:|+| ocp rtn | 0.575163636 | 0.5734 | 0.4582 | 0.7630 | 0.8080 | 0.5011 | 0.5554 | 0.6216 | 0.3080 | 0.7606 | 0.2766 | 0.7009 |+| rceil rtn | 0.6031 | 0.6002 | 0.4710 | 0.7816 | 0.8235 | 0.5792 | 0.5661 | 0.6647 | 0.3260 | 0.7780 | 0.3390 | 0.7048 |+| rceil 7.25 rtn | 0.601372727 | 0.6057 | 0.4753 | 0.7803 | 0.8242 | 0.5663 | 0.5657 | 0.6637 | 0.3340 | 0.7769 | 0.3158 | 0.7072 |+| ocp iters 200 | 0.605645455 | 0.6160 | 0.4659 | 0.8009 | 0.8257 | 0.6035 | 0.5577 | 0.6831 | 0.3000 | 0.7709 | 0.3415 | 0.6969 |+| rceil iters 200 | 0.607863636 | 0.6133 | 0.4846 | 0.7959 | 0.8300 | 0.6156 | 0.5628 | 0.6645 | 0.3120 | 0.7835 | 0.3329 | 0.6914 |+| rceil 7.25 iters 200 | 0.614072727 | 0.6149 | 0.4804 | 0.7984 | 0.8303 | 0.6262 | 0.5670 | 0.6829 | 0.3300 | 0.7807 | 0.3305 | 0.7135 |++### Average accuracy of hellaswag,lambada_openai,mmlu,piqa,winogrande.We evaluated using a fake model since we currently have no access to devices for running the real models. However, we have verified that in most cases the fake model closely matches the real model.diff --git a/test/unit/test_cpu/quantization/test_mxfp_nvfp.py b/test/unit/test_cpu/quantization/test_mxfp_nvfp.pyindex a985d4e6c..051b71e26 100644--- a/test/unit/test_cpu/quantization/test_mxfp_nvfp.py+++ b/test/unit/test_cpu/quantization/test_mxfp_nvfp.py@@ -54,6 +54,7 @@ class TestAutoRoundFP:nsamples=2,dataset=dataloader,layer_config=layer_config,+ disable_opt_rtn=True,trust_remote_code=False,)compressed_model, _ = autoround.quantize()@@ -172,6 +173,7 @@ class TestAutoRoundFP:iters=0,seqlen=2,layer_config=layer_config,+ disable_opt_rtn=True,dataset=dataloader,)quantized_model_path = self.save_dir@@ -377,6 +379,7 @@ class TestAutoRoundFP:scheme=scheme,iters=0,seqlen=2,+ disable_opt_rtn=True,dataset=dataloader,static_kv_dtype=static_kv_dtype,static_attention_dtype=static_attention_dtype,