PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Important Update]change mxfp4 to ocp standard definition and fix scale dtype issue (#1181)
Wenhua Cheng94497f3b53c3
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 07fea80eb..742243bdd 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -252,7 +252,10 @@ class BaseCompressor(object):gguf_scheme_name = get_gguf_scheme(self.scheme)# GGUF uses fp32 scale dtype as default- scale_dtype = kwargs.pop("scale_dtype", "fp32") if gguf_scheme_name else kwargs.pop("scale_dtype", "fp16")+ scale_dtype = kwargs.pop("scale_dtype", None)+ if scale_dtype is None:+ scale_dtype = "fp32" if gguf_scheme_name else "fp16"+# Extra/legacy kwargs for backward compatibility# Major version releases may pack them with extra configuration optionsamp = kwargs.pop("amp", True)diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 56cf72b2e..afbd35053 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -166,6 +166,18 @@ W8A16 = QuantizationScheme.from_dict()MXFP4 = QuantizationScheme.from_dict(+ {+ "bits": 4,+ "group_size": 32,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp",+ "act_group_size": 32,+ "act_sym": True,+ }+)++MXFP4_RCEIL = QuantizationScheme.from_dict({"bits": 4,"group_size": 32,@@ -177,6 +189,7 @@ MXFP4 = QuantizationScheme.from_dict(})+MXFP8 = QuantizationScheme.from_dict({"bits": 8,@@ -250,6 +263,7 @@ PRESET_SCHEMES = {"W3A16": W3A16,"W8A16": W8A16,"MXFP4": MXFP4,+ "MXFP4_RCEIL": MXFP4_RCEIL,"MXFP8": MXFP8,"NVFP4": NVFP4,"FPW8A16": FPW8A16,diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex edfbdcc93..17079cb80 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -118,13 +118,14 @@ AutoRound supports several Schemes:- **W8A16**(bits:8,group_size:128,sym:True,act_bits:16)- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)-- **GGUF:Q4_K_M**(all Q*_K,Q*_0,Q*_1 are supported)+- **GGUF:Q4_K_M**(all Q*_K,Q*_0,Q*_1 provided by llamacpp are supported)- **Mixed Bits Weight only**- **NVFP4**(Experimental feature, recommend exporting to `llm_compressor` format.data_type nvfp4,act_data_type nvfp4,static_global_scale,group_size 16)-- **MXFP4**(**Research feature,no real kernel**, data_type mxfp,act_data_type mxfp_rceil,bits 4, act_bits 4, group_size 32)-- **MXFP8**(**Research feature,no real kernel**, data_type mxfp,act_data_type mxfp_rceil,group_size 32)-- **FPW8A16**(**Research feature,no real kernel**, data_type fp8,group_size 0->per tensor )-- **FP8_STATIC**(**Research feature,no real kernel**, data_type:fp8,act_data_type:fp8,group_size -1 ->per channel, act_group_size=0->per tensor)+- **MXFP4**(**Research feature, no real kernel**, Standard MXFP4, data_type mxfp,act_data_type mxfp,bits 4, act_bits 4, group_size 32)+- **MXFP4_RCEIL**(**Research feature,no real kernel**, NVIDIA's variant, data_type mxfp,act_data_type mxfp_rceil,bits 4, act_bits 4, group_size 32)+- **MXFP8**(**Research feature, no real kernel**, data_type mxfp,act_data_type mxfp_rceil,group_size 32)+- **FPW8A16**(**Research feature, no real kernel**, data_type fp8,group_size 0->per tensor )+- **FP8_STATIC**(**Research feature, no real kernel**, data_type:fp8,act_data_type:fp8,group_size -1 ->per channel, act_group_size=0->per tensor)Besides, you could modify the `group_size`, `bits`, `sym` and many other configs you want, though there are maybe no real kernels.diff --git a/test/test_cpu/test_act_quantization.py b/test/test_cpu/test_act_quantization.pyindex 31ba51f1b..8ecce2152 100644--- a/test/test_cpu/test_act_quantization.py+++ b/test/test_cpu/test_act_quantization.py@@ -158,7 +158,7 @@ class TestAutoRoundAct(unittest.TestCase):# check inblock layer config valueskproj_config = model.config.quantization_config.extra_config["model.decoder.layers.1.self_attn.k_proj"]- assert "act_data_type" in kproj_config.keys() and kproj_config["act_data_type"] == "mx_fp_rceil"+ assert "act_data_type" in kproj_config.keys() and kproj_config["act_data_type"] == "mx_fp"assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 8assert "act_group_size" in kproj_config.keys() and kproj_config["act_group_size"] == 32assert "act_sym" in kproj_config.keys() and kproj_config["act_sym"]diff --git a/test/test_cpu/test_scheme.py b/test/test_cpu/test_scheme.pyindex c2d165639..15a8001b9 100644--- a/test/test_cpu/test_scheme.py+++ b/test/test_cpu/test_scheme.py@@ -58,7 +58,7 @@ class TestAutoRound(unittest.TestCase):self.assertEqual(ar.bits, 4)self.assertEqual(ar.act_bits, 4)self.assertEqual(ar.data_type, "mx_fp")- self.assertEqual(ar.act_data_type, "mx_fp_rceil")+ self.assertEqual(ar.act_data_type, "mx_fp")ar.quantize()def test_vllm(self):diff --git a/test/test_cuda/test_scheme.py b/test/test_cuda/test_scheme.pyindex 1c603c7ed..636876679 100644--- a/test/test_cuda/test_scheme.py+++ b/test/test_cuda/test_scheme.py@@ -42,7 +42,7 @@ class TestAutoRound(unittest.TestCase):self.assertEqual(ar.bits, 4)self.assertEqual(ar.act_bits, 4)self.assertEqual(ar.data_type, "mx_fp")- self.assertEqual(ar.act_data_type, "mx_fp_rceil")+ self.assertEqual(ar.act_data_type, "mx_fp")ar.quantize()def test_fp8_static(self):@@ -66,7 +66,7 @@ class TestAutoRound(unittest.TestCase):self.assertEqual(ar.bits, 4)self.assertEqual(ar.act_bits, 4)self.assertEqual(ar.data_type, "mx_fp")- self.assertEqual(ar.act_data_type, "mx_fp_rceil")+ self.assertEqual(ar.act_data_type, "mx_fp")ar.quantize()def test_fp8_static_rtn(self):