Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[Important Update]change mxfp4 to ocp standard definition and fix scale dtype issue (#1181)

Wenhua Cheng94497f3b53c3
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 07fea80eb..742243bdd 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -252,7 +252,10 @@ class BaseCompressor(object):
gguf_scheme_name = get_gguf_scheme(self.scheme)
# GGUF uses fp32 scale dtype as default
- scale_dtype = kwargs.pop("scale_dtype", "fp32") if gguf_scheme_name else kwargs.pop("scale_dtype", "fp16")
+ scale_dtype = kwargs.pop("scale_dtype", None)
+ if scale_dtype is None:
+ scale_dtype = "fp32" if gguf_scheme_name else "fp16"
+
# Extra/legacy kwargs for backward compatibility
# Major version releases may pack them with extra configuration options
amp = kwargs.pop("amp", True)
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 56cf72b2e..afbd35053 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -166,6 +166,18 @@ W8A16 = QuantizationScheme.from_dict(
)
MXFP4 = QuantizationScheme.from_dict(
+ {
+ "bits": 4,
+ "group_size": 32,
+ "data_type": "mx_fp",
+ "act_bits": 4,
+ "act_data_type": "mx_fp",
+ "act_group_size": 32,
+ "act_sym": True,
+ }
+)
+
+MXFP4_RCEIL = QuantizationScheme.from_dict(
{
"bits": 4,
"group_size": 32,
@@ -177,6 +189,7 @@ MXFP4 = QuantizationScheme.from_dict(
}
)
+
MXFP8 = QuantizationScheme.from_dict(
{
"bits": 8,
@@ -250,6 +263,7 @@ PRESET_SCHEMES = {
"W3A16": W3A16,
"W8A16": W8A16,
"MXFP4": MXFP4,
+ "MXFP4_RCEIL": MXFP4_RCEIL,
"MXFP8": MXFP8,
"NVFP4": NVFP4,
"FPW8A16": FPW8A16,
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index edfbdcc93..17079cb80 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -118,13 +118,14 @@ AutoRound supports several Schemes:
- **W8A16**(bits:8,group_size:128,sym:True,act_bits:16)
- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)
- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)
-- **GGUF:Q4_K_M**(all Q*_K,Q*_0,Q*_1 are supported)
+- **GGUF:Q4_K_M**(all Q*_K,Q*_0,Q*_1 provided by llamacpp are supported)
- **Mixed Bits Weight only**
- **NVFP4**(Experimental feature, recommend exporting to `llm_compressor` format.data_type nvfp4,act_data_type nvfp4,static_global_scale,group_size 16)
-- **MXFP4**(**Research feature,no real kernel**, data_type mxfp,act_data_type mxfp_rceil,bits 4, act_bits 4, group_size 32)
-- **MXFP8**(**Research feature,no real kernel**, data_type mxfp,act_data_type mxfp_rceil,group_size 32)
-- **FPW8A16**(**Research feature,no real kernel**, data_type fp8,group_size 0->per tensor )
-- **FP8_STATIC**(**Research feature,no real kernel**, data_type:fp8,act_data_type:fp8,group_size -1 ->per channel, act_group_size=0->per tensor)
+- **MXFP4**(**Research feature, no real kernel**, Standard MXFP4, data_type mxfp,act_data_type mxfp,bits 4, act_bits 4, group_size 32)
+- **MXFP4_RCEIL**(**Research feature,no real kernel**, NVIDIA's variant, data_type mxfp,act_data_type mxfp_rceil,bits 4, act_bits 4, group_size 32)
+- **MXFP8**(**Research feature, no real kernel**, data_type mxfp,act_data_type mxfp_rceil,group_size 32)
+- **FPW8A16**(**Research feature, no real kernel**, data_type fp8,group_size 0->per tensor )
+- **FP8_STATIC**(**Research feature, no real kernel**, data_type:fp8,act_data_type:fp8,group_size -1 ->per channel, act_group_size=0->per tensor)
Besides, you could modify the `group_size`, `bits`, `sym` and many other configs you want, though there are maybe no real kernels.
diff --git a/test/test_cpu/test_act_quantization.py b/test/test_cpu/test_act_quantization.py
index 31ba51f1b..8ecce2152 100644
--- a/test/test_cpu/test_act_quantization.py
+++ b/test/test_cpu/test_act_quantization.py
@@ -158,7 +158,7 @@ class TestAutoRoundAct(unittest.TestCase):
# check inblock layer config values
kproj_config = model.config.quantization_config.extra_config["model.decoder.layers.1.self_attn.k_proj"]
- assert "act_data_type" in kproj_config.keys() and kproj_config["act_data_type"] == "mx_fp_rceil"
+ assert "act_data_type" in kproj_config.keys() and kproj_config["act_data_type"] == "mx_fp"
assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 8
assert "act_group_size" in kproj_config.keys() and kproj_config["act_group_size"] == 32
assert "act_sym" in kproj_config.keys() and kproj_config["act_sym"]
diff --git a/test/test_cpu/test_scheme.py b/test/test_cpu/test_scheme.py
index c2d165639..15a8001b9 100644
--- a/test/test_cpu/test_scheme.py
+++ b/test/test_cpu/test_scheme.py
@@ -58,7 +58,7 @@ class TestAutoRound(unittest.TestCase):
self.assertEqual(ar.bits, 4)
self.assertEqual(ar.act_bits, 4)
self.assertEqual(ar.data_type, "mx_fp")
- self.assertEqual(ar.act_data_type, "mx_fp_rceil")
+ self.assertEqual(ar.act_data_type, "mx_fp")
ar.quantize()
def test_vllm(self):
diff --git a/test/test_cuda/test_scheme.py b/test/test_cuda/test_scheme.py
index 1c603c7ed..636876679 100644
--- a/test/test_cuda/test_scheme.py
+++ b/test/test_cuda/test_scheme.py
@@ -42,7 +42,7 @@ class TestAutoRound(unittest.TestCase):
self.assertEqual(ar.bits, 4)
self.assertEqual(ar.act_bits, 4)
self.assertEqual(ar.data_type, "mx_fp")
- self.assertEqual(ar.act_data_type, "mx_fp_rceil")
+ self.assertEqual(ar.act_data_type, "mx_fp")
ar.quantize()
def test_fp8_static(self):
@@ -66,7 +66,7 @@ class TestAutoRound(unittest.TestCase):
self.assertEqual(ar.bits, 4)
self.assertEqual(ar.act_bits, 4)
self.assertEqual(ar.data_type, "mx_fp")
- self.assertEqual(ar.act_data_type, "mx_fp_rceil")
+ self.assertEqual(ar.act_data_type, "mx_fp")
ar.quantize()
def test_fp8_static_rtn(self):