Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

mxfp/nvfp/fp8 support torch compile in tuning (#789)

Wenhua Cheng5f87a06beaee
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index f2b44eaef..b98a6de15 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -518,10 +518,6 @@ class AutoRound(object):
"Enabling it can reduce tuning cost by 20%, but it might throw an exception."
)
- # if self.act_bits <= 8 and self.enable_torch_compile:
- # self.enable_torch_compile = False
- # logger.warning("reset enable_torch_compile to `False` as activation quantization is enabled")
-
if self.low_cpu_mem_usage and self.enable_torch_compile:
self.enable_torch_compile = False
logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index 0c6ae265c..862ff0a9a 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -77,7 +77,6 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):
return tensor
def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs):
"""Quantize the given tensor using the specified parameters.
@@ -128,7 +127,6 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_roundin
return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None
def quant_mx_rceil(
tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs
):
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index 298d39e3d..cec9ee894 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -67,6 +67,7 @@ def calculate_gparam(tensor, group_size=16, device="cpu"):
return global_scale
def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
assert global_scale.dtype == torch.float32
assert x.ndim == 2
@@ -196,7 +197,6 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):
assert (not isinstance(global_scale, torch.Tensor)) or global_scale.dtype == torch.float32
assert x.ndim == 2
m, n = x.shape
- device = x.device
if isinstance(max_scale, torch.Tensor):
max_scale = max_scale.unsqueeze(dim=-1).to(x.device)
vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * max_scale
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 371104b9c..109c92093 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -170,6 +170,7 @@ def ceil_ste(x: torch.Tensor):
return (x.ceil() - x).detach() + x
+@torch._dynamo.disable()
def float8_e4m3fn_ste(x: torch.Tensor):
"""Straight-Through Estimator (STE) for float8.
diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
index c0d73f60a..2c148e744 100644
--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
@@ -47,7 +47,6 @@ __all__ = [
]
def pack_layer(name, model, backend):
if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature
return