PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
mxfp/nvfp/fp8 support torch compile in tuning (#789)
Wenhua Cheng5f87a06beaee
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex f2b44eaef..b98a6de15 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -518,10 +518,6 @@ class AutoRound(object):"Enabling it can reduce tuning cost by 20%, but it might throw an exception.")- # if self.act_bits <= 8 and self.enable_torch_compile:- # self.enable_torch_compile = False- # logger.warning("reset enable_torch_compile to `False` as activation quantization is enabled")-if self.low_cpu_mem_usage and self.enable_torch_compile:self.enable_torch_compile = Falselogger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex 0c6ae265c..862ff0a9a 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -77,7 +77,6 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):return tensordef quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs):"""Quantize the given tensor using the specified parameters.@@ -128,7 +127,6 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_roundinreturn tensor.to(orig_dtype), shared_exp.to(orig_dtype), Nonedef quant_mx_rceil(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs):diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex 298d39e3d..cec9ee894 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -67,6 +67,7 @@ def calculate_gparam(tensor, group_size=16, device="cpu"):return global_scaledef ref_nvfp4_quant(x, global_scale, block_size=16, v=0):assert global_scale.dtype == torch.float32assert x.ndim == 2@@ -196,7 +197,6 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):assert (not isinstance(global_scale, torch.Tensor)) or global_scale.dtype == torch.float32assert x.ndim == 2m, n = x.shape- device = x.deviceif isinstance(max_scale, torch.Tensor):max_scale = max_scale.unsqueeze(dim=-1).to(x.device)vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * max_scalediff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 371104b9c..109c92093 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -170,6 +170,7 @@ def ceil_ste(x: torch.Tensor):return (x.ceil() - x).detach() + x+@torch._dynamo.disable()def float8_e4m3fn_ste(x: torch.Tensor):"""Straight-Through Estimator (STE) for float8.diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pyindex c0d73f60a..2c148e744 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py@@ -47,7 +47,6 @@ __all__ = []def pack_layer(name, model, backend):if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this featurereturn