PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: Add NVFP4_E5M3_GS scheme with global scale support (#2531)
Xin He91a9bbfc8dd4
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 00aaa4e84..46b3e28d3 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -265,10 +265,10 @@ class AlgorithmComposer:Returns a list of hook handles that the caller must remove when done."""- from auto_round.compressors.utils import is_nv_fpfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size+ from auto_round.schemes import needs_nvfp_global_scale as is_nv_fp- is_act_nv_fp = getattr(self.block_quantizer.config, "is_act_nv_fp", False)+ is_act_nv_fp = is_nv_fp(getattr(self.block_quantizer.config, "act_data_type", None) or "")def collect_act_max(module, input, output):input = input[0] if isinstance(input, (tuple, list)) else input@@ -350,8 +350,8 @@ class AlgorithmComposer:q_inputs: Optional list of quantized input tensors; used instead of``fp_inputs`` when provided."""- from auto_round.compressors.utils import is_nv_fpfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size+ from auto_round.schemes import needs_nvfp_global_scale as is_nv_fptarget_input = q_inputs or fp_inputsact_group_size = layer.act_group_size@@ -500,10 +500,10 @@ class AlgorithmComposer:# ── Step 3.5: MoE scale alignment + global scale update ─────────────────# Must run after calibration hooks (act_max collected) and before quantize_block.if act_data_type is not None or not act_dynamic:- from auto_round.compressors.utils import is_nv_fp+ from auto_round.schemes import needs_nvfp_global_scalefrom auto_round.utils import set_amax_for_all_moe_layers- if is_nv_fp(act_data_type) or not act_dynamic:+ if needs_nvfp_global_scale(act_data_type) or not act_dynamic:set_amax_for_all_moe_layers(block, attr_name="act_max")# SVDQuant may have replaced the original linears during pre-quantize.@@ -583,13 +583,13 @@ class AlgorithmComposer:"""# Attach act_max for static activation quantization when inputs are available.if fp_inputs is not None:- from auto_round.compressors.utils import is_nv_fp+ from auto_round.schemes import needs_nvfp_global_scaleact_data_type = layer.act_data_typeif act_data_type is None:act_data_type = "fp"act_dynamic = getattr(layer, "act_dynamic", True)- if is_nv_fp(act_data_type) or not act_dynamic:+ if needs_nvfp_global_scale(act_data_type) or not act_dynamic:self._attach_act_max_for_outside_layer(layer, fp_inputs, q_inputs)# Infrastructure: move layer to the tuning device before handing off to the quantizer.diff --git a/auto_round/algorithms/quantization/config.py b/auto_round/algorithms/quantization/config.pyindex a570ec769..14416b084 100644--- a/auto_round/algorithms/quantization/config.py+++ b/auto_round/algorithms/quantization/config.py@@ -16,7 +16,7 @@ from typing import ClassVarfrom auto_round.algorithms.config import AlgorithmConfigfrom auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIGfrom auto_round.logger import logger-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import QuantizationScheme, is_nvfp4_e5m3_familyclass QuantizationConfig(AlgorithmConfig):@@ -161,6 +161,7 @@ class QuantizationConfig(AlgorithmConfig):if (self.is_act_quantizeand (not self.is_act_nv_fp or "static_gs" not in self.act_data_type)+ and not is_nvfp4_e5m3_family(self.act_data_type or "")and not self.is_act_mx_fpand not self.is_dynamic_wint8aint8and not self.is_static_afp8diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex 15c429dd9..ff95463ac 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -31,6 +31,7 @@ from auto_round.compressors.utils import (snapshot_best_params,)from auto_round.logger import logger+from auto_round.schemes import is_nvfp4_e5m3_gsfrom auto_round.utils.perf_counters import _PerfCounters@@ -769,7 +770,7 @@ class SignRoundQuantizer(BaseQuantizer):_perf.stop("tail")_log_tune_phases(_perf, self.iters)- if self.config.is_act_nv_fp:+ if self.config.is_act_nv_fp or is_nvfp4_e5m3_gs(self.config.act_data_type or ""):# enable moe experts act_max automatic generation for WrapperWALayerset_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")diff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.pyindex 5ef7e1d71..3cae1a44d 100644--- a/auto_round/algorithms/quantization/utils.py+++ b/auto_round/algorithms/quantization/utils.py@@ -14,8 +14,9 @@import torch-from auto_round.compressors.utils import check_need_act_calibration, is_nv_fp+from auto_round.compressors.utils import check_need_act_calibrationfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.schemes import needs_nvfp_global_scalefrom auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized@@ -27,7 +28,9 @@ def register_act_max_hooks(quantizer, model):if input.numel() == 0:returnmodule_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)- is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else quantizer.config.is_act_nv_fp+ is_module_act_nv_fp = needs_nvfp_global_scale(+ module_act_data_type or getattr(quantizer.config, "act_data_type", None) or ""+ )input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)act_max = torch.max(torch.abs(input), dim=-1).valuesif not hasattr(module, "act_max") or module.act_max.numel() == 0:diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex 2462b1d57..8e7ffdb72 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -48,6 +48,8 @@ weights with ``fake`` format):* Preset name: ``NVFP4_E5M3``.* ``data_type="nvfp4_v2"``, ``group_size=16``, with high-precision QDQ weights.+* Preset name: ``NVFP4_E5M3_GS`` (``data_type="nvfp4_v2_gs"``) adds a per-tensor weight global+ scale; model-free mode writes ``input_global_scale=1``.Schemes that require special packing (FP8, standard NVFP4, GGUF, INT8_W8A8,BF16, FPW8A16, ...) are **not** supported in model-free mode and will raise@@ -188,6 +190,7 @@ SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = ("MXFP4","MXFP8","NVFP4_E5M3",+ "NVFP4_E5M3_GS","BF16",)@@ -199,6 +202,8 @@ _SUPPORTED_INT_BITS: tuple[int, ...] = (2, 4, 8)_SUPPORTED_MXFP_BITS: tuple[int, ...] = (4, 8)_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"+_NVFP4_E5M3_GS_DATA_TYPE = "nvfp4_v2_gs"+_NVFP4_E5M3_FAMILY_DATA_TYPES = (_NVFP4_E5M3_DATA_TYPE, _NVFP4_E5M3_GS_DATA_TYPE)# Standalone quantization metadata used by different source formats. Only the# newly generated quantization_config.json may remain in model-free output.@@ -1756,11 +1761,13 @@ class _ModelFreeCompressorCore:packing_format = "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"elif data_type == _NVFP4_E5M3_DATA_TYPE:packing_format = "fake" if self.format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"+ elif data_type == _NVFP4_E5M3_GS_DATA_TYPE:+ packing_format = "fake" if self.format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3_gs"else:packing_format = "fake" if self.format == "fake" else "auto_round:auto_gptq"supports_opt_rtn = (is_mx_fp(data_type)- or data_type == _NVFP4_E5M3_DATA_TYPE+ or data_type in _NVFP4_E5M3_FAMILY_DATA_TYPESor _layer_config_has_mxfp(self.layer_config)or _layer_config_has_nvfp4(self.layer_config))@@ -2218,7 +2225,10 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):normalized_scheme is not None and is_mx_fp((normalized_scheme.data_type or "").lower())) or self._auto_scheme_family == "mx_fp":_accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}- elif normalized_scheme is not None and (normalized_scheme.data_type or "").lower() == _NVFP4_E5M3_DATA_TYPE:+ elif (+ normalized_scheme is not None+ and (normalized_scheme.data_type or "").lower() in _NVFP4_E5M3_FAMILY_DATA_TYPES+ ):_accepted_formats = {"fake", "llm_compressor", "auto_round", "auto_round:auto_gptq"}elif _is_full_precision_default(self.scheme_input) and _layer_config_has_mxfp(self.layer_config_input):# BF16 default with MXFP layer_config overrides.diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.pyindex 7e5e20ec9..e34c31ced 100644--- a/auto_round/compressors/orchestrator.py+++ b/auto_round/compressors/orchestrator.py@@ -34,11 +34,11 @@ from auto_round.compressors.base import BaseOrchestratorfrom auto_round.compressors.utils import (_get_quantized_layer_names_outside_blocks,immediate_pack,- is_nv_fp,)from auto_round.data_type.utils import update_block_global_scale_if_neededfrom auto_round.logger import loggerfrom auto_round.modeling.fused_moe.replace_modules import materialize_model_+from auto_round.schemes import needs_nvfp_global_scalefrom auto_round.utils import (SUPPORTED_LAYER_TYPES,check_to_quantized,@@ -726,7 +726,7 @@ class CompressionOrchestrator(BaseOrchestrator):)_zs_block_idx += 1# ── MoE scale alignment for FP8 dispatch efficiency ────────────────- if is_nv_fp(self.act_data_type) or not self.act_dynamic:+ if needs_nvfp_global_scale(self.act_data_type or "") or not self.act_dynamic:set_amax_for_all_moe_layers(block, attr_name="act_max")update_block_global_scale_if_needed(block, self.data_type, self.group_size)diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex 624e91b0c..435353444 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -53,14 +53,15 @@ FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max if hasattr(torch, "float8FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min if hasattr(torch, "float8_e4m3fn") else -448-def calculate_gparam(tensor, group_size=16, device="cpu"):+def calculate_gparam(tensor, group_size=16, device="cpu", scale_max=None):"""Calculate global scaling factor for NVFP quantization."""assert group_size == 16, f"Only group_size=16 is supported, got {group_size}"+ scale_max = FLOAT8_E4M3_MAX if scale_max is None else scale_maxif isinstance(tensor, (float, int)):tensor_amax = torch.tensor(tensor, device=device, dtype=torch.float32).abs()elif isinstance(tensor, torch.Tensor):tensor_amax = tensor.to(torch.float32).abs().max()- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)+ global_scale = scale_max * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)return global_scale@@ -233,6 +234,59 @@ def nvfp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):return qdq_res.to(orig_dtype), scale, None+def calculate_gparam_e5m3(tensor, group_size=16, device="cpu"):+ """Calculate the per-tensor global scale for NVFP4 E5M3 (UE5M3 block scales)."""+ return calculate_gparam(tensor, group_size, device, scale_max=FLOAT8_UE5M3_MAX)+++def calculate_gparam_by_dtype(tensor, data_type, group_size=16, device="cpu"):+ if "nvfp4_v2" in str(data_type).lower():+ return calculate_gparam_e5m3(tensor, group_size, device)+ return calculate_gparam(tensor, group_size, device)+++def _to_global_scale_tensor(global_scale, device):+ if not isinstance(global_scale, torch.Tensor):+ global_scale = torch.tensor(global_scale, dtype=torch.float32)+ return global_scale.to(device=device, dtype=torch.float32).reshape(-1)[:1]+++@register_dtype("nvfp4_v2_gs")+def nvfp4_v2_gs(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, **kwargs):+ """FP4 E2M1 with UE5M3 block scales and a per-tensor FP32 global scale."""+ assert group_size == 16, f"NVFP4_E5M3_GS only supports group_size=16, got {group_size}"+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if global_scale is None:+ global_scale = calculate_gparam_e5m3(tensor, group_size)+ global_scale = _to_global_scale_tensor(global_scale, tensor.device)+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.reshape(-1).to(tensor.device)+ qdq_res, scale = ref_fp4_quant(tensor, global_scale, group_size, v, max_scale)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), scale, None+++@register_dtype("nvfp4_v2_with_static_gs")+def nvfp4_v2_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, global_scale=None, **kwargs):+ """Activation QDQ for NVFP4_E5M3_GS; the global scale comes from calibrated ``tensor_max``."""+ if tensor is None or tensor.numel() == 0:+ return tensor, None, None+ assert group_size == 16, f"NVFP4_E5M3_GS only supports group_size=16, got {group_size}"+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if global_scale is None:+ if tensor_max is None:+ tensor_max = tensor.to(torch.float32).abs().max()+ elif isinstance(tensor_max, torch.Tensor):+ tensor_max = tensor_max.to(device=tensor.device, dtype=torch.float32).abs().max()+ global_scale = calculate_gparam_e5m3(tensor_max, group_size, tensor.device)+ global_scale = _to_global_scale_tensor(global_scale, tensor.device)+ qdq_res, scale = ref_fp4_quant(tensor, global_scale, group_size, v)+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_res.to(orig_dtype), scale, None++@register_dtype("nv_fp4_rtn")def nv_fp4_rtn(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, init_scale=1.0, **kwargs):orig_dtype = tensor.dtype@@ -409,6 +463,33 @@ def opt_rtn_fast_nvfp4(return nv_fp4_rtn(tensor, bits, group_size, v, global_scale, max_scale, init_scale=init_scale)+@register_dtype("opt_rtn_nvfp4_v2_gs")+def opt_rtn_nvfp4_v2_gs(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, imatrix=1.0, **kwargs):+ grouped, _, _ = reshape_pad_tensor_by_group_size(tensor, group_size)+ if not isinstance(imatrix, torch.Tensor):+ qw = 1.0+ else:+ imatrix = imatrix.reshape(1, -1)+ imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)+ imatrix = imatrix.expand(grouped.numel() // imatrix.numel(), -1)+ imatrix = imatrix.reshape(grouped.shape)+ qw = _imatrix_handle_zero(imatrix, grouped, bits, group_size)+ if global_scale is None:+ global_scale = calculate_gparam_e5m3(grouped, group_size)++ def _candidate(t, bits=4, group_size=16, v=0, max_scale=1.0):+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.reshape(-1)+ return nvfp4_v2_gs(t, bits, group_size, v, global_scale=global_scale, max_scale=max_scale)++ init_scale = search_nvfp4_scale(grouped, bits, qw, quant_func=_candidate, group_size=group_size).reshape(-1)+ if isinstance(max_scale, torch.Tensor):+ init_scale = init_scale * max_scale.reshape(-1).to(init_scale.device)+ else:+ init_scale = init_scale * max_scale+ return nvfp4_v2_gs(tensor, bits, group_size, v, global_scale=global_scale, max_scale=init_scale)++@register_dtype("rtn_nv_fp4_with_static_gs")def rtn_nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):if tensor is None or tensor.numel() == 0:diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 64f821cf2..e8eae2b94 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -22,8 +22,8 @@ import torchfrom torch.nn import Linear, Modulefrom auto_round import envs-from auto_round.compressors.utils import is_nv_fpfrom auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE+from auto_round.schemes import needs_nvfp_global_scalefrom auto_round.utils import check_to_quantized, logger@@ -563,9 +563,9 @@ def update_fused_layer_global_scales(def update_block_global_scale_if_needed(block, data_type, group_size):- from auto_round.data_type.nvfp import calculate_gparam+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype- has_nvfp = is_nv_fp(data_type)+ has_nvfp = needs_nvfp_global_scale(data_type)# Calculate block wise weight global scalefor _, m in block.named_modules():@@ -574,10 +574,10 @@ def update_block_global_scale_if_needed(block, data_type, group_size):# Check per-layer data_type for mixed-scheme scenariosmodule_data_type = getattr(m, "data_type", data_type)module_group_size = getattr(m, "group_size", group_size)- if is_nv_fp(module_data_type):+ if needs_nvfp_global_scale(module_data_type):has_nvfp = Trueif not hasattr(m, "weight_global_scale"):- weight_global_scale = calculate_gparam(m.weight, module_group_size)+ weight_global_scale = calculate_gparam_by_dtype(m.weight, module_data_type, module_group_size)m.weight_global_scale = weight_global_scaleif not has_nvfp:diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.pyindex e638f95bb..429f03afa 100644--- a/auto_round/experimental/qmodules/__init__.py+++ b/auto_round/experimental/qmodules/__init__.py@@ -21,5 +21,10 @@ from auto_round.experimental.qmodules.mx import (from auto_round.experimental.qmodules.fake import FakeActQuantLinearfrom auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinear-from auto_round.experimental.qmodules.nvfp4_e5m3 import CuteNVFP4E5M3QuantLinear, NVFP4E5M3QuantLinear+from auto_round.experimental.qmodules.nvfp4_e5m3 import (+ CuteNVFP4E5M3GSQuantLinear,+ CuteNVFP4E5M3QuantLinear,+ NVFP4E5M3GSQuantLinear,+ NVFP4E5M3QuantLinear,+)from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLineardiff --git a/auto_round/experimental/qmodules/fake.py b/auto_round/experimental/qmodules/fake.pyindex 322ab9aeb..b17d04485 100644--- a/auto_round/experimental/qmodules/fake.py+++ b/auto_round/experimental/qmodules/fake.py@@ -17,7 +17,7 @@ import torchfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental.qmodules.base import QModuleBase-from auto_round.schemes import QuantizationScheme, is_nv_fp+from auto_round.schemes import QuantizationScheme, needs_nvfp_global_scale__all__ = ["FakeActQuantLinear"]@@ -45,7 +45,7 @@ class FakeActQuantLinear(QModuleBase):self.register_parameter("bias", None)else:self.bias = torch.nn.Parameter(bias, requires_grad=False)- if is_nv_fp(config.act_data_type or ""):+ if needs_nvfp_global_scale(config.act_data_type or ""):self.register_buffer("input_global_scale", torch.ones(1, dtype=torch.float32))@classmethoddiff --git a/auto_round/experimental/qmodules/nvfp4_e5m3.py b/auto_round/experimental/qmodules/nvfp4_e5m3.pyindex 42f2b5528..ce3240104 100644--- a/auto_round/experimental/qmodules/nvfp4_e5m3.py+++ b/auto_round/experimental/qmodules/nvfp4_e5m3.py@@ -16,18 +16,25 @@ import osimport torch-from auto_round.data_type.nvfp import e5m3_to_float_tensor, nvfp4_v2+from auto_round.data_type.nvfp import e5m3_to_float_tensor, get_reciprocal, nvfp4_v2, nvfp4_v2_with_static_gsfrom auto_round.experimental.qmodules.base import QModuleBasefrom auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8from auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round_extension.cuda.cute_nvfp4_e5m3 import (+ try_cute_nvfp4_e5m3_gs_weight_dq,try_cute_nvfp4_e5m3_linear,try_cute_nvfp4_e5m3_weight_dq,+ try_cute_nvfp4_v2_gs_qdq,try_cute_nvfp4_v2_qdq,)-__all__ = ["CuteNVFP4E5M3QuantLinear", "NVFP4E5M3QuantLinear"]+__all__ = [+ "CuteNVFP4E5M3GSQuantLinear",+ "CuteNVFP4E5M3QuantLinear",+ "NVFP4E5M3GSQuantLinear",+ "NVFP4E5M3QuantLinear",+]_CACHE_WEIGHT_ENV = "AR_NVFP4_E5M3_CACHE_HP_WEIGHT"@@ -158,3 +165,54 @@ class CuteNVFP4E5M3QuantLinear(NVFP4E5M3QuantLinear):if fused_output is not None:return fused_outputreturn super().forward(input)+++class NVFP4E5M3GSQuantLinear(NVFP4E5M3QuantLinear):+ """NVFP4 E5M3 linear with per-tensor FP32 weight and input global scales."""++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.register_buffer("weight_global_scale", torch.ones((1,), dtype=torch.float32))+ self.register_buffer("input_global_scale", torch.ones((1,), dtype=torch.float32))++ def load_state_dict(self, state_dict, strict=True, assign=False):+ for name in ("weight_global_scale", "input_global_scale"):+ if name in state_dict and state_dict[name].dtype != torch.float32:+ state_dict[name] = state_dict[name].to(torch.float32)+ return super().load_state_dict(state_dict, strict, assign)++ def dequant_weight_online(self) -> torch.Tensor:+ unpacked = unpack_fp4_from_uint8(self.weight_packed, self.out_features, self.in_features, dtype=self.dtype).to(+ torch.float32+ )+ weight_global_scale = self.weight_global_scale.to(device=unpacked.device, dtype=torch.float32)+ scale = (e5m3_to_float_tensor(self.weight_scale) * get_reciprocal(weight_global_scale)).reshape(-1, 1)+ return (unpacked.reshape(-1, self.group_size) * scale).reshape(self.out_features, self.in_features)++ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:+ original_dtype = activation.dtype+ qdq_activation, _, _ = nvfp4_v2_with_static_gs(+ activation.to(torch.float32),+ bits=self.config.act_bits,+ group_size=self.config.act_group_size,+ global_scale=self.input_global_scale,+ )+ return qdq_activation.to(original_dtype)+++class CuteNVFP4E5M3GSQuantLinear(NVFP4E5M3GSQuantLinear):+ """NVFP4 E5M3 GS linear that dispatches activation QDQ and weight dequantization to CuTe."""++ def dequant_weight_online(self) -> torch.Tensor:+ cute_weight = try_cute_nvfp4_e5m3_gs_weight_dq(+ self.weight_packed, self.weight_scale, self.weight_global_scale, self.dtype+ )+ if cute_weight is not None:+ return cute_weight+ return super().dequant_weight_online()++ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:+ cute_qdq_activation = try_cute_nvfp4_v2_gs_qdq(activation, self.input_global_scale, self.config.act_group_size)+ if cute_qdq_activation is not None:+ return cute_qdq_activation+ return super().qdq_input(activation)diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.pyindex ef87ffab3..8f76d0fdb 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py@@ -24,7 +24,7 @@ import torch.nn as nnimport transformersfrom tqdm import tqdm-from auto_round.compressors.utils import is_mx_fp, is_nv_fp+from auto_round.compressors.utils import is_mx_fpfrom auto_round.export.export_to_autoround.utils import check_neq_configfrom auto_round.export.utils import (filter_quantization_config,@@ -34,7 +34,7 @@ from auto_round.export.utils import (save_pretrained_artifact,)from auto_round.logger import logger-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import QuantizationScheme, needs_nvfp_global_scalefrom auto_round.utils import (SUPPORTED_LAYER_TYPES,check_start_with_block_name,@@ -77,13 +77,13 @@ def pack_layer(name, model, backend, device=None):group_size = layer.group_sizesym = layer.sym- if is_nv_fp(act_data_type) and act_bits <= 8:+ if needs_nvfp_global_scale(act_data_type) and act_bits <= 8:input_global_scale = getattr(layer, "input_global_scale", None)if input_global_scale is None:assert hasattr(layer, "act_max")- from auto_round.data_type.nvfp import calculate_gparam+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype- input_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu")+ input_global_scale = calculate_gparam_by_dtype(layer.act_max, act_data_type, layer.group_size, "cpu")layer.input_global_scale = input_global_scaledelattr(layer, "act_max")@@ -189,7 +189,7 @@ def save_quantized_as_fp(orig_layer = m.orig_layerset_module(model, n, orig_layer)- if is_nv_fp(act_data_type) and "static_gs" in str(act_data_type).lower():+ if needs_nvfp_global_scale(act_data_type or "") and "static_gs" in str(act_data_type).lower():# Ensure all MOE layers have act_max set (needed after deep copy or for uncalibrated layers)from auto_round.utils.model import is_moe_model, set_amax_for_all_moe_layers@@ -202,9 +202,12 @@ def save_quantized_as_fp(layer = mif hasattr(layer, "act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None):assert hasattr(layer, "act_max")- from auto_round.data_type.nvfp import calculate_gparam+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype- input_global_scale = calculate_gparam(layer.act_max, layer.group_size, model.device)+ layer_act_data_type = getattr(layer, "act_data_type", None) or act_data_type+ input_global_scale = calculate_gparam_by_dtype(+ layer.act_max, layer_act_data_type, layer.group_size, model.device+ )layer.input_global_scale = input_global_scaledelattr(layer, "act_max")# update fused input_global_scalediff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pyindex d9082e1e4..f0b8457bc 100644--- a/auto_round/export/export_to_autoround/qlinear_fp.py+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -38,6 +38,7 @@ from auto_round.compressors.utils import BackendDataType, is_mx_fp, is_nv_fpfrom auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMALfrom auto_round.data_type.nvfp import cast_to_fp4, float_to_e5m3_frexp, get_reciprocalfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.schemes import is_nvfp4_e5m3_gsfrom auto_round.utils import get_packing_device, logger# from auto_round.utils import get_weight_compress_dtype@@ -74,6 +75,8 @@ class QuantLinear(nn.Module):self.is_mx = is_mx_fp(data_type)self.is_nv = is_nv_fp(data_type)self.is_nvfp4_e5m3 = data_type == "nvfp4_v2"+ self.is_nvfp4_e5m3_gs = is_nvfp4_e5m3_gs(data_type)+ self.has_global_scale = self.is_nv or self.is_nvfp4_e5m3_gsif self.is_mx:if group_size != 32:raise NotImplementedError(f"Only group_size 32 are supported for {BackendDataType.MX_FP} data type.")@@ -81,7 +84,7 @@ class QuantLinear(nn.Module):raise NotImplementedError(f"in_feature must be divisible by {group_size} for {BackendDataType.MX_FP} data type.")- if self.is_nv or self.is_nvfp4_e5m3:+ if self.is_nv or self.is_nvfp4_e5m3 or self.is_nvfp4_e5m3_gs:if group_size % 16 != 0:raise NotImplementedError(f"Only group_size 16 are supported for {BackendDataType.NV_FP} data type.")if infeatures % group_size != 0:@@ -112,7 +115,7 @@ class QuantLinear(nn.Module):dtype=torch.float16, ## TODO update to correct scale dtype for different bits),)- if self.is_nv and self.bits == 4:+ if self.has_global_scale and self.bits == 4:self.register_buffer("weight_global_scale",torch.zeros(@@ -120,7 +123,7 @@ class QuantLinear(nn.Module):dtype=torch.float32,),)- if self.is_nv and self.act_bits <= 8:+ if self.has_global_scale and self.act_bits <= 8:self.register_buffer("input_global_scale",torch.zeros(@@ -151,7 +154,7 @@ class QuantLinear(nn.Module):tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(W, self.group_size)scales = scales.to(device)- if self.is_nv:+ if self.has_global_scale:assert global_scale is not None and global_scale.numel() == 1global_scale = global_scale.reshape([1])global_scale = global_scale.to(device)@@ -169,7 +172,7 @@ class QuantLinear(nn.Module):scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)if self.is_mx:final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)- elif self.is_nvfp4_e5m3:+ elif self.is_nvfp4_e5m3 or self.is_nvfp4_e5m3_gs:final_scale = float_to_e5m3_frexp(scales.to(torch.float32))else:final_scale = scales.to(torch.float8_e4m3fn)diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.pyindex 99eb1196d..c974c65c3 100644--- a/auto_round/export/export_to_llmcompressor/config.py+++ b/auto_round/export/export_to_llmcompressor/config.py@@ -103,8 +103,8 @@ def initialize_quantization(scheme, targets=None, config_groups=None, kv_cache_s)-def initialize_nvfp4_e5m3_quantization(ignore=None):- """Build stable compressed-tensors metadata for global-scale-free NVFP4 E5M3."""+def initialize_nvfp4_e5m3_quantization(ignore=None, with_global_scale=False):+ """Build stable compressed-tensors metadata for NVFP4 E5M3 (optionally with per-tensor global scales)."""def quant_args(dynamic):return {@@ -129,7 +129,7 @@ def initialize_nvfp4_e5m3_quantization(ignore=None):"weights": quant_args(False),}},- "format": "nvfp4-e5m3-pack-quantized",+ "format": "nvfp4-e5m3-gs-pack-quantized" if with_global_scale else "nvfp4-e5m3-pack-quantized","global_compression_ratio": None,"ignore": list(dict.fromkeys(ignore or ["lm_head"])),"kv_cache_scheme": None,diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex bafa994b3..137cd0ab1 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -40,6 +40,7 @@ from auto_round.export.utils import (save_pretrained_artifact,)from auto_round.logger import logger+from auto_round.schemes import is_nvfp4_e5m3_family, is_nvfp4_e5m3_gs, needs_nvfp_global_scalefrom auto_round.utils import (SUPPORTED_LAYER_TYPES,check_start_with_block_name,@@ -91,13 +92,13 @@ def pack_layer(name, model, device=None):group_size = layer.group_sizesym = layer.sym- if is_nv_fp(act_data_type) and act_bits <= 8:+ if needs_nvfp_global_scale(act_data_type) and act_bits <= 8:input_global_scale = getattr(layer, "input_global_scale", None)if input_global_scale is None:assert hasattr(layer, "act_max")- from auto_round.data_type.nvfp import calculate_gparam+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype- input_global_scale = calculate_gparam(layer.act_max, layer.group_size) # , model.device+ input_global_scale = calculate_gparam_by_dtype(layer.act_max, act_data_type, layer.group_size)layer.input_global_scale = input_global_scaledelattr(layer, "act_max")@@ -150,6 +151,8 @@ def _get_scheme(bits, data_type, act_bits=16):return "NVFP4"if data_type == "nvfp4_v2":return "NVFP4_E5M3"+ if is_nvfp4_e5m3_gs(data_type):+ return "NVFP4_E5M3_GS"return None@@ -163,6 +166,8 @@ def _get_group_format(bits, data_type):return "nvfp4-pack-quantized"if data_type == "nvfp4_v2":return "nvfp4-e5m3-pack-quantized"+ if is_nvfp4_e5m3_gs(data_type):+ return "nvfp4-e5m3-gs-pack-quantized"return "float-quantized"@@ -356,16 +361,19 @@ def save_quantized_as_fp(orig_layer = m.orig_layerset_module(model, n, orig_layer)- if is_nv_fp(act_data_type) and "static_gs" in str(act_data_type).lower():+ if needs_nvfp_global_scale(act_data_type or "") and "static_gs" in str(act_data_type).lower():# generate static input_global_scalefor n, m in model.named_modules():if type(m) in SUPPORTED_LAYER_TYPES:layer = mif hasattr(layer, "act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None):assert hasattr(layer, "act_max")- from auto_round.data_type.nvfp import calculate_gparam+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype- input_global_scale = calculate_gparam(layer.act_max, layer.group_size, model.device)+ layer_act_data_type = getattr(layer, "act_data_type", None) or act_data_type+ input_global_scale = calculate_gparam_by_dtype(+ layer.act_max, layer_act_data_type, layer.group_size, model.device+ )layer.input_global_scale = input_global_scaledelattr(layer, "act_max")# update fused input_global_scale@@ -449,10 +457,12 @@ def save_quantized_as_fp(quantization_config.format = _get_group_format(layer_bits, layer_data_type)quantization_config = quantization_config.to_dict()quantization_config["provider"] = "auto-round"- elif data_type == "nvfp4_v2":+ elif is_nvfp4_e5m3_family(data_type):from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization- quantization_config = initialize_nvfp4_e5m3_quantization(ignore=ignore)+ quantization_config = initialize_nvfp4_e5m3_quantization(+ ignore=ignore, with_global_scale=is_nvfp4_e5m3_gs(data_type)+ )quantization_config["provider"] = "auto-round"else:scheme = _get_scheme(bits, data_type, act_bits if act_bits is not None else 16)diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.pyindex 867aff2e5..3bbe1effa 100644--- a/auto_round/export/formats/backends/autoround.py+++ b/auto_round/export/formats/backends/autoround.py@@ -46,6 +46,7 @@ class AutoRoundFormat(OutputFormat):"MXFP8","NVFP4","NVFP4_E5M3",+ "NVFP4_E5M3_GS","FPW8A16","W2A16G64","W2A16G32",@@ -76,6 +77,7 @@ class AutoRoundFormat(OutputFormat):scheme.is_nv_fp()or scheme.is_mx_fp()or scheme.data_type == BackendDataType.NVFP4_E5M3.value+ or scheme.data_type == BackendDataType.NVFP4_E5M3_GS.valueor (scheme.is_mx_int() and scheme.bits == 4) # only add mx_int4 now):self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)@@ -96,7 +98,8 @@ class AutoRoundFormat(OutputFormat):if format == "mlx":self.backend = MLXFormat("mlx", scheme, ctx)elif format.upper() not in list(BackendDataType.__members__.keys()) and format not in {- BackendDataType.NVFP4_E5M3.value+ BackendDataType.NVFP4_E5M3.value,+ BackendDataType.NVFP4_E5M3_GS.value,}:raise KeyError(f"Unsupported backend format auto_round:{format}, please check")else:@@ -141,7 +144,10 @@ class AutoRoundFormat(OutputFormat):backend = self.get_backend_name()- if self.output_format == f"auto_round:{BackendDataType.NVFP4_E5M3.value}":+ if self.output_format in (+ f"auto_round:{BackendDataType.NVFP4_E5M3.value}",+ f"auto_round:{BackendDataType.NVFP4_E5M3_GS.value}",+ ):from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layerreturn pack_layer(layer_name, model, device=device)@@ -201,6 +207,11 @@ class AutoRoundFormat(OutputFormat):backend = "auto_round:llm_compressor_nvfp4_e5m3"export_func = save_quantized_as_fp+ elif backend == f"auto_round:{BackendDataType.NVFP4_E5M3_GS.value}":+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp++ backend = "auto_round:llm_compressor_nvfp4_e5m3_gs"+ export_func = save_quantized_as_fpelif re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fpdiff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.pyindex b5e34caaa..bd01df5d0 100644--- a/auto_round/export/formats/backends/fake.py+++ b/auto_round/export/formats/backends/fake.py@@ -23,7 +23,7 @@ import torchfrom auto_round.export.formats.base import OutputFormatfrom auto_round.logger import logger-from auto_round.schemes import QuantizationScheme, is_nv_fp+from auto_round.schemes import QuantizationScheme, needs_nvfp_global_scalefrom auto_round.utils import unsupported_meta_device@@ -101,7 +101,10 @@ def _rewrite_saved_weights_without_orig_layer(output_dir: str) -> None:def _materialize_fake_input_global_scales(model: torch.nn.Module) -> None:"""Persist calibrated NVFP activation scales in fake-format checkpoints."""for layer in model.modules():- if not is_nv_fp(getattr(layer, "act_data_type", "") or "") or "input_global_scale" in layer._buffers:+ if (+ not needs_nvfp_global_scale(getattr(layer, "act_data_type", "") or "")+ or "input_global_scale" in layer._buffers+ ):continueinput_global_scale = getattr(layer, "input_global_scale", None)if not isinstance(input_global_scale, torch.Tensor):diff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.pyindex 749e41078..171ba8bc4 100644--- a/auto_round/export/formats/backends/llm_compressor.py+++ b/auto_round/export/formats/backends/llm_compressor.py@@ -31,6 +31,7 @@ class LLMCompressorFormat(OutputFormat):"MXFP8","NVFP4","NVFP4_E5M3",+ "NVFP4_E5M3_GS","FPW8A16","FP8_STATIC","INT8",@@ -61,11 +62,11 @@ class LLMCompressorFormat(OutputFormat):if re.search("^(auto_round:)?llm_compressor", format):self.output_format = formatself.backend = None- if scheme.data_type == "nvfp4_v2":+ if scheme.data_type in ("nvfp4_v2", "nvfp4_v2_gs"):from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supportedcheck_compressed_tensors_supported(raise_error=True)- self.output_format = "llm_compressor:nvfp4_v2"+ self.output_format = f"llm_compressor:{scheme.data_type}"elif scheme.is_nv_fp() or scheme.is_mx_fp() or (scheme.bits >= 16 and scheme.data_type in ("fp", "float")):from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported@@ -156,7 +157,7 @@ class LLMCompressorFormat(OutputFormat):)if scheme.act_bits <= 8 and (not scheme.is_act_standard_fp() or scheme.act_dynamic):- if scheme.act_data_type == "nvfp4_v2":+ if scheme.act_data_type in ("nvfp4_v2", "nvfp4_v2_with_static_gs"):return None, scheme, layer_config, quant_block_listif ((scheme.is_act_nv_fp() and "static_gs" in scheme.act_data_type)diff --git a/auto_round/export/formats/base.py b/auto_round/export/formats/base.pyindex 4c4b6d316..baf7977c4 100644--- a/auto_round/export/formats/base.py+++ b/auto_round/export/formats/base.py@@ -56,6 +56,7 @@ class BackendDataType(str, Enum):MXFP4 = "mxfp4"NVFP4 = "nvfp4"NVFP4_E5M3 = "nvfp4_v2"+ NVFP4_E5M3_GS = "nvfp4_v2_gs"FP8 = "fp8"MX_FP = "mx_fp"NV_FP = "nv_fp"diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex b25236007..b08f44352 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -240,6 +240,7 @@ AWQ_FORMAT = ["auto_round:auto_awq"]LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]FAKE_FORMAT = ["auto_round:fake", "auto_round:auto_gptq", "auto_round", "auto_round:auto_awq"]NVFP4_E5M3_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3"]+NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3_gs"]WOQ_DEFAULT_ACT_BITS = [None, 16, 32]# CPU backends that target Intel/x86 (ark / auto_round_kernel) cannot@@ -444,6 +445,44 @@ BackendInfos["auto_round:cute_nvfp4_e5m3"] = BackendInfo(requirements=["auto-round>0.12.0"],)+BackendInfos["auto_round:torch_nvfp4_e5m3_gs"] = BackendInfo(+ device=["xpu", "cuda", "cpu"],+ packing_format=NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT,+ sym=[True],+ compute_dtype=["float32", "float16", "bfloat16"],+ data_type=["nvfp4_v2_gs"],+ group_size=[16],+ bits=[4],+ act_bits=[4],+ act_group_size=[16],+ act_sym=[True],+ act_data_type=["nvfp4_v2_with_static_gs"],+ act_dynamic=[True],+ priority=3,+ checkers=[mxfp_nvfp_feature_checker],+ alias=["auto_round", "torch"],+ requirements=["auto-round>0.12.0"],+)++BackendInfos["auto_round:cute_nvfp4_e5m3_gs"] = BackendInfo(+ device=["cuda"],+ packing_format=NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT,+ sym=[True],+ compute_dtype=["float32", "float16", "bfloat16"],+ data_type=["nvfp4_v2_gs"],+ group_size=[16],+ bits=[4],+ act_bits=[4],+ act_group_size=[16],+ act_sym=[True],+ act_data_type=["nvfp4_v2_with_static_gs"],+ act_dynamic=[True],+ priority=6,+ checkers=[cute_nvfp4_e5m3_checker],+ alias=["cute_nvfp4_e5m3_gs"],+ requirements=["auto-round>0.12.0"],+)+BackendInfos["auto_round:tritonv2"] = BackendInfo(device=["cuda", "xpu"],data_type=["int"],@@ -920,6 +959,10 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):return ar_qmodules.MXINT4QuantLinearif "torch_mxfp4" in backend:return ar_qmodules.MXFP4QuantLinear+ if backend == "auto_round:cute_nvfp4_e5m3_gs":+ return ar_qmodules.CuteNVFP4E5M3GSQuantLinear+ if backend == "auto_round:torch_nvfp4_e5m3_gs":+ return ar_qmodules.NVFP4E5M3GSQuantLinearif backend == "auto_round:cute_nvfp4_e5m3":return ar_qmodules.CuteNVFP4E5M3QuantLinearif backend == "auto_round:torch_nvfp4_e5m3":diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex eec7967c9..945595b65 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -586,6 +586,8 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,or BackendDataType.MXINT4.value in layer_backendor layer_backend == "auto_round:torch_nvfp4_e5m3"or layer_backend == "auto_round:cute_nvfp4_e5m3"+ or layer_backend == "auto_round:torch_nvfp4_e5m3_gs"+ or layer_backend == "auto_round:cute_nvfp4_e5m3_gs"or layer_backend == "auto_round:fake"):return QuantLinear.from_original(config, layer)@@ -870,21 +872,26 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Mif isinstance(quantization_config, dict)else getattr(quantization_config, "format", None))- if config_format == "nvfp4-e5m3-pack-quantized":+ if config_format in ("nvfp4-e5m3-pack-quantized", "nvfp4-e5m3-gs-pack-quantized"):+ with_global_scale = config_format == "nvfp4-e5m3-gs-pack-quantized"config_dict = quantization_config if isinstance(quantization_config, dict) else quantization_config.to_dict()ignored = config_dict.get("ignore") or []# TODO: For experimental purpose, will delete when it's not necessary.quantization_config = SimpleNamespace(quant_method="auto-round",- packing_format="auto_round:llm_compressor_nvfp4_e5m3",+ packing_format=(+ "auto_round:llm_compressor_nvfp4_e5m3_gs"+ if with_global_scale+ else "auto_round:llm_compressor_nvfp4_e5m3"+ ),bits=4,group_size=16,sym=True,- data_type="nvfp4_v2",+ data_type="nvfp4_v2_gs" if with_global_scale else "nvfp4_v2",act_bits=4,act_group_size=16,act_sym=True,- act_data_type="nvfp4_v2",+ act_data_type="nvfp4_v2_with_static_gs" if with_global_scale else "nvfp4_v2",act_dynamic=True,extra_config={name: {"bits": 16, "act_bits": 16} for name in ignored},)diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 3b6de11c0..de0cdc140 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -189,6 +189,21 @@ def is_mx_int(backend: str) -> bool:return BackendDataType.MX_INT in backend.lower()+def is_nvfp4_e5m3_gs(backend: str) -> bool:+ """NVFP4 E5M3 with per-tensor global scale (``nvfp4_v2_gs`` / ``nvfp4_v2_with_static_gs``)."""+ backend = backend.lower()+ return "nvfp4_v2" in backend and backend.endswith("_gs")+++def is_nvfp4_e5m3_family(backend: str) -> bool:+ return "nvfp4_v2" in backend.lower()+++def needs_nvfp_global_scale(backend: str) -> bool:+ """Whether the data type uses NVFP-style per-tensor weight/input global scales."""+ return is_nv_fp(backend) or is_nvfp4_e5m3_gs(backend)++def is_standard_fp(backend: str) -> bool:backend = backend.lower()return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)@@ -785,6 +800,19 @@ NVFP4_E5M3 = QuantizationScheme.from_dict(})+NVFP4_E5M3_GS = QuantizationScheme.from_dict(+ {+ "bits": 4,+ "group_size": 16,+ "data_type": "nvfp4_v2_gs",+ "act_bits": 4,+ "act_data_type": "nvfp4_v2_with_static_gs",+ "act_group_size": 16,+ "act_sym": True,+ "act_dynamic": True,+ }+)+FPW8A16 = QuantizationScheme.from_dict({"bits": 8,@@ -895,6 +923,7 @@ PRESET_SCHEMES = {"MXFP8_RCEIL": MXFP8_RCEIL,"NVFP4": NVFP4,"NVFP4_E5M3": NVFP4_E5M3,+ "NVFP4_E5M3_GS": NVFP4_E5M3_GS,"FPW8A16": FPW8A16,# Group-size variants for the 2-8 bit int presets: the bare presets default# to group_size 128; these finer-group variants allow per-layer group-sizediff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 645553003..2a932d91a 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -439,7 +439,10 @@ def _patch_nvfp4_e5m3_compressed_tensors_quantizer():def is_nvfp4_e5m3(self):compression_config = getattr(getattr(self, "compressor", None), "quantization_config", None)- return getattr(compression_config, "format", None) == "nvfp4-e5m3-pack-quantized"+ return getattr(compression_config, "format", None) in (+ "nvfp4-e5m3-pack-quantized",+ "nvfp4-e5m3-gs-pack-quantized",+ )def patched_before(self, model, **kwargs):if not is_nvfp4_e5m3(self):diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex f388c4ec9..0b273c397 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -49,6 +49,9 @@ from auto_round.utils.path_safety import ()_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"+_NVFP4_E5M3_GS_DATA_TYPE = "nvfp4_v2_gs"+_NVFP4_E5M3_GS_ACT_DATA_TYPE = "nvfp4_v2_with_static_gs"+_NVFP4_E5M3_FAMILY_DATA_TYPES = (_NVFP4_E5M3_DATA_TYPE, _NVFP4_E5M3_GS_DATA_TYPE)# default_scheme flag restricting quantization to layers stored as NVFP4 in the source checkpoint._NVFP_SOURCE_ONLY_KEY = "_nvfp_source_only"_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")@@ -70,6 +73,7 @@ SUPPORTED_PRESET_SCHEMES = ("MXFP4","MXFP8","NVFP4_E5M3",+ "NVFP4_E5M3_GS","BF16",)@@ -1318,6 +1322,86 @@ def _quantize_weight_nvfp4_e5m3(return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}+def _qdq_weight_nvfp4_e5m3_gs(+ weight: torch.Tensor, layer_name: str, group_size: int, device: str, disable_opt_rtn: bool+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ """Return (fp32 weight, qdq weight, encoded-domain block scale, weight global scale) for NVFP4_E5M3_GS."""+ from auto_round.data_type.nvfp import calculate_gparam_e5m3, nvfp4_v2_gs, opt_rtn_nvfp4_v2_gs++ in_features = weight.shape[1]+ if group_size != 16 or in_features % group_size != 0:+ raise ValueError(+ f"NVFP4_E5M3_GS requires in_features divisible by group_size=16, got group_size={group_size}, "+ f"in_features={in_features} for '{layer_name}'."+ )+ weight_dev = weight.to(device=device, dtype=torch.float32)+ global_scale = calculate_gparam_e5m3(weight_dev, group_size).reshape(1)+ quant_func = nvfp4_v2_gs if disable_opt_rtn else opt_rtn_nvfp4_v2_gs+ qdq_weight, scale, _ = quant_func(weight_dev, bits=4, group_size=group_size, global_scale=global_scale)+ return weight_dev, qdq_weight, scale, global_scale+++def _quantize_weight_nvfp4_e5m3_gs(+ weight: torch.Tensor,+ layer_name: str,+ group_size: int = 16,+ device: str = "cpu",+ disable_opt_rtn: bool = False,+) -> dict[str, torch.Tensor]:+ """Fake-quantize a 2D weight tensor to NVFP4_E5M3_GS; model-free uses a fixed input global scale of 1."""+ _, qdq_weight, _, _ = _qdq_weight_nvfp4_e5m3_gs(weight, layer_name, group_size, device, disable_opt_rtn)+ return {+ f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu"),+ f"{layer_name}.input_global_scale": torch.ones(1, dtype=torch.float32),+ }+++def _pack_weight_nvfp4_e5m3_gs(+ weight: torch.Tensor,+ layer_name: str,+ group_size: int = 16,+ device: str = "cpu",+ disable_opt_rtn: bool = False,+) -> dict[str, torch.Tensor]:+ """Pack FP4 E2M1 weights with UE5M3 block scales and a per-tensor weight global scale."""+ from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear++ out_features, in_features = weight.shape+ weight_dev, _, scale, global_scale = _qdq_weight_nvfp4_e5m3_gs(+ weight, layer_name, group_size, device, disable_opt_rtn+ )+ scale = scale.reshape(out_features, in_features // group_size).to(torch.float32)+ with torch.device("meta"):+ linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=torch.float32)+ linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)+ with torch.device("meta"):+ qlayer = QuantLinear(+ 4,+ group_size,+ in_features,+ out_features,+ False,+ data_type=_NVFP4_E5M3_GS_DATA_TYPE,+ act_bits=4,+ act_data_type=_NVFP4_E5M3_GS_ACT_DATA_TYPE,+ )+ qlayer.pack(+ linear,+ scale,+ global_scale=global_scale,+ input_global_scale=torch.ones(1, dtype=torch.float32),+ device=device,+ )+ out = {+ f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),+ f"{layer_name}.weight_global_scale": qlayer.weight_global_scale.to("cpu"),+ f"{layer_name}.input_global_scale": qlayer.input_global_scale.to("cpu"),+ }+ del qlayer, linear, weight_dev, scale+ return out++def _quantize_weight_int_fake(weight: torch.Tensor,layer_name: str,@@ -1481,6 +1565,24 @@ def _quantize_single_tensor(logger.warning(f"Failed to NVFP4_E5M3-quantize {layer_name}: {e}. Keeping original weight.")return layer_name, {tensor_name: tensor}, None, layer_name+ if data_type == _NVFP4_E5M3_GS_DATA_TYPE:+ try:+ quantize_e5m3_gs = (+ _quantize_weight_nvfp4_e5m3_gs if scheme.get("_output_format") == "fake" else _pack_weight_nvfp4_e5m3_gs+ )+ out = quantize_e5m3_gs(+ weight=tensor,+ layer_name=layer_name,+ group_size=group_size,+ device=device,+ disable_opt_rtn=disable_opt_rtn,+ )+ logger.debug(f"Quantized (NVFP4_E5M3_GS): {layer_name} (bits=4, group_size={group_size})")+ return layer_name, out, layer_name, None+ except Exception as e:+ logger.warning(f"Failed to NVFP4_E5M3_GS-quantize {layer_name}: {e}. Keeping original weight.")+ return layer_name, {tensor_name: tensor}, None, layer_name+# ---- Integer WOQ path ----# opt_rtn is always disabled for integer WOQ in model-free mode because# the scale search does not improve accuracy for INT quantization here.@@ -2690,11 +2792,11 @@ def _get_llm_compressor_metadata() -> dict[str, str]:return {"provider": "auto-round"}-def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str]) -> dict:- """Build compressed-tensors metadata for NVFP4 E5M3 without global scales."""+def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str], with_global_scale: bool = False) -> dict:+ """Build compressed-tensors metadata for NVFP4 E5M3 (optionally with per-tensor global scales)."""from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization- qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignored_layers)+ qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignored_layers, with_global_scale=with_global_scale)qconfig.update(_get_llm_compressor_metadata())return qconfig@@ -2745,15 +2847,16 @@ def _get_mxfp_group_scheme_and_format(tmp_qconfig = initialize_quantization(scheme="NVFP4", ignore=ignore)return tmp_qconfig.config_groups["group_0"], "nvfp4-pack-quantized"- if group_data_type == _NVFP4_E5M3_DATA_TYPE:- # NVFP4_E5M3 is not a compressed-tensors preset; use its dedicated builder.- nvfp4_dict = initialize_nvfp4_e5m3_quantization(ignore=ignore)+ if group_data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:+ # NVFP4_E5M3(_GS) is not a compressed-tensors preset; use its dedicated builder.+ with_global_scale = group_data_type == _NVFP4_E5M3_GS_DATA_TYPE+ nvfp4_dict = initialize_nvfp4_e5m3_quantization(ignore=ignore, with_global_scale=with_global_scale)# Extract the QuantizationScheme object from the dict config.from compressed_tensors.quantization import QuantizationScheme # pylint: disable=E0401raw_scheme = nvfp4_dict["config_groups"]["group_0"]scheme_obj = QuantizationScheme.model_validate(raw_scheme)- return scheme_obj, "nvfp4-e5m3-pack-quantized"+ return scheme_obj, nvfp4_dict["format"]else:scheme_name = "MXFP4" if group_bits == 4 else "MXFP8"fmt = "mxfp4-pack-quantized" if group_bits == 4 else "mxfp8-quantized"@@ -2789,7 +2892,11 @@ def _build_mxfp_quantization_config(default_data_type = (default_scheme.get("data_type") or "mx_fp").lower()is_fp_default = (bits or 0) >= 16 # BF16/FP16 full-precision default- if not is_fp_default and bits not in _SUPPORTED_MXFP_BITS and default_data_type != _NVFP4_E5M3_DATA_TYPE:+ if (+ not is_fp_default+ and bits not in _SUPPORTED_MXFP_BITS+ and default_data_type not in _NVFP4_E5M3_FAMILY_DATA_TYPES+ ):raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")# Default ignore list: any layer present in ignored_layers (deduped) that@@ -2825,13 +2932,15 @@ def _build_mxfp_quantization_config(actual_bits, actual_dt = next(iter(scheme_groups.keys()))else:actual_bits, actual_dt = bits, default_data_type- if actual_dt != _NVFP4_E5M3_DATA_TYPE and actual_bits not in _SUPPORTED_MXFP_BITS:+ if actual_dt not in _NVFP4_E5M3_FAMILY_DATA_TYPES and actual_bits not in _SUPPORTED_MXFP_BITS:raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")group_scheme, fmt = _get_mxfp_group_scheme_and_format(actual_bits, actual_dt, ignore)- if actual_dt == _NVFP4_E5M3_DATA_TYPE:+ if actual_dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization- qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignore)+ qconfig = initialize_nvfp4_e5m3_quantization(+ ignore=ignore, with_global_scale=actual_dt == _NVFP4_E5M3_GS_DATA_TYPE+ )if is_fp_default and scheme_groups:targets = _add_routed_experts_if_moe(list(quantized_layers), quantized_layers)qconfig["config_groups"]["group_0"]["targets"] = targets@@ -3049,79 +3158,45 @@ def _layer_config_has_nvfp4(layer_config: dict | None) -> bool:with a ``"scheme"`` key or a ``"data_type"`` key), or:class:`QuantizationScheme` instances."""- if not layer_config:- return False- for val in layer_config.values():- if isinstance(val, str):- try:- s = _normalize_scheme(val.upper())- dt = (s.data_type or "").lower()- if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):- return True- except Exception:- pass- elif isinstance(val, dict):- dt = (val.get("data_type") or "").lower()- if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):- return True- scheme_val = val.get("scheme")- if isinstance(scheme_val, str):- try:- s = _normalize_scheme(scheme_val.upper())- dt = (s.data_type or "").lower()- if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):- return True- except Exception:- pass- elif isinstance(val, QuantizationScheme):- dt = (val.data_type or "").lower()- if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):- return True- return False+ return _get_layer_config_nvfp4_dt(layer_config) is not Nonedef _get_layer_config_nvfp4_dt(layer_config: dict | None) -> str | None:"""Return the first NVFP4 data_type found in *layer_config*.Returns ``"nv_fp"`` for standard NVFP4, ``"nvfp4_v2"`` for NVFP4_E5M3,- or ``None`` when no NVFP4 entry is present.+ ``"nvfp4_v2_gs"`` for NVFP4_E5M3_GS, or ``None`` when no NVFP4 entry is present."""++ def _classify(dt: str) -> str | None:+ if dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:+ return dt+ if is_nv_fp(dt):+ return "nv_fp"+ return None++ def _scheme_dt(name: str) -> str:+ try:+ return (_normalize_scheme(name.upper()).data_type or "").lower()+ except Exception:+ return ""+if not layer_config:return Nonefor val in layer_config.values():+ candidates = []if isinstance(val, str):- try:- s = _normalize_scheme(val.upper())- dt = (s.data_type or "").lower()- if dt == _NVFP4_E5M3_DATA_TYPE:- return _NVFP4_E5M3_DATA_TYPE- if is_nv_fp(dt):- return "nv_fp"- except Exception:- pass+ candidates.append(_scheme_dt(val))elif isinstance(val, dict):- dt = (val.get("data_type") or "").lower()- if dt == _NVFP4_E5M3_DATA_TYPE:- return _NVFP4_E5M3_DATA_TYPE- if is_nv_fp(dt):- return "nv_fp"- scheme_val = val.get("scheme")- if isinstance(scheme_val, str):- try:- s = _normalize_scheme(scheme_val.upper())- dt = (s.data_type or "").lower()- if dt == _NVFP4_E5M3_DATA_TYPE:- return _NVFP4_E5M3_DATA_TYPE- if is_nv_fp(dt):- return "nv_fp"- except Exception:- pass+ candidates.append((val.get("data_type") or "").lower())+ if isinstance(val.get("scheme"), str):+ candidates.append(_scheme_dt(val["scheme"]))elif isinstance(val, QuantizationScheme):- dt = (val.data_type or "").lower()- if dt == _NVFP4_E5M3_DATA_TYPE:- return _NVFP4_E5M3_DATA_TYPE- if is_nv_fp(dt):- return "nv_fp"+ candidates.append((val.data_type or "").lower())+ for dt in candidates:+ result = _classify(dt)+ if result is not None:+ return resultreturn None@@ -3184,7 +3259,7 @@ def _derive_dominant_int_scheme("sym": sym,"data_type": data_type,}- if data_type == _NVFP4_E5M3_DATA_TYPE:+ if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:for act_key in ("act_bits", "act_data_type", "act_group_size", "act_sym", "act_dynamic"):value = scheme.get(act_key)if value is not None:@@ -3217,14 +3292,16 @@ def _build_quantization_config(data_type = (default_scheme.get("data_type") or "int").lower()default_bits = default_scheme.get("bits", 4)is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)- if data_type == _NVFP4_E5M3_DATA_TYPE and format == "llm_compressor":+ if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES and format == "llm_compressor":warnings.warn("LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. ""Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",UserWarning,stacklevel=2,)- return _build_nvfp4_e5m3_quantization_config(ignored_layers)+ return _build_nvfp4_e5m3_quantization_config(+ ignored_layers, with_global_scale=data_type == _NVFP4_E5M3_GS_DATA_TYPE+ )# BF16/FP16 default with NVFP4 layer_config overrides and llm_compressor format:# build an NVFP4 config that targets only the explicitly quantized layers.if is_fp_default and _layer_config_has_nvfp4(layer_config) and format == "llm_compressor":@@ -3232,19 +3309,19 @@ def _build_quantization_config(targets = list(dict.fromkeys(quantized_layers)) if quantized_layers else ["Linear"]if quantized_layers:targets = _add_routed_experts_if_moe(targets, quantized_layers)- if nvfp4_dt == _NVFP4_E5M3_DATA_TYPE:+ if nvfp4_dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:warnings.warn("LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. ""Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",UserWarning,stacklevel=2,)- # Global-scale-free variant (NVFP4_E5M3)from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization- qconfig = initialize_nvfp4_e5m3_quantization(ignore=[])+ qconfig = initialize_nvfp4_e5m3_quantization(+ ignore=[], with_global_scale=nvfp4_dt == _NVFP4_E5M3_GS_DATA_TYPE+ )qconfig["config_groups"]["group_0"]["targets"] = targets- qconfig["format"] = "nvfp4-e5m3-pack-quantized"else:# Standard NVFP4 (nv_fp, scale_dtype=float8_e4m3fn)from auto_round.export.export_to_llmcompressor.config import initialize_quantization@@ -3301,6 +3378,8 @@ def _build_quantization_config(# So we keep auto_round for asymmetric quantization to maintain compatibility with both.if data_type == _NVFP4_E5M3_DATA_TYPE:packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"+ elif data_type == _NVFP4_E5M3_GS_DATA_TYPE:+ packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3_gs"else:packing_format = _declared_int_packing(default_scheme.get("sym", True))@@ -3316,7 +3395,7 @@ def _build_quantization_config("autoround_version": __version__,}- if data_type == _NVFP4_E5M3_DATA_TYPE:+ if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:for act_key in ("act_bits", "act_data_type", "act_group_size", "act_sym", "act_dynamic"):value = default_scheme.get(act_key)if value is not None:@@ -3350,7 +3429,7 @@ def _build_quantization_config(if non_linear_re.search(layer_name):continueextra_config[layer_name] = {"bits": 16, "data_type": "float"}- if data_type == _NVFP4_E5M3_DATA_TYPE:+ if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:extra_config[layer_name].update({"act_bits": 16, "act_data_type": "float"})quantized_layer_set = set(quantized_layers)@@ -3799,6 +3878,22 @@ def _validate_supported_scheme()return+ if data_type == _NVFP4_E5M3_GS_DATA_TYPE:+ if bits != 4 or scheme_obj.group_size != 16 or act_bits != 4:+ raise ValueError(+ f"Model-free NVFP4_E5M3_GS requires bits=4, group_size=16, and act_bits=4, "+ f"but '{scheme_input}' requests bits={bits}, group_size={scheme_obj.group_size}, "+ f"act_bits={act_bits}."+ )+ act_data_type = (scheme_obj.act_data_type or "").lower()+ if act_data_type != _NVFP4_E5M3_GS_ACT_DATA_TYPE or scheme_obj.act_group_size != 16:+ raise ValueError(+ f"Model-free NVFP4_E5M3_GS requires act_data_type='{_NVFP4_E5M3_GS_ACT_DATA_TYPE}' and "+ f"act_group_size=16, but '{scheme_input}' requests act_data_type='{scheme_obj.act_data_type}', "+ f"act_group_size={scheme_obj.act_group_size}."+ )+ return+if act_bits < 16:raise ValueError(f"Model-free mode only supports weight-only quantization (WOQ) schemes "diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex afdb48dba..af0d4c7f2 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -20,9 +20,9 @@ import transformersfrom torch.functional import Fimport auto_round.envs as envs-from auto_round.compressors.utils import is_nv_fpfrom auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_sizefrom auto_round.logger import logger+from auto_round.schemes import needs_nvfp_global_scalefrom auto_round.utils import (SUPPORTED_LAYER_TYPES,check_to_quantized,@@ -108,10 +108,12 @@ class WrapperLinear(torch.nn.Module):self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)self.enable_act_quant = self.orig_layer.act_bits <= 8self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)- if is_nv_fp(self.orig_layer.data_type) and self.weight_global_scale is None:- from auto_round.data_type.nvfp import calculate_gparam+ if needs_nvfp_global_scale(self.orig_layer.data_type) and self.weight_global_scale is None:+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype- weight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)+ weight_global_scale = calculate_gparam_by_dtype(+ self.orig_layer.weight, self.orig_layer.data_type, self.orig_layer.group_size+ )self.weight_global_scale = weight_global_scaleself.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:diff --git a/auto_round_extension/cuda/cute_nvfp4_e5m3.py b/auto_round_extension/cuda/cute_nvfp4_e5m3.pyindex 4aa087d0b..69b4f674c 100644--- a/auto_round_extension/cuda/cute_nvfp4_e5m3.py+++ b/auto_round_extension/cuda/cute_nvfp4_e5m3.py@@ -12,7 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.-"""Optional CuTe DSL dispatch for NVFP4 E5M3 activation QDQ."""+"""Optional CuTe DSL dispatch for NVFP4 E5M3 activation QDQ and weight dequantization."""from functools import cache, lru_cachefrom importlib.util import find_spec@@ -166,6 +166,174 @@ def _make_weight_dq_kernel():return launch_weight_dq+_UE5M3_MAX = 114688.0+++def _make_qdq_gs_kernel():+ import cutlass+ import cutlass.cute as cute+ from cutlass._mlir_helpers import math++ @cute.kernel+ def qdq_gs_kernel(input_tensor: cute.Tensor, global_scale: cute.Tensor, output_tensor: cute.Tensor):+ thread_idx, _, _ = cute.arch.thread_idx()+ block_idx, _, _ = cute.arch.block_idx()+ group_idx = block_idx * _THREADS_PER_BLOCK + thread_idx++ if group_idx < input_tensor.shape[0]:+ gs = cutlass.Float32(global_scale[0])+ amax = cutlass.Float32(0.0)+ for column_idx in range(_GROUP_SIZE):+ value = cutlass.Float32(input_tensor[group_idx, column_idx])+ amax = cute.arch.fmax(amax, math.abs(value))++ # Mirrors ref_fp4_quant: UE5M3(gs * amax / 6), then dequant scale = scale / gs.+ scale = gs * (amax * (cutlass.Float32(1.0) / cutlass.Float32(6.0)))+ scale = cute.arch.fmin(cute.arch.fmax(scale, 0.0), _UE5M3_MAX)+ if scale >= cutlass.Float32(2.0**-14):+ exponent = math.floor(math.log2(scale)) + cutlass.Float32(1.0)+ mantissa = scale / math.exp2(exponent)+ mantissa_bits = math.roundeven((mantissa - cutlass.Float32(0.5)) * cutlass.Float32(16.0))+ if mantissa_bits == cutlass.Float32(8.0):+ mantissa_bits = cutlass.Float32(0.0)+ exponent = exponent + cutlass.Float32(1.0)+ mantissa_bits = cute.arch.fmin(cute.arch.fmax(mantissa_bits, 0.0), 7.0)+ scale = (cutlass.Float32(1.0) + mantissa_bits / cutlass.Float32(8.0)) * math.exp2(+ exponent - cutlass.Float32(1.0)+ )+ else:+ scale = math.roundeven(scale * cutlass.Float32(2.0**17)) * cutlass.Float32(2.0**-17)+ dequant_scale = scale / gs if gs > cutlass.Float32(0.0) else cutlass.Float32(0.0)++ for column_idx in range(_GROUP_SIZE):+ value = cutlass.Float32(input_tensor[group_idx, column_idx])+ scaled = value / dequant_scale if dequant_scale > cutlass.Float32(0.0) else cutlass.Float32(0.0)+ magnitude = math.abs(scaled)+ quantized = cutlass.Float32(0.0)+ if magnitude < cutlass.Float32(2.0):+ quantized = math.roundeven(magnitude * cutlass.Float32(2.0)) / cutlass.Float32(2.0)+ elif magnitude < cutlass.Float32(4.0):+ quantized = math.roundeven(magnitude)+ else:+ quantized = cutlass.Float32(2.0) * math.roundeven(magnitude / cutlass.Float32(2.0))+ quantized = cute.arch.fmin(cute.arch.fmax(quantized, 0.0), 6.0)+ if scaled < cutlass.Float32(0.0):+ quantized = -quantized+ output_tensor[group_idx, column_idx] = (quantized * dequant_scale).to(output_tensor.element_type)++ @cute.jit+ def launch_qdq_gs(input_tensor: cute.Tensor, global_scale: cute.Tensor, output_tensor: cute.Tensor):+ groups = input_tensor.shape[0]+ qdq_gs_kernel(input_tensor, global_scale, output_tensor).launch(+ grid=((groups + _THREADS_PER_BLOCK - 1) // _THREADS_PER_BLOCK, 1, 1),+ block=(_THREADS_PER_BLOCK, 1, 1),+ )++ return launch_qdq_gs+++def _make_weight_dq_gs_kernel():+ import cutlass+ import cutlass.cute as cute+ from cutlass._mlir_helpers import math++ @cute.kernel+ def weight_dq_gs_kernel(+ weight_packed: cute.Tensor, weight_scale: cute.Tensor, global_scale: cute.Tensor, output_tensor: cute.Tensor+ ):+ thread_idx, _, _ = cute.arch.thread_idx()+ block_idx, _, _ = cute.arch.block_idx()+ group_idx = block_idx * _THREADS_PER_BLOCK + thread_idx++ if group_idx < output_tensor.shape[0]:+ scale_byte = cutlass.Float32(weight_scale[group_idx, 0])+ exponent = math.floor(scale_byte / cutlass.Float32(8.0))+ mantissa = scale_byte - exponent * cutlass.Float32(8.0)+ scale = (cutlass.Float32(1.0) + mantissa / cutlass.Float32(8.0)) * math.exp2(+ exponent - cutlass.Float32(15.0)+ )+ if exponent == cutlass.Float32(0.0):+ scale = mantissa * math.exp2(cutlass.Float32(-17.0))+ gs = cutlass.Float32(global_scale[0])+ scale = scale / gs if gs > cutlass.Float32(0.0) else cutlass.Float32(0.0)++ for column_idx in range(_GROUP_SIZE):+ packed = cutlass.Float32(weight_packed[group_idx, column_idx // 2])+ packed_high = math.floor(packed / cutlass.Float32(16.0))+ nibble = packed - packed_high * cutlass.Float32(16.0)+ if column_idx % 2:+ nibble = packed_high++ sign = cutlass.Float32(1.0)+ if nibble >= cutlass.Float32(8.0):+ sign = cutlass.Float32(-1.0)+ nibble = nibble - cutlass.Float32(8.0)++ value = cutlass.Float32(0.0)+ if nibble == cutlass.Float32(1.0):+ value = cutlass.Float32(0.5)+ elif nibble == cutlass.Float32(2.0):+ value = cutlass.Float32(1.0)+ elif nibble == cutlass.Float32(3.0):+ value = cutlass.Float32(1.5)+ elif nibble == cutlass.Float32(4.0):+ value = cutlass.Float32(2.0)+ elif nibble == cutlass.Float32(5.0):+ value = cutlass.Float32(3.0)+ elif nibble == cutlass.Float32(6.0):+ value = cutlass.Float32(4.0)+ elif nibble == cutlass.Float32(7.0):+ value = cutlass.Float32(6.0)++ output_tensor[group_idx, column_idx] = (sign * value * scale).to(output_tensor.element_type)++ @cute.jit+ def launch_weight_dq_gs(+ weight_packed: cute.Tensor, weight_scale: cute.Tensor, global_scale: cute.Tensor, output_tensor: cute.Tensor+ ):+ groups = output_tensor.shape[0]+ weight_dq_gs_kernel(weight_packed, weight_scale, global_scale, output_tensor).launch(+ grid=((groups + _THREADS_PER_BLOCK - 1) // _THREADS_PER_BLOCK, 1, 1),+ block=(_THREADS_PER_BLOCK, 1, 1),+ )++ return launch_weight_dq_gs+++@cache+def _get_compiled_qdq_gs_kernel(device_index: int, dtype: torch.dtype):+ import cutlass.cute as cute+ from cutlass.cute.runtime import from_dlpack++ input_tensor = torch.empty((1, _GROUP_SIZE), device=f"cuda:{device_index}", dtype=dtype)+ global_scale = torch.ones((1,), device=f"cuda:{device_index}", dtype=torch.float32)+ output_tensor = torch.empty_like(input_tensor)+ return cute.compile(+ _make_qdq_gs_kernel(),+ from_dlpack(input_tensor).mark_layout_dynamic(),+ from_dlpack(global_scale),+ from_dlpack(output_tensor).mark_layout_dynamic(),+ )+++@cache+def _get_compiled_weight_dq_gs_kernel(device_index: int, dtype: torch.dtype):+ import cutlass.cute as cute+ from cutlass.cute.runtime import from_dlpack++ weight_packed = torch.empty((2, _GROUP_SIZE // 2), device=f"cuda:{device_index}", dtype=torch.uint8)+ weight_scale = torch.empty((2, 1), device=f"cuda:{device_index}", dtype=torch.uint8)+ global_scale = torch.ones((1,), device=f"cuda:{device_index}", dtype=torch.float32)+ output_tensor = torch.empty((2, _GROUP_SIZE), device=f"cuda:{device_index}", dtype=dtype)+ return cute.compile(+ _make_weight_dq_gs_kernel(),+ from_dlpack(weight_packed).mark_layout_dynamic(),+ from_dlpack(weight_scale).mark_layout_dynamic(),+ from_dlpack(global_scale),+ from_dlpack(output_tensor).mark_layout_dynamic(),+ )++@cachedef _get_compiled_qdq_kernel(device_index: int, dtype: torch.dtype):import cutlass.cute as cute@@ -256,6 +424,79 @@ def try_cute_nvfp4_e5m3_weight_dq(return None+def _as_cuda_global_scale(global_scale: torch.Tensor, device: torch.device) -> torch.Tensor:+ return global_scale.reshape(-1)[:1].to(device=device, dtype=torch.float32).contiguous()+++def try_cute_nvfp4_v2_gs_qdq(+ activation: torch.Tensor, global_scale: torch.Tensor, group_size: int+) -> torch.Tensor | None:+ """Run the CuTe QDQ kernel for NVFP4 E5M3 activations with a per-tensor global scale."""+ if not can_use_cute_nvfp4_v2_qdq(activation, group_size):+ return None+ if torch.cuda.is_current_stream_capturing():+ return None++ kernel_key = ("gs", activation.device.index, activation.dtype)+ if kernel_key in _FAILED_KERNEL_KEYS:+ return None++ try:+ groups = activation.numel() // group_size+ output = torch.empty_like(activation)+ compiled_qdq = _get_compiled_qdq_gs_kernel(activation.device.index, activation.dtype)+ compiled_qdq(+ activation.reshape(groups, group_size),+ _as_cuda_global_scale(global_scale, activation.device),+ output.reshape(groups, group_size),+ )+ return output+ except Exception as error:+ _FAILED_KERNEL_KEYS.add(kernel_key)+ logger.warning_once("CuTe NVFP4 E5M3 GS QDQ failed; falling back to PyTorch reference: %s", error)+ return None+++def try_cute_nvfp4_e5m3_gs_weight_dq(+ weight_packed: torch.Tensor, weight_scale: torch.Tensor, global_scale: torch.Tensor, dtype: torch.dtype+) -> torch.Tensor | None:+ """Dequantize packed FP4 E5M3 weights with a per-tensor global scale using CuTe."""+ if (+ not is_cute_dsl_available()+ or not weight_packed.is_cuda+ or weight_packed.dtype != torch.uint8+ or weight_scale.dtype != torch.uint8+ or weight_packed.ndim != 2+ or weight_scale.shape != (weight_packed.shape[0], weight_packed.shape[1] // (_GROUP_SIZE // 2))+ or dtype not in _SUPPORTED_DTYPES+ or torch.cuda.get_device_capability(weight_packed.device)[0] < 8+ or torch.cuda.is_current_stream_capturing()+ ):+ return None++ kernel_key = ("gs", weight_packed.device.index, dtype)+ if kernel_key in _FAILED_WEIGHT_DQ_KERNEL_KEYS:+ return None++ try:+ out_features = weight_packed.shape[0]+ in_features = weight_packed.shape[1] * 2+ groups = weight_scale.numel()+ output = torch.empty((groups, _GROUP_SIZE), device=weight_packed.device, dtype=dtype)+ compiled_weight_dq = _get_compiled_weight_dq_gs_kernel(weight_packed.device.index, dtype)+ compiled_weight_dq(+ weight_packed.reshape(groups, _GROUP_SIZE // 2),+ weight_scale.reshape(groups, 1),+ _as_cuda_global_scale(global_scale, weight_packed.device),+ output,+ )+ return output.reshape(out_features, in_features)+ except Exception as error:+ _FAILED_WEIGHT_DQ_KERNEL_KEYS.add(kernel_key)+ logger.warning_once("CuTe NVFP4 E5M3 GS weight DQ failed; falling back to PyTorch reference: %s", error)+ return None++def try_cute_nvfp4_e5m3_linear(activation: torch.Tensor,weight_packed: torch.Tensor,diff --git a/docs/nvfp4_e5m3.md b/docs/nvfp4_e5m3.mdindex a88aeaa66..396bc3b8f 100644--- a/docs/nvfp4_e5m3.md+++ b/docs/nvfp4_e5m3.md@@ -105,6 +105,33 @@ tokenizer = AutoTokenizer.from_pretrained("./nvfp4_e5m3_model")---+## NVFP4_E5M3_GS: E5M3 Block Scales with a Global Scale++`NVFP4_E5M3_GS` keeps the FP4 E2M1 values and UE5M3 block scales, and adds a+per-tensor FP32 global scale for weights and activations (like standard NVFP4):++$$\text{global\_scale} = \frac{114688 \times 6}{\text{amax}}, \qquad x \approx q \cdot \frac{\text{UE5M3}(\text{block\_scale})}{\text{global\_scale}}$$++| Property | Value |+|---|---|+| Preset name | `NVFP4_E5M3_GS` |+| Weight dtype | `nvfp4_v2_gs` |+| Activation dtype | `nvfp4_v2_with_static_gs` |+| Extra tensors | `.weight_global_scale`, `.input_global_scale` (FP32) |+| `auto_round` packing format | `auto_round:llm_compressor_nvfp4_e5m3_gs` |+| `llm_compressor` format | `nvfp4-e5m3-gs-pack-quantized` |+| Inference backends | `auto_round:cute_nvfp4_e5m3_gs`, `auto_round:torch_nvfp4_e5m3_gs` |++- **Regular tuning flow**: global scales are computed like NVFP4. The weight global scale comes from the weight amax, and `input_global_scale` comes from the calibrated activation amax. q/k/v and gate/up projections share the minimum global scale.+- **Model-free flow**: the weight global scale is computed per layer and `input_global_scale` is fixed to `1`.++```bash+auto-round --model <model_id_or_path> --scheme NVFP4_E5M3_GS --output_dir ./nvfp4_e5m3_gs_model+auto-round --model <model_id_or_path> --scheme NVFP4_E5M3_GS --model_free --output_dir ./nvfp4_e5m3_gs_model+```++---+## Environment Variables### `AR_NVFP4_E5M3_CACHE_HP_WEIGHT`diff --git a/docs/nvfp4_e5m3_CN.md b/docs/nvfp4_e5m3_CN.mdindex f9d5a76a0..f9836d2f7 100644--- a/docs/nvfp4_e5m3_CN.md+++ b/docs/nvfp4_e5m3_CN.md@@ -101,6 +101,32 @@ tokenizer = AutoTokenizer.from_pretrained("./nvfp4_e5m3_model")---+## NVFP4_E5M3_GS:带全局 scale 的 E5M3 块级 scale++`NVFP4_E5M3_GS` 保留 FP4 E2M1 数值和 UE5M3 块级 scale,并像标准 NVFP4 一样为权重和激活额外增加一个张量级 FP32 全局 scale:++$$\text{global\_scale} = \frac{114688 \times 6}{\text{amax}}, \qquad x \approx q \cdot \frac{\text{UE5M3}(\text{block\_scale})}{\text{global\_scale}}$$++| 参数 | 值 |+|---|---|+| 预设名称 | `NVFP4_E5M3_GS` |+| 权重数据类型 | `nvfp4_v2_gs` |+| 激活数据类型 | `nvfp4_v2_with_static_gs` |+| 额外张量 | `.weight_global_scale`、`.input_global_scale`(FP32) |+| `auto_round` 打包格式 | `auto_round:llm_compressor_nvfp4_e5m3_gs` |+| `llm_compressor` 格式 | `nvfp4-e5m3-gs-pack-quantized` |+| 推理后端 | `auto_round:cute_nvfp4_e5m3_gs`、`auto_round:torch_nvfp4_e5m3_gs` |++- **常规调优流程**:全局 scale 的计算方式与 NVFP4 一致。权重全局 scale 由权重 amax 得到,`input_global_scale` 由校准得到的激活 amax 得到。q/k/v 及 gate/up 投影共享最小的全局 scale。+- **无模型量化流程**:按层计算权重全局 scale,`input_global_scale` 固定为 `1`。++```bash+auto-round --model <model_id_or_path> --scheme NVFP4_E5M3_GS --output_dir ./nvfp4_e5m3_gs_model+auto-round --model <model_id_or_path> --scheme NVFP4_E5M3_GS --model_free --output_dir ./nvfp4_e5m3_gs_model+```++---+## 环境变量### `AR_NVFP4_E5M3_CACHE_HP_WEIGHT`diff --git a/test/unit/common/core/test_format_decoupling.py b/test/unit/common/core/test_format_decoupling.pyindex 2dea8725d..7832288dd 100644--- a/test/unit/common/core/test_format_decoupling.py+++ b/test/unit/common/core/test_format_decoupling.py@@ -94,6 +94,33 @@ def test_nvfp4_e5m3_autoround_uses_llm_compressor_packing_and_torch_fallback():)+def test_nvfp4_e5m3_gs_autoround_uses_llm_compressor_packing_and_torch_fallback():+ scheme = _resolved_scheme("NVFP4_E5M3_GS")+ resolution = resolve_formats(+ ResolvedScheme.from_scheme(scheme), format="auto_round", model=nn.Sequential(nn.Linear(16, 16))+ )++ assert resolution.formats[0].get_backend_name() == "auto_round:nvfp4_v2_gs"+ assert BackendInfos["auto_round:torch_nvfp4_e5m3_gs"].packing_format == ["auto_round:llm_compressor_nvfp4_e5m3_gs"]+ assert dynamic_import_inference_linear("auto_round:torch_nvfp4_e5m3_gs", scheme) is (+ ar_qmodules.NVFP4E5M3GSQuantLinear+ )+ assert dynamic_import_inference_linear("auto_round:cute_nvfp4_e5m3_gs", scheme) is (+ ar_qmodules.CuteNVFP4E5M3GSQuantLinear+ )+ assert (+ get_layer_backend(+ "cpu",+ "auto",+ "auto_round:llm_compressor_nvfp4_e5m3_gs",+ scheme,+ in_features=16,+ out_features=16,+ )+ == "auto_round:torch_nvfp4_e5m3_gs"+ )++def test_gguf_correction_propagates_to_scheme_and_quantize_config(monkeypatch):# W4A16's plain int-woq fields don't match every field gguf_args_check enforces# for a GGUF target; after resolving "gguf:q4_k_m" the scheme-bearing objectsdiff --git a/test/unit/common/data_type/test_nvfp.py b/test/unit/common/data_type/test_nvfp.pyindex 0db81a493..99af14a2f 100644--- a/test/unit/common/data_type/test_nvfp.py+++ b/test/unit/common/data_type/test_nvfp.py@@ -25,6 +25,8 @@ from auto_round.data_type.nvfp import (FLOAT8_E4M3_MIN,FLOAT8_UE5M3_MAX,calculate_gparam,+ calculate_gparam_by_dtype,+ calculate_gparam_e5m3,cast_to_fp4,cast_to_ue5m3,cast_to_ue5m3_ste,@@ -34,13 +36,16 @@ from auto_round.data_type.nvfp import (nv_fp4,nv_fp4_with_static_gs,nvfp4_v2,+ nvfp4_v2_gs,nvfp4_v2_with_global_scale,+ nvfp4_v2_with_static_gs,+ opt_rtn_nvfp4_v2_gs,ref_fp4_quant,ref_nvfp4_quant,search_nvfp4_scale,search_nvfp4_v2_scale,)-from auto_round.data_type.utils import update_fused_layer_global_scales+from auto_round.data_type.utils import get_quant_func, update_fused_layer_global_scales# ---------------------------------------------------------------------------# Constants@@ -749,6 +754,81 @@ class TestNvfp4V2:assert q.shape == t.shape+# ---------------------------------------------------------------------------+# NVFP4_E5M3_GS: nvfp4_v2_gs / nvfp4_v2_with_static_gs+# ---------------------------------------------------------------------------+++class TestNvfp4V2GlobalScale:+ """Test the NVFP4 E5M3 data types with a per-tensor global scale."""++ def test_calculate_gparam_e5m3_uses_ue5m3_max(self):+ t = torch.randn(4, 32)+ expected = FLOAT8_UE5M3_MAX * FLOAT4_E2M1_MAX / t.abs().max()+ torch.testing.assert_close(calculate_gparam_e5m3(t), expected)+ torch.testing.assert_close(calculate_gparam_by_dtype(t, "opt_rtn_nvfp4_v2_gs"), expected)+ torch.testing.assert_close(calculate_gparam_by_dtype(t, "nv_fp"), calculate_gparam(t))++ def test_default_global_scale_fills_ue5m3_range(self):+ t = torch.randn(8, 64)+ q, s, z = nvfp4_v2_gs(t, group_size=16)+ assert q.shape == t.shape and z is None+ assert s.max().item() == FLOAT8_UE5M3_MAX++ def test_matches_ref_with_explicit_global_scale(self):+ t = torch.randn(8, 64)+ global_scale = torch.tensor([1234.5])+ q, s, _ = nvfp4_v2_gs(t, group_size=16, global_scale=global_scale)+ ref_q, ref_s = ref_fp4_quant(t.reshape(-1, 16), global_scale, 16)+ torch.testing.assert_close(q, ref_q.reshape(t.shape))+ torch.testing.assert_close(s, ref_s)++ def test_global_scale_one_matches_nvfp4_v2(self):+ t = torch.randn(8, 64)+ q, _, _ = nvfp4_v2_gs(t, group_size=16, global_scale=torch.ones(1))+ ref, _, _ = nvfp4_v2(t, group_size=16)+ torch.testing.assert_close(q, ref)++ def test_per_group_max_scale(self):+ t = torch.randn(4, 32)+ max_scale = torch.full((8,), 0.9)+ q, _, _ = nvfp4_v2_gs(t, group_size=16, max_scale=max_scale)+ ref, _, _ = nvfp4_v2_gs(t, group_size=16, max_scale=0.9)+ torch.testing.assert_close(q, ref)++ def test_opt_rtn_does_not_increase_error(self):+ t = torch.randn(32, 128)+ baseline, _, _ = nvfp4_v2_gs(t, group_size=16)+ optimized, _, _ = opt_rtn_nvfp4_v2_gs(t, group_size=16)+ assert (optimized - t).pow(2).sum() <= (baseline - t).pow(2).sum()++ def test_opt_rtn_handles_multiple_groups(self):+ t = torch.randn(4, 32)++ optimized, scale, _ = opt_rtn_nvfp4_v2_gs(t, group_size=16)++ assert optimized.shape == t.shape+ assert scale.shape == (8, 1)+ assert torch.isfinite(optimized).all()++ def test_static_gs_tensor_max_matches_global_scale(self):+ t = torch.randn(4, 64)+ tensor_max = torch.tensor([3.0])+ from_max, _, _ = nvfp4_v2_with_static_gs(t, group_size=16, tensor_max=tensor_max)+ from_gs, _, _ = nvfp4_v2_with_static_gs(t, group_size=16, global_scale=calculate_gparam_e5m3(tensor_max))+ torch.testing.assert_close(from_max, from_gs)++ def test_static_gs_empty_tensor(self):+ t = torch.empty(0, 16)+ q, s, z = nvfp4_v2_with_static_gs(t, group_size=16)+ assert q is t and s is None and z is None++ def test_registered_quant_funcs(self):+ assert get_quant_func("nvfp4_v2_gs", 4, True, iters=0)[1] == "opt_rtn_nvfp4_v2_gs"+ assert get_quant_func("nvfp4_v2_gs", 4, True, iters=200)[1] == "nvfp4_v2_gs"+ assert get_quant_func("nvfp4_v2_with_static_gs", 4, True, iters=0)[1] == "nvfp4_v2_with_static_gs"++# ---------------------------------------------------------------------------# Cross-check: outputs are finite / dtype preserved# ---------------------------------------------------------------------------diff --git a/test/unit/common/quantization/test_nvfp4_quant_linear.py b/test/unit/common/quantization/test_nvfp4_quant_linear.pyindex 82f403330..a37373ec6 100644--- a/test/unit/common/quantization/test_nvfp4_quant_linear.py+++ b/test/unit/common/quantization/test_nvfp4_quant_linear.py@@ -5,7 +5,13 @@ import pytestimport torchfrom transformers.quantizers.auto import AutoHfQuantizer-from auto_round.data_type.nvfp import calculate_gparam, nvfp4_v2+from auto_round.data_type.nvfp import (+ calculate_gparam,+ calculate_gparam_e5m3,+ nvfp4_v2,+ nvfp4_v2_gs,+ nvfp4_v2_with_static_gs,+)from auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear@@ -57,6 +63,80 @@ def test_nvfp4_e5m3_compressed_tensors_loading_uses_no_global_scales():assert quantizer._process_model_after_weight_loading(model) is model+def test_nvfp4_e5m3_gs_compressed_tensors_loading_uses_global_scales():+ class TinyModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(model_type="tiny")+ self.model = torch.nn.Module()+ self.model.layers = torch.nn.ModuleList([torch.nn.ModuleDict({"fc": torch.nn.Linear(16, 8)})])++ quantizer = AutoHfQuantizer.from_config(initialize_nvfp4_e5m3_quantization([], with_global_scale=True))+ model = quantizer._process_model_before_weight_loading(TinyModel())+ layer = model.model.layers[0]["fc"]++ assert isinstance(layer, ar_qmodules.NVFP4E5M3GSQuantLinear)+ assert set(layer.state_dict()) == {+ "bias",+ "weight_packed",+ "weight_scale",+ "weight_global_scale",+ "input_global_scale",+ }++[email protected]_mode()+def test_nvfp4_e5m3_gs_pack_and_dequant_round_trip():+ config = PRESET_SCHEMES["NVFP4_E5M3_GS"]+ original_layer = torch.nn.Linear(64, 32, bias=False)+ weight_global_scale = calculate_gparam_e5m3(original_layer.weight)+ weight_qdq, weight_scale, _ = nvfp4_v2_gs(original_layer.weight, group_size=16, global_scale=weight_global_scale)+ activation = torch.randn(4, 64)+ input_global_scale = calculate_gparam_e5m3(activation)++ packed = _FPLinear(4, 16, 64, 32, False, data_type=config.data_type, act_bits=4)+ packed.pack(+ original_layer,+ weight_scale.reshape(32, 4),+ global_scale=weight_global_scale,+ input_global_scale=input_global_scale,+ )+ assert packed.weight_scale.dtype == torch.uint8++ layer = ar_qmodules.NVFP4E5M3GSQuantLinear.from_original(config, original_layer)+ for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):+ getattr(layer, name).copy_(getattr(packed, name))++ torch.testing.assert_close(layer.dequant_weight_online(), weight_qdq, rtol=1e-6, atol=1e-7)+ expected_activation, _, _ = nvfp4_v2_with_static_gs(activation, group_size=16, tensor_max=activation.abs().max())+ torch.testing.assert_close(layer.qdq_input(activation), expected_activation)+++def test_nvfp4_e5m3_gs_dequant_zero_weight_with_zero_global_scale():+ config = PRESET_SCHEMES["NVFP4_E5M3_GS"]+ layer = ar_qmodules.NVFP4E5M3GSQuantLinear(16, 8, config, dtype=torch.float32)+ layer.weight_packed.zero_()+ layer.weight_scale.zero_()+ layer.weight_global_scale.zero_()++ dequantized = layer.dequant_weight_online()++ assert torch.isfinite(dequantized).all()+ assert torch.equal(dequantized, torch.zeros_like(dequantized))+++def test_nvfp4_e5m3_gs_cute_falls_back_to_reference_on_cpu():+ config = PRESET_SCHEMES["NVFP4_E5M3_GS"]+ layer = ar_qmodules.CuteNVFP4E5M3GSQuantLinear(16, 8, config, dtype=torch.float32)+ layer.input_global_scale.fill_(123.0)+ activation = torch.randn(2, 16)++ expected, _, _ = nvfp4_v2_with_static_gs(activation, group_size=16, global_scale=layer.input_global_scale)++ assert torch.equal(layer.qdq_input(activation), expected)+ assert layer(activation).shape == (2, 8)++def test_nvfp4_e5m3_qdq_input_uses_reference_fallback_on_cpu():config = PRESET_SCHEMES["NVFP4_E5M3"]layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.pyindex 2a3777534..1b53add1a 100644--- a/test/unit/test_cpu/quantization/test_model_free.py+++ b/test/unit/test_cpu/quantization/test_model_free.py@@ -77,6 +77,7 @@ from auto_round.utils.model_free_utils import (_process_shard,_quantize_weight_mxfp,_quantize_weight_nvfp4_e5m3,+ _quantize_weight_nvfp4_e5m3_gs,_validate_auto_scheme_options,is_model_free_supported_scheme,)@@ -707,6 +708,59 @@ def test_nvfp4_e5m3_model_free_llm_compressor(tmp_path):assert group["input_activations"]["dynamic"] == "local"+def test_nvfp4_e5m3_gs_model_free_fake_quantization():+ from auto_round.data_type.nvfp import calculate_gparam_e5m3, nvfp4_v2_gs++ weight = torch.randn(8, 32)+ output = _quantize_weight_nvfp4_e5m3_gs(weight, "layer.fc", group_size=16, disable_opt_rtn=True)++ expected, _, _ = nvfp4_v2_gs(weight, group_size=16, global_scale=calculate_gparam_e5m3(weight))+ assert set(output) == {"layer.fc.weight", "layer.fc.input_global_scale"}+ torch.testing.assert_close(output["layer.fc.weight"], expected)+ assert torch.equal(output["layer.fc.input_global_scale"], torch.ones(1))+ assert is_model_free_supported_scheme("NVFP4_E5M3_GS")++[email protected]("fmt", ["auto_round", "llm_compressor"])+def test_nvfp4_e5m3_gs_model_free_end_to_end(tmp_path, fmt):+ from safetensors import safe_open++ from auto_round.data_type.nvfp import calculate_gparam_e5m3++ prefix = "model.layers.0.self_attn.q_proj"+ weight = torch.randn(32, 32)+ tensors = {f"{prefix}.weight": weight, "lm_head.weight": torch.randn(64, 32)}+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")++ compressor = _ModelFreeCompressorCore(+ model_name_or_path=model_dir, output_dir=output_dir, scheme="NVFP4_E5M3_GS", format=fmt+ )+ compressor.run()++ output_keys = _read_output_keys(output_dir)+ for suffix in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):+ assert f"{prefix}.{suffix}" in output_keys+ assert f"{prefix}.weight" not in output_keys+ saved = {}+ for file_name in os.listdir(output_dir):+ if file_name.endswith(".safetensors"):+ with safe_open(os.path.join(output_dir, file_name), framework="pt") as f:+ saved.update({k: f.get_tensor(k) for k in f.keys() if k.startswith(prefix)})+ assert torch.equal(saved[f"{prefix}.input_global_scale"], torch.ones(1))+ torch.testing.assert_close(saved[f"{prefix}.weight_global_scale"], calculate_gparam_e5m3(weight).reshape(1))+ assert saved[f"{prefix}.weight_scale"].dtype == torch.uint8++ quantization_config = _read_qconfig(output_dir)+ if fmt == "auto_round":+ assert quantization_config["packing_format"] == "auto_round:llm_compressor_nvfp4_e5m3_gs"+ assert quantization_config["data_type"] == "nvfp4_v2_gs"+ assert quantization_config["act_data_type"] == "nvfp4_v2_with_static_gs"+ else:+ assert quantization_config["format"] == "nvfp4-e5m3-gs-pack-quantized"+ assert quantization_config["quant_method"] == "compressed-tensors"++def test_model_free_legacy_nvfp4_is_normalized_and_passthrough(tmp_path):prefix = "model.layers.0.mlp.down_proj"tensors = {diff --git a/test/unit/test_cpu/quantization/test_nvfp4_e5m3_gs.py b/test/unit/test_cpu/quantization/test_nvfp4_e5m3_gs.pynew file mode 100644index 000000000..2652a47c1--- /dev/null+++ b/test/unit/test_cpu/quantization/test_nvfp4_e5m3_gs.py@@ -0,0 +1,123 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import json+import os++import pytest+import torch+from safetensors import safe_open++from auto_round import AutoRound+from auto_round.data_type.nvfp import FLOAT4_E2M1_MAX, FLOAT8_UE5M3_MAX++PREFIX = "model.decoder.layers.0.self_attn"+++def _read_tensors(model_dir, names):+ tensors = {}+ for file_name in os.listdir(model_dir):+ if not file_name.endswith(".safetensors"):+ continue+ with safe_open(os.path.join(model_dir, file_name), framework="pt") as f:+ for name in names:+ if name in f.keys():+ tensors[name] = f.get_tensor(name)+ return tensors+++def _quantize(model_path, dataloader, output_dir, fmt, iters=0):+ autoround = AutoRound(+ model_path,+ scheme="NVFP4_E5M3_GS",+ iters=iters,+ seqlen=8,+ nsamples=2,+ dataset=dataloader,+ )+ _, quantized_model_path = autoround.quantize_and_save(output_dir, format=fmt)+ return quantized_model_path++[email protected]("iters", [0, 1])+def test_nvfp4_e5m3_gs_auto_round_export_and_inference(tiny_opt_model_path, dataloader, tmp_path, iters):+ from transformers import AutoModelForCausalLM++ from auto_round.experimental import qmodules as ar_qmodules++ model_dir = _quantize(tiny_opt_model_path, dataloader, str(tmp_path / "out"), "auto_round", iters=iters)++ with open(os.path.join(model_dir, "config.json")) as f:+ quantization_config = json.load(f)["quantization_config"]+ assert quantization_config["packing_format"] == "auto_round:llm_compressor_nvfp4_e5m3_gs"+ assert quantization_config["data_type"] == "nvfp4_v2_gs"+ assert quantization_config["act_data_type"] == "nvfp4_v2_with_static_gs"++ names = [+ f"{PREFIX}.{proj}.{suffix}"+ for proj in ("q_proj", "k_proj", "v_proj")+ for suffix in ("weight_scale", "weight_global_scale", "input_global_scale")+ ]+ tensors = _read_tensors(model_dir, names)+ assert set(tensors) == set(names)+ for proj in ("q_proj", "k_proj", "v_proj"):+ assert tensors[f"{PREFIX}.{proj}.weight_scale"].dtype == torch.uint8+ assert tensors[f"{PREFIX}.{proj}.weight_global_scale"].dtype == torch.float32+ assert tensors[f"{PREFIX}.{proj}.input_global_scale"].item() != 1.0+ # q/k/v share one weight and one input global scale for fused kernels.+ for suffix in ("weight_global_scale", "input_global_scale"):+ assert torch.equal(tensors[f"{PREFIX}.q_proj.{suffix}"], tensors[f"{PREFIX}.k_proj.{suffix}"])+ assert torch.equal(tensors[f"{PREFIX}.q_proj.{suffix}"], tensors[f"{PREFIX}.v_proj.{suffix}"])++ model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="cpu")+ layer = model.model.decoder.layers[0].self_attn.q_proj+ assert isinstance(layer, ar_qmodules.NVFP4E5M3GSQuantLinear)+ assert torch.equal(layer.weight_global_scale.cpu(), tensors[f"{PREFIX}.q_proj.weight_global_scale"])+ with torch.no_grad():+ logits = model(torch.tensor([[2, 100, 200, 300]])).logits+ assert torch.isfinite(logits).all()+++def test_nvfp4_e5m3_gs_llm_compressor_export(tiny_opt_model_path, dataloader, tmp_path):+ model_dir = _quantize(tiny_opt_model_path, dataloader, str(tmp_path / "out"), "llm_compressor")++ with open(os.path.join(model_dir, "config.json")) as f:+ quantization_config = json.load(f)["quantization_config"]+ assert quantization_config["format"] == "nvfp4-e5m3-gs-pack-quantized"+ assert quantization_config["quant_method"] == "compressed-tensors"++ names = [f"{PREFIX}.q_proj.weight_global_scale", f"{PREFIX}.q_proj.input_global_scale"]+ tensors = _read_tensors(model_dir, names)+ assert set(tensors) == set(names)+++def test_nvfp4_e5m3_gs_weight_global_scale_uses_ue5m3_range(tiny_opt_model_path, dataloader):+ from transformers import AutoModelForCausalLM++ orig_weight = AutoModelForCausalLM.from_pretrained(tiny_opt_model_path).model.decoder.layers[0].fc2.weight+ autoround = AutoRound(+ tiny_opt_model_path,+ scheme="NVFP4_E5M3_GS",+ iters=0,+ seqlen=8,+ nsamples=2,+ dataset=dataloader,+ model_dtype=torch.float32,+ amp=False,+ )+ model, _ = autoround.quantize()+ layer = model.model.decoder.layers[0].fc2+ layer = getattr(layer, "orig_layer", layer)+ expected = FLOAT8_UE5M3_MAX * FLOAT4_E2M1_MAX / orig_weight.float().abs().max()+ torch.testing.assert_close(layer.weight_global_scale.float().reshape(()), expected, rtol=1e-6, atol=0)diff --git a/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.pyindex ced441ab1..f3320518d 100644--- a/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py+++ b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py@@ -1,11 +1,19 @@import pytestimport torch-from auto_round.data_type.nvfp import cast_to_fp4, e5m3_to_float_tensor, nvfp4_v2+from auto_round.data_type.nvfp import (+ calculate_gparam_e5m3,+ cast_to_fp4,+ e5m3_to_float_tensor,+ nvfp4_v2,+ nvfp4_v2_with_static_gs,+)from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8from auto_round_extension.cuda.cute_nvfp4_e5m3 import (can_use_cute_nvfp4_v2_qdq,+ try_cute_nvfp4_e5m3_gs_weight_dq,try_cute_nvfp4_e5m3_weight_dq,+ try_cute_nvfp4_v2_gs_qdq,try_cute_nvfp4_v2_qdq,)@@ -78,3 +86,78 @@ def test_nvfp4_v2_direct_scale_handles_zero_group():assert torch.equal(actual, activation)assert torch.equal(scale, torch.zeros_like(scale))++[email protected](not torch.cuda.is_available(), reason="CUDA is required")+ ("dtype", "atol"),+ [(torch.float32, 1e-6), (torch.float16, 2e-3), (torch.bfloat16, 2e-2)],+)+def test_cute_nvfp4_v2_gs_qdq_matches_reference(dtype, atol):+ if torch.cuda.get_device_capability()[0] < 8:+ pytest.skip("CuTe QDQ requires SM80 or newer")++ torch.manual_seed(0)+ activation = torch.randn(64, 16, device="cuda", dtype=dtype)+ activation[0] = 0+ if not can_use_cute_nvfp4_v2_qdq(activation, 16):+ pytest.skip("CuTe DSL is not available")++ # Calibrated max below the runtime max exercises the UE5M3 upper clamp.+ global_scale = calculate_gparam_e5m3(activation.float().abs().max() * 0.5).reshape(1)+ expected, _, _ = nvfp4_v2_with_static_gs(activation.float(), group_size=16, global_scale=global_scale)+ actual = try_cute_nvfp4_v2_gs_qdq(activation, global_scale, 16)++ assert actual is not None+ torch.testing.assert_close(actual, expected.to(dtype), rtol=0, atol=atol)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")+def test_cute_nvfp4_v2_gs_qdq_handles_zero_global_scale():+ if torch.cuda.get_device_capability()[0] < 8:+ pytest.skip("CuTe QDQ requires SM80 or newer")++ activation = torch.zeros((4, 16), device="cuda", dtype=torch.float32)+ global_scale = torch.zeros(1, device="cuda", dtype=torch.float32)+ actual = try_cute_nvfp4_v2_gs_qdq(activation, global_scale, 16)+ if actual is None:+ pytest.skip("CuTe DSL is not available")++ assert torch.isfinite(actual).all()+ assert torch.equal(actual, activation)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])+def test_cute_nvfp4_e5m3_gs_weight_dq_matches_reference(dtype):+ if torch.cuda.get_device_capability()[0] < 8:+ pytest.skip("CuTe weight DQ requires SM80 or newer")++ torch.manual_seed(42)+ weight_packed = torch.randint(0, 256, (37, 64), device="cuda", dtype=torch.uint8)+ weight_scale = torch.randint(0, 0xFE, (37, 8), device="cuda", dtype=torch.uint8)+ global_scale = torch.tensor([12345.0], device="cuda")+ actual = try_cute_nvfp4_e5m3_gs_weight_dq(weight_packed, weight_scale, global_scale, dtype)+ if actual is None:+ pytest.skip("CuTe DSL is not available")++ unpacked = unpack_fp4_from_uint8(weight_packed, 37, 128, dtype=dtype).float()+ scale = (e5m3_to_float_tensor(weight_scale) / global_scale).reshape(-1, 1)+ expected = (unpacked.reshape(-1, 16) * scale).reshape(37, 128).to(dtype)+ torch.testing.assert_close(actual, expected, rtol=0, atol=0)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")+def test_cute_nvfp4_e5m3_gs_weight_dq_handles_zero_global_scale():+ if torch.cuda.get_device_capability()[0] < 8:+ pytest.skip("CuTe weight DQ requires SM80 or newer")++ weight_packed = torch.zeros((2, 8), device="cuda", dtype=torch.uint8)+ weight_scale = torch.zeros((2, 1), device="cuda", dtype=torch.uint8)+ global_scale = torch.zeros(1, device="cuda", dtype=torch.float32)+ actual = try_cute_nvfp4_e5m3_gs_weight_dq(weight_packed, weight_scale, global_scale, torch.float32)+ if actual is None:+ pytest.skip("CuTe DSL is not available")++ assert torch.isfinite(actual).all()+ assert torch.equal(actual, torch.zeros_like(actual))