Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: Add NVFP4_E5M3_GS scheme with global scale support (#2531)

Xin He91a9bbfc8dd4
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index 00aaa4e84..46b3e28d3 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -265,10 +265,10 @@ class AlgorithmComposer:
Returns a list of hook handles that the caller must remove when done.
"""
- from auto_round.compressors.utils import is_nv_fp
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+ from auto_round.schemes import needs_nvfp_global_scale as is_nv_fp
- is_act_nv_fp = getattr(self.block_quantizer.config, "is_act_nv_fp", False)
+ is_act_nv_fp = is_nv_fp(getattr(self.block_quantizer.config, "act_data_type", None) or "")
def collect_act_max(module, input, output):
input = input[0] if isinstance(input, (tuple, list)) else input
@@ -350,8 +350,8 @@ class AlgorithmComposer:
q_inputs: Optional list of quantized input tensors; used instead of
``fp_inputs`` when provided.
"""
- from auto_round.compressors.utils import is_nv_fp
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+ from auto_round.schemes import needs_nvfp_global_scale as is_nv_fp
target_input = q_inputs or fp_inputs
act_group_size = layer.act_group_size
@@ -500,10 +500,10 @@ class AlgorithmComposer:
# ── Step 3.5: MoE scale alignment + global scale update ─────────────────
# Must run after calibration hooks (act_max collected) and before quantize_block.
if act_data_type is not None or not act_dynamic:
- from auto_round.compressors.utils import is_nv_fp
+ from auto_round.schemes import needs_nvfp_global_scale
from auto_round.utils import set_amax_for_all_moe_layers
- if is_nv_fp(act_data_type) or not act_dynamic:
+ if needs_nvfp_global_scale(act_data_type) or not act_dynamic:
set_amax_for_all_moe_layers(block, attr_name="act_max")
# SVDQuant may have replaced the original linears during pre-quantize.
@@ -583,13 +583,13 @@ class AlgorithmComposer:
"""
# Attach act_max for static activation quantization when inputs are available.
if fp_inputs is not None:
- from auto_round.compressors.utils import is_nv_fp
+ from auto_round.schemes import needs_nvfp_global_scale
act_data_type = layer.act_data_type
if act_data_type is None:
act_data_type = "fp"
act_dynamic = getattr(layer, "act_dynamic", True)
- if is_nv_fp(act_data_type) or not act_dynamic:
+ if needs_nvfp_global_scale(act_data_type) or not act_dynamic:
self._attach_act_max_for_outside_layer(layer, fp_inputs, q_inputs)
# Infrastructure: move layer to the tuning device before handing off to the quantizer.
diff --git a/auto_round/algorithms/quantization/config.py b/auto_round/algorithms/quantization/config.py
index a570ec769..14416b084 100644
--- a/auto_round/algorithms/quantization/config.py
+++ b/auto_round/algorithms/quantization/config.py
@@ -16,7 +16,7 @@ from typing import ClassVar
from auto_round.algorithms.config import AlgorithmConfig
from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG
from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme
+from auto_round.schemes import QuantizationScheme, is_nvfp4_e5m3_family
class QuantizationConfig(AlgorithmConfig):
@@ -161,6 +161,7 @@ class QuantizationConfig(AlgorithmConfig):
if (
self.is_act_quantize
and (not self.is_act_nv_fp or "static_gs" not in self.act_data_type)
+ and not is_nvfp4_e5m3_family(self.act_data_type or "")
and not self.is_act_mx_fp
and not self.is_dynamic_wint8aint8
and not self.is_static_afp8
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index 15c429dd9..ff95463ac 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -31,6 +31,7 @@ from auto_round.compressors.utils import (
snapshot_best_params,
)
from auto_round.logger import logger
+from auto_round.schemes import is_nvfp4_e5m3_gs
from auto_round.utils.perf_counters import _PerfCounters
@@ -769,7 +770,7 @@ class SignRoundQuantizer(BaseQuantizer):
_perf.stop("tail")
_log_tune_phases(_perf, self.iters)
- if self.config.is_act_nv_fp:
+ if self.config.is_act_nv_fp or is_nvfp4_e5m3_gs(self.config.act_data_type or ""):
# enable moe experts act_max automatic generation for WrapperWALayer
set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")
diff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.py
index 5ef7e1d71..3cae1a44d 100644
--- a/auto_round/algorithms/quantization/utils.py
+++ b/auto_round/algorithms/quantization/utils.py
@@ -14,8 +14,9 @@
import torch
-from auto_round.compressors.utils import check_need_act_calibration, is_nv_fp
+from auto_round.compressors.utils import check_need_act_calibration
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+from auto_round.schemes import needs_nvfp_global_scale
from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized
@@ -27,7 +28,9 @@ def register_act_max_hooks(quantizer, model):
if input.numel() == 0:
return
module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)
- is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else quantizer.config.is_act_nv_fp
+ is_module_act_nv_fp = needs_nvfp_global_scale(
+ module_act_data_type or getattr(quantizer.config, "act_data_type", None) or ""
+ )
input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)
act_max = torch.max(torch.abs(input), dim=-1).values
if not hasattr(module, "act_max") or module.act_max.numel() == 0:
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index 2462b1d57..8e7ffdb72 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -48,6 +48,8 @@ weights with ``fake`` format):
* Preset name: ``NVFP4_E5M3``.
* ``data_type="nvfp4_v2"``, ``group_size=16``, with high-precision QDQ weights.
+* Preset name: ``NVFP4_E5M3_GS`` (``data_type="nvfp4_v2_gs"``) adds a per-tensor weight global
+ scale; model-free mode writes ``input_global_scale=1``.
Schemes that require special packing (FP8, standard NVFP4, GGUF, INT8_W8A8,
BF16, FPW8A16, ...) are **not** supported in model-free mode and will raise
@@ -188,6 +190,7 @@ SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = (
"MXFP4",
"MXFP8",
"NVFP4_E5M3",
+ "NVFP4_E5M3_GS",
"BF16",
)
@@ -199,6 +202,8 @@ _SUPPORTED_INT_BITS: tuple[int, ...] = (2, 4, 8)
_SUPPORTED_MXFP_BITS: tuple[int, ...] = (4, 8)
_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"
+_NVFP4_E5M3_GS_DATA_TYPE = "nvfp4_v2_gs"
+_NVFP4_E5M3_FAMILY_DATA_TYPES = (_NVFP4_E5M3_DATA_TYPE, _NVFP4_E5M3_GS_DATA_TYPE)
# Standalone quantization metadata used by different source formats. Only the
# newly generated quantization_config.json may remain in model-free output.
@@ -1756,11 +1761,13 @@ class _ModelFreeCompressorCore:
packing_format = "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"
elif data_type == _NVFP4_E5M3_DATA_TYPE:
packing_format = "fake" if self.format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"
+ elif data_type == _NVFP4_E5M3_GS_DATA_TYPE:
+ packing_format = "fake" if self.format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3_gs"
else:
packing_format = "fake" if self.format == "fake" else "auto_round:auto_gptq"
supports_opt_rtn = (
is_mx_fp(data_type)
- or data_type == _NVFP4_E5M3_DATA_TYPE
+ or data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES
or _layer_config_has_mxfp(self.layer_config)
or _layer_config_has_nvfp4(self.layer_config)
)
@@ -2218,7 +2225,10 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
normalized_scheme is not None and is_mx_fp((normalized_scheme.data_type or "").lower())
) or self._auto_scheme_family == "mx_fp":
_accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}
- elif normalized_scheme is not None and (normalized_scheme.data_type or "").lower() == _NVFP4_E5M3_DATA_TYPE:
+ elif (
+ normalized_scheme is not None
+ and (normalized_scheme.data_type or "").lower() in _NVFP4_E5M3_FAMILY_DATA_TYPES
+ ):
_accepted_formats = {"fake", "llm_compressor", "auto_round", "auto_round:auto_gptq"}
elif _is_full_precision_default(self.scheme_input) and _layer_config_has_mxfp(self.layer_config_input):
# BF16 default with MXFP layer_config overrides.
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.py
index 7e5e20ec9..e34c31ced 100644
--- a/auto_round/compressors/orchestrator.py
+++ b/auto_round/compressors/orchestrator.py
@@ -34,11 +34,11 @@ from auto_round.compressors.base import BaseOrchestrator
from auto_round.compressors.utils import (
_get_quantized_layer_names_outside_blocks,
immediate_pack,
- is_nv_fp,
)
from auto_round.data_type.utils import update_block_global_scale_if_needed
from auto_round.logger import logger
from auto_round.modeling.fused_moe.replace_modules import materialize_model_
+from auto_round.schemes import needs_nvfp_global_scale
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_to_quantized,
@@ -726,7 +726,7 @@ class CompressionOrchestrator(BaseOrchestrator):
)
_zs_block_idx += 1
# ── MoE scale alignment for FP8 dispatch efficiency ────────────────
- if is_nv_fp(self.act_data_type) or not self.act_dynamic:
+ if needs_nvfp_global_scale(self.act_data_type or "") or not self.act_dynamic:
set_amax_for_all_moe_layers(block, attr_name="act_max")
update_block_global_scale_if_needed(block, self.data_type, self.group_size)
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index 624e91b0c..435353444 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -53,14 +53,15 @@ FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max if hasattr(torch, "float8
FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min if hasattr(torch, "float8_e4m3fn") else -448
-def calculate_gparam(tensor, group_size=16, device="cpu"):
+def calculate_gparam(tensor, group_size=16, device="cpu", scale_max=None):
"""Calculate global scaling factor for NVFP quantization."""
assert group_size == 16, f"Only group_size=16 is supported, got {group_size}"
+ scale_max = FLOAT8_E4M3_MAX if scale_max is None else scale_max
if isinstance(tensor, (float, int)):
tensor_amax = torch.tensor(tensor, device=device, dtype=torch.float32).abs()
elif isinstance(tensor, torch.Tensor):
tensor_amax = tensor.to(torch.float32).abs().max()
- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)
+ global_scale = scale_max * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)
return global_scale
@@ -233,6 +234,59 @@ def nvfp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):
return qdq_res.to(orig_dtype), scale, None
+def calculate_gparam_e5m3(tensor, group_size=16, device="cpu"):
+ """Calculate the per-tensor global scale for NVFP4 E5M3 (UE5M3 block scales)."""
+ return calculate_gparam(tensor, group_size, device, scale_max=FLOAT8_UE5M3_MAX)
+
+
+def calculate_gparam_by_dtype(tensor, data_type, group_size=16, device="cpu"):
+ if "nvfp4_v2" in str(data_type).lower():
+ return calculate_gparam_e5m3(tensor, group_size, device)
+ return calculate_gparam(tensor, group_size, device)
+
+
+def _to_global_scale_tensor(global_scale, device):
+ if not isinstance(global_scale, torch.Tensor):
+ global_scale = torch.tensor(global_scale, dtype=torch.float32)
+ return global_scale.to(device=device, dtype=torch.float32).reshape(-1)[:1]
+
+
+@register_dtype("nvfp4_v2_gs")
+def nvfp4_v2_gs(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, **kwargs):
+ """FP4 E2M1 with UE5M3 block scales and a per-tensor FP32 global scale."""
+ assert group_size == 16, f"NVFP4_E5M3_GS only supports group_size=16, got {group_size}"
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if global_scale is None:
+ global_scale = calculate_gparam_e5m3(tensor, group_size)
+ global_scale = _to_global_scale_tensor(global_scale, tensor.device)
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.reshape(-1).to(tensor.device)
+ qdq_res, scale = ref_fp4_quant(tensor, global_scale, group_size, v, max_scale)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), scale, None
+
+
+@register_dtype("nvfp4_v2_with_static_gs")
+def nvfp4_v2_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, global_scale=None, **kwargs):
+ """Activation QDQ for NVFP4_E5M3_GS; the global scale comes from calibrated ``tensor_max``."""
+ if tensor is None or tensor.numel() == 0:
+ return tensor, None, None
+ assert group_size == 16, f"NVFP4_E5M3_GS only supports group_size=16, got {group_size}"
+ orig_dtype = tensor.dtype
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if global_scale is None:
+ if tensor_max is None:
+ tensor_max = tensor.to(torch.float32).abs().max()
+ elif isinstance(tensor_max, torch.Tensor):
+ tensor_max = tensor_max.to(device=tensor.device, dtype=torch.float32).abs().max()
+ global_scale = calculate_gparam_e5m3(tensor_max, group_size, tensor.device)
+ global_scale = _to_global_scale_tensor(global_scale, tensor.device)
+ qdq_res, scale = ref_fp4_quant(tensor, global_scale, group_size, v)
+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_res.to(orig_dtype), scale, None
+
+
@register_dtype("nv_fp4_rtn")
def nv_fp4_rtn(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, init_scale=1.0, **kwargs):
orig_dtype = tensor.dtype
@@ -409,6 +463,33 @@ def opt_rtn_fast_nvfp4(
return nv_fp4_rtn(tensor, bits, group_size, v, global_scale, max_scale, init_scale=init_scale)
+@register_dtype("opt_rtn_nvfp4_v2_gs")
+def opt_rtn_nvfp4_v2_gs(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, imatrix=1.0, **kwargs):
+ grouped, _, _ = reshape_pad_tensor_by_group_size(tensor, group_size)
+ if not isinstance(imatrix, torch.Tensor):
+ qw = 1.0
+ else:
+ imatrix = imatrix.reshape(1, -1)
+ imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)
+ imatrix = imatrix.expand(grouped.numel() // imatrix.numel(), -1)
+ imatrix = imatrix.reshape(grouped.shape)
+ qw = _imatrix_handle_zero(imatrix, grouped, bits, group_size)
+ if global_scale is None:
+ global_scale = calculate_gparam_e5m3(grouped, group_size)
+
+ def _candidate(t, bits=4, group_size=16, v=0, max_scale=1.0):
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.reshape(-1)
+ return nvfp4_v2_gs(t, bits, group_size, v, global_scale=global_scale, max_scale=max_scale)
+
+ init_scale = search_nvfp4_scale(grouped, bits, qw, quant_func=_candidate, group_size=group_size).reshape(-1)
+ if isinstance(max_scale, torch.Tensor):
+ init_scale = init_scale * max_scale.reshape(-1).to(init_scale.device)
+ else:
+ init_scale = init_scale * max_scale
+ return nvfp4_v2_gs(tensor, bits, group_size, v, global_scale=global_scale, max_scale=init_scale)
+
+
@register_dtype("rtn_nv_fp4_with_static_gs")
def rtn_nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):
if tensor is None or tensor.numel() == 0:
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 64f821cf2..e8eae2b94 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -22,8 +22,8 @@ import torch
from torch.nn import Linear, Module
from auto_round import envs
-from auto_round.compressors.utils import is_nv_fp
from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
+from auto_round.schemes import needs_nvfp_global_scale
from auto_round.utils import check_to_quantized, logger
@@ -563,9 +563,9 @@ def update_fused_layer_global_scales(
def update_block_global_scale_if_needed(block, data_type, group_size):
- from auto_round.data_type.nvfp import calculate_gparam
+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype
- has_nvfp = is_nv_fp(data_type)
+ has_nvfp = needs_nvfp_global_scale(data_type)
# Calculate block wise weight global scale
for _, m in block.named_modules():
@@ -574,10 +574,10 @@ def update_block_global_scale_if_needed(block, data_type, group_size):
# Check per-layer data_type for mixed-scheme scenarios
module_data_type = getattr(m, "data_type", data_type)
module_group_size = getattr(m, "group_size", group_size)
- if is_nv_fp(module_data_type):
+ if needs_nvfp_global_scale(module_data_type):
has_nvfp = True
if not hasattr(m, "weight_global_scale"):
- weight_global_scale = calculate_gparam(m.weight, module_group_size)
+ weight_global_scale = calculate_gparam_by_dtype(m.weight, module_data_type, module_group_size)
m.weight_global_scale = weight_global_scale
if not has_nvfp:
diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.py
index e638f95bb..429f03afa 100644
--- a/auto_round/experimental/qmodules/__init__.py
+++ b/auto_round/experimental/qmodules/__init__.py
@@ -21,5 +21,10 @@ from auto_round.experimental.qmodules.mx import (
from auto_round.experimental.qmodules.fake import FakeActQuantLinear
from auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinear
-from auto_round.experimental.qmodules.nvfp4_e5m3 import CuteNVFP4E5M3QuantLinear, NVFP4E5M3QuantLinear
+from auto_round.experimental.qmodules.nvfp4_e5m3 import (
+ CuteNVFP4E5M3GSQuantLinear,
+ CuteNVFP4E5M3QuantLinear,
+ NVFP4E5M3GSQuantLinear,
+ NVFP4E5M3QuantLinear,
+)
from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear
diff --git a/auto_round/experimental/qmodules/fake.py b/auto_round/experimental/qmodules/fake.py
index 322ab9aeb..b17d04485 100644
--- a/auto_round/experimental/qmodules/fake.py
+++ b/auto_round/experimental/qmodules/fake.py
@@ -17,7 +17,7 @@ import torch
from auto_round.data_type.utils import get_quant_func
from auto_round.experimental.qmodules.base import QModuleBase
-from auto_round.schemes import QuantizationScheme, is_nv_fp
+from auto_round.schemes import QuantizationScheme, needs_nvfp_global_scale
__all__ = ["FakeActQuantLinear"]
@@ -45,7 +45,7 @@ class FakeActQuantLinear(QModuleBase):
self.register_parameter("bias", None)
else:
self.bias = torch.nn.Parameter(bias, requires_grad=False)
- if is_nv_fp(config.act_data_type or ""):
+ if needs_nvfp_global_scale(config.act_data_type or ""):
self.register_buffer("input_global_scale", torch.ones(1, dtype=torch.float32))
@classmethod
diff --git a/auto_round/experimental/qmodules/nvfp4_e5m3.py b/auto_round/experimental/qmodules/nvfp4_e5m3.py
index 42f2b5528..ce3240104 100644
--- a/auto_round/experimental/qmodules/nvfp4_e5m3.py
+++ b/auto_round/experimental/qmodules/nvfp4_e5m3.py
@@ -16,18 +16,25 @@ import os
import torch
-from auto_round.data_type.nvfp import e5m3_to_float_tensor, nvfp4_v2
+from auto_round.data_type.nvfp import e5m3_to_float_tensor, get_reciprocal, nvfp4_v2, nvfp4_v2_with_static_gs
from auto_round.experimental.qmodules.base import QModuleBase
from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round_extension.cuda.cute_nvfp4_e5m3 import (
+ try_cute_nvfp4_e5m3_gs_weight_dq,
try_cute_nvfp4_e5m3_linear,
try_cute_nvfp4_e5m3_weight_dq,
+ try_cute_nvfp4_v2_gs_qdq,
try_cute_nvfp4_v2_qdq,
)
-__all__ = ["CuteNVFP4E5M3QuantLinear", "NVFP4E5M3QuantLinear"]
+__all__ = [
+ "CuteNVFP4E5M3GSQuantLinear",
+ "CuteNVFP4E5M3QuantLinear",
+ "NVFP4E5M3GSQuantLinear",
+ "NVFP4E5M3QuantLinear",
+]
_CACHE_WEIGHT_ENV = "AR_NVFP4_E5M3_CACHE_HP_WEIGHT"
@@ -158,3 +165,54 @@ class CuteNVFP4E5M3QuantLinear(NVFP4E5M3QuantLinear):
if fused_output is not None:
return fused_output
return super().forward(input)
+
+
+class NVFP4E5M3GSQuantLinear(NVFP4E5M3QuantLinear):
+ """NVFP4 E5M3 linear with per-tensor FP32 weight and input global scales."""
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.register_buffer("weight_global_scale", torch.ones((1,), dtype=torch.float32))
+ self.register_buffer("input_global_scale", torch.ones((1,), dtype=torch.float32))
+
+ def load_state_dict(self, state_dict, strict=True, assign=False):
+ for name in ("weight_global_scale", "input_global_scale"):
+ if name in state_dict and state_dict[name].dtype != torch.float32:
+ state_dict[name] = state_dict[name].to(torch.float32)
+ return super().load_state_dict(state_dict, strict, assign)
+
+ def dequant_weight_online(self) -> torch.Tensor:
+ unpacked = unpack_fp4_from_uint8(self.weight_packed, self.out_features, self.in_features, dtype=self.dtype).to(
+ torch.float32
+ )
+ weight_global_scale = self.weight_global_scale.to(device=unpacked.device, dtype=torch.float32)
+ scale = (e5m3_to_float_tensor(self.weight_scale) * get_reciprocal(weight_global_scale)).reshape(-1, 1)
+ return (unpacked.reshape(-1, self.group_size) * scale).reshape(self.out_features, self.in_features)
+
+ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:
+ original_dtype = activation.dtype
+ qdq_activation, _, _ = nvfp4_v2_with_static_gs(
+ activation.to(torch.float32),
+ bits=self.config.act_bits,
+ group_size=self.config.act_group_size,
+ global_scale=self.input_global_scale,
+ )
+ return qdq_activation.to(original_dtype)
+
+
+class CuteNVFP4E5M3GSQuantLinear(NVFP4E5M3GSQuantLinear):
+ """NVFP4 E5M3 GS linear that dispatches activation QDQ and weight dequantization to CuTe."""
+
+ def dequant_weight_online(self) -> torch.Tensor:
+ cute_weight = try_cute_nvfp4_e5m3_gs_weight_dq(
+ self.weight_packed, self.weight_scale, self.weight_global_scale, self.dtype
+ )
+ if cute_weight is not None:
+ return cute_weight
+ return super().dequant_weight_online()
+
+ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:
+ cute_qdq_activation = try_cute_nvfp4_v2_gs_qdq(activation, self.input_global_scale, self.config.act_group_size)
+ if cute_qdq_activation is not None:
+ return cute_qdq_activation
+ return super().qdq_input(activation)
diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
index ef87ffab3..8f76d0fdb 100644
--- a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
@@ -24,7 +24,7 @@ import torch.nn as nn
import transformers
from tqdm import tqdm
-from auto_round.compressors.utils import is_mx_fp, is_nv_fp
+from auto_round.compressors.utils import is_mx_fp
from auto_round.export.export_to_autoround.utils import check_neq_config
from auto_round.export.utils import (
filter_quantization_config,
@@ -34,7 +34,7 @@ from auto_round.export.utils import (
save_pretrained_artifact,
)
from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme
+from auto_round.schemes import QuantizationScheme, needs_nvfp_global_scale
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_start_with_block_name,
@@ -77,13 +77,13 @@ def pack_layer(name, model, backend, device=None):
group_size = layer.group_size
sym = layer.sym
- if is_nv_fp(act_data_type) and act_bits <= 8:
+ if needs_nvfp_global_scale(act_data_type) and act_bits <= 8:
input_global_scale = getattr(layer, "input_global_scale", None)
if input_global_scale is None:
assert hasattr(layer, "act_max")
- from auto_round.data_type.nvfp import calculate_gparam
+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype
- input_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu")
+ input_global_scale = calculate_gparam_by_dtype(layer.act_max, act_data_type, layer.group_size, "cpu")
layer.input_global_scale = input_global_scale
delattr(layer, "act_max")
@@ -189,7 +189,7 @@ def save_quantized_as_fp(
orig_layer = m.orig_layer
set_module(model, n, orig_layer)
- if is_nv_fp(act_data_type) and "static_gs" in str(act_data_type).lower():
+ if needs_nvfp_global_scale(act_data_type or "") and "static_gs" in str(act_data_type).lower():
# Ensure all MOE layers have act_max set (needed after deep copy or for uncalibrated layers)
from auto_round.utils.model import is_moe_model, set_amax_for_all_moe_layers
@@ -202,9 +202,12 @@ def save_quantized_as_fp(
layer = m
if hasattr(layer, "act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None):
assert hasattr(layer, "act_max")
- from auto_round.data_type.nvfp import calculate_gparam
+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype
- input_global_scale = calculate_gparam(layer.act_max, layer.group_size, model.device)
+ layer_act_data_type = getattr(layer, "act_data_type", None) or act_data_type
+ input_global_scale = calculate_gparam_by_dtype(
+ layer.act_max, layer_act_data_type, layer.group_size, model.device
+ )
layer.input_global_scale = input_global_scale
delattr(layer, "act_max")
# update fused input_global_scale
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.py
index d9082e1e4..f0b8457bc 100644
--- a/auto_round/export/export_to_autoround/qlinear_fp.py
+++ b/auto_round/export/export_to_autoround/qlinear_fp.py
@@ -38,6 +38,7 @@ from auto_round.compressors.utils import BackendDataType, is_mx_fp, is_nv_fp
from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL
from auto_round.data_type.nvfp import cast_to_fp4, float_to_e5m3_frexp, get_reciprocal
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.schemes import is_nvfp4_e5m3_gs
from auto_round.utils import get_packing_device, logger
# from auto_round.utils import get_weight_compress_dtype
@@ -74,6 +75,8 @@ class QuantLinear(nn.Module):
self.is_mx = is_mx_fp(data_type)
self.is_nv = is_nv_fp(data_type)
self.is_nvfp4_e5m3 = data_type == "nvfp4_v2"
+ self.is_nvfp4_e5m3_gs = is_nvfp4_e5m3_gs(data_type)
+ self.has_global_scale = self.is_nv or self.is_nvfp4_e5m3_gs
if self.is_mx:
if group_size != 32:
raise NotImplementedError(f"Only group_size 32 are supported for {BackendDataType.MX_FP} data type.")
@@ -81,7 +84,7 @@ class QuantLinear(nn.Module):
raise NotImplementedError(
f"in_feature must be divisible by {group_size} for {BackendDataType.MX_FP} data type."
)
- if self.is_nv or self.is_nvfp4_e5m3:
+ if self.is_nv or self.is_nvfp4_e5m3 or self.is_nvfp4_e5m3_gs:
if group_size % 16 != 0:
raise NotImplementedError(f"Only group_size 16 are supported for {BackendDataType.NV_FP} data type.")
if infeatures % group_size != 0:
@@ -112,7 +115,7 @@ class QuantLinear(nn.Module):
dtype=torch.float16, ## TODO update to correct scale dtype for different bits
),
)
- if self.is_nv and self.bits == 4:
+ if self.has_global_scale and self.bits == 4:
self.register_buffer(
"weight_global_scale",
torch.zeros(
@@ -120,7 +123,7 @@ class QuantLinear(nn.Module):
dtype=torch.float32,
),
)
- if self.is_nv and self.act_bits <= 8:
+ if self.has_global_scale and self.act_bits <= 8:
self.register_buffer(
"input_global_scale",
torch.zeros(
@@ -151,7 +154,7 @@ class QuantLinear(nn.Module):
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(W, self.group_size)
scales = scales.to(device)
- if self.is_nv:
+ if self.has_global_scale:
assert global_scale is not None and global_scale.numel() == 1
global_scale = global_scale.reshape([1])
global_scale = global_scale.to(device)
@@ -169,7 +172,7 @@ class QuantLinear(nn.Module):
scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)
if self.is_mx:
final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)
- elif self.is_nvfp4_e5m3:
+ elif self.is_nvfp4_e5m3 or self.is_nvfp4_e5m3_gs:
final_scale = float_to_e5m3_frexp(scales.to(torch.float32))
else:
final_scale = scales.to(torch.float8_e4m3fn)
diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.py
index 99eb1196d..c974c65c3 100644
--- a/auto_round/export/export_to_llmcompressor/config.py
+++ b/auto_round/export/export_to_llmcompressor/config.py
@@ -103,8 +103,8 @@ def initialize_quantization(scheme, targets=None, config_groups=None, kv_cache_s
)
-def initialize_nvfp4_e5m3_quantization(ignore=None):
- """Build stable compressed-tensors metadata for global-scale-free NVFP4 E5M3."""
+def initialize_nvfp4_e5m3_quantization(ignore=None, with_global_scale=False):
+ """Build stable compressed-tensors metadata for NVFP4 E5M3 (optionally with per-tensor global scales)."""
def quant_args(dynamic):
return {
@@ -129,7 +129,7 @@ def initialize_nvfp4_e5m3_quantization(ignore=None):
"weights": quant_args(False),
}
},
- "format": "nvfp4-e5m3-pack-quantized",
+ "format": "nvfp4-e5m3-gs-pack-quantized" if with_global_scale else "nvfp4-e5m3-pack-quantized",
"global_compression_ratio": None,
"ignore": list(dict.fromkeys(ignore or ["lm_head"])),
"kv_cache_scheme": None,
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index bafa994b3..137cd0ab1 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -40,6 +40,7 @@ from auto_round.export.utils import (
save_pretrained_artifact,
)
from auto_round.logger import logger
+from auto_round.schemes import is_nvfp4_e5m3_family, is_nvfp4_e5m3_gs, needs_nvfp_global_scale
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_start_with_block_name,
@@ -91,13 +92,13 @@ def pack_layer(name, model, device=None):
group_size = layer.group_size
sym = layer.sym
- if is_nv_fp(act_data_type) and act_bits <= 8:
+ if needs_nvfp_global_scale(act_data_type) and act_bits <= 8:
input_global_scale = getattr(layer, "input_global_scale", None)
if input_global_scale is None:
assert hasattr(layer, "act_max")
- from auto_round.data_type.nvfp import calculate_gparam
+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype
- input_global_scale = calculate_gparam(layer.act_max, layer.group_size) # , model.device
+ input_global_scale = calculate_gparam_by_dtype(layer.act_max, act_data_type, layer.group_size)
layer.input_global_scale = input_global_scale
delattr(layer, "act_max")
@@ -150,6 +151,8 @@ def _get_scheme(bits, data_type, act_bits=16):
return "NVFP4"
if data_type == "nvfp4_v2":
return "NVFP4_E5M3"
+ if is_nvfp4_e5m3_gs(data_type):
+ return "NVFP4_E5M3_GS"
return None
@@ -163,6 +166,8 @@ def _get_group_format(bits, data_type):
return "nvfp4-pack-quantized"
if data_type == "nvfp4_v2":
return "nvfp4-e5m3-pack-quantized"
+ if is_nvfp4_e5m3_gs(data_type):
+ return "nvfp4-e5m3-gs-pack-quantized"
return "float-quantized"
@@ -356,16 +361,19 @@ def save_quantized_as_fp(
orig_layer = m.orig_layer
set_module(model, n, orig_layer)
- if is_nv_fp(act_data_type) and "static_gs" in str(act_data_type).lower():
+ if needs_nvfp_global_scale(act_data_type or "") and "static_gs" in str(act_data_type).lower():
# generate static input_global_scale
for n, m in model.named_modules():
if type(m) in SUPPORTED_LAYER_TYPES:
layer = m
if hasattr(layer, "act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None):
assert hasattr(layer, "act_max")
- from auto_round.data_type.nvfp import calculate_gparam
+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype
- input_global_scale = calculate_gparam(layer.act_max, layer.group_size, model.device)
+ layer_act_data_type = getattr(layer, "act_data_type", None) or act_data_type
+ input_global_scale = calculate_gparam_by_dtype(
+ layer.act_max, layer_act_data_type, layer.group_size, model.device
+ )
layer.input_global_scale = input_global_scale
delattr(layer, "act_max")
# update fused input_global_scale
@@ -449,10 +457,12 @@ def save_quantized_as_fp(
quantization_config.format = _get_group_format(layer_bits, layer_data_type)
quantization_config = quantization_config.to_dict()
quantization_config["provider"] = "auto-round"
- elif data_type == "nvfp4_v2":
+ elif is_nvfp4_e5m3_family(data_type):
from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
- quantization_config = initialize_nvfp4_e5m3_quantization(ignore=ignore)
+ quantization_config = initialize_nvfp4_e5m3_quantization(
+ ignore=ignore, with_global_scale=is_nvfp4_e5m3_gs(data_type)
+ )
quantization_config["provider"] = "auto-round"
else:
scheme = _get_scheme(bits, data_type, act_bits if act_bits is not None else 16)
diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.py
index 867aff2e5..3bbe1effa 100644
--- a/auto_round/export/formats/backends/autoround.py
+++ b/auto_round/export/formats/backends/autoround.py
@@ -46,6 +46,7 @@ class AutoRoundFormat(OutputFormat):
"MXFP8",
"NVFP4",
"NVFP4_E5M3",
+ "NVFP4_E5M3_GS",
"FPW8A16",
"W2A16G64",
"W2A16G32",
@@ -76,6 +77,7 @@ class AutoRoundFormat(OutputFormat):
scheme.is_nv_fp()
or scheme.is_mx_fp()
or scheme.data_type == BackendDataType.NVFP4_E5M3.value
+ or scheme.data_type == BackendDataType.NVFP4_E5M3_GS.value
or (scheme.is_mx_int() and scheme.bits == 4) # only add mx_int4 now
):
self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)
@@ -96,7 +98,8 @@ class AutoRoundFormat(OutputFormat):
if format == "mlx":
self.backend = MLXFormat("mlx", scheme, ctx)
elif format.upper() not in list(BackendDataType.__members__.keys()) and format not in {
- BackendDataType.NVFP4_E5M3.value
+ BackendDataType.NVFP4_E5M3.value,
+ BackendDataType.NVFP4_E5M3_GS.value,
}:
raise KeyError(f"Unsupported backend format auto_round:{format}, please check")
else:
@@ -141,7 +144,10 @@ class AutoRoundFormat(OutputFormat):
backend = self.get_backend_name()
- if self.output_format == f"auto_round:{BackendDataType.NVFP4_E5M3.value}":
+ if self.output_format in (
+ f"auto_round:{BackendDataType.NVFP4_E5M3.value}",
+ f"auto_round:{BackendDataType.NVFP4_E5M3_GS.value}",
+ ):
from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer
return pack_layer(layer_name, model, device=device)
@@ -201,6 +207,11 @@ class AutoRoundFormat(OutputFormat):
backend = "auto_round:llm_compressor_nvfp4_e5m3"
export_func = save_quantized_as_fp
+ elif backend == f"auto_round:{BackendDataType.NVFP4_E5M3_GS.value}":
+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp
+
+ backend = "auto_round:llm_compressor_nvfp4_e5m3_gs"
+ export_func = save_quantized_as_fp
elif re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):
from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp
diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.py
index b5e34caaa..bd01df5d0 100644
--- a/auto_round/export/formats/backends/fake.py
+++ b/auto_round/export/formats/backends/fake.py
@@ -23,7 +23,7 @@ import torch
from auto_round.export.formats.base import OutputFormat
from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme, is_nv_fp
+from auto_round.schemes import QuantizationScheme, needs_nvfp_global_scale
from auto_round.utils import unsupported_meta_device
@@ -101,7 +101,10 @@ def _rewrite_saved_weights_without_orig_layer(output_dir: str) -> None:
def _materialize_fake_input_global_scales(model: torch.nn.Module) -> None:
"""Persist calibrated NVFP activation scales in fake-format checkpoints."""
for layer in model.modules():
- if not is_nv_fp(getattr(layer, "act_data_type", "") or "") or "input_global_scale" in layer._buffers:
+ if (
+ not needs_nvfp_global_scale(getattr(layer, "act_data_type", "") or "")
+ or "input_global_scale" in layer._buffers
+ ):
continue
input_global_scale = getattr(layer, "input_global_scale", None)
if not isinstance(input_global_scale, torch.Tensor):
diff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.py
index 749e41078..171ba8bc4 100644
--- a/auto_round/export/formats/backends/llm_compressor.py
+++ b/auto_round/export/formats/backends/llm_compressor.py
@@ -31,6 +31,7 @@ class LLMCompressorFormat(OutputFormat):
"MXFP8",
"NVFP4",
"NVFP4_E5M3",
+ "NVFP4_E5M3_GS",
"FPW8A16",
"FP8_STATIC",
"INT8",
@@ -61,11 +62,11 @@ class LLMCompressorFormat(OutputFormat):
if re.search("^(auto_round:)?llm_compressor", format):
self.output_format = format
self.backend = None
- if scheme.data_type == "nvfp4_v2":
+ if scheme.data_type in ("nvfp4_v2", "nvfp4_v2_gs"):
from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
check_compressed_tensors_supported(raise_error=True)
- self.output_format = "llm_compressor:nvfp4_v2"
+ self.output_format = f"llm_compressor:{scheme.data_type}"
elif scheme.is_nv_fp() or scheme.is_mx_fp() or (scheme.bits >= 16 and scheme.data_type in ("fp", "float")):
from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
@@ -156,7 +157,7 @@ class LLMCompressorFormat(OutputFormat):
)
if scheme.act_bits <= 8 and (not scheme.is_act_standard_fp() or scheme.act_dynamic):
- if scheme.act_data_type == "nvfp4_v2":
+ if scheme.act_data_type in ("nvfp4_v2", "nvfp4_v2_with_static_gs"):
return None, scheme, layer_config, quant_block_list
if (
(scheme.is_act_nv_fp() and "static_gs" in scheme.act_data_type)
diff --git a/auto_round/export/formats/base.py b/auto_round/export/formats/base.py
index 4c4b6d316..baf7977c4 100644
--- a/auto_round/export/formats/base.py
+++ b/auto_round/export/formats/base.py
@@ -56,6 +56,7 @@ class BackendDataType(str, Enum):
MXFP4 = "mxfp4"
NVFP4 = "nvfp4"
NVFP4_E5M3 = "nvfp4_v2"
+ NVFP4_E5M3_GS = "nvfp4_v2_gs"
FP8 = "fp8"
MX_FP = "mx_fp"
NV_FP = "nv_fp"
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index b25236007..b08f44352 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -240,6 +240,7 @@ AWQ_FORMAT = ["auto_round:auto_awq"]
LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]
FAKE_FORMAT = ["auto_round:fake", "auto_round:auto_gptq", "auto_round", "auto_round:auto_awq"]
NVFP4_E5M3_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3"]
+NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3_gs"]
WOQ_DEFAULT_ACT_BITS = [None, 16, 32]
# CPU backends that target Intel/x86 (ark / auto_round_kernel) cannot
@@ -444,6 +445,44 @@ BackendInfos["auto_round:cute_nvfp4_e5m3"] = BackendInfo(
requirements=["auto-round>0.12.0"],
)
+BackendInfos["auto_round:torch_nvfp4_e5m3_gs"] = BackendInfo(
+ device=["xpu", "cuda", "cpu"],
+ packing_format=NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT,
+ sym=[True],
+ compute_dtype=["float32", "float16", "bfloat16"],
+ data_type=["nvfp4_v2_gs"],
+ group_size=[16],
+ bits=[4],
+ act_bits=[4],
+ act_group_size=[16],
+ act_sym=[True],
+ act_data_type=["nvfp4_v2_with_static_gs"],
+ act_dynamic=[True],
+ priority=3,
+ checkers=[mxfp_nvfp_feature_checker],
+ alias=["auto_round", "torch"],
+ requirements=["auto-round>0.12.0"],
+)
+
+BackendInfos["auto_round:cute_nvfp4_e5m3_gs"] = BackendInfo(
+ device=["cuda"],
+ packing_format=NVFP4_E5M3_GS_LLM_COMPRESSOR_FORMAT,
+ sym=[True],
+ compute_dtype=["float32", "float16", "bfloat16"],
+ data_type=["nvfp4_v2_gs"],
+ group_size=[16],
+ bits=[4],
+ act_bits=[4],
+ act_group_size=[16],
+ act_sym=[True],
+ act_data_type=["nvfp4_v2_with_static_gs"],
+ act_dynamic=[True],
+ priority=6,
+ checkers=[cute_nvfp4_e5m3_checker],
+ alias=["cute_nvfp4_e5m3_gs"],
+ requirements=["auto-round>0.12.0"],
+)
+
BackendInfos["auto_round:tritonv2"] = BackendInfo(
device=["cuda", "xpu"],
data_type=["int"],
@@ -920,6 +959,10 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):
return ar_qmodules.MXINT4QuantLinear
if "torch_mxfp4" in backend:
return ar_qmodules.MXFP4QuantLinear
+ if backend == "auto_round:cute_nvfp4_e5m3_gs":
+ return ar_qmodules.CuteNVFP4E5M3GSQuantLinear
+ if backend == "auto_round:torch_nvfp4_e5m3_gs":
+ return ar_qmodules.NVFP4E5M3GSQuantLinear
if backend == "auto_round:cute_nvfp4_e5m3":
return ar_qmodules.CuteNVFP4E5M3QuantLinear
if backend == "auto_round:torch_nvfp4_e5m3":
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index eec7967c9..945595b65 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -586,6 +586,8 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,
or BackendDataType.MXINT4.value in layer_backend
or layer_backend == "auto_round:torch_nvfp4_e5m3"
or layer_backend == "auto_round:cute_nvfp4_e5m3"
+ or layer_backend == "auto_round:torch_nvfp4_e5m3_gs"
+ or layer_backend == "auto_round:cute_nvfp4_e5m3_gs"
or layer_backend == "auto_round:fake"
):
return QuantLinear.from_original(config, layer)
@@ -870,21 +872,26 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
if isinstance(quantization_config, dict)
else getattr(quantization_config, "format", None)
)
- if config_format == "nvfp4-e5m3-pack-quantized":
+ if config_format in ("nvfp4-e5m3-pack-quantized", "nvfp4-e5m3-gs-pack-quantized"):
+ with_global_scale = config_format == "nvfp4-e5m3-gs-pack-quantized"
config_dict = quantization_config if isinstance(quantization_config, dict) else quantization_config.to_dict()
ignored = config_dict.get("ignore") or []
# TODO: For experimental purpose, will delete when it's not necessary.
quantization_config = SimpleNamespace(
quant_method="auto-round",
- packing_format="auto_round:llm_compressor_nvfp4_e5m3",
+ packing_format=(
+ "auto_round:llm_compressor_nvfp4_e5m3_gs"
+ if with_global_scale
+ else "auto_round:llm_compressor_nvfp4_e5m3"
+ ),
bits=4,
group_size=16,
sym=True,
- data_type="nvfp4_v2",
+ data_type="nvfp4_v2_gs" if with_global_scale else "nvfp4_v2",
act_bits=4,
act_group_size=16,
act_sym=True,
- act_data_type="nvfp4_v2",
+ act_data_type="nvfp4_v2_with_static_gs" if with_global_scale else "nvfp4_v2",
act_dynamic=True,
extra_config={name: {"bits": 16, "act_bits": 16} for name in ignored},
)
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 3b6de11c0..de0cdc140 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -189,6 +189,21 @@ def is_mx_int(backend: str) -> bool:
return BackendDataType.MX_INT in backend.lower()
+def is_nvfp4_e5m3_gs(backend: str) -> bool:
+ """NVFP4 E5M3 with per-tensor global scale (``nvfp4_v2_gs`` / ``nvfp4_v2_with_static_gs``)."""
+ backend = backend.lower()
+ return "nvfp4_v2" in backend and backend.endswith("_gs")
+
+
+def is_nvfp4_e5m3_family(backend: str) -> bool:
+ return "nvfp4_v2" in backend.lower()
+
+
+def needs_nvfp_global_scale(backend: str) -> bool:
+ """Whether the data type uses NVFP-style per-tensor weight/input global scales."""
+ return is_nv_fp(backend) or is_nvfp4_e5m3_gs(backend)
+
+
def is_standard_fp(backend: str) -> bool:
backend = backend.lower()
return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)
@@ -785,6 +800,19 @@ NVFP4_E5M3 = QuantizationScheme.from_dict(
}
)
+NVFP4_E5M3_GS = QuantizationScheme.from_dict(
+ {
+ "bits": 4,
+ "group_size": 16,
+ "data_type": "nvfp4_v2_gs",
+ "act_bits": 4,
+ "act_data_type": "nvfp4_v2_with_static_gs",
+ "act_group_size": 16,
+ "act_sym": True,
+ "act_dynamic": True,
+ }
+)
+
FPW8A16 = QuantizationScheme.from_dict(
{
"bits": 8,
@@ -895,6 +923,7 @@ PRESET_SCHEMES = {
"MXFP8_RCEIL": MXFP8_RCEIL,
"NVFP4": NVFP4,
"NVFP4_E5M3": NVFP4_E5M3,
+ "NVFP4_E5M3_GS": NVFP4_E5M3_GS,
"FPW8A16": FPW8A16,
# Group-size variants for the 2-8 bit int presets: the bare presets default
# to group_size 128; these finer-group variants allow per-layer group-size
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index 645553003..2a932d91a 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -439,7 +439,10 @@ def _patch_nvfp4_e5m3_compressed_tensors_quantizer():
def is_nvfp4_e5m3(self):
compression_config = getattr(getattr(self, "compressor", None), "quantization_config", None)
- return getattr(compression_config, "format", None) == "nvfp4-e5m3-pack-quantized"
+ return getattr(compression_config, "format", None) in (
+ "nvfp4-e5m3-pack-quantized",
+ "nvfp4-e5m3-gs-pack-quantized",
+ )
def patched_before(self, model, **kwargs):
if not is_nvfp4_e5m3(self):
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.py
index f388c4ec9..0b273c397 100644
--- a/auto_round/utils/model_free_utils.py
+++ b/auto_round/utils/model_free_utils.py
@@ -49,6 +49,9 @@ from auto_round.utils.path_safety import (
)
_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"
+_NVFP4_E5M3_GS_DATA_TYPE = "nvfp4_v2_gs"
+_NVFP4_E5M3_GS_ACT_DATA_TYPE = "nvfp4_v2_with_static_gs"
+_NVFP4_E5M3_FAMILY_DATA_TYPES = (_NVFP4_E5M3_DATA_TYPE, _NVFP4_E5M3_GS_DATA_TYPE)
# default_scheme flag restricting quantization to layers stored as NVFP4 in the source checkpoint.
_NVFP_SOURCE_ONLY_KEY = "_nvfp_source_only"
_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")
@@ -70,6 +73,7 @@ SUPPORTED_PRESET_SCHEMES = (
"MXFP4",
"MXFP8",
"NVFP4_E5M3",
+ "NVFP4_E5M3_GS",
"BF16",
)
@@ -1318,6 +1322,86 @@ def _quantize_weight_nvfp4_e5m3(
return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}
+def _qdq_weight_nvfp4_e5m3_gs(
+ weight: torch.Tensor, layer_name: str, group_size: int, device: str, disable_opt_rtn: bool
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Return (fp32 weight, qdq weight, encoded-domain block scale, weight global scale) for NVFP4_E5M3_GS."""
+ from auto_round.data_type.nvfp import calculate_gparam_e5m3, nvfp4_v2_gs, opt_rtn_nvfp4_v2_gs
+
+ in_features = weight.shape[1]
+ if group_size != 16 or in_features % group_size != 0:
+ raise ValueError(
+ f"NVFP4_E5M3_GS requires in_features divisible by group_size=16, got group_size={group_size}, "
+ f"in_features={in_features} for '{layer_name}'."
+ )
+ weight_dev = weight.to(device=device, dtype=torch.float32)
+ global_scale = calculate_gparam_e5m3(weight_dev, group_size).reshape(1)
+ quant_func = nvfp4_v2_gs if disable_opt_rtn else opt_rtn_nvfp4_v2_gs
+ qdq_weight, scale, _ = quant_func(weight_dev, bits=4, group_size=group_size, global_scale=global_scale)
+ return weight_dev, qdq_weight, scale, global_scale
+
+
+def _quantize_weight_nvfp4_e5m3_gs(
+ weight: torch.Tensor,
+ layer_name: str,
+ group_size: int = 16,
+ device: str = "cpu",
+ disable_opt_rtn: bool = False,
+) -> dict[str, torch.Tensor]:
+ """Fake-quantize a 2D weight tensor to NVFP4_E5M3_GS; model-free uses a fixed input global scale of 1."""
+ _, qdq_weight, _, _ = _qdq_weight_nvfp4_e5m3_gs(weight, layer_name, group_size, device, disable_opt_rtn)
+ return {
+ f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu"),
+ f"{layer_name}.input_global_scale": torch.ones(1, dtype=torch.float32),
+ }
+
+
+def _pack_weight_nvfp4_e5m3_gs(
+ weight: torch.Tensor,
+ layer_name: str,
+ group_size: int = 16,
+ device: str = "cpu",
+ disable_opt_rtn: bool = False,
+) -> dict[str, torch.Tensor]:
+ """Pack FP4 E2M1 weights with UE5M3 block scales and a per-tensor weight global scale."""
+ from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
+
+ out_features, in_features = weight.shape
+ weight_dev, _, scale, global_scale = _qdq_weight_nvfp4_e5m3_gs(
+ weight, layer_name, group_size, device, disable_opt_rtn
+ )
+ scale = scale.reshape(out_features, in_features // group_size).to(torch.float32)
+ with torch.device("meta"):
+ linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=torch.float32)
+ linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)
+ with torch.device("meta"):
+ qlayer = QuantLinear(
+ 4,
+ group_size,
+ in_features,
+ out_features,
+ False,
+ data_type=_NVFP4_E5M3_GS_DATA_TYPE,
+ act_bits=4,
+ act_data_type=_NVFP4_E5M3_GS_ACT_DATA_TYPE,
+ )
+ qlayer.pack(
+ linear,
+ scale,
+ global_scale=global_scale,
+ input_global_scale=torch.ones(1, dtype=torch.float32),
+ device=device,
+ )
+ out = {
+ f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),
+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
+ f"{layer_name}.weight_global_scale": qlayer.weight_global_scale.to("cpu"),
+ f"{layer_name}.input_global_scale": qlayer.input_global_scale.to("cpu"),
+ }
+ del qlayer, linear, weight_dev, scale
+ return out
+
+
def _quantize_weight_int_fake(
weight: torch.Tensor,
layer_name: str,
@@ -1481,6 +1565,24 @@ def _quantize_single_tensor(
logger.warning(f"Failed to NVFP4_E5M3-quantize {layer_name}: {e}. Keeping original weight.")
return layer_name, {tensor_name: tensor}, None, layer_name
+ if data_type == _NVFP4_E5M3_GS_DATA_TYPE:
+ try:
+ quantize_e5m3_gs = (
+ _quantize_weight_nvfp4_e5m3_gs if scheme.get("_output_format") == "fake" else _pack_weight_nvfp4_e5m3_gs
+ )
+ out = quantize_e5m3_gs(
+ weight=tensor,
+ layer_name=layer_name,
+ group_size=group_size,
+ device=device,
+ disable_opt_rtn=disable_opt_rtn,
+ )
+ logger.debug(f"Quantized (NVFP4_E5M3_GS): {layer_name} (bits=4, group_size={group_size})")
+ return layer_name, out, layer_name, None
+ except Exception as e:
+ logger.warning(f"Failed to NVFP4_E5M3_GS-quantize {layer_name}: {e}. Keeping original weight.")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
# ---- Integer WOQ path ----
# opt_rtn is always disabled for integer WOQ in model-free mode because
# the scale search does not improve accuracy for INT quantization here.
@@ -2690,11 +2792,11 @@ def _get_llm_compressor_metadata() -> dict[str, str]:
return {"provider": "auto-round"}
-def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str]) -> dict:
- """Build compressed-tensors metadata for NVFP4 E5M3 without global scales."""
+def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str], with_global_scale: bool = False) -> dict:
+ """Build compressed-tensors metadata for NVFP4 E5M3 (optionally with per-tensor global scales)."""
from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
- qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignored_layers)
+ qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignored_layers, with_global_scale=with_global_scale)
qconfig.update(_get_llm_compressor_metadata())
return qconfig
@@ -2745,15 +2847,16 @@ def _get_mxfp_group_scheme_and_format(
tmp_qconfig = initialize_quantization(scheme="NVFP4", ignore=ignore)
return tmp_qconfig.config_groups["group_0"], "nvfp4-pack-quantized"
- if group_data_type == _NVFP4_E5M3_DATA_TYPE:
- # NVFP4_E5M3 is not a compressed-tensors preset; use its dedicated builder.
- nvfp4_dict = initialize_nvfp4_e5m3_quantization(ignore=ignore)
+ if group_data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:
+ # NVFP4_E5M3(_GS) is not a compressed-tensors preset; use its dedicated builder.
+ with_global_scale = group_data_type == _NVFP4_E5M3_GS_DATA_TYPE
+ nvfp4_dict = initialize_nvfp4_e5m3_quantization(ignore=ignore, with_global_scale=with_global_scale)
# Extract the QuantizationScheme object from the dict config.
from compressed_tensors.quantization import QuantizationScheme # pylint: disable=E0401
raw_scheme = nvfp4_dict["config_groups"]["group_0"]
scheme_obj = QuantizationScheme.model_validate(raw_scheme)
- return scheme_obj, "nvfp4-e5m3-pack-quantized"
+ return scheme_obj, nvfp4_dict["format"]
else:
scheme_name = "MXFP4" if group_bits == 4 else "MXFP8"
fmt = "mxfp4-pack-quantized" if group_bits == 4 else "mxfp8-quantized"
@@ -2789,7 +2892,11 @@ def _build_mxfp_quantization_config(
default_data_type = (default_scheme.get("data_type") or "mx_fp").lower()
is_fp_default = (bits or 0) >= 16 # BF16/FP16 full-precision default
- if not is_fp_default and bits not in _SUPPORTED_MXFP_BITS and default_data_type != _NVFP4_E5M3_DATA_TYPE:
+ if (
+ not is_fp_default
+ and bits not in _SUPPORTED_MXFP_BITS
+ and default_data_type not in _NVFP4_E5M3_FAMILY_DATA_TYPES
+ ):
raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")
# Default ignore list: any layer present in ignored_layers (deduped) that
@@ -2825,13 +2932,15 @@ def _build_mxfp_quantization_config(
actual_bits, actual_dt = next(iter(scheme_groups.keys()))
else:
actual_bits, actual_dt = bits, default_data_type
- if actual_dt != _NVFP4_E5M3_DATA_TYPE and actual_bits not in _SUPPORTED_MXFP_BITS:
+ if actual_dt not in _NVFP4_E5M3_FAMILY_DATA_TYPES and actual_bits not in _SUPPORTED_MXFP_BITS:
raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")
group_scheme, fmt = _get_mxfp_group_scheme_and_format(actual_bits, actual_dt, ignore)
- if actual_dt == _NVFP4_E5M3_DATA_TYPE:
+ if actual_dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:
from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
- qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignore)
+ qconfig = initialize_nvfp4_e5m3_quantization(
+ ignore=ignore, with_global_scale=actual_dt == _NVFP4_E5M3_GS_DATA_TYPE
+ )
if is_fp_default and scheme_groups:
targets = _add_routed_experts_if_moe(list(quantized_layers), quantized_layers)
qconfig["config_groups"]["group_0"]["targets"] = targets
@@ -3049,79 +3158,45 @@ def _layer_config_has_nvfp4(layer_config: dict | None) -> bool:
with a ``"scheme"`` key or a ``"data_type"`` key), or
:class:`QuantizationScheme` instances.
"""
- if not layer_config:
- return False
- for val in layer_config.values():
- if isinstance(val, str):
- try:
- s = _normalize_scheme(val.upper())
- dt = (s.data_type or "").lower()
- if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):
- return True
- except Exception:
- pass
- elif isinstance(val, dict):
- dt = (val.get("data_type") or "").lower()
- if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):
- return True
- scheme_val = val.get("scheme")
- if isinstance(scheme_val, str):
- try:
- s = _normalize_scheme(scheme_val.upper())
- dt = (s.data_type or "").lower()
- if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):
- return True
- except Exception:
- pass
- elif isinstance(val, QuantizationScheme):
- dt = (val.data_type or "").lower()
- if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):
- return True
- return False
+ return _get_layer_config_nvfp4_dt(layer_config) is not None
def _get_layer_config_nvfp4_dt(layer_config: dict | None) -> str | None:
"""Return the first NVFP4 data_type found in *layer_config*.
Returns ``"nv_fp"`` for standard NVFP4, ``"nvfp4_v2"`` for NVFP4_E5M3,
- or ``None`` when no NVFP4 entry is present.
+ ``"nvfp4_v2_gs"`` for NVFP4_E5M3_GS, or ``None`` when no NVFP4 entry is present.
"""
+
+ def _classify(dt: str) -> str | None:
+ if dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:
+ return dt
+ if is_nv_fp(dt):
+ return "nv_fp"
+ return None
+
+ def _scheme_dt(name: str) -> str:
+ try:
+ return (_normalize_scheme(name.upper()).data_type or "").lower()
+ except Exception:
+ return ""
+
if not layer_config:
return None
for val in layer_config.values():
+ candidates = []
if isinstance(val, str):
- try:
- s = _normalize_scheme(val.upper())
- dt = (s.data_type or "").lower()
- if dt == _NVFP4_E5M3_DATA_TYPE:
- return _NVFP4_E5M3_DATA_TYPE
- if is_nv_fp(dt):
- return "nv_fp"
- except Exception:
- pass
+ candidates.append(_scheme_dt(val))
elif isinstance(val, dict):
- dt = (val.get("data_type") or "").lower()
- if dt == _NVFP4_E5M3_DATA_TYPE:
- return _NVFP4_E5M3_DATA_TYPE
- if is_nv_fp(dt):
- return "nv_fp"
- scheme_val = val.get("scheme")
- if isinstance(scheme_val, str):
- try:
- s = _normalize_scheme(scheme_val.upper())
- dt = (s.data_type or "").lower()
- if dt == _NVFP4_E5M3_DATA_TYPE:
- return _NVFP4_E5M3_DATA_TYPE
- if is_nv_fp(dt):
- return "nv_fp"
- except Exception:
- pass
+ candidates.append((val.get("data_type") or "").lower())
+ if isinstance(val.get("scheme"), str):
+ candidates.append(_scheme_dt(val["scheme"]))
elif isinstance(val, QuantizationScheme):
- dt = (val.data_type or "").lower()
- if dt == _NVFP4_E5M3_DATA_TYPE:
- return _NVFP4_E5M3_DATA_TYPE
- if is_nv_fp(dt):
- return "nv_fp"
+ candidates.append((val.data_type or "").lower())
+ for dt in candidates:
+ result = _classify(dt)
+ if result is not None:
+ return result
return None
@@ -3184,7 +3259,7 @@ def _derive_dominant_int_scheme(
"sym": sym,
"data_type": data_type,
}
- if data_type == _NVFP4_E5M3_DATA_TYPE:
+ if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:
for act_key in ("act_bits", "act_data_type", "act_group_size", "act_sym", "act_dynamic"):
value = scheme.get(act_key)
if value is not None:
@@ -3217,14 +3292,16 @@ def _build_quantization_config(
data_type = (default_scheme.get("data_type") or "int").lower()
default_bits = default_scheme.get("bits", 4)
is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)
- if data_type == _NVFP4_E5M3_DATA_TYPE and format == "llm_compressor":
+ if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES and format == "llm_compressor":
warnings.warn(
"LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "
"Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",
UserWarning,
stacklevel=2,
)
- return _build_nvfp4_e5m3_quantization_config(ignored_layers)
+ return _build_nvfp4_e5m3_quantization_config(
+ ignored_layers, with_global_scale=data_type == _NVFP4_E5M3_GS_DATA_TYPE
+ )
# BF16/FP16 default with NVFP4 layer_config overrides and llm_compressor format:
# build an NVFP4 config that targets only the explicitly quantized layers.
if is_fp_default and _layer_config_has_nvfp4(layer_config) and format == "llm_compressor":
@@ -3232,19 +3309,19 @@ def _build_quantization_config(
targets = list(dict.fromkeys(quantized_layers)) if quantized_layers else ["Linear"]
if quantized_layers:
targets = _add_routed_experts_if_moe(targets, quantized_layers)
- if nvfp4_dt == _NVFP4_E5M3_DATA_TYPE:
+ if nvfp4_dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:
warnings.warn(
"LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "
"Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",
UserWarning,
stacklevel=2,
)
- # Global-scale-free variant (NVFP4_E5M3)
from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
- qconfig = initialize_nvfp4_e5m3_quantization(ignore=[])
+ qconfig = initialize_nvfp4_e5m3_quantization(
+ ignore=[], with_global_scale=nvfp4_dt == _NVFP4_E5M3_GS_DATA_TYPE
+ )
qconfig["config_groups"]["group_0"]["targets"] = targets
- qconfig["format"] = "nvfp4-e5m3-pack-quantized"
else:
# Standard NVFP4 (nv_fp, scale_dtype=float8_e4m3fn)
from auto_round.export.export_to_llmcompressor.config import initialize_quantization
@@ -3301,6 +3378,8 @@ def _build_quantization_config(
# So we keep auto_round for asymmetric quantization to maintain compatibility with both.
if data_type == _NVFP4_E5M3_DATA_TYPE:
packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"
+ elif data_type == _NVFP4_E5M3_GS_DATA_TYPE:
+ packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3_gs"
else:
packing_format = _declared_int_packing(default_scheme.get("sym", True))
@@ -3316,7 +3395,7 @@ def _build_quantization_config(
"autoround_version": __version__,
}
- if data_type == _NVFP4_E5M3_DATA_TYPE:
+ if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:
for act_key in ("act_bits", "act_data_type", "act_group_size", "act_sym", "act_dynamic"):
value = default_scheme.get(act_key)
if value is not None:
@@ -3350,7 +3429,7 @@ def _build_quantization_config(
if non_linear_re.search(layer_name):
continue
extra_config[layer_name] = {"bits": 16, "data_type": "float"}
- if data_type == _NVFP4_E5M3_DATA_TYPE:
+ if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:
extra_config[layer_name].update({"act_bits": 16, "act_data_type": "float"})
quantized_layer_set = set(quantized_layers)
@@ -3799,6 +3878,22 @@ def _validate_supported_scheme(
)
return
+ if data_type == _NVFP4_E5M3_GS_DATA_TYPE:
+ if bits != 4 or scheme_obj.group_size != 16 or act_bits != 4:
+ raise ValueError(
+ f"Model-free NVFP4_E5M3_GS requires bits=4, group_size=16, and act_bits=4, "
+ f"but '{scheme_input}' requests bits={bits}, group_size={scheme_obj.group_size}, "
+ f"act_bits={act_bits}."
+ )
+ act_data_type = (scheme_obj.act_data_type or "").lower()
+ if act_data_type != _NVFP4_E5M3_GS_ACT_DATA_TYPE or scheme_obj.act_group_size != 16:
+ raise ValueError(
+ f"Model-free NVFP4_E5M3_GS requires act_data_type='{_NVFP4_E5M3_GS_ACT_DATA_TYPE}' and "
+ f"act_group_size=16, but '{scheme_input}' requests act_data_type='{scheme_obj.act_data_type}', "
+ f"act_group_size={scheme_obj.act_group_size}."
+ )
+ return
+
if act_bits < 16:
raise ValueError(
f"Model-free mode only supports weight-only quantization (WOQ) schemes "
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index afdb48dba..af0d4c7f2 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -20,9 +20,9 @@ import transformers
from torch.functional import F
import auto_round.envs as envs
-from auto_round.compressors.utils import is_nv_fp
from auto_round.data_type import get_quant_func, reshape_pad_tensor_by_group_size
from auto_round.logger import logger
+from auto_round.schemes import needs_nvfp_global_scale
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_to_quantized,
@@ -108,10 +108,12 @@ class WrapperLinear(torch.nn.Module):
self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
self.enable_act_quant = self.orig_layer.act_bits <= 8
self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)
- if is_nv_fp(self.orig_layer.data_type) and self.weight_global_scale is None:
- from auto_round.data_type.nvfp import calculate_gparam
+ if needs_nvfp_global_scale(self.orig_layer.data_type) and self.weight_global_scale is None:
+ from auto_round.data_type.nvfp import calculate_gparam_by_dtype
- weight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)
+ weight_global_scale = calculate_gparam_by_dtype(
+ self.orig_layer.weight, self.orig_layer.data_type, self.orig_layer.group_size
+ )
self.weight_global_scale = weight_global_scale
self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)
if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:
diff --git a/auto_round_extension/cuda/cute_nvfp4_e5m3.py b/auto_round_extension/cuda/cute_nvfp4_e5m3.py
index 4aa087d0b..69b4f674c 100644
--- a/auto_round_extension/cuda/cute_nvfp4_e5m3.py
+++ b/auto_round_extension/cuda/cute_nvfp4_e5m3.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Optional CuTe DSL dispatch for NVFP4 E5M3 activation QDQ."""
+"""Optional CuTe DSL dispatch for NVFP4 E5M3 activation QDQ and weight dequantization."""
from functools import cache, lru_cache
from importlib.util import find_spec
@@ -166,6 +166,174 @@ def _make_weight_dq_kernel():
return launch_weight_dq
+_UE5M3_MAX = 114688.0
+
+
+def _make_qdq_gs_kernel():
+ import cutlass
+ import cutlass.cute as cute
+ from cutlass._mlir_helpers import math
+
+ @cute.kernel
+ def qdq_gs_kernel(input_tensor: cute.Tensor, global_scale: cute.Tensor, output_tensor: cute.Tensor):
+ thread_idx, _, _ = cute.arch.thread_idx()
+ block_idx, _, _ = cute.arch.block_idx()
+ group_idx = block_idx * _THREADS_PER_BLOCK + thread_idx
+
+ if group_idx < input_tensor.shape[0]:
+ gs = cutlass.Float32(global_scale[0])
+ amax = cutlass.Float32(0.0)
+ for column_idx in range(_GROUP_SIZE):
+ value = cutlass.Float32(input_tensor[group_idx, column_idx])
+ amax = cute.arch.fmax(amax, math.abs(value))
+
+ # Mirrors ref_fp4_quant: UE5M3(gs * amax / 6), then dequant scale = scale / gs.
+ scale = gs * (amax * (cutlass.Float32(1.0) / cutlass.Float32(6.0)))
+ scale = cute.arch.fmin(cute.arch.fmax(scale, 0.0), _UE5M3_MAX)
+ if scale >= cutlass.Float32(2.0**-14):
+ exponent = math.floor(math.log2(scale)) + cutlass.Float32(1.0)
+ mantissa = scale / math.exp2(exponent)
+ mantissa_bits = math.roundeven((mantissa - cutlass.Float32(0.5)) * cutlass.Float32(16.0))
+ if mantissa_bits == cutlass.Float32(8.0):
+ mantissa_bits = cutlass.Float32(0.0)
+ exponent = exponent + cutlass.Float32(1.0)
+ mantissa_bits = cute.arch.fmin(cute.arch.fmax(mantissa_bits, 0.0), 7.0)
+ scale = (cutlass.Float32(1.0) + mantissa_bits / cutlass.Float32(8.0)) * math.exp2(
+ exponent - cutlass.Float32(1.0)
+ )
+ else:
+ scale = math.roundeven(scale * cutlass.Float32(2.0**17)) * cutlass.Float32(2.0**-17)
+ dequant_scale = scale / gs if gs > cutlass.Float32(0.0) else cutlass.Float32(0.0)
+
+ for column_idx in range(_GROUP_SIZE):
+ value = cutlass.Float32(input_tensor[group_idx, column_idx])
+ scaled = value / dequant_scale if dequant_scale > cutlass.Float32(0.0) else cutlass.Float32(0.0)
+ magnitude = math.abs(scaled)
+ quantized = cutlass.Float32(0.0)
+ if magnitude < cutlass.Float32(2.0):
+ quantized = math.roundeven(magnitude * cutlass.Float32(2.0)) / cutlass.Float32(2.0)
+ elif magnitude < cutlass.Float32(4.0):
+ quantized = math.roundeven(magnitude)
+ else:
+ quantized = cutlass.Float32(2.0) * math.roundeven(magnitude / cutlass.Float32(2.0))
+ quantized = cute.arch.fmin(cute.arch.fmax(quantized, 0.0), 6.0)
+ if scaled < cutlass.Float32(0.0):
+ quantized = -quantized
+ output_tensor[group_idx, column_idx] = (quantized * dequant_scale).to(output_tensor.element_type)
+
+ @cute.jit
+ def launch_qdq_gs(input_tensor: cute.Tensor, global_scale: cute.Tensor, output_tensor: cute.Tensor):
+ groups = input_tensor.shape[0]
+ qdq_gs_kernel(input_tensor, global_scale, output_tensor).launch(
+ grid=((groups + _THREADS_PER_BLOCK - 1) // _THREADS_PER_BLOCK, 1, 1),
+ block=(_THREADS_PER_BLOCK, 1, 1),
+ )
+
+ return launch_qdq_gs
+
+
+def _make_weight_dq_gs_kernel():
+ import cutlass
+ import cutlass.cute as cute
+ from cutlass._mlir_helpers import math
+
+ @cute.kernel
+ def weight_dq_gs_kernel(
+ weight_packed: cute.Tensor, weight_scale: cute.Tensor, global_scale: cute.Tensor, output_tensor: cute.Tensor
+ ):
+ thread_idx, _, _ = cute.arch.thread_idx()
+ block_idx, _, _ = cute.arch.block_idx()
+ group_idx = block_idx * _THREADS_PER_BLOCK + thread_idx
+
+ if group_idx < output_tensor.shape[0]:
+ scale_byte = cutlass.Float32(weight_scale[group_idx, 0])
+ exponent = math.floor(scale_byte / cutlass.Float32(8.0))
+ mantissa = scale_byte - exponent * cutlass.Float32(8.0)
+ scale = (cutlass.Float32(1.0) + mantissa / cutlass.Float32(8.0)) * math.exp2(
+ exponent - cutlass.Float32(15.0)
+ )
+ if exponent == cutlass.Float32(0.0):
+ scale = mantissa * math.exp2(cutlass.Float32(-17.0))
+ gs = cutlass.Float32(global_scale[0])
+ scale = scale / gs if gs > cutlass.Float32(0.0) else cutlass.Float32(0.0)
+
+ for column_idx in range(_GROUP_SIZE):
+ packed = cutlass.Float32(weight_packed[group_idx, column_idx // 2])
+ packed_high = math.floor(packed / cutlass.Float32(16.0))
+ nibble = packed - packed_high * cutlass.Float32(16.0)
+ if column_idx % 2:
+ nibble = packed_high
+
+ sign = cutlass.Float32(1.0)
+ if nibble >= cutlass.Float32(8.0):
+ sign = cutlass.Float32(-1.0)
+ nibble = nibble - cutlass.Float32(8.0)
+
+ value = cutlass.Float32(0.0)
+ if nibble == cutlass.Float32(1.0):
+ value = cutlass.Float32(0.5)
+ elif nibble == cutlass.Float32(2.0):
+ value = cutlass.Float32(1.0)
+ elif nibble == cutlass.Float32(3.0):
+ value = cutlass.Float32(1.5)
+ elif nibble == cutlass.Float32(4.0):
+ value = cutlass.Float32(2.0)
+ elif nibble == cutlass.Float32(5.0):
+ value = cutlass.Float32(3.0)
+ elif nibble == cutlass.Float32(6.0):
+ value = cutlass.Float32(4.0)
+ elif nibble == cutlass.Float32(7.0):
+ value = cutlass.Float32(6.0)
+
+ output_tensor[group_idx, column_idx] = (sign * value * scale).to(output_tensor.element_type)
+
+ @cute.jit
+ def launch_weight_dq_gs(
+ weight_packed: cute.Tensor, weight_scale: cute.Tensor, global_scale: cute.Tensor, output_tensor: cute.Tensor
+ ):
+ groups = output_tensor.shape[0]
+ weight_dq_gs_kernel(weight_packed, weight_scale, global_scale, output_tensor).launch(
+ grid=((groups + _THREADS_PER_BLOCK - 1) // _THREADS_PER_BLOCK, 1, 1),
+ block=(_THREADS_PER_BLOCK, 1, 1),
+ )
+
+ return launch_weight_dq_gs
+
+
+@cache
+def _get_compiled_qdq_gs_kernel(device_index: int, dtype: torch.dtype):
+ import cutlass.cute as cute
+ from cutlass.cute.runtime import from_dlpack
+
+ input_tensor = torch.empty((1, _GROUP_SIZE), device=f"cuda:{device_index}", dtype=dtype)
+ global_scale = torch.ones((1,), device=f"cuda:{device_index}", dtype=torch.float32)
+ output_tensor = torch.empty_like(input_tensor)
+ return cute.compile(
+ _make_qdq_gs_kernel(),
+ from_dlpack(input_tensor).mark_layout_dynamic(),
+ from_dlpack(global_scale),
+ from_dlpack(output_tensor).mark_layout_dynamic(),
+ )
+
+
+@cache
+def _get_compiled_weight_dq_gs_kernel(device_index: int, dtype: torch.dtype):
+ import cutlass.cute as cute
+ from cutlass.cute.runtime import from_dlpack
+
+ weight_packed = torch.empty((2, _GROUP_SIZE // 2), device=f"cuda:{device_index}", dtype=torch.uint8)
+ weight_scale = torch.empty((2, 1), device=f"cuda:{device_index}", dtype=torch.uint8)
+ global_scale = torch.ones((1,), device=f"cuda:{device_index}", dtype=torch.float32)
+ output_tensor = torch.empty((2, _GROUP_SIZE), device=f"cuda:{device_index}", dtype=dtype)
+ return cute.compile(
+ _make_weight_dq_gs_kernel(),
+ from_dlpack(weight_packed).mark_layout_dynamic(),
+ from_dlpack(weight_scale).mark_layout_dynamic(),
+ from_dlpack(global_scale),
+ from_dlpack(output_tensor).mark_layout_dynamic(),
+ )
+
+
@cache
def _get_compiled_qdq_kernel(device_index: int, dtype: torch.dtype):
import cutlass.cute as cute
@@ -256,6 +424,79 @@ def try_cute_nvfp4_e5m3_weight_dq(
return None
+def _as_cuda_global_scale(global_scale: torch.Tensor, device: torch.device) -> torch.Tensor:
+ return global_scale.reshape(-1)[:1].to(device=device, dtype=torch.float32).contiguous()
+
+
+def try_cute_nvfp4_v2_gs_qdq(
+ activation: torch.Tensor, global_scale: torch.Tensor, group_size: int
+) -> torch.Tensor | None:
+ """Run the CuTe QDQ kernel for NVFP4 E5M3 activations with a per-tensor global scale."""
+ if not can_use_cute_nvfp4_v2_qdq(activation, group_size):
+ return None
+ if torch.cuda.is_current_stream_capturing():
+ return None
+
+ kernel_key = ("gs", activation.device.index, activation.dtype)
+ if kernel_key in _FAILED_KERNEL_KEYS:
+ return None
+
+ try:
+ groups = activation.numel() // group_size
+ output = torch.empty_like(activation)
+ compiled_qdq = _get_compiled_qdq_gs_kernel(activation.device.index, activation.dtype)
+ compiled_qdq(
+ activation.reshape(groups, group_size),
+ _as_cuda_global_scale(global_scale, activation.device),
+ output.reshape(groups, group_size),
+ )
+ return output
+ except Exception as error:
+ _FAILED_KERNEL_KEYS.add(kernel_key)
+ logger.warning_once("CuTe NVFP4 E5M3 GS QDQ failed; falling back to PyTorch reference: %s", error)
+ return None
+
+
+def try_cute_nvfp4_e5m3_gs_weight_dq(
+ weight_packed: torch.Tensor, weight_scale: torch.Tensor, global_scale: torch.Tensor, dtype: torch.dtype
+) -> torch.Tensor | None:
+ """Dequantize packed FP4 E5M3 weights with a per-tensor global scale using CuTe."""
+ if (
+ not is_cute_dsl_available()
+ or not weight_packed.is_cuda
+ or weight_packed.dtype != torch.uint8
+ or weight_scale.dtype != torch.uint8
+ or weight_packed.ndim != 2
+ or weight_scale.shape != (weight_packed.shape[0], weight_packed.shape[1] // (_GROUP_SIZE // 2))
+ or dtype not in _SUPPORTED_DTYPES
+ or torch.cuda.get_device_capability(weight_packed.device)[0] < 8
+ or torch.cuda.is_current_stream_capturing()
+ ):
+ return None
+
+ kernel_key = ("gs", weight_packed.device.index, dtype)
+ if kernel_key in _FAILED_WEIGHT_DQ_KERNEL_KEYS:
+ return None
+
+ try:
+ out_features = weight_packed.shape[0]
+ in_features = weight_packed.shape[1] * 2
+ groups = weight_scale.numel()
+ output = torch.empty((groups, _GROUP_SIZE), device=weight_packed.device, dtype=dtype)
+ compiled_weight_dq = _get_compiled_weight_dq_gs_kernel(weight_packed.device.index, dtype)
+ compiled_weight_dq(
+ weight_packed.reshape(groups, _GROUP_SIZE // 2),
+ weight_scale.reshape(groups, 1),
+ _as_cuda_global_scale(global_scale, weight_packed.device),
+ output,
+ )
+ return output.reshape(out_features, in_features)
+ except Exception as error:
+ _FAILED_WEIGHT_DQ_KERNEL_KEYS.add(kernel_key)
+ logger.warning_once("CuTe NVFP4 E5M3 GS weight DQ failed; falling back to PyTorch reference: %s", error)
+ return None
+
+
def try_cute_nvfp4_e5m3_linear(
activation: torch.Tensor,
weight_packed: torch.Tensor,
diff --git a/docs/nvfp4_e5m3.md b/docs/nvfp4_e5m3.md
index a88aeaa66..396bc3b8f 100644
--- a/docs/nvfp4_e5m3.md
+++ b/docs/nvfp4_e5m3.md
@@ -105,6 +105,33 @@ tokenizer = AutoTokenizer.from_pretrained("./nvfp4_e5m3_model")
---
+## NVFP4_E5M3_GS: E5M3 Block Scales with a Global Scale
+
+`NVFP4_E5M3_GS` keeps the FP4 E2M1 values and UE5M3 block scales, and adds a
+per-tensor FP32 global scale for weights and activations (like standard NVFP4):
+
+$$\text{global\_scale} = \frac{114688 \times 6}{\text{amax}}, \qquad x \approx q \cdot \frac{\text{UE5M3}(\text{block\_scale})}{\text{global\_scale}}$$
+
+| Property | Value |
+|---|---|
+| Preset name | `NVFP4_E5M3_GS` |
+| Weight dtype | `nvfp4_v2_gs` |
+| Activation dtype | `nvfp4_v2_with_static_gs` |
+| Extra tensors | `.weight_global_scale`, `.input_global_scale` (FP32) |
+| `auto_round` packing format | `auto_round:llm_compressor_nvfp4_e5m3_gs` |
+| `llm_compressor` format | `nvfp4-e5m3-gs-pack-quantized` |
+| Inference backends | `auto_round:cute_nvfp4_e5m3_gs`, `auto_round:torch_nvfp4_e5m3_gs` |
+
+- **Regular tuning flow**: global scales are computed like NVFP4. The weight global scale comes from the weight amax, and `input_global_scale` comes from the calibrated activation amax. q/k/v and gate/up projections share the minimum global scale.
+- **Model-free flow**: the weight global scale is computed per layer and `input_global_scale` is fixed to `1`.
+
+```bash
+auto-round --model <model_id_or_path> --scheme NVFP4_E5M3_GS --output_dir ./nvfp4_e5m3_gs_model
+auto-round --model <model_id_or_path> --scheme NVFP4_E5M3_GS --model_free --output_dir ./nvfp4_e5m3_gs_model
+```
+
+---
+
## Environment Variables
### `AR_NVFP4_E5M3_CACHE_HP_WEIGHT`
diff --git a/docs/nvfp4_e5m3_CN.md b/docs/nvfp4_e5m3_CN.md
index f9d5a76a0..f9836d2f7 100644
--- a/docs/nvfp4_e5m3_CN.md
+++ b/docs/nvfp4_e5m3_CN.md
@@ -101,6 +101,32 @@ tokenizer = AutoTokenizer.from_pretrained("./nvfp4_e5m3_model")
---
+## NVFP4_E5M3_GS:带全局 scale 的 E5M3 块级 scale
+
+`NVFP4_E5M3_GS` 保留 FP4 E2M1 数值和 UE5M3 块级 scale,并像标准 NVFP4 一样为权重和激活额外增加一个张量级 FP32 全局 scale:
+
+$$\text{global\_scale} = \frac{114688 \times 6}{\text{amax}}, \qquad x \approx q \cdot \frac{\text{UE5M3}(\text{block\_scale})}{\text{global\_scale}}$$
+
+| 参数 | 值 |
+|---|---|
+| 预设名称 | `NVFP4_E5M3_GS` |
+| 权重数据类型 | `nvfp4_v2_gs` |
+| 激活数据类型 | `nvfp4_v2_with_static_gs` |
+| 额外张量 | `.weight_global_scale`、`.input_global_scale`(FP32) |
+| `auto_round` 打包格式 | `auto_round:llm_compressor_nvfp4_e5m3_gs` |
+| `llm_compressor` 格式 | `nvfp4-e5m3-gs-pack-quantized` |
+| 推理后端 | `auto_round:cute_nvfp4_e5m3_gs`、`auto_round:torch_nvfp4_e5m3_gs` |
+
+- **常规调优流程**:全局 scale 的计算方式与 NVFP4 一致。权重全局 scale 由权重 amax 得到,`input_global_scale` 由校准得到的激活 amax 得到。q/k/v 及 gate/up 投影共享最小的全局 scale。
+- **无模型量化流程**:按层计算权重全局 scale,`input_global_scale` 固定为 `1`。
+
+```bash
+auto-round --model <model_id_or_path> --scheme NVFP4_E5M3_GS --output_dir ./nvfp4_e5m3_gs_model
+auto-round --model <model_id_or_path> --scheme NVFP4_E5M3_GS --model_free --output_dir ./nvfp4_e5m3_gs_model
+```
+
+---
+
## 环境变量
### `AR_NVFP4_E5M3_CACHE_HP_WEIGHT`
diff --git a/test/unit/common/core/test_format_decoupling.py b/test/unit/common/core/test_format_decoupling.py
index 2dea8725d..7832288dd 100644
--- a/test/unit/common/core/test_format_decoupling.py
+++ b/test/unit/common/core/test_format_decoupling.py
@@ -94,6 +94,33 @@ def test_nvfp4_e5m3_autoround_uses_llm_compressor_packing_and_torch_fallback():
)
+def test_nvfp4_e5m3_gs_autoround_uses_llm_compressor_packing_and_torch_fallback():
+ scheme = _resolved_scheme("NVFP4_E5M3_GS")
+ resolution = resolve_formats(
+ ResolvedScheme.from_scheme(scheme), format="auto_round", model=nn.Sequential(nn.Linear(16, 16))
+ )
+
+ assert resolution.formats[0].get_backend_name() == "auto_round:nvfp4_v2_gs"
+ assert BackendInfos["auto_round:torch_nvfp4_e5m3_gs"].packing_format == ["auto_round:llm_compressor_nvfp4_e5m3_gs"]
+ assert dynamic_import_inference_linear("auto_round:torch_nvfp4_e5m3_gs", scheme) is (
+ ar_qmodules.NVFP4E5M3GSQuantLinear
+ )
+ assert dynamic_import_inference_linear("auto_round:cute_nvfp4_e5m3_gs", scheme) is (
+ ar_qmodules.CuteNVFP4E5M3GSQuantLinear
+ )
+ assert (
+ get_layer_backend(
+ "cpu",
+ "auto",
+ "auto_round:llm_compressor_nvfp4_e5m3_gs",
+ scheme,
+ in_features=16,
+ out_features=16,
+ )
+ == "auto_round:torch_nvfp4_e5m3_gs"
+ )
+
+
def test_gguf_correction_propagates_to_scheme_and_quantize_config(monkeypatch):
# W4A16's plain int-woq fields don't match every field gguf_args_check enforces
# for a GGUF target; after resolving "gguf:q4_k_m" the scheme-bearing objects
diff --git a/test/unit/common/data_type/test_nvfp.py b/test/unit/common/data_type/test_nvfp.py
index 0db81a493..99af14a2f 100644
--- a/test/unit/common/data_type/test_nvfp.py
+++ b/test/unit/common/data_type/test_nvfp.py
@@ -25,6 +25,8 @@ from auto_round.data_type.nvfp import (
FLOAT8_E4M3_MIN,
FLOAT8_UE5M3_MAX,
calculate_gparam,
+ calculate_gparam_by_dtype,
+ calculate_gparam_e5m3,
cast_to_fp4,
cast_to_ue5m3,
cast_to_ue5m3_ste,
@@ -34,13 +36,16 @@ from auto_round.data_type.nvfp import (
nv_fp4,
nv_fp4_with_static_gs,
nvfp4_v2,
+ nvfp4_v2_gs,
nvfp4_v2_with_global_scale,
+ nvfp4_v2_with_static_gs,
+ opt_rtn_nvfp4_v2_gs,
ref_fp4_quant,
ref_nvfp4_quant,
search_nvfp4_scale,
search_nvfp4_v2_scale,
)
-from auto_round.data_type.utils import update_fused_layer_global_scales
+from auto_round.data_type.utils import get_quant_func, update_fused_layer_global_scales
# ---------------------------------------------------------------------------
# Constants
@@ -749,6 +754,81 @@ class TestNvfp4V2:
assert q.shape == t.shape
+# ---------------------------------------------------------------------------
+# NVFP4_E5M3_GS: nvfp4_v2_gs / nvfp4_v2_with_static_gs
+# ---------------------------------------------------------------------------
+
+
+class TestNvfp4V2GlobalScale:
+ """Test the NVFP4 E5M3 data types with a per-tensor global scale."""
+
+ def test_calculate_gparam_e5m3_uses_ue5m3_max(self):
+ t = torch.randn(4, 32)
+ expected = FLOAT8_UE5M3_MAX * FLOAT4_E2M1_MAX / t.abs().max()
+ torch.testing.assert_close(calculate_gparam_e5m3(t), expected)
+ torch.testing.assert_close(calculate_gparam_by_dtype(t, "opt_rtn_nvfp4_v2_gs"), expected)
+ torch.testing.assert_close(calculate_gparam_by_dtype(t, "nv_fp"), calculate_gparam(t))
+
+ def test_default_global_scale_fills_ue5m3_range(self):
+ t = torch.randn(8, 64)
+ q, s, z = nvfp4_v2_gs(t, group_size=16)
+ assert q.shape == t.shape and z is None
+ assert s.max().item() == FLOAT8_UE5M3_MAX
+
+ def test_matches_ref_with_explicit_global_scale(self):
+ t = torch.randn(8, 64)
+ global_scale = torch.tensor([1234.5])
+ q, s, _ = nvfp4_v2_gs(t, group_size=16, global_scale=global_scale)
+ ref_q, ref_s = ref_fp4_quant(t.reshape(-1, 16), global_scale, 16)
+ torch.testing.assert_close(q, ref_q.reshape(t.shape))
+ torch.testing.assert_close(s, ref_s)
+
+ def test_global_scale_one_matches_nvfp4_v2(self):
+ t = torch.randn(8, 64)
+ q, _, _ = nvfp4_v2_gs(t, group_size=16, global_scale=torch.ones(1))
+ ref, _, _ = nvfp4_v2(t, group_size=16)
+ torch.testing.assert_close(q, ref)
+
+ def test_per_group_max_scale(self):
+ t = torch.randn(4, 32)
+ max_scale = torch.full((8,), 0.9)
+ q, _, _ = nvfp4_v2_gs(t, group_size=16, max_scale=max_scale)
+ ref, _, _ = nvfp4_v2_gs(t, group_size=16, max_scale=0.9)
+ torch.testing.assert_close(q, ref)
+
+ def test_opt_rtn_does_not_increase_error(self):
+ t = torch.randn(32, 128)
+ baseline, _, _ = nvfp4_v2_gs(t, group_size=16)
+ optimized, _, _ = opt_rtn_nvfp4_v2_gs(t, group_size=16)
+ assert (optimized - t).pow(2).sum() <= (baseline - t).pow(2).sum()
+
+ def test_opt_rtn_handles_multiple_groups(self):
+ t = torch.randn(4, 32)
+
+ optimized, scale, _ = opt_rtn_nvfp4_v2_gs(t, group_size=16)
+
+ assert optimized.shape == t.shape
+ assert scale.shape == (8, 1)
+ assert torch.isfinite(optimized).all()
+
+ def test_static_gs_tensor_max_matches_global_scale(self):
+ t = torch.randn(4, 64)
+ tensor_max = torch.tensor([3.0])
+ from_max, _, _ = nvfp4_v2_with_static_gs(t, group_size=16, tensor_max=tensor_max)
+ from_gs, _, _ = nvfp4_v2_with_static_gs(t, group_size=16, global_scale=calculate_gparam_e5m3(tensor_max))
+ torch.testing.assert_close(from_max, from_gs)
+
+ def test_static_gs_empty_tensor(self):
+ t = torch.empty(0, 16)
+ q, s, z = nvfp4_v2_with_static_gs(t, group_size=16)
+ assert q is t and s is None and z is None
+
+ def test_registered_quant_funcs(self):
+ assert get_quant_func("nvfp4_v2_gs", 4, True, iters=0)[1] == "opt_rtn_nvfp4_v2_gs"
+ assert get_quant_func("nvfp4_v2_gs", 4, True, iters=200)[1] == "nvfp4_v2_gs"
+ assert get_quant_func("nvfp4_v2_with_static_gs", 4, True, iters=0)[1] == "nvfp4_v2_with_static_gs"
+
+
# ---------------------------------------------------------------------------
# Cross-check: outputs are finite / dtype preserved
# ---------------------------------------------------------------------------
diff --git a/test/unit/common/quantization/test_nvfp4_quant_linear.py b/test/unit/common/quantization/test_nvfp4_quant_linear.py
index 82f403330..a37373ec6 100644
--- a/test/unit/common/quantization/test_nvfp4_quant_linear.py
+++ b/test/unit/common/quantization/test_nvfp4_quant_linear.py
@@ -5,7 +5,13 @@ import pytest
import torch
from transformers.quantizers.auto import AutoHfQuantizer
-from auto_round.data_type.nvfp import calculate_gparam, nvfp4_v2
+from auto_round.data_type.nvfp import (
+ calculate_gparam,
+ calculate_gparam_e5m3,
+ nvfp4_v2,
+ nvfp4_v2_gs,
+ nvfp4_v2_with_static_gs,
+)
from auto_round.data_type.utils import get_quant_func
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear
@@ -57,6 +63,80 @@ def test_nvfp4_e5m3_compressed_tensors_loading_uses_no_global_scales():
assert quantizer._process_model_after_weight_loading(model) is model
+def test_nvfp4_e5m3_gs_compressed_tensors_loading_uses_global_scales():
+ class TinyModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(model_type="tiny")
+ self.model = torch.nn.Module()
+ self.model.layers = torch.nn.ModuleList([torch.nn.ModuleDict({"fc": torch.nn.Linear(16, 8)})])
+
+ quantizer = AutoHfQuantizer.from_config(initialize_nvfp4_e5m3_quantization([], with_global_scale=True))
+ model = quantizer._process_model_before_weight_loading(TinyModel())
+ layer = model.model.layers[0]["fc"]
+
+ assert isinstance(layer, ar_qmodules.NVFP4E5M3GSQuantLinear)
+ assert set(layer.state_dict()) == {
+ "bias",
+ "weight_packed",
+ "weight_scale",
+ "weight_global_scale",
+ "input_global_scale",
+ }
+
+
+def test_nvfp4_e5m3_gs_pack_and_dequant_round_trip():
+ config = PRESET_SCHEMES["NVFP4_E5M3_GS"]
+ original_layer = torch.nn.Linear(64, 32, bias=False)
+ weight_global_scale = calculate_gparam_e5m3(original_layer.weight)
+ weight_qdq, weight_scale, _ = nvfp4_v2_gs(original_layer.weight, group_size=16, global_scale=weight_global_scale)
+ activation = torch.randn(4, 64)
+ input_global_scale = calculate_gparam_e5m3(activation)
+
+ packed = _FPLinear(4, 16, 64, 32, False, data_type=config.data_type, act_bits=4)
+ packed.pack(
+ original_layer,
+ weight_scale.reshape(32, 4),
+ global_scale=weight_global_scale,
+ input_global_scale=input_global_scale,
+ )
+ assert packed.weight_scale.dtype == torch.uint8
+
+ layer = ar_qmodules.NVFP4E5M3GSQuantLinear.from_original(config, original_layer)
+ for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):
+ getattr(layer, name).copy_(getattr(packed, name))
+
+ torch.testing.assert_close(layer.dequant_weight_online(), weight_qdq, rtol=1e-6, atol=1e-7)
+ expected_activation, _, _ = nvfp4_v2_with_static_gs(activation, group_size=16, tensor_max=activation.abs().max())
+ torch.testing.assert_close(layer.qdq_input(activation), expected_activation)
+
+
+def test_nvfp4_e5m3_gs_dequant_zero_weight_with_zero_global_scale():
+ config = PRESET_SCHEMES["NVFP4_E5M3_GS"]
+ layer = ar_qmodules.NVFP4E5M3GSQuantLinear(16, 8, config, dtype=torch.float32)
+ layer.weight_packed.zero_()
+ layer.weight_scale.zero_()
+ layer.weight_global_scale.zero_()
+
+ dequantized = layer.dequant_weight_online()
+
+ assert torch.isfinite(dequantized).all()
+ assert torch.equal(dequantized, torch.zeros_like(dequantized))
+
+
+def test_nvfp4_e5m3_gs_cute_falls_back_to_reference_on_cpu():
+ config = PRESET_SCHEMES["NVFP4_E5M3_GS"]
+ layer = ar_qmodules.CuteNVFP4E5M3GSQuantLinear(16, 8, config, dtype=torch.float32)
+ layer.input_global_scale.fill_(123.0)
+ activation = torch.randn(2, 16)
+
+ expected, _, _ = nvfp4_v2_with_static_gs(activation, group_size=16, global_scale=layer.input_global_scale)
+
+ assert torch.equal(layer.qdq_input(activation), expected)
+ assert layer(activation).shape == (2, 8)
+
+
def test_nvfp4_e5m3_qdq_input_uses_reference_fallback_on_cpu():
config = PRESET_SCHEMES["NVFP4_E5M3"]
layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.py
index 2a3777534..1b53add1a 100644
--- a/test/unit/test_cpu/quantization/test_model_free.py
+++ b/test/unit/test_cpu/quantization/test_model_free.py
@@ -77,6 +77,7 @@ from auto_round.utils.model_free_utils import (
_process_shard,
_quantize_weight_mxfp,
_quantize_weight_nvfp4_e5m3,
+ _quantize_weight_nvfp4_e5m3_gs,
_validate_auto_scheme_options,
is_model_free_supported_scheme,
)
@@ -707,6 +708,59 @@ def test_nvfp4_e5m3_model_free_llm_compressor(tmp_path):
assert group["input_activations"]["dynamic"] == "local"
+def test_nvfp4_e5m3_gs_model_free_fake_quantization():
+ from auto_round.data_type.nvfp import calculate_gparam_e5m3, nvfp4_v2_gs
+
+ weight = torch.randn(8, 32)
+ output = _quantize_weight_nvfp4_e5m3_gs(weight, "layer.fc", group_size=16, disable_opt_rtn=True)
+
+ expected, _, _ = nvfp4_v2_gs(weight, group_size=16, global_scale=calculate_gparam_e5m3(weight))
+ assert set(output) == {"layer.fc.weight", "layer.fc.input_global_scale"}
+ torch.testing.assert_close(output["layer.fc.weight"], expected)
+ assert torch.equal(output["layer.fc.input_global_scale"], torch.ones(1))
+ assert is_model_free_supported_scheme("NVFP4_E5M3_GS")
+
+
[email protected]("fmt", ["auto_round", "llm_compressor"])
+def test_nvfp4_e5m3_gs_model_free_end_to_end(tmp_path, fmt):
+ from safetensors import safe_open
+
+ from auto_round.data_type.nvfp import calculate_gparam_e5m3
+
+ prefix = "model.layers.0.self_attn.q_proj"
+ weight = torch.randn(32, 32)
+ tensors = {f"{prefix}.weight": weight, "lm_head.weight": torch.randn(64, 32)}
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+
+ compressor = _ModelFreeCompressorCore(
+ model_name_or_path=model_dir, output_dir=output_dir, scheme="NVFP4_E5M3_GS", format=fmt
+ )
+ compressor.run()
+
+ output_keys = _read_output_keys(output_dir)
+ for suffix in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):
+ assert f"{prefix}.{suffix}" in output_keys
+ assert f"{prefix}.weight" not in output_keys
+ saved = {}
+ for file_name in os.listdir(output_dir):
+ if file_name.endswith(".safetensors"):
+ with safe_open(os.path.join(output_dir, file_name), framework="pt") as f:
+ saved.update({k: f.get_tensor(k) for k in f.keys() if k.startswith(prefix)})
+ assert torch.equal(saved[f"{prefix}.input_global_scale"], torch.ones(1))
+ torch.testing.assert_close(saved[f"{prefix}.weight_global_scale"], calculate_gparam_e5m3(weight).reshape(1))
+ assert saved[f"{prefix}.weight_scale"].dtype == torch.uint8
+
+ quantization_config = _read_qconfig(output_dir)
+ if fmt == "auto_round":
+ assert quantization_config["packing_format"] == "auto_round:llm_compressor_nvfp4_e5m3_gs"
+ assert quantization_config["data_type"] == "nvfp4_v2_gs"
+ assert quantization_config["act_data_type"] == "nvfp4_v2_with_static_gs"
+ else:
+ assert quantization_config["format"] == "nvfp4-e5m3-gs-pack-quantized"
+ assert quantization_config["quant_method"] == "compressed-tensors"
+
+
def test_model_free_legacy_nvfp4_is_normalized_and_passthrough(tmp_path):
prefix = "model.layers.0.mlp.down_proj"
tensors = {
diff --git a/test/unit/test_cpu/quantization/test_nvfp4_e5m3_gs.py b/test/unit/test_cpu/quantization/test_nvfp4_e5m3_gs.py
new file mode 100644
index 000000000..2652a47c1
--- /dev/null
+++ b/test/unit/test_cpu/quantization/test_nvfp4_e5m3_gs.py
@@ -0,0 +1,123 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+import pytest
+import torch
+from safetensors import safe_open
+
+from auto_round import AutoRound
+from auto_round.data_type.nvfp import FLOAT4_E2M1_MAX, FLOAT8_UE5M3_MAX
+
+PREFIX = "model.decoder.layers.0.self_attn"
+
+
+def _read_tensors(model_dir, names):
+ tensors = {}
+ for file_name in os.listdir(model_dir):
+ if not file_name.endswith(".safetensors"):
+ continue
+ with safe_open(os.path.join(model_dir, file_name), framework="pt") as f:
+ for name in names:
+ if name in f.keys():
+ tensors[name] = f.get_tensor(name)
+ return tensors
+
+
+def _quantize(model_path, dataloader, output_dir, fmt, iters=0):
+ autoround = AutoRound(
+ model_path,
+ scheme="NVFP4_E5M3_GS",
+ iters=iters,
+ seqlen=8,
+ nsamples=2,
+ dataset=dataloader,
+ )
+ _, quantized_model_path = autoround.quantize_and_save(output_dir, format=fmt)
+ return quantized_model_path
+
+
[email protected]("iters", [0, 1])
+def test_nvfp4_e5m3_gs_auto_round_export_and_inference(tiny_opt_model_path, dataloader, tmp_path, iters):
+ from transformers import AutoModelForCausalLM
+
+ from auto_round.experimental import qmodules as ar_qmodules
+
+ model_dir = _quantize(tiny_opt_model_path, dataloader, str(tmp_path / "out"), "auto_round", iters=iters)
+
+ with open(os.path.join(model_dir, "config.json")) as f:
+ quantization_config = json.load(f)["quantization_config"]
+ assert quantization_config["packing_format"] == "auto_round:llm_compressor_nvfp4_e5m3_gs"
+ assert quantization_config["data_type"] == "nvfp4_v2_gs"
+ assert quantization_config["act_data_type"] == "nvfp4_v2_with_static_gs"
+
+ names = [
+ f"{PREFIX}.{proj}.{suffix}"
+ for proj in ("q_proj", "k_proj", "v_proj")
+ for suffix in ("weight_scale", "weight_global_scale", "input_global_scale")
+ ]
+ tensors = _read_tensors(model_dir, names)
+ assert set(tensors) == set(names)
+ for proj in ("q_proj", "k_proj", "v_proj"):
+ assert tensors[f"{PREFIX}.{proj}.weight_scale"].dtype == torch.uint8
+ assert tensors[f"{PREFIX}.{proj}.weight_global_scale"].dtype == torch.float32
+ assert tensors[f"{PREFIX}.{proj}.input_global_scale"].item() != 1.0
+ # q/k/v share one weight and one input global scale for fused kernels.
+ for suffix in ("weight_global_scale", "input_global_scale"):
+ assert torch.equal(tensors[f"{PREFIX}.q_proj.{suffix}"], tensors[f"{PREFIX}.k_proj.{suffix}"])
+ assert torch.equal(tensors[f"{PREFIX}.q_proj.{suffix}"], tensors[f"{PREFIX}.v_proj.{suffix}"])
+
+ model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="cpu")
+ layer = model.model.decoder.layers[0].self_attn.q_proj
+ assert isinstance(layer, ar_qmodules.NVFP4E5M3GSQuantLinear)
+ assert torch.equal(layer.weight_global_scale.cpu(), tensors[f"{PREFIX}.q_proj.weight_global_scale"])
+ with torch.no_grad():
+ logits = model(torch.tensor([[2, 100, 200, 300]])).logits
+ assert torch.isfinite(logits).all()
+
+
+def test_nvfp4_e5m3_gs_llm_compressor_export(tiny_opt_model_path, dataloader, tmp_path):
+ model_dir = _quantize(tiny_opt_model_path, dataloader, str(tmp_path / "out"), "llm_compressor")
+
+ with open(os.path.join(model_dir, "config.json")) as f:
+ quantization_config = json.load(f)["quantization_config"]
+ assert quantization_config["format"] == "nvfp4-e5m3-gs-pack-quantized"
+ assert quantization_config["quant_method"] == "compressed-tensors"
+
+ names = [f"{PREFIX}.q_proj.weight_global_scale", f"{PREFIX}.q_proj.input_global_scale"]
+ tensors = _read_tensors(model_dir, names)
+ assert set(tensors) == set(names)
+
+
+def test_nvfp4_e5m3_gs_weight_global_scale_uses_ue5m3_range(tiny_opt_model_path, dataloader):
+ from transformers import AutoModelForCausalLM
+
+ orig_weight = AutoModelForCausalLM.from_pretrained(tiny_opt_model_path).model.decoder.layers[0].fc2.weight
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ scheme="NVFP4_E5M3_GS",
+ iters=0,
+ seqlen=8,
+ nsamples=2,
+ dataset=dataloader,
+ model_dtype=torch.float32,
+ amp=False,
+ )
+ model, _ = autoround.quantize()
+ layer = model.model.decoder.layers[0].fc2
+ layer = getattr(layer, "orig_layer", layer)
+ expected = FLOAT8_UE5M3_MAX * FLOAT4_E2M1_MAX / orig_weight.float().abs().max()
+ torch.testing.assert_close(layer.weight_global_scale.float().reshape(()), expected, rtol=1e-6, atol=0)
diff --git a/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py
index ced441ab1..f3320518d 100644
--- a/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py
+++ b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py
@@ -1,11 +1,19 @@
import pytest
import torch
-from auto_round.data_type.nvfp import cast_to_fp4, e5m3_to_float_tensor, nvfp4_v2
+from auto_round.data_type.nvfp import (
+ calculate_gparam_e5m3,
+ cast_to_fp4,
+ e5m3_to_float_tensor,
+ nvfp4_v2,
+ nvfp4_v2_with_static_gs,
+)
from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
from auto_round_extension.cuda.cute_nvfp4_e5m3 import (
can_use_cute_nvfp4_v2_qdq,
+ try_cute_nvfp4_e5m3_gs_weight_dq,
try_cute_nvfp4_e5m3_weight_dq,
+ try_cute_nvfp4_v2_gs_qdq,
try_cute_nvfp4_v2_qdq,
)
@@ -78,3 +86,78 @@ def test_nvfp4_v2_direct_scale_handles_zero_group():
assert torch.equal(actual, activation)
assert torch.equal(scale, torch.zeros_like(scale))
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
+ ("dtype", "atol"),
+ [(torch.float32, 1e-6), (torch.float16, 2e-3), (torch.bfloat16, 2e-2)],
+)
+def test_cute_nvfp4_v2_gs_qdq_matches_reference(dtype, atol):
+ if torch.cuda.get_device_capability()[0] < 8:
+ pytest.skip("CuTe QDQ requires SM80 or newer")
+
+ torch.manual_seed(0)
+ activation = torch.randn(64, 16, device="cuda", dtype=dtype)
+ activation[0] = 0
+ if not can_use_cute_nvfp4_v2_qdq(activation, 16):
+ pytest.skip("CuTe DSL is not available")
+
+ # Calibrated max below the runtime max exercises the UE5M3 upper clamp.
+ global_scale = calculate_gparam_e5m3(activation.float().abs().max() * 0.5).reshape(1)
+ expected, _, _ = nvfp4_v2_with_static_gs(activation.float(), group_size=16, global_scale=global_scale)
+ actual = try_cute_nvfp4_v2_gs_qdq(activation, global_scale, 16)
+
+ assert actual is not None
+ torch.testing.assert_close(actual, expected.to(dtype), rtol=0, atol=atol)
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
+def test_cute_nvfp4_v2_gs_qdq_handles_zero_global_scale():
+ if torch.cuda.get_device_capability()[0] < 8:
+ pytest.skip("CuTe QDQ requires SM80 or newer")
+
+ activation = torch.zeros((4, 16), device="cuda", dtype=torch.float32)
+ global_scale = torch.zeros(1, device="cuda", dtype=torch.float32)
+ actual = try_cute_nvfp4_v2_gs_qdq(activation, global_scale, 16)
+ if actual is None:
+ pytest.skip("CuTe DSL is not available")
+
+ assert torch.isfinite(actual).all()
+ assert torch.equal(actual, activation)
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_cute_nvfp4_e5m3_gs_weight_dq_matches_reference(dtype):
+ if torch.cuda.get_device_capability()[0] < 8:
+ pytest.skip("CuTe weight DQ requires SM80 or newer")
+
+ torch.manual_seed(42)
+ weight_packed = torch.randint(0, 256, (37, 64), device="cuda", dtype=torch.uint8)
+ weight_scale = torch.randint(0, 0xFE, (37, 8), device="cuda", dtype=torch.uint8)
+ global_scale = torch.tensor([12345.0], device="cuda")
+ actual = try_cute_nvfp4_e5m3_gs_weight_dq(weight_packed, weight_scale, global_scale, dtype)
+ if actual is None:
+ pytest.skip("CuTe DSL is not available")
+
+ unpacked = unpack_fp4_from_uint8(weight_packed, 37, 128, dtype=dtype).float()
+ scale = (e5m3_to_float_tensor(weight_scale) / global_scale).reshape(-1, 1)
+ expected = (unpacked.reshape(-1, 16) * scale).reshape(37, 128).to(dtype)
+ torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
+def test_cute_nvfp4_e5m3_gs_weight_dq_handles_zero_global_scale():
+ if torch.cuda.get_device_capability()[0] < 8:
+ pytest.skip("CuTe weight DQ requires SM80 or newer")
+
+ weight_packed = torch.zeros((2, 8), device="cuda", dtype=torch.uint8)
+ weight_scale = torch.zeros((2, 1), device="cuda", dtype=torch.uint8)
+ global_scale = torch.zeros(1, device="cuda", dtype=torch.float32)
+ actual = try_cute_nvfp4_e5m3_gs_weight_dq(weight_packed, weight_scale, global_scale, torch.float32)
+ if actual is None:
+ pytest.skip("CuTe DSL is not available")
+
+ assert torch.isfinite(actual).all()
+ assert torch.equal(actual, torch.zeros_like(actual))