Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: add support for FP8 model loading and ragged-tile dequantization fallback (#2158)

Xin He40635d0273a6
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index 7d5dd76f4..a32e8dc03 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -41,6 +41,7 @@ from auto_round.algorithms.config_resolver import (
resolve_shared_config_values,
split_quantization_configs,
)
+from auto_round.algorithms.utils import _has_nvfp4_layer
from auto_round.logger import logger
from auto_round.utils import clear_memory
from auto_round.utils.device_manager import device_manager
@@ -202,8 +203,9 @@ class AlgorithmComposer:
):
logger.info("Block-forward torch.compile is disabled because an enabled rotation is incompatible.")
- if "nv_fp" in orchestrator.data_type:
+ if _has_nvfp4_layer(orchestrator):
can_compile_block_forward = False
+ logger.info("Block-forward torch.compile is disabled because at least one quantized layer uses NVFP4.")
# Bind compressor-level infrastructure (set before _build_quantizer is called).
self.block_forward = (
@@ -223,6 +225,7 @@ class AlgorithmComposer:
Returns a list of hook handles that the caller must remove when done.
"""
+ from auto_round.compressors.utils import is_nv_fp
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
is_act_nv_fp = getattr(self.block_quantizer.config, "is_act_nv_fp", False)
@@ -231,16 +234,18 @@ class AlgorithmComposer:
input = input[0] if isinstance(input, (tuple, list)) else input
if input.numel() == 0:
return
+ module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)
+ is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else is_act_nv_fp
input, _, _ = reshape_pad_tensor_by_group_size(input, module.act_group_size)
act_max = torch.max(torch.abs(input), dim=-1).values
if not hasattr(module, "act_max") or module.act_max.numel() == 0:
module.act_max = act_max
- if is_act_nv_fp:
+ if is_module_act_nv_fp:
max_val = act_max.max()
module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
return
act_max = act_max.to(module.act_max.device)
- if is_act_nv_fp:
+ if is_module_act_nv_fp:
max_val = torch.max(act_max.max(), module.act_max.max())
module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
else:
diff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.py
index ac896183c..5ef7e1d71 100644
--- a/auto_round/algorithms/quantization/utils.py
+++ b/auto_round/algorithms/quantization/utils.py
@@ -14,7 +14,7 @@
import torch
-from auto_round.compressors.utils import check_need_act_calibration
+from auto_round.compressors.utils import check_need_act_calibration, is_nv_fp
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized
@@ -26,16 +26,18 @@ def register_act_max_hooks(quantizer, model):
input = input[0]
if input.numel() == 0:
return
+ module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)
+ is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else quantizer.config.is_act_nv_fp
input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)
act_max = torch.max(torch.abs(input), dim=-1).values
if not hasattr(module, "act_max") or module.act_max.numel() == 0:
module.act_max = act_max
- if quantizer.config.is_act_nv_fp:
+ if is_module_act_nv_fp:
max_val = act_max.max()
module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
else:
act_max = act_max.to(module.act_max.device)
- if quantizer.config.is_act_nv_fp:
+ if is_module_act_nv_fp:
max_val = torch.max(act_max.max(), module.act_max.max())
module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
else:
diff --git a/auto_round/algorithms/utils.py b/auto_round/algorithms/utils.py
new file mode 100644
index 000000000..76ce0d304
--- /dev/null
+++ b/auto_round/algorithms/utils.py
@@ -0,0 +1,48 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Shared utilities for the algorithms package."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from auto_round.algorithms.base import BaseOrchestrator
+
+
+def _is_nvfp4_value(value: Any) -> bool:
+ """Return True when a raw config value indicates NVFP4."""
+ if not isinstance(value, str):
+ return False
+ value = value.lower()
+ return "nv_fp" in value or "nvfp4" in value
+
+
+def _has_nvfp4_layer(orchestrator: "BaseOrchestrator") -> bool:
+ """Whether global or per-layer config enables any NVFP4 quantization."""
+ if _is_nvfp4_value(getattr(orchestrator, "data_type", "")):
+ return True
+
+ layer_config = getattr(orchestrator, "layer_config", None)
+ if not isinstance(layer_config, dict):
+ return False
+
+ for config in layer_config.values():
+ if not isinstance(config, dict):
+ continue
+ if _is_nvfp4_value(config.get("data_type")) or _is_nvfp4_value(config.get("act_data_type")):
+ return True
+ if _is_nvfp4_value(config.get("scheme")):
+ return True
+ return False
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 8183dd0e9..0b0695c9a 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -534,8 +534,58 @@ class BaseOrchestrator(object):
):
return True
+ # Layer-level scheme overrides can request static-activation paths
+ # (e.g., global MXFP8 + local NVFP4 experts). Those still need
+ # calibration data even when top-level scheme looks dynamic.
+ if self._layer_config_needs_calibration(check_need_act_calibration):
+ return True
+
return False
+ def _layer_config_needs_calibration(self, check_need_act_calibration) -> bool:
+ """Return True if any raw layer_config entry implies activation calibration."""
+ layer_cfg = self.layer_config
+ if not isinstance(layer_cfg, dict) or not layer_cfg:
+ return False
+
+ def _entry_needs_calibration(entry) -> bool:
+ if entry is None:
+ return False
+
+ candidates = []
+ if isinstance(entry, (str, QuantizationScheme)):
+ candidates.append(entry)
+ elif isinstance(entry, dict):
+ if "scheme" in entry:
+ candidates.append(entry.get("scheme"))
+ candidates.append(entry)
+ else:
+ return False
+
+ for candidate in candidates:
+ if candidate is None:
+ continue
+ try:
+ _, _, attrs = parse_scheme(candidate, {})
+ except Exception: # noqa: BLE001
+ continue
+
+ cand_act_bits = attrs.get("act_bits")
+ cand_act_data_type = attrs.get("act_data_type")
+ cand_act_dynamic = attrs.get("act_dynamic")
+ is_cand_act_quant = cand_act_bits is not None and cand_act_bits <= 8
+ if is_cand_act_quant and check_need_act_calibration(
+ cand_act_dynamic,
+ cand_act_data_type,
+ cand_act_bits if cand_act_bits is not None else 16,
+ static_kv_dtype=self.static_kv_dtype,
+ static_attention_dtype=self.static_attention_dtype,
+ ):
+ return True
+ return False
+
+ return any(_entry_needs_calibration(v) for v in layer_cfg.values())
+
# ── Convenience properties ────────────────────────────────────────────────
@property
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 7e4be30db..c412b2445 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -20,6 +20,7 @@ from typing import List, Union
import torch
from torch.nn import Linear, Module
+from auto_round import envs
from auto_round.compressors.utils import is_nv_fp
from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
from auto_round.utils import check_to_quantized, logger
@@ -442,9 +443,12 @@ def update_fused_layer_global_scales(
For MLP layers:
- gate_proj and up_proj share a single global scale.
- This behavior is currently required by vLLM and may become optional
- in the future.
+ Set ``AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE=0`` to retain per-projection
+ global scales. The default keeps scales compatible with vLLM fused kernels.
"""
+ if not envs.AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE:
+ return
+
global_scale_name = f"{base_name}_global_scale"
def _collect_scales(mods: List[Module]) -> List[torch.Tensor]:
@@ -459,14 +463,14 @@ def update_fused_layer_global_scales(
return scales
def _is_attention_module(module: Module):
- return "attention" in module.__class__.__name__.lower() and (
- hasattr(module, "k_proj") or hasattr(module, "v_proj") or hasattr(module, "qkv_proj")
- )
+ return all(hasattr(module, projection) for projection in ("q_proj", "k_proj", "v_proj"))
def _is_mlp_module(module: Module):
- return "mlp" in module.__class__.__name__.lower() and (
- hasattr(module, "gate_proj") and hasattr(module, "up_proj")
- )
+ return all(hasattr(module, projection) for projection in ("gate_proj", "up_proj"))
+
+ def _is_moe_expert_module(module: Module):
+ """Check for MoE expert naming: w1 (gate) and w3 (up)."""
+ return all(hasattr(module, projection) for projection in ("w1", "w3"))
def _update_global_scales(modules: List[Module]):
"""Update global scales for a list of modules."""
@@ -487,28 +491,39 @@ def update_fused_layer_global_scales(
# ---------------- Attention ----------------
if _is_attention_module(submodule):
- # Already fused
- if hasattr(submodule, "qkv_proj"):
- return
_update_global_scales([submodule.q_proj, submodule.k_proj, submodule.v_proj])
return
# ---------------- MLP ----------------
if _is_mlp_module(submodule):
_update_global_scales([submodule.gate_proj, submodule.up_proj])
+ return
+ # ---------------- MoE Expert (w1/w3) ----------------
+ if _is_moe_expert_module(submodule):
+ _update_global_scales([submodule.w1, submodule.w3])
-def update_block_global_scale_if_needed(block, data_type, group_size):
- if not is_nv_fp(data_type):
- return
+def update_block_global_scale_if_needed(block, data_type, group_size):
from auto_round.data_type.nvfp import calculate_gparam
+ has_nvfp = is_nv_fp(data_type)
+
# Calculate block wise weight global scale
for _, m in block.named_modules():
- if check_to_quantized(m) and not hasattr(m, "weight_global_scale"):
- weight_global_scale = calculate_gparam(m.weight, group_size)
- setattr(m, "weight_global_scale", weight_global_scale)
+ if not check_to_quantized(m):
+ continue
+ # Check per-layer data_type for mixed-scheme scenarios
+ module_data_type = getattr(m, "data_type", data_type)
+ module_group_size = getattr(m, "group_size", group_size)
+ if is_nv_fp(module_data_type):
+ has_nvfp = True
+ if not hasattr(m, "weight_global_scale"):
+ weight_global_scale = calculate_gparam(m.weight, module_group_size)
+ setattr(m, "weight_global_scale", weight_global_scale)
+
+ if not has_nvfp:
+ return
# Update fused layer global scales
for module in block.modules():
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 7c727443c..2401c8ce2 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -31,6 +31,8 @@ if TYPE_CHECKING:
AR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = False
AR_DISK_STREAM_MODEL: bool = False
AR_RESUME_DIR: Optional[str] = None
+ AR_FORCE_MOE_ROUTING_ALL_EXPERTS: bool = False
+ AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = True
def _get_optional_positive_int_env(name: str) -> Optional[int]:
@@ -123,6 +125,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
# instead of restarting the whole tuning pass from block 0 after a
# crash/kill. See auto_round/utils/resume.py.
"AR_RESUME_DIR": lambda: os.getenv("AR_RESUME_DIR", None),
+ # When enabled, MoE routing can be overridden in selected model wrappers
+ # to rotate token assignments across all experts for calibration coverage.
+ "AR_FORCE_MOE_ROUTING_ALL_EXPERTS": lambda: os.getenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "0").lower()
+ in ("1", "true", "yes"),
+ # vLLM fused kernels require q/k/v and gate/up projections to use one
+ # weight global scale. Disable only for runtimes without that requirement.
+ "AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE": lambda: os.getenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "1").lower()
+ not in ("0", "false", "no", "off"),
}
diff --git a/auto_round/modeling/fp8_quant.py b/auto_round/modeling/fp8_quant.py
index a5490c844..3f4a25057 100644
--- a/auto_round/modeling/fp8_quant.py
+++ b/auto_round/modeling/fp8_quant.py
@@ -93,12 +93,78 @@ def oot_replace_with_fp8_linear(
_orig_validate_environment = OriginalFineGrainedFP8HfQuantizer.validate_environment
+def _patch_fp8_dequantize_for_ragged_tiles():
+ """Patch transformers FP8 dequantization to handle over-provisioned scale grids.
+
+ Some pre-quantized checkpoints keep FP8 weights in non-padded shape while the
+ corresponding ``weight_scale_inv`` grid is computed on padded block tiles.
+ Example: weight ``(576, 7168)`` with scale grid ``(5, 56)`` for block size
+ ``(128, 128)`` (5 rows of tiles implies padded rows to 640).
+
+ Upstream strict divisibility checks fail on those tensors. We keep upstream
+ behavior by default and only fall back on this specific shape mismatch.
+ """
+
+ try:
+ import transformers.integrations.finegrained_fp8 as transformers_fp8
+
+ from auto_round.utils.weight_handler import _dequant_fp8_linear_weight
+ except ImportError:
+ return
+
+ if not hasattr(transformers_fp8, "Fp8Dequantize"):
+ return
+
+ cls = transformers_fp8.Fp8Dequantize
+ if hasattr(cls, "_auto_round_orig_dequantize_one"):
+ return
+
+ if not hasattr(cls, "_dequantize_one"):
+ return
+
+ cls._auto_round_orig_dequantize_one = cls._dequantize_one
+
+ def _dequantize_one_with_padding_fallback(self, quantized, scales, output_dtype=None):
+ try:
+ return cls._auto_round_orig_dequantize_one(self, quantized, scales, output_dtype=output_dtype)
+ except ValueError as e:
+ if "not divisible by scale grid" not in str(e):
+ raise
+
+ # Match upstream MXFP8 handling for uint8 E8M0 scales.
+ if scales.dtype == torch.uint8:
+ scales = (scales.to(torch.float32) - 127.0).exp2()
+
+ block_size = None
+ qcfg = getattr(self.hf_quantizer, "quantization_config", None)
+ if qcfg is not None:
+ if isinstance(qcfg, dict):
+ block_size = qcfg.get("weight_block_size")
+ else:
+ block_size = getattr(qcfg, "weight_block_size", None)
+
+ # Falls back to AutoRound's robust dequantizer that supports
+ # over-provisioned scale grids by padding then slicing back.
+ dequantized = _dequant_fp8_linear_weight(quantized, scales, block_size=block_size)
+ if output_dtype is not None:
+ dequantized = dequantized.to(output_dtype)
+ return dequantized
+
+ cls._dequantize_one = _dequantize_one_with_padding_fallback
+ auto_round_logger.debug("Applied FP8 ragged-tile dequantization fallback patch.")
+
+
@override_cuda_device_capability()
def oot_validate_environment(self, *args, **kwargs):
return _orig_validate_environment(self, *args, **kwargs)
def apply_fp8_expert_replacement_patch():
+ # Apply the ragged-tile dequantization fallback unconditionally so that
+ # dequantize-on-load (FineGrainedFP8Config(dequantize=True)) works on
+ # CPU and HPU as well as CUDA.
+ _patch_fp8_dequantize_for_ragged_tiles()
+
if is_transformers_version_greater_or_equal_5() and torch.cuda.is_available():
try:
import transformers.integrations.finegrained_fp8 as transformers_fp8
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index 3cf25784e..cc856c5c5 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -35,6 +35,7 @@ import torch
from torch import nn
from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
+from auto_round.modeling.fused_moe.utils import build_forced_routing, force_all_experts_routing_enabled
from auto_round.utils import clear_memory, logger
from auto_round.utils.device import memory_monitor
@@ -217,73 +218,81 @@ def linear_loop_experts_forward(
num_tokens = hidden_states.size(0)
num_experts = self.num_experts
- # Reshape for easier indexing
- # S is the number of selected token-expert pairs (S = num_tokens * num_top_k)
- token_idx = torch.arange(num_tokens, device=device).unsqueeze(1).expand(-1, num_top_k).reshape(-1) # (S,)
- sample_weights = top_k_weights.reshape(-1).to(hidden_states.dtype) # (S,)
- expert_ids = top_k_index.reshape(-1) # (S,)
-
- # Get current hidden states for selected samples
- selected_hidden_states = hidden_states[token_idx] # (S, hidden_dim)
-
- # Group token-expert pairs by expert using a single sort, then run each
- # expert on a contiguous *static* slice, instead of doing a per-expert
- # nonzero()/boolean-mask lookup inside the Python loop.
- #
- # Why: repeatedly issuing dynamic-shape gather kernels (nonzero(),
- # index_select() or boolean-mask indexing) once per expert in a loop
- # reliably triggers a driver-level bug on some XPU builds - either a
- # "vectorized gather kernel index out of bounds" device-side assertion or
- # a hard UR_RESULT_ERROR_DEVICE_LOST - even when every index is valid.
- # This was confirmed with minimal PyTorch-only reproductions unrelated to
- # this model/library, so working around it in application code is the
- # practical fix. Sorting once needs only two dynamic gather/scatter calls
- # in total (regardless of num_experts): one index_select to group tokens
- # by expert, and one index_copy_ to scatter results back to their
- # original positions. Each expert's slice `permuted[start:end]` is a
- # plain view (no gather kernel involved), so this is also efficient.
- sort_order = torch.argsort(expert_ids)
- permuted_hidden_states = selected_hidden_states.index_select(0, sort_order) # (S, hidden_dim)
-
- # Per-expert token counts, computed once on host to drive static slicing.
- counts = torch.bincount(expert_ids, minlength=num_experts).tolist()
-
- out_permuted = torch.zeros_like(permuted_hidden_states)
- start = 0
- for expert_idx, count in enumerate(counts):
- if count == 0:
- continue
- end = start + count
- expert_input = permuted_hidden_states[start:end] # static slice/view, no gather kernel
-
- # Get this expert's container with its projection layers
- expert = getattr(self, str(expert_idx))
- gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)
- up_out = expert.up_proj(expert_input) # (num_samples, intermediate_dim)
-
- # Apply gating
- if hasattr(self, "_apply_gate"):
- gate_up_out = torch.cat([gate_out, up_out], dim=-1)
- gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)
- else:
- gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)
-
- # Down projection
- expert_out = expert.down_proj(gated_out) # (num_samples, hidden_dim)
-
- out_permuted[start:end] = expert_out.to(out_permuted.dtype)
- start = end
-
- # Scatter results back to their original (pre-sort) token-expert order.
- out_per_sample = torch.empty_like(out_permuted)
- out_per_sample.index_copy_(0, sort_order, out_permuted)
-
- # Apply routing weights
- out_per_sample = out_per_sample * sample_weights.unsqueeze(-1) # (S, hidden_dim)
-
- # Accumulate results using deterministic reshape+sum instead of index_add_
- # (index_add_ with duplicate indices is non-deterministic on CUDA due to atomicAdd)
- final_hidden_states = out_per_sample.view(num_tokens, num_top_k, hidden_dim).sum(dim=1)
+ def _run_experts_with_routes(route_indices: torch.Tensor, route_weights: torch.Tensor) -> torch.Tensor:
+ # Reshape for easier indexing
+ # S is the number of selected token-expert pairs (S = num_tokens * num_top_k)
+ token_idx = torch.arange(num_tokens, device=device).unsqueeze(1).expand(-1, num_top_k).reshape(-1) # (S,)
+ sample_weights = route_weights.reshape(-1).to(hidden_states.dtype) # (S,)
+ expert_ids = route_indices.reshape(-1) # (S,)
+
+ # Get current hidden states for selected samples
+ selected_hidden_states = hidden_states[token_idx] # (S, hidden_dim)
+
+ # Group token-expert pairs by expert using a single sort, then run each
+ # expert on a contiguous *static* slice, instead of doing a per-expert
+ # nonzero()/boolean-mask lookup inside the Python loop.
+ sort_order = torch.argsort(expert_ids)
+ permuted_hidden_states = selected_hidden_states.index_select(0, sort_order) # (S, hidden_dim)
+
+ # Per-expert token counts, computed once on host to drive static slicing.
+ counts = torch.bincount(expert_ids, minlength=num_experts).tolist()
+
+ out_permuted = torch.zeros_like(permuted_hidden_states)
+ start = 0
+ for expert_idx, count in enumerate(counts):
+ if count == 0:
+ continue
+ end = start + count
+ expert_input = permuted_hidden_states[start:end] # static slice/view, no gather kernel
+
+ # Get this expert's container with its projection layers
+ expert = getattr(self, str(expert_idx))
+ gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)
+ up_out = expert.up_proj(expert_input) # (num_samples, intermediate_dim)
+
+ # Apply gating
+ if hasattr(self, "_apply_gate"):
+ gate_up_out = torch.cat([gate_out, up_out], dim=-1)
+ gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)
+ else:
+ gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)
+
+ # Down projection
+ expert_out = expert.down_proj(gated_out) # (num_samples, hidden_dim)
+
+ out_permuted[start:end] = expert_out.to(out_permuted.dtype)
+ start = end
+
+ # Scatter results back to their original (pre-sort) token-expert order.
+ out_per_sample = torch.empty_like(out_permuted)
+ out_per_sample.index_copy_(0, sort_order, out_permuted)
+
+ # Apply routing weights
+ out_per_sample = out_per_sample * sample_weights.unsqueeze(-1) # (S, hidden_dim)
+
+ # Accumulate results using deterministic reshape+sum instead of index_add_
+ # (index_add_ with duplicate indices is non-deterministic on CUDA due to atomicAdd)
+ return out_per_sample.view(num_tokens, num_top_k, hidden_dim).sum(dim=1)
+
+ # Main output path: always preserve original router selection.
+ final_hidden_states = _run_experts_with_routes(top_k_index, top_k_weights)
+
+ # Auxiliary coverage path: when enabled, run forced-routing once to trigger
+ # expert hooks/stat collection, but discard this output to avoid changing
+ # model semantics.
+ if force_all_experts_routing_enabled():
+ forced_indices, forced_weights = build_forced_routing(
+ module=self,
+ routing_scores=None,
+ top_k=num_top_k,
+ num_experts=num_experts,
+ dtype=hidden_states.dtype,
+ num_tokens=num_tokens,
+ device=device,
+ normalize=True,
+ )
+ with torch.no_grad():
+ _ = _run_experts_with_routes(forced_indices, forced_weights)
# Reshape back to original format if input was [batch_size, seq_len, hidden_dim]
if batch_size is not None:
diff --git a/auto_round/modeling/fused_moe/qwen3_5_moe.py b/auto_round/modeling/fused_moe/qwen3_5_moe.py
index eb0a2eea3..e96c3224e 100644
--- a/auto_round/modeling/fused_moe/qwen3_5_moe.py
+++ b/auto_round/modeling/fused_moe/qwen3_5_moe.py
@@ -8,7 +8,12 @@ from transformers.utils.versions import require_version
from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
-from auto_round.modeling.fused_moe.utils import _update_parameter, sequential_moe_forward
+from auto_round.modeling.fused_moe.utils import (
+ _update_parameter,
+ build_forced_routing,
+ force_all_experts_routing_enabled,
+ sequential_moe_forward,
+)
from auto_round.utils import clear_memory, unsupported_meta_device
require_version("transformers>=5.2.0")
@@ -43,7 +48,26 @@ class LinearQwen3_5MoeSparseMoeBlock(ReplacementModuleBase):
batch_size, sequence_length, hidden_dim = hidden_states.shape
hidden_states_reshaped = hidden_states.view(-1, hidden_dim)
shared_expert_output = self.shared_expert(hidden_states_reshaped)
- _, routing_weights, selected_experts = self.gate(hidden_states_reshaped)
+ router_logits, routing_weights, selected_experts = self.gate(hidden_states_reshaped)
+ if force_all_experts_routing_enabled():
+ routing_scores = (
+ torch.softmax(router_logits, dim=-1, dtype=torch.float)
+ if router_logits is not None
+ else torch.full(
+ (hidden_states_reshaped.size(0), self.num_experts),
+ 1.0 / float(self.num_experts),
+ device=hidden_states_reshaped.device,
+ dtype=torch.float,
+ )
+ )
+ selected_experts, routing_weights = build_forced_routing(
+ module=self,
+ routing_scores=routing_scores,
+ top_k=routing_weights.size(-1),
+ num_experts=self.num_experts,
+ dtype=hidden_states_reshaped.dtype,
+ normalize=True,
+ )
expert_output = self.experts_forward(hidden_states_reshaped, selected_experts, routing_weights)
shared_expert_output = F.sigmoid(self.shared_expert_gate(hidden_states_reshaped)) * shared_expert_output
diff --git a/auto_round/modeling/fused_moe/utils.py b/auto_round/modeling/fused_moe/utils.py
index 05b1ad3d6..2befcbafc 100644
--- a/auto_round/modeling/fused_moe/utils.py
+++ b/auto_round/modeling/fused_moe/utils.py
@@ -15,6 +15,8 @@
import torch
import torch.nn as nn
+from auto_round import envs
+
def _update_parameter(
module: torch.nn.Module,
@@ -125,3 +127,85 @@ def get_num_experts(original: torch.nn.Module) -> int:
"Unsupported MoE experts layout: expected fused gate_up_proj/down_proj "
"or linearized gate_proj/up_proj/down_proj experts"
)
+
+
+def force_all_experts_routing_enabled() -> bool:
+ """Return whether MoE wrappers should override router top-k for coverage."""
+ return envs.AR_FORCE_MOE_ROUTING_ALL_EXPERTS
+
+
+def build_forced_expert_indices(
+ *,
+ module: torch.nn.Module,
+ num_tokens: int,
+ top_k: int,
+ num_experts: int,
+ device: torch.device,
+) -> torch.Tensor:
+ """Build deterministic rotating expert indices to cover all experts over time."""
+ if num_tokens <= 0:
+ raise ValueError("num_tokens must be > 0")
+ if top_k <= 0:
+ raise ValueError("top_k must be > 0")
+ if num_experts <= 0:
+ raise ValueError("num_experts must be > 0")
+
+ top_k = min(top_k, num_experts)
+ total_pairs = num_tokens * top_k
+ offset = int(getattr(module, "_auto_round_forced_route_offset", 0))
+ forced = (torch.arange(total_pairs, device=device) + offset) % num_experts
+ forced = forced.view(num_tokens, top_k)
+ module._auto_round_forced_route_offset = (offset + total_pairs) % num_experts
+ return forced
+
+
+def build_forced_routing(
+ *,
+ module: torch.nn.Module,
+ routing_scores: torch.Tensor | None,
+ top_k: int,
+ num_experts: int,
+ dtype: torch.dtype,
+ num_tokens: int | None = None,
+ device: torch.device | None = None,
+ normalize: bool = True,
+ scaling_factor: float = 1.0,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Create forced top-k indices and matching routing weights.
+
+ When scores are unavailable, fall back to uniform weights.
+ """
+ if routing_scores is None:
+ if num_tokens is None or device is None:
+ raise ValueError("num_tokens and device are required when routing_scores is None")
+ else:
+ num_tokens = int(routing_scores.shape[0])
+ device = routing_scores.device
+
+ assert num_tokens is not None
+ assert device is not None
+ forced_indices = build_forced_expert_indices(
+ module=module,
+ num_tokens=num_tokens,
+ top_k=top_k,
+ num_experts=num_experts,
+ device=device,
+ )
+
+ if routing_scores is not None:
+ weights = routing_scores.gather(1, forced_indices)
+ else:
+ denom = float(min(top_k, num_experts))
+ weights = torch.full(
+ (num_tokens, min(top_k, num_experts)),
+ 1.0 / denom,
+ device=forced_indices.device,
+ dtype=torch.float32,
+ )
+
+ if normalize:
+ weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20)
+ if scaling_factor != 1.0:
+ weights = weights * scaling_factor
+
+ return forced_indices, weights.to(dtype)
diff --git a/auto_round/modeling/unfused_moe/deepseek_v3.py b/auto_round/modeling/unfused_moe/deepseek_v3.py
index cfcc617a6..9017bb095 100644
--- a/auto_round/modeling/unfused_moe/deepseek_v3.py
+++ b/auto_round/modeling/unfused_moe/deepseek_v3.py
@@ -15,7 +15,11 @@
import torch
import torch.nn as nn
-from auto_round.modeling.fused_moe.utils import sequential_moe_forward
+from auto_round.modeling.fused_moe.utils import (
+ build_forced_routing,
+ force_all_experts_routing_enabled,
+ sequential_moe_forward,
+)
class LinearDeepseekV3MoE(nn.Module):
@@ -53,6 +57,17 @@ class LinearDeepseekV3MoE(nn.Module):
def route_tokens_to_experts(self, router_logits):
router_logits = router_logits.sigmoid()
+ if force_all_experts_routing_enabled():
+ return build_forced_routing(
+ module=self,
+ routing_scores=router_logits,
+ top_k=self.top_k,
+ num_experts=self.num_experts,
+ dtype=router_logits.dtype,
+ normalize=self.norm_topk_prob,
+ scaling_factor=self.routed_scaling_factor,
+ )
+
router_logits_for_choice = router_logits + self.gate.e_score_correction_bias
group_scores = (
router_logits_for_choice.view(-1, self.n_group, self.n_routed_experts // self.n_group)
@@ -81,7 +96,17 @@ class LinearDeepseekV3MoE(nn.Module):
orig_shape = hidden_states.shape
router_logits = self.gate(hidden_states)
if isinstance(router_logits, tuple): # transformers >= 5.13.0
- _, topk_weights, topk_indices = router_logits
+ base_router_logits, topk_weights, topk_indices = router_logits
+ if force_all_experts_routing_enabled():
+ topk_indices, topk_weights = build_forced_routing(
+ module=self,
+ routing_scores=base_router_logits.sigmoid(),
+ top_k=self.top_k,
+ num_experts=self.num_experts,
+ dtype=hidden_states.dtype,
+ normalize=self.norm_topk_prob,
+ scaling_factor=self.routed_scaling_factor,
+ )
else:
topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
diff --git a/auto_round/modeling/unfused_moe/qwen3_moe.py b/auto_round/modeling/unfused_moe/qwen3_moe.py
index d1e9d7a39..2c3921014 100644
--- a/auto_round/modeling/unfused_moe/qwen3_moe.py
+++ b/auto_round/modeling/unfused_moe/qwen3_moe.py
@@ -16,7 +16,11 @@ import torch
import torch.nn as nn
from torch.nn import functional as F
-from auto_round.modeling.fused_moe.utils import sequential_moe_forward
+from auto_round.modeling.fused_moe.utils import (
+ build_forced_routing,
+ force_all_experts_routing_enabled,
+ sequential_moe_forward,
+)
class LinearQwen3MoeSparseMoeBlock(nn.Module):
@@ -42,11 +46,21 @@ class LinearQwen3MoeSparseMoeBlock(nn.Module):
router_logits = self.gate(hidden_states)
routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
- routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
- if self.norm_topk_prob: # only diff with mixtral sparse moe block!
- routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
- # we cast back to the input dtype
- routing_weights = routing_weights.to(hidden_states.dtype)
+ if force_all_experts_routing_enabled():
+ selected_experts, routing_weights = build_forced_routing(
+ module=self,
+ routing_scores=routing_weights,
+ top_k=self.top_k,
+ num_experts=self.num_experts,
+ dtype=hidden_states.dtype,
+ normalize=self.norm_topk_prob,
+ )
+ else:
+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
+ if self.norm_topk_prob: # only diff with mixtral sparse moe block!
+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
+ # we cast back to the input dtype
+ routing_weights = routing_weights.to(hidden_states.dtype)
final_hidden_states = sequential_moe_forward(
hidden_states, selected_experts, routing_weights, self.experts, self.num_experts
diff --git a/auto_round/modeling/unfused_moe/qwen3_next.py b/auto_round/modeling/unfused_moe/qwen3_next.py
index df0f7a960..3441d66da 100644
--- a/auto_round/modeling/unfused_moe/qwen3_next.py
+++ b/auto_round/modeling/unfused_moe/qwen3_next.py
@@ -16,7 +16,11 @@ import torch
import torch.nn as nn
from torch.nn import functional as F
-from auto_round.modeling.fused_moe.utils import sequential_moe_forward
+from auto_round.modeling.fused_moe.utils import (
+ build_forced_routing,
+ force_all_experts_routing_enabled,
+ sequential_moe_forward,
+)
class LinearQwen3NextSparseMoeBlock(nn.Module):
@@ -45,11 +49,21 @@ class LinearQwen3NextSparseMoeBlock(nn.Module):
router_logits = self.gate(hidden_states)
routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
- routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
- if self.norm_topk_prob:
- routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
- # we cast back to the input dtype
- routing_weights = routing_weights.to(hidden_states.dtype)
+ if force_all_experts_routing_enabled():
+ selected_experts, routing_weights = build_forced_routing(
+ module=self,
+ routing_scores=routing_weights,
+ top_k=self.top_k,
+ num_experts=self.num_experts,
+ dtype=hidden_states.dtype,
+ normalize=self.norm_topk_prob,
+ )
+ else:
+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
+ if self.norm_topk_prob:
+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
+ # we cast back to the input dtype
+ routing_weights = routing_weights.to(hidden_states.dtype)
final_hidden_states = sequential_moe_forward(
hidden_states, selected_experts, routing_weights, self.experts, self.num_experts
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 758fc466b..8901e52ef 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -289,6 +289,7 @@ def _check_accelerate_version():
_MXFP4_SUPPORTED_MODEL_TYPES = {"gpt_oss"}
+_FP8_SUPPORTED_MODEL_TYPES = {"deepseek_v32"}
def _is_mxfp4_model(model_path, trust_remote_code=True):
@@ -320,6 +321,38 @@ def _is_mxfp4_model(model_path, trust_remote_code=True):
return quant_method == "mxfp4" and model_type in _MXFP4_SUPPORTED_MODEL_TYPES
+def _is_fp8_model(model_path, trust_remote_code=True):
+ """Check if a model is an FP8 quantized model supported for direct loading.
+
+ Only checks when transformers >= 4.56.0. Returns False immediately for older versions,
+ adding zero overhead to non-FP8 model loading.
+ """
+ if version.parse(transformers.__version__) < version.parse("4.56.0"):
+ return False
+
+ from transformers import AutoConfig
+
+ try: # in case of config loading failure for new models
+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
+ except Exception:
+ return False
+
+ model_type = getattr(config, "model_type", "")
+ if model_type not in _FP8_SUPPORTED_MODEL_TYPES:
+ return False
+
+ quant_config = getattr(config, "quantization_config", None)
+ if quant_config is None:
+ return False
+
+ quant_method = (
+ quant_config.get("quant_method", "")
+ if isinstance(quant_config, dict)
+ else getattr(quant_config, "quant_method", "")
+ )
+ return quant_method == "fp8" and model_type in _FP8_SUPPORTED_MODEL_TYPES
+
+
def llm_load_model(
pretrained_model_name_or_path: str,
platform: str = "hf",
@@ -373,6 +406,14 @@ def llm_load_model(
}
load_kwargs.update(kwargs)
+ if version.parse(transformers.__version__) >= version.parse("4.56.0"):
+ is_fp8 = _is_fp8_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+ if is_fp8 and "quantization_config" not in load_kwargs:
+ from transformers import FineGrainedFP8Config
+
+ load_kwargs["quantization_config"] = FineGrainedFP8Config(dequantize=True)
+ logger.info("Detected FP8 quantized model, using FineGrainedFP8Config(dequantize=True) for loading.")
+
if version.parse(transformers.__version__) >= version.parse("5.0.0"):
is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
if is_mxfp4:
diff --git a/docs/environments.md b/docs/environments.md
index 47f36cb30..6d16b3130 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -30,6 +30,16 @@ export AR_LOG_LEVEL=DEBUG
export AR_ENABLE_COMPILE_PACKING=1
```
+### AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE
+- **Description**: Makes fused NVFP4 weight projections use one shared weight global scale. This applies to `q_proj`/`k_proj`/`v_proj` and `gate_proj`/`up_proj`, as required by vLLM fused kernels.
+- **Default**: `True` (equivalent to `"1"`)
+- **Valid Values**: `"0"`, `"false"`, `"no"`, or `"off"` (case-insensitive) disable sharing; any other value enables it.
+- **Usage**: Disable only when exporting for a runtime that does not require fused projections to share a global scale.
+
+```bash
+export AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE=0
+```
+
### AR_USE_MODELSCOPE
- **Description**: Controls whether to use ModelScope for model downloads
- **Default**: `False`
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index 365b3b46d..8648b4d76 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -30,6 +30,16 @@ export AR_LOG_LEVEL=DEBUG
export AR_ENABLE_COMPILE_PACKING=1
```
+### AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE
+- **描述**:让 fused NVFP4 权重投影共用一个 weight global scale。该设置作用于 `q_proj`/`k_proj`/`v_proj` 与 `gate_proj`/`up_proj`,以满足 vLLM fused kernel 的要求。
+- **默认值**:`True`(等价于 `"1"`)
+- **有效值**:`"0"`、`"false"`、`"no"` 或 `"off"`(不区分大小写)表示关闭共享;其他值表示启用。
+- **用途**:仅当导出的运行时不要求 fused 投影使用统一 global scale 时关闭。
+
+```bash
+export AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE=0
+```
+
### AR_USE_MODELSCOPE
- **描述**:控制是否使用 ModelScope 下载模型
- **默认值**:`False`
diff --git a/test/unit/test_cpu/core/test_pipeline_fail_fast.py b/test/unit/test_cpu/core/test_pipeline_fail_fast.py
index ceb1f8ce3..f49fef09a 100644
--- a/test/unit/test_cpu/core/test_pipeline_fail_fast.py
+++ b/test/unit/test_cpu/core/test_pipeline_fail_fast.py
@@ -1,10 +1,12 @@
"""Fast unit tests for algorithm registry and bundle construction."""
+from types import SimpleNamespace
+
import pytest
from auto_round import AutoRound as NewAutoRound
from auto_round import AWQConfig, OptimizedRTNConfig, RotationConfig, RTNConfig, SignRoundConfig, SpinQuantConfig
-from auto_round.algorithms.composer import AlgorithmComposer, _can_compile_block_forward
+from auto_round.algorithms.composer import AlgorithmComposer, _can_compile_block_forward, _has_nvfp4_layer
from auto_round.algorithms.config_resolver import (
get_algorithm_class,
resolve_shared_config_values,
@@ -70,6 +72,37 @@ def test_hadamard_disables_only_block_forward_compile():
assert not _can_compile_block_forward(quantizer, [], user_enabled=False)
+def test_detect_nvfp4_from_layer_config_scheme_override():
+ orchestrator = SimpleNamespace(
+ data_type="mx_fp",
+ layer_config={"mlp.experts": {"scheme": "NVFP4"}},
+ )
+ assert _has_nvfp4_layer(orchestrator)
+
+
+def test_detect_nvfp4_from_layer_config_data_type_override():
+ orchestrator = SimpleNamespace(
+ data_type="mx_fp",
+ layer_config={"mlp.experts": {"data_type": "nv_fp"}},
+ )
+ assert _has_nvfp4_layer(orchestrator)
+
+
+def test_needs_calibration_data_when_layer_config_overrides_to_nvfp4():
+ class _DummyCompressor:
+ _needs_calibration_data = Compressor._needs_calibration_data
+ _layer_config_needs_calibration = Compressor._layer_config_needs_calibration
+
+ compressor = _DummyCompressor()
+ compressor._alg_configs = []
+ compressor.scheme = "MXFP8"
+ compressor.layer_config = {"mlp.experts": {"scheme": "NVFP4"}}
+ compressor.static_kv_dtype = None
+ compressor.static_attention_dtype = None
+
+ assert compressor._needs_calibration_data() is True
+
+
def test_registry_builtin_aliases_and_unknown():
assert isinstance(_r.resolve_alg_config("RTN"), RTNConfig)
assert isinstance(_r.resolve_alg_config("awq"), AWQConfig)
diff --git a/test/unit/test_cpu/data_type/test_nvfp.py b/test/unit/test_cpu/data_type/test_nvfp.py
index 444a14aad..930dd0a28 100644
--- a/test/unit/test_cpu/data_type/test_nvfp.py
+++ b/test/unit/test_cpu/data_type/test_nvfp.py
@@ -17,6 +17,7 @@ import math
import pytest
import torch
+from torch import nn
from auto_round.data_type.nvfp import (
FLOAT4_E2M1_MAX,
@@ -38,6 +39,7 @@ from auto_round.data_type.nvfp import (
ref_nvfp4_quant,
search_nvfp4_scale,
)
+from auto_round.data_type.utils import update_fused_layer_global_scales
# ---------------------------------------------------------------------------
# Constants
@@ -177,6 +179,52 @@ class TestCalculateGparam:
assert g.item() == pytest.approx(expected, rel=1e-5)
+# ---------------------------------------------------------------------------
+# fused global scale
+# ---------------------------------------------------------------------------
+
+
+class TestFusedLayerGlobalScales:
+ @staticmethod
+ def _projection(scale: float) -> nn.Module:
+ projection = nn.Module()
+ projection.weight_global_scale = torch.tensor([scale])
+ return projection
+
+ def test_fused_projections_share_minimum_scale_by_default(self, monkeypatch):
+ monkeypatch.delenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", raising=False)
+ attention = nn.Module()
+ attention.q_proj = self._projection(3.0)
+ attention.k_proj = self._projection(1.0)
+ attention.v_proj = self._projection(2.0)
+ mlp = nn.Module()
+ mlp.gate_proj = self._projection(4.0)
+ mlp.up_proj = self._projection(0.5)
+
+ update_fused_layer_global_scales(attention)
+ update_fused_layer_global_scales(mlp)
+
+ assert all(
+ proj.weight_global_scale.item() == 1.0 for proj in (attention.q_proj, attention.k_proj, attention.v_proj)
+ )
+ assert all(proj.weight_global_scale.item() == 0.5 for proj in (mlp.gate_proj, mlp.up_proj))
+
+ def test_fused_projection_scale_update_can_be_disabled(self, monkeypatch):
+ monkeypatch.setenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "0")
+ attention = nn.Module()
+ attention.q_proj = self._projection(3.0)
+ attention.k_proj = self._projection(1.0)
+ attention.v_proj = self._projection(2.0)
+
+ update_fused_layer_global_scales(attention)
+
+ assert [proj.weight_global_scale.item() for proj in (attention.q_proj, attention.k_proj, attention.v_proj)] == [
+ 3.0,
+ 1.0,
+ 2.0,
+ ]
+
+
# ---------------------------------------------------------------------------
# ref_nvfp4_quant
# ---------------------------------------------------------------------------
diff --git a/test/unit/test_cpu/models/test_moe_experts_interface.py b/test/unit/test_cpu/models/test_moe_experts_interface.py
index 6ce9cf29e..f8c811985 100644
--- a/test/unit/test_cpu/models/test_moe_experts_interface.py
+++ b/test/unit/test_cpu/models/test_moe_experts_interface.py
@@ -225,6 +225,47 @@ def test_linear_loop_forward():
assert not torch.allclose(output, torch.zeros_like(output)), "Output is all zeros"
+def test_linear_loop_force_routing_preserves_original_output(monkeypatch):
+ """Forced expert coverage should not change the returned MoE output."""
+ from auto_round.modeling.fused_moe.moe_experts_interface import (
+ _unfuse_experts_weights_inplace,
+ linear_loop_experts_forward,
+ )
+
+ num_experts = 4
+ hidden_dim = 16
+ intermediate_dim = 32
+ num_tokens = 9
+ top_k = 2
+
+ class MockExperts(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
+ self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
+ self.act_fn = nn.SiLU()
+ self.num_experts = num_experts
+ self.has_bias = False
+ self.is_transposed = False
+
+ torch.manual_seed(7)
+ module = MockExperts()
+ _unfuse_experts_weights_inplace(module, check_decorator=False)
+
+ hidden_states = torch.randn(num_tokens, hidden_dim)
+ top_k_index = torch.randint(0, num_experts, (num_tokens, top_k))
+ top_k_weights = torch.softmax(torch.randn(num_tokens, top_k), dim=-1)
+
+ monkeypatch.setenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "0")
+ output_normal = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)
+
+ monkeypatch.setenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "1")
+ output_forced = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)
+
+ assert torch.allclose(output_normal, output_forced, atol=1e-6, rtol=1e-5)
+ assert hasattr(module, "_auto_round_forced_route_offset")
+
+
def test_prepare_model_for_moe_quantization():
"""Test the full prepare_model_for_moe_quantization flow."""
from auto_round.modeling.fused_moe.moe_experts_interface import (
diff --git a/test/unit/test_cpu/utils/test_model_utils.py b/test/unit/test_cpu/utils/test_model_utils.py
index 17e7b6e54..8a49f9296 100644
--- a/test/unit/test_cpu/utils/test_model_utils.py
+++ b/test/unit/test_cpu/utils/test_model_utils.py
@@ -289,6 +289,71 @@ class TestIsMxfp4Model:
assert result is True
+class TestIsFp8Model:
+ """Test _is_fp8_model function."""
+
+ def test_fp8_model_returns_false_for_old_transformers(self):
+ """Test _is_fp8_model returns False immediately for transformers < 4.56.0."""
+ from auto_round.utils.model import _is_fp8_model
+
+ with patch("auto_round.utils.model.version") as mock_version:
+ mock_version.parse.side_effect = lambda v: __import__("packaging.version", fromlist=["parse"]).parse(v)
+ # Simulate old transformers by making the version check fail
+ import packaging.version
+
+ with patch("auto_round.utils.model.transformers") as mock_tf:
+ mock_tf.__version__ = "4.50.0"
+ result = _is_fp8_model("test/model", trust_remote_code=True)
+ assert result is False
+
+ def test_fp8_model_returns_false_for_unsupported_model_type(self):
+ """Test _is_fp8_model returns False for model types not in _FP8_SUPPORTED_MODEL_TYPES."""
+ from auto_round.utils.model import _is_fp8_model
+
+ with patch("transformers.AutoConfig.from_pretrained") as mock_config:
+ mock_config_obj = MagicMock()
+ mock_config_obj.model_type = "opt" # Not in _FP8_SUPPORTED_MODEL_TYPES
+ mock_config_obj.quantization_config = {"quant_method": "fp8"}
+ mock_config.return_value = mock_config_obj
+
+ result = _is_fp8_model("test/model", trust_remote_code=True)
+ assert result is False
+
+ def test_fp8_model_returns_false_when_no_quant_config(self):
+ """Test _is_fp8_model returns False when quantization_config is None."""
+ from auto_round.utils.model import _is_fp8_model
+
+ with patch("transformers.AutoConfig.from_pretrained") as mock_config:
+ mock_config_obj = MagicMock()
+ mock_config_obj.model_type = "deepseek_v32"
+ mock_config_obj.quantization_config = None
+ mock_config.return_value = mock_config_obj
+
+ result = _is_fp8_model("test/model", trust_remote_code=True)
+ assert result is False
+
+ def test_fp8_model_returns_true_for_valid_config(self):
+ """Test _is_fp8_model returns True for a valid FP8 checkpoint (deepseek_v32 + fp8)."""
+ from auto_round.utils.model import _is_fp8_model
+
+ with patch("transformers.AutoConfig.from_pretrained") as mock_config:
+ mock_config_obj = MagicMock()
+ mock_config_obj.model_type = "deepseek_v32"
+ mock_config_obj.quantization_config = {"quant_method": "fp8"}
+ mock_config.return_value = mock_config_obj
+
+ result = _is_fp8_model("test/model", trust_remote_code=True)
+ assert result is True
+
+ def test_fp8_model_swallows_config_load_errors(self):
+ """Test _is_fp8_model returns False (not raises) when AutoConfig.from_pretrained fails."""
+ from auto_round.utils.model import _is_fp8_model
+
+ with patch("transformers.AutoConfig.from_pretrained", side_effect=OSError("network error")):
+ result = _is_fp8_model("bad/path", trust_remote_code=True)
+ assert result is False
+
+
class TestGetNestedAttr:
"""Test get_nested_attr function."""