PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: add support for FP8 model loading and ragged-tile dequantization fallback (#2158)
Xin He40635d0273a6
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 7d5dd76f4..a32e8dc03 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -41,6 +41,7 @@ from auto_round.algorithms.config_resolver import (resolve_shared_config_values,split_quantization_configs,)+from auto_round.algorithms.utils import _has_nvfp4_layerfrom auto_round.logger import loggerfrom auto_round.utils import clear_memoryfrom auto_round.utils.device_manager import device_manager@@ -202,8 +203,9 @@ class AlgorithmComposer:):logger.info("Block-forward torch.compile is disabled because an enabled rotation is incompatible.")- if "nv_fp" in orchestrator.data_type:+ if _has_nvfp4_layer(orchestrator):can_compile_block_forward = False+ logger.info("Block-forward torch.compile is disabled because at least one quantized layer uses NVFP4.")# Bind compressor-level infrastructure (set before _build_quantizer is called).self.block_forward = (@@ -223,6 +225,7 @@ class AlgorithmComposer:Returns a list of hook handles that the caller must remove when done."""+ from auto_round.compressors.utils import is_nv_fpfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_sizeis_act_nv_fp = getattr(self.block_quantizer.config, "is_act_nv_fp", False)@@ -231,16 +234,18 @@ class AlgorithmComposer:input = input[0] if isinstance(input, (tuple, list)) else inputif input.numel() == 0:return+ module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)+ is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else is_act_nv_fpinput, _, _ = reshape_pad_tensor_by_group_size(input, module.act_group_size)act_max = torch.max(torch.abs(input), dim=-1).valuesif not hasattr(module, "act_max") or module.act_max.numel() == 0:module.act_max = act_max- if is_act_nv_fp:+ if is_module_act_nv_fp:max_val = act_max.max()module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_valreturnact_max = act_max.to(module.act_max.device)- if is_act_nv_fp:+ if is_module_act_nv_fp:max_val = torch.max(act_max.max(), module.act_max.max())module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_valelse:diff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.pyindex ac896183c..5ef7e1d71 100644--- a/auto_round/algorithms/quantization/utils.py+++ b/auto_round/algorithms/quantization/utils.py@@ -14,7 +14,7 @@import torch-from auto_round.compressors.utils import check_need_act_calibration+from auto_round.compressors.utils import check_need_act_calibration, is_nv_fpfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized@@ -26,16 +26,18 @@ def register_act_max_hooks(quantizer, model):input = input[0]if input.numel() == 0:return+ module_act_data_type = getattr(module, "act_data_type", None) or getattr(module, "data_type", None)+ is_module_act_nv_fp = is_nv_fp(module_act_data_type) if module_act_data_type else quantizer.config.is_act_nv_fpinput, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)act_max = torch.max(torch.abs(input), dim=-1).valuesif not hasattr(module, "act_max") or module.act_max.numel() == 0:module.act_max = act_max- if quantizer.config.is_act_nv_fp:+ if is_module_act_nv_fp:max_val = act_max.max()module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_valelse:act_max = act_max.to(module.act_max.device)- if quantizer.config.is_act_nv_fp:+ if is_module_act_nv_fp:max_val = torch.max(act_max.max(), module.act_max.max())module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_valelse:diff --git a/auto_round/algorithms/utils.py b/auto_round/algorithms/utils.pynew file mode 100644index 000000000..76ce0d304--- /dev/null+++ b/auto_round/algorithms/utils.py@@ -0,0 +1,48 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Shared utilities for the algorithms package."""++from __future__ import annotations++from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from auto_round.algorithms.base import BaseOrchestrator+++def _is_nvfp4_value(value: Any) -> bool:+ """Return True when a raw config value indicates NVFP4."""+ if not isinstance(value, str):+ return False+ value = value.lower()+ return "nv_fp" in value or "nvfp4" in value+++def _has_nvfp4_layer(orchestrator: "BaseOrchestrator") -> bool:+ """Whether global or per-layer config enables any NVFP4 quantization."""+ if _is_nvfp4_value(getattr(orchestrator, "data_type", "")):+ return True++ layer_config = getattr(orchestrator, "layer_config", None)+ if not isinstance(layer_config, dict):+ return False++ for config in layer_config.values():+ if not isinstance(config, dict):+ continue+ if _is_nvfp4_value(config.get("data_type")) or _is_nvfp4_value(config.get("act_data_type")):+ return True+ if _is_nvfp4_value(config.get("scheme")):+ return True+ return Falsediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 8183dd0e9..0b0695c9a 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -534,8 +534,58 @@ class BaseOrchestrator(object):):return True+ # Layer-level scheme overrides can request static-activation paths+ # (e.g., global MXFP8 + local NVFP4 experts). Those still need+ # calibration data even when top-level scheme looks dynamic.+ if self._layer_config_needs_calibration(check_need_act_calibration):+ return True+return False+ def _layer_config_needs_calibration(self, check_need_act_calibration) -> bool:+ """Return True if any raw layer_config entry implies activation calibration."""+ layer_cfg = self.layer_config+ if not isinstance(layer_cfg, dict) or not layer_cfg:+ return False++ def _entry_needs_calibration(entry) -> bool:+ if entry is None:+ return False++ candidates = []+ if isinstance(entry, (str, QuantizationScheme)):+ candidates.append(entry)+ elif isinstance(entry, dict):+ if "scheme" in entry:+ candidates.append(entry.get("scheme"))+ candidates.append(entry)+ else:+ return False++ for candidate in candidates:+ if candidate is None:+ continue+ try:+ _, _, attrs = parse_scheme(candidate, {})+ except Exception: # noqa: BLE001+ continue++ cand_act_bits = attrs.get("act_bits")+ cand_act_data_type = attrs.get("act_data_type")+ cand_act_dynamic = attrs.get("act_dynamic")+ is_cand_act_quant = cand_act_bits is not None and cand_act_bits <= 8+ if is_cand_act_quant and check_need_act_calibration(+ cand_act_dynamic,+ cand_act_data_type,+ cand_act_bits if cand_act_bits is not None else 16,+ static_kv_dtype=self.static_kv_dtype,+ static_attention_dtype=self.static_attention_dtype,+ ):+ return True+ return False++ return any(_entry_needs_calibration(v) for v in layer_cfg.values())+# ── Convenience properties ────────────────────────────────────────────────@propertydiff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 7e4be30db..c412b2445 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -20,6 +20,7 @@ from typing import List, Unionimport torchfrom torch.nn import Linear, Module+from auto_round import envsfrom auto_round.compressors.utils import is_nv_fpfrom auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEfrom auto_round.utils import check_to_quantized, logger@@ -442,9 +443,12 @@ def update_fused_layer_global_scales(For MLP layers:- gate_proj and up_proj share a single global scale.- This behavior is currently required by vLLM and may become optional- in the future.+ Set ``AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE=0`` to retain per-projection+ global scales. The default keeps scales compatible with vLLM fused kernels."""+ if not envs.AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE:+ return+global_scale_name = f"{base_name}_global_scale"def _collect_scales(mods: List[Module]) -> List[torch.Tensor]:@@ -459,14 +463,14 @@ def update_fused_layer_global_scales(return scalesdef _is_attention_module(module: Module):- return "attention" in module.__class__.__name__.lower() and (- hasattr(module, "k_proj") or hasattr(module, "v_proj") or hasattr(module, "qkv_proj")- )+ return all(hasattr(module, projection) for projection in ("q_proj", "k_proj", "v_proj"))def _is_mlp_module(module: Module):- return "mlp" in module.__class__.__name__.lower() and (- hasattr(module, "gate_proj") and hasattr(module, "up_proj")- )+ return all(hasattr(module, projection) for projection in ("gate_proj", "up_proj"))++ def _is_moe_expert_module(module: Module):+ """Check for MoE expert naming: w1 (gate) and w3 (up)."""+ return all(hasattr(module, projection) for projection in ("w1", "w3"))def _update_global_scales(modules: List[Module]):"""Update global scales for a list of modules."""@@ -487,28 +491,39 @@ def update_fused_layer_global_scales(# ---------------- Attention ----------------if _is_attention_module(submodule):- # Already fused- if hasattr(submodule, "qkv_proj"):- return_update_global_scales([submodule.q_proj, submodule.k_proj, submodule.v_proj])return# ---------------- MLP ----------------if _is_mlp_module(submodule):_update_global_scales([submodule.gate_proj, submodule.up_proj])+ return+ # ---------------- MoE Expert (w1/w3) ----------------+ if _is_moe_expert_module(submodule):+ _update_global_scales([submodule.w1, submodule.w3])-def update_block_global_scale_if_needed(block, data_type, group_size):- if not is_nv_fp(data_type):- return+def update_block_global_scale_if_needed(block, data_type, group_size):from auto_round.data_type.nvfp import calculate_gparam+ has_nvfp = is_nv_fp(data_type)+# Calculate block wise weight global scalefor _, m in block.named_modules():- if check_to_quantized(m) and not hasattr(m, "weight_global_scale"):- weight_global_scale = calculate_gparam(m.weight, group_size)- setattr(m, "weight_global_scale", weight_global_scale)+ if not check_to_quantized(m):+ continue+ # Check per-layer data_type for mixed-scheme scenarios+ module_data_type = getattr(m, "data_type", data_type)+ module_group_size = getattr(m, "group_size", group_size)+ if is_nv_fp(module_data_type):+ has_nvfp = True+ if not hasattr(m, "weight_global_scale"):+ weight_global_scale = calculate_gparam(m.weight, module_group_size)+ setattr(m, "weight_global_scale", weight_global_scale)++ if not has_nvfp:+ return# Update fused layer global scalesfor module in block.modules():diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 7c727443c..2401c8ce2 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -31,6 +31,8 @@ if TYPE_CHECKING:AR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = FalseAR_DISK_STREAM_MODEL: bool = FalseAR_RESUME_DIR: Optional[str] = None+ AR_FORCE_MOE_ROUTING_ALL_EXPERTS: bool = False+ AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = Truedef _get_optional_positive_int_env(name: str) -> Optional[int]:@@ -123,6 +125,14 @@ environment_variables: dict[str, Callable[[], Any]] = {# instead of restarting the whole tuning pass from block 0 after a# crash/kill. See auto_round/utils/resume.py."AR_RESUME_DIR": lambda: os.getenv("AR_RESUME_DIR", None),+ # When enabled, MoE routing can be overridden in selected model wrappers+ # to rotate token assignments across all experts for calibration coverage.+ "AR_FORCE_MOE_ROUTING_ALL_EXPERTS": lambda: os.getenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "0").lower()+ in ("1", "true", "yes"),+ # vLLM fused kernels require q/k/v and gate/up projections to use one+ # weight global scale. Disable only for runtimes without that requirement.+ "AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE": lambda: os.getenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "1").lower()+ not in ("0", "false", "no", "off"),}diff --git a/auto_round/modeling/fp8_quant.py b/auto_round/modeling/fp8_quant.pyindex a5490c844..3f4a25057 100644--- a/auto_round/modeling/fp8_quant.py+++ b/auto_round/modeling/fp8_quant.py@@ -93,12 +93,78 @@ def oot_replace_with_fp8_linear(_orig_validate_environment = OriginalFineGrainedFP8HfQuantizer.validate_environment+def _patch_fp8_dequantize_for_ragged_tiles():+ """Patch transformers FP8 dequantization to handle over-provisioned scale grids.++ Some pre-quantized checkpoints keep FP8 weights in non-padded shape while the+ corresponding ``weight_scale_inv`` grid is computed on padded block tiles.+ Example: weight ``(576, 7168)`` with scale grid ``(5, 56)`` for block size+ ``(128, 128)`` (5 rows of tiles implies padded rows to 640).++ Upstream strict divisibility checks fail on those tensors. We keep upstream+ behavior by default and only fall back on this specific shape mismatch.+ """++ try:+ import transformers.integrations.finegrained_fp8 as transformers_fp8++ from auto_round.utils.weight_handler import _dequant_fp8_linear_weight+ except ImportError:+ return++ if not hasattr(transformers_fp8, "Fp8Dequantize"):+ return++ cls = transformers_fp8.Fp8Dequantize+ if hasattr(cls, "_auto_round_orig_dequantize_one"):+ return++ if not hasattr(cls, "_dequantize_one"):+ return++ cls._auto_round_orig_dequantize_one = cls._dequantize_one++ def _dequantize_one_with_padding_fallback(self, quantized, scales, output_dtype=None):+ try:+ return cls._auto_round_orig_dequantize_one(self, quantized, scales, output_dtype=output_dtype)+ except ValueError as e:+ if "not divisible by scale grid" not in str(e):+ raise++ # Match upstream MXFP8 handling for uint8 E8M0 scales.+ if scales.dtype == torch.uint8:+ scales = (scales.to(torch.float32) - 127.0).exp2()++ block_size = None+ qcfg = getattr(self.hf_quantizer, "quantization_config", None)+ if qcfg is not None:+ if isinstance(qcfg, dict):+ block_size = qcfg.get("weight_block_size")+ else:+ block_size = getattr(qcfg, "weight_block_size", None)++ # Falls back to AutoRound's robust dequantizer that supports+ # over-provisioned scale grids by padding then slicing back.+ dequantized = _dequant_fp8_linear_weight(quantized, scales, block_size=block_size)+ if output_dtype is not None:+ dequantized = dequantized.to(output_dtype)+ return dequantized++ cls._dequantize_one = _dequantize_one_with_padding_fallback+ auto_round_logger.debug("Applied FP8 ragged-tile dequantization fallback patch.")++@override_cuda_device_capability()def oot_validate_environment(self, *args, **kwargs):return _orig_validate_environment(self, *args, **kwargs)def apply_fp8_expert_replacement_patch():+ # Apply the ragged-tile dequantization fallback unconditionally so that+ # dequantize-on-load (FineGrainedFP8Config(dequantize=True)) works on+ # CPU and HPU as well as CUDA.+ _patch_fp8_dequantize_for_ragged_tiles()+if is_transformers_version_greater_or_equal_5() and torch.cuda.is_available():try:import transformers.integrations.finegrained_fp8 as transformers_fp8diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.pyindex 3cf25784e..cc856c5c5 100644--- a/auto_round/modeling/fused_moe/moe_experts_interface.py+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py@@ -35,6 +35,7 @@ import torchfrom torch import nnfrom auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec+from auto_round.modeling.fused_moe.utils import build_forced_routing, force_all_experts_routing_enabledfrom auto_round.utils import clear_memory, loggerfrom auto_round.utils.device import memory_monitor@@ -217,73 +218,81 @@ def linear_loop_experts_forward(num_tokens = hidden_states.size(0)num_experts = self.num_experts- # Reshape for easier indexing- # S is the number of selected token-expert pairs (S = num_tokens * num_top_k)- token_idx = torch.arange(num_tokens, device=device).unsqueeze(1).expand(-1, num_top_k).reshape(-1) # (S,)- sample_weights = top_k_weights.reshape(-1).to(hidden_states.dtype) # (S,)- expert_ids = top_k_index.reshape(-1) # (S,)-- # Get current hidden states for selected samples- selected_hidden_states = hidden_states[token_idx] # (S, hidden_dim)-- # Group token-expert pairs by expert using a single sort, then run each- # expert on a contiguous *static* slice, instead of doing a per-expert- # nonzero()/boolean-mask lookup inside the Python loop.- #- # Why: repeatedly issuing dynamic-shape gather kernels (nonzero(),- # index_select() or boolean-mask indexing) once per expert in a loop- # reliably triggers a driver-level bug on some XPU builds - either a- # "vectorized gather kernel index out of bounds" device-side assertion or- # a hard UR_RESULT_ERROR_DEVICE_LOST - even when every index is valid.- # This was confirmed with minimal PyTorch-only reproductions unrelated to- # this model/library, so working around it in application code is the- # practical fix. Sorting once needs only two dynamic gather/scatter calls- # in total (regardless of num_experts): one index_select to group tokens- # by expert, and one index_copy_ to scatter results back to their- # original positions. Each expert's slice `permuted[start:end]` is a- # plain view (no gather kernel involved), so this is also efficient.- sort_order = torch.argsort(expert_ids)- permuted_hidden_states = selected_hidden_states.index_select(0, sort_order) # (S, hidden_dim)-- # Per-expert token counts, computed once on host to drive static slicing.- counts = torch.bincount(expert_ids, minlength=num_experts).tolist()-- out_permuted = torch.zeros_like(permuted_hidden_states)- start = 0- for expert_idx, count in enumerate(counts):- if count == 0:- continue- end = start + count- expert_input = permuted_hidden_states[start:end] # static slice/view, no gather kernel-- # Get this expert's container with its projection layers- expert = getattr(self, str(expert_idx))- gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)- up_out = expert.up_proj(expert_input) # (num_samples, intermediate_dim)-- # Apply gating- if hasattr(self, "_apply_gate"):- gate_up_out = torch.cat([gate_out, up_out], dim=-1)- gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)- else:- gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)-- # Down projection- expert_out = expert.down_proj(gated_out) # (num_samples, hidden_dim)-- out_permuted[start:end] = expert_out.to(out_permuted.dtype)- start = end-- # Scatter results back to their original (pre-sort) token-expert order.- out_per_sample = torch.empty_like(out_permuted)- out_per_sample.index_copy_(0, sort_order, out_permuted)-- # Apply routing weights- out_per_sample = out_per_sample * sample_weights.unsqueeze(-1) # (S, hidden_dim)-- # Accumulate results using deterministic reshape+sum instead of index_add_- # (index_add_ with duplicate indices is non-deterministic on CUDA due to atomicAdd)- final_hidden_states = out_per_sample.view(num_tokens, num_top_k, hidden_dim).sum(dim=1)+ def _run_experts_with_routes(route_indices: torch.Tensor, route_weights: torch.Tensor) -> torch.Tensor:+ # Reshape for easier indexing+ # S is the number of selected token-expert pairs (S = num_tokens * num_top_k)+ token_idx = torch.arange(num_tokens, device=device).unsqueeze(1).expand(-1, num_top_k).reshape(-1) # (S,)+ sample_weights = route_weights.reshape(-1).to(hidden_states.dtype) # (S,)+ expert_ids = route_indices.reshape(-1) # (S,)++ # Get current hidden states for selected samples+ selected_hidden_states = hidden_states[token_idx] # (S, hidden_dim)++ # Group token-expert pairs by expert using a single sort, then run each+ # expert on a contiguous *static* slice, instead of doing a per-expert+ # nonzero()/boolean-mask lookup inside the Python loop.+ sort_order = torch.argsort(expert_ids)+ permuted_hidden_states = selected_hidden_states.index_select(0, sort_order) # (S, hidden_dim)++ # Per-expert token counts, computed once on host to drive static slicing.+ counts = torch.bincount(expert_ids, minlength=num_experts).tolist()++ out_permuted = torch.zeros_like(permuted_hidden_states)+ start = 0+ for expert_idx, count in enumerate(counts):+ if count == 0:+ continue+ end = start + count+ expert_input = permuted_hidden_states[start:end] # static slice/view, no gather kernel++ # Get this expert's container with its projection layers+ expert = getattr(self, str(expert_idx))+ gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)+ up_out = expert.up_proj(expert_input) # (num_samples, intermediate_dim)++ # Apply gating+ if hasattr(self, "_apply_gate"):+ gate_up_out = torch.cat([gate_out, up_out], dim=-1)+ gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)+ else:+ gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)++ # Down projection+ expert_out = expert.down_proj(gated_out) # (num_samples, hidden_dim)++ out_permuted[start:end] = expert_out.to(out_permuted.dtype)+ start = end++ # Scatter results back to their original (pre-sort) token-expert order.+ out_per_sample = torch.empty_like(out_permuted)+ out_per_sample.index_copy_(0, sort_order, out_permuted)++ # Apply routing weights+ out_per_sample = out_per_sample * sample_weights.unsqueeze(-1) # (S, hidden_dim)++ # Accumulate results using deterministic reshape+sum instead of index_add_+ # (index_add_ with duplicate indices is non-deterministic on CUDA due to atomicAdd)+ return out_per_sample.view(num_tokens, num_top_k, hidden_dim).sum(dim=1)++ # Main output path: always preserve original router selection.+ final_hidden_states = _run_experts_with_routes(top_k_index, top_k_weights)++ # Auxiliary coverage path: when enabled, run forced-routing once to trigger+ # expert hooks/stat collection, but discard this output to avoid changing+ # model semantics.+ if force_all_experts_routing_enabled():+ forced_indices, forced_weights = build_forced_routing(+ module=self,+ routing_scores=None,+ top_k=num_top_k,+ num_experts=num_experts,+ dtype=hidden_states.dtype,+ num_tokens=num_tokens,+ device=device,+ normalize=True,+ )+ with torch.no_grad():+ _ = _run_experts_with_routes(forced_indices, forced_weights)# Reshape back to original format if input was [batch_size, seq_len, hidden_dim]if batch_size is not None:diff --git a/auto_round/modeling/fused_moe/qwen3_5_moe.py b/auto_round/modeling/fused_moe/qwen3_5_moe.pyindex eb0a2eea3..e96c3224e 100644--- a/auto_round/modeling/fused_moe/qwen3_5_moe.py+++ b/auto_round/modeling/fused_moe/qwen3_5_moe.py@@ -8,7 +8,12 @@ from transformers.utils.versions import require_versionfrom auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_specfrom auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase-from auto_round.modeling.fused_moe.utils import _update_parameter, sequential_moe_forward+from auto_round.modeling.fused_moe.utils import (+ _update_parameter,+ build_forced_routing,+ force_all_experts_routing_enabled,+ sequential_moe_forward,+)from auto_round.utils import clear_memory, unsupported_meta_devicerequire_version("transformers>=5.2.0")@@ -43,7 +48,26 @@ class LinearQwen3_5MoeSparseMoeBlock(ReplacementModuleBase):batch_size, sequence_length, hidden_dim = hidden_states.shapehidden_states_reshaped = hidden_states.view(-1, hidden_dim)shared_expert_output = self.shared_expert(hidden_states_reshaped)- _, routing_weights, selected_experts = self.gate(hidden_states_reshaped)+ router_logits, routing_weights, selected_experts = self.gate(hidden_states_reshaped)+ if force_all_experts_routing_enabled():+ routing_scores = (+ torch.softmax(router_logits, dim=-1, dtype=torch.float)+ if router_logits is not None+ else torch.full(+ (hidden_states_reshaped.size(0), self.num_experts),+ 1.0 / float(self.num_experts),+ device=hidden_states_reshaped.device,+ dtype=torch.float,+ )+ )+ selected_experts, routing_weights = build_forced_routing(+ module=self,+ routing_scores=routing_scores,+ top_k=routing_weights.size(-1),+ num_experts=self.num_experts,+ dtype=hidden_states_reshaped.dtype,+ normalize=True,+ )expert_output = self.experts_forward(hidden_states_reshaped, selected_experts, routing_weights)shared_expert_output = F.sigmoid(self.shared_expert_gate(hidden_states_reshaped)) * shared_expert_outputdiff --git a/auto_round/modeling/fused_moe/utils.py b/auto_round/modeling/fused_moe/utils.pyindex 05b1ad3d6..2befcbafc 100644--- a/auto_round/modeling/fused_moe/utils.py+++ b/auto_round/modeling/fused_moe/utils.py@@ -15,6 +15,8 @@import torchimport torch.nn as nn+from auto_round import envs+def _update_parameter(module: torch.nn.Module,@@ -125,3 +127,85 @@ def get_num_experts(original: torch.nn.Module) -> int:"Unsupported MoE experts layout: expected fused gate_up_proj/down_proj ""or linearized gate_proj/up_proj/down_proj experts")+++def force_all_experts_routing_enabled() -> bool:+ """Return whether MoE wrappers should override router top-k for coverage."""+ return envs.AR_FORCE_MOE_ROUTING_ALL_EXPERTS+++def build_forced_expert_indices(+ *,+ module: torch.nn.Module,+ num_tokens: int,+ top_k: int,+ num_experts: int,+ device: torch.device,+) -> torch.Tensor:+ """Build deterministic rotating expert indices to cover all experts over time."""+ if num_tokens <= 0:+ raise ValueError("num_tokens must be > 0")+ if top_k <= 0:+ raise ValueError("top_k must be > 0")+ if num_experts <= 0:+ raise ValueError("num_experts must be > 0")++ top_k = min(top_k, num_experts)+ total_pairs = num_tokens * top_k+ offset = int(getattr(module, "_auto_round_forced_route_offset", 0))+ forced = (torch.arange(total_pairs, device=device) + offset) % num_experts+ forced = forced.view(num_tokens, top_k)+ module._auto_round_forced_route_offset = (offset + total_pairs) % num_experts+ return forced+++def build_forced_routing(+ *,+ module: torch.nn.Module,+ routing_scores: torch.Tensor | None,+ top_k: int,+ num_experts: int,+ dtype: torch.dtype,+ num_tokens: int | None = None,+ device: torch.device | None = None,+ normalize: bool = True,+ scaling_factor: float = 1.0,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Create forced top-k indices and matching routing weights.++ When scores are unavailable, fall back to uniform weights.+ """+ if routing_scores is None:+ if num_tokens is None or device is None:+ raise ValueError("num_tokens and device are required when routing_scores is None")+ else:+ num_tokens = int(routing_scores.shape[0])+ device = routing_scores.device++ assert num_tokens is not None+ assert device is not None+ forced_indices = build_forced_expert_indices(+ module=module,+ num_tokens=num_tokens,+ top_k=top_k,+ num_experts=num_experts,+ device=device,+ )++ if routing_scores is not None:+ weights = routing_scores.gather(1, forced_indices)+ else:+ denom = float(min(top_k, num_experts))+ weights = torch.full(+ (num_tokens, min(top_k, num_experts)),+ 1.0 / denom,+ device=forced_indices.device,+ dtype=torch.float32,+ )++ if normalize:+ weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20)+ if scaling_factor != 1.0:+ weights = weights * scaling_factor++ return forced_indices, weights.to(dtype)diff --git a/auto_round/modeling/unfused_moe/deepseek_v3.py b/auto_round/modeling/unfused_moe/deepseek_v3.pyindex cfcc617a6..9017bb095 100644--- a/auto_round/modeling/unfused_moe/deepseek_v3.py+++ b/auto_round/modeling/unfused_moe/deepseek_v3.py@@ -15,7 +15,11 @@import torchimport torch.nn as nn-from auto_round.modeling.fused_moe.utils import sequential_moe_forward+from auto_round.modeling.fused_moe.utils import (+ build_forced_routing,+ force_all_experts_routing_enabled,+ sequential_moe_forward,+)class LinearDeepseekV3MoE(nn.Module):@@ -53,6 +57,17 @@ class LinearDeepseekV3MoE(nn.Module):def route_tokens_to_experts(self, router_logits):router_logits = router_logits.sigmoid()+ if force_all_experts_routing_enabled():+ return build_forced_routing(+ module=self,+ routing_scores=router_logits,+ top_k=self.top_k,+ num_experts=self.num_experts,+ dtype=router_logits.dtype,+ normalize=self.norm_topk_prob,+ scaling_factor=self.routed_scaling_factor,+ )+router_logits_for_choice = router_logits + self.gate.e_score_correction_biasgroup_scores = (router_logits_for_choice.view(-1, self.n_group, self.n_routed_experts // self.n_group)@@ -81,7 +96,17 @@ class LinearDeepseekV3MoE(nn.Module):orig_shape = hidden_states.shaperouter_logits = self.gate(hidden_states)if isinstance(router_logits, tuple): # transformers >= 5.13.0- _, topk_weights, topk_indices = router_logits+ base_router_logits, topk_weights, topk_indices = router_logits+ if force_all_experts_routing_enabled():+ topk_indices, topk_weights = build_forced_routing(+ module=self,+ routing_scores=base_router_logits.sigmoid(),+ top_k=self.top_k,+ num_experts=self.num_experts,+ dtype=hidden_states.dtype,+ normalize=self.norm_topk_prob,+ scaling_factor=self.routed_scaling_factor,+ )else:topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)hidden_states = hidden_states.view(-1, hidden_states.shape[-1])diff --git a/auto_round/modeling/unfused_moe/qwen3_moe.py b/auto_round/modeling/unfused_moe/qwen3_moe.pyindex d1e9d7a39..2c3921014 100644--- a/auto_round/modeling/unfused_moe/qwen3_moe.py+++ b/auto_round/modeling/unfused_moe/qwen3_moe.py@@ -16,7 +16,11 @@ import torchimport torch.nn as nnfrom torch.nn import functional as F-from auto_round.modeling.fused_moe.utils import sequential_moe_forward+from auto_round.modeling.fused_moe.utils import (+ build_forced_routing,+ force_all_experts_routing_enabled,+ sequential_moe_forward,+)class LinearQwen3MoeSparseMoeBlock(nn.Module):@@ -42,11 +46,21 @@ class LinearQwen3MoeSparseMoeBlock(nn.Module):router_logits = self.gate(hidden_states)routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)- routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)- if self.norm_topk_prob: # only diff with mixtral sparse moe block!- routing_weights /= routing_weights.sum(dim=-1, keepdim=True)- # we cast back to the input dtype- routing_weights = routing_weights.to(hidden_states.dtype)+ if force_all_experts_routing_enabled():+ selected_experts, routing_weights = build_forced_routing(+ module=self,+ routing_scores=routing_weights,+ top_k=self.top_k,+ num_experts=self.num_experts,+ dtype=hidden_states.dtype,+ normalize=self.norm_topk_prob,+ )+ else:+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)+ if self.norm_topk_prob: # only diff with mixtral sparse moe block!+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)+ # we cast back to the input dtype+ routing_weights = routing_weights.to(hidden_states.dtype)final_hidden_states = sequential_moe_forward(hidden_states, selected_experts, routing_weights, self.experts, self.num_expertsdiff --git a/auto_round/modeling/unfused_moe/qwen3_next.py b/auto_round/modeling/unfused_moe/qwen3_next.pyindex df0f7a960..3441d66da 100644--- a/auto_round/modeling/unfused_moe/qwen3_next.py+++ b/auto_round/modeling/unfused_moe/qwen3_next.py@@ -16,7 +16,11 @@ import torchimport torch.nn as nnfrom torch.nn import functional as F-from auto_round.modeling.fused_moe.utils import sequential_moe_forward+from auto_round.modeling.fused_moe.utils import (+ build_forced_routing,+ force_all_experts_routing_enabled,+ sequential_moe_forward,+)class LinearQwen3NextSparseMoeBlock(nn.Module):@@ -45,11 +49,21 @@ class LinearQwen3NextSparseMoeBlock(nn.Module):router_logits = self.gate(hidden_states)routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)- routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)- if self.norm_topk_prob:- routing_weights /= routing_weights.sum(dim=-1, keepdim=True)- # we cast back to the input dtype- routing_weights = routing_weights.to(hidden_states.dtype)+ if force_all_experts_routing_enabled():+ selected_experts, routing_weights = build_forced_routing(+ module=self,+ routing_scores=routing_weights,+ top_k=self.top_k,+ num_experts=self.num_experts,+ dtype=hidden_states.dtype,+ normalize=self.norm_topk_prob,+ )+ else:+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)+ if self.norm_topk_prob:+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)+ # we cast back to the input dtype+ routing_weights = routing_weights.to(hidden_states.dtype)final_hidden_states = sequential_moe_forward(hidden_states, selected_experts, routing_weights, self.experts, self.num_expertsdiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 758fc466b..8901e52ef 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -289,6 +289,7 @@ def _check_accelerate_version():_MXFP4_SUPPORTED_MODEL_TYPES = {"gpt_oss"}+_FP8_SUPPORTED_MODEL_TYPES = {"deepseek_v32"}def _is_mxfp4_model(model_path, trust_remote_code=True):@@ -320,6 +321,38 @@ def _is_mxfp4_model(model_path, trust_remote_code=True):return quant_method == "mxfp4" and model_type in _MXFP4_SUPPORTED_MODEL_TYPES+def _is_fp8_model(model_path, trust_remote_code=True):+ """Check if a model is an FP8 quantized model supported for direct loading.++ Only checks when transformers >= 4.56.0. Returns False immediately for older versions,+ adding zero overhead to non-FP8 model loading.+ """+ if version.parse(transformers.__version__) < version.parse("4.56.0"):+ return False++ from transformers import AutoConfig++ try: # in case of config loading failure for new models+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)+ except Exception:+ return False++ model_type = getattr(config, "model_type", "")+ if model_type not in _FP8_SUPPORTED_MODEL_TYPES:+ return False++ quant_config = getattr(config, "quantization_config", None)+ if quant_config is None:+ return False++ quant_method = (+ quant_config.get("quant_method", "")+ if isinstance(quant_config, dict)+ else getattr(quant_config, "quant_method", "")+ )+ return quant_method == "fp8" and model_type in _FP8_SUPPORTED_MODEL_TYPES++def llm_load_model(pretrained_model_name_or_path: str,platform: str = "hf",@@ -373,6 +406,14 @@ def llm_load_model(}load_kwargs.update(kwargs)+ if version.parse(transformers.__version__) >= version.parse("4.56.0"):+ is_fp8 = _is_fp8_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)+ if is_fp8 and "quantization_config" not in load_kwargs:+ from transformers import FineGrainedFP8Config++ load_kwargs["quantization_config"] = FineGrainedFP8Config(dequantize=True)+ logger.info("Detected FP8 quantized model, using FineGrainedFP8Config(dequantize=True) for loading.")+if version.parse(transformers.__version__) >= version.parse("5.0.0"):is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)if is_mxfp4:diff --git a/docs/environments.md b/docs/environments.mdindex 47f36cb30..6d16b3130 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -30,6 +30,16 @@ export AR_LOG_LEVEL=DEBUGexport AR_ENABLE_COMPILE_PACKING=1```+### AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE+- **Description**: Makes fused NVFP4 weight projections use one shared weight global scale. This applies to `q_proj`/`k_proj`/`v_proj` and `gate_proj`/`up_proj`, as required by vLLM fused kernels.+- **Default**: `True` (equivalent to `"1"`)+- **Valid Values**: `"0"`, `"false"`, `"no"`, or `"off"` (case-insensitive) disable sharing; any other value enables it.+- **Usage**: Disable only when exporting for a runtime that does not require fused projections to share a global scale.++```bash+export AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE=0+```+### AR_USE_MODELSCOPE- **Description**: Controls whether to use ModelScope for model downloads- **Default**: `False`diff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex 365b3b46d..8648b4d76 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -30,6 +30,16 @@ export AR_LOG_LEVEL=DEBUGexport AR_ENABLE_COMPILE_PACKING=1```+### AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE+- **描述**:让 fused NVFP4 权重投影共用一个 weight global scale。该设置作用于 `q_proj`/`k_proj`/`v_proj` 与 `gate_proj`/`up_proj`,以满足 vLLM fused kernel 的要求。+- **默认值**:`True`(等价于 `"1"`)+- **有效值**:`"0"`、`"false"`、`"no"` 或 `"off"`(不区分大小写)表示关闭共享;其他值表示启用。+- **用途**:仅当导出的运行时不要求 fused 投影使用统一 global scale 时关闭。++```bash+export AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE=0+```+### AR_USE_MODELSCOPE- **描述**:控制是否使用 ModelScope 下载模型- **默认值**:`False`diff --git a/test/unit/test_cpu/core/test_pipeline_fail_fast.py b/test/unit/test_cpu/core/test_pipeline_fail_fast.pyindex ceb1f8ce3..f49fef09a 100644--- a/test/unit/test_cpu/core/test_pipeline_fail_fast.py+++ b/test/unit/test_cpu/core/test_pipeline_fail_fast.py@@ -1,10 +1,12 @@"""Fast unit tests for algorithm registry and bundle construction."""+from types import SimpleNamespace+import pytestfrom auto_round import AutoRound as NewAutoRoundfrom auto_round import AWQConfig, OptimizedRTNConfig, RotationConfig, RTNConfig, SignRoundConfig, SpinQuantConfig-from auto_round.algorithms.composer import AlgorithmComposer, _can_compile_block_forward+from auto_round.algorithms.composer import AlgorithmComposer, _can_compile_block_forward, _has_nvfp4_layerfrom auto_round.algorithms.config_resolver import (get_algorithm_class,resolve_shared_config_values,@@ -70,6 +72,37 @@ def test_hadamard_disables_only_block_forward_compile():assert not _can_compile_block_forward(quantizer, [], user_enabled=False)+def test_detect_nvfp4_from_layer_config_scheme_override():+ orchestrator = SimpleNamespace(+ data_type="mx_fp",+ layer_config={"mlp.experts": {"scheme": "NVFP4"}},+ )+ assert _has_nvfp4_layer(orchestrator)+++def test_detect_nvfp4_from_layer_config_data_type_override():+ orchestrator = SimpleNamespace(+ data_type="mx_fp",+ layer_config={"mlp.experts": {"data_type": "nv_fp"}},+ )+ assert _has_nvfp4_layer(orchestrator)+++def test_needs_calibration_data_when_layer_config_overrides_to_nvfp4():+ class _DummyCompressor:+ _needs_calibration_data = Compressor._needs_calibration_data+ _layer_config_needs_calibration = Compressor._layer_config_needs_calibration++ compressor = _DummyCompressor()+ compressor._alg_configs = []+ compressor.scheme = "MXFP8"+ compressor.layer_config = {"mlp.experts": {"scheme": "NVFP4"}}+ compressor.static_kv_dtype = None+ compressor.static_attention_dtype = None++ assert compressor._needs_calibration_data() is True++def test_registry_builtin_aliases_and_unknown():assert isinstance(_r.resolve_alg_config("RTN"), RTNConfig)assert isinstance(_r.resolve_alg_config("awq"), AWQConfig)diff --git a/test/unit/test_cpu/data_type/test_nvfp.py b/test/unit/test_cpu/data_type/test_nvfp.pyindex 444a14aad..930dd0a28 100644--- a/test/unit/test_cpu/data_type/test_nvfp.py+++ b/test/unit/test_cpu/data_type/test_nvfp.py@@ -17,6 +17,7 @@ import mathimport pytestimport torch+from torch import nnfrom auto_round.data_type.nvfp import (FLOAT4_E2M1_MAX,@@ -38,6 +39,7 @@ from auto_round.data_type.nvfp import (ref_nvfp4_quant,search_nvfp4_scale,)+from auto_round.data_type.utils import update_fused_layer_global_scales# ---------------------------------------------------------------------------# Constants@@ -177,6 +179,52 @@ class TestCalculateGparam:assert g.item() == pytest.approx(expected, rel=1e-5)+# ---------------------------------------------------------------------------+# fused global scale+# ---------------------------------------------------------------------------+++class TestFusedLayerGlobalScales:+ @staticmethod+ def _projection(scale: float) -> nn.Module:+ projection = nn.Module()+ projection.weight_global_scale = torch.tensor([scale])+ return projection++ def test_fused_projections_share_minimum_scale_by_default(self, monkeypatch):+ monkeypatch.delenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", raising=False)+ attention = nn.Module()+ attention.q_proj = self._projection(3.0)+ attention.k_proj = self._projection(1.0)+ attention.v_proj = self._projection(2.0)+ mlp = nn.Module()+ mlp.gate_proj = self._projection(4.0)+ mlp.up_proj = self._projection(0.5)++ update_fused_layer_global_scales(attention)+ update_fused_layer_global_scales(mlp)++ assert all(+ proj.weight_global_scale.item() == 1.0 for proj in (attention.q_proj, attention.k_proj, attention.v_proj)+ )+ assert all(proj.weight_global_scale.item() == 0.5 for proj in (mlp.gate_proj, mlp.up_proj))++ def test_fused_projection_scale_update_can_be_disabled(self, monkeypatch):+ monkeypatch.setenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "0")+ attention = nn.Module()+ attention.q_proj = self._projection(3.0)+ attention.k_proj = self._projection(1.0)+ attention.v_proj = self._projection(2.0)++ update_fused_layer_global_scales(attention)++ assert [proj.weight_global_scale.item() for proj in (attention.q_proj, attention.k_proj, attention.v_proj)] == [+ 3.0,+ 1.0,+ 2.0,+ ]++# ---------------------------------------------------------------------------# ref_nvfp4_quant# ---------------------------------------------------------------------------diff --git a/test/unit/test_cpu/models/test_moe_experts_interface.py b/test/unit/test_cpu/models/test_moe_experts_interface.pyindex 6ce9cf29e..f8c811985 100644--- a/test/unit/test_cpu/models/test_moe_experts_interface.py+++ b/test/unit/test_cpu/models/test_moe_experts_interface.py@@ -225,6 +225,47 @@ def test_linear_loop_forward():assert not torch.allclose(output, torch.zeros_like(output)), "Output is all zeros"+def test_linear_loop_force_routing_preserves_original_output(monkeypatch):+ """Forced expert coverage should not change the returned MoE output."""+ from auto_round.modeling.fused_moe.moe_experts_interface import (+ _unfuse_experts_weights_inplace,+ linear_loop_experts_forward,+ )++ num_experts = 4+ hidden_dim = 16+ intermediate_dim = 32+ num_tokens = 9+ top_k = 2++ class MockExperts(nn.Module):+ def __init__(self):+ super().__init__()+ self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))+ self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))+ self.act_fn = nn.SiLU()+ self.num_experts = num_experts+ self.has_bias = False+ self.is_transposed = False++ torch.manual_seed(7)+ module = MockExperts()+ _unfuse_experts_weights_inplace(module, check_decorator=False)++ hidden_states = torch.randn(num_tokens, hidden_dim)+ top_k_index = torch.randint(0, num_experts, (num_tokens, top_k))+ top_k_weights = torch.softmax(torch.randn(num_tokens, top_k), dim=-1)++ monkeypatch.setenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "0")+ output_normal = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)++ monkeypatch.setenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "1")+ output_forced = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)++ assert torch.allclose(output_normal, output_forced, atol=1e-6, rtol=1e-5)+ assert hasattr(module, "_auto_round_forced_route_offset")++def test_prepare_model_for_moe_quantization():"""Test the full prepare_model_for_moe_quantization flow."""from auto_round.modeling.fused_moe.moe_experts_interface import (diff --git a/test/unit/test_cpu/utils/test_model_utils.py b/test/unit/test_cpu/utils/test_model_utils.pyindex 17e7b6e54..8a49f9296 100644--- a/test/unit/test_cpu/utils/test_model_utils.py+++ b/test/unit/test_cpu/utils/test_model_utils.py@@ -289,6 +289,71 @@ class TestIsMxfp4Model:assert result is True+class TestIsFp8Model:+ """Test _is_fp8_model function."""++ def test_fp8_model_returns_false_for_old_transformers(self):+ """Test _is_fp8_model returns False immediately for transformers < 4.56.0."""+ from auto_round.utils.model import _is_fp8_model++ with patch("auto_round.utils.model.version") as mock_version:+ mock_version.parse.side_effect = lambda v: __import__("packaging.version", fromlist=["parse"]).parse(v)+ # Simulate old transformers by making the version check fail+ import packaging.version++ with patch("auto_round.utils.model.transformers") as mock_tf:+ mock_tf.__version__ = "4.50.0"+ result = _is_fp8_model("test/model", trust_remote_code=True)+ assert result is False++ def test_fp8_model_returns_false_for_unsupported_model_type(self):+ """Test _is_fp8_model returns False for model types not in _FP8_SUPPORTED_MODEL_TYPES."""+ from auto_round.utils.model import _is_fp8_model++ with patch("transformers.AutoConfig.from_pretrained") as mock_config:+ mock_config_obj = MagicMock()+ mock_config_obj.model_type = "opt" # Not in _FP8_SUPPORTED_MODEL_TYPES+ mock_config_obj.quantization_config = {"quant_method": "fp8"}+ mock_config.return_value = mock_config_obj++ result = _is_fp8_model("test/model", trust_remote_code=True)+ assert result is False++ def test_fp8_model_returns_false_when_no_quant_config(self):+ """Test _is_fp8_model returns False when quantization_config is None."""+ from auto_round.utils.model import _is_fp8_model++ with patch("transformers.AutoConfig.from_pretrained") as mock_config:+ mock_config_obj = MagicMock()+ mock_config_obj.model_type = "deepseek_v32"+ mock_config_obj.quantization_config = None+ mock_config.return_value = mock_config_obj++ result = _is_fp8_model("test/model", trust_remote_code=True)+ assert result is False++ def test_fp8_model_returns_true_for_valid_config(self):+ """Test _is_fp8_model returns True for a valid FP8 checkpoint (deepseek_v32 + fp8)."""+ from auto_round.utils.model import _is_fp8_model++ with patch("transformers.AutoConfig.from_pretrained") as mock_config:+ mock_config_obj = MagicMock()+ mock_config_obj.model_type = "deepseek_v32"+ mock_config_obj.quantization_config = {"quant_method": "fp8"}+ mock_config.return_value = mock_config_obj++ result = _is_fp8_model("test/model", trust_remote_code=True)+ assert result is True++ def test_fp8_model_swallows_config_load_errors(self):+ """Test _is_fp8_model returns False (not raises) when AutoConfig.from_pretrained fails."""+ from auto_round.utils.model import _is_fp8_model++ with patch("transformers.AutoConfig.from_pretrained", side_effect=OSError("network error")):+ result = _is_fp8_model("bad/path", trust_remote_code=True)+ assert result is False++class TestGetNestedAttr:"""Test get_nested_attr function."""