Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[risky]reduce ram usage of loading moe without monkey patch and fix qwen3-flash-next group_size issue (#2291)

Wenhua Cheng7d8905efd0de
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index b85b157c7..e7d4dae5f 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -2326,18 +2326,18 @@ def _load_scheme_worker_model(model_name, use_model_replacements, low_cpu_mem_us
def _load_disk_stream_scheme_worker_model(model_name, use_model_replacements=False):
"""Build an isolated meta model and checkpoint index for a scoring worker."""
+ from auto_round.modeling.fused_moe.replace_modules import _handle_moe_modules
from auto_round.utils.disk_stream_util import build_meta_model
+ # Build the meta skeleton, then structurally unfuse the fused ``transformers>=5`` MoE
+ # experts (fused 3D expert params -> per-expert ``nn.Linear``) while everything is
+ # still on meta, so per-expert quant layers resolve; weights stay on meta and are
+ # filled per-block from the checkpoint index during scoring. The unfuse must happen
+ # here in the worker: with the custom MoE replacements disabled, the meta-skeleton
+ # build no longer guarantees it, and leaving the experts fused makes the per-expert
+ # quant layers unresolvable during scoring.
model, tokenizer, disk_index = build_meta_model(model_name)
- # The regular load pipeline applies custom replacements, among them the
- # structural MoE unfusing that turns fused expert containers into per-expert
- # Linear modules; the meta-skeleton build skips that pipeline. Unfuse here
- # so per-expert quant layers resolve -- weights stay on meta and are filled
- # per-block from the checkpoint index during scoring.
- from auto_round.modeling.fused_moe.replace_modules import _handle_moe_modules
-
- unfused = _handle_moe_modules(model)
- logger.info("disk-stream scoring worker: structural MoE unfuse produced %d unfused experts modules", len(unfused))
+ _handle_moe_modules(model)
if use_model_replacements:
from auto_round.special_model_handler import _handle_special_model, update_module
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 562547469..b4c9437b2 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -451,6 +451,17 @@ class BaseOrchestrator(object):
# blocks already hold real weights (see OffloadManager._reload).
if self.model_context.disk_stream_model_dir is not None:
self._offloader.model_dir = self.model_context.disk_stream_model_dir
+ # A meta skeleton (explicit AR_DISK_STREAM_MODEL=1, or auto-selected for
+ # fused-MoE checkpoints -- signalled by `_disk_stream_index`) leaves every
+ # block on the meta device, so per-block reload from disk is mandatory: the
+ # tuning loop must materialize each block before moving it to the compute
+ # device. The offloader was created with `enabled=low_cpu_mem_usage`, which
+ # can be False (e.g. user-supplied, or GGUF forcing it off later); in that
+ # case reload() would no-op and `block.to(device)` would crash with
+ # "Cannot copy out of meta tensor". Force it enabled here whenever streaming
+ # is active so the reload path stays available regardless of low_cpu_mem_usage.
+ if getattr(self.model_context, "_disk_stream_index", None) is not None:
+ self._offloader.enabled = True
# Alternatively, you can use CompressContext.create_context
self.compress_context = CompressContext(
low_cpu_mem_usage,
diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.py
index f3cde69ec..48df479ed 100644
--- a/auto_round/compressors/layer_config_resolver.py
+++ b/auto_round/compressors/layer_config_resolver.py
@@ -534,6 +534,20 @@ def apply_plan_to_model(model, plan: ResolvedQuantizationConfig) -> None:
"""Apply a resolved plan at the single explicit module-attribute write boundary."""
scheme_keys = tuple(field.name for field in fields(QuantizationScheme)) + ("scale_dtype",)
for module_name, module in model.named_modules():
+ # AutoRound only ever writes scheme attributes onto quantization-target
+ # layers (plus model-level keys such as ``rotation_config`` on the root
+ # module). Some scheme keys share a name with attributes a model class
+ # legitimately owns -- e.g. a grouped RMSNorm's own ``group_size`` -- so
+ # blindly clearing them from every module would corrupt the model
+ # (``AttributeError`` on the norm's next forward). Scope the reset to the
+ # same modules that receive the plan below.
+ is_quant_target = (
+ isinstance(module, SUPPORTED_LAYER_TYPES)
+ or isinstance(module, torch.nn.Embedding)
+ or module.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES
+ )
+ if module_name != "" and not is_quant_target:
+ continue
for key in scheme_keys:
if module_name == "" and key == "rotation_config":
continue
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.py
index dea73bec6..f1a4215f5 100644
--- a/auto_round/compressors/orchestrator.py
+++ b/auto_round/compressors/orchestrator.py
@@ -241,19 +241,24 @@ class CompressionOrchestrator(BaseOrchestrator):
modules = [get_module(model, n) for n in names]
m = WrapperMultiblock(modules)
- # Also reload when `AR_DISK_STREAM_MODEL` is set even if
- # `low_cpu_mem_usage` has been forced False (e.g. GGUF export --
- # see base.py's `_finalize_compress_context`, which disables
- # `low_cpu_mem_usage` for gguf formats for reasons unrelated to disk
- # streaming). Under streaming, a block starts on the meta device
- # regardless of `low_cpu_mem_usage`, which only ever controlled whether
- # to *free* it again after use -- without this, the block below is never
- # materialized at all and `m.to(device)` crashes with "Cannot copy out
- # of meta tensor". The block intentionally stays real afterward (no
- # matching post-tune offload runs when `low_cpu_mem_usage` is False --
- # see the `is_immediate_saving`-adjacent offload call further down),
- # matching upstream's own choice not to cycle blocks for these formats.
- if self.compress_context.low_cpu_mem_usage or envs.AR_DISK_STREAM_MODEL:
+ # Also reload when disk streaming is active even if `low_cpu_mem_usage`
+ # has been forced False (e.g. GGUF export -- see base.py's
+ # `_finalize_compress_context`, which disables `low_cpu_mem_usage` for
+ # gguf formats for reasons unrelated to disk streaming). Disk streaming
+ # can be turned on explicitly via `AR_DISK_STREAM_MODEL=1` *or* chosen
+ # automatically for fused-MoE checkpoints (see ModelContext's
+ # `_should_use_meta_skeleton`); either way the model was built as a meta
+ # skeleton and `_disk_stream_index` is set. Under streaming, a block
+ # starts on the meta device regardless of `low_cpu_mem_usage`, which only
+ # ever controlled whether to *free* it again after use -- without this,
+ # the block below is never materialized at all and `m.to(device)` crashes
+ # with "Cannot copy out of meta tensor". The block intentionally stays
+ # real afterward (no matching post-tune offload runs when
+ # `low_cpu_mem_usage` is False -- see the `is_immediate_saving`-adjacent
+ # offload call further down), matching upstream's own choice not to cycle
+ # blocks for these formats.
+ disk_streaming = getattr(self.model_context, "_disk_stream_index", None) is not None
+ if self.compress_context.low_cpu_mem_usage or envs.AR_DISK_STREAM_MODEL or disk_streaming:
if nblocks == 1:
self._offloader.reload(model, n)
else:
@@ -440,6 +445,18 @@ class CompressionOrchestrator(BaseOrchestrator):
pbar.set_description(f"Quantizing {block_name}")
block = get_module(self.model, block_name)
+ # ── Infrastructure: reload from disk when streaming ───────
+ # Fused-MoE checkpoints (and explicit `AR_DISK_STREAM_MODEL=1`)
+ # build an all-meta skeleton to reduce RAM: each decoder block
+ # starts on the meta device and its real weights must be read
+ # back from the checkpoint before quantization. The data-driven
+ # path does this same reload; without it here the zero-shot
+ # (RTN) path leaves the block on meta and `layer.to(device)`
+ # crashes with "Cannot copy out of meta tensor".
+ disk_streaming = getattr(self.model_context, "_disk_stream_index", None) is not None
+ if self.compress_context.low_cpu_mem_usage or envs.AR_DISK_STREAM_MODEL or disk_streaming:
+ self._offloader.reload(self.model, block_name)
+
# ── Infrastructure: materialize ───────────────────────────
materialize_model_(block)
diff --git a/auto_round/context/model.py b/auto_round/context/model.py
index 63f62ba7e..0b399bf6d 100644
--- a/auto_round/context/model.py
+++ b/auto_round/context/model.py
@@ -18,6 +18,7 @@ import os
from typing import Any, Callable, Optional, Union
import torch
+import transformers
from packaging import version
from transformers import AutoConfig
@@ -183,7 +184,7 @@ class ModelContext(BaseContext):
# __init__, exactly like the text path. Falls back to the
# full mllm_load_model on any failure.
loaded_via_meta = False
- if envs.AR_DISK_STREAM_MODEL and os.path.isdir(self.model):
+ if self._should_use_meta_skeleton(self._peek_config()):
try:
self.model, self.tokenizer, self._disk_stream_index = self._build_disk_stream_model(
self.disk_stream_model_dir
@@ -197,8 +198,7 @@ class ModelContext(BaseContext):
loaded_via_meta = True
except Exception:
logger.warning(
- "AR_DISK_STREAM_MODEL requested but building a multimodal meta "
- "skeleton for %s failed; falling back to a full CPU load.",
+ "Building a multimodal meta skeleton for %s failed; " "falling back to a full CPU load.",
self.disk_stream_model_dir,
exc_info=True,
)
@@ -224,7 +224,6 @@ class ModelContext(BaseContext):
self.is_model_patched = apply_model_monkey_patches(
model_name=self.model, trust_remote_code=self.trust_remote_code
)
- import transformers
if (
not self.is_model_patched
@@ -248,15 +247,14 @@ class ModelContext(BaseContext):
gc.collect()
_force_trim_malloc()
- if envs.AR_DISK_STREAM_MODEL:
+ if self._should_use_meta_skeleton(config):
try:
self.model, self.tokenizer, self._disk_stream_index = self._build_disk_stream_model(
self.disk_stream_model_dir
)
except Exception:
logger.warning(
- "AR_DISK_STREAM_MODEL requested but building a meta skeleton for %s failed; "
- "falling back to a normal full CPU load.",
+ "Building a meta skeleton for %s failed; falling back to a normal full CPU load.",
self.disk_stream_model_dir,
exc_info=True,
)
@@ -290,6 +288,90 @@ class ModelContext(BaseContext):
if hasattr(m, "tuning_device"):
delattr(m, "tuning_device")
+ def _peek_config(self):
+ """Best-effort config lookup before the model object exists."""
+ if self.config is not None:
+ return self.config
+ if not isinstance(self.model, str):
+ return getattr(self.model, "config", None)
+ try:
+ return AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)
+ except Exception as exc:
+ logger.debug("could not peek the config of %s: %s", self.model, exc)
+ return None
+
+ def _should_use_meta_skeleton(self, config) -> bool:
+ """Whether to build a meta skeleton instead of a full CPU load.
+
+ Deliberately narrow: this only targets ``transformers>=5`` models whose experts are
+ stored as a *fused* 3D ``nn.Parameter``. For those, the checkpoint holds one 2D
+ tensor per expert, ``from_pretrained`` stacks them into the fused parameter, and
+ AutoRound then has to split it back into per-expert ``nn.Linear`` to quantize it --
+ and splitting real tensors needs the fused tensor and its per-expert copies alive
+ at once, i.e. ~2x one experts module on top of an already fully resident model.
+
+ Building on meta makes that unfuse allocation-free and lets the tuning loop
+ materialize one block at a time from the checkpoint. Every other model (dense, or
+ a MoE that already ships as ``ModuleList`` of ``Linear``) has nothing to gain and
+ keeps the ordinary load path. ``AR_DISK_STREAM_MODEL=1`` forces this on for any
+ model; ``AR_DISABLE_AUTO_META_LOAD=1`` turns the automatic choice off.
+ """
+ if envs.AR_DISK_STREAM_MODEL:
+ return True
+ if envs.AR_DISABLE_AUTO_META_LOAD:
+ return False
+ if not isinstance(self.model, str):
+ return False
+ if version.parse(transformers.__version__) < version.parse("5.0.0"):
+ return False
+ if config is None:
+ return False
+
+ from auto_round.modeling.fused_moe.moe_experts_interface import config_has_fused_moe_experts
+ from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES
+
+ if not config_has_fused_moe_experts(config):
+ return False
+ # Families with a dedicated replacement drive their own (already memory-aware)
+ # materialization; do not reroute them through the meta skeleton.
+ model_type = getattr(config, "model_type", None)
+ if model_type in BUILTIN_MODULES:
+ logger.debug("meta-skeleton load skipped: %s has a dedicated MoE replacement", model_type)
+ return False
+
+ checkpoint_dir = self._resolve_local_checkpoint_dir()
+ if checkpoint_dir is None:
+ logger.debug("meta-skeleton load skipped: %s is not addressable as a local directory", self.model)
+ return False
+ self.disk_stream_model_dir = checkpoint_dir
+ logger.info(
+ "Fused-MoE checkpoint detected: building a meta skeleton and materializing weights per block "
+ "(set AR_DISABLE_AUTO_META_LOAD=1 to load the whole model on CPU instead)."
+ )
+ return True
+
+ def _resolve_local_checkpoint_dir(self) -> Optional[str]:
+ """Return a local directory holding the checkpoint shards, or ``None``.
+
+ ``SafetensorsIndex`` reads shards by path, so a hub repo id has to be resolved to
+ a local snapshot first. Only the metadata + weights are fetched, and nothing is
+ read into memory here -- ``from_pretrained`` would have downloaded the same files.
+ """
+ if os.path.isdir(self.model):
+ return self.model
+ if envs.AR_USE_MODELSCOPE:
+ return None
+ try:
+ from huggingface_hub import snapshot_download
+
+ return snapshot_download(
+ self.model,
+ allow_patterns=["*.safetensors", "*.safetensors.index.json", "*.json", "*.txt", "*.model"],
+ )
+ except Exception as exc:
+ logger.debug("could not resolve %s to a local snapshot: %s", self.model, exc)
+ return None
+
def _build_disk_stream_model(self, model_name: str):
"""Build an all-meta skeleton instead of
fully materializing the checkpoint on CPU RAM. Left fully meta here
diff --git a/auto_round/envs.py b/auto_round/envs.py
index a30d016e3..f62f6f308 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = True
AR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = False
AR_DISK_STREAM_MODEL: bool = False
+ AR_DISABLE_AUTO_META_LOAD: bool = False
AR_RESUME_DIR: Optional[str] = None
AR_FORCE_MOE_ROUTING_ALL_EXPERTS: bool = False
AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = True
@@ -126,6 +127,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
# block-by-block from disk during quantization instead of being fully
# materialized on CPU RAM up front.
"AR_DISK_STREAM_MODEL": lambda: os.getenv("AR_DISK_STREAM_MODEL", "0").lower() in ("1", "true", "yes"),
+ # AutoRound builds a meta skeleton automatically for transformers>=5 MoE checkpoints
+ # (fused 3D experts must be split into per-expert Linear to be quantizable, and doing
+ # that on real tensors costs ~2x one experts module on top of a fully resident model).
+ # Set this to fall back to the old behavior of loading the whole model on CPU first.
+ "AR_DISABLE_AUTO_META_LOAD": lambda: os.getenv("AR_DISABLE_AUTO_META_LOAD", "0").lower() in ("1", "true", "yes"),
# When set to a directory path, the per-block tuning loop checkpoints its
# progress there after each completed block, and resumes from the first
# not-yet-completed block on a fresh run against the same directory --
diff --git a/auto_round/export/export_to_gguf/conversion/base.py b/auto_round/export/export_to_gguf/conversion/base.py
index 56547ace0..db916b51e 100644
--- a/auto_round/export/export_to_gguf/conversion/base.py
+++ b/auto_round/export/export_to_gguf/conversion/base.py
@@ -234,7 +234,7 @@ class ModelBase:
index_file = self.dir_model / index_name
if index_file.is_file():
- logger.info(f"gguf: loading model weight map from '{index_name}'")
+ logger.debug(f"gguf: loading model weight map from '{index_name}'")
with open(index_file, "r", encoding="utf-8") as f:
index: dict[str, Any] = json.load(f)
weight_map = index.get("weight_map")
@@ -249,7 +249,7 @@ class ModelBase:
weight_map = {}
for part_name in part_names:
- logger.info(f"gguf: indexing model part '{part_name}'")
+ logger.debug(f"gguf: indexing model part '{part_name}'")
ctx: ContextManager[Any]
if is_safetensors:
ctx = cast(ContextManager[Any], gguf.utility.SafetensorsLocal(self.dir_model / part_name))
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index 5393d183d..7d2dd065e 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -326,33 +326,48 @@ def _detect_expert_projections(module: nn.Module) -> dict[str, dict]:
This function scans the module for any 3D nn.Parameter attributes.
It first checks known projection names, then discovers any unknown 3D parameters.
+ Reads ``module._parameters`` directly instead of ``getattr``/``dir()``. This runs once
+ per module in the model, and after unfusing a large MoE that is tens of thousands of
+ modules: ``dir()`` on an ``nn.Module`` is expensive, and every miss goes through
+ ``nn.Module.__getattr__``, which searches three dicts before raising. A parameter is
+ always registered in ``_parameters``, so the cheap membership test is equivalent.
+
Returns:
Dict mapping projection names to their config, only for projections that exist
as 3D nn.Parameter in the module.
"""
+ parameters = getattr(module, "_parameters", None)
+ if not parameters:
+ return {}
+
+ # Fast reject: an experts module owns at least one 3D parameter. Almost every module
+ # in the model (Linear, norms, containers) fails here in a single dict scan.
+ three_dim = {
+ name: param
+ for name, param in parameters.items()
+ if param is not None and not name.startswith("_") and param.dim() == 3
+ }
+ if not three_dim:
+ return {}
+
detected = {}
# First, check known projection patterns
for proj_name, config in KNOWN_PROJECTION_PATTERNS.items():
- param = getattr(module, proj_name, None)
- if param is not None and isinstance(param, nn.Parameter) and param.dim() == 3:
+ if proj_name in three_dim:
detected[proj_name] = config
- # If no known patterns found, scan for any 3D Parameter (future-proofing),
- # but exclude known non-expert 3D parameters that exist on some transformer
- # architectures (e.g. scale_shift_table on WanTransformer3DModel).
+ # If no known patterns found, take any 3D parameter (future-proofing), but exclude
+ # known non-expert 3D parameters that exist on some transformer architectures
+ # (e.g. scale_shift_table on WanTransformer3DModel).
_NON_EXPERT_3D_PARAMS = frozenset(["scale_shift_table"])
if not detected:
- for attr_name in dir(module):
- if attr_name.startswith("_"):
- continue
- if attr_name in _NON_EXPERT_3D_PARAMS:
+ for name in three_dim:
+ if name in _NON_EXPERT_3D_PARAMS:
continue
- param = getattr(module, attr_name, None)
- if param is not None and isinstance(param, nn.Parameter) and param.dim() == 3:
- # Use default config for unknown projections
- logger.debug(f"Discovered unknown 3D projection: {attr_name}")
- detected[attr_name] = {"is_input_proj": True, "output_multiplier": 1}
+ # Use default config for unknown projections
+ logger.debug(f"Discovered unknown 3D projection: {name}")
+ detected[name] = {"is_input_proj": True, "output_multiplier": 1}
return detected
@@ -385,6 +400,27 @@ def _infer_dimensions(param: nn.Parameter, config: dict, is_transposed: bool) ->
return in_features, out_features
+def _new_meta_linear(in_features: int, out_features: int, has_bias: bool, dtype: torch.dtype) -> nn.Linear:
+ """Build an ``nn.Linear`` shell on meta without running its initializer.
+
+ ``nn.Linear.__init__`` calls ``reset_parameters`` -> ``kaiming_uniform_``, whose result
+ is always discarded here: the weight is either left on meta for the checkpoint loader
+ to fill, or replaced with a checkpoint slice on the very next line. With hundreds of
+ experts per layer (256 x 3 projections x 40 layers for Qwen3.5-MoE) that initializer
+ dominates the unfuse, so construct the module directly instead.
+ """
+ linear = nn.Linear.__new__(nn.Linear)
+ nn.Module.__init__(linear)
+ linear.in_features = in_features
+ linear.out_features = out_features
+ linear.weight = nn.Parameter(torch.empty((out_features, in_features), dtype=dtype, device="meta"))
+ if has_bias:
+ linear.bias = nn.Parameter(torch.empty(out_features, dtype=dtype, device="meta"))
+ else:
+ linear.register_parameter("bias", None)
+ return linear
+
+
def _unfuse_single_projection(
module: nn.Module,
proj_name: str,
@@ -456,7 +492,7 @@ def _unfuse_single_projection(
linears = []
for i in range(num_experts):
# meta device: creates the module structure without allocating weight storage
- linear = nn.Linear(in_features, out_features, bias=has_bias, dtype=dtype, device="meta")
+ linear = _new_meta_linear(in_features, out_features, has_bias, dtype)
if not is_meta:
# Direct parameter assignment — no copy, just references the CPU tensor slice
@@ -482,6 +518,33 @@ def _unfuse_single_projection(
_logged_memory_before_replacement = False # To ensure we only log memory once before replacements
+def _warn_if_unfusing_allocates(module: nn.Module, detected_projections: dict[str, dict]) -> None:
+ """Warn when the fused expert weights are not on meta.
+
+ Unfusing is free on meta. On a real device the fused tensor and the per-expert copies
+ must coexist for the duration of the split, so the peak is inherently ~2x one experts
+ module on top of an already resident model.
+ """
+ params = [
+ param
+ for param in (getattr(module, name, None) for name in detected_projections)
+ if isinstance(param, nn.Parameter) and param.device.type != "meta"
+ ]
+ if not params:
+ return
+
+ total_bytes = sum(param.numel() * param.element_size() for param in params)
+ logger.warning(
+ "[MoE Prep] fused expert weights are on %s (%.2f GB for '%s' alone), not meta. "
+ "Splitting them in memory needs the fused tensor and its per-expert copies at the "
+ "same time. AutoRound normally avoids this by building a meta skeleton and reading "
+ "each expert from disk; that requires a local checkpoint directory.",
+ params[0].device.type,
+ total_bytes / 1024**3,
+ module.__class__.__name__,
+ )
+
+
def _unfuse_experts_weights_inplace(
module: nn.Module,
check_decorator: bool = True,
@@ -521,6 +584,7 @@ def _unfuse_experts_weights_inplace(
global _logged_memory_before_replacement
if not _logged_memory_before_replacement:
_logged_memory_before_replacement = True
+ _warn_if_unfusing_allocates(module, detected_projections)
memory_monitor.update()
memory_monitor.log_summary("Before applying custom replacements")
@@ -697,3 +761,82 @@ def prepare_model_for_moe_quantization(
def is_linear_loop_available() -> bool:
"""Check if linear_loop experts implementation is available."""
return HAS_EXPERTS_INTERFACE
+
+
+def _config_model_types(config) -> list[str]:
+ """Collect ``model_type`` from a config and every nested sub-config.
+
+ A VLM keeps the MoE rules on the text sub-config
+ (``text_config.model_type == 'qwen3_5_moe_text'``), so the top-level type alone is
+ not enough. Uses ``sub_configs``/``get_text_config`` when available (the supported
+ transformers API) and falls back to scanning ``__dict__`` for config-like values.
+ """
+ found: list[str] = []
+ seen: set[int] = set()
+
+ def _walk(node) -> None:
+ if node is None or id(node) in seen:
+ return
+ seen.add(id(node))
+
+ model_type = getattr(node, "model_type", None)
+ if isinstance(model_type, str) and model_type not in found:
+ found.append(model_type)
+
+ children = []
+ # Preferred: the declared sub-config names (transformers >= 4.4x).
+ for name in getattr(type(node), "sub_configs", None) or ():
+ children.append(getattr(node, name, None))
+ get_text_config = getattr(node, "get_text_config", None)
+ if callable(get_text_config):
+ try:
+ children.append(get_text_config())
+ except Exception: # pragma: no cover - defensive
+ pass
+ # Fallback for plain/duck-typed configs.
+ for value in (vars(node).values() if hasattr(node, "__dict__") else ()):
+ if getattr(value, "model_type", None) is not None:
+ children.append(value)
+
+ for child in children:
+ if child is not None and child is not node:
+ _walk(child)
+
+ _walk(config)
+ return found
+
+
+def config_has_fused_moe_experts(config) -> bool:
+ """Whether loading this config would build fused 3D expert parameters.
+
+ ``transformers>=5`` describes such models with a ``WeightConverter`` that stacks the
+ checkpoint's per-expert 2D tensors into one fused parameter (``MergeModulelist``).
+ That converter is exactly the signal we care about: its presence means
+ ``from_pretrained`` would produce a fused ``nn.Parameter`` that AutoRound then has to
+ split back into per-expert ``nn.Linear`` -- the expensive case worth building on meta
+ for. MoE models that already ship as ``ModuleList`` of ``Linear`` have no such
+ converter and are deliberately left on the ordinary load path.
+ """
+ try:
+ from transformers.conversion_mapping import get_checkpoint_conversion_mapping
+ from transformers.core_model_loading import MergeModulelist, WeightConverter
+ except ImportError: # pragma: no cover - transformers < 5
+ return False
+
+ for model_type in _config_model_types(config):
+ mapping = get_checkpoint_conversion_mapping(model_type)
+ if not mapping:
+ continue
+ for entry in mapping:
+ if not isinstance(entry, WeightConverter):
+ continue
+ if not any(isinstance(op, MergeModulelist) for op in entry.operations):
+ continue
+ if any("experts" in pattern for pattern in entry.source_patterns):
+ logger.debug(
+ "[MoE Prep] '%s' merges per-expert checkpoint tensors into a fused parameter (%s)",
+ model_type,
+ entry.target_patterns,
+ )
+ return True
+ return False
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index 1276b89e5..3896402c2 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -13,12 +13,12 @@
# limitations under the License.
from abc import ABC, abstractmethod
+from contextlib import contextmanager
from dataclasses import dataclass
from typing import Dict, Type
import torch
from tqdm import tqdm
-from transformers import PreTrainedModel
from auto_round.modeling.fused_moe.utils import is_linearized_layout
from auto_round.utils import (
@@ -121,13 +121,46 @@ def is_custom_model(model: torch.nn.Module) -> bool:
return False
-def _log_first_moe_block(model: torch.nn.Module, label: str) -> None:
- """Log the first experts module found in the model for debugging."""
+def _find_first_moe_block(model: torch.nn.Module) -> tuple[str, torch.nn.Module] | tuple[None, None]:
+ """Return ``(name, module)`` of the first experts-like module, or ``(None, None)``."""
for name, module in model.named_modules():
if name.endswith(".experts") or name.endswith(".moe"):
- logger.info(f"Experts ({label}) [{name}] ({module.__class__.__name__}):\n{module}")
- return True
- return False
+ return name, module
+ return None, None
+
+
+def _describe_moe_block(model: torch.nn.Module, name: str) -> str | None:
+ """Render the experts module at ``name`` for the before/after comparison."""
+ try:
+ module = model.get_submodule(name)
+ except AttributeError:
+ return None
+ return f"({module.__class__.__name__}):\n{module}"
+
+
+@contextmanager
+def log_moe_block_transition(model: torch.nn.Module, label: str = "replacement"):
+ """Log the first experts module before and after a structural change.
+
+ The structural unfuse happens in one of two places depending on how the model was
+ built: on the meta skeleton (``build_meta_model``) or during ``apply_replacements``.
+ Both wrap themselves in this so the transition is reported wherever it actually
+ occurs, and only when something really changed -- printing an identical before/after
+ pair (which is what the other path sees, since it then has nothing left to do) is
+ just noise.
+ """
+ name, _ = _find_first_moe_block(model)
+ before = _describe_moe_block(model, name) if name is not None else None
+ try:
+ yield
+ finally:
+ if name is not None:
+ after = _describe_moe_block(model, name)
+ if after is not None and after != before:
+ logger.info(f"Experts (before {label}) [{name}] {before}")
+ logger.info(f"Experts (after {label}) [{name}] {after}")
+ else:
+ logger.debug(f"Experts [{name}] left unchanged by {label} {before}")
@dump_mem_usage("Materializing model", log_level="debug")
@@ -138,18 +171,27 @@ def materialize_model_(model: torch.nn.Module) -> None:
model.apply(_materialize_module)
- # check if any module on meta device remains
- found_meta = False
- for name, param in model.named_parameters():
- if param.device.type == "meta":
- logger.warning(f"Parameter {name} is still on meta device after materialization.")
- found_meta = True
- for name, buffer in model.named_buffers():
- if buffer.device.type == "meta":
- logger.warning(f"Buffer {name} is still on meta device after materialization.")
- found_meta = True
- if not found_meta:
- logger.debug("All parameters and buffers have been materialized from meta device.")
+ # Any tensor still on meta after per-block materialization means the meta-skeleton
+ # loader could not resolve it from the checkpoint (e.g. a checkpoint-conversion
+ # layout AutoRound does not reverse yet). Leaving it on meta only defers the failure
+ # to a cryptic "Cannot copy out of meta tensor" crash on the next `.to(device)`, so
+ # fail fast with an actionable message.
+ still_meta = [name for name, param in model.named_parameters() if param.device.type == "meta"]
+ still_meta += [name for name, buffer in model.named_buffers() if buffer.device.type == "meta"]
+ if still_meta:
+ from auto_round import envs
+
+ preview = ", ".join(still_meta[:8]) + (f" (+{len(still_meta) - 8} more)" if len(still_meta) > 8 else "")
+ # The auto meta-skeleton is only enabled for fused-MoE checkpoints; an explicit
+ # AR_DISK_STREAM_MODEL=1 forces it for anything else. Point at whichever toggle
+ # actually turned it on so the hint stays correct (AR_DISABLE_AUTO_META_LOAD is
+ # the MoE-only auto path).
+ hint = "unset AR_DISK_STREAM_MODEL" if envs.AR_DISK_STREAM_MODEL else "set AR_DISABLE_AUTO_META_LOAD=1"
+ raise RuntimeError(
+ f"Failed to materialize {len(still_meta)} checkpoint tensor(s), still on meta: {preview}. "
+ f"To load the whole model on CPU instead, {hint} (uses more RAM)."
+ )
+ logger.debug("All parameters and buffers have been materialized from meta device.")
release_original_module_(model)
@@ -319,25 +361,14 @@ def apply_replacements(
The model with modules replaced.
"""
_import_required_replacements(model)
- _raw_expert_is_logged = False
-
- # Custom replacements first
- if is_custom_model(model):
-
- if not _raw_expert_is_logged:
- _raw_expert_is_logged = _log_first_moe_block(model, "before replacement")
-
- _apply_custom_replacements(model, gguf_export=gguf_export)
-
- if auto_detect_moe and is_transformers_version_greater_or_equal_5():
-
- if not _raw_expert_is_logged:
- _raw_expert_is_logged = _log_first_moe_block(model, "before replacement")
- _handle_moe_modules(model)
+ with log_moe_block_transition(model, "replacement"):
+ # Custom replacements first
+ if is_custom_model(model):
+ _apply_custom_replacements(model, gguf_export=gguf_export)
- if _raw_expert_is_logged:
- _log_first_moe_block(model, "after replacement/skip")
+ if auto_detect_moe and is_transformers_version_greater_or_equal_5():
+ _handle_moe_modules(model)
return model
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 4e1212640..73750a8de 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -22,7 +22,7 @@ import torch
from auto_round.export.formats import OutputFormat
from auto_round.modeling.fused_moe.replace_modules import apply_replacements, release_original_module_
-from auto_round.utils import is_moe_model_via_config, logger
+from auto_round.utils import is_moe_model_via_config, logger, torch_version_at_least
mllms_with_limited_bs = (
"llava",
@@ -1217,24 +1217,27 @@ def get_torch_compile_off_reason(model) -> str | None:
# DeepSeek V2/V3/V3.2/V4 and the GLM-5.3-Flash family (``glm_moe_dsa`` text /
# ``glm5_next`` multimodal) share a DSA indexer + fused MoE with shape-dependent
-# control flow. Compiling the tuning graph there hits dynamo's recompile_limit and
-# provides little benefit, so keep those runs in eager mode.
-register_torch_compile_off(
- matchers=[
- ModelTypeMatcher("deepseek", mode="in"),
- ArchitectureMatcher("Deepseek", mode="in"),
- ArchitectureMatcher("DeepSeek", mode="in"),
- ],
- reason="the DeepSeek architecture is incompatible with torch.compile during tuning",
-)
-register_torch_compile_off(
- matchers=[
- ModelTypeMatcher(r"glm_moe_dsa", mode="full"),
- ModelTypeMatcher(r"glm5_next", mode="full"),
- ArchitectureMatcher(r"Glm5Next", mode="in"),
- ],
- reason="the GLM-5 architecture is incompatible with torch.compile during tuning",
-)
+# control flow. On torch < 2.14.0 compiling the tuning graph there hits dynamo's
+# recompile_limit and provides little benefit, so keep those runs in eager mode.
+# From torch 2.14.0 onward these constraints are resolved, so we no longer force
+# ``torch.compile`` off and instead honor the default / user-provided setting.
+if not torch_version_at_least("2.14.0"):
+ register_torch_compile_off(
+ matchers=[
+ ModelTypeMatcher("deepseek", mode="in"),
+ ArchitectureMatcher("Deepseek", mode="in"),
+ ArchitectureMatcher("DeepSeek", mode="in"),
+ ],
+ reason="the DeepSeek architecture is incompatible with torch.compile during tuning on torch < 2.14.0",
+ )
+ register_torch_compile_off(
+ matchers=[
+ ModelTypeMatcher(r"glm_moe_dsa", mode="full"),
+ ModelTypeMatcher(r"glm5_next", mode="full"),
+ ArchitectureMatcher(r"Glm5Next", mode="in"),
+ ],
+ reason="the GLM-5 architecture is incompatible with torch.compile during tuning on torch < 2.14.0",
+ )
def _attach_gemma4_rotary_emb(model):
diff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.py
index 3da7864e6..4751fac17 100644
--- a/auto_round/utils/disk_stream_util.py
+++ b/auto_round/utils/disk_stream_util.py
@@ -13,6 +13,7 @@ from __future__ import annotations
import json
import logging
import re
+from functools import lru_cache
from pathlib import Path
from typing import Dict
@@ -77,6 +78,17 @@ class SafetensorsIndex:
return [n for n in self.weight_map if n == prefix or n.startswith(dotted)]
+@lru_cache(maxsize=8)
+def get_safetensors_index(checkpoint_dir: str) -> "SafetensorsIndex":
+ """Shared ``SafetensorsIndex`` per checkpoint directory.
+
+ Only the (cheap) name->shard map is cached; no ``safe_open`` handle is kept, so this
+ does not reintroduce the mmap-residency problem ``SafetensorsIndex`` avoids. Callers
+ used to build a fresh index for every block, re-reading the index JSON 40+ times.
+ """
+ return SafetensorsIndex(str(checkpoint_dir))
+
+
# Model-side module names sometimes differ from the checkpoint-side tensor
# names. Transformers already maintains the authoritative per-family renames
# (``transformers/conversion_mapping.py``, checkpoint -> model direction);
@@ -84,12 +96,22 @@ class SafetensorsIndex:
# are inverted here. A rename is applied only when the model-side name is
# absent from the index AND the renamed candidate exists in it, so checkpoints
# whose names already match the model are untouched.
-from functools import lru_cache
@lru_cache(maxsize=None)
def _reverse_renamings_for(model_type):
- """Invert the checkpoint-conversion WeightRenaming entries for one family."""
+ """Invert the checkpoint-conversion WeightRenaming entries for one family.
+
+ ``transformers`` owns the checkpoint->model renames, so reversing them is best left to
+ ``WeightTransform.reverse_transform()``: it handles capturing groups and the
+ ``PrefixChange`` special cases correctly. Naively reusing a ``target_pattern`` as a
+ regex pattern does not -- the targets are *replacement* strings and a ``\\1``
+ backreference in one (e.g. ``model.language_model.\\1``) raises
+ ``re.PatternError: invalid group reference``.
+
+ Returns reversed ``WeightTransform`` objects; call ``rename_source_key(model_side_name)``
+ on them to get the checkpoint-side name.
+ """
if not model_type:
return ()
try:
@@ -99,40 +121,303 @@ def _reverse_renamings_for(model_type):
mapping = get_checkpoint_conversion_mapping(model_type)
if not mapping:
return ()
- renamings = []
+ reversed_transforms = []
for entry in mapping:
if not isinstance(entry, WeightRenaming):
- # Other converter types (fusions, prefix moves) are not invertible
- # by simple regex substitution and are skipped.
+ # Other converter types (expert fusions) are handled separately, by
+ # `_renamed_expert_candidates` and `materialize_module`'s fused lookup.
continue
- for target in entry.target_patterns:
- for source in entry.source_patterns:
- renamings.append((target, source))
- return tuple(renamings)
+ try:
+ reversed_transforms.append(entry.reverse_transform())
+ except Exception as exc: # pragma: no cover - not every transform is reversible
+ logger.debug("skipping non-reversible rename %s: %s", entry, exc)
+ return tuple(reversed_transforms)
@lru_cache(maxsize=None)
-def _index_model_type(index):
- config_path = Path(index.checkpoint_dir) / "config.json"
+def _model_types_for_dir(checkpoint_dir: str):
+ """Every model_type in the checkpoint's config, including nested sub-configs.
+
+ A VLM config nests the text model (``text_config.model_type = 'qwen3_5_moe_text'``)
+ and that is where the MoE expert conversion rules are registered, so the top-level
+ ``model_type`` alone is not enough to resolve expert tensor names.
+
+ Keyed by directory rather than by ``SafetensorsIndex`` instance: callers build a fresh
+ index per block, so caching on the object would both miss every time and pin every
+ index (and its ``weight_map``) in memory for the whole run.
+ """
try:
- with open(config_path) as f:
- return json.load(f).get("model_type")
+ with open(Path(checkpoint_dir) / "config.json") as f:
+ config = json.load(f)
except OSError:
+ return ()
+
+ found: list[str] = []
+
+ def _walk(node):
+ if not isinstance(node, dict):
+ return
+ model_type = node.get("model_type")
+ if isinstance(model_type, str) and model_type not in found:
+ found.append(model_type)
+ for value in node.values():
+ if isinstance(value, dict):
+ _walk(value)
+
+ _walk(config)
+ return tuple(found)
+
+
+def _index_model_type(index):
+ model_types = _model_types_for_dir(str(getattr(index, "checkpoint_dir", None)))
+ return model_types[0] if model_types else None
+
+
+def _index_model_types(index):
+ # Some lightweight/test indices carry no ``checkpoint_dir``; degrade to no
+ # known model types rather than raising AttributeError.
+ return _model_types_for_dir(str(getattr(index, "checkpoint_dir", None)))
+
+
+# Model-side per-expert projection -> (fused checkpoint projection, index within it).
+# ``gate_proj``/``up_proj`` come from splitting a fused ``gate_up_proj`` in that order,
+# which matches the order of the corresponding WeightConverter ``source_patterns``.
+_MODEL_PROJ_TO_FUSED = {
+ "gate_proj": ("gate_up_proj", 0),
+ "up_proj": ("gate_up_proj", 1),
+ "down_proj": ("down_proj", 0),
+}
+
+# On-disk layout of fused expert tensors, keyed by checkpoint ``model_type``.
+# Two axes vary between families:
+# * ``checkpoint_transposed`` -- the per-expert 2-D slice is stored as
+# ``[in_features, out_features]`` (matmul layout, ``x @ W``) instead of the
+# ``nn.Linear`` ``[out_features, in_features]`` weight layout, so it must be
+# transposed before being loaded into the unfused per-expert Linear.
+# * ``gate_up_interleaved`` -- the fused ``gate_up_proj`` interleaves the gate
+# and up columns (``gate = fused[..., 0::2]``, ``up = fused[..., 1::2]``)
+# instead of storing them as two contiguous halves.
+# The transpose axis is normally inferred from the *target* parameter shape and
+# does not need this table; the table is only consulted (a) to pick the
+# interleave axis, which is invisible to shapes, and (b) as a tiebreak for the
+# rare square/ambiguous shapes where transposed and non-transposed slices share
+# the same shape. Families not listed here default to the non-transposed,
+# contiguous ``[N, 2*inter, hidden]`` layout (qwen3_moe / phimoe / mixtral-style).
+_FUSED_EXPERT_LAYOUTS = {
+ # model_type: (checkpoint_transposed, gate_up_interleaved)
+ "gpt_oss": (True, True),
+ "llama4": (True, False),
+}
+
+
+def _fused_expert_layout(index) -> tuple[bool, bool]:
+ """Return the ``(checkpoint_transposed, gate_up_interleaved)`` hint for ``index``.
+
+ The transpose flag is only a *hint* (shape inference takes precedence); the
+ interleave flag is authoritative since it cannot be recovered from shapes.
+ Unknown model types default to ``(False, False)``.
+ """
+ for model_type in _index_model_types(index):
+ if model_type in _FUSED_EXPERT_LAYOUTS:
+ return _FUSED_EXPERT_LAYOUTS[model_type]
+ return (False, False)
+
+
+def _slice_fused_expert(fused, proj, attr, is_gate_up, target_shape, transposed_hint, interleaved):
+ """Slice one unfused per-expert tensor out of a fused per-expert tensor.
+
+ ``fused`` is a single expert's slice of the fused checkpoint tensor (2-D for a
+ weight, 1-D for a bias). ``target_shape`` is the shape the unfused
+ ``nn.Linear`` parameter expects. Whether the checkpoint stores the weight
+ transposed is inferred from ``target_shape`` first; ``transposed_hint`` only
+ breaks ambiguous (square) ties, and ``interleaved`` selects the gate/up split
+ order. Returns ``None`` when no slice can produce ``target_shape`` (the caller
+ then leaves the parameter on meta and reports an actionable error).
+ """
+ if attr == "bias":
+ # 1-D per-expert bias; only gate/up need splitting.
+ if not is_gate_up:
+ return fused.contiguous()
+ if interleaved:
+ return (fused[0::2] if proj == "gate_proj" else fused[1::2]).contiguous()
+ half = fused.shape[0] // 2
+ return (fused[:half] if proj == "gate_proj" else fused[half:]).contiguous()
+
+ if fused.ndim != 2:
return None
+ if not is_gate_up: # down_proj weight, target (out, in)
+ reversed_shape = (target_shape[1], target_shape[0])
+ if target_shape == reversed_shape:
+ transposed = transposed_hint # square: shape can't decide
+ elif tuple(fused.shape) == target_shape:
+ transposed = False
+ elif tuple(fused.shape) == reversed_shape:
+ transposed = True
+ else:
+ return None
+ return fused.t().contiguous() if transposed else fused.contiguous()
+
+ # gate/up weight, target (out, in) = (inter, hidden)
+ out, in_features = target_shape
+ non_transposed = tuple(fused.shape) == (2 * out, in_features)
+ transposed = tuple(fused.shape) == (in_features, 2 * out)
+ if non_transposed and transposed:
+ transposed = transposed_hint # square-ish (2*out == in): shape can't decide
+ non_transposed = not transposed
+ if non_transposed:
+ return (fused[:out] if proj == "gate_proj" else fused[out:]).contiguous()
+ if transposed:
+ # fused is [in_features, 2*out]; split along the 2*out dim, then transpose
+ # back to the [out, in_features] nn.Linear weight layout.
+ if interleaved:
+ sliced = fused[:, 0::2] if proj == "gate_proj" else fused[:, 1::2]
+ else:
+ sliced = fused[:, :out] if proj == "gate_proj" else fused[:, out:]
+ return sliced.t().contiguous()
+ return None
+
+
+_MODEL_SIDE_EXPERT_RE = re.compile(
+ r"^(?P<prefix>.*\.experts)\.(?P<expert>\d+)\.(?P<proj>[A-Za-z0-9_]+)\.(?P<attr>weight|bias)$"
+)
+
+
+@lru_cache(maxsize=None)
+def _expert_projection_renames_for(model_type):
+ """Map a fused projection to the checkpoint-side per-expert projection names.
+
+ ``transformers`` describes MoE experts with a ``WeightConverter`` whose
+ ``source_patterns`` are the per-expert checkpoint keys and whose single
+ ``target_pattern`` is the fused model-side parameter, e.g. for phimoe::
+
+ source_patterns = [".experts.*.w1.weight", ".experts.*.w3.weight"]
+ target_patterns = ".experts.gate_up_proj"
+
+ After AutoRound unfuses the experts the model-side names are
+ ``experts.<i>.gate_proj.weight`` / ``experts.<i>.up_proj.weight``, so we need
+ ``(gate_up_proj, 0) -> "w1.weight"`` and ``(gate_up_proj, 1) -> "w3.weight"`` to find
+ the tensors back on disk. Returns a tuple of ``((fused, index), suffix)`` pairs.
+ """
+ if not model_type:
+ return ()
+ try:
+ from transformers.conversion_mapping import get_checkpoint_conversion_mapping
+ from transformers.core_model_loading import MergeModulelist, WeightConverter
+ except ImportError: # pragma: no cover - transformers < 5
+ return ()
+
+ mapping = get_checkpoint_conversion_mapping(model_type)
+ if not mapping:
+ return ()
+
+ renames: dict[tuple[str, int], str] = {}
+ for entry in mapping:
+ if not isinstance(entry, WeightConverter):
+ continue
+ # Only expert merges (`experts.*.<proj>` -> fused 3D) are relevant here.
+ if not any(isinstance(op, MergeModulelist) for op in entry.operations):
+ continue
+ if len(entry.target_patterns) != 1:
+ continue
+ fused = entry.target_patterns[0].rstrip("$").rsplit(".", 1)[-1]
+ for position, source_pattern in enumerate(entry.source_patterns):
+ # ".experts.*.w1.weight" -> "w1.weight"
+ _, star, suffix = source_pattern.partition("*.")
+ if not star:
+ continue
+ renames.setdefault((fused, position), suffix.rstrip("$"))
+ return tuple(renames.items())
+
+
+@lru_cache(maxsize=None)
+def _concat_converters_for(model_type):
+ """Model-side params assembled by concatenating several checkpoint tensors.
+
+ Some families describe a single model-side parameter as the concatenation of
+ several separately-stored checkpoint tensors, via a ``WeightConverter`` whose
+ ``operations`` contain a ``Concatenate`` (and *no* ``MergeModulelist`` -- that is the
+ expert-fusion case, handled elsewhere). GLM-5.3-Flash (``glm5_next``) is the motivating
+ example: ``self_attn.conv1d.weight`` is ``cat([q_conv1d.weight, k_conv1d.weight,
+ v_conv1d.weight], dim=0)``. The meta skeleton only knows the fused model-side name, so
+ map it back to its checkpoint-side sources to materialize it.
+
+ Returns a tuple of ``(target_suffix, (source_suffixes...), concat_dim)`` triples, using
+ only literal (wildcard-free) patterns -- wildcard/expert converters are handled by the
+ expert-specific helpers.
+ """
+ if not model_type:
+ return ()
+ try:
+ from transformers.conversion_mapping import get_checkpoint_conversion_mapping
+ from transformers.core_model_loading import Concatenate, MergeModulelist, WeightConverter
+ except ImportError: # pragma: no cover - transformers < 5
+ return ()
+
+ mapping = get_checkpoint_conversion_mapping(model_type)
+ if not mapping:
+ return ()
+
+ converters = []
+ for entry in mapping:
+ if not isinstance(entry, WeightConverter):
+ continue
+ # Expert fusions (MergeModulelist) are handled by `_renamed_expert_candidates`
+ # and `materialize_module`'s fused lookup.
+ if any(isinstance(op, MergeModulelist) for op in entry.operations):
+ continue
+ concat = next((op for op in entry.operations if isinstance(op, Concatenate)), None)
+ if concat is None or len(entry.target_patterns) != 1:
+ continue
+ target = entry.target_patterns[0].rstrip("$")
+ sources = tuple(src.rstrip("$") for src in entry.source_patterns)
+ # Only literal patterns here; wildcard (`*`) converters are per-expert.
+ if "*" in target or any("*" in src for src in sources):
+ continue
+ converters.append((target, sources, getattr(concat, "dim", 0)))
+ return tuple(converters)
+
+
+def _renamed_expert_candidates(index, full_name: str):
+ """Checkpoint-name candidates for an unfused per-expert parameter."""
+ match = _MODEL_SIDE_EXPERT_RE.match(full_name)
+ if match is None:
+ return
+
+ fused_position = _MODEL_PROJ_TO_FUSED.get(match["proj"])
+ if fused_position is None:
+ return
+ prefix, expert = match["prefix"], match["expert"]
+
+ for model_type in _index_model_types(index):
+ for key, suffix in _expert_projection_renames_for(model_type):
+ if key != fused_position:
+ continue
+ # "w1.weight" keeps the checkpoint's own attribute name; drop it for a bias.
+ suffix_attr = suffix.rsplit(".", 1)
+ if len(suffix_attr) == 2:
+ suffix = f"{suffix_attr[0]}.{match['attr']}"
+ yield f"{prefix}.{expert}.{suffix}"
+
def _resolve_checkpoint_name(index, full_name: str):
"""Map a model-side parameter name to its checkpoint-side tensor name."""
if index.has_tensor(full_name):
return full_name
- for target_pattern, source_replacement in _reverse_renamings_for(_index_model_type(index)):
- # ``source_replacement`` arrives regex-escaped (``mlp\.router\.gate``);
- # as a ``re.sub`` replacement those backslashes would land in the string
- # verbatim, so unescape it to the plain checkpoint-side spelling.
- replacement = re.sub(r"\\(.)", r"\1", source_replacement)
- candidate = re.sub(target_pattern, lambda _m: replacement, full_name)
- if candidate != full_name and index.has_tensor(candidate):
+ # Unfused MoE experts whose checkpoint spells the projections differently
+ # (e.g. phimoe's w1/w2/w3, mixtral-style checkpoints).
+ for candidate in _renamed_expert_candidates(index, full_name):
+ if index.has_tensor(candidate):
return candidate
+ for model_type in _index_model_types(index):
+ for transform in _reverse_renamings_for(model_type):
+ try:
+ candidate, _ = transform.rename_source_key(full_name)
+ except Exception as exc: # pragma: no cover - defensive, never fail a load here
+ logger.debug("rename %s via %s failed: %s", full_name, transform, exc)
+ continue
+ if candidate != full_name and index.has_tensor(candidate):
+ return candidate
return None
@@ -160,24 +445,70 @@ def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIn
# calibration/eval forwards) previously left those params on meta -- the
# meta-ness then propagated silently until a crash far downstream. Map
# each unfused name onto its fused on-disk tensor and slice.
- _FUSED_RE = _re.compile(r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$")
+ #
+ # The exact slicing depends on the family's fused layout: qwen3_moe stores
+ # ``[N, 2*inter, hidden]`` (contiguous halves, no transpose), while gpt_oss /
+ # llama4 store ``[N, hidden, 2*inter]`` (matmul layout, needs transpose) and
+ # gpt_oss additionally interleaves the gate/up columns. Rather than key the
+ # whole decision off ``model_type`` (which does not generalise to new
+ # families), the transpose is inferred from the *target* parameter shape and
+ # only the interleave axis (invisible to shapes) falls back to a small
+ # model_type hint table (see ``_slice_fused_expert``). Biases (gpt_oss) follow
+ # the same split as their weight.
+ _FUSED_RE = _re.compile(r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.(weight|bias)$")
_fused_cache: dict = {}
+ _layout_cache: list = [] # lazily-resolved (transposed_hint, interleaved)
- def _fused_lookup(full_name: str):
+ def _fused_lookup(full_name: str, target_shape):
m = _FUSED_RE.match(full_name)
if not m:
return None
- prefix, expert_idx, proj = m.group(1), int(m.group(2)), m.group(3)
- fused_name = f"{prefix}.gate_up_proj" if proj in ("gate_proj", "up_proj") else f"{prefix}.down_proj"
+ prefix, expert_idx, proj, attr = m.group(1), int(m.group(2)), m.group(3), m.group(4)
+ is_gate_up = proj in ("gate_proj", "up_proj")
+ if is_gate_up:
+ fused_name = f"{prefix}.gate_up_proj" if attr == "weight" else f"{prefix}.gate_up_proj_bias"
+ else:
+ fused_name = f"{prefix}.down_proj" if attr == "weight" else f"{prefix}.down_proj_bias"
if not index.has_tensor(fused_name):
return None
if fused_name not in _fused_cache:
_fused_cache[fused_name] = index.read_tensors([fused_name], device=device)[fused_name]
fused = _fused_cache[fused_name][expert_idx]
- if proj == "down_proj":
- return fused.contiguous()
- inter = fused.shape[0] // 2
- return (fused[:inter] if proj == "gate_proj" else fused[inter:]).contiguous()
+ # Resolve the model_type layout hint lazily and only once, and only when a
+ # fused expert tensor is actually being sliced -- reading the model type
+ # touches ``index.checkpoint_dir``/config, which cheap/fake indices used by
+ # non-fused (dense) materialization do not populate.
+ if not _layout_cache:
+ _layout_cache.append(_fused_expert_layout(index))
+ transposed_hint, interleaved = _layout_cache[0]
+ value = _slice_fused_expert(fused, proj, attr, is_gate_up, tuple(target_shape), transposed_hint, interleaved)
+ # Guard against a slice that does not match the destination parameter: it
+ # is safer to leave the param on meta (the caller then raises an
+ # actionable "fall back to a full CPU load" error) than to hand accelerate
+ # a mismatched tensor and crash with a cryptic ValueError.
+ if value is not None and tuple(value.shape) != tuple(target_shape):
+ return None
+ return value
+
+ def _concat_lookup(full_name: str):
+ """Assemble a model-side param that the checkpoint stores as several tensors
+ concatenated by a ``WeightConverter`` (e.g. GLM-5.3 ``q/k/v_conv1d`` ->
+ ``conv1d``). Returns the concatenated tensor, or ``None`` when no converter
+ applies or a source tensor is missing."""
+ for model_type in _index_model_types(index):
+ for target_suffix, source_suffixes, dim in _concat_converters_for(model_type):
+ if not full_name.endswith(target_suffix):
+ continue
+ prefix = full_name[: -len(target_suffix)]
+ # Guard against a spurious mid-token match (require a clean boundary).
+ if prefix and not prefix.endswith("."):
+ continue
+ source_names = [f"{prefix}{suffix}" for suffix in source_suffixes]
+ if not all(index.has_tensor(name) for name in source_names):
+ continue
+ read = index.read_tensors(source_names, device=device)
+ return torch.cat([read[name] for name in source_names], dim=dim).contiguous()
+ return None
targets = [] # (param_name, full_checkpoint_name, declared_meta_dtype)
fused_targets = [] # (param_name, sliced_value)
@@ -187,7 +518,9 @@ def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIn
full_name = f"{module_name}.{name}".replace(".orig_layer.", ".")
resolved_name = _resolve_checkpoint_name(index, full_name)
if resolved_name is None:
- sliced = _fused_lookup(full_name)
+ sliced = _fused_lookup(full_name, tensor.shape)
+ if sliced is None:
+ sliced = _concat_lookup(full_name)
if sliced is not None:
fused_targets.append((name, sliced))
continue
@@ -196,7 +529,18 @@ def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIn
targets.append((name, resolved_name, tensor.dtype))
for name, value in fused_targets:
- set_module_tensor_to_device(module, name, device, value=value, dtype=value.dtype)
+ try:
+ set_module_tensor_to_device(module, name, device, value=value, dtype=value.dtype)
+ except (ValueError, RuntimeError) as exc:
+ # A fused-expert slice whose shape does not match its destination
+ # cannot be reconciled from the checkpoint; rather than surface the
+ # raw accelerate error, point the user at the non-meta fallback.
+ raise RuntimeError(
+ f"Failed to materialize fused MoE parameter '{module_name}.{name}' from the checkpoint "
+ f"(value shape {tuple(value.shape)}). This usually means the model's fused-expert layout "
+ "is not yet recognised by the meta-skeleton loader. Set AR_DISABLE_AUTO_META_LOAD=1 to load "
+ "the whole model on CPU instead (uses more RAM)."
+ ) from exc
_fused_cache.clear()
if not targets:
@@ -242,7 +586,63 @@ def total_resident_bytes(model: nn.Module) -> int:
return total
-def build_meta_model(model_name: str, trust_remote_code: bool = True):
+def unfuse_meta_moe_(model: nn.Module) -> list[str]:
+ """Turn ``transformers>=5`` fused 3D expert parameters into per-expert ``nn.Linear``.
+
+ ``transformers>=5`` declares MoE experts as a single fused ``nn.Parameter`` of shape
+ ``(num_experts, ...)``, which AutoRound cannot quantize (it is not an ``nn.Linear``).
+ The regular load pipeline fixes this *after* a full CPU load; on a meta skeleton we
+ can do it up-front for free -- meta tensors own no storage, so unfusing here only
+ rewires ``nn.Module`` objects and costs ~0 RAM.
+
+ Doing it before any weight is read is also what keeps the later per-block
+ materialization cheap: the model-side names become ``experts.<i>.<proj>.weight``,
+ which is exactly the layout the checkpoints use, so ``materialize_module`` reads one
+ small 2D tensor per expert instead of forcing a whole fused ``(num_experts, ...)``
+ tensor to be assembled in RAM.
+
+ No modeling class is monkey-patched: the per-expert forward is dispatched through
+ ``transformers.integrations.moe.ALL_EXPERTS_FUNCTIONS`` (the public experts-interface
+ registry) via ``config._experts_implementation``.
+
+ Returns the list of unfused module names (empty when there is nothing to do).
+ """
+ try:
+ from auto_round.modeling.fused_moe.replace_modules import (
+ _handle_moe_modules,
+ is_custom_model,
+ log_moe_block_transition,
+ )
+ except ImportError: # pragma: no cover - transformers < 5 or partial install
+ return []
+
+ # Model families with a dedicated replacement (llama4, qwen3_5_moe, step3p5, ...) must
+ # go through `apply_replacements`, which runs its custom `from_original` against the
+ # *fused* module. Unfusing them here with the generic linear_loop path would leave
+ # that replacement facing a module whose fused params no longer exist.
+ if is_custom_model(model):
+ logger.debug(
+ "meta skeleton: %s has a dedicated MoE replacement, deferring the unfuse to apply_replacements",
+ getattr(getattr(model, "config", None), "model_type", None),
+ )
+ return []
+
+ try:
+ # This is where the structural change actually happens for a meta-built model, so
+ # report the before/after here rather than in `apply_replacements` (which by then
+ # has nothing left to do).
+ with log_moe_block_transition(model, "unfuse"):
+ unfused = _handle_moe_modules(model)
+ except Exception: # pragma: no cover - never block the meta build on this
+ logger.warning("Structural MoE unfuse on the meta skeleton failed", exc_info=True)
+ return []
+
+ if unfused:
+ logger.info("meta skeleton: structural MoE unfuse produced %d unfused experts modules", len(unfused))
+ return unfused
+
+
+def build_meta_model(model_name: str, trust_remote_code: bool = True, unfuse_moe: bool = True):
"""Build a meta-device model skeleton (~0 RAM) plus its tokenizer and a
SafetensorsIndex for on-demand materialization, instead of AutoRound's own
``llm_load_model(model_name, device_map="cpu")`` which fully materializes the
@@ -250,6 +650,10 @@ def build_meta_model(model_name: str, trust_remote_code: bool = True):
only covers the common local-directory ``AutoModelForCausalLM`` case (no
bagel/glm/mxfp4/HPU special-casing) -- callers should fall back to
``llm_load_model`` for anything this doesn't handle.
+
+ When ``unfuse_moe`` is set (the default) the fused ``transformers>=5`` MoE experts
+ are split into per-expert ``nn.Linear`` while still on meta, so the returned skeleton
+ is immediately quantizable and every later materialization is per-expert.
"""
from accelerate import init_empty_weights
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
@@ -269,6 +673,12 @@ def build_meta_model(model_name: str, trust_remote_code: bool = True):
model = model_cls(config)
else:
model = AutoModelForCausalLM.from_config(config, trust_remote_code=trust_remote_code)
+
+ # Unfuse while everything is still meta: free, and it makes the model-side
+ # parameter names line up 1:1 with the per-expert checkpoint tensors.
+ if unfuse_moe:
+ unfuse_meta_moe_(model)
+
# Constructing from config does not tie weights (that happens in
# from_pretrained), so tied params such as lm_head.weight -- absent from
# the checkpoint by design -- stay separate meta tensors and later crash
@@ -296,25 +706,38 @@ def materialize_non_block_params(
def _in_block(name: str) -> bool:
return any(name == p or name.startswith(p + ".") for p in block_prefixes)
+ # Tied output embeddings (lm_head.weight tied to embed_tokens.weight) are absent from
+ # the checkpoint by design -- the tie is re-established below, which makes them real.
+ # Do not warn about those, or every tied model reports a scary "leaving on meta" line
+ # for a parameter that is fine a few statements later.
+ tied_keys = set(getattr(model, "_tied_weights_keys", None) or ())
+
+ def _is_tied(name: str) -> bool:
+ return name in tied_keys or any(name.startswith(key.rstrip("*")) for key in tied_keys if key.endswith("*"))
+
targets = [] # (param_name, full_checkpoint_name)
+ deferred_tied = []
for name, tensor in list(model.named_parameters()) + list(model.named_buffers()):
if str(tensor.device) != "meta" or _in_block(name):
continue
full_name = name.replace(".orig_layer.", ".")
resolved_name = _resolve_checkpoint_name(index, full_name)
if resolved_name is None:
- logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)
+ if _is_tied(name):
+ deferred_tied.append(name)
+ else:
+ logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)
continue
targets.append((name, resolved_name))
- if not targets:
- return
- values = index.read_tensors([full_name for _, full_name in targets], device=device)
- for name, full_name in targets:
- # See the matching comment in materialize_module() -- explicit dtype=
- # is required so the checkpoint's real dtype wins over whatever the
- # meta skeleton happened to declare.
- set_module_tensor_to_device(model, name, device, value=values[full_name], dtype=values[full_name].dtype)
+ if targets:
+ values = index.read_tensors([full_name for _, full_name in targets], device=device)
+ for name, full_name in targets:
+ # See the matching comment in materialize_module() -- explicit dtype=
+ # is required so the checkpoint's real dtype wins over whatever the
+ # meta skeleton happened to declare.
+ set_module_tensor_to_device(model, name, device, value=values[full_name], dtype=values[full_name].dtype)
+
# set_module_tensor_to_device replaces parameter objects, which breaks
# weight tying: a tied lm_head still references the old (meta) parameter
# while the embedding points at the new real one. Re-tie so every tied
@@ -322,6 +745,20 @@ def materialize_non_block_params(
if getattr(getattr(model, "config", None), "tie_word_embeddings", False):
model.tie_weights()
+ # Anything declared tied that the re-tie did not actually make real is a genuine
+ # problem (quantizing it would fail on a meta tensor), so surface it now.
+ still_meta = [name for name in deferred_tied if _param_or_buffer(model, name) is None]
+ if still_meta:
+ logger.warning("No checkpoint tensor found for tied %s, leaving on meta", ", ".join(still_meta))
+
+
+def _param_or_buffer(model: nn.Module, name: str):
+ """Return the named parameter/buffer when it is materialized, else ``None``."""
+ for candidate_name, tensor in list(model.named_parameters()) + list(model.named_buffers()):
+ if candidate_name == name:
+ return None if tensor.device.type == "meta" else tensor
+ return None
+
class stream_block_forward:
"""Context manager: wrap every top-level decoder block's ``forward`` so it
diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.py
index 4f5ee3481..0cea78e8c 100644
--- a/auto_round/utils/offload.py
+++ b/auto_round/utils/offload.py
@@ -255,9 +255,9 @@ def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nn
# ``mlp.router.gate`` behind a ``mlp.gate`` module) and leaves them on
# meta until a downstream ``.to(device)`` crashes.
try:
- from auto_round.utils.disk_stream_util import SafetensorsIndex, materialize_module
+ from auto_round.utils.disk_stream_util import get_safetensors_index, materialize_module
- materialize_module(block, block_name, SafetensorsIndex(model_dir), device="cpu")
+ materialize_module(block, block_name, get_safetensors_index(model_dir), device="cpu")
if not _has_meta(block):
return
except FileNotFoundError:
diff --git a/test/unit/common/models/test_special_model_handler.py b/test/unit/common/models/test_special_model_handler.py
index 606a365ff..ad938ead5 100644
--- a/test/unit/common/models/test_special_model_handler.py
+++ b/test/unit/common/models/test_special_model_handler.py
@@ -983,26 +983,40 @@ class TestGetPredefinedIgnoreLayers:
class TestTorchCompileOff:
- """DeepSeek / GLM-5.3-Flash DSA families must not run under torch.compile."""
+ """DeepSeek / GLM-5.3-Flash DSA families must not run under torch.compile on torch < 2.14.0.
+
+ From torch 2.14.0 onward the recompile_limit issue is resolved, so these families no
+ longer force ``torch.compile`` off and instead honor the default / user-provided setting.
+ """
def _cfg(self, model_type="", architectures=None):
from types import SimpleNamespace
return SimpleNamespace(config=SimpleNamespace(model_type=model_type, architectures=architectures or []))
+ def _expected_off(self):
+ from auto_round.utils import torch_version_at_least
+
+ # On torch >= 2.14.0 the rules are not registered, so the reason is ``None``.
+ return not torch_version_at_least("2.14.0")
+
def test_deepseek_disabled(self):
from auto_round.special_model_handler import get_torch_compile_off_reason
for mt in ("deepseek_v3", "deepseek_v32"):
- assert get_torch_compile_off_reason(self._cfg(model_type=mt)) is not None
+ off = get_torch_compile_off_reason(self._cfg(model_type=mt)) is not None
+ assert off == self._expected_off()
def test_glm5_family_disabled(self):
from auto_round.special_model_handler import get_torch_compile_off_reason
- assert get_torch_compile_off_reason(self._cfg(model_type="glm5_next")) is not None
- assert get_torch_compile_off_reason(self._cfg(model_type="glm_moe_dsa")) is not None
+ expected = self._expected_off()
+ assert (get_torch_compile_off_reason(self._cfg(model_type="glm5_next")) is not None) == expected
+ assert (get_torch_compile_off_reason(self._cfg(model_type="glm_moe_dsa")) is not None) == expected
# Architecture name alone is enough (config before weights are materialized).
- assert get_torch_compile_off_reason(self._cfg(architectures=["Glm5NextForConditionalGeneration"])) is not None
+ assert (
+ get_torch_compile_off_reason(self._cfg(architectures=["Glm5NextForConditionalGeneration"])) is not None
+ ) == expected
def test_other_models_unaffected(self):
from auto_round.special_model_handler import get_torch_compile_off_reason
diff --git a/test/unit/common/utils/test_apply.py b/test/unit/common/utils/test_apply.py
index 6e8c1c9d0..3bc1c71e6 100644
--- a/test/unit/common/utils/test_apply.py
+++ b/test/unit/common/utils/test_apply.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import torch
import torch.nn as nn
from auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme
@@ -31,3 +32,34 @@ def test_apply_plan_is_the_explicit_model_write_boundary():
assert model[0].bits == 4
assert model[0].data_type == "int"
+
+
+def test_apply_plan_preserves_non_target_module_own_attributes():
+ """A non-quantized module's own attribute that happens to share a scheme-key
+ name (e.g. a grouped RMSNorm's ``group_size``) must not be stripped.
+
+ Regression: ``apply_plan_to_model`` used to ``delattr`` every scheme-key-named
+ attribute from *every* module, deleting ``Qwen4ExpTextRMSNorm.group_size`` and
+ crashing the norm's next forward with ``AttributeError``.
+ """
+
+ class GroupedRMSNorm(nn.Module):
+ def __init__(self, dim, group_size):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(dim))
+ self.group_size = group_size # legitimately owned by the model
+
+ model = nn.Sequential(nn.Linear(32, 32), GroupedRMSNorm(32, group_size=8))
+ plan = ResolvedQuantizationConfig(
+ scheme=ResolvedScheme.from_scheme(QuantizationScheme()),
+ formats=(),
+ layer_config={"0": {"bits": 4, "group_size": 128, "data_type": "int"}},
+ )
+
+ apply_plan_to_model(model, plan)
+
+ # Quantization target still receives the plan.
+ assert model[0].bits == 4
+ assert model[0].group_size == 128
+ # The norm keeps its own group_size, untouched by the scheme-key reset.
+ assert model[1].group_size == 8
diff --git a/test/unit/test_cpu/models/test_fused_moe_detection.py b/test/unit/test_cpu/models/test_fused_moe_detection.py
new file mode 100644
index 000000000..1f158c14d
--- /dev/null
+++ b/test/unit/test_cpu/models/test_fused_moe_detection.py
@@ -0,0 +1,100 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The meta-skeleton load path must only trigger for fused-3D MoE checkpoints.
+
+Building on meta pays off exactly when ``transformers`` would stack the checkpoint's
+per-expert 2D tensors into a fused 3D ``nn.Parameter`` that AutoRound then has to split
+back apart. Everything else (dense models, MoE families with a dedicated AutoRound
+replacement) must keep the ordinary load path, so the blast radius stays small.
+"""
+
+import pytest
+import transformers
+from packaging import version
+
+from auto_round.modeling.fused_moe.moe_experts_interface import (
+ _config_model_types,
+ config_has_fused_moe_experts,
+)
+from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES
+
+pytestmark = pytest.mark.skipif(
+ version.parse(transformers.__version__) < version.parse("5.0.0"),
+ reason="fused 3D MoE experts only exist in transformers>=5",
+)
+
+
+class _FakeConfig:
+ """Duck-typed stand-in so the test does not depend on every config class existing."""
+
+ def __init__(self, model_type, **sub_configs):
+ self.model_type = model_type
+ for name, value in sub_configs.items():
+ setattr(self, name, value)
+
+
+# Fused 3D experts: `from_pretrained` merges `experts.<i>.<proj>` into one parameter.
+FUSED_MOE_TYPES = ["qwen3_moe", "qwen2_moe", "deepseek_v3", "glm4_moe", "phimoe", "mixtral"]
+# Dense models have no expert conversion at all.
+DENSE_TYPES = ["llama", "qwen2", "gemma2", "mistral"]
+# The checkpoint already stores the fused 3D tensor, so there is no per-expert merge.
+ALREADY_FUSED_CHECKPOINT_TYPES = ["qwen3_vl_moe", "gpt_oss"]
+
+
[email protected]("model_type", FUSED_MOE_TYPES)
+def test_fused_moe_families_are_detected(model_type):
+ assert config_has_fused_moe_experts(_FakeConfig(model_type)) is True
+
+
[email protected]("model_type", DENSE_TYPES)
+def test_dense_models_are_not_detected(model_type):
+ assert config_has_fused_moe_experts(_FakeConfig(model_type)) is False
+
+
[email protected]("model_type", ALREADY_FUSED_CHECKPOINT_TYPES)
+def test_checkpoints_without_a_per_expert_merge_are_not_detected(model_type):
+ assert config_has_fused_moe_experts(_FakeConfig(model_type)) is False
+
+
+def test_detection_reaches_nested_text_config():
+ """A VLM registers the MoE rules on its text sub-config, not the top level."""
+ vlm = _FakeConfig("qwen3_5_moe", text_config=_FakeConfig("qwen3_5_moe_text"))
+
+ assert config_has_fused_moe_experts(_FakeConfig("qwen3_5_moe")) is False, "precondition: top level alone is inert"
+ assert config_has_fused_moe_experts(vlm) is True
+ assert "qwen3_5_moe_text" in _config_model_types(vlm)
+
+
+def test_detection_uses_real_transformers_configs():
+ """Same behaviour with genuine config objects, not just duck-typed ones."""
+ from transformers import LlamaConfig, Qwen3MoeConfig
+
+ assert config_has_fused_moe_experts(Qwen3MoeConfig()) is True
+ assert config_has_fused_moe_experts(LlamaConfig()) is False
+
+
+def test_families_with_a_dedicated_replacement_are_left_alone():
+ """`ModelContext` excludes these; they drive their own memory-aware materialization."""
+ expected = {
+ "llama4",
+ "deepseek_v2",
+ "step3p5",
+ "qwen3_omni_moe",
+ "qwen3_5_moe",
+ "qwen3_5_moe_text",
+ }
+ if version.parse(transformers.__version__) < version.parse("5.0.0"):
+ # These two only get a dedicated replacement on the pre-5.0 linear_loop path.
+ expected |= {"qwen3_vl_moe", "gpt_oss"}
+ assert set(BUILTIN_MODULES) == expected