PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[risky]reduce ram usage of loading moe without monkey patch and fix qwen3-flash-next group_size issue (#2291)
Wenhua Cheng7d8905efd0de
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex b85b157c7..e7d4dae5f 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -2326,18 +2326,18 @@ def _load_scheme_worker_model(model_name, use_model_replacements, low_cpu_mem_usdef _load_disk_stream_scheme_worker_model(model_name, use_model_replacements=False):"""Build an isolated meta model and checkpoint index for a scoring worker."""+ from auto_round.modeling.fused_moe.replace_modules import _handle_moe_modulesfrom auto_round.utils.disk_stream_util import build_meta_model+ # Build the meta skeleton, then structurally unfuse the fused ``transformers>=5`` MoE+ # experts (fused 3D expert params -> per-expert ``nn.Linear``) while everything is+ # still on meta, so per-expert quant layers resolve; weights stay on meta and are+ # filled per-block from the checkpoint index during scoring. The unfuse must happen+ # here in the worker: with the custom MoE replacements disabled, the meta-skeleton+ # build no longer guarantees it, and leaving the experts fused makes the per-expert+ # quant layers unresolvable during scoring.model, tokenizer, disk_index = build_meta_model(model_name)- # The regular load pipeline applies custom replacements, among them the- # structural MoE unfusing that turns fused expert containers into per-expert- # Linear modules; the meta-skeleton build skips that pipeline. Unfuse here- # so per-expert quant layers resolve -- weights stay on meta and are filled- # per-block from the checkpoint index during scoring.- from auto_round.modeling.fused_moe.replace_modules import _handle_moe_modules-- unfused = _handle_moe_modules(model)- logger.info("disk-stream scoring worker: structural MoE unfuse produced %d unfused experts modules", len(unfused))+ _handle_moe_modules(model)if use_model_replacements:from auto_round.special_model_handler import _handle_special_model, update_modulediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 562547469..b4c9437b2 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -451,6 +451,17 @@ class BaseOrchestrator(object):# blocks already hold real weights (see OffloadManager._reload).if self.model_context.disk_stream_model_dir is not None:self._offloader.model_dir = self.model_context.disk_stream_model_dir+ # A meta skeleton (explicit AR_DISK_STREAM_MODEL=1, or auto-selected for+ # fused-MoE checkpoints -- signalled by `_disk_stream_index`) leaves every+ # block on the meta device, so per-block reload from disk is mandatory: the+ # tuning loop must materialize each block before moving it to the compute+ # device. The offloader was created with `enabled=low_cpu_mem_usage`, which+ # can be False (e.g. user-supplied, or GGUF forcing it off later); in that+ # case reload() would no-op and `block.to(device)` would crash with+ # "Cannot copy out of meta tensor". Force it enabled here whenever streaming+ # is active so the reload path stays available regardless of low_cpu_mem_usage.+ if getattr(self.model_context, "_disk_stream_index", None) is not None:+ self._offloader.enabled = True# Alternatively, you can use CompressContext.create_contextself.compress_context = CompressContext(low_cpu_mem_usage,diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.pyindex f3cde69ec..48df479ed 100644--- a/auto_round/compressors/layer_config_resolver.py+++ b/auto_round/compressors/layer_config_resolver.py@@ -534,6 +534,20 @@ def apply_plan_to_model(model, plan: ResolvedQuantizationConfig) -> None:"""Apply a resolved plan at the single explicit module-attribute write boundary."""scheme_keys = tuple(field.name for field in fields(QuantizationScheme)) + ("scale_dtype",)for module_name, module in model.named_modules():+ # AutoRound only ever writes scheme attributes onto quantization-target+ # layers (plus model-level keys such as ``rotation_config`` on the root+ # module). Some scheme keys share a name with attributes a model class+ # legitimately owns -- e.g. a grouped RMSNorm's own ``group_size`` -- so+ # blindly clearing them from every module would corrupt the model+ # (``AttributeError`` on the norm's next forward). Scope the reset to the+ # same modules that receive the plan below.+ is_quant_target = (+ isinstance(module, SUPPORTED_LAYER_TYPES)+ or isinstance(module, torch.nn.Embedding)+ or module.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES+ )+ if module_name != "" and not is_quant_target:+ continuefor key in scheme_keys:if module_name == "" and key == "rotation_config":continuediff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.pyindex dea73bec6..f1a4215f5 100644--- a/auto_round/compressors/orchestrator.py+++ b/auto_round/compressors/orchestrator.py@@ -241,19 +241,24 @@ class CompressionOrchestrator(BaseOrchestrator):modules = [get_module(model, n) for n in names]m = WrapperMultiblock(modules)- # Also reload when `AR_DISK_STREAM_MODEL` is set even if- # `low_cpu_mem_usage` has been forced False (e.g. GGUF export --- # see base.py's `_finalize_compress_context`, which disables- # `low_cpu_mem_usage` for gguf formats for reasons unrelated to disk- # streaming). Under streaming, a block starts on the meta device- # regardless of `low_cpu_mem_usage`, which only ever controlled whether- # to *free* it again after use -- without this, the block below is never- # materialized at all and `m.to(device)` crashes with "Cannot copy out- # of meta tensor". The block intentionally stays real afterward (no- # matching post-tune offload runs when `low_cpu_mem_usage` is False --- # see the `is_immediate_saving`-adjacent offload call further down),- # matching upstream's own choice not to cycle blocks for these formats.- if self.compress_context.low_cpu_mem_usage or envs.AR_DISK_STREAM_MODEL:+ # Also reload when disk streaming is active even if `low_cpu_mem_usage`+ # has been forced False (e.g. GGUF export -- see base.py's+ # `_finalize_compress_context`, which disables `low_cpu_mem_usage` for+ # gguf formats for reasons unrelated to disk streaming). Disk streaming+ # can be turned on explicitly via `AR_DISK_STREAM_MODEL=1` *or* chosen+ # automatically for fused-MoE checkpoints (see ModelContext's+ # `_should_use_meta_skeleton`); either way the model was built as a meta+ # skeleton and `_disk_stream_index` is set. Under streaming, a block+ # starts on the meta device regardless of `low_cpu_mem_usage`, which only+ # ever controlled whether to *free* it again after use -- without this,+ # the block below is never materialized at all and `m.to(device)` crashes+ # with "Cannot copy out of meta tensor". The block intentionally stays+ # real afterward (no matching post-tune offload runs when+ # `low_cpu_mem_usage` is False -- see the `is_immediate_saving`-adjacent+ # offload call further down), matching upstream's own choice not to cycle+ # blocks for these formats.+ disk_streaming = getattr(self.model_context, "_disk_stream_index", None) is not None+ if self.compress_context.low_cpu_mem_usage or envs.AR_DISK_STREAM_MODEL or disk_streaming:if nblocks == 1:self._offloader.reload(model, n)else:@@ -440,6 +445,18 @@ class CompressionOrchestrator(BaseOrchestrator):pbar.set_description(f"Quantizing {block_name}")block = get_module(self.model, block_name)+ # ── Infrastructure: reload from disk when streaming ───────+ # Fused-MoE checkpoints (and explicit `AR_DISK_STREAM_MODEL=1`)+ # build an all-meta skeleton to reduce RAM: each decoder block+ # starts on the meta device and its real weights must be read+ # back from the checkpoint before quantization. The data-driven+ # path does this same reload; without it here the zero-shot+ # (RTN) path leaves the block on meta and `layer.to(device)`+ # crashes with "Cannot copy out of meta tensor".+ disk_streaming = getattr(self.model_context, "_disk_stream_index", None) is not None+ if self.compress_context.low_cpu_mem_usage or envs.AR_DISK_STREAM_MODEL or disk_streaming:+ self._offloader.reload(self.model, block_name)+# ── Infrastructure: materialize ───────────────────────────materialize_model_(block)diff --git a/auto_round/context/model.py b/auto_round/context/model.pyindex 63f62ba7e..0b399bf6d 100644--- a/auto_round/context/model.py+++ b/auto_round/context/model.py@@ -18,6 +18,7 @@ import osfrom typing import Any, Callable, Optional, Unionimport torch+import transformersfrom packaging import versionfrom transformers import AutoConfig@@ -183,7 +184,7 @@ class ModelContext(BaseContext):# __init__, exactly like the text path. Falls back to the# full mllm_load_model on any failure.loaded_via_meta = False- if envs.AR_DISK_STREAM_MODEL and os.path.isdir(self.model):+ if self._should_use_meta_skeleton(self._peek_config()):try:self.model, self.tokenizer, self._disk_stream_index = self._build_disk_stream_model(self.disk_stream_model_dir@@ -197,8 +198,7 @@ class ModelContext(BaseContext):loaded_via_meta = Trueexcept Exception:logger.warning(- "AR_DISK_STREAM_MODEL requested but building a multimodal meta "- "skeleton for %s failed; falling back to a full CPU load.",+ "Building a multimodal meta skeleton for %s failed; " "falling back to a full CPU load.",self.disk_stream_model_dir,exc_info=True,)@@ -224,7 +224,6 @@ class ModelContext(BaseContext):self.is_model_patched = apply_model_monkey_patches(model_name=self.model, trust_remote_code=self.trust_remote_code)- import transformersif (not self.is_model_patched@@ -248,15 +247,14 @@ class ModelContext(BaseContext):gc.collect()_force_trim_malloc()- if envs.AR_DISK_STREAM_MODEL:+ if self._should_use_meta_skeleton(config):try:self.model, self.tokenizer, self._disk_stream_index = self._build_disk_stream_model(self.disk_stream_model_dir)except Exception:logger.warning(- "AR_DISK_STREAM_MODEL requested but building a meta skeleton for %s failed; "- "falling back to a normal full CPU load.",+ "Building a meta skeleton for %s failed; falling back to a normal full CPU load.",self.disk_stream_model_dir,exc_info=True,)@@ -290,6 +288,90 @@ class ModelContext(BaseContext):if hasattr(m, "tuning_device"):delattr(m, "tuning_device")+ def _peek_config(self):+ """Best-effort config lookup before the model object exists."""+ if self.config is not None:+ return self.config+ if not isinstance(self.model, str):+ return getattr(self.model, "config", None)+ try:+ return AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)+ except Exception as exc:+ logger.debug("could not peek the config of %s: %s", self.model, exc)+ return None++ def _should_use_meta_skeleton(self, config) -> bool:+ """Whether to build a meta skeleton instead of a full CPU load.++ Deliberately narrow: this only targets ``transformers>=5`` models whose experts are+ stored as a *fused* 3D ``nn.Parameter``. For those, the checkpoint holds one 2D+ tensor per expert, ``from_pretrained`` stacks them into the fused parameter, and+ AutoRound then has to split it back into per-expert ``nn.Linear`` to quantize it --+ and splitting real tensors needs the fused tensor and its per-expert copies alive+ at once, i.e. ~2x one experts module on top of an already fully resident model.++ Building on meta makes that unfuse allocation-free and lets the tuning loop+ materialize one block at a time from the checkpoint. Every other model (dense, or+ a MoE that already ships as ``ModuleList`` of ``Linear``) has nothing to gain and+ keeps the ordinary load path. ``AR_DISK_STREAM_MODEL=1`` forces this on for any+ model; ``AR_DISABLE_AUTO_META_LOAD=1`` turns the automatic choice off.+ """+ if envs.AR_DISK_STREAM_MODEL:+ return True+ if envs.AR_DISABLE_AUTO_META_LOAD:+ return False+ if not isinstance(self.model, str):+ return False+ if version.parse(transformers.__version__) < version.parse("5.0.0"):+ return False+ if config is None:+ return False++ from auto_round.modeling.fused_moe.moe_experts_interface import config_has_fused_moe_experts+ from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES++ if not config_has_fused_moe_experts(config):+ return False+ # Families with a dedicated replacement drive their own (already memory-aware)+ # materialization; do not reroute them through the meta skeleton.+ model_type = getattr(config, "model_type", None)+ if model_type in BUILTIN_MODULES:+ logger.debug("meta-skeleton load skipped: %s has a dedicated MoE replacement", model_type)+ return False++ checkpoint_dir = self._resolve_local_checkpoint_dir()+ if checkpoint_dir is None:+ logger.debug("meta-skeleton load skipped: %s is not addressable as a local directory", self.model)+ return False+ self.disk_stream_model_dir = checkpoint_dir+ logger.info(+ "Fused-MoE checkpoint detected: building a meta skeleton and materializing weights per block "+ "(set AR_DISABLE_AUTO_META_LOAD=1 to load the whole model on CPU instead)."+ )+ return True++ def _resolve_local_checkpoint_dir(self) -> Optional[str]:+ """Return a local directory holding the checkpoint shards, or ``None``.++ ``SafetensorsIndex`` reads shards by path, so a hub repo id has to be resolved to+ a local snapshot first. Only the metadata + weights are fetched, and nothing is+ read into memory here -- ``from_pretrained`` would have downloaded the same files.+ """+ if os.path.isdir(self.model):+ return self.model+ if envs.AR_USE_MODELSCOPE:+ return None+ try:+ from huggingface_hub import snapshot_download++ return snapshot_download(+ self.model,+ allow_patterns=["*.safetensors", "*.safetensors.index.json", "*.json", "*.txt", "*.model"],+ )+ except Exception as exc:+ logger.debug("could not resolve %s to a local snapshot: %s", self.model, exc)+ return None+def _build_disk_stream_model(self, model_name: str):"""Build an all-meta skeleton instead offully materializing the checkpoint on CPU RAM. Left fully meta herediff --git a/auto_round/envs.py b/auto_round/envs.pyindex a30d016e3..f62f6f308 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -32,6 +32,7 @@ if TYPE_CHECKING:AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = TrueAR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = FalseAR_DISK_STREAM_MODEL: bool = False+ AR_DISABLE_AUTO_META_LOAD: bool = FalseAR_RESUME_DIR: Optional[str] = NoneAR_FORCE_MOE_ROUTING_ALL_EXPERTS: bool = FalseAR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = True@@ -126,6 +127,11 @@ environment_variables: dict[str, Callable[[], Any]] = {# block-by-block from disk during quantization instead of being fully# materialized on CPU RAM up front."AR_DISK_STREAM_MODEL": lambda: os.getenv("AR_DISK_STREAM_MODEL", "0").lower() in ("1", "true", "yes"),+ # AutoRound builds a meta skeleton automatically for transformers>=5 MoE checkpoints+ # (fused 3D experts must be split into per-expert Linear to be quantizable, and doing+ # that on real tensors costs ~2x one experts module on top of a fully resident model).+ # Set this to fall back to the old behavior of loading the whole model on CPU first.+ "AR_DISABLE_AUTO_META_LOAD": lambda: os.getenv("AR_DISABLE_AUTO_META_LOAD", "0").lower() in ("1", "true", "yes"),# When set to a directory path, the per-block tuning loop checkpoints its# progress there after each completed block, and resumes from the first# not-yet-completed block on a fresh run against the same directory --diff --git a/auto_round/export/export_to_gguf/conversion/base.py b/auto_round/export/export_to_gguf/conversion/base.pyindex 56547ace0..db916b51e 100644--- a/auto_round/export/export_to_gguf/conversion/base.py+++ b/auto_round/export/export_to_gguf/conversion/base.py@@ -234,7 +234,7 @@ class ModelBase:index_file = self.dir_model / index_nameif index_file.is_file():- logger.info(f"gguf: loading model weight map from '{index_name}'")+ logger.debug(f"gguf: loading model weight map from '{index_name}'")with open(index_file, "r", encoding="utf-8") as f:index: dict[str, Any] = json.load(f)weight_map = index.get("weight_map")@@ -249,7 +249,7 @@ class ModelBase:weight_map = {}for part_name in part_names:- logger.info(f"gguf: indexing model part '{part_name}'")+ logger.debug(f"gguf: indexing model part '{part_name}'")ctx: ContextManager[Any]if is_safetensors:ctx = cast(ContextManager[Any], gguf.utility.SafetensorsLocal(self.dir_model / part_name))diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.pyindex 5393d183d..7d2dd065e 100644--- a/auto_round/modeling/fused_moe/moe_experts_interface.py+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py@@ -326,33 +326,48 @@ def _detect_expert_projections(module: nn.Module) -> dict[str, dict]:This function scans the module for any 3D nn.Parameter attributes.It first checks known projection names, then discovers any unknown 3D parameters.+ Reads ``module._parameters`` directly instead of ``getattr``/``dir()``. This runs once+ per module in the model, and after unfusing a large MoE that is tens of thousands of+ modules: ``dir()`` on an ``nn.Module`` is expensive, and every miss goes through+ ``nn.Module.__getattr__``, which searches three dicts before raising. A parameter is+ always registered in ``_parameters``, so the cheap membership test is equivalent.+Returns:Dict mapping projection names to their config, only for projections that existas 3D nn.Parameter in the module."""+ parameters = getattr(module, "_parameters", None)+ if not parameters:+ return {}++ # Fast reject: an experts module owns at least one 3D parameter. Almost every module+ # in the model (Linear, norms, containers) fails here in a single dict scan.+ three_dim = {+ name: param+ for name, param in parameters.items()+ if param is not None and not name.startswith("_") and param.dim() == 3+ }+ if not three_dim:+ return {}+detected = {}# First, check known projection patternsfor proj_name, config in KNOWN_PROJECTION_PATTERNS.items():- param = getattr(module, proj_name, None)- if param is not None and isinstance(param, nn.Parameter) and param.dim() == 3:+ if proj_name in three_dim:detected[proj_name] = config- # If no known patterns found, scan for any 3D Parameter (future-proofing),- # but exclude known non-expert 3D parameters that exist on some transformer- # architectures (e.g. scale_shift_table on WanTransformer3DModel).+ # If no known patterns found, take any 3D parameter (future-proofing), but exclude+ # known non-expert 3D parameters that exist on some transformer architectures+ # (e.g. scale_shift_table on WanTransformer3DModel)._NON_EXPERT_3D_PARAMS = frozenset(["scale_shift_table"])if not detected:- for attr_name in dir(module):- if attr_name.startswith("_"):- continue- if attr_name in _NON_EXPERT_3D_PARAMS:+ for name in three_dim:+ if name in _NON_EXPERT_3D_PARAMS:continue- param = getattr(module, attr_name, None)- if param is not None and isinstance(param, nn.Parameter) and param.dim() == 3:- # Use default config for unknown projections- logger.debug(f"Discovered unknown 3D projection: {attr_name}")- detected[attr_name] = {"is_input_proj": True, "output_multiplier": 1}+ # Use default config for unknown projections+ logger.debug(f"Discovered unknown 3D projection: {name}")+ detected[name] = {"is_input_proj": True, "output_multiplier": 1}return detected@@ -385,6 +400,27 @@ def _infer_dimensions(param: nn.Parameter, config: dict, is_transposed: bool) ->return in_features, out_features+def _new_meta_linear(in_features: int, out_features: int, has_bias: bool, dtype: torch.dtype) -> nn.Linear:+ """Build an ``nn.Linear`` shell on meta without running its initializer.++ ``nn.Linear.__init__`` calls ``reset_parameters`` -> ``kaiming_uniform_``, whose result+ is always discarded here: the weight is either left on meta for the checkpoint loader+ to fill, or replaced with a checkpoint slice on the very next line. With hundreds of+ experts per layer (256 x 3 projections x 40 layers for Qwen3.5-MoE) that initializer+ dominates the unfuse, so construct the module directly instead.+ """+ linear = nn.Linear.__new__(nn.Linear)+ nn.Module.__init__(linear)+ linear.in_features = in_features+ linear.out_features = out_features+ linear.weight = nn.Parameter(torch.empty((out_features, in_features), dtype=dtype, device="meta"))+ if has_bias:+ linear.bias = nn.Parameter(torch.empty(out_features, dtype=dtype, device="meta"))+ else:+ linear.register_parameter("bias", None)+ return linear++def _unfuse_single_projection(module: nn.Module,proj_name: str,@@ -456,7 +492,7 @@ def _unfuse_single_projection(linears = []for i in range(num_experts):# meta device: creates the module structure without allocating weight storage- linear = nn.Linear(in_features, out_features, bias=has_bias, dtype=dtype, device="meta")+ linear = _new_meta_linear(in_features, out_features, has_bias, dtype)if not is_meta:# Direct parameter assignment — no copy, just references the CPU tensor slice@@ -482,6 +518,33 @@ def _unfuse_single_projection(_logged_memory_before_replacement = False # To ensure we only log memory once before replacements+def _warn_if_unfusing_allocates(module: nn.Module, detected_projections: dict[str, dict]) -> None:+ """Warn when the fused expert weights are not on meta.++ Unfusing is free on meta. On a real device the fused tensor and the per-expert copies+ must coexist for the duration of the split, so the peak is inherently ~2x one experts+ module on top of an already resident model.+ """+ params = [+ param+ for param in (getattr(module, name, None) for name in detected_projections)+ if isinstance(param, nn.Parameter) and param.device.type != "meta"+ ]+ if not params:+ return++ total_bytes = sum(param.numel() * param.element_size() for param in params)+ logger.warning(+ "[MoE Prep] fused expert weights are on %s (%.2f GB for '%s' alone), not meta. "+ "Splitting them in memory needs the fused tensor and its per-expert copies at the "+ "same time. AutoRound normally avoids this by building a meta skeleton and reading "+ "each expert from disk; that requires a local checkpoint directory.",+ params[0].device.type,+ total_bytes / 1024**3,+ module.__class__.__name__,+ )++def _unfuse_experts_weights_inplace(module: nn.Module,check_decorator: bool = True,@@ -521,6 +584,7 @@ def _unfuse_experts_weights_inplace(global _logged_memory_before_replacementif not _logged_memory_before_replacement:_logged_memory_before_replacement = True+ _warn_if_unfusing_allocates(module, detected_projections)memory_monitor.update()memory_monitor.log_summary("Before applying custom replacements")@@ -697,3 +761,82 @@ def prepare_model_for_moe_quantization(def is_linear_loop_available() -> bool:"""Check if linear_loop experts implementation is available."""return HAS_EXPERTS_INTERFACE+++def _config_model_types(config) -> list[str]:+ """Collect ``model_type`` from a config and every nested sub-config.++ A VLM keeps the MoE rules on the text sub-config+ (``text_config.model_type == 'qwen3_5_moe_text'``), so the top-level type alone is+ not enough. Uses ``sub_configs``/``get_text_config`` when available (the supported+ transformers API) and falls back to scanning ``__dict__`` for config-like values.+ """+ found: list[str] = []+ seen: set[int] = set()++ def _walk(node) -> None:+ if node is None or id(node) in seen:+ return+ seen.add(id(node))++ model_type = getattr(node, "model_type", None)+ if isinstance(model_type, str) and model_type not in found:+ found.append(model_type)++ children = []+ # Preferred: the declared sub-config names (transformers >= 4.4x).+ for name in getattr(type(node), "sub_configs", None) or ():+ children.append(getattr(node, name, None))+ get_text_config = getattr(node, "get_text_config", None)+ if callable(get_text_config):+ try:+ children.append(get_text_config())+ except Exception: # pragma: no cover - defensive+ pass+ # Fallback for plain/duck-typed configs.+ for value in (vars(node).values() if hasattr(node, "__dict__") else ()):+ if getattr(value, "model_type", None) is not None:+ children.append(value)++ for child in children:+ if child is not None and child is not node:+ _walk(child)++ _walk(config)+ return found+++def config_has_fused_moe_experts(config) -> bool:+ """Whether loading this config would build fused 3D expert parameters.++ ``transformers>=5`` describes such models with a ``WeightConverter`` that stacks the+ checkpoint's per-expert 2D tensors into one fused parameter (``MergeModulelist``).+ That converter is exactly the signal we care about: its presence means+ ``from_pretrained`` would produce a fused ``nn.Parameter`` that AutoRound then has to+ split back into per-expert ``nn.Linear`` -- the expensive case worth building on meta+ for. MoE models that already ship as ``ModuleList`` of ``Linear`` have no such+ converter and are deliberately left on the ordinary load path.+ """+ try:+ from transformers.conversion_mapping import get_checkpoint_conversion_mapping+ from transformers.core_model_loading import MergeModulelist, WeightConverter+ except ImportError: # pragma: no cover - transformers < 5+ return False++ for model_type in _config_model_types(config):+ mapping = get_checkpoint_conversion_mapping(model_type)+ if not mapping:+ continue+ for entry in mapping:+ if not isinstance(entry, WeightConverter):+ continue+ if not any(isinstance(op, MergeModulelist) for op in entry.operations):+ continue+ if any("experts" in pattern for pattern in entry.source_patterns):+ logger.debug(+ "[MoE Prep] '%s' merges per-expert checkpoint tensors into a fused parameter (%s)",+ model_type,+ entry.target_patterns,+ )+ return True+ return Falsediff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex 1276b89e5..3896402c2 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -13,12 +13,12 @@# limitations under the License.from abc import ABC, abstractmethod+from contextlib import contextmanagerfrom dataclasses import dataclassfrom typing import Dict, Typeimport torchfrom tqdm import tqdm-from transformers import PreTrainedModelfrom auto_round.modeling.fused_moe.utils import is_linearized_layoutfrom auto_round.utils import (@@ -121,13 +121,46 @@ def is_custom_model(model: torch.nn.Module) -> bool:return False-def _log_first_moe_block(model: torch.nn.Module, label: str) -> None:- """Log the first experts module found in the model for debugging."""+def _find_first_moe_block(model: torch.nn.Module) -> tuple[str, torch.nn.Module] | tuple[None, None]:+ """Return ``(name, module)`` of the first experts-like module, or ``(None, None)``."""for name, module in model.named_modules():if name.endswith(".experts") or name.endswith(".moe"):- logger.info(f"Experts ({label}) [{name}] ({module.__class__.__name__}):\n{module}")- return True- return False+ return name, module+ return None, None+++def _describe_moe_block(model: torch.nn.Module, name: str) -> str | None:+ """Render the experts module at ``name`` for the before/after comparison."""+ try:+ module = model.get_submodule(name)+ except AttributeError:+ return None+ return f"({module.__class__.__name__}):\n{module}"+++@contextmanager+def log_moe_block_transition(model: torch.nn.Module, label: str = "replacement"):+ """Log the first experts module before and after a structural change.++ The structural unfuse happens in one of two places depending on how the model was+ built: on the meta skeleton (``build_meta_model``) or during ``apply_replacements``.+ Both wrap themselves in this so the transition is reported wherever it actually+ occurs, and only when something really changed -- printing an identical before/after+ pair (which is what the other path sees, since it then has nothing left to do) is+ just noise.+ """+ name, _ = _find_first_moe_block(model)+ before = _describe_moe_block(model, name) if name is not None else None+ try:+ yield+ finally:+ if name is not None:+ after = _describe_moe_block(model, name)+ if after is not None and after != before:+ logger.info(f"Experts (before {label}) [{name}] {before}")+ logger.info(f"Experts (after {label}) [{name}] {after}")+ else:+ logger.debug(f"Experts [{name}] left unchanged by {label} {before}")@dump_mem_usage("Materializing model", log_level="debug")@@ -138,18 +171,27 @@ def materialize_model_(model: torch.nn.Module) -> None:model.apply(_materialize_module)- # check if any module on meta device remains- found_meta = False- for name, param in model.named_parameters():- if param.device.type == "meta":- logger.warning(f"Parameter {name} is still on meta device after materialization.")- found_meta = True- for name, buffer in model.named_buffers():- if buffer.device.type == "meta":- logger.warning(f"Buffer {name} is still on meta device after materialization.")- found_meta = True- if not found_meta:- logger.debug("All parameters and buffers have been materialized from meta device.")+ # Any tensor still on meta after per-block materialization means the meta-skeleton+ # loader could not resolve it from the checkpoint (e.g. a checkpoint-conversion+ # layout AutoRound does not reverse yet). Leaving it on meta only defers the failure+ # to a cryptic "Cannot copy out of meta tensor" crash on the next `.to(device)`, so+ # fail fast with an actionable message.+ still_meta = [name for name, param in model.named_parameters() if param.device.type == "meta"]+ still_meta += [name for name, buffer in model.named_buffers() if buffer.device.type == "meta"]+ if still_meta:+ from auto_round import envs++ preview = ", ".join(still_meta[:8]) + (f" (+{len(still_meta) - 8} more)" if len(still_meta) > 8 else "")+ # The auto meta-skeleton is only enabled for fused-MoE checkpoints; an explicit+ # AR_DISK_STREAM_MODEL=1 forces it for anything else. Point at whichever toggle+ # actually turned it on so the hint stays correct (AR_DISABLE_AUTO_META_LOAD is+ # the MoE-only auto path).+ hint = "unset AR_DISK_STREAM_MODEL" if envs.AR_DISK_STREAM_MODEL else "set AR_DISABLE_AUTO_META_LOAD=1"+ raise RuntimeError(+ f"Failed to materialize {len(still_meta)} checkpoint tensor(s), still on meta: {preview}. "+ f"To load the whole model on CPU instead, {hint} (uses more RAM)."+ )+ logger.debug("All parameters and buffers have been materialized from meta device.")release_original_module_(model)@@ -319,25 +361,14 @@ def apply_replacements(The model with modules replaced."""_import_required_replacements(model)- _raw_expert_is_logged = False-- # Custom replacements first- if is_custom_model(model):-- if not _raw_expert_is_logged:- _raw_expert_is_logged = _log_first_moe_block(model, "before replacement")-- _apply_custom_replacements(model, gguf_export=gguf_export)-- if auto_detect_moe and is_transformers_version_greater_or_equal_5():-- if not _raw_expert_is_logged:- _raw_expert_is_logged = _log_first_moe_block(model, "before replacement")- _handle_moe_modules(model)+ with log_moe_block_transition(model, "replacement"):+ # Custom replacements first+ if is_custom_model(model):+ _apply_custom_replacements(model, gguf_export=gguf_export)- if _raw_expert_is_logged:- _log_first_moe_block(model, "after replacement/skip")+ if auto_detect_moe and is_transformers_version_greater_or_equal_5():+ _handle_moe_modules(model)return modeldiff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 4e1212640..73750a8de 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -22,7 +22,7 @@ import torchfrom auto_round.export.formats import OutputFormatfrom auto_round.modeling.fused_moe.replace_modules import apply_replacements, release_original_module_-from auto_round.utils import is_moe_model_via_config, logger+from auto_round.utils import is_moe_model_via_config, logger, torch_version_at_leastmllms_with_limited_bs = ("llava",@@ -1217,24 +1217,27 @@ def get_torch_compile_off_reason(model) -> str | None:# DeepSeek V2/V3/V3.2/V4 and the GLM-5.3-Flash family (``glm_moe_dsa`` text /# ``glm5_next`` multimodal) share a DSA indexer + fused MoE with shape-dependent-# control flow. Compiling the tuning graph there hits dynamo's recompile_limit and-# provides little benefit, so keep those runs in eager mode.-register_torch_compile_off(- matchers=[- ModelTypeMatcher("deepseek", mode="in"),- ArchitectureMatcher("Deepseek", mode="in"),- ArchitectureMatcher("DeepSeek", mode="in"),- ],- reason="the DeepSeek architecture is incompatible with torch.compile during tuning",-)-register_torch_compile_off(- matchers=[- ModelTypeMatcher(r"glm_moe_dsa", mode="full"),- ModelTypeMatcher(r"glm5_next", mode="full"),- ArchitectureMatcher(r"Glm5Next", mode="in"),- ],- reason="the GLM-5 architecture is incompatible with torch.compile during tuning",-)+# control flow. On torch < 2.14.0 compiling the tuning graph there hits dynamo's+# recompile_limit and provides little benefit, so keep those runs in eager mode.+# From torch 2.14.0 onward these constraints are resolved, so we no longer force+# ``torch.compile`` off and instead honor the default / user-provided setting.+if not torch_version_at_least("2.14.0"):+ register_torch_compile_off(+ matchers=[+ ModelTypeMatcher("deepseek", mode="in"),+ ArchitectureMatcher("Deepseek", mode="in"),+ ArchitectureMatcher("DeepSeek", mode="in"),+ ],+ reason="the DeepSeek architecture is incompatible with torch.compile during tuning on torch < 2.14.0",+ )+ register_torch_compile_off(+ matchers=[+ ModelTypeMatcher(r"glm_moe_dsa", mode="full"),+ ModelTypeMatcher(r"glm5_next", mode="full"),+ ArchitectureMatcher(r"Glm5Next", mode="in"),+ ],+ reason="the GLM-5 architecture is incompatible with torch.compile during tuning on torch < 2.14.0",+ )def _attach_gemma4_rotary_emb(model):diff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.pyindex 3da7864e6..4751fac17 100644--- a/auto_round/utils/disk_stream_util.py+++ b/auto_round/utils/disk_stream_util.py@@ -13,6 +13,7 @@ from __future__ import annotationsimport jsonimport loggingimport re+from functools import lru_cachefrom pathlib import Pathfrom typing import Dict@@ -77,6 +78,17 @@ class SafetensorsIndex:return [n for n in self.weight_map if n == prefix or n.startswith(dotted)]+@lru_cache(maxsize=8)+def get_safetensors_index(checkpoint_dir: str) -> "SafetensorsIndex":+ """Shared ``SafetensorsIndex`` per checkpoint directory.++ Only the (cheap) name->shard map is cached; no ``safe_open`` handle is kept, so this+ does not reintroduce the mmap-residency problem ``SafetensorsIndex`` avoids. Callers+ used to build a fresh index for every block, re-reading the index JSON 40+ times.+ """+ return SafetensorsIndex(str(checkpoint_dir))++# Model-side module names sometimes differ from the checkpoint-side tensor# names. Transformers already maintains the authoritative per-family renames# (``transformers/conversion_mapping.py``, checkpoint -> model direction);@@ -84,12 +96,22 @@ class SafetensorsIndex:# are inverted here. A rename is applied only when the model-side name is# absent from the index AND the renamed candidate exists in it, so checkpoints# whose names already match the model are untouched.-from functools import lru_cache@lru_cache(maxsize=None)def _reverse_renamings_for(model_type):- """Invert the checkpoint-conversion WeightRenaming entries for one family."""+ """Invert the checkpoint-conversion WeightRenaming entries for one family.++ ``transformers`` owns the checkpoint->model renames, so reversing them is best left to+ ``WeightTransform.reverse_transform()``: it handles capturing groups and the+ ``PrefixChange`` special cases correctly. Naively reusing a ``target_pattern`` as a+ regex pattern does not -- the targets are *replacement* strings and a ``\\1``+ backreference in one (e.g. ``model.language_model.\\1``) raises+ ``re.PatternError: invalid group reference``.++ Returns reversed ``WeightTransform`` objects; call ``rename_source_key(model_side_name)``+ on them to get the checkpoint-side name.+ """if not model_type:return ()try:@@ -99,40 +121,303 @@ def _reverse_renamings_for(model_type):mapping = get_checkpoint_conversion_mapping(model_type)if not mapping:return ()- renamings = []+ reversed_transforms = []for entry in mapping:if not isinstance(entry, WeightRenaming):- # Other converter types (fusions, prefix moves) are not invertible- # by simple regex substitution and are skipped.+ # Other converter types (expert fusions) are handled separately, by+ # `_renamed_expert_candidates` and `materialize_module`'s fused lookup.continue- for target in entry.target_patterns:- for source in entry.source_patterns:- renamings.append((target, source))- return tuple(renamings)+ try:+ reversed_transforms.append(entry.reverse_transform())+ except Exception as exc: # pragma: no cover - not every transform is reversible+ logger.debug("skipping non-reversible rename %s: %s", entry, exc)+ return tuple(reversed_transforms)@lru_cache(maxsize=None)-def _index_model_type(index):- config_path = Path(index.checkpoint_dir) / "config.json"+def _model_types_for_dir(checkpoint_dir: str):+ """Every model_type in the checkpoint's config, including nested sub-configs.++ A VLM config nests the text model (``text_config.model_type = 'qwen3_5_moe_text'``)+ and that is where the MoE expert conversion rules are registered, so the top-level+ ``model_type`` alone is not enough to resolve expert tensor names.++ Keyed by directory rather than by ``SafetensorsIndex`` instance: callers build a fresh+ index per block, so caching on the object would both miss every time and pin every+ index (and its ``weight_map``) in memory for the whole run.+ """try:- with open(config_path) as f:- return json.load(f).get("model_type")+ with open(Path(checkpoint_dir) / "config.json") as f:+ config = json.load(f)except OSError:+ return ()++ found: list[str] = []++ def _walk(node):+ if not isinstance(node, dict):+ return+ model_type = node.get("model_type")+ if isinstance(model_type, str) and model_type not in found:+ found.append(model_type)+ for value in node.values():+ if isinstance(value, dict):+ _walk(value)++ _walk(config)+ return tuple(found)+++def _index_model_type(index):+ model_types = _model_types_for_dir(str(getattr(index, "checkpoint_dir", None)))+ return model_types[0] if model_types else None+++def _index_model_types(index):+ # Some lightweight/test indices carry no ``checkpoint_dir``; degrade to no+ # known model types rather than raising AttributeError.+ return _model_types_for_dir(str(getattr(index, "checkpoint_dir", None)))+++# Model-side per-expert projection -> (fused checkpoint projection, index within it).+# ``gate_proj``/``up_proj`` come from splitting a fused ``gate_up_proj`` in that order,+# which matches the order of the corresponding WeightConverter ``source_patterns``.+_MODEL_PROJ_TO_FUSED = {+ "gate_proj": ("gate_up_proj", 0),+ "up_proj": ("gate_up_proj", 1),+ "down_proj": ("down_proj", 0),+}++# On-disk layout of fused expert tensors, keyed by checkpoint ``model_type``.+# Two axes vary between families:+# * ``checkpoint_transposed`` -- the per-expert 2-D slice is stored as+# ``[in_features, out_features]`` (matmul layout, ``x @ W``) instead of the+# ``nn.Linear`` ``[out_features, in_features]`` weight layout, so it must be+# transposed before being loaded into the unfused per-expert Linear.+# * ``gate_up_interleaved`` -- the fused ``gate_up_proj`` interleaves the gate+# and up columns (``gate = fused[..., 0::2]``, ``up = fused[..., 1::2]``)+# instead of storing them as two contiguous halves.+# The transpose axis is normally inferred from the *target* parameter shape and+# does not need this table; the table is only consulted (a) to pick the+# interleave axis, which is invisible to shapes, and (b) as a tiebreak for the+# rare square/ambiguous shapes where transposed and non-transposed slices share+# the same shape. Families not listed here default to the non-transposed,+# contiguous ``[N, 2*inter, hidden]`` layout (qwen3_moe / phimoe / mixtral-style).+_FUSED_EXPERT_LAYOUTS = {+ # model_type: (checkpoint_transposed, gate_up_interleaved)+ "gpt_oss": (True, True),+ "llama4": (True, False),+}+++def _fused_expert_layout(index) -> tuple[bool, bool]:+ """Return the ``(checkpoint_transposed, gate_up_interleaved)`` hint for ``index``.++ The transpose flag is only a *hint* (shape inference takes precedence); the+ interleave flag is authoritative since it cannot be recovered from shapes.+ Unknown model types default to ``(False, False)``.+ """+ for model_type in _index_model_types(index):+ if model_type in _FUSED_EXPERT_LAYOUTS:+ return _FUSED_EXPERT_LAYOUTS[model_type]+ return (False, False)+++def _slice_fused_expert(fused, proj, attr, is_gate_up, target_shape, transposed_hint, interleaved):+ """Slice one unfused per-expert tensor out of a fused per-expert tensor.++ ``fused`` is a single expert's slice of the fused checkpoint tensor (2-D for a+ weight, 1-D for a bias). ``target_shape`` is the shape the unfused+ ``nn.Linear`` parameter expects. Whether the checkpoint stores the weight+ transposed is inferred from ``target_shape`` first; ``transposed_hint`` only+ breaks ambiguous (square) ties, and ``interleaved`` selects the gate/up split+ order. Returns ``None`` when no slice can produce ``target_shape`` (the caller+ then leaves the parameter on meta and reports an actionable error).+ """+ if attr == "bias":+ # 1-D per-expert bias; only gate/up need splitting.+ if not is_gate_up:+ return fused.contiguous()+ if interleaved:+ return (fused[0::2] if proj == "gate_proj" else fused[1::2]).contiguous()+ half = fused.shape[0] // 2+ return (fused[:half] if proj == "gate_proj" else fused[half:]).contiguous()++ if fused.ndim != 2:return None+ if not is_gate_up: # down_proj weight, target (out, in)+ reversed_shape = (target_shape[1], target_shape[0])+ if target_shape == reversed_shape:+ transposed = transposed_hint # square: shape can't decide+ elif tuple(fused.shape) == target_shape:+ transposed = False+ elif tuple(fused.shape) == reversed_shape:+ transposed = True+ else:+ return None+ return fused.t().contiguous() if transposed else fused.contiguous()++ # gate/up weight, target (out, in) = (inter, hidden)+ out, in_features = target_shape+ non_transposed = tuple(fused.shape) == (2 * out, in_features)+ transposed = tuple(fused.shape) == (in_features, 2 * out)+ if non_transposed and transposed:+ transposed = transposed_hint # square-ish (2*out == in): shape can't decide+ non_transposed = not transposed+ if non_transposed:+ return (fused[:out] if proj == "gate_proj" else fused[out:]).contiguous()+ if transposed:+ # fused is [in_features, 2*out]; split along the 2*out dim, then transpose+ # back to the [out, in_features] nn.Linear weight layout.+ if interleaved:+ sliced = fused[:, 0::2] if proj == "gate_proj" else fused[:, 1::2]+ else:+ sliced = fused[:, :out] if proj == "gate_proj" else fused[:, out:]+ return sliced.t().contiguous()+ return None+++_MODEL_SIDE_EXPERT_RE = re.compile(+ r"^(?P<prefix>.*\.experts)\.(?P<expert>\d+)\.(?P<proj>[A-Za-z0-9_]+)\.(?P<attr>weight|bias)$"+)+++@lru_cache(maxsize=None)+def _expert_projection_renames_for(model_type):+ """Map a fused projection to the checkpoint-side per-expert projection names.++ ``transformers`` describes MoE experts with a ``WeightConverter`` whose+ ``source_patterns`` are the per-expert checkpoint keys and whose single+ ``target_pattern`` is the fused model-side parameter, e.g. for phimoe::++ source_patterns = [".experts.*.w1.weight", ".experts.*.w3.weight"]+ target_patterns = ".experts.gate_up_proj"++ After AutoRound unfuses the experts the model-side names are+ ``experts.<i>.gate_proj.weight`` / ``experts.<i>.up_proj.weight``, so we need+ ``(gate_up_proj, 0) -> "w1.weight"`` and ``(gate_up_proj, 1) -> "w3.weight"`` to find+ the tensors back on disk. Returns a tuple of ``((fused, index), suffix)`` pairs.+ """+ if not model_type:+ return ()+ try:+ from transformers.conversion_mapping import get_checkpoint_conversion_mapping+ from transformers.core_model_loading import MergeModulelist, WeightConverter+ except ImportError: # pragma: no cover - transformers < 5+ return ()++ mapping = get_checkpoint_conversion_mapping(model_type)+ if not mapping:+ return ()++ renames: dict[tuple[str, int], str] = {}+ for entry in mapping:+ if not isinstance(entry, WeightConverter):+ continue+ # Only expert merges (`experts.*.<proj>` -> fused 3D) are relevant here.+ if not any(isinstance(op, MergeModulelist) for op in entry.operations):+ continue+ if len(entry.target_patterns) != 1:+ continue+ fused = entry.target_patterns[0].rstrip("$").rsplit(".", 1)[-1]+ for position, source_pattern in enumerate(entry.source_patterns):+ # ".experts.*.w1.weight" -> "w1.weight"+ _, star, suffix = source_pattern.partition("*.")+ if not star:+ continue+ renames.setdefault((fused, position), suffix.rstrip("$"))+ return tuple(renames.items())+++@lru_cache(maxsize=None)+def _concat_converters_for(model_type):+ """Model-side params assembled by concatenating several checkpoint tensors.++ Some families describe a single model-side parameter as the concatenation of+ several separately-stored checkpoint tensors, via a ``WeightConverter`` whose+ ``operations`` contain a ``Concatenate`` (and *no* ``MergeModulelist`` -- that is the+ expert-fusion case, handled elsewhere). GLM-5.3-Flash (``glm5_next``) is the motivating+ example: ``self_attn.conv1d.weight`` is ``cat([q_conv1d.weight, k_conv1d.weight,+ v_conv1d.weight], dim=0)``. The meta skeleton only knows the fused model-side name, so+ map it back to its checkpoint-side sources to materialize it.++ Returns a tuple of ``(target_suffix, (source_suffixes...), concat_dim)`` triples, using+ only literal (wildcard-free) patterns -- wildcard/expert converters are handled by the+ expert-specific helpers.+ """+ if not model_type:+ return ()+ try:+ from transformers.conversion_mapping import get_checkpoint_conversion_mapping+ from transformers.core_model_loading import Concatenate, MergeModulelist, WeightConverter+ except ImportError: # pragma: no cover - transformers < 5+ return ()++ mapping = get_checkpoint_conversion_mapping(model_type)+ if not mapping:+ return ()++ converters = []+ for entry in mapping:+ if not isinstance(entry, WeightConverter):+ continue+ # Expert fusions (MergeModulelist) are handled by `_renamed_expert_candidates`+ # and `materialize_module`'s fused lookup.+ if any(isinstance(op, MergeModulelist) for op in entry.operations):+ continue+ concat = next((op for op in entry.operations if isinstance(op, Concatenate)), None)+ if concat is None or len(entry.target_patterns) != 1:+ continue+ target = entry.target_patterns[0].rstrip("$")+ sources = tuple(src.rstrip("$") for src in entry.source_patterns)+ # Only literal patterns here; wildcard (`*`) converters are per-expert.+ if "*" in target or any("*" in src for src in sources):+ continue+ converters.append((target, sources, getattr(concat, "dim", 0)))+ return tuple(converters)+++def _renamed_expert_candidates(index, full_name: str):+ """Checkpoint-name candidates for an unfused per-expert parameter."""+ match = _MODEL_SIDE_EXPERT_RE.match(full_name)+ if match is None:+ return++ fused_position = _MODEL_PROJ_TO_FUSED.get(match["proj"])+ if fused_position is None:+ return+ prefix, expert = match["prefix"], match["expert"]++ for model_type in _index_model_types(index):+ for key, suffix in _expert_projection_renames_for(model_type):+ if key != fused_position:+ continue+ # "w1.weight" keeps the checkpoint's own attribute name; drop it for a bias.+ suffix_attr = suffix.rsplit(".", 1)+ if len(suffix_attr) == 2:+ suffix = f"{suffix_attr[0]}.{match['attr']}"+ yield f"{prefix}.{expert}.{suffix}"+def _resolve_checkpoint_name(index, full_name: str):"""Map a model-side parameter name to its checkpoint-side tensor name."""if index.has_tensor(full_name):return full_name- for target_pattern, source_replacement in _reverse_renamings_for(_index_model_type(index)):- # ``source_replacement`` arrives regex-escaped (``mlp\.router\.gate``);- # as a ``re.sub`` replacement those backslashes would land in the string- # verbatim, so unescape it to the plain checkpoint-side spelling.- replacement = re.sub(r"\\(.)", r"\1", source_replacement)- candidate = re.sub(target_pattern, lambda _m: replacement, full_name)- if candidate != full_name and index.has_tensor(candidate):+ # Unfused MoE experts whose checkpoint spells the projections differently+ # (e.g. phimoe's w1/w2/w3, mixtral-style checkpoints).+ for candidate in _renamed_expert_candidates(index, full_name):+ if index.has_tensor(candidate):return candidate+ for model_type in _index_model_types(index):+ for transform in _reverse_renamings_for(model_type):+ try:+ candidate, _ = transform.rename_source_key(full_name)+ except Exception as exc: # pragma: no cover - defensive, never fail a load here+ logger.debug("rename %s via %s failed: %s", full_name, transform, exc)+ continue+ if candidate != full_name and index.has_tensor(candidate):+ return candidatereturn None@@ -160,24 +445,70 @@ def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIn# calibration/eval forwards) previously left those params on meta -- the# meta-ness then propagated silently until a crash far downstream. Map# each unfused name onto its fused on-disk tensor and slice.- _FUSED_RE = _re.compile(r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$")+ #+ # The exact slicing depends on the family's fused layout: qwen3_moe stores+ # ``[N, 2*inter, hidden]`` (contiguous halves, no transpose), while gpt_oss /+ # llama4 store ``[N, hidden, 2*inter]`` (matmul layout, needs transpose) and+ # gpt_oss additionally interleaves the gate/up columns. Rather than key the+ # whole decision off ``model_type`` (which does not generalise to new+ # families), the transpose is inferred from the *target* parameter shape and+ # only the interleave axis (invisible to shapes) falls back to a small+ # model_type hint table (see ``_slice_fused_expert``). Biases (gpt_oss) follow+ # the same split as their weight.+ _FUSED_RE = _re.compile(r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.(weight|bias)$")_fused_cache: dict = {}+ _layout_cache: list = [] # lazily-resolved (transposed_hint, interleaved)- def _fused_lookup(full_name: str):+ def _fused_lookup(full_name: str, target_shape):m = _FUSED_RE.match(full_name)if not m:return None- prefix, expert_idx, proj = m.group(1), int(m.group(2)), m.group(3)- fused_name = f"{prefix}.gate_up_proj" if proj in ("gate_proj", "up_proj") else f"{prefix}.down_proj"+ prefix, expert_idx, proj, attr = m.group(1), int(m.group(2)), m.group(3), m.group(4)+ is_gate_up = proj in ("gate_proj", "up_proj")+ if is_gate_up:+ fused_name = f"{prefix}.gate_up_proj" if attr == "weight" else f"{prefix}.gate_up_proj_bias"+ else:+ fused_name = f"{prefix}.down_proj" if attr == "weight" else f"{prefix}.down_proj_bias"if not index.has_tensor(fused_name):return Noneif fused_name not in _fused_cache:_fused_cache[fused_name] = index.read_tensors([fused_name], device=device)[fused_name]fused = _fused_cache[fused_name][expert_idx]- if proj == "down_proj":- return fused.contiguous()- inter = fused.shape[0] // 2- return (fused[:inter] if proj == "gate_proj" else fused[inter:]).contiguous()+ # Resolve the model_type layout hint lazily and only once, and only when a+ # fused expert tensor is actually being sliced -- reading the model type+ # touches ``index.checkpoint_dir``/config, which cheap/fake indices used by+ # non-fused (dense) materialization do not populate.+ if not _layout_cache:+ _layout_cache.append(_fused_expert_layout(index))+ transposed_hint, interleaved = _layout_cache[0]+ value = _slice_fused_expert(fused, proj, attr, is_gate_up, tuple(target_shape), transposed_hint, interleaved)+ # Guard against a slice that does not match the destination parameter: it+ # is safer to leave the param on meta (the caller then raises an+ # actionable "fall back to a full CPU load" error) than to hand accelerate+ # a mismatched tensor and crash with a cryptic ValueError.+ if value is not None and tuple(value.shape) != tuple(target_shape):+ return None+ return value++ def _concat_lookup(full_name: str):+ """Assemble a model-side param that the checkpoint stores as several tensors+ concatenated by a ``WeightConverter`` (e.g. GLM-5.3 ``q/k/v_conv1d`` ->+ ``conv1d``). Returns the concatenated tensor, or ``None`` when no converter+ applies or a source tensor is missing."""+ for model_type in _index_model_types(index):+ for target_suffix, source_suffixes, dim in _concat_converters_for(model_type):+ if not full_name.endswith(target_suffix):+ continue+ prefix = full_name[: -len(target_suffix)]+ # Guard against a spurious mid-token match (require a clean boundary).+ if prefix and not prefix.endswith("."):+ continue+ source_names = [f"{prefix}{suffix}" for suffix in source_suffixes]+ if not all(index.has_tensor(name) for name in source_names):+ continue+ read = index.read_tensors(source_names, device=device)+ return torch.cat([read[name] for name in source_names], dim=dim).contiguous()+ return Nonetargets = [] # (param_name, full_checkpoint_name, declared_meta_dtype)fused_targets = [] # (param_name, sliced_value)@@ -187,7 +518,9 @@ def materialize_module(module: nn.Module, module_name: str, index: SafetensorsInfull_name = f"{module_name}.{name}".replace(".orig_layer.", ".")resolved_name = _resolve_checkpoint_name(index, full_name)if resolved_name is None:- sliced = _fused_lookup(full_name)+ sliced = _fused_lookup(full_name, tensor.shape)+ if sliced is None:+ sliced = _concat_lookup(full_name)if sliced is not None:fused_targets.append((name, sliced))continue@@ -196,7 +529,18 @@ def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIntargets.append((name, resolved_name, tensor.dtype))for name, value in fused_targets:- set_module_tensor_to_device(module, name, device, value=value, dtype=value.dtype)+ try:+ set_module_tensor_to_device(module, name, device, value=value, dtype=value.dtype)+ except (ValueError, RuntimeError) as exc:+ # A fused-expert slice whose shape does not match its destination+ # cannot be reconciled from the checkpoint; rather than surface the+ # raw accelerate error, point the user at the non-meta fallback.+ raise RuntimeError(+ f"Failed to materialize fused MoE parameter '{module_name}.{name}' from the checkpoint "+ f"(value shape {tuple(value.shape)}). This usually means the model's fused-expert layout "+ "is not yet recognised by the meta-skeleton loader. Set AR_DISABLE_AUTO_META_LOAD=1 to load "+ "the whole model on CPU instead (uses more RAM)."+ ) from exc_fused_cache.clear()if not targets:@@ -242,7 +586,63 @@ def total_resident_bytes(model: nn.Module) -> int:return total-def build_meta_model(model_name: str, trust_remote_code: bool = True):+def unfuse_meta_moe_(model: nn.Module) -> list[str]:+ """Turn ``transformers>=5`` fused 3D expert parameters into per-expert ``nn.Linear``.++ ``transformers>=5`` declares MoE experts as a single fused ``nn.Parameter`` of shape+ ``(num_experts, ...)``, which AutoRound cannot quantize (it is not an ``nn.Linear``).+ The regular load pipeline fixes this *after* a full CPU load; on a meta skeleton we+ can do it up-front for free -- meta tensors own no storage, so unfusing here only+ rewires ``nn.Module`` objects and costs ~0 RAM.++ Doing it before any weight is read is also what keeps the later per-block+ materialization cheap: the model-side names become ``experts.<i>.<proj>.weight``,+ which is exactly the layout the checkpoints use, so ``materialize_module`` reads one+ small 2D tensor per expert instead of forcing a whole fused ``(num_experts, ...)``+ tensor to be assembled in RAM.++ No modeling class is monkey-patched: the per-expert forward is dispatched through+ ``transformers.integrations.moe.ALL_EXPERTS_FUNCTIONS`` (the public experts-interface+ registry) via ``config._experts_implementation``.++ Returns the list of unfused module names (empty when there is nothing to do).+ """+ try:+ from auto_round.modeling.fused_moe.replace_modules import (+ _handle_moe_modules,+ is_custom_model,+ log_moe_block_transition,+ )+ except ImportError: # pragma: no cover - transformers < 5 or partial install+ return []++ # Model families with a dedicated replacement (llama4, qwen3_5_moe, step3p5, ...) must+ # go through `apply_replacements`, which runs its custom `from_original` against the+ # *fused* module. Unfusing them here with the generic linear_loop path would leave+ # that replacement facing a module whose fused params no longer exist.+ if is_custom_model(model):+ logger.debug(+ "meta skeleton: %s has a dedicated MoE replacement, deferring the unfuse to apply_replacements",+ getattr(getattr(model, "config", None), "model_type", None),+ )+ return []++ try:+ # This is where the structural change actually happens for a meta-built model, so+ # report the before/after here rather than in `apply_replacements` (which by then+ # has nothing left to do).+ with log_moe_block_transition(model, "unfuse"):+ unfused = _handle_moe_modules(model)+ except Exception: # pragma: no cover - never block the meta build on this+ logger.warning("Structural MoE unfuse on the meta skeleton failed", exc_info=True)+ return []++ if unfused:+ logger.info("meta skeleton: structural MoE unfuse produced %d unfused experts modules", len(unfused))+ return unfused+++def build_meta_model(model_name: str, trust_remote_code: bool = True, unfuse_moe: bool = True):"""Build a meta-device model skeleton (~0 RAM) plus its tokenizer and aSafetensorsIndex for on-demand materialization, instead of AutoRound's own``llm_load_model(model_name, device_map="cpu")`` which fully materializes the@@ -250,6 +650,10 @@ def build_meta_model(model_name: str, trust_remote_code: bool = True):only covers the common local-directory ``AutoModelForCausalLM`` case (nobagel/glm/mxfp4/HPU special-casing) -- callers should fall back to``llm_load_model`` for anything this doesn't handle.++ When ``unfuse_moe`` is set (the default) the fused ``transformers>=5`` MoE experts+ are split into per-expert ``nn.Linear`` while still on meta, so the returned skeleton+ is immediately quantizable and every later materialization is per-expert."""from accelerate import init_empty_weightsfrom transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer@@ -269,6 +673,12 @@ def build_meta_model(model_name: str, trust_remote_code: bool = True):model = model_cls(config)else:model = AutoModelForCausalLM.from_config(config, trust_remote_code=trust_remote_code)++ # Unfuse while everything is still meta: free, and it makes the model-side+ # parameter names line up 1:1 with the per-expert checkpoint tensors.+ if unfuse_moe:+ unfuse_meta_moe_(model)+# Constructing from config does not tie weights (that happens in# from_pretrained), so tied params such as lm_head.weight -- absent from# the checkpoint by design -- stay separate meta tensors and later crash@@ -296,25 +706,38 @@ def materialize_non_block_params(def _in_block(name: str) -> bool:return any(name == p or name.startswith(p + ".") for p in block_prefixes)+ # Tied output embeddings (lm_head.weight tied to embed_tokens.weight) are absent from+ # the checkpoint by design -- the tie is re-established below, which makes them real.+ # Do not warn about those, or every tied model reports a scary "leaving on meta" line+ # for a parameter that is fine a few statements later.+ tied_keys = set(getattr(model, "_tied_weights_keys", None) or ())++ def _is_tied(name: str) -> bool:+ return name in tied_keys or any(name.startswith(key.rstrip("*")) for key in tied_keys if key.endswith("*"))+targets = [] # (param_name, full_checkpoint_name)+ deferred_tied = []for name, tensor in list(model.named_parameters()) + list(model.named_buffers()):if str(tensor.device) != "meta" or _in_block(name):continuefull_name = name.replace(".orig_layer.", ".")resolved_name = _resolve_checkpoint_name(index, full_name)if resolved_name is None:- logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)+ if _is_tied(name):+ deferred_tied.append(name)+ else:+ logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)continuetargets.append((name, resolved_name))- if not targets:- return- values = index.read_tensors([full_name for _, full_name in targets], device=device)- for name, full_name in targets:- # See the matching comment in materialize_module() -- explicit dtype=- # is required so the checkpoint's real dtype wins over whatever the- # meta skeleton happened to declare.- set_module_tensor_to_device(model, name, device, value=values[full_name], dtype=values[full_name].dtype)+ if targets:+ values = index.read_tensors([full_name for _, full_name in targets], device=device)+ for name, full_name in targets:+ # See the matching comment in materialize_module() -- explicit dtype=+ # is required so the checkpoint's real dtype wins over whatever the+ # meta skeleton happened to declare.+ set_module_tensor_to_device(model, name, device, value=values[full_name], dtype=values[full_name].dtype)+# set_module_tensor_to_device replaces parameter objects, which breaks# weight tying: a tied lm_head still references the old (meta) parameter# while the embedding points at the new real one. Re-tie so every tied@@ -322,6 +745,20 @@ def materialize_non_block_params(if getattr(getattr(model, "config", None), "tie_word_embeddings", False):model.tie_weights()+ # Anything declared tied that the re-tie did not actually make real is a genuine+ # problem (quantizing it would fail on a meta tensor), so surface it now.+ still_meta = [name for name in deferred_tied if _param_or_buffer(model, name) is None]+ if still_meta:+ logger.warning("No checkpoint tensor found for tied %s, leaving on meta", ", ".join(still_meta))+++def _param_or_buffer(model: nn.Module, name: str):+ """Return the named parameter/buffer when it is materialized, else ``None``."""+ for candidate_name, tensor in list(model.named_parameters()) + list(model.named_buffers()):+ if candidate_name == name:+ return None if tensor.device.type == "meta" else tensor+ return None+class stream_block_forward:"""Context manager: wrap every top-level decoder block's ``forward`` so itdiff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.pyindex 4f5ee3481..0cea78e8c 100644--- a/auto_round/utils/offload.py+++ b/auto_round/utils/offload.py@@ -255,9 +255,9 @@ def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nn# ``mlp.router.gate`` behind a ``mlp.gate`` module) and leaves them on# meta until a downstream ``.to(device)`` crashes.try:- from auto_round.utils.disk_stream_util import SafetensorsIndex, materialize_module+ from auto_round.utils.disk_stream_util import get_safetensors_index, materialize_module- materialize_module(block, block_name, SafetensorsIndex(model_dir), device="cpu")+ materialize_module(block, block_name, get_safetensors_index(model_dir), device="cpu")if not _has_meta(block):returnexcept FileNotFoundError:diff --git a/test/unit/common/models/test_special_model_handler.py b/test/unit/common/models/test_special_model_handler.pyindex 606a365ff..ad938ead5 100644--- a/test/unit/common/models/test_special_model_handler.py+++ b/test/unit/common/models/test_special_model_handler.py@@ -983,26 +983,40 @@ class TestGetPredefinedIgnoreLayers:class TestTorchCompileOff:- """DeepSeek / GLM-5.3-Flash DSA families must not run under torch.compile."""+ """DeepSeek / GLM-5.3-Flash DSA families must not run under torch.compile on torch < 2.14.0.++ From torch 2.14.0 onward the recompile_limit issue is resolved, so these families no+ longer force ``torch.compile`` off and instead honor the default / user-provided setting.+ """def _cfg(self, model_type="", architectures=None):from types import SimpleNamespacereturn SimpleNamespace(config=SimpleNamespace(model_type=model_type, architectures=architectures or []))+ def _expected_off(self):+ from auto_round.utils import torch_version_at_least++ # On torch >= 2.14.0 the rules are not registered, so the reason is ``None``.+ return not torch_version_at_least("2.14.0")+def test_deepseek_disabled(self):from auto_round.special_model_handler import get_torch_compile_off_reasonfor mt in ("deepseek_v3", "deepseek_v32"):- assert get_torch_compile_off_reason(self._cfg(model_type=mt)) is not None+ off = get_torch_compile_off_reason(self._cfg(model_type=mt)) is not None+ assert off == self._expected_off()def test_glm5_family_disabled(self):from auto_round.special_model_handler import get_torch_compile_off_reason- assert get_torch_compile_off_reason(self._cfg(model_type="glm5_next")) is not None- assert get_torch_compile_off_reason(self._cfg(model_type="glm_moe_dsa")) is not None+ expected = self._expected_off()+ assert (get_torch_compile_off_reason(self._cfg(model_type="glm5_next")) is not None) == expected+ assert (get_torch_compile_off_reason(self._cfg(model_type="glm_moe_dsa")) is not None) == expected# Architecture name alone is enough (config before weights are materialized).- assert get_torch_compile_off_reason(self._cfg(architectures=["Glm5NextForConditionalGeneration"])) is not None+ assert (+ get_torch_compile_off_reason(self._cfg(architectures=["Glm5NextForConditionalGeneration"])) is not None+ ) == expecteddef test_other_models_unaffected(self):from auto_round.special_model_handler import get_torch_compile_off_reasondiff --git a/test/unit/common/utils/test_apply.py b/test/unit/common/utils/test_apply.pyindex 6e8c1c9d0..3bc1c71e6 100644--- a/test/unit/common/utils/test_apply.py+++ b/test/unit/common/utils/test_apply.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import torchimport torch.nn as nnfrom auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme@@ -31,3 +32,34 @@ def test_apply_plan_is_the_explicit_model_write_boundary():assert model[0].bits == 4assert model[0].data_type == "int"+++def test_apply_plan_preserves_non_target_module_own_attributes():+ """A non-quantized module's own attribute that happens to share a scheme-key+ name (e.g. a grouped RMSNorm's ``group_size``) must not be stripped.++ Regression: ``apply_plan_to_model`` used to ``delattr`` every scheme-key-named+ attribute from *every* module, deleting ``Qwen4ExpTextRMSNorm.group_size`` and+ crashing the norm's next forward with ``AttributeError``.+ """++ class GroupedRMSNorm(nn.Module):+ def __init__(self, dim, group_size):+ super().__init__()+ self.weight = nn.Parameter(torch.ones(dim))+ self.group_size = group_size # legitimately owned by the model++ model = nn.Sequential(nn.Linear(32, 32), GroupedRMSNorm(32, group_size=8))+ plan = ResolvedQuantizationConfig(+ scheme=ResolvedScheme.from_scheme(QuantizationScheme()),+ formats=(),+ layer_config={"0": {"bits": 4, "group_size": 128, "data_type": "int"}},+ )++ apply_plan_to_model(model, plan)++ # Quantization target still receives the plan.+ assert model[0].bits == 4+ assert model[0].group_size == 128+ # The norm keeps its own group_size, untouched by the scheme-key reset.+ assert model[1].group_size == 8diff --git a/test/unit/test_cpu/models/test_fused_moe_detection.py b/test/unit/test_cpu/models/test_fused_moe_detection.pynew file mode 100644index 000000000..1f158c14d--- /dev/null+++ b/test/unit/test_cpu/models/test_fused_moe_detection.py@@ -0,0 +1,100 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""The meta-skeleton load path must only trigger for fused-3D MoE checkpoints.++Building on meta pays off exactly when ``transformers`` would stack the checkpoint's+per-expert 2D tensors into a fused 3D ``nn.Parameter`` that AutoRound then has to split+back apart. Everything else (dense models, MoE families with a dedicated AutoRound+replacement) must keep the ordinary load path, so the blast radius stays small.+"""++import pytest+import transformers+from packaging import version++from auto_round.modeling.fused_moe.moe_experts_interface import (+ _config_model_types,+ config_has_fused_moe_experts,+)+from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES++pytestmark = pytest.mark.skipif(+ version.parse(transformers.__version__) < version.parse("5.0.0"),+ reason="fused 3D MoE experts only exist in transformers>=5",+)+++class _FakeConfig:+ """Duck-typed stand-in so the test does not depend on every config class existing."""++ def __init__(self, model_type, **sub_configs):+ self.model_type = model_type+ for name, value in sub_configs.items():+ setattr(self, name, value)+++# Fused 3D experts: `from_pretrained` merges `experts.<i>.<proj>` into one parameter.+FUSED_MOE_TYPES = ["qwen3_moe", "qwen2_moe", "deepseek_v3", "glm4_moe", "phimoe", "mixtral"]+# Dense models have no expert conversion at all.+DENSE_TYPES = ["llama", "qwen2", "gemma2", "mistral"]+# The checkpoint already stores the fused 3D tensor, so there is no per-expert merge.+ALREADY_FUSED_CHECKPOINT_TYPES = ["qwen3_vl_moe", "gpt_oss"]++[email protected]("model_type", FUSED_MOE_TYPES)+def test_fused_moe_families_are_detected(model_type):+ assert config_has_fused_moe_experts(_FakeConfig(model_type)) is True++[email protected]("model_type", DENSE_TYPES)+def test_dense_models_are_not_detected(model_type):+ assert config_has_fused_moe_experts(_FakeConfig(model_type)) is False++[email protected]("model_type", ALREADY_FUSED_CHECKPOINT_TYPES)+def test_checkpoints_without_a_per_expert_merge_are_not_detected(model_type):+ assert config_has_fused_moe_experts(_FakeConfig(model_type)) is False+++def test_detection_reaches_nested_text_config():+ """A VLM registers the MoE rules on its text sub-config, not the top level."""+ vlm = _FakeConfig("qwen3_5_moe", text_config=_FakeConfig("qwen3_5_moe_text"))++ assert config_has_fused_moe_experts(_FakeConfig("qwen3_5_moe")) is False, "precondition: top level alone is inert"+ assert config_has_fused_moe_experts(vlm) is True+ assert "qwen3_5_moe_text" in _config_model_types(vlm)+++def test_detection_uses_real_transformers_configs():+ """Same behaviour with genuine config objects, not just duck-typed ones."""+ from transformers import LlamaConfig, Qwen3MoeConfig++ assert config_has_fused_moe_experts(Qwen3MoeConfig()) is True+ assert config_has_fused_moe_experts(LlamaConfig()) is False+++def test_families_with_a_dedicated_replacement_are_left_alone():+ """`ModelContext` excludes these; they drive their own memory-aware materialization."""+ expected = {+ "llama4",+ "deepseek_v2",+ "step3p5",+ "qwen3_omni_moe",+ "qwen3_5_moe",+ "qwen3_5_moe_text",+ }+ if version.parse(transformers.__version__) < version.parse("5.0.0"):+ # These two only get a dedicated replacement on the pre-5.0 linear_loop path.+ expected |= {"qwen3_vl_moe", "gpt_oss"}+ assert set(BUILTIN_MODULES) == expected