PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Update MOE linear_loop implementation for speedup and matching name in vLLM (#1478)
Xin Hed1501180a706
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex cc8b11a93..39964319a 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -161,6 +161,8 @@ class ShardWriter:for pname, tensor in full_sd.items():if pname in all_saved_names:continue+ if tensor.device.type == "meta":+ continuelayer_name = ".".join(pname.split(".")[:-1])if self.lm_head_name is not None and layer_name == self.lm_head_name and tie_word_embeddings:lm_head_module = get_module(self.model, self.lm_head_name)diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 3bdacf096..aec990b37 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -19,59 +19,6 @@ import torch.nn as nnfrom auto_round.utils import copy_python_files_from_model_cache, logger, unsupported_meta_device-def _has_unfused_moe_experts(model: nn.Module) -> bool:- """Check if model has unfused MOE experts (nn.ModuleList instead of 3D Parameter).-- This is used to detect if we need to bypass transformers' weight conversion- during save_pretrained.- """- for module in model.modules():- if hasattr(module, "gate_up_proj") and isinstance(module.gate_up_proj, nn.ModuleList):- if hasattr(module, "down_proj") and isinstance(module.down_proj, nn.ModuleList):- return True- return False---def _save_model_state_dict(- model: nn.Module,- save_dir: str,- max_shard_size: str = "5GB",- safe_serialization: bool = True,-):- """Save model using state_dict directly, bypassing transformers' weight conversion.-- This is needed for models with unfused MOE experts where transformers'- revert_weight_conversion expects 3D tensor format but we have ModuleList.- """- import torch- from safetensors.torch import save_file-- os.makedirs(save_dir, exist_ok=True)-- # Save config- if hasattr(model, "config") and model.config is not None:- model.config.save_pretrained(save_dir)-- if hasattr(model, "generation_config") and model.generation_config is not None:- try:- model.generation_config.save_pretrained(save_dir)- except Exception:- pass # generation_config save can fail for some models-- # Get state dict- state_dict = model.state_dict()-- # Save weights- if safe_serialization:- # Save as safetensors- weights_file = os.path.join(save_dir, "model.safetensors")- save_file(state_dict, weights_file)- else:- # Save as pytorch bin- weights_file = os.path.join(save_dir, "pytorch_model.bin")- torch.save(state_dict, weights_file)--def save_model(model: nn.Module,save_dir: str,@@ -101,10 +48,6 @@ def save_model("""os.makedirs(save_dir, exist_ok=True)- # Check if model has unfused MOE experts - if so, we need to bypass transformers'- # weight conversion which expects original 3D tensor format- has_unfused_experts = _has_unfused_moe_experts(model)-if unsupported_meta_device(model):if hasattr(model, "config") and model.config is not None:model.config.save_pretrained(save_dir)@@ -113,11 +56,6 @@ def save_model(if hasattr(model, "generation_config"):setattr(model.generation_config, "do_sample", True)model.generation_config.save_pretrained(save_dir)- elif has_unfused_experts:- # For models with unfused MOE experts, save state_dict directly to avoid- # transformers' revert_weight_conversion which expects 3D tensor format- logger.info("Saving model with unfused MOE experts using state_dict (bypassing weight conversion)")- _save_model_state_dict(model, save_dir, max_shard_size, safe_serialization)else:try:model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 8c8b8782f..1e4324244 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -41,6 +41,7 @@ from auto_round.utils import (get_block_names,get_module,is_hpex_available,+ is_transformers_version_greater_or_equal_5,set_module,)@@ -570,6 +571,23 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:logger.warning(f"Forced model to {target_dtype}")+def disable_moe_conversion_mapping(model):+ """Disables MoE-specific checkpoint conversion mappings to prevent unintended weight merging."""+ from transformers.conversion_mapping import (+ get_checkpoint_conversion_mapping,+ register_checkpoint_conversion_mapping,+ )+ from transformers.core_model_loading import WeightRenaming++ model_type = getattr(model.config, "model_type", None)+ if model_type is not None:+ conversions = get_checkpoint_conversion_mapping(model_type)+ if conversions is not None:+ # 只保留 WeightRenaming,跳过 WeightConverter(MoE merge 操作)+ filtered = [c for c in conversions if isinstance(c, WeightRenaming)]+ register_checkpoint_conversion_mapping(model_type, mapping=filtered, overwrite=True)++def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Module, list]:"""Converts a HuggingFace model into an AutoRound model by replacing layers with quantized layers.@@ -590,6 +608,8 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.MNotImplementedError: If the GPTQ model uses an unsupported `g_idx`.ValueError: If quantization backend is not properly specified."""+ if is_transformers_version_greater_or_equal_5():+ disable_moe_conversion_mapping(model)quantization_config = model.config.quantization_config# Check desc_act + static_groupsdiff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.pyindex aaef9ab8b..85b294759 100644--- a/auto_round/modeling/fused_moe/moe_experts_interface.py+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py@@ -31,10 +31,12 @@ Usage:# Now the model uses linear_loop forward which supports quantized nn.Linear layers"""+import torchfrom torch import nnfrom auto_round.utils import clear_memory, logger+from auto_round.utils.device import memory_monitortry:from transformers.integrations.moe import ALL_EXPERTS_FUNCTIONS@@ -54,7 +56,10 @@ LINEAR_LOOP_IMPL = "linear_loop"# output_multiplier: output dimension multiplier (e.g., 2 for fused gate+up projection)KNOWN_PROJECTION_PATTERNS = {# Transformers 5.0+ standard (Qwen3-MoE, etc.)- "gate_up_proj": {"is_input_proj": True, "output_multiplier": 2}, # hidden -> 2*intermediate+ # gate_up_proj is auto-split into gate_proj + up_proj during unfusing+ "gate_up_proj": {"is_input_proj": True, "output_multiplier": 2, "split_into": ["gate_proj", "up_proj"]},+ "gate_proj": {"is_input_proj": True, "output_multiplier": 1}, # hidden -> intermediate (gate)+ "up_proj": {"is_input_proj": True, "output_multiplier": 1}, # hidden -> intermediate (up)"down_proj": {"is_input_proj": False, "output_multiplier": 1}, # intermediate -> hidden# Mixtral-style"w1": {"is_input_proj": True, "output_multiplier": 1}, # gate: hidden -> intermediate@@ -67,6 +72,103 @@ KNOWN_PROJECTION_PATTERNS = {}+class _ExpertContainer(nn.Module):+ """Lightweight container for a single expert's projection layers.++ Each expert has its projections (e.g., gate_proj, up_proj, down_proj)+ as direct attributes. When attached as numbered children of the experts+ module (e.g., module.add_module("0", container)), PyTorch naturally+ produces state_dict keys like: {prefix}0.gate_proj.weight+ which matches the standard checkpoint format without any hooks.+ """++ pass+++def _install_compact_expert_repr(module: nn.Module) -> None:+ """Install compact __repr__ on the module's class.++ Collapses identical _ExpertContainer children into a range display,+ similar to nn.ModuleList::++ GptOssExperts(+ (0-63): 64 x _ExpertContainer(+ (gate_proj): Linear(...)+ (up_proj): Linear(...)+ (down_proj): Linear(...)+ )+ )+ """+ cls = module.__class__+ if "_compact_expert_repr_installed" in cls.__dict__:+ return++ _original_repr = cls.__repr__++ def _compact_repr(self):+ from torch.nn.modules.module import _addindent++ # Separate expert containers from other children+ expert_items = []+ other_items = []+ for key, child in self._modules.items():+ if isinstance(child, _ExpertContainer):+ expert_items.append((key, child))+ else:+ other_items.append((key, child))++ if not expert_items:+ return _original_repr(self)++ lines = []+ extra_repr = self.extra_repr()+ if extra_repr:+ lines.extend(extra_repr.split("\n"))++ # Print non-expert children normally+ for key, child in other_items:+ mod_str = repr(child)+ mod_str = _addindent(mod_str, 2)+ lines.append(f"({key}): {mod_str}")++ # Collapse expert containers into a single range line+ first_key = expert_items[0][0]+ last_key = expert_items[-1][0]+ sample_repr = repr(expert_items[0][1])+ n = len(expert_items)+ sample_repr = _addindent(sample_repr, 2)+ if n > 1:+ lines.append(f"({first_key}-{last_key}): {n} x {sample_repr}")+ else:+ lines.append(f"({first_key}): {sample_repr}")++ main_str = self._get_name() + "("+ if lines:+ main_str += "\n " + "\n ".join(lines) + "\n"+ main_str += ")"+ return main_str++ cls.__repr__ = _compact_repr+ cls._compact_expert_repr_installed = True++ # Make the module subscriptable and iterable: module[0], module[1], len(module), for e in module+ if not hasattr(cls, "__getitem__") or "__getitem__" not in cls.__dict__:++ def _getitem(self, idx):+ return getattr(self, str(idx))++ def _len(self):+ return self.num_experts++ def _iter(self):+ for i in range(self.num_experts):+ yield getattr(self, str(i))++ cls.__getitem__ = _getitem+ cls.__len__ = _len+ cls.__iter__ = _iter++def linear_loop_experts_forward(self: nn.Module,hidden_states: torch.Tensor,@@ -75,13 +177,16 @@ def linear_loop_experts_forward() -> torch.Tensor:"""Forward using individual nn.Linear layers per expert.- This implementation loops over experts and uses self.gate_up_proj[i] and- self.down_proj[i] as nn.Linear layers (or quantized equivalents), enabling- proper quantization support.+ This implementation loops over experts and accesses per-expert containers+ (self._modules["0"], self._modules["1"], ...) each with gate_proj,+ up_proj, and down_proj as nn.Linear layers (or quantized equivalents),+ enabling proper quantization support.- Expected module attributes:- - gate_up_proj: nn.ModuleList of nn.Linear (in_features=hidden_dim, out_features=2*intermediate_dim)- - down_proj: nn.ModuleList of nn.Linear (in_features=intermediate_dim, out_features=hidden_dim)+ Expected module structure:+ - Numbered children (0, 1, ..., num_experts-1), each an _ExpertContainer with:+ - gate_proj: nn.Linear (in_features=hidden_dim, out_features=intermediate_dim)+ - up_proj: nn.Linear (in_features=hidden_dim, out_features=intermediate_dim)+ - down_proj: nn.Linear (in_features=intermediate_dim, out_features=hidden_dim)- act_fn: activation function- num_experts: number of experts- _apply_gate: optional custom gating function@@ -133,18 +238,20 @@ def linear_loop_experts_forward(expert_input = selected_hidden_states[mask] # (num_samples_for_expert, hidden_dim)- # Use nn.Linear layers for this expert- gate_up_out = self.gate_up_proj[expert_idx](expert_input) # (num_samples, 2*intermediate_dim)+ # Get this expert's container with its projection layers+ expert = getattr(self, str(expert_idx))+ gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)+ up_out = expert.up_proj(expert_input) # (num_samples, intermediate_dim)# Apply gatingif hasattr(self, "_apply_gate"):+ gate_up_out = torch.cat([gate_out, up_out], dim=-1)gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)else:- gate, up = gate_up_out.chunk(2, dim=-1)- gated_out = self.act_fn(gate) * up+ gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)# Down projection- expert_out = self.down_proj[expert_idx](gated_out) # (num_samples, hidden_dim)+ expert_out = expert.down_proj(gated_out) # (num_samples, hidden_dim)# Store resultsout_per_sample[mask] = expert_out.to(out_per_sample.dtype)@@ -264,8 +371,14 @@ def _unfuse_single_projection(is_transposed: bool,dtype: torch.dtype,target_device: torch.device,-) -> nn.ModuleList | None:- """Unfuse a single projection from 3D Parameter to ModuleList of Linear layers.+) -> list | None:+ """Unfuse a single projection from 3D Parameter to a list of Linear layers.++ Optimized to minimize device allocations and copies:+ - Moves the full 3D tensor to CPU in a single transfer+ - Performs batch transpose on CPU if needed+ - Creates Linear shells on meta device (no allocation)+ - Directly assigns weight slices as Parameters (zero-copy on CPU)Args:module: The experts module@@ -276,7 +389,7 @@ def _unfuse_single_projection(target_device: Device for the Linear layersReturns:- ModuleList of Linear layers, or None if projection doesn't exist+ List of Linear layers, or None if projection doesn't exist"""param = getattr(module, proj_name, None)if param is None or not isinstance(param, nn.Parameter) or param.dim() != 3:@@ -293,32 +406,53 @@ def _unfuse_single_projection(bias_param = getattr(module, bias_name, None)has_bias = bias_param is not None- # Create ModuleList- linears = nn.ModuleList()source_device = param.device-+ is_meta = source_device.type == "meta"++ # Prepare weight slices on CPU in batch (single D2H transfer + batch transpose)+ if not is_meta:+ # Single transfer: GPU -> CPU (or no-op if already on CPU)+ weights_cpu = param.data.cpu() # (num_experts, dim1, dim2)+ if is_transposed:+ # Batch transpose: (num_experts, in, out) -> (num_experts, out, in)+ weights_cpu = weights_cpu.transpose(1, 2)+ # Ensure contiguous layout so unbind produces contiguous 2D slices.+ # This is needed when the param comes from a chunk split (Phase 1)+ # which produces non-contiguous views even on CPU.+ if not weights_cpu.is_contiguous():+ weights_cpu = weights_cpu.contiguous()+ # Unbind into a tuple of 2D tensors (zero-copy views since contiguous)+ weight_slices = weights_cpu.unbind(0)++ if has_bias:+ bias_cpu = bias_param.data.cpu()+ # Ensure contiguous — bias may come from a chunk split (Phase 1)+ if not bias_cpu.is_contiguous():+ bias_cpu = bias_cpu.contiguous()+ bias_slices = bias_cpu.unbind(0)++ # Create Linear shells on meta device (no memory allocation)+ linears = []for i in range(num_experts):- linear = nn.Linear(in_features, out_features, bias=has_bias, dtype=dtype, device=target_device)-- # Copy weights if not on meta device- if source_device.type != "meta":- if is_transposed:- linear.weight.data.copy_(param[i].t())- else:- linear.weight.data.copy_(param[i])+ # meta device: creates the module structure without allocating weight storage+ linear = nn.Linear(in_features, out_features, bias=has_bias, dtype=dtype, device="meta")+ if not is_meta:+ # Direct parameter assignment — no copy, just references the CPU tensor slice+ linear.weight = nn.Parameter(weight_slices[i])if has_bias:- linear.bias.data.copy_(bias_param[i])+ linear.bias = nn.Parameter(bias_slices[i])linears.append(linear)- # Release original parameter memory using to_empty- if source_device.type != "meta":+ # Release original parameter memory+ if not is_meta:try:param.data = param.data.to_empty(device="meta")+ if has_bias:+ bias_param.data = bias_param.data.to_empty(device="meta")logger.debug(f"Released memory for {proj_name} using to_empty(device='meta')")except Exception:- # Fallback: just deletepassreturn linears@@ -329,10 +463,10 @@ def _unfuse_experts_weights_inplace(check_decorator: bool = True,projection_names: list[str] | None = None,) -> bool:- """Convert fused 3D expert weights to nn.ModuleList of nn.Linear layers.+ """Convert fused 3D expert weights to per-expert containers of nn.Linear layers.This function modifies the module in-place, replacing fused 3D Parameters- with nn.ModuleList[nn.Linear] for each detected projection.+ with numbered _ExpertContainer children, each holding nn.Linear projections.Args:module: The experts module to unfuse@@ -375,22 +509,90 @@ def _unfuse_experts_weights_inplace(dtype = first_param.dtypetarget_device = first_param.device if first_param.device.type != "meta" else "cpu"- # Unfuse each projection- unfused_count = 0+ # Phase 1: Split fused projections (e.g., gate_up_proj -> gate_proj + up_proj) into separate 3D params+ extra_projections = {}+ fused_to_remove = []+ for proj_name, config in detected_projections.items():+ split_into = config.get("split_into")+ if not split_into:+ continue+ param = getattr(module, proj_name, None)+ if param is None or not isinstance(param, nn.Parameter) or param.dim() != 3:+ continue+ # Split along output dimension+ split_dim = 2 if is_transposed else 1+ split_params = param.chunk(len(split_into), dim=split_dim)++ # Also split bias if present (e.g., gate_up_proj_bias -> gate_proj_bias + up_proj_bias)+ bias_name = f"{proj_name}_bias"+ bias_param = getattr(module, bias_name, None)+ bias_splits = None+ if bias_param is not None and isinstance(bias_param, nn.Parameter) and bias_param.dim() == 2:+ bias_splits = bias_param.chunk(len(split_into), dim=1)++ for i, (split_name, split_param) in enumerate(zip(split_into, split_params)):+ # Avoid .contiguous() here — _unfuse_single_projection will handle it+ # during batch transpose/unbind, saving a full-tensor copy+ setattr(module, split_name, nn.Parameter(split_param))+ if bias_splits is not None:+ setattr(module, f"{split_name}_bias", nn.Parameter(bias_splits[i]))+ extra_projections[split_name] = KNOWN_PROJECTION_PATTERNS.get(+ split_name, {"is_input_proj": True, "output_multiplier": 1}+ )+ delattr(module, proj_name)+ if bias_param is not None:+ delattr(module, bias_name)+ fused_to_remove.append(proj_name)+ logger.debug(f"Split {proj_name} -> {split_into}: {num_experts} experts")++ # Remove fused entries and add split entries+ for name in fused_to_remove:+ del detected_projections[name]+ detected_projections.update(extra_projections)++ # Phase 2: Unfuse all 3D params into per-expert Linear layers+ proj_linears = {} # proj_name -> [Linear_expert0, Linear_expert1, ...]for proj_name in detected_projections:linears = _unfuse_single_projection(module, proj_name, num_experts, is_transposed, dtype, target_device)if linears is not None:- # Delete original parameter and set new ModuleListdelattr(module, proj_name)- setattr(module, proj_name, linears)- unfused_count += 1- logger.debug(f"Unfused {proj_name}: {num_experts} experts")+ # Also remove the bias parameter if it exists (already absorbed into Linear.bias)+ bias_name = f"{proj_name}_bias"+ if hasattr(module, bias_name):+ delattr(module, bias_name)+ proj_linears[proj_name] = linears++ if not proj_linears:+ return False++ # Phase 3: Assemble per-expert containers as direct numbered children.+ # This produces state_dict keys like: {prefix}0.gate_proj.weight+ # which naturally matches the standard checkpoint format — no hooks needed.+ for expert_idx in range(num_experts):+ container = _ExpertContainer()+ for proj_name, linears in proj_linears.items():+ setattr(container, proj_name, linears[expert_idx])+ module.add_module(str(expert_idx), container)# Ensure num_experts is setif not hasattr(module, "num_experts"):module.num_experts = num_experts- return unfused_count > 0+ # Mark as unfused for detection during save+ module._unfused_experts = True++ # Mark all unfused modules as already initialized so transformers'+ # _initialize_weights skips them. Without this, the model's _init_weights+ # tries to access the original fused attributes (e.g., gate_up_proj) which+ # no longer exist after unfusing.+ module._is_hf_initialized = True+ for child in module.modules():+ child._is_hf_initialized = True++ # Install compact repr to collapse identical expert containers in print output+ _install_compact_expert_repr(module)++ return Truedef prepare_model_for_moe_quantization(model: nn.Module, implementation: str = LINEAR_LOOP_IMPL) -> list[str]:@@ -399,7 +601,7 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = LThis function:1. Registers the linear_loop experts implementation with transformers2. Sets model.config._experts_implementation = implementation- 3. Unfuses all fused MOE expert weights into nn.ModuleList[nn.Linear]+ 3. Unfuses all fused MOE expert weights into per-expert containers of nn.LinearAfter calling this function, the model's forward pass will use individualnn.Linear layers per expert, which can be quantized normally.@@ -411,7 +613,6 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = LReturns:List of module names that were unfused"""- # Register our custom implementationif not register_linear_loop_experts():raise RuntimeError("Failed to register linear_loop experts implementation. "@@ -423,13 +624,13 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = Lfor name, module in model.named_modules():if _unfuse_experts_weights_inplace(module):unfused_modules.append(name)- logger.debug(f"Unfused expert weights in: {name}")+ logger.debug(f"[MoE Prep] Unfused '{name}'")# Only set config if we actually unfused something# Models that don't support the decorator (like Llama4) won't have anything unfused# and should use full module replacement insteadif unfused_modules:- logger.info(f"Unfused {len(unfused_modules)} MOE experts modules for quantization")+ logger.info(f"[MoE Prep] Unfused {len(unfused_modules)} MOE experts modules")clear_memory()# Set config for linear_loop forwarddiff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex ac6bb8ee2..f84515348 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -20,32 +20,35 @@ import torchfrom tqdm import tqdmfrom transformers import PreTrainedModel-# Import constant for expert implementation name-from auto_round.modeling.fused_moe.moe_experts_interface import LINEAR_LOOP_IMPL-from auto_round.utils import LazyImport, dump_mem_usage, dump_memory_usage_ctx, global_state, logger+from auto_round.utils import (+ LazyImport,+ dump_mem_usage,+ dump_memory_usage_ctx,+ global_state,+ is_transformers_version_greater_or_equal_5,+ logger,+)BUILTIN_MODULES = {+ # Llama4 has no use_experts_implementation, needs custom replacement to handle fused MoE blocks."llama4": LazyImport("auto_round.modeling.fused_moe.llama4"),- "gpt_oss": LazyImport("auto_round.modeling.fused_moe.gpt_oss"),- "qwen3_vl_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe"),+ # DeepseekV2Attention enables q_scale calibration for deepseek v2 on Gaudi (#1299)"deepseek_v2": LazyImport("auto_round.modeling.fused_moe.deepseek_v2"),+ # supports transformers >= 5.0.0"qwen3_5_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),"qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),}-def _has_custom_replacement_only(model: torch.nn.Module) -> bool:- """Check if all MOE-like modules in model have custom replacements registered.-- If all MOE modules are covered by BUILTIN_MODULES, we can skip the linear_loop path entirely.- """- if hasattr(model, "config") and hasattr(model.config, "model_type"):- model_type = model.config.model_type- if model_type in BUILTIN_MODULES:- return True- return False+# transformers >= 5.0.0 supports the general linear_loop experts interface.+if not is_transformers_version_greater_or_equal_5():+ # transformers == 5.2.0: IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)+ BUILTIN_MODULES["qwen3_vl_moe"] = LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe")+ # transformers == 5.2.0: IndexError: index 4 is out of bounds for dimension 0 with size 4+ BUILTIN_MODULES["gpt_oss"] = LazyImport("auto_round.modeling.fused_moe.gpt_oss")+@dump_mem_usage("Applying general replacements")def _handle_moe_modules(model: torch.nn.Module) -> list[str]:"""Handle fused MOE modules using transformers' linear_loop backend.@@ -77,40 +80,29 @@ def _handle_moe_modules(model: torch.nn.Module) -> list[str]:def _import_required_replacements(model: torch.nn.Module) -> None:"""Scan model and trigger lazy imports for registered replacement modules."""- imported = set()- if hasattr(model, "config") and hasattr(model.config, "model_type"):- model_type = model.config.model_type- if model_type in BUILTIN_MODULES:- _ = BUILTIN_MODULES[model_type].__name__ # or any attribute- imported.add(model_type)- logger.debug(f"Loaded replacement module for {model_type}")-+ if not is_custom_model(model):+ return+ model_type = model.config.model_type+ _ = BUILTIN_MODULES[model_type].__name__ # Trigger lazy import+ logger.debug(f"Loaded replacement module for {model_type}")-def _should_skip_moe_replacement(module: torch.nn.Module, model: torch.nn.Module) -> bool:- """Skip MOE replacement if linear_loop experts are already unfused.- This is only true when:- 1. model.config._experts_implementation == "linear_loop" (set by prepare_model_for_moe_quantization)- 2. The experts' gate_up_proj and down_proj are already nn.ModuleList+def is_custom_model(model: torch.nn.Module) -> bool:+ """Check if the model has a custom replacement registered via BUILTIN_MODULES.- Note: _experts_implementation is only set when the experts class supports- @use_experts_implementation decorator, so we don't need to check that again here.+ Returns True if the model's model_type matches a key in BUILTIN_MODULES."""- if not hasattr(model, "config"):- return False- if getattr(model.config, "_experts_implementation", None) != LINEAR_LOOP_IMPL:- return False- experts = getattr(module, "experts", None)- if experts is None:- return False- gate_up = getattr(experts, "gate_up_proj", None)- down = getattr(experts, "down_proj", None)- result = isinstance(gate_up, torch.nn.ModuleList) and isinstance(down, torch.nn.ModuleList)- logger.debug(- f"_should_skip_moe_replacement for {module.__class__.__name__}: "- f"gate_up type={type(gate_up).__name__}, down type={type(down).__name__}, skip={result}"- )- return result+ if hasattr(model, "config") and hasattr(model.config, "model_type"):+ return model.config.model_type in BUILTIN_MODULES+ return False+++def _log_first_moe_block(model: torch.nn.Module, label: str) -> None:+ """Log the first experts module found in the model for debugging."""+ for name, module in model.named_modules():+ if name.endswith(".experts"):+ logger.info(f"Experts ({label}) [{name}] ({module.__class__.__name__}):\n{module}")+ return@dump_mem_usage("Materializing model", log_level="debug")@@ -144,16 +136,6 @@ def release_original_module_(model: torch.nn.Module) -> None:model.apply(_clear_source_module)-def _has_meta_params_or_buffers(model: PreTrainedModel) -> bool:- for _, param in model.named_parameters():- if param.device.type == "meta":- return True- for _, buffer in model.named_buffers():- if buffer.device.type == "meta":- return True- return False--def safe_to_cpu_(model: torch.nn.Module) -> None:# If no replacement happened, move model to CPU directlyif global_state.replaced_module_count == 0:@@ -309,11 +291,29 @@ def apply_replacements("""_import_required_replacements(model)- # Auto-detect and handle fused MOE modules if enabled- # Skip if all MOE modules already have custom replacements registered- if auto_detect_moe and not _has_custom_replacement_only(model):+ _log_first_moe_block(model, "before replacement")++ # Custom replacements first+ if is_custom_model(model):+ _apply_custom_replacements(model)+ if auto_detect_moe and is_transformers_version_greater_or_equal_5():_handle_moe_modules(model)+ _log_first_moe_block(model, "after replacement")++ return model+++@dump_mem_usage("Applying custom replacements")+def _apply_custom_replacements(model: torch.nn.Module) -> list:+ """Scan model and replace registered modules with custom implementations.++ Args:+ model: The model to scan and apply replacements to (modified in-place).++ Returns:+ List of (name, replacement_class) tuples for replaced modules.+ """replaced = []# Step 1: Collect all modules that need replacement@@ -324,9 +324,6 @@ def apply_replacements(if isinstance(module, ReplacementModuleBase):continueclass_name = module.__class__.__name__- if ReplacementModuleBase.is_registered(class_name) and _should_skip_moe_replacement(module, model):- logger.debug(f"Skipping replacement for {name}: linear_loop experts already unfused")- continueif ReplacementModuleBase.is_registered(class_name) and ReplacementModuleBase.get_replacement_class(class_name).is_to_be_replaced(module):@@ -363,7 +360,7 @@ def apply_replacements(global_state.replaced_module_count = len(replaced)logger.info(f"Replaced {len(replaced)} modules")- return model+ return replaced@dataclassdiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 3d010d150..baceddecd 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1237,10 +1237,7 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_naexpert_input_proj_names = get_expert_input_proj_names(sub_module)# Check experts structure and handle accordingly- if _is_unfused_experts_module(sub_module.experts):- # Unfused experts: gate_up_proj/down_proj are nn.ModuleList- _set_amax_for_unfused_experts(sub_module.experts, attr_name=attr_name)- elif _is_fused_experts_module(sub_module.experts):+ if _is_fused_experts_module(sub_module.experts):# Fused experts: 3D Parameters (e.g., DeepseekV2Experts)# For fused experts, act_max is set on the parent MOE module, not individual experts# Skip processing here as they don't have individual Linear layers to calibrate@@ -1281,13 +1278,6 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_na)-def _is_unfused_experts_module(module: torch.nn.Module) -> bool:- """Check if the module is an unfused experts module (has ModuleList gate_up_proj/down_proj)."""- if not hasattr(module, "gate_up_proj") or not hasattr(module, "down_proj"):- return False- return isinstance(module.gate_up_proj, torch.nn.ModuleList) and isinstance(module.down_proj, torch.nn.ModuleList)--def _is_fused_experts_module(module: torch.nn.Module) -> bool:"""Check if the module is a fused experts module (has 3D Parameter gate_up_proj/down_proj)."""if not hasattr(module, "gate_up_proj") or not hasattr(module, "down_proj"):@@ -1300,29 +1290,6 @@ def _is_fused_experts_module(module: torch.nn.Module) -> bool:)-def _set_amax_for_unfused_experts(experts_module: torch.nn.Module, attr_name: str = "act_max"):- """Set amax for unfused experts module with ModuleList attributes.-- This handles experts modules that have been unfused to have:- - gate_up_proj: nn.ModuleList of nn.Linear (input projections, unified scale)- - down_proj: nn.ModuleList of nn.Linear (output projections)- """- if hasattr(experts_module, "gate_up_proj") and isinstance(experts_module.gate_up_proj, torch.nn.ModuleList):- unify_scale = envs.AR_ENABLE_UNIFY_MOE_INPUT_SCALE- set_amax_for_uncalibrated_experts(- list(experts_module.gate_up_proj),- attr_name=attr_name,- unify_all=unify_scale,- )-- if hasattr(experts_module, "down_proj") and isinstance(experts_module.down_proj, torch.nn.ModuleList):- set_amax_for_uncalibrated_experts(- list(experts_module.down_proj),- attr_name=attr_name,- unify_all=False,- )--def _set_amax_for_moe_auxiliary_layers(moe_module: torch.nn.Module, attr_name: str = "act_max"):"""Set amax for auxiliary layers in MOE modules (gate/router, shared_experts).@@ -1393,17 +1360,8 @@ def _get_reference_amax_from_experts(moe_module: torch.nn.Module, attr_name: strexperts = moe_module.experts- # Handle unfused experts (ModuleList)- if _is_unfused_experts_module(experts):- for proj_list in [getattr(experts, "gate_up_proj", None), getattr(experts, "down_proj", None)]:- if proj_list is not None and isinstance(proj_list, torch.nn.ModuleList):- for layer in proj_list:- amax = get_nested_attr(layer, attr_name)- if amax is not None:- amax_values.append(amax)-# Handle iterable experts (list of modules)- elif isinstance(experts, collections.abc.Iterable):+ if isinstance(experts, collections.abc.Iterable):expert_linear_names = get_expert_linear_names(moe_module)for expert in experts:for linear_name in expert_linear_names:diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.pyindex 37b8a8a4d..1d6bdcceb 100644--- a/test/test_cpu/quantization/test_mxfp_nvfp.py+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py@@ -54,44 +54,7 @@ class TestAutoRoundFP:)compressed_model, _ = autoround.quantize()moe = compressed_model.model.layers[1].mlp- experts = moe.experts-- def _has_act_max(layer):- if layer is None:- return False- if hasattr(layer, "orig_layer"):- layer = layer.orig_layer- return hasattr(layer, "act_max")-- found_act_max = False- if hasattr(experts, "gate_up_proj") and isinstance(experts.gate_up_proj, torch.nn.ModuleList):- if len(experts.gate_up_proj) > 0:- found_act_max = _has_act_max(experts.gate_up_proj[0])- elif isinstance(experts, collections.abc.Iterable):- first_expert = next(iter(experts), None)- if first_expert is not None:- for linear_name in [- "gate_proj",- "up_proj",- "down_proj",- "linear_fc1",- "linear_fc2",- "w1",- "w2",- "w3",- ]:- if hasattr(first_expert, linear_name):- found_act_max = _has_act_max(getattr(first_expert, linear_name))- if found_act_max:- break- elif hasattr(moe, "act_max"):- found_act_max = True-- assert found_act_max, "Missing act_max on MOE expert layers"- lm_head = compressed_model.lm_head- assert hasattr(lm_head, "orig_layer") and hasattr(- lm_head.orig_layer, "act_max"- ), "Illegal NVFP4 quantization for lm_head layer"+ assert hasattr(moe.experts[0].gate_proj.orig_layer, "act_max")shutil.rmtree(self.save_dir, ignore_errors=True)def test_nvfp4_moe_actmax_ar(self, tiny_deepseek_v2_model_path, dataloader):@@ -363,6 +326,8 @@ class TestAutoRoundFP:model_name = tiny_qwen_moe_model_pathlayer_config = {"layers.0": {"bits": 16, "act_bits": 16},+ # general recipe allows moe quantization+ "layers.1.mlp.experts": {"bits": 16, "act_bits": 16},}scheme = "nvfp4"autoround = AutoRound(diff --git a/test/test_cuda/models/test_moe_experts_interface.py b/test/test_cuda/models/test_moe_experts_interface.pyindex a292d4294..85605eda0 100644--- a/test/test_cuda/models/test_moe_experts_interface.py+++ b/test/test_cuda/models/test_moe_experts_interface.py@@ -79,22 +79,26 @@ def test_unfuse_experts_weights():success = _unfuse_experts_weights_inplace(module, check_decorator=False)assert success, "Failed to unfuse weights"- # Verify structure- assert isinstance(module.gate_up_proj, nn.ModuleList)- assert isinstance(module.down_proj, nn.ModuleList)- assert len(module.gate_up_proj) == num_experts- assert len(module.down_proj) == num_experts+ # Verify structure - per-expert containers with individual Linear layers+ for i in range(num_experts):+ expert = getattr(module, str(i))+ assert isinstance(expert.gate_proj, nn.Linear), f"Expert {i} gate_proj is not nn.Linear"+ assert isinstance(expert.up_proj, nn.Linear), f"Expert {i} up_proj is not nn.Linear"+ assert isinstance(expert.down_proj, nn.Linear), f"Expert {i} down_proj is not nn.Linear"# Verify weights are preserved+ # gate_up_proj was (num_experts, 2*intermediate, hidden), not transposed+ # Split along dim 1: gate -> [:intermediate_dim], up -> [intermediate_dim:]for i in range(num_experts):- # gate_up: original (2*intermediate, hidden), linear.weight should be same+ expert = getattr(module, str(i))assert torch.allclose(- module.gate_up_proj[i].weight.data, original_gate_up[i], atol=1e-6- ), f"gate_up weight mismatch for expert {i}"-- # down: original (hidden, intermediate), linear.weight should be same+ expert.gate_proj.weight.data, original_gate_up[i, :intermediate_dim, :], atol=1e-6+ ), f"gate weight mismatch for expert {i}"+ assert torch.allclose(+ expert.up_proj.weight.data, original_gate_up[i, intermediate_dim:, :], atol=1e-6+ ), f"up weight mismatch for expert {i}"assert torch.allclose(- module.down_proj[i].weight.data, original_down[i], atol=1e-6+ expert.down_proj.weight.data, original_down[i], atol=1e-6), f"down weight mismatch for expert {i}"@@ -128,20 +132,27 @@ def test_unfuse_experts_weights_transposed():success = _unfuse_experts_weights_inplace(module, check_decorator=False)assert success, "Failed to unfuse transposed weights"- # Verify structure- assert isinstance(module.gate_up_proj, nn.ModuleList)- assert isinstance(module.down_proj, nn.ModuleList)+ # Verify structure - per-expert containers with individual Linear layers+ for i in range(num_experts):+ expert = getattr(module, str(i))+ assert isinstance(expert.gate_proj, nn.Linear), f"Expert {i} gate_proj is not nn.Linear"+ assert isinstance(expert.up_proj, nn.Linear), f"Expert {i} up_proj is not nn.Linear"+ assert isinstance(expert.down_proj, nn.Linear), f"Expert {i} down_proj is not nn.Linear"# Verify weights are correctly transposed+ # gate_up_proj was (num_experts, hidden, 2*intermediate), transposed+ # Split along dim 2: gate -> [:, :intermediate_dim], up -> [:, intermediate_dim:]+ # Then transposed: (hidden, intermediate) -> (intermediate, hidden)for i in range(num_experts):- # gate_up: original (hidden, 2*intermediate), should be transposed to (2*intermediate, hidden)+ expert = getattr(module, str(i))assert torch.allclose(- module.gate_up_proj[i].weight.data, original_gate_up[i].t(), atol=1e-6- ), f"gate_up weight mismatch for expert {i}"-- # down: original (intermediate, hidden), should be transposed to (hidden, intermediate)+ expert.gate_proj.weight.data, original_gate_up[i, :, :intermediate_dim].t(), atol=1e-6+ ), f"gate weight mismatch for expert {i}"+ assert torch.allclose(+ expert.up_proj.weight.data, original_gate_up[i, :, intermediate_dim:].t(), atol=1e-6+ ), f"up weight mismatch for expert {i}"assert torch.allclose(- module.down_proj[i].weight.data, original_down[i].t(), atol=1e-6+ expert.down_proj.weight.data, original_down[i].t(), atol=1e-6), f"down weight mismatch for expert {i}"@@ -236,4 +247,6 @@ def test_prepare_model_for_moe_quantization():# Verifyassert model.config._experts_implementation == "linear_loop"assert len(unfused) == 1- assert isinstance(model.layer["experts"].gate_up_proj, nn.ModuleList)+ experts = model.layer["experts"]+ expert_0 = getattr(experts, "0")+ assert isinstance(expert_0.gate_proj, nn.Linear)diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.pyindex 8c275d595..be71eaf8a 100644--- a/test/test_cuda/models/test_moe_model.py+++ b/test/test_cuda/models/test_moe_model.py@@ -2,14 +2,11 @@ import shutilimport pytestimport torch-import torch.nn as nnfrom transformers import AutoConfig, AutoProcessor, AutoTokenizer, Llama4ForConditionalGenerationfrom transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM-from transformers.models.qwen3.modeling_qwen3 import Qwen3Config, Qwen3ForCausalLM, Qwen3MLPfrom transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGenerationfrom auto_round import AutoRound-from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBasefrom ...helpers import check_version@@ -135,47 +132,6 @@ def test_llama4(setup_llama4):shutil.rmtree(output_dir, ignore_errors=True)-class NewQwen3MLP(ReplacementModuleBase):- def __init__(self, original: Qwen3MLP, config: Qwen3Config):- super().__init__(original)- self.config = config- self.hidden_size = config.hidden_size- self.intermediate_size = config.intermediate_size- self.new_gate_proj = original.gate_proj- self.new_up_proj = original.up_proj- self.new_down_proj = original.down_proj- self.act_fn = nn.GELU()-- def forward(self, x):- gate = self.new_gate_proj(x)- up = self.new_up_proj(x)- gate_act = self.act_fn(gate)- act = gate_act * up- out = self.new_down_proj(act)- return out-- @classmethod- def original_module_class(cls) -> str:- return "Qwen3MLP"-- @classmethod- def from_original(cls, original: Qwen3MLP, config: Qwen3Config):- return cls(original, config)---def setup_qwen3():- """Fixture to set up the qwen3 model and tokenizer."""- model_name = "/models/Qwen3-0.6B"- tokenizer = AutoTokenizer.from_pretrained(model_name)- config = AutoConfig.from_pretrained(model_name)- config.num_hidden_layers = 2 # Reduce layers for testing- config.layer_types = config.layer_types[:2] # Reduce layers for testing- model = Qwen3ForCausalLM(config)- output_dir = "test_quantized_qwen3"- return model, tokenizer, output_dir, config--@pytest.fixturedef setup_qwen35_moe():"""Fixture to set up the Qwen3.5 MoE model, tokenizer, and processor."""@@ -224,6 +180,7 @@ def test_qwen3_5_moe(setup_qwen35_moe):shutil.rmtree(output_dir, ignore_errors=True)+# using general recipe when transformers >= 5.0.0def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):model, tokenizer, processor, output_dir, config = setup_qwen3_vl_moeautoround = AutoRound(@@ -257,34 +214,4 @@ def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):print(tokenizer.decode(loaded_model.generate(**inputs, max_new_tokens=50)[0]))# clean the output directory after testshutil.rmtree(output_dir, ignore_errors=True)---def has_module(model: torch.nn.Module, module_name: str) -> bool:- for n, m in model.named_modules():- if module_name in n:- return True- return False---def test_register_module_out_of_tree_base():- from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase-- for name, subclass in ReplacementModuleBase._replacement_registry.items():- if name == "Qwen3MLP":- assert subclass == NewQwen3MLP, "Qwen3MLP not registered correctly."---def test_register_module_out_of_tree_model(setup_qwen3):- model, tokenizer, output_dir, config = setup_qwen3- quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")- # Ensure the quantized model is not None- assert quantized_model is not None, "Quantized model should not be None."- check_module_names = ["new_gate_proj", "new_up_proj", "new_down_proj"]- for name in check_module_names:- assert has_module(quantized_model, name), f"Module {name} not found in quantized model."- loaded_model = Qwen3ForCausalLM.from_pretrained(output_dir)- quantized_model.to("cuda")-- for name in check_module_names:- assert has_module(loaded_model, name), f"Module {name} not found in loaded model."loaded_model.to("cuda")