Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Update MOE linear_loop implementation for speedup and matching name in vLLM (#1478)

Xin Hed1501180a706
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
index cc8b11a93..39964319a 100644
--- a/auto_round/compressors/shard_writer.py
+++ b/auto_round/compressors/shard_writer.py
@@ -161,6 +161,8 @@ class ShardWriter:
for pname, tensor in full_sd.items():
if pname in all_saved_names:
continue
+ if tensor.device.type == "meta":
+ continue
layer_name = ".".join(pname.split(".")[:-1])
if self.lm_head_name is not None and layer_name == self.lm_head_name and tie_word_embeddings:
lm_head_module = get_module(self.model, self.lm_head_name)
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 3bdacf096..aec990b37 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -19,59 +19,6 @@ import torch.nn as nn
from auto_round.utils import copy_python_files_from_model_cache, logger, unsupported_meta_device
-def _has_unfused_moe_experts(model: nn.Module) -> bool:
- """Check if model has unfused MOE experts (nn.ModuleList instead of 3D Parameter).
-
- This is used to detect if we need to bypass transformers' weight conversion
- during save_pretrained.
- """
- for module in model.modules():
- if hasattr(module, "gate_up_proj") and isinstance(module.gate_up_proj, nn.ModuleList):
- if hasattr(module, "down_proj") and isinstance(module.down_proj, nn.ModuleList):
- return True
- return False
-
-
-def _save_model_state_dict(
- model: nn.Module,
- save_dir: str,
- max_shard_size: str = "5GB",
- safe_serialization: bool = True,
-):
- """Save model using state_dict directly, bypassing transformers' weight conversion.
-
- This is needed for models with unfused MOE experts where transformers'
- revert_weight_conversion expects 3D tensor format but we have ModuleList.
- """
- import torch
- from safetensors.torch import save_file
-
- os.makedirs(save_dir, exist_ok=True)
-
- # Save config
- if hasattr(model, "config") and model.config is not None:
- model.config.save_pretrained(save_dir)
-
- if hasattr(model, "generation_config") and model.generation_config is not None:
- try:
- model.generation_config.save_pretrained(save_dir)
- except Exception:
- pass # generation_config save can fail for some models
-
- # Get state dict
- state_dict = model.state_dict()
-
- # Save weights
- if safe_serialization:
- # Save as safetensors
- weights_file = os.path.join(save_dir, "model.safetensors")
- save_file(state_dict, weights_file)
- else:
- # Save as pytorch bin
- weights_file = os.path.join(save_dir, "pytorch_model.bin")
- torch.save(state_dict, weights_file)
-
-
def save_model(
model: nn.Module,
save_dir: str,
@@ -101,10 +48,6 @@ def save_model(
"""
os.makedirs(save_dir, exist_ok=True)
- # Check if model has unfused MOE experts - if so, we need to bypass transformers'
- # weight conversion which expects original 3D tensor format
- has_unfused_experts = _has_unfused_moe_experts(model)
-
if unsupported_meta_device(model):
if hasattr(model, "config") and model.config is not None:
model.config.save_pretrained(save_dir)
@@ -113,11 +56,6 @@ def save_model(
if hasattr(model, "generation_config"):
setattr(model.generation_config, "do_sample", True)
model.generation_config.save_pretrained(save_dir)
- elif has_unfused_experts:
- # For models with unfused MOE experts, save state_dict directly to avoid
- # transformers' revert_weight_conversion which expects 3D tensor format
- logger.info("Saving model with unfused MOE experts using state_dict (bypassing weight conversion)")
- _save_model_state_dict(model, save_dir, max_shard_size, safe_serialization)
else:
try:
model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 8c8b8782f..1e4324244 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -41,6 +41,7 @@ from auto_round.utils import (
get_block_names,
get_module,
is_hpex_available,
+ is_transformers_version_greater_or_equal_5,
set_module,
)
@@ -570,6 +571,23 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
logger.warning(f"Forced model to {target_dtype}")
+def disable_moe_conversion_mapping(model):
+ """Disables MoE-specific checkpoint conversion mappings to prevent unintended weight merging."""
+ from transformers.conversion_mapping import (
+ get_checkpoint_conversion_mapping,
+ register_checkpoint_conversion_mapping,
+ )
+ from transformers.core_model_loading import WeightRenaming
+
+ model_type = getattr(model.config, "model_type", None)
+ if model_type is not None:
+ conversions = get_checkpoint_conversion_mapping(model_type)
+ if conversions is not None:
+ # 只保留 WeightRenaming,跳过 WeightConverter(MoE merge 操作)
+ filtered = [c for c in conversions if isinstance(c, WeightRenaming)]
+ register_checkpoint_conversion_mapping(model_type, mapping=filtered, overwrite=True)
+
+
def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Module, list]:
"""Converts a HuggingFace model into an AutoRound model by replacing layers with quantized layers.
@@ -590,6 +608,8 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
NotImplementedError: If the GPTQ model uses an unsupported `g_idx`.
ValueError: If quantization backend is not properly specified.
"""
+ if is_transformers_version_greater_or_equal_5():
+ disable_moe_conversion_mapping(model)
quantization_config = model.config.quantization_config
# Check desc_act + static_groups
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index aaef9ab8b..85b294759 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -31,10 +31,12 @@ Usage:
# Now the model uses linear_loop forward which supports quantized nn.Linear layers
"""
+
import torch
from torch import nn
from auto_round.utils import clear_memory, logger
+from auto_round.utils.device import memory_monitor
try:
from transformers.integrations.moe import ALL_EXPERTS_FUNCTIONS
@@ -54,7 +56,10 @@ LINEAR_LOOP_IMPL = "linear_loop"
# output_multiplier: output dimension multiplier (e.g., 2 for fused gate+up projection)
KNOWN_PROJECTION_PATTERNS = {
# Transformers 5.0+ standard (Qwen3-MoE, etc.)
- "gate_up_proj": {"is_input_proj": True, "output_multiplier": 2}, # hidden -> 2*intermediate
+ # gate_up_proj is auto-split into gate_proj + up_proj during unfusing
+ "gate_up_proj": {"is_input_proj": True, "output_multiplier": 2, "split_into": ["gate_proj", "up_proj"]},
+ "gate_proj": {"is_input_proj": True, "output_multiplier": 1}, # hidden -> intermediate (gate)
+ "up_proj": {"is_input_proj": True, "output_multiplier": 1}, # hidden -> intermediate (up)
"down_proj": {"is_input_proj": False, "output_multiplier": 1}, # intermediate -> hidden
# Mixtral-style
"w1": {"is_input_proj": True, "output_multiplier": 1}, # gate: hidden -> intermediate
@@ -67,6 +72,103 @@ KNOWN_PROJECTION_PATTERNS = {
}
+class _ExpertContainer(nn.Module):
+ """Lightweight container for a single expert's projection layers.
+
+ Each expert has its projections (e.g., gate_proj, up_proj, down_proj)
+ as direct attributes. When attached as numbered children of the experts
+ module (e.g., module.add_module("0", container)), PyTorch naturally
+ produces state_dict keys like: {prefix}0.gate_proj.weight
+ which matches the standard checkpoint format without any hooks.
+ """
+
+ pass
+
+
+def _install_compact_expert_repr(module: nn.Module) -> None:
+ """Install compact __repr__ on the module's class.
+
+ Collapses identical _ExpertContainer children into a range display,
+ similar to nn.ModuleList::
+
+ GptOssExperts(
+ (0-63): 64 x _ExpertContainer(
+ (gate_proj): Linear(...)
+ (up_proj): Linear(...)
+ (down_proj): Linear(...)
+ )
+ )
+ """
+ cls = module.__class__
+ if "_compact_expert_repr_installed" in cls.__dict__:
+ return
+
+ _original_repr = cls.__repr__
+
+ def _compact_repr(self):
+ from torch.nn.modules.module import _addindent
+
+ # Separate expert containers from other children
+ expert_items = []
+ other_items = []
+ for key, child in self._modules.items():
+ if isinstance(child, _ExpertContainer):
+ expert_items.append((key, child))
+ else:
+ other_items.append((key, child))
+
+ if not expert_items:
+ return _original_repr(self)
+
+ lines = []
+ extra_repr = self.extra_repr()
+ if extra_repr:
+ lines.extend(extra_repr.split("\n"))
+
+ # Print non-expert children normally
+ for key, child in other_items:
+ mod_str = repr(child)
+ mod_str = _addindent(mod_str, 2)
+ lines.append(f"({key}): {mod_str}")
+
+ # Collapse expert containers into a single range line
+ first_key = expert_items[0][0]
+ last_key = expert_items[-1][0]
+ sample_repr = repr(expert_items[0][1])
+ n = len(expert_items)
+ sample_repr = _addindent(sample_repr, 2)
+ if n > 1:
+ lines.append(f"({first_key}-{last_key}): {n} x {sample_repr}")
+ else:
+ lines.append(f"({first_key}): {sample_repr}")
+
+ main_str = self._get_name() + "("
+ if lines:
+ main_str += "\n " + "\n ".join(lines) + "\n"
+ main_str += ")"
+ return main_str
+
+ cls.__repr__ = _compact_repr
+ cls._compact_expert_repr_installed = True
+
+ # Make the module subscriptable and iterable: module[0], module[1], len(module), for e in module
+ if not hasattr(cls, "__getitem__") or "__getitem__" not in cls.__dict__:
+
+ def _getitem(self, idx):
+ return getattr(self, str(idx))
+
+ def _len(self):
+ return self.num_experts
+
+ def _iter(self):
+ for i in range(self.num_experts):
+ yield getattr(self, str(i))
+
+ cls.__getitem__ = _getitem
+ cls.__len__ = _len
+ cls.__iter__ = _iter
+
+
def linear_loop_experts_forward(
self: nn.Module,
hidden_states: torch.Tensor,
@@ -75,13 +177,16 @@ def linear_loop_experts_forward(
) -> torch.Tensor:
"""Forward using individual nn.Linear layers per expert.
- This implementation loops over experts and uses self.gate_up_proj[i] and
- self.down_proj[i] as nn.Linear layers (or quantized equivalents), enabling
- proper quantization support.
+ This implementation loops over experts and accesses per-expert containers
+ (self._modules["0"], self._modules["1"], ...) each with gate_proj,
+ up_proj, and down_proj as nn.Linear layers (or quantized equivalents),
+ enabling proper quantization support.
- Expected module attributes:
- - gate_up_proj: nn.ModuleList of nn.Linear (in_features=hidden_dim, out_features=2*intermediate_dim)
- - down_proj: nn.ModuleList of nn.Linear (in_features=intermediate_dim, out_features=hidden_dim)
+ Expected module structure:
+ - Numbered children (0, 1, ..., num_experts-1), each an _ExpertContainer with:
+ - gate_proj: nn.Linear (in_features=hidden_dim, out_features=intermediate_dim)
+ - up_proj: nn.Linear (in_features=hidden_dim, out_features=intermediate_dim)
+ - down_proj: nn.Linear (in_features=intermediate_dim, out_features=hidden_dim)
- act_fn: activation function
- num_experts: number of experts
- _apply_gate: optional custom gating function
@@ -133,18 +238,20 @@ def linear_loop_experts_forward(
expert_input = selected_hidden_states[mask] # (num_samples_for_expert, hidden_dim)
- # Use nn.Linear layers for this expert
- gate_up_out = self.gate_up_proj[expert_idx](expert_input) # (num_samples, 2*intermediate_dim)
+ # Get this expert's container with its projection layers
+ expert = getattr(self, str(expert_idx))
+ gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)
+ up_out = expert.up_proj(expert_input) # (num_samples, intermediate_dim)
# Apply gating
if hasattr(self, "_apply_gate"):
+ gate_up_out = torch.cat([gate_out, up_out], dim=-1)
gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)
else:
- gate, up = gate_up_out.chunk(2, dim=-1)
- gated_out = self.act_fn(gate) * up
+ gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)
# Down projection
- expert_out = self.down_proj[expert_idx](gated_out) # (num_samples, hidden_dim)
+ expert_out = expert.down_proj(gated_out) # (num_samples, hidden_dim)
# Store results
out_per_sample[mask] = expert_out.to(out_per_sample.dtype)
@@ -264,8 +371,14 @@ def _unfuse_single_projection(
is_transposed: bool,
dtype: torch.dtype,
target_device: torch.device,
-) -> nn.ModuleList | None:
- """Unfuse a single projection from 3D Parameter to ModuleList of Linear layers.
+) -> list | None:
+ """Unfuse a single projection from 3D Parameter to a list of Linear layers.
+
+ Optimized to minimize device allocations and copies:
+ - Moves the full 3D tensor to CPU in a single transfer
+ - Performs batch transpose on CPU if needed
+ - Creates Linear shells on meta device (no allocation)
+ - Directly assigns weight slices as Parameters (zero-copy on CPU)
Args:
module: The experts module
@@ -276,7 +389,7 @@ def _unfuse_single_projection(
target_device: Device for the Linear layers
Returns:
- ModuleList of Linear layers, or None if projection doesn't exist
+ List of Linear layers, or None if projection doesn't exist
"""
param = getattr(module, proj_name, None)
if param is None or not isinstance(param, nn.Parameter) or param.dim() != 3:
@@ -293,32 +406,53 @@ def _unfuse_single_projection(
bias_param = getattr(module, bias_name, None)
has_bias = bias_param is not None
- # Create ModuleList
- linears = nn.ModuleList()
source_device = param.device
-
+ is_meta = source_device.type == "meta"
+
+ # Prepare weight slices on CPU in batch (single D2H transfer + batch transpose)
+ if not is_meta:
+ # Single transfer: GPU -> CPU (or no-op if already on CPU)
+ weights_cpu = param.data.cpu() # (num_experts, dim1, dim2)
+ if is_transposed:
+ # Batch transpose: (num_experts, in, out) -> (num_experts, out, in)
+ weights_cpu = weights_cpu.transpose(1, 2)
+ # Ensure contiguous layout so unbind produces contiguous 2D slices.
+ # This is needed when the param comes from a chunk split (Phase 1)
+ # which produces non-contiguous views even on CPU.
+ if not weights_cpu.is_contiguous():
+ weights_cpu = weights_cpu.contiguous()
+ # Unbind into a tuple of 2D tensors (zero-copy views since contiguous)
+ weight_slices = weights_cpu.unbind(0)
+
+ if has_bias:
+ bias_cpu = bias_param.data.cpu()
+ # Ensure contiguous — bias may come from a chunk split (Phase 1)
+ if not bias_cpu.is_contiguous():
+ bias_cpu = bias_cpu.contiguous()
+ bias_slices = bias_cpu.unbind(0)
+
+ # Create Linear shells on meta device (no memory allocation)
+ linears = []
for i in range(num_experts):
- linear = nn.Linear(in_features, out_features, bias=has_bias, dtype=dtype, device=target_device)
-
- # Copy weights if not on meta device
- if source_device.type != "meta":
- if is_transposed:
- linear.weight.data.copy_(param[i].t())
- else:
- linear.weight.data.copy_(param[i])
+ # meta device: creates the module structure without allocating weight storage
+ linear = nn.Linear(in_features, out_features, bias=has_bias, dtype=dtype, device="meta")
+ if not is_meta:
+ # Direct parameter assignment — no copy, just references the CPU tensor slice
+ linear.weight = nn.Parameter(weight_slices[i])
if has_bias:
- linear.bias.data.copy_(bias_param[i])
+ linear.bias = nn.Parameter(bias_slices[i])
linears.append(linear)
- # Release original parameter memory using to_empty
- if source_device.type != "meta":
+ # Release original parameter memory
+ if not is_meta:
try:
param.data = param.data.to_empty(device="meta")
+ if has_bias:
+ bias_param.data = bias_param.data.to_empty(device="meta")
logger.debug(f"Released memory for {proj_name} using to_empty(device='meta')")
except Exception:
- # Fallback: just delete
pass
return linears
@@ -329,10 +463,10 @@ def _unfuse_experts_weights_inplace(
check_decorator: bool = True,
projection_names: list[str] | None = None,
) -> bool:
- """Convert fused 3D expert weights to nn.ModuleList of nn.Linear layers.
+ """Convert fused 3D expert weights to per-expert containers of nn.Linear layers.
This function modifies the module in-place, replacing fused 3D Parameters
- with nn.ModuleList[nn.Linear] for each detected projection.
+ with numbered _ExpertContainer children, each holding nn.Linear projections.
Args:
module: The experts module to unfuse
@@ -375,22 +509,90 @@ def _unfuse_experts_weights_inplace(
dtype = first_param.dtype
target_device = first_param.device if first_param.device.type != "meta" else "cpu"
- # Unfuse each projection
- unfused_count = 0
+ # Phase 1: Split fused projections (e.g., gate_up_proj -> gate_proj + up_proj) into separate 3D params
+ extra_projections = {}
+ fused_to_remove = []
+ for proj_name, config in detected_projections.items():
+ split_into = config.get("split_into")
+ if not split_into:
+ continue
+ param = getattr(module, proj_name, None)
+ if param is None or not isinstance(param, nn.Parameter) or param.dim() != 3:
+ continue
+ # Split along output dimension
+ split_dim = 2 if is_transposed else 1
+ split_params = param.chunk(len(split_into), dim=split_dim)
+
+ # Also split bias if present (e.g., gate_up_proj_bias -> gate_proj_bias + up_proj_bias)
+ bias_name = f"{proj_name}_bias"
+ bias_param = getattr(module, bias_name, None)
+ bias_splits = None
+ if bias_param is not None and isinstance(bias_param, nn.Parameter) and bias_param.dim() == 2:
+ bias_splits = bias_param.chunk(len(split_into), dim=1)
+
+ for i, (split_name, split_param) in enumerate(zip(split_into, split_params)):
+ # Avoid .contiguous() here — _unfuse_single_projection will handle it
+ # during batch transpose/unbind, saving a full-tensor copy
+ setattr(module, split_name, nn.Parameter(split_param))
+ if bias_splits is not None:
+ setattr(module, f"{split_name}_bias", nn.Parameter(bias_splits[i]))
+ extra_projections[split_name] = KNOWN_PROJECTION_PATTERNS.get(
+ split_name, {"is_input_proj": True, "output_multiplier": 1}
+ )
+ delattr(module, proj_name)
+ if bias_param is not None:
+ delattr(module, bias_name)
+ fused_to_remove.append(proj_name)
+ logger.debug(f"Split {proj_name} -> {split_into}: {num_experts} experts")
+
+ # Remove fused entries and add split entries
+ for name in fused_to_remove:
+ del detected_projections[name]
+ detected_projections.update(extra_projections)
+
+ # Phase 2: Unfuse all 3D params into per-expert Linear layers
+ proj_linears = {} # proj_name -> [Linear_expert0, Linear_expert1, ...]
for proj_name in detected_projections:
linears = _unfuse_single_projection(module, proj_name, num_experts, is_transposed, dtype, target_device)
if linears is not None:
- # Delete original parameter and set new ModuleList
delattr(module, proj_name)
- setattr(module, proj_name, linears)
- unfused_count += 1
- logger.debug(f"Unfused {proj_name}: {num_experts} experts")
+ # Also remove the bias parameter if it exists (already absorbed into Linear.bias)
+ bias_name = f"{proj_name}_bias"
+ if hasattr(module, bias_name):
+ delattr(module, bias_name)
+ proj_linears[proj_name] = linears
+
+ if not proj_linears:
+ return False
+
+ # Phase 3: Assemble per-expert containers as direct numbered children.
+ # This produces state_dict keys like: {prefix}0.gate_proj.weight
+ # which naturally matches the standard checkpoint format — no hooks needed.
+ for expert_idx in range(num_experts):
+ container = _ExpertContainer()
+ for proj_name, linears in proj_linears.items():
+ setattr(container, proj_name, linears[expert_idx])
+ module.add_module(str(expert_idx), container)
# Ensure num_experts is set
if not hasattr(module, "num_experts"):
module.num_experts = num_experts
- return unfused_count > 0
+ # Mark as unfused for detection during save
+ module._unfused_experts = True
+
+ # Mark all unfused modules as already initialized so transformers'
+ # _initialize_weights skips them. Without this, the model's _init_weights
+ # tries to access the original fused attributes (e.g., gate_up_proj) which
+ # no longer exist after unfusing.
+ module._is_hf_initialized = True
+ for child in module.modules():
+ child._is_hf_initialized = True
+
+ # Install compact repr to collapse identical expert containers in print output
+ _install_compact_expert_repr(module)
+
+ return True
def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = LINEAR_LOOP_IMPL) -> list[str]:
@@ -399,7 +601,7 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = L
This function:
1. Registers the linear_loop experts implementation with transformers
2. Sets model.config._experts_implementation = implementation
- 3. Unfuses all fused MOE expert weights into nn.ModuleList[nn.Linear]
+ 3. Unfuses all fused MOE expert weights into per-expert containers of nn.Linear
After calling this function, the model's forward pass will use individual
nn.Linear layers per expert, which can be quantized normally.
@@ -411,7 +613,6 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = L
Returns:
List of module names that were unfused
"""
- # Register our custom implementation
if not register_linear_loop_experts():
raise RuntimeError(
"Failed to register linear_loop experts implementation. "
@@ -423,13 +624,13 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = L
for name, module in model.named_modules():
if _unfuse_experts_weights_inplace(module):
unfused_modules.append(name)
- logger.debug(f"Unfused expert weights in: {name}")
+ logger.debug(f"[MoE Prep] Unfused '{name}'")
# Only set config if we actually unfused something
# Models that don't support the decorator (like Llama4) won't have anything unfused
# and should use full module replacement instead
if unfused_modules:
- logger.info(f"Unfused {len(unfused_modules)} MOE experts modules for quantization")
+ logger.info(f"[MoE Prep] Unfused {len(unfused_modules)} MOE experts modules")
clear_memory()
# Set config for linear_loop forward
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index ac6bb8ee2..f84515348 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -20,32 +20,35 @@ import torch
from tqdm import tqdm
from transformers import PreTrainedModel
-# Import constant for expert implementation name
-from auto_round.modeling.fused_moe.moe_experts_interface import LINEAR_LOOP_IMPL
-from auto_round.utils import LazyImport, dump_mem_usage, dump_memory_usage_ctx, global_state, logger
+from auto_round.utils import (
+ LazyImport,
+ dump_mem_usage,
+ dump_memory_usage_ctx,
+ global_state,
+ is_transformers_version_greater_or_equal_5,
+ logger,
+)
BUILTIN_MODULES = {
+ # Llama4 has no use_experts_implementation, needs custom replacement to handle fused MoE blocks.
"llama4": LazyImport("auto_round.modeling.fused_moe.llama4"),
- "gpt_oss": LazyImport("auto_round.modeling.fused_moe.gpt_oss"),
- "qwen3_vl_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe"),
+ # DeepseekV2Attention enables q_scale calibration for deepseek v2 on Gaudi (#1299)
"deepseek_v2": LazyImport("auto_round.modeling.fused_moe.deepseek_v2"),
+ # supports transformers >= 5.0.0
"qwen3_5_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
"qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
}
-def _has_custom_replacement_only(model: torch.nn.Module) -> bool:
- """Check if all MOE-like modules in model have custom replacements registered.
-
- If all MOE modules are covered by BUILTIN_MODULES, we can skip the linear_loop path entirely.
- """
- if hasattr(model, "config") and hasattr(model.config, "model_type"):
- model_type = model.config.model_type
- if model_type in BUILTIN_MODULES:
- return True
- return False
+# transformers >= 5.0.0 supports the general linear_loop experts interface.
+if not is_transformers_version_greater_or_equal_5():
+ # transformers == 5.2.0: IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)
+ BUILTIN_MODULES["qwen3_vl_moe"] = LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe")
+ # transformers == 5.2.0: IndexError: index 4 is out of bounds for dimension 0 with size 4
+ BUILTIN_MODULES["gpt_oss"] = LazyImport("auto_round.modeling.fused_moe.gpt_oss")
+@dump_mem_usage("Applying general replacements")
def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
"""Handle fused MOE modules using transformers' linear_loop backend.
@@ -77,40 +80,29 @@ def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
def _import_required_replacements(model: torch.nn.Module) -> None:
"""Scan model and trigger lazy imports for registered replacement modules."""
- imported = set()
- if hasattr(model, "config") and hasattr(model.config, "model_type"):
- model_type = model.config.model_type
- if model_type in BUILTIN_MODULES:
- _ = BUILTIN_MODULES[model_type].__name__ # or any attribute
- imported.add(model_type)
- logger.debug(f"Loaded replacement module for {model_type}")
-
+ if not is_custom_model(model):
+ return
+ model_type = model.config.model_type
+ _ = BUILTIN_MODULES[model_type].__name__ # Trigger lazy import
+ logger.debug(f"Loaded replacement module for {model_type}")
-def _should_skip_moe_replacement(module: torch.nn.Module, model: torch.nn.Module) -> bool:
- """Skip MOE replacement if linear_loop experts are already unfused.
- This is only true when:
- 1. model.config._experts_implementation == "linear_loop" (set by prepare_model_for_moe_quantization)
- 2. The experts' gate_up_proj and down_proj are already nn.ModuleList
+def is_custom_model(model: torch.nn.Module) -> bool:
+ """Check if the model has a custom replacement registered via BUILTIN_MODULES.
- Note: _experts_implementation is only set when the experts class supports
- @use_experts_implementation decorator, so we don't need to check that again here.
+ Returns True if the model's model_type matches a key in BUILTIN_MODULES.
"""
- if not hasattr(model, "config"):
- return False
- if getattr(model.config, "_experts_implementation", None) != LINEAR_LOOP_IMPL:
- return False
- experts = getattr(module, "experts", None)
- if experts is None:
- return False
- gate_up = getattr(experts, "gate_up_proj", None)
- down = getattr(experts, "down_proj", None)
- result = isinstance(gate_up, torch.nn.ModuleList) and isinstance(down, torch.nn.ModuleList)
- logger.debug(
- f"_should_skip_moe_replacement for {module.__class__.__name__}: "
- f"gate_up type={type(gate_up).__name__}, down type={type(down).__name__}, skip={result}"
- )
- return result
+ if hasattr(model, "config") and hasattr(model.config, "model_type"):
+ return model.config.model_type in BUILTIN_MODULES
+ return False
+
+
+def _log_first_moe_block(model: torch.nn.Module, label: str) -> None:
+ """Log the first experts module found in the model for debugging."""
+ for name, module in model.named_modules():
+ if name.endswith(".experts"):
+ logger.info(f"Experts ({label}) [{name}] ({module.__class__.__name__}):\n{module}")
+ return
@dump_mem_usage("Materializing model", log_level="debug")
@@ -144,16 +136,6 @@ def release_original_module_(model: torch.nn.Module) -> None:
model.apply(_clear_source_module)
-def _has_meta_params_or_buffers(model: PreTrainedModel) -> bool:
- for _, param in model.named_parameters():
- if param.device.type == "meta":
- return True
- for _, buffer in model.named_buffers():
- if buffer.device.type == "meta":
- return True
- return False
-
-
def safe_to_cpu_(model: torch.nn.Module) -> None:
# If no replacement happened, move model to CPU directly
if global_state.replaced_module_count == 0:
@@ -309,11 +291,29 @@ def apply_replacements(
"""
_import_required_replacements(model)
- # Auto-detect and handle fused MOE modules if enabled
- # Skip if all MOE modules already have custom replacements registered
- if auto_detect_moe and not _has_custom_replacement_only(model):
+ _log_first_moe_block(model, "before replacement")
+
+ # Custom replacements first
+ if is_custom_model(model):
+ _apply_custom_replacements(model)
+ if auto_detect_moe and is_transformers_version_greater_or_equal_5():
_handle_moe_modules(model)
+ _log_first_moe_block(model, "after replacement")
+
+ return model
+
+
+@dump_mem_usage("Applying custom replacements")
+def _apply_custom_replacements(model: torch.nn.Module) -> list:
+ """Scan model and replace registered modules with custom implementations.
+
+ Args:
+ model: The model to scan and apply replacements to (modified in-place).
+
+ Returns:
+ List of (name, replacement_class) tuples for replaced modules.
+ """
replaced = []
# Step 1: Collect all modules that need replacement
@@ -324,9 +324,6 @@ def apply_replacements(
if isinstance(module, ReplacementModuleBase):
continue
class_name = module.__class__.__name__
- if ReplacementModuleBase.is_registered(class_name) and _should_skip_moe_replacement(module, model):
- logger.debug(f"Skipping replacement for {name}: linear_loop experts already unfused")
- continue
if ReplacementModuleBase.is_registered(class_name) and ReplacementModuleBase.get_replacement_class(
class_name
).is_to_be_replaced(module):
@@ -363,7 +360,7 @@ def apply_replacements(
global_state.replaced_module_count = len(replaced)
logger.info(f"Replaced {len(replaced)} modules")
- return model
+ return replaced
@dataclass
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 3d010d150..baceddecd 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1237,10 +1237,7 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_na
expert_input_proj_names = get_expert_input_proj_names(sub_module)
# Check experts structure and handle accordingly
- if _is_unfused_experts_module(sub_module.experts):
- # Unfused experts: gate_up_proj/down_proj are nn.ModuleList
- _set_amax_for_unfused_experts(sub_module.experts, attr_name=attr_name)
- elif _is_fused_experts_module(sub_module.experts):
+ if _is_fused_experts_module(sub_module.experts):
# Fused experts: 3D Parameters (e.g., DeepseekV2Experts)
# For fused experts, act_max is set on the parent MOE module, not individual experts
# Skip processing here as they don't have individual Linear layers to calibrate
@@ -1281,13 +1278,6 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_na
)
-def _is_unfused_experts_module(module: torch.nn.Module) -> bool:
- """Check if the module is an unfused experts module (has ModuleList gate_up_proj/down_proj)."""
- if not hasattr(module, "gate_up_proj") or not hasattr(module, "down_proj"):
- return False
- return isinstance(module.gate_up_proj, torch.nn.ModuleList) and isinstance(module.down_proj, torch.nn.ModuleList)
-
-
def _is_fused_experts_module(module: torch.nn.Module) -> bool:
"""Check if the module is a fused experts module (has 3D Parameter gate_up_proj/down_proj)."""
if not hasattr(module, "gate_up_proj") or not hasattr(module, "down_proj"):
@@ -1300,29 +1290,6 @@ def _is_fused_experts_module(module: torch.nn.Module) -> bool:
)
-def _set_amax_for_unfused_experts(experts_module: torch.nn.Module, attr_name: str = "act_max"):
- """Set amax for unfused experts module with ModuleList attributes.
-
- This handles experts modules that have been unfused to have:
- - gate_up_proj: nn.ModuleList of nn.Linear (input projections, unified scale)
- - down_proj: nn.ModuleList of nn.Linear (output projections)
- """
- if hasattr(experts_module, "gate_up_proj") and isinstance(experts_module.gate_up_proj, torch.nn.ModuleList):
- unify_scale = envs.AR_ENABLE_UNIFY_MOE_INPUT_SCALE
- set_amax_for_uncalibrated_experts(
- list(experts_module.gate_up_proj),
- attr_name=attr_name,
- unify_all=unify_scale,
- )
-
- if hasattr(experts_module, "down_proj") and isinstance(experts_module.down_proj, torch.nn.ModuleList):
- set_amax_for_uncalibrated_experts(
- list(experts_module.down_proj),
- attr_name=attr_name,
- unify_all=False,
- )
-
-
def _set_amax_for_moe_auxiliary_layers(moe_module: torch.nn.Module, attr_name: str = "act_max"):
"""Set amax for auxiliary layers in MOE modules (gate/router, shared_experts).
@@ -1393,17 +1360,8 @@ def _get_reference_amax_from_experts(moe_module: torch.nn.Module, attr_name: str
experts = moe_module.experts
- # Handle unfused experts (ModuleList)
- if _is_unfused_experts_module(experts):
- for proj_list in [getattr(experts, "gate_up_proj", None), getattr(experts, "down_proj", None)]:
- if proj_list is not None and isinstance(proj_list, torch.nn.ModuleList):
- for layer in proj_list:
- amax = get_nested_attr(layer, attr_name)
- if amax is not None:
- amax_values.append(amax)
-
# Handle iterable experts (list of modules)
- elif isinstance(experts, collections.abc.Iterable):
+ if isinstance(experts, collections.abc.Iterable):
expert_linear_names = get_expert_linear_names(moe_module)
for expert in experts:
for linear_name in expert_linear_names:
diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.py
index 37b8a8a4d..1d6bdcceb 100644
--- a/test/test_cpu/quantization/test_mxfp_nvfp.py
+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py
@@ -54,44 +54,7 @@ class TestAutoRoundFP:
)
compressed_model, _ = autoround.quantize()
moe = compressed_model.model.layers[1].mlp
- experts = moe.experts
-
- def _has_act_max(layer):
- if layer is None:
- return False
- if hasattr(layer, "orig_layer"):
- layer = layer.orig_layer
- return hasattr(layer, "act_max")
-
- found_act_max = False
- if hasattr(experts, "gate_up_proj") and isinstance(experts.gate_up_proj, torch.nn.ModuleList):
- if len(experts.gate_up_proj) > 0:
- found_act_max = _has_act_max(experts.gate_up_proj[0])
- elif isinstance(experts, collections.abc.Iterable):
- first_expert = next(iter(experts), None)
- if first_expert is not None:
- for linear_name in [
- "gate_proj",
- "up_proj",
- "down_proj",
- "linear_fc1",
- "linear_fc2",
- "w1",
- "w2",
- "w3",
- ]:
- if hasattr(first_expert, linear_name):
- found_act_max = _has_act_max(getattr(first_expert, linear_name))
- if found_act_max:
- break
- elif hasattr(moe, "act_max"):
- found_act_max = True
-
- assert found_act_max, "Missing act_max on MOE expert layers"
- lm_head = compressed_model.lm_head
- assert hasattr(lm_head, "orig_layer") and hasattr(
- lm_head.orig_layer, "act_max"
- ), "Illegal NVFP4 quantization for lm_head layer"
+ assert hasattr(moe.experts[0].gate_proj.orig_layer, "act_max")
shutil.rmtree(self.save_dir, ignore_errors=True)
def test_nvfp4_moe_actmax_ar(self, tiny_deepseek_v2_model_path, dataloader):
@@ -363,6 +326,8 @@ class TestAutoRoundFP:
model_name = tiny_qwen_moe_model_path
layer_config = {
"layers.0": {"bits": 16, "act_bits": 16},
+ # general recipe allows moe quantization
+ "layers.1.mlp.experts": {"bits": 16, "act_bits": 16},
}
scheme = "nvfp4"
autoround = AutoRound(
diff --git a/test/test_cuda/models/test_moe_experts_interface.py b/test/test_cuda/models/test_moe_experts_interface.py
index a292d4294..85605eda0 100644
--- a/test/test_cuda/models/test_moe_experts_interface.py
+++ b/test/test_cuda/models/test_moe_experts_interface.py
@@ -79,22 +79,26 @@ def test_unfuse_experts_weights():
success = _unfuse_experts_weights_inplace(module, check_decorator=False)
assert success, "Failed to unfuse weights"
- # Verify structure
- assert isinstance(module.gate_up_proj, nn.ModuleList)
- assert isinstance(module.down_proj, nn.ModuleList)
- assert len(module.gate_up_proj) == num_experts
- assert len(module.down_proj) == num_experts
+ # Verify structure - per-expert containers with individual Linear layers
+ for i in range(num_experts):
+ expert = getattr(module, str(i))
+ assert isinstance(expert.gate_proj, nn.Linear), f"Expert {i} gate_proj is not nn.Linear"
+ assert isinstance(expert.up_proj, nn.Linear), f"Expert {i} up_proj is not nn.Linear"
+ assert isinstance(expert.down_proj, nn.Linear), f"Expert {i} down_proj is not nn.Linear"
# Verify weights are preserved
+ # gate_up_proj was (num_experts, 2*intermediate, hidden), not transposed
+ # Split along dim 1: gate -> [:intermediate_dim], up -> [intermediate_dim:]
for i in range(num_experts):
- # gate_up: original (2*intermediate, hidden), linear.weight should be same
+ expert = getattr(module, str(i))
assert torch.allclose(
- module.gate_up_proj[i].weight.data, original_gate_up[i], atol=1e-6
- ), f"gate_up weight mismatch for expert {i}"
-
- # down: original (hidden, intermediate), linear.weight should be same
+ expert.gate_proj.weight.data, original_gate_up[i, :intermediate_dim, :], atol=1e-6
+ ), f"gate weight mismatch for expert {i}"
+ assert torch.allclose(
+ expert.up_proj.weight.data, original_gate_up[i, intermediate_dim:, :], atol=1e-6
+ ), f"up weight mismatch for expert {i}"
assert torch.allclose(
- module.down_proj[i].weight.data, original_down[i], atol=1e-6
+ expert.down_proj.weight.data, original_down[i], atol=1e-6
), f"down weight mismatch for expert {i}"
@@ -128,20 +132,27 @@ def test_unfuse_experts_weights_transposed():
success = _unfuse_experts_weights_inplace(module, check_decorator=False)
assert success, "Failed to unfuse transposed weights"
- # Verify structure
- assert isinstance(module.gate_up_proj, nn.ModuleList)
- assert isinstance(module.down_proj, nn.ModuleList)
+ # Verify structure - per-expert containers with individual Linear layers
+ for i in range(num_experts):
+ expert = getattr(module, str(i))
+ assert isinstance(expert.gate_proj, nn.Linear), f"Expert {i} gate_proj is not nn.Linear"
+ assert isinstance(expert.up_proj, nn.Linear), f"Expert {i} up_proj is not nn.Linear"
+ assert isinstance(expert.down_proj, nn.Linear), f"Expert {i} down_proj is not nn.Linear"
# Verify weights are correctly transposed
+ # gate_up_proj was (num_experts, hidden, 2*intermediate), transposed
+ # Split along dim 2: gate -> [:, :intermediate_dim], up -> [:, intermediate_dim:]
+ # Then transposed: (hidden, intermediate) -> (intermediate, hidden)
for i in range(num_experts):
- # gate_up: original (hidden, 2*intermediate), should be transposed to (2*intermediate, hidden)
+ expert = getattr(module, str(i))
assert torch.allclose(
- module.gate_up_proj[i].weight.data, original_gate_up[i].t(), atol=1e-6
- ), f"gate_up weight mismatch for expert {i}"
-
- # down: original (intermediate, hidden), should be transposed to (hidden, intermediate)
+ expert.gate_proj.weight.data, original_gate_up[i, :, :intermediate_dim].t(), atol=1e-6
+ ), f"gate weight mismatch for expert {i}"
+ assert torch.allclose(
+ expert.up_proj.weight.data, original_gate_up[i, :, intermediate_dim:].t(), atol=1e-6
+ ), f"up weight mismatch for expert {i}"
assert torch.allclose(
- module.down_proj[i].weight.data, original_down[i].t(), atol=1e-6
+ expert.down_proj.weight.data, original_down[i].t(), atol=1e-6
), f"down weight mismatch for expert {i}"
@@ -236,4 +247,6 @@ def test_prepare_model_for_moe_quantization():
# Verify
assert model.config._experts_implementation == "linear_loop"
assert len(unfused) == 1
- assert isinstance(model.layer["experts"].gate_up_proj, nn.ModuleList)
+ experts = model.layer["experts"]
+ expert_0 = getattr(experts, "0")
+ assert isinstance(expert_0.gate_proj, nn.Linear)
diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.py
index 8c275d595..be71eaf8a 100644
--- a/test/test_cuda/models/test_moe_model.py
+++ b/test/test_cuda/models/test_moe_model.py
@@ -2,14 +2,11 @@ import shutil
import pytest
import torch
-import torch.nn as nn
from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Llama4ForConditionalGeneration
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM
-from transformers.models.qwen3.modeling_qwen3 import Qwen3Config, Qwen3ForCausalLM, Qwen3MLP
from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
from auto_round import AutoRound
-from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
from ...helpers import check_version
@@ -135,47 +132,6 @@ def test_llama4(setup_llama4):
shutil.rmtree(output_dir, ignore_errors=True)
-class NewQwen3MLP(ReplacementModuleBase):
- def __init__(self, original: Qwen3MLP, config: Qwen3Config):
- super().__init__(original)
- self.config = config
- self.hidden_size = config.hidden_size
- self.intermediate_size = config.intermediate_size
- self.new_gate_proj = original.gate_proj
- self.new_up_proj = original.up_proj
- self.new_down_proj = original.down_proj
- self.act_fn = nn.GELU()
-
- def forward(self, x):
- gate = self.new_gate_proj(x)
- up = self.new_up_proj(x)
- gate_act = self.act_fn(gate)
- act = gate_act * up
- out = self.new_down_proj(act)
- return out
-
- @classmethod
- def original_module_class(cls) -> str:
- return "Qwen3MLP"
-
- @classmethod
- def from_original(cls, original: Qwen3MLP, config: Qwen3Config):
- return cls(original, config)
-
-
-def setup_qwen3():
- """Fixture to set up the qwen3 model and tokenizer."""
- model_name = "/models/Qwen3-0.6B"
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- config = AutoConfig.from_pretrained(model_name)
- config.num_hidden_layers = 2 # Reduce layers for testing
- config.layer_types = config.layer_types[:2] # Reduce layers for testing
- model = Qwen3ForCausalLM(config)
- output_dir = "test_quantized_qwen3"
- return model, tokenizer, output_dir, config
-
-
@pytest.fixture
def setup_qwen35_moe():
"""Fixture to set up the Qwen3.5 MoE model, tokenizer, and processor."""
@@ -224,6 +180,7 @@ def test_qwen3_5_moe(setup_qwen35_moe):
shutil.rmtree(output_dir, ignore_errors=True)
+# using general recipe when transformers >= 5.0.0
def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):
model, tokenizer, processor, output_dir, config = setup_qwen3_vl_moe
autoround = AutoRound(
@@ -257,34 +214,4 @@ def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):
print(tokenizer.decode(loaded_model.generate(**inputs, max_new_tokens=50)[0]))
# clean the output directory after test
shutil.rmtree(output_dir, ignore_errors=True)
-
-
-def has_module(model: torch.nn.Module, module_name: str) -> bool:
- for n, m in model.named_modules():
- if module_name in n:
- return True
- return False
-
-
-def test_register_module_out_of_tree_base():
- from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
-
- for name, subclass in ReplacementModuleBase._replacement_registry.items():
- if name == "Qwen3MLP":
- assert subclass == NewQwen3MLP, "Qwen3MLP not registered correctly."
-
-
-def test_register_module_out_of_tree_model(setup_qwen3):
- model, tokenizer, output_dir, config = setup_qwen3
- quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")
- # Ensure the quantized model is not None
- assert quantized_model is not None, "Quantized model should not be None."
- check_module_names = ["new_gate_proj", "new_up_proj", "new_down_proj"]
- for name in check_module_names:
- assert has_module(quantized_model, name), f"Module {name} not found in quantized model."
- loaded_model = Qwen3ForCausalLM.from_pretrained(output_dir)
- quantized_model.to("cuda")
-
- for name in check_module_names:
- assert has_module(loaded_model, name), f"Module {name} not found in loaded model."
loaded_model.to("cuda")