Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Skip fused_moe replacement when llm-compressor oneshot already linearizes experts (#1966)

Wang, Chang09d3dec19d0b
diff --git a/auto_round/modeling/fused_moe/llama4.py b/auto_round/modeling/fused_moe/llama4.py
index 488acdaac..342978aac 100644
--- a/auto_round/modeling/fused_moe/llama4.py
+++ b/auto_round/modeling/fused_moe/llama4.py
@@ -31,14 +31,12 @@ from auto_round.utils import clear_memory, unsupported_meta_device
class SequentialLlama4TextExperts(torch.nn.ModuleList):
def __init__(self, config, original):
self.num_experts = original.gate_up_proj.shape[0]
- target_device = next(original.parameters()).device
with no_init_weights(), torch.device("meta"):
super().__init__([Llama4TextMLP(config) for _ in range(self.num_experts)])
def _materialize_weights(self, original) -> None:
if not unsupported_meta_device(original):
intermediate_size = original.down_proj.shape[1]
-
for i in range(self.num_experts):
gate_up = original.gate_up_proj[i]
down = original.down_proj[i]
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index 6f98d83c0..a276a5bff 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -20,6 +20,7 @@ import torch
from tqdm import tqdm
from transformers import PreTrainedModel
+from auto_round.modeling.fused_moe.utils import is_linearized_layout
from auto_round.utils import (
LazyImport,
dump_mem_usage,
@@ -325,7 +326,7 @@ def apply_replacements(
_handle_moe_modules(model)
if _raw_expert_is_logged:
- _log_first_moe_block(model, "after replacement")
+ _log_first_moe_block(model, "after replacement/skip")
return model
@@ -345,6 +346,7 @@ def _apply_custom_replacements(model: torch.nn.Module) -> list:
# Step 1: Collect all modules that need replacement
logger.debug("Scanning for modules to replace")
modules_to_replace = []
+ skipped_modules = []
for name, module in model.named_modules():
# skip replaced modules
if isinstance(module, ReplacementModuleBase):
@@ -352,8 +354,18 @@ def _apply_custom_replacements(model: torch.nn.Module) -> list:
class_name = module.__class__.__name__
if ReplacementModuleBase.is_registered(class_name):
replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)
+ # If llm-compressor already linearized experts, skip custom replacement.
+ if hasattr(module, "experts") and is_linearized_layout(module.experts):
+ skipped_modules.append((name, class_name, "skipped because linearize_moe"))
+ continue
if replacement_cls.is_to_be_replaced(module):
modules_to_replace.append((name, module, class_name))
+ else:
+ skipped_modules.append((name, class_name, "skipped because replacement criteria not met"))
+
+ if skipped_modules:
+ for name, class_name, reason in skipped_modules:
+ logger.info(f"Skipped replacement for {name} ({class_name}): {reason}")
# Step 2: Replace modules
if modules_to_replace:
diff --git a/auto_round/modeling/fused_moe/utils.py b/auto_round/modeling/fused_moe/utils.py
index 015dccf5c..260cf3cec 100644
--- a/auto_round/modeling/fused_moe/utils.py
+++ b/auto_round/modeling/fused_moe/utils.py
@@ -24,3 +24,32 @@ def _update_parameter(
old_param = getattr(module, name)
new_param = nn.Parameter(data, requires_grad=old_param.requires_grad)
setattr(module, name, new_param)
+
+
+def is_fused_layout(original: torch.nn.Module) -> bool:
+ """Check if MoE experts use fused gate_up_proj/down_proj layout."""
+ return hasattr(original, "gate_up_proj") and hasattr(original, "down_proj")
+
+
+def is_linearized_layout(original: torch.nn.Module) -> bool:
+ """Check if MoE experts use linearized layout with individual gate_proj/up_proj/down_proj."""
+ if not isinstance(original, torch.nn.ModuleList) or len(original) == 0:
+ return False
+ first_expert = original[0]
+ return all(hasattr(first_expert, attr) for attr in ("gate_proj", "up_proj", "down_proj"))
+
+
+def get_num_experts(original: torch.nn.Module) -> int:
+ """Get the number of experts from either fused or linearized layout."""
+ if is_fused_layout(original):
+ return original.gate_up_proj.shape[0]
+ if is_linearized_layout(original):
+ # Count only numeric keys (expert modules), exclude 'act_fn' etc.
+ if hasattr(original, "_modules"):
+ numeric_keys = [k for k in original._modules.keys() if k.isdigit()]
+ return len(numeric_keys)
+ return len(original)
+ raise AttributeError(
+ "Unsupported MoE experts layout: expected fused gate_up_proj/down_proj "
+ "or linearized gate_proj/up_proj/down_proj experts"
+ )
diff --git a/test/test_cpu/models/test_moe_experts_interface.py b/test/test_cpu/models/test_moe_experts_interface.py
index 85605eda0..6ac653ebd 100644
--- a/test/test_cpu/models/test_moe_experts_interface.py
+++ b/test/test_cpu/models/test_moe_experts_interface.py
@@ -25,6 +25,15 @@ import pytest
import torch
from torch import nn
+from auto_round.modeling.fused_moe.replace_modules import (
+ ReplacementModuleBase,
+ _apply_custom_replacements,
+)
+from auto_round.modeling.fused_moe.utils import (
+ get_num_experts,
+ is_linearized_layout,
+)
+
def _skip_if_no_linear_loop():
from auto_round.modeling.fused_moe.moe_experts_interface import is_linear_loop_available
@@ -250,3 +259,62 @@ def test_prepare_model_for_moe_quantization():
experts = model.layer["experts"]
expert_0 = getattr(experts, "0")
assert isinstance(expert_0.gate_proj, nn.Linear)
+
+
+class _TinyExpert(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_proj = nn.Linear(4, 8, bias=False)
+ self.up_proj = nn.Linear(4, 8, bias=False)
+ self.down_proj = nn.Linear(8, 4, bias=False)
+
+
+def _build_linearized_experts(num_experts: int, with_act_fn: bool = True) -> nn.ModuleList:
+ experts = nn.ModuleList([_TinyExpert() for _ in range(num_experts)])
+ if with_act_fn:
+ experts.act_fn = nn.SiLU()
+ return experts
+
+
+def test_utils_get_num_experts_ignores_non_numeric_modules_in_linearized_layout():
+ experts = _build_linearized_experts(num_experts=3, with_act_fn=True)
+
+ assert is_linearized_layout(experts)
+ assert len(experts) == 4 # 3 experts + act_fn module
+ assert get_num_experts(experts) == 3
+
+
+def test_apply_custom_replacements_skips_linearized_moe_and_logs_reason(monkeypatch):
+ messages = []
+
+ # Capture info logs to assert the skip reason is explicit.
+ from auto_round.modeling.fused_moe import replace_modules as rm
+
+ monkeypatch.setattr(rm.logger, "info", lambda msg: messages.append(str(msg)))
+
+ class DummyMoEForLinearizeSkipUT(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = _build_linearized_experts(num_experts=2, with_act_fn=True)
+
+ class DummyReplacementForLinearizeSkipUT(ReplacementModuleBase):
+ @classmethod
+ def original_module_class(cls) -> str:
+ return "DummyMoEForLinearizeSkipUT"
+
+ @classmethod
+ def from_original(cls, original: nn.Module, config):
+ return cls(original)
+
+ def __init__(self, original: nn.Module):
+ super().__init__(original)
+
+ model = nn.Module()
+ model.config = type("Cfg", (), {"model_type": "unit_test"})()
+ model.block = DummyMoEForLinearizeSkipUT()
+
+ replaced = _apply_custom_replacements(model)
+
+ assert replaced == []
+ assert isinstance(model.block, DummyMoEForLinearizeSkipUT)
+ assert any("skipped because linearize_moe" in msg for msg in messages)