PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Skip fused_moe replacement when llm-compressor oneshot already linearizes experts (#1966)
Wang, Chang09d3dec19d0b
diff --git a/auto_round/modeling/fused_moe/llama4.py b/auto_round/modeling/fused_moe/llama4.pyindex 488acdaac..342978aac 100644--- a/auto_round/modeling/fused_moe/llama4.py+++ b/auto_round/modeling/fused_moe/llama4.py@@ -31,14 +31,12 @@ from auto_round.utils import clear_memory, unsupported_meta_deviceclass SequentialLlama4TextExperts(torch.nn.ModuleList):def __init__(self, config, original):self.num_experts = original.gate_up_proj.shape[0]- target_device = next(original.parameters()).devicewith no_init_weights(), torch.device("meta"):super().__init__([Llama4TextMLP(config) for _ in range(self.num_experts)])def _materialize_weights(self, original) -> None:if not unsupported_meta_device(original):intermediate_size = original.down_proj.shape[1]-for i in range(self.num_experts):gate_up = original.gate_up_proj[i]down = original.down_proj[i]diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex 6f98d83c0..a276a5bff 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -20,6 +20,7 @@ import torchfrom tqdm import tqdmfrom transformers import PreTrainedModel+from auto_round.modeling.fused_moe.utils import is_linearized_layoutfrom auto_round.utils import (LazyImport,dump_mem_usage,@@ -325,7 +326,7 @@ def apply_replacements(_handle_moe_modules(model)if _raw_expert_is_logged:- _log_first_moe_block(model, "after replacement")+ _log_first_moe_block(model, "after replacement/skip")return model@@ -345,6 +346,7 @@ def _apply_custom_replacements(model: torch.nn.Module) -> list:# Step 1: Collect all modules that need replacementlogger.debug("Scanning for modules to replace")modules_to_replace = []+ skipped_modules = []for name, module in model.named_modules():# skip replaced modulesif isinstance(module, ReplacementModuleBase):@@ -352,8 +354,18 @@ def _apply_custom_replacements(model: torch.nn.Module) -> list:class_name = module.__class__.__name__if ReplacementModuleBase.is_registered(class_name):replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)+ # If llm-compressor already linearized experts, skip custom replacement.+ if hasattr(module, "experts") and is_linearized_layout(module.experts):+ skipped_modules.append((name, class_name, "skipped because linearize_moe"))+ continueif replacement_cls.is_to_be_replaced(module):modules_to_replace.append((name, module, class_name))+ else:+ skipped_modules.append((name, class_name, "skipped because replacement criteria not met"))++ if skipped_modules:+ for name, class_name, reason in skipped_modules:+ logger.info(f"Skipped replacement for {name} ({class_name}): {reason}")# Step 2: Replace modulesif modules_to_replace:diff --git a/auto_round/modeling/fused_moe/utils.py b/auto_round/modeling/fused_moe/utils.pyindex 015dccf5c..260cf3cec 100644--- a/auto_round/modeling/fused_moe/utils.py+++ b/auto_round/modeling/fused_moe/utils.py@@ -24,3 +24,32 @@ def _update_parameter(old_param = getattr(module, name)new_param = nn.Parameter(data, requires_grad=old_param.requires_grad)setattr(module, name, new_param)+++def is_fused_layout(original: torch.nn.Module) -> bool:+ """Check if MoE experts use fused gate_up_proj/down_proj layout."""+ return hasattr(original, "gate_up_proj") and hasattr(original, "down_proj")+++def is_linearized_layout(original: torch.nn.Module) -> bool:+ """Check if MoE experts use linearized layout with individual gate_proj/up_proj/down_proj."""+ if not isinstance(original, torch.nn.ModuleList) or len(original) == 0:+ return False+ first_expert = original[0]+ return all(hasattr(first_expert, attr) for attr in ("gate_proj", "up_proj", "down_proj"))+++def get_num_experts(original: torch.nn.Module) -> int:+ """Get the number of experts from either fused or linearized layout."""+ if is_fused_layout(original):+ return original.gate_up_proj.shape[0]+ if is_linearized_layout(original):+ # Count only numeric keys (expert modules), exclude 'act_fn' etc.+ if hasattr(original, "_modules"):+ numeric_keys = [k for k in original._modules.keys() if k.isdigit()]+ return len(numeric_keys)+ return len(original)+ raise AttributeError(+ "Unsupported MoE experts layout: expected fused gate_up_proj/down_proj "+ "or linearized gate_proj/up_proj/down_proj experts"+ )diff --git a/test/test_cpu/models/test_moe_experts_interface.py b/test/test_cpu/models/test_moe_experts_interface.pyindex 85605eda0..6ac653ebd 100644--- a/test/test_cpu/models/test_moe_experts_interface.py+++ b/test/test_cpu/models/test_moe_experts_interface.py@@ -25,6 +25,15 @@ import pytestimport torchfrom torch import nn+from auto_round.modeling.fused_moe.replace_modules import (+ ReplacementModuleBase,+ _apply_custom_replacements,+)+from auto_round.modeling.fused_moe.utils import (+ get_num_experts,+ is_linearized_layout,+)+def _skip_if_no_linear_loop():from auto_round.modeling.fused_moe.moe_experts_interface import is_linear_loop_available@@ -250,3 +259,62 @@ def test_prepare_model_for_moe_quantization():experts = model.layer["experts"]expert_0 = getattr(experts, "0")assert isinstance(expert_0.gate_proj, nn.Linear)+++class _TinyExpert(nn.Module):+ def __init__(self):+ super().__init__()+ self.gate_proj = nn.Linear(4, 8, bias=False)+ self.up_proj = nn.Linear(4, 8, bias=False)+ self.down_proj = nn.Linear(8, 4, bias=False)+++def _build_linearized_experts(num_experts: int, with_act_fn: bool = True) -> nn.ModuleList:+ experts = nn.ModuleList([_TinyExpert() for _ in range(num_experts)])+ if with_act_fn:+ experts.act_fn = nn.SiLU()+ return experts+++def test_utils_get_num_experts_ignores_non_numeric_modules_in_linearized_layout():+ experts = _build_linearized_experts(num_experts=3, with_act_fn=True)++ assert is_linearized_layout(experts)+ assert len(experts) == 4 # 3 experts + act_fn module+ assert get_num_experts(experts) == 3+++def test_apply_custom_replacements_skips_linearized_moe_and_logs_reason(monkeypatch):+ messages = []++ # Capture info logs to assert the skip reason is explicit.+ from auto_round.modeling.fused_moe import replace_modules as rm++ monkeypatch.setattr(rm.logger, "info", lambda msg: messages.append(str(msg)))++ class DummyMoEForLinearizeSkipUT(nn.Module):+ def __init__(self):+ super().__init__()+ self.experts = _build_linearized_experts(num_experts=2, with_act_fn=True)++ class DummyReplacementForLinearizeSkipUT(ReplacementModuleBase):+ @classmethod+ def original_module_class(cls) -> str:+ return "DummyMoEForLinearizeSkipUT"++ @classmethod+ def from_original(cls, original: nn.Module, config):+ return cls(original)++ def __init__(self, original: nn.Module):+ super().__init__(original)++ model = nn.Module()+ model.config = type("Cfg", (), {"model_type": "unit_test"})()+ model.block = DummyMoEForLinearizeSkipUT()++ replaced = _apply_custom_replacements(model)++ assert replaced == []+ assert isinstance(model.block, DummyMoEForLinearizeSkipUT)+ assert any("skipped because linearize_moe" in msg for msg in messages)