PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: implement Kimi remote MoE with grad-aware inference and add uni… (#2496)
Xin He88dc729ddb42
diff --git a/auto_round/modeling/fused_moe/kimi.py b/auto_round/modeling/fused_moe/kimi.pynew file mode 100644index 000000000..5c528d89e--- /dev/null+++ b/auto_round/modeling/fused_moe/kimi.py@@ -0,0 +1,74 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from types import MethodType++import torch++from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase+++def _get_moe_infer_impl(module: torch.nn.Module):+ """Return the undecorated body of the remote ``@torch.no_grad()`` ``moe_infer``, or None."""+ return getattr(getattr(type(module), "moe_infer", None), "__wrapped__", None)+++def moe_infer_with_grad(module, x, topk_ids, topk_weight):+ """Differentiable routed-expert inference, used while AutoRound tunes a block."""+ return _get_moe_infer_impl(module)(module, x, topk_ids, topk_weight)+++def moe_infer(module, x, topk_ids, topk_weight):+ if torch.is_grad_enabled():+ return moe_infer_with_grad(module, x, topk_ids, topk_weight)+ with torch.no_grad():+ return _get_moe_infer_impl(module)(module, x, topk_ids, topk_weight)+++class _KimiMoEGradHook:+ """Expose a grad-aware ``moe_infer`` on Kimi remote MoE blocks.++ The remote ``moe_infer`` is decorated with ``@torch.no_grad()`` and is the only path+ used in eval mode, so routed experts would never receive tuning gradients.+ """++ skip_linearized_experts = False++ def __init__(self, original, config):+ super().__init__(original)++ @classmethod+ def is_to_be_replaced(cls, original: torch.nn.Module) -> bool:+ return cls.is_registered(original.__class__.__name__) and callable(_get_moe_infer_impl(original))++ @classmethod+ def from_original(cls, original: torch.nn.Module, config, **kwargs) -> torch.nn.Module:+ original.moe_infer = MethodType(moe_infer, original)+ return original+++class KimiVLDeepseekV3MoE(_KimiMoEGradHook, ReplacementModuleBase):+ """Kimi-VL remote ``DeepseekV3MoE``."""++ @classmethod+ def original_module_class(cls) -> str:+ return "DeepseekV3MoE"+++class KimiLinearSparseMoeBlock(_KimiMoEGradHook, ReplacementModuleBase):+ """Kimi-Linear / Kimi-K3 remote ``KimiSparseMoeBlock``."""++ @classmethod+ def original_module_class(cls) -> str:+ return "KimiSparseMoeBlock"diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex 5048e8ab8..124c81e57 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -41,6 +41,10 @@ BUILTIN_MODULES = {"step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),# Qwen3-Omni MoE: thinker (no shared expert)"qwen3_omni_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_omni"),+ # Kimi remote MoE: routed experts run under @torch.no_grad() in eval mode+ "kimi_vl": LazyImport("auto_round.modeling.fused_moe.kimi"),+ "kimi_linear": LazyImport("auto_round.modeling.fused_moe.kimi"),+ "kimi_k3": LazyImport("auto_round.modeling.fused_moe.kimi"),}_FUSED_MOE_PROJECTION_NAMES = frozenset({"gate_up_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"})@@ -227,6 +231,8 @@ class ReplacementModuleBase(ABC, torch.nn.Module):# Registry: module class name -> replacement module class_replacement_registry: dict[str, type["ReplacementModuleBase"]] = {}supports_gguf_fused_moe: bool = False+ # Set False for replacements that only patch behavior and also apply to linearized experts.+ skip_linearized_experts: bool = Truedef __init_subclass__(cls, **kwargs):"""Automatically register subclasses in the replacement registry."""@@ -400,7 +406,11 @@ def _apply_custom_replacements(model: torch.nn.Module, gguf_export: bool = Falseskipped_modules.append((name, class_name, "skipped because GGUF fusion spec is unavailable"))continue# If llm-compressor already linearized experts, skip custom replacement.- if hasattr(module, "experts") and is_linearized_layout(module.experts):+ if (+ replacement_cls.skip_linearized_experts+ and hasattr(module, "experts")+ and is_linearized_layout(module.experts)+ ):skipped_modules.append((name, class_name, "skipped because linearize_moe"))continueif replacement_cls.is_to_be_replaced(module):diff --git a/test/unit/common/models/test_kimi_moe_grad.py b/test/unit/common/models/test_kimi_moe_grad.pynew file mode 100644index 000000000..4d4f8e513--- /dev/null+++ b/test/unit/common/models/test_kimi_moe_grad.py@@ -0,0 +1,129 @@+import types++import pytest+import torch+from torch import nn++from auto_round.special_model_handler import update_module+++class _MLP(nn.Module):+ def __init__(self, hidden, inter):+ super().__init__()+ self.gate_proj = nn.Linear(hidden, inter, bias=False)+ self.up_proj = nn.Linear(hidden, inter, bias=False)+ self.down_proj = nn.Linear(inter, hidden, bias=False)++ def forward(self, x):+ return self.down_proj(nn.functional.silu(self.gate_proj(x)) * self.up_proj(x))+++class _MoEGate(nn.Module):+ def __init__(self, hidden, n_experts, top_k):+ super().__init__()+ self.top_k = top_k+ self.weight = nn.Parameter(torch.randn(n_experts, hidden))++ def forward(self, x):+ scores = nn.functional.linear(x.view(-1, x.shape[-1]), self.weight).softmax(dim=-1)+ topk_weight, topk_idx = torch.topk(scores, k=self.top_k, dim=-1)+ return topk_idx, topk_weight+++class DeepseekV3MoE(nn.Module):+ """Same eval-mode forward/moe_infer structure as Kimi-VL's remote DeepseekV3MoE."""++ def __init__(self, hidden=16, inter=8, n_experts=4, top_k=2):+ super().__init__()+ self.ep_size, self.ep_rank, self.experts_per_rank = 1, 0, n_experts+ self.experts = nn.ModuleList([_MLP(hidden, inter) for _ in range(n_experts)])+ self.gate = _MoEGate(hidden, n_experts, top_k)+ self.shared_experts = _MLP(hidden, inter)++ def forward(self, hidden_states):+ identity = hidden_states+ orig_shape = hidden_states.shape+ topk_idx, topk_weight = self.gate(hidden_states)+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])+ y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)+ return y + self.shared_experts(identity)++ @torch.no_grad()+ def moe_infer(self, x, topk_ids, topk_weight):+ cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts)))+ cnts.scatter_(1, topk_ids, 1)+ tokens_per_expert = cnts.sum(dim=0).cpu().numpy()+ idxs = topk_ids.view(-1).argsort()+ sorted_tokens = x[idxs // topk_ids.shape[1]]+ outputs, start_idx = [], 0+ for i, num_tokens in enumerate(tokens_per_expert):+ end_idx = start_idx + num_tokens+ if num_tokens == 0:+ continue+ outputs.append(self.experts[i](sorted_tokens[start_idx:end_idx]))+ start_idx = end_idx+ outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)+ new_x = torch.empty_like(outs)+ new_x[idxs] = outs+ return (+ new_x.view(*topk_ids.shape, -1)+ .type(topk_weight.dtype)+ .mul_(topk_weight.unsqueeze(dim=-1))+ .sum(dim=1)+ .type(new_x.dtype)+ )+++class KimiSparseMoeBlock(DeepseekV3MoE):+ """Kimi-Linear / Kimi-K3 remote MoE block shares the same eval-mode moe_infer structure."""+++def _build_model(model_type, moe_cls):+ torch.manual_seed(0)+ model = nn.Module()+ model.config = types.SimpleNamespace(model_type=model_type)+ model.mlp = moe_cls()+ model.eval()+ for p in model.parameters():+ p.requires_grad_(False)+ return model+++def _routed_expert_loss(model, x):+ routed = list(model.mlp.experts.parameters())+ for p in routed:+ p.requires_grad_(True)+ with torch.enable_grad():+ loss = model.mlp(x).pow(2).mean()+ return loss, routed+++ "model_type, moe_cls, patched",+ [+ ("kimi_vl", DeepseekV3MoE, True),+ ("kimi_linear", KimiSparseMoeBlock, True),+ ("kimi_k3", KimiSparseMoeBlock, True),+ ("deepseek_v3", DeepseekV3MoE, False),+ ],+)+def test_kimi_routed_experts_receive_grad(model_type, moe_cls, patched):+ model = _build_model(model_type, moe_cls)+ x = torch.randn(2, 8, 16)+ with torch.no_grad():+ ref = model.mlp(x)++ model = update_module(model, cleanup_original=False)+ loss, routed = _routed_expert_loss(model, x)++ if not patched:+ assert not loss.requires_grad+ return+ loss.backward()+ assert all(p.grad is not None and p.grad.abs().sum() > 0 for p in routed)+ assert not model.training++ with torch.no_grad():+ out = model.mlp(x)+ assert not out.requires_grad+ assert torch.equal(out, ref)diff --git a/test/unit/test_cpu/models/test_fused_moe_detection.py b/test/unit/test_cpu/models/test_fused_moe_detection.pyindex 9d82a1948..9a6ece0f8 100644--- a/test/unit/test_cpu/models/test_fused_moe_detection.py+++ b/test/unit/test_cpu/models/test_fused_moe_detection.py@@ -114,6 +114,9 @@ def test_families_with_a_dedicated_replacement_are_left_alone():"llama4","deepseek_v2","step3p5",+ "kimi_vl",+ "kimi_linear",+ "kimi_k3","qwen3_omni_moe","qwen3_5_moe","qwen3_5_moe_text",