Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: implement Kimi remote MoE with grad-aware inference and add uni… (#2496)

Xin He88dc729ddb42
diff --git a/auto_round/modeling/fused_moe/kimi.py b/auto_round/modeling/fused_moe/kimi.py
new file mode 100644
index 000000000..5c528d89e
--- /dev/null
+++ b/auto_round/modeling/fused_moe/kimi.py
@@ -0,0 +1,74 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from types import MethodType
+
+import torch
+
+from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
+
+
+def _get_moe_infer_impl(module: torch.nn.Module):
+ """Return the undecorated body of the remote ``@torch.no_grad()`` ``moe_infer``, or None."""
+ return getattr(getattr(type(module), "moe_infer", None), "__wrapped__", None)
+
+
+def moe_infer_with_grad(module, x, topk_ids, topk_weight):
+ """Differentiable routed-expert inference, used while AutoRound tunes a block."""
+ return _get_moe_infer_impl(module)(module, x, topk_ids, topk_weight)
+
+
+def moe_infer(module, x, topk_ids, topk_weight):
+ if torch.is_grad_enabled():
+ return moe_infer_with_grad(module, x, topk_ids, topk_weight)
+ with torch.no_grad():
+ return _get_moe_infer_impl(module)(module, x, topk_ids, topk_weight)
+
+
+class _KimiMoEGradHook:
+ """Expose a grad-aware ``moe_infer`` on Kimi remote MoE blocks.
+
+ The remote ``moe_infer`` is decorated with ``@torch.no_grad()`` and is the only path
+ used in eval mode, so routed experts would never receive tuning gradients.
+ """
+
+ skip_linearized_experts = False
+
+ def __init__(self, original, config):
+ super().__init__(original)
+
+ @classmethod
+ def is_to_be_replaced(cls, original: torch.nn.Module) -> bool:
+ return cls.is_registered(original.__class__.__name__) and callable(_get_moe_infer_impl(original))
+
+ @classmethod
+ def from_original(cls, original: torch.nn.Module, config, **kwargs) -> torch.nn.Module:
+ original.moe_infer = MethodType(moe_infer, original)
+ return original
+
+
+class KimiVLDeepseekV3MoE(_KimiMoEGradHook, ReplacementModuleBase):
+ """Kimi-VL remote ``DeepseekV3MoE``."""
+
+ @classmethod
+ def original_module_class(cls) -> str:
+ return "DeepseekV3MoE"
+
+
+class KimiLinearSparseMoeBlock(_KimiMoEGradHook, ReplacementModuleBase):
+ """Kimi-Linear / Kimi-K3 remote ``KimiSparseMoeBlock``."""
+
+ @classmethod
+ def original_module_class(cls) -> str:
+ return "KimiSparseMoeBlock"
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index 5048e8ab8..124c81e57 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -41,6 +41,10 @@ BUILTIN_MODULES = {
"step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),
# Qwen3-Omni MoE: thinker (no shared expert)
"qwen3_omni_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_omni"),
+ # Kimi remote MoE: routed experts run under @torch.no_grad() in eval mode
+ "kimi_vl": LazyImport("auto_round.modeling.fused_moe.kimi"),
+ "kimi_linear": LazyImport("auto_round.modeling.fused_moe.kimi"),
+ "kimi_k3": LazyImport("auto_round.modeling.fused_moe.kimi"),
}
_FUSED_MOE_PROJECTION_NAMES = frozenset({"gate_up_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"})
@@ -227,6 +231,8 @@ class ReplacementModuleBase(ABC, torch.nn.Module):
# Registry: module class name -> replacement module class
_replacement_registry: dict[str, type["ReplacementModuleBase"]] = {}
supports_gguf_fused_moe: bool = False
+ # Set False for replacements that only patch behavior and also apply to linearized experts.
+ skip_linearized_experts: bool = True
def __init_subclass__(cls, **kwargs):
"""Automatically register subclasses in the replacement registry."""
@@ -400,7 +406,11 @@ def _apply_custom_replacements(model: torch.nn.Module, gguf_export: bool = False
skipped_modules.append((name, class_name, "skipped because GGUF fusion spec is unavailable"))
continue
# If llm-compressor already linearized experts, skip custom replacement.
- if hasattr(module, "experts") and is_linearized_layout(module.experts):
+ if (
+ replacement_cls.skip_linearized_experts
+ and hasattr(module, "experts")
+ and is_linearized_layout(module.experts)
+ ):
skipped_modules.append((name, class_name, "skipped because linearize_moe"))
continue
if replacement_cls.is_to_be_replaced(module):
diff --git a/test/unit/common/models/test_kimi_moe_grad.py b/test/unit/common/models/test_kimi_moe_grad.py
new file mode 100644
index 000000000..4d4f8e513
--- /dev/null
+++ b/test/unit/common/models/test_kimi_moe_grad.py
@@ -0,0 +1,129 @@
+import types
+
+import pytest
+import torch
+from torch import nn
+
+from auto_round.special_model_handler import update_module
+
+
+class _MLP(nn.Module):
+ def __init__(self, hidden, inter):
+ super().__init__()
+ self.gate_proj = nn.Linear(hidden, inter, bias=False)
+ self.up_proj = nn.Linear(hidden, inter, bias=False)
+ self.down_proj = nn.Linear(inter, hidden, bias=False)
+
+ def forward(self, x):
+ return self.down_proj(nn.functional.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class _MoEGate(nn.Module):
+ def __init__(self, hidden, n_experts, top_k):
+ super().__init__()
+ self.top_k = top_k
+ self.weight = nn.Parameter(torch.randn(n_experts, hidden))
+
+ def forward(self, x):
+ scores = nn.functional.linear(x.view(-1, x.shape[-1]), self.weight).softmax(dim=-1)
+ topk_weight, topk_idx = torch.topk(scores, k=self.top_k, dim=-1)
+ return topk_idx, topk_weight
+
+
+class DeepseekV3MoE(nn.Module):
+ """Same eval-mode forward/moe_infer structure as Kimi-VL's remote DeepseekV3MoE."""
+
+ def __init__(self, hidden=16, inter=8, n_experts=4, top_k=2):
+ super().__init__()
+ self.ep_size, self.ep_rank, self.experts_per_rank = 1, 0, n_experts
+ self.experts = nn.ModuleList([_MLP(hidden, inter) for _ in range(n_experts)])
+ self.gate = _MoEGate(hidden, n_experts, top_k)
+ self.shared_experts = _MLP(hidden, inter)
+
+ def forward(self, hidden_states):
+ identity = hidden_states
+ orig_shape = hidden_states.shape
+ topk_idx, topk_weight = self.gate(hidden_states)
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+ y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
+ return y + self.shared_experts(identity)
+
+ @torch.no_grad()
+ def moe_infer(self, x, topk_ids, topk_weight):
+ cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts)))
+ cnts.scatter_(1, topk_ids, 1)
+ tokens_per_expert = cnts.sum(dim=0).cpu().numpy()
+ idxs = topk_ids.view(-1).argsort()
+ sorted_tokens = x[idxs // topk_ids.shape[1]]
+ outputs, start_idx = [], 0
+ for i, num_tokens in enumerate(tokens_per_expert):
+ end_idx = start_idx + num_tokens
+ if num_tokens == 0:
+ continue
+ outputs.append(self.experts[i](sorted_tokens[start_idx:end_idx]))
+ start_idx = end_idx
+ outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
+ new_x = torch.empty_like(outs)
+ new_x[idxs] = outs
+ return (
+ new_x.view(*topk_ids.shape, -1)
+ .type(topk_weight.dtype)
+ .mul_(topk_weight.unsqueeze(dim=-1))
+ .sum(dim=1)
+ .type(new_x.dtype)
+ )
+
+
+class KimiSparseMoeBlock(DeepseekV3MoE):
+ """Kimi-Linear / Kimi-K3 remote MoE block shares the same eval-mode moe_infer structure."""
+
+
+def _build_model(model_type, moe_cls):
+ torch.manual_seed(0)
+ model = nn.Module()
+ model.config = types.SimpleNamespace(model_type=model_type)
+ model.mlp = moe_cls()
+ model.eval()
+ for p in model.parameters():
+ p.requires_grad_(False)
+ return model
+
+
+def _routed_expert_loss(model, x):
+ routed = list(model.mlp.experts.parameters())
+ for p in routed:
+ p.requires_grad_(True)
+ with torch.enable_grad():
+ loss = model.mlp(x).pow(2).mean()
+ return loss, routed
+
+
+ "model_type, moe_cls, patched",
+ [
+ ("kimi_vl", DeepseekV3MoE, True),
+ ("kimi_linear", KimiSparseMoeBlock, True),
+ ("kimi_k3", KimiSparseMoeBlock, True),
+ ("deepseek_v3", DeepseekV3MoE, False),
+ ],
+)
+def test_kimi_routed_experts_receive_grad(model_type, moe_cls, patched):
+ model = _build_model(model_type, moe_cls)
+ x = torch.randn(2, 8, 16)
+ with torch.no_grad():
+ ref = model.mlp(x)
+
+ model = update_module(model, cleanup_original=False)
+ loss, routed = _routed_expert_loss(model, x)
+
+ if not patched:
+ assert not loss.requires_grad
+ return
+ loss.backward()
+ assert all(p.grad is not None and p.grad.abs().sum() > 0 for p in routed)
+ assert not model.training
+
+ with torch.no_grad():
+ out = model.mlp(x)
+ assert not out.requires_grad
+ assert torch.equal(out, ref)
diff --git a/test/unit/test_cpu/models/test_fused_moe_detection.py b/test/unit/test_cpu/models/test_fused_moe_detection.py
index 9d82a1948..9a6ece0f8 100644
--- a/test/unit/test_cpu/models/test_fused_moe_detection.py
+++ b/test/unit/test_cpu/models/test_fused_moe_detection.py
@@ -114,6 +114,9 @@ def test_families_with_a_dedicated_replacement_are_left_alone():
"llama4",
"deepseek_v2",
"step3p5",
+ "kimi_vl",
+ "kimi_linear",
+ "kimi_k3",
"qwen3_omni_moe",
"qwen3_5_moe",
"qwen3_5_moe_text",