Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

58.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Training forwards for remote DeepSeek-V3-style MoE code (Kimi-K2.7) and the gradient
checkpointing flag on a remote composition. Runs offline on CPU with a synthetic copy of the
port's `MoEGate` / `DeepseekV3MoE` idioms defined in a `transformers_modules` namespace."""

import math
import os
import sys
import tempfile
import types

import pytest
import torch
from torch import nn
from torch.nn import functional as F

# Import unsloth first to set UNSLOTH_IS_PRESENT env var.
import unsloth  # noqa: F401
from unsloth.models.remote_moe_shims import (
    is_remote_deepseek_gate,
    is_remote_deepseek_moe,
    prepare_remote_moe_for_training,
)
from unsloth.models.loader_utils import enable_composite_gradient_checkpointing


def _remote_module(name = "transformers_modules.tiny_kimi.modeling_deepseek", edits = ()):
    """The port's classes, verbatim in the parts that matter, under a remote-code module name.
    ``edits`` are (old, new) source replacements for ports derived from it."""
    src = """
import math, torch
import torch.nn.functional as F
from torch import nn

class Cfg:
    # A PretrainedConfig in the hub code: PEFT 0.18 reads model.config.to_dict().
    def to_dict(self):
        return dict(vars(self))

    def __init__(self):
        self.hidden_size = 16; self.n_routed_experts = 8; self.num_experts_per_tok = 2
        self.moe_intermediate_size = 8; self.n_shared_experts = 1; self.routed_scaling_factor = 2.5
        self.scoring_func = "sigmoid"; self.seq_aux = True; self.topk_method = "noaux_tc"
        self.n_group = 1; self.topk_group = 1; self.norm_topk_prob = True; self.hidden_act = "silu"

class MLP(nn.Module):
    def __init__(self, config, intermediate_size):
        super().__init__()
        self.gate_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False)
        self.up_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False)
        self.down_proj = nn.Linear(intermediate_size, config.hidden_size, bias=False)
    def forward(self, x):
        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))

class MoEGate(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config; self.top_k = config.num_experts_per_tok; self.n_routed_experts = config.n_routed_experts
        self.routed_scaling_factor = config.routed_scaling_factor; self.scoring_func = config.scoring_func
        self.topk_method = config.topk_method; self.n_group = config.n_group; self.topk_group = config.topk_group
        self.norm_topk_prob = config.norm_topk_prob; self.gating_dim = config.hidden_size
        self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim)))
        self.e_score_correction_bias = nn.Parameter(torch.zeros((self.n_routed_experts)))
        nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
    def forward(self, hidden_states):
        bsz, seq_len, h = hidden_states.shape
        hidden_states = hidden_states.view(-1, h)
        logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32), None)
        scores = logits.sigmoid()
        assert not self.training
        scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)
        group_scores = scores_for_choice.view(bsz * seq_len, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
        group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
        group_mask = torch.zeros_like(group_scores); group_mask.scatter_(1, group_idx, 1)
        score_mask = group_mask.unsqueeze(-1).expand(bsz * seq_len, self.n_group, self.n_routed_experts // self.n_group).reshape(bsz * seq_len, -1)
        tmp_scores = scores_for_choice.masked_fill(~score_mask.bool(), 0.0)
        _, topk_idx = torch.topk(tmp_scores, k=self.top_k, dim=-1, sorted=False)
        topk_weight = scores.gather(1, topk_idx)
        if self.top_k > 1 and self.norm_topk_prob:
            topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)
        return topk_idx, topk_weight * self.routed_scaling_factor

class DeepseekV3MoE(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config; self.num_experts_per_tok = config.num_experts_per_tok
        self.ep_size = 1; self.experts_per_rank = config.n_routed_experts; self.ep_rank = 0
        self.experts = nn.ModuleList([MLP(config, config.moe_intermediate_size) for _ in range(config.n_routed_experts)])
        self.gate = MoEGate(config)
        self.shared_experts = MLP(config, config.moe_intermediate_size * config.n_shared_experts)
    def forward(self, hidden_states):
        identity = hidden_states; orig_shape = hidden_states.shape
        topk_idx, topk_weight = self.gate(hidden_states)
        hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
        if not self.training:
            y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
        if self.config.n_shared_experts is not None:
            y = y + self.shared_experts(identity)
        return y
    @torch.no_grad()
    def moe_infer(self, x, topk_ids, topk_weight):
        cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts))); cnts.scatter_(1, topk_ids, 1)
        tokens_per_expert = cnts.sum(dim=0); idxs = topk_ids.view(-1).argsort()
        sorted_tokens = x[idxs // topk_ids.shape[1]]
        outputs = []; start_idx = 0
        for i, num_tokens in enumerate(tokens_per_expert.cpu().numpy()):
            end_idx = start_idx + num_tokens
            if num_tokens == 0: continue
            outputs.append(self.experts[i](sorted_tokens[start_idx:end_idx])); start_idx = end_idx
        outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
        new_x = torch.empty_like(outs); new_x[idxs] = outs
        return (new_x.view(*topk_ids.shape, -1).type(topk_weight.dtype).mul_(topk_weight.unsqueeze(dim=-1)).sum(dim=1).type(new_x.dtype))
"""
    for before, after in edits:
        assert before in src, before
        src = src.replace(before, after)
    mod = types.ModuleType(name)
    # The shim predicate reads forward's source, so register the exec'd fixture in linecache.
    import linecache

    filename = f"<{name}>"
    linecache.cache[filename] = (len(src), None, src.splitlines(True), filename)
    exec(compile(src, filename, "exec"), mod.__dict__)
    sys.modules[name] = mod
    gate_cls = getattr(mod, "MoEGate", None) or mod.KimiMoEGate
    for cls in (gate_cls, getattr(mod, "DeepseekV3MoE", None) or mod.SarvamMLAMoE, mod.MLP):
        cls.__module__ = name
    return mod


# sarvam-105b: renamed block, `else:` still calls `moe_infer`, `num_shared_experts` key.
_SARVAM_EDITS = (
    ("class DeepseekV3MoE(nn.Module):", "class SarvamMLAMoE(nn.Module):"),
    (
        "        if not self.training:\n            y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)\n",
        "        if not self.training:\n            y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)\n"
        "        else:\n            y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)\n",
    ),
    (
        "        if self.config.n_shared_experts is not None:",
        "        if self.shared_experts is not None:",
    ),
)


def test_detection_is_structural_and_ignores_native_classes():
    mod = _remote_module()
    block = mod.DeepseekV3MoE(mod.Cfg())
    assert is_remote_deepseek_gate(block.gate)
    assert is_remote_deepseek_moe(block)

    class MoEGate(nn.Module):  # same name, not remote code
        pass

    assert not is_remote_deepseek_gate(MoEGate())
    assert not is_remote_deepseek_moe(nn.Linear(2, 2))


def test_training_forward_matches_eval_and_backpropagates():
    torch.manual_seed(0)
    mod = _remote_module("transformers_modules.tiny_kimi_b.modeling_deepseek")
    block = mod.DeepseekV3MoE(mod.Cfg())
    x = torch.randn(2, 5, 16)
    block.train()
    with pytest.raises(AssertionError):
        block(x)
    patched = prepare_remote_moe_for_training(block, verbose = False)
    assert sorted(patched) == ["DeepseekV3MoE", "MoEGate"]
    assert prepare_remote_moe_for_training(block, verbose = False) == []
    block.eval()
    with torch.no_grad():
        reference = block(x)
    block.train()
    out = block(x)
    assert torch.allclose(out, reference, atol = 1e-5, rtol = 1e-5)
    out.float().pow(2).sum().backward()
    assert block.gate.weight.grad is not None and torch.isfinite(block.gate.weight.grad).all()
    expert_grads = [
        e.down_proj.weight.grad for e in block.experts if e.down_proj.weight.grad is not None
    ]
    assert expert_grads and any(g.abs().sum() > 0 for g in expert_grads)
    assert block.shared_experts.down_proj.weight.grad is not None
    assert block.gate.training and block.training


def test_a_gate_adding_its_bias_in_place_still_backpropagates():
    """Kimi-Linear's gate (renamed KimiMoEGate) adds e_score_correction_bias in place to the sigmoid
    output that the sigmoid's backward saved; training through the shim must not trip autograd's
    version check."""
    inplace = (
        ("class MoEGate(nn.Module):", "class KimiMoEGate(nn.Module):"),
        ("self.gate = MoEGate(config)", "self.gate = KimiMoEGate(config)"),
        (
            "scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)",
            "scores_for_choice = scores.view(bsz * seq_len, -1)\n"
            "        scores_for_choice += self.e_score_correction_bias.unsqueeze(0)",
        ),
    )
    grads = []
    for name, edits in (("tiny_kimi_c", ()), ("tiny_kimi_inplace", inplace)):
        mod = _remote_module(f"transformers_modules.{name}.modeling_deepseek", edits)
        torch.manual_seed(0)
        block = mod.DeepseekV3MoE(mod.Cfg())
        assert len(prepare_remote_moe_for_training(block, verbose = False)) == 2
        block.train()
        x = torch.randn(2, 5, 16, requires_grad = True)
        block(x).float().pow(2).sum().backward()
        grads.append((x.grad, block.gate.weight.grad))
    # The bias is zero, so the in-place port routes and weighs exactly like the reference one.
    for got, expected in zip(grads[1], grads[0]):
        torch.testing.assert_close(got, expected)


def test_shims_reach_a_module_behind_an_accelerate_hook():
    """A device_map load wraps forward in an accelerate hook keeping the original as _old_forward,
    so a class-level shim is never called; Kimi-K2.7-Code on four GPUs hit the training assert."""
    from accelerate.hooks import ModelHook, add_hook_to_module

    torch.manual_seed(0)
    mod = _remote_module("transformers_modules.tiny_kimi_c.modeling_deepseek")
    block = mod.DeepseekV3MoE(mod.Cfg())
    for sub in (block, block.gate):
        add_hook_to_module(sub, ModelHook())
    assert "_old_forward" in vars(block.gate)
    block.train()
    x = torch.randn(2, 5, 16)
    with pytest.raises(AssertionError):
        block(x)
    prepare_remote_moe_for_training(block, verbose = False)
    out = block(x)
    block.eval()
    with torch.no_grad():
        reference = block(x)
    assert torch.allclose(out, reference, atol = 1e-5, rtol = 1e-5)
    # A hook attached after the classes were shimmed is rebound too.
    block2 = mod.DeepseekV3MoE(mod.Cfg())
    add_hook_to_module(block2.gate, ModelHook())
    block2.train()
    assert prepare_remote_moe_for_training(block2, verbose = False) == []
    block2(x)


def test_composite_gradient_checkpointing_flag():
    from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedModel, PretrainedConfig

    class OuterConfig(PretrainedConfig):
        model_type = "outer_test"

    class Outer(PreTrainedModel):
        config_class = OuterConfig

        def __init__(self, config):
            super().__init__(config)
            self.language_model = LlamaForCausalLM(
                LlamaConfig(
                    hidden_size = 8,
                    num_hidden_layers = 1,
                    num_attention_heads = 2,
                    intermediate_size = 8,
                    vocab_size = 16,
                )
            )

    Outer.supports_gradient_checkpointing = False
    outer = Outer(OuterConfig())
    with pytest.raises(ValueError):
        outer.gradient_checkpointing_enable()
    assert enable_composite_gradient_checkpointing(outer, verbose = False)
    assert not enable_composite_gradient_checkpointing(outer, verbose = False)
    outer.gradient_checkpointing_enable()
    assert outer.language_model.model.gradient_checkpointing
    assert not enable_composite_gradient_checkpointing(nn.Linear(2, 2), verbose = False)


def test_a_training_capable_remote_moe_is_left_alone():
    """The standard DeepSeek-V2/V3 implementation branches on self.training and its gate
    returns an auxiliary loss; shimming it would replace a working training path."""
    import sys, types
    from unsloth.models.remote_moe_shims import is_remote_deepseek_moe

    module = types.ModuleType("transformers_modules.capable.modeling_deepseek")

    class DeepseekV3MoE(nn.Module):
        def __init__(self):
            super().__init__()
            self.experts = nn.ModuleList([nn.Linear(2, 2)])
            self.gate = nn.Linear(2, 1)

        def forward(self, hidden_states):
            if self.training:
                return hidden_states * 2
            else:
                return self.moe_infer(hidden_states)

        def moe_infer(self, x):
            return x

    DeepseekV3MoE.__module__ = module.__name__
    sys.modules[module.__name__] = module
    try:
        assert not is_remote_deepseek_moe(DeepseekV3MoE())
    finally:
        sys.modules.pop(module.__name__, None)


def test_the_gate_of_a_training_capable_block_is_not_patched():
    """A training-capable block's gate computes its auxiliary loss in train mode; the gate is
    patched only when its block is the inference-only port."""
    import linecache, sys, types
    from unsloth.models.remote_moe_shims import prepare_remote_moe_for_training

    name = "transformers_modules.capable2.modeling_deepseek"
    src = """
import torch
from torch import nn
class MoEGate(nn.Module):
    def __init__(self):
        super().__init__()
        self.topk_method = "greedy"; self.n_group = 1; self.topk_group = 1
        self.routed_scaling_factor = 1.0; self.norm_topk_prob = False; self.top_k = 1
    def forward(self, x):
        return x
class DeepseekV3MoE(nn.Module):
    def __init__(self):
        super().__init__()
        self.gate = MoEGate(); self.experts = nn.ModuleList([nn.Linear(2, 2)])
    def forward(self, x):
        if self.training:
            return x * 2
        else:
            return self.moe_infer(x)
    def moe_infer(self, x):
        return x
"""
    filename = f"<{name}>"
    linecache.cache[filename] = (len(src), None, src.splitlines(True), filename)
    mod = types.ModuleType(name)
    exec(compile(src, filename, "exec"), mod.__dict__)
    for cls in (mod.MoEGate, mod.DeepseekV3MoE):
        cls.__module__ = name
    sys.modules[name] = mod
    try:
        block = mod.DeepseekV3MoE()
        assert prepare_remote_moe_for_training(block, verbose = False) == []
        assert not getattr(mod.MoEGate.forward, "_unsloth_remote_moe_shim", False)
    finally:
        sys.modules.pop(name, None)


@pytest.mark.parametrize("shared", [True, False])
def test_a_renamed_port_with_a_no_grad_else_branch_is_shimmed(shared):
    """sarvam-105b trains through the same shims: its block is not called DeepseekV3MoE, its
    `else:` branch computes the output under `@torch.no_grad()` (so LoRA on the experts would
    get no gradient), and its shared expert may be None."""
    torch.manual_seed(0)
    mod = _remote_module(
        f"transformers_modules.tiny_sarvam_{int(shared)}.modeling_sarvam_moe", _SARVAM_EDITS
    )
    block = mod.SarvamMLAMoE(mod.Cfg())
    if not shared:
        block.shared_experts = None
    assert is_remote_deepseek_moe(block)
    x = torch.randn(2, 5, 16)
    block.train()
    with pytest.raises(AssertionError):
        block(x)
    patched = prepare_remote_moe_for_training(block, verbose = False)
    assert sorted(patched) == ["MoEGate", "SarvamMLAMoE"]
    block.eval()
    with torch.no_grad():
        reference = block(x)
    block.train()
    out = block(x)
    assert torch.allclose(out, reference, atol = 1e-5, rtol = 1e-5)
    out.float().pow(2).sum().backward()
    assert any(
        e.down_proj.weight.grad is not None and e.down_proj.weight.grad.abs().sum() > 0
        for e in block.experts
    )
    if shared:
        assert block.shared_experts.down_proj.weight.grad is not None


def test_an_eval_first_training_capable_remote_moe_is_left_alone():
    """`if not self.training: moe_infer(...) else: <training dispatch>` trains on its own."""
    import sys, types
    from unsloth.models.remote_moe_shims import is_remote_deepseek_moe

    module = types.ModuleType("transformers_modules.eval_first.modeling_deepseek")

    class EvalFirstMoE(nn.Module):
        def __init__(self):
            super().__init__()
            self.experts = nn.ModuleList([nn.Linear(2, 2)])
            self.gate = nn.Linear(2, 1)

        def forward(self, hidden_states):
            if not self.training:
                y = self.moe_infer(hidden_states)
            else:
                y = self.experts[0](hidden_states)
            return y

        def moe_infer(self, x):
            return x

    EvalFirstMoE.__module__ = module.__name__
    sys.modules[module.__name__] = module
    try:
        assert not is_remote_deepseek_moe(EvalFirstMoE())
    finally:
        sys.modules.pop(module.__name__, None)


@pytest.mark.parametrize(
    "training_branch, shimmed",
    [
        ('raise NotImplementedError("Training mode is not supported")', True),
        ("pass", True),
        ("y = self.experts[0](hidden_states)", False),
    ],
)
def test_a_training_branch_that_only_raises_is_not_a_dispatch(training_branch, shimmed):
    """Kimi-K3's port: `if not self.training: moe_infer(...) else: raise ...` is inference-only."""
    import sys, textwrap, types
    from unsloth.models.remote_moe_shims import is_remote_deepseek_moe

    module = types.ModuleType("transformers_modules.raise_only.modeling_kimi")
    source = textwrap.dedent(f"""
        class RaiseOnlyMoE(nn.Module):
            def __init__(self):
                super().__init__()
                self.experts = nn.ModuleList([nn.Linear(2, 2)])
                self.gate = nn.Linear(2, 1)

            def forward(self, hidden_states):
                y = hidden_states
                if not self.training:
                    y = self.moe_infer(hidden_states)
                else:
                    {training_branch}
                return y

            def moe_infer(self, x):
                return x
    """)
    path = os.path.join(tempfile.mkdtemp(dir = os.environ.get("TMPDIR")), "modeling_kimi.py")
    with open(path, "w") as f:
        f.write("import torch.nn as nn\n" + source)
    namespace = {"nn": nn, "__name__": module.__name__}
    exec(compile("import torch.nn as nn\n" + source, path, "exec"), namespace)
    RaiseOnlyMoE = namespace["RaiseOnlyMoE"]
    RaiseOnlyMoE.__module__ = module.__name__
    sys.modules[module.__name__] = module
    try:
        assert is_remote_deepseek_moe(RaiseOnlyMoE()) is shimmed
    finally:
        sys.modules.pop(module.__name__, None)


@pytest.mark.parametrize(
    "predicate, trains_in_body",
    [
        ("self.training and self.ep_size == 1", True),
        ("self.ep_size == 1 and self.training", True),
        ("not self.training and self.ep_size == 1", False),
        ("self.training or self.ep_size == 2", True),
        ("(self.training and self.ep_size == 1) or self.use_fused", True),
        ("not (self.training and self.ep_size == 1)", False),
        ("not (not self.training or self.ep_size != 1)", True),
    ],
)
def test_a_compound_training_predicate_is_recognised(predicate, trains_in_body):
    """`if self.training and self.ep_size == 1:` dispatches its own experts in training."""
    import sys, textwrap, types
    from unsloth.models.remote_moe_shims import is_remote_deepseek_moe

    module = types.ModuleType("transformers_modules.compound.modeling_deepseek")
    train, infer = "y = self.experts[0](hidden_states)", "y = self.moe_infer(hidden_states)"
    body, other = (train, infer) if trains_in_body else (infer, train)
    source = textwrap.dedent(f"""
        class CompoundMoE(nn.Module):
            def __init__(self):
                super().__init__()
                self.experts = nn.ModuleList([nn.Linear(2, 2)])
                self.gate = nn.Linear(2, 1)
                self.ep_size = 1
                self.use_fused = False

            def forward(self, hidden_states):
                if {predicate}:
                    {body}
                else:
                    {other}
                return y

            def moe_infer(self, x):
                return x
    """)
    path = os.path.join(tempfile.mkdtemp(dir = os.environ.get("TMPDIR")), "modeling_compound.py")
    with open(path, "w") as f:
        f.write("import torch.nn as nn\n" + source)
    module.__file__ = path
    namespace = {"nn": nn, "__name__": module.__name__}
    exec(compile("import torch.nn as nn\n" + source, path, "exec"), namespace)
    CompoundMoE = namespace["CompoundMoE"]
    CompoundMoE.__module__ = module.__name__
    sys.modules[module.__name__] = module
    try:
        assert not is_remote_deepseek_moe(CompoundMoE())
    finally:
        sys.modules.pop(module.__name__, None)


_KIMI_SRC = """
import torch
import torch.nn.functional as F
from torch import nn

class Cfg:
    hidden_size = 16; routed_expert_hidden_size = 8; moe_intermediate_size = 12
    num_experts = 6; num_experts_per_token = 2; num_shared_experts = 1; routed_scaling_factor = 2.0

class KimiRMSNorm(nn.Module):
    def __init__(self, n, eps=1e-6):
        super().__init__(); self.weight = nn.Parameter(torch.ones(n)); self.variance_epsilon = eps
    def forward(self, x):
        v = x.float().pow(2).mean(-1, keepdim=True)
        return (self.weight * (x.float() * torch.rsqrt(v + self.variance_epsilon))).to(x.dtype)

class KimiMLP(nn.Module):
    def __init__(self, hidden, inter):
        super().__init__()
        self.w1 = nn.Linear(hidden, inter, bias=False); self.w3 = nn.Linear(hidden, inter, bias=False)
        self.w2 = nn.Linear(inter, hidden, bias=False)
    def forward(self, x):
        return self.w2(F.silu(self.w1(x)) * self.w3(x))

class KimiMoEGate(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.top_k = config.num_experts_per_token; self.routed_scaling_factor = config.routed_scaling_factor
        self.weight = nn.Parameter(torch.randn(config.num_experts, config.hidden_size) * 0.3)
        self.e_score_correction_bias = nn.Parameter(torch.zeros(config.num_experts))
    def forward(self, hidden_states):
        bsz, seq_len, h = hidden_states.shape
        logits = F.linear(hidden_states.view(-1, h).float(), self.weight.float())
        scores = logits.sigmoid()
        assert not self.training
        _, topk_idx = torch.topk(scores + self.e_score_correction_bias, k=self.top_k, dim=-1, sorted=False)
        topk_weight = scores.gather(1, topk_idx)
        topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)
        return topk_idx, topk_weight * self.routed_scaling_factor

class KimiSparseMoeBlock(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config; self.ep_size = 1; self.ep_rank = 0; self.experts_per_rank = config.num_experts
        self.experts = nn.ModuleList([KimiMLP(config.routed_expert_hidden_size, config.moe_intermediate_size) for _ in range(config.num_experts)])
        self.gate = KimiMoEGate(config)
        self.shared_experts = KimiMLP(config.hidden_size, config.moe_intermediate_size)
        self.routed_expert_down_proj = nn.Linear(config.hidden_size, config.routed_expert_hidden_size, bias=False)
        self.routed_expert_up_proj = nn.Linear(config.routed_expert_hidden_size, config.hidden_size, bias=False)
        self.routed_expert_norm = KimiRMSNorm(config.routed_expert_hidden_size)
    def forward(self, hidden_states):
        identity = hidden_states
        orig_shape = hidden_states.shape
        topk_idx, topk_weight = self.gate(hidden_states)
        hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
        hidden_states = self.routed_expert_down_proj(hidden_states)
        if not self.training:
            y = self.moe_infer(hidden_states, topk_idx, topk_weight)
        else:
            raise NotImplementedError("Training mode is not supported in KimiSparseMoeBlock")
        y = self.routed_expert_norm(y)
        y = self.routed_expert_up_proj(y)
        y = y.view(*orig_shape)
        if self.config.num_shared_experts is not None:
            y = y + self.shared_experts(identity)
        return y
    @torch.no_grad()
    def moe_infer(self, x, topk_ids, topk_weight):
        cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts))); cnts.scatter_(1, topk_ids, 1)
        tokens_per_expert = cnts.sum(dim=0); idxs = topk_ids.view(-1).argsort()
        sorted_tokens = x[idxs // topk_ids.shape[1]]
        outputs = []; start_idx = 0
        for i, num_tokens in enumerate(tokens_per_expert.cpu().numpy()):
            end_idx = start_idx + num_tokens
            if num_tokens == 0: continue
            outputs.append(self.experts[i](sorted_tokens[start_idx:end_idx])); start_idx = end_idx
        outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
        new_x = torch.empty_like(outs); new_x[idxs] = outs
        return new_x.view(*topk_ids.shape, -1).type(topk_weight.dtype).mul_(topk_weight.unsqueeze(dim=-1)).sum(dim=1).type(new_x.dtype)
"""


def _kimi_module(name):
    import linecache

    filename = f"<{name}>"
    linecache.cache[filename] = (len(_KIMI_SRC), None, _KIMI_SRC.splitlines(True), filename)
    mod = types.ModuleType(name)
    exec(compile(_KIMI_SRC, filename, "exec"), mod.__dict__)
    for cls in (mod.KimiRMSNorm, mod.KimiMLP, mod.KimiMoEGate, mod.KimiSparseMoeBlock):
        cls.__module__ = name
    sys.modules[name] = mod
    return mod


def test_a_latent_moe_trains_through_its_own_projections():
    torch.manual_seed(0)
    mod = _kimi_module("transformers_modules.tiny_kimi_k3.modeling_kimi_linear")
    block = mod.KimiSparseMoeBlock(mod.Cfg())
    x = torch.randn(2, 5, 16)
    block.train()
    with pytest.raises((AssertionError, NotImplementedError)):
        block(x)
    # The renamed gate is shimmed through its inference-only block (in-place-safe backward).
    assert sorted(prepare_remote_moe_for_training(block, verbose = False)) == [
        "KimiMoEGate",
        "KimiSparseMoeBlock",
    ]
    block.eval()
    with torch.no_grad():
        reference = block(x)
    block.train()
    out = block(x)
    assert torch.allclose(out, reference, atol = 1e-5, rtol = 1e-5)
    out.float().pow(2).sum().backward()
    for p in (
        block.gate.weight,
        block.routed_expert_down_proj.weight,
        block.routed_expert_up_proj.weight,
        block.routed_expert_norm.weight,
        block.shared_experts.w2.weight,
    ):
        assert p.grad is not None and torch.isfinite(p.grad).all() and p.grad.abs().sum() > 0
    assert any(
        e.w2.weight.grad is not None and e.w2.weight.grad.abs().sum() > 0 for e in block.experts
    )
    assert block.training and block.gate.training
    assert "moe_infer" not in vars(block)
    block.eval()
    with torch.no_grad():
        assert torch.allclose(block(x), reference)


_KIMI_INPLACE = (
    ("class MoEGate(nn.Module):", "class KimiMoEGate(nn.Module):"),
    ("self.gate = MoEGate(config)", "self.gate = KimiMoEGate(config)"),
    (
        "scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)",
        "scores_for_choice = scores.view(bsz * seq_len, -1)\n"
        "        scores_for_choice += self.e_score_correction_bias.unsqueeze(0)",
    ),
)
# The same gate written out of place. The in-place add goes through a view, so Kimi's `scores`
# (later gathered into the routing weights) carries the bias too: keep that.
_KIMI_OUT_OF_PLACE = _KIMI_INPLACE[:2] + (
    (
        "scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)",
        "scores = scores + self.e_score_correction_bias.unsqueeze(0)\n"
        "        scores_for_choice = scores.view(bsz * seq_len, -1)",
    ),
    ("torch.float32", "torch.float64"),
)


def _kimi_gate_block(name, edits, scoring, dtype):
    if scoring == "softmax":
        edits = edits + (("scores = logits.sigmoid()", "scores = logits.softmax(dim=-1)"),)
    mod = _remote_module(f"transformers_modules.{name}.modeling_deepseek", edits)
    torch.manual_seed(0)
    block = mod.DeepseekV3MoE(mod.Cfg())
    with torch.no_grad():
        block.gate.e_score_correction_bias.normal_(0, 0.3)  # nonzero: Kimi's real bias
    assert sorted(prepare_remote_moe_for_training(block, verbose = False)) == [
        "DeepseekV3MoE",
        "KimiMoEGate",
    ]
    return block.to(dtype).train()


def _checkpointed(block, x, mode):
    from torch.utils.checkpoint import checkpoint

    if mode == "none":
        return block(x)
    if mode == "reentrant":
        return checkpoint(block, x, use_reentrant = True)
    if mode == "non_reentrant":
        return checkpoint(block, x, use_reentrant = False)
    # How vision.py runs non-reentrant checkpointing under DDP: inside the zoo offload hooks.
    from unsloth_zoo.gradient_checkpointing import non_reentrant_offload_hooks

    with non_reentrant_offload_hooks():
        return checkpoint(block, x, use_reentrant = False)


@pytest.mark.parametrize("scoring", ["sigmoid", "softmax"])
@pytest.mark.parametrize("mode", ["none", "reentrant", "non_reentrant", "non_reentrant_offload"])
def test_an_in_place_kimi_gate_matches_an_out_of_place_oracle(mode, scoring):
    """Kimi-Linear adds its (nonzero) score bias in place to the sigmoid / softmax output that op's
    backward saved. Without a copy, plain and reentrant backward raise autograd's version error and
    non-reentrant checkpointing silently differentiates the biased scores."""
    if mode == "non_reentrant_offload":
        pytest.importorskip("unsloth_zoo.gradient_checkpointing").non_reentrant_offload_hooks
    x = torch.randn(3, 7, 16, dtype = torch.float64, generator = torch.Generator().manual_seed(1))
    oracle = _kimi_gate_block(f"kimi_oop_{scoring}", _KIMI_OUT_OF_PLACE, scoring, torch.float64)
    x_oracle = x.clone().requires_grad_()
    expected = oracle(x_oracle)
    expected.pow(2).sum().backward()

    block = _kimi_gate_block(
        f"kimi_inplace_{mode}_{scoring}", _KIMI_INPLACE, scoring, torch.float32
    )
    x_block = x.float().requires_grad_()
    out = _checkpointed(block, x_block, mode)
    out.pow(2).sum().backward()
    torch.testing.assert_close(out.double(), expected, rtol = 1e-5, atol = 1e-6)
    torch.testing.assert_close(x_block.grad.double(), x_oracle.grad, rtol = 1e-4, atol = 1e-6)
    for (name, got), want in zip(block.named_parameters(), oracle.parameters()):
        assert (got.grad is None) == (want.grad is None), name
        if got.grad is not None:
            torch.testing.assert_close(
                got.grad.double(), want.grad, rtol = 1e-4, atol = 1e-6, msg = lambda m: f"{name}: {m}"
            )


def test_the_gate_leaves_its_saved_tensors_to_checkpointing():
    """Inside non-reentrant checkpointing (Unsloth's DDP / context parallel default) every saved
    tensor belongs to the checkpoint, which drops it until backward. saved_tensors_hooks pushed
    inside the gate would win over the checkpoint's and keep the gate's saved tensors (its fp32
    input copy, scores, weights) alive through the whole forward."""
    import gc
    import weakref

    from torch.utils._python_dispatch import TorchDispatchMode
    from torch.utils.checkpoint import checkpoint

    class Created(TorchDispatchMode):
        def __init__(self):
            super().__init__()
            self.refs = []

        def __torch_dispatch__(
            self,
            func,
            types,
            args = (),
            kwargs = None,
        ):
            out = func(*args, **(kwargs or {}))
            for t in out if isinstance(out, (tuple, list)) else (out,):
                if isinstance(t, torch.Tensor):
                    self.refs.append(weakref.ref(t))
            return out

    block = _kimi_gate_block("kimi_resident", _KIMI_INPLACE, "sigmoid", torch.float32)
    x = torch.randn(2, 64, 16, requires_grad = True)
    with Created() as created:
        out = checkpoint(block, x, use_reentrant = False)
    gc.collect()
    keep = {t.untyped_storage().data_ptr() for t in (out, x, *block.parameters())}
    resident = {
        t.untyped_storage().data_ptr(): t.untyped_storage().nbytes()
        for t in (r() for r in created.refs)
        if t is not None and t.untyped_storage().data_ptr() not in keep
    }
    assert sum(resident.values()) == 0, resident
    out.pow(2).sum().backward()
    assert x.grad is not None and block.gate.weight.grad is not None


def test_a_non_remote_gate_is_not_patched():
    mod = _remote_module("transformers_modules.kimi_local_gate.modeling_deepseek", _KIMI_INPLACE)
    mod.KimiMoEGate.__module__ = "my_project.modeling"  # a local class an inference-only port uses
    original = mod.KimiMoEGate.forward
    block = mod.DeepseekV3MoE(mod.Cfg())
    assert prepare_remote_moe_for_training(block, verbose = False) == ["DeepseekV3MoE"]
    assert mod.KimiMoEGate.forward is original
    assert prepare_remote_moe_for_training(block, verbose = False) == []


def _moe_ddp_entry(rank, world, init_file, fn, args):
    import datetime

    import torch.distributed as dist

    torch.set_num_threads(1)
    dist.init_process_group(
        "gloo",
        init_method = f"file://{init_file}",
        rank = rank,
        world_size = world,
        timeout = datetime.timedelta(seconds = 60),
    )
    try:
        fn(rank, world, *args)
    finally:
        dist.destroy_process_group()


def _moe_spawn(fn, world, *args):
    import torch.multiprocessing as mp
    if "fork" not in mp.get_all_start_methods():
        pytest.skip("needs the fork start method")
    with tempfile.TemporaryDirectory() as d:
        mp.start_processes(
            _moe_ddp_entry,
            args = (world, os.path.join(d, "init"), fn, args),
            nprocs = world,
            join = True,
            start_method = "fork",
        )


def _routed_tokens(experts, seed):
    # The gate below scores expert e by feature e, so these tokens pick exactly `experts`.
    x = 0.1 * torch.randn(1, 6, 16, generator = torch.Generator().manual_seed(seed))
    for boost, expert in zip((5.0, 4.0), experts):
        x[..., expert] += boost
    return x


def _complementary_block(lora):
    mod = _remote_module("transformers_modules.kimi_ddp.modeling_deepseek", _KIMI_INPLACE)
    torch.manual_seed(0)
    block = mod.DeepseekV3MoE(mod.Cfg())
    with torch.no_grad():
        block.gate.weight.copy_(torch.eye(8, 16))
    prepare_remote_moe_for_training(block, verbose = False)
    if lora:
        from peft import LoraConfig, inject_adapter_in_model

        config = LoraConfig(r = 2, target_modules = r".*experts\.\d+\.(gate_proj|up_proj|down_proj)")
        block = inject_adapter_in_model(config, block)
        for name, param in block.named_parameters():
            param.requires_grad_("lora_" in name)
            if "lora_B" in name:
                with torch.no_grad():
                    param.normal_(0, 0.1)
    return block.train()


# Per step, the experts each rank routes to: disjoint across ranks and changing every step, so
# with the shard-local routing of context parallelism each rank leaves experts its peer trains.
_ROUTES = (((0, 1), (2, 3)), ((2, 3), (4, 5)), ((6, 7), (0, 1)))


def _moe_ddp_worker(rank, world, cp_active, lora):
    import copy

    import torch.distributed as dist
    from torch.nn.parallel import DistributedDataParallel

    from unsloth import context_parallel as cp

    block = _complementary_block(lora)
    reference = copy.deepcopy(block)
    model = DistributedDataParallel(block, find_unused_parameters = False)
    params = [p for p in block.parameters() if p.requires_grad]
    optimizer = torch.optim.SGD(params, lr = 0.1)
    ref_optimizer = torch.optim.SGD([p for p in reference.parameters() if p.requires_grad], lr = 0.1)
    cp._ACTIVE_MANAGER = object() if cp_active else None
    try:
        for step, routes in enumerate(_ROUTES):
            optimizer.zero_grad(set_to_none = True)
            model(_routed_tokens(routes[rank], step * 10 + rank)).pow(2).sum().backward()
            # DDP averages over ranks: the reference sees both ranks' tokens at half weight.
            ref_optimizer.zero_grad(set_to_none = True)
            for peer in range(world):
                loss = reference(_routed_tokens(routes[peer], step * 10 + peer)).pow(2).sum()
                (loss / world).backward()
            used = {e for pair in routes for e in pair}
            for (name, got), want in zip(block.named_parameters(), reference.parameters()):
                if not got.requires_grad:
                    continue
                expert = int(name.split(".")[1]) if name.startswith("experts.") else None
                assert (
                    got.grad is not None
                ), f"{name}: no gradient"  # a zero gradient, never None, for DDP
                if want.grad is None:
                    assert expert is not None and expert not in used, name
                    assert not got.grad.any(), name
                else:
                    torch.testing.assert_close(got.grad, want.grad, msg = lambda m: f"{name}: {m}")
            optimizer.step()
            ref_optimizer.step()
        for (name, got), want in zip(block.named_parameters(), reference.parameters()):
            torch.testing.assert_close(got, want, msg = lambda m: f"{name} after 3 steps: {m}")
    finally:
        cp._ACTIVE_MANAGER = None


@pytest.mark.parametrize("lora", [False, True], ids = ["full", "lora_experts"])
def test_context_parallel_ranks_routing_disjoint_experts_train_under_ddp(lora):
    """Under context parallelism each rank routes its own sequence shard, so an expert idle on one
    rank trains on its peer. DDP with find_unused_parameters = False needs a gradient from every
    rank: idle experts run on no tokens for a zero gradient, over several steps of shifting routes."""
    _moe_spawn(_moe_ddp_worker, 2, True, lora)


def test_without_context_parallelism_idle_experts_stall_ddp():
    # Negative control: the CP-off dispatch leaves idle experts' gradients None, so DDP without
    # find_unused_parameters never reduces that bucket (local gradients) and refuses the next step.
    # Whichever rank fails first is the error spawn reports: the rank that used an expert sees its
    # unreduced gradient (not close), the rank that left it idle sees no gradient at all.
    with pytest.raises(
        Exception, match = "not close|finished reduction|did not receive grad|no gradient"
    ):
        _moe_spawn(_moe_ddp_worker, 2, False, False)


# Per optimizer step, two gradient accumulation micro-batches, the experts each rank routes to.
# Step 0: 6 and 7 idle everywhere; 4 and 5 used in the second micro-batch only. Step 1: 2 to 5
# idle everywhere after training at step 0 (AdamW momentum and weight decay would move them).
# Step 2: 4 to 7 idle everywhere.
_GA_ROUTES = (
    (((0, 1), (2, 3)), ((0, 1), (4, 5))),
    (((6, 7), (0, 1)), ((0, 1), (6, 7))),
    (((2, 3), (2, 3)), ((0, 1), (2, 3))),
)


def _moe_adamw_worker(rank, world, lora, hook):
    import copy

    from torch.nn.parallel import DistributedDataParallel

    from unsloth import context_parallel as cp

    block = _complementary_block(lora)
    reference = copy.deepcopy(block)
    model = DistributedDataParallel(block, find_unused_parameters = False)
    adamw = dict(lr = 1e-2, weight_decay = 0.1)
    optimizer = torch.optim.AdamW([p for p in block.parameters() if p.requires_grad], **adamw)
    ref_optimizer = torch.optim.AdamW(
        [p for p in reference.parameters() if p.requires_grad], **adamw
    )
    if hook:
        assert cp.install_idle_expert_grad_hook(optimizer, model)
        assert cp.install_idle_expert_grad_hook(optimizer, model)  # once per optimizer
        assert len(optimizer._optimizer_step_pre_hooks) == 1
    cp._ACTIVE_MANAGER = object()
    try:
        for step, micro_routes in enumerate(_GA_ROUTES):
            optimizer.zero_grad(set_to_none = True)
            ref_optimizer.zero_grad(set_to_none = True)
            for micro, routes in enumerate(micro_routes):
                seed = step * 100 + micro * 10
                loss = model(_routed_tokens(routes[rank], seed + rank)).pow(2).sum()
                (loss / len(micro_routes)).backward()
                cp._ACTIVE_MANAGER = None  # the single-process reference
                for peer in range(world):
                    loss = reference(_routed_tokens(routes[peer], seed + peer)).pow(2).sum()
                    (loss / (world * len(micro_routes))).backward()
                cp._ACTIVE_MANAGER = object()
            used = {e for routes in micro_routes for pair in routes for e in pair}
            before = {n: p.detach().clone() for n, p in block.named_parameters()}
            optimizer.step()
            ref_optimizer.step()
            for (name, got), want in zip(block.named_parameters(), reference.parameters()):
                if not got.requires_grad:
                    continue
                expert = int(name.split(".")[1]) if name.startswith("experts.") else None
                if hook and expert is not None:
                    assert (got.grad is None) == (expert not in used), (step, name)
                if expert is not None and expert not in used:
                    assert torch.equal(got, before[name]), f"idle {name} moved at step {step}"
                torch.testing.assert_close(got, want, msg = lambda m: f"{name} step {step}: {m}")
    finally:
        cp._ACTIVE_MANAGER = None


@pytest.mark.parametrize("lora", [False, True], ids = ["full", "lora_experts"])
def test_context_parallel_idle_experts_follow_single_process_adamw(lora):
    """An expert no rank routes to runs on no tokens under CP, so DDP hands it a zero gradient.
    The optimizer step pre-hook must turn that back into None so AdamW (weight decay, moments,
    step count) treats it like the single-process run, over gradient accumulation micro-batches
    where an expert used in only one micro-batch keeps its gradient."""
    _moe_spawn(_moe_adamw_worker, 2, lora, True)


def test_without_the_hook_idle_experts_move_under_adamw():
    # Negative control: the zero gradients alone decay idle experts and advance their moments.
    with pytest.raises(Exception, match = "moved at step|not close"):
        _moe_spawn(_moe_adamw_worker, 2, False, False)


def test_idle_expert_hook_is_inert_without_every_expert_blocks_and_when_all_are_routed():
    from unsloth import context_parallel as cp

    optimizer = torch.optim.AdamW(nn.Linear(2, 2).parameters())
    assert not cp.install_idle_expert_grad_hook(optimizer, nn.Linear(2, 2))
    assert not optimizer._optimizer_step_pre_hooks
    block = _complementary_block(False)
    cp._ACTIVE_MANAGER = object()
    try:
        x = torch.cat([_routed_tokens((e, (e + 1) % 8), e) for e in range(0, 8, 2)], dim = 1)
        block(x).pow(2).sum().backward()
        with torch.no_grad():  # a graph-free forward does not count
            block(_routed_tokens((0, 1), 0))
    finally:
        cp._ACTIVE_MANAGER = None
    assert block._unsloth_cp_expert_tokens.gt(0).all()
    grads = {n: p.grad.clone() for n, p in block.named_parameters() if p.grad is not None}
    assert cp.drop_idle_expert_grads([block]) == 0
    for n, p in block.named_parameters():
        if n in grads:
            assert torch.equal(p.grad, grads[n]), n
    # The next window forgets only experts whose gradients were cleared since (zero_grad).
    for expert in block.experts[:4]:
        expert.zero_grad(set_to_none = True)
    cp.start_expert_window([block])
    assert block._unsloth_cp_expert_tokens.gt(0).tolist() == [False] * 4 + [True] * 4
    block.zero_grad(set_to_none = True)
    cp.start_expert_window([block])
    assert "_unsloth_cp_expert_tokens" not in block.__dict__
    # No counts recorded (CP off): the step keeps every gradient, also if a CP hook is installed.
    block(_routed_tokens((0, 1), 0)).pow(2).sum().backward()
    assert "_unsloth_cp_expert_tokens" not in block.__dict__
    assert cp.drop_idle_expert_grads([block]) == 0
    assert all(p.grad is not None for p in block.experts[0].parameters())


@pytest.mark.parametrize("skipped", [False, True], ids = ["stepped", "amp_skipped"])
def test_an_optimizer_reused_without_context_parallelism_keeps_every_gradient(skipped):
    """The idle-expert step pre-hook stays on the optimizer. A later trainer without CP reusing it
    must keep the gradients of experts the CP window never routed to (one process has them), also
    when the last CP step was skipped (GradScaler overflow: the pre-hook never ran)."""
    from unsloth import context_parallel as cp

    block = _complementary_block(False)
    optimizer = torch.optim.AdamW(block.parameters(), lr = 1e-2, weight_decay = 0.1)
    assert cp.install_idle_expert_grad_hook(optimizer, block, window = 0)
    cp._ACTIVE_MANAGER = object()
    try:
        block(_routed_tokens((0, 2), 0)).pow(2).sum().backward()  # CP step: experts 0 and 2
    finally:
        cp._ACTIVE_MANAGER = None
    if not skipped:
        optimizer.step()  # the hook drops the other experts' zero gradients
        assert block.experts[1].down_proj.weight.grad is None
    optimizer.zero_grad(set_to_none = True)
    # The same optimizer, CP off: experts 1 and 3 only.
    block(_routed_tokens((1, 3), 1)).pow(2).sum().backward()
    grads = {n: p.grad.clone() for n, p in block.named_parameters() if p.grad is not None}
    assert any(n.startswith("experts.1.") for n in grads)
    before = {n: p.detach().clone() for n, p in block.named_parameters()}
    optimizer.step()
    for name, param in block.named_parameters():
        assert (param.grad is None) == (name not in grads), name
        if name.startswith(("experts.1.", "experts.3.")):
            assert torch.equal(param.grad, grads[name]), name
            assert not torch.equal(param, before[name]), f"{name} did not train"


def test_an_expert_parallel_block_is_not_counted_as_running_every_expert():
    """ep_size > 1 keeps the remote forward (its own expert all-to-all), not the zero-token
    dispatch, so the context parallel MoE refusal must still apply to it."""
    from unsloth import context_parallel as cp

    mod = _remote_module("transformers_modules.kimi_ep.modeling_deepseek", _KIMI_INPLACE)
    block = mod.DeepseekV3MoE(mod.Cfg())
    prepare_remote_moe_for_training(block, verbose = False)
    assert cp._runs_every_expert(block)
    block.ep_size = 2
    assert not cp._runs_every_expert(block)
    block.train()
    with pytest.raises(Exception):  # the port has no training path of its own
        block(torch.randn(1, 3, 16))


def _ulysses_manager(
    size,
    rank,
    backend = "ulysses",
):
    from unsloth import context_parallel as cp

    manager = object.__new__(cp.ContextParallelManager)
    manager.size = size
    manager.backend = backend
    if backend == "ring":
        from torch.distributed.device_mesh import init_device_mesh
        manager.mesh = init_device_mesh("cpu", (size,))
    else:
        manager.mesh = types.SimpleNamespace(get_group = lambda: None, get_local_rank = lambda: rank)
    return manager


def _pad_routed_embedding():
    # Token t routes to experts _PAD_ROUTES[t]; token 0 is the id context parallelism pads with.
    weight = torch.cat([_routed_tokens(experts, t)[0, :1] for t, experts in enumerate(_PAD_ROUTES)])
    return nn.Embedding.from_pretrained(weight, freeze = True)


_PAD_ROUTES = ((6, 7), (0, 1), (2, 3), (4, 5))


def _moe_cp_padding_worker(rank, world, lora, backend):
    """Five real tokens over 2 ranks: the manager pads with id 0 (Ulysses: one token, on rank 1's
    shard; ring: three, load balanced over both), the only token routed to experts 6 and 7. One
    process never routes to them."""
    import copy

    import torch.distributed as dist

    from unsloth import context_parallel as cp

    block = _complementary_block(lora)
    reference = copy.deepcopy(block)
    embed = _pad_routed_embedding()
    ids = torch.tensor([[1, 2, 3, 1, 2]])
    shift = F.pad(ids, (0, 1), value = -100)[:, 1:]
    count = shift.ne(-100).sum()

    def loss_of(module, input_ids, shift_labels):
        out = module(embed(input_ids)).pow(2).sum(-1)
        return (out * shift_labels.ne(-100)).sum() / count

    adamw = dict(lr = 1e-2, weight_decay = 0.1)
    optimizer = torch.optim.AdamW([p for p in block.parameters() if p.requires_grad], **adamw)
    ref_optimizer = torch.optim.AdamW(
        [p for p in reference.parameters() if p.requires_grad], **adamw
    )
    assert cp.install_idle_expert_grad_hook(optimizer, block)
    for step in range(2):
        optimizer.zero_grad(set_to_none = True)
        ref_optimizer.zero_grad(set_to_none = True)
        loss_of(reference, ids, shift).backward()
        manager = _ulysses_manager(world, rank, backend)
        inputs = {"input_ids": ids.clone(), "labels": ids.clone()}
        with manager.apply(inputs):
            local = inputs["input_ids"]
            assert local.shape == ((1, 3) if backend == "ulysses" else (1, 4))
            # Synthetic pad tokens are on this shard, and they really run through the MoE.
            assert bool(local.eq(0).any()) == (backend == "ring" or rank == world - 1)
            loss_of(block, local, inputs["shift_labels"]).backward()
        for param in block.parameters():
            if param.requires_grad:
                assert param.grad is not None  # every expert ran, as DDP needs
                dist.all_reduce(param.grad)
        before = {n: p.detach().clone() for n, p in block.named_parameters()}
        optimizer.step()
        ref_optimizer.step()
        for (name, got), want in zip(block.named_parameters(), reference.parameters()):
            if not got.requires_grad:
                continue
            expert = int(name.split(".")[1]) if name.startswith("experts.") else None
            if expert in (6, 7):
                assert want.grad is None, name
                assert got.grad is None, f"{name}: a pad-only expert kept its zero gradient"
                assert torch.equal(got, before[name]), f"pad-only {name} moved at step {step}"
            assert (got.grad is None) == (want.grad is None), name
            torch.testing.assert_close(got, want, msg = lambda m: f"{name} step {step}: {m}")
        for p, q in zip(
            optimizer.param_groups[0]["params"], ref_optimizer.param_groups[0]["params"]
        ):
            assert (p in optimizer.state) == (q in ref_optimizer.state)


@pytest.mark.parametrize(
    "lora, backend",
    [(False, "ulysses"), (True, "ulysses"), (False, "ring")],
    ids = ["full-ulysses", "lora_experts-ulysses", "full-ring"],
)
def test_context_parallel_padding_does_not_count_as_expert_usage(lora, backend):
    """When the sequence length does not divide the CP size, the manager appends synthetic pad
    tokens. Their routes must not mark an expert as used: one process never sees them, so an
    expert only they pick has a None gradient there and AdamW must leave it alone under CP too."""
    _moe_spawn(_moe_cp_padding_worker, 2, lora, backend)


# Per optimizer step, two gradient accumulation micro-batches, the experts each rank routes to.
# Step 0 overflows under fp16 loss scaling, so GradScaler skips optimizer.step() (and its pre-hook).
# Step 1: 0 to 3 idle everywhere; they were used only in the discarded window. Step 2: 4 to 7 idle.
_AMP_ROUTES = (
    (((0, 1), (2, 3)), ((0, 1), (2, 3))),
    (((4, 5), (6, 7)), ((4, 5), (6, 7))),
    (((0, 1), (0, 1)), ((2, 3), (0, 1))),
)


class _CheckpointedLayer(nn.Module):
    def __init__(self, block, gc):
        super().__init__()
        self.block = block
        self.gc = gc

    def forward(self, x):
        if self.gc:
            from torch.utils.checkpoint import checkpoint
            return checkpoint(self.block, x, use_reentrant = False)
        return self.block(x)


def _moe_amp_skip_worker(rank, world, set_to_none, gc):
    """Trainer.training_step (Unsloth's CP wrapper) over DDP, GradScaler skipping the first step,
    against one process seeing every rank's tokens with the same scaler."""
    import copy

    import trl
    from torch.nn.parallel import DistributedDataParallel

    from unsloth import context_parallel as cp

    block = _complementary_block(False)
    reference = _CheckpointedLayer(copy.deepcopy(block), gc)
    model = DistributedDataParallel(_CheckpointedLayer(block, gc), find_unused_parameters = False)
    adamw = dict(lr = 1e-2, weight_decay = 0.1)
    optimizer = torch.optim.AdamW([p for p in block.parameters() if p.requires_grad], **adamw)
    ref_optimizer = torch.optim.AdamW(
        [p for p in reference.parameters() if p.requires_grad], **adamw
    )
    scaler = torch.amp.GradScaler("cpu", init_scale = 1.0)
    ref_scaler = torch.amp.GradScaler("cpu", init_scale = 1.0)
    micros = len(_AMP_ROUTES[0])

    class Trainer:
        def __init__(self):
            pass

        def training_step(
            self,
            model,
            inputs,
            num_items_in_batch = None,
        ):
            loss = model(_routed_tokens(self.route, self.seed)).pow(2).sum() / micros
            scaler.scale(loss).backward()

        def prediction_step(self, model, inputs, prediction_loss_only):
            return (None, None, None)

        def train(self):
            pass

        def evaluate(self):
            pass

        def evaluation_loop(self, *a, **k):
            pass

    trl.SFTTrainer = Trainer
    cp.patch_sft_trainer(install_cp_methods = True)
    trainer = Trainer()
    trainer.model, trainer.optimizer = model, optimizer
    trainer.state = types.SimpleNamespace(global_step = 0)
    trainer._context_parallel_manager = _ulysses_manager(world, rank)
    probe = next(n for n, p in block.named_parameters() if p.requires_grad)
    for step, micro_routes in enumerate(_AMP_ROUTES):
        for micro, routes in enumerate(micro_routes):
            seed = step * 100 + micro * 10
            trainer.route, trainer.seed = routes[rank], seed + rank
            trainer.training_step(model, {"input_ids": torch.ones(1, 4, dtype = torch.long)}, 4)
            for peer in range(world):
                loss = reference(_routed_tokens(routes[peer], seed + peer)).pow(2).sum()
                ref_scaler.scale(loss / (world * micros)).backward()
        if step == 0:  # an fp16 overflow on every rank
            for module in (block, reference.block):
                module.get_parameter(probe).grad.view(-1)[0] = float("inf")
        before = {n: p.detach().clone() for n, p in block.named_parameters()}
        scaler.step(optimizer)
        ref_scaler.step(ref_optimizer)
        scaler.update()
        ref_scaler.update()
        if step == 0:
            assert not optimizer.state and not ref_optimizer.state  # really skipped
            assert scaler.get_scale() < 1.0
        else:
            for (name, got), want in zip(block.named_parameters(), reference.block.parameters()):
                if got.requires_grad:
                    assert (got.grad is None) == (want.grad is None), (step, name)
        for (name, got), want in zip(block.named_parameters(), reference.block.parameters()):
            if not got.requires_grad:
                continue
            expert = int(name.split(".")[1]) if name.startswith("experts.") else None
            if step == 2 and set_to_none and expert is not None and expert >= 4:
                assert torch.equal(got, before[name]), f"idle {name} moved at step {step}"
            torch.testing.assert_close(got, want, msg = lambda m: f"{name} step {step}: {m}")
        # Trainer's model.zero_grad() (set_to_none = True); set_to_none = False keeps zero tensors.
        model.zero_grad(set_to_none = set_to_none)
        reference.zero_grad(set_to_none = set_to_none)
        trainer.state.global_step += 1


@pytest.mark.parametrize("gc", [False, True], ids = ["plain", "checkpointed"])
@pytest.mark.parametrize("set_to_none", [True, False], ids = ["set_to_none", "zeroed"])
def test_a_skipped_amp_step_does_not_leak_expert_usage(set_to_none, gc):
    """fp16 GradScaler skips optimizer.step() on overflow, so the step pre-hook never runs for that
    window. Its expert usage must not carry into the next window (an expert used only there would
    keep a zero gradient and AdamW would decay it), with gradient accumulation and checkpoint
    recompute; with zero_grad(set_to_none = False) one process keeps zero gradients, which CP
    must mirror instead of dropping them."""
    _moe_spawn(_moe_amp_skip_worker, 2, set_to_none, gc)


def test_real_token_masks_never_mark_a_real_token_as_padding():
    """The MoE sees only a shard's token count: masks of graph-building forwards of one size are
    OR-ed, and one with nothing appended makes every token of that size real."""
    manager = _ulysses_manager(2, 1)
    odd = {"input_ids": torch.ones(1, 5, dtype = torch.long)}
    manager._prepare_inputs(odd)
    manager._prepare_inputs(odd)  # prediction_step, then apply: still knows the real length
    manager._ulysses_shard(odd)
    assert manager.real_token_mask(3).tolist() == [True, True, False]
    assert manager.real_token_mask(4) is None
    even = {"input_ids": torch.ones(1, 6, dtype = torch.long)}
    manager._prepare_inputs(even)
    manager._ulysses_shard(even)
    assert manager.real_token_mask(3) is None
    manager._clear_real_tokens()
    odd = {"input_ids": torch.ones(1, 5, dtype = torch.long)}
    with torch.no_grad():  # nothing to recompute: not noted
        manager._prepare_inputs(odd)
        manager._ulysses_shard(odd)
    assert manager.real_token_mask(3) is None