Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: Ulysses for MiniMax (M1 Lightning hybrid, M2) (#15881)

Immanuel Sunfa82bcc7ec09
diff --git a/tests/test_context_parallel_rl_ulysses_only.py b/tests/test_context_parallel_rl_ulysses_only.py
new file mode 100644
index 00000000000..ab847412091
--- /dev/null
+++ b/tests/test_context_parallel_rl_ulysses_only.py
@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""DPO / GRPO context parallelism shards Unsloth's Llama-style decoders only: MiniMax-M2 runs on main's
+generic Hugging Face Ulysses route and the MiniMax-M1 Lightning hybrid on its mixer route, so both must
+be refused there, while SFT accepts them and Llama-style RL stays accepted."""
+
+import types
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+from unsloth.models.llama import LlamaAttention_fast_forward
+
+transformers = pytest.importorskip("transformers")
+
+RL_REFUSAL = "DPO / GRPO context parallelism supports Unsloth's Llama-style"
+RL_HYBRID_REFUSAL = "DPO / GRPO context parallelism does not support linear attention hybrids"
+
+
+def _trainer():
+ return types.SimpleNamespace(accelerator = None, train_dataset = None, eval_dataset = None)
+
+
+def _rl(model, size = 2):
+ # What the DPO / GRPO patched __init__ runs: allow_fsdp2 (and so allow_hybrids) off.
+ return cp._check_cp_trainer(_trainer(), size, "ulysses", [model, None])
+
+
+def _sft(model, size = 2):
+ return cp._check_cp_trainer(_trainer(), size, "ulysses", [model], allow_fsdp2 = True)
+
+
+def _fake_model(attn_cls):
+ class Layer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.self_attn = attn_cls()
+
+ class Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([Layer(), Layer()])
+ self.config = types.SimpleNamespace(
+ model_type = "fake",
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ layer_types = ["full_attention"] * 2,
+ )
+
+ return Model()
+
+
+def _llama_style():
+ return _fake_model(
+ type("LlamaAttention", (torch.nn.Module,), {"forward": LlamaAttention_fast_forward})
+ )
+
+
+def _minimax_m1():
+ config = transformers.MiniMaxConfig(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 32,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ num_local_experts = 2,
+ num_experts_per_tok = 1,
+ layer_types = ["linear_attention", "full_attention"],
+ block_size = 4,
+ )
+ return transformers.MiniMaxForCausalLM(config)
+
+
+def _minimax_m2():
+ config = transformers.MiniMaxM2Config(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 32,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ num_local_experts = 2,
+ num_experts_per_tok = 1,
+ )
+ return transformers.MiniMaxM2ForCausalLM(config)
+
+
[email protected](not hasattr(transformers, "MiniMaxM2Config"), reason = "no MiniMax-M2")
+def test_rl_refuses_real_minimax_m2_and_sft_accepts_it():
+ model = _minimax_m2()
+ assert not cp._stateful_mixers(model)
+ # main's generic route: every attention layer goes through ALL_ATTENTION_FUNCTIONS.
+ assert cp._generic_hf_attention(model)
+ _sft(model)
+ with pytest.raises(NotImplementedError, match = RL_REFUSAL):
+ _rl(model)
+ with pytest.raises(NotImplementedError, match = RL_REFUSAL):
+ cp._check_cp_trainer(_trainer(), 2, "ulysses", [_llama_style(), model])
+
+
[email protected](not hasattr(transformers, "MiniMaxConfig"), reason = "no MiniMax")
+def test_rl_refuses_the_minimax_m1_lightning_hybrid():
+ model = _minimax_m1()
+ assert cp._stateful_mixers(model)
+ with pytest.raises(NotImplementedError, match = RL_HYBRID_REFUSAL):
+ _rl(model)
+
+
+def test_rl_keeps_accepting_llama_style_attention():
+ model = _llama_style()
+ assert not cp._generic_hf_attention(model)
+ _rl(model)
+ _sft(model)
diff --git a/tests/test_parallel_capabilities_evidence.py b/tests/test_parallel_capabilities_evidence.py
index cfef87515e0..70a2812d4dd 100644
--- a/tests/test_parallel_capabilities_evidence.py
+++ b/tests/test_parallel_capabilities_evidence.py
@@ -303,6 +303,56 @@ def test_kimi_linear_moe_under_cp_follows_its_gloo_evidence(mode, expert_lora):
caps.check(caps.Query(**kw, **extra))
[email protected]("family", ["minimax", "minimax_m2"])
+ "mode, expert_lora",
+ [("full", False), ("lora", False), ("lora", True)],
+ ids = ["full", "lora", "lora+experts"],
+)
+def test_minimax_moe_under_cp_follows_its_gloo_evidence(family, mode, expert_lora):
+ # tests/test_ulysses_minimax_parity.py::test_minimax_model_matches_upstream: sdpa / eager, none / HF
+ # checkpointing, cp 2 and dp 2 x cp 2. Unsloth's checkpoint, FSDP2 x cp and the router loss stay refused.
+ kw = dict(
+ trainer = "sft",
+ family = family,
+ moe = True,
+ mode = mode,
+ expert_lora = expert_lora,
+ cp = 2,
+ cp_backend = "ulysses",
+ torch = (2, 13),
+ )
+ for extra in ({}, {"memory": "gc"}, {"attention": "eager"}, {"dp": 2}):
+ query = caps.Query(**kw, **extra)
+ status, cell = caps.cell_status(query)
+ assert status == "supported" and "test_minimax_model_matches_upstream" in cell.parity, (
+ extra,
+ cell,
+ )
+ caps.check(query)
+ for extra, match in (
+ ({"memory": "unsloth_offload"}, "Unsloth checkpoint"),
+ ({"fsdp2": True}, "fsdp2_cp"),
+ ({"router_aux_loss": True}, "router auxiliary loss"),
+ ({"compile": "block"}, "no numeric parity"),
+ ({"mode": "qlora"}, "4-bit experts"),
+ ):
+ with pytest.raises(NotImplementedError, match = match):
+ caps.check(caps.Query(**{**kw, **extra}))
+
+
+def test_minimax_evidence_ids_are_real_parity_cases():
+ # Every case a MiniMax cell names exists in the parity test, so a renamed case cannot leave a cell
+ # pointing at nothing.
+ import re
+
+ from tests import test_ulysses_minimax_parity as parity
+ for cell in caps.CELLS:
+ if cell.parity and "test_minimax_model_matches_upstream[" in cell.parity:
+ ids = re.search(r"\[(.*)\]", cell.parity).group(1).split("|")
+ assert set(ids) <= set(parity.MODEL_CASES), (cell.name, ids)
+
+
def test_unsloth_checkpoint_keeps_an_unverified_moe_row_refused_under_strict(
clean_env, monkeypatch
):
diff --git a/tests/test_ulysses_minimax.py b/tests/test_ulysses_minimax.py
new file mode 100644
index 00000000000..7530aae37cb
--- /dev/null
+++ b/tests/test_ulysses_minimax.py
@@ -0,0 +1,190 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Ulysses context parallelism for MiniMax (M1 Lightning hybrid, M2 full attention), on CPU gloo ranks."""
+
+import os
+import tempfile
+import types
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import torch.nn.functional as F
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+from unsloth.utils import attention_dispatch as ad
+from unsloth.utils import ulysses_hybrid as uh
+
+transformers = pytest.importorskip("transformers")
+if not hasattr(transformers, "MiniMaxConfig"):
+ pytest.skip("transformers without MiniMax", allow_module_level = True)
+KINDS = ["m1", "m2"] if hasattr(transformers, "MiniMaxM2Config") else ["m1"]
+
+
+def _config(kind):
+ common = dict(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 32,
+ num_hidden_layers = 3,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ num_local_experts = 2,
+ num_experts_per_tok = 1,
+ attention_dropout = 0.0,
+ tie_word_embeddings = False,
+ )
+ if kind == "m1":
+ common.update(
+ layer_types = ["linear_attention", "full_attention", "linear_attention"],
+ # Small blocks so 13 tokens cross several Lightning blocks and the shard seam.
+ block_size = 4,
+ )
+ return transformers.MiniMaxConfig(**common), transformers.MiniMaxForCausalLM
+ common.update(
+ rope_parameters = {
+ "rope_type": "default",
+ "rope_theta": 10000.0,
+ "partial_rotary_factor": 0.5,
+ }
+ )
+ return transformers.MiniMaxM2Config(**common), transformers.MiniMaxM2ForCausalLM
+
+
+def _model(kind):
+ torch.manual_seed(0)
+ config, cls = _config(kind)
+ model = cls(config).train()
+ model.config._attn_implementation = "sdpa"
+ for module in model.modules():
+ # Grouped expert kernels may touch CUDA in a forked CPU worker.
+ if hasattr(module, "set_experts_implementation"):
+ module.set_experts_implementation("eager")
+ return model
+
+
+def _per_expert_modules(model):
+ return any(
+ isinstance(getattr(m, "experts", None), torch.nn.ModuleList) for m in model.modules()
+ )
+
+
+def _manager(size, rank):
+ manager = object.__new__(cp.ContextParallelManager)
+ manager.size = size
+ manager.backend = "ulysses"
+ manager.mesh = types.SimpleNamespace(get_group = lambda: None, get_local_rank = lambda: rank)
+ return manager
+
+
+def _loss(logits, shift_labels, count):
+ return (
+ F.cross_entropy(
+ logits.float().flatten(0, 1), shift_labels.flatten(), ignore_index = -100, reduction = "sum"
+ )
+ / count
+ )
+
+
+def _parity_worker(rank, world, kind, control):
+ ad.HAS_FLASH_ATTENTION = False
+ ad.HAS_XFORMERS = False
+ model = _model(kind)
+ ids = torch.randint(0, 64, (2, 13), generator = torch.Generator().manual_seed(1))
+ shift = F.pad(ids, (0, 1), value = -100)[:, 1:]
+ count = shift.ne(-100).sum()
+ _loss(model(input_ids = ids, use_cache = False).logits, shift, count).backward()
+ reference = {n: p.grad.clone() for n, p in model.named_parameters()}
+ model.zero_grad(set_to_none = True)
+
+ manager = _manager(world, rank)
+ manager.attach_attention_hooks(model)
+ if control:
+ for module in model.modules():
+ if getattr(module, "_unsloth_cp_orig_forward", None) is not None:
+ module.forward = module._unsloth_cp_orig_forward
+ inputs = {"input_ids": ids.clone(), "labels": ids.clone()}
+ manager._prepare_inputs(inputs)
+ with manager.apply(inputs):
+ out = model(
+ input_ids = inputs["input_ids"], position_ids = inputs["position_ids"], use_cache = False
+ )
+ _loss(out.logits, inputs["shift_labels"], count).backward()
+ for name, param in model.named_parameters():
+ grad = torch.zeros_like(param) if param.grad is None else param.grad
+ dist.all_reduce(grad)
+ torch.testing.assert_close(grad, reference[name], rtol = 1e-4, atol = 1e-6, msg = name)
+
+
+def _spawn(fn, world, *args):
+ # Fork: a spawned child would re-import unsloth without the conftest CUDA spoof.
+ if "fork" not in mp.get_all_start_methods():
+ pytest.skip("needs the fork start method")
+ with tempfile.TemporaryDirectory() as d:
+ mp.start_processes(
+ _entry,
+ args = (world, os.path.join(d, "init"), fn, args),
+ nprocs = world,
+ join = True,
+ start_method = "fork",
+ )
+
+
+def _entry(rank, world, init_file, fn, args):
+ torch.set_num_threads(1)
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
+ try:
+ fn(rank, world, *args)
+ finally:
+ dist.destroy_process_group()
+
+
[email protected]("kind", KINDS)
+def test_ulysses_minimax_gradients_match_one_rank(kind):
+ _spawn(_parity_worker, 2, kind, False)
+
+
+def test_shard_local_lightning_is_wrong():
+ # Negative control: the stock Lightning forward on a shard restarts its decayed state at the seam.
+ with pytest.raises(Exception, match = "AssertionError"):
+ _spawn(_parity_worker, 2, "m1", True)
+
+
+def test_minimax_gates():
+ model = _model("m1")
+ lightning = model.model.layers[0].self_attn
+ assert uh.is_ulysses_lightning(lightning) and uh.is_ulysses_mixer(lightning)
+ assert cp._unsupported_attention(model.config, "ulysses") is None
+ if _per_expert_modules(model):
+ # transformers 4.x MiniMax: experts a shard never routes to would stall DDP's reduction.
+ assert not cp._supports_context_parallel(model, "ulysses")
+ with pytest.raises(NotImplementedError, match = "per-expert modules"):
+ cp._check_cp_model(model, 2, "ulysses")
+ else:
+ assert cp._supports_context_parallel(model, "ulysses")
+ assert not cp._supports_context_parallel(model, "ring")
+ cp._check_stateful_mixers(model, "ulysses", 2)
+ with pytest.raises(ValueError, match = "num_attention_heads"):
+ cp._check_stateful_mixers(model, "ulysses", 8)
+
+
[email protected]("m2" not in KINDS, reason = "transformers without MiniMax-M2")
+def test_minimax_m2_runs_ulysses_without_a_stateful_mixer():
+ model = _model("m2")
+ assert not cp._stateful_mixers(model)
+ assert cp._supports_context_parallel(model, "ulysses")
+ model.config.output_router_logits = True
+ assert not cp._supports_context_parallel(model, "ulysses")
+
+
+def test_lightning_refuses_packing_and_cache():
+ lightning = _model("m1").model.layers[0].self_attn
+ h = torch.zeros(1, 4, 32)
+ with pytest.raises(NotImplementedError, match = "packing"):
+ uh.ulysses_lightning_forward(lightning, h, packed_seq_lengths = torch.tensor([4]), group = None)
+ with pytest.raises(RuntimeError, match = "cache"):
+ uh.ulysses_lightning_forward(lightning, h, past_key_values = object(), group = None)
diff --git a/tests/test_ulysses_minimax_parity.py b/tests/test_ulysses_minimax_parity.py
new file mode 100644
index 00000000000..83dad174d79
--- /dev/null
+++ b/tests/test_ulysses_minimax_parity.py
@@ -0,0 +1,548 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""MiniMax Ulysses parity against the upstream full-sequence forward in float64, on CPU gloo ranks.
+
+Lightning attention is checked block by block (lengths below / at / above the block size, seams
+inside a block and exactly between blocks, a partial final block), then whole M1 hybrid and M2
+models with padding, a rank without labels, a label across the seam, global position offsets,
+GQA / MQA, LoRA, one AdamW step, CP = 2 / 4 and DP2 x CP2. Each mutant of the Lightning forward
+and each negative control must break parity."""
+
+import copy
+import inspect
+import os
+import tempfile
+import textwrap
+import types
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import torch.nn.functional as F
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+from unsloth.utils import attention_dispatch as ad
+from unsloth.utils import ulysses_hybrid as uh
+
+transformers = pytest.importorskip("transformers")
+if not hasattr(transformers, "MiniMaxConfig"):
+ pytest.skip("transformers without MiniMax", allow_module_level = True)
+HAS_M2 = hasattr(transformers, "MiniMaxM2Config")
+# transformers 5 fuses the experts into 3D weights; 4.x keeps per-expert modules (refused under CP).
+FUSED_EXPERTS = hasattr(
+ getattr(getattr(transformers.models, "minimax", None), "modeling_minimax", None),
+ "MiniMaxExperts",
+)
+
+TOL = dict(rtol = 1e-8, atol = 1e-10)
+# A broken variant must fail on numbers (assert_close with these names), not crash on a shape.
+PARITY_FAILURE = r"AssertionError: (output|input grad|logits)"
+SEQ = 22
+
+
+def _config(
+ kind,
+ block_size = 4,
+ heads = 4,
+ kv_heads = 2,
+ layer_types = None,
+):
+ common = dict(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 32,
+ num_attention_heads = heads,
+ num_key_value_heads = kv_heads,
+ head_dim = 8,
+ num_local_experts = 2,
+ num_experts_per_tok = 1,
+ attention_dropout = 0.0,
+ tie_word_embeddings = False,
+ )
+ if kind == "m1":
+ layer_types = layer_types or [
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ ]
+ common.update(
+ num_hidden_layers = len(layer_types), layer_types = layer_types, block_size = block_size
+ )
+ return transformers.MiniMaxConfig(**common), transformers.MiniMaxForCausalLM
+ common.update(
+ num_hidden_layers = 2,
+ rope_parameters = {
+ "rope_type": "default",
+ "rope_theta": 10000.0,
+ "partial_rotary_factor": 0.5,
+ },
+ )
+ return transformers.MiniMaxM2Config(**common), transformers.MiniMaxM2ForCausalLM
+
+
+def _float64_router(router):
+ """The stock routers take their sigmoid / softmax in fp32 (router_logits.float()). Elementwise fp32 kernels
+ round the vectorised body and the scalar tail differently, so a shard (a different token count)
+ can move a routing weight by one fp32 ulp, about 1e-7 relative after backward: far above the
+ float64 tolerance and unrelated to CP. Keep this one cast in the router's own dtype."""
+ cls = type(router)
+ src = textwrap.dedent(inspect.getsource(cls.forward))
+ if "router_logits.float()" not in src:
+ return
+ if "_float64_forward" not in cls.__dict__:
+ namespace = dict(vars(inspect.getmodule(cls)))
+ exec(src.replace("router_logits.float()", "router_logits"), namespace)
+ cls._float64_forward = namespace["forward"]
+ router.forward = types.MethodType(cls._float64_forward, router)
+
+
+def _model(
+ kind,
+ lora = False,
+ checkpoint = None,
+ attn = "sdpa",
+ **config_kwargs,
+):
+ torch.manual_seed(0)
+ config, cls = _config(kind, **config_kwargs)
+ model = cls(config).train()
+ if checkpoint is not None:
+ # The recompute repeats every all-to-all in backward, in the same order on each rank.
+ model.gradient_checkpointing_enable({"use_reentrant": checkpoint == "reentrant"})
+ model.config._attn_implementation = attn
+ for module in model.modules():
+ if hasattr(module, "set_experts_implementation"):
+ module.set_experts_implementation("eager")
+ if type(module).__name__ in ("MiniMaxTopKRouter", "MiniMaxM2TopKRouter"):
+ _float64_router(module)
+ if lora:
+ from peft import LoraConfig, get_peft_model
+
+ targets = ["q_proj", "k_proj", "v_proj", "o_proj", "qkv_proj", "out_proj", "output_gate"]
+ extra = {}
+ if lora == "experts":
+ # LoRA on the routed experts too: the fused 3D expert weights.
+ extra["target_parameters"] = ["experts.gate_up_proj", "experts.down_proj"]
+ # init_lora_weights = False: B is random, so the adapters change the forward.
+ model = get_peft_model(
+ model,
+ LoraConfig(r = 4, lora_alpha = 8, target_modules = targets, init_lora_weights = False, **extra),
+ )
+ if lora == "experts":
+ # Engaged: the fused expert weights carry adapters (PEFT matches no parameter silently).
+ assert any(
+ "experts" in name and "lora_" in name and param.requires_grad
+ for name, param in model.named_parameters()
+ )
+ return model.double()
+
+
+def _mutate(old, new):
+ """Replace ulysses_lightning_forward in this (forked) worker by a source-edited copy."""
+ src = textwrap.dedent(inspect.getsource(uh.ulysses_lightning_forward))
+ assert old in src, old
+ namespace = dict(vars(uh))
+ exec(compile(src.replace(old, new), uh.__file__, "exec"), namespace)
+ uh.ulysses_lightning_forward = namespace["ulysses_lightning_forward"]
+
+
+MUTANTS = {
+ # Every rank uses rank 0's per-head slopes / decays.
+ "slope_rate_slice": ("module.slope_rate[head]", "module.slope_rate[:heads]"),
+ "diagonal_decay_slice": ("module.diagonal_decay[:, head]", "module.diagonal_decay[:, :heads]"),
+ "query_decay_slice": ("module.query_decay[head]", "module.query_decay[:heads]"),
+ # A partial block read with the tail of the query decay instead of its head.
+ "partial_block_query_decay": ("query_decay[:, :n]", "query_decay[:, -n:]"),
+ # The carried state is not decayed between blocks.
+ "undecayed_state": ("state * torch.exp(-slope * n)", "state"),
+ # q | k | v as three contiguous thirds instead of interleaved per head.
+ "qkv_layout": (
+ "qkv.reshape(bsz, local_len, module.num_attention_heads, 3 * dim)",
+ "qkv.reshape(bsz, local_len, 3, module.num_attention_heads, dim).transpose(2, 3)"
+ ".reshape(bsz, local_len, module.num_attention_heads, 3 * dim)",
+ ),
+ # Output gate applied before the full-width norm.
+ "gate_before_norm": (
+ "out = module.norm(out.reshape(bsz, local_len, -1))\n"
+ " out = torch.sigmoid(module.output_gate(hidden_states)) * out",
+ "out = torch.sigmoid(module.output_gate(hidden_states)) * out.reshape(bsz, local_len, -1)\n"
+ " out = module.norm(out)",
+ ),
+ # Blocks restart at the shard start instead of the global sequence start.
+ "shard_aligned_blocks": (
+ "block, total = module.block_size, q.shape[2]",
+ "block, total = local_len, q.shape[2]",
+ ),
+}
+
+
+def _lightning_worker(rank, world, seq_len, block_size, mutant):
+ if mutant is not None and mutant != "shard_local":
+ _mutate(*MUTANTS[mutant])
+ config, _ = _config("m1", block_size = block_size)
+ from transformers.models.minimax.modeling_minimax import MiniMaxLightningAttention
+
+ torch.manual_seed(0)
+ # layer_idx 2 of 4: the slope factor is neither 1 nor ~0.
+ module = MiniMaxLightningAttention(config, layer_idx = 2).double()
+ for p in module.parameters():
+ p.data.normal_(0, 0.3)
+ g = torch.Generator().manual_seed(1)
+ x = torch.randn(2, seq_len, config.hidden_size, generator = g, dtype = torch.float64)
+ cot = torch.randn(2, seq_len, config.hidden_size, generator = g, dtype = torch.float64)
+
+ x_ref = x.clone().requires_grad_()
+ ref = module(x_ref, None, None)[0]
+ (ref * cot).sum().backward()
+ ref_grads = {n: p.grad.clone() for n, p in module.named_parameters()}
+ module.zero_grad(set_to_none = True)
+
+ local = seq_len // world
+ shard = slice(rank * local, (rank + 1) * local)
+ x_local = x[:, shard].clone().requires_grad_()
+ if mutant == "shard_local":
+ out = module(x_local, None, None)[0]
+ else:
+ out = uh.ulysses_lightning_forward(module, x_local, group = None)[0]
+ (out * cot[:, shard]).sum().backward()
+ torch.testing.assert_close(out, ref[:, shard], **TOL, msg = "output")
+ torch.testing.assert_close(x_local.grad, x_ref.grad[:, shard], **TOL, msg = "input grad")
+ for name, param in module.named_parameters():
+ dist.all_reduce(param.grad)
+ torch.testing.assert_close(param.grad, ref_grads[name], **TOL, msg = name)
+
+
+# (seq_len, block_size, world): what the seam and the final block look like.
+LIGHTNING_CASES = {
+ "below_block": (4, 8, 2), # one partial block, seam inside it
+ "at_block": (8, 8, 2), # one full block, seam inside it
+ "above_block_partial": (12, 8, 2), # seam inside block 0, partial final block
+ "seam_between_blocks": (16, 4, 2), # seam exactly at a block boundary
+ "many_blocks_partial": (22, 4, 2), # 5 full blocks + 2, seam inside block 2
+ "cp4_seams_inside": (20, 3, 4), # seams at 5 / 10 / 15, final block of 2
+ "cp4_seams_between": (16, 4, 4), # every seam on a block boundary
+}
+
+
+def _spawn(fn, world, *args):
+ # Fork: a spawned child would re-import unsloth without the conftest CUDA spoof.
+ if "fork" not in mp.get_all_start_methods():
+ pytest.skip("needs the fork start method")
+ with tempfile.TemporaryDirectory() as d:
+ mp.start_processes(
+ _entry,
+ args = (world, os.path.join(d, "init"), fn, args),
+ nprocs = world,
+ join = True,
+ start_method = "fork",
+ )
+
+
+def _entry(rank, world, init_file, fn, args):
+ torch.set_num_threads(1)
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
+ try:
+ fn(rank, world, *args)
+ finally:
+ dist.destroy_process_group()
+
+
[email protected]("case", list(LIGHTNING_CASES))
+def test_lightning_matches_upstream_full_sequence(case):
+ seq_len, block_size, world = LIGHTNING_CASES[case]
+ _spawn(_lightning_worker, world, seq_len, block_size, None)
+
+
+# Each mutant on a case where it changes the math (a full-size final block hides partial-block slips).
+MUTANT_CASES = {
+ "shard_local": "many_blocks_partial",
+ "slope_rate_slice": "many_blocks_partial",
+ "diagonal_decay_slice": "many_blocks_partial",
+ "query_decay_slice": "many_blocks_partial",
+ "partial_block_query_decay": "above_block_partial",
+ "undecayed_state": "seam_between_blocks",
+ "qkv_layout": "at_block",
+ "gate_before_norm": "below_block",
+ "shard_aligned_blocks": "above_block_partial",
+}
+
+
[email protected]("mutant", list(MUTANT_CASES))
+def test_lightning_mutants_break_parity(mutant):
+ seq_len, block_size, world = LIGHTNING_CASES[MUTANT_CASES[mutant]]
+ with pytest.raises(Exception, match = PARITY_FAILURE):
+ _spawn(_lightning_worker, world, seq_len, block_size, mutant)
+
+
+def _batch(dp_rank, vocab = 64):
+ """Right padded rows of unequal length; labels stop before position 12, so the CP=2 rank 1
+ shard has none and position 10's target (token 11) crosses the CP=2 seam."""
+ g = torch.Generator().manual_seed(10 + dp_rank)
+ ids = torch.randint(1, vocab, (2, SEQ), generator = g)
+ lengths = [(SEQ, 9), (17, SEQ)][dp_rank]
+ mask = torch.zeros(2, SEQ, dtype = torch.long)
+ for row, n in enumerate(lengths):
+ mask[row, :n] = 1
+ ids = ids * mask
+ labels = ids.masked_fill(mask == 0, -100)
+ labels[:, 12:] = -100
+ # Nonzero global offsets: RoPE must see these positions on every shard.
+ offsets = torch.tensor([[5], [17]]) + 3 * dp_rank
+ positions = torch.arange(SEQ).expand(2, -1) + offsets
+ return {"input_ids": ids, "attention_mask": mask, "labels": labels, "position_ids": positions}
+
+
+def _ce_sum(logits, shift_labels):
+ return F.cross_entropy(
+ logits.flatten(0, 1), shift_labels.flatten(), ignore_index = -100, reduction = "sum"
+ )
+
+
+def _model_worker(rank, world, kind, cp_size, lora, control, model_kwargs):
+ ad.HAS_FLASH_ATTENTION = False
+ ad.HAS_XFORMERS = False
+ cp.DEVICE_TYPE_TORCH = "cpu"
+ model = _model(kind, lora = lora, **model_kwargs)
+ reference = copy.deepcopy(model)
+ dp_size, dp_rank = world // cp_size, rank // cp_size
+ batches = [_batch(d) for d in range(dp_size)]
+ shift = [F.pad(b["labels"], (0, 1), value = -100)[:, 1:] for b in batches]
+ count = sum(int(s.ne(-100).sum()) for s in shift)
+
+ ref_logits = []
+ ref_loss = 0
+ for b, s in zip(batches, shift):
+ # No labels: a loss-computing forward may hand back placeholder logits.
+ inputs = {k: v for k, v in b.items() if k != "labels"}
+ logits = reference(**inputs, use_cache = False).logits
+ ref_logits.append(logits.detach())
+ ref_loss = ref_loss + _ce_sum(logits, s)
+ ref_loss = ref_loss / count
+ ref_loss.backward()
+
+ manager = cp.ContextParallelManager(cp_size, "ulysses")
+ cp_rank = manager.mesh.get_local_rank()
+ manager.attach_attention_hooks(model)
+ if control == "shard_local":
+ for module in model.modules():
+ if getattr(module, "_unsloth_cp_orig_forward", None) is not None:
+ module.forward = module._unsloth_cp_orig_forward
+ module._unsloth_cp_ulysses_attention = False
+ if control == "local_positions":
+ shard = manager._ulysses_shard
+
+ def local_positions(inputs):
+ shard(inputs)
+ p = inputs["position_ids"]
+ inputs["position_ids"] = p - p[:, :1] + batches[dp_rank]["position_ids"][:, :1]
+
+ manager._ulysses_shard = local_positions
+ if control == "qk_norm_after_exchange":
+ # What normalising q / k on the received head group (after the exchange) would compute.
+ for module in model.modules():
+ for norm in (getattr(module, "q_norm", None), getattr(module, "k_norm", None)):
+ if norm is None:
+ continue
+
+ def grouped(
+ self,
+ x,
+ size = cp_size,
+ ):
+ parts = x.chunk(size, dim = -1)
+ weights = self.weight.chunk(size)
+ out = []
+ for part, weight in zip(parts, weights):
+ var = part.pow(2).mean(-1, keepdim = True)
+ out.append(weight * part * torch.rsqrt(var + self.variance_epsilon))
+ return torch.cat(out, dim = -1)
+
+ norm.forward = types.MethodType(grouped, norm)
+
+ calls = {"lightning": 0, "attention": 0}
+ for key, fn_name in (
+ ("lightning", "ulysses_lightning_forward"),
+ ("attention", "_ulysses_hf_attention"),
+ ):
+
+ def counted(
+ *args,
+ _fn = getattr(uh, fn_name),
+ _key = key,
+ **kwargs,
+ ):
+ calls[_key] += 1
+ return _fn(*args, **kwargs)
+
+ setattr(uh, fn_name, counted)
+
+ inputs = {k: v.clone() for k, v in batches[dp_rank].items()}
+ with manager.apply(inputs):
+ out = model(
+ input_ids = inputs["input_ids"],
+ attention_mask = inputs["attention_mask"],
+ position_ids = inputs["position_ids"],
+ use_cache = False,
+ )
+ local_loss = _ce_sum(out.logits, inputs["shift_labels"]) / count
+ local_loss.backward()
+
+ if control is None:
+ # The Ulysses paths ran once per layer of their kind: no silent fallback.
+ layer_types = getattr(model.config, "layer_types", None) or (
+ ["full_attention"] * model.config.num_hidden_layers
+ )
+ # Checkpointing recomputes each layer once in backward.
+ runs = 2 if model_kwargs.get("checkpoint") else 1
+ assert calls["lightning"] == runs * layer_types.count("linear_attention"), calls
+ assert calls["attention"] == runs * layer_types.count("full_attention"), calls
+
+ # Outputs: valid positions of the gathered CP shards against the reference.
+ gathered = [torch.empty_like(out.logits) for _ in range(cp_size)]
+ dist.all_gather(gathered, out.logits.detach().contiguous(), group = manager.group)
+ logits = torch.cat(gathered, dim = 1)[:, :SEQ]
+ valid = batches[dp_rank]["attention_mask"].bool()
+ torch.testing.assert_close(logits[valid], ref_logits[dp_rank][valid], **TOL, msg = "logits")
+ assert cp_rank == rank % cp_size
+
+ loss = local_loss.detach().clone()
+ dist.all_reduce(loss)
+ torch.testing.assert_close(loss, ref_loss.detach(), **TOL, msg = "loss")
+
+ # Gradients: None on every rank iff None in the reference; zero stays zero; else the value.
+ ref_params = dict(reference.named_parameters())
+ for name, param in model.named_parameters():
+ ref_grad = ref_params[name].grad
+ present = torch.tensor(float(param.grad is not None))
+ dist.all_reduce(present)
+ if ref_grad is None:
+ assert present.item() == 0, f"{name}: None in the reference, set under CP"
+ continue
+ assert present.item() > 0, f"{name}: set in the reference, None on every CP rank"
+ grad = torch.zeros_like(param) if param.grad is None else param.grad
+ dist.all_reduce(grad)
+ param.grad = grad
+ if not ref_grad.any():
+ assert not grad.any(), f"{name}: zero in the reference, nonzero under CP"
+ torch.testing.assert_close(grad, ref_grad, **TOL, msg = lambda m: f"{name}: {m}")
+
+ trainable = lambda m: [p for p in m.parameters() if p.requires_grad]
+ for m in (model, reference):
+ torch.optim.AdamW(trainable(m), lr = 1e-2, weight_decay = 0.1).step()
+ for (name, p), q in zip(model.named_parameters(), reference.parameters()):
+ torch.testing.assert_close(p, q, **TOL, msg = f"after AdamW: {name}")
+
+
+# (kind, cp_size, world, lora, model kwargs)
+MODEL_CASES = {
+ "m1_cp2": ("m1", 2, 2, False, {}),
+ "m1_cp2_lora": ("m1", 2, 2, True, {}),
+ "m1_cp4": ("m1", 4, 4, False, {}),
+ "m1_dp2_cp2": ("m1", 2, 4, False, {}),
+ "m1_cp2_block_8": ("m1", 2, 2, False, {"block_size": 8}),
+ "m1_cp2_lora_checkpoint": ("m1", 2, 2, True, {"checkpoint": "non_reentrant"}),
+ "m2_cp2_lora_reentrant": ("m2", 2, 2, True, {"checkpoint": "reentrant"}),
+ "m2_cp2": ("m2", 2, 2, False, {}),
+ "m2_cp2_lora": ("m2", 2, 2, True, {}),
+ "m2_cp4_gqa": ("m2", 4, 4, False, {}),
+ "m2_cp2_mqa": ("m2", 2, 2, False, {"kv_heads": 1}),
+ "m2_dp2_cp2": ("m2", 2, 4, False, {}),
+ # MiniMax with full attention layers only: no Lightning mixer, so main's generic Ulysses route alone.
+ "m1_full_attention_only_cp2": ("m1", 2, 2, False, {"layer_types": ["full_attention"] * 2}),
+ # The parallel_capabilities MoE x cp cells (aux loss off): eager attention, HF checkpointing,
+ # full finetuning, attention LoRA and LoRA on the fused experts, CP 2 and DP2 x CP2.
+ "m1_cp2_eager_checkpoint": (
+ "m1",
+ 2,
+ 2,
+ False,
+ {"attn": "eager", "checkpoint": "non_reentrant"},
+ ),
+ "m1_cp2_lora_eager": ("m1", 2, 2, True, {"attn": "eager"}),
+ "m1_cp2_lora_experts": ("m1", 2, 2, "experts", {}),
+ "m1_dp2_cp2_lora_experts_eager_checkpoint": (
+ "m1",
+ 2,
+ 4,
+ "experts",
+ {"attn": "eager", "checkpoint": "non_reentrant"},
+ ),
+ "m2_cp2_eager_checkpoint": (
+ "m2",
+ 2,
+ 2,
+ False,
+ {"attn": "eager", "checkpoint": "non_reentrant"},
+ ),
+ "m2_cp2_lora_eager": ("m2", 2, 2, True, {"attn": "eager"}),
+ "m2_cp2_lora_experts": ("m2", 2, 2, "experts", {}),
+ "m2_dp2_cp2_lora_experts_eager_checkpoint": (
+ "m2",
+ 2,
+ 4,
+ "experts",
+ {"attn": "eager", "checkpoint": "non_reentrant"},
+ ),
+}
+
+
[email protected]("case", list(MODEL_CASES))
+def test_minimax_model_matches_upstream(case):
+ kind, cp_size, world, lora, kwargs = MODEL_CASES[case]
+ if kind == "m2" and not HAS_M2:
+ pytest.skip("transformers without MiniMax-M2")
+ if lora == "experts" and not FUSED_EXPERTS:
+ pytest.skip(
+ "transformers without fused MiniMax experts (target_parameters has nothing to match)"
+ )
+ _spawn(_model_worker, world, kind, cp_size, lora, None, kwargs)
+
+
+CONTROLS = [
+ ("m1", "shard_local"),
+ ("m2", "shard_local"),
+ ("m1", "local_positions"),
+ ("m2", "local_positions"),
+ ("m2", "qk_norm_after_exchange"),
+]
+
+
[email protected]("kind, control", CONTROLS)
+def test_minimax_negative_controls_break_parity(kind, control):
+ if kind == "m2" and not HAS_M2:
+ pytest.skip("transformers without MiniMax-M2")
+ with pytest.raises(Exception, match = PARITY_FAILURE):
+ _spawn(_model_worker, 2, kind, 2, False, control, {})
+
+
[email protected](not HAS_M2, reason = "transformers without MiniMax-M2")
+ "heads, kv_heads, size, match",
+ [
+ (4, 2, 8, "number of attention heads"),
+ (6, 3, 2, "divide one another"),
+ (4, 3, 2, "multiple of key/value heads"),
+ ],
+)
+def test_minimax_m2_head_counts_are_validated(heads, kv_heads, size, match):
+ config, cls = _config("m2", heads = heads, kv_heads = kv_heads)
+ model = cls(config)
+ with pytest.raises(ValueError, match = match):
+ cp._check_cp_model(model, size, "ulysses")
+
+
+def test_minimax_m1_head_counts_are_validated():
+ config, cls = _config("m1", heads = 4, kv_heads = 4)
+ model = cls(config)
+ if any(isinstance(getattr(m, "experts", None), torch.nn.ModuleList) for m in model.modules()):
+ # transformers 4.x MiniMax keeps per-expert modules, which CP refuses after the head check.
+ with pytest.raises(NotImplementedError, match = "per-expert modules"):
+ cp._check_cp_model(model, 4, "ulysses")
+ else:
+ cp._check_cp_model(model, 4, "ulysses")
+ with pytest.raises(ValueError, match = "number of attention heads"):
+ cp._check_cp_model(model, 8, "ulysses")
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 27411cfd022..ef762071465 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -473,8 +473,8 @@ def _check_stateful_mixers(model, backend, size) -> None:
unsupported = sorted({type(m).__name__ for m, _ in mixers if not is_ulysses_mixer(m)})
if unsupported:
raise NotImplementedError(
- "Unsloth: Ulysses context parallelism supports Gated DeltaNet and Kimi delta "
- f"linear attention only, not {unsupported}."
+ "Unsloth: Ulysses context parallelism supports Gated DeltaNet, Kimi delta and "
+ f"Lightning linear attention only, not {unsupported}."
)
for module, _ in mixers:
check_ulysses_mixer_heads(module, size)
@@ -1520,9 +1520,9 @@ def _check_cp_model(
if not _supports_context_parallel(model, backend):
raise NotImplementedError(
"Unsloth: context parallelism supports Unsloth's fast attention (Llama-style, Qwen3, Mistral, Granite, "
- "Cohere) with either backend, "
- "and with Ulysses any Hugging Face attention model (attention through ALL_ATTENTION_FUNCTIONS) "
- "and the Qwen3-Next / Qwen3.5 Gated DeltaNet and Kimi-Linear hybrids."
+ "Cohere) with either backend, and with Ulysses any Hugging Face attention model (attention through "
+ "ALL_ATTENTION_FUNCTIONS, e.g. MiniMax-M2) and the Qwen3-Next / Qwen3.5 Gated DeltaNet, Kimi-Linear and "
+ "MiniMax-M1 Lightning hybrids."
)
diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.py
index 9a4c034f0b0..60995583aab 100644
--- a/unsloth/parallel_capabilities.py
+++ b/unsloth/parallel_capabilities.py
@@ -814,6 +814,8 @@ MOE_FAMILIES = (
"gpt_oss",
"deepseek_v3",
"kimi_linear",
+ "minimax",
+ "minimax_m2",
"olmoe",
"mixtral",
"qwen2_moe",
@@ -875,6 +877,11 @@ _MOE_PP = "tests/test_pp_moe_aux_gloo.py::test_pp2_moe_aux_matches_gradient_accu
# fp64 against one process, the idle-expert hook and one AdamW step. HF checkpointing only (no Unsloth checkpoint run).
_KIMI_MOE = "tests/test_ulysses_kimi.py::test_kimi_moe_under_ulysses_matches_one_process"
_KIMI_EAGER = "eager: remote code, no compiled region"
+# MiniMax (transformers' MiniMaxForCausalLM / MiniMaxM2ForCausalLM, fused experts): M1 Lightning + full attention
+# layers, M2 full attention through the generic route, the MoE block with routed / unrouted experts, right padded
+# rows, fp64 against one process, every gradient and one AdamW step. HF checkpointing only.
+_MINIMAX = "tests/test_ulysses_minimax_parity.py::test_minimax_model_matches_upstream"
+_MINIMAX_EAGER = "eager: block compile not measured for this family"
# Families whose cp rows also ran Unsloth's checkpoint (path A and B); the others match HF checkpointing only.
_MOE_UNSLOTH_GC_FAMILIES = ("qwen3_moe",)
# (family, precision, aux, mode) -> (parity, engagement, compile) for the rows a test proves.
@@ -915,6 +922,36 @@ _MOE_EVIDENCE = {
)
* 2
+ (_KIMI_EAGER,),
+ ("minimax", "full", False, "cp"): (
+ _MINIMAX + "[m1_cp2|m1_cp4|m1_dp2_cp2|m1_cp2_eager_checkpoint]",
+ )
+ * 2
+ + (_MINIMAX_EAGER,),
+ ("minimax", "lora", False, "cp"): (
+ _MINIMAX + "[m1_cp2_lora|m1_cp2_lora_checkpoint|m1_cp2_lora_eager]",
+ )
+ * 2
+ + (_MINIMAX_EAGER,),
+ ("minimax", "lora+experts", False, "cp"): (
+ _MINIMAX + "[m1_cp2_lora_experts|m1_dp2_cp2_lora_experts_eager_checkpoint]",
+ )
+ * 2
+ + (_MINIMAX_EAGER,),
+ ("minimax_m2", "full", False, "cp"): (
+ _MINIMAX + "[m2_cp2|m2_cp4_gqa|m2_dp2_cp2|m2_cp2_eager_checkpoint]",
+ )
+ * 2
+ + (_MINIMAX_EAGER,),
+ ("minimax_m2", "lora", False, "cp"): (
+ _MINIMAX + "[m2_cp2_lora|m2_cp2_lora_reentrant|m2_cp2_lora_eager]",
+ )
+ * 2
+ + (_MINIMAX_EAGER,),
+ ("minimax_m2", "lora+experts", False, "cp"): (
+ _MINIMAX + "[m2_cp2_lora_experts|m2_dp2_cp2_lora_experts_eager_checkpoint]",
+ )
+ * 2
+ + (_MINIMAX_EAGER,),
}
diff --git a/unsloth/utils/ulysses_hybrid.py b/unsloth/utils/ulysses_hybrid.py
index 69fbcf1b2d4..0fb47c07ac6 100644
--- a/unsloth/utils/ulysses_hybrid.py
+++ b/unsloth/utils/ulysses_hybrid.py
@@ -2,8 +2,8 @@
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Ulysses context parallelism for Hugging Face attention (any model whose attention calls ALL_ATTENTION_FUNCTIONS
-runs its own attention function after the exchange) and for hybrid models (Gated DeltaNet, Kimi delta attention),
-whose scans need the full sequence so they run on heads."""
+runs its own attention function after the exchange) and for hybrid models (Gated DeltaNet, Kimi delta and MiniMax
+Lightning attention), whose scans need the full sequence so they run on heads."""
import builtins
import functools
@@ -19,11 +19,13 @@ __all__ = [
"GATED_DELTA_ATTENTION_CLASSES",
"is_ulysses_gated_delta",
"is_ulysses_kimi_delta",
+ "is_ulysses_lightning",
"is_ulysses_mixer",
"check_gated_delta_heads",
"check_ulysses_mixer_heads",
"ulysses_gated_delta_forward",
"ulysses_kimi_delta_forward",
+ "ulysses_lightning_forward",
"wrap_gated_delta_forward",
"wrap_ulysses_mixer_forward",
"install_hf_attention_wrapper",
@@ -56,8 +58,20 @@ def is_ulysses_kimi_delta(module) -> bool:
return hasattr(module, "f_a_proj") or hasattr(module, "f_proj")
+def is_ulysses_lightning(module) -> bool:
+ if not type(module).__name__.endswith("LightningAttention"):
+ return False
+ needed = ("qkv_proj", "norm", "output_gate", "out_proj", "block_size", "num_attention_heads")
+ buffers = ("slope_rate", "query_decay", "key_decay", "diagonal_decay")
+ return all(hasattr(module, n) for n in needed + buffers)
+
+
def is_ulysses_mixer(module) -> bool:
- return is_ulysses_gated_delta(module) or is_ulysses_kimi_delta(module)
+ return (
+ is_ulysses_gated_delta(module)
+ or is_ulysses_kimi_delta(module)
+ or is_ulysses_lightning(module)
+ )
def check_gated_delta_heads(module, cp_size) -> None:
@@ -71,12 +85,16 @@ def check_gated_delta_heads(module, cp_size) -> None:
def check_ulysses_mixer_heads(module, cp_size) -> None:
- if not is_ulysses_kimi_delta(module):
+ if is_ulysses_kimi_delta(module):
+ name, heads = "Kimi delta attention num_heads", module.num_heads
+ elif is_ulysses_lightning(module):
+ name, heads = "Lightning attention num_attention_heads", module.num_attention_heads
+ else:
return check_gated_delta_heads(module, cp_size)
- if module.num_heads % cp_size != 0:
+ if heads % cp_size != 0:
raise ValueError(
- f"Unsloth: Ulysses context parallelism needs Kimi delta attention num_heads "
- f"({module.num_heads}) to be divisible by context_parallel_size ({cp_size})."
+ f"Unsloth: Ulysses context parallelism needs {name} ({heads}) to be divisible by "
+ f"context_parallel_size ({cp_size})."
)
@@ -419,6 +437,68 @@ def ulysses_kimi_delta_forward(
return out
+def ulysses_lightning_forward(
+ module,
+ hidden_states,
+ position_embeddings = None,
+ attention_mask = None,
+ past_key_values = None,
+ *,
+ group,
+ **kwargs,
+):
+ """Training forward of a MiniMax LightningAttention on a contiguous sequence shard."""
+ if past_key_values is not None:
+ raise RuntimeError("Unsloth: Ulysses context parallelism does not support a cache.")
+ if kwargs.get("packed_seq_lengths") is not None:
+ raise NotImplementedError(
+ "Unsloth: Ulysses context parallelism does not support packing for Lightning attention."
+ )
+ size = dist.get_world_size(group)
+ rank = dist.get_rank(group)
+ check_ulysses_mixer_heads(module, size)
+ bsz, local_len, _ = hidden_states.shape
+ heads, dim = module.num_attention_heads // size, module.head_dim
+
+ qkv = module.act_fn(module.qkv_proj(hidden_states))
+ if isinstance(attention_mask, torch.Tensor) and attention_mask.ndim == 2:
+ # The stock forward zeroes padded values; zeroing q / k too changes nothing downstream.
+ qkv = qkv * attention_mask[:, -local_len:, None].to(qkv.dtype)
+ # [B, T/N, H, 3D] -> [B, T, H/N, 3D]: one exchange for q, k and v of each head.
+ qkv = _AllToAll.apply(
+ qkv.reshape(bsz, local_len, module.num_attention_heads, 3 * dim), 2, 1, group
+ )
+ q, k, v = (x.transpose(1, 2) for x in torch.split(qkv, dim, dim = -1))
+
+ head = slice(rank * heads, (rank + 1) * heads)
+ dtype = v.dtype
+ slope = module.slope_rate[head].to(dtype)
+ query_decay = module.query_decay[head].to(dtype)
+ key_decay = module.key_decay[head].to(dtype)
+ diagonal_decay = module.diagonal_decay[:, head].to(dtype)
+ block, total = module.block_size, q.shape[2]
+
+ state = v.new_zeros(bsz, heads, dim, dim)
+ outs = []
+ for start in range(0, total, block):
+ end = min(start + block, total)
+ n = end - start
+ qb, kb, vb = q[:, :, start:end], k[:, :, start:end], v[:, :, start:end]
+ intra = torch.matmul(
+ torch.matmul(qb, kb.transpose(-1, -2)) * diagonal_decay[:, :, :n, :n], vb
+ )
+ inter = torch.matmul(qb * query_decay[:, :n], state)
+ outs.append(inter + intra)
+ state = state * torch.exp(-slope * n) + torch.matmul(
+ (kb * key_decay[:, -n:]).transpose(-1, -2), vb
+ )
+ out = _AllToAll.apply(torch.cat(outs, dim = 2).transpose(1, 2), 1, 2, group)
+
+ out = module.norm(out.reshape(bsz, local_len, -1))
+ out = torch.sigmoid(module.output_gate(hidden_states)) * out
+ return module.out_proj(out), None
+
+
def _wrap_ulysses_forward(module, get_cp_manager, name) -> None:
if getattr(module, "_unsloth_cp_orig_forward", None) is not None:
return
@@ -453,6 +533,8 @@ def wrap_gated_delta_forward(module, get_cp_manager) -> None:
def wrap_ulysses_mixer_forward(module, get_cp_manager) -> None:
if is_ulysses_kimi_delta(module):
return _wrap_ulysses_forward(module, get_cp_manager, "ulysses_kimi_delta_forward")
+ if is_ulysses_lightning(module):
+ return _wrap_ulysses_forward(module, get_cp_manager, "ulysses_lightning_forward")
return wrap_gated_delta_forward(module, get_cp_manager)