Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallel: Ulysses for MiniMax (M1 Lightning hybrid, M2) (#15881)
Immanuel Sunfa82bcc7ec09
diff --git a/tests/test_context_parallel_rl_ulysses_only.py b/tests/test_context_parallel_rl_ulysses_only.pynew file mode 100644index 00000000000..ab847412091--- /dev/null+++ b/tests/test_context_parallel_rl_ulysses_only.py@@ -0,0 +1,119 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""DPO / GRPO context parallelism shards Unsloth's Llama-style decoders only: MiniMax-M2 runs on main's+generic Hugging Face Ulysses route and the MiniMax-M1 Lightning hybrid on its mixer route, so both must+be refused there, while SFT accepts them and Llama-style RL stays accepted."""++import types++import pytest+import torch++import unsloth # noqa: F401+from unsloth import context_parallel as cp+from unsloth.models.llama import LlamaAttention_fast_forward++transformers = pytest.importorskip("transformers")++RL_REFUSAL = "DPO / GRPO context parallelism supports Unsloth's Llama-style"+RL_HYBRID_REFUSAL = "DPO / GRPO context parallelism does not support linear attention hybrids"+++def _trainer():+ return types.SimpleNamespace(accelerator = None, train_dataset = None, eval_dataset = None)+++def _rl(model, size = 2):+ # What the DPO / GRPO patched __init__ runs: allow_fsdp2 (and so allow_hybrids) off.+ return cp._check_cp_trainer(_trainer(), size, "ulysses", [model, None])+++def _sft(model, size = 2):+ return cp._check_cp_trainer(_trainer(), size, "ulysses", [model], allow_fsdp2 = True)+++def _fake_model(attn_cls):+ class Layer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.self_attn = attn_cls()++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList([Layer(), Layer()])+ self.config = types.SimpleNamespace(+ model_type = "fake",+ num_attention_heads = 4,+ num_key_value_heads = 2,+ layer_types = ["full_attention"] * 2,+ )++ return Model()+++def _llama_style():+ return _fake_model(+ type("LlamaAttention", (torch.nn.Module,), {"forward": LlamaAttention_fast_forward})+ )+++def _minimax_m1():+ config = transformers.MiniMaxConfig(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 32,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ num_local_experts = 2,+ num_experts_per_tok = 1,+ layer_types = ["linear_attention", "full_attention"],+ block_size = 4,+ )+ return transformers.MiniMaxForCausalLM(config)+++def _minimax_m2():+ config = transformers.MiniMaxM2Config(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 32,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ num_local_experts = 2,+ num_experts_per_tok = 1,+ )+ return transformers.MiniMaxM2ForCausalLM(config)++[email protected](not hasattr(transformers, "MiniMaxM2Config"), reason = "no MiniMax-M2")+def test_rl_refuses_real_minimax_m2_and_sft_accepts_it():+ model = _minimax_m2()+ assert not cp._stateful_mixers(model)+ # main's generic route: every attention layer goes through ALL_ATTENTION_FUNCTIONS.+ assert cp._generic_hf_attention(model)+ _sft(model)+ with pytest.raises(NotImplementedError, match = RL_REFUSAL):+ _rl(model)+ with pytest.raises(NotImplementedError, match = RL_REFUSAL):+ cp._check_cp_trainer(_trainer(), 2, "ulysses", [_llama_style(), model])++[email protected](not hasattr(transformers, "MiniMaxConfig"), reason = "no MiniMax")+def test_rl_refuses_the_minimax_m1_lightning_hybrid():+ model = _minimax_m1()+ assert cp._stateful_mixers(model)+ with pytest.raises(NotImplementedError, match = RL_HYBRID_REFUSAL):+ _rl(model)+++def test_rl_keeps_accepting_llama_style_attention():+ model = _llama_style()+ assert not cp._generic_hf_attention(model)+ _rl(model)+ _sft(model)diff --git a/tests/test_parallel_capabilities_evidence.py b/tests/test_parallel_capabilities_evidence.pyindex cfef87515e0..70a2812d4dd 100644--- a/tests/test_parallel_capabilities_evidence.py+++ b/tests/test_parallel_capabilities_evidence.py@@ -303,6 +303,56 @@ def test_kimi_linear_moe_under_cp_follows_its_gloo_evidence(mode, expert_lora):caps.check(caps.Query(**kw, **extra))[email protected]("family", ["minimax", "minimax_m2"])+ "mode, expert_lora",+ [("full", False), ("lora", False), ("lora", True)],+ ids = ["full", "lora", "lora+experts"],+)+def test_minimax_moe_under_cp_follows_its_gloo_evidence(family, mode, expert_lora):+ # tests/test_ulysses_minimax_parity.py::test_minimax_model_matches_upstream: sdpa / eager, none / HF+ # checkpointing, cp 2 and dp 2 x cp 2. Unsloth's checkpoint, FSDP2 x cp and the router loss stay refused.+ kw = dict(+ trainer = "sft",+ family = family,+ moe = True,+ mode = mode,+ expert_lora = expert_lora,+ cp = 2,+ cp_backend = "ulysses",+ torch = (2, 13),+ )+ for extra in ({}, {"memory": "gc"}, {"attention": "eager"}, {"dp": 2}):+ query = caps.Query(**kw, **extra)+ status, cell = caps.cell_status(query)+ assert status == "supported" and "test_minimax_model_matches_upstream" in cell.parity, (+ extra,+ cell,+ )+ caps.check(query)+ for extra, match in (+ ({"memory": "unsloth_offload"}, "Unsloth checkpoint"),+ ({"fsdp2": True}, "fsdp2_cp"),+ ({"router_aux_loss": True}, "router auxiliary loss"),+ ({"compile": "block"}, "no numeric parity"),+ ({"mode": "qlora"}, "4-bit experts"),+ ):+ with pytest.raises(NotImplementedError, match = match):+ caps.check(caps.Query(**{**kw, **extra}))+++def test_minimax_evidence_ids_are_real_parity_cases():+ # Every case a MiniMax cell names exists in the parity test, so a renamed case cannot leave a cell+ # pointing at nothing.+ import re++ from tests import test_ulysses_minimax_parity as parity+ for cell in caps.CELLS:+ if cell.parity and "test_minimax_model_matches_upstream[" in cell.parity:+ ids = re.search(r"\[(.*)\]", cell.parity).group(1).split("|")+ assert set(ids) <= set(parity.MODEL_CASES), (cell.name, ids)++def test_unsloth_checkpoint_keeps_an_unverified_moe_row_refused_under_strict(clean_env, monkeypatch):diff --git a/tests/test_ulysses_minimax.py b/tests/test_ulysses_minimax.pynew file mode 100644index 00000000000..7530aae37cb--- /dev/null+++ b/tests/test_ulysses_minimax.py@@ -0,0 +1,190 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Ulysses context parallelism for MiniMax (M1 Lightning hybrid, M2 full attention), on CPU gloo ranks."""++import os+import tempfile+import types++import pytest+import torch+import torch.distributed as dist+import torch.multiprocessing as mp+import torch.nn.functional as F++import unsloth # noqa: F401+from unsloth import context_parallel as cp+from unsloth.utils import attention_dispatch as ad+from unsloth.utils import ulysses_hybrid as uh++transformers = pytest.importorskip("transformers")+if not hasattr(transformers, "MiniMaxConfig"):+ pytest.skip("transformers without MiniMax", allow_module_level = True)+KINDS = ["m1", "m2"] if hasattr(transformers, "MiniMaxM2Config") else ["m1"]+++def _config(kind):+ common = dict(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 32,+ num_hidden_layers = 3,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ num_local_experts = 2,+ num_experts_per_tok = 1,+ attention_dropout = 0.0,+ tie_word_embeddings = False,+ )+ if kind == "m1":+ common.update(+ layer_types = ["linear_attention", "full_attention", "linear_attention"],+ # Small blocks so 13 tokens cross several Lightning blocks and the shard seam.+ block_size = 4,+ )+ return transformers.MiniMaxConfig(**common), transformers.MiniMaxForCausalLM+ common.update(+ rope_parameters = {+ "rope_type": "default",+ "rope_theta": 10000.0,+ "partial_rotary_factor": 0.5,+ }+ )+ return transformers.MiniMaxM2Config(**common), transformers.MiniMaxM2ForCausalLM+++def _model(kind):+ torch.manual_seed(0)+ config, cls = _config(kind)+ model = cls(config).train()+ model.config._attn_implementation = "sdpa"+ for module in model.modules():+ # Grouped expert kernels may touch CUDA in a forked CPU worker.+ if hasattr(module, "set_experts_implementation"):+ module.set_experts_implementation("eager")+ return model+++def _per_expert_modules(model):+ return any(+ isinstance(getattr(m, "experts", None), torch.nn.ModuleList) for m in model.modules()+ )+++def _manager(size, rank):+ manager = object.__new__(cp.ContextParallelManager)+ manager.size = size+ manager.backend = "ulysses"+ manager.mesh = types.SimpleNamespace(get_group = lambda: None, get_local_rank = lambda: rank)+ return manager+++def _loss(logits, shift_labels, count):+ return (+ F.cross_entropy(+ logits.float().flatten(0, 1), shift_labels.flatten(), ignore_index = -100, reduction = "sum"+ )+ / count+ )+++def _parity_worker(rank, world, kind, control):+ ad.HAS_FLASH_ATTENTION = False+ ad.HAS_XFORMERS = False+ model = _model(kind)+ ids = torch.randint(0, 64, (2, 13), generator = torch.Generator().manual_seed(1))+ shift = F.pad(ids, (0, 1), value = -100)[:, 1:]+ count = shift.ne(-100).sum()+ _loss(model(input_ids = ids, use_cache = False).logits, shift, count).backward()+ reference = {n: p.grad.clone() for n, p in model.named_parameters()}+ model.zero_grad(set_to_none = True)++ manager = _manager(world, rank)+ manager.attach_attention_hooks(model)+ if control:+ for module in model.modules():+ if getattr(module, "_unsloth_cp_orig_forward", None) is not None:+ module.forward = module._unsloth_cp_orig_forward+ inputs = {"input_ids": ids.clone(), "labels": ids.clone()}+ manager._prepare_inputs(inputs)+ with manager.apply(inputs):+ out = model(+ input_ids = inputs["input_ids"], position_ids = inputs["position_ids"], use_cache = False+ )+ _loss(out.logits, inputs["shift_labels"], count).backward()+ for name, param in model.named_parameters():+ grad = torch.zeros_like(param) if param.grad is None else param.grad+ dist.all_reduce(grad)+ torch.testing.assert_close(grad, reference[name], rtol = 1e-4, atol = 1e-6, msg = name)+++def _spawn(fn, world, *args):+ # Fork: a spawned child would re-import unsloth without the conftest CUDA spoof.+ if "fork" not in mp.get_all_start_methods():+ pytest.skip("needs the fork start method")+ with tempfile.TemporaryDirectory() as d:+ mp.start_processes(+ _entry,+ args = (world, os.path.join(d, "init"), fn, args),+ nprocs = world,+ join = True,+ start_method = "fork",+ )+++def _entry(rank, world, init_file, fn, args):+ torch.set_num_threads(1)+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)+ try:+ fn(rank, world, *args)+ finally:+ dist.destroy_process_group()++[email protected]("kind", KINDS)+def test_ulysses_minimax_gradients_match_one_rank(kind):+ _spawn(_parity_worker, 2, kind, False)+++def test_shard_local_lightning_is_wrong():+ # Negative control: the stock Lightning forward on a shard restarts its decayed state at the seam.+ with pytest.raises(Exception, match = "AssertionError"):+ _spawn(_parity_worker, 2, "m1", True)+++def test_minimax_gates():+ model = _model("m1")+ lightning = model.model.layers[0].self_attn+ assert uh.is_ulysses_lightning(lightning) and uh.is_ulysses_mixer(lightning)+ assert cp._unsupported_attention(model.config, "ulysses") is None+ if _per_expert_modules(model):+ # transformers 4.x MiniMax: experts a shard never routes to would stall DDP's reduction.+ assert not cp._supports_context_parallel(model, "ulysses")+ with pytest.raises(NotImplementedError, match = "per-expert modules"):+ cp._check_cp_model(model, 2, "ulysses")+ else:+ assert cp._supports_context_parallel(model, "ulysses")+ assert not cp._supports_context_parallel(model, "ring")+ cp._check_stateful_mixers(model, "ulysses", 2)+ with pytest.raises(ValueError, match = "num_attention_heads"):+ cp._check_stateful_mixers(model, "ulysses", 8)++[email protected]("m2" not in KINDS, reason = "transformers without MiniMax-M2")+def test_minimax_m2_runs_ulysses_without_a_stateful_mixer():+ model = _model("m2")+ assert not cp._stateful_mixers(model)+ assert cp._supports_context_parallel(model, "ulysses")+ model.config.output_router_logits = True+ assert not cp._supports_context_parallel(model, "ulysses")+++def test_lightning_refuses_packing_and_cache():+ lightning = _model("m1").model.layers[0].self_attn+ h = torch.zeros(1, 4, 32)+ with pytest.raises(NotImplementedError, match = "packing"):+ uh.ulysses_lightning_forward(lightning, h, packed_seq_lengths = torch.tensor([4]), group = None)+ with pytest.raises(RuntimeError, match = "cache"):+ uh.ulysses_lightning_forward(lightning, h, past_key_values = object(), group = None)diff --git a/tests/test_ulysses_minimax_parity.py b/tests/test_ulysses_minimax_parity.pynew file mode 100644index 00000000000..83dad174d79--- /dev/null+++ b/tests/test_ulysses_minimax_parity.py@@ -0,0 +1,548 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""MiniMax Ulysses parity against the upstream full-sequence forward in float64, on CPU gloo ranks.++Lightning attention is checked block by block (lengths below / at / above the block size, seams+inside a block and exactly between blocks, a partial final block), then whole M1 hybrid and M2+models with padding, a rank without labels, a label across the seam, global position offsets,+GQA / MQA, LoRA, one AdamW step, CP = 2 / 4 and DP2 x CP2. Each mutant of the Lightning forward+and each negative control must break parity."""++import copy+import inspect+import os+import tempfile+import textwrap+import types++import pytest+import torch+import torch.distributed as dist+import torch.multiprocessing as mp+import torch.nn.functional as F++import unsloth # noqa: F401+from unsloth import context_parallel as cp+from unsloth.utils import attention_dispatch as ad+from unsloth.utils import ulysses_hybrid as uh++transformers = pytest.importorskip("transformers")+if not hasattr(transformers, "MiniMaxConfig"):+ pytest.skip("transformers without MiniMax", allow_module_level = True)+HAS_M2 = hasattr(transformers, "MiniMaxM2Config")+# transformers 5 fuses the experts into 3D weights; 4.x keeps per-expert modules (refused under CP).+FUSED_EXPERTS = hasattr(+ getattr(getattr(transformers.models, "minimax", None), "modeling_minimax", None),+ "MiniMaxExperts",+)++TOL = dict(rtol = 1e-8, atol = 1e-10)+# A broken variant must fail on numbers (assert_close with these names), not crash on a shape.+PARITY_FAILURE = r"AssertionError: (output|input grad|logits)"+SEQ = 22+++def _config(+ kind,+ block_size = 4,+ heads = 4,+ kv_heads = 2,+ layer_types = None,+):+ common = dict(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 32,+ num_attention_heads = heads,+ num_key_value_heads = kv_heads,+ head_dim = 8,+ num_local_experts = 2,+ num_experts_per_tok = 1,+ attention_dropout = 0.0,+ tie_word_embeddings = False,+ )+ if kind == "m1":+ layer_types = layer_types or [+ "linear_attention",+ "full_attention",+ "linear_attention",+ "linear_attention",+ ]+ common.update(+ num_hidden_layers = len(layer_types), layer_types = layer_types, block_size = block_size+ )+ return transformers.MiniMaxConfig(**common), transformers.MiniMaxForCausalLM+ common.update(+ num_hidden_layers = 2,+ rope_parameters = {+ "rope_type": "default",+ "rope_theta": 10000.0,+ "partial_rotary_factor": 0.5,+ },+ )+ return transformers.MiniMaxM2Config(**common), transformers.MiniMaxM2ForCausalLM+++def _float64_router(router):+ """The stock routers take their sigmoid / softmax in fp32 (router_logits.float()). Elementwise fp32 kernels+ round the vectorised body and the scalar tail differently, so a shard (a different token count)+ can move a routing weight by one fp32 ulp, about 1e-7 relative after backward: far above the+ float64 tolerance and unrelated to CP. Keep this one cast in the router's own dtype."""+ cls = type(router)+ src = textwrap.dedent(inspect.getsource(cls.forward))+ if "router_logits.float()" not in src:+ return+ if "_float64_forward" not in cls.__dict__:+ namespace = dict(vars(inspect.getmodule(cls)))+ exec(src.replace("router_logits.float()", "router_logits"), namespace)+ cls._float64_forward = namespace["forward"]+ router.forward = types.MethodType(cls._float64_forward, router)+++def _model(+ kind,+ lora = False,+ checkpoint = None,+ attn = "sdpa",+ **config_kwargs,+):+ torch.manual_seed(0)+ config, cls = _config(kind, **config_kwargs)+ model = cls(config).train()+ if checkpoint is not None:+ # The recompute repeats every all-to-all in backward, in the same order on each rank.+ model.gradient_checkpointing_enable({"use_reentrant": checkpoint == "reentrant"})+ model.config._attn_implementation = attn+ for module in model.modules():+ if hasattr(module, "set_experts_implementation"):+ module.set_experts_implementation("eager")+ if type(module).__name__ in ("MiniMaxTopKRouter", "MiniMaxM2TopKRouter"):+ _float64_router(module)+ if lora:+ from peft import LoraConfig, get_peft_model++ targets = ["q_proj", "k_proj", "v_proj", "o_proj", "qkv_proj", "out_proj", "output_gate"]+ extra = {}+ if lora == "experts":+ # LoRA on the routed experts too: the fused 3D expert weights.+ extra["target_parameters"] = ["experts.gate_up_proj", "experts.down_proj"]+ # init_lora_weights = False: B is random, so the adapters change the forward.+ model = get_peft_model(+ model,+ LoraConfig(r = 4, lora_alpha = 8, target_modules = targets, init_lora_weights = False, **extra),+ )+ if lora == "experts":+ # Engaged: the fused expert weights carry adapters (PEFT matches no parameter silently).+ assert any(+ "experts" in name and "lora_" in name and param.requires_grad+ for name, param in model.named_parameters()+ )+ return model.double()+++def _mutate(old, new):+ """Replace ulysses_lightning_forward in this (forked) worker by a source-edited copy."""+ src = textwrap.dedent(inspect.getsource(uh.ulysses_lightning_forward))+ assert old in src, old+ namespace = dict(vars(uh))+ exec(compile(src.replace(old, new), uh.__file__, "exec"), namespace)+ uh.ulysses_lightning_forward = namespace["ulysses_lightning_forward"]+++MUTANTS = {+ # Every rank uses rank 0's per-head slopes / decays.+ "slope_rate_slice": ("module.slope_rate[head]", "module.slope_rate[:heads]"),+ "diagonal_decay_slice": ("module.diagonal_decay[:, head]", "module.diagonal_decay[:, :heads]"),+ "query_decay_slice": ("module.query_decay[head]", "module.query_decay[:heads]"),+ # A partial block read with the tail of the query decay instead of its head.+ "partial_block_query_decay": ("query_decay[:, :n]", "query_decay[:, -n:]"),+ # The carried state is not decayed between blocks.+ "undecayed_state": ("state * torch.exp(-slope * n)", "state"),+ # q | k | v as three contiguous thirds instead of interleaved per head.+ "qkv_layout": (+ "qkv.reshape(bsz, local_len, module.num_attention_heads, 3 * dim)",+ "qkv.reshape(bsz, local_len, 3, module.num_attention_heads, dim).transpose(2, 3)"+ ".reshape(bsz, local_len, module.num_attention_heads, 3 * dim)",+ ),+ # Output gate applied before the full-width norm.+ "gate_before_norm": (+ "out = module.norm(out.reshape(bsz, local_len, -1))\n"+ " out = torch.sigmoid(module.output_gate(hidden_states)) * out",+ "out = torch.sigmoid(module.output_gate(hidden_states)) * out.reshape(bsz, local_len, -1)\n"+ " out = module.norm(out)",+ ),+ # Blocks restart at the shard start instead of the global sequence start.+ "shard_aligned_blocks": (+ "block, total = module.block_size, q.shape[2]",+ "block, total = local_len, q.shape[2]",+ ),+}+++def _lightning_worker(rank, world, seq_len, block_size, mutant):+ if mutant is not None and mutant != "shard_local":+ _mutate(*MUTANTS[mutant])+ config, _ = _config("m1", block_size = block_size)+ from transformers.models.minimax.modeling_minimax import MiniMaxLightningAttention++ torch.manual_seed(0)+ # layer_idx 2 of 4: the slope factor is neither 1 nor ~0.+ module = MiniMaxLightningAttention(config, layer_idx = 2).double()+ for p in module.parameters():+ p.data.normal_(0, 0.3)+ g = torch.Generator().manual_seed(1)+ x = torch.randn(2, seq_len, config.hidden_size, generator = g, dtype = torch.float64)+ cot = torch.randn(2, seq_len, config.hidden_size, generator = g, dtype = torch.float64)++ x_ref = x.clone().requires_grad_()+ ref = module(x_ref, None, None)[0]+ (ref * cot).sum().backward()+ ref_grads = {n: p.grad.clone() for n, p in module.named_parameters()}+ module.zero_grad(set_to_none = True)++ local = seq_len // world+ shard = slice(rank * local, (rank + 1) * local)+ x_local = x[:, shard].clone().requires_grad_()+ if mutant == "shard_local":+ out = module(x_local, None, None)[0]+ else:+ out = uh.ulysses_lightning_forward(module, x_local, group = None)[0]+ (out * cot[:, shard]).sum().backward()+ torch.testing.assert_close(out, ref[:, shard], **TOL, msg = "output")+ torch.testing.assert_close(x_local.grad, x_ref.grad[:, shard], **TOL, msg = "input grad")+ for name, param in module.named_parameters():+ dist.all_reduce(param.grad)+ torch.testing.assert_close(param.grad, ref_grads[name], **TOL, msg = name)+++# (seq_len, block_size, world): what the seam and the final block look like.+LIGHTNING_CASES = {+ "below_block": (4, 8, 2), # one partial block, seam inside it+ "at_block": (8, 8, 2), # one full block, seam inside it+ "above_block_partial": (12, 8, 2), # seam inside block 0, partial final block+ "seam_between_blocks": (16, 4, 2), # seam exactly at a block boundary+ "many_blocks_partial": (22, 4, 2), # 5 full blocks + 2, seam inside block 2+ "cp4_seams_inside": (20, 3, 4), # seams at 5 / 10 / 15, final block of 2+ "cp4_seams_between": (16, 4, 4), # every seam on a block boundary+}+++def _spawn(fn, world, *args):+ # Fork: a spawned child would re-import unsloth without the conftest CUDA spoof.+ if "fork" not in mp.get_all_start_methods():+ pytest.skip("needs the fork start method")+ with tempfile.TemporaryDirectory() as d:+ mp.start_processes(+ _entry,+ args = (world, os.path.join(d, "init"), fn, args),+ nprocs = world,+ join = True,+ start_method = "fork",+ )+++def _entry(rank, world, init_file, fn, args):+ torch.set_num_threads(1)+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)+ try:+ fn(rank, world, *args)+ finally:+ dist.destroy_process_group()++[email protected]("case", list(LIGHTNING_CASES))+def test_lightning_matches_upstream_full_sequence(case):+ seq_len, block_size, world = LIGHTNING_CASES[case]+ _spawn(_lightning_worker, world, seq_len, block_size, None)+++# Each mutant on a case where it changes the math (a full-size final block hides partial-block slips).+MUTANT_CASES = {+ "shard_local": "many_blocks_partial",+ "slope_rate_slice": "many_blocks_partial",+ "diagonal_decay_slice": "many_blocks_partial",+ "query_decay_slice": "many_blocks_partial",+ "partial_block_query_decay": "above_block_partial",+ "undecayed_state": "seam_between_blocks",+ "qkv_layout": "at_block",+ "gate_before_norm": "below_block",+ "shard_aligned_blocks": "above_block_partial",+}++[email protected]("mutant", list(MUTANT_CASES))+def test_lightning_mutants_break_parity(mutant):+ seq_len, block_size, world = LIGHTNING_CASES[MUTANT_CASES[mutant]]+ with pytest.raises(Exception, match = PARITY_FAILURE):+ _spawn(_lightning_worker, world, seq_len, block_size, mutant)+++def _batch(dp_rank, vocab = 64):+ """Right padded rows of unequal length; labels stop before position 12, so the CP=2 rank 1+ shard has none and position 10's target (token 11) crosses the CP=2 seam."""+ g = torch.Generator().manual_seed(10 + dp_rank)+ ids = torch.randint(1, vocab, (2, SEQ), generator = g)+ lengths = [(SEQ, 9), (17, SEQ)][dp_rank]+ mask = torch.zeros(2, SEQ, dtype = torch.long)+ for row, n in enumerate(lengths):+ mask[row, :n] = 1+ ids = ids * mask+ labels = ids.masked_fill(mask == 0, -100)+ labels[:, 12:] = -100+ # Nonzero global offsets: RoPE must see these positions on every shard.+ offsets = torch.tensor([[5], [17]]) + 3 * dp_rank+ positions = torch.arange(SEQ).expand(2, -1) + offsets+ return {"input_ids": ids, "attention_mask": mask, "labels": labels, "position_ids": positions}+++def _ce_sum(logits, shift_labels):+ return F.cross_entropy(+ logits.flatten(0, 1), shift_labels.flatten(), ignore_index = -100, reduction = "sum"+ )+++def _model_worker(rank, world, kind, cp_size, lora, control, model_kwargs):+ ad.HAS_FLASH_ATTENTION = False+ ad.HAS_XFORMERS = False+ cp.DEVICE_TYPE_TORCH = "cpu"+ model = _model(kind, lora = lora, **model_kwargs)+ reference = copy.deepcopy(model)+ dp_size, dp_rank = world // cp_size, rank // cp_size+ batches = [_batch(d) for d in range(dp_size)]+ shift = [F.pad(b["labels"], (0, 1), value = -100)[:, 1:] for b in batches]+ count = sum(int(s.ne(-100).sum()) for s in shift)++ ref_logits = []+ ref_loss = 0+ for b, s in zip(batches, shift):+ # No labels: a loss-computing forward may hand back placeholder logits.+ inputs = {k: v for k, v in b.items() if k != "labels"}+ logits = reference(**inputs, use_cache = False).logits+ ref_logits.append(logits.detach())+ ref_loss = ref_loss + _ce_sum(logits, s)+ ref_loss = ref_loss / count+ ref_loss.backward()++ manager = cp.ContextParallelManager(cp_size, "ulysses")+ cp_rank = manager.mesh.get_local_rank()+ manager.attach_attention_hooks(model)+ if control == "shard_local":+ for module in model.modules():+ if getattr(module, "_unsloth_cp_orig_forward", None) is not None:+ module.forward = module._unsloth_cp_orig_forward+ module._unsloth_cp_ulysses_attention = False+ if control == "local_positions":+ shard = manager._ulysses_shard++ def local_positions(inputs):+ shard(inputs)+ p = inputs["position_ids"]+ inputs["position_ids"] = p - p[:, :1] + batches[dp_rank]["position_ids"][:, :1]++ manager._ulysses_shard = local_positions+ if control == "qk_norm_after_exchange":+ # What normalising q / k on the received head group (after the exchange) would compute.+ for module in model.modules():+ for norm in (getattr(module, "q_norm", None), getattr(module, "k_norm", None)):+ if norm is None:+ continue++ def grouped(+ self,+ x,+ size = cp_size,+ ):+ parts = x.chunk(size, dim = -1)+ weights = self.weight.chunk(size)+ out = []+ for part, weight in zip(parts, weights):+ var = part.pow(2).mean(-1, keepdim = True)+ out.append(weight * part * torch.rsqrt(var + self.variance_epsilon))+ return torch.cat(out, dim = -1)++ norm.forward = types.MethodType(grouped, norm)++ calls = {"lightning": 0, "attention": 0}+ for key, fn_name in (+ ("lightning", "ulysses_lightning_forward"),+ ("attention", "_ulysses_hf_attention"),+ ):++ def counted(+ *args,+ _fn = getattr(uh, fn_name),+ _key = key,+ **kwargs,+ ):+ calls[_key] += 1+ return _fn(*args, **kwargs)++ setattr(uh, fn_name, counted)++ inputs = {k: v.clone() for k, v in batches[dp_rank].items()}+ with manager.apply(inputs):+ out = model(+ input_ids = inputs["input_ids"],+ attention_mask = inputs["attention_mask"],+ position_ids = inputs["position_ids"],+ use_cache = False,+ )+ local_loss = _ce_sum(out.logits, inputs["shift_labels"]) / count+ local_loss.backward()++ if control is None:+ # The Ulysses paths ran once per layer of their kind: no silent fallback.+ layer_types = getattr(model.config, "layer_types", None) or (+ ["full_attention"] * model.config.num_hidden_layers+ )+ # Checkpointing recomputes each layer once in backward.+ runs = 2 if model_kwargs.get("checkpoint") else 1+ assert calls["lightning"] == runs * layer_types.count("linear_attention"), calls+ assert calls["attention"] == runs * layer_types.count("full_attention"), calls++ # Outputs: valid positions of the gathered CP shards against the reference.+ gathered = [torch.empty_like(out.logits) for _ in range(cp_size)]+ dist.all_gather(gathered, out.logits.detach().contiguous(), group = manager.group)+ logits = torch.cat(gathered, dim = 1)[:, :SEQ]+ valid = batches[dp_rank]["attention_mask"].bool()+ torch.testing.assert_close(logits[valid], ref_logits[dp_rank][valid], **TOL, msg = "logits")+ assert cp_rank == rank % cp_size++ loss = local_loss.detach().clone()+ dist.all_reduce(loss)+ torch.testing.assert_close(loss, ref_loss.detach(), **TOL, msg = "loss")++ # Gradients: None on every rank iff None in the reference; zero stays zero; else the value.+ ref_params = dict(reference.named_parameters())+ for name, param in model.named_parameters():+ ref_grad = ref_params[name].grad+ present = torch.tensor(float(param.grad is not None))+ dist.all_reduce(present)+ if ref_grad is None:+ assert present.item() == 0, f"{name}: None in the reference, set under CP"+ continue+ assert present.item() > 0, f"{name}: set in the reference, None on every CP rank"+ grad = torch.zeros_like(param) if param.grad is None else param.grad+ dist.all_reduce(grad)+ param.grad = grad+ if not ref_grad.any():+ assert not grad.any(), f"{name}: zero in the reference, nonzero under CP"+ torch.testing.assert_close(grad, ref_grad, **TOL, msg = lambda m: f"{name}: {m}")++ trainable = lambda m: [p for p in m.parameters() if p.requires_grad]+ for m in (model, reference):+ torch.optim.AdamW(trainable(m), lr = 1e-2, weight_decay = 0.1).step()+ for (name, p), q in zip(model.named_parameters(), reference.parameters()):+ torch.testing.assert_close(p, q, **TOL, msg = f"after AdamW: {name}")+++# (kind, cp_size, world, lora, model kwargs)+MODEL_CASES = {+ "m1_cp2": ("m1", 2, 2, False, {}),+ "m1_cp2_lora": ("m1", 2, 2, True, {}),+ "m1_cp4": ("m1", 4, 4, False, {}),+ "m1_dp2_cp2": ("m1", 2, 4, False, {}),+ "m1_cp2_block_8": ("m1", 2, 2, False, {"block_size": 8}),+ "m1_cp2_lora_checkpoint": ("m1", 2, 2, True, {"checkpoint": "non_reentrant"}),+ "m2_cp2_lora_reentrant": ("m2", 2, 2, True, {"checkpoint": "reentrant"}),+ "m2_cp2": ("m2", 2, 2, False, {}),+ "m2_cp2_lora": ("m2", 2, 2, True, {}),+ "m2_cp4_gqa": ("m2", 4, 4, False, {}),+ "m2_cp2_mqa": ("m2", 2, 2, False, {"kv_heads": 1}),+ "m2_dp2_cp2": ("m2", 2, 4, False, {}),+ # MiniMax with full attention layers only: no Lightning mixer, so main's generic Ulysses route alone.+ "m1_full_attention_only_cp2": ("m1", 2, 2, False, {"layer_types": ["full_attention"] * 2}),+ # The parallel_capabilities MoE x cp cells (aux loss off): eager attention, HF checkpointing,+ # full finetuning, attention LoRA and LoRA on the fused experts, CP 2 and DP2 x CP2.+ "m1_cp2_eager_checkpoint": (+ "m1",+ 2,+ 2,+ False,+ {"attn": "eager", "checkpoint": "non_reentrant"},+ ),+ "m1_cp2_lora_eager": ("m1", 2, 2, True, {"attn": "eager"}),+ "m1_cp2_lora_experts": ("m1", 2, 2, "experts", {}),+ "m1_dp2_cp2_lora_experts_eager_checkpoint": (+ "m1",+ 2,+ 4,+ "experts",+ {"attn": "eager", "checkpoint": "non_reentrant"},+ ),+ "m2_cp2_eager_checkpoint": (+ "m2",+ 2,+ 2,+ False,+ {"attn": "eager", "checkpoint": "non_reentrant"},+ ),+ "m2_cp2_lora_eager": ("m2", 2, 2, True, {"attn": "eager"}),+ "m2_cp2_lora_experts": ("m2", 2, 2, "experts", {}),+ "m2_dp2_cp2_lora_experts_eager_checkpoint": (+ "m2",+ 2,+ 4,+ "experts",+ {"attn": "eager", "checkpoint": "non_reentrant"},+ ),+}++[email protected]("case", list(MODEL_CASES))+def test_minimax_model_matches_upstream(case):+ kind, cp_size, world, lora, kwargs = MODEL_CASES[case]+ if kind == "m2" and not HAS_M2:+ pytest.skip("transformers without MiniMax-M2")+ if lora == "experts" and not FUSED_EXPERTS:+ pytest.skip(+ "transformers without fused MiniMax experts (target_parameters has nothing to match)"+ )+ _spawn(_model_worker, world, kind, cp_size, lora, None, kwargs)+++CONTROLS = [+ ("m1", "shard_local"),+ ("m2", "shard_local"),+ ("m1", "local_positions"),+ ("m2", "local_positions"),+ ("m2", "qk_norm_after_exchange"),+]++[email protected]("kind, control", CONTROLS)+def test_minimax_negative_controls_break_parity(kind, control):+ if kind == "m2" and not HAS_M2:+ pytest.skip("transformers without MiniMax-M2")+ with pytest.raises(Exception, match = PARITY_FAILURE):+ _spawn(_model_worker, 2, kind, 2, False, control, {})++[email protected](not HAS_M2, reason = "transformers without MiniMax-M2")+ "heads, kv_heads, size, match",+ [+ (4, 2, 8, "number of attention heads"),+ (6, 3, 2, "divide one another"),+ (4, 3, 2, "multiple of key/value heads"),+ ],+)+def test_minimax_m2_head_counts_are_validated(heads, kv_heads, size, match):+ config, cls = _config("m2", heads = heads, kv_heads = kv_heads)+ model = cls(config)+ with pytest.raises(ValueError, match = match):+ cp._check_cp_model(model, size, "ulysses")+++def test_minimax_m1_head_counts_are_validated():+ config, cls = _config("m1", heads = 4, kv_heads = 4)+ model = cls(config)+ if any(isinstance(getattr(m, "experts", None), torch.nn.ModuleList) for m in model.modules()):+ # transformers 4.x MiniMax keeps per-expert modules, which CP refuses after the head check.+ with pytest.raises(NotImplementedError, match = "per-expert modules"):+ cp._check_cp_model(model, 4, "ulysses")+ else:+ cp._check_cp_model(model, 4, "ulysses")+ with pytest.raises(ValueError, match = "number of attention heads"):+ cp._check_cp_model(model, 8, "ulysses")diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 27411cfd022..ef762071465 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -473,8 +473,8 @@ def _check_stateful_mixers(model, backend, size) -> None:unsupported = sorted({type(m).__name__ for m, _ in mixers if not is_ulysses_mixer(m)})if unsupported:raise NotImplementedError(- "Unsloth: Ulysses context parallelism supports Gated DeltaNet and Kimi delta "- f"linear attention only, not {unsupported}."+ "Unsloth: Ulysses context parallelism supports Gated DeltaNet, Kimi delta and "+ f"Lightning linear attention only, not {unsupported}.")for module, _ in mixers:check_ulysses_mixer_heads(module, size)@@ -1520,9 +1520,9 @@ def _check_cp_model(if not _supports_context_parallel(model, backend):raise NotImplementedError("Unsloth: context parallelism supports Unsloth's fast attention (Llama-style, Qwen3, Mistral, Granite, "- "Cohere) with either backend, "- "and with Ulysses any Hugging Face attention model (attention through ALL_ATTENTION_FUNCTIONS) "- "and the Qwen3-Next / Qwen3.5 Gated DeltaNet and Kimi-Linear hybrids."+ "Cohere) with either backend, and with Ulysses any Hugging Face attention model (attention through "+ "ALL_ATTENTION_FUNCTIONS, e.g. MiniMax-M2) and the Qwen3-Next / Qwen3.5 Gated DeltaNet, Kimi-Linear and "+ "MiniMax-M1 Lightning hybrids.")diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.pyindex 9a4c034f0b0..60995583aab 100644--- a/unsloth/parallel_capabilities.py+++ b/unsloth/parallel_capabilities.py@@ -814,6 +814,8 @@ MOE_FAMILIES = ("gpt_oss","deepseek_v3","kimi_linear",+ "minimax",+ "minimax_m2","olmoe","mixtral","qwen2_moe",@@ -875,6 +877,11 @@ _MOE_PP = "tests/test_pp_moe_aux_gloo.py::test_pp2_moe_aux_matches_gradient_accu# fp64 against one process, the idle-expert hook and one AdamW step. HF checkpointing only (no Unsloth checkpoint run)._KIMI_MOE = "tests/test_ulysses_kimi.py::test_kimi_moe_under_ulysses_matches_one_process"_KIMI_EAGER = "eager: remote code, no compiled region"+# MiniMax (transformers' MiniMaxForCausalLM / MiniMaxM2ForCausalLM, fused experts): M1 Lightning + full attention+# layers, M2 full attention through the generic route, the MoE block with routed / unrouted experts, right padded+# rows, fp64 against one process, every gradient and one AdamW step. HF checkpointing only.+_MINIMAX = "tests/test_ulysses_minimax_parity.py::test_minimax_model_matches_upstream"+_MINIMAX_EAGER = "eager: block compile not measured for this family"# Families whose cp rows also ran Unsloth's checkpoint (path A and B); the others match HF checkpointing only._MOE_UNSLOTH_GC_FAMILIES = ("qwen3_moe",)# (family, precision, aux, mode) -> (parity, engagement, compile) for the rows a test proves.@@ -915,6 +922,36 @@ _MOE_EVIDENCE = {)* 2+ (_KIMI_EAGER,),+ ("minimax", "full", False, "cp"): (+ _MINIMAX + "[m1_cp2|m1_cp4|m1_dp2_cp2|m1_cp2_eager_checkpoint]",+ )+ * 2+ + (_MINIMAX_EAGER,),+ ("minimax", "lora", False, "cp"): (+ _MINIMAX + "[m1_cp2_lora|m1_cp2_lora_checkpoint|m1_cp2_lora_eager]",+ )+ * 2+ + (_MINIMAX_EAGER,),+ ("minimax", "lora+experts", False, "cp"): (+ _MINIMAX + "[m1_cp2_lora_experts|m1_dp2_cp2_lora_experts_eager_checkpoint]",+ )+ * 2+ + (_MINIMAX_EAGER,),+ ("minimax_m2", "full", False, "cp"): (+ _MINIMAX + "[m2_cp2|m2_cp4_gqa|m2_dp2_cp2|m2_cp2_eager_checkpoint]",+ )+ * 2+ + (_MINIMAX_EAGER,),+ ("minimax_m2", "lora", False, "cp"): (+ _MINIMAX + "[m2_cp2_lora|m2_cp2_lora_reentrant|m2_cp2_lora_eager]",+ )+ * 2+ + (_MINIMAX_EAGER,),+ ("minimax_m2", "lora+experts", False, "cp"): (+ _MINIMAX + "[m2_cp2_lora_experts|m2_dp2_cp2_lora_experts_eager_checkpoint]",+ )+ * 2+ + (_MINIMAX_EAGER,),}diff --git a/unsloth/utils/ulysses_hybrid.py b/unsloth/utils/ulysses_hybrid.pyindex 69fbcf1b2d4..0fb47c07ac6 100644--- a/unsloth/utils/ulysses_hybrid.py+++ b/unsloth/utils/ulysses_hybrid.py@@ -2,8 +2,8 @@# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved."""Ulysses context parallelism for Hugging Face attention (any model whose attention calls ALL_ATTENTION_FUNCTIONS-runs its own attention function after the exchange) and for hybrid models (Gated DeltaNet, Kimi delta attention),-whose scans need the full sequence so they run on heads."""+runs its own attention function after the exchange) and for hybrid models (Gated DeltaNet, Kimi delta and MiniMax+Lightning attention), whose scans need the full sequence so they run on heads."""import builtinsimport functools@@ -19,11 +19,13 @@ __all__ = ["GATED_DELTA_ATTENTION_CLASSES","is_ulysses_gated_delta","is_ulysses_kimi_delta",+ "is_ulysses_lightning","is_ulysses_mixer","check_gated_delta_heads","check_ulysses_mixer_heads","ulysses_gated_delta_forward","ulysses_kimi_delta_forward",+ "ulysses_lightning_forward","wrap_gated_delta_forward","wrap_ulysses_mixer_forward","install_hf_attention_wrapper",@@ -56,8 +58,20 @@ def is_ulysses_kimi_delta(module) -> bool:return hasattr(module, "f_a_proj") or hasattr(module, "f_proj")+def is_ulysses_lightning(module) -> bool:+ if not type(module).__name__.endswith("LightningAttention"):+ return False+ needed = ("qkv_proj", "norm", "output_gate", "out_proj", "block_size", "num_attention_heads")+ buffers = ("slope_rate", "query_decay", "key_decay", "diagonal_decay")+ return all(hasattr(module, n) for n in needed + buffers)++def is_ulysses_mixer(module) -> bool:- return is_ulysses_gated_delta(module) or is_ulysses_kimi_delta(module)+ return (+ is_ulysses_gated_delta(module)+ or is_ulysses_kimi_delta(module)+ or is_ulysses_lightning(module)+ )def check_gated_delta_heads(module, cp_size) -> None:@@ -71,12 +85,16 @@ def check_gated_delta_heads(module, cp_size) -> None:def check_ulysses_mixer_heads(module, cp_size) -> None:- if not is_ulysses_kimi_delta(module):+ if is_ulysses_kimi_delta(module):+ name, heads = "Kimi delta attention num_heads", module.num_heads+ elif is_ulysses_lightning(module):+ name, heads = "Lightning attention num_attention_heads", module.num_attention_heads+ else:return check_gated_delta_heads(module, cp_size)- if module.num_heads % cp_size != 0:+ if heads % cp_size != 0:raise ValueError(- f"Unsloth: Ulysses context parallelism needs Kimi delta attention num_heads "- f"({module.num_heads}) to be divisible by context_parallel_size ({cp_size})."+ f"Unsloth: Ulysses context parallelism needs {name} ({heads}) to be divisible by "+ f"context_parallel_size ({cp_size}).")@@ -419,6 +437,68 @@ def ulysses_kimi_delta_forward(return out+def ulysses_lightning_forward(+ module,+ hidden_states,+ position_embeddings = None,+ attention_mask = None,+ past_key_values = None,+ *,+ group,+ **kwargs,+):+ """Training forward of a MiniMax LightningAttention on a contiguous sequence shard."""+ if past_key_values is not None:+ raise RuntimeError("Unsloth: Ulysses context parallelism does not support a cache.")+ if kwargs.get("packed_seq_lengths") is not None:+ raise NotImplementedError(+ "Unsloth: Ulysses context parallelism does not support packing for Lightning attention."+ )+ size = dist.get_world_size(group)+ rank = dist.get_rank(group)+ check_ulysses_mixer_heads(module, size)+ bsz, local_len, _ = hidden_states.shape+ heads, dim = module.num_attention_heads // size, module.head_dim++ qkv = module.act_fn(module.qkv_proj(hidden_states))+ if isinstance(attention_mask, torch.Tensor) and attention_mask.ndim == 2:+ # The stock forward zeroes padded values; zeroing q / k too changes nothing downstream.+ qkv = qkv * attention_mask[:, -local_len:, None].to(qkv.dtype)+ # [B, T/N, H, 3D] -> [B, T, H/N, 3D]: one exchange for q, k and v of each head.+ qkv = _AllToAll.apply(+ qkv.reshape(bsz, local_len, module.num_attention_heads, 3 * dim), 2, 1, group+ )+ q, k, v = (x.transpose(1, 2) for x in torch.split(qkv, dim, dim = -1))++ head = slice(rank * heads, (rank + 1) * heads)+ dtype = v.dtype+ slope = module.slope_rate[head].to(dtype)+ query_decay = module.query_decay[head].to(dtype)+ key_decay = module.key_decay[head].to(dtype)+ diagonal_decay = module.diagonal_decay[:, head].to(dtype)+ block, total = module.block_size, q.shape[2]++ state = v.new_zeros(bsz, heads, dim, dim)+ outs = []+ for start in range(0, total, block):+ end = min(start + block, total)+ n = end - start+ qb, kb, vb = q[:, :, start:end], k[:, :, start:end], v[:, :, start:end]+ intra = torch.matmul(+ torch.matmul(qb, kb.transpose(-1, -2)) * diagonal_decay[:, :, :n, :n], vb+ )+ inter = torch.matmul(qb * query_decay[:, :n], state)+ outs.append(inter + intra)+ state = state * torch.exp(-slope * n) + torch.matmul(+ (kb * key_decay[:, -n:]).transpose(-1, -2), vb+ )+ out = _AllToAll.apply(torch.cat(outs, dim = 2).transpose(1, 2), 1, 2, group)++ out = module.norm(out.reshape(bsz, local_len, -1))+ out = torch.sigmoid(module.output_gate(hidden_states)) * out+ return module.out_proj(out), None++def _wrap_ulysses_forward(module, get_cp_manager, name) -> None:if getattr(module, "_unsloth_cp_orig_forward", None) is not None:return@@ -453,6 +533,8 @@ def wrap_gated_delta_forward(module, get_cp_manager) -> None:def wrap_ulysses_mixer_forward(module, get_cp_manager) -> None:if is_ulysses_kimi_delta(module):return _wrap_ulysses_forward(module, get_cp_manager, "ulysses_kimi_delta_forward")+ if is_ulysses_lightning(module):+ return _wrap_ulysses_forward(module, get_cp_manager, "ulysses_lightning_forward")return wrap_gated_delta_forward(module, get_cp_manager)