Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallel (Ulysses) for DPO and GRPO training (#15041)
Daniel Han1d3e5f56e63f
diff --git a/tests/test_context_parallel_rl.py b/tests/test_context_parallel_rl.pynew file mode 100644index 00000000000..e7c83179075--- /dev/null+++ b/tests/test_context_parallel_rl.py@@ -0,0 +1,718 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Context parallelism for DPO / GRPO: decoder sharding + hidden-state gather, peer sync, refusals (CPU, gloo)."""++import contextlib+import dataclasses as _dc+import os+import tempfile+import types++import pytest+import torch+import torch.distributed as dist+import torch.multiprocessing as mp+from transformers.modeling_outputs import BaseModelOutputWithPast++import unsloth # noqa: F401+from unsloth import context_parallel as cp+++class _Decoder(torch.nn.Module):+ """Causal toy decoder: position-dependent, mixes earlier tokens, returns an HF-style output."""++ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.embed_tokens = torch.nn.Embedding(50, 8)+ self.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8)])++ def forward(+ self,+ input_ids = None,+ attention_mask = None,+ position_ids = None,+ output_hidden_states = False,+ **kwargs,+ ):+ if cp.get_cp_manager() is not None:+ # Per-shard masks differ across peers (only the last holds padding): the hook must drop it.+ assert attention_mask is None+ h = self.embed_tokens(input_ids) * (1 + 0.01 * position_ids[..., None].float())+ first = h+ # Sum over earlier tokens of the FULL sequence (stands in for Ulysses attention).+ manager = cp.get_cp_manager()+ if manager is not None:+ rank = dist.get_rank()+ full = cp.gather_sequence(h, manager)+ prefix = full.cumsum(1)[:, rank * h.shape[1] : (rank + 1) * h.shape[1]]+ else:+ prefix = h.cumsum(1)+ h = torch.tanh(self.layers[0](prefix))+ return BaseModelOutputWithPast(+ last_hidden_state = h, hidden_states = (first, h) if output_hidden_states else None+ )+++def _manager(rank, size = 2):+ manager = object.__new__(cp.ContextParallelManager)+ manager.size = size+ manager.backend = "ulysses"+ manager.mesh = types.SimpleNamespace(+ get_local_rank = lambda: rank, get_group = lambda: dist.group.WORLD+ )+ return manager+++def _worker(rank, world, init_file, queue):+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)+ try:+ torch.manual_seed(1)+ ids = torch.randint(1, 50, (2, 7))+ mask = torch.ones(2, 7, dtype = torch.long)+ mask[1, 5:] = 0+ weight = torch.randn(2, 7, 8)++ reference = _Decoder()+ full = reference(+ input_ids = ids, attention_mask = mask, position_ids = torch.arange(7).expand(2, 7)+ )+ (full.last_hidden_state * weight).sum().backward()++ model = _Decoder()+ manager = _manager(rank, world)+ cp.attach_rl_hooks(manager, torch.nn.ModuleDict({"model": model}))+ with cp._manager_scope(manager):+ out = model(input_ids = ids, attention_mask = mask, output_hidden_states = True)+ # Every peer computes the same loss on the gathered (unpadded) sequence.+ (out.last_hidden_state * weight).sum().backward()+ assert out.last_hidden_state.shape == (2, 7, 8)+ assert out.hidden_states[-1] is out.last_hidden_state+ assert out.hidden_states[0].shape == (2, 7, 8)+ # Trimmed from the padded gather as fresh tensors: compiled consumers must not guard on a view base.+ assert out.last_hidden_state._base is None and out.hidden_states[0]._base is None+ torch.testing.assert_close(+ out.last_hidden_state, full.last_hidden_state, rtol = 1e-5, atol = 1e-6+ )+ # DDP's average over the CP group, as the trainer gets it.+ for p in model.parameters():+ dist.all_reduce(p.grad)+ p.grad /= world+ for p, q in zip(model.parameters(), reference.parameters()):+ torch.testing.assert_close(p.grad, q.grad, rtol = 1e-5, atol = 1e-6)++ # Outside the scope the hooks pass through untouched.+ plain = model(input_ids = ids, attention_mask = mask, position_ids = torch.arange(7).expand(2, 7))+ torch.testing.assert_close(plain.last_hidden_state, full.last_hidden_state)+ assert model._unsloth_cp_rl_stack == []++ # Peers adopt rank 0's object; rank 1's own differs.+ mine = {"ids": torch.full((2,), rank), "text": [f"r{rank}"], "pair": (rank, None)}+ got = cp._broadcast_from_group_root(mine, manager, torch.device("cpu"))+ assert got["text"] == ["r0"] and got["pair"] == (0, None)+ assert torch.equal(got["ids"], torch.zeros(2, dtype = torch.long))+ # The root's inference-mode flag reaches every peer (compiled forwards guard on it).+ with torch.inference_mode():+ scored = {"ids": torch.full((2,), rank)}+ got = cp._broadcast_from_group_root(scored, manager, torch.device("cpu"))+ assert got["ids"].is_inference() and torch.equal(+ got["ids"], torch.zeros(2, dtype = torch.long)+ )+ got = cp._broadcast_from_group_root({"ids": torch.ones(1)}, manager, torch.device("cpu"))+ assert not got["ids"].is_inference()++ cp._check_same_prompts(["same"], manager)+ with pytest.raises(RuntimeError, match = "different prompts"):+ cp._check_same_prompts([f"prompt {rank}"], manager)+ queue.put((rank, "ok"))+ except BaseException as error: # pragma: no cover - reported to the parent+ import traceback+ queue.put((rank, traceback.format_exc()))+ finally:+ dist.destroy_process_group()+++def test_decoder_shard_gather_matches_full_sequence_with_grads():+ if "fork" not in mp.get_all_start_methods():+ pytest.skip("needs fork")+ ctx = mp.get_context("fork")+ queue = ctx.Queue()+ with tempfile.TemporaryDirectory() as tmp:+ init_file = os.path.join(tmp, "init")+ procs = [ctx.Process(target = _worker, args = (r, 2, init_file, queue)) for r in range(2)]+ for p in procs:+ p.start()+ results = dict(queue.get(timeout = 120) for _ in procs)+ for p in procs:+ p.join(30)+ assert results == {0: "ok", 1: "ok"}, results+++def test_decoder_hook_refuses_cache_and_prefix_grouper():+ model = _Decoder()+ manager = _manager(0)+ cp.attach_rl_hooks(manager, torch.nn.ModuleDict({"model": model}))+ ids = torch.ones(1, 4, dtype = torch.long)+ with cp._manager_scope(manager):+ with pytest.raises(NotImplementedError, match = "KV cache"):+ model(input_ids = ids, past_key_values = ((torch.zeros(1),),))+ with pytest.raises(NotImplementedError, match = "prefix_seg_info"):+ model(input_ids = ids, prefix_seg_info = object())+++def _args(**kw):+ base = dict(+ context_parallel_size = 2,+ context_parallel_backend = "ulysses",+ per_device_train_batch_size = 4,+ per_device_eval_batch_size = 4,+ gradient_accumulation_steps = 1,+ steps_per_generation = 1,+ num_generations = 4,+ )+ base.update(kw)+ return types.SimpleNamespace(**base)+++ "kind, attrs, message",+ [+ ("dpo_trainer", {"padding_free": True}, "padding_free"),+ ("dpo_trainer", {"precompute_ref_log_probs": True}, "precompute_ref_log_probs"),+ ("dpo_trainer", {"generate_during_eval": True}, "generate_during_eval"),+ ("grpo_trainer", {"use_vllm": True}, "vLLM"),+ ("grpo_trainer", {"per_device_train_batch_size": 3}, "multiples of num_generations"),+ ("grpo_trainer", {"scale_rewards": "batch"}, "per-group reward scaling"),+ ("grpo_trainer", {"multi_objective_aggregation": "normalize_then_sum"}, "per-group"),+ ],+)+def test_unsupported_rl_configs_are_refused(kind, attrs, message):+ trainer = types.SimpleNamespace(args = _args(**attrs), _generate = None, eval_dataset = None)+ with pytest.raises(NotImplementedError, match = message):+ cp._check_rl_config(trainer, kind)+++def test_grpo_without_generate_hook_is_refused():+ trainer = types.SimpleNamespace(args = _args(), eval_dataset = None)+ with pytest.raises(NotImplementedError, match = "TRL >= 0.24"):+ cp._check_rl_config(trainer, "grpo_trainer")+++def _fake_trainer_class(log):+ class Sampler:+ mini_repeat_count = 4+ batch_size = 8++ class Trainer:+ def __init__(self, args):+ self.args = args+ self.model = torch.nn.Linear(1, 1)+ self.accelerator = types.SimpleNamespace(device = torch.device("cpu"))++ def training_step(self, model, inputs):+ self._prepare_inputs(inputs)+ log.append(("compute_loss_returned", self.compute_loss(model, inputs)))+ log.append(("backward", cp.get_cp_manager()))+ return 0.0++ def prediction_step(self, model, inputs, prediction_loss_only):+ log.append(("predict", cp.get_cp_manager()))++ def _prepare_inputs(self, inputs):+ self._generate(["p"])+ self._get_per_token_logps_and_entropies()+ return {"rows": [cp.get_cp_manager() is None]}++ def _generate(self, prompts):+ log.append(("generate", cp.get_cp_manager()))+ return (prompts,)++ def _get_per_token_logps_and_entropies(self):+ log.append(("logps", cp.get_cp_manager()))++ def compute_loss(self, model, inputs):+ log.append(("loss", cp.get_cp_manager()))+ return 1.0++ def _get_train_sampler(self, *a):+ return Sampler()++ def train(self):+ return "trained"++ def evaluate(self, eval_dataset = None):+ return "evaluated"++ def evaluation_loop(self):+ return "looped"++ return Trainer++[email protected]("kind", ["dpo_trainer", "grpo_trainer"])+def test_rl_scopes_keep_generation_unsharded(monkeypatch, kind):+ log = []+ manager = types.SimpleNamespace(size = 2, backend = "ulysses")+ monkeypatch.setattr(cp, "_check_cp_runtime", lambda *a: None)+ monkeypatch.setattr(cp, "_check_cp_trainer", lambda *a: None)+ monkeypatch.setattr(cp, "_install_accelerator_state", lambda *a: None)+ monkeypatch.setattr(cp, "ContextParallelManager", lambda size, backend: manager)+ monkeypatch.setattr(cp, "attach_rl_hooks", lambda m, model: log.append(("hooked", m)))+ monkeypatch.setattr(cp, "_decoders", lambda model: [])+ monkeypatch.setattr(cp, "_check_same_prompts", lambda prompts, m: log.append(("same", prompts)))+ monkeypatch.setattr(cp, "_broadcast_from_group_root", lambda obj, m, device: obj)+ Trainer = _fake_trainer_class(log)+ cp.patch_rl_trainer(Trainer, kind)+ trainer = Trainer(_args())+ assert trainer._context_parallel_manager is manager+ assert cp.get_cp_manager() is None+ trainer.training_step(trainer.model, {})+ assert cp.get_cp_manager() is None+ seen = {name: value for name, value in log if name in ("generate", "logps", "loss", "backward")}+ assert seen["generate"] is (None if kind == "grpo_trainer" else manager)+ assert seen["logps"] is manager and seen["loss"] is manager and seen["backward"] is manager+ trainer.prediction_step(trainer.model, {}, True)+ assert dict(log)["predict"] is (manager if kind == "dpo_trainer" else None)+ if kind == "grpo_trainer":+ assert trainer._get_train_sampler().batch_size == 4+ assert ("same", ["p"]) in log+++def test_rl_trainers_refuse_ring(monkeypatch):+ Trainer = _fake_trainer_class([])+ cp.patch_rl_trainer(Trainer, "dpo_trainer")+ with pytest.raises(NotImplementedError, match = "ulysses"):+ Trainer(_args(context_parallel_backend = "ring"))+++def test_rl_trainers_without_cp_are_untouched(monkeypatch):+ monkeypatch.setattr(cp, "_install_accelerator_state", lambda *a: None)+ log = []+ Trainer = _fake_trainer_class(log)+ originals = {+ name: Trainer.__dict__[name] for name in ("training_step", "compute_loss", "_generate")+ }+ cp.patch_rl_trainer(Trainer, "grpo_trainer")+ trainer = Trainer(_args(context_parallel_size = 1))+ assert trainer._context_parallel_manager is None+ # Without CP only __init__ is wrapped: the step methods stay the original functions.+ assert all(Trainer.__dict__[name] is fn for name, fn in originals.items())+ trainer.training_step(trainer.model, {})+ assert all(value is None for name, value in log if name != "compute_loss_returned")+ assert trainer._get_train_sampler().batch_size == 8+++# Objective parity: a Llama-shaped decoder (RoPE on global positions, GQA, real Ulysses all-to-all) under the+# RL hooks, scored with TRL's DPO sigmoid loss and GRPO's default dapo loss, against the same model without CP.+_H, _NH, _NKV, _D, _V = 16, 4, 2, 4, 40+++def _rope(x, pos):+ inv = 1.0 / (10000 ** (torch.arange(0, _D, 2, dtype = torch.float64) / _D))+ ang = (pos[..., None].double() * inv)[:, None].to(x.dtype)+ x1, x2 = x[..., ::2], x[..., 1::2]+ return torch.stack(+ [x1 * ang.cos() - x2 * ang.sin(), x1 * ang.sin() + x2 * ang.cos()], -1+ ).flatten(-2)+++def _causal_attention(*, config, context, Q, K, V):+ K = K.repeat_interleave(config.n_groups, 1)+ V = V.repeat_interleave(config.n_groups, 1)+ return torch.nn.functional.scaled_dot_product_attention(Q, K, V, is_causal = True).transpose(1, 2)+++@_dc.dataclass+class _UlyssesConfig:+ n_kv_heads: int+ n_groups: int+++@_dc.dataclass+class _UlyssesContext:+ bsz: int+ q_len: int+ kv_seq_len: int+ n_heads: int+ head_dim: int+ attention_mask: object = None+ causal_mask: object = None+ prefix_seg_info: object = None+++class _Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q = torch.nn.Linear(_H, _NH * _D, bias = False)+ self.kv = torch.nn.Linear(_H, 2 * _NKV * _D, bias = False)+ self.o = torch.nn.Linear(_NH * _D, _H, bias = False)+ self.up = torch.nn.Linear(_H, 2 * _H, bias = False)+ self.down = torch.nn.Linear(2 * _H, _H, bias = False)++ def forward(self, x, pos):+ from unsloth.utils import ulysses++ B, T, _ = x.shape+ q = _rope(self.q(x).view(B, T, _NH, _D).transpose(1, 2), pos)+ k, v = self.kv(x).view(B, T, 2, _NKV, _D).unbind(2)+ k, v = _rope(k.transpose(1, 2), pos), v.transpose(1, 2)+ config = _UlyssesConfig(_NKV, _NH // _NKV)+ manager = cp.get_cp_manager()+ if manager is None:+ a = _causal_attention(config = config, context = None, Q = q, K = k, V = v)+ else:+ a = ulysses.ulysses_attention(+ _causal_attention, config, _UlyssesContext(B, T, T, _NH, _D), q, k, v, manager.group+ )+ x = x + self.o(a.reshape(B, T, -1))+ return x + self.down(torch.nn.functional.silu(self.up(x)))+++class _LlamaLike(torch.nn.Module):+ def __init__(self, seed, dtype):+ super().__init__()+ torch.manual_seed(seed)+ self.model = torch.nn.Module()+ self.model.embed_tokens = torch.nn.Embedding(_V, _H)+ self.model.layers = torch.nn.ModuleList([_Block(), _Block()])+ self.model.forward = self._decode+ self.lm_head = torch.nn.Linear(_H, _V, bias = False)+ with torch.no_grad():+ for p in self.parameters():+ p.normal_(0, 0.4)+ self.to(dtype)++ def _decode(+ self,+ input_ids = None,+ attention_mask = None,+ position_ids = None,+ **kwargs,+ ):+ if position_ids is None:+ position_ids = torch.arange(input_ids.shape[1]).expand_as(input_ids)+ x = self.model.embed_tokens(input_ids)+ for block in self.model.layers:+ x = block(x, position_ids)+ return BaseModelOutputWithPast(last_hidden_state = torch.nn.functional.rms_norm(x, (_H,)))++ def token_logps(self, ids, mask):+ h = self.model(input_ids = ids, attention_mask = mask).last_hidden_state+ return (+ torch.log_softmax(self.lm_head(h)[:, :-1], -1).gather(-1, ids[:, 1:, None]).squeeze(-1)+ )+++def _right_pad(rows):+ ids = torch.zeros(len(rows), max(map(len, rows)), dtype = torch.long)+ mask = torch.zeros_like(ids)+ for i, row in enumerate(rows):+ ids[i, : len(row)], mask[i, : len(row)] = torch.tensor(row), 1+ return ids, mask+++def _dpo_sigmoid(+ policy,+ ref,+ batch,+ world,+ beta = 0.1,+):+ # trl DPOTrainer._compute_loss: completion sums, rows [chosen, rejected], -logsigmoid(beta * delta).mean()+ ids, mask, completion = batch+ logps = (policy.token_logps(ids, mask) * completion).sum(1)+ with torch.no_grad():+ ref_logps = (ref.token_logps(ids, mask) * completion).sum(1)+ (c, r), (rc, rr) = logps.chunk(2), ref_logps.chunk(2)+ return -torch.nn.functional.logsigmoid(beta * ((c - rc) - (r - rr))).mean()+++def _grpo_dapo(+ policy,+ ref,+ batch,+ world,+ beta = 0.04,+ eps = 0.2,+):+ # trl GRPOTrainer (loss_type "dapo", num_iterations 1): token sum over gather(mask.sum()).sum() / num_processes+ ids, mask, completion, rewards = batch+ adv = ((rewards - rewards.mean()) / (rewards.std() + 1e-4)).to(policy.lm_head.weight.dtype)[+ :, None+ ]+ completion = completion.to(adv.dtype)+ counts = [completion.sum().float()]+ if world > 1: # accelerator.gather over every process: CP peers each count the full rows+ counts = [torch.zeros(()) for _ in range(world)]+ dist.all_gather(counts, completion.sum().float())+ normalizer = torch.stack(counts).sum().clamp(min = 1.0) / world+ logps = policy.token_logps(ids, mask)+ with torch.no_grad():+ ref_logps = ref.token_logps(ids, mask)+ ratio = torch.exp(logps - logps.detach())+ loss = -torch.min(ratio * adv, ratio.clamp(1 - eps, 1 + eps) * adv)+ loss = loss + beta * (torch.exp(ref_logps - logps) - (ref_logps - logps) - 1)+ return (loss * completion).sum() / normalizer+++def _objective_worker(rank, world, init_file, queue):+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)+ try:+ torch.set_num_threads(1)+ g = torch.Generator().manual_seed(3)+ rand = lambda n: torch.randint(1, _V, (n,), generator = g).tolist()+ prompts = [rand(7), rand(12)]+ chosen, rejected = (+ [p + rand(n) for p, n in zip(prompts, (9, 5))],+ [p + rand(n) for p, n in zip(prompts, (4, 14))],+ )+ ids, mask = _right_pad(+ chosen + rejected+ ) # 26 tokens: odd row lengths, the pad tail crosses the shard+ completion = mask.clone()+ for i, n in enumerate([7, 12, 7, 12]):+ completion[i, :n] = 0+ dpo = (ids, mask, completion[:, 1:])+ prompt = rand(9)+ g_ids, g_mask = _right_pad([prompt + rand(n) for n in (11, 3, 17, 8)])+ g_completion = g_mask.clone()+ g_completion[:, :9] = 0+ grpo = (g_ids, g_mask, g_completion[:, 1:], torch.tensor([0.3, 1.0, 0.1, 0.65]))+ tolerances = {torch.float64: 1e-12, torch.float32: 1e-6}+ for dtype, tol in tolerances.items():+ for objective, batch in ((_dpo_sigmoid, dpo), (_grpo_dapo, grpo)):+ results = []+ for use_cp in (False, True):+ policy, ref = _LlamaLike(0, dtype), _LlamaLike(1, dtype).requires_grad_(False)+ scope = contextlib.nullcontext()+ if use_cp:+ manager = _manager(rank, world)+ cp.attach_rl_hooks(manager, policy)+ cp.attach_rl_hooks(manager, ref)+ scope = cp._manager_scope(manager)+ with scope:+ loss = objective(policy, ref, batch, world if use_cp else 1)+ loss.backward()+ grads = torch.cat([p.grad.flatten() for p in policy.parameters()])+ if use_cp: # DDP's average over the CP group+ dist.all_reduce(grads)+ grads /= world+ results.append((loss.detach(), grads))+ (loss_ref, grad_ref), (loss_cp, grad_cp) = results+ name = f"{objective.__name__} {dtype}"+ assert abs(float(loss_cp - loss_ref)) <= tol * abs(float(loss_ref)), name+ grad_rel = float((grad_cp - grad_ref).norm() / grad_ref.norm())+ assert grad_rel <= tol, f"{name}: grad rel {grad_rel:.3e}"+ queue.put((rank, "ok"))+ except BaseException: # pragma: no cover - reported to the parent+ import traceback+ queue.put((rank, traceback.format_exc()))+ finally:+ dist.destroy_process_group()+++def test_dpo_and_grpo_objectives_match_without_cp_in_fp64_and_fp32():+ if "fork" not in mp.get_all_start_methods():+ pytest.skip("needs fork")+ ctx = mp.get_context("fork")+ queue = ctx.Queue()+ with tempfile.TemporaryDirectory() as tmp:+ init_file = os.path.join(tmp, "init")+ procs = [+ ctx.Process(target = _objective_worker, args = (r, 2, init_file, queue)) for r in range(2)+ ]+ for p in procs:+ p.start()+ results = dict(queue.get(timeout = 180) for _ in procs)+ for p in procs:+ p.join(30)+ assert results == {0: "ok", 1: "ok"}, results+++# Review follow-ups: evaluation entry checks, pre-CP trainers, split batches, the GRPO DAPO token count.+def _cp_fake_trainer(+ monkeypatch,+ kind,+ log,+ installs = None,+):+ manager = types.SimpleNamespace(size = 2, backend = "ulysses")+ monkeypatch.setattr(cp, "_check_cp_runtime", lambda *a: None)+ monkeypatch.setattr(cp, "_check_cp_trainer", lambda *a: None)+ monkeypatch.setattr(+ cp,+ "_install_accelerator_state",+ lambda trainer: (installs if installs is not None else []).append(trainer),+ )+ monkeypatch.setattr(cp, "ContextParallelManager", lambda size, backend: manager)+ monkeypatch.setattr(cp, "attach_rl_hooks", lambda m, model: None)+ monkeypatch.setattr(cp, "_decoders", lambda model: [])+ Trainer = _fake_trainer_class(log)+ Trainer.predict = lambda self, test_dataset: "predicted"+ cp.patch_rl_trainer(Trainer, kind)+ return Trainer+++def test_grpo_evaluate_checks_generation_groups_of_a_late_eval_dataset(monkeypatch):+ # No eval_dataset at construction, so __init__ cannot check the eval batch; evaluate() must.+ Trainer = _cp_fake_trainer(monkeypatch, "grpo_trainer", [])+ trainer = Trainer(_args(per_device_eval_batch_size = 3))+ trainer.eval_dataset = None+ with pytest.raises(NotImplementedError, match = "multiples of num_generations"):+ trainer.evaluate(eval_dataset = [{"prompt": "p"}] * 3)+ ok = Trainer(_args(per_device_eval_batch_size = 8))+ ok.eval_dataset = None+ assert ok.evaluate(eval_dataset = [{"prompt": "p"}] * 8) == "evaluated"++[email protected]("kind", ["dpo_trainer", "grpo_trainer"])+def test_rl_evaluate_and_predict_refuse_iterable_datasets(monkeypatch, kind):+ class Stream(torch.utils.data.IterableDataset):+ def __iter__(self):+ return iter(range(4))++ Trainer = _cp_fake_trainer(monkeypatch, kind, [])+ trainer = Trainer(_args())+ trainer.eval_dataset = None+ with pytest.raises(NotImplementedError, match = "iterable"):+ trainer.evaluate(eval_dataset = Stream())+ trainer.eval_dataset = {"held_out": Stream()}+ with pytest.raises(NotImplementedError, match = "iterable"):+ trainer.evaluate(eval_dataset = "held_out")+ with pytest.raises(NotImplementedError, match = "iterable"):+ trainer.predict(Stream())+ trainer.eval_dataset = None+ assert trainer.evaluate(eval_dataset = [1, 2, 3, 4]) == "evaluated"+ assert trainer.predict([1, 2, 3, 4]) == "predicted"+++def test_trainers_built_before_cp_restore_the_shared_accelerator_state(monkeypatch):+ # The generated RL trainers bind self.train on the instance in __init__ (rl.py: MethodType(...)):+ # a trainer built before CP was enabled keeps that plain method, so it must still drop the CP mesh+ # a later CP trainer installed on the shared AcceleratorState.+ installs = []+ Trainer = _cp_fake_trainer(monkeypatch, "dpo_trainer", [], installs)++ def bind_train(trainer):+ trainer.train = types.MethodType(type(trainer).train, trainer)++ plain = Trainer(_args(context_parallel_size = 1))+ bind_train(plain)+ cp_trainer = Trainer(_args())+ bind_train(cp_trainer)+ installs.clear()+ assert plain.train() == "trained"+ assert plain in installs, "the pre-CP trainer's bound train() skipped the accelerator restore"+ installs.clear()+ assert cp_trainer.train() == "trained" and cp_trainer in installs+++def test_grpo_refuses_split_batches():+ # split_batches hands each data parallel slice per_device_train_batch_size / D rows, so a slice holds part+ # of a generation group and TRL's gather would group peer copies together.+ trainer = types.SimpleNamespace(+ args = _args(),+ _generate = None,+ eval_dataset = None,+ accelerator = types.SimpleNamespace(split_batches = True),+ )+ with pytest.raises(NotImplementedError, match = "split_batches"):+ cp._check_rl_config(trainer, "grpo_trainer")+ trainer.accelerator.split_batches = False+ cp._check_rl_config(trainer, "grpo_trainer")+++def _dapo_count_worker(rank, world, init_file, queue):+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)+ try:+ manager = _manager(rank, world)+ # Forked and discarded: patch the module directly (no monkeypatch fixture in the child).+ cp._check_cp_runtime = cp._check_cp_trainer = cp._install_accelerator_state = (+ lambda *a: None+ )+ cp.ContextParallelManager = lambda size, backend: manager+ cp.attach_rl_hooks = lambda m, model: None++ def gather(t):+ parts = [torch.empty_like(t) for _ in range(world)]+ dist.all_gather(parts, t)+ return torch.cat(parts)++ # Each rank samples its own completions (rank 0: 2 tokens each, rank 1: 4); peers then adopt rank 0's.+ completion_ids = [[1] * (2 + 2 * rank) for _ in range(4)]+ tool_mask = [[1] * (1 + rank) + [0] * (1 + rank) for _ in range(4)]+ lengths = torch.tensor([len(ids) for ids in completion_ids])+ tool_lengths = torch.tensor([sum(m) for m in tool_mask])+ layouts = {+ # TRL 0.24 / 0.25: prompt_ids, completion_ids, total_completion_tokens, logprobs, extra+ 5: (lambda: (["p"], completion_ids, gather(lengths).sum(), None, {}), 2, 8 * world),+ # TRL 0.27 - 1.0: prompt_ids, completion_ids, tool_mask, completions, count, logprobs, extra+ 7: (+ lambda: (+ ["p"],+ completion_ids,+ tool_mask,+ ["c"],+ gather(tool_lengths).sum(),+ None,+ {},+ ),+ 4,+ 4 * world,+ ),+ # TRL 1.7 - 1.11: the 7 above plus images, tool_images+ 9: (+ lambda: (+ ["p"],+ completion_ids,+ None,+ ["c"],+ gather(lengths).sum(),+ None,+ {},+ None,+ None,+ ),+ 4,+ 8 * world,+ ),+ # TRL >= 1.9 counts after _generate returns: nothing to fix+ 8: (lambda: (["p"], completion_ids, None, ["c"], None, {}, None, None), None, None),+ }+ for size, (generate, index, expected) in layouts.items():+ Trainer = _fake_trainer_class([])+ Trainer._generate = lambda self, prompts, generate = generate: generate()+ cp.patch_rl_trainer(Trainer, "grpo_trainer")+ trainer = Trainer(_args())+ trainer.accelerator = types.SimpleNamespace(device = torch.device("cpu"), gather = gather)+ out = trainer._generate(["p"])+ assert type(out) is tuple and len(out) == size+ assert out[1] == [[1] * 2] * 4 # rank 0's completions on every peer+ if index is not None:+ # TRL's count is gather(lengths).sum() over every process, of the completions actually scored.+ assert int(out[index]) == expected, (size, int(out[index]), expected)+ queue.put((rank, "ok"))+ except BaseException: # pragma: no cover - reported to the parent+ import traceback+ queue.put((rank, traceback.format_exc()))+ finally:+ dist.destroy_process_group()+++def test_grpo_dapo_token_count_follows_the_adopted_completions():+ if "fork" not in mp.get_all_start_methods():+ pytest.skip("needs fork")+ ctx = mp.get_context("fork")+ queue = ctx.Queue()+ with tempfile.TemporaryDirectory() as tmp:+ init_file = os.path.join(tmp, "init")+ procs = [+ ctx.Process(target = _dapo_count_worker, args = (r, 2, init_file, queue)) for r in range(2)+ ]+ for p in procs:+ p.start()+ results = dict(queue.get(timeout = 120) for _ in procs)+ for p in procs:+ p.join(30)+ assert results == {0: "ok", 1: "ok"}, resultsdiff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 8185f2b4f43..c1da4d31727 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -396,8 +396,7 @@ class ContextParallelManager:finally:_cp_options.rotate_method = previous- @contextlib.contextmanager- def _ulysses_scope(self, inputs: dict) -> Iterator[None]:+ def _ulysses_shard(self, inputs: dict) -> None:# Contiguous shards; packed_seq_lengths stays global.rank = self.mesh.get_local_rank()for name in _BUFFER_NAMES:@@ -405,6 +404,10 @@ class ContextParallelManager:if isinstance(tensor, torch.Tensor) and tensor.ndim > 1:local = tensor.shape[1] // self.sizeinputs[name] = tensor[:, rank * local : (rank + 1) * local].contiguous()++ @contextlib.contextmanager+ def _ulysses_scope(self, inputs: dict) -> Iterator[None]:+ self._ulysses_shard(inputs)global _ACTIVE_MANAGERprevious, _ACTIVE_MANAGER = _ACTIVE_MANAGER, self# Some decoders (transformers 5.3 / 5.5) do not forward packed_seq_lengths to linear_attn.@@ -592,6 +595,141 @@ def _restore_parallelism_config(state) -> None:_INSTALLED_PARALLELISM_CONFIG.clear()+def _install_accelerator_state(self):+ manager = getattr(self, "_context_parallel_manager", None)+ accelerator = getattr(self, "accelerator", None)+ state = getattr(accelerator, "state", None)+ if manager is None:+ if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:+ # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.+ state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None+ _INSTALLED_MESH.clear()+ _restore_parallelism_config(state)+ return+ if accelerator is None:+ return+ current = getattr(accelerator.state, "device_mesh", None)+ if _INSTALLED_MESH and current is _INSTALLED_MESH[0]:+ current = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None+ accelerator.state.device_mesh = manager.device_mesh+ _INSTALLED_MESH[:] = [manager.device_mesh, current]+ if getattr(manager, "fsdp2", False):+ _install_fsdp2_parallelism_config(accelerator.state, manager)+ else:+ _restore_parallelism_config(accelerator.state)+ # PEFT makes the Trainer ask DDP for find_unused_parameters, which breaks reentrant checkpointing.+ if getattr(self.args, "ddp_find_unused_parameters", None) in (None, False):+ self.args.ddp_find_unused_parameters = False+ handler = getattr(accelerator, "ddp_handler", None)+ if handler is not None:+ handler.find_unused_parameters = False+++def _check_cp_runtime(self, size: int, backend: str) -> None:+ # train() would swap in a fresh model without the attention hooks or the support check.+ if getattr(self, "model_init", None) is not None:+ raise NotImplementedError("Unsloth: context parallelism does not support model_init.")+ if not _torch_supports_context_parallel():+ raise RuntimeError(+ f"Unsloth: context_parallel_size > 1 needs PyTorch >= 2.7, found {torch.__version__}."+ )+ if not (dist.is_available() and dist.is_initialized()):+ raise RuntimeError(+ "Unsloth: context_parallel_size > 1 needs torch.distributed. "+ f"Launch with torchrun or accelerate launch using a multiple of {size} processes."+ )+ world_size = dist.get_world_size()+ if world_size % size != 0:+ raise RuntimeError(+ f"Unsloth: world size {world_size} is not a multiple of context_parallel_size {size}."+ )+ if (+ backend == "ring"+ and torch.cuda.is_available()+ and torch.cuda.get_device_capability()[0] < 8+ ):+ # Pure torch ring attention fails in backward without flash (2x T4, torch 2.10).+ raise NotImplementedError(+ "Unsloth: context parallelism needs flash attention (compute capability >= 8.0)."+ )+++def _check_cp_model(+ model,+ size: int,+ backend: str,+ allow_hybrids: bool = True,+) -> None:+ model_config = getattr(model, "config", None)+ unsupported = _unsupported_attention(model_config, backend)+ if unsupported is not None:+ raise NotImplementedError(+ f"Unsloth: {backend} context parallelism does not support {unsupported}."+ )+ if backend == "ulysses":+ # Every rank raises here, before any collective.+ text_config = getattr(model_config, "text_config", None) or model_config+ ulysses_head_plan(+ text_config.num_attention_heads,+ getattr(text_config, "num_key_value_heads", None) or text_config.num_attention_heads,+ size,+ )+ if _stateful_mixers(model) and not allow_hybrids:+ raise NotImplementedError(+ "Unsloth: DPO / GRPO context parallelism does not support Gated DeltaNet hybrids yet."+ )+ _check_stateful_mixers(model, backend, size)+ blocker = _hybrid_config_blocker(model) if _stateful_mixers(model) else None+ if blocker is not None:+ raise NotImplementedError(+ f"Unsloth: context parallelism on Gated DeltaNet hybrids does not support {blocker}."+ )+ if not _supports_context_parallel(model, backend):+ raise NotImplementedError(+ "Unsloth: context parallelism currently supports Llama-style attention "+ "(Llama, Qwen2, Gemma), and Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids with Ulysses."+ )+++def _check_cp_trainer(+ self,+ size: int,+ backend: str,+ models,+ allow_fsdp2: bool = False,+) -> bool:+ """Shared trainer checks; returns whether the trainer runs under FSDP2."""+ import accelerate+ from packaging.version import Version++ if Version(accelerate.__version__) < Version("1.10.0"):+ raise NotImplementedError("Unsloth: context parallelism needs accelerate >= 1.10.0.")+ for model in models:+ if model is not None:+ # SFT shards the inputs of the whole model; the RL hooks only shard Llama-style decoders.+ _check_cp_model(model, size, backend, allow_hybrids = allow_fsdp2)+ accelerator = getattr(self, "accelerator", None)+ distributed_type = getattr(getattr(accelerator, "distributed_type", None), "name", "NO")+ fsdp2 = allow_fsdp2 and _is_fsdp2(accelerator, distributed_type)+ if distributed_type != "NO" and not distributed_type.startswith("MULTI_") and not fsdp2:+ if allow_fsdp2:+ raise NotImplementedError(+ f"Unsloth: context parallelism supports DDP and FSDP2 only, not {distributed_type}"+ + (" version 1." if distributed_type == "FSDP" else ".")+ )+ raise NotImplementedError(+ f"Unsloth: DPO / GRPO context parallelism supports DDP only, not {distributed_type}."+ )+ if fsdp2:+ _check_fsdp2_parallelism_config(accelerator)+ if getattr(accelerator, "dispatch_batches", None):+ raise NotImplementedError("Unsloth: context parallelism does not support dispatch_batches.")+ _refuse_iterable_datasets(+ getattr(self, "train_dataset", None), getattr(self, "eval_dataset", None)+ )+ return fsdp2++def patch_sft_trainer(install_cp_methods: bool = False) -> None:# Only __init__ is wrapped up front; the other wrappers go on the class once a trainer enables# CP. With CP off they still slowed torch 2.10 LoRA steps by ~35% (gemma-4-E2B, L4 / A100).@@ -624,35 +762,6 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:if has_total_train_batch_size:original_get_total_train_batch_size = _original("get_total_train_batch_size")- def _install_accelerator_state(self):- manager = getattr(self, "_context_parallel_manager", None)- accelerator = getattr(self, "accelerator", None)- state = getattr(accelerator, "state", None)- if manager is None:- if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:- # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.- state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None- _INSTALLED_MESH.clear()- _restore_parallelism_config(state)- return- if accelerator is None:- return- current = getattr(accelerator.state, "device_mesh", None)- if _INSTALLED_MESH and current is _INSTALLED_MESH[0]:- current = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None- accelerator.state.device_mesh = manager.device_mesh- _INSTALLED_MESH[:] = [manager.device_mesh, current]- if getattr(manager, "fsdp2", False):- _install_fsdp2_parallelism_config(accelerator.state, manager)- else:- _restore_parallelism_config(accelerator.state)- # PEFT makes the Trainer ask DDP for find_unused_parameters, which breaks reentrant checkpointing.- if getattr(self.args, "ddp_find_unused_parameters", None) in (None, False):- self.args.ddp_find_unused_parameters = False- handler = getattr(accelerator, "ddp_handler", None)- if handler is not None:- handler.find_unused_parameters = False-@functools.wraps(original_init)def patched_init(self, *args, **kwargs):config = kwargs["args"] if "args" in kwargs else (args[1] if len(args) > 1 else None)@@ -686,32 +795,7 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]pre_cp_trainers.append(weakref.ref(self))return- # train() would swap in a fresh model without the attention hooks or the support check.- if getattr(self, "model_init", None) is not None:- raise NotImplementedError("Unsloth: context parallelism does not support model_init.")- if not _torch_supports_context_parallel():- raise RuntimeError(- f"Unsloth: context_parallel_size > 1 needs PyTorch >= 2.7, found {torch.__version__}."- )- if not (dist.is_available() and dist.is_initialized()):- raise RuntimeError(- "Unsloth: context_parallel_size > 1 needs torch.distributed. "- f"Launch with torchrun or accelerate launch using a multiple of {size} processes."- )- world_size = dist.get_world_size()- if world_size % size != 0:- raise RuntimeError(- f"Unsloth: world size {world_size} is not a multiple of context_parallel_size {size}."- )- if (- backend == "ring"- and torch.cuda.is_available()- and torch.cuda.get_device_capability()[0] < 8- ):- # Pure torch ring attention fails in backward without flash (2x T4, torch 2.10).- raise NotImplementedError(- "Unsloth: context parallelism needs flash attention (compute capability >= 8.0)."- )+ _check_cp_runtime(self, size, backend)# These recompute loss outside our forward from sharded labels, losing cross-shard targets.if (getattr(self.args, "label_smoothing_factor", 0)@@ -735,53 +819,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:raise NotImplementedError("Unsloth: context parallelism needs average_tokens_across_devices = True.")- import accelerate- from packaging.version import Version-- if Version(accelerate.__version__) < Version("1.10.0"):- raise NotImplementedError("Unsloth: context parallelism needs accelerate >= 1.10.0.")- model_config = getattr(self.model, "config", None)- unsupported = _unsupported_attention(model_config, backend)- if unsupported is not None:- raise NotImplementedError(- f"Unsloth: {backend} context parallelism does not support {unsupported}."- )- if backend == "ulysses":- # Every rank raises here, before any collective.- text_config = getattr(model_config, "text_config", None) or model_config- ulysses_head_plan(- text_config.num_attention_heads,- getattr(text_config, "num_key_value_heads", None)- or text_config.num_attention_heads,- size,- )- _check_stateful_mixers(self.model, backend, size)- blocker = _hybrid_config_blocker(self.model) if _stateful_mixers(self.model) else None- if blocker is not None:- raise NotImplementedError(- f"Unsloth: context parallelism on Gated DeltaNet hybrids does not support {blocker}."- )- if not _supports_context_parallel(self.model, backend):- raise NotImplementedError(- "Unsloth: context parallelism currently supports Llama-style attention "- "(Llama, Qwen2, Gemma), and Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids with Ulysses."- )- accelerator = getattr(self, "accelerator", None)- distributed_type = getattr(getattr(accelerator, "distributed_type", None), "name", "NO")- fsdp2 = _is_fsdp2(accelerator, distributed_type)- if distributed_type != "NO" and not distributed_type.startswith("MULTI_") and not fsdp2:- raise NotImplementedError(- f"Unsloth: context parallelism supports DDP and FSDP2 only, not {distributed_type}"- + (" version 1." if distributed_type == "FSDP" else ".")- )- if fsdp2:- _check_fsdp2_parallelism_config(accelerator)- if getattr(accelerator, "dispatch_batches", None):- raise NotImplementedError(- "Unsloth: context parallelism does not support dispatch_batches."- )- _refuse_iterable_datasets(- getattr(self, "train_dataset", None), getattr(self, "eval_dataset", None)+ fsdp2 = _check_cp_trainer(+ self, size, backend, [getattr(self, "model", None)], allow_fsdp2 = True)manager = ContextParallelManager(size, backend, rotate_method, fsdp2 = fsdp2)self._context_parallel_manager = manager@@ -796,7 +835,7 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:f"Unsloth: Context parallelism enabled with size = {size} ({backend}"+ (f", rotate_method = {rotate_method})" if backend == "ring" else ")")+ (- f", FSDP2 sharding over dp_shard = {world_size // size} x cp = {size}."+ f", FSDP2 sharding over dp_shard = {dist.get_world_size() // size} x cp = {size}."if fsdp2else ".")@@ -956,3 +995,496 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:trainer_cls.__unsloth_context_parallel__ = Trueif install_cp_methods:install_methods()+++# DPO / GRPO: the loss reads per-token logps of whole sequences (sums over completions, group masks),+# so only the decoder is sharded; its last hidden states are gathered back to full length before lm_head.+class _GatherSequence(torch.autograd.Function):+ @staticmethod+ def forward(ctx, x, group, rank, size):+ ctx.group, ctx.rank, ctx.local = group, rank, x.shape[1]+ x = x.contiguous()+ parts = [torch.empty_like(x) for _ in range(size)]+ dist.all_gather(parts, x, group = group)+ return torch.cat(parts, dim = 1)++ @staticmethod+ def backward(ctx, grad):+ # Every peer computes the same loss on the gathered sequence: summing their gradients makes each+ # shard's parameter grads N x its share, and DDP's average over the group then yields one copy.+ grad = grad.clone(memory_format = torch.contiguous_format)+ dist.all_reduce(grad, group = ctx.group)+ return grad.narrow(1, ctx.rank * ctx.local, ctx.local), None, None, None+++def gather_sequence(x, manager):+ return _GatherSequence.apply(x, manager.group, manager.mesh.get_local_rank(), manager.size)+++def _rl_decoder_pre_hook(module, args, kwargs):+ manager = get_cp_manager()+ stack = module._unsloth_cp_rl_stack+ if manager is None or manager is not module._unsloth_cp_rl_manager:+ stack.append(None)+ return None+ if args or kwargs.get("past_key_values") is not None or kwargs.get("input_ids") is None:+ raise NotImplementedError(+ "Unsloth: DPO / GRPO context parallelism shards keyword input_ids training forwards "+ "only (no positional inputs, inputs_embeds or KV cache)."+ )+ for key in ("prefix_seg_info", "output_attentions"):+ if kwargs.get(key):+ raise NotImplementedError(f"Unsloth: context parallelism does not support {key}.")+ batch = {+ name: kwargs.get(name)+ for name in ("input_ids", "attention_mask", "position_ids", "packed_seq_lengths")+ }+ seq_len = batch["input_ids"].shape[1]+ manager._prepare_inputs(batch)+ manager._ulysses_shard(batch)+ for name in ("input_ids", "position_ids"):+ if batch.get(name) is not None:+ # Fresh in the ambient grad / inference mode: a shard view keeps its base's inference flag.+ kwargs[name] = batch[name].clone()+ # Validated right padding needs no mask under Ulysses (attention drops it anyway). Passing the shard+ # would differ per peer (only the last shard holds padding) and recompile the compiled decoder+ # blocks on that rank alone.+ if "attention_mask" in kwargs:+ kwargs["attention_mask"] = None+ stack.append(seq_len)+ return args, kwargs+++def _rl_decoder_post_hook(module, args, kwargs, output):+ seq_len = module._unsloth_cp_rl_stack.pop() if module._unsloth_cp_rl_stack else None+ if seq_len is None:+ return None+ manager = module._unsloth_cp_rl_manager+ last = output[0]++ def _full(x):+ # A fresh tensor, not a view of the padded gather: the compiled log-softmax would otherwise+ # guard on the view's base and recompile under CP only.+ x = gather_sequence(x, manager)+ return x if x.shape[1] == seq_len else x[:, :seq_len].clone()++ hidden = _full(last)+ layers = output.get("hidden_states") if not isinstance(output, tuple) else None+ if isinstance(output, tuple):+ if kwargs.get("output_hidden_states"):+ raise NotImplementedError(+ "Unsloth: context parallelism needs return_dict = True with output_hidden_states."+ )+ return (hidden, *output[1:])+ output[next(iter(output.keys()))] = hidden+ if layers is not None:+ # GRPO's hidden-state forward asks for these and reads the last one (the same tensor as above).+ output["hidden_states"] = tuple(+ hidden if layer is last else _full(layer) for layer in layers+ )+ return output+++def _decoders(model):+ # The text decoder of each CausalLM (its embed_tokens + layers), under any PEFT / DDP wrapping.+ found = []+ for name, module in model.named_modules():+ if hasattr(module, "embed_tokens") and hasattr(module, "layers"):+ if not any(name.startswith(prefix + ".") for prefix, _ in found):+ found.append((name, module))+ return [module for _, module in found]+++def attach_rl_hooks(manager, model) -> None:+ decoders = _decoders(model)+ if not decoders:+ raise NotImplementedError("Unsloth: context parallelism could not find the text decoder.")+ for decoder in decoders:+ decoder._unsloth_cp_rl_manager = manager+ decoder._unsloth_cp_rl_stack = []+ if not getattr(decoder, "_unsloth_cp_rl_hooked", False):+ decoder.register_forward_pre_hook(_rl_decoder_pre_hook, with_kwargs = True)+ decoder.register_forward_hook(_rl_decoder_post_hook, with_kwargs = True)+ decoder._unsloth_cp_rl_hooked = True+ manager.attach_attention_hooks(model)+++def _activate(manager) -> None:+ global _ACTIVE_MANAGER+ _ACTIVE_MANAGER = manager+++def _manager_scope(manager) -> Iterator[None]:+ previous = get_cp_manager()+ _activate(manager)+ try:+ yield+ finally:+ _activate(previous)+++def _map_leaves(+ obj,+ fn,+ leaf = torch.Tensor,+):+ if isinstance(obj, leaf):+ return fn(obj)+ if isinstance(obj, (list, tuple)):+ mapped = [_map_leaves(x, fn, leaf) for x in obj]+ return type(obj)(*mapped) if hasattr(obj, "_fields") else type(obj)(mapped)+ if isinstance(obj, dict):+ return type(obj)((k, _map_leaves(v, fn, leaf)) for k, v in obj.items())+ return obj+++class _OnDevice:+ def __init__(self, tensor):+ self.tensor, self.on_accelerator = tensor.cpu(), tensor.device.type != "cpu"+ self.inference = tensor.is_inference()++ def restore(self, device):+ # Peers get the root's inference-mode flag too: compiled forwards guard on it, so a mismatch+ # recompiles on every peer but the root.+ with torch.inference_mode(self.inference):+ tensor = self.tensor.to(device) if self.on_accelerator else self.tensor+ return tensor.clone() if self.inference and not tensor.is_inference() else tensor+++def _broadcast_from_group_root(obj, manager, device):+ """CP rank 0 sends its object (tensors through CPU) so every peer scores the same samples."""+ payload = [_map_leaves(obj, _OnDevice)]+ group = manager.group+ dist.broadcast_object_list(payload, src = dist.get_global_rank(group, 0), group = group)+ return _map_leaves(+ payload[0],+ lambda x: x.restore(device),+ leaf = _OnDevice,+ )+++def _check_same_prompts(prompts, manager) -> None:+ import hashlib+ import pickle++ digest = hashlib.sha256(pickle.dumps(prompts)).hexdigest()+ digests = [None] * manager.size+ dist.all_gather_object(digests, digest, group = manager.group)+ if len(set(digests)) != 1:+ raise RuntimeError(+ "Unsloth: context parallel peers received different prompts; the dataloader must shard "+ "by data parallel rank only."+ )+++def _grpo_eval_groups_fit(args) -> bool:+ generations = (+ getattr(args, "num_generations_eval", None) or getattr(args, "num_generations", None) or 1+ )+ return args.per_device_eval_batch_size % generations == 0+++def _check_rl_eval_entry(self, kind: str, dataset) -> None:+ # evaluate() / predict() may bring a dataset __init__ never saw.+ _refuse_iterable_datasets(dataset)+ if kind == "grpo_trainer" and dataset is not None and not _grpo_eval_groups_fit(self.args):+ raise NotImplementedError(+ "Unsloth: GRPO context parallelism needs per_device_eval_batch_size to be multiples of "+ "num_generations (num_generations_eval)."+ )+++def _recount_completion_tokens(trainer, result):+ """TRL 0.24 - 1.8 return the DAPO token count from _generate (index 2 of 5 in 0.24 / 0.25, index 4 of+ 7 or 9 in 0.26 - 1.8), gathered from every rank's own completions before CP peers adopted rank 0's:+ recount it from the adopted ones. TRL >= 1.9 returns 8 items and counts after _generate."""+ if not isinstance(result, tuple) or len(result) not in (5, 7, 9):+ return result+ index = 2 if len(result) == 5 else 4+ count = result[index]+ if not (isinstance(count, torch.Tensor) and count.ndim == 0):+ return result+ tool_mask = result[2] if len(result) > 5 else None+ if tool_mask is not None: # TRL counts only model-generated tokens+ local = sum(sum(mask) for mask in tool_mask)+ else:+ local = sum(len(ids) for ids in result[1])+ # As TRL: summed over every process (DAPO later divides by num_processes, which cancels CP copies).+ total = trainer.accelerator.gather(+ torch.tensor([int(local)], device = count.device, dtype = count.dtype)+ ).sum()+ return (*result[:index], total, *result[index + 1 :])+++def _check_rl_config(self, kind: str) -> None:+ args = self.args+ if kind == "dpo_trainer":+ for name in ("padding_free", "precompute_ref_log_probs", "generate_during_eval"):+ if getattr(args, name, False):+ raise NotImplementedError(+ f"Unsloth: DPO context parallelism does not support {name} = True."+ )+ return+ if getattr(self, "use_vllm", False) or getattr(args, "use_vllm", False):+ raise NotImplementedError(+ "Unsloth: GRPO context parallelism does not support vLLM generation "+ "(fast_inference / use_vllm); generate with model.generate instead."+ )+ if getattr(args, "scale_rewards", None) == "batch" or (+ getattr(args, "multi_objective_aggregation", None) == "normalize_then_sum"+ ):+ # Batch-wide reward std over TRL's gather would count each peer's copy (Bessel bias).+ raise NotImplementedError(+ "Unsloth: GRPO context parallelism needs per-group reward scaling "+ "(scale_rewards = 'group' or 'none', multi_objective_aggregation = 'sum_then_normalize')."+ )+ if not hasattr(self, "_generate"):+ raise NotImplementedError("Unsloth: GRPO context parallelism needs TRL >= 0.24.")+ if getattr(getattr(self, "accelerator", None), "split_batches", False):+ # Each data parallel slice would get per_device_train_batch_size / D rows, part of a generation+ # group, and TRL's reward gather would then group peer copies together.+ raise NotImplementedError(+ "Unsloth: GRPO context parallelism does not support split_batches."+ )+ num_generations = getattr(args, "num_generations", None) or 1+ steps = getattr(args, "steps_per_generation", None) or args.gradient_accumulation_steps+ local = args.per_device_train_batch_size * steps+ if local % num_generations or (+ getattr(self, "eval_dataset", None) is not None and not _grpo_eval_groups_fit(args)+ ):+ # Peers duplicate their rank's slice in TRL's global reward gather; a group split across+ # ranks would mix copies into its mean and std.+ raise NotImplementedError(+ "Unsloth: GRPO context parallelism needs per_device_train_batch_size x "+ "steps_per_generation (and per_device_eval_batch_size) to be multiples of num_generations."+ )+++def patch_rl_trainer(trainer_cls, kind: str) -> None:+ """Context parallelism for DPOTrainer / GRPOTrainer training forwards (Ulysses backend)."""+ if trainer_cls.__dict__.get("__unsloth_context_parallel__"):+ return+ original_init = trainer_cls.__init__++ def _original(name):+ if name in trainer_cls.__dict__:+ return trainer_cls.__dict__[name]+ if getattr(trainer_cls, name, None) is None:+ return None++ @functools.wraps(getattr(trainer_cls, name))+ def inherited(self, *args, **kwargs):+ return getattr(super(trainer_cls, self), name)(*args, **kwargs)++ return inherited++ originals = {+ name: _original(name)+ for name in (+ "training_step",+ "prediction_step",+ "train",+ "evaluate",+ "predict",+ "evaluation_loop",+ "compute_loss",+ "_generate",+ "_prepare_inputs",+ "_get_per_token_logps",+ "_get_per_token_logps_and_entropies",+ "_get_train_sampler",+ )+ }+ # GRPO generates (and scores rewards) inside training_step, so it switches CP on only for the+ # logprob forwards and from compute_loss until backward ends; DPO has no unsharded forward.+ activate_at_step = kind != "grpo_trainer"++ @functools.wraps(original_init)+ def patched_init(self, *args, **kwargs):+ config = kwargs.get("args")+ if config is None:+ config = next((x for x in args if hasattr(x, "context_parallel_size")), None)+ try:+ requested = context_parallel_options(config)[0]+ except ValueError:+ requested = 1 # Raised again after the original __init__.+ if requested > 1:+ # Before the original __init__, so instance wrappers made there capture the CP methods.+ install_methods()+ original_init(self, *args, **kwargs)+ self._context_parallel_manager = None+ size, backend, _ = context_parallel_options(getattr(self, "args", None))+ if size <= 1:+ _install_accelerator_state(self)+ if not trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):+ # The generated trainers bind self.train in __init__ (rl.py), around the plain method.+ pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]+ pre_cp_trainers.append(weakref.ref(self))+ return+ if backend != "ulysses":+ raise NotImplementedError(+ "Unsloth: DPO / GRPO context parallelism needs context_parallel_backend = 'ulysses'."+ )+ _check_cp_runtime(self, size, backend)+ _check_rl_config(self, kind)+ ref_model = getattr(self, "ref_model", None)+ _check_cp_trainer(self, size, backend, [self.model, ref_model])+ manager = ContextParallelManager(size, backend)+ self._context_parallel_manager = manager+ attach_rl_hooks(manager, self.model)+ if kind == "grpo_trainer":+ # GRPO's packed and shared-prefix forwards pick their path per step from self-checks;+ # under CP every peer must run the same forwards, so keep the padded path.+ unwrap = getattr(getattr(self, "accelerator", None), "unwrap_model", None)+ for module in (self.model, unwrap(self.model) if unwrap else self.model):+ module._unsloth_seq_packing_grad_ok = False+ module._unsloth_seq_packing_nograd_ok = False+ module._unsloth_prefix_grouper_nograd_disabled = True+ if isinstance(ref_model, torch.nn.Module):+ attach_rl_hooks(manager, ref_model)+ _install_accelerator_state(self)+ print(f"Unsloth: Context parallelism enabled with size = {size} (ulysses, {kind}).")++ def _step_wrapper(name):+ original = originals[name]++ @functools.wraps(original)+ def wrapped(self, *args, **kwargs):+ manager = getattr(self, "_context_parallel_manager", None)+ if manager is None:+ return original(self, *args, **kwargs)+ for decoder in _decoders(self.model):+ decoder._unsloth_cp_rl_stack.clear()+ # Backward, and checkpoint recompute in it, run Ulysses attention too, so the scope spans them.+ with _manager_scope(manager if activate_at_step else None):+ return original(self, *args, **kwargs)++ return wrapped++ def _scoped(name, sticky = False):+ original = originals[name]++ @functools.wraps(original)+ def wrapped(self, *args, **kwargs):+ manager = getattr(self, "_context_parallel_manager", None)+ if manager is None:+ return original(self, *args, **kwargs)+ if sticky:+ # Left on: training_step's scope restores it after backward.+ _activate(manager)+ return original(self, *args, **kwargs)+ with _manager_scope(manager):+ return original(self, *args, **kwargs)++ return wrapped++ def patched_generate(self, prompts, *args, **kwargs):+ manager = getattr(self, "_context_parallel_manager", None)+ if manager is None:+ return originals["_generate"](self, prompts, *args, **kwargs)+ _check_same_prompts(prompts, manager)+ with _manager_scope(None):+ result = originals["_generate"](self, prompts, *args, **kwargs)+ result = _broadcast_from_group_root(result, manager, self.accelerator.device)+ return _recount_completion_tokens(self, result)++ @functools.wraps(originals["_prepare_inputs"])+ def patched_prepare_inputs(self, *args, **kwargs):+ inputs = originals["_prepare_inputs"](self, *args, **kwargs)+ manager = getattr(self, "_context_parallel_manager", None)+ if manager is None:+ return inputs+ # TRL shuffles the scored batch with the process RNG; peers must hold the same rows in the same order.+ return _broadcast_from_group_root(inputs, manager, self.accelerator.device)++ @functools.wraps(originals["_get_train_sampler"])+ def patched_train_sampler(self, *args, **kwargs):+ sampler = originals["_get_train_sampler"](self, *args, **kwargs)+ manager = getattr(self, "_context_parallel_manager", None)+ # TRL sizes a generation batch for every process; CP peers share one data parallel slice.+ if manager is not None and hasattr(sampler, "mini_repeat_count"):+ sampler.batch_size = sampler.batch_size // manager.size+ return sampler++ @functools.wraps(originals["train"])+ def patched_train(self, *args, **kwargs):+ if getattr(self, "_context_parallel_manager", None) is not None:+ _refuse_iterable_datasets(getattr(self, "train_dataset", None))+ _install_accelerator_state(self)+ return originals["train"](self, *args, **kwargs)++ @functools.wraps(originals["evaluate"])+ def patched_evaluate(self, *args, **kwargs):+ if getattr(self, "_context_parallel_manager", None) is not None:+ dataset = args[0] if args else kwargs.get("eval_dataset")+ if dataset is None or isinstance(dataset, str):+ current = getattr(self, "eval_dataset", None)+ dataset = (+ current.get(dataset)+ if isinstance(dataset, str) and isinstance(current, dict)+ else current+ )+ _check_rl_eval_entry(self, kind, dataset)+ _install_accelerator_state(self)+ return originals["evaluate"](self, *args, **kwargs)++ def patched_predict(self, test_dataset, *args, **kwargs):+ if getattr(self, "_context_parallel_manager", None) is not None:+ _check_rl_eval_entry(self, kind, test_dataset)+ _install_accelerator_state(self)+ return originals["predict"](self, test_dataset, *args, **kwargs)++ @functools.wraps(originals["evaluation_loop"])+ def patched_evaluation_loop(self, *args, **kwargs):+ manager = getattr(self, "_context_parallel_manager", None)+ if manager is not None:+ self.gather_function = _cp_gather_for_metrics(self.accelerator, manager.size)+ return originals["evaluation_loop"](self, *args, **kwargs)++ # Non-CP trainers built before CP was enabled: like SFT, repair what their __init__ bound on the+ # instance, so they still drop the mesh a CP trainer installs on the shared AcceleratorState.+ pre_cp_trainers = [] # weakref.ref per trainer++ def _restoring(trainer, bound):+ @functools.wraps(bound)+ def call(*args, **kwargs):+ _install_accelerator_state(trainer)+ return bound(*args, **kwargs)++ return call++ def install_methods():+ # Only __init__ is wrapped up front (as for SFT): the wrappers below go on the class once a+ # trainer enables CP, so trainers without CP keep their original methods.+ if trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):+ return+ for trainer in [ref() for ref in pre_cp_trainers]:+ if trainer is None:+ continue+ for name in ("train", "evaluate", "predict"):+ bound = trainer.__dict__.get(name)+ if bound is not None:+ trainer.__dict__[name] = _restoring(trainer, bound)+ pre_cp_trainers.clear()+ trainer_cls.training_step = _step_wrapper("training_step")+ trainer_cls.prediction_step = _step_wrapper("prediction_step")+ trainer_cls.train = patched_train+ trainer_cls.evaluate = patched_evaluate+ if originals["predict"] is not None:+ trainer_cls.predict = functools.wraps(originals["predict"])(patched_predict)+ trainer_cls.evaluation_loop = patched_evaluation_loop+ if kind == "grpo_trainer":+ trainer_cls.compute_loss = _scoped("compute_loss", sticky = True)+ for name in ("_get_per_token_logps", "_get_per_token_logps_and_entropies"):+ if originals[name] is not None:+ setattr(trainer_cls, name, _scoped(name))+ if originals["_generate"] is not None:+ trainer_cls._generate = functools.wraps(originals["_generate"])(patched_generate)+ trainer_cls._prepare_inputs = patched_prepare_inputs+ if originals["_get_train_sampler"] is not None:+ trainer_cls._get_train_sampler = patched_train_sampler+ trainer_cls.__unsloth_context_parallel_methods__ = True++ trainer_cls.__init__ = patched_init+ trainer_cls.__unsloth_context_parallel__ = Truediff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 7dd73b69332..2412b633b01 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -3431,6 +3431,23 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"\n self.context_parallel_backend = context_parallel_backend""\n self.context_parallel_rotate_method = context_parallel_rotate_method")+ elif (+ trainer_file in ("dpo_trainer", "grpo_trainer") and "context_parallel_size" not in call_args+ ):+ max_seq_length_pre += """+ context_parallel_size : Optional[int] = field(+ default = 1,+ metadata = {'help': 'Ranks per context parallel group (training forwards only). 1 disables it.'},+ )+ context_parallel_backend : Optional[str] = field(+ default = 'ulysses',+ metadata = {'help': 'Context parallel attention backend; preference and RL trainers support ulysses.'},+ )"""+ max_seq_length_call += "context_parallel_size = 1,context_parallel_backend = 'ulysses',"+ max_seq_length_post += (+ "\n self.context_parallel_size = context_parallel_size"+ "\n self.context_parallel_backend = context_parallel_backend"+ )if "output_dir" in call_args:saving_check = (@@ -4014,6 +4031,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):_wrap_grpo_ddp_gradient_sync(getattr(created_module, f"Unsloth{RLTrainer_name}"))except Exception as e:logger.info(f"Unsloth: Could not wrap GRPO DDP gradient sync for {RLTrainer_name}: {e}")+ if trainer_file in ("dpo_trainer", "grpo_trainer"):+ try:+ from ..context_parallel import patch_rl_trainer+ patch_rl_trainer(getattr(created_module, f"Unsloth{RLTrainer_name}"), trainer_file)+ except Exception as e:+ logger.warning(f"Unsloth: Could not add context parallelism to {RLTrainer_name}: {e}")if trainer_file == "ppo_trainer":try:_wrap_ppo_train(getattr(created_module, f"Unsloth{RLTrainer_name}"))