Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel (Ulysses) for DPO and GRPO training (#15041)

Daniel Han1d3e5f56e63f
diff --git a/tests/test_context_parallel_rl.py b/tests/test_context_parallel_rl.py
new file mode 100644
index 00000000000..e7c83179075
--- /dev/null
+++ b/tests/test_context_parallel_rl.py
@@ -0,0 +1,718 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Context parallelism for DPO / GRPO: decoder sharding + hidden-state gather, peer sync, refusals (CPU, gloo)."""
+
+import contextlib
+import dataclasses as _dc
+import os
+import tempfile
+import types
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+from transformers.modeling_outputs import BaseModelOutputWithPast
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+
+
+class _Decoder(torch.nn.Module):
+ """Causal toy decoder: position-dependent, mixes earlier tokens, returns an HF-style output."""
+
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.embed_tokens = torch.nn.Embedding(50, 8)
+ self.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8)])
+
+ def forward(
+ self,
+ input_ids = None,
+ attention_mask = None,
+ position_ids = None,
+ output_hidden_states = False,
+ **kwargs,
+ ):
+ if cp.get_cp_manager() is not None:
+ # Per-shard masks differ across peers (only the last holds padding): the hook must drop it.
+ assert attention_mask is None
+ h = self.embed_tokens(input_ids) * (1 + 0.01 * position_ids[..., None].float())
+ first = h
+ # Sum over earlier tokens of the FULL sequence (stands in for Ulysses attention).
+ manager = cp.get_cp_manager()
+ if manager is not None:
+ rank = dist.get_rank()
+ full = cp.gather_sequence(h, manager)
+ prefix = full.cumsum(1)[:, rank * h.shape[1] : (rank + 1) * h.shape[1]]
+ else:
+ prefix = h.cumsum(1)
+ h = torch.tanh(self.layers[0](prefix))
+ return BaseModelOutputWithPast(
+ last_hidden_state = h, hidden_states = (first, h) if output_hidden_states else None
+ )
+
+
+def _manager(rank, size = 2):
+ manager = object.__new__(cp.ContextParallelManager)
+ manager.size = size
+ manager.backend = "ulysses"
+ manager.mesh = types.SimpleNamespace(
+ get_local_rank = lambda: rank, get_group = lambda: dist.group.WORLD
+ )
+ return manager
+
+
+def _worker(rank, world, init_file, queue):
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
+ try:
+ torch.manual_seed(1)
+ ids = torch.randint(1, 50, (2, 7))
+ mask = torch.ones(2, 7, dtype = torch.long)
+ mask[1, 5:] = 0
+ weight = torch.randn(2, 7, 8)
+
+ reference = _Decoder()
+ full = reference(
+ input_ids = ids, attention_mask = mask, position_ids = torch.arange(7).expand(2, 7)
+ )
+ (full.last_hidden_state * weight).sum().backward()
+
+ model = _Decoder()
+ manager = _manager(rank, world)
+ cp.attach_rl_hooks(manager, torch.nn.ModuleDict({"model": model}))
+ with cp._manager_scope(manager):
+ out = model(input_ids = ids, attention_mask = mask, output_hidden_states = True)
+ # Every peer computes the same loss on the gathered (unpadded) sequence.
+ (out.last_hidden_state * weight).sum().backward()
+ assert out.last_hidden_state.shape == (2, 7, 8)
+ assert out.hidden_states[-1] is out.last_hidden_state
+ assert out.hidden_states[0].shape == (2, 7, 8)
+ # Trimmed from the padded gather as fresh tensors: compiled consumers must not guard on a view base.
+ assert out.last_hidden_state._base is None and out.hidden_states[0]._base is None
+ torch.testing.assert_close(
+ out.last_hidden_state, full.last_hidden_state, rtol = 1e-5, atol = 1e-6
+ )
+ # DDP's average over the CP group, as the trainer gets it.
+ for p in model.parameters():
+ dist.all_reduce(p.grad)
+ p.grad /= world
+ for p, q in zip(model.parameters(), reference.parameters()):
+ torch.testing.assert_close(p.grad, q.grad, rtol = 1e-5, atol = 1e-6)
+
+ # Outside the scope the hooks pass through untouched.
+ plain = model(input_ids = ids, attention_mask = mask, position_ids = torch.arange(7).expand(2, 7))
+ torch.testing.assert_close(plain.last_hidden_state, full.last_hidden_state)
+ assert model._unsloth_cp_rl_stack == []
+
+ # Peers adopt rank 0's object; rank 1's own differs.
+ mine = {"ids": torch.full((2,), rank), "text": [f"r{rank}"], "pair": (rank, None)}
+ got = cp._broadcast_from_group_root(mine, manager, torch.device("cpu"))
+ assert got["text"] == ["r0"] and got["pair"] == (0, None)
+ assert torch.equal(got["ids"], torch.zeros(2, dtype = torch.long))
+ # The root's inference-mode flag reaches every peer (compiled forwards guard on it).
+ with torch.inference_mode():
+ scored = {"ids": torch.full((2,), rank)}
+ got = cp._broadcast_from_group_root(scored, manager, torch.device("cpu"))
+ assert got["ids"].is_inference() and torch.equal(
+ got["ids"], torch.zeros(2, dtype = torch.long)
+ )
+ got = cp._broadcast_from_group_root({"ids": torch.ones(1)}, manager, torch.device("cpu"))
+ assert not got["ids"].is_inference()
+
+ cp._check_same_prompts(["same"], manager)
+ with pytest.raises(RuntimeError, match = "different prompts"):
+ cp._check_same_prompts([f"prompt {rank}"], manager)
+ queue.put((rank, "ok"))
+ except BaseException as error: # pragma: no cover - reported to the parent
+ import traceback
+ queue.put((rank, traceback.format_exc()))
+ finally:
+ dist.destroy_process_group()
+
+
+def test_decoder_shard_gather_matches_full_sequence_with_grads():
+ if "fork" not in mp.get_all_start_methods():
+ pytest.skip("needs fork")
+ ctx = mp.get_context("fork")
+ queue = ctx.Queue()
+ with tempfile.TemporaryDirectory() as tmp:
+ init_file = os.path.join(tmp, "init")
+ procs = [ctx.Process(target = _worker, args = (r, 2, init_file, queue)) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = dict(queue.get(timeout = 120) for _ in procs)
+ for p in procs:
+ p.join(30)
+ assert results == {0: "ok", 1: "ok"}, results
+
+
+def test_decoder_hook_refuses_cache_and_prefix_grouper():
+ model = _Decoder()
+ manager = _manager(0)
+ cp.attach_rl_hooks(manager, torch.nn.ModuleDict({"model": model}))
+ ids = torch.ones(1, 4, dtype = torch.long)
+ with cp._manager_scope(manager):
+ with pytest.raises(NotImplementedError, match = "KV cache"):
+ model(input_ids = ids, past_key_values = ((torch.zeros(1),),))
+ with pytest.raises(NotImplementedError, match = "prefix_seg_info"):
+ model(input_ids = ids, prefix_seg_info = object())
+
+
+def _args(**kw):
+ base = dict(
+ context_parallel_size = 2,
+ context_parallel_backend = "ulysses",
+ per_device_train_batch_size = 4,
+ per_device_eval_batch_size = 4,
+ gradient_accumulation_steps = 1,
+ steps_per_generation = 1,
+ num_generations = 4,
+ )
+ base.update(kw)
+ return types.SimpleNamespace(**base)
+
+
+ "kind, attrs, message",
+ [
+ ("dpo_trainer", {"padding_free": True}, "padding_free"),
+ ("dpo_trainer", {"precompute_ref_log_probs": True}, "precompute_ref_log_probs"),
+ ("dpo_trainer", {"generate_during_eval": True}, "generate_during_eval"),
+ ("grpo_trainer", {"use_vllm": True}, "vLLM"),
+ ("grpo_trainer", {"per_device_train_batch_size": 3}, "multiples of num_generations"),
+ ("grpo_trainer", {"scale_rewards": "batch"}, "per-group reward scaling"),
+ ("grpo_trainer", {"multi_objective_aggregation": "normalize_then_sum"}, "per-group"),
+ ],
+)
+def test_unsupported_rl_configs_are_refused(kind, attrs, message):
+ trainer = types.SimpleNamespace(args = _args(**attrs), _generate = None, eval_dataset = None)
+ with pytest.raises(NotImplementedError, match = message):
+ cp._check_rl_config(trainer, kind)
+
+
+def test_grpo_without_generate_hook_is_refused():
+ trainer = types.SimpleNamespace(args = _args(), eval_dataset = None)
+ with pytest.raises(NotImplementedError, match = "TRL >= 0.24"):
+ cp._check_rl_config(trainer, "grpo_trainer")
+
+
+def _fake_trainer_class(log):
+ class Sampler:
+ mini_repeat_count = 4
+ batch_size = 8
+
+ class Trainer:
+ def __init__(self, args):
+ self.args = args
+ self.model = torch.nn.Linear(1, 1)
+ self.accelerator = types.SimpleNamespace(device = torch.device("cpu"))
+
+ def training_step(self, model, inputs):
+ self._prepare_inputs(inputs)
+ log.append(("compute_loss_returned", self.compute_loss(model, inputs)))
+ log.append(("backward", cp.get_cp_manager()))
+ return 0.0
+
+ def prediction_step(self, model, inputs, prediction_loss_only):
+ log.append(("predict", cp.get_cp_manager()))
+
+ def _prepare_inputs(self, inputs):
+ self._generate(["p"])
+ self._get_per_token_logps_and_entropies()
+ return {"rows": [cp.get_cp_manager() is None]}
+
+ def _generate(self, prompts):
+ log.append(("generate", cp.get_cp_manager()))
+ return (prompts,)
+
+ def _get_per_token_logps_and_entropies(self):
+ log.append(("logps", cp.get_cp_manager()))
+
+ def compute_loss(self, model, inputs):
+ log.append(("loss", cp.get_cp_manager()))
+ return 1.0
+
+ def _get_train_sampler(self, *a):
+ return Sampler()
+
+ def train(self):
+ return "trained"
+
+ def evaluate(self, eval_dataset = None):
+ return "evaluated"
+
+ def evaluation_loop(self):
+ return "looped"
+
+ return Trainer
+
+
[email protected]("kind", ["dpo_trainer", "grpo_trainer"])
+def test_rl_scopes_keep_generation_unsharded(monkeypatch, kind):
+ log = []
+ manager = types.SimpleNamespace(size = 2, backend = "ulysses")
+ monkeypatch.setattr(cp, "_check_cp_runtime", lambda *a: None)
+ monkeypatch.setattr(cp, "_check_cp_trainer", lambda *a: None)
+ monkeypatch.setattr(cp, "_install_accelerator_state", lambda *a: None)
+ monkeypatch.setattr(cp, "ContextParallelManager", lambda size, backend: manager)
+ monkeypatch.setattr(cp, "attach_rl_hooks", lambda m, model: log.append(("hooked", m)))
+ monkeypatch.setattr(cp, "_decoders", lambda model: [])
+ monkeypatch.setattr(cp, "_check_same_prompts", lambda prompts, m: log.append(("same", prompts)))
+ monkeypatch.setattr(cp, "_broadcast_from_group_root", lambda obj, m, device: obj)
+ Trainer = _fake_trainer_class(log)
+ cp.patch_rl_trainer(Trainer, kind)
+ trainer = Trainer(_args())
+ assert trainer._context_parallel_manager is manager
+ assert cp.get_cp_manager() is None
+ trainer.training_step(trainer.model, {})
+ assert cp.get_cp_manager() is None
+ seen = {name: value for name, value in log if name in ("generate", "logps", "loss", "backward")}
+ assert seen["generate"] is (None if kind == "grpo_trainer" else manager)
+ assert seen["logps"] is manager and seen["loss"] is manager and seen["backward"] is manager
+ trainer.prediction_step(trainer.model, {}, True)
+ assert dict(log)["predict"] is (manager if kind == "dpo_trainer" else None)
+ if kind == "grpo_trainer":
+ assert trainer._get_train_sampler().batch_size == 4
+ assert ("same", ["p"]) in log
+
+
+def test_rl_trainers_refuse_ring(monkeypatch):
+ Trainer = _fake_trainer_class([])
+ cp.patch_rl_trainer(Trainer, "dpo_trainer")
+ with pytest.raises(NotImplementedError, match = "ulysses"):
+ Trainer(_args(context_parallel_backend = "ring"))
+
+
+def test_rl_trainers_without_cp_are_untouched(monkeypatch):
+ monkeypatch.setattr(cp, "_install_accelerator_state", lambda *a: None)
+ log = []
+ Trainer = _fake_trainer_class(log)
+ originals = {
+ name: Trainer.__dict__[name] for name in ("training_step", "compute_loss", "_generate")
+ }
+ cp.patch_rl_trainer(Trainer, "grpo_trainer")
+ trainer = Trainer(_args(context_parallel_size = 1))
+ assert trainer._context_parallel_manager is None
+ # Without CP only __init__ is wrapped: the step methods stay the original functions.
+ assert all(Trainer.__dict__[name] is fn for name, fn in originals.items())
+ trainer.training_step(trainer.model, {})
+ assert all(value is None for name, value in log if name != "compute_loss_returned")
+ assert trainer._get_train_sampler().batch_size == 8
+
+
+# Objective parity: a Llama-shaped decoder (RoPE on global positions, GQA, real Ulysses all-to-all) under the
+# RL hooks, scored with TRL's DPO sigmoid loss and GRPO's default dapo loss, against the same model without CP.
+_H, _NH, _NKV, _D, _V = 16, 4, 2, 4, 40
+
+
+def _rope(x, pos):
+ inv = 1.0 / (10000 ** (torch.arange(0, _D, 2, dtype = torch.float64) / _D))
+ ang = (pos[..., None].double() * inv)[:, None].to(x.dtype)
+ x1, x2 = x[..., ::2], x[..., 1::2]
+ return torch.stack(
+ [x1 * ang.cos() - x2 * ang.sin(), x1 * ang.sin() + x2 * ang.cos()], -1
+ ).flatten(-2)
+
+
+def _causal_attention(*, config, context, Q, K, V):
+ K = K.repeat_interleave(config.n_groups, 1)
+ V = V.repeat_interleave(config.n_groups, 1)
+ return torch.nn.functional.scaled_dot_product_attention(Q, K, V, is_causal = True).transpose(1, 2)
+
+
+@_dc.dataclass
+class _UlyssesConfig:
+ n_kv_heads: int
+ n_groups: int
+
+
+@_dc.dataclass
+class _UlyssesContext:
+ bsz: int
+ q_len: int
+ kv_seq_len: int
+ n_heads: int
+ head_dim: int
+ attention_mask: object = None
+ causal_mask: object = None
+ prefix_seg_info: object = None
+
+
+class _Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q = torch.nn.Linear(_H, _NH * _D, bias = False)
+ self.kv = torch.nn.Linear(_H, 2 * _NKV * _D, bias = False)
+ self.o = torch.nn.Linear(_NH * _D, _H, bias = False)
+ self.up = torch.nn.Linear(_H, 2 * _H, bias = False)
+ self.down = torch.nn.Linear(2 * _H, _H, bias = False)
+
+ def forward(self, x, pos):
+ from unsloth.utils import ulysses
+
+ B, T, _ = x.shape
+ q = _rope(self.q(x).view(B, T, _NH, _D).transpose(1, 2), pos)
+ k, v = self.kv(x).view(B, T, 2, _NKV, _D).unbind(2)
+ k, v = _rope(k.transpose(1, 2), pos), v.transpose(1, 2)
+ config = _UlyssesConfig(_NKV, _NH // _NKV)
+ manager = cp.get_cp_manager()
+ if manager is None:
+ a = _causal_attention(config = config, context = None, Q = q, K = k, V = v)
+ else:
+ a = ulysses.ulysses_attention(
+ _causal_attention, config, _UlyssesContext(B, T, T, _NH, _D), q, k, v, manager.group
+ )
+ x = x + self.o(a.reshape(B, T, -1))
+ return x + self.down(torch.nn.functional.silu(self.up(x)))
+
+
+class _LlamaLike(torch.nn.Module):
+ def __init__(self, seed, dtype):
+ super().__init__()
+ torch.manual_seed(seed)
+ self.model = torch.nn.Module()
+ self.model.embed_tokens = torch.nn.Embedding(_V, _H)
+ self.model.layers = torch.nn.ModuleList([_Block(), _Block()])
+ self.model.forward = self._decode
+ self.lm_head = torch.nn.Linear(_H, _V, bias = False)
+ with torch.no_grad():
+ for p in self.parameters():
+ p.normal_(0, 0.4)
+ self.to(dtype)
+
+ def _decode(
+ self,
+ input_ids = None,
+ attention_mask = None,
+ position_ids = None,
+ **kwargs,
+ ):
+ if position_ids is None:
+ position_ids = torch.arange(input_ids.shape[1]).expand_as(input_ids)
+ x = self.model.embed_tokens(input_ids)
+ for block in self.model.layers:
+ x = block(x, position_ids)
+ return BaseModelOutputWithPast(last_hidden_state = torch.nn.functional.rms_norm(x, (_H,)))
+
+ def token_logps(self, ids, mask):
+ h = self.model(input_ids = ids, attention_mask = mask).last_hidden_state
+ return (
+ torch.log_softmax(self.lm_head(h)[:, :-1], -1).gather(-1, ids[:, 1:, None]).squeeze(-1)
+ )
+
+
+def _right_pad(rows):
+ ids = torch.zeros(len(rows), max(map(len, rows)), dtype = torch.long)
+ mask = torch.zeros_like(ids)
+ for i, row in enumerate(rows):
+ ids[i, : len(row)], mask[i, : len(row)] = torch.tensor(row), 1
+ return ids, mask
+
+
+def _dpo_sigmoid(
+ policy,
+ ref,
+ batch,
+ world,
+ beta = 0.1,
+):
+ # trl DPOTrainer._compute_loss: completion sums, rows [chosen, rejected], -logsigmoid(beta * delta).mean()
+ ids, mask, completion = batch
+ logps = (policy.token_logps(ids, mask) * completion).sum(1)
+ with torch.no_grad():
+ ref_logps = (ref.token_logps(ids, mask) * completion).sum(1)
+ (c, r), (rc, rr) = logps.chunk(2), ref_logps.chunk(2)
+ return -torch.nn.functional.logsigmoid(beta * ((c - rc) - (r - rr))).mean()
+
+
+def _grpo_dapo(
+ policy,
+ ref,
+ batch,
+ world,
+ beta = 0.04,
+ eps = 0.2,
+):
+ # trl GRPOTrainer (loss_type "dapo", num_iterations 1): token sum over gather(mask.sum()).sum() / num_processes
+ ids, mask, completion, rewards = batch
+ adv = ((rewards - rewards.mean()) / (rewards.std() + 1e-4)).to(policy.lm_head.weight.dtype)[
+ :, None
+ ]
+ completion = completion.to(adv.dtype)
+ counts = [completion.sum().float()]
+ if world > 1: # accelerator.gather over every process: CP peers each count the full rows
+ counts = [torch.zeros(()) for _ in range(world)]
+ dist.all_gather(counts, completion.sum().float())
+ normalizer = torch.stack(counts).sum().clamp(min = 1.0) / world
+ logps = policy.token_logps(ids, mask)
+ with torch.no_grad():
+ ref_logps = ref.token_logps(ids, mask)
+ ratio = torch.exp(logps - logps.detach())
+ loss = -torch.min(ratio * adv, ratio.clamp(1 - eps, 1 + eps) * adv)
+ loss = loss + beta * (torch.exp(ref_logps - logps) - (ref_logps - logps) - 1)
+ return (loss * completion).sum() / normalizer
+
+
+def _objective_worker(rank, world, init_file, queue):
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
+ try:
+ torch.set_num_threads(1)
+ g = torch.Generator().manual_seed(3)
+ rand = lambda n: torch.randint(1, _V, (n,), generator = g).tolist()
+ prompts = [rand(7), rand(12)]
+ chosen, rejected = (
+ [p + rand(n) for p, n in zip(prompts, (9, 5))],
+ [p + rand(n) for p, n in zip(prompts, (4, 14))],
+ )
+ ids, mask = _right_pad(
+ chosen + rejected
+ ) # 26 tokens: odd row lengths, the pad tail crosses the shard
+ completion = mask.clone()
+ for i, n in enumerate([7, 12, 7, 12]):
+ completion[i, :n] = 0
+ dpo = (ids, mask, completion[:, 1:])
+ prompt = rand(9)
+ g_ids, g_mask = _right_pad([prompt + rand(n) for n in (11, 3, 17, 8)])
+ g_completion = g_mask.clone()
+ g_completion[:, :9] = 0
+ grpo = (g_ids, g_mask, g_completion[:, 1:], torch.tensor([0.3, 1.0, 0.1, 0.65]))
+ tolerances = {torch.float64: 1e-12, torch.float32: 1e-6}
+ for dtype, tol in tolerances.items():
+ for objective, batch in ((_dpo_sigmoid, dpo), (_grpo_dapo, grpo)):
+ results = []
+ for use_cp in (False, True):
+ policy, ref = _LlamaLike(0, dtype), _LlamaLike(1, dtype).requires_grad_(False)
+ scope = contextlib.nullcontext()
+ if use_cp:
+ manager = _manager(rank, world)
+ cp.attach_rl_hooks(manager, policy)
+ cp.attach_rl_hooks(manager, ref)
+ scope = cp._manager_scope(manager)
+ with scope:
+ loss = objective(policy, ref, batch, world if use_cp else 1)
+ loss.backward()
+ grads = torch.cat([p.grad.flatten() for p in policy.parameters()])
+ if use_cp: # DDP's average over the CP group
+ dist.all_reduce(grads)
+ grads /= world
+ results.append((loss.detach(), grads))
+ (loss_ref, grad_ref), (loss_cp, grad_cp) = results
+ name = f"{objective.__name__} {dtype}"
+ assert abs(float(loss_cp - loss_ref)) <= tol * abs(float(loss_ref)), name
+ grad_rel = float((grad_cp - grad_ref).norm() / grad_ref.norm())
+ assert grad_rel <= tol, f"{name}: grad rel {grad_rel:.3e}"
+ queue.put((rank, "ok"))
+ except BaseException: # pragma: no cover - reported to the parent
+ import traceback
+ queue.put((rank, traceback.format_exc()))
+ finally:
+ dist.destroy_process_group()
+
+
+def test_dpo_and_grpo_objectives_match_without_cp_in_fp64_and_fp32():
+ if "fork" not in mp.get_all_start_methods():
+ pytest.skip("needs fork")
+ ctx = mp.get_context("fork")
+ queue = ctx.Queue()
+ with tempfile.TemporaryDirectory() as tmp:
+ init_file = os.path.join(tmp, "init")
+ procs = [
+ ctx.Process(target = _objective_worker, args = (r, 2, init_file, queue)) for r in range(2)
+ ]
+ for p in procs:
+ p.start()
+ results = dict(queue.get(timeout = 180) for _ in procs)
+ for p in procs:
+ p.join(30)
+ assert results == {0: "ok", 1: "ok"}, results
+
+
+# Review follow-ups: evaluation entry checks, pre-CP trainers, split batches, the GRPO DAPO token count.
+def _cp_fake_trainer(
+ monkeypatch,
+ kind,
+ log,
+ installs = None,
+):
+ manager = types.SimpleNamespace(size = 2, backend = "ulysses")
+ monkeypatch.setattr(cp, "_check_cp_runtime", lambda *a: None)
+ monkeypatch.setattr(cp, "_check_cp_trainer", lambda *a: None)
+ monkeypatch.setattr(
+ cp,
+ "_install_accelerator_state",
+ lambda trainer: (installs if installs is not None else []).append(trainer),
+ )
+ monkeypatch.setattr(cp, "ContextParallelManager", lambda size, backend: manager)
+ monkeypatch.setattr(cp, "attach_rl_hooks", lambda m, model: None)
+ monkeypatch.setattr(cp, "_decoders", lambda model: [])
+ Trainer = _fake_trainer_class(log)
+ Trainer.predict = lambda self, test_dataset: "predicted"
+ cp.patch_rl_trainer(Trainer, kind)
+ return Trainer
+
+
+def test_grpo_evaluate_checks_generation_groups_of_a_late_eval_dataset(monkeypatch):
+ # No eval_dataset at construction, so __init__ cannot check the eval batch; evaluate() must.
+ Trainer = _cp_fake_trainer(monkeypatch, "grpo_trainer", [])
+ trainer = Trainer(_args(per_device_eval_batch_size = 3))
+ trainer.eval_dataset = None
+ with pytest.raises(NotImplementedError, match = "multiples of num_generations"):
+ trainer.evaluate(eval_dataset = [{"prompt": "p"}] * 3)
+ ok = Trainer(_args(per_device_eval_batch_size = 8))
+ ok.eval_dataset = None
+ assert ok.evaluate(eval_dataset = [{"prompt": "p"}] * 8) == "evaluated"
+
+
[email protected]("kind", ["dpo_trainer", "grpo_trainer"])
+def test_rl_evaluate_and_predict_refuse_iterable_datasets(monkeypatch, kind):
+ class Stream(torch.utils.data.IterableDataset):
+ def __iter__(self):
+ return iter(range(4))
+
+ Trainer = _cp_fake_trainer(monkeypatch, kind, [])
+ trainer = Trainer(_args())
+ trainer.eval_dataset = None
+ with pytest.raises(NotImplementedError, match = "iterable"):
+ trainer.evaluate(eval_dataset = Stream())
+ trainer.eval_dataset = {"held_out": Stream()}
+ with pytest.raises(NotImplementedError, match = "iterable"):
+ trainer.evaluate(eval_dataset = "held_out")
+ with pytest.raises(NotImplementedError, match = "iterable"):
+ trainer.predict(Stream())
+ trainer.eval_dataset = None
+ assert trainer.evaluate(eval_dataset = [1, 2, 3, 4]) == "evaluated"
+ assert trainer.predict([1, 2, 3, 4]) == "predicted"
+
+
+def test_trainers_built_before_cp_restore_the_shared_accelerator_state(monkeypatch):
+ # The generated RL trainers bind self.train on the instance in __init__ (rl.py: MethodType(...)):
+ # a trainer built before CP was enabled keeps that plain method, so it must still drop the CP mesh
+ # a later CP trainer installed on the shared AcceleratorState.
+ installs = []
+ Trainer = _cp_fake_trainer(monkeypatch, "dpo_trainer", [], installs)
+
+ def bind_train(trainer):
+ trainer.train = types.MethodType(type(trainer).train, trainer)
+
+ plain = Trainer(_args(context_parallel_size = 1))
+ bind_train(plain)
+ cp_trainer = Trainer(_args())
+ bind_train(cp_trainer)
+ installs.clear()
+ assert plain.train() == "trained"
+ assert plain in installs, "the pre-CP trainer's bound train() skipped the accelerator restore"
+ installs.clear()
+ assert cp_trainer.train() == "trained" and cp_trainer in installs
+
+
+def test_grpo_refuses_split_batches():
+ # split_batches hands each data parallel slice per_device_train_batch_size / D rows, so a slice holds part
+ # of a generation group and TRL's gather would group peer copies together.
+ trainer = types.SimpleNamespace(
+ args = _args(),
+ _generate = None,
+ eval_dataset = None,
+ accelerator = types.SimpleNamespace(split_batches = True),
+ )
+ with pytest.raises(NotImplementedError, match = "split_batches"):
+ cp._check_rl_config(trainer, "grpo_trainer")
+ trainer.accelerator.split_batches = False
+ cp._check_rl_config(trainer, "grpo_trainer")
+
+
+def _dapo_count_worker(rank, world, init_file, queue):
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
+ try:
+ manager = _manager(rank, world)
+ # Forked and discarded: patch the module directly (no monkeypatch fixture in the child).
+ cp._check_cp_runtime = cp._check_cp_trainer = cp._install_accelerator_state = (
+ lambda *a: None
+ )
+ cp.ContextParallelManager = lambda size, backend: manager
+ cp.attach_rl_hooks = lambda m, model: None
+
+ def gather(t):
+ parts = [torch.empty_like(t) for _ in range(world)]
+ dist.all_gather(parts, t)
+ return torch.cat(parts)
+
+ # Each rank samples its own completions (rank 0: 2 tokens each, rank 1: 4); peers then adopt rank 0's.
+ completion_ids = [[1] * (2 + 2 * rank) for _ in range(4)]
+ tool_mask = [[1] * (1 + rank) + [0] * (1 + rank) for _ in range(4)]
+ lengths = torch.tensor([len(ids) for ids in completion_ids])
+ tool_lengths = torch.tensor([sum(m) for m in tool_mask])
+ layouts = {
+ # TRL 0.24 / 0.25: prompt_ids, completion_ids, total_completion_tokens, logprobs, extra
+ 5: (lambda: (["p"], completion_ids, gather(lengths).sum(), None, {}), 2, 8 * world),
+ # TRL 0.27 - 1.0: prompt_ids, completion_ids, tool_mask, completions, count, logprobs, extra
+ 7: (
+ lambda: (
+ ["p"],
+ completion_ids,
+ tool_mask,
+ ["c"],
+ gather(tool_lengths).sum(),
+ None,
+ {},
+ ),
+ 4,
+ 4 * world,
+ ),
+ # TRL 1.7 - 1.11: the 7 above plus images, tool_images
+ 9: (
+ lambda: (
+ ["p"],
+ completion_ids,
+ None,
+ ["c"],
+ gather(lengths).sum(),
+ None,
+ {},
+ None,
+ None,
+ ),
+ 4,
+ 8 * world,
+ ),
+ # TRL >= 1.9 counts after _generate returns: nothing to fix
+ 8: (lambda: (["p"], completion_ids, None, ["c"], None, {}, None, None), None, None),
+ }
+ for size, (generate, index, expected) in layouts.items():
+ Trainer = _fake_trainer_class([])
+ Trainer._generate = lambda self, prompts, generate = generate: generate()
+ cp.patch_rl_trainer(Trainer, "grpo_trainer")
+ trainer = Trainer(_args())
+ trainer.accelerator = types.SimpleNamespace(device = torch.device("cpu"), gather = gather)
+ out = trainer._generate(["p"])
+ assert type(out) is tuple and len(out) == size
+ assert out[1] == [[1] * 2] * 4 # rank 0's completions on every peer
+ if index is not None:
+ # TRL's count is gather(lengths).sum() over every process, of the completions actually scored.
+ assert int(out[index]) == expected, (size, int(out[index]), expected)
+ queue.put((rank, "ok"))
+ except BaseException: # pragma: no cover - reported to the parent
+ import traceback
+ queue.put((rank, traceback.format_exc()))
+ finally:
+ dist.destroy_process_group()
+
+
+def test_grpo_dapo_token_count_follows_the_adopted_completions():
+ if "fork" not in mp.get_all_start_methods():
+ pytest.skip("needs fork")
+ ctx = mp.get_context("fork")
+ queue = ctx.Queue()
+ with tempfile.TemporaryDirectory() as tmp:
+ init_file = os.path.join(tmp, "init")
+ procs = [
+ ctx.Process(target = _dapo_count_worker, args = (r, 2, init_file, queue)) for r in range(2)
+ ]
+ for p in procs:
+ p.start()
+ results = dict(queue.get(timeout = 120) for _ in procs)
+ for p in procs:
+ p.join(30)
+ assert results == {0: "ok", 1: "ok"}, results
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 8185f2b4f43..c1da4d31727 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -396,8 +396,7 @@ class ContextParallelManager:
finally:
_cp_options.rotate_method = previous
- @contextlib.contextmanager
- def _ulysses_scope(self, inputs: dict) -> Iterator[None]:
+ def _ulysses_shard(self, inputs: dict) -> None:
# Contiguous shards; packed_seq_lengths stays global.
rank = self.mesh.get_local_rank()
for name in _BUFFER_NAMES:
@@ -405,6 +404,10 @@ class ContextParallelManager:
if isinstance(tensor, torch.Tensor) and tensor.ndim > 1:
local = tensor.shape[1] // self.size
inputs[name] = tensor[:, rank * local : (rank + 1) * local].contiguous()
+
+ @contextlib.contextmanager
+ def _ulysses_scope(self, inputs: dict) -> Iterator[None]:
+ self._ulysses_shard(inputs)
global _ACTIVE_MANAGER
previous, _ACTIVE_MANAGER = _ACTIVE_MANAGER, self
# Some decoders (transformers 5.3 / 5.5) do not forward packed_seq_lengths to linear_attn.
@@ -592,6 +595,141 @@ def _restore_parallelism_config(state) -> None:
_INSTALLED_PARALLELISM_CONFIG.clear()
+def _install_accelerator_state(self):
+ manager = getattr(self, "_context_parallel_manager", None)
+ accelerator = getattr(self, "accelerator", None)
+ state = getattr(accelerator, "state", None)
+ if manager is None:
+ if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:
+ # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.
+ state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None
+ _INSTALLED_MESH.clear()
+ _restore_parallelism_config(state)
+ return
+ if accelerator is None:
+ return
+ current = getattr(accelerator.state, "device_mesh", None)
+ if _INSTALLED_MESH and current is _INSTALLED_MESH[0]:
+ current = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None
+ accelerator.state.device_mesh = manager.device_mesh
+ _INSTALLED_MESH[:] = [manager.device_mesh, current]
+ if getattr(manager, "fsdp2", False):
+ _install_fsdp2_parallelism_config(accelerator.state, manager)
+ else:
+ _restore_parallelism_config(accelerator.state)
+ # PEFT makes the Trainer ask DDP for find_unused_parameters, which breaks reentrant checkpointing.
+ if getattr(self.args, "ddp_find_unused_parameters", None) in (None, False):
+ self.args.ddp_find_unused_parameters = False
+ handler = getattr(accelerator, "ddp_handler", None)
+ if handler is not None:
+ handler.find_unused_parameters = False
+
+
+def _check_cp_runtime(self, size: int, backend: str) -> None:
+ # train() would swap in a fresh model without the attention hooks or the support check.
+ if getattr(self, "model_init", None) is not None:
+ raise NotImplementedError("Unsloth: context parallelism does not support model_init.")
+ if not _torch_supports_context_parallel():
+ raise RuntimeError(
+ f"Unsloth: context_parallel_size > 1 needs PyTorch >= 2.7, found {torch.__version__}."
+ )
+ if not (dist.is_available() and dist.is_initialized()):
+ raise RuntimeError(
+ "Unsloth: context_parallel_size > 1 needs torch.distributed. "
+ f"Launch with torchrun or accelerate launch using a multiple of {size} processes."
+ )
+ world_size = dist.get_world_size()
+ if world_size % size != 0:
+ raise RuntimeError(
+ f"Unsloth: world size {world_size} is not a multiple of context_parallel_size {size}."
+ )
+ if (
+ backend == "ring"
+ and torch.cuda.is_available()
+ and torch.cuda.get_device_capability()[0] < 8
+ ):
+ # Pure torch ring attention fails in backward without flash (2x T4, torch 2.10).
+ raise NotImplementedError(
+ "Unsloth: context parallelism needs flash attention (compute capability >= 8.0)."
+ )
+
+
+def _check_cp_model(
+ model,
+ size: int,
+ backend: str,
+ allow_hybrids: bool = True,
+) -> None:
+ model_config = getattr(model, "config", None)
+ unsupported = _unsupported_attention(model_config, backend)
+ if unsupported is not None:
+ raise NotImplementedError(
+ f"Unsloth: {backend} context parallelism does not support {unsupported}."
+ )
+ if backend == "ulysses":
+ # Every rank raises here, before any collective.
+ text_config = getattr(model_config, "text_config", None) or model_config
+ ulysses_head_plan(
+ text_config.num_attention_heads,
+ getattr(text_config, "num_key_value_heads", None) or text_config.num_attention_heads,
+ size,
+ )
+ if _stateful_mixers(model) and not allow_hybrids:
+ raise NotImplementedError(
+ "Unsloth: DPO / GRPO context parallelism does not support Gated DeltaNet hybrids yet."
+ )
+ _check_stateful_mixers(model, backend, size)
+ blocker = _hybrid_config_blocker(model) if _stateful_mixers(model) else None
+ if blocker is not None:
+ raise NotImplementedError(
+ f"Unsloth: context parallelism on Gated DeltaNet hybrids does not support {blocker}."
+ )
+ if not _supports_context_parallel(model, backend):
+ raise NotImplementedError(
+ "Unsloth: context parallelism currently supports Llama-style attention "
+ "(Llama, Qwen2, Gemma), and Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids with Ulysses."
+ )
+
+
+def _check_cp_trainer(
+ self,
+ size: int,
+ backend: str,
+ models,
+ allow_fsdp2: bool = False,
+) -> bool:
+ """Shared trainer checks; returns whether the trainer runs under FSDP2."""
+ import accelerate
+ from packaging.version import Version
+
+ if Version(accelerate.__version__) < Version("1.10.0"):
+ raise NotImplementedError("Unsloth: context parallelism needs accelerate >= 1.10.0.")
+ for model in models:
+ if model is not None:
+ # SFT shards the inputs of the whole model; the RL hooks only shard Llama-style decoders.
+ _check_cp_model(model, size, backend, allow_hybrids = allow_fsdp2)
+ accelerator = getattr(self, "accelerator", None)
+ distributed_type = getattr(getattr(accelerator, "distributed_type", None), "name", "NO")
+ fsdp2 = allow_fsdp2 and _is_fsdp2(accelerator, distributed_type)
+ if distributed_type != "NO" and not distributed_type.startswith("MULTI_") and not fsdp2:
+ if allow_fsdp2:
+ raise NotImplementedError(
+ f"Unsloth: context parallelism supports DDP and FSDP2 only, not {distributed_type}"
+ + (" version 1." if distributed_type == "FSDP" else ".")
+ )
+ raise NotImplementedError(
+ f"Unsloth: DPO / GRPO context parallelism supports DDP only, not {distributed_type}."
+ )
+ if fsdp2:
+ _check_fsdp2_parallelism_config(accelerator)
+ if getattr(accelerator, "dispatch_batches", None):
+ raise NotImplementedError("Unsloth: context parallelism does not support dispatch_batches.")
+ _refuse_iterable_datasets(
+ getattr(self, "train_dataset", None), getattr(self, "eval_dataset", None)
+ )
+ return fsdp2
+
+
def patch_sft_trainer(install_cp_methods: bool = False) -> None:
# Only __init__ is wrapped up front; the other wrappers go on the class once a trainer enables
# CP. With CP off they still slowed torch 2.10 LoRA steps by ~35% (gemma-4-E2B, L4 / A100).
@@ -624,35 +762,6 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
if has_total_train_batch_size:
original_get_total_train_batch_size = _original("get_total_train_batch_size")
- def _install_accelerator_state(self):
- manager = getattr(self, "_context_parallel_manager", None)
- accelerator = getattr(self, "accelerator", None)
- state = getattr(accelerator, "state", None)
- if manager is None:
- if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:
- # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.
- state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None
- _INSTALLED_MESH.clear()
- _restore_parallelism_config(state)
- return
- if accelerator is None:
- return
- current = getattr(accelerator.state, "device_mesh", None)
- if _INSTALLED_MESH and current is _INSTALLED_MESH[0]:
- current = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None
- accelerator.state.device_mesh = manager.device_mesh
- _INSTALLED_MESH[:] = [manager.device_mesh, current]
- if getattr(manager, "fsdp2", False):
- _install_fsdp2_parallelism_config(accelerator.state, manager)
- else:
- _restore_parallelism_config(accelerator.state)
- # PEFT makes the Trainer ask DDP for find_unused_parameters, which breaks reentrant checkpointing.
- if getattr(self.args, "ddp_find_unused_parameters", None) in (None, False):
- self.args.ddp_find_unused_parameters = False
- handler = getattr(accelerator, "ddp_handler", None)
- if handler is not None:
- handler.find_unused_parameters = False
-
@functools.wraps(original_init)
def patched_init(self, *args, **kwargs):
config = kwargs["args"] if "args" in kwargs else (args[1] if len(args) > 1 else None)
@@ -686,32 +795,7 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]
pre_cp_trainers.append(weakref.ref(self))
return
- # train() would swap in a fresh model without the attention hooks or the support check.
- if getattr(self, "model_init", None) is not None:
- raise NotImplementedError("Unsloth: context parallelism does not support model_init.")
- if not _torch_supports_context_parallel():
- raise RuntimeError(
- f"Unsloth: context_parallel_size > 1 needs PyTorch >= 2.7, found {torch.__version__}."
- )
- if not (dist.is_available() and dist.is_initialized()):
- raise RuntimeError(
- "Unsloth: context_parallel_size > 1 needs torch.distributed. "
- f"Launch with torchrun or accelerate launch using a multiple of {size} processes."
- )
- world_size = dist.get_world_size()
- if world_size % size != 0:
- raise RuntimeError(
- f"Unsloth: world size {world_size} is not a multiple of context_parallel_size {size}."
- )
- if (
- backend == "ring"
- and torch.cuda.is_available()
- and torch.cuda.get_device_capability()[0] < 8
- ):
- # Pure torch ring attention fails in backward without flash (2x T4, torch 2.10).
- raise NotImplementedError(
- "Unsloth: context parallelism needs flash attention (compute capability >= 8.0)."
- )
+ _check_cp_runtime(self, size, backend)
# These recompute loss outside our forward from sharded labels, losing cross-shard targets.
if (
getattr(self.args, "label_smoothing_factor", 0)
@@ -735,53 +819,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
raise NotImplementedError(
"Unsloth: context parallelism needs average_tokens_across_devices = True."
)
- import accelerate
- from packaging.version import Version
-
- if Version(accelerate.__version__) < Version("1.10.0"):
- raise NotImplementedError("Unsloth: context parallelism needs accelerate >= 1.10.0.")
- model_config = getattr(self.model, "config", None)
- unsupported = _unsupported_attention(model_config, backend)
- if unsupported is not None:
- raise NotImplementedError(
- f"Unsloth: {backend} context parallelism does not support {unsupported}."
- )
- if backend == "ulysses":
- # Every rank raises here, before any collective.
- text_config = getattr(model_config, "text_config", None) or model_config
- ulysses_head_plan(
- text_config.num_attention_heads,
- getattr(text_config, "num_key_value_heads", None)
- or text_config.num_attention_heads,
- size,
- )
- _check_stateful_mixers(self.model, backend, size)
- blocker = _hybrid_config_blocker(self.model) if _stateful_mixers(self.model) else None
- if blocker is not None:
- raise NotImplementedError(
- f"Unsloth: context parallelism on Gated DeltaNet hybrids does not support {blocker}."
- )
- if not _supports_context_parallel(self.model, backend):
- raise NotImplementedError(
- "Unsloth: context parallelism currently supports Llama-style attention "
- "(Llama, Qwen2, Gemma), and Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids with Ulysses."
- )
- accelerator = getattr(self, "accelerator", None)
- distributed_type = getattr(getattr(accelerator, "distributed_type", None), "name", "NO")
- fsdp2 = _is_fsdp2(accelerator, distributed_type)
- if distributed_type != "NO" and not distributed_type.startswith("MULTI_") and not fsdp2:
- raise NotImplementedError(
- f"Unsloth: context parallelism supports DDP and FSDP2 only, not {distributed_type}"
- + (" version 1." if distributed_type == "FSDP" else ".")
- )
- if fsdp2:
- _check_fsdp2_parallelism_config(accelerator)
- if getattr(accelerator, "dispatch_batches", None):
- raise NotImplementedError(
- "Unsloth: context parallelism does not support dispatch_batches."
- )
- _refuse_iterable_datasets(
- getattr(self, "train_dataset", None), getattr(self, "eval_dataset", None)
+ fsdp2 = _check_cp_trainer(
+ self, size, backend, [getattr(self, "model", None)], allow_fsdp2 = True
)
manager = ContextParallelManager(size, backend, rotate_method, fsdp2 = fsdp2)
self._context_parallel_manager = manager
@@ -796,7 +835,7 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
f"Unsloth: Context parallelism enabled with size = {size} ({backend}"
+ (f", rotate_method = {rotate_method})" if backend == "ring" else ")")
+ (
- f", FSDP2 sharding over dp_shard = {world_size // size} x cp = {size}."
+ f", FSDP2 sharding over dp_shard = {dist.get_world_size() // size} x cp = {size}."
if fsdp2
else "."
)
@@ -956,3 +995,496 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
trainer_cls.__unsloth_context_parallel__ = True
if install_cp_methods:
install_methods()
+
+
+# DPO / GRPO: the loss reads per-token logps of whole sequences (sums over completions, group masks),
+# so only the decoder is sharded; its last hidden states are gathered back to full length before lm_head.
+class _GatherSequence(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x, group, rank, size):
+ ctx.group, ctx.rank, ctx.local = group, rank, x.shape[1]
+ x = x.contiguous()
+ parts = [torch.empty_like(x) for _ in range(size)]
+ dist.all_gather(parts, x, group = group)
+ return torch.cat(parts, dim = 1)
+
+ @staticmethod
+ def backward(ctx, grad):
+ # Every peer computes the same loss on the gathered sequence: summing their gradients makes each
+ # shard's parameter grads N x its share, and DDP's average over the group then yields one copy.
+ grad = grad.clone(memory_format = torch.contiguous_format)
+ dist.all_reduce(grad, group = ctx.group)
+ return grad.narrow(1, ctx.rank * ctx.local, ctx.local), None, None, None
+
+
+def gather_sequence(x, manager):
+ return _GatherSequence.apply(x, manager.group, manager.mesh.get_local_rank(), manager.size)
+
+
+def _rl_decoder_pre_hook(module, args, kwargs):
+ manager = get_cp_manager()
+ stack = module._unsloth_cp_rl_stack
+ if manager is None or manager is not module._unsloth_cp_rl_manager:
+ stack.append(None)
+ return None
+ if args or kwargs.get("past_key_values") is not None or kwargs.get("input_ids") is None:
+ raise NotImplementedError(
+ "Unsloth: DPO / GRPO context parallelism shards keyword input_ids training forwards "
+ "only (no positional inputs, inputs_embeds or KV cache)."
+ )
+ for key in ("prefix_seg_info", "output_attentions"):
+ if kwargs.get(key):
+ raise NotImplementedError(f"Unsloth: context parallelism does not support {key}.")
+ batch = {
+ name: kwargs.get(name)
+ for name in ("input_ids", "attention_mask", "position_ids", "packed_seq_lengths")
+ }
+ seq_len = batch["input_ids"].shape[1]
+ manager._prepare_inputs(batch)
+ manager._ulysses_shard(batch)
+ for name in ("input_ids", "position_ids"):
+ if batch.get(name) is not None:
+ # Fresh in the ambient grad / inference mode: a shard view keeps its base's inference flag.
+ kwargs[name] = batch[name].clone()
+ # Validated right padding needs no mask under Ulysses (attention drops it anyway). Passing the shard
+ # would differ per peer (only the last shard holds padding) and recompile the compiled decoder
+ # blocks on that rank alone.
+ if "attention_mask" in kwargs:
+ kwargs["attention_mask"] = None
+ stack.append(seq_len)
+ return args, kwargs
+
+
+def _rl_decoder_post_hook(module, args, kwargs, output):
+ seq_len = module._unsloth_cp_rl_stack.pop() if module._unsloth_cp_rl_stack else None
+ if seq_len is None:
+ return None
+ manager = module._unsloth_cp_rl_manager
+ last = output[0]
+
+ def _full(x):
+ # A fresh tensor, not a view of the padded gather: the compiled log-softmax would otherwise
+ # guard on the view's base and recompile under CP only.
+ x = gather_sequence(x, manager)
+ return x if x.shape[1] == seq_len else x[:, :seq_len].clone()
+
+ hidden = _full(last)
+ layers = output.get("hidden_states") if not isinstance(output, tuple) else None
+ if isinstance(output, tuple):
+ if kwargs.get("output_hidden_states"):
+ raise NotImplementedError(
+ "Unsloth: context parallelism needs return_dict = True with output_hidden_states."
+ )
+ return (hidden, *output[1:])
+ output[next(iter(output.keys()))] = hidden
+ if layers is not None:
+ # GRPO's hidden-state forward asks for these and reads the last one (the same tensor as above).
+ output["hidden_states"] = tuple(
+ hidden if layer is last else _full(layer) for layer in layers
+ )
+ return output
+
+
+def _decoders(model):
+ # The text decoder of each CausalLM (its embed_tokens + layers), under any PEFT / DDP wrapping.
+ found = []
+ for name, module in model.named_modules():
+ if hasattr(module, "embed_tokens") and hasattr(module, "layers"):
+ if not any(name.startswith(prefix + ".") for prefix, _ in found):
+ found.append((name, module))
+ return [module for _, module in found]
+
+
+def attach_rl_hooks(manager, model) -> None:
+ decoders = _decoders(model)
+ if not decoders:
+ raise NotImplementedError("Unsloth: context parallelism could not find the text decoder.")
+ for decoder in decoders:
+ decoder._unsloth_cp_rl_manager = manager
+ decoder._unsloth_cp_rl_stack = []
+ if not getattr(decoder, "_unsloth_cp_rl_hooked", False):
+ decoder.register_forward_pre_hook(_rl_decoder_pre_hook, with_kwargs = True)
+ decoder.register_forward_hook(_rl_decoder_post_hook, with_kwargs = True)
+ decoder._unsloth_cp_rl_hooked = True
+ manager.attach_attention_hooks(model)
+
+
+def _activate(manager) -> None:
+ global _ACTIVE_MANAGER
+ _ACTIVE_MANAGER = manager
+
+
+def _manager_scope(manager) -> Iterator[None]:
+ previous = get_cp_manager()
+ _activate(manager)
+ try:
+ yield
+ finally:
+ _activate(previous)
+
+
+def _map_leaves(
+ obj,
+ fn,
+ leaf = torch.Tensor,
+):
+ if isinstance(obj, leaf):
+ return fn(obj)
+ if isinstance(obj, (list, tuple)):
+ mapped = [_map_leaves(x, fn, leaf) for x in obj]
+ return type(obj)(*mapped) if hasattr(obj, "_fields") else type(obj)(mapped)
+ if isinstance(obj, dict):
+ return type(obj)((k, _map_leaves(v, fn, leaf)) for k, v in obj.items())
+ return obj
+
+
+class _OnDevice:
+ def __init__(self, tensor):
+ self.tensor, self.on_accelerator = tensor.cpu(), tensor.device.type != "cpu"
+ self.inference = tensor.is_inference()
+
+ def restore(self, device):
+ # Peers get the root's inference-mode flag too: compiled forwards guard on it, so a mismatch
+ # recompiles on every peer but the root.
+ with torch.inference_mode(self.inference):
+ tensor = self.tensor.to(device) if self.on_accelerator else self.tensor
+ return tensor.clone() if self.inference and not tensor.is_inference() else tensor
+
+
+def _broadcast_from_group_root(obj, manager, device):
+ """CP rank 0 sends its object (tensors through CPU) so every peer scores the same samples."""
+ payload = [_map_leaves(obj, _OnDevice)]
+ group = manager.group
+ dist.broadcast_object_list(payload, src = dist.get_global_rank(group, 0), group = group)
+ return _map_leaves(
+ payload[0],
+ lambda x: x.restore(device),
+ leaf = _OnDevice,
+ )
+
+
+def _check_same_prompts(prompts, manager) -> None:
+ import hashlib
+ import pickle
+
+ digest = hashlib.sha256(pickle.dumps(prompts)).hexdigest()
+ digests = [None] * manager.size
+ dist.all_gather_object(digests, digest, group = manager.group)
+ if len(set(digests)) != 1:
+ raise RuntimeError(
+ "Unsloth: context parallel peers received different prompts; the dataloader must shard "
+ "by data parallel rank only."
+ )
+
+
+def _grpo_eval_groups_fit(args) -> bool:
+ generations = (
+ getattr(args, "num_generations_eval", None) or getattr(args, "num_generations", None) or 1
+ )
+ return args.per_device_eval_batch_size % generations == 0
+
+
+def _check_rl_eval_entry(self, kind: str, dataset) -> None:
+ # evaluate() / predict() may bring a dataset __init__ never saw.
+ _refuse_iterable_datasets(dataset)
+ if kind == "grpo_trainer" and dataset is not None and not _grpo_eval_groups_fit(self.args):
+ raise NotImplementedError(
+ "Unsloth: GRPO context parallelism needs per_device_eval_batch_size to be multiples of "
+ "num_generations (num_generations_eval)."
+ )
+
+
+def _recount_completion_tokens(trainer, result):
+ """TRL 0.24 - 1.8 return the DAPO token count from _generate (index 2 of 5 in 0.24 / 0.25, index 4 of
+ 7 or 9 in 0.26 - 1.8), gathered from every rank's own completions before CP peers adopted rank 0's:
+ recount it from the adopted ones. TRL >= 1.9 returns 8 items and counts after _generate."""
+ if not isinstance(result, tuple) or len(result) not in (5, 7, 9):
+ return result
+ index = 2 if len(result) == 5 else 4
+ count = result[index]
+ if not (isinstance(count, torch.Tensor) and count.ndim == 0):
+ return result
+ tool_mask = result[2] if len(result) > 5 else None
+ if tool_mask is not None: # TRL counts only model-generated tokens
+ local = sum(sum(mask) for mask in tool_mask)
+ else:
+ local = sum(len(ids) for ids in result[1])
+ # As TRL: summed over every process (DAPO later divides by num_processes, which cancels CP copies).
+ total = trainer.accelerator.gather(
+ torch.tensor([int(local)], device = count.device, dtype = count.dtype)
+ ).sum()
+ return (*result[:index], total, *result[index + 1 :])
+
+
+def _check_rl_config(self, kind: str) -> None:
+ args = self.args
+ if kind == "dpo_trainer":
+ for name in ("padding_free", "precompute_ref_log_probs", "generate_during_eval"):
+ if getattr(args, name, False):
+ raise NotImplementedError(
+ f"Unsloth: DPO context parallelism does not support {name} = True."
+ )
+ return
+ if getattr(self, "use_vllm", False) or getattr(args, "use_vllm", False):
+ raise NotImplementedError(
+ "Unsloth: GRPO context parallelism does not support vLLM generation "
+ "(fast_inference / use_vllm); generate with model.generate instead."
+ )
+ if getattr(args, "scale_rewards", None) == "batch" or (
+ getattr(args, "multi_objective_aggregation", None) == "normalize_then_sum"
+ ):
+ # Batch-wide reward std over TRL's gather would count each peer's copy (Bessel bias).
+ raise NotImplementedError(
+ "Unsloth: GRPO context parallelism needs per-group reward scaling "
+ "(scale_rewards = 'group' or 'none', multi_objective_aggregation = 'sum_then_normalize')."
+ )
+ if not hasattr(self, "_generate"):
+ raise NotImplementedError("Unsloth: GRPO context parallelism needs TRL >= 0.24.")
+ if getattr(getattr(self, "accelerator", None), "split_batches", False):
+ # Each data parallel slice would get per_device_train_batch_size / D rows, part of a generation
+ # group, and TRL's reward gather would then group peer copies together.
+ raise NotImplementedError(
+ "Unsloth: GRPO context parallelism does not support split_batches."
+ )
+ num_generations = getattr(args, "num_generations", None) or 1
+ steps = getattr(args, "steps_per_generation", None) or args.gradient_accumulation_steps
+ local = args.per_device_train_batch_size * steps
+ if local % num_generations or (
+ getattr(self, "eval_dataset", None) is not None and not _grpo_eval_groups_fit(args)
+ ):
+ # Peers duplicate their rank's slice in TRL's global reward gather; a group split across
+ # ranks would mix copies into its mean and std.
+ raise NotImplementedError(
+ "Unsloth: GRPO context parallelism needs per_device_train_batch_size x "
+ "steps_per_generation (and per_device_eval_batch_size) to be multiples of num_generations."
+ )
+
+
+def patch_rl_trainer(trainer_cls, kind: str) -> None:
+ """Context parallelism for DPOTrainer / GRPOTrainer training forwards (Ulysses backend)."""
+ if trainer_cls.__dict__.get("__unsloth_context_parallel__"):
+ return
+ original_init = trainer_cls.__init__
+
+ def _original(name):
+ if name in trainer_cls.__dict__:
+ return trainer_cls.__dict__[name]
+ if getattr(trainer_cls, name, None) is None:
+ return None
+
+ @functools.wraps(getattr(trainer_cls, name))
+ def inherited(self, *args, **kwargs):
+ return getattr(super(trainer_cls, self), name)(*args, **kwargs)
+
+ return inherited
+
+ originals = {
+ name: _original(name)
+ for name in (
+ "training_step",
+ "prediction_step",
+ "train",
+ "evaluate",
+ "predict",
+ "evaluation_loop",
+ "compute_loss",
+ "_generate",
+ "_prepare_inputs",
+ "_get_per_token_logps",
+ "_get_per_token_logps_and_entropies",
+ "_get_train_sampler",
+ )
+ }
+ # GRPO generates (and scores rewards) inside training_step, so it switches CP on only for the
+ # logprob forwards and from compute_loss until backward ends; DPO has no unsharded forward.
+ activate_at_step = kind != "grpo_trainer"
+
+ @functools.wraps(original_init)
+ def patched_init(self, *args, **kwargs):
+ config = kwargs.get("args")
+ if config is None:
+ config = next((x for x in args if hasattr(x, "context_parallel_size")), None)
+ try:
+ requested = context_parallel_options(config)[0]
+ except ValueError:
+ requested = 1 # Raised again after the original __init__.
+ if requested > 1:
+ # Before the original __init__, so instance wrappers made there capture the CP methods.
+ install_methods()
+ original_init(self, *args, **kwargs)
+ self._context_parallel_manager = None
+ size, backend, _ = context_parallel_options(getattr(self, "args", None))
+ if size <= 1:
+ _install_accelerator_state(self)
+ if not trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):
+ # The generated trainers bind self.train in __init__ (rl.py), around the plain method.
+ pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]
+ pre_cp_trainers.append(weakref.ref(self))
+ return
+ if backend != "ulysses":
+ raise NotImplementedError(
+ "Unsloth: DPO / GRPO context parallelism needs context_parallel_backend = 'ulysses'."
+ )
+ _check_cp_runtime(self, size, backend)
+ _check_rl_config(self, kind)
+ ref_model = getattr(self, "ref_model", None)
+ _check_cp_trainer(self, size, backend, [self.model, ref_model])
+ manager = ContextParallelManager(size, backend)
+ self._context_parallel_manager = manager
+ attach_rl_hooks(manager, self.model)
+ if kind == "grpo_trainer":
+ # GRPO's packed and shared-prefix forwards pick their path per step from self-checks;
+ # under CP every peer must run the same forwards, so keep the padded path.
+ unwrap = getattr(getattr(self, "accelerator", None), "unwrap_model", None)
+ for module in (self.model, unwrap(self.model) if unwrap else self.model):
+ module._unsloth_seq_packing_grad_ok = False
+ module._unsloth_seq_packing_nograd_ok = False
+ module._unsloth_prefix_grouper_nograd_disabled = True
+ if isinstance(ref_model, torch.nn.Module):
+ attach_rl_hooks(manager, ref_model)
+ _install_accelerator_state(self)
+ print(f"Unsloth: Context parallelism enabled with size = {size} (ulysses, {kind}).")
+
+ def _step_wrapper(name):
+ original = originals[name]
+
+ @functools.wraps(original)
+ def wrapped(self, *args, **kwargs):
+ manager = getattr(self, "_context_parallel_manager", None)
+ if manager is None:
+ return original(self, *args, **kwargs)
+ for decoder in _decoders(self.model):
+ decoder._unsloth_cp_rl_stack.clear()
+ # Backward, and checkpoint recompute in it, run Ulysses attention too, so the scope spans them.
+ with _manager_scope(manager if activate_at_step else None):
+ return original(self, *args, **kwargs)
+
+ return wrapped
+
+ def _scoped(name, sticky = False):
+ original = originals[name]
+
+ @functools.wraps(original)
+ def wrapped(self, *args, **kwargs):
+ manager = getattr(self, "_context_parallel_manager", None)
+ if manager is None:
+ return original(self, *args, **kwargs)
+ if sticky:
+ # Left on: training_step's scope restores it after backward.
+ _activate(manager)
+ return original(self, *args, **kwargs)
+ with _manager_scope(manager):
+ return original(self, *args, **kwargs)
+
+ return wrapped
+
+ def patched_generate(self, prompts, *args, **kwargs):
+ manager = getattr(self, "_context_parallel_manager", None)
+ if manager is None:
+ return originals["_generate"](self, prompts, *args, **kwargs)
+ _check_same_prompts(prompts, manager)
+ with _manager_scope(None):
+ result = originals["_generate"](self, prompts, *args, **kwargs)
+ result = _broadcast_from_group_root(result, manager, self.accelerator.device)
+ return _recount_completion_tokens(self, result)
+
+ @functools.wraps(originals["_prepare_inputs"])
+ def patched_prepare_inputs(self, *args, **kwargs):
+ inputs = originals["_prepare_inputs"](self, *args, **kwargs)
+ manager = getattr(self, "_context_parallel_manager", None)
+ if manager is None:
+ return inputs
+ # TRL shuffles the scored batch with the process RNG; peers must hold the same rows in the same order.
+ return _broadcast_from_group_root(inputs, manager, self.accelerator.device)
+
+ @functools.wraps(originals["_get_train_sampler"])
+ def patched_train_sampler(self, *args, **kwargs):
+ sampler = originals["_get_train_sampler"](self, *args, **kwargs)
+ manager = getattr(self, "_context_parallel_manager", None)
+ # TRL sizes a generation batch for every process; CP peers share one data parallel slice.
+ if manager is not None and hasattr(sampler, "mini_repeat_count"):
+ sampler.batch_size = sampler.batch_size // manager.size
+ return sampler
+
+ @functools.wraps(originals["train"])
+ def patched_train(self, *args, **kwargs):
+ if getattr(self, "_context_parallel_manager", None) is not None:
+ _refuse_iterable_datasets(getattr(self, "train_dataset", None))
+ _install_accelerator_state(self)
+ return originals["train"](self, *args, **kwargs)
+
+ @functools.wraps(originals["evaluate"])
+ def patched_evaluate(self, *args, **kwargs):
+ if getattr(self, "_context_parallel_manager", None) is not None:
+ dataset = args[0] if args else kwargs.get("eval_dataset")
+ if dataset is None or isinstance(dataset, str):
+ current = getattr(self, "eval_dataset", None)
+ dataset = (
+ current.get(dataset)
+ if isinstance(dataset, str) and isinstance(current, dict)
+ else current
+ )
+ _check_rl_eval_entry(self, kind, dataset)
+ _install_accelerator_state(self)
+ return originals["evaluate"](self, *args, **kwargs)
+
+ def patched_predict(self, test_dataset, *args, **kwargs):
+ if getattr(self, "_context_parallel_manager", None) is not None:
+ _check_rl_eval_entry(self, kind, test_dataset)
+ _install_accelerator_state(self)
+ return originals["predict"](self, test_dataset, *args, **kwargs)
+
+ @functools.wraps(originals["evaluation_loop"])
+ def patched_evaluation_loop(self, *args, **kwargs):
+ manager = getattr(self, "_context_parallel_manager", None)
+ if manager is not None:
+ self.gather_function = _cp_gather_for_metrics(self.accelerator, manager.size)
+ return originals["evaluation_loop"](self, *args, **kwargs)
+
+ # Non-CP trainers built before CP was enabled: like SFT, repair what their __init__ bound on the
+ # instance, so they still drop the mesh a CP trainer installs on the shared AcceleratorState.
+ pre_cp_trainers = [] # weakref.ref per trainer
+
+ def _restoring(trainer, bound):
+ @functools.wraps(bound)
+ def call(*args, **kwargs):
+ _install_accelerator_state(trainer)
+ return bound(*args, **kwargs)
+
+ return call
+
+ def install_methods():
+ # Only __init__ is wrapped up front (as for SFT): the wrappers below go on the class once a
+ # trainer enables CP, so trainers without CP keep their original methods.
+ if trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):
+ return
+ for trainer in [ref() for ref in pre_cp_trainers]:
+ if trainer is None:
+ continue
+ for name in ("train", "evaluate", "predict"):
+ bound = trainer.__dict__.get(name)
+ if bound is not None:
+ trainer.__dict__[name] = _restoring(trainer, bound)
+ pre_cp_trainers.clear()
+ trainer_cls.training_step = _step_wrapper("training_step")
+ trainer_cls.prediction_step = _step_wrapper("prediction_step")
+ trainer_cls.train = patched_train
+ trainer_cls.evaluate = patched_evaluate
+ if originals["predict"] is not None:
+ trainer_cls.predict = functools.wraps(originals["predict"])(patched_predict)
+ trainer_cls.evaluation_loop = patched_evaluation_loop
+ if kind == "grpo_trainer":
+ trainer_cls.compute_loss = _scoped("compute_loss", sticky = True)
+ for name in ("_get_per_token_logps", "_get_per_token_logps_and_entropies"):
+ if originals[name] is not None:
+ setattr(trainer_cls, name, _scoped(name))
+ if originals["_generate"] is not None:
+ trainer_cls._generate = functools.wraps(originals["_generate"])(patched_generate)
+ trainer_cls._prepare_inputs = patched_prepare_inputs
+ if originals["_get_train_sampler"] is not None:
+ trainer_cls._get_train_sampler = patched_train_sampler
+ trainer_cls.__unsloth_context_parallel_methods__ = True
+
+ trainer_cls.__init__ = patched_init
+ trainer_cls.__unsloth_context_parallel__ = True
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 7dd73b69332..2412b633b01 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -3431,6 +3431,23 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"\n self.context_parallel_backend = context_parallel_backend"
"\n self.context_parallel_rotate_method = context_parallel_rotate_method"
)
+ elif (
+ trainer_file in ("dpo_trainer", "grpo_trainer") and "context_parallel_size" not in call_args
+ ):
+ max_seq_length_pre += """
+ context_parallel_size : Optional[int] = field(
+ default = 1,
+ metadata = {'help': 'Ranks per context parallel group (training forwards only). 1 disables it.'},
+ )
+ context_parallel_backend : Optional[str] = field(
+ default = 'ulysses',
+ metadata = {'help': 'Context parallel attention backend; preference and RL trainers support ulysses.'},
+ )"""
+ max_seq_length_call += "context_parallel_size = 1,context_parallel_backend = 'ulysses',"
+ max_seq_length_post += (
+ "\n self.context_parallel_size = context_parallel_size"
+ "\n self.context_parallel_backend = context_parallel_backend"
+ )
if "output_dir" in call_args:
saving_check = (
@@ -4014,6 +4031,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
_wrap_grpo_ddp_gradient_sync(getattr(created_module, f"Unsloth{RLTrainer_name}"))
except Exception as e:
logger.info(f"Unsloth: Could not wrap GRPO DDP gradient sync for {RLTrainer_name}: {e}")
+ if trainer_file in ("dpo_trainer", "grpo_trainer"):
+ try:
+ from ..context_parallel import patch_rl_trainer
+ patch_rl_trainer(getattr(created_module, f"Unsloth{RLTrainer_name}"), trainer_file)
+ except Exception as e:
+ logger.warning(f"Unsloth: Could not add context parallelism to {RLTrainer_name}: {e}")
if trainer_file == "ppo_trainer":
try:
_wrap_ppo_train(getattr(created_module, f"Unsloth{RLTrainer_name}"))