Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallel: Ulysses backend with packing and varlen attention (#13848)
Daniel Hanace96a57aa7c
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.pyindex 4bc322af359..d2826794232 100644--- a/tests/test_context_parallel.py+++ b/tests/test_context_parallel.py@@ -544,7 +544,10 @@ def test_find_unused_parameters_is_off_under_cp_unless_the_user_set_it(monkeypatcp,"ContextParallelManager",lambda size, *rest: types.SimpleNamespace(- size = size, device_mesh = object(), attach_attention_hooks = lambda model: None+ size = size,+ backend = "ring",+ device_mesh = object(),+ attach_attention_hooks = lambda model: None,),)args = types.SimpleNamespace(@@ -605,7 +608,10 @@ def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):trainer = object.__new__(Trainer)trainer.accelerator = types.SimpleNamespace(state = state)trainer.args = types.SimpleNamespace(ddp_find_unused_parameters = None)- trainer._context_parallel_manager = types.SimpleNamespace(size = 2, device_mesh = mesh)+ trainer._context_parallel_manager = types.SimpleNamespace(+ size = 2, backend = "ring", device_mesh = mesh+ )+ # A later non-CP trainer cleared the shared state; train() must put the CP mesh back.assert trainer.train() == "trained"assert state.device_mesh is meshassert trainer.args.ddp_find_unused_parameters is False@@ -746,7 +752,10 @@ def _gated_trainer():built.append((size, backend, rotate_method))return types.SimpleNamespace(- size = size, device_mesh = object(), attach_attention_hooks = lambda model: None+ size = size,+ backend = backend,+ device_mesh = object(),+ attach_attention_hooks = lambda model: None,)monkeypatch.setattr(cp, "ContextParallelManager", manager)@@ -771,7 +780,6 @@ def _gated_trainer(({"context_parallel_size": "2"}, ValueError),({"context_parallel_backend": "tree"}, ValueError),({"context_parallel_rotate_method": "ring"}, ValueError),- ({"context_parallel_backend": "ulysses"}, NotImplementedError),],)def test_bad_cp_options_fail_before_the_trainer_builds(monkeypatch, overrides, error):@@ -1030,6 +1038,99 @@ def test_generated_sft_config_carries_the_cp_options(tmp_path):assert cp.context_parallel_options(default) == (1, "ring", "allgather")+class _FakeCPMesh:+ def __init__(self, rank):+ self.rank = rank++ def get_local_rank(self):+ return self.rank++ def get_group(self):+ return ("group", self.rank)+++def _ulysses_manager(rank, size = 2):+ manager = _manager(size)+ manager.backend = "ulysses"+ manager.mesh = _FakeCPMesh(rank)+ return manager+++def test_ulysses_shards_contiguously_after_global_shift():+ labels = torch.tensor([[-100, 5, 6, 7, 8, 9, 10]])+ shards = []+ for rank in range(2):+ inputs = {"input_ids": torch.arange(1, 8)[None], "labels": labels.clone()}+ manager = _ulysses_manager(rank)+ with manager.apply(inputs):+ assert cp.get_cp_manager() is manager+ shards.append(inputs)+ assert cp.get_cp_manager() is None+ assert [s["input_ids"].tolist() for s in shards] == [[[1, 2, 3, 4]], [[5, 6, 7, 0]]]+ assert [s["shift_labels"].tolist() for s in shards] == [[[5, 6, 7, 8]], [[9, 10, -100, -100]]]+ assert [s["position_ids"].tolist() for s in shards] == [[[0, 1, 2, 3]], [[4, 5, 6, 7]]]+++def test_ulysses_packed_batch_keeps_global_lengths_and_masks_document_starts():+ lengths = torch.tensor([3, 4], dtype = torch.int32)+ labels = torch.tensor([[10, 11, 12, 20, 21, 22, 23]])+ shards = []+ for rank in range(2):+ inputs = {+ "input_ids": torch.arange(7)[None],+ "labels": labels.clone(),+ "packed_seq_lengths": lengths,+ }+ with _ulysses_manager(rank).apply(inputs):+ shards.append(inputs)+ assert all(s["packed_seq_lengths"] is lengths for s in shards)+ assert torch.cat([s["position_ids"] for s in shards], 1).tolist() == [[0, 1, 2, 0, 1, 2, 3, 4]]+ shift = torch.cat([s["shift_labels"] for s in shards], 1).tolist()+ assert shift == [[11, 12, -100, 21, 22, 23, -100, -100]]+++def test_ulysses_refuses_packed_lengths_longer_than_the_row():+ inputs = {+ "input_ids": torch.ones(1, 4, dtype = torch.long),+ "position_ids": torch.arange(4)[None],+ "packed_seq_lengths": torch.tensor([3, 10**9]),+ }+ with pytest.raises(ValueError, match = "packed row has 4 tokens"):+ _ulysses_manager(0)._prepare_inputs(inputs)+++def test_ring_refuses_packed_batches():+ inputs = {+ "input_ids": torch.ones(1, 4, dtype = torch.long),+ "packed_seq_lengths": torch.tensor([4]),+ }+ manager = _manager()+ manager.backend = "ring"+ with pytest.raises(ValueError, match = "ring context parallelism does not support packing"):+ manager._prepare_inputs(inputs)+++def test_ulysses_keeps_varlen_and_routes_attention_through_the_exchange(monkeypatch):+ calls = []++ def fake_ulysses(run_attention_fn, config, context, Q, K, V, group):+ calls.append((run_attention_fn, group))+ return "exchanged"++ monkeypatch.setattr(ad, "ulysses_attention", fake_ulysses)+ manager = _ulysses_manager(1)+ monkeypatch.setattr(cp, "_ACTIVE_MANAGER", manager)+ if ad.HAS_FLASH_ATTENTION:+ assert ad.select_attention_backend(True) == ad.FLASH_VARLEN+ else:+ assert ad.select_attention_backend(True) != ad.FLASH_VARLEN+ assert ad.run_attention(config = None, context = None, Q = 1, K = 2, V = 3) == "exchanged"+ assert calls == [(ad._run_attention, ("group", 1))]+ manager.backend = "ring"+ with pytest.raises(ValueError, match = "context_parallel_backend = 'ulysses'"):+ ad.select_attention_backend(True)++def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):import trl@@ -1050,3 +1151,35 @@ def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):cp.patch_sft_trainer()monkeypatch.setattr(Base, "train", lambda self: "patched later")assert Trainer().train() == "patched later"+++def test_ulysses_head_counts_are_checked_at_init(monkeypatch):+ import types++ config = types.SimpleNamespace(num_attention_heads = 3, num_key_value_heads = 3)+ model = types.SimpleNamespace(config = config)+ Trainer, _, built = _gated_trainer(monkeypatch, model = model, context_parallel_backend = "ulysses")+ with pytest.raises(ValueError, match = "divisible by context_parallel_size"):+ Trainer()+ assert built == []+ config.num_attention_heads = config.num_key_value_heads = 4+ Trainer()+ assert built == [(2, "ulysses", "allgather")]++[email protected]("backend", ["ring", "ulysses"])+def test_only_ring_needs_sm80(monkeypatch, backend):+ import types++ config = types.SimpleNamespace(num_attention_heads = 4, num_key_value_heads = 4)+ Trainer, _, built = _gated_trainer(+ monkeypatch, model = types.SimpleNamespace(config = config), context_parallel_backend = backend+ )+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (7, 5))+ if backend == "ring":+ with pytest.raises(NotImplementedError, match = "compute capability"):+ Trainer()+ else:+ Trainer()+ assert built == [(2, "ulysses", "allgather")]diff --git a/tests/test_ulysses.py b/tests/test_ulysses.pynew file mode 100644index 00000000000..ff24ed813b2--- /dev/null+++ b/tests/test_ulysses.py@@ -0,0 +1,317 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Ulysses sequence parallel exchange and KV replication, on CPU gloo ranks."""++import importlib.util+import os+import pathlib+import tempfile+from dataclasses import dataclass+from typing import Any, Optional++import pytest+import torch+import torch.distributed as dist+import torch.multiprocessing as mp+import torch.nn.functional as F++_MODULE = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "utils" / "ulysses.py"+++def _load():+ spec = importlib.util.spec_from_file_location("_ulysses_under_test", _MODULE)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++U = _load()+++@dataclass+class _Config:+ backend: str+ n_kv_heads: int+ n_groups: int+ flash_dense_kwargs: Optional[dict] = None+ flash_varlen_kwargs: Optional[dict] = None+ sdpa_kwargs: Optional[dict] = None+ xformers_kwargs: Optional[dict] = None+++@dataclass+class _Context:+ bsz: int+ q_len: int+ kv_seq_len: int+ n_heads: int+ head_dim: int+ requires_grad: bool+ seq_info: Any+ attention_mask: Any+ causal_mask: Any+ sliding_window: Optional[int] = None+ prefix_seg_info: Any = None+ is_causal: bool = True+++def _segment_mask(lengths, total):+ lengths = [int(x) for x in lengths]+ if sum(lengths) < total:+ lengths.append(total - sum(lengths))+ doc = torch.repeat_interleave(torch.arange(len(lengths)), torch.tensor(lengths))+ pos = torch.arange(total)+ return (doc[:, None] == doc[None, :]) & (pos[None, :] <= pos[:, None])+++def _fake_run_attention(*, config, context, Q, K, V):+ assert Q.shape == (context.bsz, context.n_heads, context.q_len, context.head_dim), Q.shape+ assert K.shape == (context.bsz, config.n_kv_heads, context.kv_seq_len, context.head_dim)+ assert context.n_heads == config.n_kv_heads * config.n_groups+ K = K.repeat_interleave(config.n_groups, dim = 1)+ V = V.repeat_interleave(config.n_groups, dim = 1)+ if context.seq_info is None:+ out = F.scaled_dot_product_attention(Q, K, V, is_causal = True)+ else:+ mask = _segment_mask(context.seq_info[0].tolist(), context.q_len)+ out = F.scaled_dot_product_attention(Q, K, V, attn_mask = mask)+ return out.transpose(1, 2)+++def _reference(+ Q,+ K,+ V,+ lengths = None,+):+ G = Q.shape[1] // K.shape[1]+ K = K.repeat_interleave(G, dim = 1)+ V = V.repeat_interleave(G, dim = 1)+ if lengths is None:+ out = F.scaled_dot_product_attention(Q, K, V, is_causal = True)+ else:+ out = F.scaled_dot_product_attention(Q, K, V, attn_mask = _segment_mask(lengths, Q.shape[2]))+ return out.transpose(1, 2)+++def _spawn(+ fn,+ world,+ *args,+ start_method = "spawn",+):+ with tempfile.TemporaryDirectory() as d:+ init_file = os.path.join(d, "init")+ mp.start_processes(+ _entry,+ args = (world, init_file, fn, args),+ nprocs = world,+ join = True,+ start_method = start_method,+ )+++def _entry(rank, world, init_file, fn, args):+ torch.set_num_threads(1)+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)+ try:+ fn(rank, world, *args)+ finally:+ dist.destroy_process_group()+++def _exchange_worker(rank, world):+ torch.manual_seed(0)+ B, H, T, D = 2, 2 * world, 3 * world, 4+ full = torch.randn(B, H, T, D, dtype = torch.float64)+ S = T // world+ local = full[:, :, rank * S : (rank + 1) * S]+ heads = U.seq_to_head(local, None)+ h = H // world+ torch.testing.assert_close(heads, full[:, rank * h : (rank + 1) * h], rtol = 0, atol = 0)+ back = U.head_to_seq(heads.transpose(1, 2), None)+ torch.testing.assert_close(back, local.transpose(1, 2), rtol = 0, atol = 0)+++def _attention_worker(rank, world, n_heads, n_kv, raw_len, lengths, dtype):+ torch.manual_seed(1234)+ B = 1 if lengths is not None else 2+ D = 8+ T = -(-raw_len // world) * world+ Q = torch.randn(B, n_heads, T, D, dtype = dtype)+ K = torch.randn(B, n_kv, T, D, dtype = dtype)+ V = torch.randn(B, n_kv, T, D, dtype = dtype)+ W = torch.randn(B, T, n_heads, D, dtype = dtype)+ Q[:, :, raw_len:] = 0+ K[:, :, raw_len:] = 0+ V[:, :, raw_len:] = 0++ Qr, Kr, Vr = (x.clone().requires_grad_(True) for x in (Q, K, V))+ ref = _reference(Qr, Kr, Vr, lengths)+ (ref * W).sum().backward()++ S = T // world+ sl = slice(rank * S, (rank + 1) * S)+ Ql, Kl, Vl = (x[:, :, sl].clone().requires_grad_(True) for x in (Q, K, V))+ seq_info = None+ if lengths is not None:+ lens = torch.tensor(lengths, dtype = torch.int32)+ cu = torch.zeros(len(lengths) + 1, dtype = torch.int32)+ cu[1:] = torch.cumsum(lens, 0)+ seq_info = (lens, cu, int(lens.max()))+ config = _Config("sdpa", n_kv_heads = n_kv, n_groups = n_heads // n_kv)+ context = _Context(+ bsz = B,+ q_len = S,+ kv_seq_len = S,+ n_heads = n_heads,+ head_dim = D,+ requires_grad = True,+ seq_info = seq_info,+ attention_mask = None,+ causal_mask = None,+ )+ out = U.ulysses_attention(_fake_run_attention, config, context, Ql, Kl, Vl, None)+ assert out.shape == (B, S, n_heads, D)+ tol = dict(rtol = 1e-10, atol = 1e-10) if dtype == torch.float64 else dict(rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(out, ref[:, sl].detach(), **tol)+ (out * W[:, sl]).sum().backward()+ torch.testing.assert_close(Ql.grad, Qr.grad[:, :, sl], **tol)+ torch.testing.assert_close(Kl.grad, Kr.grad[:, :, sl], **tol)+ torch.testing.assert_close(Vl.grad, Vr.grad[:, :, sl], **tol)+++_CASES = {+ 2: [(4, 4, torch.float64), (4, 2, torch.float64), (4, 1, torch.float64)],+ 4: [(8, 1, torch.float64), (8, 2, torch.float32), (8, 8, torch.float32)],+}+_PACKED = {2: (4, 2), 4: (8, 1)}+++def _all_cases_worker(rank, world):+ _exchange_worker(rank, world)+ for n_heads, n_kv, dtype in _CASES[world]:+ _attention_worker(rank, world, n_heads, n_kv, 4 * world + 3, None, dtype)+ raw_len = 6 * world + 1+ lengths = [5, 3 * world - 2, raw_len - 3 * world - 3]+ n_heads, n_kv = _PACKED[world]+ _attention_worker(rank, world, n_heads, n_kv, raw_len, lengths, torch.float64)++[email protected]("world", [2, 4])+def test_ulysses_exchange_and_attention_match_full_sequence(world):+ _spawn(_all_cases_worker, world)+++def test_head_plan():+ assert U.ulysses_head_plan(8, 8, 2) == (1, 4, 4, 1)+ assert U.ulysses_head_plan(8, 2, 2) == (1, 4, 1, 4)+ assert U.ulysses_head_plan(8, 2, 4) == (2, 2, 1, 2)+ assert U.ulysses_head_plan(8, 1, 8) == (8, 1, 1, 1)+ assert U.ulysses_head_plan(32, 8, 4) == (1, 8, 2, 4)+ with pytest.raises(ValueError, match = "attention heads"):+ U.ulysses_head_plan(6, 2, 4)+ with pytest.raises(ValueError, match = "divide one another"):+ U.ulysses_head_plan(24, 6, 4)+ with pytest.raises(ValueError, match = "multiple of key/value"):+ U.ulysses_head_plan(8, 3, 2)+++def test_ulysses_attention_refuses_masks_and_cache():+ config = _Config("sdpa", n_kv_heads = 2, n_groups = 1)++ def ctx(**kw):+ base = dict(+ bsz = 1,+ q_len = 4,+ kv_seq_len = 4,+ n_heads = 2,+ head_dim = 4,+ requires_grad = False,+ seq_info = None,+ attention_mask = None,+ causal_mask = None,+ )+ base.update(kw)+ return _Context(**base)++ x = torch.zeros(1, 2, 4, 4)+ for bad, match in [+ (ctx(attention_mask = torch.ones(1, 4)), "dense attention mask"),+ (ctx(causal_mask = torch.ones(4, 4)), "dense attention mask"),+ (ctx(prefix_seg_info = object()), "PrefixGrouper"),+ (ctx(kv_seq_len = 8), "KV cache"),+ ]:+ with pytest.raises(RuntimeError, match = match):+ U.ulysses_attention(_fake_run_attention, config, bad, x, x, x, None)+++def _real_dispatch_worker(rank, world, run_attention, AttentionConfig, AttentionContext):+ torch.manual_seed(7)+ B, H, Hkv, T, D = 2, 4, 2, 4 * world, 8+ Q = torch.randn(B, H, T, D)+ K = torch.randn(B, Hkv, T, D)+ V = torch.randn(B, Hkv, T, D)+ ref = _reference(Q, K, V)+ S = T // world+ sl = slice(rank * S, (rank + 1) * S)+ config = AttentionConfig(backend = "sdpa", n_kv_heads = Hkv, n_groups = H // Hkv)+ context = AttentionContext(+ bsz = B,+ q_len = S,+ kv_seq_len = S,+ n_heads = H,+ head_dim = D,+ requires_grad = False,+ seq_info = None,+ attention_mask = None,+ causal_mask = None,+ )+ out = U.ulysses_attention(+ run_attention, config, context, Q[:, :, sl], K[:, :, sl], V[:, :, sl], None+ )+ torch.testing.assert_close(out, ref[:, sl], rtol = 1e-5, atol = 1e-5)++ lengths = torch.tensor([3, T - 5], dtype = torch.int32)+ cu_seqlens = torch.zeros(3, dtype = torch.int32)+ cu_seqlens[1:] = torch.cumsum(lengths, 0)+ Q, K, V = Q[:1], K[:1], V[:1]+ ref = _reference(Q, K, V, lengths.tolist())+ context = AttentionContext(+ bsz = 1,+ q_len = S,+ kv_seq_len = S,+ n_heads = H,+ head_dim = D,+ requires_grad = False,+ seq_info = (lengths, cu_seqlens, int(lengths.max())),+ attention_mask = None,+ causal_mask = None,+ )+ out = U.ulysses_attention(+ run_attention, config, context, Q[:, :, sl], K[:, :, sl], V[:, :, sl], None+ )+ torch.testing.assert_close(out, ref[:, sl], rtol = 1e-5, atol = 1e-5)+++def test_real_run_attention_sdpa_path():+ try:+ from unsloth.utils.attention_dispatch import (+ AttentionConfig,+ AttentionContext,+ _run_attention,+ )+ except Exception as exc: # the unsloth package import needs an accelerator+ pytest.skip(f"unsloth.utils.attention_dispatch does not import here: {type(exc).__name__}")+ # Forked, not spawned: a fresh interpreter would re-import unsloth without the test GPU spoof.+ if "fork" not in mp.get_all_start_methods():+ pytest.skip("needs the fork start method")+ _spawn(+ _real_dispatch_worker,+ 2,+ _run_attention,+ AttentionConfig,+ AttentionContext,+ start_method = "fork",+ )diff --git a/unsloth-cli.py b/unsloth-cli.pyindex ffdaad7ec26..92a85e7e2e3 100644--- a/unsloth-cli.py+++ b/unsloth-cli.py@@ -477,14 +477,16 @@ if __name__ == "__main__":"--context_parallel_size",type = int,default = 1,- help = "Ranks per context parallel group (SDPA ring attention, torchrun, PyTorch >= 2.7).",+ help = "Ranks per context parallel group (torchrun, PyTorch >= 2.7).",)training_group.add_argument("--context_parallel_backend",type = str,default = "ring",- choices = ["ring"],- help = "Context parallel attention backend.",+ choices = ["ring", "ulysses"],+ help = "Context parallel attention backend: ring (SDPA ring attention) or ulysses "+ "(all-to-all over heads; keeps packing and the usual attention kernels, needs heads "+ "divisible by the size).",)training_group.add_argument("--context_parallel_rotate_method",diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 1307178e448..58c2dfc41a5 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -84,14 +84,11 @@ def context_parallel_options(config) -> tuple:"Unsloth: context_parallel_rotate_method must be one of "f"{_ROTATE_METHODS}, got {rotate_method!r}.")- if size > 1 and backend == "ulysses":- raise NotImplementedError(- "Unsloth: context_parallel_backend = 'ulysses' is not available yet; use 'ring'."- )return size, backend, rotate_method-def _ring_unsupported_attention(config) -> Optional[str]:+def _unsupported_attention(config) -> Optional[str]:+ # Ring and Llama's forward ignore mask, window and softcap, so both backends refuse them.configs = [config]text_config = getattr(config, "text_config", None)if text_config is not None and text_config is not config:@@ -127,6 +124,7 @@ def _unregister_cp_sharding_rules() -> None:from .device_type import DEVICE_TYPE_TORCH+from .utils.ulysses import _document_positions, ulysses_head_plan# A module global, not a ContextVar: CUDA runs backward (and checkpoint recompute) on another thread._ACTIVE_MANAGER: Optional["ContextParallelManager"] = None@@ -154,13 +152,15 @@ def _supports_context_parallel(model) -> bool:def _self_attn_pre_forward_hook(_module, module_args, module_kwargs):- # Ring attention only supports is_causal with no mask; right padding is already -100 in labels.if get_cp_manager() is not None and "attention_mask" in module_kwargs:module_kwargs["attention_mask"] = Nonereturn module_args, module_kwargsclass ContextParallelManager:+ backend = "ring"+ rotate_method = "allgather"+def __init__(self,size: int,@@ -178,6 +178,10 @@ class ContextParallelManager:)self.mesh = self.device_mesh["cp"]+ @property+ def group(self):+ return self.mesh.get_group()+def attach_attention_hooks(self, model: torch.nn.Module) -> None:for name, module in model.named_modules():if name.endswith("self_attn") and not getattr(module, "_unsloth_cp_hooked", False):@@ -193,7 +197,24 @@ class ContextParallelManager:"Unsloth: context parallelism needs input_ids batches (not inputs_embeds).")bsz, seq_len = input_ids.shape- if inputs.get("position_ids") is None:+ packed_seq_lengths = inputs.get("packed_seq_lengths")+ ulysses = self.backend == "ulysses"+ if packed_seq_lengths is not None and not ulysses:+ raise ValueError("Unsloth: ring context parallelism does not support packing.")+ if packed_seq_lengths is not None:+ if bsz != 1:+ raise ValueError(f"Unsloth: packed batches must be a single row, got {bsz} rows.")+ used = int(torch.as_tensor(packed_seq_lengths).sum())+ if used > seq_len:+ raise ValueError(+ f"Unsloth: packed_seq_lengths sum to {used} but the packed row has {seq_len} tokens."+ )+ # .get: a collator may carry the key with None, which would leave positions shard-local.+ if inputs.get("position_ids") is None and packed_seq_lengths is not None:+ inputs["position_ids"] = _document_positions(+ packed_seq_lengths, seq_len, input_ids.device+ )+ elif inputs.get("position_ids") is None:inputs["position_ids"] = (torch.arange(seq_len, device = input_ids.device).expand(bsz, -1).contiguous())@@ -206,8 +227,11 @@ class ContextParallelManager:)labels = inputs.get("labels")if inputs.get("shift_labels") is None and labels is not None:+ # Shard-local labels cannot see document starts on another shard: mask them here.+ from .utils.packing import mask_packed_boundary_labels+ labels = mask_packed_boundary_labels(labels, packed_seq_lengths)inputs["shift_labels"] = F.pad(labels, (0, 1), value = -100)[:, 1:].contiguous()- pad = (-seq_len) % (2 * self.size)+ pad = (-seq_len) % (self.size if ulysses else 2 * self.size)if pad:for name in _BUFFER_NAMES:tensor = inputs.get(name)@@ -221,7 +245,8 @@ class ContextParallelManager:@contextlib.contextmanagerdef _rotate_method_scope(self) -> Iterator[None]:- rotate_method = getattr(self, "rotate_method", "allgather")+ # Process-wide torch state, read in forward and backward: set per step, restored after.+ rotate_method = self.rotate_methodif _cp_options is None or set_rotate_method is None:if rotate_method != "allgather":raise RuntimeError(@@ -236,9 +261,29 @@ class ContextParallelManager:finally:_cp_options.rotate_method = previous+ @contextlib.contextmanager+ def _ulysses_scope(self, inputs: dict) -> Iterator[None]:+ # Contiguous shards; packed_seq_lengths stays global.+ rank = self.mesh.get_local_rank()+ for name in _BUFFER_NAMES:+ tensor = inputs.get(name)+ if isinstance(tensor, torch.Tensor) and tensor.ndim > 1:+ local = tensor.shape[1] // self.size+ inputs[name] = tensor[:, rank * local : (rank + 1) * local].contiguous()+ global _ACTIVE_MANAGER+ previous, _ACTIVE_MANAGER = _ACTIVE_MANAGER, self+ try:+ yield+ finally:+ _ACTIVE_MANAGER = previous+@contextlib.contextmanagerdef apply(self, inputs: dict) -> Iterator[None]:self._prepare_inputs(inputs)+ if self.backend == "ulysses":+ with self._ulysses_scope(inputs):+ yield+ returnbuffers = [inputs[name]for name in _BUFFER_NAMES@@ -387,7 +432,11 @@ def patch_sft_trainer() -> None:raise RuntimeError(f"Unsloth: world size {world_size} is not a multiple of context_parallel_size {size}.")- if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] < 8:+ if (+ backend == "ring"+ and torch.cuda.is_available()+ and torch.cuda.get_device_capability()[0] < 8+ ):# Pure torch ring attention fails in backward without flash (2x T4, torch 2.10).raise NotImplementedError("Unsloth: context parallelism needs flash attention (compute capability >= 8.0)."@@ -420,10 +469,20 @@ def patch_sft_trainer() -> None:if Version(accelerate.__version__) < Version("1.10.0"):raise NotImplementedError("Unsloth: context parallelism needs accelerate >= 1.10.0.")- unsupported = _ring_unsupported_attention(getattr(self.model, "config", None))+ model_config = getattr(self.model, "config", None)+ unsupported = _unsupported_attention(model_config)if unsupported is not None:raise NotImplementedError(- f"Unsloth: ring context parallelism does not support {unsupported}."+ f"Unsloth: {backend} context parallelism does not support {unsupported}."+ )+ if backend == "ulysses":+ # Every rank raises here, before any collective.+ text_config = getattr(model_config, "text_config", None) or model_config+ ulysses_head_plan(+ text_config.num_attention_heads,+ getattr(text_config, "num_key_value_heads", None)+ or text_config.num_attention_heads,+ size,)if not _supports_context_parallel(self.model):raise NotImplementedError(@@ -452,8 +511,8 @@ def patch_sft_trainer() -> None:_scope_flash_attention_impl(self.model, self)print(- f"Unsloth: Context parallelism enabled with size = {size} "- f"({backend}, rotate_method = {rotate_method})."+ f"Unsloth: Context parallelism enabled with size = {size} ({backend}"+ + (f", rotate_method = {rotate_method})." if backend == "ring" else ")."))@functools.wraps(original_prediction_step)diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 5a7e9c76d5e..dcd054682df 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -3325,11 +3325,11 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):max_seq_length_pre += """context_parallel_size : Optional[int] = field(default = 1,- metadata = {'help': 'Ranks per context parallel group (SDPA ring attention). 1 disables it.'},+ metadata = {'help': 'Ranks per context parallel group. 1 disables it.'},)context_parallel_backend : Optional[str] = field(default = 'ring',- metadata = {'help': 'Context parallel attention backend: ring.'},+ metadata = {'help': 'Context parallel attention backend: ring or ulysses (keeps packing).'},)context_parallel_rotate_method : Optional[str] = field(default = 'allgather',diff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 72e914dd00b..99b598235e2 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -1863,14 +1863,16 @@ def _patch_sft_trainer_auto_packing(trl_module):# several classes, so a "yes" from the resolved one is not proof about the# instance. A correct "no" has already turned both flags off, which is the# condition the post-init check skips on, so leaving it armed is free.- is_context_parallel = (getattr(config_arg, "context_parallel_size", 1) or 1) > 1+ is_ring_context_parallel = (getattr(config_arg, "context_parallel_size", 1) or 1) > 1 and (+ getattr(config_arg, "context_parallel_backend", None) or "ring"+ ) == "ring"# Read before __init__: the compiled trainer ends it with model.for_inference(), which sets the flag.eval_logits_wanted = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" or _wants_eval_logits(original_init, args, kwargs)blocked = ((data_collator is not None)- or is_context_parallel+ or is_ring_context_parallelor (is_processor and not processor_text_only)or is_auto_processor_vlmor is_vision_dataset@@ -1886,7 +1888,8 @@ def _patch_sft_trainer_auto_packing(trl_module):setattr(config_arg, "packing", False)if hasattr(config_arg, "padding_free"):setattr(config_arg, "padding_free", False)- if is_context_parallel and getattr(config_arg, "eval_packing", None):+ # TRL resolves eval_packing on its own; packed eval batches would hit ring attention's varlen refusal.+ if is_ring_context_parallel and getattr(config_arg, "eval_packing", None):setattr(config_arg, "eval_packing", False)if blocked and requested_pack:@@ -1903,8 +1906,8 @@ def _patch_sft_trainer_auto_packing(trl_module):reason = "hybrid linear-attention model"elif is_unsupported_model:reason = f"unsupported model type(s): {', '.join(model_types)}"- elif is_context_parallel:- reason = "context parallelism"+ elif is_ring_context_parallel:+ reason = "ring context parallelism"elif forward_rejects_packing:# Name the real blocker, else this falls through to the# UNSLOTH_RETURN_LOGITS branch and points at an unset flag. For a stringdiff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 72d693997fc..25670f919f7 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -26,6 +26,7 @@ import torch.nn.functional as Ffrom ..device_type import DEVICE_TYPEfrom ..models._utils import *from ..context_parallel import get_cp_manager+from .ulysses import ulysses_attentionfrom ..utils.packing import (build_sdpa_packed_attention_mask,build_xformers_block_causal_mask,@@ -528,13 +529,19 @@ class AttentionContext:is_causal: bool = True+def _ring_attention_active() -> bool:+ manager = get_cp_manager()+ return manager is not None and manager.backend == "ring"++def select_attention_backend(use_varlen: bool = False) -> str:"""Return attention backend based on availability / priority order."""- if get_cp_manager() is not None:+ if _ring_attention_active():if use_varlen:raise ValueError(- "Unsloth: context parallelism does not support packing / padding-free."+ "Unsloth: ring context parallelism does not support packing / padding-free; "+ "use context_parallel_backend = 'ulysses'.")return SDPAif HAS_FLASH_ATTENTION:@@ -611,7 +618,15 @@ def run_attention(Varlen flash is preferred for packed batches as it avoids padding; xFormersand SDPA handle packing via a block-diagonal mask."""+ manager = get_cp_manager()+ if manager is not None and manager.backend == "ulysses":+ return ulysses_attention(_run_attention, config, context, Q, K, V, manager.group)+ return _run_attention(config = config, context = context, Q = Q, K = K, V = V)++def _run_attention(+ *, config: AttentionConfig, context: AttentionContext, Q: Tensor, K: Tensor, V: Tensor+) -> Tensor:# PrefixGrouper shared-prefix attention (GRPO dedup): Q/K/V here are [bsz, H, T, D] while the kernel# takes and returns [1, T, H, D], matching the other backends.if context.prefix_seg_info is not None:@@ -732,11 +747,13 @@ def run_attention(# DoRA promotes q/k/v_proj outputs to fp32, which FlashAttention rejects (as does the xformers# flash-2 op on sm_100+), so downcast any fp32 Q/K/V to a supported dtype (#1013).- cp_out_dtype = Q.dtype if backend == SDPA and get_cp_manager() is not None else None+ # Ring attention (context parallelism) likewise needs a flash-eligible dtype.+ ring = backend == SDPA and _ring_attention_active()+ cp_out_dtype = Q.dtype if ring else Noneif (backend in (FLASH_DENSE, FLASH_VARLEN)or (backend == XFORMERS and xformers_fp32_unsupported)- or (backend == SDPA and get_cp_manager() is not None)+ or ring) and torch.float32 in (Q.dtype,K.dtype,@@ -973,7 +990,7 @@ def run_attention(# Flash takes GQA natively, halving the ring's K/V peak; UNSLOTH_CP_GQA=0 expands.if (use_sdpa_gqa- and get_cp_manager() is not None+ and _ring_attention_active()and os.environ.get("UNSLOTH_CP_GQA", "1") == "0"):use_sdpa_gqa = Falsediff --git a/unsloth/utils/ulysses.py b/unsloth/utils/ulysses.pynew file mode 100644index 00000000000..36e03a37455--- /dev/null+++ b/unsloth/utils/ulysses.py@@ -0,0 +1,131 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Ulysses sequence parallelism: all-to-all from sequence shards to head shards around attention."""++from dataclasses import replace+from typing import NamedTuple++import torch+import torch.distributed as dist++__all__ = [+ "UlyssesHeadPlan",+ "ulysses_head_plan",+ "seq_to_head",+ "head_to_seq",+ "ulysses_attention",+]+++def _all_to_all(x, scatter_dim, gather_dim, group):+ world = dist.get_world_size(group)+ if world == 1:+ return x+ if x.shape[scatter_dim] % world != 0:+ raise ValueError(+ f"Unsloth: Ulysses cannot split dim {scatter_dim} of size {x.shape[scatter_dim]} "+ f"across {world} ranks."+ )+ send = torch.stack(x.chunk(world, dim = scatter_dim), dim = 0)+ recv = torch.empty_like(send)+ dist.all_to_all_single(recv, send, group = group)+ return torch.cat(recv.unbind(0), dim = gather_dim)+++class _AllToAll(torch.autograd.Function):+ @staticmethod+ def forward(ctx, x, scatter_dim, gather_dim, group):+ ctx.scatter_dim = scatter_dim+ ctx.gather_dim = gather_dim+ ctx.group = group+ return _all_to_all(x, scatter_dim, gather_dim, group)++ @staticmethod+ def backward(ctx, grad):+ return _all_to_all(grad, ctx.gather_dim, ctx.scatter_dim, ctx.group), None, None, None+++def seq_to_head(x, group):+ """[B, H, T/N, D] sequence shard -> [B, H/N, T, D] head shard (rank r gets heads r*H/N ...)."""+ return _AllToAll.apply(x, 1, 2, group)+++def head_to_seq(x, group):+ """run_attention output [B, T, H/N, D] -> [B, T/N, H, D] (the local shard's run_attention layout)."""+ return _AllToAll.apply(x, 1, 2, group)+++class UlyssesHeadPlan(NamedTuple):+ kv_replication: int+ local_heads: int+ local_kv_heads: int+ local_groups: int+++def ulysses_head_plan(n_heads, n_kv_heads, cp_size):+ if n_heads % cp_size != 0:+ raise ValueError(+ f"Unsloth: Ulysses context parallelism needs the number of attention heads ({n_heads}) "+ f"to be divisible by context_parallel_size ({cp_size})."+ )+ if n_heads % n_kv_heads != 0:+ raise ValueError(+ f"Unsloth: attention heads ({n_heads}) must be a multiple of key/value heads ({n_kv_heads})."+ )+ if n_kv_heads % cp_size == 0:+ replication = 1+ elif cp_size % n_kv_heads == 0:+ replication = cp_size // n_kv_heads+ else:+ raise ValueError(+ f"Unsloth: Ulysses context parallelism needs key/value heads ({n_kv_heads}) and "+ f"context_parallel_size ({cp_size}) to divide one another."+ )+ local_heads = n_heads // cp_size+ local_kv_heads = n_kv_heads * replication // cp_size+ return UlyssesHeadPlan(replication, local_heads, local_kv_heads, local_heads // local_kv_heads)+++def ulysses_attention(run_attention_fn, config, context, Q, K, V, group):+ """Q [B, H, T/N, D], K/V [B, Hkv, T/N, D] (post RoPE) -> [B, T/N, H, D]."""+ if context.attention_mask is not None or isinstance(context.causal_mask, torch.Tensor):+ raise RuntimeError(+ "Unsloth: Ulysses context parallelism does not take a dense attention mask; padding is "+ "handled by ignoring padded labels and packing by global sequence lengths."+ )+ if context.prefix_seg_info is not None:+ raise RuntimeError("Unsloth: Ulysses context parallelism does not support PrefixGrouper.")+ if context.q_len != context.kv_seq_len:+ raise RuntimeError("Unsloth: Ulysses context parallelism does not support a KV cache.")++ size = dist.get_world_size(group)+ plan = ulysses_head_plan(context.n_heads, config.n_kv_heads, size)+ if plan.kv_replication > 1:+ # Consecutive copies keep query head h on kv head h // n_groups after the split.+ K = K.repeat_interleave(plan.kv_replication, dim = 1)+ V = V.repeat_interleave(plan.kv_replication, dim = 1)++ Q = seq_to_head(Q, group)+ K = seq_to_head(K, group)+ V = seq_to_head(V, group)++ config = replace(config, n_kv_heads = plan.local_kv_heads, n_groups = plan.local_groups)+ context = replace(+ context,+ q_len = context.q_len * size,+ kv_seq_len = context.kv_seq_len * size,+ n_heads = plan.local_heads,+ )+ A = run_attention_fn(config = config, context = context, Q = Q, K = K, V = V)+ A = A.reshape(context.bsz, context.q_len, plan.local_heads, context.head_dim)+ return head_to_seq(A, group)+++def _document_positions(packed_seq_lengths, total, device):+ lengths = [int(x) for x in torch.as_tensor(packed_seq_lengths).reshape(-1).tolist()]+ pieces = [torch.arange(length, device = device) for length in lengths]+ used = sum(lengths)+ if used < total:+ pieces.append(torch.arange(total - used, device = device))+ return torch.cat(pieces)[None]