Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: Ulysses backend with packing and varlen attention (#13848)

Daniel Hanace96a57aa7c
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
index 4bc322af359..d2826794232 100644
--- a/tests/test_context_parallel.py
+++ b/tests/test_context_parallel.py
@@ -544,7 +544,10 @@ def test_find_unused_parameters_is_off_under_cp_unless_the_user_set_it(monkeypat
cp,
"ContextParallelManager",
lambda size, *rest: types.SimpleNamespace(
- size = size, device_mesh = object(), attach_attention_hooks = lambda model: None
+ size = size,
+ backend = "ring",
+ device_mesh = object(),
+ attach_attention_hooks = lambda model: None,
),
)
args = types.SimpleNamespace(
@@ -605,7 +608,10 @@ def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):
trainer = object.__new__(Trainer)
trainer.accelerator = types.SimpleNamespace(state = state)
trainer.args = types.SimpleNamespace(ddp_find_unused_parameters = None)
- trainer._context_parallel_manager = types.SimpleNamespace(size = 2, device_mesh = mesh)
+ trainer._context_parallel_manager = types.SimpleNamespace(
+ size = 2, backend = "ring", device_mesh = mesh
+ )
+ # A later non-CP trainer cleared the shared state; train() must put the CP mesh back.
assert trainer.train() == "trained"
assert state.device_mesh is mesh
assert trainer.args.ddp_find_unused_parameters is False
@@ -746,7 +752,10 @@ def _gated_trainer(
):
built.append((size, backend, rotate_method))
return types.SimpleNamespace(
- size = size, device_mesh = object(), attach_attention_hooks = lambda model: None
+ size = size,
+ backend = backend,
+ device_mesh = object(),
+ attach_attention_hooks = lambda model: None,
)
monkeypatch.setattr(cp, "ContextParallelManager", manager)
@@ -771,7 +780,6 @@ def _gated_trainer(
({"context_parallel_size": "2"}, ValueError),
({"context_parallel_backend": "tree"}, ValueError),
({"context_parallel_rotate_method": "ring"}, ValueError),
- ({"context_parallel_backend": "ulysses"}, NotImplementedError),
],
)
def test_bad_cp_options_fail_before_the_trainer_builds(monkeypatch, overrides, error):
@@ -1030,6 +1038,99 @@ def test_generated_sft_config_carries_the_cp_options(tmp_path):
assert cp.context_parallel_options(default) == (1, "ring", "allgather")
+class _FakeCPMesh:
+ def __init__(self, rank):
+ self.rank = rank
+
+ def get_local_rank(self):
+ return self.rank
+
+ def get_group(self):
+ return ("group", self.rank)
+
+
+def _ulysses_manager(rank, size = 2):
+ manager = _manager(size)
+ manager.backend = "ulysses"
+ manager.mesh = _FakeCPMesh(rank)
+ return manager
+
+
+def test_ulysses_shards_contiguously_after_global_shift():
+ labels = torch.tensor([[-100, 5, 6, 7, 8, 9, 10]])
+ shards = []
+ for rank in range(2):
+ inputs = {"input_ids": torch.arange(1, 8)[None], "labels": labels.clone()}
+ manager = _ulysses_manager(rank)
+ with manager.apply(inputs):
+ assert cp.get_cp_manager() is manager
+ shards.append(inputs)
+ assert cp.get_cp_manager() is None
+ assert [s["input_ids"].tolist() for s in shards] == [[[1, 2, 3, 4]], [[5, 6, 7, 0]]]
+ assert [s["shift_labels"].tolist() for s in shards] == [[[5, 6, 7, 8]], [[9, 10, -100, -100]]]
+ assert [s["position_ids"].tolist() for s in shards] == [[[0, 1, 2, 3]], [[4, 5, 6, 7]]]
+
+
+def test_ulysses_packed_batch_keeps_global_lengths_and_masks_document_starts():
+ lengths = torch.tensor([3, 4], dtype = torch.int32)
+ labels = torch.tensor([[10, 11, 12, 20, 21, 22, 23]])
+ shards = []
+ for rank in range(2):
+ inputs = {
+ "input_ids": torch.arange(7)[None],
+ "labels": labels.clone(),
+ "packed_seq_lengths": lengths,
+ }
+ with _ulysses_manager(rank).apply(inputs):
+ shards.append(inputs)
+ assert all(s["packed_seq_lengths"] is lengths for s in shards)
+ assert torch.cat([s["position_ids"] for s in shards], 1).tolist() == [[0, 1, 2, 0, 1, 2, 3, 4]]
+ shift = torch.cat([s["shift_labels"] for s in shards], 1).tolist()
+ assert shift == [[11, 12, -100, 21, 22, 23, -100, -100]]
+
+
+def test_ulysses_refuses_packed_lengths_longer_than_the_row():
+ inputs = {
+ "input_ids": torch.ones(1, 4, dtype = torch.long),
+ "position_ids": torch.arange(4)[None],
+ "packed_seq_lengths": torch.tensor([3, 10**9]),
+ }
+ with pytest.raises(ValueError, match = "packed row has 4 tokens"):
+ _ulysses_manager(0)._prepare_inputs(inputs)
+
+
+def test_ring_refuses_packed_batches():
+ inputs = {
+ "input_ids": torch.ones(1, 4, dtype = torch.long),
+ "packed_seq_lengths": torch.tensor([4]),
+ }
+ manager = _manager()
+ manager.backend = "ring"
+ with pytest.raises(ValueError, match = "ring context parallelism does not support packing"):
+ manager._prepare_inputs(inputs)
+
+
+def test_ulysses_keeps_varlen_and_routes_attention_through_the_exchange(monkeypatch):
+ calls = []
+
+ def fake_ulysses(run_attention_fn, config, context, Q, K, V, group):
+ calls.append((run_attention_fn, group))
+ return "exchanged"
+
+ monkeypatch.setattr(ad, "ulysses_attention", fake_ulysses)
+ manager = _ulysses_manager(1)
+ monkeypatch.setattr(cp, "_ACTIVE_MANAGER", manager)
+ if ad.HAS_FLASH_ATTENTION:
+ assert ad.select_attention_backend(True) == ad.FLASH_VARLEN
+ else:
+ assert ad.select_attention_backend(True) != ad.FLASH_VARLEN
+ assert ad.run_attention(config = None, context = None, Q = 1, K = 2, V = 3) == "exchanged"
+ assert calls == [(ad._run_attention, ("group", 1))]
+ manager.backend = "ring"
+ with pytest.raises(ValueError, match = "context_parallel_backend = 'ulysses'"):
+ ad.select_attention_backend(True)
+
+
def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):
import trl
@@ -1050,3 +1151,35 @@ def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):
cp.patch_sft_trainer()
monkeypatch.setattr(Base, "train", lambda self: "patched later")
assert Trainer().train() == "patched later"
+
+
+def test_ulysses_head_counts_are_checked_at_init(monkeypatch):
+ import types
+
+ config = types.SimpleNamespace(num_attention_heads = 3, num_key_value_heads = 3)
+ model = types.SimpleNamespace(config = config)
+ Trainer, _, built = _gated_trainer(monkeypatch, model = model, context_parallel_backend = "ulysses")
+ with pytest.raises(ValueError, match = "divisible by context_parallel_size"):
+ Trainer()
+ assert built == []
+ config.num_attention_heads = config.num_key_value_heads = 4
+ Trainer()
+ assert built == [(2, "ulysses", "allgather")]
+
+
[email protected]("backend", ["ring", "ulysses"])
+def test_only_ring_needs_sm80(monkeypatch, backend):
+ import types
+
+ config = types.SimpleNamespace(num_attention_heads = 4, num_key_value_heads = 4)
+ Trainer, _, built = _gated_trainer(
+ monkeypatch, model = types.SimpleNamespace(config = config), context_parallel_backend = backend
+ )
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (7, 5))
+ if backend == "ring":
+ with pytest.raises(NotImplementedError, match = "compute capability"):
+ Trainer()
+ else:
+ Trainer()
+ assert built == [(2, "ulysses", "allgather")]
diff --git a/tests/test_ulysses.py b/tests/test_ulysses.py
new file mode 100644
index 00000000000..ff24ed813b2
--- /dev/null
+++ b/tests/test_ulysses.py
@@ -0,0 +1,317 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Ulysses sequence parallel exchange and KV replication, on CPU gloo ranks."""
+
+import importlib.util
+import os
+import pathlib
+import tempfile
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import torch.nn.functional as F
+
+_MODULE = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "utils" / "ulysses.py"
+
+
+def _load():
+ spec = importlib.util.spec_from_file_location("_ulysses_under_test", _MODULE)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+U = _load()
+
+
+@dataclass
+class _Config:
+ backend: str
+ n_kv_heads: int
+ n_groups: int
+ flash_dense_kwargs: Optional[dict] = None
+ flash_varlen_kwargs: Optional[dict] = None
+ sdpa_kwargs: Optional[dict] = None
+ xformers_kwargs: Optional[dict] = None
+
+
+@dataclass
+class _Context:
+ bsz: int
+ q_len: int
+ kv_seq_len: int
+ n_heads: int
+ head_dim: int
+ requires_grad: bool
+ seq_info: Any
+ attention_mask: Any
+ causal_mask: Any
+ sliding_window: Optional[int] = None
+ prefix_seg_info: Any = None
+ is_causal: bool = True
+
+
+def _segment_mask(lengths, total):
+ lengths = [int(x) for x in lengths]
+ if sum(lengths) < total:
+ lengths.append(total - sum(lengths))
+ doc = torch.repeat_interleave(torch.arange(len(lengths)), torch.tensor(lengths))
+ pos = torch.arange(total)
+ return (doc[:, None] == doc[None, :]) & (pos[None, :] <= pos[:, None])
+
+
+def _fake_run_attention(*, config, context, Q, K, V):
+ assert Q.shape == (context.bsz, context.n_heads, context.q_len, context.head_dim), Q.shape
+ assert K.shape == (context.bsz, config.n_kv_heads, context.kv_seq_len, context.head_dim)
+ assert context.n_heads == config.n_kv_heads * config.n_groups
+ K = K.repeat_interleave(config.n_groups, dim = 1)
+ V = V.repeat_interleave(config.n_groups, dim = 1)
+ if context.seq_info is None:
+ out = F.scaled_dot_product_attention(Q, K, V, is_causal = True)
+ else:
+ mask = _segment_mask(context.seq_info[0].tolist(), context.q_len)
+ out = F.scaled_dot_product_attention(Q, K, V, attn_mask = mask)
+ return out.transpose(1, 2)
+
+
+def _reference(
+ Q,
+ K,
+ V,
+ lengths = None,
+):
+ G = Q.shape[1] // K.shape[1]
+ K = K.repeat_interleave(G, dim = 1)
+ V = V.repeat_interleave(G, dim = 1)
+ if lengths is None:
+ out = F.scaled_dot_product_attention(Q, K, V, is_causal = True)
+ else:
+ out = F.scaled_dot_product_attention(Q, K, V, attn_mask = _segment_mask(lengths, Q.shape[2]))
+ return out.transpose(1, 2)
+
+
+def _spawn(
+ fn,
+ world,
+ *args,
+ start_method = "spawn",
+):
+ with tempfile.TemporaryDirectory() as d:
+ init_file = os.path.join(d, "init")
+ mp.start_processes(
+ _entry,
+ args = (world, init_file, fn, args),
+ nprocs = world,
+ join = True,
+ start_method = start_method,
+ )
+
+
+def _entry(rank, world, init_file, fn, args):
+ torch.set_num_threads(1)
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
+ try:
+ fn(rank, world, *args)
+ finally:
+ dist.destroy_process_group()
+
+
+def _exchange_worker(rank, world):
+ torch.manual_seed(0)
+ B, H, T, D = 2, 2 * world, 3 * world, 4
+ full = torch.randn(B, H, T, D, dtype = torch.float64)
+ S = T // world
+ local = full[:, :, rank * S : (rank + 1) * S]
+ heads = U.seq_to_head(local, None)
+ h = H // world
+ torch.testing.assert_close(heads, full[:, rank * h : (rank + 1) * h], rtol = 0, atol = 0)
+ back = U.head_to_seq(heads.transpose(1, 2), None)
+ torch.testing.assert_close(back, local.transpose(1, 2), rtol = 0, atol = 0)
+
+
+def _attention_worker(rank, world, n_heads, n_kv, raw_len, lengths, dtype):
+ torch.manual_seed(1234)
+ B = 1 if lengths is not None else 2
+ D = 8
+ T = -(-raw_len // world) * world
+ Q = torch.randn(B, n_heads, T, D, dtype = dtype)
+ K = torch.randn(B, n_kv, T, D, dtype = dtype)
+ V = torch.randn(B, n_kv, T, D, dtype = dtype)
+ W = torch.randn(B, T, n_heads, D, dtype = dtype)
+ Q[:, :, raw_len:] = 0
+ K[:, :, raw_len:] = 0
+ V[:, :, raw_len:] = 0
+
+ Qr, Kr, Vr = (x.clone().requires_grad_(True) for x in (Q, K, V))
+ ref = _reference(Qr, Kr, Vr, lengths)
+ (ref * W).sum().backward()
+
+ S = T // world
+ sl = slice(rank * S, (rank + 1) * S)
+ Ql, Kl, Vl = (x[:, :, sl].clone().requires_grad_(True) for x in (Q, K, V))
+ seq_info = None
+ if lengths is not None:
+ lens = torch.tensor(lengths, dtype = torch.int32)
+ cu = torch.zeros(len(lengths) + 1, dtype = torch.int32)
+ cu[1:] = torch.cumsum(lens, 0)
+ seq_info = (lens, cu, int(lens.max()))
+ config = _Config("sdpa", n_kv_heads = n_kv, n_groups = n_heads // n_kv)
+ context = _Context(
+ bsz = B,
+ q_len = S,
+ kv_seq_len = S,
+ n_heads = n_heads,
+ head_dim = D,
+ requires_grad = True,
+ seq_info = seq_info,
+ attention_mask = None,
+ causal_mask = None,
+ )
+ out = U.ulysses_attention(_fake_run_attention, config, context, Ql, Kl, Vl, None)
+ assert out.shape == (B, S, n_heads, D)
+ tol = dict(rtol = 1e-10, atol = 1e-10) if dtype == torch.float64 else dict(rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(out, ref[:, sl].detach(), **tol)
+ (out * W[:, sl]).sum().backward()
+ torch.testing.assert_close(Ql.grad, Qr.grad[:, :, sl], **tol)
+ torch.testing.assert_close(Kl.grad, Kr.grad[:, :, sl], **tol)
+ torch.testing.assert_close(Vl.grad, Vr.grad[:, :, sl], **tol)
+
+
+_CASES = {
+ 2: [(4, 4, torch.float64), (4, 2, torch.float64), (4, 1, torch.float64)],
+ 4: [(8, 1, torch.float64), (8, 2, torch.float32), (8, 8, torch.float32)],
+}
+_PACKED = {2: (4, 2), 4: (8, 1)}
+
+
+def _all_cases_worker(rank, world):
+ _exchange_worker(rank, world)
+ for n_heads, n_kv, dtype in _CASES[world]:
+ _attention_worker(rank, world, n_heads, n_kv, 4 * world + 3, None, dtype)
+ raw_len = 6 * world + 1
+ lengths = [5, 3 * world - 2, raw_len - 3 * world - 3]
+ n_heads, n_kv = _PACKED[world]
+ _attention_worker(rank, world, n_heads, n_kv, raw_len, lengths, torch.float64)
+
+
[email protected]("world", [2, 4])
+def test_ulysses_exchange_and_attention_match_full_sequence(world):
+ _spawn(_all_cases_worker, world)
+
+
+def test_head_plan():
+ assert U.ulysses_head_plan(8, 8, 2) == (1, 4, 4, 1)
+ assert U.ulysses_head_plan(8, 2, 2) == (1, 4, 1, 4)
+ assert U.ulysses_head_plan(8, 2, 4) == (2, 2, 1, 2)
+ assert U.ulysses_head_plan(8, 1, 8) == (8, 1, 1, 1)
+ assert U.ulysses_head_plan(32, 8, 4) == (1, 8, 2, 4)
+ with pytest.raises(ValueError, match = "attention heads"):
+ U.ulysses_head_plan(6, 2, 4)
+ with pytest.raises(ValueError, match = "divide one another"):
+ U.ulysses_head_plan(24, 6, 4)
+ with pytest.raises(ValueError, match = "multiple of key/value"):
+ U.ulysses_head_plan(8, 3, 2)
+
+
+def test_ulysses_attention_refuses_masks_and_cache():
+ config = _Config("sdpa", n_kv_heads = 2, n_groups = 1)
+
+ def ctx(**kw):
+ base = dict(
+ bsz = 1,
+ q_len = 4,
+ kv_seq_len = 4,
+ n_heads = 2,
+ head_dim = 4,
+ requires_grad = False,
+ seq_info = None,
+ attention_mask = None,
+ causal_mask = None,
+ )
+ base.update(kw)
+ return _Context(**base)
+
+ x = torch.zeros(1, 2, 4, 4)
+ for bad, match in [
+ (ctx(attention_mask = torch.ones(1, 4)), "dense attention mask"),
+ (ctx(causal_mask = torch.ones(4, 4)), "dense attention mask"),
+ (ctx(prefix_seg_info = object()), "PrefixGrouper"),
+ (ctx(kv_seq_len = 8), "KV cache"),
+ ]:
+ with pytest.raises(RuntimeError, match = match):
+ U.ulysses_attention(_fake_run_attention, config, bad, x, x, x, None)
+
+
+def _real_dispatch_worker(rank, world, run_attention, AttentionConfig, AttentionContext):
+ torch.manual_seed(7)
+ B, H, Hkv, T, D = 2, 4, 2, 4 * world, 8
+ Q = torch.randn(B, H, T, D)
+ K = torch.randn(B, Hkv, T, D)
+ V = torch.randn(B, Hkv, T, D)
+ ref = _reference(Q, K, V)
+ S = T // world
+ sl = slice(rank * S, (rank + 1) * S)
+ config = AttentionConfig(backend = "sdpa", n_kv_heads = Hkv, n_groups = H // Hkv)
+ context = AttentionContext(
+ bsz = B,
+ q_len = S,
+ kv_seq_len = S,
+ n_heads = H,
+ head_dim = D,
+ requires_grad = False,
+ seq_info = None,
+ attention_mask = None,
+ causal_mask = None,
+ )
+ out = U.ulysses_attention(
+ run_attention, config, context, Q[:, :, sl], K[:, :, sl], V[:, :, sl], None
+ )
+ torch.testing.assert_close(out, ref[:, sl], rtol = 1e-5, atol = 1e-5)
+
+ lengths = torch.tensor([3, T - 5], dtype = torch.int32)
+ cu_seqlens = torch.zeros(3, dtype = torch.int32)
+ cu_seqlens[1:] = torch.cumsum(lengths, 0)
+ Q, K, V = Q[:1], K[:1], V[:1]
+ ref = _reference(Q, K, V, lengths.tolist())
+ context = AttentionContext(
+ bsz = 1,
+ q_len = S,
+ kv_seq_len = S,
+ n_heads = H,
+ head_dim = D,
+ requires_grad = False,
+ seq_info = (lengths, cu_seqlens, int(lengths.max())),
+ attention_mask = None,
+ causal_mask = None,
+ )
+ out = U.ulysses_attention(
+ run_attention, config, context, Q[:, :, sl], K[:, :, sl], V[:, :, sl], None
+ )
+ torch.testing.assert_close(out, ref[:, sl], rtol = 1e-5, atol = 1e-5)
+
+
+def test_real_run_attention_sdpa_path():
+ try:
+ from unsloth.utils.attention_dispatch import (
+ AttentionConfig,
+ AttentionContext,
+ _run_attention,
+ )
+ except Exception as exc: # the unsloth package import needs an accelerator
+ pytest.skip(f"unsloth.utils.attention_dispatch does not import here: {type(exc).__name__}")
+ # Forked, not spawned: a fresh interpreter would re-import unsloth without the test GPU spoof.
+ if "fork" not in mp.get_all_start_methods():
+ pytest.skip("needs the fork start method")
+ _spawn(
+ _real_dispatch_worker,
+ 2,
+ _run_attention,
+ AttentionConfig,
+ AttentionContext,
+ start_method = "fork",
+ )
diff --git a/unsloth-cli.py b/unsloth-cli.py
index ffdaad7ec26..92a85e7e2e3 100644
--- a/unsloth-cli.py
+++ b/unsloth-cli.py
@@ -477,14 +477,16 @@ if __name__ == "__main__":
"--context_parallel_size",
type = int,
default = 1,
- help = "Ranks per context parallel group (SDPA ring attention, torchrun, PyTorch >= 2.7).",
+ help = "Ranks per context parallel group (torchrun, PyTorch >= 2.7).",
)
training_group.add_argument(
"--context_parallel_backend",
type = str,
default = "ring",
- choices = ["ring"],
- help = "Context parallel attention backend.",
+ choices = ["ring", "ulysses"],
+ help = "Context parallel attention backend: ring (SDPA ring attention) or ulysses "
+ "(all-to-all over heads; keeps packing and the usual attention kernels, needs heads "
+ "divisible by the size).",
)
training_group.add_argument(
"--context_parallel_rotate_method",
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 1307178e448..58c2dfc41a5 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -84,14 +84,11 @@ def context_parallel_options(config) -> tuple:
"Unsloth: context_parallel_rotate_method must be one of "
f"{_ROTATE_METHODS}, got {rotate_method!r}."
)
- if size > 1 and backend == "ulysses":
- raise NotImplementedError(
- "Unsloth: context_parallel_backend = 'ulysses' is not available yet; use 'ring'."
- )
return size, backend, rotate_method
-def _ring_unsupported_attention(config) -> Optional[str]:
+def _unsupported_attention(config) -> Optional[str]:
+ # Ring and Llama's forward ignore mask, window and softcap, so both backends refuse them.
configs = [config]
text_config = getattr(config, "text_config", None)
if text_config is not None and text_config is not config:
@@ -127,6 +124,7 @@ def _unregister_cp_sharding_rules() -> None:
from .device_type import DEVICE_TYPE_TORCH
+from .utils.ulysses import _document_positions, ulysses_head_plan
# A module global, not a ContextVar: CUDA runs backward (and checkpoint recompute) on another thread.
_ACTIVE_MANAGER: Optional["ContextParallelManager"] = None
@@ -154,13 +152,15 @@ def _supports_context_parallel(model) -> bool:
def _self_attn_pre_forward_hook(_module, module_args, module_kwargs):
- # Ring attention only supports is_causal with no mask; right padding is already -100 in labels.
if get_cp_manager() is not None and "attention_mask" in module_kwargs:
module_kwargs["attention_mask"] = None
return module_args, module_kwargs
class ContextParallelManager:
+ backend = "ring"
+ rotate_method = "allgather"
+
def __init__(
self,
size: int,
@@ -178,6 +178,10 @@ class ContextParallelManager:
)
self.mesh = self.device_mesh["cp"]
+ @property
+ def group(self):
+ return self.mesh.get_group()
+
def attach_attention_hooks(self, model: torch.nn.Module) -> None:
for name, module in model.named_modules():
if name.endswith("self_attn") and not getattr(module, "_unsloth_cp_hooked", False):
@@ -193,7 +197,24 @@ class ContextParallelManager:
"Unsloth: context parallelism needs input_ids batches (not inputs_embeds)."
)
bsz, seq_len = input_ids.shape
- if inputs.get("position_ids") is None:
+ packed_seq_lengths = inputs.get("packed_seq_lengths")
+ ulysses = self.backend == "ulysses"
+ if packed_seq_lengths is not None and not ulysses:
+ raise ValueError("Unsloth: ring context parallelism does not support packing.")
+ if packed_seq_lengths is not None:
+ if bsz != 1:
+ raise ValueError(f"Unsloth: packed batches must be a single row, got {bsz} rows.")
+ used = int(torch.as_tensor(packed_seq_lengths).sum())
+ if used > seq_len:
+ raise ValueError(
+ f"Unsloth: packed_seq_lengths sum to {used} but the packed row has {seq_len} tokens."
+ )
+ # .get: a collator may carry the key with None, which would leave positions shard-local.
+ if inputs.get("position_ids") is None and packed_seq_lengths is not None:
+ inputs["position_ids"] = _document_positions(
+ packed_seq_lengths, seq_len, input_ids.device
+ )
+ elif inputs.get("position_ids") is None:
inputs["position_ids"] = (
torch.arange(seq_len, device = input_ids.device).expand(bsz, -1).contiguous()
)
@@ -206,8 +227,11 @@ class ContextParallelManager:
)
labels = inputs.get("labels")
if inputs.get("shift_labels") is None and labels is not None:
+ # Shard-local labels cannot see document starts on another shard: mask them here.
+ from .utils.packing import mask_packed_boundary_labels
+ labels = mask_packed_boundary_labels(labels, packed_seq_lengths)
inputs["shift_labels"] = F.pad(labels, (0, 1), value = -100)[:, 1:].contiguous()
- pad = (-seq_len) % (2 * self.size)
+ pad = (-seq_len) % (self.size if ulysses else 2 * self.size)
if pad:
for name in _BUFFER_NAMES:
tensor = inputs.get(name)
@@ -221,7 +245,8 @@ class ContextParallelManager:
@contextlib.contextmanager
def _rotate_method_scope(self) -> Iterator[None]:
- rotate_method = getattr(self, "rotate_method", "allgather")
+ # Process-wide torch state, read in forward and backward: set per step, restored after.
+ rotate_method = self.rotate_method
if _cp_options is None or set_rotate_method is None:
if rotate_method != "allgather":
raise RuntimeError(
@@ -236,9 +261,29 @@ class ContextParallelManager:
finally:
_cp_options.rotate_method = previous
+ @contextlib.contextmanager
+ def _ulysses_scope(self, inputs: dict) -> Iterator[None]:
+ # Contiguous shards; packed_seq_lengths stays global.
+ rank = self.mesh.get_local_rank()
+ for name in _BUFFER_NAMES:
+ tensor = inputs.get(name)
+ if isinstance(tensor, torch.Tensor) and tensor.ndim > 1:
+ local = tensor.shape[1] // self.size
+ inputs[name] = tensor[:, rank * local : (rank + 1) * local].contiguous()
+ global _ACTIVE_MANAGER
+ previous, _ACTIVE_MANAGER = _ACTIVE_MANAGER, self
+ try:
+ yield
+ finally:
+ _ACTIVE_MANAGER = previous
+
@contextlib.contextmanager
def apply(self, inputs: dict) -> Iterator[None]:
self._prepare_inputs(inputs)
+ if self.backend == "ulysses":
+ with self._ulysses_scope(inputs):
+ yield
+ return
buffers = [
inputs[name]
for name in _BUFFER_NAMES
@@ -387,7 +432,11 @@ def patch_sft_trainer() -> None:
raise RuntimeError(
f"Unsloth: world size {world_size} is not a multiple of context_parallel_size {size}."
)
- if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] < 8:
+ if (
+ backend == "ring"
+ and torch.cuda.is_available()
+ and torch.cuda.get_device_capability()[0] < 8
+ ):
# Pure torch ring attention fails in backward without flash (2x T4, torch 2.10).
raise NotImplementedError(
"Unsloth: context parallelism needs flash attention (compute capability >= 8.0)."
@@ -420,10 +469,20 @@ def patch_sft_trainer() -> None:
if Version(accelerate.__version__) < Version("1.10.0"):
raise NotImplementedError("Unsloth: context parallelism needs accelerate >= 1.10.0.")
- unsupported = _ring_unsupported_attention(getattr(self.model, "config", None))
+ model_config = getattr(self.model, "config", None)
+ unsupported = _unsupported_attention(model_config)
if unsupported is not None:
raise NotImplementedError(
- f"Unsloth: ring context parallelism does not support {unsupported}."
+ f"Unsloth: {backend} context parallelism does not support {unsupported}."
+ )
+ if backend == "ulysses":
+ # Every rank raises here, before any collective.
+ text_config = getattr(model_config, "text_config", None) or model_config
+ ulysses_head_plan(
+ text_config.num_attention_heads,
+ getattr(text_config, "num_key_value_heads", None)
+ or text_config.num_attention_heads,
+ size,
)
if not _supports_context_parallel(self.model):
raise NotImplementedError(
@@ -452,8 +511,8 @@ def patch_sft_trainer() -> None:
_scope_flash_attention_impl(self.model, self)
print(
- f"Unsloth: Context parallelism enabled with size = {size} "
- f"({backend}, rotate_method = {rotate_method})."
+ f"Unsloth: Context parallelism enabled with size = {size} ({backend}"
+ + (f", rotate_method = {rotate_method})." if backend == "ring" else ").")
)
@functools.wraps(original_prediction_step)
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 5a7e9c76d5e..dcd054682df 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -3325,11 +3325,11 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
max_seq_length_pre += """
context_parallel_size : Optional[int] = field(
default = 1,
- metadata = {'help': 'Ranks per context parallel group (SDPA ring attention). 1 disables it.'},
+ metadata = {'help': 'Ranks per context parallel group. 1 disables it.'},
)
context_parallel_backend : Optional[str] = field(
default = 'ring',
- metadata = {'help': 'Context parallel attention backend: ring.'},
+ metadata = {'help': 'Context parallel attention backend: ring or ulysses (keeps packing).'},
)
context_parallel_rotate_method : Optional[str] = field(
default = 'allgather',
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 72e914dd00b..99b598235e2 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -1863,14 +1863,16 @@ def _patch_sft_trainer_auto_packing(trl_module):
# several classes, so a "yes" from the resolved one is not proof about the
# instance. A correct "no" has already turned both flags off, which is the
# condition the post-init check skips on, so leaving it armed is free.
- is_context_parallel = (getattr(config_arg, "context_parallel_size", 1) or 1) > 1
+ is_ring_context_parallel = (getattr(config_arg, "context_parallel_size", 1) or 1) > 1 and (
+ getattr(config_arg, "context_parallel_backend", None) or "ring"
+ ) == "ring"
# Read before __init__: the compiled trainer ends it with model.for_inference(), which sets the flag.
eval_logits_wanted = os.environ.get(
"UNSLOTH_RETURN_LOGITS", "0"
) == "1" or _wants_eval_logits(original_init, args, kwargs)
blocked = (
(data_collator is not None)
- or is_context_parallel
+ or is_ring_context_parallel
or (is_processor and not processor_text_only)
or is_auto_processor_vlm
or is_vision_dataset
@@ -1886,7 +1888,8 @@ def _patch_sft_trainer_auto_packing(trl_module):
setattr(config_arg, "packing", False)
if hasattr(config_arg, "padding_free"):
setattr(config_arg, "padding_free", False)
- if is_context_parallel and getattr(config_arg, "eval_packing", None):
+ # TRL resolves eval_packing on its own; packed eval batches would hit ring attention's varlen refusal.
+ if is_ring_context_parallel and getattr(config_arg, "eval_packing", None):
setattr(config_arg, "eval_packing", False)
if blocked and requested_pack:
@@ -1903,8 +1906,8 @@ def _patch_sft_trainer_auto_packing(trl_module):
reason = "hybrid linear-attention model"
elif is_unsupported_model:
reason = f"unsupported model type(s): {', '.join(model_types)}"
- elif is_context_parallel:
- reason = "context parallelism"
+ elif is_ring_context_parallel:
+ reason = "ring context parallelism"
elif forward_rejects_packing:
# Name the real blocker, else this falls through to the
# UNSLOTH_RETURN_LOGITS branch and points at an unset flag. For a string
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 72d693997fc..25670f919f7 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -26,6 +26,7 @@ import torch.nn.functional as F
from ..device_type import DEVICE_TYPE
from ..models._utils import *
from ..context_parallel import get_cp_manager
+from .ulysses import ulysses_attention
from ..utils.packing import (
build_sdpa_packed_attention_mask,
build_xformers_block_causal_mask,
@@ -528,13 +529,19 @@ class AttentionContext:
is_causal: bool = True
+def _ring_attention_active() -> bool:
+ manager = get_cp_manager()
+ return manager is not None and manager.backend == "ring"
+
+
def select_attention_backend(use_varlen: bool = False) -> str:
"""Return attention backend based on availability / priority order."""
- if get_cp_manager() is not None:
+ if _ring_attention_active():
if use_varlen:
raise ValueError(
- "Unsloth: context parallelism does not support packing / padding-free."
+ "Unsloth: ring context parallelism does not support packing / padding-free; "
+ "use context_parallel_backend = 'ulysses'."
)
return SDPA
if HAS_FLASH_ATTENTION:
@@ -611,7 +618,15 @@ def run_attention(
Varlen flash is preferred for packed batches as it avoids padding; xFormers
and SDPA handle packing via a block-diagonal mask.
"""
+ manager = get_cp_manager()
+ if manager is not None and manager.backend == "ulysses":
+ return ulysses_attention(_run_attention, config, context, Q, K, V, manager.group)
+ return _run_attention(config = config, context = context, Q = Q, K = K, V = V)
+
+def _run_attention(
+ *, config: AttentionConfig, context: AttentionContext, Q: Tensor, K: Tensor, V: Tensor
+) -> Tensor:
# PrefixGrouper shared-prefix attention (GRPO dedup): Q/K/V here are [bsz, H, T, D] while the kernel
# takes and returns [1, T, H, D], matching the other backends.
if context.prefix_seg_info is not None:
@@ -732,11 +747,13 @@ def run_attention(
# DoRA promotes q/k/v_proj outputs to fp32, which FlashAttention rejects (as does the xformers
# flash-2 op on sm_100+), so downcast any fp32 Q/K/V to a supported dtype (#1013).
- cp_out_dtype = Q.dtype if backend == SDPA and get_cp_manager() is not None else None
+ # Ring attention (context parallelism) likewise needs a flash-eligible dtype.
+ ring = backend == SDPA and _ring_attention_active()
+ cp_out_dtype = Q.dtype if ring else None
if (
backend in (FLASH_DENSE, FLASH_VARLEN)
or (backend == XFORMERS and xformers_fp32_unsupported)
- or (backend == SDPA and get_cp_manager() is not None)
+ or ring
) and torch.float32 in (
Q.dtype,
K.dtype,
@@ -973,7 +990,7 @@ def run_attention(
# Flash takes GQA natively, halving the ring's K/V peak; UNSLOTH_CP_GQA=0 expands.
if (
use_sdpa_gqa
- and get_cp_manager() is not None
+ and _ring_attention_active()
and os.environ.get("UNSLOTH_CP_GQA", "1") == "0"
):
use_sdpa_gqa = False
diff --git a/unsloth/utils/ulysses.py b/unsloth/utils/ulysses.py
new file mode 100644
index 00000000000..36e03a37455
--- /dev/null
+++ b/unsloth/utils/ulysses.py
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Ulysses sequence parallelism: all-to-all from sequence shards to head shards around attention."""
+
+from dataclasses import replace
+from typing import NamedTuple
+
+import torch
+import torch.distributed as dist
+
+__all__ = [
+ "UlyssesHeadPlan",
+ "ulysses_head_plan",
+ "seq_to_head",
+ "head_to_seq",
+ "ulysses_attention",
+]
+
+
+def _all_to_all(x, scatter_dim, gather_dim, group):
+ world = dist.get_world_size(group)
+ if world == 1:
+ return x
+ if x.shape[scatter_dim] % world != 0:
+ raise ValueError(
+ f"Unsloth: Ulysses cannot split dim {scatter_dim} of size {x.shape[scatter_dim]} "
+ f"across {world} ranks."
+ )
+ send = torch.stack(x.chunk(world, dim = scatter_dim), dim = 0)
+ recv = torch.empty_like(send)
+ dist.all_to_all_single(recv, send, group = group)
+ return torch.cat(recv.unbind(0), dim = gather_dim)
+
+
+class _AllToAll(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x, scatter_dim, gather_dim, group):
+ ctx.scatter_dim = scatter_dim
+ ctx.gather_dim = gather_dim
+ ctx.group = group
+ return _all_to_all(x, scatter_dim, gather_dim, group)
+
+ @staticmethod
+ def backward(ctx, grad):
+ return _all_to_all(grad, ctx.gather_dim, ctx.scatter_dim, ctx.group), None, None, None
+
+
+def seq_to_head(x, group):
+ """[B, H, T/N, D] sequence shard -> [B, H/N, T, D] head shard (rank r gets heads r*H/N ...)."""
+ return _AllToAll.apply(x, 1, 2, group)
+
+
+def head_to_seq(x, group):
+ """run_attention output [B, T, H/N, D] -> [B, T/N, H, D] (the local shard's run_attention layout)."""
+ return _AllToAll.apply(x, 1, 2, group)
+
+
+class UlyssesHeadPlan(NamedTuple):
+ kv_replication: int
+ local_heads: int
+ local_kv_heads: int
+ local_groups: int
+
+
+def ulysses_head_plan(n_heads, n_kv_heads, cp_size):
+ if n_heads % cp_size != 0:
+ raise ValueError(
+ f"Unsloth: Ulysses context parallelism needs the number of attention heads ({n_heads}) "
+ f"to be divisible by context_parallel_size ({cp_size})."
+ )
+ if n_heads % n_kv_heads != 0:
+ raise ValueError(
+ f"Unsloth: attention heads ({n_heads}) must be a multiple of key/value heads ({n_kv_heads})."
+ )
+ if n_kv_heads % cp_size == 0:
+ replication = 1
+ elif cp_size % n_kv_heads == 0:
+ replication = cp_size // n_kv_heads
+ else:
+ raise ValueError(
+ f"Unsloth: Ulysses context parallelism needs key/value heads ({n_kv_heads}) and "
+ f"context_parallel_size ({cp_size}) to divide one another."
+ )
+ local_heads = n_heads // cp_size
+ local_kv_heads = n_kv_heads * replication // cp_size
+ return UlyssesHeadPlan(replication, local_heads, local_kv_heads, local_heads // local_kv_heads)
+
+
+def ulysses_attention(run_attention_fn, config, context, Q, K, V, group):
+ """Q [B, H, T/N, D], K/V [B, Hkv, T/N, D] (post RoPE) -> [B, T/N, H, D]."""
+ if context.attention_mask is not None or isinstance(context.causal_mask, torch.Tensor):
+ raise RuntimeError(
+ "Unsloth: Ulysses context parallelism does not take a dense attention mask; padding is "
+ "handled by ignoring padded labels and packing by global sequence lengths."
+ )
+ if context.prefix_seg_info is not None:
+ raise RuntimeError("Unsloth: Ulysses context parallelism does not support PrefixGrouper.")
+ if context.q_len != context.kv_seq_len:
+ raise RuntimeError("Unsloth: Ulysses context parallelism does not support a KV cache.")
+
+ size = dist.get_world_size(group)
+ plan = ulysses_head_plan(context.n_heads, config.n_kv_heads, size)
+ if plan.kv_replication > 1:
+ # Consecutive copies keep query head h on kv head h // n_groups after the split.
+ K = K.repeat_interleave(plan.kv_replication, dim = 1)
+ V = V.repeat_interleave(plan.kv_replication, dim = 1)
+
+ Q = seq_to_head(Q, group)
+ K = seq_to_head(K, group)
+ V = seq_to_head(V, group)
+
+ config = replace(config, n_kv_heads = plan.local_kv_heads, n_groups = plan.local_groups)
+ context = replace(
+ context,
+ q_len = context.q_len * size,
+ kv_seq_len = context.kv_seq_len * size,
+ n_heads = plan.local_heads,
+ )
+ A = run_attention_fn(config = config, context = context, Q = Q, K = K, V = V)
+ A = A.reshape(context.bsz, context.q_len, plan.local_heads, context.head_dim)
+ return head_to_seq(A, group)
+
+
+def _document_positions(packed_seq_lengths, total, device):
+ lengths = [int(x) for x in torch.as_tensor(packed_seq_lengths).reshape(-1).tolist()]
+ pieces = [torch.arange(length, device = device) for length in lengths]
+ used = sum(lengths)
+ if used < total:
+ pieces.append(torch.arange(total - used, device = device))
+ return torch.cat(pieces)[None]