Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add context parallelism support (SDPA only) (#4257)

Daniel Han493ac4dfceaf
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
new file mode 100644
index 00000000000..4bc322af359
--- /dev/null
+++ b/tests/test_context_parallel.py
@@ -0,0 +1,1052 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Context parallelism plumbing that runs without a process group (the ring itself needs >= 2 GPUs)."""
+
+import contextlib
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+from unsloth.utils import attention_dispatch as ad
+
+
+def _manager(size = 2):
+ manager = object.__new__(cp.ContextParallelManager)
+ manager.size = size
+ return manager
+
+
+def _fake_context_parallel(sharded):
+ @contextlib.contextmanager
+ def fake(mesh, buffers, buffer_seq_dims, no_restore_buffers):
+ sharded.extend(buffers)
+ yield
+
+ return fake
+
+
+def test_none_position_ids_and_shift_labels_are_rebuilt():
+ inputs = {
+ "input_ids": torch.ones(1, 4, dtype = torch.long),
+ "labels": torch.tensor([[1, 2, 3, 4]]),
+ "position_ids": None,
+ "shift_labels": None,
+ }
+ _manager()._prepare_inputs(inputs)
+ assert inputs["position_ids"].tolist() == [[0, 1, 2, 3]]
+ assert inputs["shift_labels"].tolist() == [[2, 3, 4, -100]]
+
+
+def test_labels_shift_before_sharding_and_pad_to_load_balancer_chunks():
+ manager = _manager(size = 2)
+ labels = torch.tensor([[-100, 5, 6, 7, 8, 9]])
+ inputs = {"input_ids": torch.tensor([[1, 5, 6, 7, 8, 9]]), "labels": labels.clone()}
+ manager._prepare_inputs(inputs)
+ assert inputs["shift_labels"].tolist() == [[5, 6, 7, 8, 9, -100, -100, -100]]
+ assert inputs["labels"].tolist() == [[-100, 5, 6, 7, 8, 9, -100, -100]]
+ assert inputs["position_ids"].tolist() == [[0, 1, 2, 3, 4, 5, 6, 7]]
+ assert inputs["input_ids"].shape == (1, 8)
+
+
[email protected]("mask", [[[0, 1, 1, 1]], [[1, 0, 1, 1]], [[[1, 1, 1, 1]] * 4]])
+def test_left_padded_or_holed_masks_are_refused(mask):
+ inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long), "attention_mask": torch.tensor(mask)}
+ with pytest.raises(ValueError, match = "right-padded"):
+ _manager()._prepare_inputs(inputs)
+ right = {
+ "input_ids": torch.ones(1, 4, dtype = torch.long),
+ "attention_mask": torch.tensor([[1, 1, 0, 0]]),
+ }
+ _manager()._prepare_inputs(right)
+
+
+def test_inputs_embeds_batches_are_refused():
+ with pytest.raises(ValueError, match = "input_ids"):
+ _manager()._prepare_inputs(
+ {"inputs_embeds": torch.zeros(1, 4, 8), "labels": torch.ones(1, 4)}
+ )
+
+
+def test_active_manager_resets_when_the_step_raises(monkeypatch):
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ manager = _manager()
+ manager.mesh = None
+ inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long)}
+ with pytest.raises(RuntimeError, match = "boom"):
+ with manager.apply(inputs):
+ assert cp.get_cp_manager() is manager
+ raise RuntimeError("boom")
+ assert cp.get_cp_manager() is None
+
+
+def test_attention_hook_only_drops_the_mask_inside_a_cp_step(monkeypatch):
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ mask = torch.ones(1, 4)
+ _, kwargs = cp._self_attn_pre_forward_hook(None, (), {"attention_mask": mask})
+ assert kwargs["attention_mask"] is mask
+ manager = _manager()
+ manager.mesh = None
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ _, kwargs = cp._self_attn_pre_forward_hook(None, (), {"attention_mask": mask})
+ assert kwargs["attention_mask"] is None
+
+
+def test_backend_is_sdpa_under_cp_and_packing_is_refused(monkeypatch):
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ manager = _manager()
+ manager.mesh = None
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ assert ad.select_attention_backend(False) == ad.SDPA
+ with pytest.raises(ValueError, match = "packing"):
+ ad.select_attention_backend(True)
+
+
+def test_sdpa_is_looked_up_per_call(monkeypatch):
+ # A name bound at import would keep attending to the local shard only.
+ calls = []
+ monkeypatch.setattr(
+ torch.nn.functional, "scaled_dot_product_attention", lambda *a, **k: calls.append(1)
+ )
+ ad.scaled_dot_product_attention(None, None, None)
+ assert calls == [1]
+
+
+def test_training_step_divides_the_pre_shard_token_count_by_cp_size(monkeypatch):
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ seen = []
+
+ class Trainer:
+ def __init__(self):
+ pass
+
+ def training_step(
+ self,
+ model,
+ inputs,
+ num_items_in_batch = None,
+ ):
+ seen.append(("train", num_items_in_batch))
+
+ def prediction_step(self, model, inputs, prediction_loss_only):
+ return (None, None, None)
+
+ def train(self):
+ pass
+
+ def evaluate(self):
+ pass
+
+ def evaluation_loop(self, *a, **k):
+ pass
+
+ import trl
+
+ monkeypatch.setattr(trl, "SFTTrainer", Trainer)
+ cp.patch_sft_trainer()
+ trainer = Trainer()
+ manager = _manager(size = 4)
+ manager.mesh = None
+ trainer._context_parallel_manager = manager
+ batch = lambda: {"input_ids": torch.ones(1, 8, dtype = torch.long)}
+ trainer.training_step(None, batch(), torch.tensor(12.0))
+ trainer.training_step(None, batch(), num_items_in_batch = torch.tensor(12.0))
+ trainer.training_step(None, batch(), torch.tensor(12))
+ assert [(k, float(n)) for k, n in seen] == [("train", 3.0)] * 3
+ assert seen[-1][1].dtype == torch.int64
+
+
+def test_shift_labels_is_not_an_eval_label_name():
+ # A named shift_labels made find_labels treat it as a required eval label.
+ from transformers.utils import find_labels
+ from unsloth.models.llama import CausalLM_fast_forward, LlamaModel_fast_forward_inference
+
+ class LM(torch.nn.Module):
+ forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference)
+
+ assert find_labels(LM) == ["labels"]
+
+
[email protected]("opt_out", [False, True])
+def test_gqa_stays_native_under_cp_unless_opted_out(monkeypatch, opt_out):
+ if opt_out:
+ monkeypatch.setenv("UNSLOTH_CP_GQA", "0")
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ monkeypatch.setattr(ad, "SDPA_HAS_GQA", True)
+ seen = []
+ monkeypatch.setattr(
+ ad,
+ "scaled_dot_product_attention",
+ lambda Q, K, V, **k: (seen.append((K.shape[1], k.get("enable_gqa"))), Q)[1],
+ )
+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 2, n_groups = 2)
+ context = ad.AttentionContext(
+ bsz = 1,
+ q_len = 4,
+ kv_seq_len = 4,
+ n_heads = 4,
+ head_dim = 8,
+ requires_grad = True,
+ seq_info = None,
+ attention_mask = None,
+ causal_mask = None,
+ )
+ Q, K = torch.zeros(1, 4, 4, 8), torch.zeros(1, 2, 4, 8)
+ manager = _manager()
+ manager.mesh = None
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ ad.run_attention(config = config, context = context, Q = Q, K = K, V = K)
+ assert seen == ([(4, None)] if opt_out else [(2, True)])
+
+
+def test_every_rank_builds_the_same_global_mesh(monkeypatch):
+ built = []
+
+ class FakeMesh:
+ def __init__(
+ self,
+ device_type,
+ mesh,
+ mesh_dim_names = None,
+ ):
+ built.append((mesh.tolist(), mesh_dim_names))
+
+ def __getitem__(self, name):
+ return ("submesh", name)
+
+ monkeypatch.setattr(cp, "DeviceMesh", FakeMesh)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 4)
+ for rank in range(4):
+ monkeypatch.setattr(cp.dist, "get_rank", lambda rank = rank: rank)
+ manager = cp.ContextParallelManager(2)
+ assert manager.mesh == ("submesh", "cp")
+ assert built == [([[0, 1], [2, 3]], ("dp_replicate", "cp"))] * 4
+
+
+def _patched_trainer(monkeypatch, **init_attrs):
+ import trl
+
+ class Trainer:
+ def __init__(self, *a, **k):
+ for key, value in init_attrs.items():
+ setattr(self, key, value)
+
+ def training_step(
+ self,
+ model,
+ inputs,
+ num_items_in_batch = None,
+ ):
+ pass
+
+ def prediction_step(
+ self,
+ model,
+ inputs,
+ prediction_loss_only,
+ ignore_keys = None,
+ ):
+ return (torch.tensor(1.0), None, None)
+
+ def _prepare_inputs(self, inputs):
+ return inputs
+
+ def compute_loss_context_manager(self):
+ return contextlib.nullcontext()
+
+ def compute_loss(self, model, inputs):
+ assert inputs["use_cache"] is False
+ targets = inputs["shift_labels"].ne(-100).sum()
+ return targets.float() / inputs["num_items_in_batch"]
+
+ def train(self, *a, **k):
+ return "trained"
+
+ def evaluate(self, *a, **k):
+ return "evaluated"
+
+ def predict(self, *a, **k):
+ return "predicted"
+
+ def evaluation_loop(self, *a, **k):
+ return self.gather_function(torch.arange(1.0, 3.0))
+
+ monkeypatch.setattr(trl, "SFTTrainer", Trainer)
+ cp.patch_sft_trainer()
+ return Trainer
+
+
[email protected]("world_size", [2, 4])
+def test_eval_loss_is_the_token_mean_even_with_an_empty_shard(monkeypatch, world_size):
+ import types
+
+ @contextlib.contextmanager
+ def first_half(mesh, buffers, buffer_seq_dims, no_restore_buffers):
+ for buffer in buffers:
+ buffer.data = buffer[:, : buffer.shape[1] // 2].clone()
+ yield
+
+ monkeypatch.setattr(cp, "context_parallel", first_half)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: world_size)
+ Trainer = _patched_trainer(monkeypatch)
+ trainer = object.__new__(Trainer)
+ manager = _manager(size = 2)
+ manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")
+ trainer._context_parallel_manager = manager
+ counts = []
+
+ def all_reduce(tensor, group = None):
+ assert group == "cp"
+ counts.append(group)
+ tensor.add_(2 / 2)
+
+ monkeypatch.setattr(cp.dist, "all_reduce", all_reduce)
+ labels = torch.tensor([[-100] * 7 + [1, 1]])
+ batch = lambda: {"input_ids": torch.ones(1, 9, dtype = torch.long), "labels": labels.clone()}
+ loss, logits, labels_out = trainer.prediction_step(None, batch(), True)
+ assert counts == ["cp"] and loss.item() == pytest.approx(1.0)
+ assert logits is None and labels_out is None
+ with pytest.raises(NotImplementedError, match = "loss-only"):
+ trainer.prediction_step(None, batch(), False)
+
+
+ "attrs",
+ [
+ {"label_smoothing_factor": 0.1},
+ {"compute_loss_func": lambda *a, **k: 0},
+ {"loss_type": "chunked_nll"},
+ ],
+)
+def test_label_dropping_loss_paths_are_refused(monkeypatch, attrs):
+ import types
+
+ monkeypatch.setattr(cp.dist, "is_available", lambda: True)
+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
+ for key in ("label_smoothing_factor", "loss_type"):
+ if key in attrs:
+ setattr(args, key, attrs[key])
+ init = {"args": args}
+ if "compute_loss_func" in attrs:
+ init["compute_loss_func"] = attrs["compute_loss_func"]
+ Trainer = _patched_trainer(monkeypatch, **init)
+ with pytest.raises(NotImplementedError, match = "loss_type = 'nll'"):
+ Trainer()
+
+
+def test_unaveraged_token_counts_are_refused(monkeypatch):
+ import types
+
+ monkeypatch.setattr(cp.dist, "is_available", lambda: True)
+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ args = types.SimpleNamespace(
+ context_parallel_size = 2,
+ label_smoothing_factor = 0.0,
+ average_tokens_across_devices = False,
+ )
+ Trainer = _patched_trainer(monkeypatch, args = args)
+ with pytest.raises(NotImplementedError, match = "average_tokens_across_devices"):
+ Trainer()
+
+
+def test_old_accelerate_is_refused(monkeypatch):
+ import types, accelerate
+
+ monkeypatch.setattr(cp.dist, "is_available", lambda: True)
+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(accelerate, "__version__", "1.9.0")
+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
+ Trainer = _patched_trainer(monkeypatch, args = args)
+ with pytest.raises(NotImplementedError, match = "accelerate >= 1.10.0"):
+ Trainer()
+
+
+def test_fp32_qkv_is_downcast_under_cp(monkeypatch):
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ seen = []
+ monkeypatch.setattr(
+ ad, "scaled_dot_product_attention", lambda Q, K, V, **k: (seen.append(Q.dtype), Q)[1]
+ )
+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 4, n_groups = 1)
+ context = ad.AttentionContext(
+ bsz = 1,
+ q_len = 4,
+ kv_seq_len = 4,
+ n_heads = 4,
+ head_dim = 8,
+ requires_grad = True,
+ seq_info = None,
+ attention_mask = None,
+ causal_mask = None,
+ )
+ Q = torch.zeros(1, 4, 4, 8, dtype = torch.float32)
+ ad.run_attention(config = config, context = context, Q = Q, K = Q, V = Q)
+ manager = _manager()
+ manager.mesh = None
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ out = ad.run_attention(config = config, context = context, Q = Q, K = Q, V = Q)
+ assert seen[0] == torch.float32 and seen[1] in (torch.bfloat16, torch.float16)
+ assert out.dtype == torch.float32
+
+
[email protected]("distributed_type", ["DEEPSPEED", "FSDP"])
+def test_deepspeed_and_fsdp_are_refused(monkeypatch, distributed_type):
+ import types
+
+ monkeypatch.setattr(cp.dist, "is_available", lambda: True)
+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(cp, "_supports_context_parallel", lambda model: True)
+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
+ accelerator = types.SimpleNamespace(
+ distributed_type = types.SimpleNamespace(name = distributed_type)
+ )
+ Trainer = _patched_trainer(monkeypatch, args = args, accelerator = accelerator, model = None)
+ with pytest.raises(NotImplementedError, match = "DDP only"):
+ Trainer()
+
+
+def _cp_env(monkeypatch):
+ monkeypatch.setattr(cp.dist, "is_available", lambda: True)
+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(cp, "_supports_context_parallel", lambda model: True)
+
+
[email protected]("case", ["iterable", "dispatch", "eval_dict"])
+def test_dispatched_or_iterable_loaders_are_refused(monkeypatch, case):
+ import types
+
+ _cp_env(monkeypatch)
+
+ class Stream(torch.utils.data.IterableDataset):
+ def __iter__(self):
+ return iter(())
+
+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
+ accelerator = types.SimpleNamespace(
+ distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),
+ dispatch_batches = case == "dispatch",
+ )
+ Trainer = _patched_trainer(
+ monkeypatch,
+ args = args,
+ accelerator = accelerator,
+ model = None,
+ train_dataset = Stream() if case == "iterable" else [1],
+ eval_dataset = {"a": [1], "b": Stream()} if case == "eval_dict" else None,
+ )
+ with pytest.raises(NotImplementedError, match = "iterable datasets|dispatch_batches"):
+ Trainer()
+
+
+def test_iterable_eval_dataset_passed_to_evaluate_is_refused(monkeypatch):
+ class Stream(torch.utils.data.IterableDataset):
+ def __iter__(self):
+ return iter(())
+
+ Trainer = _patched_trainer(monkeypatch)
+ trainer = object.__new__(Trainer)
+ trainer._context_parallel_manager = _manager()
+ with pytest.raises(NotImplementedError, match = "iterable datasets"):
+ trainer.evaluate(Stream())
+ with pytest.raises(NotImplementedError, match = "iterable datasets"):
+ trainer.evaluate(eval_dataset = {"a": Stream()})
+ trainer._context_parallel_manager = None
+ assert trainer.evaluate(Stream()) == "evaluated"
+
+
+def test_iterable_predict_dataset_is_refused(monkeypatch):
+ class Stream(torch.utils.data.IterableDataset):
+ def __iter__(self):
+ return iter(())
+
+ Trainer = _patched_trainer(monkeypatch)
+ trainer = object.__new__(Trainer)
+ trainer._context_parallel_manager = _manager()
+ with pytest.raises(NotImplementedError, match = "iterable datasets"):
+ trainer.predict(Stream())
+ trainer._context_parallel_manager = None
+ assert trainer.predict(Stream()) == "predicted"
+
+
+def test_sliding_window_models_are_refused(monkeypatch):
+ import types
+
+ _cp_env(monkeypatch)
+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
+ accelerator = types.SimpleNamespace(distributed_type = types.SimpleNamespace(name = "MULTI_GPU"))
+ model = types.SimpleNamespace(config = types.SimpleNamespace(sliding_window = 4096))
+ Trainer = _patched_trainer(monkeypatch, args = args, accelerator = accelerator, model = model)
+ with pytest.raises(NotImplementedError, match = "sliding window"):
+ Trainer()
+
+
+def test_a_later_non_cp_trainer_drops_the_installed_mesh(monkeypatch):
+ import types
+
+ mesh = object()
+ monkeypatch.setattr(cp, "_INSTALLED_MESH", [mesh])
+ state = types.SimpleNamespace(device_mesh = mesh)
+ args = types.SimpleNamespace(context_parallel_size = 1)
+ Trainer = _patched_trainer(
+ monkeypatch, args = args, accelerator = types.SimpleNamespace(state = state)
+ )
+ Trainer()
+ assert state.device_mesh is None
+ other = object()
+ state.device_mesh = other
+ Trainer()
+ assert state.device_mesh is other
+
+
+def test_every_attention_layer_must_be_the_llama_forward():
+ from unsloth.models.llama import LlamaAttention_fast_forward
+
+ class Llama(torch.nn.Module):
+ forward = LlamaAttention_fast_forward
+
+ class Other(torch.nn.Module):
+ def forward(self, x):
+ return x
+
+ def model(*attns):
+ root = torch.nn.Module()
+ root.layers = torch.nn.ModuleList()
+ for attn in attns:
+ layer = torch.nn.Module()
+ layer.self_attn = attn()
+ root.layers.append(layer)
+ return root
+
+ assert cp._supports_context_parallel(model(Llama, Llama))
+ assert not cp._supports_context_parallel(model(Llama, Other))
+ assert not cp._supports_context_parallel(model())
+
+
[email protected]("user_value", [None, True])
+def test_find_unused_parameters_is_off_under_cp_unless_the_user_set_it(monkeypatch, user_value):
+ import types
+
+ _cp_env(monkeypatch)
+ monkeypatch.setattr(
+ cp,
+ "ContextParallelManager",
+ lambda size, *rest: types.SimpleNamespace(
+ size = size, device_mesh = object(), attach_attention_hooks = lambda model: None
+ ),
+ )
+ args = types.SimpleNamespace(
+ context_parallel_size = 2,
+ label_smoothing_factor = 0.0,
+ ddp_find_unused_parameters = user_value,
+ )
+ handler = types.SimpleNamespace(
+ find_unused_parameters = True if user_value is None else user_value
+ )
+ accelerator = types.SimpleNamespace(
+ distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),
+ state = types.SimpleNamespace(device_mesh = None),
+ ddp_handler = handler,
+ )
+ Trainer = _patched_trainer(
+ monkeypatch, args = args, accelerator = accelerator, model = None, train_dataset = [1]
+ )
+ Trainer()
+ expected = False if user_value is None else True
+ # 5.x reads the handler built at init; 4.x rebuilds it from the argument at train time.
+ assert handler.find_unused_parameters is expected
+ assert args.ddp_find_unused_parameters is expected
+
+
+def test_active_manager_is_visible_from_the_autograd_device_thread(monkeypatch):
+ import threading
+
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ manager = _manager()
+ manager.mesh = None
+ seen = []
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ thread = threading.Thread(target = lambda: seen.append(cp.get_cp_manager()))
+ thread.start()
+ thread.join()
+ assert seen == [manager]
+ assert cp.get_cp_manager() is None
+
+
+def test_transformers_cp_size_is_refused(monkeypatch):
+ import types
+
+ args = types.SimpleNamespace(
+ context_parallel_size = 1, parallelism_config = types.SimpleNamespace(cp_size = 2)
+ )
+ Trainer = _patched_trainer(monkeypatch, args = args)
+ with pytest.raises(NotImplementedError, match = "context_parallel_size"):
+ Trainer()
+
+
+def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):
+ import types
+
+ mesh = object()
+ state = types.SimpleNamespace(device_mesh = None)
+ Trainer = _patched_trainer(monkeypatch, args = types.SimpleNamespace(context_parallel_size = 1))
+ trainer = object.__new__(Trainer)
+ trainer.accelerator = types.SimpleNamespace(state = state)
+ trainer.args = types.SimpleNamespace(ddp_find_unused_parameters = None)
+ trainer._context_parallel_manager = types.SimpleNamespace(size = 2, device_mesh = mesh)
+ assert trainer.train() == "trained"
+ assert state.device_mesh is mesh
+ assert trainer.args.ddp_find_unused_parameters is False
+
+
+def test_attention_hooks_are_not_stacked_by_a_second_manager():
+ model = torch.nn.Module()
+ model.self_attn = torch.nn.Linear(2, 2)
+ for _ in range(2):
+ _manager().attach_attention_hooks(model)
+ assert len(model.self_attn._forward_pre_hooks) == 1
+
+
+def test_sdpa_is_restored_when_the_cp_step_raises(monkeypatch):
+ import torch.nn.functional as F
+
+ original = F.scaled_dot_product_attention
+
+ @contextlib.contextmanager
+ def leaky(mesh, buffers, buffer_seq_dims, no_restore_buffers):
+ F.scaled_dot_product_attention = lambda *a, **k: None # torch < 2.13
+ yield
+ F.scaled_dot_product_attention = original
+
+ monkeypatch.setattr(cp, "context_parallel", leaky)
+ manager = _manager()
+ manager.mesh = None
+ with pytest.raises(RuntimeError):
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ raise RuntimeError("OOM")
+ assert F.scaled_dot_product_attention is original
+
+
[email protected]("remainder", [0, 3])
+def test_eval_gather_keeps_one_loss_per_cp_group(monkeypatch, remainder):
+ import types
+
+ per_rank = [torch.tensor([10.0 * r + 1, 10.0 * r + 2]) for r in (0, 0, 1, 1, 2, 2)]
+ accelerator = types.SimpleNamespace(
+ gather = lambda t: torch.cat(per_rank),
+ gather_for_metrics = lambda t, *a, **k: t,
+ gradient_state = types.SimpleNamespace(end_of_dataloader = True, remainder = remainder),
+ )
+ Trainer = _patched_trainer(monkeypatch)
+ trainer = object.__new__(Trainer)
+ trainer.accelerator = accelerator
+ trainer._context_parallel_manager = _manager(size = 2)
+ out = trainer.evaluation_loop()
+ expected = [1.0, 2.0, 11.0, 12.0, 21.0, 22.0]
+ assert out.tolist() == (expected[:remainder] if remainder else expected)
+
+
+def test_replaced_iterable_train_dataset_is_refused_at_train(monkeypatch):
+ class Stream(torch.utils.data.IterableDataset):
+ def __iter__(self):
+ return iter(())
+
+ Trainer = _patched_trainer(monkeypatch)
+ trainer = object.__new__(Trainer)
+ trainer._context_parallel_manager = _manager()
+ trainer.train_dataset = Stream()
+ with pytest.raises(NotImplementedError, match = "iterable datasets"):
+ trainer.train()
+
+
+def test_null_inputs_embeds_placeholder_is_allowed():
+ inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long), "inputs_embeds": None}
+ _manager()._prepare_inputs(inputs)
+ assert inputs["position_ids"].shape == (1, 4)
+
+
+def test_replaced_iterable_eval_dataset_is_refused_at_evaluate(monkeypatch):
+ class Stream(torch.utils.data.IterableDataset):
+ def __iter__(self):
+ return iter(())
+
+ Trainer = _patched_trainer(monkeypatch)
+ trainer = object.__new__(Trainer)
+ trainer._context_parallel_manager = _manager()
+ trainer.eval_dataset = Stream()
+ with pytest.raises(NotImplementedError, match = "iterable datasets"):
+ trainer.evaluate()
+ trainer.eval_dataset = {"a": [1], "b": Stream()}
+ with pytest.raises(NotImplementedError, match = "iterable datasets"):
+ trainer.evaluate("b")
+ assert trainer.evaluate("a") == "evaluated"
+
+
+def test_model_init_is_refused(monkeypatch):
+ import types
+
+ _cp_env(monkeypatch)
+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
+ Trainer = _patched_trainer(monkeypatch, args = args, model_init = lambda: None)
+ with pytest.raises(NotImplementedError, match = "model_init"):
+ Trainer()
+
+
+def test_cp_dtensor_dispatcher_is_torn_down_when_the_step_raises(monkeypatch):
+ dispatcher = cp.DTensor._op_dispatcher
+ original = dispatcher._custom_op_handlers
+ unregistered = []
+ monkeypatch.setattr(cp, "_unregister_cp_sharding_rules", lambda: unregistered.append(1))
+
+ @contextlib.contextmanager
+ def leaky(mesh, buffers, buffer_seq_dims, no_restore_buffers):
+ dispatcher._custom_op_handlers = {**original, "cp": None} # torch < 2.13: no finally
+ yield
+ dispatcher._custom_op_handlers = original
+
+ monkeypatch.setattr(cp, "context_parallel", leaky)
+ manager = _manager()
+ manager.mesh = None
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ pass
+ assert dispatcher._custom_op_handlers is original and unregistered == []
+ with pytest.raises(RuntimeError):
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ raise RuntimeError("OOM")
+ assert dispatcher._custom_op_handlers is original and unregistered == [1]
+
+
+def _gated_trainer(
+ monkeypatch,
+ model = None,
+ **args_overrides,
+):
+ """A trainer that clears every CP gate, with the manager stubbed (no process group)."""
+ import types
+
+ _cp_env(monkeypatch)
+ built = []
+
+ def manager(
+ size,
+ backend = "ring",
+ rotate_method = "allgather",
+ ):
+ built.append((size, backend, rotate_method))
+ return types.SimpleNamespace(
+ size = size, device_mesh = object(), attach_attention_hooks = lambda model: None
+ )
+
+ monkeypatch.setattr(cp, "ContextParallelManager", manager)
+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
+ for key, value in args_overrides.items():
+ setattr(args, key, value)
+ accelerator = types.SimpleNamespace(
+ distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),
+ state = types.SimpleNamespace(device_mesh = None),
+ )
+ init = {"args": args, "accelerator": accelerator, "model": model, "train_dataset": [1]}
+ return _patched_trainer(monkeypatch, **init), args, built
+
+
+ "overrides, error",
+ [
+ ({"context_parallel_size": True}, ValueError),
+ ({"context_parallel_size": 0}, ValueError),
+ ({"context_parallel_size": -2}, ValueError),
+ ({"context_parallel_size": 2.0}, ValueError),
+ ({"context_parallel_size": "2"}, ValueError),
+ ({"context_parallel_backend": "tree"}, ValueError),
+ ({"context_parallel_rotate_method": "ring"}, ValueError),
+ ({"context_parallel_backend": "ulysses"}, NotImplementedError),
+ ],
+)
+def test_bad_cp_options_fail_before_the_trainer_builds(monkeypatch, overrides, error):
+ import types, trl
+
+ seen = []
+
+ class Trainer:
+ def __init__(self, *a, **k):
+ seen.append(1)
+
+ training_step = prediction_step = train = evaluate = evaluation_loop = None
+
+ monkeypatch.setattr(trl, "SFTTrainer", Trainer)
+ cp.patch_sft_trainer()
+ args = types.SimpleNamespace(**{"context_parallel_size": 2, **overrides})
+ with pytest.raises(error):
+ Trainer(None, args)
+ with pytest.raises(error):
+ Trainer(None, args = args)
+ assert seen == []
+
+
+def test_cp_option_defaults_and_size_one():
+ import types
+
+ assert cp.context_parallel_options(None) == (1, "ring", "allgather")
+ config = types.SimpleNamespace(context_parallel_size = 1, context_parallel_backend = "ulysses")
+ assert cp.context_parallel_options(config) == (1, "ulysses", "allgather")
+ config = types.SimpleNamespace(
+ context_parallel_size = 4, context_parallel_rotate_method = "alltoall"
+ )
+ assert cp.context_parallel_options(config) == (4, "ring", "alltoall")
+
+
+def test_manager_gets_the_backend_and_rotate_method(monkeypatch):
+ Trainer, _, built = _gated_trainer(monkeypatch, context_parallel_rotate_method = "alltoall")
+ Trainer()
+ assert built == [(2, "ring", "alltoall")]
+
+
[email protected]("version", ["2.6.0", "2.6.0+cu124", "2.5.1"])
+def test_torch_older_than_2_7_is_refused(monkeypatch, version):
+ Trainer, _, _ = _gated_trainer(monkeypatch)
+ monkeypatch.setattr(cp.torch, "__version__", version)
+ with pytest.raises(RuntimeError, match = "PyTorch >= 2.7"):
+ Trainer()
+
+
+def test_torch_2_7_dev_builds_pass_the_gate(monkeypatch):
+ monkeypatch.setattr(cp.torch, "__version__", "2.7.0a0+git1234")
+ assert cp._torch_supports_context_parallel()
+
+
+ "config, reason",
+ [
+ ({"layer_types": ["full_attention", "sliding_attention"]}, "sliding_attention"),
+ ({"layer_types": ["chunked_attention", "full_attention"]}, "chunked_attention"),
+ ({"attn_logit_softcapping": 50.0}, "softcapping"),
+ ({"attention_chunk_size": 8192}, "chunked attention"),
+ ({"use_sliding_window": True}, "sliding window"),
+ ({"text_config": {"layer_types": ["sliding_attention"]}}, "sliding_attention"),
+ ],
+)
+def test_windowed_or_softcapped_attention_is_refused_for_ring(monkeypatch, config, reason):
+ import types
+
+ if "text_config" in config:
+ config = {"text_config": types.SimpleNamespace(**config["text_config"])}
+ model = types.SimpleNamespace(config = types.SimpleNamespace(**config))
+ Trainer, _, _ = _gated_trainer(monkeypatch, model = model)
+ with pytest.raises(NotImplementedError, match = reason):
+ Trainer()
+
+
+def test_full_attention_layer_types_pass(monkeypatch):
+ import types
+
+ config = types.SimpleNamespace(
+ layer_types = ["full_attention"] * 2, sliding_window = None, use_sliding_window = False
+ )
+ Trainer, _, built = _gated_trainer(monkeypatch, model = types.SimpleNamespace(config = config))
+ Trainer()
+ assert built
+
+
+def test_legacy_prediction_loop_is_refused(monkeypatch):
+ Trainer, _, built = _gated_trainer(monkeypatch, use_legacy_prediction_loop = True)
+ with pytest.raises(NotImplementedError, match = "use_legacy_prediction_loop"):
+ Trainer()
+ assert built == []
+
+
+def test_model_without_loss_kwargs_is_refused(monkeypatch):
+ Trainer, _, _ = _gated_trainer(monkeypatch)
+ monkeypatch.setattr(Trainer, "model_accepts_loss_kwargs", False, raising = False)
+ with pytest.raises(NotImplementedError, match = "loss kwargs"):
+ Trainer()
+
+
+def test_flash_attention_impl_is_scoped_to_train_without_packing(monkeypatch):
+ from unsloth.utils import packing
+
+ scoped = []
+ monkeypatch.setattr(
+ packing, "_scope_flash_attention_impl", lambda model, trainer: scoped.append(trainer)
+ )
+ Trainer, _, _ = _gated_trainer(monkeypatch)
+ trainer = Trainer()
+ assert scoped == [trainer]
+
+
+def test_training_step_refuses_a_missing_token_count_with_labels(monkeypatch):
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ Trainer = _patched_trainer(monkeypatch)
+ trainer = object.__new__(Trainer)
+ manager = _manager()
+ manager.mesh = None
+ trainer._context_parallel_manager = manager
+ batch = {"input_ids": torch.ones(1, 4, dtype = torch.long), "labels": torch.ones(1, 4)}
+ with pytest.raises(RuntimeError, match = "num_items_in_batch"):
+ trainer.training_step(None, dict(batch))
+ with pytest.raises(RuntimeError, match = "num_items_in_batch"):
+ trainer.training_step(None, dict(batch), None)
+ shifted = {"input_ids": torch.ones(1, 4, dtype = torch.long), "shift_labels": torch.ones(1, 4)}
+ with pytest.raises(RuntimeError, match = "num_items_in_batch"):
+ trainer.training_step(None, shifted)
+ assert cp.get_cp_manager() is None
+ trainer.training_step(None, {"input_ids": torch.ones(1, 4, dtype = torch.long)})
+ trainer.training_step(None, dict(batch), torch.tensor(4.0))
+
+
[email protected]("raises", [False, True])
+def test_rotate_method_is_set_for_the_step_and_restored(monkeypatch, raises):
+ if cp._cp_options is None:
+ pytest.skip("this torch has no ring rotate option")
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ manager = _manager()
+ manager.mesh = None
+ manager.rotate_method = "alltoall"
+ before = cp._cp_options.rotate_method
+ seen = []
+ with contextlib.suppress(RuntimeError):
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ seen.append(cp._cp_options.rotate_method.name)
+ if raises:
+ raise RuntimeError("boom")
+ assert seen == ["ALL_TO_ALL"]
+ assert cp._cp_options.rotate_method == before
+
+
[email protected]("raises", [False, True])
+def test_ring_pins_sdpa_to_kernels_the_dispatcher_rings(monkeypatch, raises):
+ if cp.sdpa_kernel is None:
+ pytest.skip("no torch.nn.attention.sdpa_kernel")
+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
+ manager = _manager()
+ manager.mesh = None
+ cuda = torch.backends.cuda
+ before = (cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled(), cuda.math_sdp_enabled())
+ seen = []
+ with contextlib.suppress(RuntimeError):
+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
+ seen.append(
+ (
+ cuda.flash_sdp_enabled(),
+ cuda.mem_efficient_sdp_enabled(),
+ cuda.math_sdp_enabled(),
+ )
+ )
+ if raises:
+ raise RuntimeError("boom")
+ assert seen == [(True, True, False)]
+ assert (
+ cuda.flash_sdp_enabled(),
+ cuda.mem_efficient_sdp_enabled(),
+ cuda.math_sdp_enabled(),
+ ) == before
+
+
+def test_cudnn_is_pinned_only_where_the_ring_handles_it():
+ backends = cp._ring_sdpa_backends()
+ if backends is None:
+ pytest.skip("no SDPBackend")
+ from torch.nn.attention import SDPBackend
+
+ assert backends[:2] == [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]
+ from packaging.version import Version
+
+ if Version(Version(torch.__version__).base_version) >= Version("2.13"):
+ assert SDPBackend.CUDNN_ATTENTION in backends
+
+
+def test_unsloth_training_arguments_keep_the_cp_options():
+ from unsloth.trainer import UnslothTrainingArguments
+ args = UnslothTrainingArguments(
+ output_dir = "unused",
+ context_parallel_size = 2,
+ context_parallel_rotate_method = "alltoall",
+ report_to = "none",
+ )
+ assert cp.context_parallel_options(args) == (2, "ring", "alltoall")
+
+
+def test_cli_forwards_the_cp_options():
+ import importlib.util, pathlib, types
+
+ path = pathlib.Path(__file__).resolve().parents[1] / "unsloth-cli.py"
+ spec = importlib.util.spec_from_file_location("unsloth_cli_for_test", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ args = types.SimpleNamespace(
+ per_device_train_batch_size = 1,
+ gradient_accumulation_steps = 1,
+ warmup_steps = 0,
+ max_steps = 1,
+ learning_rate = 1e-4,
+ logging_steps = 1,
+ optim = "adamw_torch",
+ weight_decay = 0.0,
+ lr_scheduler_type = "linear",
+ seed = 0,
+ output_dir = "unused",
+ report_to = "none",
+ max_seq_length = 64,
+ packing = False,
+ per_device_eval_batch_size = 4,
+ context_parallel_size = 2,
+ context_parallel_backend = "ring",
+ context_parallel_rotate_method = "alltoall",
+ )
+ config = module._build_sft_config(lambda **kwargs: kwargs, args, False, True)
+ assert config["context_parallel_size"] == 2
+ assert config["context_parallel_backend"] == "ring"
+ assert config["context_parallel_rotate_method"] == "alltoall"
+ assert config["average_tokens_across_devices"] is True
+
+
+def test_generated_sft_config_carries_the_cp_options(tmp_path):
+ import dataclasses
+ import trl.trainer.sft_config as config_module
+
+ names = {f.name for f in dataclasses.fields(config_module.SFTConfig)}
+ if "context_parallel_size" not in names:
+ pytest.skip("SFTConfig is not the Unsloth-generated class in this process")
+ assert {"context_parallel_backend", "context_parallel_rotate_method"} <= names
+ config = config_module.SFTConfig(
+ output_dir = str(tmp_path),
+ context_parallel_size = 2,
+ context_parallel_rotate_method = "alltoall",
+ report_to = "none",
+ )
+ assert cp.context_parallel_options(config) == (2, "ring", "alltoall")
+ default = config_module.SFTConfig(output_dir = str(tmp_path), report_to = "none")
+ assert cp.context_parallel_options(default) == (1, "ring", "allgather")
+
+
+def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):
+ import trl
+
+ class Base:
+ def train(self):
+ return "base"
+
+ def training_step(self, *a, **k):
+ pass
+
+ evaluate = evaluation_loop = prediction_step = training_step
+
+ class Trainer(Base):
+ def __init__(self):
+ pass
+
+ monkeypatch.setattr(trl, "SFTTrainer", Trainer)
+ cp.patch_sft_trainer()
+ monkeypatch.setattr(Base, "train", lambda self: "patched later")
+ assert Trainer().train() == "patched later"
diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py
index ac53406e29f..cf8c6353555 100644
--- a/tests/utils/test_packing.py
+++ b/tests/utils/test_packing.py
@@ -2108,6 +2108,24 @@ def test_packing_skip_warning_keeps_custom_collator_reason(monkeypatch, caplog):
assert "UNSLOTH_RETURN_LOGITS" not in messages[0]
+def test_context_parallel_disables_eval_packing(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)
+ fake_trainer = _patch_fake_sft_trainer()
+ config = SimpleNamespace(
+ packing = True,
+ eval_packing = True,
+ padding_free = None,
+ remove_unused_columns = True,
+ context_parallel_size = 2,
+ )
+ fake_trainer(
+ model = _warn_text_model(),
+ args = config,
+ train_dataset = Dataset.from_dict({"text": ["sample"]}),
+ )
+ assert config.packing is False and config.eval_packing is False
+
+
# Fused-CE paths shift internally, so they mask boundaries on unshifted labels.
def test_mask_packed_boundary_labels_masks_next_document_first_token():
labels = torch.arange(6, dtype = torch.long).view(1, 6)
diff --git a/unsloth-cli.py b/unsloth-cli.py
index 87dc719cecf..ffdaad7ec26 100644
--- a/unsloth-cli.py
+++ b/unsloth-cli.py
@@ -186,6 +186,16 @@ def _build_sft_config(SFTConfig, args, is_mlx, bf16_supported):
dataset_num_proc = 2,
packing = args.packing,
)
+ if getattr(args, "context_parallel_size", 1) > 1:
+ config_kwargs["context_parallel_size"] = args.context_parallel_size
+ config_kwargs["context_parallel_backend"] = getattr(
+ args, "context_parallel_backend", "ring"
+ )
+ config_kwargs["context_parallel_rotate_method"] = getattr(
+ args, "context_parallel_rotate_method", "allgather"
+ )
+ # Required under CP; older transformers releases default it to False.
+ config_kwargs["average_tokens_across_devices"] = True
if is_mlx:
if args.per_device_eval_batch_size != 4:
print("Warning: --per_device_eval_batch_size is ignored on MLX without eval data.")
@@ -463,6 +473,26 @@ if __name__ == "__main__":
action = "store_true",
help = "Enable padding-free sample packing via TRL's bin packer.",
)
+ training_group.add_argument(
+ "--context_parallel_size",
+ type = int,
+ default = 1,
+ help = "Ranks per context parallel group (SDPA ring attention, torchrun, PyTorch >= 2.7).",
+ )
+ training_group.add_argument(
+ "--context_parallel_backend",
+ type = str,
+ default = "ring",
+ choices = ["ring"],
+ help = "Context parallel attention backend.",
+ )
+ training_group.add_argument(
+ "--context_parallel_rotate_method",
+ type = str,
+ default = "allgather",
+ choices = ["allgather", "alltoall"],
+ help = "How ring attention rotates KV shards between ranks.",
+ )
report_group = parser.add_argument_group("📊 Report Options")
report_group.add_argument(
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 5aa2536177f..7ae16996db7 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -771,3 +771,6 @@ from unsloth_zoo.rl_environments import (
# inspect.getsource() and corrupts downstream drift detectors.
if os.environ.get("UNSLOTH_ALLOW_CPU", "0") != "1":
_patch_trl_trainer()
+
+ from .context_parallel import patch_sft_trainer
+ patch_sft_trainer()
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
new file mode 100644
index 00000000000..1307178e448
--- /dev/null
+++ b/unsloth/context_parallel.py
@@ -0,0 +1,549 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+from __future__ import annotations
+
+import contextlib
+import functools
+from typing import Iterator, Optional
+
+import torch
+import torch.nn.functional as F
+import torch.distributed as dist
+
+try:
+ from torch.distributed.tensor.experimental import context_parallel
+ from torch.distributed.device_mesh import DeviceMesh
+ from torch.distributed.tensor import DTensor
+except (ImportError, AttributeError):
+ context_parallel = None
+ DeviceMesh = None
+ DTensor = None
+
+try:
+ # Re-exported from _context_parallel._attention on torch >= 2.10; defined here on 2.7 - 2.9.
+ from torch.distributed.tensor.experimental._attention import _cp_options, set_rotate_method
+except (ImportError, AttributeError):
+ _cp_options = None
+ set_rotate_method = None
+
+try:
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+except ImportError:
+ SDPBackend = None
+ sdpa_kernel = None
+
+_CP_BACKENDS = ("ring", "ulysses")
+_ROTATE_METHODS = ("allgather", "alltoall")
+
+
+def _torch_supports_context_parallel() -> bool:
+ from packaging.version import Version
+ return context_parallel is not None and Version(
+ Version(torch.__version__).base_version
+ ) >= Version("2.7.0")
+
+
+def _ring_sdpa_backends():
+ # The math kernel is not ringed: it would attend over the local shard only.
+ if SDPBackend is None:
+ return None
+ backends = [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]
+ cudnn = getattr(SDPBackend, "CUDNN_ATTENTION", None)
+ op = getattr(torch.ops.aten, "_scaled_dot_product_cudnn_attention", None)
+ if cudnn is not None and op is not None:
+ try:
+ from torch.distributed.tensor.experimental._context_parallel import _attention
+ except ImportError:
+ try:
+ from torch.distributed.tensor.experimental import _attention
+ except ImportError:
+ _attention = None
+ handled = getattr(_attention, "custom_ops", None)
+ if isinstance(handled, dict) and op.default in handled:
+ backends.append(cudnn)
+ return backends
+
+
+def context_parallel_options(config) -> tuple:
+ """(size, backend, rotate_method) from a training config, validated before any dataset work."""
+ size = getattr(config, "context_parallel_size", 1)
+ size = 1 if size is None else size
+ if isinstance(size, bool) or not isinstance(size, int) or size < 1:
+ raise ValueError(
+ f"Unsloth: context_parallel_size must be a positive integer, got {size!r}."
+ )
+ backend = getattr(config, "context_parallel_backend", None) or "ring"
+ if backend not in _CP_BACKENDS:
+ raise ValueError(
+ f"Unsloth: context_parallel_backend must be one of {_CP_BACKENDS}, got {backend!r}."
+ )
+ rotate_method = getattr(config, "context_parallel_rotate_method", None) or "allgather"
+ if rotate_method not in _ROTATE_METHODS:
+ raise ValueError(
+ "Unsloth: context_parallel_rotate_method must be one of "
+ f"{_ROTATE_METHODS}, got {rotate_method!r}."
+ )
+ if size > 1 and backend == "ulysses":
+ raise NotImplementedError(
+ "Unsloth: context_parallel_backend = 'ulysses' is not available yet; use 'ring'."
+ )
+ return size, backend, rotate_method
+
+
+def _ring_unsupported_attention(config) -> Optional[str]:
+ configs = [config]
+ text_config = getattr(config, "text_config", None)
+ if text_config is not None and text_config is not config:
+ configs.append(text_config)
+ for c in configs:
+ if c is None:
+ continue
+ layer_types = getattr(c, "layer_types", None)
+ if layer_types and any(t != "full_attention" for t in layer_types):
+ kinds = sorted({t for t in layer_types if t != "full_attention"})
+ return f"{', '.join(kinds)} layers"
+ # Any positive window, even with use_sliding_window off: Unsloth's masks read it directly.
+ sliding_window = getattr(c, "sliding_window", None)
+ if (isinstance(sliding_window, int) and sliding_window > 0) or getattr(
+ c, "use_sliding_window", None
+ ) is True:
+ return "sliding window attention"
+ if getattr(c, "attention_chunk_size", None):
+ return "chunked attention"
+ if getattr(c, "attn_logit_softcapping", None):
+ return "attention logit softcapping"
+ return None
+
+
+def _unregister_cp_sharding_rules() -> None:
+ try:
+ from torch.distributed.tensor.experimental._context_parallel._sharding_rules import (
+ unregister_cp_sharding_rules,
+ )
+ except ImportError:
+ return
+ unregister_cp_sharding_rules(clear_the_cache = False)
+
+
+from .device_type import DEVICE_TYPE_TORCH
+
+# A module global, not a ContextVar: CUDA runs backward (and checkpoint recompute) on another thread.
+_ACTIVE_MANAGER: Optional["ContextParallelManager"] = None
+
+# shift_labels is built before sharding so each shard's last token keeps its next-token target.
+_BUFFER_NAMES = ("input_ids", "attention_mask", "labels", "position_ids", "shift_labels")
+_PAD_VALUES = {"labels": -100, "shift_labels": -100, "attention_mask": 0, "input_ids": 0}
+
+
+_INSTALLED_MESH = []
+
+
+def get_cp_manager() -> Optional["ContextParallelManager"]:
+ return _ACTIVE_MANAGER
+
+
+def _supports_context_parallel(model) -> bool:
+ from .models.llama import LlamaAttention_fast_forward
+ forwards = [
+ getattr(type(module).forward, "__func__", type(module).forward)
+ for name, module in model.named_modules()
+ if name.endswith("self_attn")
+ ]
+ return bool(forwards) and all(f is LlamaAttention_fast_forward for f in forwards)
+
+
+def _self_attn_pre_forward_hook(_module, module_args, module_kwargs):
+ # Ring attention only supports is_causal with no mask; right padding is already -100 in labels.
+ if get_cp_manager() is not None and "attention_mask" in module_kwargs:
+ module_kwargs["attention_mask"] = None
+ return module_args, module_kwargs
+
+
+class ContextParallelManager:
+ def __init__(
+ self,
+ size: int,
+ backend: str = "ring",
+ rotate_method: str = "allgather",
+ ):
+ self.size = size
+ self.backend = backend
+ self.rotate_method = rotate_method
+ world_size = dist.get_world_size()
+ self.device_mesh = DeviceMesh(
+ DEVICE_TYPE_TORCH,
+ torch.arange(world_size).reshape(world_size // size, size),
+ mesh_dim_names = ("dp_replicate", "cp"),
+ )
+ self.mesh = self.device_mesh["cp"]
+
+ def attach_attention_hooks(self, model: torch.nn.Module) -> None:
+ for name, module in model.named_modules():
+ if name.endswith("self_attn") and not getattr(module, "_unsloth_cp_hooked", False):
+ module.register_forward_pre_hook(
+ _self_attn_pre_forward_hook, with_kwargs = True, prepend = True
+ )
+ module._unsloth_cp_hooked = True
+
+ def _prepare_inputs(self, inputs: dict) -> None:
+ input_ids = inputs.get("input_ids")
+ if not isinstance(input_ids, torch.Tensor) or inputs.get("inputs_embeds") is not None:
+ raise ValueError(
+ "Unsloth: context parallelism needs input_ids batches (not inputs_embeds)."
+ )
+ bsz, seq_len = input_ids.shape
+ if inputs.get("position_ids") is None:
+ inputs["position_ids"] = (
+ torch.arange(seq_len, device = input_ids.device).expand(bsz, -1).contiguous()
+ )
+ mask = inputs.get("attention_mask")
+ if isinstance(mask, torch.Tensor) and (
+ mask.ndim != 2 or (mask[:, 1:] > mask[:, :-1]).any()
+ ):
+ raise ValueError(
+ "Unsloth: context parallelism needs 2D right-padded attention masks without holes."
+ )
+ labels = inputs.get("labels")
+ if inputs.get("shift_labels") is None and labels is not None:
+ inputs["shift_labels"] = F.pad(labels, (0, 1), value = -100)[:, 1:].contiguous()
+ pad = (-seq_len) % (2 * self.size)
+ if pad:
+ for name in _BUFFER_NAMES:
+ tensor = inputs.get(name)
+ if not isinstance(tensor, torch.Tensor) or tensor.ndim < 2:
+ continue
+ if name == "position_ids":
+ extra = tensor[:, -1:] + torch.arange(1, pad + 1, device = tensor.device)
+ inputs[name] = torch.cat([tensor, extra.to(tensor.dtype)], dim = 1)
+ else:
+ inputs[name] = F.pad(tensor, (0, pad), value = _PAD_VALUES.get(name, 0))
+
+ @contextlib.contextmanager
+ def _rotate_method_scope(self) -> Iterator[None]:
+ rotate_method = getattr(self, "rotate_method", "allgather")
+ if _cp_options is None or set_rotate_method is None:
+ if rotate_method != "allgather":
+ raise RuntimeError(
+ f"Unsloth: this PyTorch cannot set the ring rotate method to {rotate_method!r}."
+ )
+ yield
+ return
+ previous = _cp_options.rotate_method
+ set_rotate_method(rotate_method)
+ try:
+ yield
+ finally:
+ _cp_options.rotate_method = previous
+
+ @contextlib.contextmanager
+ def apply(self, inputs: dict) -> Iterator[None]:
+ self._prepare_inputs(inputs)
+ buffers = [
+ inputs[name]
+ for name in _BUFFER_NAMES
+ if isinstance(inputs.get(name), torch.Tensor) and inputs[name].ndim > 1
+ ]
+ global _ACTIVE_MANAGER
+ previous, _ACTIVE_MANAGER = _ACTIVE_MANAGER, self
+ # torch < 2.13 leaves its CP dispatcher installed when the step raises.
+ sdpa = F.scaled_dot_product_attention
+ dispatcher = getattr(DTensor, "_op_dispatcher", None)
+ handlers = getattr(dispatcher, "_custom_op_handlers", None)
+ ring_backends = _ring_sdpa_backends()
+ raised = False
+ try:
+ with (
+ (
+ sdpa_kernel(ring_backends)
+ if ring_backends is not None
+ else contextlib.nullcontext()
+ ),
+ self._rotate_method_scope(),
+ context_parallel(
+ self.mesh,
+ buffers = buffers,
+ buffer_seq_dims = [1] * len(buffers),
+ no_restore_buffers = set(buffers),
+ ),
+ ):
+ yield
+ except BaseException:
+ raised = True
+ raise
+ finally:
+ _ACTIVE_MANAGER = previous
+ F.scaled_dot_product_attention = sdpa
+ if raised:
+ if handlers is not None:
+ dispatcher._custom_op_handlers = handlers
+ _unregister_cp_sharding_rules()
+
+
+def _refuse_iterable_datasets(*datasets) -> None:
+ # accelerate's batch dispatcher (dispatch_batches, the default for iterable datasets) ignores cp.
+ for dataset in datasets:
+ for d in dataset.values() if isinstance(dataset, dict) else [dataset]:
+ if (
+ isinstance(d, torch.utils.data.IterableDataset)
+ or "IterableDataset" in type(d).__name__
+ ):
+ raise NotImplementedError(
+ "Unsloth: context parallelism does not support iterable datasets."
+ )
+
+
+def _cp_gather_for_metrics(accelerator, size: int):
+ # Keep one loss per CP group, else last-batch truncation keeps duplicates and drops replicas.
+ def gather(tensor, *args, **kwargs):
+ if not isinstance(tensor, torch.Tensor) or tensor.ndim == 0:
+ return accelerator.gather_for_metrics(tensor, *args, **kwargs)
+ gathered = accelerator.gather(tensor)
+ gathered = gathered.reshape(-1, size, *tensor.shape)[:, 0].reshape(-1, *tensor.shape[1:])
+ state = accelerator.gradient_state
+ if state.end_of_dataloader and state.remainder > 0:
+ gathered = gathered[: state.remainder]
+ return gathered
+
+ return gather
+
+
+def patch_sft_trainer() -> None:
+ import trl
+
+ trainer_cls = trl.SFTTrainer
+ if trainer_cls.__dict__.get("__unsloth_context_parallel__"):
+ return
+
+ def _original(name):
+ # Inherited methods are looked up per call: Trainer.train etc. may be patched after import.
+ if name in trainer_cls.__dict__:
+ return trainer_cls.__dict__[name]
+
+ @functools.wraps(getattr(trainer_cls, name, None))
+ def inherited(self, *args, **kwargs):
+ return getattr(super(trainer_cls, self), name)(*args, **kwargs)
+
+ return inherited
+
+ original_init = trainer_cls.__init__
+ original_prediction_step = _original("prediction_step")
+ original_training_step = _original("training_step")
+ original_train = _original("train")
+ original_evaluate = _original("evaluate")
+ original_predict = _original("predict")
+ original_evaluation_loop = _original("evaluation_loop")
+
+ def _install_accelerator_state(self):
+ manager = getattr(self, "_context_parallel_manager", None)
+ accelerator = getattr(self, "accelerator", None)
+ state = getattr(accelerator, "state", None)
+ if manager is None:
+ if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:
+ state.device_mesh = None
+ return
+ if accelerator is None:
+ return
+ accelerator.state.device_mesh = manager.device_mesh
+ _INSTALLED_MESH[:] = [manager.device_mesh]
+ # PEFT makes the Trainer ask DDP for find_unused_parameters, which breaks reentrant checkpointing.
+ if getattr(self.args, "ddp_find_unused_parameters", None) in (None, False):
+ self.args.ddp_find_unused_parameters = False
+ handler = getattr(accelerator, "ddp_handler", None)
+ if handler is not None:
+ handler.find_unused_parameters = False
+
+ @functools.wraps(original_init)
+ def patched_init(self, *args, **kwargs):
+ config = kwargs["args"] if "args" in kwargs else (args[1] if len(args) > 1 else None)
+ context_parallel_options(config)
+ original_init(self, *args, **kwargs)
+ self._context_parallel_manager = None
+ # Unsloth's attention ignores transformers' own CP: it would stay local to each shard.
+ parallelism_config = getattr(getattr(self, "args", None), "parallelism_config", None)
+ if (getattr(parallelism_config, "cp_size", 1) or 1) > 1:
+ raise NotImplementedError(
+ "Unsloth: use SFTConfig(context_parallel_size = N) instead of "
+ "parallelism_config with cp_size > 1."
+ )
+ size, backend, rotate_method = context_parallel_options(getattr(self, "args", None))
+ if size <= 1:
+ _install_accelerator_state(self)
+ return
+ # train() would swap in a fresh model without the attention hooks or the support check.
+ if getattr(self, "model_init", None) is not None:
+ raise NotImplementedError("Unsloth: context parallelism does not support model_init.")
+ if not _torch_supports_context_parallel():
+ raise RuntimeError(
+ f"Unsloth: context_parallel_size > 1 needs PyTorch >= 2.7, found {torch.__version__}."
+ )
+ if not (dist.is_available() and dist.is_initialized()):
+ raise RuntimeError(
+ "Unsloth: context_parallel_size > 1 needs torch.distributed. "
+ f"Launch with torchrun or accelerate launch using a multiple of {size} processes."
+ )
+ world_size = dist.get_world_size()
+ if world_size % size != 0:
+ raise RuntimeError(
+ f"Unsloth: world size {world_size} is not a multiple of context_parallel_size {size}."
+ )
+ if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] < 8:
+ # Pure torch ring attention fails in backward without flash (2x T4, torch 2.10).
+ raise NotImplementedError(
+ "Unsloth: context parallelism needs flash attention (compute capability >= 8.0)."
+ )
+ # These recompute loss outside our forward from sharded labels, losing cross-shard targets.
+ if (
+ getattr(self.args, "label_smoothing_factor", 0)
+ or getattr(self, "compute_loss_func", None)
+ or getattr(self.args, "loss_type", None) not in (None, "nll")
+ ):
+ raise NotImplementedError(
+ "Unsloth: context parallelism needs loss_type = 'nll' without "
+ "label_smoothing_factor or compute_loss_func."
+ )
+ if getattr(self, "model_accepts_loss_kwargs", True) is False:
+ raise NotImplementedError(
+ "Unsloth: context parallelism needs a model whose forward accepts loss kwargs "
+ "(num_items_in_batch)."
+ )
+ if getattr(self.args, "use_legacy_prediction_loop", False):
+ raise NotImplementedError(
+ "Unsloth: context parallelism needs use_legacy_prediction_loop = False."
+ )
+ if getattr(self.args, "average_tokens_across_devices", True) is False:
+ raise NotImplementedError(
+ "Unsloth: context parallelism needs average_tokens_across_devices = True."
+ )
+ import accelerate
+ from packaging.version import Version
+
+ if Version(accelerate.__version__) < Version("1.10.0"):
+ raise NotImplementedError("Unsloth: context parallelism needs accelerate >= 1.10.0.")
+ unsupported = _ring_unsupported_attention(getattr(self.model, "config", None))
+ if unsupported is not None:
+ raise NotImplementedError(
+ f"Unsloth: ring context parallelism does not support {unsupported}."
+ )
+ if not _supports_context_parallel(self.model):
+ raise NotImplementedError(
+ "Unsloth: context parallelism currently supports Llama-style attention only "
+ "(Llama, Qwen2, Gemma)."
+ )
+ accelerator = getattr(self, "accelerator", None)
+ distributed_type = getattr(getattr(accelerator, "distributed_type", None), "name", "NO")
+ if distributed_type != "NO" and not distributed_type.startswith("MULTI_"):
+ raise NotImplementedError(
+ f"Unsloth: context parallelism supports DDP only, not {distributed_type}."
+ )
+ if getattr(accelerator, "dispatch_batches", None):
+ raise NotImplementedError(
+ "Unsloth: context parallelism does not support dispatch_batches."
+ )
+ _refuse_iterable_datasets(
+ getattr(self, "train_dataset", None), getattr(self, "eval_dataset", None)
+ )
+ manager = ContextParallelManager(size, backend, rotate_method)
+ self._context_parallel_manager = manager
+ manager.attach_attention_hooks(self.model)
+ _install_accelerator_state(self)
+ # The ring calls the flash ops FA3 / FA4 override, so scope them to train() here too.
+ from .utils.packing import _scope_flash_attention_impl
+
+ _scope_flash_attention_impl(self.model, self)
+ print(
+ f"Unsloth: Context parallelism enabled with size = {size} "
+ f"({backend}, rotate_method = {rotate_method})."
+ )
+
+ @functools.wraps(original_prediction_step)
+ def patched_prediction_step(self, model, inputs, *args, **kwargs):
+ manager = getattr(self, "_context_parallel_manager", None)
+ prediction_loss_only = args[0] if args else kwargs.get("prediction_loss_only", True)
+ if manager is not None and not prediction_loss_only:
+ raise NotImplementedError(
+ "Unsloth: context parallelism supports loss-only evaluation; "
+ "compute_metrics / predict() need context_parallel_size = 1."
+ )
+ if manager is None:
+ return original_prediction_step(self, model, inputs, *args, **kwargs)
+ inputs = self._prepare_inputs(inputs)
+ manager._prepare_inputs(inputs)
+ shift_labels = inputs.get("shift_labels")
+ if not isinstance(shift_labels, torch.Tensor):
+ with manager.apply(inputs):
+ return original_prediction_step(self, model, inputs, *args, **kwargs)
+ inputs["num_items_in_batch"] = shift_labels.ne(-100).sum().clamp_min(1)
+ inputs.setdefault("use_cache", False)
+ with manager.apply(inputs), torch.no_grad(), self.compute_loss_context_manager():
+ loss = self.compute_loss(model, inputs).detach()
+ dist.all_reduce(loss, group = manager.mesh.get_group())
+ return loss, None, None
+
+ @functools.wraps(original_train)
+ def patched_train(self, *args, **kwargs):
+ if getattr(self, "_context_parallel_manager", None) is not None:
+ _refuse_iterable_datasets(getattr(self, "train_dataset", None))
+ _install_accelerator_state(self)
+ return original_train(self, *args, **kwargs)
+
+ @functools.wraps(original_evaluate)
+ def patched_evaluate(self, *args, **kwargs):
+ if getattr(self, "_context_parallel_manager", None) is not None:
+ eval_dataset = args[0] if args else kwargs.get("eval_dataset")
+ if eval_dataset is None or isinstance(eval_dataset, str):
+ current = self.eval_dataset
+ eval_dataset = (
+ current.get(eval_dataset)
+ if isinstance(eval_dataset, str) and isinstance(current, dict)
+ else current
+ )
+ _refuse_iterable_datasets(eval_dataset)
+ _install_accelerator_state(self)
+ return original_evaluate(self, *args, **kwargs)
+
+ @functools.wraps(original_predict)
+ def patched_predict(self, test_dataset, *args, **kwargs):
+ if getattr(self, "_context_parallel_manager", None) is not None:
+ _refuse_iterable_datasets(test_dataset)
+ _install_accelerator_state(self)
+ return original_predict(self, test_dataset, *args, **kwargs)
+
+ @functools.wraps(original_evaluation_loop)
+ def patched_evaluation_loop(self, *args, **kwargs):
+ manager = getattr(self, "_context_parallel_manager", None)
+ if manager is not None:
+ self.gather_function = _cp_gather_for_metrics(self.accelerator, manager.size)
+ return original_evaluation_loop(self, *args, **kwargs)
+
+ @functools.wraps(original_training_step)
+ def patched_training_step(self, model, inputs, *args, **kwargs):
+ manager = getattr(self, "_context_parallel_manager", None)
+ if manager is not None:
+ num_items = args[0] if args else kwargs.get("num_items_in_batch")
+ if (
+ num_items is None
+ and hasattr(inputs, "get")
+ and (inputs.get("labels") is not None or inputs.get("shift_labels") is not None)
+ ):
+ raise RuntimeError(
+ "Unsloth: context parallelism needs num_items_in_batch in training_step; "
+ "this transformers / trainer path did not pass one."
+ )
+ # HF summed the count over all D x N ranks while CP peers share a batch: divide by N (exact).
+ if args and args[0] is not None:
+ args = (args[0] // manager.size, *args[1:])
+ elif kwargs.get("num_items_in_batch") is not None:
+ kwargs["num_items_in_batch"] = kwargs["num_items_in_batch"] // manager.size
+ # Forward and backward both inside, so gradient checkpointing recomputes with ring attention.
+ with manager.apply(inputs) if manager else contextlib.nullcontext():
+ return original_training_step(self, model, inputs, *args, **kwargs)
+
+ trainer_cls.__init__ = patched_init
+ trainer_cls.prediction_step = patched_prediction_step
+ trainer_cls.train = patched_train
+ trainer_cls.evaluate = patched_evaluate
+ trainer_cls.predict = patched_predict
+ trainer_cls.evaluation_loop = patched_evaluation_loop
+ trainer_cls.training_step = patched_training_step
+ trainer_cls.__unsloth_context_parallel__ = True
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 6e377f0d18d..160cbad9f5c 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -146,6 +146,7 @@ from ..kernels import *
from ..kernels.utils import _has_active_lora_bias, has_mxfp4_base
from ..kernels.bnb_override import install_bnb_nf4_override as _install_bnb_nf4_override
from ..tokenizer_utils import *
+from ..context_parallel import get_cp_manager
from .vision import FastBaseModel, _is_text_seq2seq_config
from .vision import (
_resolve_offload_embedding,
@@ -956,12 +957,15 @@ def LlamaAttention_fast_forward(
if past_key_value is not None:
kv_seq_len += past_key_value[0].shape[-2]
- if position_embeddings and kv_seq_len <= position_embeddings[0].shape[0]:
+ cp_manager = get_cp_manager()
+ rope_seq_len = kv_seq_len * cp_manager.size if cp_manager is not None else kv_seq_len
+
+ if position_embeddings and rope_seq_len <= position_embeddings[0].shape[0]:
cos, sin = position_embeddings
else:
rotary_emb = self.rotary_emb
- rotary_emb.extend_rope_embedding(V, seq_len = kv_seq_len)
- cos, sin = rotary_emb.get_cached(kv_seq_len, Q.device.index)
+ rotary_emb.extend_rope_embedding(V, seq_len = rope_seq_len)
+ cos, sin = rotary_emb.get_cached(rope_seq_len, Q.device.index)
cos = cos.to(device = Q.device, dtype = Q.dtype)
sin = sin.to(device = Q.device, dtype = Q.dtype)
@@ -1731,6 +1735,8 @@ def CausalLM_fast_forward(fast_forward_inference):
*args,
**kwargs,
) -> Union[Tuple, CausalLMOutputWithPast]:
+ # Not a named parameter: HF's find_labels would make it a required eval label.
+ shift_labels = kwargs.pop("shift_labels", None)
past_key_values = _cache_as_legacy_tuple(past_key_values)
if past_key_values is not None and len(past_key_values) == 0:
past_key_values = None
@@ -1797,6 +1803,8 @@ def CausalLM_fast_forward(fast_forward_inference):
hidden_states = hidden_states.to(lm_head_device)
if labels is not None:
labels = labels.to(lm_head_device)
+ if shift_labels is not None:
+ shift_labels = shift_labels.to(lm_head_device)
# Output last hidden states without logits if asked.
if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":
@@ -1854,7 +1862,7 @@ def CausalLM_fast_forward(fast_forward_inference):
hidden_states = hidden_states,
lm_head_weight = lm_head,
lm_head_bias = None,
- labels = labels,
+ labels = labels if shift_labels is None else shift_labels,
mask = None,
n_items = n_items,
scaling = getattr(self, "accelerator_scaler", None),
@@ -1863,6 +1871,7 @@ def CausalLM_fast_forward(fast_forward_inference):
logit_softcapping = logit_softcapping,
logit_scale_multiply = logit_scale_multiply,
logit_scale_divide = logit_scale_divide,
+ shift_labels = shift_labels is None,
)
if not return_dict:
# Fused CE never materializes logits; use EMPTY_LOGITS like the return_dict branch below (#2068).
@@ -1885,15 +1894,16 @@ def CausalLM_fast_forward(fast_forward_inference):
# Same answer the fused branch above reads, so the two branches cannot drift apart.
logit_softcapping, logit_scaling = resolve_logit_scaling(self.config)
- if labels is not None:
+ if labels is not None or shift_labels is not None:
shift_logits = logits
- shift_labels = torch.empty_like(labels)
- shift_labels[..., :-1] = labels[..., 1:]
- shift_labels[..., -1] = -100
- mask_packed_sequence_boundaries(
- shift_labels,
- kwargs.get("packed_seq_lengths"),
- )
+ if shift_labels is None:
+ shift_labels = torch.empty_like(labels)
+ shift_labels[..., :-1] = labels[..., 1:]
+ shift_labels[..., -1] = -100
+ mask_packed_sequence_boundaries(
+ shift_labels,
+ kwargs.get("packed_seq_lengths"),
+ )
n_items = kwargs.get("num_items_in_batch", None)
if n_items is None:
n_items = kwargs.get("n_items", None)
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 36d4499cbba..0bbd6eeffe6 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -3203,6 +3203,32 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
max_seq_length_call = ""
max_seq_length_post = ""
+ # Read by unsloth.context_parallel; a field here keeps SFTConfig itself unpatched (pickling).
+ if trainer_file == "sft_trainer" and "context_parallel_size" not in call_args:
+ max_seq_length_pre += """
+ context_parallel_size : Optional[int] = field(
+ default = 1,
+ metadata = {'help': 'Ranks per context parallel group (SDPA ring attention). 1 disables it.'},
+ )
+ context_parallel_backend : Optional[str] = field(
+ default = 'ring',
+ metadata = {'help': 'Context parallel attention backend: ring.'},
+ )
+ context_parallel_rotate_method : Optional[str] = field(
+ default = 'allgather',
+ metadata = {'help': 'Ring KV rotation: allgather or alltoall.'},
+ )"""
+ max_seq_length_call += (
+ "context_parallel_size = 1,"
+ "context_parallel_backend = 'ring',"
+ "context_parallel_rotate_method = 'allgather',"
+ )
+ max_seq_length_post += (
+ "\n self.context_parallel_size = context_parallel_size"
+ "\n self.context_parallel_backend = context_parallel_backend"
+ "\n self.context_parallel_rotate_method = context_parallel_rotate_method"
+ )
+
if "output_dir" in call_args:
saving_check = (
"if output_dir is None and save_strategy == 'steps' and save_steps == 500:\n"
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 9d99105e874..72e914dd00b 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -940,7 +940,19 @@ class UnslothTrainingArguments(TrainingArguments):
self.q_galore_config = q_galore_config
self.embedding_learning_rate = embedding_learning_rate
self.loraplus_lr_ratio = loraplus_lr_ratio
+ # Not fields of the pristine SFTConfig captured at import.
+ context_parallel = {
+ "context_parallel_size": kwargs.pop("context_parallel_size", 1),
+ "context_parallel_backend": kwargs.pop("context_parallel_backend", "ring"),
+ "context_parallel_rotate_method": kwargs.pop(
+ "context_parallel_rotate_method", "allgather"
+ ),
+ }
+ fields = getattr(TrainingArguments, "__dataclass_fields__", {})
+ kwargs.update({k: v for k, v in context_parallel.items() if k in fields})
super().__init__(*args, **kwargs)
+ for key, value in context_parallel.items():
+ setattr(self, key, value)
self.embedding_learning_rate = embedding_learning_rate
self.loraplus_lr_ratio = loraplus_lr_ratio
if self.eval_steps is not None and self.eval_strategy != "steps":
@@ -1851,12 +1863,14 @@ def _patch_sft_trainer_auto_packing(trl_module):
# several classes, so a "yes" from the resolved one is not proof about the
# instance. A correct "no" has already turned both flags off, which is the
# condition the post-init check skips on, so leaving it armed is free.
+ is_context_parallel = (getattr(config_arg, "context_parallel_size", 1) or 1) > 1
# Read before __init__: the compiled trainer ends it with model.for_inference(), which sets the flag.
eval_logits_wanted = os.environ.get(
"UNSLOTH_RETURN_LOGITS", "0"
) == "1" or _wants_eval_logits(original_init, args, kwargs)
blocked = (
(data_collator is not None)
+ or is_context_parallel
or (is_processor and not processor_text_only)
or is_auto_processor_vlm
or is_vision_dataset
@@ -1872,6 +1886,8 @@ def _patch_sft_trainer_auto_packing(trl_module):
setattr(config_arg, "packing", False)
if hasattr(config_arg, "padding_free"):
setattr(config_arg, "padding_free", False)
+ if is_context_parallel and getattr(config_arg, "eval_packing", None):
+ setattr(config_arg, "eval_packing", False)
if blocked and requested_pack:
reason = "custom data collator"
@@ -1887,6 +1903,8 @@ def _patch_sft_trainer_auto_packing(trl_module):
reason = "hybrid linear-attention model"
elif is_unsupported_model:
reason = f"unsupported model type(s): {', '.join(model_types)}"
+ elif is_context_parallel:
+ reason = "context parallelism"
elif forward_rejects_packing:
# Name the real blocker, else this falls through to the
# UNSLOTH_RETURN_LOGITS branch and points at an unset flag. For a string
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index e5195ed2669..72d693997fc 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -21,10 +21,11 @@ from typing import Any, Optional, Tuple
import torch
from torch import Tensor
-from torch.nn.functional import scaled_dot_product_attention
+import torch.nn.functional as F
from ..device_type import DEVICE_TYPE
from ..models._utils import *
+from ..context_parallel import get_cp_manager
from ..utils.packing import (
build_sdpa_packed_attention_mask,
build_xformers_block_causal_mask,
@@ -91,6 +92,11 @@ _XFORMERS_FP32_UNSUPPORTED = (
)
+def scaled_dot_product_attention(*args, **kwargs):
+ # Resolved per call: context_parallel monkey patches torch.nn.functional.
+ return F.scaled_dot_product_attention(*args, **kwargs)
+
+
def _per_device_backends(capabilities):
"""(flash runs, xformers runs, xformers rejects fp32, bf16 kernels) per CUDA device index, or None
when every device has device 0's capability class, so homogeneous hosts never consult the table.
@@ -158,7 +164,7 @@ def _backend_for_device(
return backend
-SDPA_HAS_GQA = "enable_gqa" in (scaled_dot_product_attention.__doc__ or "")
+SDPA_HAS_GQA = "enable_gqa" in (F.scaled_dot_product_attention.__doc__ or "")
# Packed SDPA runs per segment: a dense (T, T) mask + enable_gqa leaves only the math kernel.
_SDPA_PACKED_SEGMENTS = os.environ.get("UNSLOTH_SDPA_PACKED_SEGMENTS", "1").lower() not in (
@@ -525,6 +531,12 @@ class AttentionContext:
def select_attention_backend(use_varlen: bool = False) -> str:
"""Return attention backend based on availability / priority order."""
+ if get_cp_manager() is not None:
+ if use_varlen:
+ raise ValueError(
+ "Unsloth: context parallelism does not support packing / padding-free."
+ )
+ return SDPA
if HAS_FLASH_ATTENTION:
if use_varlen:
return FLASH_VARLEN
@@ -720,9 +732,11 @@ def run_attention(
# DoRA promotes q/k/v_proj outputs to fp32, which FlashAttention rejects (as does the xformers
# flash-2 op on sm_100+), so downcast any fp32 Q/K/V to a supported dtype (#1013).
+ cp_out_dtype = Q.dtype if backend == SDPA and get_cp_manager() is not None else None
if (
backend in (FLASH_DENSE, FLASH_VARLEN)
or (backend == XFORMERS and xformers_fp32_unsupported)
+ or (backend == SDPA and get_cp_manager() is not None)
) and torch.float32 in (
Q.dtype,
K.dtype,
@@ -956,6 +970,13 @@ def run_attention(
kwargs.setdefault("is_causal", is_causal_local)
use_sdpa_gqa = SDPA_HAS_GQA and config.n_groups != 1
+ # Flash takes GQA natively, halving the ring's K/V peak; UNSLOTH_CP_GQA=0 expands.
+ if (
+ use_sdpa_gqa
+ and get_cp_manager() is not None
+ and os.environ.get("UNSLOTH_CP_GQA", "1") == "0"
+ ):
+ use_sdpa_gqa = False
if (
use_sdpa_gqa
and (not requires_grad)
@@ -970,7 +991,8 @@ def run_attention(
if use_sdpa_gqa:
kwargs.setdefault("enable_gqa", True)
out = scaled_dot_product_attention(Q, K, V, **kwargs)
- return out.transpose(1, 2)
+ out = out.transpose(1, 2)
+ return out if cp_out_dtype is None else out.to(cp_out_dtype)
K_mod = K
V_mod = V
@@ -990,7 +1012,8 @@ def run_attention(
V_mod.contiguous(),
**kwargs,
)
- return out.transpose(1, 2).contiguous()
+ out = out.transpose(1, 2).contiguous()
+ return out if cp_out_dtype is None else out.to(cp_out_dtype)
__all__ = [