Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add context parallelism support (SDPA only) (#4257)
Daniel Han493ac4dfceaf
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.pynew file mode 100644index 00000000000..4bc322af359--- /dev/null+++ b/tests/test_context_parallel.py@@ -0,0 +1,1052 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Context parallelism plumbing that runs without a process group (the ring itself needs >= 2 GPUs)."""++import contextlib++import pytest+import torch++import unsloth # noqa: F401+from unsloth import context_parallel as cp+from unsloth.utils import attention_dispatch as ad+++def _manager(size = 2):+ manager = object.__new__(cp.ContextParallelManager)+ manager.size = size+ return manager+++def _fake_context_parallel(sharded):+ @contextlib.contextmanager+ def fake(mesh, buffers, buffer_seq_dims, no_restore_buffers):+ sharded.extend(buffers)+ yield++ return fake+++def test_none_position_ids_and_shift_labels_are_rebuilt():+ inputs = {+ "input_ids": torch.ones(1, 4, dtype = torch.long),+ "labels": torch.tensor([[1, 2, 3, 4]]),+ "position_ids": None,+ "shift_labels": None,+ }+ _manager()._prepare_inputs(inputs)+ assert inputs["position_ids"].tolist() == [[0, 1, 2, 3]]+ assert inputs["shift_labels"].tolist() == [[2, 3, 4, -100]]+++def test_labels_shift_before_sharding_and_pad_to_load_balancer_chunks():+ manager = _manager(size = 2)+ labels = torch.tensor([[-100, 5, 6, 7, 8, 9]])+ inputs = {"input_ids": torch.tensor([[1, 5, 6, 7, 8, 9]]), "labels": labels.clone()}+ manager._prepare_inputs(inputs)+ assert inputs["shift_labels"].tolist() == [[5, 6, 7, 8, 9, -100, -100, -100]]+ assert inputs["labels"].tolist() == [[-100, 5, 6, 7, 8, 9, -100, -100]]+ assert inputs["position_ids"].tolist() == [[0, 1, 2, 3, 4, 5, 6, 7]]+ assert inputs["input_ids"].shape == (1, 8)++[email protected]("mask", [[[0, 1, 1, 1]], [[1, 0, 1, 1]], [[[1, 1, 1, 1]] * 4]])+def test_left_padded_or_holed_masks_are_refused(mask):+ inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long), "attention_mask": torch.tensor(mask)}+ with pytest.raises(ValueError, match = "right-padded"):+ _manager()._prepare_inputs(inputs)+ right = {+ "input_ids": torch.ones(1, 4, dtype = torch.long),+ "attention_mask": torch.tensor([[1, 1, 0, 0]]),+ }+ _manager()._prepare_inputs(right)+++def test_inputs_embeds_batches_are_refused():+ with pytest.raises(ValueError, match = "input_ids"):+ _manager()._prepare_inputs(+ {"inputs_embeds": torch.zeros(1, 4, 8), "labels": torch.ones(1, 4)}+ )+++def test_active_manager_resets_when_the_step_raises(monkeypatch):+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ manager = _manager()+ manager.mesh = None+ inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long)}+ with pytest.raises(RuntimeError, match = "boom"):+ with manager.apply(inputs):+ assert cp.get_cp_manager() is manager+ raise RuntimeError("boom")+ assert cp.get_cp_manager() is None+++def test_attention_hook_only_drops_the_mask_inside_a_cp_step(monkeypatch):+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ mask = torch.ones(1, 4)+ _, kwargs = cp._self_attn_pre_forward_hook(None, (), {"attention_mask": mask})+ assert kwargs["attention_mask"] is mask+ manager = _manager()+ manager.mesh = None+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ _, kwargs = cp._self_attn_pre_forward_hook(None, (), {"attention_mask": mask})+ assert kwargs["attention_mask"] is None+++def test_backend_is_sdpa_under_cp_and_packing_is_refused(monkeypatch):+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ manager = _manager()+ manager.mesh = None+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ assert ad.select_attention_backend(False) == ad.SDPA+ with pytest.raises(ValueError, match = "packing"):+ ad.select_attention_backend(True)+++def test_sdpa_is_looked_up_per_call(monkeypatch):+ # A name bound at import would keep attending to the local shard only.+ calls = []+ monkeypatch.setattr(+ torch.nn.functional, "scaled_dot_product_attention", lambda *a, **k: calls.append(1)+ )+ ad.scaled_dot_product_attention(None, None, None)+ assert calls == [1]+++def test_training_step_divides_the_pre_shard_token_count_by_cp_size(monkeypatch):+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ seen = []++ class Trainer:+ def __init__(self):+ pass++ def training_step(+ self,+ model,+ inputs,+ num_items_in_batch = None,+ ):+ seen.append(("train", num_items_in_batch))++ def prediction_step(self, model, inputs, prediction_loss_only):+ return (None, None, None)++ def train(self):+ pass++ def evaluate(self):+ pass++ def evaluation_loop(self, *a, **k):+ pass++ import trl++ monkeypatch.setattr(trl, "SFTTrainer", Trainer)+ cp.patch_sft_trainer()+ trainer = Trainer()+ manager = _manager(size = 4)+ manager.mesh = None+ trainer._context_parallel_manager = manager+ batch = lambda: {"input_ids": torch.ones(1, 8, dtype = torch.long)}+ trainer.training_step(None, batch(), torch.tensor(12.0))+ trainer.training_step(None, batch(), num_items_in_batch = torch.tensor(12.0))+ trainer.training_step(None, batch(), torch.tensor(12))+ assert [(k, float(n)) for k, n in seen] == [("train", 3.0)] * 3+ assert seen[-1][1].dtype == torch.int64+++def test_shift_labels_is_not_an_eval_label_name():+ # A named shift_labels made find_labels treat it as a required eval label.+ from transformers.utils import find_labels+ from unsloth.models.llama import CausalLM_fast_forward, LlamaModel_fast_forward_inference++ class LM(torch.nn.Module):+ forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference)++ assert find_labels(LM) == ["labels"]++[email protected]("opt_out", [False, True])+def test_gqa_stays_native_under_cp_unless_opted_out(monkeypatch, opt_out):+ if opt_out:+ monkeypatch.setenv("UNSLOTH_CP_GQA", "0")+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ monkeypatch.setattr(ad, "SDPA_HAS_GQA", True)+ seen = []+ monkeypatch.setattr(+ ad,+ "scaled_dot_product_attention",+ lambda Q, K, V, **k: (seen.append((K.shape[1], k.get("enable_gqa"))), Q)[1],+ )+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 2, n_groups = 2)+ context = ad.AttentionContext(+ bsz = 1,+ q_len = 4,+ kv_seq_len = 4,+ n_heads = 4,+ head_dim = 8,+ requires_grad = True,+ seq_info = None,+ attention_mask = None,+ causal_mask = None,+ )+ Q, K = torch.zeros(1, 4, 4, 8), torch.zeros(1, 2, 4, 8)+ manager = _manager()+ manager.mesh = None+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ ad.run_attention(config = config, context = context, Q = Q, K = K, V = K)+ assert seen == ([(4, None)] if opt_out else [(2, True)])+++def test_every_rank_builds_the_same_global_mesh(monkeypatch):+ built = []++ class FakeMesh:+ def __init__(+ self,+ device_type,+ mesh,+ mesh_dim_names = None,+ ):+ built.append((mesh.tolist(), mesh_dim_names))++ def __getitem__(self, name):+ return ("submesh", name)++ monkeypatch.setattr(cp, "DeviceMesh", FakeMesh)+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 4)+ for rank in range(4):+ monkeypatch.setattr(cp.dist, "get_rank", lambda rank = rank: rank)+ manager = cp.ContextParallelManager(2)+ assert manager.mesh == ("submesh", "cp")+ assert built == [([[0, 1], [2, 3]], ("dp_replicate", "cp"))] * 4+++def _patched_trainer(monkeypatch, **init_attrs):+ import trl++ class Trainer:+ def __init__(self, *a, **k):+ for key, value in init_attrs.items():+ setattr(self, key, value)++ def training_step(+ self,+ model,+ inputs,+ num_items_in_batch = None,+ ):+ pass++ def prediction_step(+ self,+ model,+ inputs,+ prediction_loss_only,+ ignore_keys = None,+ ):+ return (torch.tensor(1.0), None, None)++ def _prepare_inputs(self, inputs):+ return inputs++ def compute_loss_context_manager(self):+ return contextlib.nullcontext()++ def compute_loss(self, model, inputs):+ assert inputs["use_cache"] is False+ targets = inputs["shift_labels"].ne(-100).sum()+ return targets.float() / inputs["num_items_in_batch"]++ def train(self, *a, **k):+ return "trained"++ def evaluate(self, *a, **k):+ return "evaluated"++ def predict(self, *a, **k):+ return "predicted"++ def evaluation_loop(self, *a, **k):+ return self.gather_function(torch.arange(1.0, 3.0))++ monkeypatch.setattr(trl, "SFTTrainer", Trainer)+ cp.patch_sft_trainer()+ return Trainer++[email protected]("world_size", [2, 4])+def test_eval_loss_is_the_token_mean_even_with_an_empty_shard(monkeypatch, world_size):+ import types++ @contextlib.contextmanager+ def first_half(mesh, buffers, buffer_seq_dims, no_restore_buffers):+ for buffer in buffers:+ buffer.data = buffer[:, : buffer.shape[1] // 2].clone()+ yield++ monkeypatch.setattr(cp, "context_parallel", first_half)+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: world_size)+ Trainer = _patched_trainer(monkeypatch)+ trainer = object.__new__(Trainer)+ manager = _manager(size = 2)+ manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")+ trainer._context_parallel_manager = manager+ counts = []++ def all_reduce(tensor, group = None):+ assert group == "cp"+ counts.append(group)+ tensor.add_(2 / 2)++ monkeypatch.setattr(cp.dist, "all_reduce", all_reduce)+ labels = torch.tensor([[-100] * 7 + [1, 1]])+ batch = lambda: {"input_ids": torch.ones(1, 9, dtype = torch.long), "labels": labels.clone()}+ loss, logits, labels_out = trainer.prediction_step(None, batch(), True)+ assert counts == ["cp"] and loss.item() == pytest.approx(1.0)+ assert logits is None and labels_out is None+ with pytest.raises(NotImplementedError, match = "loss-only"):+ trainer.prediction_step(None, batch(), False)+++ "attrs",+ [+ {"label_smoothing_factor": 0.1},+ {"compute_loss_func": lambda *a, **k: 0},+ {"loss_type": "chunked_nll"},+ ],+)+def test_label_dropping_loss_paths_are_refused(monkeypatch, attrs):+ import types++ monkeypatch.setattr(cp.dist, "is_available", lambda: True)+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)+ for key in ("label_smoothing_factor", "loss_type"):+ if key in attrs:+ setattr(args, key, attrs[key])+ init = {"args": args}+ if "compute_loss_func" in attrs:+ init["compute_loss_func"] = attrs["compute_loss_func"]+ Trainer = _patched_trainer(monkeypatch, **init)+ with pytest.raises(NotImplementedError, match = "loss_type = 'nll'"):+ Trainer()+++def test_unaveraged_token_counts_are_refused(monkeypatch):+ import types++ monkeypatch.setattr(cp.dist, "is_available", lambda: True)+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ args = types.SimpleNamespace(+ context_parallel_size = 2,+ label_smoothing_factor = 0.0,+ average_tokens_across_devices = False,+ )+ Trainer = _patched_trainer(monkeypatch, args = args)+ with pytest.raises(NotImplementedError, match = "average_tokens_across_devices"):+ Trainer()+++def test_old_accelerate_is_refused(monkeypatch):+ import types, accelerate++ monkeypatch.setattr(cp.dist, "is_available", lambda: True)+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.setattr(accelerate, "__version__", "1.9.0")+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)+ Trainer = _patched_trainer(monkeypatch, args = args)+ with pytest.raises(NotImplementedError, match = "accelerate >= 1.10.0"):+ Trainer()+++def test_fp32_qkv_is_downcast_under_cp(monkeypatch):+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ seen = []+ monkeypatch.setattr(+ ad, "scaled_dot_product_attention", lambda Q, K, V, **k: (seen.append(Q.dtype), Q)[1]+ )+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 4, n_groups = 1)+ context = ad.AttentionContext(+ bsz = 1,+ q_len = 4,+ kv_seq_len = 4,+ n_heads = 4,+ head_dim = 8,+ requires_grad = True,+ seq_info = None,+ attention_mask = None,+ causal_mask = None,+ )+ Q = torch.zeros(1, 4, 4, 8, dtype = torch.float32)+ ad.run_attention(config = config, context = context, Q = Q, K = Q, V = Q)+ manager = _manager()+ manager.mesh = None+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ out = ad.run_attention(config = config, context = context, Q = Q, K = Q, V = Q)+ assert seen[0] == torch.float32 and seen[1] in (torch.bfloat16, torch.float16)+ assert out.dtype == torch.float32++[email protected]("distributed_type", ["DEEPSPEED", "FSDP"])+def test_deepspeed_and_fsdp_are_refused(monkeypatch, distributed_type):+ import types++ monkeypatch.setattr(cp.dist, "is_available", lambda: True)+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.setattr(cp, "_supports_context_parallel", lambda model: True)+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)+ accelerator = types.SimpleNamespace(+ distributed_type = types.SimpleNamespace(name = distributed_type)+ )+ Trainer = _patched_trainer(monkeypatch, args = args, accelerator = accelerator, model = None)+ with pytest.raises(NotImplementedError, match = "DDP only"):+ Trainer()+++def _cp_env(monkeypatch):+ monkeypatch.setattr(cp.dist, "is_available", lambda: True)+ monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.setattr(cp, "_supports_context_parallel", lambda model: True)++[email protected]("case", ["iterable", "dispatch", "eval_dict"])+def test_dispatched_or_iterable_loaders_are_refused(monkeypatch, case):+ import types++ _cp_env(monkeypatch)++ class Stream(torch.utils.data.IterableDataset):+ def __iter__(self):+ return iter(())++ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)+ accelerator = types.SimpleNamespace(+ distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),+ dispatch_batches = case == "dispatch",+ )+ Trainer = _patched_trainer(+ monkeypatch,+ args = args,+ accelerator = accelerator,+ model = None,+ train_dataset = Stream() if case == "iterable" else [1],+ eval_dataset = {"a": [1], "b": Stream()} if case == "eval_dict" else None,+ )+ with pytest.raises(NotImplementedError, match = "iterable datasets|dispatch_batches"):+ Trainer()+++def test_iterable_eval_dataset_passed_to_evaluate_is_refused(monkeypatch):+ class Stream(torch.utils.data.IterableDataset):+ def __iter__(self):+ return iter(())++ Trainer = _patched_trainer(monkeypatch)+ trainer = object.__new__(Trainer)+ trainer._context_parallel_manager = _manager()+ with pytest.raises(NotImplementedError, match = "iterable datasets"):+ trainer.evaluate(Stream())+ with pytest.raises(NotImplementedError, match = "iterable datasets"):+ trainer.evaluate(eval_dataset = {"a": Stream()})+ trainer._context_parallel_manager = None+ assert trainer.evaluate(Stream()) == "evaluated"+++def test_iterable_predict_dataset_is_refused(monkeypatch):+ class Stream(torch.utils.data.IterableDataset):+ def __iter__(self):+ return iter(())++ Trainer = _patched_trainer(monkeypatch)+ trainer = object.__new__(Trainer)+ trainer._context_parallel_manager = _manager()+ with pytest.raises(NotImplementedError, match = "iterable datasets"):+ trainer.predict(Stream())+ trainer._context_parallel_manager = None+ assert trainer.predict(Stream()) == "predicted"+++def test_sliding_window_models_are_refused(monkeypatch):+ import types++ _cp_env(monkeypatch)+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)+ accelerator = types.SimpleNamespace(distributed_type = types.SimpleNamespace(name = "MULTI_GPU"))+ model = types.SimpleNamespace(config = types.SimpleNamespace(sliding_window = 4096))+ Trainer = _patched_trainer(monkeypatch, args = args, accelerator = accelerator, model = model)+ with pytest.raises(NotImplementedError, match = "sliding window"):+ Trainer()+++def test_a_later_non_cp_trainer_drops_the_installed_mesh(monkeypatch):+ import types++ mesh = object()+ monkeypatch.setattr(cp, "_INSTALLED_MESH", [mesh])+ state = types.SimpleNamespace(device_mesh = mesh)+ args = types.SimpleNamespace(context_parallel_size = 1)+ Trainer = _patched_trainer(+ monkeypatch, args = args, accelerator = types.SimpleNamespace(state = state)+ )+ Trainer()+ assert state.device_mesh is None+ other = object()+ state.device_mesh = other+ Trainer()+ assert state.device_mesh is other+++def test_every_attention_layer_must_be_the_llama_forward():+ from unsloth.models.llama import LlamaAttention_fast_forward++ class Llama(torch.nn.Module):+ forward = LlamaAttention_fast_forward++ class Other(torch.nn.Module):+ def forward(self, x):+ return x++ def model(*attns):+ root = torch.nn.Module()+ root.layers = torch.nn.ModuleList()+ for attn in attns:+ layer = torch.nn.Module()+ layer.self_attn = attn()+ root.layers.append(layer)+ return root++ assert cp._supports_context_parallel(model(Llama, Llama))+ assert not cp._supports_context_parallel(model(Llama, Other))+ assert not cp._supports_context_parallel(model())++[email protected]("user_value", [None, True])+def test_find_unused_parameters_is_off_under_cp_unless_the_user_set_it(monkeypatch, user_value):+ import types++ _cp_env(monkeypatch)+ monkeypatch.setattr(+ cp,+ "ContextParallelManager",+ lambda size, *rest: types.SimpleNamespace(+ size = size, device_mesh = object(), attach_attention_hooks = lambda model: None+ ),+ )+ args = types.SimpleNamespace(+ context_parallel_size = 2,+ label_smoothing_factor = 0.0,+ ddp_find_unused_parameters = user_value,+ )+ handler = types.SimpleNamespace(+ find_unused_parameters = True if user_value is None else user_value+ )+ accelerator = types.SimpleNamespace(+ distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),+ state = types.SimpleNamespace(device_mesh = None),+ ddp_handler = handler,+ )+ Trainer = _patched_trainer(+ monkeypatch, args = args, accelerator = accelerator, model = None, train_dataset = [1]+ )+ Trainer()+ expected = False if user_value is None else True+ # 5.x reads the handler built at init; 4.x rebuilds it from the argument at train time.+ assert handler.find_unused_parameters is expected+ assert args.ddp_find_unused_parameters is expected+++def test_active_manager_is_visible_from_the_autograd_device_thread(monkeypatch):+ import threading++ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ manager = _manager()+ manager.mesh = None+ seen = []+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ thread = threading.Thread(target = lambda: seen.append(cp.get_cp_manager()))+ thread.start()+ thread.join()+ assert seen == [manager]+ assert cp.get_cp_manager() is None+++def test_transformers_cp_size_is_refused(monkeypatch):+ import types++ args = types.SimpleNamespace(+ context_parallel_size = 1, parallelism_config = types.SimpleNamespace(cp_size = 2)+ )+ Trainer = _patched_trainer(monkeypatch, args = args)+ with pytest.raises(NotImplementedError, match = "context_parallel_size"):+ Trainer()+++def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):+ import types++ mesh = object()+ state = types.SimpleNamespace(device_mesh = None)+ Trainer = _patched_trainer(monkeypatch, args = types.SimpleNamespace(context_parallel_size = 1))+ trainer = object.__new__(Trainer)+ trainer.accelerator = types.SimpleNamespace(state = state)+ trainer.args = types.SimpleNamespace(ddp_find_unused_parameters = None)+ trainer._context_parallel_manager = types.SimpleNamespace(size = 2, device_mesh = mesh)+ assert trainer.train() == "trained"+ assert state.device_mesh is mesh+ assert trainer.args.ddp_find_unused_parameters is False+++def test_attention_hooks_are_not_stacked_by_a_second_manager():+ model = torch.nn.Module()+ model.self_attn = torch.nn.Linear(2, 2)+ for _ in range(2):+ _manager().attach_attention_hooks(model)+ assert len(model.self_attn._forward_pre_hooks) == 1+++def test_sdpa_is_restored_when_the_cp_step_raises(monkeypatch):+ import torch.nn.functional as F++ original = F.scaled_dot_product_attention++ @contextlib.contextmanager+ def leaky(mesh, buffers, buffer_seq_dims, no_restore_buffers):+ F.scaled_dot_product_attention = lambda *a, **k: None # torch < 2.13+ yield+ F.scaled_dot_product_attention = original++ monkeypatch.setattr(cp, "context_parallel", leaky)+ manager = _manager()+ manager.mesh = None+ with pytest.raises(RuntimeError):+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ raise RuntimeError("OOM")+ assert F.scaled_dot_product_attention is original++[email protected]("remainder", [0, 3])+def test_eval_gather_keeps_one_loss_per_cp_group(monkeypatch, remainder):+ import types++ per_rank = [torch.tensor([10.0 * r + 1, 10.0 * r + 2]) for r in (0, 0, 1, 1, 2, 2)]+ accelerator = types.SimpleNamespace(+ gather = lambda t: torch.cat(per_rank),+ gather_for_metrics = lambda t, *a, **k: t,+ gradient_state = types.SimpleNamespace(end_of_dataloader = True, remainder = remainder),+ )+ Trainer = _patched_trainer(monkeypatch)+ trainer = object.__new__(Trainer)+ trainer.accelerator = accelerator+ trainer._context_parallel_manager = _manager(size = 2)+ out = trainer.evaluation_loop()+ expected = [1.0, 2.0, 11.0, 12.0, 21.0, 22.0]+ assert out.tolist() == (expected[:remainder] if remainder else expected)+++def test_replaced_iterable_train_dataset_is_refused_at_train(monkeypatch):+ class Stream(torch.utils.data.IterableDataset):+ def __iter__(self):+ return iter(())++ Trainer = _patched_trainer(monkeypatch)+ trainer = object.__new__(Trainer)+ trainer._context_parallel_manager = _manager()+ trainer.train_dataset = Stream()+ with pytest.raises(NotImplementedError, match = "iterable datasets"):+ trainer.train()+++def test_null_inputs_embeds_placeholder_is_allowed():+ inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long), "inputs_embeds": None}+ _manager()._prepare_inputs(inputs)+ assert inputs["position_ids"].shape == (1, 4)+++def test_replaced_iterable_eval_dataset_is_refused_at_evaluate(monkeypatch):+ class Stream(torch.utils.data.IterableDataset):+ def __iter__(self):+ return iter(())++ Trainer = _patched_trainer(monkeypatch)+ trainer = object.__new__(Trainer)+ trainer._context_parallel_manager = _manager()+ trainer.eval_dataset = Stream()+ with pytest.raises(NotImplementedError, match = "iterable datasets"):+ trainer.evaluate()+ trainer.eval_dataset = {"a": [1], "b": Stream()}+ with pytest.raises(NotImplementedError, match = "iterable datasets"):+ trainer.evaluate("b")+ assert trainer.evaluate("a") == "evaluated"+++def test_model_init_is_refused(monkeypatch):+ import types++ _cp_env(monkeypatch)+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)+ Trainer = _patched_trainer(monkeypatch, args = args, model_init = lambda: None)+ with pytest.raises(NotImplementedError, match = "model_init"):+ Trainer()+++def test_cp_dtensor_dispatcher_is_torn_down_when_the_step_raises(monkeypatch):+ dispatcher = cp.DTensor._op_dispatcher+ original = dispatcher._custom_op_handlers+ unregistered = []+ monkeypatch.setattr(cp, "_unregister_cp_sharding_rules", lambda: unregistered.append(1))++ @contextlib.contextmanager+ def leaky(mesh, buffers, buffer_seq_dims, no_restore_buffers):+ dispatcher._custom_op_handlers = {**original, "cp": None} # torch < 2.13: no finally+ yield+ dispatcher._custom_op_handlers = original++ monkeypatch.setattr(cp, "context_parallel", leaky)+ manager = _manager()+ manager.mesh = None+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ pass+ assert dispatcher._custom_op_handlers is original and unregistered == []+ with pytest.raises(RuntimeError):+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ raise RuntimeError("OOM")+ assert dispatcher._custom_op_handlers is original and unregistered == [1]+++def _gated_trainer(+ monkeypatch,+ model = None,+ **args_overrides,+):+ """A trainer that clears every CP gate, with the manager stubbed (no process group)."""+ import types++ _cp_env(monkeypatch)+ built = []++ def manager(+ size,+ backend = "ring",+ rotate_method = "allgather",+ ):+ built.append((size, backend, rotate_method))+ return types.SimpleNamespace(+ size = size, device_mesh = object(), attach_attention_hooks = lambda model: None+ )++ monkeypatch.setattr(cp, "ContextParallelManager", manager)+ args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)+ for key, value in args_overrides.items():+ setattr(args, key, value)+ accelerator = types.SimpleNamespace(+ distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),+ state = types.SimpleNamespace(device_mesh = None),+ )+ init = {"args": args, "accelerator": accelerator, "model": model, "train_dataset": [1]}+ return _patched_trainer(monkeypatch, **init), args, built+++ "overrides, error",+ [+ ({"context_parallel_size": True}, ValueError),+ ({"context_parallel_size": 0}, ValueError),+ ({"context_parallel_size": -2}, ValueError),+ ({"context_parallel_size": 2.0}, ValueError),+ ({"context_parallel_size": "2"}, ValueError),+ ({"context_parallel_backend": "tree"}, ValueError),+ ({"context_parallel_rotate_method": "ring"}, ValueError),+ ({"context_parallel_backend": "ulysses"}, NotImplementedError),+ ],+)+def test_bad_cp_options_fail_before_the_trainer_builds(monkeypatch, overrides, error):+ import types, trl++ seen = []++ class Trainer:+ def __init__(self, *a, **k):+ seen.append(1)++ training_step = prediction_step = train = evaluate = evaluation_loop = None++ monkeypatch.setattr(trl, "SFTTrainer", Trainer)+ cp.patch_sft_trainer()+ args = types.SimpleNamespace(**{"context_parallel_size": 2, **overrides})+ with pytest.raises(error):+ Trainer(None, args)+ with pytest.raises(error):+ Trainer(None, args = args)+ assert seen == []+++def test_cp_option_defaults_and_size_one():+ import types++ assert cp.context_parallel_options(None) == (1, "ring", "allgather")+ config = types.SimpleNamespace(context_parallel_size = 1, context_parallel_backend = "ulysses")+ assert cp.context_parallel_options(config) == (1, "ulysses", "allgather")+ config = types.SimpleNamespace(+ context_parallel_size = 4, context_parallel_rotate_method = "alltoall"+ )+ assert cp.context_parallel_options(config) == (4, "ring", "alltoall")+++def test_manager_gets_the_backend_and_rotate_method(monkeypatch):+ Trainer, _, built = _gated_trainer(monkeypatch, context_parallel_rotate_method = "alltoall")+ Trainer()+ assert built == [(2, "ring", "alltoall")]++[email protected]("version", ["2.6.0", "2.6.0+cu124", "2.5.1"])+def test_torch_older_than_2_7_is_refused(monkeypatch, version):+ Trainer, _, _ = _gated_trainer(monkeypatch)+ monkeypatch.setattr(cp.torch, "__version__", version)+ with pytest.raises(RuntimeError, match = "PyTorch >= 2.7"):+ Trainer()+++def test_torch_2_7_dev_builds_pass_the_gate(monkeypatch):+ monkeypatch.setattr(cp.torch, "__version__", "2.7.0a0+git1234")+ assert cp._torch_supports_context_parallel()+++ "config, reason",+ [+ ({"layer_types": ["full_attention", "sliding_attention"]}, "sliding_attention"),+ ({"layer_types": ["chunked_attention", "full_attention"]}, "chunked_attention"),+ ({"attn_logit_softcapping": 50.0}, "softcapping"),+ ({"attention_chunk_size": 8192}, "chunked attention"),+ ({"use_sliding_window": True}, "sliding window"),+ ({"text_config": {"layer_types": ["sliding_attention"]}}, "sliding_attention"),+ ],+)+def test_windowed_or_softcapped_attention_is_refused_for_ring(monkeypatch, config, reason):+ import types++ if "text_config" in config:+ config = {"text_config": types.SimpleNamespace(**config["text_config"])}+ model = types.SimpleNamespace(config = types.SimpleNamespace(**config))+ Trainer, _, _ = _gated_trainer(monkeypatch, model = model)+ with pytest.raises(NotImplementedError, match = reason):+ Trainer()+++def test_full_attention_layer_types_pass(monkeypatch):+ import types++ config = types.SimpleNamespace(+ layer_types = ["full_attention"] * 2, sliding_window = None, use_sliding_window = False+ )+ Trainer, _, built = _gated_trainer(monkeypatch, model = types.SimpleNamespace(config = config))+ Trainer()+ assert built+++def test_legacy_prediction_loop_is_refused(monkeypatch):+ Trainer, _, built = _gated_trainer(monkeypatch, use_legacy_prediction_loop = True)+ with pytest.raises(NotImplementedError, match = "use_legacy_prediction_loop"):+ Trainer()+ assert built == []+++def test_model_without_loss_kwargs_is_refused(monkeypatch):+ Trainer, _, _ = _gated_trainer(monkeypatch)+ monkeypatch.setattr(Trainer, "model_accepts_loss_kwargs", False, raising = False)+ with pytest.raises(NotImplementedError, match = "loss kwargs"):+ Trainer()+++def test_flash_attention_impl_is_scoped_to_train_without_packing(monkeypatch):+ from unsloth.utils import packing++ scoped = []+ monkeypatch.setattr(+ packing, "_scope_flash_attention_impl", lambda model, trainer: scoped.append(trainer)+ )+ Trainer, _, _ = _gated_trainer(monkeypatch)+ trainer = Trainer()+ assert scoped == [trainer]+++def test_training_step_refuses_a_missing_token_count_with_labels(monkeypatch):+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ Trainer = _patched_trainer(monkeypatch)+ trainer = object.__new__(Trainer)+ manager = _manager()+ manager.mesh = None+ trainer._context_parallel_manager = manager+ batch = {"input_ids": torch.ones(1, 4, dtype = torch.long), "labels": torch.ones(1, 4)}+ with pytest.raises(RuntimeError, match = "num_items_in_batch"):+ trainer.training_step(None, dict(batch))+ with pytest.raises(RuntimeError, match = "num_items_in_batch"):+ trainer.training_step(None, dict(batch), None)+ shifted = {"input_ids": torch.ones(1, 4, dtype = torch.long), "shift_labels": torch.ones(1, 4)}+ with pytest.raises(RuntimeError, match = "num_items_in_batch"):+ trainer.training_step(None, shifted)+ assert cp.get_cp_manager() is None+ trainer.training_step(None, {"input_ids": torch.ones(1, 4, dtype = torch.long)})+ trainer.training_step(None, dict(batch), torch.tensor(4.0))++[email protected]("raises", [False, True])+def test_rotate_method_is_set_for_the_step_and_restored(monkeypatch, raises):+ if cp._cp_options is None:+ pytest.skip("this torch has no ring rotate option")+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ manager = _manager()+ manager.mesh = None+ manager.rotate_method = "alltoall"+ before = cp._cp_options.rotate_method+ seen = []+ with contextlib.suppress(RuntimeError):+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ seen.append(cp._cp_options.rotate_method.name)+ if raises:+ raise RuntimeError("boom")+ assert seen == ["ALL_TO_ALL"]+ assert cp._cp_options.rotate_method == before++[email protected]("raises", [False, True])+def test_ring_pins_sdpa_to_kernels_the_dispatcher_rings(monkeypatch, raises):+ if cp.sdpa_kernel is None:+ pytest.skip("no torch.nn.attention.sdpa_kernel")+ monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))+ manager = _manager()+ manager.mesh = None+ cuda = torch.backends.cuda+ before = (cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled(), cuda.math_sdp_enabled())+ seen = []+ with contextlib.suppress(RuntimeError):+ with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):+ seen.append(+ (+ cuda.flash_sdp_enabled(),+ cuda.mem_efficient_sdp_enabled(),+ cuda.math_sdp_enabled(),+ )+ )+ if raises:+ raise RuntimeError("boom")+ assert seen == [(True, True, False)]+ assert (+ cuda.flash_sdp_enabled(),+ cuda.mem_efficient_sdp_enabled(),+ cuda.math_sdp_enabled(),+ ) == before+++def test_cudnn_is_pinned_only_where_the_ring_handles_it():+ backends = cp._ring_sdpa_backends()+ if backends is None:+ pytest.skip("no SDPBackend")+ from torch.nn.attention import SDPBackend++ assert backends[:2] == [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]+ from packaging.version import Version++ if Version(Version(torch.__version__).base_version) >= Version("2.13"):+ assert SDPBackend.CUDNN_ATTENTION in backends+++def test_unsloth_training_arguments_keep_the_cp_options():+ from unsloth.trainer import UnslothTrainingArguments+ args = UnslothTrainingArguments(+ output_dir = "unused",+ context_parallel_size = 2,+ context_parallel_rotate_method = "alltoall",+ report_to = "none",+ )+ assert cp.context_parallel_options(args) == (2, "ring", "alltoall")+++def test_cli_forwards_the_cp_options():+ import importlib.util, pathlib, types++ path = pathlib.Path(__file__).resolve().parents[1] / "unsloth-cli.py"+ spec = importlib.util.spec_from_file_location("unsloth_cli_for_test", path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ args = types.SimpleNamespace(+ per_device_train_batch_size = 1,+ gradient_accumulation_steps = 1,+ warmup_steps = 0,+ max_steps = 1,+ learning_rate = 1e-4,+ logging_steps = 1,+ optim = "adamw_torch",+ weight_decay = 0.0,+ lr_scheduler_type = "linear",+ seed = 0,+ output_dir = "unused",+ report_to = "none",+ max_seq_length = 64,+ packing = False,+ per_device_eval_batch_size = 4,+ context_parallel_size = 2,+ context_parallel_backend = "ring",+ context_parallel_rotate_method = "alltoall",+ )+ config = module._build_sft_config(lambda **kwargs: kwargs, args, False, True)+ assert config["context_parallel_size"] == 2+ assert config["context_parallel_backend"] == "ring"+ assert config["context_parallel_rotate_method"] == "alltoall"+ assert config["average_tokens_across_devices"] is True+++def test_generated_sft_config_carries_the_cp_options(tmp_path):+ import dataclasses+ import trl.trainer.sft_config as config_module++ names = {f.name for f in dataclasses.fields(config_module.SFTConfig)}+ if "context_parallel_size" not in names:+ pytest.skip("SFTConfig is not the Unsloth-generated class in this process")+ assert {"context_parallel_backend", "context_parallel_rotate_method"} <= names+ config = config_module.SFTConfig(+ output_dir = str(tmp_path),+ context_parallel_size = 2,+ context_parallel_rotate_method = "alltoall",+ report_to = "none",+ )+ assert cp.context_parallel_options(config) == (2, "ring", "alltoall")+ default = config_module.SFTConfig(output_dir = str(tmp_path), report_to = "none")+ assert cp.context_parallel_options(default) == (1, "ring", "allgather")+++def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):+ import trl++ class Base:+ def train(self):+ return "base"++ def training_step(self, *a, **k):+ pass++ evaluate = evaluation_loop = prediction_step = training_step++ class Trainer(Base):+ def __init__(self):+ pass++ monkeypatch.setattr(trl, "SFTTrainer", Trainer)+ cp.patch_sft_trainer()+ monkeypatch.setattr(Base, "train", lambda self: "patched later")+ assert Trainer().train() == "patched later"diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.pyindex ac53406e29f..cf8c6353555 100644--- a/tests/utils/test_packing.py+++ b/tests/utils/test_packing.py@@ -2108,6 +2108,24 @@ def test_packing_skip_warning_keeps_custom_collator_reason(monkeypatch, caplog):assert "UNSLOTH_RETURN_LOGITS" not in messages[0]+def test_context_parallel_disables_eval_packing(monkeypatch):+ monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)+ fake_trainer = _patch_fake_sft_trainer()+ config = SimpleNamespace(+ packing = True,+ eval_packing = True,+ padding_free = None,+ remove_unused_columns = True,+ context_parallel_size = 2,+ )+ fake_trainer(+ model = _warn_text_model(),+ args = config,+ train_dataset = Dataset.from_dict({"text": ["sample"]}),+ )+ assert config.packing is False and config.eval_packing is False++# Fused-CE paths shift internally, so they mask boundaries on unshifted labels.def test_mask_packed_boundary_labels_masks_next_document_first_token():labels = torch.arange(6, dtype = torch.long).view(1, 6)diff --git a/unsloth-cli.py b/unsloth-cli.pyindex 87dc719cecf..ffdaad7ec26 100644--- a/unsloth-cli.py+++ b/unsloth-cli.py@@ -186,6 +186,16 @@ def _build_sft_config(SFTConfig, args, is_mlx, bf16_supported):dataset_num_proc = 2,packing = args.packing,)+ if getattr(args, "context_parallel_size", 1) > 1:+ config_kwargs["context_parallel_size"] = args.context_parallel_size+ config_kwargs["context_parallel_backend"] = getattr(+ args, "context_parallel_backend", "ring"+ )+ config_kwargs["context_parallel_rotate_method"] = getattr(+ args, "context_parallel_rotate_method", "allgather"+ )+ # Required under CP; older transformers releases default it to False.+ config_kwargs["average_tokens_across_devices"] = Trueif is_mlx:if args.per_device_eval_batch_size != 4:print("Warning: --per_device_eval_batch_size is ignored on MLX without eval data.")@@ -463,6 +473,26 @@ if __name__ == "__main__":action = "store_true",help = "Enable padding-free sample packing via TRL's bin packer.",)+ training_group.add_argument(+ "--context_parallel_size",+ type = int,+ default = 1,+ help = "Ranks per context parallel group (SDPA ring attention, torchrun, PyTorch >= 2.7).",+ )+ training_group.add_argument(+ "--context_parallel_backend",+ type = str,+ default = "ring",+ choices = ["ring"],+ help = "Context parallel attention backend.",+ )+ training_group.add_argument(+ "--context_parallel_rotate_method",+ type = str,+ default = "allgather",+ choices = ["allgather", "alltoall"],+ help = "How ring attention rotates KV shards between ranks.",+ )report_group = parser.add_argument_group("📊 Report Options")report_group.add_argument(diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 5aa2536177f..7ae16996db7 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -771,3 +771,6 @@ from unsloth_zoo.rl_environments import (# inspect.getsource() and corrupts downstream drift detectors.if os.environ.get("UNSLOTH_ALLOW_CPU", "0") != "1":_patch_trl_trainer()++ from .context_parallel import patch_sft_trainer+ patch_sft_trainer()diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pynew file mode 100644index 00000000000..1307178e448--- /dev/null+++ b/unsloth/context_parallel.py@@ -0,0 +1,549 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++from __future__ import annotations++import contextlib+import functools+from typing import Iterator, Optional++import torch+import torch.nn.functional as F+import torch.distributed as dist++try:+ from torch.distributed.tensor.experimental import context_parallel+ from torch.distributed.device_mesh import DeviceMesh+ from torch.distributed.tensor import DTensor+except (ImportError, AttributeError):+ context_parallel = None+ DeviceMesh = None+ DTensor = None++try:+ # Re-exported from _context_parallel._attention on torch >= 2.10; defined here on 2.7 - 2.9.+ from torch.distributed.tensor.experimental._attention import _cp_options, set_rotate_method+except (ImportError, AttributeError):+ _cp_options = None+ set_rotate_method = None++try:+ from torch.nn.attention import SDPBackend, sdpa_kernel+except ImportError:+ SDPBackend = None+ sdpa_kernel = None++_CP_BACKENDS = ("ring", "ulysses")+_ROTATE_METHODS = ("allgather", "alltoall")+++def _torch_supports_context_parallel() -> bool:+ from packaging.version import Version+ return context_parallel is not None and Version(+ Version(torch.__version__).base_version+ ) >= Version("2.7.0")+++def _ring_sdpa_backends():+ # The math kernel is not ringed: it would attend over the local shard only.+ if SDPBackend is None:+ return None+ backends = [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]+ cudnn = getattr(SDPBackend, "CUDNN_ATTENTION", None)+ op = getattr(torch.ops.aten, "_scaled_dot_product_cudnn_attention", None)+ if cudnn is not None and op is not None:+ try:+ from torch.distributed.tensor.experimental._context_parallel import _attention+ except ImportError:+ try:+ from torch.distributed.tensor.experimental import _attention+ except ImportError:+ _attention = None+ handled = getattr(_attention, "custom_ops", None)+ if isinstance(handled, dict) and op.default in handled:+ backends.append(cudnn)+ return backends+++def context_parallel_options(config) -> tuple:+ """(size, backend, rotate_method) from a training config, validated before any dataset work."""+ size = getattr(config, "context_parallel_size", 1)+ size = 1 if size is None else size+ if isinstance(size, bool) or not isinstance(size, int) or size < 1:+ raise ValueError(+ f"Unsloth: context_parallel_size must be a positive integer, got {size!r}."+ )+ backend = getattr(config, "context_parallel_backend", None) or "ring"+ if backend not in _CP_BACKENDS:+ raise ValueError(+ f"Unsloth: context_parallel_backend must be one of {_CP_BACKENDS}, got {backend!r}."+ )+ rotate_method = getattr(config, "context_parallel_rotate_method", None) or "allgather"+ if rotate_method not in _ROTATE_METHODS:+ raise ValueError(+ "Unsloth: context_parallel_rotate_method must be one of "+ f"{_ROTATE_METHODS}, got {rotate_method!r}."+ )+ if size > 1 and backend == "ulysses":+ raise NotImplementedError(+ "Unsloth: context_parallel_backend = 'ulysses' is not available yet; use 'ring'."+ )+ return size, backend, rotate_method+++def _ring_unsupported_attention(config) -> Optional[str]:+ configs = [config]+ text_config = getattr(config, "text_config", None)+ if text_config is not None and text_config is not config:+ configs.append(text_config)+ for c in configs:+ if c is None:+ continue+ layer_types = getattr(c, "layer_types", None)+ if layer_types and any(t != "full_attention" for t in layer_types):+ kinds = sorted({t for t in layer_types if t != "full_attention"})+ return f"{', '.join(kinds)} layers"+ # Any positive window, even with use_sliding_window off: Unsloth's masks read it directly.+ sliding_window = getattr(c, "sliding_window", None)+ if (isinstance(sliding_window, int) and sliding_window > 0) or getattr(+ c, "use_sliding_window", None+ ) is True:+ return "sliding window attention"+ if getattr(c, "attention_chunk_size", None):+ return "chunked attention"+ if getattr(c, "attn_logit_softcapping", None):+ return "attention logit softcapping"+ return None+++def _unregister_cp_sharding_rules() -> None:+ try:+ from torch.distributed.tensor.experimental._context_parallel._sharding_rules import (+ unregister_cp_sharding_rules,+ )+ except ImportError:+ return+ unregister_cp_sharding_rules(clear_the_cache = False)+++from .device_type import DEVICE_TYPE_TORCH++# A module global, not a ContextVar: CUDA runs backward (and checkpoint recompute) on another thread.+_ACTIVE_MANAGER: Optional["ContextParallelManager"] = None++# shift_labels is built before sharding so each shard's last token keeps its next-token target.+_BUFFER_NAMES = ("input_ids", "attention_mask", "labels", "position_ids", "shift_labels")+_PAD_VALUES = {"labels": -100, "shift_labels": -100, "attention_mask": 0, "input_ids": 0}+++_INSTALLED_MESH = []+++def get_cp_manager() -> Optional["ContextParallelManager"]:+ return _ACTIVE_MANAGER+++def _supports_context_parallel(model) -> bool:+ from .models.llama import LlamaAttention_fast_forward+ forwards = [+ getattr(type(module).forward, "__func__", type(module).forward)+ for name, module in model.named_modules()+ if name.endswith("self_attn")+ ]+ return bool(forwards) and all(f is LlamaAttention_fast_forward for f in forwards)+++def _self_attn_pre_forward_hook(_module, module_args, module_kwargs):+ # Ring attention only supports is_causal with no mask; right padding is already -100 in labels.+ if get_cp_manager() is not None and "attention_mask" in module_kwargs:+ module_kwargs["attention_mask"] = None+ return module_args, module_kwargs+++class ContextParallelManager:+ def __init__(+ self,+ size: int,+ backend: str = "ring",+ rotate_method: str = "allgather",+ ):+ self.size = size+ self.backend = backend+ self.rotate_method = rotate_method+ world_size = dist.get_world_size()+ self.device_mesh = DeviceMesh(+ DEVICE_TYPE_TORCH,+ torch.arange(world_size).reshape(world_size // size, size),+ mesh_dim_names = ("dp_replicate", "cp"),+ )+ self.mesh = self.device_mesh["cp"]++ def attach_attention_hooks(self, model: torch.nn.Module) -> None:+ for name, module in model.named_modules():+ if name.endswith("self_attn") and not getattr(module, "_unsloth_cp_hooked", False):+ module.register_forward_pre_hook(+ _self_attn_pre_forward_hook, with_kwargs = True, prepend = True+ )+ module._unsloth_cp_hooked = True++ def _prepare_inputs(self, inputs: dict) -> None:+ input_ids = inputs.get("input_ids")+ if not isinstance(input_ids, torch.Tensor) or inputs.get("inputs_embeds") is not None:+ raise ValueError(+ "Unsloth: context parallelism needs input_ids batches (not inputs_embeds)."+ )+ bsz, seq_len = input_ids.shape+ if inputs.get("position_ids") is None:+ inputs["position_ids"] = (+ torch.arange(seq_len, device = input_ids.device).expand(bsz, -1).contiguous()+ )+ mask = inputs.get("attention_mask")+ if isinstance(mask, torch.Tensor) and (+ mask.ndim != 2 or (mask[:, 1:] > mask[:, :-1]).any()+ ):+ raise ValueError(+ "Unsloth: context parallelism needs 2D right-padded attention masks without holes."+ )+ labels = inputs.get("labels")+ if inputs.get("shift_labels") is None and labels is not None:+ inputs["shift_labels"] = F.pad(labels, (0, 1), value = -100)[:, 1:].contiguous()+ pad = (-seq_len) % (2 * self.size)+ if pad:+ for name in _BUFFER_NAMES:+ tensor = inputs.get(name)+ if not isinstance(tensor, torch.Tensor) or tensor.ndim < 2:+ continue+ if name == "position_ids":+ extra = tensor[:, -1:] + torch.arange(1, pad + 1, device = tensor.device)+ inputs[name] = torch.cat([tensor, extra.to(tensor.dtype)], dim = 1)+ else:+ inputs[name] = F.pad(tensor, (0, pad), value = _PAD_VALUES.get(name, 0))++ @contextlib.contextmanager+ def _rotate_method_scope(self) -> Iterator[None]:+ rotate_method = getattr(self, "rotate_method", "allgather")+ if _cp_options is None or set_rotate_method is None:+ if rotate_method != "allgather":+ raise RuntimeError(+ f"Unsloth: this PyTorch cannot set the ring rotate method to {rotate_method!r}."+ )+ yield+ return+ previous = _cp_options.rotate_method+ set_rotate_method(rotate_method)+ try:+ yield+ finally:+ _cp_options.rotate_method = previous++ @contextlib.contextmanager+ def apply(self, inputs: dict) -> Iterator[None]:+ self._prepare_inputs(inputs)+ buffers = [+ inputs[name]+ for name in _BUFFER_NAMES+ if isinstance(inputs.get(name), torch.Tensor) and inputs[name].ndim > 1+ ]+ global _ACTIVE_MANAGER+ previous, _ACTIVE_MANAGER = _ACTIVE_MANAGER, self+ # torch < 2.13 leaves its CP dispatcher installed when the step raises.+ sdpa = F.scaled_dot_product_attention+ dispatcher = getattr(DTensor, "_op_dispatcher", None)+ handlers = getattr(dispatcher, "_custom_op_handlers", None)+ ring_backends = _ring_sdpa_backends()+ raised = False+ try:+ with (+ (+ sdpa_kernel(ring_backends)+ if ring_backends is not None+ else contextlib.nullcontext()+ ),+ self._rotate_method_scope(),+ context_parallel(+ self.mesh,+ buffers = buffers,+ buffer_seq_dims = [1] * len(buffers),+ no_restore_buffers = set(buffers),+ ),+ ):+ yield+ except BaseException:+ raised = True+ raise+ finally:+ _ACTIVE_MANAGER = previous+ F.scaled_dot_product_attention = sdpa+ if raised:+ if handlers is not None:+ dispatcher._custom_op_handlers = handlers+ _unregister_cp_sharding_rules()+++def _refuse_iterable_datasets(*datasets) -> None:+ # accelerate's batch dispatcher (dispatch_batches, the default for iterable datasets) ignores cp.+ for dataset in datasets:+ for d in dataset.values() if isinstance(dataset, dict) else [dataset]:+ if (+ isinstance(d, torch.utils.data.IterableDataset)+ or "IterableDataset" in type(d).__name__+ ):+ raise NotImplementedError(+ "Unsloth: context parallelism does not support iterable datasets."+ )+++def _cp_gather_for_metrics(accelerator, size: int):+ # Keep one loss per CP group, else last-batch truncation keeps duplicates and drops replicas.+ def gather(tensor, *args, **kwargs):+ if not isinstance(tensor, torch.Tensor) or tensor.ndim == 0:+ return accelerator.gather_for_metrics(tensor, *args, **kwargs)+ gathered = accelerator.gather(tensor)+ gathered = gathered.reshape(-1, size, *tensor.shape)[:, 0].reshape(-1, *tensor.shape[1:])+ state = accelerator.gradient_state+ if state.end_of_dataloader and state.remainder > 0:+ gathered = gathered[: state.remainder]+ return gathered++ return gather+++def patch_sft_trainer() -> None:+ import trl++ trainer_cls = trl.SFTTrainer+ if trainer_cls.__dict__.get("__unsloth_context_parallel__"):+ return++ def _original(name):+ # Inherited methods are looked up per call: Trainer.train etc. may be patched after import.+ if name in trainer_cls.__dict__:+ return trainer_cls.__dict__[name]++ @functools.wraps(getattr(trainer_cls, name, None))+ def inherited(self, *args, **kwargs):+ return getattr(super(trainer_cls, self), name)(*args, **kwargs)++ return inherited++ original_init = trainer_cls.__init__+ original_prediction_step = _original("prediction_step")+ original_training_step = _original("training_step")+ original_train = _original("train")+ original_evaluate = _original("evaluate")+ original_predict = _original("predict")+ original_evaluation_loop = _original("evaluation_loop")++ def _install_accelerator_state(self):+ manager = getattr(self, "_context_parallel_manager", None)+ accelerator = getattr(self, "accelerator", None)+ state = getattr(accelerator, "state", None)+ if manager is None:+ if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:+ state.device_mesh = None+ return+ if accelerator is None:+ return+ accelerator.state.device_mesh = manager.device_mesh+ _INSTALLED_MESH[:] = [manager.device_mesh]+ # PEFT makes the Trainer ask DDP for find_unused_parameters, which breaks reentrant checkpointing.+ if getattr(self.args, "ddp_find_unused_parameters", None) in (None, False):+ self.args.ddp_find_unused_parameters = False+ handler = getattr(accelerator, "ddp_handler", None)+ if handler is not None:+ handler.find_unused_parameters = False++ @functools.wraps(original_init)+ def patched_init(self, *args, **kwargs):+ config = kwargs["args"] if "args" in kwargs else (args[1] if len(args) > 1 else None)+ context_parallel_options(config)+ original_init(self, *args, **kwargs)+ self._context_parallel_manager = None+ # Unsloth's attention ignores transformers' own CP: it would stay local to each shard.+ parallelism_config = getattr(getattr(self, "args", None), "parallelism_config", None)+ if (getattr(parallelism_config, "cp_size", 1) or 1) > 1:+ raise NotImplementedError(+ "Unsloth: use SFTConfig(context_parallel_size = N) instead of "+ "parallelism_config with cp_size > 1."+ )+ size, backend, rotate_method = context_parallel_options(getattr(self, "args", None))+ if size <= 1:+ _install_accelerator_state(self)+ return+ # train() would swap in a fresh model without the attention hooks or the support check.+ if getattr(self, "model_init", None) is not None:+ raise NotImplementedError("Unsloth: context parallelism does not support model_init.")+ if not _torch_supports_context_parallel():+ raise RuntimeError(+ f"Unsloth: context_parallel_size > 1 needs PyTorch >= 2.7, found {torch.__version__}."+ )+ if not (dist.is_available() and dist.is_initialized()):+ raise RuntimeError(+ "Unsloth: context_parallel_size > 1 needs torch.distributed. "+ f"Launch with torchrun or accelerate launch using a multiple of {size} processes."+ )+ world_size = dist.get_world_size()+ if world_size % size != 0:+ raise RuntimeError(+ f"Unsloth: world size {world_size} is not a multiple of context_parallel_size {size}."+ )+ if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] < 8:+ # Pure torch ring attention fails in backward without flash (2x T4, torch 2.10).+ raise NotImplementedError(+ "Unsloth: context parallelism needs flash attention (compute capability >= 8.0)."+ )+ # These recompute loss outside our forward from sharded labels, losing cross-shard targets.+ if (+ getattr(self.args, "label_smoothing_factor", 0)+ or getattr(self, "compute_loss_func", None)+ or getattr(self.args, "loss_type", None) not in (None, "nll")+ ):+ raise NotImplementedError(+ "Unsloth: context parallelism needs loss_type = 'nll' without "+ "label_smoothing_factor or compute_loss_func."+ )+ if getattr(self, "model_accepts_loss_kwargs", True) is False:+ raise NotImplementedError(+ "Unsloth: context parallelism needs a model whose forward accepts loss kwargs "+ "(num_items_in_batch)."+ )+ if getattr(self.args, "use_legacy_prediction_loop", False):+ raise NotImplementedError(+ "Unsloth: context parallelism needs use_legacy_prediction_loop = False."+ )+ if getattr(self.args, "average_tokens_across_devices", True) is False:+ raise NotImplementedError(+ "Unsloth: context parallelism needs average_tokens_across_devices = True."+ )+ import accelerate+ from packaging.version import Version++ if Version(accelerate.__version__) < Version("1.10.0"):+ raise NotImplementedError("Unsloth: context parallelism needs accelerate >= 1.10.0.")+ unsupported = _ring_unsupported_attention(getattr(self.model, "config", None))+ if unsupported is not None:+ raise NotImplementedError(+ f"Unsloth: ring context parallelism does not support {unsupported}."+ )+ if not _supports_context_parallel(self.model):+ raise NotImplementedError(+ "Unsloth: context parallelism currently supports Llama-style attention only "+ "(Llama, Qwen2, Gemma)."+ )+ accelerator = getattr(self, "accelerator", None)+ distributed_type = getattr(getattr(accelerator, "distributed_type", None), "name", "NO")+ if distributed_type != "NO" and not distributed_type.startswith("MULTI_"):+ raise NotImplementedError(+ f"Unsloth: context parallelism supports DDP only, not {distributed_type}."+ )+ if getattr(accelerator, "dispatch_batches", None):+ raise NotImplementedError(+ "Unsloth: context parallelism does not support dispatch_batches."+ )+ _refuse_iterable_datasets(+ getattr(self, "train_dataset", None), getattr(self, "eval_dataset", None)+ )+ manager = ContextParallelManager(size, backend, rotate_method)+ self._context_parallel_manager = manager+ manager.attach_attention_hooks(self.model)+ _install_accelerator_state(self)+ # The ring calls the flash ops FA3 / FA4 override, so scope them to train() here too.+ from .utils.packing import _scope_flash_attention_impl++ _scope_flash_attention_impl(self.model, self)+ print(+ f"Unsloth: Context parallelism enabled with size = {size} "+ f"({backend}, rotate_method = {rotate_method})."+ )++ @functools.wraps(original_prediction_step)+ def patched_prediction_step(self, model, inputs, *args, **kwargs):+ manager = getattr(self, "_context_parallel_manager", None)+ prediction_loss_only = args[0] if args else kwargs.get("prediction_loss_only", True)+ if manager is not None and not prediction_loss_only:+ raise NotImplementedError(+ "Unsloth: context parallelism supports loss-only evaluation; "+ "compute_metrics / predict() need context_parallel_size = 1."+ )+ if manager is None:+ return original_prediction_step(self, model, inputs, *args, **kwargs)+ inputs = self._prepare_inputs(inputs)+ manager._prepare_inputs(inputs)+ shift_labels = inputs.get("shift_labels")+ if not isinstance(shift_labels, torch.Tensor):+ with manager.apply(inputs):+ return original_prediction_step(self, model, inputs, *args, **kwargs)+ inputs["num_items_in_batch"] = shift_labels.ne(-100).sum().clamp_min(1)+ inputs.setdefault("use_cache", False)+ with manager.apply(inputs), torch.no_grad(), self.compute_loss_context_manager():+ loss = self.compute_loss(model, inputs).detach()+ dist.all_reduce(loss, group = manager.mesh.get_group())+ return loss, None, None++ @functools.wraps(original_train)+ def patched_train(self, *args, **kwargs):+ if getattr(self, "_context_parallel_manager", None) is not None:+ _refuse_iterable_datasets(getattr(self, "train_dataset", None))+ _install_accelerator_state(self)+ return original_train(self, *args, **kwargs)++ @functools.wraps(original_evaluate)+ def patched_evaluate(self, *args, **kwargs):+ if getattr(self, "_context_parallel_manager", None) is not None:+ eval_dataset = args[0] if args else kwargs.get("eval_dataset")+ if eval_dataset is None or isinstance(eval_dataset, str):+ current = self.eval_dataset+ eval_dataset = (+ current.get(eval_dataset)+ if isinstance(eval_dataset, str) and isinstance(current, dict)+ else current+ )+ _refuse_iterable_datasets(eval_dataset)+ _install_accelerator_state(self)+ return original_evaluate(self, *args, **kwargs)++ @functools.wraps(original_predict)+ def patched_predict(self, test_dataset, *args, **kwargs):+ if getattr(self, "_context_parallel_manager", None) is not None:+ _refuse_iterable_datasets(test_dataset)+ _install_accelerator_state(self)+ return original_predict(self, test_dataset, *args, **kwargs)++ @functools.wraps(original_evaluation_loop)+ def patched_evaluation_loop(self, *args, **kwargs):+ manager = getattr(self, "_context_parallel_manager", None)+ if manager is not None:+ self.gather_function = _cp_gather_for_metrics(self.accelerator, manager.size)+ return original_evaluation_loop(self, *args, **kwargs)++ @functools.wraps(original_training_step)+ def patched_training_step(self, model, inputs, *args, **kwargs):+ manager = getattr(self, "_context_parallel_manager", None)+ if manager is not None:+ num_items = args[0] if args else kwargs.get("num_items_in_batch")+ if (+ num_items is None+ and hasattr(inputs, "get")+ and (inputs.get("labels") is not None or inputs.get("shift_labels") is not None)+ ):+ raise RuntimeError(+ "Unsloth: context parallelism needs num_items_in_batch in training_step; "+ "this transformers / trainer path did not pass one."+ )+ # HF summed the count over all D x N ranks while CP peers share a batch: divide by N (exact).+ if args and args[0] is not None:+ args = (args[0] // manager.size, *args[1:])+ elif kwargs.get("num_items_in_batch") is not None:+ kwargs["num_items_in_batch"] = kwargs["num_items_in_batch"] // manager.size+ # Forward and backward both inside, so gradient checkpointing recomputes with ring attention.+ with manager.apply(inputs) if manager else contextlib.nullcontext():+ return original_training_step(self, model, inputs, *args, **kwargs)++ trainer_cls.__init__ = patched_init+ trainer_cls.prediction_step = patched_prediction_step+ trainer_cls.train = patched_train+ trainer_cls.evaluate = patched_evaluate+ trainer_cls.predict = patched_predict+ trainer_cls.evaluation_loop = patched_evaluation_loop+ trainer_cls.training_step = patched_training_step+ trainer_cls.__unsloth_context_parallel__ = Truediff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 6e377f0d18d..160cbad9f5c 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -146,6 +146,7 @@ from ..kernels import *from ..kernels.utils import _has_active_lora_bias, has_mxfp4_basefrom ..kernels.bnb_override import install_bnb_nf4_override as _install_bnb_nf4_overridefrom ..tokenizer_utils import *+from ..context_parallel import get_cp_managerfrom .vision import FastBaseModel, _is_text_seq2seq_configfrom .vision import (_resolve_offload_embedding,@@ -956,12 +957,15 @@ def LlamaAttention_fast_forward(if past_key_value is not None:kv_seq_len += past_key_value[0].shape[-2]- if position_embeddings and kv_seq_len <= position_embeddings[0].shape[0]:+ cp_manager = get_cp_manager()+ rope_seq_len = kv_seq_len * cp_manager.size if cp_manager is not None else kv_seq_len++ if position_embeddings and rope_seq_len <= position_embeddings[0].shape[0]:cos, sin = position_embeddingselse:rotary_emb = self.rotary_emb- rotary_emb.extend_rope_embedding(V, seq_len = kv_seq_len)- cos, sin = rotary_emb.get_cached(kv_seq_len, Q.device.index)+ rotary_emb.extend_rope_embedding(V, seq_len = rope_seq_len)+ cos, sin = rotary_emb.get_cached(rope_seq_len, Q.device.index)cos = cos.to(device = Q.device, dtype = Q.dtype)sin = sin.to(device = Q.device, dtype = Q.dtype)@@ -1731,6 +1735,8 @@ def CausalLM_fast_forward(fast_forward_inference):*args,**kwargs,) -> Union[Tuple, CausalLMOutputWithPast]:+ # Not a named parameter: HF's find_labels would make it a required eval label.+ shift_labels = kwargs.pop("shift_labels", None)past_key_values = _cache_as_legacy_tuple(past_key_values)if past_key_values is not None and len(past_key_values) == 0:past_key_values = None@@ -1797,6 +1803,8 @@ def CausalLM_fast_forward(fast_forward_inference):hidden_states = hidden_states.to(lm_head_device)if labels is not None:labels = labels.to(lm_head_device)+ if shift_labels is not None:+ shift_labels = shift_labels.to(lm_head_device)# Output last hidden states without logits if asked.if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":@@ -1854,7 +1862,7 @@ def CausalLM_fast_forward(fast_forward_inference):hidden_states = hidden_states,lm_head_weight = lm_head,lm_head_bias = None,- labels = labels,+ labels = labels if shift_labels is None else shift_labels,mask = None,n_items = n_items,scaling = getattr(self, "accelerator_scaler", None),@@ -1863,6 +1871,7 @@ def CausalLM_fast_forward(fast_forward_inference):logit_softcapping = logit_softcapping,logit_scale_multiply = logit_scale_multiply,logit_scale_divide = logit_scale_divide,+ shift_labels = shift_labels is None,)if not return_dict:# Fused CE never materializes logits; use EMPTY_LOGITS like the return_dict branch below (#2068).@@ -1885,15 +1894,16 @@ def CausalLM_fast_forward(fast_forward_inference):# Same answer the fused branch above reads, so the two branches cannot drift apart.logit_softcapping, logit_scaling = resolve_logit_scaling(self.config)- if labels is not None:+ if labels is not None or shift_labels is not None:shift_logits = logits- shift_labels = torch.empty_like(labels)- shift_labels[..., :-1] = labels[..., 1:]- shift_labels[..., -1] = -100- mask_packed_sequence_boundaries(- shift_labels,- kwargs.get("packed_seq_lengths"),- )+ if shift_labels is None:+ shift_labels = torch.empty_like(labels)+ shift_labels[..., :-1] = labels[..., 1:]+ shift_labels[..., -1] = -100+ mask_packed_sequence_boundaries(+ shift_labels,+ kwargs.get("packed_seq_lengths"),+ )n_items = kwargs.get("num_items_in_batch", None)if n_items is None:n_items = kwargs.get("n_items", None)diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 36d4499cbba..0bbd6eeffe6 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -3203,6 +3203,32 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):max_seq_length_call = ""max_seq_length_post = ""+ # Read by unsloth.context_parallel; a field here keeps SFTConfig itself unpatched (pickling).+ if trainer_file == "sft_trainer" and "context_parallel_size" not in call_args:+ max_seq_length_pre += """+ context_parallel_size : Optional[int] = field(+ default = 1,+ metadata = {'help': 'Ranks per context parallel group (SDPA ring attention). 1 disables it.'},+ )+ context_parallel_backend : Optional[str] = field(+ default = 'ring',+ metadata = {'help': 'Context parallel attention backend: ring.'},+ )+ context_parallel_rotate_method : Optional[str] = field(+ default = 'allgather',+ metadata = {'help': 'Ring KV rotation: allgather or alltoall.'},+ )"""+ max_seq_length_call += (+ "context_parallel_size = 1,"+ "context_parallel_backend = 'ring',"+ "context_parallel_rotate_method = 'allgather',"+ )+ max_seq_length_post += (+ "\n self.context_parallel_size = context_parallel_size"+ "\n self.context_parallel_backend = context_parallel_backend"+ "\n self.context_parallel_rotate_method = context_parallel_rotate_method"+ )+if "output_dir" in call_args:saving_check = ("if output_dir is None and save_strategy == 'steps' and save_steps == 500:\n"diff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 9d99105e874..72e914dd00b 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -940,7 +940,19 @@ class UnslothTrainingArguments(TrainingArguments):self.q_galore_config = q_galore_configself.embedding_learning_rate = embedding_learning_rateself.loraplus_lr_ratio = loraplus_lr_ratio+ # Not fields of the pristine SFTConfig captured at import.+ context_parallel = {+ "context_parallel_size": kwargs.pop("context_parallel_size", 1),+ "context_parallel_backend": kwargs.pop("context_parallel_backend", "ring"),+ "context_parallel_rotate_method": kwargs.pop(+ "context_parallel_rotate_method", "allgather"+ ),+ }+ fields = getattr(TrainingArguments, "__dataclass_fields__", {})+ kwargs.update({k: v for k, v in context_parallel.items() if k in fields})super().__init__(*args, **kwargs)+ for key, value in context_parallel.items():+ setattr(self, key, value)self.embedding_learning_rate = embedding_learning_rateself.loraplus_lr_ratio = loraplus_lr_ratioif self.eval_steps is not None and self.eval_strategy != "steps":@@ -1851,12 +1863,14 @@ def _patch_sft_trainer_auto_packing(trl_module):# several classes, so a "yes" from the resolved one is not proof about the# instance. A correct "no" has already turned both flags off, which is the# condition the post-init check skips on, so leaving it armed is free.+ is_context_parallel = (getattr(config_arg, "context_parallel_size", 1) or 1) > 1# Read before __init__: the compiled trainer ends it with model.for_inference(), which sets the flag.eval_logits_wanted = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" or _wants_eval_logits(original_init, args, kwargs)blocked = ((data_collator is not None)+ or is_context_parallelor (is_processor and not processor_text_only)or is_auto_processor_vlmor is_vision_dataset@@ -1872,6 +1886,8 @@ def _patch_sft_trainer_auto_packing(trl_module):setattr(config_arg, "packing", False)if hasattr(config_arg, "padding_free"):setattr(config_arg, "padding_free", False)+ if is_context_parallel and getattr(config_arg, "eval_packing", None):+ setattr(config_arg, "eval_packing", False)if blocked and requested_pack:reason = "custom data collator"@@ -1887,6 +1903,8 @@ def _patch_sft_trainer_auto_packing(trl_module):reason = "hybrid linear-attention model"elif is_unsupported_model:reason = f"unsupported model type(s): {', '.join(model_types)}"+ elif is_context_parallel:+ reason = "context parallelism"elif forward_rejects_packing:# Name the real blocker, else this falls through to the# UNSLOTH_RETURN_LOGITS branch and points at an unset flag. For a stringdiff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex e5195ed2669..72d693997fc 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -21,10 +21,11 @@ from typing import Any, Optional, Tupleimport torchfrom torch import Tensor-from torch.nn.functional import scaled_dot_product_attention+import torch.nn.functional as Ffrom ..device_type import DEVICE_TYPEfrom ..models._utils import *+from ..context_parallel import get_cp_managerfrom ..utils.packing import (build_sdpa_packed_attention_mask,build_xformers_block_causal_mask,@@ -91,6 +92,11 @@ _XFORMERS_FP32_UNSUPPORTED = ()+def scaled_dot_product_attention(*args, **kwargs):+ # Resolved per call: context_parallel monkey patches torch.nn.functional.+ return F.scaled_dot_product_attention(*args, **kwargs)++def _per_device_backends(capabilities):"""(flash runs, xformers runs, xformers rejects fp32, bf16 kernels) per CUDA device index, or Nonewhen every device has device 0's capability class, so homogeneous hosts never consult the table.@@ -158,7 +164,7 @@ def _backend_for_device(return backend-SDPA_HAS_GQA = "enable_gqa" in (scaled_dot_product_attention.__doc__ or "")+SDPA_HAS_GQA = "enable_gqa" in (F.scaled_dot_product_attention.__doc__ or "")# Packed SDPA runs per segment: a dense (T, T) mask + enable_gqa leaves only the math kernel._SDPA_PACKED_SEGMENTS = os.environ.get("UNSLOTH_SDPA_PACKED_SEGMENTS", "1").lower() not in (@@ -525,6 +531,12 @@ class AttentionContext:def select_attention_backend(use_varlen: bool = False) -> str:"""Return attention backend based on availability / priority order."""+ if get_cp_manager() is not None:+ if use_varlen:+ raise ValueError(+ "Unsloth: context parallelism does not support packing / padding-free."+ )+ return SDPAif HAS_FLASH_ATTENTION:if use_varlen:return FLASH_VARLEN@@ -720,9 +732,11 @@ def run_attention(# DoRA promotes q/k/v_proj outputs to fp32, which FlashAttention rejects (as does the xformers# flash-2 op on sm_100+), so downcast any fp32 Q/K/V to a supported dtype (#1013).+ cp_out_dtype = Q.dtype if backend == SDPA and get_cp_manager() is not None else Noneif (backend in (FLASH_DENSE, FLASH_VARLEN)or (backend == XFORMERS and xformers_fp32_unsupported)+ or (backend == SDPA and get_cp_manager() is not None)) and torch.float32 in (Q.dtype,K.dtype,@@ -956,6 +970,13 @@ def run_attention(kwargs.setdefault("is_causal", is_causal_local)use_sdpa_gqa = SDPA_HAS_GQA and config.n_groups != 1+ # Flash takes GQA natively, halving the ring's K/V peak; UNSLOTH_CP_GQA=0 expands.+ if (+ use_sdpa_gqa+ and get_cp_manager() is not None+ and os.environ.get("UNSLOTH_CP_GQA", "1") == "0"+ ):+ use_sdpa_gqa = Falseif (use_sdpa_gqaand (not requires_grad)@@ -970,7 +991,8 @@ def run_attention(if use_sdpa_gqa:kwargs.setdefault("enable_gqa", True)out = scaled_dot_product_attention(Q, K, V, **kwargs)- return out.transpose(1, 2)+ out = out.transpose(1, 2)+ return out if cp_out_dtype is None else out.to(cp_out_dtype)K_mod = KV_mod = V@@ -990,7 +1012,8 @@ def run_attention(V_mod.contiguous(),**kwargs,)- return out.transpose(1, 2).contiguous()+ out = out.transpose(1, 2).contiguous()+ return out if cp_out_dtype is None else out.to(cp_out_dtype)__all__ = [