Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallel: compose with FSDP2 on a (dp_shard, cp) mesh (#13910)
Daniel Han4ec819800d7f
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.pyindex 9d07430819d..4ff272bd6ea 100644--- a/tests/test_context_parallel.py+++ b/tests/test_context_parallel.py@@ -399,8 +399,8 @@ def test_fp32_qkv_is_downcast_under_cp(monkeypatch):assert out.dtype == torch.float32[email protected]("distributed_type", ["DEEPSPEED", "FSDP"])-def test_deepspeed_and_fsdp_are_refused(monkeypatch, distributed_type):[email protected]("distributed_type, fsdp_version", [("DEEPSPEED", None), ("FSDP", 1)])+def test_deepspeed_and_fsdp1_are_refused(monkeypatch, distributed_type, fsdp_version):import typesmonkeypatch.setattr(cp.dist, "is_available", lambda: True)@@ -409,14 +409,65 @@ def test_deepspeed_and_fsdp_are_refused(monkeypatch, distributed_type):monkeypatch.setattr(torch.cuda, "is_available", lambda: False)monkeypatch.setattr(cp, "_supports_context_parallel", lambda model, backend = "ring": True)args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)+ plugin = None if fsdp_version is None else types.SimpleNamespace(fsdp_version = fsdp_version)accelerator = types.SimpleNamespace(- distributed_type = types.SimpleNamespace(name = distributed_type)+ distributed_type = types.SimpleNamespace(name = distributed_type),+ state = types.SimpleNamespace(fsdp_plugin = plugin),)Trainer = _patched_trainer(monkeypatch, args = args, accelerator = accelerator, model = None)- with pytest.raises(NotImplementedError, match = "DDP only"):+ with pytest.raises(NotImplementedError, match = "DDP and FSDP2 only"):Trainer()+def _fsdp2_accelerator(parallelism_config = None):+ import types+ return types.SimpleNamespace(+ distributed_type = types.SimpleNamespace(name = "FSDP"),+ state = types.SimpleNamespace(+ device_mesh = None,+ fsdp_plugin = types.SimpleNamespace(fsdp_version = 2),+ parallelism_config = parallelism_config,+ ),+ )+++def test_fsdp2_shards_over_dp_shard_cp_and_a_later_trainer_restores_the_config(monkeypatch):+ import types++ monkeypatch.setattr(cp, "_INSTALLED_MESH", [])+ monkeypatch.setattr(cp, "_INSTALLED_PARALLELISM_CONFIG", [])+ user_pc = types.SimpleNamespace(dp_replicate_size = 1, tp_size = 1, sp_size = 1, cp_size = 1)+ accelerator = _fsdp2_accelerator(user_pc)+ Trainer, _, built = _gated_trainer(monkeypatch, accelerator = accelerator)+ Trainer()+ assert built == [(2, "ring", "allgather", "fsdp2")]+ pc = accelerator.state.parallelism_config+ # What accelerate's fsdp2_prepare_model indexes the mesh with; cp stays Unsloth's own.+ assert pc.fsdp_dim_names == ["dp_shard_cp"]+ assert (pc.dp_shard_size, pc.cp_size, pc.non_data_parallel_size) == (4, 1, 1)+ assert accelerator.state.device_mesh is not None+ later = _patched_trainer(+ monkeypatch,+ args = types.SimpleNamespace(context_parallel_size = 1),+ accelerator = accelerator,+ )+ later()+ assert accelerator.state.parallelism_config is user_pc+ assert accelerator.state.device_mesh is None++[email protected]("dim", ["dp_replicate", "tp", "sp"])+def test_fsdp2_refuses_other_parallel_dims(monkeypatch, dim):+ import types++ sizes = {"dp_replicate_size": 1, "tp_size": 1, "sp_size": 1, f"{dim}_size": 2}+ accelerator = _fsdp2_accelerator(types.SimpleNamespace(cp_size = 1, **sizes))+ Trainer, _, built = _gated_trainer(monkeypatch, accelerator = accelerator)+ with pytest.raises(NotImplementedError, match = "dp_shard, cp"):+ Trainer()+ assert built == []++def _cp_env(monkeypatch):monkeypatch.setattr(cp.dist, "is_available", lambda: True)monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)@@ -512,6 +563,21 @@ def test_a_later_non_cp_trainer_drops_the_installed_mesh(monkeypatch):assert state.device_mesh is other+def test_a_later_non_cp_trainer_restores_the_mesh_cp_replaced(monkeypatch):+ import types++ # An accepted FSDP2 ParallelismConfig came with its own mesh; CP swapped both.+ prior, cp_mesh = object(), object()+ monkeypatch.setattr(cp, "_INSTALLED_MESH", [cp_mesh, prior])+ state = types.SimpleNamespace(device_mesh = cp_mesh)+ args = types.SimpleNamespace(context_parallel_size = 1)+ Trainer = _patched_trainer(+ monkeypatch, args = args, accelerator = types.SimpleNamespace(state = state)+ )+ Trainer()+ assert state.device_mesh is prior++def test_every_attention_layer_must_be_the_llama_forward():from unsloth.models.llama import LlamaAttention_fast_forward@@ -544,7 +610,7 @@ def test_find_unused_parameters_is_off_under_cp_unless_the_user_set_it(monkeypatmonkeypatch.setattr(cp,"ContextParallelManager",- lambda size, *rest: types.SimpleNamespace(+ lambda size, *rest, **kwargs: types.SimpleNamespace(size = size,backend = "ring",device_mesh = object(),@@ -738,6 +804,7 @@ def test_cp_dtensor_dispatcher_is_torn_down_when_the_step_raises(monkeypatch):def _gated_trainer(monkeypatch,model = None,+ accelerator = None,**args_overrides,):"""A trainer that clears every CP gate, with the manager stubbed (no process group)."""@@ -750,12 +817,14 @@ def _gated_trainer(size,backend = "ring",rotate_method = "allgather",+ fsdp2 = False,):- built.append((size, backend, rotate_method))+ built.append((size, backend, rotate_method) + (("fsdp2",) if fsdp2 else ()))return types.SimpleNamespace(size = size,backend = backend,- device_mesh = object(),+ fsdp2 = fsdp2,+ device_mesh = {"dp_shard_cp": types.SimpleNamespace(size = lambda: 4)},attach_attention_hooks = lambda model: None,)@@ -763,10 +832,11 @@ def _gated_trainer(args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)for key, value in args_overrides.items():setattr(args, key, value)- accelerator = types.SimpleNamespace(- distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),- state = types.SimpleNamespace(device_mesh = None),- )+ if accelerator is None:+ accelerator = types.SimpleNamespace(+ distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),+ state = types.SimpleNamespace(device_mesh = None),+ )init = {"args": args, "accelerator": accelerator, "model": model, "train_dataset": [1]}return _patched_trainer(monkeypatch, **init), args, built@@ -1184,3 +1254,31 @@ def test_only_ring_needs_sm80(monkeypatch, backend):else:Trainer()assert built == [(2, "ulysses", "allgather")]++[email protected]("fsdp2", [False, True])+def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):+ import types++ import transformers+ import trl++ # The real transformers formula: DDP leaves parallelism_config unset, FSDP2 installs cp_size = 1.+ class Trainer(transformers.Trainer):+ def __init__(self, *a, **k):+ pass++ monkeypatch.setattr(trl, "SFTTrainer", Trainer)+ cp.patch_sft_trainer()+ trainer = object.__new__(Trainer)+ pc = types.SimpleNamespace(cp_size = 1, sp_size = 1, tp_size = 1) if fsdp2 else None+ trainer.accelerator = types.SimpleNamespace(parallelism_config = pc)+ trainer.model = types.SimpleNamespace()+ trainer.is_deepspeed_enabled = False+ trainer._train_batch_size = 2+ args = types.SimpleNamespace(world_size = 8, gradient_accumulation_steps = 3)+ trainer._context_parallel_manager = None+ assert trainer.get_total_train_batch_size(args) == 2 * 3 * 8+ # D = 2 data parallel groups of N = 4 CP peers that each consume the same batch.+ trainer._context_parallel_manager = _manager(size = 4)+ assert trainer.get_total_train_batch_size(args) == 2 * 3 * 2diff --git a/tests/test_context_parallel_fsdp2_gloo.py b/tests/test_context_parallel_fsdp2_gloo.pynew file mode 100644index 00000000000..8770aa6b5e2--- /dev/null+++ b/tests/test_context_parallel_fsdp2_gloo.py@@ -0,0 +1,234 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Context parallelism under FSDP2 on a (dp_shard = 2, cp = 2) mesh, four gloo ranks on CPU.++Checks the mesh (cp peers adjacent, FSDP2 over dp_shard x cp flattened), that accelerate's data loader+hands both cp peers the same batch, and that one patched training step (Ulysses split, token count+division, accelerate's real fsdp2_prepare_model) reproduces the gradients of an unsharded model on the+whole batch. The model is token-wise, so attention exchange is not exercised here (tests/test_ulysses.py).+"""++from __future__ import annotations++import os+import socket+import types++import pytest++torch = pytest.importorskip("torch")+dist = pytest.importorskip("torch.distributed")+pytest.importorskip("torch.distributed.fsdp")+pytest.importorskip("accelerate")++if not dist.is_available() or not dist.is_gloo_available():+ pytest.skip("gloo backend unavailable", allow_module_level = True)++WORLD = 4+CP = 2+VOCAB = 50+SEQ = 13+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++class Block(torch.nn.Module):+ def __init__(self, hidden):+ super().__init__()+ self.up = torch.nn.Linear(hidden, 2 * hidden)+ self.down = torch.nn.Linear(2 * hidden, hidden)++ def forward(self, x):+ return x + self.down(torch.nn.functional.silu(self.up(x)))+++class TokenModel(torch.nn.Module):+ _no_split_modules = ["Block"]++ def __init__(self, hidden = 16):+ super().__init__()+ self.embed = torch.nn.Embedding(VOCAB, hidden)+ self.layers = torch.nn.ModuleList([Block(hidden) for _ in range(2)])+ self.head = torch.nn.Linear(hidden, VOCAB)++ def forward(self, input_ids):+ x = self.embed(input_ids)+ for layer in self.layers:+ x = layer(x)+ return self.head(x)+++def _batches():+ g = torch.Generator().manual_seed(5)+ ids = torch.randint(0, VOCAB, (WORLD, SEQ), generator = g)+ labels = ids.clone()+ labels[0, 2:9] = -100+ # Row 1: every target sits in the first half, so its second cp shard counts zero tokens.+ labels[1, SEQ // 2 :] = -100+ return ids, labels+++def _accelerator_less_import():+ # Spawned ranks skip conftest, whose CUDA spoof lets `import unsloth` run on CPU-only runners.+ if torch.cuda.is_available():+ return+ import importlib.util+ import pathlib++ path = pathlib.Path(__file__).resolve().parent / "conftest.py"+ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", path)+ spec.loader.exec_module(importlib.util.module_from_spec(spec))+++def _worker(rank, port, out_dir):+ _accelerator_less_import()+ os.environ.update(+ RANK = str(rank),+ LOCAL_RANK = str(rank),+ WORLD_SIZE = str(WORLD),+ LOCAL_WORLD_SIZE = str(WORLD),+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ ACCELERATE_USE_CPU = "true",+ )+ import json++ dist.init_process_group("gloo", rank = rank, world_size = WORLD)+ try:+ import trl+ from torch.distributed.tensor import DTensor++ import accelerate.accelerator as accelerator_module+ from accelerate.data_loader import prepare_data_loader+ from accelerate.utils import FullyShardedDataParallelPlugin+ from unsloth import context_parallel as cp++ cp.DEVICE_TYPE_TORCH = "cpu"+ result = {}+ manager = cp.ContextParallelManager(CP, "ulysses", fsdp2 = True)+ mesh = manager.device_mesh+ result["names"] = list(mesh.mesh_dim_names)+ result["cp_group"] = manager.mesh.mesh.tolist()+ result["dp_shard_group"] = mesh["dp_shard"].mesh.tolist()+ result["fsdp_group"] = mesh["dp_shard_cp"].mesh.tolist()++ state = types.SimpleNamespace(device_mesh = mesh, parallelism_config = None)+ cp._install_fsdp2_parallelism_config(state, manager)+ pc = state.parallelism_config++ loader = prepare_data_loader(+ torch.utils.data.DataLoader(list(range(8)), batch_size = 1),+ num_processes = WORLD,+ process_index = rank,+ put_on_device = False,+ torch_device_mesh = mesh,+ )+ result["samples"] = [int(x) for batch in loader for x in batch]++ torch.manual_seed(0)+ model = TokenModel()+ reference = TokenModel()+ reference.load_state_dict(model.state_dict())+ plugin = FullyShardedDataParallelPlugin(+ fsdp_version = 2,+ auto_wrap_policy = "TRANSFORMER_BASED_WRAP",+ transformer_cls_names_to_wrap = ["Block"],+ reshard_after_forward = True,+ )+ accelerator = types.SimpleNamespace(+ state = types.SimpleNamespace(fsdp_plugin = plugin),+ torch_device_mesh = mesh,+ parallelism_config = pc,+ mixed_precision = "no",+ device = torch.device("cpu"),+ is_main_process = rank == 0,+ )+ model = accelerator_module.fsdp2_prepare_model(accelerator, model)+ result["param_mesh"] = next(model.parameters()).device_mesh.mesh.tolist()++ class Trainer:+ def training_step(+ self,+ model,+ inputs,+ num_items_in_batch = None,+ ):+ logits = model(inputs["input_ids"]).float()+ loss = torch.nn.functional.cross_entropy(+ logits.reshape(-1, VOCAB),+ inputs["shift_labels"].reshape(-1),+ ignore_index = -100,+ reduction = "sum",+ )+ # What transformers' compute_loss does with num_items_in_batch under DDP / FSDP.+ loss = loss / num_items_in_batch * dist.get_world_size()+ loss.backward()+ return loss.detach()++ trl.SFTTrainer = Trainer+ cp.patch_sft_trainer()+ trainer = object.__new__(Trainer)+ trainer._context_parallel_manager = manager++ ids, labels = _batches()+ dp_index = rank // CP+ # transformers sums the pre-shard counts over every rank (cp peers count the same row twice).+ count = (labels[dp_index, 1:] != -100).sum().reshape(1)+ dist.all_reduce(count)+ inputs = {+ "input_ids": ids[dp_index : dp_index + 1],+ "labels": labels[dp_index : dp_index + 1],+ }+ trainer.training_step(model, inputs, count.clone())+ result["local_tokens"] = int(inputs["input_ids"].shape[1])++ rows = slice(0, WORLD // CP)+ logits = reference(ids[rows]).float()+ shift = torch.nn.functional.pad(labels[rows], (0, 1), value = -100)[:, 1:]+ ref_loss = torch.nn.functional.cross_entropy(+ logits.reshape(-1, VOCAB), shift.reshape(-1), ignore_index = -100+ )+ ref_loss.backward()+ ref = dict(reference.named_parameters())+ worst = 0.0+ for name, p in model.named_parameters():+ g = p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad+ r = ref[name].grad+ worst = max(worst, ((g - r).abs().max() / (r.abs().max() + 1e-12)).item())+ result["grad_rel_err"] = worst+ gathered = [None] * WORLD+ dist.all_gather_object(gathered, result)+ if rank == 0:+ with open(os.path.join(out_dir, "result.json"), "w") as f:+ json.dump(gathered, f)+ finally:+ dist.destroy_process_group()+++def test_cp_composes_with_fsdp2_on_a_dp_shard_cp_mesh(tmp_path):+ import json++ import torch.multiprocessing as mp++ mp.spawn(_worker, args = (_free_port(), str(tmp_path)), nprocs = WORLD, join = True)+ with open(tmp_path / "result.json") as f:+ results = json.load(f)+ for rank, r in enumerate(results):+ peer = rank - rank % CP+ assert r["names"] == ["dp_shard", "cp"]+ assert r["cp_group"] == [peer, peer + 1]+ assert r["dp_shard_group"] == [rank % CP, rank % CP + CP]+ assert r["fsdp_group"] == list(range(WORLD))+ assert r["param_mesh"] == list(range(WORLD)), "FSDP2 must shard over dp_shard x cp"+ assert r["local_tokens"] == (SEQ + 1) // CP+ assert r["grad_rel_err"] < 1e-5, r["grad_rel_err"]+ # cp peers read the same rows, dp_shard ranks different ones.+ samples = [r["samples"] for r in results]+ assert samples[0] == samples[1] and samples[2] == samples[3]+ assert samples[0] != samples[2]+ assert sorted(samples[0] + samples[2]) == list(range(8))diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 6527588de96..8be15ea869b 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -148,6 +148,7 @@ _PAD_VALUES = {"labels": -100, "shift_labels": -100, "attention_mask": 0, "input_INSTALLED_MESH = []+_INSTALLED_PARALLELISM_CONFIG = []def get_cp_manager() -> Optional["ContextParallelManager"]:@@ -236,16 +237,22 @@ class ContextParallelManager:size: int,backend: str = "ring",rotate_method: str = "allgather",+ fsdp2: bool = False,):self.size = sizeself.backend = backendself.rotate_method = rotate_method+ self.fsdp2 = fsdp2world_size = dist.get_world_size()+ # FSDP2 shards parameters over dp_shard x cp flattened (torchtitan, accelerate's dp_shard_cp):+ # CP peers hold different tokens, so their gradients must be reduced like data parallel ones.self.device_mesh = DeviceMesh(DEVICE_TYPE_TORCH,torch.arange(world_size).reshape(world_size // size, size),- mesh_dim_names = ("dp_replicate", "cp"),+ mesh_dim_names = ("dp_shard" if fsdp2 else "dp_replicate", "cp"),)+ if fsdp2:+ self.device_mesh[("dp_shard", "cp")]._flatten("dp_shard_cp")self.mesh = self.device_mesh["cp"]@property@@ -461,6 +468,53 @@ def _cp_gather_for_metrics(accelerator, size: int):return gather+def _is_fsdp2(accelerator, distributed_type: str) -> bool:+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)+ return distributed_type == "FSDP" and getattr(plugin, "fsdp_version", 1) == 2+++def _check_fsdp2_parallelism_config(accelerator) -> None:+ pc = getattr(getattr(accelerator, "state", None), "parallelism_config", None)+ if pc is None:+ return+ others = {name: getattr(pc, f"{name}_size", 1) or 1 for name in ("dp_replicate", "tp", "sp")}+ others = {name: n for name, n in others.items() if n > 1}+ if others:+ raise NotImplementedError(+ "Unsloth: context parallelism with FSDP2 supports a (dp_shard, cp) mesh only, "+ f"not parallelism_config {others}."+ )+++def _install_fsdp2_parallelism_config(state, manager) -> None:+ # accelerate's fsdp2_prepare_model shards over mesh[parallelism_config.fsdp_dim_names], i.e.+ # "dp_shard_cp" (accelerate >= 1.10). cp_size stays 1 here: cp_size > 1 would make accelerate attach+ # its own ring hooks and transformers divide num_items_in_batch by cp, both done by Unsloth already.+ from accelerate.parallelism_config import ParallelismConfig++ current = getattr(state, "parallelism_config", None)+ if _INSTALLED_PARALLELISM_CONFIG and current is _INSTALLED_PARALLELISM_CONFIG[1]:+ previous = _INSTALLED_PARALLELISM_CONFIG[0]+ else:+ previous = current+ pc = ParallelismConfig(+ dp_replicate_size = 1,+ dp_shard_size = manager.device_mesh["dp_shard_cp"].size(),+ tp_size = 1,+ cp_size = 1,+ )+ state.parallelism_config = pc+ _INSTALLED_PARALLELISM_CONFIG[:] = [previous, pc]+++def _restore_parallelism_config(state) -> None:+ if _INSTALLED_PARALLELISM_CONFIG and (+ getattr(state, "parallelism_config", None) is _INSTALLED_PARALLELISM_CONFIG[1]+ ):+ state.parallelism_config = _INSTALLED_PARALLELISM_CONFIG[0]+ _INSTALLED_PARALLELISM_CONFIG.clear()++def patch_sft_trainer() -> None:import trl@@ -486,6 +540,9 @@ def patch_sft_trainer() -> None:original_evaluate = _original("evaluate")original_predict = _original("predict")original_evaluation_loop = _original("evaluation_loop")+ has_total_train_batch_size = hasattr(trainer_cls, "get_total_train_batch_size")+ if has_total_train_batch_size:+ original_get_total_train_batch_size = _original("get_total_train_batch_size")def _install_accelerator_state(self):manager = getattr(self, "_context_parallel_manager", None)@@ -493,12 +550,22 @@ def patch_sft_trainer() -> None:state = getattr(accelerator, "state", None)if manager is None:if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:- state.device_mesh = None+ # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.+ state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None+ _INSTALLED_MESH.clear()+ _restore_parallelism_config(state)returnif accelerator is None:return+ current = getattr(accelerator.state, "device_mesh", None)+ if _INSTALLED_MESH and current is _INSTALLED_MESH[0]:+ current = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else Noneaccelerator.state.device_mesh = manager.device_mesh- _INSTALLED_MESH[:] = [manager.device_mesh]+ _INSTALLED_MESH[:] = [manager.device_mesh, current]+ if getattr(manager, "fsdp2", False):+ _install_fsdp2_parallelism_config(accelerator.state, manager)+ else:+ _restore_parallelism_config(accelerator.state)# PEFT makes the Trainer ask DDP for find_unused_parameters, which breaks reentrant checkpointing.if getattr(self.args, "ddp_find_unused_parameters", None) in (None, False):self.args.ddp_find_unused_parameters = False@@ -605,10 +672,14 @@ def patch_sft_trainer() -> None:)accelerator = getattr(self, "accelerator", None)distributed_type = getattr(getattr(accelerator, "distributed_type", None), "name", "NO")- if distributed_type != "NO" and not distributed_type.startswith("MULTI_"):+ fsdp2 = _is_fsdp2(accelerator, distributed_type)+ if distributed_type != "NO" and not distributed_type.startswith("MULTI_") and not fsdp2:raise NotImplementedError(- f"Unsloth: context parallelism supports DDP only, not {distributed_type}."+ f"Unsloth: context parallelism supports DDP and FSDP2 only, not {distributed_type}"+ + (" version 1." if distributed_type == "FSDP" else "."))+ if fsdp2:+ _check_fsdp2_parallelism_config(accelerator)if getattr(accelerator, "dispatch_batches", None):raise NotImplementedError("Unsloth: context parallelism does not support dispatch_batches."@@ -616,7 +687,7 @@ def patch_sft_trainer() -> None:_refuse_iterable_datasets(getattr(self, "train_dataset", None), getattr(self, "eval_dataset", None))- manager = ContextParallelManager(size, backend, rotate_method)+ manager = ContextParallelManager(size, backend, rotate_method, fsdp2 = fsdp2)self._context_parallel_manager = managermanager.attach_attention_hooks(self.model)_install_accelerator_state(self)@@ -626,7 +697,12 @@ def patch_sft_trainer() -> None:_scope_flash_attention_impl(self.model, self)print(f"Unsloth: Context parallelism enabled with size = {size} ({backend}"- + (f", rotate_method = {rotate_method})." if backend == "ring" else ").")+ + (f", rotate_method = {rotate_method})" if backend == "ring" else ")")+ + (+ f", FSDP2 sharding over dp_shard = {world_size // size} x cp = {size}."+ if fsdp2+ else "."+ ))@functools.wraps(original_prediction_step)@@ -715,6 +791,20 @@ def patch_sft_trainer() -> None:with manager.apply(inputs) if manager else contextlib.nullcontext():return original_training_step(self, model, inputs, *args, **kwargs)+ if has_total_train_batch_size:++ @functools.wraps(original_get_total_train_batch_size)+ def patched_get_total_train_batch_size(self, *args, **kwargs):+ total = original_get_total_train_batch_size(self, *args, **kwargs)+ manager = getattr(self, "_context_parallel_manager", None)+ # CP peers share one batch, but the parallelism_config Unsloth exposes (DDP: none,+ # FSDP2: cp_size = 1) makes transformers count all D x N ranks as data parallel.+ if manager is not None and manager.size > 1:+ total = total // manager.size+ return total++ trainer_cls.get_total_train_batch_size = patched_get_total_train_batch_size+trainer_cls.__init__ = patched_inittrainer_cls.prediction_step = patched_prediction_steptrainer_cls.train = patched_train