Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: compose with FSDP2 on a (dp_shard, cp) mesh (#13910)

Daniel Han4ec819800d7f
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
index 9d07430819d..4ff272bd6ea 100644
--- a/tests/test_context_parallel.py
+++ b/tests/test_context_parallel.py
@@ -399,8 +399,8 @@ def test_fp32_qkv_is_downcast_under_cp(monkeypatch):
assert out.dtype == torch.float32
[email protected]("distributed_type", ["DEEPSPEED", "FSDP"])
-def test_deepspeed_and_fsdp_are_refused(monkeypatch, distributed_type):
[email protected]("distributed_type, fsdp_version", [("DEEPSPEED", None), ("FSDP", 1)])
+def test_deepspeed_and_fsdp1_are_refused(monkeypatch, distributed_type, fsdp_version):
import types
monkeypatch.setattr(cp.dist, "is_available", lambda: True)
@@ -409,14 +409,65 @@ def test_deepspeed_and_fsdp_are_refused(monkeypatch, distributed_type):
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
monkeypatch.setattr(cp, "_supports_context_parallel", lambda model, backend = "ring": True)
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
+ plugin = None if fsdp_version is None else types.SimpleNamespace(fsdp_version = fsdp_version)
accelerator = types.SimpleNamespace(
- distributed_type = types.SimpleNamespace(name = distributed_type)
+ distributed_type = types.SimpleNamespace(name = distributed_type),
+ state = types.SimpleNamespace(fsdp_plugin = plugin),
)
Trainer = _patched_trainer(monkeypatch, args = args, accelerator = accelerator, model = None)
- with pytest.raises(NotImplementedError, match = "DDP only"):
+ with pytest.raises(NotImplementedError, match = "DDP and FSDP2 only"):
Trainer()
+def _fsdp2_accelerator(parallelism_config = None):
+ import types
+ return types.SimpleNamespace(
+ distributed_type = types.SimpleNamespace(name = "FSDP"),
+ state = types.SimpleNamespace(
+ device_mesh = None,
+ fsdp_plugin = types.SimpleNamespace(fsdp_version = 2),
+ parallelism_config = parallelism_config,
+ ),
+ )
+
+
+def test_fsdp2_shards_over_dp_shard_cp_and_a_later_trainer_restores_the_config(monkeypatch):
+ import types
+
+ monkeypatch.setattr(cp, "_INSTALLED_MESH", [])
+ monkeypatch.setattr(cp, "_INSTALLED_PARALLELISM_CONFIG", [])
+ user_pc = types.SimpleNamespace(dp_replicate_size = 1, tp_size = 1, sp_size = 1, cp_size = 1)
+ accelerator = _fsdp2_accelerator(user_pc)
+ Trainer, _, built = _gated_trainer(monkeypatch, accelerator = accelerator)
+ Trainer()
+ assert built == [(2, "ring", "allgather", "fsdp2")]
+ pc = accelerator.state.parallelism_config
+ # What accelerate's fsdp2_prepare_model indexes the mesh with; cp stays Unsloth's own.
+ assert pc.fsdp_dim_names == ["dp_shard_cp"]
+ assert (pc.dp_shard_size, pc.cp_size, pc.non_data_parallel_size) == (4, 1, 1)
+ assert accelerator.state.device_mesh is not None
+ later = _patched_trainer(
+ monkeypatch,
+ args = types.SimpleNamespace(context_parallel_size = 1),
+ accelerator = accelerator,
+ )
+ later()
+ assert accelerator.state.parallelism_config is user_pc
+ assert accelerator.state.device_mesh is None
+
+
[email protected]("dim", ["dp_replicate", "tp", "sp"])
+def test_fsdp2_refuses_other_parallel_dims(monkeypatch, dim):
+ import types
+
+ sizes = {"dp_replicate_size": 1, "tp_size": 1, "sp_size": 1, f"{dim}_size": 2}
+ accelerator = _fsdp2_accelerator(types.SimpleNamespace(cp_size = 1, **sizes))
+ Trainer, _, built = _gated_trainer(monkeypatch, accelerator = accelerator)
+ with pytest.raises(NotImplementedError, match = "dp_shard, cp"):
+ Trainer()
+ assert built == []
+
+
def _cp_env(monkeypatch):
monkeypatch.setattr(cp.dist, "is_available", lambda: True)
monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
@@ -512,6 +563,21 @@ def test_a_later_non_cp_trainer_drops_the_installed_mesh(monkeypatch):
assert state.device_mesh is other
+def test_a_later_non_cp_trainer_restores_the_mesh_cp_replaced(monkeypatch):
+ import types
+
+ # An accepted FSDP2 ParallelismConfig came with its own mesh; CP swapped both.
+ prior, cp_mesh = object(), object()
+ monkeypatch.setattr(cp, "_INSTALLED_MESH", [cp_mesh, prior])
+ state = types.SimpleNamespace(device_mesh = cp_mesh)
+ args = types.SimpleNamespace(context_parallel_size = 1)
+ Trainer = _patched_trainer(
+ monkeypatch, args = args, accelerator = types.SimpleNamespace(state = state)
+ )
+ Trainer()
+ assert state.device_mesh is prior
+
+
def test_every_attention_layer_must_be_the_llama_forward():
from unsloth.models.llama import LlamaAttention_fast_forward
@@ -544,7 +610,7 @@ def test_find_unused_parameters_is_off_under_cp_unless_the_user_set_it(monkeypat
monkeypatch.setattr(
cp,
"ContextParallelManager",
- lambda size, *rest: types.SimpleNamespace(
+ lambda size, *rest, **kwargs: types.SimpleNamespace(
size = size,
backend = "ring",
device_mesh = object(),
@@ -738,6 +804,7 @@ def test_cp_dtensor_dispatcher_is_torn_down_when_the_step_raises(monkeypatch):
def _gated_trainer(
monkeypatch,
model = None,
+ accelerator = None,
**args_overrides,
):
"""A trainer that clears every CP gate, with the manager stubbed (no process group)."""
@@ -750,12 +817,14 @@ def _gated_trainer(
size,
backend = "ring",
rotate_method = "allgather",
+ fsdp2 = False,
):
- built.append((size, backend, rotate_method))
+ built.append((size, backend, rotate_method) + (("fsdp2",) if fsdp2 else ()))
return types.SimpleNamespace(
size = size,
backend = backend,
- device_mesh = object(),
+ fsdp2 = fsdp2,
+ device_mesh = {"dp_shard_cp": types.SimpleNamespace(size = lambda: 4)},
attach_attention_hooks = lambda model: None,
)
@@ -763,10 +832,11 @@ def _gated_trainer(
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
for key, value in args_overrides.items():
setattr(args, key, value)
- accelerator = types.SimpleNamespace(
- distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),
- state = types.SimpleNamespace(device_mesh = None),
- )
+ if accelerator is None:
+ accelerator = types.SimpleNamespace(
+ distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),
+ state = types.SimpleNamespace(device_mesh = None),
+ )
init = {"args": args, "accelerator": accelerator, "model": model, "train_dataset": [1]}
return _patched_trainer(monkeypatch, **init), args, built
@@ -1184,3 +1254,31 @@ def test_only_ring_needs_sm80(monkeypatch, backend):
else:
Trainer()
assert built == [(2, "ulysses", "allgather")]
+
+
[email protected]("fsdp2", [False, True])
+def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):
+ import types
+
+ import transformers
+ import trl
+
+ # The real transformers formula: DDP leaves parallelism_config unset, FSDP2 installs cp_size = 1.
+ class Trainer(transformers.Trainer):
+ def __init__(self, *a, **k):
+ pass
+
+ monkeypatch.setattr(trl, "SFTTrainer", Trainer)
+ cp.patch_sft_trainer()
+ trainer = object.__new__(Trainer)
+ pc = types.SimpleNamespace(cp_size = 1, sp_size = 1, tp_size = 1) if fsdp2 else None
+ trainer.accelerator = types.SimpleNamespace(parallelism_config = pc)
+ trainer.model = types.SimpleNamespace()
+ trainer.is_deepspeed_enabled = False
+ trainer._train_batch_size = 2
+ args = types.SimpleNamespace(world_size = 8, gradient_accumulation_steps = 3)
+ trainer._context_parallel_manager = None
+ assert trainer.get_total_train_batch_size(args) == 2 * 3 * 8
+ # D = 2 data parallel groups of N = 4 CP peers that each consume the same batch.
+ trainer._context_parallel_manager = _manager(size = 4)
+ assert trainer.get_total_train_batch_size(args) == 2 * 3 * 2
diff --git a/tests/test_context_parallel_fsdp2_gloo.py b/tests/test_context_parallel_fsdp2_gloo.py
new file mode 100644
index 00000000000..8770aa6b5e2
--- /dev/null
+++ b/tests/test_context_parallel_fsdp2_gloo.py
@@ -0,0 +1,234 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Context parallelism under FSDP2 on a (dp_shard = 2, cp = 2) mesh, four gloo ranks on CPU.
+
+Checks the mesh (cp peers adjacent, FSDP2 over dp_shard x cp flattened), that accelerate's data loader
+hands both cp peers the same batch, and that one patched training step (Ulysses split, token count
+division, accelerate's real fsdp2_prepare_model) reproduces the gradients of an unsharded model on the
+whole batch. The model is token-wise, so attention exchange is not exercised here (tests/test_ulysses.py).
+"""
+
+from __future__ import annotations
+
+import os
+import socket
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+dist = pytest.importorskip("torch.distributed")
+pytest.importorskip("torch.distributed.fsdp")
+pytest.importorskip("accelerate")
+
+if not dist.is_available() or not dist.is_gloo_available():
+ pytest.skip("gloo backend unavailable", allow_module_level = True)
+
+WORLD = 4
+CP = 2
+VOCAB = 50
+SEQ = 13
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+class Block(torch.nn.Module):
+ def __init__(self, hidden):
+ super().__init__()
+ self.up = torch.nn.Linear(hidden, 2 * hidden)
+ self.down = torch.nn.Linear(2 * hidden, hidden)
+
+ def forward(self, x):
+ return x + self.down(torch.nn.functional.silu(self.up(x)))
+
+
+class TokenModel(torch.nn.Module):
+ _no_split_modules = ["Block"]
+
+ def __init__(self, hidden = 16):
+ super().__init__()
+ self.embed = torch.nn.Embedding(VOCAB, hidden)
+ self.layers = torch.nn.ModuleList([Block(hidden) for _ in range(2)])
+ self.head = torch.nn.Linear(hidden, VOCAB)
+
+ def forward(self, input_ids):
+ x = self.embed(input_ids)
+ for layer in self.layers:
+ x = layer(x)
+ return self.head(x)
+
+
+def _batches():
+ g = torch.Generator().manual_seed(5)
+ ids = torch.randint(0, VOCAB, (WORLD, SEQ), generator = g)
+ labels = ids.clone()
+ labels[0, 2:9] = -100
+ # Row 1: every target sits in the first half, so its second cp shard counts zero tokens.
+ labels[1, SEQ // 2 :] = -100
+ return ids, labels
+
+
+def _accelerator_less_import():
+ # Spawned ranks skip conftest, whose CUDA spoof lets `import unsloth` run on CPU-only runners.
+ if torch.cuda.is_available():
+ return
+ import importlib.util
+ import pathlib
+
+ path = pathlib.Path(__file__).resolve().parent / "conftest.py"
+ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", path)
+ spec.loader.exec_module(importlib.util.module_from_spec(spec))
+
+
+def _worker(rank, port, out_dir):
+ _accelerator_less_import()
+ os.environ.update(
+ RANK = str(rank),
+ LOCAL_RANK = str(rank),
+ WORLD_SIZE = str(WORLD),
+ LOCAL_WORLD_SIZE = str(WORLD),
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ ACCELERATE_USE_CPU = "true",
+ )
+ import json
+
+ dist.init_process_group("gloo", rank = rank, world_size = WORLD)
+ try:
+ import trl
+ from torch.distributed.tensor import DTensor
+
+ import accelerate.accelerator as accelerator_module
+ from accelerate.data_loader import prepare_data_loader
+ from accelerate.utils import FullyShardedDataParallelPlugin
+ from unsloth import context_parallel as cp
+
+ cp.DEVICE_TYPE_TORCH = "cpu"
+ result = {}
+ manager = cp.ContextParallelManager(CP, "ulysses", fsdp2 = True)
+ mesh = manager.device_mesh
+ result["names"] = list(mesh.mesh_dim_names)
+ result["cp_group"] = manager.mesh.mesh.tolist()
+ result["dp_shard_group"] = mesh["dp_shard"].mesh.tolist()
+ result["fsdp_group"] = mesh["dp_shard_cp"].mesh.tolist()
+
+ state = types.SimpleNamespace(device_mesh = mesh, parallelism_config = None)
+ cp._install_fsdp2_parallelism_config(state, manager)
+ pc = state.parallelism_config
+
+ loader = prepare_data_loader(
+ torch.utils.data.DataLoader(list(range(8)), batch_size = 1),
+ num_processes = WORLD,
+ process_index = rank,
+ put_on_device = False,
+ torch_device_mesh = mesh,
+ )
+ result["samples"] = [int(x) for batch in loader for x in batch]
+
+ torch.manual_seed(0)
+ model = TokenModel()
+ reference = TokenModel()
+ reference.load_state_dict(model.state_dict())
+ plugin = FullyShardedDataParallelPlugin(
+ fsdp_version = 2,
+ auto_wrap_policy = "TRANSFORMER_BASED_WRAP",
+ transformer_cls_names_to_wrap = ["Block"],
+ reshard_after_forward = True,
+ )
+ accelerator = types.SimpleNamespace(
+ state = types.SimpleNamespace(fsdp_plugin = plugin),
+ torch_device_mesh = mesh,
+ parallelism_config = pc,
+ mixed_precision = "no",
+ device = torch.device("cpu"),
+ is_main_process = rank == 0,
+ )
+ model = accelerator_module.fsdp2_prepare_model(accelerator, model)
+ result["param_mesh"] = next(model.parameters()).device_mesh.mesh.tolist()
+
+ class Trainer:
+ def training_step(
+ self,
+ model,
+ inputs,
+ num_items_in_batch = None,
+ ):
+ logits = model(inputs["input_ids"]).float()
+ loss = torch.nn.functional.cross_entropy(
+ logits.reshape(-1, VOCAB),
+ inputs["shift_labels"].reshape(-1),
+ ignore_index = -100,
+ reduction = "sum",
+ )
+ # What transformers' compute_loss does with num_items_in_batch under DDP / FSDP.
+ loss = loss / num_items_in_batch * dist.get_world_size()
+ loss.backward()
+ return loss.detach()
+
+ trl.SFTTrainer = Trainer
+ cp.patch_sft_trainer()
+ trainer = object.__new__(Trainer)
+ trainer._context_parallel_manager = manager
+
+ ids, labels = _batches()
+ dp_index = rank // CP
+ # transformers sums the pre-shard counts over every rank (cp peers count the same row twice).
+ count = (labels[dp_index, 1:] != -100).sum().reshape(1)
+ dist.all_reduce(count)
+ inputs = {
+ "input_ids": ids[dp_index : dp_index + 1],
+ "labels": labels[dp_index : dp_index + 1],
+ }
+ trainer.training_step(model, inputs, count.clone())
+ result["local_tokens"] = int(inputs["input_ids"].shape[1])
+
+ rows = slice(0, WORLD // CP)
+ logits = reference(ids[rows]).float()
+ shift = torch.nn.functional.pad(labels[rows], (0, 1), value = -100)[:, 1:]
+ ref_loss = torch.nn.functional.cross_entropy(
+ logits.reshape(-1, VOCAB), shift.reshape(-1), ignore_index = -100
+ )
+ ref_loss.backward()
+ ref = dict(reference.named_parameters())
+ worst = 0.0
+ for name, p in model.named_parameters():
+ g = p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad
+ r = ref[name].grad
+ worst = max(worst, ((g - r).abs().max() / (r.abs().max() + 1e-12)).item())
+ result["grad_rel_err"] = worst
+ gathered = [None] * WORLD
+ dist.all_gather_object(gathered, result)
+ if rank == 0:
+ with open(os.path.join(out_dir, "result.json"), "w") as f:
+ json.dump(gathered, f)
+ finally:
+ dist.destroy_process_group()
+
+
+def test_cp_composes_with_fsdp2_on_a_dp_shard_cp_mesh(tmp_path):
+ import json
+
+ import torch.multiprocessing as mp
+
+ mp.spawn(_worker, args = (_free_port(), str(tmp_path)), nprocs = WORLD, join = True)
+ with open(tmp_path / "result.json") as f:
+ results = json.load(f)
+ for rank, r in enumerate(results):
+ peer = rank - rank % CP
+ assert r["names"] == ["dp_shard", "cp"]
+ assert r["cp_group"] == [peer, peer + 1]
+ assert r["dp_shard_group"] == [rank % CP, rank % CP + CP]
+ assert r["fsdp_group"] == list(range(WORLD))
+ assert r["param_mesh"] == list(range(WORLD)), "FSDP2 must shard over dp_shard x cp"
+ assert r["local_tokens"] == (SEQ + 1) // CP
+ assert r["grad_rel_err"] < 1e-5, r["grad_rel_err"]
+ # cp peers read the same rows, dp_shard ranks different ones.
+ samples = [r["samples"] for r in results]
+ assert samples[0] == samples[1] and samples[2] == samples[3]
+ assert samples[0] != samples[2]
+ assert sorted(samples[0] + samples[2]) == list(range(8))
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 6527588de96..8be15ea869b 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -148,6 +148,7 @@ _PAD_VALUES = {"labels": -100, "shift_labels": -100, "attention_mask": 0, "input
_INSTALLED_MESH = []
+_INSTALLED_PARALLELISM_CONFIG = []
def get_cp_manager() -> Optional["ContextParallelManager"]:
@@ -236,16 +237,22 @@ class ContextParallelManager:
size: int,
backend: str = "ring",
rotate_method: str = "allgather",
+ fsdp2: bool = False,
):
self.size = size
self.backend = backend
self.rotate_method = rotate_method
+ self.fsdp2 = fsdp2
world_size = dist.get_world_size()
+ # FSDP2 shards parameters over dp_shard x cp flattened (torchtitan, accelerate's dp_shard_cp):
+ # CP peers hold different tokens, so their gradients must be reduced like data parallel ones.
self.device_mesh = DeviceMesh(
DEVICE_TYPE_TORCH,
torch.arange(world_size).reshape(world_size // size, size),
- mesh_dim_names = ("dp_replicate", "cp"),
+ mesh_dim_names = ("dp_shard" if fsdp2 else "dp_replicate", "cp"),
)
+ if fsdp2:
+ self.device_mesh[("dp_shard", "cp")]._flatten("dp_shard_cp")
self.mesh = self.device_mesh["cp"]
@property
@@ -461,6 +468,53 @@ def _cp_gather_for_metrics(accelerator, size: int):
return gather
+def _is_fsdp2(accelerator, distributed_type: str) -> bool:
+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)
+ return distributed_type == "FSDP" and getattr(plugin, "fsdp_version", 1) == 2
+
+
+def _check_fsdp2_parallelism_config(accelerator) -> None:
+ pc = getattr(getattr(accelerator, "state", None), "parallelism_config", None)
+ if pc is None:
+ return
+ others = {name: getattr(pc, f"{name}_size", 1) or 1 for name in ("dp_replicate", "tp", "sp")}
+ others = {name: n for name, n in others.items() if n > 1}
+ if others:
+ raise NotImplementedError(
+ "Unsloth: context parallelism with FSDP2 supports a (dp_shard, cp) mesh only, "
+ f"not parallelism_config {others}."
+ )
+
+
+def _install_fsdp2_parallelism_config(state, manager) -> None:
+ # accelerate's fsdp2_prepare_model shards over mesh[parallelism_config.fsdp_dim_names], i.e.
+ # "dp_shard_cp" (accelerate >= 1.10). cp_size stays 1 here: cp_size > 1 would make accelerate attach
+ # its own ring hooks and transformers divide num_items_in_batch by cp, both done by Unsloth already.
+ from accelerate.parallelism_config import ParallelismConfig
+
+ current = getattr(state, "parallelism_config", None)
+ if _INSTALLED_PARALLELISM_CONFIG and current is _INSTALLED_PARALLELISM_CONFIG[1]:
+ previous = _INSTALLED_PARALLELISM_CONFIG[0]
+ else:
+ previous = current
+ pc = ParallelismConfig(
+ dp_replicate_size = 1,
+ dp_shard_size = manager.device_mesh["dp_shard_cp"].size(),
+ tp_size = 1,
+ cp_size = 1,
+ )
+ state.parallelism_config = pc
+ _INSTALLED_PARALLELISM_CONFIG[:] = [previous, pc]
+
+
+def _restore_parallelism_config(state) -> None:
+ if _INSTALLED_PARALLELISM_CONFIG and (
+ getattr(state, "parallelism_config", None) is _INSTALLED_PARALLELISM_CONFIG[1]
+ ):
+ state.parallelism_config = _INSTALLED_PARALLELISM_CONFIG[0]
+ _INSTALLED_PARALLELISM_CONFIG.clear()
+
+
def patch_sft_trainer() -> None:
import trl
@@ -486,6 +540,9 @@ def patch_sft_trainer() -> None:
original_evaluate = _original("evaluate")
original_predict = _original("predict")
original_evaluation_loop = _original("evaluation_loop")
+ has_total_train_batch_size = hasattr(trainer_cls, "get_total_train_batch_size")
+ if has_total_train_batch_size:
+ original_get_total_train_batch_size = _original("get_total_train_batch_size")
def _install_accelerator_state(self):
manager = getattr(self, "_context_parallel_manager", None)
@@ -493,12 +550,22 @@ def patch_sft_trainer() -> None:
state = getattr(accelerator, "state", None)
if manager is None:
if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:
- state.device_mesh = None
+ # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.
+ state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None
+ _INSTALLED_MESH.clear()
+ _restore_parallelism_config(state)
return
if accelerator is None:
return
+ current = getattr(accelerator.state, "device_mesh", None)
+ if _INSTALLED_MESH and current is _INSTALLED_MESH[0]:
+ current = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None
accelerator.state.device_mesh = manager.device_mesh
- _INSTALLED_MESH[:] = [manager.device_mesh]
+ _INSTALLED_MESH[:] = [manager.device_mesh, current]
+ if getattr(manager, "fsdp2", False):
+ _install_fsdp2_parallelism_config(accelerator.state, manager)
+ else:
+ _restore_parallelism_config(accelerator.state)
# PEFT makes the Trainer ask DDP for find_unused_parameters, which breaks reentrant checkpointing.
if getattr(self.args, "ddp_find_unused_parameters", None) in (None, False):
self.args.ddp_find_unused_parameters = False
@@ -605,10 +672,14 @@ def patch_sft_trainer() -> None:
)
accelerator = getattr(self, "accelerator", None)
distributed_type = getattr(getattr(accelerator, "distributed_type", None), "name", "NO")
- if distributed_type != "NO" and not distributed_type.startswith("MULTI_"):
+ fsdp2 = _is_fsdp2(accelerator, distributed_type)
+ if distributed_type != "NO" and not distributed_type.startswith("MULTI_") and not fsdp2:
raise NotImplementedError(
- f"Unsloth: context parallelism supports DDP only, not {distributed_type}."
+ f"Unsloth: context parallelism supports DDP and FSDP2 only, not {distributed_type}"
+ + (" version 1." if distributed_type == "FSDP" else ".")
)
+ if fsdp2:
+ _check_fsdp2_parallelism_config(accelerator)
if getattr(accelerator, "dispatch_batches", None):
raise NotImplementedError(
"Unsloth: context parallelism does not support dispatch_batches."
@@ -616,7 +687,7 @@ def patch_sft_trainer() -> None:
_refuse_iterable_datasets(
getattr(self, "train_dataset", None), getattr(self, "eval_dataset", None)
)
- manager = ContextParallelManager(size, backend, rotate_method)
+ manager = ContextParallelManager(size, backend, rotate_method, fsdp2 = fsdp2)
self._context_parallel_manager = manager
manager.attach_attention_hooks(self.model)
_install_accelerator_state(self)
@@ -626,7 +697,12 @@ def patch_sft_trainer() -> None:
_scope_flash_attention_impl(self.model, self)
print(
f"Unsloth: Context parallelism enabled with size = {size} ({backend}"
- + (f", rotate_method = {rotate_method})." if backend == "ring" else ").")
+ + (f", rotate_method = {rotate_method})" if backend == "ring" else ")")
+ + (
+ f", FSDP2 sharding over dp_shard = {world_size // size} x cp = {size}."
+ if fsdp2
+ else "."
+ )
)
@functools.wraps(original_prediction_step)
@@ -715,6 +791,20 @@ def patch_sft_trainer() -> None:
with manager.apply(inputs) if manager else contextlib.nullcontext():
return original_training_step(self, model, inputs, *args, **kwargs)
+ if has_total_train_batch_size:
+
+ @functools.wraps(original_get_total_train_batch_size)
+ def patched_get_total_train_batch_size(self, *args, **kwargs):
+ total = original_get_total_train_batch_size(self, *args, **kwargs)
+ manager = getattr(self, "_context_parallel_manager", None)
+ # CP peers share one batch, but the parallelism_config Unsloth exposes (DDP: none,
+ # FSDP2: cp_size = 1) makes transformers count all D x N ranks as data parallel.
+ if manager is not None and manager.size > 1:
+ total = total // manager.size
+ return total
+
+ trainer_cls.get_total_train_batch_size = patched_get_total_train_batch_size
+
trainer_cls.__init__ = patched_init
trainer_cls.prediction_step = patched_prediction_step
trainer_cls.train = patched_train