Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Refuse accelerate config cp_size under Unsloth SFT, not only TrainingArguments (#15212)

Daniel Hanaf99ed7b84ce
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
index 3e728900bd0..50b955e3199 100644
--- a/tests/test_context_parallel.py
+++ b/tests/test_context_parallel.py
@@ -673,6 +673,28 @@ def test_transformers_cp_size_is_refused(monkeypatch):
Trainer()
[email protected]("cp_size", [1, 2])
+def test_accelerate_config_cp_size_is_refused(monkeypatch, cp_size):
+ # `accelerate launch` with parallelism_config_cp_size: the Accelerator builds its
+ # ParallelismConfig from the environment and TrainingArguments.parallelism_config stays None.
+ import types
+
+ accelerator = types.SimpleNamespace(
+ parallelism_config = types.SimpleNamespace(cp_size = cp_size),
+ state = types.SimpleNamespace(device_mesh = None),
+ )
+ Trainer = _patched_trainer(
+ monkeypatch,
+ args = types.SimpleNamespace(context_parallel_size = 1, parallelism_config = None),
+ accelerator = accelerator,
+ )
+ if cp_size == 1:
+ Trainer()
+ return
+ with pytest.raises(NotImplementedError, match = "accelerate config"):
+ Trainer()
+
+
def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):
import types
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index a4859535690..555fcd2349f 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -781,12 +781,24 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
original_init(self, *args, **kwargs)
self._context_parallel_manager = None
# Unsloth's attention ignores transformers' own CP: it would stay local to each shard.
- parallelism_config = getattr(getattr(self, "args", None), "parallelism_config", None)
- if (getattr(parallelism_config, "cp_size", 1) or 1) > 1:
- raise NotImplementedError(
- "Unsloth: use SFTConfig(context_parallel_size = N) instead of "
- "parallelism_config with cp_size > 1."
- )
+ # `accelerate launch` with parallelism_config_cp_size builds it from the environment
+ # (ACCELERATE_USE_PARALLELISM_CONFIG), leaving args.parallelism_config None.
+ for source, parallelism_config in (
+ (
+ "parallelism_config",
+ getattr(getattr(self, "args", None), "parallelism_config", None),
+ ),
+ (
+ "the accelerate config (parallelism_config_cp_size)",
+ getattr(getattr(self, "accelerator", None), "parallelism_config", None),
+ ),
+ ):
+ cp_size = getattr(parallelism_config, "cp_size", 1) or 1
+ if cp_size > 1:
+ raise NotImplementedError(
+ f"Unsloth: use SFTConfig(context_parallel_size = {cp_size}) instead of "
+ f"{source} with cp_size > 1."
+ )
size, backend, rotate_method = context_parallel_options(getattr(self, "args", None))
if size <= 1:
_install_accelerator_state(self)