Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Refuse accelerate config cp_size under Unsloth SFT, not only TrainingArguments (#15212)
Daniel Hanaf99ed7b84ce
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.pyindex 3e728900bd0..50b955e3199 100644--- a/tests/test_context_parallel.py+++ b/tests/test_context_parallel.py@@ -673,6 +673,28 @@ def test_transformers_cp_size_is_refused(monkeypatch):Trainer()[email protected]("cp_size", [1, 2])+def test_accelerate_config_cp_size_is_refused(monkeypatch, cp_size):+ # `accelerate launch` with parallelism_config_cp_size: the Accelerator builds its+ # ParallelismConfig from the environment and TrainingArguments.parallelism_config stays None.+ import types++ accelerator = types.SimpleNamespace(+ parallelism_config = types.SimpleNamespace(cp_size = cp_size),+ state = types.SimpleNamespace(device_mesh = None),+ )+ Trainer = _patched_trainer(+ monkeypatch,+ args = types.SimpleNamespace(context_parallel_size = 1, parallelism_config = None),+ accelerator = accelerator,+ )+ if cp_size == 1:+ Trainer()+ return+ with pytest.raises(NotImplementedError, match = "accelerate config"):+ Trainer()++def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):import typesdiff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex a4859535690..555fcd2349f 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -781,12 +781,24 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:original_init(self, *args, **kwargs)self._context_parallel_manager = None# Unsloth's attention ignores transformers' own CP: it would stay local to each shard.- parallelism_config = getattr(getattr(self, "args", None), "parallelism_config", None)- if (getattr(parallelism_config, "cp_size", 1) or 1) > 1:- raise NotImplementedError(- "Unsloth: use SFTConfig(context_parallel_size = N) instead of "- "parallelism_config with cp_size > 1."- )+ # `accelerate launch` with parallelism_config_cp_size builds it from the environment+ # (ACCELERATE_USE_PARALLELISM_CONFIG), leaving args.parallelism_config None.+ for source, parallelism_config in (+ (+ "parallelism_config",+ getattr(getattr(self, "args", None), "parallelism_config", None),+ ),+ (+ "the accelerate config (parallelism_config_cp_size)",+ getattr(getattr(self, "accelerator", None), "parallelism_config", None),+ ),+ ):+ cp_size = getattr(parallelism_config, "cp_size", 1) or 1+ if cp_size > 1:+ raise NotImplementedError(+ f"Unsloth: use SFTConfig(context_parallel_size = {cp_size}) instead of "+ f"{source} with cp_size > 1."+ )size, backend, rotate_method = context_parallel_options(getattr(self, "args", None))if size <= 1:_install_accelerator_state(self)