Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallel: wrap trainer methods only once a trainer enables CP (torch 2.10 step time) (#14152)
Daniel Han40e20017af2a
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.pyindex 4ff272bd6ea..3e728900bd0 100644--- a/tests/test_context_parallel.py+++ b/tests/test_context_parallel.py@@ -146,7 +146,7 @@ def test_training_step_divides_the_pre_shard_token_count_by_cp_size(monkeypatch)import trlmonkeypatch.setattr(trl, "SFTTrainer", Trainer)- cp.patch_sft_trainer()+ cp.patch_sft_trainer(install_cp_methods = True)trainer = Trainer()manager = _manager(size = 4)manager.mesh = None@@ -226,13 +226,20 @@ def test_every_rank_builds_the_same_global_mesh(monkeypatch):assert built == [([[0, 1], [2, 3]], ("dp_replicate", "cp"))] * 4-def _patched_trainer(monkeypatch, **init_attrs):+def _patched_trainer(+ monkeypatch,+ install_cp_methods = True,+ on_init = None,+ **init_attrs,+):import trlclass Trainer:def __init__(self, *a, **k):for key, value in init_attrs.items():setattr(self, key, value)+ if on_init is not None:+ on_init(self)def training_step(self,@@ -275,7 +282,7 @@ def _patched_trainer(monkeypatch, **init_attrs):return self.gather_function(torch.arange(1.0, 3.0))monkeypatch.setattr(trl, "SFTTrainer", Trainer)- cp.patch_sft_trainer()+ cp.patch_sft_trainer(install_cp_methods = install_cp_methods)return Trainer@@ -805,6 +812,8 @@ def _gated_trainer(monkeypatch,model = None,accelerator = None,+ install_cp_methods = True,+ on_init = None,**args_overrides,):"""A trainer that clears every CP gate, with the manager stubbed (no process group)."""@@ -838,7 +847,37 @@ def _gated_trainer(state = types.SimpleNamespace(device_mesh = None),)init = {"args": args, "accelerator": accelerator, "model": model, "train_dataset": [1]}- return _patched_trainer(monkeypatch, **init), args, built+ return _patched_trainer(monkeypatch, install_cp_methods, on_init, **init), args, built+++_CP_METHODS = (+ "train",+ "training_step",+ "prediction_step",+ "evaluate",+ "predict",+ "evaluation_loop",+)+++def test_cp_methods_are_wrapped_only_once_a_trainer_enables_cp(monkeypatch):+ # With CP off the wrappers slowed torch 2.10 LoRA steps by ~35%, so only __init__ is patched.+ import types++ Trainer = _patched_trainer(monkeypatch, install_cp_methods = False)+ assert hasattr(Trainer.__init__, "__wrapped__")+ assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)+ trainer = Trainer(args = types.SimpleNamespace(context_parallel_size = 1))+ assert trainer._context_parallel_manager is None+ assert trainer.train() == "trained"+ assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)++ Trainer, _, built = _gated_trainer(monkeypatch, install_cp_methods = False)+ plain = {name: Trainer.__dict__[name] for name in _CP_METHODS}+ Trainer()+ assert built == [(2, "ring", "allgather")]+ for name in _CP_METHODS:+ assert Trainer.__dict__[name].__wrapped__ is plain[name]@pytest.mark.parametrize(@@ -1219,7 +1258,7 @@ def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):passmonkeypatch.setattr(trl, "SFTTrainer", Trainer)- cp.patch_sft_trainer()+ cp.patch_sft_trainer(install_cp_methods = True)monkeypatch.setattr(Base, "train", lambda self: "patched later")assert Trainer().train() == "patched later"@@ -1269,7 +1308,7 @@ def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):passmonkeypatch.setattr(trl, "SFTTrainer", Trainer)- cp.patch_sft_trainer()+ cp.patch_sft_trainer(install_cp_methods = True)trainer = object.__new__(Trainer)pc = types.SimpleNamespace(cp_size = 1, sp_size = 1, tp_size = 1) if fsdp2 else Nonetrainer.accelerator = types.SimpleNamespace(parallelism_config = pc)@@ -1282,3 +1321,89 @@ def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):# D = 2 data parallel groups of N = 4 CP peers that each consume the same batch.trainer._context_parallel_manager = _manager(size = 4)assert trainer.get_total_train_batch_size(args) == 2 * 3 * 2+++def test_instance_wrappers_made_in_init_capture_the_cp_methods(monkeypatch):+ # Auto packing (self.train) and the token budget (self.training_step) wrap the class methods on+ # the instance inside __init__; with lazy installation they must already see the CP wrappers.+ import functools++ captured = {}++ def on_init(self):+ for name in ("train", "training_step"):+ method = getattr(type(self), name)+ captured[name] = method+ setattr(self, name, functools.partial(method, self))++ import types++ for forwarded in (False, True):+ captured.clear()+ Trainer, args, built = _gated_trainer(+ monkeypatch, install_cp_methods = False, on_init = on_init+ )+ plain = {name: Trainer.__dict__[name] for name in ("train", "training_step")}+ if forwarded:+ # SFTTrainer(model, args = base_args, context_parallel_size = 2): moved into args in __init__+ Trainer(+ None, args = types.SimpleNamespace(context_parallel_size = 1), context_parallel_size = 2+ )+ else:+ Trainer(+ None, args = args+ ) # SFTTrainer(model, args = SFTConfig(context_parallel_size = 2))+ assert built == [(2, "ring", "allgather")]+ for name, method in captured.items():+ assert method is not plain[name] and method.__wrapped__ is plain[name]++[email protected]("unhashable", [False, True])+def test_a_trainer_built_before_cp_still_restores_the_accelerator_state(monkeypatch, unhashable):+ # A non-CP trainer whose __init__ wrapped self.train (auto packing) before any CP trainer existed+ # must still drop the mesh a later CP trainer installs, as patched_train does.+ import functools+ import types++ plain_args = [types.SimpleNamespace(context_parallel_size = 1)]++ def on_init(self):+ if plain_args: # the first trainer has CP off+ self.args = plain_args.pop()+ self.train = functools.partial(type(self).train, self)++ if unhashable:+ # A subclass defining __eq__ without __hash__ is unhashable.+ on_init_plain = on_init++ def on_init(self):+ type(self).__eq__ = lambda a, b: a is b+ type(self).__hash__ = None+ on_init_plain(self)++ Trainer, args, built = _gated_trainer(monkeypatch, install_cp_methods = False, on_init = on_init)+ restored = []+ monkeypatch.setattr(cp, "_restore_parallelism_config", lambda state: restored.append(state))+ early = Trainer(None, args = types.SimpleNamespace(context_parallel_size = 1))+ assert early._context_parallel_manager is None+ early.accelerator = types.SimpleNamespace(state = types.SimpleNamespace(device_mesh = None))+ Trainer(None, args = args) # enables CP: installs the class wrappers+ assert built == [(2, "ring", "allgather")]+ restored.clear()+ assert early.train() == "trained"+ assert restored == [early.accelerator.state]++[email protected]("forwarded", [1, None])+def test_a_forwarded_cp_size_of_one_installs_no_wrappers(monkeypatch, forwarded):+ # SFTTrainer(..., args = config_with_cp_2, context_parallel_size = 1): the keyword wins in+ # _backwards_compatible_trainer, so CP is off and the class must stay unwrapped.+ import types++ def on_init(self):+ self.args = types.SimpleNamespace(context_parallel_size = 1)++ Trainer, args, built = _gated_trainer(monkeypatch, install_cp_methods = False, on_init = on_init)+ Trainer(None, args = args, context_parallel_size = forwarded)+ assert built == []+ assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)diff --git a/tests/test_context_parallel_fsdp2_gloo.py b/tests/test_context_parallel_fsdp2_gloo.pyindex 8770aa6b5e2..3956f1e99a6 100644--- a/tests/test_context_parallel_fsdp2_gloo.py+++ b/tests/test_context_parallel_fsdp2_gloo.py@@ -171,7 +171,7 @@ def _worker(rank, port, out_dir):return loss.detach()trl.SFTTrainer = Trainer- cp.patch_sft_trainer()+ cp.patch_sft_trainer(install_cp_methods = True)trainer = object.__new__(Trainer)trainer._context_parallel_manager = managerdiff --git a/tests/test_context_parallel_fsdp2_save_gloo.py b/tests/test_context_parallel_fsdp2_save_gloo.pyindex a0b6b3bd5dc..6f7a1b3bd55 100644--- a/tests/test_context_parallel_fsdp2_save_gloo.py+++ b/tests/test_context_parallel_fsdp2_save_gloo.py@@ -103,7 +103,7 @@ def _worker(rank, port, out_dir, user_config):passtrl.SFTTrainer = SFTTrainer- cp.patch_sft_trainer()+ cp.patch_sft_trainer(install_cp_methods = True)class Acc:parallelism_config = property(lambda self: self.state.parallelism_config)diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 9b34ebbd90b..b44fc969a6f 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport contextlibimport functools+import weakreffrom typing import Iterator, Optionalimport torch@@ -544,7 +545,9 @@ def _restore_parallelism_config(state) -> None:_INSTALLED_PARALLELISM_CONFIG.clear()-def patch_sft_trainer() -> None:+def patch_sft_trainer(install_cp_methods: bool = False) -> None:+ # Only __init__ is wrapped up front; the other wrappers go on the class once a trainer enables+ # CP. With CP off they still slowed torch 2.10 LoRA steps by ~35% (gemma-4-E2B, L4 / A100).import trltrainer_cls = trl.SFTTrainer@@ -606,7 +609,19 @@ def patch_sft_trainer() -> None:@functools.wraps(original_init)def patched_init(self, *args, **kwargs):config = kwargs["args"] if "args" in kwargs else (args[1] if len(args) > 1 else None)- context_parallel_options(config)+ # SFTTrainer(..., args = base_args, context_parallel_size = N) is moved into args only+ # inside the original __init__ (_backwards_compatible_trainer), so read it here too.+ # The forwarded value wins there, so it decides when given.+ requested = context_parallel_options(config)[0]+ if "context_parallel_size" in kwargs:+ # An explicit None opts out like 1; a non-integer fails validation later anyway.+ forwarded = kwargs["context_parallel_size"]+ valid = isinstance(forwarded, int) and not isinstance(forwarded, bool)+ requested = forwarded if valid else 1+ if requested > 1:+ # Before the original __init__: Unsloth wraps instance methods there (auto packing's+ # train reset, the token budget's training_step), and they must capture the CP ones.+ install_methods()original_init(self, *args, **kwargs)self._context_parallel_manager = None# Unsloth's attention ignores transformers' own CP: it would stay local to each shard.@@ -619,6 +634,10 @@ def patch_sft_trainer() -> None:size, backend, rotate_method = context_parallel_options(getattr(self, "args", None))if size <= 1:_install_accelerator_state(self)+ if not trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):+ # By identity: a subclass may define __eq__ without __hash__.+ pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]+ pre_cp_trainers.append(weakref.ref(self))return# train() would swap in a fresh model without the attention hooks or the support check.if getattr(self, "model_init", None) is not None:@@ -719,6 +738,7 @@ def patch_sft_trainer() -> None:)manager = ContextParallelManager(size, backend, rotate_method, fsdp2 = fsdp2)self._context_parallel_manager = manager+ install_methods()manager.attach_attention_hooks(self.model)_install_accelerator_state(self)# The ring calls the flash ops FA3 / FA4 override, so scope them to train() here too.@@ -847,14 +867,43 @@ def patch_sft_trainer() -> None:total = total // manager.sizereturn total- trainer_cls.get_total_train_batch_size = patched_get_total_train_batch_size+ # Non-CP trainers built before any trainer enabled CP: their __init__ may have wrapped train /+ # evaluate / predict on the instance around the plain methods (auto packing's train reset).+ pre_cp_trainers = [] # weakref.ref per trainer++ def _restoring(trainer, bound):+ @functools.wraps(bound)+ def call(*args, **kwargs):+ # What patched_train / evaluate / predict do for a non-CP trainer: drop the mesh and+ # parallelism config a CP trainer installed on the shared accelerator state.+ _install_accelerator_state(trainer)+ return bound(*args, **kwargs)++ return call++ def install_methods():+ if trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):+ return+ for trainer in [ref() for ref in pre_cp_trainers]:+ if trainer is None:+ continue+ for name in ("train", "evaluate", "predict"):+ bound = trainer.__dict__.get(name)+ if bound is not None:+ trainer.__dict__[name] = _restoring(trainer, bound)+ pre_cp_trainers.clear()+ trainer_cls.prediction_step = patched_prediction_step+ trainer_cls.train = patched_train+ trainer_cls.evaluate = patched_evaluate+ trainer_cls.predict = patched_predict+ trainer_cls.evaluation_loop = patched_evaluation_loop+ trainer_cls.save_model = patched_save_model+ trainer_cls.training_step = patched_training_step+ if has_total_train_batch_size:+ trainer_cls.get_total_train_batch_size = patched_get_total_train_batch_size+ trainer_cls.__unsloth_context_parallel_methods__ = Truetrainer_cls.__init__ = patched_init- trainer_cls.prediction_step = patched_prediction_step- trainer_cls.train = patched_train- trainer_cls.evaluate = patched_evaluate- trainer_cls.predict = patched_predict- trainer_cls.evaluation_loop = patched_evaluation_loop- trainer_cls.save_model = patched_save_model- trainer_cls.training_step = patched_training_steptrainer_cls.__unsloth_context_parallel__ = True+ if install_cp_methods:+ install_methods()