Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: wrap trainer methods only once a trainer enables CP (torch 2.10 step time) (#14152)

Daniel Han40e20017af2a
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
index 4ff272bd6ea..3e728900bd0 100644
--- a/tests/test_context_parallel.py
+++ b/tests/test_context_parallel.py
@@ -146,7 +146,7 @@ def test_training_step_divides_the_pre_shard_token_count_by_cp_size(monkeypatch)
import trl
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
- cp.patch_sft_trainer()
+ cp.patch_sft_trainer(install_cp_methods = True)
trainer = Trainer()
manager = _manager(size = 4)
manager.mesh = None
@@ -226,13 +226,20 @@ def test_every_rank_builds_the_same_global_mesh(monkeypatch):
assert built == [([[0, 1], [2, 3]], ("dp_replicate", "cp"))] * 4
-def _patched_trainer(monkeypatch, **init_attrs):
+def _patched_trainer(
+ monkeypatch,
+ install_cp_methods = True,
+ on_init = None,
+ **init_attrs,
+):
import trl
class Trainer:
def __init__(self, *a, **k):
for key, value in init_attrs.items():
setattr(self, key, value)
+ if on_init is not None:
+ on_init(self)
def training_step(
self,
@@ -275,7 +282,7 @@ def _patched_trainer(monkeypatch, **init_attrs):
return self.gather_function(torch.arange(1.0, 3.0))
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
- cp.patch_sft_trainer()
+ cp.patch_sft_trainer(install_cp_methods = install_cp_methods)
return Trainer
@@ -805,6 +812,8 @@ def _gated_trainer(
monkeypatch,
model = None,
accelerator = None,
+ install_cp_methods = True,
+ on_init = None,
**args_overrides,
):
"""A trainer that clears every CP gate, with the manager stubbed (no process group)."""
@@ -838,7 +847,37 @@ def _gated_trainer(
state = types.SimpleNamespace(device_mesh = None),
)
init = {"args": args, "accelerator": accelerator, "model": model, "train_dataset": [1]}
- return _patched_trainer(monkeypatch, **init), args, built
+ return _patched_trainer(monkeypatch, install_cp_methods, on_init, **init), args, built
+
+
+_CP_METHODS = (
+ "train",
+ "training_step",
+ "prediction_step",
+ "evaluate",
+ "predict",
+ "evaluation_loop",
+)
+
+
+def test_cp_methods_are_wrapped_only_once_a_trainer_enables_cp(monkeypatch):
+ # With CP off the wrappers slowed torch 2.10 LoRA steps by ~35%, so only __init__ is patched.
+ import types
+
+ Trainer = _patched_trainer(monkeypatch, install_cp_methods = False)
+ assert hasattr(Trainer.__init__, "__wrapped__")
+ assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)
+ trainer = Trainer(args = types.SimpleNamespace(context_parallel_size = 1))
+ assert trainer._context_parallel_manager is None
+ assert trainer.train() == "trained"
+ assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)
+
+ Trainer, _, built = _gated_trainer(monkeypatch, install_cp_methods = False)
+ plain = {name: Trainer.__dict__[name] for name in _CP_METHODS}
+ Trainer()
+ assert built == [(2, "ring", "allgather")]
+ for name in _CP_METHODS:
+ assert Trainer.__dict__[name].__wrapped__ is plain[name]
@pytest.mark.parametrize(
@@ -1219,7 +1258,7 @@ def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):
pass
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
- cp.patch_sft_trainer()
+ cp.patch_sft_trainer(install_cp_methods = True)
monkeypatch.setattr(Base, "train", lambda self: "patched later")
assert Trainer().train() == "patched later"
@@ -1269,7 +1308,7 @@ def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):
pass
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
- cp.patch_sft_trainer()
+ cp.patch_sft_trainer(install_cp_methods = True)
trainer = object.__new__(Trainer)
pc = types.SimpleNamespace(cp_size = 1, sp_size = 1, tp_size = 1) if fsdp2 else None
trainer.accelerator = types.SimpleNamespace(parallelism_config = pc)
@@ -1282,3 +1321,89 @@ def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):
# D = 2 data parallel groups of N = 4 CP peers that each consume the same batch.
trainer._context_parallel_manager = _manager(size = 4)
assert trainer.get_total_train_batch_size(args) == 2 * 3 * 2
+
+
+def test_instance_wrappers_made_in_init_capture_the_cp_methods(monkeypatch):
+ # Auto packing (self.train) and the token budget (self.training_step) wrap the class methods on
+ # the instance inside __init__; with lazy installation they must already see the CP wrappers.
+ import functools
+
+ captured = {}
+
+ def on_init(self):
+ for name in ("train", "training_step"):
+ method = getattr(type(self), name)
+ captured[name] = method
+ setattr(self, name, functools.partial(method, self))
+
+ import types
+
+ for forwarded in (False, True):
+ captured.clear()
+ Trainer, args, built = _gated_trainer(
+ monkeypatch, install_cp_methods = False, on_init = on_init
+ )
+ plain = {name: Trainer.__dict__[name] for name in ("train", "training_step")}
+ if forwarded:
+ # SFTTrainer(model, args = base_args, context_parallel_size = 2): moved into args in __init__
+ Trainer(
+ None, args = types.SimpleNamespace(context_parallel_size = 1), context_parallel_size = 2
+ )
+ else:
+ Trainer(
+ None, args = args
+ ) # SFTTrainer(model, args = SFTConfig(context_parallel_size = 2))
+ assert built == [(2, "ring", "allgather")]
+ for name, method in captured.items():
+ assert method is not plain[name] and method.__wrapped__ is plain[name]
+
+
[email protected]("unhashable", [False, True])
+def test_a_trainer_built_before_cp_still_restores_the_accelerator_state(monkeypatch, unhashable):
+ # A non-CP trainer whose __init__ wrapped self.train (auto packing) before any CP trainer existed
+ # must still drop the mesh a later CP trainer installs, as patched_train does.
+ import functools
+ import types
+
+ plain_args = [types.SimpleNamespace(context_parallel_size = 1)]
+
+ def on_init(self):
+ if plain_args: # the first trainer has CP off
+ self.args = plain_args.pop()
+ self.train = functools.partial(type(self).train, self)
+
+ if unhashable:
+ # A subclass defining __eq__ without __hash__ is unhashable.
+ on_init_plain = on_init
+
+ def on_init(self):
+ type(self).__eq__ = lambda a, b: a is b
+ type(self).__hash__ = None
+ on_init_plain(self)
+
+ Trainer, args, built = _gated_trainer(monkeypatch, install_cp_methods = False, on_init = on_init)
+ restored = []
+ monkeypatch.setattr(cp, "_restore_parallelism_config", lambda state: restored.append(state))
+ early = Trainer(None, args = types.SimpleNamespace(context_parallel_size = 1))
+ assert early._context_parallel_manager is None
+ early.accelerator = types.SimpleNamespace(state = types.SimpleNamespace(device_mesh = None))
+ Trainer(None, args = args) # enables CP: installs the class wrappers
+ assert built == [(2, "ring", "allgather")]
+ restored.clear()
+ assert early.train() == "trained"
+ assert restored == [early.accelerator.state]
+
+
[email protected]("forwarded", [1, None])
+def test_a_forwarded_cp_size_of_one_installs_no_wrappers(monkeypatch, forwarded):
+ # SFTTrainer(..., args = config_with_cp_2, context_parallel_size = 1): the keyword wins in
+ # _backwards_compatible_trainer, so CP is off and the class must stay unwrapped.
+ import types
+
+ def on_init(self):
+ self.args = types.SimpleNamespace(context_parallel_size = 1)
+
+ Trainer, args, built = _gated_trainer(monkeypatch, install_cp_methods = False, on_init = on_init)
+ Trainer(None, args = args, context_parallel_size = forwarded)
+ assert built == []
+ assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)
diff --git a/tests/test_context_parallel_fsdp2_gloo.py b/tests/test_context_parallel_fsdp2_gloo.py
index 8770aa6b5e2..3956f1e99a6 100644
--- a/tests/test_context_parallel_fsdp2_gloo.py
+++ b/tests/test_context_parallel_fsdp2_gloo.py
@@ -171,7 +171,7 @@ def _worker(rank, port, out_dir):
return loss.detach()
trl.SFTTrainer = Trainer
- cp.patch_sft_trainer()
+ cp.patch_sft_trainer(install_cp_methods = True)
trainer = object.__new__(Trainer)
trainer._context_parallel_manager = manager
diff --git a/tests/test_context_parallel_fsdp2_save_gloo.py b/tests/test_context_parallel_fsdp2_save_gloo.py
index a0b6b3bd5dc..6f7a1b3bd55 100644
--- a/tests/test_context_parallel_fsdp2_save_gloo.py
+++ b/tests/test_context_parallel_fsdp2_save_gloo.py
@@ -103,7 +103,7 @@ def _worker(rank, port, out_dir, user_config):
pass
trl.SFTTrainer = SFTTrainer
- cp.patch_sft_trainer()
+ cp.patch_sft_trainer(install_cp_methods = True)
class Acc:
parallelism_config = property(lambda self: self.state.parallelism_config)
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 9b34ebbd90b..b44fc969a6f 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import contextlib
import functools
+import weakref
from typing import Iterator, Optional
import torch
@@ -544,7 +545,9 @@ def _restore_parallelism_config(state) -> None:
_INSTALLED_PARALLELISM_CONFIG.clear()
-def patch_sft_trainer() -> None:
+def patch_sft_trainer(install_cp_methods: bool = False) -> None:
+ # Only __init__ is wrapped up front; the other wrappers go on the class once a trainer enables
+ # CP. With CP off they still slowed torch 2.10 LoRA steps by ~35% (gemma-4-E2B, L4 / A100).
import trl
trainer_cls = trl.SFTTrainer
@@ -606,7 +609,19 @@ def patch_sft_trainer() -> None:
@functools.wraps(original_init)
def patched_init(self, *args, **kwargs):
config = kwargs["args"] if "args" in kwargs else (args[1] if len(args) > 1 else None)
- context_parallel_options(config)
+ # SFTTrainer(..., args = base_args, context_parallel_size = N) is moved into args only
+ # inside the original __init__ (_backwards_compatible_trainer), so read it here too.
+ # The forwarded value wins there, so it decides when given.
+ requested = context_parallel_options(config)[0]
+ if "context_parallel_size" in kwargs:
+ # An explicit None opts out like 1; a non-integer fails validation later anyway.
+ forwarded = kwargs["context_parallel_size"]
+ valid = isinstance(forwarded, int) and not isinstance(forwarded, bool)
+ requested = forwarded if valid else 1
+ if requested > 1:
+ # Before the original __init__: Unsloth wraps instance methods there (auto packing's
+ # train reset, the token budget's training_step), and they must capture the CP ones.
+ install_methods()
original_init(self, *args, **kwargs)
self._context_parallel_manager = None
# Unsloth's attention ignores transformers' own CP: it would stay local to each shard.
@@ -619,6 +634,10 @@ def patch_sft_trainer() -> None:
size, backend, rotate_method = context_parallel_options(getattr(self, "args", None))
if size <= 1:
_install_accelerator_state(self)
+ if not trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):
+ # By identity: a subclass may define __eq__ without __hash__.
+ pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]
+ pre_cp_trainers.append(weakref.ref(self))
return
# train() would swap in a fresh model without the attention hooks or the support check.
if getattr(self, "model_init", None) is not None:
@@ -719,6 +738,7 @@ def patch_sft_trainer() -> None:
)
manager = ContextParallelManager(size, backend, rotate_method, fsdp2 = fsdp2)
self._context_parallel_manager = manager
+ install_methods()
manager.attach_attention_hooks(self.model)
_install_accelerator_state(self)
# The ring calls the flash ops FA3 / FA4 override, so scope them to train() here too.
@@ -847,14 +867,43 @@ def patch_sft_trainer() -> None:
total = total // manager.size
return total
- trainer_cls.get_total_train_batch_size = patched_get_total_train_batch_size
+ # Non-CP trainers built before any trainer enabled CP: their __init__ may have wrapped train /
+ # evaluate / predict on the instance around the plain methods (auto packing's train reset).
+ pre_cp_trainers = [] # weakref.ref per trainer
+
+ def _restoring(trainer, bound):
+ @functools.wraps(bound)
+ def call(*args, **kwargs):
+ # What patched_train / evaluate / predict do for a non-CP trainer: drop the mesh and
+ # parallelism config a CP trainer installed on the shared accelerator state.
+ _install_accelerator_state(trainer)
+ return bound(*args, **kwargs)
+
+ return call
+
+ def install_methods():
+ if trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):
+ return
+ for trainer in [ref() for ref in pre_cp_trainers]:
+ if trainer is None:
+ continue
+ for name in ("train", "evaluate", "predict"):
+ bound = trainer.__dict__.get(name)
+ if bound is not None:
+ trainer.__dict__[name] = _restoring(trainer, bound)
+ pre_cp_trainers.clear()
+ trainer_cls.prediction_step = patched_prediction_step
+ trainer_cls.train = patched_train
+ trainer_cls.evaluate = patched_evaluate
+ trainer_cls.predict = patched_predict
+ trainer_cls.evaluation_loop = patched_evaluation_loop
+ trainer_cls.save_model = patched_save_model
+ trainer_cls.training_step = patched_training_step
+ if has_total_train_batch_size:
+ trainer_cls.get_total_train_batch_size = patched_get_total_train_batch_size
+ trainer_cls.__unsloth_context_parallel_methods__ = True
trainer_cls.__init__ = patched_init
- trainer_cls.prediction_step = patched_prediction_step
- trainer_cls.train = patched_train
- trainer_cls.evaluate = patched_evaluate
- trainer_cls.predict = patched_predict
- trainer_cls.evaluation_loop = patched_evaluation_loop
- trainer_cls.save_model = patched_save_model
- trainer_cls.training_step = patched_training_step
trainer_cls.__unsloth_context_parallel__ = True
+ if install_cp_methods:
+ install_methods()