Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallelism: generic Ulysses on any Hugging Face attention model, per-instance trainer wrappers (#16558)

Daniel Han7dd74a5342f9
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index 8607413cebe..adc6206dd7e 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -1840,3 +1840,33 @@ def test_graphs_off_compile_gets_the_same_region_limit(monkeypatch):
for kwargs in seen:
assert kwargs["isolate_recompiles"] is True
assert kwargs["recompile_limit"] == M._region_recompile_limit(2) == 48
+
+
+def test_parametrized_class_is_reused_per_module(monkeypatch):
+ """PEFT's ParamWrapper registers a parametrization per forward: each module keeps one
+ Parametrized<Class> (a stable type id for Dynamo), and two modules never share one."""
+ from torch.nn.utils import parametrize
+
+ monkeypatch.setattr(parametrize, "_inject_new_class", parametrize._inject_new_class)
+ bc = _load()
+ bc._stable_parametrized_classes()
+ bc._stable_parametrized_classes() # idempotent
+
+ class Double(torch.nn.Module):
+ def forward(self, w):
+ return 2 * w
+
+ a, b = torch.nn.Linear(3, 3), torch.nn.Linear(3, 3)
+ seen = []
+ for _ in range(3):
+ parametrize.register_parametrization(a, "weight", Double())
+ seen.append(type(a))
+ expected = 2 * a.parametrizations["weight"].original
+ assert torch.equal(a.weight, expected)
+ parametrize.remove_parametrizations(a, "weight", leave_parametrized = False)
+ assert type(a) is torch.nn.Linear and "weight" in a._parameters
+ assert seen[0] is seen[1] is seen[2] and seen[0] is not torch.nn.Linear
+ parametrize.register_parametrization(b, "weight", Double())
+ assert type(b) is not seen[0] and torch.equal(
+ b.weight, 2 * b.parametrizations["weight"].original
+ )
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
index 0b0be959228..c87ed36a87e 100644
--- a/tests/test_context_parallel.py
+++ b/tests/test_context_parallel.py
@@ -299,7 +299,7 @@ def test_eval_loss_is_the_token_mean_even_with_an_empty_shard(monkeypatch, world
monkeypatch.setattr(cp, "context_parallel", first_half)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: world_size)
Trainer = _patched_trainer(monkeypatch)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
manager = _manager(size = 2)
manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")
trainer._context_parallel_manager = manager
@@ -446,19 +446,25 @@ def test_fsdp2_shards_over_dp_shard_cp_and_a_later_trainer_restores_the_config(m
user_pc = types.SimpleNamespace(dp_replicate_size = 1, tp_size = 1, sp_size = 1, cp_size = 1)
accelerator = _fsdp2_accelerator(user_pc)
Trainer, _, built = _gated_trainer(monkeypatch, accelerator = accelerator)
- Trainer()
+ trainer = Trainer()
assert built == [(2, "ulysses", "allgather", "fsdp2")] # the default backend: auto
- pc = accelerator.state.parallelism_config
- # What accelerate's fsdp2_prepare_model indexes the mesh with; cp stays Unsloth's own.
- assert pc.fsdp_dim_names == ["dp_shard_cp"]
- assert (pc.dp_shard_size, pc.cp_size, pc.non_data_parallel_size) == (4, 1, 1)
- assert accelerator.state.device_mesh is not None
- later = _patched_trainer(
- monkeypatch,
- args = types.SimpleNamespace(context_parallel_size = 1),
- accelerator = accelerator,
- )
- later()
+ # Installed only while a CP entry point (train / evaluate / predict / save_model) runs.
+ assert accelerator.state.parallelism_config is user_pc
+ assert accelerator.state.device_mesh is None
+ with cp._cp_state_scope(trainer):
+ pc = accelerator.state.parallelism_config
+ # What accelerate's fsdp2_prepare_model indexes the mesh with; cp stays Unsloth's own.
+ assert pc.fsdp_dim_names == ["dp_shard_cp"]
+ assert (pc.dp_shard_size, pc.cp_size, pc.non_data_parallel_size) == (4, 1, 1)
+ assert accelerator.state.device_mesh is not None
+ later = _patched_trainer(
+ monkeypatch,
+ args = types.SimpleNamespace(context_parallel_size = 1),
+ accelerator = accelerator,
+ )
+ later()
+ assert accelerator.state.parallelism_config is user_pc
+ assert accelerator.state.device_mesh is None
assert accelerator.state.parallelism_config is user_pc
assert accelerator.state.device_mesh is None
@@ -516,7 +522,7 @@ def test_iterable_eval_dataset_passed_to_evaluate_is_refused(monkeypatch):
return iter(())
Trainer = _patched_trainer(monkeypatch)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = _manager()
with pytest.raises(NotImplementedError, match = "iterable datasets"):
trainer.evaluate(Stream())
@@ -532,7 +538,7 @@ def test_iterable_predict_dataset_is_refused(monkeypatch):
return iter(())
Trainer = _patched_trainer(monkeypatch)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = _manager()
with pytest.raises(NotImplementedError, match = "iterable datasets"):
trainer.predict(Stream())
@@ -701,15 +707,21 @@ def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):
mesh = object()
state = types.SimpleNamespace(device_mesh = None)
Trainer = _patched_trainer(monkeypatch, args = types.SimpleNamespace(context_parallel_size = 1))
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer.accelerator = types.SimpleNamespace(state = state)
trainer.args = types.SimpleNamespace(ddp_find_unused_parameters = None)
trainer._context_parallel_manager = types.SimpleNamespace(
size = 2, backend = "ring", device_mesh = mesh
)
- # A later non-CP trainer cleared the shared state; train() must put the CP mesh back.
+ # A later non-CP trainer cleared the shared state; train() must put the CP mesh back while it runs.
+ seen = []
+ release = cp._release_accelerator_state
+ monkeypatch.setattr(
+ cp, "_release_accelerator_state", lambda s: (seen.append(s.device_mesh), release(s))
+ )
assert trainer.train() == "trained"
- assert state.device_mesh is mesh
+ assert seen == [mesh]
+ assert state.device_mesh is None
assert trainer.args.ddp_find_unused_parameters is False
@@ -752,7 +764,7 @@ def test_eval_gather_keeps_one_loss_per_cp_group(monkeypatch, remainder):
gradient_state = types.SimpleNamespace(end_of_dataloader = True, remainder = remainder),
)
Trainer = _patched_trainer(monkeypatch)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer.accelerator = accelerator
trainer._context_parallel_manager = _manager(size = 2)
out = trainer.evaluation_loop()
@@ -766,7 +778,7 @@ def test_replaced_iterable_train_dataset_is_refused_at_train(monkeypatch):
return iter(())
Trainer = _patched_trainer(monkeypatch)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = _manager()
trainer.train_dataset = Stream()
with pytest.raises(NotImplementedError, match = "iterable datasets"):
@@ -785,7 +797,7 @@ def test_replaced_iterable_eval_dataset_is_refused_at_evaluate(monkeypatch):
return iter(())
Trainer = _patched_trainer(monkeypatch)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = _manager()
trainer.eval_dataset = Stream()
with pytest.raises(NotImplementedError, match = "iterable datasets"):
@@ -882,8 +894,11 @@ _CP_METHODS = (
)
-def test_cp_methods_are_wrapped_only_once_a_trainer_enables_cp(monkeypatch):
- # With CP off the wrappers slowed torch 2.10 LoRA steps by ~35%, so only __init__ is patched.
+def test_cp_methods_are_bound_only_on_the_trainer_that_enables_cp(monkeypatch):
+ import inspect
+
+ # With CP off class wrappers slowed torch 2.10 LoRA steps by ~35%, so only __init__ is patched on the
+ # class and the CP wrappers are bound per trainer that enables CP.
import types
Trainer = _patched_trainer(monkeypatch, install_cp_methods = False)
@@ -892,14 +907,16 @@ def test_cp_methods_are_wrapped_only_once_a_trainer_enables_cp(monkeypatch):
trainer = Trainer(args = types.SimpleNamespace(context_parallel_size = 1))
assert trainer._context_parallel_manager is None
assert trainer.train() == "trained"
- assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)
+ assert not any(name in vars(trainer) for name in _CP_METHODS)
Trainer, _, built = _gated_trainer(monkeypatch, install_cp_methods = False)
plain = {name: Trainer.__dict__[name] for name in _CP_METHODS}
- Trainer()
+ cp_trainer = Trainer()
assert built == [(2, "ulysses", "allgather")] # the default backend: auto
for name in _CP_METHODS:
- assert Trainer.__dict__[name].__wrapped__ is plain[name]
+ assert Trainer.__dict__[name] is plain[name]
+ # Bound per instance (train is wrapped again by the flash attention scope).
+ assert inspect.unwrap(vars(cp_trainer)[name]) is plain[name]
@pytest.mark.parametrize(
@@ -1030,7 +1047,7 @@ def test_flash_attention_impl_is_scoped_to_train_without_packing(monkeypatch):
def test_training_step_refuses_a_missing_token_count_with_labels(monkeypatch):
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
Trainer = _patched_trainer(monkeypatch)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
manager = _manager()
manager.mesh = None
trainer._context_parallel_manager = manager
@@ -1333,7 +1350,7 @@ def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
cp.patch_sft_trainer(install_cp_methods = True)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
pc = types.SimpleNamespace(cp_size = 1, sp_size = 1, tp_size = 1) if fsdp2 else None
trainer.accelerator = types.SimpleNamespace(parallelism_config = pc)
trainer.model = types.SimpleNamespace()
@@ -1356,9 +1373,9 @@ def test_instance_wrappers_made_in_init_capture_the_cp_methods(monkeypatch):
def on_init(self):
for name in ("train", "training_step"):
- method = getattr(type(self), name)
+ method = getattr(self, name)
captured[name] = method
- setattr(self, name, functools.partial(method, self))
+ setattr(self, name, functools.partial(method))
import types
@@ -1379,7 +1396,9 @@ def test_instance_wrappers_made_in_init_capture_the_cp_methods(monkeypatch):
) # SFTTrainer(model, args = SFTConfig(context_parallel_size = 2))
assert built == [(2, "ulysses", "allgather")] # the default backend: auto
for name, method in captured.items():
- assert method is not plain[name] and method.__wrapped__ is plain[name]
+ assert method.__func__ is not plain[name] and method.__func__.__wrapped__ is plain[name]
+ # The class keeps its own methods.
+ assert {name: Trainer.__dict__[name] for name in plain} == plain
@pytest.mark.parametrize("unhashable", [False, True])
@@ -1406,16 +1425,16 @@ def test_a_trainer_built_before_cp_still_restores_the_accelerator_state(monkeypa
on_init_plain(self)
Trainer, args, built = _gated_trainer(monkeypatch, install_cp_methods = False, on_init = on_init)
- restored = []
- monkeypatch.setattr(cp, "_restore_parallelism_config", lambda state: restored.append(state))
+ plain_train = Trainer.__dict__["train"]
early = Trainer(None, args = types.SimpleNamespace(context_parallel_size = 1))
assert early._context_parallel_manager is None
- early.accelerator = types.SimpleNamespace(state = types.SimpleNamespace(device_mesh = None))
- Trainer(None, args = args) # enables CP: installs the class wrappers
+ Trainer(None, args = args) # enables CP: binds the wrappers on that trainer only
assert built == [(2, "ulysses", "allgather")] # the default backend: auto
- restored.clear()
+ # The CP trainer put the shared state back at the end of its __init__ ...
+ assert early.accelerator.state.device_mesh is None
+ # ... and the earlier trainer runs the class's own train.
+ assert Trainer.__dict__["train"] is plain_train and early.train.func is plain_train
assert early.train() == "trained"
- assert restored == [early.accelerator.state]
@pytest.mark.parametrize("forwarded", [1, None])
diff --git a/tests/test_context_parallel_fsdp2_gloo.py b/tests/test_context_parallel_fsdp2_gloo.py
index 9eacb00ef04..06373b6c42f 100644
--- a/tests/test_context_parallel_fsdp2_gloo.py
+++ b/tests/test_context_parallel_fsdp2_gloo.py
@@ -170,7 +170,7 @@ def _worker(rank, port, out_dir):
trl.SFTTrainer = Trainer
cp.patch_sft_trainer(install_cp_methods = True)
- trainer = object.__new__(Trainer)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = manager
ids, labels = _batches()
diff --git a/tests/test_context_parallel_fsdp2_save_gloo.py b/tests/test_context_parallel_fsdp2_save_gloo.py
index 18670ab3945..ff1fede8686 100644
--- a/tests/test_context_parallel_fsdp2_save_gloo.py
+++ b/tests/test_context_parallel_fsdp2_save_gloo.py
@@ -6,7 +6,7 @@
Unsloth installs a ParallelismConfig for the (dp_shard, cp) mesh. transformers < 5 dispatches
save_model on that config before FSDP, skipping the full state dict gather, so PEFT is handed DTensor
shards and fails. The patched save_model must write whole adapter tensors equal to the unsharded ones,
-and leave the installed config in place afterwards.
+and put back the config that was there before the CP trainer installed its own.
"""
from __future__ import annotations
@@ -83,8 +83,8 @@ def _worker(
# A dp_shard only ParallelismConfig the user passed: still non-None for transformers 4.x.
from accelerate.parallelism_config import ParallelismConfig
state.parallelism_config = ParallelismConfig(dp_shard_size = 2)
+ user = state.parallelism_config
cp._install_fsdp2_parallelism_config(state, manager)
- installed = state.parallelism_config
torch.manual_seed(0)
cfg = LlamaConfig(
@@ -119,7 +119,7 @@ def _worker(
def unwrap_model(self, model, **kwargs):
return model
- trainer = object.__new__(SFTTrainer)
+ trainer = cp._bind_cp_methods(object.__new__(SFTTrainer))
trainer.model = trainer.model_wrapped = model
trainer.is_fsdp_enabled = True
trainer.is_deepspeed_enabled = False
@@ -145,7 +145,8 @@ def _worker(
result["error"] = None
except Exception as e:
result["error"] = f"{type(e).__name__}: {e}"
- result["config_kept"] = state.parallelism_config is installed
+ # The CP save_model installs its config only while it runs (_cp_state_scope), then puts the user's back.
+ result["config_kept"] = state.parallelism_config is user
result["full_state_dicts"] = len(full_state_dicts)
dist.barrier()
if rank == 0 and result["error"] is None:
diff --git a/tests/test_context_parallel_rl.py b/tests/test_context_parallel_rl.py
index eebb4fe6e34..063e0d56b7f 100644
--- a/tests/test_context_parallel_rl.py
+++ b/tests/test_context_parallel_rl.py
@@ -636,24 +636,27 @@ def test_rl_evaluate_and_predict_refuse_iterable_datasets(monkeypatch, kind):
assert trainer.predict([1, 2, 3, 4]) == "predicted"
-def test_trainers_built_before_cp_restore_the_shared_accelerator_state(monkeypatch):
- # The generated RL trainers bind self.train on the instance in __init__ (rl.py: MethodType(...)):
- # a trainer built before CP was enabled keeps that plain method, so it must still drop the CP mesh
- # a later CP trainer installed on the shared AcceleratorState.
+def test_trainers_built_before_cp_keep_the_plain_methods(monkeypatch):
+ # The generated RL trainers bind self.train on the instance in __init__ (rl.py: MethodType(...)), from
+ # the instance's CP train when one is bound, else from the class. A trainer built before CP keeps the
+ # class's own train (no CP wrapper runs for it); the CP trainer installs its state only inside train.
installs = []
Trainer = _cp_fake_trainer(monkeypatch, "dpo_trainer", [], installs)
+ plain_train = Trainer.__dict__.get("train")
def bind_train(trainer):
- trainer.train = types.MethodType(type(trainer).train, trainer)
+ bound = getattr(trainer.__dict__.get("train"), "__func__", None) or type(trainer).train
+ trainer.train = types.MethodType(bound, trainer)
plain = Trainer(_args(context_parallel_size = 1))
bind_train(plain)
cp_trainer = Trainer(_args())
bind_train(cp_trainer)
+ assert Trainer.__dict__.get("train") is plain_train
+ assert plain.train.__func__ is type(plain).train
installs.clear()
assert plain.train() == "trained"
- assert plain in installs, "the pre-CP trainer's bound train() skipped the accelerator restore"
- installs.clear()
+ assert installs == []
assert cp_trainer.train() == "trained" and cp_trainer in installs
diff --git a/tests/test_cp_instance_overrides_gloo.py b/tests/test_cp_instance_overrides_gloo.py
new file mode 100644
index 00000000000..d28890aa521
--- /dev/null
+++ b/tests/test_cp_instance_overrides_gloo.py
@@ -0,0 +1,254 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A context parallel SFTTrainer binds its wrappers on itself only: a plain SFTTrainer built later in the same
+process (same gloo ranks) runs the class's original methods, and no CP trainer wrapper is called during its
+steps. With class-level wrappers every later trainer paid for them (about 35% of a torch 2.10 LoRA step with CP
+off); here that is settled by call counts, not timing."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+import tempfile # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_parallel_cp_compose_gloo import _ENV_KEYS, SEQ, _checkpoint # noqa: E402
+from test_tp_native_gloo import _free_port, _import_checkout_unsloth # noqa: E402
+
+# The trainer-level CP wrappers (the attention / data hooks are per model and not counted here).
+_WRAPPERS = {
+ "patched_training_step",
+ "patched_prediction_step",
+ "patched_train",
+ "patched_evaluate",
+ "patched_predict",
+ "patched_evaluation_loop",
+ "patched_save_model",
+ "_save_model",
+ "patched_get_total_train_batch_size",
+ "_install_accelerator_state",
+ "_cp_state_scope",
+ "call", # the old pre-CP trainers' restore wrapper
+}
+_METHODS = (
+ "training_step",
+ "prediction_step",
+ "train",
+ "evaluate",
+ "predict",
+ "evaluation_loop",
+ "save_model",
+ "get_total_train_batch_size",
+)
+
+
+def _case(rank, world, tmp, ckpt):
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ import datasets
+ import torch.distributed as dist
+ import transformers
+ import trl
+ from peft import LoraConfig, get_peft_model
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ from unsloth import context_parallel as cpm
+ from unsloth.models._utils import patch_gradient_accumulation_fix
+
+ patch_gradient_accumulation_fix(transformers.Trainer)
+ cpm.patch_sft_trainer()
+ cpm.DEVICE_TYPE_TORCH = "cpu"
+ from unsloth.utils import attention_dispatch
+
+ attention_dispatch.HAS_XFORMERS = False
+ attention_dispatch.HAS_FLASH_ATTENTION = False
+ cls = trl.SFTTrainer
+ class_before = {name: cls.__dict__.get(name) for name in _METHODS}
+ resolved_before = {name: getattr(cls, name, None) for name in _METHODS}
+
+ def model():
+ m = AutoModelForCausalLM.from_pretrained(
+ ckpt, torch_dtype = torch.float32, attn_implementation = "sdpa"
+ )
+ torch.manual_seed(1)
+ return get_peft_model(
+ m, LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", lora_dropout = 0.0)
+ )
+
+ g = torch.Generator().manual_seed(1)
+ data = datasets.Dataset.from_dict(
+ {"input_ids": torch.randint(0, 1000, (16, SEQ), generator = g).tolist()}
+ )
+
+ def collate(rows):
+ ids = torch.tensor([r["input_ids"] for r in rows])
+ return {"input_ids": ids, "labels": ids.clone()}
+
+ def config(name, cp):
+ args = trl.SFTConfig(
+ output_dir = os.path.join(tmp, name),
+ per_device_train_batch_size = 2,
+ gradient_accumulation_steps = 2,
+ max_steps = 2,
+ learning_rate = 1e-3,
+ optim = "adamw_torch",
+ logging_steps = 1,
+ save_strategy = "no",
+ report_to = [],
+ use_cpu = True,
+ seed = 0,
+ remove_unused_columns = False,
+ disable_tqdm = True,
+ dataset_kwargs = {"skip_prepare_dataset": True},
+ )
+ if cp > 1:
+ args.context_parallel_size = cp
+ args.context_parallel_backend = "ulysses"
+ return args
+
+ tokenizer = AutoTokenizer.from_pretrained(ckpt)
+ calls = {}
+
+ def profile(frame, event, arg):
+ if (
+ event == "call"
+ and frame.f_code.co_filename == cpm.__file__
+ and frame.f_code.co_name in _WRAPPERS
+ ):
+ calls[frame.f_code.co_name] = calls.get(frame.f_code.co_name, 0) + 1
+
+ def run(trainer):
+ calls.clear()
+ sys.setprofile(profile)
+ try:
+ trainer.train()
+ finally:
+ sys.setprofile(None)
+ return dict(calls)
+
+ cp_model = model() # Hugging Face attention: the trainer installs generic Ulysses itself
+ cp_trainer = cls(
+ model = cp_model,
+ args = config("cp", 2),
+ train_dataset = data,
+ data_collator = collate,
+ processing_class = tokenizer,
+ )
+ assert cp_trainer._context_parallel_manager is not None
+ cp_calls = run(cp_trainer)
+
+ plain = cls(
+ model = model(),
+ args = config("plain", 1),
+ train_dataset = data,
+ data_collator = collate,
+ processing_class = tokenizer,
+ )
+ plain_calls = run(plain)
+ # What a plain trainer resolves each method to, and whether a CP wrapper sits behind it.
+ plain_wrapped = []
+ for name in _METHODS:
+ method = getattr(plain, name, None)
+ chain, f = [], getattr(method, "__func__", method)
+ while f is not None and len(chain) < 20:
+ chain.append(f)
+ f = getattr(
+ getattr(f, "__wrapped__", None), "__func__", getattr(f, "__wrapped__", None)
+ )
+ if any(
+ getattr(getattr(c, "__code__", None), "co_filename", None) == cpm.__file__
+ for c in chain
+ ):
+ plain_wrapped.append(name)
+ res = {
+ "cp_calls": cp_calls,
+ "plain_calls": plain_calls,
+ "plain_wrapped": plain_wrapped,
+ "class_unchanged": [n for n in _METHODS if cls.__dict__.get(n) is class_before[n]],
+ "resolved_unchanged": [n for n in _METHODS if getattr(cls, n, None) is resolved_before[n]],
+ "plain_losses": [h["loss"] for h in plain.state.log_history if "loss" in h],
+ }
+ if dist.is_initialized():
+ dist.barrier()
+ return res
+
+
+def _worker(rank, world, port, tmp, ckpt):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ for key in _ENV_KEYS:
+ os.environ.pop(key, None)
+ torch.set_num_threads(1)
+ import torch.distributed as dist
+
+ try:
+ try:
+ res = _case(rank, world, tmp, ckpt)
+ except Exception as e:
+ import traceback
+ res = {"error": f"{type(e).__name__}: {e}", "trace": traceback.format_exc()}
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ if dist.is_initialized():
+ try:
+ dist.destroy_process_group()
+ except Exception:
+ pass
+
+
+def test_a_plain_trainer_after_a_cp_trainer_runs_the_original_methods():
+ ctx = multiprocessing.get_context("spawn")
+ port, world = _free_port(), 2
+ with tempfile.TemporaryDirectory() as tmp:
+ ckpt = _checkpoint(tmp)
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, tmp, ckpt)) for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(900)
+ if p.is_alive():
+ p.kill()
+ results = []
+ for r in range(world):
+ path = os.path.join(tmp, f"result_{r}.pt")
+ results.append(
+ torch.load(path, weights_only = False)
+ if os.path.exists(path)
+ else {"error": f"rank {r}"}
+ )
+ if "skip" in results[0]:
+ pytest.skip(results[0]["skip"])
+ for rank, res in enumerate(results):
+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))
+ print(
+ f"rank {rank}: cp trainer wrapper calls {res['cp_calls']}, plain trainer {res['plain_calls']}"
+ )
+ # The CP trainer engaged its wrappers (the counter can see them) ...
+ assert (
+ res["cp_calls"].get("patched_training_step", 0) >= 4
+ and res["cp_calls"].get("patched_train") == 1
+ )
+ # ... the plain trainer built after it called none, and resolves to the class's original methods.
+ assert res["plain_calls"] == {}, res["plain_calls"]
+ assert res["plain_wrapped"] == [], res["plain_wrapped"]
+ assert res["class_unchanged"] == list(_METHODS)
+ assert res["resolved_unchanged"] == list(_METHODS)
+ assert len(res["plain_losses"]) == 2
diff --git a/tests/test_cp_trainer_refusal.py b/tests/test_cp_trainer_refusal.py
new file mode 100644
index 00000000000..87a5d9698cc
--- /dev/null
+++ b/tests/test_cp_trainer_refusal.py
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_CP_SIZE on a trainer without a context parallel wrapper: the refusal names the real cause (CPU backend,
+non-SFT / DPO / GRPO trainer, failed patched-trainer build), and the build skips unsloth_zoo's CPU trainer quietly."""
+
+import sys
+import types
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+
+
+def cp_env(monkeypatch):
+ for name in ("UNSLOTH_PARALLEL", "UNSLOTH_TP_SIZE", "UNSLOTH_PP_SIZE", "UNSLOTH_ALLOW_CPU"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setenv("UNSLOTH_CP_SIZE", "2")
+ return monkeypatch
+
+
+def _refusal(trainer_cls):
+ from unsloth.models import _utils
+
+ trainer = object.__new__(trainer_cls)
+ with pytest.raises(NotImplementedError) as info:
+ _utils._parallel_capabilities_pre(trainer, trainer_cls.__init__, (), {}, None)
+ return str(info.value)
+
+
+def _sft_like():
+ import trl
+
+ class SFTTrainer(trl.SFTTrainer): # what TRL's own class looks like: no CP wrapper on it
+ pass
+
+ SFTTrainer.__unsloth_context_parallel__ = False
+ return SFTTrainer
+
+
+def test_cpu_backend_names_the_backend_not_sfttrainer(cp_env):
+ cp_env.setitem(sys.modules, "unsloth._cpu_init", types.ModuleType("unsloth._cpu_init"))
+ message = _refusal(_sft_like())
+ assert "CPU backend" in message and "no GPU was found" in message, message
+ assert "Instead: launch on GPUs" in message and "use SFTTrainer" not in message, message
+
+
+def test_sft_trainer_whose_patched_build_failed_names_the_build_error(cp_env):
+ from unsloth.models import rl
+
+ cp_env.delitem(sys.modules, "unsloth._cpu_init", raising = False)
+ cp_env.setitem(
+ rl.PATCHED_TRAINER_BUILD_ERRORS, "sft_trainer", "OSError: source code not available"
+ )
+ message = _refusal(_sft_like())
+ assert (
+ "Unsloth's patched SFTTrainer could not be built (OSError: source code not available)"
+ in message
+ )
+ assert "use SFTTrainer" not in message, message
+
+
+def test_allow_cpu_names_the_variable(cp_env):
+ cp_env.delitem(sys.modules, "unsloth._cpu_init", raising = False)
+ cp_env.setenv("UNSLOTH_ALLOW_CPU", "1")
+ message = _refusal(_sft_like())
+ assert "UNSLOTH_ALLOW_CPU=1 skips Unsloth's trainer patches" in message, message
+
+
+def test_patched_trainer_build_skips_the_zoo_cpu_trainer(monkeypatch):
+ """unsloth_zoo.cpu replaces trl.trainer.sft_trainer.SFTTrainer with a subclass inspect cannot read; the build
+ used to fail on it with "OSError: source code not available" and warn. It now keeps it, quietly."""
+ from trl.trainer import sft_trainer
+
+ from unsloth.models import rl
+
+ base = sft_trainer.SFTTrainer
+ while getattr(base, "_unsloth_cpu", False) or base.__name__.startswith("Unsloth"):
+ base = base.__mro__[1]
+ namespace = {"_unsloth_cpu": True, "__module__": "unsloth_zoo.cpu.trainer"}
+ cpu_cls = type("SFTTrainer", (base,), namespace) # no __firstlineno__, as in the rehearsal
+ with pytest.raises((OSError, TypeError)):
+ __import__("inspect").getsource(cpu_cls)
+ monkeypatch.setattr(sft_trainer, "SFTTrainer", cpu_cls)
+ warnings = []
+ monkeypatch.setattr(rl.logger, "warning_once", lambda msg, *a, **k: warnings.append(msg))
+ monkeypatch.setattr(rl, "PATCHED_TRAINER_BUILD_ERRORS", {}, raising = False)
+ rl._patch_trl_rl_trainers("sft_trainer")
+ assert not [w for w in warnings if "Could not build" in w], warnings
+ assert sft_trainer.SFTTrainer is cpu_cls
+ assert "CPU backend" in rl.PATCHED_TRAINER_BUILD_ERRORS.get("sft_trainer", "")
diff --git a/tests/test_cp_ulysses_generic_gloo.py b/tests/test_cp_ulysses_generic_gloo.py
new file mode 100644
index 00000000000..63a6ae17fc5
--- /dev/null
+++ b/tests/test_cp_ulysses_generic_gloo.py
@@ -0,0 +1,305 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Ulysses context parallelism on any Hugging Face attention model, through the model's own attention function
+(ALL_ATTENTION_FUNCTIONS, or the modeling module's eager_attention_forward): scaling, softcap, sliding window and
+attention sinks survive the exchange. Two gloo CPU ranks vs one process: the loss and every parameter gradient.
+
+Configs are the tiny test repos' (tiny_models.py: hf-internal-testing/tiny-random-Gemma3ForCausalLM,
+trl-internal-testing/tiny-Qwen3ForCausalLM, trl-internal-testing/tiny-GptOssForCausalLM) with random weights, a
+small vocabulary, and a window shorter than a shard so it crosses the seam."""
+
+import os
+import tempfile
+import types
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import torch.nn.functional as F
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+from unsloth.utils import attention_dispatch as ad
+from unsloth.utils import ulysses_hybrid as uh
+
+transformers = pytest.importorskip("transformers")
+
+SEQ = 16
+WINDOW = 5 # < SEQ / 2: windows cross the shard seam
+REPOS = {
+ "gemma3": "hf-internal-testing/tiny-random-Gemma3ForCausalLM",
+ "gemma2": "trl-internal-testing/tiny-Gemma2ForCausalLM",
+ "qwen3": "trl-internal-testing/tiny-Qwen3ForCausalLM",
+ "gpt_oss": "trl-internal-testing/tiny-GptOssForCausalLM",
+}
+
+
+def _config(kind):
+ from transformers import AutoConfig
+
+ try:
+ config = AutoConfig.from_pretrained(REPOS[kind])
+ except Exception as e: # offline host without the tiny repo cached
+ pytest.skip(f"{REPOS[kind]}: {e}")
+ config.vocab_size = 96
+ config.num_hidden_layers = 2
+ config.attention_dropout = 0.0
+ config.use_cache = False
+ config.pad_token_id = 0
+ if kind in ("gemma3", "gemma2"):
+ # Gemma3: 2 heads / 1 kv head, so the kv head is replicated over cp 2, and the 1 / sqrt(256) scale
+ # (head_dim 8) is off the default the old path refused. Gemma2: attention logit softcapping.
+ config.layer_types = ["sliding_attention", "full_attention"]
+ config.sliding_window = WINDOW
+ config.attn_logit_softcapping = 0.05 # small enough to bend the random tiny logits
+ config.final_logit_softcapping = None
+ elif kind == "gpt_oss":
+ config.layer_types = ["sliding_attention", "full_attention"]
+ config.sliding_window = WINDOW
+ config.output_router_logits = False
+ return config
+
+
+def _model(kind, implementation):
+ from transformers import AutoModelForCausalLM
+
+ torch.manual_seed(0)
+ model = AutoModelForCausalLM.from_config(_config(kind), attn_implementation = implementation)
+ model.config._attn_implementation = implementation
+ for name, p in model.named_parameters():
+ if name.endswith("sinks"):
+ with torch.no_grad():
+ p.normal_(0, 1) # zero-initialised: make every head's sink differ
+ return model.float().train()
+
+
+def _manager(size, rank):
+ manager = object.__new__(cp.ContextParallelManager)
+ manager.size = size
+ manager.backend = "ulysses"
+ manager.packed_seq_lengths = None
+ manager.mesh = types.SimpleNamespace(get_group = lambda: None, get_local_rank = lambda: rank)
+ return manager
+
+
+def _loss(logits, shift_labels, count):
+ return (
+ F.cross_entropy(
+ logits.float().flatten(0, 1), shift_labels.flatten(), ignore_index = -100, reduction = "sum"
+ )
+ / count
+ )
+
+
+def _batch(packed):
+ ids = torch.randint(
+ 0, 96, (1 if packed else 2, SEQ), generator = torch.Generator().manual_seed(1)
+ )
+ labels = ids.clone()
+ labels[labels % 5 == 0] = -100
+ return ids, labels
+
+
+def _reference(model, ids, labels, lengths):
+ """One process: the whole rows, or every packed document as its own row."""
+ if lengths is None:
+ shift = F.pad(labels, (0, 1), value = -100)[:, 1:]
+ count = shift.ne(-100).sum()
+ _loss(model(input_ids = ids, use_cache = False).logits, shift, count).backward()
+ return count
+ pieces, start = [], 0
+ for n in lengths:
+ doc_ids, doc_labels = ids[:, start : start + n], labels[:, start : start + n]
+ pieces.append((doc_ids, F.pad(doc_labels, (0, 1), value = -100)[:, 1:]))
+ start += n
+ count = sum(shift.ne(-100).sum() for _, shift in pieces)
+ total = sum(
+ _loss(model(input_ids = doc_ids, use_cache = False).logits, shift, count)
+ for doc_ids, shift in pieces
+ )
+ total.backward()
+ return count
+
+
+def _parity_worker(
+ rank,
+ world,
+ kind,
+ implementation,
+ packed,
+ out_dir,
+ control = None,
+):
+ ad.HAS_FLASH_ATTENTION = False
+ ad.HAS_XFORMERS = False
+ model = _model(kind, implementation)
+ assert cp._generic_hf_attention(model) and cp._supports_context_parallel(model, "ulysses")
+ cp._check_cp_model(model, world, "ulysses")
+ ids, labels = _batch(packed)
+ lengths = [6, 7, 3] if packed else None
+ count = _reference(model, ids, labels, lengths)
+ reference = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
+ model.zero_grad(set_to_none = True)
+
+ calls, lengths_seen = [], set()
+ original = uh._ulysses_hf_attention
+
+ def counted(module, query, *args, **kwargs):
+ fn = kwargs["attention_fn"]
+
+ def seen(shard, q, *a, **k):
+ lengths_seen.add(int(q.shape[2])) # the model's own function sees the whole sequence
+ return fn(shard, q, *a, **k)
+
+ kwargs["attention_fn"] = seen
+ calls.append(1)
+ return original(module, query, *args, **kwargs)
+
+ uh._ulysses_hf_attention = counted
+ manager = _manager(world, rank)
+ manager.attach_attention_hooks(model)
+ if control == "window_dropped":
+ uh._sliding_window = lambda module, kwargs: None
+ elif control == "softcap_dropped":
+ original_counted = uh._ulysses_hf_attention
+
+ def no_softcap(*args, **kwargs):
+ kwargs.pop("softcap", None)
+ return original_counted(*args, **kwargs)
+
+ uh._ulysses_hf_attention = no_softcap
+ elif control == "sinks_unsliced":
+ original_init = uh._HeadShard.__init__
+
+ def full_sinks(self, module, **overrides):
+ overrides.pop("sinks", None)
+ original_init(self, module, **overrides)
+
+ uh._HeadShard.__init__ = full_sinks
+ inputs = {"input_ids": ids.clone(), "labels": labels.clone()}
+ if packed:
+ inputs["packed_seq_lengths"] = torch.tensor(lengths, dtype = torch.int32)
+ manager._prepare_inputs(inputs)
+ with manager.apply(inputs):
+ out = model(
+ input_ids = inputs["input_ids"],
+ position_ids = inputs["position_ids"],
+ use_cache = False,
+ )
+ loss = _loss(out.logits, inputs["shift_labels"], count)
+ loss.backward()
+ dist.all_reduce(loss.detach())
+ uh._ulysses_hf_attention = original
+ assert len(calls) == model.config.num_hidden_layers, calls
+ assert lengths_seen == {SEQ}, lengths_seen
+ assert inputs["input_ids"].shape[1] == SEQ // world
+ worst = 0.0
+ for name, param in model.named_parameters():
+ if not param.requires_grad:
+ continue
+ grad = torch.zeros_like(param) if param.grad is None else param.grad.clone()
+ dist.all_reduce(grad)
+ want = reference.get(name, torch.zeros_like(param))
+ torch.testing.assert_close(grad, want, rtol = 1e-4, atol = 1e-6, msg = name)
+ worst = max(worst, float((grad - want).abs().max()))
+ if rank == 0 and out_dir:
+ with open(
+ os.path.join(out_dir, f"{kind}_{implementation}_{'packed' if packed else 'rows'}.txt"),
+ "w",
+ ) as f:
+ f.write(
+ f"loss {float(loss):.6f} ulysses calls {len(calls)} max grad diff {worst:.3e}\n"
+ )
+
+
+def _spawn(fn, world, *args):
+ # Fork: a spawned child would re-import unsloth without the conftest CUDA spoof.
+ if "fork" not in mp.get_all_start_methods():
+ pytest.skip("needs the fork start method")
+ with tempfile.TemporaryDirectory() as d:
+ mp.start_processes(
+ _entry,
+ args = (world, os.path.join(d, "init"), fn, args),
+ nprocs = world,
+ join = True,
+ start_method = "fork",
+ )
+
+
+def _entry(rank, world, init_file, fn, args):
+ torch.set_num_threads(1)
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
+ try:
+ fn(rank, world, *args)
+ finally:
+ dist.destroy_process_group()
+
+
+_OUT = os.environ.get("UNSLOTH_TEST_PARITY_OUT", "")
+
+
+ "kind, implementation, packed",
+ [
+ ("gemma3", "eager", False), # sliding window + non-default scale + replicated kv head
+ ("gemma2", "eager", False), # attention logit softcapping + sliding window
+ ("gemma3", "sdpa", False), # sliding window through sdpa's mask
+ ("gemma3", "eager", True), # window and packed documents in one mask
+ ("qwen3", "sdpa", False), # plain causal: sdpa's is_causal path
+ ("qwen3", "sdpa", True),
+ ("gpt_oss", "eager", False), # attention sinks (local heads' slice) + sliding window
+ ],
+)
+def test_generic_ulysses_matches_one_process(kind, implementation, packed):
+ _config(kind) # skip before forking when the repo is unavailable
+ _spawn(_parity_worker, 2, kind, implementation, packed, _OUT)
+
+
+ "kind, control",
+ [("gemma3", "window_dropped"), ("gemma2", "softcap_dropped"), ("gpt_oss", "sinks_unsliced")],
+)
+def test_each_feature_is_load_bearing(kind, control):
+ """Negative controls: dropping the window or the softcap after the exchange, or handing every head's sinks
+ to a head shard, breaks parity, so the cases above prove the features are passed through."""
+ _config(kind)
+ with pytest.raises(Exception, match = "AssertionError|IndexError|RuntimeError|Mismatch"):
+ _spawn(_parity_worker, 2, kind, "eager", False, "", control)
+
+
+def test_windowed_hf_models_pass_the_ulysses_check_but_not_ring():
+ model = _model("gemma2", "eager")
+ assert cp._unsupported_attention(model.config, "ulysses", generic = True) is None
+ softcapped = types.SimpleNamespace(attn_logit_softcapping = 3.0)
+ assert "softcapping" in cp._unsupported_attention(softcapped, "ring")
+ assert cp._unsupported_attention(softcapped, "ulysses", generic = True) is None
+ assert "sliding" in cp._unsupported_attention(model.config, "ring")
+ assert "sliding" in cp._unsupported_attention(model.config, "ulysses") # Unsloth's fast forward
+ with pytest.raises(NotImplementedError, match = "sliding"):
+ cp._check_cp_model(model, 2, "ring")
+ cp._check_cp_model(model, 2, "ulysses")
+ # DPO / GRPO keep the Llama-style rule.
+ with pytest.raises(NotImplementedError):
+ cp._check_cp_model(model, 2, "ulysses", allow_hybrids = False)
+
+
+def test_chunked_attention_stays_refused():
+ model = _model("qwen3", "sdpa")
+ model.config.attention_chunk_size = 8
+ with pytest.raises(NotImplementedError, match = "chunked"):
+ cp._check_cp_model(model, 2, "ulysses")
+
+
+def test_unsloth_fast_forwards_are_not_generic():
+ from unsloth.models.llama import LlamaAttention_fast_forward
+
+ class Fast(torch.nn.Module):
+ forward = LlamaAttention_fast_forward
+
+ root = torch.nn.Module()
+ root.self_attn = Fast()
+ assert not uh.is_hf_registry_attention(root.self_attn)
+ assert not cp._generic_hf_attention(root)
+ assert cp._supports_context_parallel(root, "ulysses") # the fast Llama path, as before
diff --git a/tests/test_fsdp2_bnb_optim_kept.py b/tests/test_fsdp2_bnb_optim_kept.py
new file mode 100644
index 00000000000..33db6c485de
--- /dev/null
+++ b/tests/test_fsdp2_bnb_optim_kept.py
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""accelerate's FSDP2 keeps optim = 'adamw_8bit': a bitsandbytes Adam / AdamW steps each rank's local shard after
+Accelerator._prepare_fsdp2, and a bitsandbytes optimizer that cannot is refused before the first step."""
+
+from __future__ import annotations
+
+
+import pytest
+
+pytest.importorskip("trl")
+
+
+def test_accelerate_fsdp2_prepare_localizes_bitsandbytes_adamw():
+ bnb = pytest.importorskip("bitsandbytes")
+ import torch
+
+ from unsloth.models.fsdp2 import _localize_or_refuse_bnb
+ from unsloth.models.fsdp2_bnb_optim import is_local_shard_bnb_optimizer
+
+ p = torch.nn.Parameter(torch.zeros(8))
+ opt = bnb.optim.AdamW8bit([p], lr = 1e-3)
+ _localize_or_refuse_bnb([p, opt])
+ assert is_local_shard_bnb_optimizer(opt)
+
+ torch_opt = torch.optim.AdamW([torch.nn.Parameter(torch.zeros(8))])
+ _localize_or_refuse_bnb([torch_opt]) # torch optimizers pass untouched
+ assert not is_local_shard_bnb_optimizer(torch_opt)
+
+
+def test_accelerate_fsdp2_refuses_bnb_that_cannot_step_local_shards(monkeypatch):
+ bnb = pytest.importorskip("bitsandbytes")
+ import torch
+
+ from unsloth.models.fsdp2 import _localize_or_refuse_bnb
+
+ lion = bnb.optim.Lion8bit([torch.nn.Parameter(torch.zeros(8))], lr = 1e-4)
+ with pytest.raises(ValueError, match = "Unsloth: FSDP2 .*Lion8bit.*adamw_torch"):
+ _localize_or_refuse_bnb([lion])
+
+ monkeypatch.setenv("UNSLOTH_FSDP2_BNB_LOCAL", "0")
+ opt = bnb.optim.AdamW8bit([torch.nn.Parameter(torch.zeros(8))], lr = 1e-3)
+ with pytest.raises(ValueError, match = "UNSLOTH_FSDP2_BNB_LOCAL=0"):
+ _localize_or_refuse_bnb([opt])
diff --git a/tests/test_fsdp2_default_optim.py b/tests/test_fsdp2_default_optim.py
index 1888fdb5ed7..1236d769869 100644
--- a/tests/test_fsdp2_default_optim.py
+++ b/tests/test_fsdp2_default_optim.py
@@ -1,7 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""unsloth#3551: Unsloth's adamw_8bit default cannot step FSDP2's DTensor parameters."""
+"""unsloth#3551: Unsloth's adamw_8bit default under FSDP2. The generated configs used to switch it to
+adamw_torch_fused; FSDP2 (native and accelerate's) now steps bitsandbytes Adam / AdamW on each rank's
+local shard, so the configs keep it (tests/test_fsdp2_bnb_optim_kept.py)."""
from __future__ import annotations
@@ -45,6 +47,7 @@ _PROBE = textwrap.dedent(
"grpo_fsdp2_args": optim(GRPOConfig, {}, **fsdp2_args),
"grpo_fsdp2_json": optim(GRPOConfig, {}, fsdp = "full_shard", fsdp_config = fsdp2_json),
"grpo_fsdp2_explicit_torch": optim(GRPOConfig, {"FSDP_VERSION": "2"}, optim = "adamw_torch"),
+ "sft_fsdp2_paged": optim(SFTConfig, {"FSDP_VERSION": "2"}, optim = "paged_adamw_8bit"),
"grpo_fsdp_unversioned": optim(GRPOConfig, {}, fsdp = "full_shard"),
}))
"""
@@ -61,6 +64,7 @@ def probe(tmp_path_factory):
)
lines = [line for line in run.stdout.splitlines() if line.startswith("PROBE ")]
assert lines, f"probe failed:\n{run.stdout[-3000:]}\n{run.stderr[-3000:]}"
+ assert "Switching optim" not in run.stdout
return json.loads(lines[-1][len("PROBE ") :])
@@ -71,18 +75,15 @@ def test_without_fsdp2_the_default_is_unchanged(probe):
@pytest.mark.parametrize("case", ["grpo_fsdp2", "sft_fsdp2", "grpo_fsdp2_args", "grpo_fsdp2_json"])
-def test_fsdp2_gets_an_optimizer_that_steps_dtensors(probe, case):
- assert probe[case] == "adamw_torch_fused"
+def test_fsdp2_keeps_adamw_8bit(probe, case):
+ # Stepped on the local shards (fsdp2_bnb_optim), not silently switched to torch's AdamW.
+ assert probe[case] == "adamw_8bit"
-def test_fsdp_without_a_version_follows_the_transformers_default(probe):
- """TrainingArguments picks FSDP2 when fsdp_config names no version from transformers 5.10.0, FSDP1 before."""
- from transformers import __version__
- from unsloth_zoo.utils import Version
+def test_fsdp_without_a_version_keeps_adamw_8bit(probe):
+ assert probe["grpo_fsdp_unversioned"] == "adamw_8bit"
- fsdp2 = Version(__version__) >= Version("5.10.0")
- assert probe["grpo_fsdp_unversioned"] == ("adamw_torch_fused" if fsdp2 else "adamw_8bit")
-
-def test_an_explicit_non_bnb_choice_is_kept(probe):
+def test_explicit_choices_are_kept(probe):
assert probe["grpo_fsdp2_explicit_torch"] == "adamw_torch"
+ assert probe["sft_fsdp2_paged"] == "paged_adamw_8bit"
diff --git a/tests/test_parallel_cp_compose_gloo.py b/tests/test_parallel_cp_compose_gloo.py
index 62db8ec2592..73ead931c99 100644
--- a/tests/test_parallel_cp_compose_gloo.py
+++ b/tests/test_parallel_cp_compose_gloo.py
@@ -5,10 +5,10 @@
the same script in one process: losses, per-parameter gradients of the first optimizer step and the adapters
after the run.
-CPU ranks cannot run Unsloth's fast Llama attention (it needs CUDA), so the attention is the Hugging Face one with
-the Ulysses wrapper hybrids use (utils/ulysses_hybrid.install_hf_attention_wrapper), and the model support check is
-opened for it. Everything else (the env layout, the mesh, tensor parallel, the CP sharding, the token count and
-the gradient reduction) is the production path."""
+CPU ranks cannot run Unsloth's fast Llama attention (it needs CUDA), so the attention is the Hugging Face one, which
+the production support check accepts and the trainer routes through generic Ulysses (ALL_ATTENTION_FUNCTIONS). The
+env layout, the mesh, tensor parallel, the CP sharding, the token count and the gradient reduction are all the
+production path; nothing is patched open."""
import multiprocessing
import os
@@ -196,7 +196,6 @@ def _case_train(
from unsloth import context_parallel as cpm
from unsloth.models import pp_native, tp_env, tp_native
from unsloth.models._utils import patch_gradient_accumulation_fix
- from unsloth.utils.ulysses_hybrid import install_hf_attention_wrapper
patch_gradient_accumulation_fix(transformers.Trainer)
cpm.patch_sft_trainer()
@@ -220,8 +219,6 @@ def _case_train(
torch.backends.cuda.enable_mem_efficient_sdp(False)
torch.backends.cuda.enable_cudnn_sdp(False)
torch.use_deterministic_algorithms(True, warn_only = True)
- # The test seam: CPU ranks run the Hugging Face attention through the Ulysses wrapper.
- cpm._supports_context_parallel = lambda model, backend = "ring": True
if bypass_pre or bypass_post:
# Evidence runs only: switch off the aux-loss guards to show what each one catches.
from unsloth import parallel_capabilities
@@ -283,9 +280,6 @@ def _case_train(
moe_calls = _count_moe_backend_calls() if count else None
if router_aux:
model.config.output_router_logits = True
- for name, module in model.named_modules():
- if name.endswith("self_attn"):
- install_hf_attention_wrapper(module, "sdpa", cpm.get_cp_manager)
compiled = _block_compile_on_cpu(model) if compile else []
# Data parallel replicas read different rows; cp / tp / pp peers share theirs.
diff --git a/tests/test_parallel_env.py b/tests/test_parallel_env.py
index 7b97dcc86d0..cf323bd7994 100644
--- a/tests/test_parallel_env.py
+++ b/tests/test_parallel_env.py
@@ -442,7 +442,7 @@ def test_tensor_parallel_alone_keeps_what_tp_native_supports():
(dict(cp = 2, cp_backend = "ring", hybrid = True), "cp backend", "cp_backend=ulysses"),
(dict(cp = 2, cp_backend = "ring", trainer = "dpo"), "cp backend", "cp_backend=ulysses"),
(dict(cp = 2, cp_backend = "ring", sm = 7), "cp backend", "cp_backend=ulysses"),
- (dict(cp = 2, cp_backend = "ulysses", windowed = True), "cp x attention", "drop cp"),
+ (dict(cp = 2, cp_backend = "ring", windowed = True), "cp x attention", "cp_backend=ulysses"),
(dict(cp = 2, torch = (2, 6)), "cp x torch", "upgrade torch"),
(dict(pp = 2, trainer = "grpo"), "pp x trainer", "drop pp"),
(dict(pp = 2, packing = True), "pp x packing", "packing = False"),
@@ -467,6 +467,7 @@ def test_every_refusal_names_its_axis_and_the_alternative(query, axis, instead):
dict(cp = 2, cp_backend = "ulysses", packing = True, hybrid = True),
dict(cp = 2, cp_backend = "ulysses", trainer = "grpo"),
dict(cp = 2, cp_backend = "ring", sm = 9),
+ dict(cp = 2, cp_backend = "ulysses", windowed = True), # Ulysses runs the model's own attention
dict(pp = 2, tp = 2, dp = 2, fsdp2 = True),
dict(tp = 2, mode = "qlora"),
dict(dp = 4, mode = "full", fsdp2 = True),
@@ -517,7 +518,10 @@ def test_cp_env_with_a_plain_trainer_is_refused(clean_env):
def __init__(self):
pass
- with pytest.raises(NotImplementedError, match = r"cp x trainer: .*Plain would train every rank"):
+ with pytest.raises(
+ NotImplementedError,
+ match = r"cp x trainer: .*Plain is not one of the trainers .*Instead: use SFTTrainer",
+ ):
_utils._parallel_capabilities_pre(Plain(), Plain.__init__, (), {}, None)
@@ -578,6 +582,9 @@ def test_every_contract_axis_has_declared_features():
} <= names
for f in caps.FEATURES:
assert f.module and f.scope in ("instance", "class"), f
+ # Context parallel binds its trainer wrappers per instance now (context_parallel._bind_instance_methods):
+ # no feature still patches a class.
+ assert f.scope == "instance", f.name
assert set(f.hooks) | set(f.exclusive) <= set(caps.HOOKS), (f.name, f.hooks, f.exclusive)
assert set(f.consumes) | set(f.uses) | set(f.produces) <= set(caps.METADATA), f.name
assert set(f.mesh) <= {"dp", "tp", "pp", "cp"}, f.name
@@ -590,7 +597,7 @@ def test_every_contract_axis_has_declared_features():
[
(
dict(pp = 2, cp = 2, cp_backend = "ulysses"),
- r"pp x cp.ulysses: both run Trainer.training_step",
+ r"pp x cp.ulysses: both run trainer.training_step",
),
(dict(attention = "flash_varlen"), r"flash_varlen consumes cu_seqlens"),
],
diff --git a/tests/test_ulysses_hybrid.py b/tests/test_ulysses_hybrid.py
index fe794b46b54..c7b7c1d5656 100644
--- a/tests/test_ulysses_hybrid.py
+++ b/tests/test_ulysses_hybrid.py
@@ -335,11 +335,26 @@ def test_hybrid_gates():
cp._check_stateful_mixers(model, "ulysses", 2)
-def test_eager_attention_is_refused():
+def test_flex_attention_is_refused():
+ # Eager is routed through the modeling module's eager_attention_forward now (test_cp_ulysses_generic_gloo.py).
model = _model(KINDS[0])
attn = model.model.layers[1].self_attn
with pytest.raises(NotImplementedError, match = "attn_implementation"):
- uh.install_hf_attention_wrapper(attn, "eager", cp.get_cp_manager)
+ uh.install_hf_attention_wrapper(attn, "flex_attention", cp.get_cp_manager)
+
+
+def test_bidirectional_attention_is_refused():
+ # The rebuilt mask is causal: a bidirectional layer (Gemma3 use_bidirectional_attention) would train causal.
+ model = _model(KINDS[0])
+ attn = model.model.layers[1].self_attn
+ attn.is_causal = False
+ with pytest.raises(NotImplementedError, match = "only supports causal attention"):
+ uh.install_hf_attention_wrapper(attn, "sdpa", cp.get_cp_manager)
+ assert not getattr(attn, "_unsloth_cp_ulysses_attention", False)
+ attn.is_causal = True
+ with pytest.raises(NotImplementedError, match = "only supports causal attention"):
+ uh._refuse_non_causal(attn, False) # a per-call is_causal = False override
+ uh._refuse_non_causal(attn, None)
def test_hybrid_packing_metadata_is_off_under_context_parallelism():
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 5170f377ff8..059978036d0 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -6,7 +6,6 @@ from __future__ import annotations
import contextlib
import functools
import types
-import weakref
from typing import Iterator, Optional
import torch
@@ -146,9 +145,17 @@ def resolve_cp_backend(
return "ulysses"
-def _unsupported_attention(config, backend = "ring") -> Optional[str]:
- # Ring and Llama's forward ignore mask, window and softcap, so both backends refuse them.
+def _unsupported_attention(
+ config,
+ backend = "ring",
+ generic = False,
+) -> Optional[str]:
+ # Ring and Unsloth's fast Llama forward ignore mask, window and softcap, so they refuse them. Ulysses
+ # around the model's own attention function (`generic`) keeps the window and the softcap.
allowed = {"full_attention", "linear_attention"} if backend == "ulysses" else {"full_attention"}
+ windows_ok = generic and backend == "ulysses"
+ if windows_ok:
+ allowed = allowed | {"sliding_attention"}
configs = [config]
text_config = getattr(config, "text_config", None)
if text_config is not None and text_config is not config:
@@ -167,13 +174,14 @@ def _unsupported_attention(config, backend = "ring") -> Optional[str]:
return f"{', '.join(kinds)} layers{hint}"
# Any positive window, even with use_sliding_window off: Unsloth's masks read it directly.
sliding_window = getattr(c, "sliding_window", None)
- if (isinstance(sliding_window, int) and sliding_window > 0) or getattr(
- c, "use_sliding_window", None
- ) is True:
+ if not windows_ok and (
+ (isinstance(sliding_window, int) and sliding_window > 0)
+ or getattr(c, "use_sliding_window", None) is True
+ ):
return "sliding window attention"
if getattr(c, "attention_chunk_size", None):
return "chunked attention"
- if getattr(c, "attn_logit_softcapping", None):
+ if not windows_ok and getattr(c, "attn_logit_softcapping", None):
return "attention logit softcapping"
return None
@@ -194,6 +202,7 @@ from .utils.ulysses_hybrid import (
GATED_DELTA_ATTENTION_CLASSES,
check_gated_delta_heads,
install_hf_attention_wrapper,
+ is_hf_registry_attention,
is_ulysses_gated_delta,
wrap_gated_delta_forward,
)
@@ -258,20 +267,43 @@ def _hybrid_config_blocker(model) -> Optional[str]:
return None
-def _supports_context_parallel(model, backend = "ring") -> bool:
+def _attention_modules(model) -> list:
+ if not hasattr(model, "named_modules"):
+ return []
+ return [module for name, module in model.named_modules() if name.endswith("self_attn")]
+
+
+def _uses_fast_llama_attention(model) -> bool:
from .models.llama import LlamaAttention_fast_forward, runs_fast_forward
+ modules = _attention_modules(model)
+ return bool(modules) and all(runs_fast_forward(m, LlamaAttention_fast_forward) for m in modules)
+
+
+def _generic_hf_attention(model) -> bool:
+ """Every attention layer is a Hugging Face forward through ALL_ATTENTION_FUNCTIONS, so Ulysses can run the
+ model's own attention function (scaling, softcap, sliding window, sinks) after the exchange."""
+ if not isinstance(model, torch.nn.Module) or _stateful_mixers(model):
+ return False
+ modules = _attention_modules(model)
+ return bool(modules) and all(is_hf_registry_attention(m) for m in modules)
+
- modules = [module for name, module in model.named_modules() if name.endswith("self_attn")]
- if bool(modules) and all(runs_fast_forward(m, LlamaAttention_fast_forward) for m in modules):
+def _supports_context_parallel(
+ model,
+ backend = "ring",
+ generic = True,
+) -> bool:
+ if _uses_fast_llama_attention(model):
return not _stateful_mixers(model)
+ if backend != "ulysses" or _hybrid_config_blocker(model) is not None:
+ return False
mixers = _stateful_mixers(model)
- return (
- backend == "ulysses"
- and bool(mixers)
- and all(is_ulysses_gated_delta(m) for m, _ in mixers)
- and all(type(m).__name__ in GATED_DELTA_ATTENTION_CLASSES for m in modules)
- and _hybrid_config_blocker(model) is None
- )
+ if mixers:
+ # Hybrids: the family-specific linear attention forwards, and their HF full attention layers.
+ return all(is_ulysses_gated_delta(m) for m, _ in mixers) and all(
+ type(m).__name__ in GATED_DELTA_ATTENTION_CLASSES for m in _attention_modules(model)
+ )
+ return generic and _generic_hf_attention(model)
def _check_stateful_mixers(model, backend, size) -> None:
@@ -351,10 +383,13 @@ class ContextParallelManager:
for module, _ in _stateful_mixers(model):
if is_ulysses_gated_delta(module):
wrap_gated_delta_forward(module, get_cp_manager)
+ # Unsloth's fast Llama forward has its own Ulysses path; every other Hugging Face attention layer
+ # runs its own attention function after the exchange.
+ generic = not _uses_fast_llama_attention(model)
for name, module in model.named_modules():
- if (
- name.endswith("self_attn")
- and type(module).__name__ in GATED_DELTA_ATTENTION_CLASSES
+ if name.endswith("self_attn") and (
+ type(module).__name__ in GATED_DELTA_ATTENTION_CLASSES
+ or (generic and is_hf_registry_attention(module))
):
install_hf_attention_wrapper(
module, module.config._attn_implementation, get_cp_manager
@@ -666,16 +701,62 @@ def _restore_parallelism_config(state) -> None:
_INSTALLED_PARALLELISM_CONFIG.clear()
+def _release_accelerator_state(state) -> None:
+ """Put back the AcceleratorState mesh / parallelism config a CP trainer installed (no-op otherwise)."""
+ if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:
+ # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.
+ state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None
+ _INSTALLED_MESH.clear()
+ _restore_parallelism_config(state)
+
+
+def _cp_state_scope(self):
+ """A CP trainer's entry point (train / evaluate / predict / save_model): its mesh and parallelism config
+ sit on the shared AcceleratorState only while it runs, so a trainer without CP built in the same process
+ keeps the class's original methods and still sees the state it expects. Nested calls (train ->
+ save_model) restore only when the outermost one returns."""
+ depth = self.__dict__.get("_unsloth_cp_state_depth", 0)
+ self._unsloth_cp_state_depth = depth + 1
+ _install_accelerator_state(self)
+ try:
+ yield
+ finally:
+ self._unsloth_cp_state_depth = depth
+ if depth == 0 and getattr(self, "_context_parallel_manager", None) is not None:
+ _release_accelerator_state(getattr(getattr(self, "accelerator", None), "state", None))
+
+
+def _bind_instance_methods(trainer, methods: dict) -> None:
+ """Bind the CP wrappers on this trainer only (types.MethodType); the class keeps its methods."""
+ import types
+
+ bound = {name: types.MethodType(fn, trainer) for name, fn in methods.items() if fn is not None}
+ trainer.__dict__.update(bound)
+ trainer.__dict__["_unsloth_cp_bound_methods"] = bound
+
+
+def _bind_cp_methods(trainer):
+ """Bind the CP wrappers of the trainer's (patched) class on a trainer built without __init__ (tests)."""
+ _bind_instance_methods(trainer, getattr(type(trainer), "_unsloth_cp_methods"))
+ return trainer
+
+
+def _unbind_instance_methods(trainer) -> None:
+ """Drop the CP wrappers bound by _bind_instance_methods; an attribute Unsloth re-wrapped since (it then
+ holds our bound method inside, which passes through without a manager) is left alone."""
+ bound = trainer.__dict__.pop("_unsloth_cp_bound_methods", None) or {}
+ for name, method in bound.items():
+ if trainer.__dict__.get(name) is method:
+ del trainer.__dict__[name]
+
+
def _install_accelerator_state(self):
manager = getattr(self, "_context_parallel_manager", None)
accelerator = getattr(self, "accelerator", None)
state = getattr(accelerator, "state", None)
if manager is None:
- if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:
- # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.
- state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None
- _INSTALLED_MESH.clear()
- _restore_parallelism_config(state)
+ _release_accelerator_state(state)
return
if accelerator is None:
return
@@ -1121,7 +1202,15 @@ def _check_cp_model(
tp: int = 1,
) -> None:
model_config = getattr(model, "config", None)
- unsupported = _unsupported_attention(model_config, backend)
+ # DPO / GRPO (allow_hybrids False) keep the Llama-style rule: their decoder hooks are not verified on
+ # the generic path.
+ generic = (
+ allow_hybrids
+ and backend == "ulysses"
+ and not _uses_fast_llama_attention(model)
+ and _generic_hf_attention(model)
+ )
+ unsupported = _unsupported_attention(model_config, backend, generic = generic)
if unsupported is not None:
raise NotImplementedError(
f"Unsloth: {backend} context parallelism does not support {unsupported}."
@@ -1142,10 +1231,21 @@ def _check_cp_model(
raise NotImplementedError(
f"Unsloth: context parallelism on Gated DeltaNet hybrids does not support {blocker}."
)
+ blocker = _hybrid_config_blocker(model) if generic else None
+ if blocker is not None:
+ raise NotImplementedError(
+ f"Unsloth: Ulysses context parallelism does not support {blocker}."
+ )
+ if not allow_hybrids and not _uses_fast_llama_attention(model) and _generic_hf_attention(model):
+ raise NotImplementedError(
+ "Unsloth: DPO / GRPO context parallelism supports Unsloth's Llama-style attention only; generic "
+ "Ulysses on other Hugging Face attention models is verified for SFTTrainer."
+ )
if not _supports_context_parallel(model, backend):
raise NotImplementedError(
- "Unsloth: context parallelism currently supports Llama-style attention "
- "(Llama, Qwen2, Gemma), and Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids with Ulysses."
+ "Unsloth: context parallelism supports Unsloth's Llama-style attention with either backend, "
+ "and with Ulysses any Hugging Face attention model (attention through ALL_ATTENTION_FUNCTIONS) "
+ "and the Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids."
)
@@ -1231,8 +1331,10 @@ def _check_cp_trainer(
def patch_sft_trainer(install_cp_methods: bool = False) -> None:
- # Only __init__ is wrapped up front; the other wrappers go on the class once a trainer enables
- # CP. With CP off they still slowed torch 2.10 LoRA steps by ~35% (gemma-4-E2B, L4 / A100).
+ # Only __init__ is patched on the class. The CP wrappers are bound per trainer (types.MethodType) when it
+ # asks for CP, so trainers without CP, built before or after one with it, run the class's own methods:
+ # class wrappers slowed torch 2.10 LoRA steps by ~35% with CP off (gemma-4-E2B, L4 / A100).
+ # install_cp_methods = True binds them on every trainer (a test seam for trainers given a manager by hand).
import trl
trainer_cls = trl.SFTTrainer
@@ -1306,11 +1408,23 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
peft_config = kwargs.get("peft_config"),
backend = kwargs.get("context_parallel_backend"),
)
+ if requested > 1 or install_cp_methods:
# Before the original __init__: Unsloth wraps instance methods there (auto packing's
# train reset, the token budget's training_step), and they must capture the CP ones.
- install_methods()
- original_init(self, *args, **kwargs)
+ _bind_instance_methods(self, cp_methods)
+ try:
+ original_init(self, *args, **kwargs)
+ except BaseException:
+ _unbind_instance_methods(self)
+ raise
self._context_parallel_manager = None
+ try:
+ _cp_init_tail(self)
+ except BaseException:
+ _unbind_instance_methods(self)
+ raise
+
+ def _cp_init_tail(self):
# Unsloth's attention ignores transformers' own CP: it would stay local to each shard.
# `accelerate launch` with parallelism_config_cp_size builds it from the environment
# (ACCELERATE_USE_PARALLELISM_CONFIG), leaving args.parallelism_config None.
@@ -1342,10 +1456,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
self.args.context_parallel_backend = backend
if size <= 1:
_install_accelerator_state(self)
- if not trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):
- # By identity: a subclass may define __eq__ without __hash__.
- pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]
- pre_cp_trainers.append(weakref.ref(self))
+ if not install_cp_methods:
+ _unbind_instance_methods(self)
return
_check_cp_runtime(self, size, backend)
_refuse_router_aux_loss_after_init(self, size)
@@ -1390,7 +1502,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
)
_check_native_fsdp2_reduces_over_cp(self.model, manager)
self._context_parallel_manager = manager
- install_methods()
+ if "_unsloth_cp_bound_methods" not in self.__dict__:
+ _bind_instance_methods(self, cp_methods)
manager.attach_attention_hooks(self.model)
_install_accelerator_state(self)
# The ring calls the flash ops FA3 / FA4 override, so scope them to train() here too.
@@ -1409,6 +1522,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
else "."
)
)
+ # Installed again by each CP entry point (_cp_state_scope) while it runs.
+ _release_accelerator_state(getattr(getattr(self, "accelerator", None), "state", None))
@functools.wraps(original_prediction_step)
def patched_prediction_step(self, model, inputs, *args, **kwargs):
@@ -1438,8 +1553,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
def patched_train(self, *args, **kwargs):
if getattr(self, "_context_parallel_manager", None) is not None:
_refuse_iterable_datasets(getattr(self, "train_dataset", None))
- _install_accelerator_state(self)
- return original_train(self, *args, **kwargs)
+ with _cp_state_scope(self):
+ return original_train(self, *args, **kwargs)
@functools.wraps(original_evaluate)
def patched_evaluate(self, *args, **kwargs):
@@ -1453,18 +1568,22 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
else current
)
_refuse_iterable_datasets(eval_dataset)
- _install_accelerator_state(self)
- return original_evaluate(self, *args, **kwargs)
+ with _cp_state_scope(self):
+ return original_evaluate(self, *args, **kwargs)
@functools.wraps(original_predict)
def patched_predict(self, test_dataset, *args, **kwargs):
if getattr(self, "_context_parallel_manager", None) is not None:
_refuse_iterable_datasets(test_dataset)
- _install_accelerator_state(self)
- return original_predict(self, test_dataset, *args, **kwargs)
+ with _cp_state_scope(self):
+ return original_predict(self, test_dataset, *args, **kwargs)
@functools.wraps(original_save_model)
def patched_save_model(self, *args, **kwargs):
+ with _cp_state_scope(self):
+ return _save_model(self, *args, **kwargs)
+
+ def _save_model(self, *args, **kwargs):
state = getattr(getattr(self, "accelerator", None), "state", None)
installed = _INSTALLED_PARALLELISM_CONFIG[1] if _INSTALLED_PARALLELISM_CONFIG else None
if installed is None or getattr(state, "parallelism_config", None) is not installed:
@@ -1524,46 +1643,21 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
total = total // manager.size
return total
- # Non-CP trainers built before any trainer enabled CP: their __init__ may have wrapped train /
- # evaluate / predict on the instance around the plain methods (auto packing's train reset).
- pre_cp_trainers = [] # weakref.ref per trainer
-
- def _restoring(trainer, bound):
- @functools.wraps(bound)
- def call(*args, **kwargs):
- # What patched_train / evaluate / predict do for a non-CP trainer: drop the mesh and
- # parallelism config a CP trainer installed on the shared accelerator state.
- _install_accelerator_state(trainer)
- return bound(*args, **kwargs)
-
- return call
-
- def install_methods():
- if trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):
- return
- for trainer in [ref() for ref in pre_cp_trainers]:
- if trainer is None:
- continue
- for name in ("train", "evaluate", "predict"):
- bound = trainer.__dict__.get(name)
- if bound is not None:
- trainer.__dict__[name] = _restoring(trainer, bound)
- pre_cp_trainers.clear()
- trainer_cls.prediction_step = patched_prediction_step
- trainer_cls.train = patched_train
- trainer_cls.evaluate = patched_evaluate
- trainer_cls.predict = patched_predict
- trainer_cls.evaluation_loop = patched_evaluation_loop
- trainer_cls.save_model = patched_save_model
- trainer_cls.training_step = patched_training_step
- if has_total_train_batch_size:
- trainer_cls.get_total_train_batch_size = patched_get_total_train_batch_size
- trainer_cls.__unsloth_context_parallel_methods__ = True
+ cp_methods = {
+ "prediction_step": patched_prediction_step,
+ "train": patched_train,
+ "evaluate": patched_evaluate,
+ "predict": patched_predict,
+ "evaluation_loop": patched_evaluation_loop,
+ "save_model": patched_save_model,
+ "training_step": patched_training_step,
+ }
+ if has_total_train_batch_size:
+ cp_methods["get_total_train_batch_size"] = patched_get_total_train_batch_size
trainer_cls.__init__ = patched_init
+ trainer_cls._unsloth_cp_methods = cp_methods
trainer_cls.__unsloth_context_parallel__ = True
- if install_cp_methods:
- install_methods()
# Backends with single-tensor collectives; others (MPI) keep the list path.
@@ -1982,9 +2076,20 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:
backend = kwargs.get("context_parallel_backend"),
)
# Before the original __init__, so instance wrappers made there capture the CP methods.
- install_methods()
- original_init(self, *args, **kwargs)
+ _bind_instance_methods(self, cp_methods)
+ try:
+ original_init(self, *args, **kwargs)
+ except BaseException:
+ _unbind_instance_methods(self)
+ raise
self._context_parallel_manager = None
+ try:
+ _cp_init_tail(self)
+ except BaseException:
+ _unbind_instance_methods(self)
+ raise
+
+ def _cp_init_tail(self):
_refuse_transformers_cp(
self,
"DPOConfig" if kind == "dpo_trainer" else "GRPOConfig",
@@ -1994,10 +2099,7 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:
backend = resolve_cp_backend(backend, size, rl = True)
if size <= 1:
_install_accelerator_state(self)
- if not trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):
- # The generated trainers bind self.train in __init__ (rl.py), around the plain method.
- pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]
- pre_cp_trainers.append(weakref.ref(self))
+ _unbind_instance_methods(self)
return
if backend != "ulysses":
raise NotImplementedError(
@@ -2010,6 +2112,8 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:
_check_cp_trainer(self, size, backend, [self.model, ref_model])
manager = ContextParallelManager(size, backend)
self._context_parallel_manager = manager
+ if "_unsloth_cp_bound_methods" not in self.__dict__:
+ _bind_instance_methods(self, cp_methods)
attach_rl_hooks(manager, self.model)
if kind == "grpo_trainer":
# GRPO's packed and shared-prefix forwards pick their path per step from self-checks;
@@ -2021,7 +2125,6 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:
module._unsloth_prefix_grouper_nograd_disabled = True
if isinstance(ref_model, torch.nn.Module):
attach_rl_hooks(manager, ref_model)
- _install_accelerator_state(self)
print(f"Unsloth: Context parallelism enabled with size = {size} (ulysses, {kind}).")
def _step_wrapper(name):
@@ -2089,8 +2192,8 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:
def patched_train(self, *args, **kwargs):
if getattr(self, "_context_parallel_manager", None) is not None:
_refuse_iterable_datasets(getattr(self, "train_dataset", None))
- _install_accelerator_state(self)
- return originals["train"](self, *args, **kwargs)
+ with _cp_state_scope(self):
+ return originals["train"](self, *args, **kwargs)
@functools.wraps(originals["evaluate"])
def patched_evaluate(self, *args, **kwargs):
@@ -2104,14 +2207,14 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:
else current
)
_check_rl_eval_entry(self, kind, dataset)
- _install_accelerator_state(self)
- return originals["evaluate"](self, *args, **kwargs)
+ with _cp_state_scope(self):
+ return originals["evaluate"](self, *args, **kwargs)
def patched_predict(self, test_dataset, *args, **kwargs):
if getattr(self, "_context_parallel_manager", None) is not None:
_check_rl_eval_entry(self, kind, test_dataset)
- _install_accelerator_state(self)
- return originals["predict"](self, test_dataset, *args, **kwargs)
+ with _cp_state_scope(self):
+ return originals["predict"](self, test_dataset, *args, **kwargs)
@functools.wraps(originals["evaluation_loop"])
def patched_evaluation_loop(self, *args, **kwargs):
@@ -2120,49 +2223,28 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:
self.gather_function = _cp_gather_for_metrics(self.accelerator, manager.size)
return originals["evaluation_loop"](self, *args, **kwargs)
- # Non-CP trainers built before CP was enabled: like SFT, repair what their __init__ bound on the
- # instance, so they still drop the mesh a CP trainer installs on the shared AcceleratorState.
- pre_cp_trainers = [] # weakref.ref per trainer
-
- def _restoring(trainer, bound):
- @functools.wraps(bound)
- def call(*args, **kwargs):
- _install_accelerator_state(trainer)
- return bound(*args, **kwargs)
-
- return call
-
- def install_methods():
- # Only __init__ is wrapped up front (as for SFT): the wrappers below go on the class once a
- # trainer enables CP, so trainers without CP keep their original methods.
- if trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):
- return
- for trainer in [ref() for ref in pre_cp_trainers]:
- if trainer is None:
- continue
- for name in ("train", "evaluate", "predict"):
- bound = trainer.__dict__.get(name)
- if bound is not None:
- trainer.__dict__[name] = _restoring(trainer, bound)
- pre_cp_trainers.clear()
- trainer_cls.training_step = _step_wrapper("training_step")
- trainer_cls.prediction_step = _step_wrapper("prediction_step")
- trainer_cls.train = patched_train
- trainer_cls.evaluate = patched_evaluate
- if originals["predict"] is not None:
- trainer_cls.predict = functools.wraps(originals["predict"])(patched_predict)
- trainer_cls.evaluation_loop = patched_evaluation_loop
- if kind == "grpo_trainer":
- trainer_cls.compute_loss = _scoped("compute_loss", sticky = True)
- for name in ("_get_per_token_logps", "_get_per_token_logps_and_entropies"):
- if originals[name] is not None:
- setattr(trainer_cls, name, _scoped(name))
- if originals["_generate"] is not None:
- trainer_cls._generate = functools.wraps(originals["_generate"])(patched_generate)
- trainer_cls._prepare_inputs = patched_prepare_inputs
- if originals["_get_train_sampler"] is not None:
- trainer_cls._get_train_sampler = patched_train_sampler
- trainer_cls.__unsloth_context_parallel_methods__ = True
-
+ # Bound per trainer that enables CP (types.MethodType, as for SFT): trainers without CP keep the
+ # class's own methods.
+ cp_methods = {
+ "training_step": _step_wrapper("training_step"),
+ "prediction_step": _step_wrapper("prediction_step"),
+ "train": patched_train,
+ "evaluate": patched_evaluate,
+ "evaluation_loop": patched_evaluation_loop,
+ }
+ if originals["predict"] is not None:
+ cp_methods["predict"] = functools.wraps(originals["predict"])(patched_predict)
+ if kind == "grpo_trainer":
+ cp_methods["compute_loss"] = _scoped("compute_loss", sticky = True)
+ for name in ("_get_per_token_logps", "_get_per_token_logps_and_entropies"):
+ if originals[name] is not None:
+ cp_methods[name] = _scoped(name)
+ if originals["_generate"] is not None:
+ cp_methods["_generate"] = functools.wraps(originals["_generate"])(patched_generate)
+ cp_methods["_prepare_inputs"] = patched_prepare_inputs
+ if originals["_get_train_sampler"] is not None:
+ cp_methods["_get_train_sampler"] = patched_train_sampler
+
+ trainer_cls._unsloth_cp_methods = cp_methods
trainer_cls.__init__ = patched_init
trainer_cls.__unsloth_context_parallel__ = True
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index ff6b617964a..e3df9074d52 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -1145,6 +1145,46 @@ def _prepare_gdn(layer, hidden_states):
_GDN_OP[0] = False
+# PEFT's ParamWrapper (target_parameters on a non-expert weight, e.g. a MoE router) registers a torch
+# parametrization on every forward and removes it after, and torch builds a new Parametrized<Class>
+# subclass per registration. Dynamo guards a module's type id, so each call recompiles the module's
+# forward whenever the fresh class lands at an address it has not seen (how many do depends on the
+# allocator: a different code path shifted the count). One class per module instance keeps the id.
+_PARAMETRIZED_CLASSES = weakref.WeakKeyDictionary()
+
+
+def _stable_parametrized_classes():
+ try:
+ from torch.nn.utils import parametrize
+ except Exception:
+ return
+ original = getattr(parametrize, "_inject_new_class", None)
+ if original is None or getattr(original, "_unsloth_stable", False):
+ return
+
+ # Eager: the class swap is the graph break PEFT's registration already takes, and Dynamo must
+ # not fold the cache lookup into a graph.
+ @torch.compiler.disable
+ @functools.wraps(original)
+ def _inject_new_class(module):
+ cls = module.__class__
+ try:
+ cached = _PARAMETRIZED_CLASSES.get(module)
+ except TypeError: # not weak-referenceable / hashable
+ return original(module)
+ if cached is not None and cached.__bases__ == (cls,):
+ module.__class__ = cached
+ return
+ original(module)
+ try:
+ _PARAMETRIZED_CLASSES[module] = module.__class__
+ except TypeError:
+ pass
+
+ _inject_new_class._unsloth_stable = True
+ parametrize._inject_new_class = _inject_new_class
+
+
# zoo's Gemma-4 sliding router scans the mask (data dependent) and declines while compiling, so
# the block runs the scan eagerly and a router wrapper reads the verdict stored on the mask.
_BAND_MODULE = "unsloth_zoo.temporary_patches.gemma4_banded_attention"
@@ -1956,6 +1996,7 @@ def patch_block_compile(model):
_rebind_class(cls)
_flex_split_hook()
_gemma4_router_hook()
+ _stable_parametrized_classes()
if any(type(m).__name__ == "Linear4bit" for m in model.modules()):
_nf4_in_blocks()
kinds = {}
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c1ec368f2ea..ff1161bc73e 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -6635,6 +6635,59 @@ def _prebuilt_optimizer(trainer, init, args, kwargs):
return None
+def _cp_trainer_refusal(trainer_cls, cp) -> str:
+ """Why a trainer class carries no context parallel wrapper under UNSLOTH_CP_SIZE, and what to do."""
+ import sys
+
+ name = trainer_cls.__name__
+ bases = {c.__name__.removeprefix("Unsloth").removeprefix("_CPU") for c in trainer_cls.__mro__}
+ kind = next((k for k in ("SFTTrainer", "DPOTrainer", "GRPOTrainer") if k in bases), None)
+ head = (
+ f"Unsloth: cp x trainer: context parallel (UNSLOTH_CP_SIZE={cp}) was requested, but {name} "
+ )
+ if sys.modules.get("unsloth._cpu_init") is not None or getattr(
+ trainer_cls, "_unsloth_cpu", False
+ ):
+ return (
+ head
+ + "runs on Unsloth's CPU backend (no GPU was found), which uses TRL's own trainers and does not "
+ "wire context parallel; every rank would train on the whole sequence. Instead: launch on GPUs "
+ "(torchrun with CUDA devices visible), or drop cp."
+ )
+ if kind is None:
+ return (
+ head
+ + "is not one of the trainers context parallel hooks into (SFTTrainer, DPOTrainer, GRPOTrainer); "
+ "every rank would train on the whole sequence. Instead: use SFTTrainer (it takes a plain Trainer "
+ "script's arguments), or drop cp."
+ )
+ if os.environ.get("UNSLOTH_ALLOW_CPU", "0") == "1":
+ reason = "UNSLOTH_ALLOW_CPU=1 skips Unsloth's trainer patches"
+ fix = "unset UNSLOTH_ALLOW_CPU"
+ else:
+ try:
+ from .rl import PATCHED_TRAINER_BUILD_ERRORS
+ except Exception:
+ PATCHED_TRAINER_BUILD_ERRORS = {}
+ file = {
+ "SFTTrainer": "sft_trainer",
+ "DPOTrainer": "dpo_trainer",
+ "GRPOTrainer": "grpo_trainer",
+ }[kind]
+ built = PATCHED_TRAINER_BUILD_ERRORS.get(file)
+ reason = (
+ f"Unsloth's patched {kind} could not be built ({built})"
+ if built
+ else f"Unsloth's context parallel wrapper was not installed on {name} (import unsloth before trl)"
+ )
+ fix = "import unsloth before trl / transformers, and check the warning above for the build failure"
+ return (
+ head
+ + f"has no context parallel wrapper: {reason}; every rank would train on the whole sequence. "
+ f"Instead: {fix}, or drop cp."
+ )
+
+
def _optimizer_kwargs_hold_params(trainer, init, args, kwargs):
"""Trainer(optimizer_cls_and_kwargs = (cls, {"params": ...})) carries parameters chosen before __init__."""
value = kwargs.get("optimizer_cls_and_kwargs")
@@ -6679,11 +6732,7 @@ def _parallel_capabilities_pre(trainer, init, args, kwargs, model):
cp = env_cp_size()
if cp > 1 and not getattr(type(trainer), "__unsloth_context_parallel__", False):
- raise NotImplementedError(
- f"Unsloth: cp x trainer: context parallel (UNSLOTH_CP_SIZE={cp}) runs through SFTTrainer, "
- f"DPOTrainer and GRPOTrainer; {type(trainer).__name__} would train every rank on the whole sequence. "
- "Instead: use SFTTrainer (a plain Trainer script: SFTTrainer takes the same arguments), or drop cp."
- )
+ raise NotImplementedError(_cp_trainer_refusal(type(trainer), cp))
from .tp_env import env_layout
layout = env_layout()
diff --git a/unsloth/models/fsdp2.py b/unsloth/models/fsdp2.py
index 3d088aad4fd..ea03997a5f8 100644
--- a/unsloth/models/fsdp2.py
+++ b/unsloth/models/fsdp2.py
@@ -452,6 +452,79 @@ def install_fsdp2_prepare_hook():
setattr(fsdp2_prepare_model, _PATCHED, True)
accelerator_module.fsdp2_prepare_model = fsdp2_prepare_model
+ install_fsdp2_bnb_optimizer_hook()
+ return True
+
+
+_BNB_ALTERNATIVE = (
+ "Use optim = 'adamw_8bit' / 'paged_adamw_8bit' (bitsandbytes Adam / AdamW step each rank's local "
+ "shard), or optim = 'adamw_torch' / 'adamw_torch_fused'."
+)
+
+
+def _localize_or_refuse_bnb(objects):
+ """accelerate's FSDP2 swaps the optimizer's parameters for DTensors in Accelerator._prepare_fsdp2;
+ bitsandbytes' kernels take plain tensors. A bitsandbytes Adam / AdamW is made to step the local
+ shards (fsdp2_bnb_optim, as native FSDP2 does); any other bitsandbytes optimizer is refused here,
+ before its first step would fail with "got mixed torch.Tensor and DTensor" (unsloth#3551)."""
+ from .fsdp2_native import _inner_optimizer, _is_bnb_optimizer_class, _optimizer_on_cpu
+ for obj in objects:
+ if not isinstance(obj, torch.optim.Optimizer):
+ continue
+ inner = _inner_optimizer(obj)
+ if not _is_bnb_optimizer_class(type(inner)):
+ continue
+ from .fsdp2_bnb_optim import (
+ bnb_local_shards_enabled,
+ bnb_steps_cpu_tensors,
+ localize_bnb_optimizer,
+ )
+
+ name = type(inner).__name__
+ if not bnb_local_shards_enabled():
+ raise ValueError(
+ f"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' {name} cannot step, "
+ "and UNSLOTH_FSDP2_BNB_LOCAL=0 turns off stepping the local shards. Unset it, or use "
+ "optim = 'adamw_torch' / 'adamw_torch_fused'."
+ )
+ if _optimizer_on_cpu(inner) and not bnb_steps_cpu_tensors():
+ raise ValueError(
+ f"Unsloth: FSDP2 keeps the parameters on the CPU (CPU offload), and this bitsandbytes "
+ f"cannot step CPU tensors with {name}. Upgrade bitsandbytes (>= 0.50), turn CPU offload "
+ "off, or use optim = 'adamw_torch'."
+ )
+ if not localize_bnb_optimizer(inner):
+ raise ValueError(
+ f"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' {name} cannot step "
+ f"(only bitsandbytes Adam / AdamW step local shards). {_BNB_ALTERNATIVE}"
+ )
+ _print_rank0(
+ f"Unsloth: FSDP2 keeps the bitsandbytes optimizer ({name}): it steps each rank's local "
+ "shard of the DTensor parameters (8-bit state per shard)."
+ )
+
+
+def install_fsdp2_bnb_optimizer_hook():
+ """Wrap Accelerator._prepare_fsdp2: after it maps the optimizer onto the DTensor parameters,
+ localize (or refuse) a bitsandbytes optimizer."""
+ try:
+ from accelerate import Accelerator
+ except Exception:
+ return False
+ original = getattr(Accelerator, "_prepare_fsdp2", None)
+ if original is None:
+ return False
+ if getattr(original, _PATCHED, False):
+ return True
+
+ @functools.wraps(original)
+ def _prepare_fsdp2(self, *args, **kwargs):
+ result = original(self, *args, **kwargs)
+ _localize_or_refuse_bnb(result if isinstance(result, (tuple, list)) else (result,))
+ return result
+
+ setattr(_prepare_fsdp2, _PATCHED, True)
+ Accelerator._prepare_fsdp2 = _prepare_fsdp2
return True
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index a08aa0e9d7d..1fc2da89a79 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2433,11 +2433,16 @@ def _patch_ppo_policy_value_wrapper(module):
wrapper.gradient_checkpointing_disable = gradient_checkpointing_disable
+# trainer_file -> "<ExceptionType>: <message>" of the last failed build, so a later refusal can name the cause.
+PATCHED_TRAINER_BUILD_ERRORS = {}
+
+
def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
# Defensive wrapper matching patch_trl_rl_trainers()'s try/except, so direct callers do not see exceptions from the impl on TRL versions that rename or move classes.
try:
return _patch_trl_rl_trainers_impl(trainer_file)
except Exception as e:
+ PATCHED_TRAINER_BUILD_ERRORS[trainer_file] = f"{type(e).__name__}: {e}"
# Warning, not info: the impl RETURNS for the benign case, so reaching here means generation failed and the run silently falls back to trl's trainer, losing Unsloth's compute_loss, bf16/fp16 fixup and dataset handling.
logger.warning_once(
f"Unsloth: Could not build the patched trl.trainer.{trainer_file}, "
@@ -2532,6 +2537,17 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
f"Unsloth: Could not load {RLTrainer_name} from trl.trainer.{trainer_file}: {e}"
)
return
+ if getattr(RLTrainer, "_unsloth_cpu", False):
+ # Unsloth's CPU backend (no accelerator found) runs TRL's own trainer behind unsloth_zoo.cpu's defaults
+ # subclass by design; that subclass has no readable source, so building from it would only fail.
+ PATCHED_TRAINER_BUILD_ERRORS[trainer_file] = (
+ "the CPU backend keeps TRL's own trainer (no GPU was found)"
+ )
+ logger.info(
+ f"Unsloth: CPU backend: trl.trainer.{trainer_file} keeps TRL's own {RLTrainer_name} "
+ "with unsloth_zoo.cpu's defaults."
+ )
+ return
_config_resolved_module = None
try:
RLConfig = eval(f"trl.trainer.{trainer_file}.{RLConfig_name}")
@@ -3295,7 +3311,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
if "model" in call_args:
training_check = (
"if hasattr(self, 'train'):\n"
- " self.train = MethodType(prepare_for_training_mode(self.__class__.train), self)\n"
+ # A context parallel trainer binds its train wrapper on the instance (context_parallel.py).
+ " self.train = MethodType(prepare_for_training_mode(getattr(self.__dict__.get('train'), '__func__', None) or self.__class__.train), self)\n"
"pass\n"
)
RLTrainer_post += training_check
@@ -3475,35 +3492,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += learning_rate_check
- # bitsandbytes' 8-bit optimizers step plain tensors, but FSDP2 shards parameters as DTensors, so the
- # first step dies with "got mixed torch.Tensor and DTensor" (unsloth#3551).
+ # optim = 'adamw_8bit' is kept under FSDP2: native FSDP2 (fsdp2_native._swap_bnb_optimizer) and
+ # accelerate's FSDP2 (fsdp2._localize_or_refuse_bnb after Accelerator._prepare_fsdp2) step bitsandbytes
+ # Adam / AdamW on each rank's local shard, and refuse the bitsandbytes optimizers that cannot (unsloth#3551).
if "optim" in call_args:
- extra_args += "_unsloth_fsdp2 = os.environ.get('FSDP_VERSION', '').strip() == '2'\n"
- if "fsdp" in call_args and "fsdp_config" in call_args:
- # TrainingArguments(fsdp = ..., fsdp_config = {"fsdp_version": 2} or a JSON path) without the launcher.
- extra_args += (
- "_unsloth_fsdp_cfg = fsdp_config\n"
- "if fsdp and isinstance(_unsloth_fsdp_cfg, str) and os.path.isfile(_unsloth_fsdp_cfg):\n"
- " try:\n"
- " import json as _unsloth_json\n"
- " with open(_unsloth_fsdp_cfg, encoding = 'utf-8') as _unsloth_f:\n"
- " _unsloth_fsdp_cfg = _unsloth_json.load(_unsloth_f)\n"
- " except Exception:\n"
- " _unsloth_fsdp_cfg = None\n"
- "if fsdp and _unsloth_fsdp_cfg is None:\n"
- " _unsloth_fsdp_cfg = {}\n"
- "if fsdp and isinstance(_unsloth_fsdp_cfg, dict):\n"
- f" _unsloth_fsdp2 = _unsloth_fsdp2 or str(_unsloth_fsdp_cfg.get('fsdp_version', _unsloth_fsdp_cfg.get('version', {_FSDP_DEFAULT_VERSION}))) == '2'\n"
- )
extra_args += (
"_unsloth_flash_adamw = str(getattr(optim, 'value', optim)).strip().lower() == 'adamw_flash'\n"
"if _unsloth_flash_adamw: optim = 'adamw_torch'\n"
)
- extra_args += (
- "if _unsloth_fsdp2 and str(getattr(optim, 'value', optim)) == 'adamw_8bit':\n"
- " print(\"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' 8-bit optimizers cannot step. Switching optim = 'adamw_8bit' to 'adamw_torch_fused'.\")\n"
- " optim = 'adamw_torch_fused'\n"
- )
# Fix num_train_epochs = None causing a TypeError in Trainer.__init__, which does `args.num_train_epochs > 0`.
if "num_train_epochs" in call_args:
diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.py
index 30a19625172..a2120010b63 100644
--- a/unsloth/parallel_capabilities.py
+++ b/unsloth/parallel_capabilities.py
@@ -180,10 +180,10 @@ RULES = (
),
Rule(
"cp x attention",
- "context parallel on sliding window / chunked / softcapped attention",
- lambda q: q.cp > 1 and q.windowed,
- "Unsloth's context parallel attention drops the window / chunk / softcap",
- "drop cp for this model",
+ "ring context parallel on sliding window / chunked / softcapped attention",
+ lambda q: q.cp > 1 and q.windowed and q.cp_backend == "ring",
+ "ring attention drops the window / chunk / softcap (Ulysses runs the model's own attention function)",
+ "cp_backend=ulysses (or auto), or drop cp for this model",
),
Rule(
"cp x torch",
@@ -291,8 +291,10 @@ HOOKS = (
"from_pretrained",
"get_peft_model",
"Trainer.__init__",
- "Trainer.training_step",
- "Trainer.prediction_step",
+ # Lower case: bound on one trainer instance (types.MethodType), never on the class. pp_env and the CP
+ # managers both bind these, so two of them on one trainer still cannot compose.
+ "trainer.training_step",
+ "trainer.prediction_step",
"Trainer.save_model",
"Accelerator.prepare_model",
"Accelerator.prepare_data_loader",
@@ -382,7 +384,7 @@ FEATURES = (
"Accelerator.clip_grad_norm_",
"Trainer.save_model",
),
- exclusive = ("Trainer.training_step", "Trainer.prediction_step"),
+ exclusive = ("trainer.training_step", "trainer.prediction_step"),
),
Feature(
"cp.ulysses",
@@ -394,8 +396,7 @@ FEATURES = (
uses = ("packed_seq_lengths",),
produces = ("seq_shard", "shift_labels", "position_ids"),
hooks = ("Trainer.__init__", "attention_function"),
- exclusive = ("Trainer.training_step", "Trainer.prediction_step"),
- scope = "class",
+ exclusive = ("trainer.training_step", "trainer.prediction_step"),
),
Feature(
"cp.ring",
@@ -406,8 +407,7 @@ FEATURES = (
consumes = ("input_ids", "labels", "position_ids", "num_items_in_batch"),
produces = ("seq_shard", "shift_labels", "cp_permutation", "position_ids"),
hooks = ("Trainer.__init__", "attention_function"),
- exclusive = ("Trainer.training_step", "Trainer.prediction_step"),
- scope = "class",
+ exclusive = ("trainer.training_step", "trainer.prediction_step"),
),
# Precision.
Feature(
diff --git a/unsloth/utils/ulysses_hybrid.py b/unsloth/utils/ulysses_hybrid.py
index e742c05bb6f..fd256e2c6a2 100644
--- a/unsloth/utils/ulysses_hybrid.py
+++ b/unsloth/utils/ulysses_hybrid.py
@@ -1,7 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""Ulysses context parallelism for hybrid models: scans need the full sequence, so they run on heads."""
+"""Ulysses context parallelism for Hugging Face attention (any model whose attention calls ALL_ATTENTION_FUNCTIONS
+runs its own attention function after the exchange) and for hybrid models, whose scans need the full sequence so
+they run on heads."""
import functools
import inspect
@@ -10,7 +12,7 @@ import torch
import torch.distributed as dist
import torch.nn.functional as F
-from .ulysses import _AllToAll, ulysses_attention
+from .ulysses import _AllToAll
__all__ = [
"GATED_DELTA_ATTENTION_CLASSES",
@@ -19,6 +21,8 @@ __all__ = [
"ulysses_gated_delta_forward",
"wrap_gated_delta_forward",
"install_hf_attention_wrapper",
+ "is_hf_registry_attention",
+ "hf_attention_forward",
]
# Hybrid full attention layers: Hugging Face forwards that call ALL_ATTENTION_FUNCTIONS after RoPE.
@@ -226,63 +230,215 @@ def wrap_gated_delta_forward(module, get_cp_manager) -> None:
module.forward = wrapped
-def _ulysses_hf_attention(module, query, key, value, attention_mask, *, group, **kwargs):
- from .attention_dispatch import (
- AttentionConfig,
- AttentionContext,
- _run_attention,
- select_attention_backend,
+# Keyword arguments that describe the local sequence shard: rebuilt for the whole sequence after the exchange.
+_SHARD_LOCAL_KWARGS = (
+ "position_ids",
+ "cu_seq_lens_q",
+ "cu_seq_lens_k",
+ "max_length_q",
+ "max_length_k",
+ "packed_seq_lengths",
+)
+
+
+def _calls_attention_registry(function) -> bool:
+ try:
+ function = inspect.unwrap(function)
+ except Exception:
+ return False
+ code = getattr(function, "__code__", None)
+ return code is not None and "ALL_ATTENTION_FUNCTIONS" in code.co_names
+
+
+def hf_attention_forward(module):
+ """The Hugging Face forward of this attention class that calls ALL_ATTENTION_FUNCTIONS (eager falls back to
+ the modeling module's eager_attention_forward): the class forward, or the original unsloth_zoo stashed when
+ it patched the class (`_original_<module>_<Class>_forward`, e.g. Gemma3's fp32 SDPA forward). None for
+ Unsloth's fast forwards and custom kernels."""
+ cls = type(module)
+ forward = getattr(cls, "forward", None)
+ if _calls_attention_registry(forward):
+ return forward
+ for klass in cls.__mro__:
+ for name, value in vars(klass).items():
+ if (
+ name.startswith("_original_")
+ and name.endswith(f"_{klass.__name__}_forward")
+ and _calls_attention_registry(value)
+ ):
+ return value
+ return None
+
+
+def is_hf_registry_attention(module) -> bool:
+ return hf_attention_forward(module) is not None
+
+
+def _forward_globals(module) -> dict:
+ return getattr(inspect.unwrap(hf_attention_forward(module)), "__globals__", {})
+
+
+def _route_hf_forward(module, get_cp_manager) -> None:
+ """When the class forward is unsloth_zoo's replacement, run the stashed Hugging Face forward on this module
+ while a Ulysses manager is active (so attention goes through the registry), and the class forward otherwise."""
+ hf_forward = hf_attention_forward(module)
+ if hf_forward is None or hf_forward is getattr(type(module), "forward", None):
+ return
+ if "_unsloth_cp_hf_forward" in vars(module):
+ return
+ import types
+
+ default = module.forward
+ hf_bound = types.MethodType(hf_forward, module)
+
+ @functools.wraps(default)
+ def forward(*args, **kwargs):
+ manager = get_cp_manager()
+ if manager is not None and manager.backend == "ulysses":
+ return hf_bound(*args, **kwargs)
+ return default(*args, **kwargs)
+
+ module._unsloth_cp_hf_forward = hf_bound
+ module.forward = forward
+
+
+class _HeadShard:
+ """The attention module as its own attention function sees it on a head shard: the local key / value
+ group count and the local heads' sinks; everything else is the module's."""
+
+ # Plain attribute access only: torch.compile cannot trace object.__getattribute__ / object.__setattr__.
+ def __init__(self, module, **overrides):
+ self._unsloth_module = module
+ self.__dict__.update(overrides)
+
+ def __getattr__(self, name):
+ # Only called for names the shard does not hold itself.
+ if name == "_unsloth_module":
+ raise AttributeError(name)
+ return getattr(self._unsloth_module, name)
+
+
+def _sliding_window(module, kwargs):
+ window = (
+ kwargs["sliding_window"]
+ if "sliding_window" in kwargs
+ else getattr(module, "sliding_window", None)
)
- from .packing import get_packed_info_from_kwargs
+ return (
+ window if isinstance(window, int) and not isinstance(window, bool) and window > 0 else None
+ )
+
- bsz, n_heads, q_len, head_dim = query.shape
- scaling = kwargs.get("scaling")
- if scaling is not None and abs(scaling * head_dim**0.5 - 1) > 1e-6:
+def _refuse_non_causal(module, is_causal = None) -> None:
+ # The rebuilt whole-sequence mask is causal, and bidirectional windows differ per model (Gemma3 `<`, the
+ # generic overlay `<=`), so non-causal attention would silently train causal.
+ if is_causal is None:
+ is_causal = getattr(module, "is_causal", True)
+ if is_causal is False:
raise NotImplementedError(
- "Unsloth: Ulysses context parallelism needs the default attention scale 1/sqrt(head_dim)."
+ f"Unsloth: Ulysses context parallelism only supports causal attention, but {type(module).__name__} "
+ "is bidirectional (for example use_bidirectional_attention = True). Turn off context parallelism "
+ "for this model."
)
+
+
+def _whole_sequence_mask(total, device, dtype, window, cu_seqlens):
+ """Additive [1, 1, T, T] mask over the gathered sequence: causal, the sliding window (key > query - window,
+ as transformers' masks), and packed documents closed off from one another."""
+ idx = torch.arange(total, device = device)
+ allowed = idx[None, :] <= idx[:, None]
+ if window is not None:
+ allowed = allowed & (idx[:, None] - idx[None, :] < window)
+ if cu_seqlens is not None:
+ doc = torch.bucketize(idx, cu_seqlens[1:].to(device = device, dtype = idx.dtype), right = True)
+ allowed = allowed & (doc[:, None] == doc[None, :])
+ mask = torch.zeros(total, total, device = device, dtype = dtype)
+ return mask.masked_fill_(~allowed, torch.finfo(dtype).min)[None, None]
+
+
+def _is_flash(implementation) -> bool:
+ # flash_attention_2 / 3, and hub kernels ("kernels-community/...") that take transformers' flash kwargs.
+ return "flash" in str(implementation) or "/" in str(implementation)
+
+
+def _ulysses_hf_attention(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ *,
+ group,
+ attention_fn,
+ implementation,
+ packed_seq_lengths = None,
+ **kwargs,
+):
+ """Ulysses around the model's own attention function: q / k / v go from sequence shards to head shards,
+ the function runs over the whole sequence with its own scaling, softcap, sliding window and the local
+ heads' sinks, and the output goes back to sequence shards."""
+ from .packing import get_packed_info_from_kwargs
+ from .ulysses import head_to_seq, seq_to_head, ulysses_head_plan
+
if module.training and kwargs.get("dropout"):
raise NotImplementedError(
"Unsloth: Ulysses context parallelism does not support attention dropout."
)
- seq_info = get_packed_info_from_kwargs(kwargs, query.device)
- n_kv_heads = key.shape[1]
- config = AttentionConfig(
- backend = select_attention_backend(use_varlen = seq_info is not None),
- n_kv_heads = n_kv_heads,
- n_groups = n_heads // n_kv_heads,
- flash_dense_kwargs = {"causal": True},
- flash_varlen_kwargs = {"dropout_p": 0.0, "causal": True},
- )
- context = AttentionContext(
- bsz = bsz,
- q_len = q_len,
- kv_seq_len = key.shape[2],
- n_heads = n_heads,
- head_dim = head_dim,
- requires_grad = query.requires_grad,
- seq_info = seq_info,
- attention_mask = None,
- causal_mask = None,
+ _refuse_non_causal(module, kwargs.get("is_causal"))
+ bsz, n_heads, local_len, _ = query.shape
+ size = dist.get_world_size(group)
+ rank = dist.get_rank(group)
+ plan = ulysses_head_plan(n_heads, key.shape[1], size)
+ if plan.kv_replication > 1:
+ # Consecutive copies keep query head h on kv head h // n_groups after the split.
+ key = key.repeat_interleave(plan.kv_replication, dim = 1)
+ value = value.repeat_interleave(plan.kv_replication, dim = 1)
+ query, key, value = (seq_to_head(x, group) for x in (query, key, value))
+ total = local_len * size
+ heads = slice(rank * plan.local_heads, (rank + 1) * plan.local_heads)
+
+ for name in _SHARD_LOCAL_KWARGS:
+ kwargs.pop(name, None)
+ seq_info = (
+ None
+ if packed_seq_lengths is None
+ else get_packed_info_from_kwargs({"packed_seq_lengths": packed_seq_lengths}, query.device)
)
- return ulysses_attention(_run_attention, config, context, query, key, value, group), None
-
-
-def install_hf_attention_wrapper(module, implementation, get_cp_manager) -> None:
- """Route this module's ALL_ATTENTION_FUNCTIONS[implementation] call through Ulysses while active."""
- from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+ cu_seqlens = None if seq_info is None else seq_info[1]
+ if cu_seqlens is not None and int(cu_seqlens[-1]) < total:
+ # Padding to the cp multiple sits after the last document: its own segment.
+ cu_seqlens = torch.cat([cu_seqlens, cu_seqlens.new_tensor([total])])
+
+ overrides = {"num_key_value_groups": plan.local_groups}
+ sinks = getattr(module, "sinks", None)
+ if isinstance(sinks, torch.Tensor) and sinks.ndim >= 1 and sinks.shape[0] == n_heads:
+ overrides["sinks"] = sinks[heads]
+ s_aux = kwargs.get("s_aux")
+ if isinstance(s_aux, torch.Tensor) and s_aux.ndim >= 1 and s_aux.shape[0] == n_heads:
+ kwargs["s_aux"] = s_aux[heads]
+ shard = _HeadShard(module, **overrides)
+
+ window = _sliding_window(module, kwargs)
+ if _is_flash(implementation):
+ # Flash takes the window and the softcap as arguments, and documents as cu_seqlens.
+ mask = None
+ if cu_seqlens is not None:
+ cu = cu_seqlens.to(device = query.device, dtype = torch.int32)
+ longest = int((cu[1:] - cu[:-1]).max())
+ kwargs.update(
+ cu_seq_lens_q = cu, cu_seq_lens_k = cu, max_length_q = longest, max_length_k = longest
+ )
+ elif implementation == "sdpa" and cu_seqlens is None and window is None:
+ mask = None # sdpa's is_causal path
+ else:
+ mask = _whole_sequence_mask(total, query.device, query.dtype, window, cu_seqlens)
+ out, _ = attention_fn(shard, query, key, value, mask, **kwargs)
+ # [B, T, H / N, D] -> [B, T / N, H, D]
+ out = out.reshape(bsz, total, plan.local_heads, -1)
+ return head_to_seq(out, group), None
- # "eager" is a per-model function some transformers releases call without the registry.
- current = None if implementation == "eager" else ALL_ATTENTION_FUNCTIONS.get(implementation)
- if current is None:
- raise NotImplementedError(
- "Unsloth: Ulysses context parallelism for hybrid models needs attn_implementation "
- f"'sdpa' or a flash attention backend, not {implementation!r}."
- )
- module._unsloth_cp_ulysses_attention = True
- if getattr(current, "_unsloth_cp_ulysses", False):
- return
+def _ulysses_attention_wrapper(current, implementation, get_cp_manager):
@functools.wraps(current)
def attention(module, query, key, value, attention_mask, *args, **kwargs):
manager = get_cp_manager()
@@ -294,9 +450,55 @@ def install_hf_attention_wrapper(module, implementation, get_cp_manager) -> None
return current(module, query, key, value, attention_mask, *args, **kwargs)
if args:
raise TypeError("Unsloth: Ulysses attention expects keyword arguments after the mask.")
+ if kwargs.get("packed_seq_lengths") is None:
+ # Some decoders do not forward packed_seq_lengths to the attention layers.
+ kwargs["packed_seq_lengths"] = getattr(manager, "packed_seq_lengths", None)
return _ulysses_hf_attention(
- module, query, key, value, attention_mask, group = manager.group, **kwargs
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ group = manager.group,
+ attention_fn = current,
+ implementation = implementation,
+ **kwargs,
)
attention._unsloth_cp_ulysses = True
- ALL_ATTENTION_FUNCTIONS[implementation] = attention
+ return attention
+
+
+def install_hf_attention_wrapper(module, implementation, get_cp_manager) -> None:
+ """Route this module's attention function (ALL_ATTENTION_FUNCTIONS[implementation], or the modeling
+ module's eager_attention_forward for "eager") through Ulysses while a Ulysses manager is active."""
+ _refuse_non_causal(module)
+ _route_hf_forward(module, get_cp_manager)
+ if implementation == "eager":
+ globs = _forward_globals(module)
+ current = globs.get("eager_attention_forward")
+ if current is None:
+ raise NotImplementedError(
+ "Unsloth: Ulysses context parallelism found no eager_attention_forward for "
+ f"{type(module).__name__}; use attn_implementation = 'sdpa'."
+ )
+ if not getattr(current, "_unsloth_cp_ulysses", False):
+ globs["eager_attention_forward"] = _ulysses_attention_wrapper(
+ current, implementation, get_cp_manager
+ )
+ module._unsloth_cp_ulysses_attention = True
+ return
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+
+ current = ALL_ATTENTION_FUNCTIONS.get(implementation)
+ if current is None or implementation == "flex_attention":
+ raise NotImplementedError(
+ "Unsloth: Ulysses context parallelism needs attn_implementation 'sdpa', 'eager' or a flash "
+ f"attention backend, not {implementation!r}."
+ )
+ module._unsloth_cp_ulysses_attention = True
+ if getattr(current, "_unsloth_cp_ulysses", False):
+ return
+ ALL_ATTENTION_FUNCTIONS[implementation] = _ulysses_attention_wrapper(
+ current, implementation, get_cp_manager
+ )