Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallelism: generic Ulysses on any Hugging Face attention model, per-instance trainer wrappers (#16558)
Daniel Han7dd74a5342f9
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex 8607413cebe..adc6206dd7e 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -1840,3 +1840,33 @@ def test_graphs_off_compile_gets_the_same_region_limit(monkeypatch):for kwargs in seen:assert kwargs["isolate_recompiles"] is Trueassert kwargs["recompile_limit"] == M._region_recompile_limit(2) == 48+++def test_parametrized_class_is_reused_per_module(monkeypatch):+ """PEFT's ParamWrapper registers a parametrization per forward: each module keeps one+ Parametrized<Class> (a stable type id for Dynamo), and two modules never share one."""+ from torch.nn.utils import parametrize++ monkeypatch.setattr(parametrize, "_inject_new_class", parametrize._inject_new_class)+ bc = _load()+ bc._stable_parametrized_classes()+ bc._stable_parametrized_classes() # idempotent++ class Double(torch.nn.Module):+ def forward(self, w):+ return 2 * w++ a, b = torch.nn.Linear(3, 3), torch.nn.Linear(3, 3)+ seen = []+ for _ in range(3):+ parametrize.register_parametrization(a, "weight", Double())+ seen.append(type(a))+ expected = 2 * a.parametrizations["weight"].original+ assert torch.equal(a.weight, expected)+ parametrize.remove_parametrizations(a, "weight", leave_parametrized = False)+ assert type(a) is torch.nn.Linear and "weight" in a._parameters+ assert seen[0] is seen[1] is seen[2] and seen[0] is not torch.nn.Linear+ parametrize.register_parametrization(b, "weight", Double())+ assert type(b) is not seen[0] and torch.equal(+ b.weight, 2 * b.parametrizations["weight"].original+ )diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.pyindex 0b0be959228..c87ed36a87e 100644--- a/tests/test_context_parallel.py+++ b/tests/test_context_parallel.py@@ -299,7 +299,7 @@ def test_eval_loss_is_the_token_mean_even_with_an_empty_shard(monkeypatch, worldmonkeypatch.setattr(cp, "context_parallel", first_half)monkeypatch.setattr(cp.dist, "get_world_size", lambda: world_size)Trainer = _patched_trainer(monkeypatch)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))manager = _manager(size = 2)manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")trainer._context_parallel_manager = manager@@ -446,19 +446,25 @@ def test_fsdp2_shards_over_dp_shard_cp_and_a_later_trainer_restores_the_config(muser_pc = types.SimpleNamespace(dp_replicate_size = 1, tp_size = 1, sp_size = 1, cp_size = 1)accelerator = _fsdp2_accelerator(user_pc)Trainer, _, built = _gated_trainer(monkeypatch, accelerator = accelerator)- Trainer()+ trainer = Trainer()assert built == [(2, "ulysses", "allgather", "fsdp2")] # the default backend: auto- pc = accelerator.state.parallelism_config- # What accelerate's fsdp2_prepare_model indexes the mesh with; cp stays Unsloth's own.- assert pc.fsdp_dim_names == ["dp_shard_cp"]- assert (pc.dp_shard_size, pc.cp_size, pc.non_data_parallel_size) == (4, 1, 1)- assert accelerator.state.device_mesh is not None- later = _patched_trainer(- monkeypatch,- args = types.SimpleNamespace(context_parallel_size = 1),- accelerator = accelerator,- )- later()+ # Installed only while a CP entry point (train / evaluate / predict / save_model) runs.+ assert accelerator.state.parallelism_config is user_pc+ assert accelerator.state.device_mesh is None+ with cp._cp_state_scope(trainer):+ pc = accelerator.state.parallelism_config+ # What accelerate's fsdp2_prepare_model indexes the mesh with; cp stays Unsloth's own.+ assert pc.fsdp_dim_names == ["dp_shard_cp"]+ assert (pc.dp_shard_size, pc.cp_size, pc.non_data_parallel_size) == (4, 1, 1)+ assert accelerator.state.device_mesh is not None+ later = _patched_trainer(+ monkeypatch,+ args = types.SimpleNamespace(context_parallel_size = 1),+ accelerator = accelerator,+ )+ later()+ assert accelerator.state.parallelism_config is user_pc+ assert accelerator.state.device_mesh is Noneassert accelerator.state.parallelism_config is user_pcassert accelerator.state.device_mesh is None@@ -516,7 +522,7 @@ def test_iterable_eval_dataset_passed_to_evaluate_is_refused(monkeypatch):return iter(())Trainer = _patched_trainer(monkeypatch)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))trainer._context_parallel_manager = _manager()with pytest.raises(NotImplementedError, match = "iterable datasets"):trainer.evaluate(Stream())@@ -532,7 +538,7 @@ def test_iterable_predict_dataset_is_refused(monkeypatch):return iter(())Trainer = _patched_trainer(monkeypatch)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))trainer._context_parallel_manager = _manager()with pytest.raises(NotImplementedError, match = "iterable datasets"):trainer.predict(Stream())@@ -701,15 +707,21 @@ def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):mesh = object()state = types.SimpleNamespace(device_mesh = None)Trainer = _patched_trainer(monkeypatch, args = types.SimpleNamespace(context_parallel_size = 1))- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))trainer.accelerator = types.SimpleNamespace(state = state)trainer.args = types.SimpleNamespace(ddp_find_unused_parameters = None)trainer._context_parallel_manager = types.SimpleNamespace(size = 2, backend = "ring", device_mesh = mesh)- # A later non-CP trainer cleared the shared state; train() must put the CP mesh back.+ # A later non-CP trainer cleared the shared state; train() must put the CP mesh back while it runs.+ seen = []+ release = cp._release_accelerator_state+ monkeypatch.setattr(+ cp, "_release_accelerator_state", lambda s: (seen.append(s.device_mesh), release(s))+ )assert trainer.train() == "trained"- assert state.device_mesh is mesh+ assert seen == [mesh]+ assert state.device_mesh is Noneassert trainer.args.ddp_find_unused_parameters is False@@ -752,7 +764,7 @@ def test_eval_gather_keeps_one_loss_per_cp_group(monkeypatch, remainder):gradient_state = types.SimpleNamespace(end_of_dataloader = True, remainder = remainder),)Trainer = _patched_trainer(monkeypatch)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))trainer.accelerator = acceleratortrainer._context_parallel_manager = _manager(size = 2)out = trainer.evaluation_loop()@@ -766,7 +778,7 @@ def test_replaced_iterable_train_dataset_is_refused_at_train(monkeypatch):return iter(())Trainer = _patched_trainer(monkeypatch)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))trainer._context_parallel_manager = _manager()trainer.train_dataset = Stream()with pytest.raises(NotImplementedError, match = "iterable datasets"):@@ -785,7 +797,7 @@ def test_replaced_iterable_eval_dataset_is_refused_at_evaluate(monkeypatch):return iter(())Trainer = _patched_trainer(monkeypatch)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))trainer._context_parallel_manager = _manager()trainer.eval_dataset = Stream()with pytest.raises(NotImplementedError, match = "iterable datasets"):@@ -882,8 +894,11 @@ _CP_METHODS = ()-def test_cp_methods_are_wrapped_only_once_a_trainer_enables_cp(monkeypatch):- # With CP off the wrappers slowed torch 2.10 LoRA steps by ~35%, so only __init__ is patched.+def test_cp_methods_are_bound_only_on_the_trainer_that_enables_cp(monkeypatch):+ import inspect++ # With CP off class wrappers slowed torch 2.10 LoRA steps by ~35%, so only __init__ is patched on the+ # class and the CP wrappers are bound per trainer that enables CP.import typesTrainer = _patched_trainer(monkeypatch, install_cp_methods = False)@@ -892,14 +907,16 @@ def test_cp_methods_are_wrapped_only_once_a_trainer_enables_cp(monkeypatch):trainer = Trainer(args = types.SimpleNamespace(context_parallel_size = 1))assert trainer._context_parallel_manager is Noneassert trainer.train() == "trained"- assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)+ assert not any(name in vars(trainer) for name in _CP_METHODS)Trainer, _, built = _gated_trainer(monkeypatch, install_cp_methods = False)plain = {name: Trainer.__dict__[name] for name in _CP_METHODS}- Trainer()+ cp_trainer = Trainer()assert built == [(2, "ulysses", "allgather")] # the default backend: autofor name in _CP_METHODS:- assert Trainer.__dict__[name].__wrapped__ is plain[name]+ assert Trainer.__dict__[name] is plain[name]+ # Bound per instance (train is wrapped again by the flash attention scope).+ assert inspect.unwrap(vars(cp_trainer)[name]) is plain[name]@pytest.mark.parametrize(@@ -1030,7 +1047,7 @@ def test_flash_attention_impl_is_scoped_to_train_without_packing(monkeypatch):def test_training_step_refuses_a_missing_token_count_with_labels(monkeypatch):monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))Trainer = _patched_trainer(monkeypatch)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))manager = _manager()manager.mesh = Nonetrainer._context_parallel_manager = manager@@ -1333,7 +1350,7 @@ def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):monkeypatch.setattr(trl, "SFTTrainer", Trainer)cp.patch_sft_trainer(install_cp_methods = True)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))pc = types.SimpleNamespace(cp_size = 1, sp_size = 1, tp_size = 1) if fsdp2 else Nonetrainer.accelerator = types.SimpleNamespace(parallelism_config = pc)trainer.model = types.SimpleNamespace()@@ -1356,9 +1373,9 @@ def test_instance_wrappers_made_in_init_capture_the_cp_methods(monkeypatch):def on_init(self):for name in ("train", "training_step"):- method = getattr(type(self), name)+ method = getattr(self, name)captured[name] = method- setattr(self, name, functools.partial(method, self))+ setattr(self, name, functools.partial(method))import types@@ -1379,7 +1396,9 @@ def test_instance_wrappers_made_in_init_capture_the_cp_methods(monkeypatch):) # SFTTrainer(model, args = SFTConfig(context_parallel_size = 2))assert built == [(2, "ulysses", "allgather")] # the default backend: autofor name, method in captured.items():- assert method is not plain[name] and method.__wrapped__ is plain[name]+ assert method.__func__ is not plain[name] and method.__func__.__wrapped__ is plain[name]+ # The class keeps its own methods.+ assert {name: Trainer.__dict__[name] for name in plain} == plain@pytest.mark.parametrize("unhashable", [False, True])@@ -1406,16 +1425,16 @@ def test_a_trainer_built_before_cp_still_restores_the_accelerator_state(monkeypaon_init_plain(self)Trainer, args, built = _gated_trainer(monkeypatch, install_cp_methods = False, on_init = on_init)- restored = []- monkeypatch.setattr(cp, "_restore_parallelism_config", lambda state: restored.append(state))+ plain_train = Trainer.__dict__["train"]early = Trainer(None, args = types.SimpleNamespace(context_parallel_size = 1))assert early._context_parallel_manager is None- early.accelerator = types.SimpleNamespace(state = types.SimpleNamespace(device_mesh = None))- Trainer(None, args = args) # enables CP: installs the class wrappers+ Trainer(None, args = args) # enables CP: binds the wrappers on that trainer onlyassert built == [(2, "ulysses", "allgather")] # the default backend: auto- restored.clear()+ # The CP trainer put the shared state back at the end of its __init__ ...+ assert early.accelerator.state.device_mesh is None+ # ... and the earlier trainer runs the class's own train.+ assert Trainer.__dict__["train"] is plain_train and early.train.func is plain_trainassert early.train() == "trained"- assert restored == [early.accelerator.state]@pytest.mark.parametrize("forwarded", [1, None])diff --git a/tests/test_context_parallel_fsdp2_gloo.py b/tests/test_context_parallel_fsdp2_gloo.pyindex 9eacb00ef04..06373b6c42f 100644--- a/tests/test_context_parallel_fsdp2_gloo.py+++ b/tests/test_context_parallel_fsdp2_gloo.py@@ -170,7 +170,7 @@ def _worker(rank, port, out_dir):trl.SFTTrainer = Trainercp.patch_sft_trainer(install_cp_methods = True)- trainer = object.__new__(Trainer)+ trainer = cp._bind_cp_methods(object.__new__(Trainer))trainer._context_parallel_manager = managerids, labels = _batches()diff --git a/tests/test_context_parallel_fsdp2_save_gloo.py b/tests/test_context_parallel_fsdp2_save_gloo.pyindex 18670ab3945..ff1fede8686 100644--- a/tests/test_context_parallel_fsdp2_save_gloo.py+++ b/tests/test_context_parallel_fsdp2_save_gloo.py@@ -6,7 +6,7 @@Unsloth installs a ParallelismConfig for the (dp_shard, cp) mesh. transformers < 5 dispatchessave_model on that config before FSDP, skipping the full state dict gather, so PEFT is handed DTensorshards and fails. The patched save_model must write whole adapter tensors equal to the unsharded ones,-and leave the installed config in place afterwards.+and put back the config that was there before the CP trainer installed its own."""from __future__ import annotations@@ -83,8 +83,8 @@ def _worker(# A dp_shard only ParallelismConfig the user passed: still non-None for transformers 4.x.from accelerate.parallelism_config import ParallelismConfigstate.parallelism_config = ParallelismConfig(dp_shard_size = 2)+ user = state.parallelism_configcp._install_fsdp2_parallelism_config(state, manager)- installed = state.parallelism_configtorch.manual_seed(0)cfg = LlamaConfig(@@ -119,7 +119,7 @@ def _worker(def unwrap_model(self, model, **kwargs):return model- trainer = object.__new__(SFTTrainer)+ trainer = cp._bind_cp_methods(object.__new__(SFTTrainer))trainer.model = trainer.model_wrapped = modeltrainer.is_fsdp_enabled = Truetrainer.is_deepspeed_enabled = False@@ -145,7 +145,8 @@ def _worker(result["error"] = Noneexcept Exception as e:result["error"] = f"{type(e).__name__}: {e}"- result["config_kept"] = state.parallelism_config is installed+ # The CP save_model installs its config only while it runs (_cp_state_scope), then puts the user's back.+ result["config_kept"] = state.parallelism_config is userresult["full_state_dicts"] = len(full_state_dicts)dist.barrier()if rank == 0 and result["error"] is None:diff --git a/tests/test_context_parallel_rl.py b/tests/test_context_parallel_rl.pyindex eebb4fe6e34..063e0d56b7f 100644--- a/tests/test_context_parallel_rl.py+++ b/tests/test_context_parallel_rl.py@@ -636,24 +636,27 @@ def test_rl_evaluate_and_predict_refuse_iterable_datasets(monkeypatch, kind):assert trainer.predict([1, 2, 3, 4]) == "predicted"-def test_trainers_built_before_cp_restore_the_shared_accelerator_state(monkeypatch):- # The generated RL trainers bind self.train on the instance in __init__ (rl.py: MethodType(...)):- # a trainer built before CP was enabled keeps that plain method, so it must still drop the CP mesh- # a later CP trainer installed on the shared AcceleratorState.+def test_trainers_built_before_cp_keep_the_plain_methods(monkeypatch):+ # The generated RL trainers bind self.train on the instance in __init__ (rl.py: MethodType(...)), from+ # the instance's CP train when one is bound, else from the class. A trainer built before CP keeps the+ # class's own train (no CP wrapper runs for it); the CP trainer installs its state only inside train.installs = []Trainer = _cp_fake_trainer(monkeypatch, "dpo_trainer", [], installs)+ plain_train = Trainer.__dict__.get("train")def bind_train(trainer):- trainer.train = types.MethodType(type(trainer).train, trainer)+ bound = getattr(trainer.__dict__.get("train"), "__func__", None) or type(trainer).train+ trainer.train = types.MethodType(bound, trainer)plain = Trainer(_args(context_parallel_size = 1))bind_train(plain)cp_trainer = Trainer(_args())bind_train(cp_trainer)+ assert Trainer.__dict__.get("train") is plain_train+ assert plain.train.__func__ is type(plain).traininstalls.clear()assert plain.train() == "trained"- assert plain in installs, "the pre-CP trainer's bound train() skipped the accelerator restore"- installs.clear()+ assert installs == []assert cp_trainer.train() == "trained" and cp_trainer in installsdiff --git a/tests/test_cp_instance_overrides_gloo.py b/tests/test_cp_instance_overrides_gloo.pynew file mode 100644index 00000000000..d28890aa521--- /dev/null+++ b/tests/test_cp_instance_overrides_gloo.py@@ -0,0 +1,254 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A context parallel SFTTrainer binds its wrappers on itself only: a plain SFTTrainer built later in the same+process (same gloo ranks) runs the class's original methods, and no CP trainer wrapper is called during its+steps. With class-level wrappers every later trainer paid for them (about 35% of a torch 2.10 LoRA step with CP+off); here that is settled by call counts, not timing."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import sys # noqa: E402+import tempfile # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_parallel_cp_compose_gloo import _ENV_KEYS, SEQ, _checkpoint # noqa: E402+from test_tp_native_gloo import _free_port, _import_checkout_unsloth # noqa: E402++# The trainer-level CP wrappers (the attention / data hooks are per model and not counted here).+_WRAPPERS = {+ "patched_training_step",+ "patched_prediction_step",+ "patched_train",+ "patched_evaluate",+ "patched_predict",+ "patched_evaluation_loop",+ "patched_save_model",+ "_save_model",+ "patched_get_total_train_batch_size",+ "_install_accelerator_state",+ "_cp_state_scope",+ "call", # the old pre-CP trainers' restore wrapper+}+_METHODS = (+ "training_step",+ "prediction_step",+ "train",+ "evaluate",+ "predict",+ "evaluation_loop",+ "save_model",+ "get_total_train_batch_size",+)+++def _case(rank, world, tmp, ckpt):+ error = _import_checkout_unsloth()+ if error is not None:+ return {"skip": error}+ import datasets+ import torch.distributed as dist+ import transformers+ import trl+ from peft import LoraConfig, get_peft_model+ from transformers import AutoModelForCausalLM, AutoTokenizer++ from unsloth import context_parallel as cpm+ from unsloth.models._utils import patch_gradient_accumulation_fix++ patch_gradient_accumulation_fix(transformers.Trainer)+ cpm.patch_sft_trainer()+ cpm.DEVICE_TYPE_TORCH = "cpu"+ from unsloth.utils import attention_dispatch++ attention_dispatch.HAS_XFORMERS = False+ attention_dispatch.HAS_FLASH_ATTENTION = False+ cls = trl.SFTTrainer+ class_before = {name: cls.__dict__.get(name) for name in _METHODS}+ resolved_before = {name: getattr(cls, name, None) for name in _METHODS}++ def model():+ m = AutoModelForCausalLM.from_pretrained(+ ckpt, torch_dtype = torch.float32, attn_implementation = "sdpa"+ )+ torch.manual_seed(1)+ return get_peft_model(+ m, LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", lora_dropout = 0.0)+ )++ g = torch.Generator().manual_seed(1)+ data = datasets.Dataset.from_dict(+ {"input_ids": torch.randint(0, 1000, (16, SEQ), generator = g).tolist()}+ )++ def collate(rows):+ ids = torch.tensor([r["input_ids"] for r in rows])+ return {"input_ids": ids, "labels": ids.clone()}++ def config(name, cp):+ args = trl.SFTConfig(+ output_dir = os.path.join(tmp, name),+ per_device_train_batch_size = 2,+ gradient_accumulation_steps = 2,+ max_steps = 2,+ learning_rate = 1e-3,+ optim = "adamw_torch",+ logging_steps = 1,+ save_strategy = "no",+ report_to = [],+ use_cpu = True,+ seed = 0,+ remove_unused_columns = False,+ disable_tqdm = True,+ dataset_kwargs = {"skip_prepare_dataset": True},+ )+ if cp > 1:+ args.context_parallel_size = cp+ args.context_parallel_backend = "ulysses"+ return args++ tokenizer = AutoTokenizer.from_pretrained(ckpt)+ calls = {}++ def profile(frame, event, arg):+ if (+ event == "call"+ and frame.f_code.co_filename == cpm.__file__+ and frame.f_code.co_name in _WRAPPERS+ ):+ calls[frame.f_code.co_name] = calls.get(frame.f_code.co_name, 0) + 1++ def run(trainer):+ calls.clear()+ sys.setprofile(profile)+ try:+ trainer.train()+ finally:+ sys.setprofile(None)+ return dict(calls)++ cp_model = model() # Hugging Face attention: the trainer installs generic Ulysses itself+ cp_trainer = cls(+ model = cp_model,+ args = config("cp", 2),+ train_dataset = data,+ data_collator = collate,+ processing_class = tokenizer,+ )+ assert cp_trainer._context_parallel_manager is not None+ cp_calls = run(cp_trainer)++ plain = cls(+ model = model(),+ args = config("plain", 1),+ train_dataset = data,+ data_collator = collate,+ processing_class = tokenizer,+ )+ plain_calls = run(plain)+ # What a plain trainer resolves each method to, and whether a CP wrapper sits behind it.+ plain_wrapped = []+ for name in _METHODS:+ method = getattr(plain, name, None)+ chain, f = [], getattr(method, "__func__", method)+ while f is not None and len(chain) < 20:+ chain.append(f)+ f = getattr(+ getattr(f, "__wrapped__", None), "__func__", getattr(f, "__wrapped__", None)+ )+ if any(+ getattr(getattr(c, "__code__", None), "co_filename", None) == cpm.__file__+ for c in chain+ ):+ plain_wrapped.append(name)+ res = {+ "cp_calls": cp_calls,+ "plain_calls": plain_calls,+ "plain_wrapped": plain_wrapped,+ "class_unchanged": [n for n in _METHODS if cls.__dict__.get(n) is class_before[n]],+ "resolved_unchanged": [n for n in _METHODS if getattr(cls, n, None) is resolved_before[n]],+ "plain_losses": [h["loss"] for h in plain.state.log_history if "loss" in h],+ }+ if dist.is_initialized():+ dist.barrier()+ return res+++def _worker(rank, world, port, tmp, ckpt):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ )+ for key in _ENV_KEYS:+ os.environ.pop(key, None)+ torch.set_num_threads(1)+ import torch.distributed as dist++ try:+ try:+ res = _case(rank, world, tmp, ckpt)+ except Exception as e:+ import traceback+ res = {"error": f"{type(e).__name__}: {e}", "trace": traceback.format_exc()}+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))+ finally:+ if dist.is_initialized():+ try:+ dist.destroy_process_group()+ except Exception:+ pass+++def test_a_plain_trainer_after_a_cp_trainer_runs_the_original_methods():+ ctx = multiprocessing.get_context("spawn")+ port, world = _free_port(), 2+ with tempfile.TemporaryDirectory() as tmp:+ ckpt = _checkpoint(tmp)+ procs = [+ ctx.Process(target = _worker, args = (r, world, port, tmp, ckpt)) for r in range(world)+ ]+ for p in procs:+ p.start()+ for p in procs:+ p.join(900)+ if p.is_alive():+ p.kill()+ results = []+ for r in range(world):+ path = os.path.join(tmp, f"result_{r}.pt")+ results.append(+ torch.load(path, weights_only = False)+ if os.path.exists(path)+ else {"error": f"rank {r}"}+ )+ if "skip" in results[0]:+ pytest.skip(results[0]["skip"])+ for rank, res in enumerate(results):+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))+ print(+ f"rank {rank}: cp trainer wrapper calls {res['cp_calls']}, plain trainer {res['plain_calls']}"+ )+ # The CP trainer engaged its wrappers (the counter can see them) ...+ assert (+ res["cp_calls"].get("patched_training_step", 0) >= 4+ and res["cp_calls"].get("patched_train") == 1+ )+ # ... the plain trainer built after it called none, and resolves to the class's original methods.+ assert res["plain_calls"] == {}, res["plain_calls"]+ assert res["plain_wrapped"] == [], res["plain_wrapped"]+ assert res["class_unchanged"] == list(_METHODS)+ assert res["resolved_unchanged"] == list(_METHODS)+ assert len(res["plain_losses"]) == 2diff --git a/tests/test_cp_trainer_refusal.py b/tests/test_cp_trainer_refusal.pynew file mode 100644index 00000000000..87a5d9698cc--- /dev/null+++ b/tests/test_cp_trainer_refusal.py@@ -0,0 +1,93 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_CP_SIZE on a trainer without a context parallel wrapper: the refusal names the real cause (CPU backend,+non-SFT / DPO / GRPO trainer, failed patched-trainer build), and the build skips unsloth_zoo's CPU trainer quietly."""++import sys+import types++import pytest+import torch++import unsloth # noqa: F401+++def cp_env(monkeypatch):+ for name in ("UNSLOTH_PARALLEL", "UNSLOTH_TP_SIZE", "UNSLOTH_PP_SIZE", "UNSLOTH_ALLOW_CPU"):+ monkeypatch.delenv(name, raising = False)+ monkeypatch.setenv("UNSLOTH_CP_SIZE", "2")+ return monkeypatch+++def _refusal(trainer_cls):+ from unsloth.models import _utils++ trainer = object.__new__(trainer_cls)+ with pytest.raises(NotImplementedError) as info:+ _utils._parallel_capabilities_pre(trainer, trainer_cls.__init__, (), {}, None)+ return str(info.value)+++def _sft_like():+ import trl++ class SFTTrainer(trl.SFTTrainer): # what TRL's own class looks like: no CP wrapper on it+ pass++ SFTTrainer.__unsloth_context_parallel__ = False+ return SFTTrainer+++def test_cpu_backend_names_the_backend_not_sfttrainer(cp_env):+ cp_env.setitem(sys.modules, "unsloth._cpu_init", types.ModuleType("unsloth._cpu_init"))+ message = _refusal(_sft_like())+ assert "CPU backend" in message and "no GPU was found" in message, message+ assert "Instead: launch on GPUs" in message and "use SFTTrainer" not in message, message+++def test_sft_trainer_whose_patched_build_failed_names_the_build_error(cp_env):+ from unsloth.models import rl++ cp_env.delitem(sys.modules, "unsloth._cpu_init", raising = False)+ cp_env.setitem(+ rl.PATCHED_TRAINER_BUILD_ERRORS, "sft_trainer", "OSError: source code not available"+ )+ message = _refusal(_sft_like())+ assert (+ "Unsloth's patched SFTTrainer could not be built (OSError: source code not available)"+ in message+ )+ assert "use SFTTrainer" not in message, message+++def test_allow_cpu_names_the_variable(cp_env):+ cp_env.delitem(sys.modules, "unsloth._cpu_init", raising = False)+ cp_env.setenv("UNSLOTH_ALLOW_CPU", "1")+ message = _refusal(_sft_like())+ assert "UNSLOTH_ALLOW_CPU=1 skips Unsloth's trainer patches" in message, message+++def test_patched_trainer_build_skips_the_zoo_cpu_trainer(monkeypatch):+ """unsloth_zoo.cpu replaces trl.trainer.sft_trainer.SFTTrainer with a subclass inspect cannot read; the build+ used to fail on it with "OSError: source code not available" and warn. It now keeps it, quietly."""+ from trl.trainer import sft_trainer++ from unsloth.models import rl++ base = sft_trainer.SFTTrainer+ while getattr(base, "_unsloth_cpu", False) or base.__name__.startswith("Unsloth"):+ base = base.__mro__[1]+ namespace = {"_unsloth_cpu": True, "__module__": "unsloth_zoo.cpu.trainer"}+ cpu_cls = type("SFTTrainer", (base,), namespace) # no __firstlineno__, as in the rehearsal+ with pytest.raises((OSError, TypeError)):+ __import__("inspect").getsource(cpu_cls)+ monkeypatch.setattr(sft_trainer, "SFTTrainer", cpu_cls)+ warnings = []+ monkeypatch.setattr(rl.logger, "warning_once", lambda msg, *a, **k: warnings.append(msg))+ monkeypatch.setattr(rl, "PATCHED_TRAINER_BUILD_ERRORS", {}, raising = False)+ rl._patch_trl_rl_trainers("sft_trainer")+ assert not [w for w in warnings if "Could not build" in w], warnings+ assert sft_trainer.SFTTrainer is cpu_cls+ assert "CPU backend" in rl.PATCHED_TRAINER_BUILD_ERRORS.get("sft_trainer", "")diff --git a/tests/test_cp_ulysses_generic_gloo.py b/tests/test_cp_ulysses_generic_gloo.pynew file mode 100644index 00000000000..63a6ae17fc5--- /dev/null+++ b/tests/test_cp_ulysses_generic_gloo.py@@ -0,0 +1,305 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Ulysses context parallelism on any Hugging Face attention model, through the model's own attention function+(ALL_ATTENTION_FUNCTIONS, or the modeling module's eager_attention_forward): scaling, softcap, sliding window and+attention sinks survive the exchange. Two gloo CPU ranks vs one process: the loss and every parameter gradient.++Configs are the tiny test repos' (tiny_models.py: hf-internal-testing/tiny-random-Gemma3ForCausalLM,+trl-internal-testing/tiny-Qwen3ForCausalLM, trl-internal-testing/tiny-GptOssForCausalLM) with random weights, a+small vocabulary, and a window shorter than a shard so it crosses the seam."""++import os+import tempfile+import types++import pytest+import torch+import torch.distributed as dist+import torch.multiprocessing as mp+import torch.nn.functional as F++import unsloth # noqa: F401+from unsloth import context_parallel as cp+from unsloth.utils import attention_dispatch as ad+from unsloth.utils import ulysses_hybrid as uh++transformers = pytest.importorskip("transformers")++SEQ = 16+WINDOW = 5 # < SEQ / 2: windows cross the shard seam+REPOS = {+ "gemma3": "hf-internal-testing/tiny-random-Gemma3ForCausalLM",+ "gemma2": "trl-internal-testing/tiny-Gemma2ForCausalLM",+ "qwen3": "trl-internal-testing/tiny-Qwen3ForCausalLM",+ "gpt_oss": "trl-internal-testing/tiny-GptOssForCausalLM",+}+++def _config(kind):+ from transformers import AutoConfig++ try:+ config = AutoConfig.from_pretrained(REPOS[kind])+ except Exception as e: # offline host without the tiny repo cached+ pytest.skip(f"{REPOS[kind]}: {e}")+ config.vocab_size = 96+ config.num_hidden_layers = 2+ config.attention_dropout = 0.0+ config.use_cache = False+ config.pad_token_id = 0+ if kind in ("gemma3", "gemma2"):+ # Gemma3: 2 heads / 1 kv head, so the kv head is replicated over cp 2, and the 1 / sqrt(256) scale+ # (head_dim 8) is off the default the old path refused. Gemma2: attention logit softcapping.+ config.layer_types = ["sliding_attention", "full_attention"]+ config.sliding_window = WINDOW+ config.attn_logit_softcapping = 0.05 # small enough to bend the random tiny logits+ config.final_logit_softcapping = None+ elif kind == "gpt_oss":+ config.layer_types = ["sliding_attention", "full_attention"]+ config.sliding_window = WINDOW+ config.output_router_logits = False+ return config+++def _model(kind, implementation):+ from transformers import AutoModelForCausalLM++ torch.manual_seed(0)+ model = AutoModelForCausalLM.from_config(_config(kind), attn_implementation = implementation)+ model.config._attn_implementation = implementation+ for name, p in model.named_parameters():+ if name.endswith("sinks"):+ with torch.no_grad():+ p.normal_(0, 1) # zero-initialised: make every head's sink differ+ return model.float().train()+++def _manager(size, rank):+ manager = object.__new__(cp.ContextParallelManager)+ manager.size = size+ manager.backend = "ulysses"+ manager.packed_seq_lengths = None+ manager.mesh = types.SimpleNamespace(get_group = lambda: None, get_local_rank = lambda: rank)+ return manager+++def _loss(logits, shift_labels, count):+ return (+ F.cross_entropy(+ logits.float().flatten(0, 1), shift_labels.flatten(), ignore_index = -100, reduction = "sum"+ )+ / count+ )+++def _batch(packed):+ ids = torch.randint(+ 0, 96, (1 if packed else 2, SEQ), generator = torch.Generator().manual_seed(1)+ )+ labels = ids.clone()+ labels[labels % 5 == 0] = -100+ return ids, labels+++def _reference(model, ids, labels, lengths):+ """One process: the whole rows, or every packed document as its own row."""+ if lengths is None:+ shift = F.pad(labels, (0, 1), value = -100)[:, 1:]+ count = shift.ne(-100).sum()+ _loss(model(input_ids = ids, use_cache = False).logits, shift, count).backward()+ return count+ pieces, start = [], 0+ for n in lengths:+ doc_ids, doc_labels = ids[:, start : start + n], labels[:, start : start + n]+ pieces.append((doc_ids, F.pad(doc_labels, (0, 1), value = -100)[:, 1:]))+ start += n+ count = sum(shift.ne(-100).sum() for _, shift in pieces)+ total = sum(+ _loss(model(input_ids = doc_ids, use_cache = False).logits, shift, count)+ for doc_ids, shift in pieces+ )+ total.backward()+ return count+++def _parity_worker(+ rank,+ world,+ kind,+ implementation,+ packed,+ out_dir,+ control = None,+):+ ad.HAS_FLASH_ATTENTION = False+ ad.HAS_XFORMERS = False+ model = _model(kind, implementation)+ assert cp._generic_hf_attention(model) and cp._supports_context_parallel(model, "ulysses")+ cp._check_cp_model(model, world, "ulysses")+ ids, labels = _batch(packed)+ lengths = [6, 7, 3] if packed else None+ count = _reference(model, ids, labels, lengths)+ reference = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}+ model.zero_grad(set_to_none = True)++ calls, lengths_seen = [], set()+ original = uh._ulysses_hf_attention++ def counted(module, query, *args, **kwargs):+ fn = kwargs["attention_fn"]++ def seen(shard, q, *a, **k):+ lengths_seen.add(int(q.shape[2])) # the model's own function sees the whole sequence+ return fn(shard, q, *a, **k)++ kwargs["attention_fn"] = seen+ calls.append(1)+ return original(module, query, *args, **kwargs)++ uh._ulysses_hf_attention = counted+ manager = _manager(world, rank)+ manager.attach_attention_hooks(model)+ if control == "window_dropped":+ uh._sliding_window = lambda module, kwargs: None+ elif control == "softcap_dropped":+ original_counted = uh._ulysses_hf_attention++ def no_softcap(*args, **kwargs):+ kwargs.pop("softcap", None)+ return original_counted(*args, **kwargs)++ uh._ulysses_hf_attention = no_softcap+ elif control == "sinks_unsliced":+ original_init = uh._HeadShard.__init__++ def full_sinks(self, module, **overrides):+ overrides.pop("sinks", None)+ original_init(self, module, **overrides)++ uh._HeadShard.__init__ = full_sinks+ inputs = {"input_ids": ids.clone(), "labels": labels.clone()}+ if packed:+ inputs["packed_seq_lengths"] = torch.tensor(lengths, dtype = torch.int32)+ manager._prepare_inputs(inputs)+ with manager.apply(inputs):+ out = model(+ input_ids = inputs["input_ids"],+ position_ids = inputs["position_ids"],+ use_cache = False,+ )+ loss = _loss(out.logits, inputs["shift_labels"], count)+ loss.backward()+ dist.all_reduce(loss.detach())+ uh._ulysses_hf_attention = original+ assert len(calls) == model.config.num_hidden_layers, calls+ assert lengths_seen == {SEQ}, lengths_seen+ assert inputs["input_ids"].shape[1] == SEQ // world+ worst = 0.0+ for name, param in model.named_parameters():+ if not param.requires_grad:+ continue+ grad = torch.zeros_like(param) if param.grad is None else param.grad.clone()+ dist.all_reduce(grad)+ want = reference.get(name, torch.zeros_like(param))+ torch.testing.assert_close(grad, want, rtol = 1e-4, atol = 1e-6, msg = name)+ worst = max(worst, float((grad - want).abs().max()))+ if rank == 0 and out_dir:+ with open(+ os.path.join(out_dir, f"{kind}_{implementation}_{'packed' if packed else 'rows'}.txt"),+ "w",+ ) as f:+ f.write(+ f"loss {float(loss):.6f} ulysses calls {len(calls)} max grad diff {worst:.3e}\n"+ )+++def _spawn(fn, world, *args):+ # Fork: a spawned child would re-import unsloth without the conftest CUDA spoof.+ if "fork" not in mp.get_all_start_methods():+ pytest.skip("needs the fork start method")+ with tempfile.TemporaryDirectory() as d:+ mp.start_processes(+ _entry,+ args = (world, os.path.join(d, "init"), fn, args),+ nprocs = world,+ join = True,+ start_method = "fork",+ )+++def _entry(rank, world, init_file, fn, args):+ torch.set_num_threads(1)+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)+ try:+ fn(rank, world, *args)+ finally:+ dist.destroy_process_group()+++_OUT = os.environ.get("UNSLOTH_TEST_PARITY_OUT", "")+++ "kind, implementation, packed",+ [+ ("gemma3", "eager", False), # sliding window + non-default scale + replicated kv head+ ("gemma2", "eager", False), # attention logit softcapping + sliding window+ ("gemma3", "sdpa", False), # sliding window through sdpa's mask+ ("gemma3", "eager", True), # window and packed documents in one mask+ ("qwen3", "sdpa", False), # plain causal: sdpa's is_causal path+ ("qwen3", "sdpa", True),+ ("gpt_oss", "eager", False), # attention sinks (local heads' slice) + sliding window+ ],+)+def test_generic_ulysses_matches_one_process(kind, implementation, packed):+ _config(kind) # skip before forking when the repo is unavailable+ _spawn(_parity_worker, 2, kind, implementation, packed, _OUT)+++ "kind, control",+ [("gemma3", "window_dropped"), ("gemma2", "softcap_dropped"), ("gpt_oss", "sinks_unsliced")],+)+def test_each_feature_is_load_bearing(kind, control):+ """Negative controls: dropping the window or the softcap after the exchange, or handing every head's sinks+ to a head shard, breaks parity, so the cases above prove the features are passed through."""+ _config(kind)+ with pytest.raises(Exception, match = "AssertionError|IndexError|RuntimeError|Mismatch"):+ _spawn(_parity_worker, 2, kind, "eager", False, "", control)+++def test_windowed_hf_models_pass_the_ulysses_check_but_not_ring():+ model = _model("gemma2", "eager")+ assert cp._unsupported_attention(model.config, "ulysses", generic = True) is None+ softcapped = types.SimpleNamespace(attn_logit_softcapping = 3.0)+ assert "softcapping" in cp._unsupported_attention(softcapped, "ring")+ assert cp._unsupported_attention(softcapped, "ulysses", generic = True) is None+ assert "sliding" in cp._unsupported_attention(model.config, "ring")+ assert "sliding" in cp._unsupported_attention(model.config, "ulysses") # Unsloth's fast forward+ with pytest.raises(NotImplementedError, match = "sliding"):+ cp._check_cp_model(model, 2, "ring")+ cp._check_cp_model(model, 2, "ulysses")+ # DPO / GRPO keep the Llama-style rule.+ with pytest.raises(NotImplementedError):+ cp._check_cp_model(model, 2, "ulysses", allow_hybrids = False)+++def test_chunked_attention_stays_refused():+ model = _model("qwen3", "sdpa")+ model.config.attention_chunk_size = 8+ with pytest.raises(NotImplementedError, match = "chunked"):+ cp._check_cp_model(model, 2, "ulysses")+++def test_unsloth_fast_forwards_are_not_generic():+ from unsloth.models.llama import LlamaAttention_fast_forward++ class Fast(torch.nn.Module):+ forward = LlamaAttention_fast_forward++ root = torch.nn.Module()+ root.self_attn = Fast()+ assert not uh.is_hf_registry_attention(root.self_attn)+ assert not cp._generic_hf_attention(root)+ assert cp._supports_context_parallel(root, "ulysses") # the fast Llama path, as beforediff --git a/tests/test_fsdp2_bnb_optim_kept.py b/tests/test_fsdp2_bnb_optim_kept.pynew file mode 100644index 00000000000..33db6c485de--- /dev/null+++ b/tests/test_fsdp2_bnb_optim_kept.py@@ -0,0 +1,45 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""accelerate's FSDP2 keeps optim = 'adamw_8bit': a bitsandbytes Adam / AdamW steps each rank's local shard after+Accelerator._prepare_fsdp2, and a bitsandbytes optimizer that cannot is refused before the first step."""++from __future__ import annotations+++import pytest++pytest.importorskip("trl")+++def test_accelerate_fsdp2_prepare_localizes_bitsandbytes_adamw():+ bnb = pytest.importorskip("bitsandbytes")+ import torch++ from unsloth.models.fsdp2 import _localize_or_refuse_bnb+ from unsloth.models.fsdp2_bnb_optim import is_local_shard_bnb_optimizer++ p = torch.nn.Parameter(torch.zeros(8))+ opt = bnb.optim.AdamW8bit([p], lr = 1e-3)+ _localize_or_refuse_bnb([p, opt])+ assert is_local_shard_bnb_optimizer(opt)++ torch_opt = torch.optim.AdamW([torch.nn.Parameter(torch.zeros(8))])+ _localize_or_refuse_bnb([torch_opt]) # torch optimizers pass untouched+ assert not is_local_shard_bnb_optimizer(torch_opt)+++def test_accelerate_fsdp2_refuses_bnb_that_cannot_step_local_shards(monkeypatch):+ bnb = pytest.importorskip("bitsandbytes")+ import torch++ from unsloth.models.fsdp2 import _localize_or_refuse_bnb++ lion = bnb.optim.Lion8bit([torch.nn.Parameter(torch.zeros(8))], lr = 1e-4)+ with pytest.raises(ValueError, match = "Unsloth: FSDP2 .*Lion8bit.*adamw_torch"):+ _localize_or_refuse_bnb([lion])++ monkeypatch.setenv("UNSLOTH_FSDP2_BNB_LOCAL", "0")+ opt = bnb.optim.AdamW8bit([torch.nn.Parameter(torch.zeros(8))], lr = 1e-3)+ with pytest.raises(ValueError, match = "UNSLOTH_FSDP2_BNB_LOCAL=0"):+ _localize_or_refuse_bnb([opt])diff --git a/tests/test_fsdp2_default_optim.py b/tests/test_fsdp2_default_optim.pyindex 1888fdb5ed7..1236d769869 100644--- a/tests/test_fsdp2_default_optim.py+++ b/tests/test_fsdp2_default_optim.py@@ -1,7 +1,9 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""unsloth#3551: Unsloth's adamw_8bit default cannot step FSDP2's DTensor parameters."""+"""unsloth#3551: Unsloth's adamw_8bit default under FSDP2. The generated configs used to switch it to+adamw_torch_fused; FSDP2 (native and accelerate's) now steps bitsandbytes Adam / AdamW on each rank's+local shard, so the configs keep it (tests/test_fsdp2_bnb_optim_kept.py)."""from __future__ import annotations@@ -45,6 +47,7 @@ _PROBE = textwrap.dedent("grpo_fsdp2_args": optim(GRPOConfig, {}, **fsdp2_args),"grpo_fsdp2_json": optim(GRPOConfig, {}, fsdp = "full_shard", fsdp_config = fsdp2_json),"grpo_fsdp2_explicit_torch": optim(GRPOConfig, {"FSDP_VERSION": "2"}, optim = "adamw_torch"),+ "sft_fsdp2_paged": optim(SFTConfig, {"FSDP_VERSION": "2"}, optim = "paged_adamw_8bit"),"grpo_fsdp_unversioned": optim(GRPOConfig, {}, fsdp = "full_shard"),}))"""@@ -61,6 +64,7 @@ def probe(tmp_path_factory):)lines = [line for line in run.stdout.splitlines() if line.startswith("PROBE ")]assert lines, f"probe failed:\n{run.stdout[-3000:]}\n{run.stderr[-3000:]}"+ assert "Switching optim" not in run.stdoutreturn json.loads(lines[-1][len("PROBE ") :])@@ -71,18 +75,15 @@ def test_without_fsdp2_the_default_is_unchanged(probe):@pytest.mark.parametrize("case", ["grpo_fsdp2", "sft_fsdp2", "grpo_fsdp2_args", "grpo_fsdp2_json"])-def test_fsdp2_gets_an_optimizer_that_steps_dtensors(probe, case):- assert probe[case] == "adamw_torch_fused"+def test_fsdp2_keeps_adamw_8bit(probe, case):+ # Stepped on the local shards (fsdp2_bnb_optim), not silently switched to torch's AdamW.+ assert probe[case] == "adamw_8bit"-def test_fsdp_without_a_version_follows_the_transformers_default(probe):- """TrainingArguments picks FSDP2 when fsdp_config names no version from transformers 5.10.0, FSDP1 before."""- from transformers import __version__- from unsloth_zoo.utils import Version+def test_fsdp_without_a_version_keeps_adamw_8bit(probe):+ assert probe["grpo_fsdp_unversioned"] == "adamw_8bit"- fsdp2 = Version(__version__) >= Version("5.10.0")- assert probe["grpo_fsdp_unversioned"] == ("adamw_torch_fused" if fsdp2 else "adamw_8bit")--def test_an_explicit_non_bnb_choice_is_kept(probe):+def test_explicit_choices_are_kept(probe):assert probe["grpo_fsdp2_explicit_torch"] == "adamw_torch"+ assert probe["sft_fsdp2_paged"] == "paged_adamw_8bit"diff --git a/tests/test_parallel_cp_compose_gloo.py b/tests/test_parallel_cp_compose_gloo.pyindex 62db8ec2592..73ead931c99 100644--- a/tests/test_parallel_cp_compose_gloo.py+++ b/tests/test_parallel_cp_compose_gloo.py@@ -5,10 +5,10 @@the same script in one process: losses, per-parameter gradients of the first optimizer step and the adaptersafter the run.-CPU ranks cannot run Unsloth's fast Llama attention (it needs CUDA), so the attention is the Hugging Face one with-the Ulysses wrapper hybrids use (utils/ulysses_hybrid.install_hf_attention_wrapper), and the model support check is-opened for it. Everything else (the env layout, the mesh, tensor parallel, the CP sharding, the token count and-the gradient reduction) is the production path."""+CPU ranks cannot run Unsloth's fast Llama attention (it needs CUDA), so the attention is the Hugging Face one, which+the production support check accepts and the trainer routes through generic Ulysses (ALL_ATTENTION_FUNCTIONS). The+env layout, the mesh, tensor parallel, the CP sharding, the token count and the gradient reduction are all the+production path; nothing is patched open."""import multiprocessingimport os@@ -196,7 +196,6 @@ def _case_train(from unsloth import context_parallel as cpmfrom unsloth.models import pp_native, tp_env, tp_nativefrom unsloth.models._utils import patch_gradient_accumulation_fix- from unsloth.utils.ulysses_hybrid import install_hf_attention_wrapperpatch_gradient_accumulation_fix(transformers.Trainer)cpm.patch_sft_trainer()@@ -220,8 +219,6 @@ def _case_train(torch.backends.cuda.enable_mem_efficient_sdp(False)torch.backends.cuda.enable_cudnn_sdp(False)torch.use_deterministic_algorithms(True, warn_only = True)- # The test seam: CPU ranks run the Hugging Face attention through the Ulysses wrapper.- cpm._supports_context_parallel = lambda model, backend = "ring": Trueif bypass_pre or bypass_post:# Evidence runs only: switch off the aux-loss guards to show what each one catches.from unsloth import parallel_capabilities@@ -283,9 +280,6 @@ def _case_train(moe_calls = _count_moe_backend_calls() if count else Noneif router_aux:model.config.output_router_logits = True- for name, module in model.named_modules():- if name.endswith("self_attn"):- install_hf_attention_wrapper(module, "sdpa", cpm.get_cp_manager)compiled = _block_compile_on_cpu(model) if compile else []# Data parallel replicas read different rows; cp / tp / pp peers share theirs.diff --git a/tests/test_parallel_env.py b/tests/test_parallel_env.pyindex 7b97dcc86d0..cf323bd7994 100644--- a/tests/test_parallel_env.py+++ b/tests/test_parallel_env.py@@ -442,7 +442,7 @@ def test_tensor_parallel_alone_keeps_what_tp_native_supports():(dict(cp = 2, cp_backend = "ring", hybrid = True), "cp backend", "cp_backend=ulysses"),(dict(cp = 2, cp_backend = "ring", trainer = "dpo"), "cp backend", "cp_backend=ulysses"),(dict(cp = 2, cp_backend = "ring", sm = 7), "cp backend", "cp_backend=ulysses"),- (dict(cp = 2, cp_backend = "ulysses", windowed = True), "cp x attention", "drop cp"),+ (dict(cp = 2, cp_backend = "ring", windowed = True), "cp x attention", "cp_backend=ulysses"),(dict(cp = 2, torch = (2, 6)), "cp x torch", "upgrade torch"),(dict(pp = 2, trainer = "grpo"), "pp x trainer", "drop pp"),(dict(pp = 2, packing = True), "pp x packing", "packing = False"),@@ -467,6 +467,7 @@ def test_every_refusal_names_its_axis_and_the_alternative(query, axis, instead):dict(cp = 2, cp_backend = "ulysses", packing = True, hybrid = True),dict(cp = 2, cp_backend = "ulysses", trainer = "grpo"),dict(cp = 2, cp_backend = "ring", sm = 9),+ dict(cp = 2, cp_backend = "ulysses", windowed = True), # Ulysses runs the model's own attentiondict(pp = 2, tp = 2, dp = 2, fsdp2 = True),dict(tp = 2, mode = "qlora"),dict(dp = 4, mode = "full", fsdp2 = True),@@ -517,7 +518,10 @@ def test_cp_env_with_a_plain_trainer_is_refused(clean_env):def __init__(self):pass- with pytest.raises(NotImplementedError, match = r"cp x trainer: .*Plain would train every rank"):+ with pytest.raises(+ NotImplementedError,+ match = r"cp x trainer: .*Plain is not one of the trainers .*Instead: use SFTTrainer",+ ):_utils._parallel_capabilities_pre(Plain(), Plain.__init__, (), {}, None)@@ -578,6 +582,9 @@ def test_every_contract_axis_has_declared_features():} <= namesfor f in caps.FEATURES:assert f.module and f.scope in ("instance", "class"), f+ # Context parallel binds its trainer wrappers per instance now (context_parallel._bind_instance_methods):+ # no feature still patches a class.+ assert f.scope == "instance", f.nameassert set(f.hooks) | set(f.exclusive) <= set(caps.HOOKS), (f.name, f.hooks, f.exclusive)assert set(f.consumes) | set(f.uses) | set(f.produces) <= set(caps.METADATA), f.nameassert set(f.mesh) <= {"dp", "tp", "pp", "cp"}, f.name@@ -590,7 +597,7 @@ def test_every_contract_axis_has_declared_features():[(dict(pp = 2, cp = 2, cp_backend = "ulysses"),- r"pp x cp.ulysses: both run Trainer.training_step",+ r"pp x cp.ulysses: both run trainer.training_step",),(dict(attention = "flash_varlen"), r"flash_varlen consumes cu_seqlens"),],diff --git a/tests/test_ulysses_hybrid.py b/tests/test_ulysses_hybrid.pyindex fe794b46b54..c7b7c1d5656 100644--- a/tests/test_ulysses_hybrid.py+++ b/tests/test_ulysses_hybrid.py@@ -335,11 +335,26 @@ def test_hybrid_gates():cp._check_stateful_mixers(model, "ulysses", 2)-def test_eager_attention_is_refused():+def test_flex_attention_is_refused():+ # Eager is routed through the modeling module's eager_attention_forward now (test_cp_ulysses_generic_gloo.py).model = _model(KINDS[0])attn = model.model.layers[1].self_attnwith pytest.raises(NotImplementedError, match = "attn_implementation"):- uh.install_hf_attention_wrapper(attn, "eager", cp.get_cp_manager)+ uh.install_hf_attention_wrapper(attn, "flex_attention", cp.get_cp_manager)+++def test_bidirectional_attention_is_refused():+ # The rebuilt mask is causal: a bidirectional layer (Gemma3 use_bidirectional_attention) would train causal.+ model = _model(KINDS[0])+ attn = model.model.layers[1].self_attn+ attn.is_causal = False+ with pytest.raises(NotImplementedError, match = "only supports causal attention"):+ uh.install_hf_attention_wrapper(attn, "sdpa", cp.get_cp_manager)+ assert not getattr(attn, "_unsloth_cp_ulysses_attention", False)+ attn.is_causal = True+ with pytest.raises(NotImplementedError, match = "only supports causal attention"):+ uh._refuse_non_causal(attn, False) # a per-call is_causal = False override+ uh._refuse_non_causal(attn, None)def test_hybrid_packing_metadata_is_off_under_context_parallelism():diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 5170f377ff8..059978036d0 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -6,7 +6,6 @@ from __future__ import annotationsimport contextlibimport functoolsimport types-import weakreffrom typing import Iterator, Optionalimport torch@@ -146,9 +145,17 @@ def resolve_cp_backend(return "ulysses"-def _unsupported_attention(config, backend = "ring") -> Optional[str]:- # Ring and Llama's forward ignore mask, window and softcap, so both backends refuse them.+def _unsupported_attention(+ config,+ backend = "ring",+ generic = False,+) -> Optional[str]:+ # Ring and Unsloth's fast Llama forward ignore mask, window and softcap, so they refuse them. Ulysses+ # around the model's own attention function (`generic`) keeps the window and the softcap.allowed = {"full_attention", "linear_attention"} if backend == "ulysses" else {"full_attention"}+ windows_ok = generic and backend == "ulysses"+ if windows_ok:+ allowed = allowed | {"sliding_attention"}configs = [config]text_config = getattr(config, "text_config", None)if text_config is not None and text_config is not config:@@ -167,13 +174,14 @@ def _unsupported_attention(config, backend = "ring") -> Optional[str]:return f"{', '.join(kinds)} layers{hint}"# Any positive window, even with use_sliding_window off: Unsloth's masks read it directly.sliding_window = getattr(c, "sliding_window", None)- if (isinstance(sliding_window, int) and sliding_window > 0) or getattr(- c, "use_sliding_window", None- ) is True:+ if not windows_ok and (+ (isinstance(sliding_window, int) and sliding_window > 0)+ or getattr(c, "use_sliding_window", None) is True+ ):return "sliding window attention"if getattr(c, "attention_chunk_size", None):return "chunked attention"- if getattr(c, "attn_logit_softcapping", None):+ if not windows_ok and getattr(c, "attn_logit_softcapping", None):return "attention logit softcapping"return None@@ -194,6 +202,7 @@ from .utils.ulysses_hybrid import (GATED_DELTA_ATTENTION_CLASSES,check_gated_delta_heads,install_hf_attention_wrapper,+ is_hf_registry_attention,is_ulysses_gated_delta,wrap_gated_delta_forward,)@@ -258,20 +267,43 @@ def _hybrid_config_blocker(model) -> Optional[str]:return None-def _supports_context_parallel(model, backend = "ring") -> bool:+def _attention_modules(model) -> list:+ if not hasattr(model, "named_modules"):+ return []+ return [module for name, module in model.named_modules() if name.endswith("self_attn")]+++def _uses_fast_llama_attention(model) -> bool:from .models.llama import LlamaAttention_fast_forward, runs_fast_forward+ modules = _attention_modules(model)+ return bool(modules) and all(runs_fast_forward(m, LlamaAttention_fast_forward) for m in modules)+++def _generic_hf_attention(model) -> bool:+ """Every attention layer is a Hugging Face forward through ALL_ATTENTION_FUNCTIONS, so Ulysses can run the+ model's own attention function (scaling, softcap, sliding window, sinks) after the exchange."""+ if not isinstance(model, torch.nn.Module) or _stateful_mixers(model):+ return False+ modules = _attention_modules(model)+ return bool(modules) and all(is_hf_registry_attention(m) for m in modules)+- modules = [module for name, module in model.named_modules() if name.endswith("self_attn")]- if bool(modules) and all(runs_fast_forward(m, LlamaAttention_fast_forward) for m in modules):+def _supports_context_parallel(+ model,+ backend = "ring",+ generic = True,+) -> bool:+ if _uses_fast_llama_attention(model):return not _stateful_mixers(model)+ if backend != "ulysses" or _hybrid_config_blocker(model) is not None:+ return Falsemixers = _stateful_mixers(model)- return (- backend == "ulysses"- and bool(mixers)- and all(is_ulysses_gated_delta(m) for m, _ in mixers)- and all(type(m).__name__ in GATED_DELTA_ATTENTION_CLASSES for m in modules)- and _hybrid_config_blocker(model) is None- )+ if mixers:+ # Hybrids: the family-specific linear attention forwards, and their HF full attention layers.+ return all(is_ulysses_gated_delta(m) for m, _ in mixers) and all(+ type(m).__name__ in GATED_DELTA_ATTENTION_CLASSES for m in _attention_modules(model)+ )+ return generic and _generic_hf_attention(model)def _check_stateful_mixers(model, backend, size) -> None:@@ -351,10 +383,13 @@ class ContextParallelManager:for module, _ in _stateful_mixers(model):if is_ulysses_gated_delta(module):wrap_gated_delta_forward(module, get_cp_manager)+ # Unsloth's fast Llama forward has its own Ulysses path; every other Hugging Face attention layer+ # runs its own attention function after the exchange.+ generic = not _uses_fast_llama_attention(model)for name, module in model.named_modules():- if (- name.endswith("self_attn")- and type(module).__name__ in GATED_DELTA_ATTENTION_CLASSES+ if name.endswith("self_attn") and (+ type(module).__name__ in GATED_DELTA_ATTENTION_CLASSES+ or (generic and is_hf_registry_attention(module))):install_hf_attention_wrapper(module, module.config._attn_implementation, get_cp_manager@@ -666,16 +701,62 @@ def _restore_parallelism_config(state) -> None:_INSTALLED_PARALLELISM_CONFIG.clear()+def _release_accelerator_state(state) -> None:+ """Put back the AcceleratorState mesh / parallelism config a CP trainer installed (no-op otherwise)."""+ if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:+ # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.+ state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None+ _INSTALLED_MESH.clear()+ _restore_parallelism_config(state)+++def _cp_state_scope(self):+ """A CP trainer's entry point (train / evaluate / predict / save_model): its mesh and parallelism config+ sit on the shared AcceleratorState only while it runs, so a trainer without CP built in the same process+ keeps the class's original methods and still sees the state it expects. Nested calls (train ->+ save_model) restore only when the outermost one returns."""+ depth = self.__dict__.get("_unsloth_cp_state_depth", 0)+ self._unsloth_cp_state_depth = depth + 1+ _install_accelerator_state(self)+ try:+ yield+ finally:+ self._unsloth_cp_state_depth = depth+ if depth == 0 and getattr(self, "_context_parallel_manager", None) is not None:+ _release_accelerator_state(getattr(getattr(self, "accelerator", None), "state", None))+++def _bind_instance_methods(trainer, methods: dict) -> None:+ """Bind the CP wrappers on this trainer only (types.MethodType); the class keeps its methods."""+ import types++ bound = {name: types.MethodType(fn, trainer) for name, fn in methods.items() if fn is not None}+ trainer.__dict__.update(bound)+ trainer.__dict__["_unsloth_cp_bound_methods"] = bound+++def _bind_cp_methods(trainer):+ """Bind the CP wrappers of the trainer's (patched) class on a trainer built without __init__ (tests)."""+ _bind_instance_methods(trainer, getattr(type(trainer), "_unsloth_cp_methods"))+ return trainer+++def _unbind_instance_methods(trainer) -> None:+ """Drop the CP wrappers bound by _bind_instance_methods; an attribute Unsloth re-wrapped since (it then+ holds our bound method inside, which passes through without a manager) is left alone."""+ bound = trainer.__dict__.pop("_unsloth_cp_bound_methods", None) or {}+ for name, method in bound.items():+ if trainer.__dict__.get(name) is method:+ del trainer.__dict__[name]++def _install_accelerator_state(self):manager = getattr(self, "_context_parallel_manager", None)accelerator = getattr(self, "accelerator", None)state = getattr(accelerator, "state", None)if manager is None:- if _INSTALLED_MESH and getattr(state, "device_mesh", None) is _INSTALLED_MESH[0]:- # The mesh an accepted FSDP2 ParallelismConfig indexes, restored with that config.- state.device_mesh = _INSTALLED_MESH[1] if len(_INSTALLED_MESH) > 1 else None- _INSTALLED_MESH.clear()- _restore_parallelism_config(state)+ _release_accelerator_state(state)returnif accelerator is None:return@@ -1121,7 +1202,15 @@ def _check_cp_model(tp: int = 1,) -> None:model_config = getattr(model, "config", None)- unsupported = _unsupported_attention(model_config, backend)+ # DPO / GRPO (allow_hybrids False) keep the Llama-style rule: their decoder hooks are not verified on+ # the generic path.+ generic = (+ allow_hybrids+ and backend == "ulysses"+ and not _uses_fast_llama_attention(model)+ and _generic_hf_attention(model)+ )+ unsupported = _unsupported_attention(model_config, backend, generic = generic)if unsupported is not None:raise NotImplementedError(f"Unsloth: {backend} context parallelism does not support {unsupported}."@@ -1142,10 +1231,21 @@ def _check_cp_model(raise NotImplementedError(f"Unsloth: context parallelism on Gated DeltaNet hybrids does not support {blocker}.")+ blocker = _hybrid_config_blocker(model) if generic else None+ if blocker is not None:+ raise NotImplementedError(+ f"Unsloth: Ulysses context parallelism does not support {blocker}."+ )+ if not allow_hybrids and not _uses_fast_llama_attention(model) and _generic_hf_attention(model):+ raise NotImplementedError(+ "Unsloth: DPO / GRPO context parallelism supports Unsloth's Llama-style attention only; generic "+ "Ulysses on other Hugging Face attention models is verified for SFTTrainer."+ )if not _supports_context_parallel(model, backend):raise NotImplementedError(- "Unsloth: context parallelism currently supports Llama-style attention "- "(Llama, Qwen2, Gemma), and Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids with Ulysses."+ "Unsloth: context parallelism supports Unsloth's Llama-style attention with either backend, "+ "and with Ulysses any Hugging Face attention model (attention through ALL_ATTENTION_FUNCTIONS) "+ "and the Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids.")@@ -1231,8 +1331,10 @@ def _check_cp_trainer(def patch_sft_trainer(install_cp_methods: bool = False) -> None:- # Only __init__ is wrapped up front; the other wrappers go on the class once a trainer enables- # CP. With CP off they still slowed torch 2.10 LoRA steps by ~35% (gemma-4-E2B, L4 / A100).+ # Only __init__ is patched on the class. The CP wrappers are bound per trainer (types.MethodType) when it+ # asks for CP, so trainers without CP, built before or after one with it, run the class's own methods:+ # class wrappers slowed torch 2.10 LoRA steps by ~35% with CP off (gemma-4-E2B, L4 / A100).+ # install_cp_methods = True binds them on every trainer (a test seam for trainers given a manager by hand).import trltrainer_cls = trl.SFTTrainer@@ -1306,11 +1408,23 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:peft_config = kwargs.get("peft_config"),backend = kwargs.get("context_parallel_backend"),)+ if requested > 1 or install_cp_methods:# Before the original __init__: Unsloth wraps instance methods there (auto packing's# train reset, the token budget's training_step), and they must capture the CP ones.- install_methods()- original_init(self, *args, **kwargs)+ _bind_instance_methods(self, cp_methods)+ try:+ original_init(self, *args, **kwargs)+ except BaseException:+ _unbind_instance_methods(self)+ raiseself._context_parallel_manager = None+ try:+ _cp_init_tail(self)+ except BaseException:+ _unbind_instance_methods(self)+ raise++ def _cp_init_tail(self):# Unsloth's attention ignores transformers' own CP: it would stay local to each shard.# `accelerate launch` with parallelism_config_cp_size builds it from the environment# (ACCELERATE_USE_PARALLELISM_CONFIG), leaving args.parallelism_config None.@@ -1342,10 +1456,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:self.args.context_parallel_backend = backendif size <= 1:_install_accelerator_state(self)- if not trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):- # By identity: a subclass may define __eq__ without __hash__.- pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]- pre_cp_trainers.append(weakref.ref(self))+ if not install_cp_methods:+ _unbind_instance_methods(self)return_check_cp_runtime(self, size, backend)_refuse_router_aux_loss_after_init(self, size)@@ -1390,7 +1502,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:)_check_native_fsdp2_reduces_over_cp(self.model, manager)self._context_parallel_manager = manager- install_methods()+ if "_unsloth_cp_bound_methods" not in self.__dict__:+ _bind_instance_methods(self, cp_methods)manager.attach_attention_hooks(self.model)_install_accelerator_state(self)# The ring calls the flash ops FA3 / FA4 override, so scope them to train() here too.@@ -1409,6 +1522,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:else "."))+ # Installed again by each CP entry point (_cp_state_scope) while it runs.+ _release_accelerator_state(getattr(getattr(self, "accelerator", None), "state", None))@functools.wraps(original_prediction_step)def patched_prediction_step(self, model, inputs, *args, **kwargs):@@ -1438,8 +1553,8 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:def patched_train(self, *args, **kwargs):if getattr(self, "_context_parallel_manager", None) is not None:_refuse_iterable_datasets(getattr(self, "train_dataset", None))- _install_accelerator_state(self)- return original_train(self, *args, **kwargs)+ with _cp_state_scope(self):+ return original_train(self, *args, **kwargs)@functools.wraps(original_evaluate)def patched_evaluate(self, *args, **kwargs):@@ -1453,18 +1568,22 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:else current)_refuse_iterable_datasets(eval_dataset)- _install_accelerator_state(self)- return original_evaluate(self, *args, **kwargs)+ with _cp_state_scope(self):+ return original_evaluate(self, *args, **kwargs)@functools.wraps(original_predict)def patched_predict(self, test_dataset, *args, **kwargs):if getattr(self, "_context_parallel_manager", None) is not None:_refuse_iterable_datasets(test_dataset)- _install_accelerator_state(self)- return original_predict(self, test_dataset, *args, **kwargs)+ with _cp_state_scope(self):+ return original_predict(self, test_dataset, *args, **kwargs)@functools.wraps(original_save_model)def patched_save_model(self, *args, **kwargs):+ with _cp_state_scope(self):+ return _save_model(self, *args, **kwargs)++ def _save_model(self, *args, **kwargs):state = getattr(getattr(self, "accelerator", None), "state", None)installed = _INSTALLED_PARALLELISM_CONFIG[1] if _INSTALLED_PARALLELISM_CONFIG else Noneif installed is None or getattr(state, "parallelism_config", None) is not installed:@@ -1524,46 +1643,21 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:total = total // manager.sizereturn total- # Non-CP trainers built before any trainer enabled CP: their __init__ may have wrapped train /- # evaluate / predict on the instance around the plain methods (auto packing's train reset).- pre_cp_trainers = [] # weakref.ref per trainer-- def _restoring(trainer, bound):- @functools.wraps(bound)- def call(*args, **kwargs):- # What patched_train / evaluate / predict do for a non-CP trainer: drop the mesh and- # parallelism config a CP trainer installed on the shared accelerator state.- _install_accelerator_state(trainer)- return bound(*args, **kwargs)-- return call-- def install_methods():- if trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):- return- for trainer in [ref() for ref in pre_cp_trainers]:- if trainer is None:- continue- for name in ("train", "evaluate", "predict"):- bound = trainer.__dict__.get(name)- if bound is not None:- trainer.__dict__[name] = _restoring(trainer, bound)- pre_cp_trainers.clear()- trainer_cls.prediction_step = patched_prediction_step- trainer_cls.train = patched_train- trainer_cls.evaluate = patched_evaluate- trainer_cls.predict = patched_predict- trainer_cls.evaluation_loop = patched_evaluation_loop- trainer_cls.save_model = patched_save_model- trainer_cls.training_step = patched_training_step- if has_total_train_batch_size:- trainer_cls.get_total_train_batch_size = patched_get_total_train_batch_size- trainer_cls.__unsloth_context_parallel_methods__ = True+ cp_methods = {+ "prediction_step": patched_prediction_step,+ "train": patched_train,+ "evaluate": patched_evaluate,+ "predict": patched_predict,+ "evaluation_loop": patched_evaluation_loop,+ "save_model": patched_save_model,+ "training_step": patched_training_step,+ }+ if has_total_train_batch_size:+ cp_methods["get_total_train_batch_size"] = patched_get_total_train_batch_sizetrainer_cls.__init__ = patched_init+ trainer_cls._unsloth_cp_methods = cp_methodstrainer_cls.__unsloth_context_parallel__ = True- if install_cp_methods:- install_methods()# Backends with single-tensor collectives; others (MPI) keep the list path.@@ -1982,9 +2076,20 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:backend = kwargs.get("context_parallel_backend"),)# Before the original __init__, so instance wrappers made there capture the CP methods.- install_methods()- original_init(self, *args, **kwargs)+ _bind_instance_methods(self, cp_methods)+ try:+ original_init(self, *args, **kwargs)+ except BaseException:+ _unbind_instance_methods(self)+ raiseself._context_parallel_manager = None+ try:+ _cp_init_tail(self)+ except BaseException:+ _unbind_instance_methods(self)+ raise++ def _cp_init_tail(self):_refuse_transformers_cp(self,"DPOConfig" if kind == "dpo_trainer" else "GRPOConfig",@@ -1994,10 +2099,7 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:backend = resolve_cp_backend(backend, size, rl = True)if size <= 1:_install_accelerator_state(self)- if not trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):- # The generated trainers bind self.train in __init__ (rl.py), around the plain method.- pre_cp_trainers[:] = [ref for ref in pre_cp_trainers if ref() is not None]- pre_cp_trainers.append(weakref.ref(self))+ _unbind_instance_methods(self)returnif backend != "ulysses":raise NotImplementedError(@@ -2010,6 +2112,8 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:_check_cp_trainer(self, size, backend, [self.model, ref_model])manager = ContextParallelManager(size, backend)self._context_parallel_manager = manager+ if "_unsloth_cp_bound_methods" not in self.__dict__:+ _bind_instance_methods(self, cp_methods)attach_rl_hooks(manager, self.model)if kind == "grpo_trainer":# GRPO's packed and shared-prefix forwards pick their path per step from self-checks;@@ -2021,7 +2125,6 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:module._unsloth_prefix_grouper_nograd_disabled = Trueif isinstance(ref_model, torch.nn.Module):attach_rl_hooks(manager, ref_model)- _install_accelerator_state(self)print(f"Unsloth: Context parallelism enabled with size = {size} (ulysses, {kind}).")def _step_wrapper(name):@@ -2089,8 +2192,8 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:def patched_train(self, *args, **kwargs):if getattr(self, "_context_parallel_manager", None) is not None:_refuse_iterable_datasets(getattr(self, "train_dataset", None))- _install_accelerator_state(self)- return originals["train"](self, *args, **kwargs)+ with _cp_state_scope(self):+ return originals["train"](self, *args, **kwargs)@functools.wraps(originals["evaluate"])def patched_evaluate(self, *args, **kwargs):@@ -2104,14 +2207,14 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:else current)_check_rl_eval_entry(self, kind, dataset)- _install_accelerator_state(self)- return originals["evaluate"](self, *args, **kwargs)+ with _cp_state_scope(self):+ return originals["evaluate"](self, *args, **kwargs)def patched_predict(self, test_dataset, *args, **kwargs):if getattr(self, "_context_parallel_manager", None) is not None:_check_rl_eval_entry(self, kind, test_dataset)- _install_accelerator_state(self)- return originals["predict"](self, test_dataset, *args, **kwargs)+ with _cp_state_scope(self):+ return originals["predict"](self, test_dataset, *args, **kwargs)@functools.wraps(originals["evaluation_loop"])def patched_evaluation_loop(self, *args, **kwargs):@@ -2120,49 +2223,28 @@ def patch_rl_trainer(trainer_cls, kind: str) -> None:self.gather_function = _cp_gather_for_metrics(self.accelerator, manager.size)return originals["evaluation_loop"](self, *args, **kwargs)- # Non-CP trainers built before CP was enabled: like SFT, repair what their __init__ bound on the- # instance, so they still drop the mesh a CP trainer installs on the shared AcceleratorState.- pre_cp_trainers = [] # weakref.ref per trainer-- def _restoring(trainer, bound):- @functools.wraps(bound)- def call(*args, **kwargs):- _install_accelerator_state(trainer)- return bound(*args, **kwargs)-- return call-- def install_methods():- # Only __init__ is wrapped up front (as for SFT): the wrappers below go on the class once a- # trainer enables CP, so trainers without CP keep their original methods.- if trainer_cls.__dict__.get("__unsloth_context_parallel_methods__"):- return- for trainer in [ref() for ref in pre_cp_trainers]:- if trainer is None:- continue- for name in ("train", "evaluate", "predict"):- bound = trainer.__dict__.get(name)- if bound is not None:- trainer.__dict__[name] = _restoring(trainer, bound)- pre_cp_trainers.clear()- trainer_cls.training_step = _step_wrapper("training_step")- trainer_cls.prediction_step = _step_wrapper("prediction_step")- trainer_cls.train = patched_train- trainer_cls.evaluate = patched_evaluate- if originals["predict"] is not None:- trainer_cls.predict = functools.wraps(originals["predict"])(patched_predict)- trainer_cls.evaluation_loop = patched_evaluation_loop- if kind == "grpo_trainer":- trainer_cls.compute_loss = _scoped("compute_loss", sticky = True)- for name in ("_get_per_token_logps", "_get_per_token_logps_and_entropies"):- if originals[name] is not None:- setattr(trainer_cls, name, _scoped(name))- if originals["_generate"] is not None:- trainer_cls._generate = functools.wraps(originals["_generate"])(patched_generate)- trainer_cls._prepare_inputs = patched_prepare_inputs- if originals["_get_train_sampler"] is not None:- trainer_cls._get_train_sampler = patched_train_sampler- trainer_cls.__unsloth_context_parallel_methods__ = True-+ # Bound per trainer that enables CP (types.MethodType, as for SFT): trainers without CP keep the+ # class's own methods.+ cp_methods = {+ "training_step": _step_wrapper("training_step"),+ "prediction_step": _step_wrapper("prediction_step"),+ "train": patched_train,+ "evaluate": patched_evaluate,+ "evaluation_loop": patched_evaluation_loop,+ }+ if originals["predict"] is not None:+ cp_methods["predict"] = functools.wraps(originals["predict"])(patched_predict)+ if kind == "grpo_trainer":+ cp_methods["compute_loss"] = _scoped("compute_loss", sticky = True)+ for name in ("_get_per_token_logps", "_get_per_token_logps_and_entropies"):+ if originals[name] is not None:+ cp_methods[name] = _scoped(name)+ if originals["_generate"] is not None:+ cp_methods["_generate"] = functools.wraps(originals["_generate"])(patched_generate)+ cp_methods["_prepare_inputs"] = patched_prepare_inputs+ if originals["_get_train_sampler"] is not None:+ cp_methods["_get_train_sampler"] = patched_train_sampler++ trainer_cls._unsloth_cp_methods = cp_methodstrainer_cls.__init__ = patched_inittrainer_cls.__unsloth_context_parallel__ = Truediff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex ff6b617964a..e3df9074d52 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -1145,6 +1145,46 @@ def _prepare_gdn(layer, hidden_states):_GDN_OP[0] = False+# PEFT's ParamWrapper (target_parameters on a non-expert weight, e.g. a MoE router) registers a torch+# parametrization on every forward and removes it after, and torch builds a new Parametrized<Class>+# subclass per registration. Dynamo guards a module's type id, so each call recompiles the module's+# forward whenever the fresh class lands at an address it has not seen (how many do depends on the+# allocator: a different code path shifted the count). One class per module instance keeps the id.+_PARAMETRIZED_CLASSES = weakref.WeakKeyDictionary()+++def _stable_parametrized_classes():+ try:+ from torch.nn.utils import parametrize+ except Exception:+ return+ original = getattr(parametrize, "_inject_new_class", None)+ if original is None or getattr(original, "_unsloth_stable", False):+ return++ # Eager: the class swap is the graph break PEFT's registration already takes, and Dynamo must+ # not fold the cache lookup into a graph.+ @torch.compiler.disable+ @functools.wraps(original)+ def _inject_new_class(module):+ cls = module.__class__+ try:+ cached = _PARAMETRIZED_CLASSES.get(module)+ except TypeError: # not weak-referenceable / hashable+ return original(module)+ if cached is not None and cached.__bases__ == (cls,):+ module.__class__ = cached+ return+ original(module)+ try:+ _PARAMETRIZED_CLASSES[module] = module.__class__+ except TypeError:+ pass++ _inject_new_class._unsloth_stable = True+ parametrize._inject_new_class = _inject_new_class++# zoo's Gemma-4 sliding router scans the mask (data dependent) and declines while compiling, so# the block runs the scan eagerly and a router wrapper reads the verdict stored on the mask._BAND_MODULE = "unsloth_zoo.temporary_patches.gemma4_banded_attention"@@ -1956,6 +1996,7 @@ def patch_block_compile(model):_rebind_class(cls)_flex_split_hook()_gemma4_router_hook()+ _stable_parametrized_classes()if any(type(m).__name__ == "Linear4bit" for m in model.modules()):_nf4_in_blocks()kinds = {}diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex c1ec368f2ea..ff1161bc73e 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -6635,6 +6635,59 @@ def _prebuilt_optimizer(trainer, init, args, kwargs):return None+def _cp_trainer_refusal(trainer_cls, cp) -> str:+ """Why a trainer class carries no context parallel wrapper under UNSLOTH_CP_SIZE, and what to do."""+ import sys++ name = trainer_cls.__name__+ bases = {c.__name__.removeprefix("Unsloth").removeprefix("_CPU") for c in trainer_cls.__mro__}+ kind = next((k for k in ("SFTTrainer", "DPOTrainer", "GRPOTrainer") if k in bases), None)+ head = (+ f"Unsloth: cp x trainer: context parallel (UNSLOTH_CP_SIZE={cp}) was requested, but {name} "+ )+ if sys.modules.get("unsloth._cpu_init") is not None or getattr(+ trainer_cls, "_unsloth_cpu", False+ ):+ return (+ head+ + "runs on Unsloth's CPU backend (no GPU was found), which uses TRL's own trainers and does not "+ "wire context parallel; every rank would train on the whole sequence. Instead: launch on GPUs "+ "(torchrun with CUDA devices visible), or drop cp."+ )+ if kind is None:+ return (+ head+ + "is not one of the trainers context parallel hooks into (SFTTrainer, DPOTrainer, GRPOTrainer); "+ "every rank would train on the whole sequence. Instead: use SFTTrainer (it takes a plain Trainer "+ "script's arguments), or drop cp."+ )+ if os.environ.get("UNSLOTH_ALLOW_CPU", "0") == "1":+ reason = "UNSLOTH_ALLOW_CPU=1 skips Unsloth's trainer patches"+ fix = "unset UNSLOTH_ALLOW_CPU"+ else:+ try:+ from .rl import PATCHED_TRAINER_BUILD_ERRORS+ except Exception:+ PATCHED_TRAINER_BUILD_ERRORS = {}+ file = {+ "SFTTrainer": "sft_trainer",+ "DPOTrainer": "dpo_trainer",+ "GRPOTrainer": "grpo_trainer",+ }[kind]+ built = PATCHED_TRAINER_BUILD_ERRORS.get(file)+ reason = (+ f"Unsloth's patched {kind} could not be built ({built})"+ if built+ else f"Unsloth's context parallel wrapper was not installed on {name} (import unsloth before trl)"+ )+ fix = "import unsloth before trl / transformers, and check the warning above for the build failure"+ return (+ head+ + f"has no context parallel wrapper: {reason}; every rank would train on the whole sequence. "+ f"Instead: {fix}, or drop cp."+ )++def _optimizer_kwargs_hold_params(trainer, init, args, kwargs):"""Trainer(optimizer_cls_and_kwargs = (cls, {"params": ...})) carries parameters chosen before __init__."""value = kwargs.get("optimizer_cls_and_kwargs")@@ -6679,11 +6732,7 @@ def _parallel_capabilities_pre(trainer, init, args, kwargs, model):cp = env_cp_size()if cp > 1 and not getattr(type(trainer), "__unsloth_context_parallel__", False):- raise NotImplementedError(- f"Unsloth: cp x trainer: context parallel (UNSLOTH_CP_SIZE={cp}) runs through SFTTrainer, "- f"DPOTrainer and GRPOTrainer; {type(trainer).__name__} would train every rank on the whole sequence. "- "Instead: use SFTTrainer (a plain Trainer script: SFTTrainer takes the same arguments), or drop cp."- )+ raise NotImplementedError(_cp_trainer_refusal(type(trainer), cp))from .tp_env import env_layoutlayout = env_layout()diff --git a/unsloth/models/fsdp2.py b/unsloth/models/fsdp2.pyindex 3d088aad4fd..ea03997a5f8 100644--- a/unsloth/models/fsdp2.py+++ b/unsloth/models/fsdp2.py@@ -452,6 +452,79 @@ def install_fsdp2_prepare_hook():setattr(fsdp2_prepare_model, _PATCHED, True)accelerator_module.fsdp2_prepare_model = fsdp2_prepare_model+ install_fsdp2_bnb_optimizer_hook()+ return True+++_BNB_ALTERNATIVE = (+ "Use optim = 'adamw_8bit' / 'paged_adamw_8bit' (bitsandbytes Adam / AdamW step each rank's local "+ "shard), or optim = 'adamw_torch' / 'adamw_torch_fused'."+)+++def _localize_or_refuse_bnb(objects):+ """accelerate's FSDP2 swaps the optimizer's parameters for DTensors in Accelerator._prepare_fsdp2;+ bitsandbytes' kernels take plain tensors. A bitsandbytes Adam / AdamW is made to step the local+ shards (fsdp2_bnb_optim, as native FSDP2 does); any other bitsandbytes optimizer is refused here,+ before its first step would fail with "got mixed torch.Tensor and DTensor" (unsloth#3551)."""+ from .fsdp2_native import _inner_optimizer, _is_bnb_optimizer_class, _optimizer_on_cpu+ for obj in objects:+ if not isinstance(obj, torch.optim.Optimizer):+ continue+ inner = _inner_optimizer(obj)+ if not _is_bnb_optimizer_class(type(inner)):+ continue+ from .fsdp2_bnb_optim import (+ bnb_local_shards_enabled,+ bnb_steps_cpu_tensors,+ localize_bnb_optimizer,+ )++ name = type(inner).__name__+ if not bnb_local_shards_enabled():+ raise ValueError(+ f"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' {name} cannot step, "+ "and UNSLOTH_FSDP2_BNB_LOCAL=0 turns off stepping the local shards. Unset it, or use "+ "optim = 'adamw_torch' / 'adamw_torch_fused'."+ )+ if _optimizer_on_cpu(inner) and not bnb_steps_cpu_tensors():+ raise ValueError(+ f"Unsloth: FSDP2 keeps the parameters on the CPU (CPU offload), and this bitsandbytes "+ f"cannot step CPU tensors with {name}. Upgrade bitsandbytes (>= 0.50), turn CPU offload "+ "off, or use optim = 'adamw_torch'."+ )+ if not localize_bnb_optimizer(inner):+ raise ValueError(+ f"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' {name} cannot step "+ f"(only bitsandbytes Adam / AdamW step local shards). {_BNB_ALTERNATIVE}"+ )+ _print_rank0(+ f"Unsloth: FSDP2 keeps the bitsandbytes optimizer ({name}): it steps each rank's local "+ "shard of the DTensor parameters (8-bit state per shard)."+ )+++def install_fsdp2_bnb_optimizer_hook():+ """Wrap Accelerator._prepare_fsdp2: after it maps the optimizer onto the DTensor parameters,+ localize (or refuse) a bitsandbytes optimizer."""+ try:+ from accelerate import Accelerator+ except Exception:+ return False+ original = getattr(Accelerator, "_prepare_fsdp2", None)+ if original is None:+ return False+ if getattr(original, _PATCHED, False):+ return True++ @functools.wraps(original)+ def _prepare_fsdp2(self, *args, **kwargs):+ result = original(self, *args, **kwargs)+ _localize_or_refuse_bnb(result if isinstance(result, (tuple, list)) else (result,))+ return result++ setattr(_prepare_fsdp2, _PATCHED, True)+ Accelerator._prepare_fsdp2 = _prepare_fsdp2return Truediff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex a08aa0e9d7d..1fc2da89a79 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2433,11 +2433,16 @@ def _patch_ppo_policy_value_wrapper(module):wrapper.gradient_checkpointing_disable = gradient_checkpointing_disable+# trainer_file -> "<ExceptionType>: <message>" of the last failed build, so a later refusal can name the cause.+PATCHED_TRAINER_BUILD_ERRORS = {}++def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):# Defensive wrapper matching patch_trl_rl_trainers()'s try/except, so direct callers do not see exceptions from the impl on TRL versions that rename or move classes.try:return _patch_trl_rl_trainers_impl(trainer_file)except Exception as e:+ PATCHED_TRAINER_BUILD_ERRORS[trainer_file] = f"{type(e).__name__}: {e}"# Warning, not info: the impl RETURNS for the benign case, so reaching here means generation failed and the run silently falls back to trl's trainer, losing Unsloth's compute_loss, bf16/fp16 fixup and dataset handling.logger.warning_once(f"Unsloth: Could not build the patched trl.trainer.{trainer_file}, "@@ -2532,6 +2537,17 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):f"Unsloth: Could not load {RLTrainer_name} from trl.trainer.{trainer_file}: {e}")return+ if getattr(RLTrainer, "_unsloth_cpu", False):+ # Unsloth's CPU backend (no accelerator found) runs TRL's own trainer behind unsloth_zoo.cpu's defaults+ # subclass by design; that subclass has no readable source, so building from it would only fail.+ PATCHED_TRAINER_BUILD_ERRORS[trainer_file] = (+ "the CPU backend keeps TRL's own trainer (no GPU was found)"+ )+ logger.info(+ f"Unsloth: CPU backend: trl.trainer.{trainer_file} keeps TRL's own {RLTrainer_name} "+ "with unsloth_zoo.cpu's defaults."+ )+ return_config_resolved_module = Nonetry:RLConfig = eval(f"trl.trainer.{trainer_file}.{RLConfig_name}")@@ -3295,7 +3311,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):if "model" in call_args:training_check = ("if hasattr(self, 'train'):\n"- " self.train = MethodType(prepare_for_training_mode(self.__class__.train), self)\n"+ # A context parallel trainer binds its train wrapper on the instance (context_parallel.py).+ " self.train = MethodType(prepare_for_training_mode(getattr(self.__dict__.get('train'), '__func__', None) or self.__class__.train), self)\n""pass\n")RLTrainer_post += training_check@@ -3475,35 +3492,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += learning_rate_check- # bitsandbytes' 8-bit optimizers step plain tensors, but FSDP2 shards parameters as DTensors, so the- # first step dies with "got mixed torch.Tensor and DTensor" (unsloth#3551).+ # optim = 'adamw_8bit' is kept under FSDP2: native FSDP2 (fsdp2_native._swap_bnb_optimizer) and+ # accelerate's FSDP2 (fsdp2._localize_or_refuse_bnb after Accelerator._prepare_fsdp2) step bitsandbytes+ # Adam / AdamW on each rank's local shard, and refuse the bitsandbytes optimizers that cannot (unsloth#3551).if "optim" in call_args:- extra_args += "_unsloth_fsdp2 = os.environ.get('FSDP_VERSION', '').strip() == '2'\n"- if "fsdp" in call_args and "fsdp_config" in call_args:- # TrainingArguments(fsdp = ..., fsdp_config = {"fsdp_version": 2} or a JSON path) without the launcher.- extra_args += (- "_unsloth_fsdp_cfg = fsdp_config\n"- "if fsdp and isinstance(_unsloth_fsdp_cfg, str) and os.path.isfile(_unsloth_fsdp_cfg):\n"- " try:\n"- " import json as _unsloth_json\n"- " with open(_unsloth_fsdp_cfg, encoding = 'utf-8') as _unsloth_f:\n"- " _unsloth_fsdp_cfg = _unsloth_json.load(_unsloth_f)\n"- " except Exception:\n"- " _unsloth_fsdp_cfg = None\n"- "if fsdp and _unsloth_fsdp_cfg is None:\n"- " _unsloth_fsdp_cfg = {}\n"- "if fsdp and isinstance(_unsloth_fsdp_cfg, dict):\n"- f" _unsloth_fsdp2 = _unsloth_fsdp2 or str(_unsloth_fsdp_cfg.get('fsdp_version', _unsloth_fsdp_cfg.get('version', {_FSDP_DEFAULT_VERSION}))) == '2'\n"- )extra_args += ("_unsloth_flash_adamw = str(getattr(optim, 'value', optim)).strip().lower() == 'adamw_flash'\n""if _unsloth_flash_adamw: optim = 'adamw_torch'\n")- extra_args += (- "if _unsloth_fsdp2 and str(getattr(optim, 'value', optim)) == 'adamw_8bit':\n"- " print(\"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' 8-bit optimizers cannot step. Switching optim = 'adamw_8bit' to 'adamw_torch_fused'.\")\n"- " optim = 'adamw_torch_fused'\n"- )# Fix num_train_epochs = None causing a TypeError in Trainer.__init__, which does `args.num_train_epochs > 0`.if "num_train_epochs" in call_args:diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.pyindex 30a19625172..a2120010b63 100644--- a/unsloth/parallel_capabilities.py+++ b/unsloth/parallel_capabilities.py@@ -180,10 +180,10 @@ RULES = (),Rule("cp x attention",- "context parallel on sliding window / chunked / softcapped attention",- lambda q: q.cp > 1 and q.windowed,- "Unsloth's context parallel attention drops the window / chunk / softcap",- "drop cp for this model",+ "ring context parallel on sliding window / chunked / softcapped attention",+ lambda q: q.cp > 1 and q.windowed and q.cp_backend == "ring",+ "ring attention drops the window / chunk / softcap (Ulysses runs the model's own attention function)",+ "cp_backend=ulysses (or auto), or drop cp for this model",),Rule("cp x torch",@@ -291,8 +291,10 @@ HOOKS = ("from_pretrained","get_peft_model","Trainer.__init__",- "Trainer.training_step",- "Trainer.prediction_step",+ # Lower case: bound on one trainer instance (types.MethodType), never on the class. pp_env and the CP+ # managers both bind these, so two of them on one trainer still cannot compose.+ "trainer.training_step",+ "trainer.prediction_step","Trainer.save_model","Accelerator.prepare_model","Accelerator.prepare_data_loader",@@ -382,7 +384,7 @@ FEATURES = ("Accelerator.clip_grad_norm_","Trainer.save_model",),- exclusive = ("Trainer.training_step", "Trainer.prediction_step"),+ exclusive = ("trainer.training_step", "trainer.prediction_step"),),Feature("cp.ulysses",@@ -394,8 +396,7 @@ FEATURES = (uses = ("packed_seq_lengths",),produces = ("seq_shard", "shift_labels", "position_ids"),hooks = ("Trainer.__init__", "attention_function"),- exclusive = ("Trainer.training_step", "Trainer.prediction_step"),- scope = "class",+ exclusive = ("trainer.training_step", "trainer.prediction_step"),),Feature("cp.ring",@@ -406,8 +407,7 @@ FEATURES = (consumes = ("input_ids", "labels", "position_ids", "num_items_in_batch"),produces = ("seq_shard", "shift_labels", "cp_permutation", "position_ids"),hooks = ("Trainer.__init__", "attention_function"),- exclusive = ("Trainer.training_step", "Trainer.prediction_step"),- scope = "class",+ exclusive = ("trainer.training_step", "trainer.prediction_step"),),# Precision.Feature(diff --git a/unsloth/utils/ulysses_hybrid.py b/unsloth/utils/ulysses_hybrid.pyindex e742c05bb6f..fd256e2c6a2 100644--- a/unsloth/utils/ulysses_hybrid.py+++ b/unsloth/utils/ulysses_hybrid.py@@ -1,7 +1,9 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""Ulysses context parallelism for hybrid models: scans need the full sequence, so they run on heads."""+"""Ulysses context parallelism for Hugging Face attention (any model whose attention calls ALL_ATTENTION_FUNCTIONS+runs its own attention function after the exchange) and for hybrid models, whose scans need the full sequence so+they run on heads."""import functoolsimport inspect@@ -10,7 +12,7 @@ import torchimport torch.distributed as distimport torch.nn.functional as F-from .ulysses import _AllToAll, ulysses_attention+from .ulysses import _AllToAll__all__ = ["GATED_DELTA_ATTENTION_CLASSES",@@ -19,6 +21,8 @@ __all__ = ["ulysses_gated_delta_forward","wrap_gated_delta_forward","install_hf_attention_wrapper",+ "is_hf_registry_attention",+ "hf_attention_forward",]# Hybrid full attention layers: Hugging Face forwards that call ALL_ATTENTION_FUNCTIONS after RoPE.@@ -226,63 +230,215 @@ def wrap_gated_delta_forward(module, get_cp_manager) -> None:module.forward = wrapped-def _ulysses_hf_attention(module, query, key, value, attention_mask, *, group, **kwargs):- from .attention_dispatch import (- AttentionConfig,- AttentionContext,- _run_attention,- select_attention_backend,+# Keyword arguments that describe the local sequence shard: rebuilt for the whole sequence after the exchange.+_SHARD_LOCAL_KWARGS = (+ "position_ids",+ "cu_seq_lens_q",+ "cu_seq_lens_k",+ "max_length_q",+ "max_length_k",+ "packed_seq_lengths",+)+++def _calls_attention_registry(function) -> bool:+ try:+ function = inspect.unwrap(function)+ except Exception:+ return False+ code = getattr(function, "__code__", None)+ return code is not None and "ALL_ATTENTION_FUNCTIONS" in code.co_names+++def hf_attention_forward(module):+ """The Hugging Face forward of this attention class that calls ALL_ATTENTION_FUNCTIONS (eager falls back to+ the modeling module's eager_attention_forward): the class forward, or the original unsloth_zoo stashed when+ it patched the class (`_original_<module>_<Class>_forward`, e.g. Gemma3's fp32 SDPA forward). None for+ Unsloth's fast forwards and custom kernels."""+ cls = type(module)+ forward = getattr(cls, "forward", None)+ if _calls_attention_registry(forward):+ return forward+ for klass in cls.__mro__:+ for name, value in vars(klass).items():+ if (+ name.startswith("_original_")+ and name.endswith(f"_{klass.__name__}_forward")+ and _calls_attention_registry(value)+ ):+ return value+ return None+++def is_hf_registry_attention(module) -> bool:+ return hf_attention_forward(module) is not None+++def _forward_globals(module) -> dict:+ return getattr(inspect.unwrap(hf_attention_forward(module)), "__globals__", {})+++def _route_hf_forward(module, get_cp_manager) -> None:+ """When the class forward is unsloth_zoo's replacement, run the stashed Hugging Face forward on this module+ while a Ulysses manager is active (so attention goes through the registry), and the class forward otherwise."""+ hf_forward = hf_attention_forward(module)+ if hf_forward is None or hf_forward is getattr(type(module), "forward", None):+ return+ if "_unsloth_cp_hf_forward" in vars(module):+ return+ import types++ default = module.forward+ hf_bound = types.MethodType(hf_forward, module)++ @functools.wraps(default)+ def forward(*args, **kwargs):+ manager = get_cp_manager()+ if manager is not None and manager.backend == "ulysses":+ return hf_bound(*args, **kwargs)+ return default(*args, **kwargs)++ module._unsloth_cp_hf_forward = hf_bound+ module.forward = forward+++class _HeadShard:+ """The attention module as its own attention function sees it on a head shard: the local key / value+ group count and the local heads' sinks; everything else is the module's."""++ # Plain attribute access only: torch.compile cannot trace object.__getattribute__ / object.__setattr__.+ def __init__(self, module, **overrides):+ self._unsloth_module = module+ self.__dict__.update(overrides)++ def __getattr__(self, name):+ # Only called for names the shard does not hold itself.+ if name == "_unsloth_module":+ raise AttributeError(name)+ return getattr(self._unsloth_module, name)+++def _sliding_window(module, kwargs):+ window = (+ kwargs["sliding_window"]+ if "sliding_window" in kwargs+ else getattr(module, "sliding_window", None))- from .packing import get_packed_info_from_kwargs+ return (+ window if isinstance(window, int) and not isinstance(window, bool) and window > 0 else None+ )+- bsz, n_heads, q_len, head_dim = query.shape- scaling = kwargs.get("scaling")- if scaling is not None and abs(scaling * head_dim**0.5 - 1) > 1e-6:+def _refuse_non_causal(module, is_causal = None) -> None:+ # The rebuilt whole-sequence mask is causal, and bidirectional windows differ per model (Gemma3 `<`, the+ # generic overlay `<=`), so non-causal attention would silently train causal.+ if is_causal is None:+ is_causal = getattr(module, "is_causal", True)+ if is_causal is False:raise NotImplementedError(- "Unsloth: Ulysses context parallelism needs the default attention scale 1/sqrt(head_dim)."+ f"Unsloth: Ulysses context parallelism only supports causal attention, but {type(module).__name__} "+ "is bidirectional (for example use_bidirectional_attention = True). Turn off context parallelism "+ "for this model.")+++def _whole_sequence_mask(total, device, dtype, window, cu_seqlens):+ """Additive [1, 1, T, T] mask over the gathered sequence: causal, the sliding window (key > query - window,+ as transformers' masks), and packed documents closed off from one another."""+ idx = torch.arange(total, device = device)+ allowed = idx[None, :] <= idx[:, None]+ if window is not None:+ allowed = allowed & (idx[:, None] - idx[None, :] < window)+ if cu_seqlens is not None:+ doc = torch.bucketize(idx, cu_seqlens[1:].to(device = device, dtype = idx.dtype), right = True)+ allowed = allowed & (doc[:, None] == doc[None, :])+ mask = torch.zeros(total, total, device = device, dtype = dtype)+ return mask.masked_fill_(~allowed, torch.finfo(dtype).min)[None, None]+++def _is_flash(implementation) -> bool:+ # flash_attention_2 / 3, and hub kernels ("kernels-community/...") that take transformers' flash kwargs.+ return "flash" in str(implementation) or "/" in str(implementation)+++def _ulysses_hf_attention(+ module,+ query,+ key,+ value,+ attention_mask,+ *,+ group,+ attention_fn,+ implementation,+ packed_seq_lengths = None,+ **kwargs,+):+ """Ulysses around the model's own attention function: q / k / v go from sequence shards to head shards,+ the function runs over the whole sequence with its own scaling, softcap, sliding window and the local+ heads' sinks, and the output goes back to sequence shards."""+ from .packing import get_packed_info_from_kwargs+ from .ulysses import head_to_seq, seq_to_head, ulysses_head_plan+if module.training and kwargs.get("dropout"):raise NotImplementedError("Unsloth: Ulysses context parallelism does not support attention dropout.")- seq_info = get_packed_info_from_kwargs(kwargs, query.device)- n_kv_heads = key.shape[1]- config = AttentionConfig(- backend = select_attention_backend(use_varlen = seq_info is not None),- n_kv_heads = n_kv_heads,- n_groups = n_heads // n_kv_heads,- flash_dense_kwargs = {"causal": True},- flash_varlen_kwargs = {"dropout_p": 0.0, "causal": True},- )- context = AttentionContext(- bsz = bsz,- q_len = q_len,- kv_seq_len = key.shape[2],- n_heads = n_heads,- head_dim = head_dim,- requires_grad = query.requires_grad,- seq_info = seq_info,- attention_mask = None,- causal_mask = None,+ _refuse_non_causal(module, kwargs.get("is_causal"))+ bsz, n_heads, local_len, _ = query.shape+ size = dist.get_world_size(group)+ rank = dist.get_rank(group)+ plan = ulysses_head_plan(n_heads, key.shape[1], size)+ if plan.kv_replication > 1:+ # Consecutive copies keep query head h on kv head h // n_groups after the split.+ key = key.repeat_interleave(plan.kv_replication, dim = 1)+ value = value.repeat_interleave(plan.kv_replication, dim = 1)+ query, key, value = (seq_to_head(x, group) for x in (query, key, value))+ total = local_len * size+ heads = slice(rank * plan.local_heads, (rank + 1) * plan.local_heads)++ for name in _SHARD_LOCAL_KWARGS:+ kwargs.pop(name, None)+ seq_info = (+ None+ if packed_seq_lengths is None+ else get_packed_info_from_kwargs({"packed_seq_lengths": packed_seq_lengths}, query.device))- return ulysses_attention(_run_attention, config, context, query, key, value, group), None---def install_hf_attention_wrapper(module, implementation, get_cp_manager) -> None:- """Route this module's ALL_ATTENTION_FUNCTIONS[implementation] call through Ulysses while active."""- from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS+ cu_seqlens = None if seq_info is None else seq_info[1]+ if cu_seqlens is not None and int(cu_seqlens[-1]) < total:+ # Padding to the cp multiple sits after the last document: its own segment.+ cu_seqlens = torch.cat([cu_seqlens, cu_seqlens.new_tensor([total])])++ overrides = {"num_key_value_groups": plan.local_groups}+ sinks = getattr(module, "sinks", None)+ if isinstance(sinks, torch.Tensor) and sinks.ndim >= 1 and sinks.shape[0] == n_heads:+ overrides["sinks"] = sinks[heads]+ s_aux = kwargs.get("s_aux")+ if isinstance(s_aux, torch.Tensor) and s_aux.ndim >= 1 and s_aux.shape[0] == n_heads:+ kwargs["s_aux"] = s_aux[heads]+ shard = _HeadShard(module, **overrides)++ window = _sliding_window(module, kwargs)+ if _is_flash(implementation):+ # Flash takes the window and the softcap as arguments, and documents as cu_seqlens.+ mask = None+ if cu_seqlens is not None:+ cu = cu_seqlens.to(device = query.device, dtype = torch.int32)+ longest = int((cu[1:] - cu[:-1]).max())+ kwargs.update(+ cu_seq_lens_q = cu, cu_seq_lens_k = cu, max_length_q = longest, max_length_k = longest+ )+ elif implementation == "sdpa" and cu_seqlens is None and window is None:+ mask = None # sdpa's is_causal path+ else:+ mask = _whole_sequence_mask(total, query.device, query.dtype, window, cu_seqlens)+ out, _ = attention_fn(shard, query, key, value, mask, **kwargs)+ # [B, T, H / N, D] -> [B, T / N, H, D]+ out = out.reshape(bsz, total, plan.local_heads, -1)+ return head_to_seq(out, group), None- # "eager" is a per-model function some transformers releases call without the registry.- current = None if implementation == "eager" else ALL_ATTENTION_FUNCTIONS.get(implementation)- if current is None:- raise NotImplementedError(- "Unsloth: Ulysses context parallelism for hybrid models needs attn_implementation "- f"'sdpa' or a flash attention backend, not {implementation!r}."- )- module._unsloth_cp_ulysses_attention = True- if getattr(current, "_unsloth_cp_ulysses", False):- return+def _ulysses_attention_wrapper(current, implementation, get_cp_manager):@functools.wraps(current)def attention(module, query, key, value, attention_mask, *args, **kwargs):manager = get_cp_manager()@@ -294,9 +450,55 @@ def install_hf_attention_wrapper(module, implementation, get_cp_manager) -> Nonereturn current(module, query, key, value, attention_mask, *args, **kwargs)if args:raise TypeError("Unsloth: Ulysses attention expects keyword arguments after the mask.")+ if kwargs.get("packed_seq_lengths") is None:+ # Some decoders do not forward packed_seq_lengths to the attention layers.+ kwargs["packed_seq_lengths"] = getattr(manager, "packed_seq_lengths", None)return _ulysses_hf_attention(- module, query, key, value, attention_mask, group = manager.group, **kwargs+ module,+ query,+ key,+ value,+ attention_mask,+ group = manager.group,+ attention_fn = current,+ implementation = implementation,+ **kwargs,)attention._unsloth_cp_ulysses = True- ALL_ATTENTION_FUNCTIONS[implementation] = attention+ return attention+++def install_hf_attention_wrapper(module, implementation, get_cp_manager) -> None:+ """Route this module's attention function (ALL_ATTENTION_FUNCTIONS[implementation], or the modeling+ module's eager_attention_forward for "eager") through Ulysses while a Ulysses manager is active."""+ _refuse_non_causal(module)+ _route_hf_forward(module, get_cp_manager)+ if implementation == "eager":+ globs = _forward_globals(module)+ current = globs.get("eager_attention_forward")+ if current is None:+ raise NotImplementedError(+ "Unsloth: Ulysses context parallelism found no eager_attention_forward for "+ f"{type(module).__name__}; use attn_implementation = 'sdpa'."+ )+ if not getattr(current, "_unsloth_cp_ulysses", False):+ globs["eager_attention_forward"] = _ulysses_attention_wrapper(+ current, implementation, get_cp_manager+ )+ module._unsloth_cp_ulysses_attention = True+ return+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS++ current = ALL_ATTENTION_FUNCTIONS.get(implementation)+ if current is None or implementation == "flex_attention":+ raise NotImplementedError(+ "Unsloth: Ulysses context parallelism needs attn_implementation 'sdpa', 'eager' or a flash "+ f"attention backend, not {implementation!r}."+ )+ module._unsloth_cp_ulysses_attention = True+ if getattr(current, "_unsloth_cp_ulysses", False):+ return+ ALL_ATTENTION_FUNCTIONS[implementation] = _ulysses_attention_wrapper(+ current, implementation, get_cp_manager+ )