Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
ORPO / CPO / DPO / KTO chunked head: fall back under torch's global module hooks (#15259)
Shubhankar Tripathy22a024f05193
diff --git a/tests/python/test_gkd_chunked_jsd.py b/tests/python/test_gkd_chunked_jsd.pyindex da36d9731bd..8818ab9390d 100644--- a/tests/python/test_gkd_chunked_jsd.py+++ b/tests/python/test_gkd_chunked_jsd.py@@ -365,6 +365,8 @@ def test_generated_items_are_self_contained():for name in ("_unsloth_gkd_chunked_loss","_unsloth_gkd_dense_head",+ "_unsloth_gkd_head_hooks",+ "_unsloth_global_module_hooks","_unsloth_gkd_logit_transforms","_unsloth_gkd_project","_unsloth_gkd_canonical",@@ -711,3 +713,79 @@ def test_fp16_grad_scaler_reaches_the_chunked_jsd(enabled, accepts_scaler, monkeassert calls[0]["scaler"] is scalerelse:assert calls[0].get("scaler") is None+++_GLOBAL_HOOKS = {+ "forward": "register_module_forward_hook",+ "forward_pre": "register_module_forward_pre_hook",+ "full_backward": "register_module_full_backward_hook",+ "full_backward_pre": "register_module_full_backward_pre_hook",+}++[email protected]("hook", list(_GLOBAL_HOOKS))+def test_global_module_hooks_fall_back(hook):+ """The chunked JSD reads the head weights directly, so a torch global module hook would never see the heads."""+ from torch.nn.modules import module as module_hooks++ student, teacher = _TinyLM(97, 16, 0.0, 1), _TinyLM(97, 24, 0.0, 2)+ layout = {"shift": "shift", "num_items_in_batch": False}+ trainer = _trainer(0.5, student, teacher)+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is not None+ handle = getattr(module_hooks, _GLOBAL_HOOKS[hook])(lambda *a: None)+ try:+ trainer = _trainer(0.5, student, teacher)+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is None+ finally:+ handle.remove()+ assert any("hooks" in reason for reason in trainer._unsloth_gkd_chunked_fallbacks)+ trainer = _trainer(0.5, student, teacher)+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is not None++[email protected]("which", ["student", "teacher"])+ "register",+ [+ "register_forward_hook",+ "register_forward_pre_hook",+ "register_full_backward_hook",+ "register_full_backward_pre_hook",+ ],+)+def test_hooked_heads_fall_back(which, register):+ """A forward / backward hook on either head (a probe, a logit filter) only runs when the head is called."""+ student, teacher = _TinyLM(97, 16, 0.0, 1), _TinyLM(97, 24, 0.0, 2)+ layout = {"shift": "shift", "num_items_in_batch": False}+ head = (student if which == "student" else teacher).lm_head+ handle = getattr(head, register)(lambda *a: None)+ trainer = _trainer(0.5, student, teacher)+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is None+ assert any("hooks" in reason for reason in trainer._unsloth_gkd_chunked_fallbacks)+ handle.remove()+ trainer = _trainer(0.5, student, teacher)+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is not None+++def test_trl_activation_offloading_head_hooks_keep_the_chunked_jsd():+ """TRL's activation_offloading pair on lm_head only pauses offloading, so it must not force the dense path."""+ student, teacher = _TinyLM(97, 16, 0.0, 1), _TinyLM(97, 24, 0.0, 2)+ layout = {"shift": "shift", "num_items_in_batch": False}+ pre, post = (lambda *a: None), (lambda *a: None)+ for hook in (pre, post):+ hook.__module__ = "trl.models.activation_offloading"+ head = student.lm_head+ handles = [+ head.register_forward_pre_hook(pre),+ head.register_forward_hook(post, always_call = True),+ ]+ try:+ trainer = _trainer(0.5, student, teacher)+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is not None+ other = head.register_forward_hook(lambda *a: None)+ trainer = _trainer(0.5, student, teacher)+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is None+ other.remove()+ finally:+ for handle in handles:+ handle.remove()diff --git a/tests/python/test_preference_chunked_logprobs.py b/tests/python/test_preference_chunked_logprobs.pyindex 07983e0ba93..5803db4935d 100644--- a/tests/python/test_preference_chunked_logprobs.py+++ b/tests/python/test_preference_chunked_logprobs.py@@ -359,6 +359,15 @@ def test_declines_without_kernel_or_on_unsupported_setups(monkeypatch):for hook in ("get_batch_loss_metrics", "get_batch_logps", "concatenated_forward"):sub = type("Sub", (cls,), {hook: lambda self, *a, **k: None})assert rl._unsloth_pref_chunked_head(_trainer(sub, "orpo"), model) is None, hook+ # A global module hook (register_module_forward_hook) fires for the head only when it is called.+ from torch.nn.modules import module as module_hooks++ handle = module_hooks.register_module_forward_hook(lambda m, i, o: None)+ try:+ assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is None+ finally:+ handle.remove()+ assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is not Nonemonkeypatch.setenv("UNSLOTH_PREFERENCE_CHUNKED", "0")assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is Nonemonkeypatch.delenv("UNSLOTH_PREFERENCE_CHUNKED")@@ -385,6 +394,7 @@ def test_generated_items_are_self_contained():"_unsloth_pref_chunked_forward","_unsloth_pref_logit_transforms","_unsloth_gkd_dense_head",+ "_unsloth_global_module_hooks","_unsloth_grpo_returns_hidden_states","_unsloth_grpo_hidden_states_signal","_unsloth_pref_returns_hidden_states",@@ -415,3 +425,64 @@ def test_square_head_needs_an_affirmative_hidden_state_signal():assert rl._unsloth_pref_returns_hidden_states(marked, out, square) is True# GRPO keeps its leniency for a zoo too old to write the marker.assert rl._unsloth_grpo_returns_hidden_states(plain, out, square) is True+++def test_every_generated_trainer_with_a_head_guard_defines_the_global_hook_check():+ # A trainer that copies a chunked-head guard into its source must copy the helper it calls.+ for trainer, items in rl.RL_PRE_ITEMS.items():+ pre = "\n".join(items)+ if any(+ f"def {guard}(" in pre+ for guard in (+ "_unsloth_dpo_chunked_head",+ "_unsloth_pref_chunked_head",+ "_unsloth_gkd_chunked_loss",+ )+ ):+ assert "def _unsloth_global_module_hooks(" in pre, trainer+++_GLOBAL_HOOKS = {+ "forward": "register_module_forward_hook",+ "forward_pre": "register_module_forward_pre_hook",+ "full_backward": "register_module_full_backward_hook",+ "full_backward_pre": "register_module_full_backward_pre_hook",+}++[email protected]("hook", list(_GLOBAL_HOOKS))+def test_global_module_hooks_see_the_head_through_trl_math(hook, monkeypatch):+ """Every torch global module hook must still fire on the head: the chunked path declines while one is registered,+ gives TRL's numbers, and engages again once it is removed."""+ from torch.nn.modules import module as module_hooks++ monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES", raising = False)+ cls, source = _trl_class("orpo")+ stock = _compile(source)+ patched = _compile(rl.orpo_trainer_concatenated_forward("concatenated_forward", source))+ model = _HiddenStatesLM(_ToyLM())+ head = model.get_output_embeddings()+ params = list(model.parameters())+ batch = _batch(61)+ seen = []+ handle = getattr(module_hooks, _GLOBAL_HOOKS[hook])(+ lambda m, *a: seen.append(1) if m is head else None+ )+ try:+ want = _run(_trainer(cls, "orpo"), stock, model, batch, params)+ assert seen, "control: the hook fires on the head under TRL's own forward"+ seen.clear()+ trainer = _trainer(cls, "orpo")+ got = _run(trainer, patched, model, batch, params)+ finally:+ handle.remove()+ assert getattr(trainer, "_unsloth_pref_chunked_calls", 0) == 0+ assert seen, "the chunked path skipped a global module hook on the head"+ assert torch.equal(got[0], want[0])+ for g, w in zip(got[2], want[2]):+ assert torch.equal(g, w)+ trainer = _trainer(cls, "orpo")+ _run(trainer, patched, model, batch, params)+ assert (+ trainer._unsloth_pref_chunked_calls == 1+ ), "removing the hook must restore the chunked path"diff --git a/tests/test_dpo_chunked_logprobs.py b/tests/test_dpo_chunked_logprobs.pyindex f7af8269ad5..ddcc5093848 100644--- a/tests/test_dpo_chunked_logprobs.py+++ b/tests/test_dpo_chunked_logprobs.py@@ -38,6 +38,7 @@ HELPERS = ("_unsloth_grpo_hidden_states_signal","_unsloth_pref_returns_hidden_states","_unsloth_gkd_dense_head",+ "_unsloth_global_module_hooks","_unsloth_gkd_logit_transforms","_unsloth_pref_logit_transforms","_unsloth_stock_logps_hook",@@ -328,6 +329,35 @@ def test_use_weighting_falls_back(env):assert torch.allclose(loss, base_loss, atol = 1e-6)+_GLOBAL_HOOKS = {+ "forward": "register_module_forward_hook",+ "forward_pre": "register_module_forward_pre_hook",+ "full_backward": "register_module_full_backward_hook",+ "full_backward_pre": "register_module_full_backward_pre_hook",+}++[email protected]("hook", list(_GLOBAL_HOOKS))+def test_global_module_hook_falls_back(env, hook):+ # torch's global module hooks apply to every module: the fused path never calls the head.+ from torch.nn.modules import module as module_hooks++ dpo_mod, forward = env+ _, base_loss, _, _ = _run(dpo_mod, forward, {}, chunked = False)+ trainer, _, _, _ = _run(dpo_mod, forward, {}, chunked = True)+ assert trainer._unsloth_dpo_chunked_calls > 0, "control: the fused path runs without hooks"+ handle = getattr(module_hooks, _GLOBAL_HOOKS[hook])(lambda *a: None)+ try:+ trainer, loss, _, _ = _run(dpo_mod, forward, {}, chunked = True)+ finally:+ handle.remove()+ assert getattr(trainer, "_unsloth_dpo_chunked_calls", 0) == 0+ assert "output head has hooks or a patched forward" in trainer._unsloth_dpo_chunked_fallbacks+ assert torch.allclose(loss, base_loss, atol = 1e-6)+ trainer, _, _, _ = _run(dpo_mod, forward, {}, chunked = True)+ assert trainer._unsloth_dpo_chunked_calls > 0, "removing the hook must restore the fused path"++TRL_TAGS = ["v0.18.2","v0.19.1",diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 2c07c25aaa2..3575b69c0dd 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -6171,6 +6171,31 @@ def _unsloth_gkd_layout(source):return {"shift": "prompt" if is_prompt else "shift", "num_items_in_batch": bool(calls[0])}+def _unsloth_global_module_hooks():+ # torch's register_module_*_hook globals apply to every module, the head included.+ from torch.nn.modules import module as _module+ return bool(+ getattr(_module, "_global_forward_hooks", None)+ or getattr(_module, "_global_forward_pre_hooks", None)+ or getattr(_module, "_global_backward_hooks", None)+ or getattr(_module, "_global_backward_pre_hooks", None)+ )+++def _unsloth_gkd_head_hooks(head):+ # TRL activation_offloading puts a pre / post forward pair on lm_head that only pauses offloading around+ # the logits; skipping both is safe and keeps the chunked JSD on, so only other hooks count.+ forward = (*head._forward_hooks.values(), *head._forward_pre_hooks.values())+ return bool(+ any(+ getattr(hook, "__module__", None) != "trl.models.activation_offloading"+ for hook in forward+ )+ or head._backward_hooks+ or getattr(head, "_backward_pre_hooks", None)+ )++def _unsloth_gkd_dense_head(model):"""The output head when it is a plain dense ``[vocab, hidden]`` projection the chunked loss can read directly.Rejects bnb heads (``nn.Linear`` subclasses with packed weights), PEFT heads and ZeRO-3 partitioned weights.@@ -6407,6 +6432,11 @@ def _unsloth_gkd_chunked_loss(self, model, inputs, num_items_in_batch, layout):return _unsloth_gkd_note_fallback(self, "output head is not a dense nn.Linear")if student_head.weight.shape[0] != teacher_head.weight.shape[0]:return _unsloth_gkd_note_fallback(self, "vocab mismatch")+ # The JSD bypasses head calls, so head / global hooks would not fire; accelerate's _hf_hook is fine.+ if _unsloth_global_module_hooks() or any(+ _unsloth_gkd_head_hooks(head) for head in (student_head, teacher_head)+ ):+ return _unsloth_gkd_note_fallback(self, "output head has forward / backward hooks")# MiniCPM3 divides hidden states by logits_scaling before lm_head; a wrapped forward's hidden_states[-1] predates it.for unwrapped in (unwrapped_student, unwrapped_teacher):for config in _unsloth_text_configs(_unsloth_get_model_config(unwrapped)):@@ -6590,6 +6620,8 @@ for _gkd_item in (_unsloth_gkd_jsd_supported,_unsloth_gkd_note_fallback,_unsloth_gkd_dense_head,+ _unsloth_gkd_head_hooks,+ _unsloth_global_module_hooks,_unsloth_gkd_logit_transforms,_unsloth_gkd_project,_unsloth_gkd_chunk_size,@@ -6685,7 +6717,8 @@ def _unsloth_pref_chunked_head(self, model):head = _unsloth_gkd_dense_head(unwrapped)if head is None:return None- # The kernel reads weight / bias directly: a hooked head (accelerate, forward / backward hooks) is not+ # The kernel reads weight / bias directly: a hooked head (accelerate, forward / backward+ # hooks, torch's global module hooks) is not# F.linear(h, W, b), and its backward hooks would never see the gradient.if (hasattr(head, "_hf_hook")@@ -6694,6 +6727,7 @@ def _unsloth_pref_chunked_head(self, model):or head._forward_pre_hooksor head._backward_hooksor getattr(head, "_backward_pre_hooks", None)+ or _unsloth_global_module_hooks()):return None# MiniCPM3 divides hidden states by logits_scaling before the head.@@ -6927,6 +6961,7 @@ for _pref_trainer in ("orpo_trainer", "cpo_trainer"):_unsloth_grpo_hidden_states_signal,_unsloth_pref_returns_hidden_states,_unsloth_gkd_dense_head,+ _unsloth_global_module_hooks,_unsloth_stock_logps_hook,_unsloth_pref_chunked_head,_unsloth_gkd_logit_transforms,@@ -7010,7 +7045,8 @@ def _unsloth_dpo_chunked_head(head = _unsloth_gkd_dense_head(unwrapped)if head is None:reason = "output head is not a dense nn.Linear"- # The kernel reads weight / bias directly: a hooked head (accelerate, forward / backward hooks) is not+ # The kernel reads weight / bias directly: a hooked head (accelerate, forward / backward+ # hooks, torch's global module hooks) is not# F.linear(h, W, b), and its backward hooks would never see the gradient.elif (hasattr(head, "_hf_hook")@@ -7019,6 +7055,7 @@ def _unsloth_dpo_chunked_head(or head._forward_pre_hooksor head._backward_hooksor getattr(head, "_backward_pre_hooks", None)+ or _unsloth_global_module_hooks()):reason = "output head has hooks or a patched forward"if reason is None:@@ -7436,6 +7473,7 @@ for _dpo_item in (_unsloth_grpo_hidden_states_signal,_unsloth_pref_returns_hidden_states,_unsloth_gkd_dense_head,+ _unsloth_global_module_hooks,_unsloth_gkd_logit_transforms,_unsloth_pref_logit_transforms,_unsloth_stock_logps_hook,@@ -7660,6 +7698,7 @@ for _kto_item in (_unsloth_grpo_hidden_states_signal,_unsloth_pref_returns_hidden_states,_unsloth_gkd_dense_head,+ _unsloth_global_module_hooks,_unsloth_gkd_logit_transforms,_unsloth_pref_logit_transforms,_unsloth_stock_logps_hook,@@ -8588,6 +8627,7 @@ for _odpo_trainer in ("bco_trainer", "online_dpo_trainer", "xpo_trainer", "nash__unsloth_grpo_hidden_states_signal,_unsloth_pref_returns_hidden_states,_unsloth_gkd_dense_head,+ _unsloth_global_module_hooks,_unsloth_gkd_logit_transforms,_unsloth_pref_logit_transforms,_unsloth_stock_logps_hook,