Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

ORPO / CPO / DPO / KTO chunked head: fall back under torch's global module hooks (#15259)

Shubhankar Tripathy22a024f05193
diff --git a/tests/python/test_gkd_chunked_jsd.py b/tests/python/test_gkd_chunked_jsd.py
index da36d9731bd..8818ab9390d 100644
--- a/tests/python/test_gkd_chunked_jsd.py
+++ b/tests/python/test_gkd_chunked_jsd.py
@@ -365,6 +365,8 @@ def test_generated_items_are_self_contained():
for name in (
"_unsloth_gkd_chunked_loss",
"_unsloth_gkd_dense_head",
+ "_unsloth_gkd_head_hooks",
+ "_unsloth_global_module_hooks",
"_unsloth_gkd_logit_transforms",
"_unsloth_gkd_project",
"_unsloth_gkd_canonical",
@@ -711,3 +713,79 @@ def test_fp16_grad_scaler_reaches_the_chunked_jsd(enabled, accepts_scaler, monke
assert calls[0]["scaler"] is scaler
else:
assert calls[0].get("scaler") is None
+
+
+_GLOBAL_HOOKS = {
+ "forward": "register_module_forward_hook",
+ "forward_pre": "register_module_forward_pre_hook",
+ "full_backward": "register_module_full_backward_hook",
+ "full_backward_pre": "register_module_full_backward_pre_hook",
+}
+
+
[email protected]("hook", list(_GLOBAL_HOOKS))
+def test_global_module_hooks_fall_back(hook):
+ """The chunked JSD reads the head weights directly, so a torch global module hook would never see the heads."""
+ from torch.nn.modules import module as module_hooks
+
+ student, teacher = _TinyLM(97, 16, 0.0, 1), _TinyLM(97, 24, 0.0, 2)
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ trainer = _trainer(0.5, student, teacher)
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is not None
+ handle = getattr(module_hooks, _GLOBAL_HOOKS[hook])(lambda *a: None)
+ try:
+ trainer = _trainer(0.5, student, teacher)
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is None
+ finally:
+ handle.remove()
+ assert any("hooks" in reason for reason in trainer._unsloth_gkd_chunked_fallbacks)
+ trainer = _trainer(0.5, student, teacher)
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is not None
+
+
[email protected]("which", ["student", "teacher"])
+ "register",
+ [
+ "register_forward_hook",
+ "register_forward_pre_hook",
+ "register_full_backward_hook",
+ "register_full_backward_pre_hook",
+ ],
+)
+def test_hooked_heads_fall_back(which, register):
+ """A forward / backward hook on either head (a probe, a logit filter) only runs when the head is called."""
+ student, teacher = _TinyLM(97, 16, 0.0, 1), _TinyLM(97, 24, 0.0, 2)
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ head = (student if which == "student" else teacher).lm_head
+ handle = getattr(head, register)(lambda *a: None)
+ trainer = _trainer(0.5, student, teacher)
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is None
+ assert any("hooks" in reason for reason in trainer._unsloth_gkd_chunked_fallbacks)
+ handle.remove()
+ trainer = _trainer(0.5, student, teacher)
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is not None
+
+
+def test_trl_activation_offloading_head_hooks_keep_the_chunked_jsd():
+ """TRL's activation_offloading pair on lm_head only pauses offloading, so it must not force the dense path."""
+ student, teacher = _TinyLM(97, 16, 0.0, 1), _TinyLM(97, 24, 0.0, 2)
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ pre, post = (lambda *a: None), (lambda *a: None)
+ for hook in (pre, post):
+ hook.__module__ = "trl.models.activation_offloading"
+ head = student.lm_head
+ handles = [
+ head.register_forward_pre_hook(pre),
+ head.register_forward_hook(post, always_call = True),
+ ]
+ try:
+ trainer = _trainer(0.5, student, teacher)
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is not None
+ other = head.register_forward_hook(lambda *a: None)
+ trainer = _trainer(0.5, student, teacher)
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is None
+ other.remove()
+ finally:
+ for handle in handles:
+ handle.remove()
diff --git a/tests/python/test_preference_chunked_logprobs.py b/tests/python/test_preference_chunked_logprobs.py
index 07983e0ba93..5803db4935d 100644
--- a/tests/python/test_preference_chunked_logprobs.py
+++ b/tests/python/test_preference_chunked_logprobs.py
@@ -359,6 +359,15 @@ def test_declines_without_kernel_or_on_unsupported_setups(monkeypatch):
for hook in ("get_batch_loss_metrics", "get_batch_logps", "concatenated_forward"):
sub = type("Sub", (cls,), {hook: lambda self, *a, **k: None})
assert rl._unsloth_pref_chunked_head(_trainer(sub, "orpo"), model) is None, hook
+ # A global module hook (register_module_forward_hook) fires for the head only when it is called.
+ from torch.nn.modules import module as module_hooks
+
+ handle = module_hooks.register_module_forward_hook(lambda m, i, o: None)
+ try:
+ assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is None
+ finally:
+ handle.remove()
+ assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is not None
monkeypatch.setenv("UNSLOTH_PREFERENCE_CHUNKED", "0")
assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is None
monkeypatch.delenv("UNSLOTH_PREFERENCE_CHUNKED")
@@ -385,6 +394,7 @@ def test_generated_items_are_self_contained():
"_unsloth_pref_chunked_forward",
"_unsloth_pref_logit_transforms",
"_unsloth_gkd_dense_head",
+ "_unsloth_global_module_hooks",
"_unsloth_grpo_returns_hidden_states",
"_unsloth_grpo_hidden_states_signal",
"_unsloth_pref_returns_hidden_states",
@@ -415,3 +425,64 @@ def test_square_head_needs_an_affirmative_hidden_state_signal():
assert rl._unsloth_pref_returns_hidden_states(marked, out, square) is True
# GRPO keeps its leniency for a zoo too old to write the marker.
assert rl._unsloth_grpo_returns_hidden_states(plain, out, square) is True
+
+
+def test_every_generated_trainer_with_a_head_guard_defines_the_global_hook_check():
+ # A trainer that copies a chunked-head guard into its source must copy the helper it calls.
+ for trainer, items in rl.RL_PRE_ITEMS.items():
+ pre = "\n".join(items)
+ if any(
+ f"def {guard}(" in pre
+ for guard in (
+ "_unsloth_dpo_chunked_head",
+ "_unsloth_pref_chunked_head",
+ "_unsloth_gkd_chunked_loss",
+ )
+ ):
+ assert "def _unsloth_global_module_hooks(" in pre, trainer
+
+
+_GLOBAL_HOOKS = {
+ "forward": "register_module_forward_hook",
+ "forward_pre": "register_module_forward_pre_hook",
+ "full_backward": "register_module_full_backward_hook",
+ "full_backward_pre": "register_module_full_backward_pre_hook",
+}
+
+
[email protected]("hook", list(_GLOBAL_HOOKS))
+def test_global_module_hooks_see_the_head_through_trl_math(hook, monkeypatch):
+ """Every torch global module hook must still fire on the head: the chunked path declines while one is registered,
+ gives TRL's numbers, and engages again once it is removed."""
+ from torch.nn.modules import module as module_hooks
+
+ monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES", raising = False)
+ cls, source = _trl_class("orpo")
+ stock = _compile(source)
+ patched = _compile(rl.orpo_trainer_concatenated_forward("concatenated_forward", source))
+ model = _HiddenStatesLM(_ToyLM())
+ head = model.get_output_embeddings()
+ params = list(model.parameters())
+ batch = _batch(61)
+ seen = []
+ handle = getattr(module_hooks, _GLOBAL_HOOKS[hook])(
+ lambda m, *a: seen.append(1) if m is head else None
+ )
+ try:
+ want = _run(_trainer(cls, "orpo"), stock, model, batch, params)
+ assert seen, "control: the hook fires on the head under TRL's own forward"
+ seen.clear()
+ trainer = _trainer(cls, "orpo")
+ got = _run(trainer, patched, model, batch, params)
+ finally:
+ handle.remove()
+ assert getattr(trainer, "_unsloth_pref_chunked_calls", 0) == 0
+ assert seen, "the chunked path skipped a global module hook on the head"
+ assert torch.equal(got[0], want[0])
+ for g, w in zip(got[2], want[2]):
+ assert torch.equal(g, w)
+ trainer = _trainer(cls, "orpo")
+ _run(trainer, patched, model, batch, params)
+ assert (
+ trainer._unsloth_pref_chunked_calls == 1
+ ), "removing the hook must restore the chunked path"
diff --git a/tests/test_dpo_chunked_logprobs.py b/tests/test_dpo_chunked_logprobs.py
index f7af8269ad5..ddcc5093848 100644
--- a/tests/test_dpo_chunked_logprobs.py
+++ b/tests/test_dpo_chunked_logprobs.py
@@ -38,6 +38,7 @@ HELPERS = (
"_unsloth_grpo_hidden_states_signal",
"_unsloth_pref_returns_hidden_states",
"_unsloth_gkd_dense_head",
+ "_unsloth_global_module_hooks",
"_unsloth_gkd_logit_transforms",
"_unsloth_pref_logit_transforms",
"_unsloth_stock_logps_hook",
@@ -328,6 +329,35 @@ def test_use_weighting_falls_back(env):
assert torch.allclose(loss, base_loss, atol = 1e-6)
+_GLOBAL_HOOKS = {
+ "forward": "register_module_forward_hook",
+ "forward_pre": "register_module_forward_pre_hook",
+ "full_backward": "register_module_full_backward_hook",
+ "full_backward_pre": "register_module_full_backward_pre_hook",
+}
+
+
[email protected]("hook", list(_GLOBAL_HOOKS))
+def test_global_module_hook_falls_back(env, hook):
+ # torch's global module hooks apply to every module: the fused path never calls the head.
+ from torch.nn.modules import module as module_hooks
+
+ dpo_mod, forward = env
+ _, base_loss, _, _ = _run(dpo_mod, forward, {}, chunked = False)
+ trainer, _, _, _ = _run(dpo_mod, forward, {}, chunked = True)
+ assert trainer._unsloth_dpo_chunked_calls > 0, "control: the fused path runs without hooks"
+ handle = getattr(module_hooks, _GLOBAL_HOOKS[hook])(lambda *a: None)
+ try:
+ trainer, loss, _, _ = _run(dpo_mod, forward, {}, chunked = True)
+ finally:
+ handle.remove()
+ assert getattr(trainer, "_unsloth_dpo_chunked_calls", 0) == 0
+ assert "output head has hooks or a patched forward" in trainer._unsloth_dpo_chunked_fallbacks
+ assert torch.allclose(loss, base_loss, atol = 1e-6)
+ trainer, _, _, _ = _run(dpo_mod, forward, {}, chunked = True)
+ assert trainer._unsloth_dpo_chunked_calls > 0, "removing the hook must restore the fused path"
+
+
TRL_TAGS = [
"v0.18.2",
"v0.19.1",
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 2c07c25aaa2..3575b69c0dd 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -6171,6 +6171,31 @@ def _unsloth_gkd_layout(source):
return {"shift": "prompt" if is_prompt else "shift", "num_items_in_batch": bool(calls[0])}
+def _unsloth_global_module_hooks():
+ # torch's register_module_*_hook globals apply to every module, the head included.
+ from torch.nn.modules import module as _module
+ return bool(
+ getattr(_module, "_global_forward_hooks", None)
+ or getattr(_module, "_global_forward_pre_hooks", None)
+ or getattr(_module, "_global_backward_hooks", None)
+ or getattr(_module, "_global_backward_pre_hooks", None)
+ )
+
+
+def _unsloth_gkd_head_hooks(head):
+ # TRL activation_offloading puts a pre / post forward pair on lm_head that only pauses offloading around
+ # the logits; skipping both is safe and keeps the chunked JSD on, so only other hooks count.
+ forward = (*head._forward_hooks.values(), *head._forward_pre_hooks.values())
+ return bool(
+ any(
+ getattr(hook, "__module__", None) != "trl.models.activation_offloading"
+ for hook in forward
+ )
+ or head._backward_hooks
+ or getattr(head, "_backward_pre_hooks", None)
+ )
+
+
def _unsloth_gkd_dense_head(model):
"""The output head when it is a plain dense ``[vocab, hidden]`` projection the chunked loss can read directly.
Rejects bnb heads (``nn.Linear`` subclasses with packed weights), PEFT heads and ZeRO-3 partitioned weights.
@@ -6407,6 +6432,11 @@ def _unsloth_gkd_chunked_loss(self, model, inputs, num_items_in_batch, layout):
return _unsloth_gkd_note_fallback(self, "output head is not a dense nn.Linear")
if student_head.weight.shape[0] != teacher_head.weight.shape[0]:
return _unsloth_gkd_note_fallback(self, "vocab mismatch")
+ # The JSD bypasses head calls, so head / global hooks would not fire; accelerate's _hf_hook is fine.
+ if _unsloth_global_module_hooks() or any(
+ _unsloth_gkd_head_hooks(head) for head in (student_head, teacher_head)
+ ):
+ return _unsloth_gkd_note_fallback(self, "output head has forward / backward hooks")
# MiniCPM3 divides hidden states by logits_scaling before lm_head; a wrapped forward's hidden_states[-1] predates it.
for unwrapped in (unwrapped_student, unwrapped_teacher):
for config in _unsloth_text_configs(_unsloth_get_model_config(unwrapped)):
@@ -6590,6 +6620,8 @@ for _gkd_item in (
_unsloth_gkd_jsd_supported,
_unsloth_gkd_note_fallback,
_unsloth_gkd_dense_head,
+ _unsloth_gkd_head_hooks,
+ _unsloth_global_module_hooks,
_unsloth_gkd_logit_transforms,
_unsloth_gkd_project,
_unsloth_gkd_chunk_size,
@@ -6685,7 +6717,8 @@ def _unsloth_pref_chunked_head(self, model):
head = _unsloth_gkd_dense_head(unwrapped)
if head is None:
return None
- # The kernel reads weight / bias directly: a hooked head (accelerate, forward / backward hooks) is not
+ # The kernel reads weight / bias directly: a hooked head (accelerate, forward / backward
+ # hooks, torch's global module hooks) is not
# F.linear(h, W, b), and its backward hooks would never see the gradient.
if (
hasattr(head, "_hf_hook")
@@ -6694,6 +6727,7 @@ def _unsloth_pref_chunked_head(self, model):
or head._forward_pre_hooks
or head._backward_hooks
or getattr(head, "_backward_pre_hooks", None)
+ or _unsloth_global_module_hooks()
):
return None
# MiniCPM3 divides hidden states by logits_scaling before the head.
@@ -6927,6 +6961,7 @@ for _pref_trainer in ("orpo_trainer", "cpo_trainer"):
_unsloth_grpo_hidden_states_signal,
_unsloth_pref_returns_hidden_states,
_unsloth_gkd_dense_head,
+ _unsloth_global_module_hooks,
_unsloth_stock_logps_hook,
_unsloth_pref_chunked_head,
_unsloth_gkd_logit_transforms,
@@ -7010,7 +7045,8 @@ def _unsloth_dpo_chunked_head(
head = _unsloth_gkd_dense_head(unwrapped)
if head is None:
reason = "output head is not a dense nn.Linear"
- # The kernel reads weight / bias directly: a hooked head (accelerate, forward / backward hooks) is not
+ # The kernel reads weight / bias directly: a hooked head (accelerate, forward / backward
+ # hooks, torch's global module hooks) is not
# F.linear(h, W, b), and its backward hooks would never see the gradient.
elif (
hasattr(head, "_hf_hook")
@@ -7019,6 +7055,7 @@ def _unsloth_dpo_chunked_head(
or head._forward_pre_hooks
or head._backward_hooks
or getattr(head, "_backward_pre_hooks", None)
+ or _unsloth_global_module_hooks()
):
reason = "output head has hooks or a patched forward"
if reason is None:
@@ -7436,6 +7473,7 @@ for _dpo_item in (
_unsloth_grpo_hidden_states_signal,
_unsloth_pref_returns_hidden_states,
_unsloth_gkd_dense_head,
+ _unsloth_global_module_hooks,
_unsloth_gkd_logit_transforms,
_unsloth_pref_logit_transforms,
_unsloth_stock_logps_hook,
@@ -7660,6 +7698,7 @@ for _kto_item in (
_unsloth_grpo_hidden_states_signal,
_unsloth_pref_returns_hidden_states,
_unsloth_gkd_dense_head,
+ _unsloth_global_module_hooks,
_unsloth_gkd_logit_transforms,
_unsloth_pref_logit_transforms,
_unsloth_stock_logps_hook,
@@ -8588,6 +8627,7 @@ for _odpo_trainer in ("bco_trainer", "online_dpo_trainer", "xpo_trainer", "nash_
_unsloth_grpo_hidden_states_signal,
_unsloth_pref_returns_hidden_states,
_unsloth_gkd_dense_head,
+ _unsloth_global_module_hooks,
_unsloth_gkd_logit_transforms,
_unsloth_pref_logit_transforms,
_unsloth_stock_logps_hook,