Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO: score a hooked output head through its own forward (#17050)
Daniel Hana1748239658b
diff --git a/tests/python/test_grpo_head_hooks.py b/tests/python/test_grpo_head_hooks.pynew file mode 100644index 00000000000..57eaf0abac9--- /dev/null+++ b/tests/python/test_grpo_head_hooks.py@@ -0,0 +1,319 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""GRPO scores with real logits from the model's own head call when the head is hooked.++The hidden-states scorers read lm_head.weight directly and never call the head, so its forward / backward hooks+and torch's global module hooks would never see it."""++from __future__ import annotations++import ast+import functools+import inspect+import os+import textwrap++import pytest+import torch++os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")+import unsloth.models.rl_replacements as rl++from torch.nn.modules import module as _torch_module++HIDDEN, VOCAB = 8, 16+++class _TinyLM(torch.nn.Module):+ """Honours UNSLOTH_RETURN_HIDDEN_STATES like Unsloth's generated forwards: hidden states, else its own head."""++ def __init__(+ self,+ hidden = HIDDEN,+ vocab = VOCAB,+ ):+ super().__init__()+ torch.manual_seed(0)+ self.embed = torch.nn.Embedding(vocab, hidden)+ self.lm_head = torch.nn.Linear(hidden, vocab, bias = False)+ self.hidden_state_calls = 0++ def get_output_embeddings(self):+ return self.lm_head++ def forward(self, input_ids):+ hidden = self.embed(input_ids)+ if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":+ self.hidden_state_calls += 1+ return type("Out", (), {"logits": hidden})()+ return type("Out", (), {"logits": self.lm_head(hidden)})()+++def _owned_by(module_name):+ def hook(*args, **kwargs):+ return None++ hook.__module__ = module_name+ return hook++[email protected](autouse = True)+def _no_global_hooks():+ tables = (+ "_global_forward_hooks",+ "_global_forward_pre_hooks",+ "_global_backward_hooks",+ "_global_backward_pre_hooks",+ )+ assert not any(getattr(_torch_module, name, None) for name in tables)+ yield+ assert not any(getattr(_torch_module, name, None) for name in tables)+++def test_plain_head_keeps_hidden_states():+ assert rl._unsloth_grpo_head_hooked(_TinyLM()) is False+++ "register",+ [+ lambda head: head.register_forward_hook(lambda m, i, o: None),+ lambda head: head.register_forward_pre_hook(lambda m, i: None),+ lambda head: head.register_full_backward_hook(lambda m, gi, go: None),+ lambda head: head.register_full_backward_pre_hook(lambda m, go: None),+ lambda head: head.register_forward_hook(+ functools.partial(lambda tag, m, i, o: None, "user")+ ),+ ],+ ids = ["forward", "forward_pre", "backward", "backward_pre", "partial"],+)+def test_hooked_heads_fall_back(register):+ model = _TinyLM()+ handle = register(model.lm_head)+ assert rl._unsloth_grpo_head_hooked(model) is True+ handle.remove()+ assert rl._unsloth_grpo_head_hooked(model) is False+++def test_patched_head_forward_falls_back():+ model = _TinyLM()+ model.lm_head.forward = lambda x: torch.nn.functional.linear(x, model.lm_head.weight) * 2+ assert rl._unsloth_grpo_head_hooked(model) is True+++ "register",+ [+ lambda: _torch_module.register_module_forward_hook(lambda m, i, o: None),+ lambda: _torch_module.register_module_forward_pre_hook(lambda m, i: None),+ lambda: _torch_module.register_module_full_backward_hook(lambda m, gi, go: None),+ lambda: _torch_module.register_module_full_backward_pre_hook(lambda m, go: None),+ ],+ ids = ["forward", "forward_pre", "backward", "backward_pre"],+)+def test_global_module_hooks_fall_back(register):+ model = _TinyLM()+ handle = register()+ try:+ assert rl._unsloth_grpo_head_hooked(model) is True+ finally:+ handle.remove()+ assert rl._unsloth_grpo_head_hooked(model) is False+++ "owner",+ [+ "unsloth_zoo.training_utils",+ "unsloth.models.vision",+ "deepspeed.runtime.zero.parameter_offload",+ "torch.distributed.fsdp._fully_shard._fsdp_state",+ ],+)+def test_internal_hooks_keep_hidden_states(owner):+ """Hooks the scorers already account for (ZeRO-3 gather, FSDP2 unshard, Unsloth's own) keep the fused route."""+ model = _TinyLM()+ model.lm_head.register_forward_pre_hook(_owned_by(owner))+ model.lm_head.register_forward_hook(_owned_by(owner))+ assert rl._unsloth_grpo_head_hooked(model) is False+ model.lm_head.register_forward_hook(lambda m, i, o: None)+ assert rl._unsloth_grpo_head_hooked(model) is True+++def test_accelerate_device_hook_keeps_hidden_states():+ hooks = pytest.importorskip("accelerate.hooks")+ model = _TinyLM()+ hooks.add_hook_to_module(model.lm_head, hooks.ModelHook())+ assert "forward" in vars(model.lm_head)+ assert rl._unsloth_grpo_head_hooked(model) is False+ model.lm_head.register_forward_hook(lambda m, i, o: None)+ assert rl._unsloth_grpo_head_hooked(model) is True+++def test_square_head_keeps_hidden_states():+ """vocab_size == hidden_size: the width dispatch could not tell real logits from hidden states."""+ model = _TinyLM(hidden = VOCAB, vocab = VOCAB)+ model.lm_head.register_forward_hook(lambda m, i, o: None)+ assert rl._unsloth_grpo_head_hooked(model) is False+++def test_models_without_an_output_head_are_left_alone():+ assert rl._unsloth_grpo_head_hooked(torch.nn.Linear(2, 2)) is False+++# zoo's grpo_accumulated_loss as unsloth embeds it: one hidden-states request, one width dispatch.+_ZOO_SHAPED = """+def grpo_accumulated_loss(trainer, input_ids):+ _prior = os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES")+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"+ try:+ unwrapped_model = trainer.model+ lm_head = unwrapped_model.get_output_embeddings().weight+ new_hidden_states_chunk = unwrapped_model(input_ids).logits+ if new_hidden_states_chunk.shape[-1] == lm_head.shape[1]:+ logits = new_hidden_states_chunk @ lm_head.t()+ else:+ logits = new_hidden_states_chunk+ return torch.log_softmax(logits.float(), dim = -1)+ finally:+ if _prior is None:+ os.environ.pop("UNSLOTH_RETURN_HIDDEN_STATES", None)+ else:+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = _prior+"""+++def _patched_gradient_function(source):+ source = rl._patch_grpo_accumulated_loss_head_hooks(+ rl._patch_grpo_accumulated_loss_hidden_states_dispatch(source)+ )+ namespace = {+ "os": os,+ "torch": torch,+ "_unsloth_grpo_returns_hidden_states": rl._unsloth_grpo_returns_hidden_states,+ "_unsloth_grpo_hidden_states_signal": rl._unsloth_grpo_hidden_states_signal,+ "_unsloth_grpo_head_hooked": rl._unsloth_grpo_head_hooked,+ }+ exec(compile(source, "<patched grpo_accumulated_loss>", "exec"), namespace)+ return namespace["grpo_accumulated_loss"]+++def test_gradient_path_runs_a_hooked_head_and_its_backward_hook():+ function = _patched_gradient_function(_ZOO_SHAPED)+ model = _TinyLM()+ trainer = type("Trainer", (), {"model": model})()+ input_ids = torch.randint(0, VOCAB, (2, 5), generator = torch.Generator().manual_seed(1))++ unhooked = function(trainer, input_ids)+ assert model.hidden_state_calls == 1++ seen = {"forward": 0, "backward": 0}+ model.lm_head.register_forward_hook(+ lambda m, i, o: seen.__setitem__("forward", seen["forward"] + 1)+ )+ model.lm_head.register_full_backward_hook(+ lambda m, gi, go: seen.__setitem__("backward", seen["backward"] + 1)+ )+ hooked = function(trainer, input_ids)+ hooked.sum().backward()+ assert model.hidden_state_calls == 1 # no new hidden-states forward+ assert seen == {"forward": 1, "backward": 1}+ torch.testing.assert_close(hooked, unhooked)+ assert os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES") is None+++def test_installed_zoo_gradient_path_requests_hidden_states_only_through_the_check():+ source = rl._patch_grpo_accumulated_loss_head_hooks(+ rl._patch_grpo_accumulated_loss_hidden_states_dispatch(rl.grpo_accumulated_loss)+ )+ assert rl._GRPO_HIDDEN_STATES_REQUEST.search(source) is None+ assert source.count("_unsloth_grpo_head_hooked(trainer.model)") >= 1+ compile(source, "<patched grpo_accumulated_loss>", "exec")+++def test_source_patch_fails_loudly_without_a_hidden_states_request():+ with pytest.raises(RuntimeError, match = "requests hidden states"):+ rl._patch_grpo_accumulated_loss_head_hooks(+ "def grpo_accumulated_loss(trainer, x):\n return x\n"+ )+++def test_generated_trainer_carries_the_check_and_the_patched_gradient_path():+ items = rl.RL_PRE_ITEMS["grpo_trainer"]+ assert sum("def _unsloth_grpo_head_hooked(" in item for item in items) == 1+ embedded = [item for item in items if "def grpo_accumulated_loss(" in item]+ assert len(embedded) == 1+ assert rl._GRPO_HIDDEN_STATES_REQUEST.search(embedded[0]) is None+++def _nograd_function_ast():+ source = rl.grpo_trainer__get_per_token_logps_and_entropies(+ "_get_per_token_logps_and_entropies", "def _get_per_token_logps_and_entropies(): pass"+ )+ tree = ast.parse(textwrap.dedent(source))+ (function,) = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.FunctionDef) and node.name == "_get_per_token_logps_and_entropies"+ ]+ return function+++def test_nograd_path_requests_hidden_states_only_for_an_unhooked_head():+ function = _nograd_function_ast()+ checks = [+ node+ for node in ast.walk(function)+ if isinstance(node, ast.Assign)+ and [ast.unparse(t) for t in node.targets] == ["_unsloth_head_hooked"]+ ]+ assert [ast.unparse(node.value) for node in checks] == [+ "_unsloth_grpo_head_hooked(unwrapped_model)"+ ]+ requests = [+ ast.unparse(node.value)+ for node in ast.walk(function)+ if isinstance(node, ast.Assign)+ and [ast.unparse(t) for t in node.targets] == ["os.environ['UNSLOTH_RETURN_HIDDEN_STATES']"]+ ]+ assert requests, "the no-grad path no longer sets UNSLOTH_RETURN_HIDDEN_STATES"+ for value in requests:+ assert value in ("'0'", "'0' if _unsloth_head_hooked else '1'"), value+ assert requests.count("'0' if _unsloth_head_hooked else '1'") >= 1+++def test_nograd_prefix_grouper_is_off_for_a_hooked_head():+ """PrefixGrouper's extract_logps always projects with lm_head.weight."""+ function = _nograd_function_ast()+ gates = [+ node+ for node in ast.walk(function)+ if isinstance(node, ast.Assign)+ and [ast.unparse(t) for t in node.targets] == ["_pg_engage"]+ and "UNSLOTH_GRPO_PREFIX_GROUPER_ON" in ast.unparse(node.value)+ ]+ assert len(gates) == 1+ assert "not _unsloth_head_hooked" in ast.unparse(gates[0].value)+++def test_helper_is_self_contained_for_the_generated_trainer():+ """Embedded via inspect.getsource: only torch may come from outside."""+ source = textwrap.dedent(inspect.getsource(rl._unsloth_grpo_head_hooked))+ tree = ast.parse(source)+ loaded = {+ n.id for n in ast.walk(tree) if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load)+ }+ stored = {+ n.id for n in ast.walk(tree) if isinstance(n, ast.Name) and not isinstance(n.ctx, ast.Load)+ }+ stored |= {a.arg for a in ast.walk(tree) if isinstance(a, ast.arg)}+ stored |= {+ alias.asname or alias.name+ for n in ast.walk(tree)+ if isinstance(n, ast.ImportFrom)+ for alias in n.names+ }+ import builtins++ free = {name for name in loaded - stored if not hasattr(builtins, name)}+ assert free == {"torch"}, freediff --git a/tests/test_grpo_accumulated_loss_hidden_states_signal.py b/tests/test_grpo_accumulated_loss_hidden_states_signal.pyindex 46834623fe3..90f51c631eb 100644--- a/tests/test_grpo_accumulated_loss_hidden_states_signal.py+++ b/tests/test_grpo_accumulated_loss_hidden_states_signal.py@@ -219,17 +219,23 @@ def test_source_patch_still_accepts_the_installed_zoo():def test_generated_trainer_embeds_the_patched_gradient_function():text = SOURCE_PATH.read_text(encoding = "utf-8")tree = ast.parse(text, filename = str(SOURCE_PATH))- calls = [- node+ appended = [+ node.args[0]for node in ast.walk(tree)if isinstance(node, ast.Call)and isinstance(node.func, ast.Attribute)and node.func.attr == "append"and ast.unparse(node.func.value) == "RL_PRE_ITEMS['grpo_trainer']"and node.args- and isinstance(node.args[0], ast.Call)- and isinstance(node.args[0].func, ast.Name)- and node.args[0].func.id == PATCH_HELPER+ ]+ # The dispatch patch may be nested inside the hooked-head patch.+ calls = [+ inner+ for item in appended+ for inner in ast.walk(item)+ if isinstance(inner, ast.Call)+ and isinstance(inner.func, ast.Name)+ and inner.func.id == PATCH_HELPER]assert len(calls) == 1- assert ast.unparse(calls[0].args[0].args[0]) == "grpo_accumulated_loss"+ assert ast.unparse(calls[0].args[0]) == "grpo_accumulated_loss"diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 3575b69c0dd..8b05ba2c85a 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -3785,6 +3785,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):vision_inputs["mm_token_type_ids"] = mm_token_type_idsunwrapped_model = self.accelerator.unwrap_model(model, keep_fp32_wrapper = False)+ # A hooked output head must run as a module: score the real logits its own call returns.+ _unsloth_head_hooked = _unsloth_grpo_head_hooked(unwrapped_model)_unsloth_grpo_refuse_boundary_blind_packing(self, model)from unsloth_zoo.rl_replacements import (@@ -3985,7 +3987,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):attention_mask_chunks,vision_chunks,)- os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0" if _unsloth_head_hooked else "1"# Sequence packing (default on; UNSLOTH_GRPO_SEQ_PACKING=0 disables): one varlen [1, sum L] forward replaces the padded [B, Lmax] loop and fixes the left-pad RoPE error. Self-verified against the per-row forward, re-checked as T grows, and falls back if a backend ignores packed_seq_lengths.logprobs = None@@ -4000,6 +4002,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):_pg_engage = (UNSLOTH_GRPO_PREFIX_GROUPER_ONand not getattr(self, "use_vllm", False)+ # Its extract_logps always projects with lm_head.weight.+ and not _unsloth_head_hookedand not getattr(unwrapped_model, "_unsloth_prefix_grouper_nograd_disabled", False))if _pg_engage:@@ -4119,7 +4123,9 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):unwrapped_model._unsloth_prefix_grouper_nograd_disabled = Trueif isinstance(_pg_err, torch.cuda.OutOfMemoryError):torch.cuda.empty_cache()- os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = (+ "0" if _unsloth_head_hooked else "1"+ )if UNSLOTH_ENABLE_LOGGING:print(f"[Unsloth] GRPO PrefixGrouper (no-grad) disabled (fell back to packed): {_pg_err!r}",@@ -4427,7 +4433,9 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):_pg_use = Falseif isinstance(_pg_err3, torch.cuda.OutOfMemoryError):torch.cuda.empty_cache()- os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = (+ "0" if _unsloth_head_hooked else "1"+ )if UNSLOTH_ENABLE_LOGGING:print(f"[Unsloth] GRPO PrefixGrouper (no-grad) verify failed (fell back to packed): {_pg_err3!r}",@@ -4649,6 +4657,45 @@ def _unsloth_grpo_returns_hidden_states(model, tensor, lm_head):return _unsloth_grpo_hidden_states_signal(model) is not False+def _unsloth_grpo_head_hooked(model):+ """True when GRPO must score real logits from the head's own call: hidden-states scorers read ``lm_head.weight``+ and bypass its hooks / patched ``forward``. Unsloth, ZeRO-3, FSDP2 and accelerate ``_hf_hook`` hooks are already+ handled. A square head keeps hidden states: the width dispatch could not tell them from logits."""+ try:+ head = model.get_output_embeddings()+ except Exception:+ return False+ if not isinstance(head, torch.nn.Module):+ return False+ weight = getattr(head, "weight", None)+ if (+ isinstance(weight, torch.Tensor)+ and weight.dim() == 2+ and weight.shape[0] == weight.shape[1]+ ):+ return False+ from torch.nn.modules import module as _module++ if (+ getattr(_module, "_global_forward_hooks", None)+ or getattr(_module, "_global_forward_pre_hooks", None)+ or getattr(_module, "_global_backward_hooks", None)+ or getattr(_module, "_global_backward_pre_hooks", None)+ ):+ return True+ if "forward" in vars(head) and not hasattr(head, "_hf_hook"):+ return True+ for _name in ("_forward_hooks", "_forward_pre_hooks", "_backward_hooks", "_backward_pre_hooks"):+ for _hook in (getattr(head, _name, None) or {}).values():+ _fn = getattr(_hook, "func", _hook) # functools.partial+ _owner = getattr(_fn, "__module__", None) or ""+ if not _owner.startswith(+ ("unsloth.", "unsloth_zoo.", "deepspeed.", "torch.distributed.")+ ):+ return True+ return False++def _unsloth_pref_returns_hidden_states(model, tensor, lm_head):"""``_unsloth_grpo_returns_hidden_states`` for the preference routes, which need a zoo that writes the hidden-statemarkers: a square head with no signal is a forward that ignored the flag, so its output is real logits."""@@ -4752,6 +4799,31 @@ def _patch_grpo_accumulated_loss_hidden_states_dispatch(function):return source+_GRPO_HIDDEN_STATES_REQUEST = re.compile(+ r"""os\.environ\[\s*(["'])UNSLOTH_RETURN_HIDDEN_STATES\1\s*\]\s*=\s*(["'])1\2"""+)+++def _patch_grpo_accumulated_loss_head_hooks(source):+ """In ``grpo_accumulated_loss``, request real logits instead of hidden states while ``_unsloth_grpo_head_hooked``."""+ if re.search(r"def grpo_accumulated_loss\(\s*trainer\s*,", source) is None:+ raise RuntimeError(+ "Unsloth: grpo_accumulated_loss in this unsloth_zoo version does not take "+ "`trainer` first. Please upgrade unsloth_zoo."+ )+ source, requests = _GRPO_HIDDEN_STATES_REQUEST.subn(+ 'os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = '+ '"0" if _unsloth_grpo_head_hooked(trainer.model) else "1"',+ source,+ )+ if requests == 0:+ raise RuntimeError(+ "Unsloth: could not find where grpo_accumulated_loss requests hidden states "+ "in this unsloth_zoo version. Please upgrade unsloth_zoo."+ )+ return source++def grpo_update_SamplingParams(SamplingParams,generation_kwargs,@@ -5054,6 +5126,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_text_configs))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_final_logit_softcapping))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_resolve_logit_scales))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_returns_hidden_states))+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_head_hooked))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_hidden_states_signal))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_mm_token_id))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_fix_mm_token_type_ids))@@ -5084,7 +5157,9 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_clear_stateful_mrRL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(grpo_compute_loss))RL_PRE_ITEMS["grpo_trainer"].append(_class_source(UnslothEfficientGRPO))RL_PRE_ITEMS["grpo_trainer"].append(- _patch_grpo_accumulated_loss_hidden_states_dispatch(grpo_accumulated_loss)+ _patch_grpo_accumulated_loss_head_hooks(+ _patch_grpo_accumulated_loss_hidden_states_dispatch(grpo_accumulated_loss)+ ))RL_PRE_ITEMS["grpo_trainer"].append(grpo_compute_loss_slow)RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(grpo_update_SamplingParams))