Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Forward output head bias in GRPO policy scoring (#13302)

cherry77-clouda4a8d8239bf4
diff --git a/tests/test_grpo_ref_model_output_head.py b/tests/test_grpo_ref_model_output_head.py
index f5e3a29c918..61c794adc5a 100644
--- a/tests/test_grpo_ref_model_output_head.py
+++ b/tests/test_grpo_ref_model_output_head.py
@@ -22,18 +22,33 @@ def _lm_head_assignment():
for node in ast.walk(ast.parse(source))
if isinstance(node, ast.FunctionDef) and node.name == "_get_per_token_logps_and_entropies"
]
- (assign,) = [
- node
- for node in ast.walk(function)
- if isinstance(node, ast.Assign)
- and any(isinstance(t, ast.Name) and t.id == "lm_head" for t in node.targets)
- ]
- return compile(textwrap.dedent(ast.get_source_segment(source, assign)), "<lm_head>", "exec")
+ segments = {}
+ for node in ast.walk(function):
+ if isinstance(node, ast.Assign):
+ for target in node.targets:
+ if isinstance(target, ast.Name) and target.id in (
+ "output_embeddings",
+ "lm_head",
+ "lm_head_bias",
+ ):
+ segments.setdefault(
+ (node.lineno, target.id), ast.get_source_segment(source, node)
+ )
+ # First binding of each name, in source order: the head lookup, before any fallback rebinding.
+ first = {}
+ for (_, name), segment in sorted(segments.items()):
+ first.setdefault(name, segment)
+ code = "\n".join(
+ textwrap.dedent(first[name])
+ for name in ("output_embeddings", "lm_head", "lm_head_bias")
+ if name in first
+ )
+ return compile(code, "<lm_head>", "exec")
-def _causal_lm(seed):
+def _causal_lm(seed, bias = False):
torch.manual_seed(seed)
- head = torch.nn.Linear(8, 16, bias = False)
+ head = torch.nn.Linear(8, 16, bias = bias)
return SimpleNamespace(get_output_embeddings = lambda: head)
@@ -42,3 +57,11 @@ def test_reference_model_scored_with_its_own_head():
namespace = {"self": SimpleNamespace(model = policy), "unwrapped_model": ref}
exec(_lm_head_assignment(), namespace)
assert namespace["lm_head"] is ref.get_output_embeddings().weight
+
+
+def test_reference_model_scored_with_its_own_head_bias():
+ policy, ref = _causal_lm(0, bias = True), _causal_lm(1, bias = True)
+ namespace = {"self": SimpleNamespace(model = policy), "unwrapped_model": ref}
+ exec(_lm_head_assignment(), namespace)
+ assert namespace["lm_head"] is ref.get_output_embeddings().weight
+ assert namespace["lm_head_bias"] is ref.get_output_embeddings().bias
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 13be72a8e84..afcd59900ce 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -67,6 +67,13 @@ try:
)
except Exception:
UNSLOTH_ZOO_HAS_MASKED_COL_GUARD = False
+# Zoo >= unslothai/unsloth-zoo#1685 scores a biased output head; mirrored via RL_PRE_ITEMS.
+try:
+ UNSLOTH_ZOO_HAS_HEAD_BIAS = (
+ "lm_head_bias" in inspect.signature(chunked_hidden_states_selective_log_softmax).parameters
+ )
+except Exception:
+ UNSLOTH_ZOO_HAS_HEAD_BIAS = False
# One-time PrefixGrouper gate; any import failure degrades to "PrefixGrouper off".
_pg_build_layout = _pg_enabled_fn = _pg_verify_on = _pg_tol_ok = _PG_TOL_KILL = None
UNSLOTH_GRPO_PREFIX_GROUPER_ON = os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER", "1").lower() not in (
@@ -2365,7 +2372,25 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
unwrapped_model = self.accelerator.unwrap_model(model, keep_fp32_wrapper = False)
- lm_head = unwrapped_model.get_output_embeddings().weight
+ from unsloth_zoo.rl_replacements import chunked_hidden_states_selective_log_softmax
+
+ output_embeddings = unwrapped_model.get_output_embeddings()
+ lm_head = output_embeddings.weight
+ lm_head_bias = getattr(output_embeddings, "bias", None)
+ if lm_head_bias is not None and not UNSLOTH_ZOO_HAS_HEAD_BIAS:
+ import warnings
+ warnings.warn(
+ "Unsloth: this model's output head has a bias, which GRPO scoring needs "
+ "a newer unsloth_zoo to include. Please upgrade: pip install -U unsloth_zoo",
+ stacklevel = 2,
+ )
+ lm_head_bias = None
+ if lm_head_bias is not None:
+ from functools import partial
+ chunked_hidden_states_selective_log_softmax = partial(
+ chunked_hidden_states_selective_log_softmax,
+ lm_head_bias = lm_head_bias,
+ )
# Size on the dtype the forward actually runs in: with autocast off that is the model's own dtype.
forward_dtype = (
@@ -3290,6 +3315,14 @@ RL_PRE_ITEMS["grpo_trainer"].append(
"except Exception:\n"
" UNSLOTH_ZOO_HAS_MASKED_COL_GUARD = False\n"
)
+RL_PRE_ITEMS["grpo_trainer"].append(
+ "try:\n"
+ " import inspect as _unsloth_inspect\n"
+ " from unsloth_zoo.rl_replacements import chunked_hidden_states_selective_log_softmax as _unsloth_chunked_lsm\n"
+ " UNSLOTH_ZOO_HAS_HEAD_BIAS = 'lm_head_bias' in _unsloth_inspect.signature(_unsloth_chunked_lsm).parameters\n"
+ "except Exception:\n"
+ " UNSLOTH_ZOO_HAS_HEAD_BIAS = False\n"
+)
RL_PRE_ITEMS["grpo_trainer"].append(
"_pg_build_layout = _pg_enabled_fn = _pg_verify_on = _pg_tol_ok = _PG_TOL_KILL = None\n"
"UNSLOTH_GRPO_PREFIX_GROUPER_ON = _unsloth_os.environ.get('UNSLOTH_GRPO_PREFIX_GROUPER', '1').lower() not in ('0', 'false', 'no', 'off')\n"