Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Forward output head bias in GRPO policy scoring (#13302)
cherry77-clouda4a8d8239bf4
diff --git a/tests/test_grpo_ref_model_output_head.py b/tests/test_grpo_ref_model_output_head.pyindex f5e3a29c918..61c794adc5a 100644--- a/tests/test_grpo_ref_model_output_head.py+++ b/tests/test_grpo_ref_model_output_head.py@@ -22,18 +22,33 @@ def _lm_head_assignment():for node in ast.walk(ast.parse(source))if isinstance(node, ast.FunctionDef) and node.name == "_get_per_token_logps_and_entropies"]- (assign,) = [- node- for node in ast.walk(function)- if isinstance(node, ast.Assign)- and any(isinstance(t, ast.Name) and t.id == "lm_head" for t in node.targets)- ]- return compile(textwrap.dedent(ast.get_source_segment(source, assign)), "<lm_head>", "exec")+ segments = {}+ for node in ast.walk(function):+ if isinstance(node, ast.Assign):+ for target in node.targets:+ if isinstance(target, ast.Name) and target.id in (+ "output_embeddings",+ "lm_head",+ "lm_head_bias",+ ):+ segments.setdefault(+ (node.lineno, target.id), ast.get_source_segment(source, node)+ )+ # First binding of each name, in source order: the head lookup, before any fallback rebinding.+ first = {}+ for (_, name), segment in sorted(segments.items()):+ first.setdefault(name, segment)+ code = "\n".join(+ textwrap.dedent(first[name])+ for name in ("output_embeddings", "lm_head", "lm_head_bias")+ if name in first+ )+ return compile(code, "<lm_head>", "exec")-def _causal_lm(seed):+def _causal_lm(seed, bias = False):torch.manual_seed(seed)- head = torch.nn.Linear(8, 16, bias = False)+ head = torch.nn.Linear(8, 16, bias = bias)return SimpleNamespace(get_output_embeddings = lambda: head)@@ -42,3 +57,11 @@ def test_reference_model_scored_with_its_own_head():namespace = {"self": SimpleNamespace(model = policy), "unwrapped_model": ref}exec(_lm_head_assignment(), namespace)assert namespace["lm_head"] is ref.get_output_embeddings().weight+++def test_reference_model_scored_with_its_own_head_bias():+ policy, ref = _causal_lm(0, bias = True), _causal_lm(1, bias = True)+ namespace = {"self": SimpleNamespace(model = policy), "unwrapped_model": ref}+ exec(_lm_head_assignment(), namespace)+ assert namespace["lm_head"] is ref.get_output_embeddings().weight+ assert namespace["lm_head_bias"] is ref.get_output_embeddings().biasdiff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 13be72a8e84..afcd59900ce 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -67,6 +67,13 @@ try:)except Exception:UNSLOTH_ZOO_HAS_MASKED_COL_GUARD = False+# Zoo >= unslothai/unsloth-zoo#1685 scores a biased output head; mirrored via RL_PRE_ITEMS.+try:+ UNSLOTH_ZOO_HAS_HEAD_BIAS = (+ "lm_head_bias" in inspect.signature(chunked_hidden_states_selective_log_softmax).parameters+ )+except Exception:+ UNSLOTH_ZOO_HAS_HEAD_BIAS = False# One-time PrefixGrouper gate; any import failure degrades to "PrefixGrouper off"._pg_build_layout = _pg_enabled_fn = _pg_verify_on = _pg_tol_ok = _PG_TOL_KILL = NoneUNSLOTH_GRPO_PREFIX_GROUPER_ON = os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER", "1").lower() not in (@@ -2365,7 +2372,25 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):unwrapped_model = self.accelerator.unwrap_model(model, keep_fp32_wrapper = False)- lm_head = unwrapped_model.get_output_embeddings().weight+ from unsloth_zoo.rl_replacements import chunked_hidden_states_selective_log_softmax++ output_embeddings = unwrapped_model.get_output_embeddings()+ lm_head = output_embeddings.weight+ lm_head_bias = getattr(output_embeddings, "bias", None)+ if lm_head_bias is not None and not UNSLOTH_ZOO_HAS_HEAD_BIAS:+ import warnings+ warnings.warn(+ "Unsloth: this model's output head has a bias, which GRPO scoring needs "+ "a newer unsloth_zoo to include. Please upgrade: pip install -U unsloth_zoo",+ stacklevel = 2,+ )+ lm_head_bias = None+ if lm_head_bias is not None:+ from functools import partial+ chunked_hidden_states_selective_log_softmax = partial(+ chunked_hidden_states_selective_log_softmax,+ lm_head_bias = lm_head_bias,+ )# Size on the dtype the forward actually runs in: with autocast off that is the model's own dtype.forward_dtype = (@@ -3290,6 +3315,14 @@ RL_PRE_ITEMS["grpo_trainer"].append("except Exception:\n"" UNSLOTH_ZOO_HAS_MASKED_COL_GUARD = False\n")+RL_PRE_ITEMS["grpo_trainer"].append(+ "try:\n"+ " import inspect as _unsloth_inspect\n"+ " from unsloth_zoo.rl_replacements import chunked_hidden_states_selective_log_softmax as _unsloth_chunked_lsm\n"+ " UNSLOTH_ZOO_HAS_HEAD_BIAS = 'lm_head_bias' in _unsloth_inspect.signature(_unsloth_chunked_lsm).parameters\n"+ "except Exception:\n"+ " UNSLOTH_ZOO_HAS_HEAD_BIAS = False\n"+)RL_PRE_ITEMS["grpo_trainer"].append("_pg_build_layout = _pg_enabled_fn = _pg_verify_on = _pg_tol_ok = _PG_TOL_KILL = None\n""UNSLOTH_GRPO_PREFIX_GROUPER_ON = _unsloth_os.environ.get('UNSLOTH_GRPO_PREFIX_GROUPER', '1').lower() not in ('0', 'false', 'no', 'off')\n"