Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO kept-set refusals: stop saying UNSLOTH_GRPO_TRUNCATED_SUPPORT=1 was set (#15307)
Daniel Hanfbc905fd02a9
diff --git a/tests/test_grpo_truncated_support_policy_gate.py b/tests/test_grpo_truncated_support_policy_gate.pyindex c0dba35cdc1..8c262d100fd 100644--- a/tests/test_grpo_truncated_support_policy_gate.py+++ b/tests/test_grpo_truncated_support_policy_gate.py@@ -161,6 +161,26 @@ def test_policies_with_their_own_kept_set_score_the_full_vocabulary(monkeypatch,assert not messages+def test_refusals_do_not_claim_the_variable_was_set(monkeypatch):+ # zoo's automatic gate means refusals fire with the variable unset: never say "ignoring ...=1".+ monkeypatch.delenv("UNSLOTH_GRPO_TRUNCATED_SUPPORT", raising = False)+ _, messages = _bind(monkeypatch, _trainer(beta = 0.04))+ refuse = _refuse_helper(monkeypatch, _SUPPORT)+ trainer = _trainer()+ trainer._unsloth_vllm_sync_skip = True+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ refuse(trainer)+ messages += [str(w.message) for w in caught]+ assert len(messages) == 2+ for message in messages:+ assert "ignoring" not in message+ assert message.startswith(+ "Unsloth: not scoring over the rollouts' top_k / top_p / min_p kept set"+ )+ assert "automatic" in message++def _own_support_logps(logits, index, top_k):"""The per-policy rule: each policy's own top-k set, plus the scored token."""kth = torch.topk(logits, top_k, dim = -1).values[:, -1:]diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 54f2032995b..c8f354f8834 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -1611,7 +1611,9 @@ def _unsloth_grpo_refuse_unscored_support(self):if mismatch:import warningswarnings.warn(- "Unsloth: ignoring UNSLOTH_GRPO_TRUNCATED_SUPPORT=1 because "+ "Unsloth: not scoring over the rollouts' top_k / top_p / min_p kept set "+ "(UNSLOTH_GRPO_TRUNCATED_SUPPORT=1, or automatic for truncated vLLM rollouts "+ "feeding the IS correction or off-policy mask) because "f"{', '.join(mismatch)}: the scored policy is not the sampler, so its top_k / top_p / ""min_p kept set differs from vLLM's and the PPO ratio would be unnormalised. ""Scoring over the full vocabulary instead.",@@ -1621,7 +1623,7 @@ def _unsloth_grpo_refuse_unscored_support(self):def _unsloth_grpo_support_mismatch(self):- """Why UNSLOTH_GRPO_TRUNCATED_SUPPORT=1 cannot score this trainer, or [] when it can.+ """Why kept-set scoring (UNSLOTH_GRPO_TRUNCATED_SUPPORT) cannot score this trainer, or [] when it can.Each scoring call rebuilds the top_k / top_p / min_p kept set from its own logits and always keeps thescored token. That is the sampler's set only while the scored weights are the rollout's. A reference@@ -3543,8 +3545,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):**kwargs,)- # Opt-in (UNSLOTH_GRPO_TRUNCATED_SUPPORT=1): score over the rollout's top_k / top_p /- # min_p kept set, matching vLLM's processed_logprobs. The zoo loss refuses to mix+ # UNSLOTH_GRPO_TRUNCATED_SUPPORT (=1 or zoo's automatic gate): score over the rollout's+ # top_k / top_p / min_p kept set, matching vLLM's processed_logprobs. The zoo loss refuses to mix# normalisations unless this marker says the old / ref logps were scored the same way.try:from unsloth_zoo.rl_replacements import grpo_truncated_support_kwargs@@ -3557,7 +3559,9 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):if not getattr(self, "_unsloth_truncated_support_refused", False):import warningswarnings.warn(- "Unsloth: ignoring UNSLOTH_GRPO_TRUNCATED_SUPPORT=1 because "+ "Unsloth: not scoring over the rollouts' top_k / top_p / min_p kept set "+ "(UNSLOTH_GRPO_TRUNCATED_SUPPORT=1, or automatic for truncated vLLM "+ "rollouts feeding the IS correction or off-policy mask) because "f"{', '.join(_support_mismatch)}: those logprobs come from weights other ""than the rollout's, whose top_k / top_p / min_p kept set differs from the ""sampler's, so per-policy kept sets give unnormalised PPO ratios and KL. "