Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO: score old and reference logprobs over the truncated sampling support when opted in (#13900)

Daniel Han65d1fdb03c19
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 55963ead6e9..d48d60bec4d 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -2514,7 +2514,10 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
unwrapped_model = self.accelerator.unwrap_model(model, keep_fp32_wrapper = False)
- from unsloth_zoo.rl_replacements import chunked_hidden_states_selective_log_softmax
+ from unsloth_zoo.rl_replacements import (
+ chunked_hidden_states_selective_log_softmax,
+ chunked_selective_log_softmax,
+ )
output_embeddings = unwrapped_model.get_output_embeddings()
lm_head = output_embeddings.weight
@@ -2533,6 +2536,26 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
chunked_hidden_states_selective_log_softmax,
lm_head_bias = lm_head_bias,
)
+ # Opt-in (UNSLOTH_GRPO_TRUNCATED_SUPPORT=1): score over the rollout's top_k / top_p /
+ # min_p kept set, matching vLLM's processed_logprobs. The zoo loss refuses to mix
+ # normalisations unless this marker says the old / ref logps were scored the same way.
+ try:
+ from unsloth_zoo.rl_replacements import grpo_truncated_support_kwargs
+ _support_kwargs = grpo_truncated_support_kwargs(self)
+ except Exception:
+ _support_kwargs = {}
+ if _support_kwargs:
+ from functools import partial
+
+ chunked_hidden_states_selective_log_softmax = partial(
+ chunked_hidden_states_selective_log_softmax,
+ **_support_kwargs,
+ )
+ chunked_selective_log_softmax = partial(
+ chunked_selective_log_softmax,
+ **_support_kwargs,
+ )
+ self._unsloth_truncated_support_scored = True
# Size on the dtype the forward actually runs in: with autocast off that is the model's own dtype.
forward_dtype = (