Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO: keep TRL's old logprob gate when using vLLM (#13856)

Daniel Han3794b271caea
diff --git a/tests/test_grpo_old_logprob_gate.py b/tests/test_grpo_old_logprob_gate.py
new file mode 100644
index 00000000000..657d9aa0274
--- /dev/null
+++ b/tests/test_grpo_old_logprob_gate.py
@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""GRPO keeps TRL's old logprob gate: rewriting it to `or (self.use_vllm)` paid a full no-grad
+forward per on-policy vLLM step with IS correction off, where old == per_token_logps.detach()."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import re
+
+import pytest
+
+
+def _patched_generate_and_score():
+ if importlib.util.find_spec("trl") is None:
+ pytest.skip("trl is not installed")
+ spec = importlib.util.find_spec("trl.trainer.grpo_trainer")
+ if spec is None or spec.origin is None:
+ pytest.skip("trl is not installed")
+ from unsloth.models.rl_replacements import grpo_trainer__generate_and_score_completions
+
+ with open(spec.origin, "r", encoding = "utf-8") as fh:
+ module_source = fh.read()
+ node = next(
+ n
+ for n in ast.walk(ast.parse(module_source))
+ if isinstance(n, ast.FunctionDef) and n.name == "_generate_and_score_completions"
+ )
+ source = " " + ast.get_source_segment(module_source, node)
+ if "generate_every != 0 or (" not in source:
+ pytest.skip("this TRL has no vLLM clause in the old logprob gate (< 0.23.0)")
+ return grpo_trainer__generate_and_score_completions("_generate_and_score_completions", source)
+
+
+def _gate_condition(patched):
+ match = re.search(
+ r"if (self\.args\.gradient_accumulation_steps % generate_every != 0 or \(.*?\)):\n",
+ patched,
+ flags = re.DOTALL,
+ )
+ assert match is not None, "old logprob gate not found in the patched function"
+ return " ".join(match.group(1).split())
+
+
+def test_old_logprob_gate_matches_trl():
+ condition = _gate_condition(_patched_generate_and_score())
+ assert "self.use_vllm and self.vllm_importance_sampling_correction" in condition, condition
+
+
+ "grad_accum, generate_every, use_vllm, is_correction, expected",
+ [
+ (4, 4, True, False, False),
+ (4, 4, True, True, True),
+ (4, 4, False, False, False),
+ (4, 8, True, False, True),
+ (4, 8, False, False, True),
+ ],
+)
+def test_old_logprob_gate_truth_table(
+ grad_accum, generate_every, use_vllm, is_correction, expected
+):
+ condition = _gate_condition(_patched_generate_and_score())
+
+ class _Args:
+ gradient_accumulation_steps = grad_accum
+
+ class _Self:
+ args = _Args()
+
+ _Self.use_vllm = use_vllm
+ _Self.vllm_importance_sampling_correction = is_correction
+ value = eval(condition, {}, {"self": _Self(), "generate_every": generate_every})
+ assert bool(value) is expected
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 91353a746f0..063edc66b82 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -1809,18 +1809,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
"image batch is being sliced there without its image counts.",
)
- pattern_to_find = re.compile(
- r"^\s*if self\.args\.gradient_accumulation_steps % generate_every != 0 or \(\s*"
- r"self\.use_vllm and self\.vllm_importance_sampling_correction\s*"
- r"\):",
- re.MULTILINE,
- )
-
- replacement_text = """
- if self.args.gradient_accumulation_steps % generate_every != 0 or (
- self.use_vllm
- ):"""
- function, num_replacements = pattern_to_find.subn(replacement_text, function)
+ # Keep TRL's old logprob gate: every old_per_token_logps consumer runs only under IS correction
+ # or treats None as per_token_logps.detach(), so forcing it on for vLLM is a wasted forward.
pattern_to_find = re.compile(
r"(^\s*)all_logprobs = \[" # Capture indentation (group 1)