Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO: keep TRL's old logprob gate when using vLLM (#13856)
Daniel Han3794b271caea
diff --git a/tests/test_grpo_old_logprob_gate.py b/tests/test_grpo_old_logprob_gate.pynew file mode 100644index 00000000000..657d9aa0274--- /dev/null+++ b/tests/test_grpo_old_logprob_gate.py@@ -0,0 +1,76 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""GRPO keeps TRL's old logprob gate: rewriting it to `or (self.use_vllm)` paid a full no-grad+forward per on-policy vLLM step with IS correction off, where old == per_token_logps.detach()."""++from __future__ import annotations++import ast+import importlib.util+import re++import pytest+++def _patched_generate_and_score():+ if importlib.util.find_spec("trl") is None:+ pytest.skip("trl is not installed")+ spec = importlib.util.find_spec("trl.trainer.grpo_trainer")+ if spec is None or spec.origin is None:+ pytest.skip("trl is not installed")+ from unsloth.models.rl_replacements import grpo_trainer__generate_and_score_completions++ with open(spec.origin, "r", encoding = "utf-8") as fh:+ module_source = fh.read()+ node = next(+ n+ for n in ast.walk(ast.parse(module_source))+ if isinstance(n, ast.FunctionDef) and n.name == "_generate_and_score_completions"+ )+ source = " " + ast.get_source_segment(module_source, node)+ if "generate_every != 0 or (" not in source:+ pytest.skip("this TRL has no vLLM clause in the old logprob gate (< 0.23.0)")+ return grpo_trainer__generate_and_score_completions("_generate_and_score_completions", source)+++def _gate_condition(patched):+ match = re.search(+ r"if (self\.args\.gradient_accumulation_steps % generate_every != 0 or \(.*?\)):\n",+ patched,+ flags = re.DOTALL,+ )+ assert match is not None, "old logprob gate not found in the patched function"+ return " ".join(match.group(1).split())+++def test_old_logprob_gate_matches_trl():+ condition = _gate_condition(_patched_generate_and_score())+ assert "self.use_vllm and self.vllm_importance_sampling_correction" in condition, condition+++ "grad_accum, generate_every, use_vllm, is_correction, expected",+ [+ (4, 4, True, False, False),+ (4, 4, True, True, True),+ (4, 4, False, False, False),+ (4, 8, True, False, True),+ (4, 8, False, False, True),+ ],+)+def test_old_logprob_gate_truth_table(+ grad_accum, generate_every, use_vllm, is_correction, expected+):+ condition = _gate_condition(_patched_generate_and_score())++ class _Args:+ gradient_accumulation_steps = grad_accum++ class _Self:+ args = _Args()++ _Self.use_vllm = use_vllm+ _Self.vllm_importance_sampling_correction = is_correction+ value = eval(condition, {}, {"self": _Self(), "generate_every": generate_every})+ assert bool(value) is expecteddiff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 91353a746f0..063edc66b82 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -1809,18 +1809,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function):"image batch is being sliced there without its image counts.",)- pattern_to_find = re.compile(- r"^\s*if self\.args\.gradient_accumulation_steps % generate_every != 0 or \(\s*"- r"self\.use_vllm and self\.vllm_importance_sampling_correction\s*"- r"\):",- re.MULTILINE,- )-- replacement_text = """- if self.args.gradient_accumulation_steps % generate_every != 0 or (- self.use_vllm- ):"""- function, num_replacements = pattern_to_find.subn(replacement_text, function)+ # Keep TRL's old logprob gate: every old_per_token_logps consumer runs only under IS correction+ # or treats None as per_token_logps.detach(), so forcing it on for vLLM is a wasted forward.pattern_to_find = re.compile(r"(^\s*)all_logprobs = \[" # Capture indentation (group 1)