Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop ignoring the min_p and stop settings the GRPO notebooks pass to vLLM (#11595)

Nilayf5109b58434d
diff --git a/tests/python/test_grpo_update_sampling_params.py b/tests/python/test_grpo_update_sampling_params.py
new file mode 100644
index 00000000000..9a2c9a01a43
--- /dev/null
+++ b/tests/python/test_grpo_update_sampling_params.py
@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+from __future__ import annotations
+
+import ast
+import inspect
+import os
+from dataclasses import dataclass, field
+
+import pytest
+
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py")
+HELPER = "grpo_update_SamplingParams"
+
+
+def _read_source() -> str:
+ with open(SOURCE_PATH, "r", encoding = "utf-8") as fh:
+ return fh.read()
+
+
+def _load_helper():
+ try:
+ import unsloth.models.rl_replacements as rl
+ except Exception:
+ rl = None
+ if rl is not None:
+ return getattr(rl, HELPER)
+ tree = ast.parse(_read_source())
+ node = next(
+ (n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == HELPER),
+ None,
+ )
+ assert node is not None, f"{HELPER} is not defined in rl_replacements.py"
+ namespace = {"inspect": inspect}
+ exec(compile(ast.Module(body = [node], type_ignores = []), SOURCE_PATH, "exec"), namespace)
+ return namespace[HELPER]
+
+
+@dataclass
+class SamplingParams:
+ n: int = 1
+ temperature: float = 1.0
+ top_p: float = 1.0
+ top_k: int = -1
+ min_p: float = 0.0
+ seed: int | None = None
+ max_tokens: int = 16
+ stop: list[str] | None = None
+ include_stop_str_in_output: bool = False
+ logprobs: int | None = None
+ _real_n: int | None = field(default = None, repr = False)
+
+
+EOS = "<|im_end|>"
+
+
+def _trl_generation_kwargs():
+ return {
+ "n": 8,
+ "repetition_penalty": 1.0,
+ "temperature": 1.0,
+ "top_p": 1.0,
+ "top_k": -1,
+ "min_p": 0.0,
+ "max_tokens": 1024,
+ "truncate_prompt_tokens": 512,
+ "guided_decoding": None,
+ "logprobs": 0,
+ }
+
+
[email protected](scope = "module")
+def helper():
+ return _load_helper()
+
+
+def test_notebook_scalar_fields_reach_generation(helper):
+ generation_kwargs = _trl_generation_kwargs()
+ user = SamplingParams(
+ min_p = 0.1,
+ top_p = 1.0,
+ top_k = -1,
+ seed = 3407,
+ stop = [EOS],
+ include_stop_str_in_output = True,
+ )
+ result = helper(SamplingParams, generation_kwargs, user)
+ assert result["min_p"] == 0.1
+ assert result["include_stop_str_in_output"] is True
+ assert result["stop"] == [EOS]
+ assert "seed" not in result
+ assert result["n"] == generation_kwargs["n"]
+ assert result["max_tokens"] == generation_kwargs["max_tokens"]
+ assert "repetition_penalty" not in result
+ assert "_real_n" not in result
+ SamplingParams(**result)
+
+
+def test_default_sampling_params_overlay_nothing(helper):
+ generation_kwargs = _trl_generation_kwargs()
+ result = helper(SamplingParams, generation_kwargs, SamplingParams())
+ expected = {
+ k: v for k, v in generation_kwargs.items() if k in SamplingParams.__dataclass_fields__
+ }
+ assert result == expected
+
+
+def test_set_kwargs_take_precedence_over_field_diff(helper):
+ generation_kwargs = _trl_generation_kwargs()
+ user = SamplingParams(min_p = 0.1, seed = 3407, n = 4)
+ user._set_kwargs = {"min_p": 0.2, "seed": 3407, "n": 4, "not_a_field": 1}
+ result = helper(SamplingParams, generation_kwargs, user)
+ assert result["min_p"] == 0.2
+ assert "seed" not in result
+ assert result["n"] == generation_kwargs["n"]
+ assert "not_a_field" not in result
+
+
[email protected]("use_set_kwargs", [False, True])
+def test_trl_owned_fields_are_not_overridden(helper, use_set_kwargs):
+ generation_kwargs = _trl_generation_kwargs()
+ overrides = {"temperature": 0.6, "max_tokens": 4096, "logprobs": 5, "min_p": 0.1}
+ user = SamplingParams(**overrides)
+ if use_set_kwargs:
+ user._set_kwargs = dict(overrides)
+ result = helper(SamplingParams, generation_kwargs, user)
+ assert result["temperature"] == generation_kwargs["temperature"]
+ assert result["max_tokens"] == generation_kwargs["max_tokens"]
+ assert result["logprobs"] == generation_kwargs["logprobs"]
+ assert result["min_p"] == 0.1
+
+
+def test_repo_injects_the_local_helper():
+ src = _read_source()
+ node = next(
+ n for n in ast.parse(src).body if isinstance(n, ast.FunctionDef) and n.name == HELPER
+ )
+ assert [a.arg for a in node.args.args] == [
+ "SamplingParams",
+ "generation_kwargs",
+ "vllm_sampling_params",
+ ]
+ assert [ast.literal_eval(d) for d in node.args.defaults] == [None]
+ assert f'RL_REPLACEMENTS["{HELPER}"]' not in src
+ assert f'RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource({HELPER}))' in src
diff --git a/tests/version_compat/test_trl_vllm_generation_lora_patch.py b/tests/version_compat/test_trl_vllm_generation_lora_patch.py
index 5b80cc38b8c..22bc8478325 100644
--- a/tests/version_compat/test_trl_vllm_generation_lora_patch.py
+++ b/tests/version_compat/test_trl_vllm_generation_lora_patch.py
@@ -559,3 +559,117 @@ def generate(self, prompts, **kwargs):
assert cls.generate(self, ["hello"]) == ["generated"]
assert _lora_requests(log) == ["LORA[%s|True]" % _lora_name()], log
+
+
+# TRL >= 0.28 builds `SamplingParams(**generation_kwargs)` inside `generate`, from the module global, and never sees
+# the trainer's `args.vllm_sampling_params`; the trainer hands them over as `_unsloth_vllm_sampling_params`.
+_GENERATE_BUILDS_SAMPLING_PARAMS = """
+def generate(self, prompts, **kwargs):
+ self.sync_weights()
+ generation_kwargs = {"n": 1, "temperature": 1.0, "min_p": 0.0, "max_tokens": 64, "logprobs": 0}
+ generation_kwargs.update(self.generation_kwargs)
+ sampling_params = SamplingParams(**generation_kwargs)
+ if self.fail:
+ raise RuntimeError("generate failed")
+ return self.llm.generate(prompts, sampling_params = sampling_params)
+"""
+
+
+class FakeSamplingParams:
+ def __init__(
+ self,
+ n = 1,
+ temperature = 1.0,
+ min_p = 0.0,
+ seed = None,
+ max_tokens = 16,
+ logprobs = None,
+ stop = None,
+ include_stop_str_in_output = False,
+ ):
+ self.n = n
+ self.temperature = temperature
+ self.min_p = min_p
+ self.seed = seed
+ self.max_tokens = max_tokens
+ self.logprobs = logprobs
+ self.stop = [] if stop is None else stop
+ self.include_stop_str_in_output = include_stop_str_in_output
+
+
+class SamplingParamsEngine(FakeEngine):
+ def generate(self, *args, **kwargs):
+ self.log.append(("sampling_params", kwargs.get("sampling_params")))
+ return ["generated"]
+
+
+def _sampling_generation(
+ monkeypatch,
+ user_sampling_params,
+ shared_weights = True,
+):
+ cls = _build_fake_trl(monkeypatch, _GENERATE_BUILDS_SAMPLING_PARAMS)
+ module = sys.modules["trl.generation.vllm_generation"]
+ module.SamplingParams = FakeSamplingParams
+ _rl_replacements().vllm_generation_init_patch()
+ log = []
+ self = _make_generation(cls, log, shared_weights = shared_weights, with_lora = shared_weights)
+ self.llm = SamplingParamsEngine(log, shared_weights = shared_weights)
+ self.generation_kwargs = {}
+ self.fail = False
+ if user_sampling_params is not None:
+ self._unsloth_vllm_sampling_params = user_sampling_params
+ return cls, module, self, log
+
+
+def _sent_sampling_params(log):
+ sent = [entry[1] for entry in log if entry[0] == "sampling_params"]
+ assert len(sent) == 1, log
+ return sent[0]
+
+
+def test_user_vllm_sampling_params_reach_the_engine(monkeypatch):
+ user = FakeSamplingParams(
+ min_p = 0.1,
+ seed = 3407,
+ temperature = 0.6,
+ stop = ["<|im_end|>"],
+ include_stop_str_in_output = True,
+ )
+ cls, module, self, log = _sampling_generation(monkeypatch, user)
+ assert cls.generate(self, ["hello"]) == ["generated"]
+ sent = _sent_sampling_params(log)
+ assert isinstance(sent, FakeSamplingParams)
+ assert sent.min_p == 0.1
+ assert sent.stop == ["<|im_end|>"]
+ assert sent.include_stop_str_in_output is True
+ # GRPOConfig still owns these.
+ assert sent.seed is None
+ assert sent.temperature == 1.0
+ assert sent.max_tokens == 64
+ assert module.SamplingParams is FakeSamplingParams
+
+
+def test_no_user_sampling_params_leaves_trl_untouched(monkeypatch):
+ cls, module, self, log = _sampling_generation(monkeypatch, None)
+ cls.generate(self, ["hello"])
+ sent = _sent_sampling_params(log)
+ assert (sent.min_p, sent.stop, sent.include_stop_str_in_output) == (0.0, [], False)
+ assert module.SamplingParams is FakeSamplingParams
+
+
+def test_server_mode_ignores_user_sampling_params(monkeypatch):
+ cls, module, self, log = _sampling_generation(
+ monkeypatch, FakeSamplingParams(min_p = 0.1), shared_weights = False
+ )
+ cls.generate(self, ["hello"])
+ assert _sent_sampling_params(log).min_p == 0.0
+ assert module.SamplingParams is FakeSamplingParams
+
+
+def test_sampling_params_restored_when_generate_raises(monkeypatch):
+ cls, module, self, log = _sampling_generation(monkeypatch, FakeSamplingParams(min_p = 0.1))
+ self.fail = True
+ with pytest.raises(RuntimeError, match = "generate failed"):
+ cls.generate(self, ["hello"])
+ assert module.SamplingParams is FakeSamplingParams
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index af772d7fe8d..d797282ad4c 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2560,6 +2560,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
RLTrainer_post += vllm_chat_template_sync
+ # TRL >= 0.28 builds SamplingParams inside VLLMGeneration, which never sees args; hand it the user's vllm_sampling_params so the generate wrapper in rl_replacements.py can apply them.
+ if trainer_file == "grpo_trainer":
+ RLTrainer_post += (
+ "if getattr(self, 'vllm_generation', None) is not None:\n"
+ " self.vllm_generation._unsloth_vllm_sampling_params = getattr(getattr(self, 'args', None), 'vllm_sampling_params', None)\n"
+ "pass\n"
+ )
+
other_metrics_processor = ""
if trainer_file in RL_METRICS_CHANGES:
process_extra_args = RL_METRICS_CHANGES[trainer_file]
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index d65c61f0bd5..69591cfb7f1 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -2747,11 +2747,46 @@ def _patch_grpo_accumulated_loss_hidden_states_dispatch(function):
return source
+def grpo_update_SamplingParams(
+ SamplingParams,
+ generation_kwargs,
+ vllm_sampling_params = None,
+):
+ good_sampling_params_keys = inspect.signature(SamplingParams).parameters.keys()
+
+ new_generation_kwargs = {}
+ for key in generation_kwargs.keys():
+ if key in good_sampling_params_keys:
+ new_generation_kwargs[key] = generation_kwargs[key]
+ generation_kwargs = new_generation_kwargs
+
+ if vllm_sampling_params is not None:
+ overwrites = getattr(vllm_sampling_params, "_set_kwargs", None)
+ if overwrites is None:
+ default_sampling_params = SamplingParams()
+ overwrites = {}
+ for key in good_sampling_params_keys:
+ if key.startswith("_") or not hasattr(vllm_sampling_params, key):
+ continue
+ overwrited_key = getattr(vllm_sampling_params, key)
+ if overwrited_key != getattr(default_sampling_params, key, None):
+ overwrites[key] = overwrited_key
+ for key, overwrited_key in overwrites.items():
+ if key in good_sampling_params_keys and key not in (
+ "seed",
+ "n",
+ "temperature",
+ "max_tokens",
+ "logprobs",
+ ):
+ generation_kwargs[key] = overwrited_key
+ return generation_kwargs
+
+
grpo_compute_loss = RL_REPLACEMENTS["grpo_compute_loss"]
grpo_compute_loss_slow = RL_REPLACEMENTS["grpo_compute_loss_slow"]
UnslothEfficientGRPO = RL_REPLACEMENTS["UnslothEfficientGRPO"]
grpo_accumulated_loss = RL_REPLACEMENTS["grpo_accumulated_loss"]
-grpo_update_SamplingParams = RL_REPLACEMENTS["grpo_update_SamplingParams"]
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_autocast))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_autocast_kwargs))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_model_config))
@@ -3586,9 +3621,28 @@ def vllm_generation_init_patch():
override("generate", wrap_generation_call)
override("chat", wrap_generation_call)
override("collective_rpc", wrap_collective_rpc)
+ # TRL >= 0.28 builds `SamplingParams(**generation_kwargs)` inside this call, the one site the older-TRL source replacement in rl.py targets; route it through the same helper for this call only.
+ user_sampling_params = getattr(self, "_unsloth_vllm_sampling_params", None)
+ original_sampling_params = vllm_generation.__dict__.get("SamplingParams", None)
+ if user_sampling_params is not None and original_sampling_params is not None:
+
+ def unsloth_sampling_params(*sp_args, **sp_kwargs):
+ if sp_args:
+ return original_sampling_params(*sp_args, **sp_kwargs)
+ return original_sampling_params(
+ **grpo_update_SamplingParams(
+ original_sampling_params, sp_kwargs, user_sampling_params
+ )
+ )
+
+ vllm_generation.SamplingParams = unsloth_sampling_params
+ else:
+ original_sampling_params = None
try:
return original_generate(self, *args, **kwargs)
finally:
+ if original_sampling_params is not None:
+ vllm_generation.SamplingParams = original_sampling_params
for name, had_own, bound in reversed(saved):
try:
if had_own: