Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Stop ignoring the min_p and stop settings the GRPO notebooks pass to vLLM (#11595)
Nilayf5109b58434d
diff --git a/tests/python/test_grpo_update_sampling_params.py b/tests/python/test_grpo_update_sampling_params.pynew file mode 100644index 00000000000..9a2c9a01a43--- /dev/null+++ b/tests/python/test_grpo_update_sampling_params.py@@ -0,0 +1,147 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++from __future__ import annotations++import ast+import inspect+import os+from dataclasses import dataclass, field++import pytest++REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))+SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py")+HELPER = "grpo_update_SamplingParams"+++def _read_source() -> str:+ with open(SOURCE_PATH, "r", encoding = "utf-8") as fh:+ return fh.read()+++def _load_helper():+ try:+ import unsloth.models.rl_replacements as rl+ except Exception:+ rl = None+ if rl is not None:+ return getattr(rl, HELPER)+ tree = ast.parse(_read_source())+ node = next(+ (n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == HELPER),+ None,+ )+ assert node is not None, f"{HELPER} is not defined in rl_replacements.py"+ namespace = {"inspect": inspect}+ exec(compile(ast.Module(body = [node], type_ignores = []), SOURCE_PATH, "exec"), namespace)+ return namespace[HELPER]+++@dataclass+class SamplingParams:+ n: int = 1+ temperature: float = 1.0+ top_p: float = 1.0+ top_k: int = -1+ min_p: float = 0.0+ seed: int | None = None+ max_tokens: int = 16+ stop: list[str] | None = None+ include_stop_str_in_output: bool = False+ logprobs: int | None = None+ _real_n: int | None = field(default = None, repr = False)+++EOS = "<|im_end|>"+++def _trl_generation_kwargs():+ return {+ "n": 8,+ "repetition_penalty": 1.0,+ "temperature": 1.0,+ "top_p": 1.0,+ "top_k": -1,+ "min_p": 0.0,+ "max_tokens": 1024,+ "truncate_prompt_tokens": 512,+ "guided_decoding": None,+ "logprobs": 0,+ }++[email protected](scope = "module")+def helper():+ return _load_helper()+++def test_notebook_scalar_fields_reach_generation(helper):+ generation_kwargs = _trl_generation_kwargs()+ user = SamplingParams(+ min_p = 0.1,+ top_p = 1.0,+ top_k = -1,+ seed = 3407,+ stop = [EOS],+ include_stop_str_in_output = True,+ )+ result = helper(SamplingParams, generation_kwargs, user)+ assert result["min_p"] == 0.1+ assert result["include_stop_str_in_output"] is True+ assert result["stop"] == [EOS]+ assert "seed" not in result+ assert result["n"] == generation_kwargs["n"]+ assert result["max_tokens"] == generation_kwargs["max_tokens"]+ assert "repetition_penalty" not in result+ assert "_real_n" not in result+ SamplingParams(**result)+++def test_default_sampling_params_overlay_nothing(helper):+ generation_kwargs = _trl_generation_kwargs()+ result = helper(SamplingParams, generation_kwargs, SamplingParams())+ expected = {+ k: v for k, v in generation_kwargs.items() if k in SamplingParams.__dataclass_fields__+ }+ assert result == expected+++def test_set_kwargs_take_precedence_over_field_diff(helper):+ generation_kwargs = _trl_generation_kwargs()+ user = SamplingParams(min_p = 0.1, seed = 3407, n = 4)+ user._set_kwargs = {"min_p": 0.2, "seed": 3407, "n": 4, "not_a_field": 1}+ result = helper(SamplingParams, generation_kwargs, user)+ assert result["min_p"] == 0.2+ assert "seed" not in result+ assert result["n"] == generation_kwargs["n"]+ assert "not_a_field" not in result++[email protected]("use_set_kwargs", [False, True])+def test_trl_owned_fields_are_not_overridden(helper, use_set_kwargs):+ generation_kwargs = _trl_generation_kwargs()+ overrides = {"temperature": 0.6, "max_tokens": 4096, "logprobs": 5, "min_p": 0.1}+ user = SamplingParams(**overrides)+ if use_set_kwargs:+ user._set_kwargs = dict(overrides)+ result = helper(SamplingParams, generation_kwargs, user)+ assert result["temperature"] == generation_kwargs["temperature"]+ assert result["max_tokens"] == generation_kwargs["max_tokens"]+ assert result["logprobs"] == generation_kwargs["logprobs"]+ assert result["min_p"] == 0.1+++def test_repo_injects_the_local_helper():+ src = _read_source()+ node = next(+ n for n in ast.parse(src).body if isinstance(n, ast.FunctionDef) and n.name == HELPER+ )+ assert [a.arg for a in node.args.args] == [+ "SamplingParams",+ "generation_kwargs",+ "vllm_sampling_params",+ ]+ assert [ast.literal_eval(d) for d in node.args.defaults] == [None]+ assert f'RL_REPLACEMENTS["{HELPER}"]' not in src+ assert f'RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource({HELPER}))' in srcdiff --git a/tests/version_compat/test_trl_vllm_generation_lora_patch.py b/tests/version_compat/test_trl_vllm_generation_lora_patch.pyindex 5b80cc38b8c..22bc8478325 100644--- a/tests/version_compat/test_trl_vllm_generation_lora_patch.py+++ b/tests/version_compat/test_trl_vllm_generation_lora_patch.py@@ -559,3 +559,117 @@ def generate(self, prompts, **kwargs):assert cls.generate(self, ["hello"]) == ["generated"]assert _lora_requests(log) == ["LORA[%s|True]" % _lora_name()], log+++# TRL >= 0.28 builds `SamplingParams(**generation_kwargs)` inside `generate`, from the module global, and never sees+# the trainer's `args.vllm_sampling_params`; the trainer hands them over as `_unsloth_vllm_sampling_params`.+_GENERATE_BUILDS_SAMPLING_PARAMS = """+def generate(self, prompts, **kwargs):+ self.sync_weights()+ generation_kwargs = {"n": 1, "temperature": 1.0, "min_p": 0.0, "max_tokens": 64, "logprobs": 0}+ generation_kwargs.update(self.generation_kwargs)+ sampling_params = SamplingParams(**generation_kwargs)+ if self.fail:+ raise RuntimeError("generate failed")+ return self.llm.generate(prompts, sampling_params = sampling_params)+"""+++class FakeSamplingParams:+ def __init__(+ self,+ n = 1,+ temperature = 1.0,+ min_p = 0.0,+ seed = None,+ max_tokens = 16,+ logprobs = None,+ stop = None,+ include_stop_str_in_output = False,+ ):+ self.n = n+ self.temperature = temperature+ self.min_p = min_p+ self.seed = seed+ self.max_tokens = max_tokens+ self.logprobs = logprobs+ self.stop = [] if stop is None else stop+ self.include_stop_str_in_output = include_stop_str_in_output+++class SamplingParamsEngine(FakeEngine):+ def generate(self, *args, **kwargs):+ self.log.append(("sampling_params", kwargs.get("sampling_params")))+ return ["generated"]+++def _sampling_generation(+ monkeypatch,+ user_sampling_params,+ shared_weights = True,+):+ cls = _build_fake_trl(monkeypatch, _GENERATE_BUILDS_SAMPLING_PARAMS)+ module = sys.modules["trl.generation.vllm_generation"]+ module.SamplingParams = FakeSamplingParams+ _rl_replacements().vllm_generation_init_patch()+ log = []+ self = _make_generation(cls, log, shared_weights = shared_weights, with_lora = shared_weights)+ self.llm = SamplingParamsEngine(log, shared_weights = shared_weights)+ self.generation_kwargs = {}+ self.fail = False+ if user_sampling_params is not None:+ self._unsloth_vllm_sampling_params = user_sampling_params+ return cls, module, self, log+++def _sent_sampling_params(log):+ sent = [entry[1] for entry in log if entry[0] == "sampling_params"]+ assert len(sent) == 1, log+ return sent[0]+++def test_user_vllm_sampling_params_reach_the_engine(monkeypatch):+ user = FakeSamplingParams(+ min_p = 0.1,+ seed = 3407,+ temperature = 0.6,+ stop = ["<|im_end|>"],+ include_stop_str_in_output = True,+ )+ cls, module, self, log = _sampling_generation(monkeypatch, user)+ assert cls.generate(self, ["hello"]) == ["generated"]+ sent = _sent_sampling_params(log)+ assert isinstance(sent, FakeSamplingParams)+ assert sent.min_p == 0.1+ assert sent.stop == ["<|im_end|>"]+ assert sent.include_stop_str_in_output is True+ # GRPOConfig still owns these.+ assert sent.seed is None+ assert sent.temperature == 1.0+ assert sent.max_tokens == 64+ assert module.SamplingParams is FakeSamplingParams+++def test_no_user_sampling_params_leaves_trl_untouched(monkeypatch):+ cls, module, self, log = _sampling_generation(monkeypatch, None)+ cls.generate(self, ["hello"])+ sent = _sent_sampling_params(log)+ assert (sent.min_p, sent.stop, sent.include_stop_str_in_output) == (0.0, [], False)+ assert module.SamplingParams is FakeSamplingParams+++def test_server_mode_ignores_user_sampling_params(monkeypatch):+ cls, module, self, log = _sampling_generation(+ monkeypatch, FakeSamplingParams(min_p = 0.1), shared_weights = False+ )+ cls.generate(self, ["hello"])+ assert _sent_sampling_params(log).min_p == 0.0+ assert module.SamplingParams is FakeSamplingParams+++def test_sampling_params_restored_when_generate_raises(monkeypatch):+ cls, module, self, log = _sampling_generation(monkeypatch, FakeSamplingParams(min_p = 0.1))+ self.fail = True+ with pytest.raises(RuntimeError, match = "generate failed"):+ cls.generate(self, ["hello"])+ assert module.SamplingParams is FakeSamplingParamsdiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex af772d7fe8d..d797282ad4c 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2560,6 +2560,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)RLTrainer_post += vllm_chat_template_sync+ # TRL >= 0.28 builds SamplingParams inside VLLMGeneration, which never sees args; hand it the user's vllm_sampling_params so the generate wrapper in rl_replacements.py can apply them.+ if trainer_file == "grpo_trainer":+ RLTrainer_post += (+ "if getattr(self, 'vllm_generation', None) is not None:\n"+ " self.vllm_generation._unsloth_vllm_sampling_params = getattr(getattr(self, 'args', None), 'vllm_sampling_params', None)\n"+ "pass\n"+ )+other_metrics_processor = ""if trainer_file in RL_METRICS_CHANGES:process_extra_args = RL_METRICS_CHANGES[trainer_file]diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex d65c61f0bd5..69591cfb7f1 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -2747,11 +2747,46 @@ def _patch_grpo_accumulated_loss_hidden_states_dispatch(function):return source+def grpo_update_SamplingParams(+ SamplingParams,+ generation_kwargs,+ vllm_sampling_params = None,+):+ good_sampling_params_keys = inspect.signature(SamplingParams).parameters.keys()++ new_generation_kwargs = {}+ for key in generation_kwargs.keys():+ if key in good_sampling_params_keys:+ new_generation_kwargs[key] = generation_kwargs[key]+ generation_kwargs = new_generation_kwargs++ if vllm_sampling_params is not None:+ overwrites = getattr(vllm_sampling_params, "_set_kwargs", None)+ if overwrites is None:+ default_sampling_params = SamplingParams()+ overwrites = {}+ for key in good_sampling_params_keys:+ if key.startswith("_") or not hasattr(vllm_sampling_params, key):+ continue+ overwrited_key = getattr(vllm_sampling_params, key)+ if overwrited_key != getattr(default_sampling_params, key, None):+ overwrites[key] = overwrited_key+ for key, overwrited_key in overwrites.items():+ if key in good_sampling_params_keys and key not in (+ "seed",+ "n",+ "temperature",+ "max_tokens",+ "logprobs",+ ):+ generation_kwargs[key] = overwrited_key+ return generation_kwargs++grpo_compute_loss = RL_REPLACEMENTS["grpo_compute_loss"]grpo_compute_loss_slow = RL_REPLACEMENTS["grpo_compute_loss_slow"]UnslothEfficientGRPO = RL_REPLACEMENTS["UnslothEfficientGRPO"]grpo_accumulated_loss = RL_REPLACEMENTS["grpo_accumulated_loss"]-grpo_update_SamplingParams = RL_REPLACEMENTS["grpo_update_SamplingParams"]RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_autocast))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_autocast_kwargs))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_model_config))@@ -3586,9 +3621,28 @@ def vllm_generation_init_patch():override("generate", wrap_generation_call)override("chat", wrap_generation_call)override("collective_rpc", wrap_collective_rpc)+ # TRL >= 0.28 builds `SamplingParams(**generation_kwargs)` inside this call, the one site the older-TRL source replacement in rl.py targets; route it through the same helper for this call only.+ user_sampling_params = getattr(self, "_unsloth_vllm_sampling_params", None)+ original_sampling_params = vllm_generation.__dict__.get("SamplingParams", None)+ if user_sampling_params is not None and original_sampling_params is not None:++ def unsloth_sampling_params(*sp_args, **sp_kwargs):+ if sp_args:+ return original_sampling_params(*sp_args, **sp_kwargs)+ return original_sampling_params(+ **grpo_update_SamplingParams(+ original_sampling_params, sp_kwargs, user_sampling_params+ )+ )++ vllm_generation.SamplingParams = unsloth_sampling_params+ else:+ original_sampling_params = Nonetry:return original_generate(self, *args, **kwargs)finally:+ if original_sampling_params is not None:+ vllm_generation.SamplingParams = original_sampling_paramsfor name, had_own, bound in reversed(saved):try:if had_own: