Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Give vLLM a valid top_k when fast_inference enables it in the GRPO trainer (#11675)
Daniel Han4c30b0d7c90c
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.ymlindex 33d93278642..8ef1e928281 100644--- a/.github/workflows/version-compat-ci.yml+++ b/.github/workflows/version-compat-ci.yml@@ -315,6 +315,7 @@ jobs:tests/version_compat/test_trl_fake_train_cpu.py \tests/version_compat/test_trl_padding_free_max_length.py \tests/version_compat/test_trl_loss_normalization_contract.py \+ tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \-v --tb=short# `main` is scheduled/dispatch-only so PR jobs stay fast and a bleeding-edge# TRL break does not red every PR. github.event_name is valid in a step if.@@ -336,6 +337,7 @@ jobs:tests/version_compat/test_trl_fake_train_cpu.py \tests/version_compat/test_trl_padding_free_max_length.py \tests/version_compat/test_trl_loss_normalization_contract.py \+ tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \-v --tb=short- name: Save the pip cachediff --git a/tests/fast_inference/test_fast_inference.py b/tests/fast_inference/test_fast_inference.pyindex b0a9585f248..d482dbd6c25 100644--- a/tests/fast_inference/test_fast_inference.py+++ b/tests/fast_inference/test_fast_inference.py@@ -141,7 +141,11 @@ def test_fast_inference():# The trainer must actually route rollouts through vLLM, otherwise it would# fall back to HF generation and never exercise WorkerLoRAManager.assert trainer.args.use_vllm, "GRPO is not configured to use vLLM"- assert getattr(trainer, "llm", None) is not None, "GRPO did not bind a vLLM engine"+ # TRL >= 0.28 keeps the engine on trainer.vllm_generation.llm instead of trainer.llm.+ engine = getattr(trainer, "llm", None) or getattr(+ getattr(trainer, "vllm_generation", None), "llm", None+ )+ assert engine is not None, "GRPO did not bind a vLLM engine"with header_footer_context("GRPO train (vLLM LoRA rollout)"):trainer_stats = trainer.train()diff --git a/tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py b/tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.pynew file mode 100644index 00000000000..6463cf59b7e--- /dev/null+++ b/tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py@@ -0,0 +1,78 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""A fast_inference model must not hand vLLM top_k=None.++Unsloth's GRPOConfig defaults top_k to None and maps it to -1 only when use_vllm is set on the+config. A notebook that gets vLLM from FastLanguageModel(fast_inference=True) never sets it, so+the trainer turns use_vllm on in __init__, after that guard ran, and TRL >= 0.27 passes the None+straight to vllm.SamplingParams, which raises TypeError. Runs the generated __init__ block.+"""++from __future__ import annotations++import importlib.util+import inspect+import os+import re+import sys+import textwrap+import types+from pathlib import Path++import pytest+++os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")+os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")++if importlib.util.find_spec("torch") is None:+ pytest.skip("torch not installed", allow_module_level = True)++_SPOOF_DIR = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(_SPOOF_DIR))+import _zoo_aggressive_cuda_spoof as _spoof # noqa: E402++_spoof.apply()+++def _vllm_setter_block():+ pytest.importorskip("trl")+ import unsloth # noqa: F401+ from trl import GRPOTrainer++ assert GRPOTrainer.__name__ == "UnslothGRPOTrainer", "GRPO patch did not apply"+ # __init__ is wrapped after generation, so read the generated module rather than the method.+ source = inspect.getsource(sys.modules[GRPOTrainer.__module__])+ match = re.search(+ r"^( +)if hasattr\(model, 'vllm_engine'\) and hasattr\(args, 'use_vllm'\):\n(?:\1 {4}.*\n)+",+ source,+ flags = re.MULTILINE,+ )+ assert match is not None, "vLLM setter block not found in the generated trainer"+ return textwrap.dedent(match.group(0))+++def _run(block, model, **args):+ args = types.SimpleNamespace(**args)+ exec(block, {"os": os}, {"model": model, "args": args})+ return args++[email protected]("top_k", [None, 0])+def test_fast_inference_model_gets_a_vllm_safe_top_k(top_k):+ engine_model = types.SimpleNamespace(vllm_engine = object())+ args = _run(_vllm_setter_block(), engine_model, use_vllm = False, top_k = top_k)+ assert args.use_vllm is True+ assert args.top_k == -1++[email protected]("top_k", [1, 20, -1])+def test_a_user_top_k_is_kept(top_k):+ engine_model = types.SimpleNamespace(vllm_engine = object())+ assert _run(_vllm_setter_block(), engine_model, use_vllm = False, top_k = top_k).top_k == top_k+++def test_a_model_without_vllm_keeps_none():+ # HF generate reads None as "use the model's generation_config", so it must survive.+ args = _run(_vllm_setter_block(), types.SimpleNamespace(), use_vllm = False, top_k = None)+ assert args.top_k is None and args.use_vllm is Falsediff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex d6be24f4feb..b95b7ebcdd4 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -3270,6 +3270,12 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import+ "if (getattr(args, 'use_vllm', False) == False):\n"+ " " * 16+ "args.use_vllm = True\n"+ # TRL >= 0.27 hands args.top_k to vLLM's SamplingParams unchanged, and it rejects None;+ # the config-time guard misses this when vLLM comes only from fast_inference.+ + " " * 12+ + "if getattr(args, 'top_k', -1) is None or getattr(args, 'top_k', -1) == 0:\n"+ + " " * 16+ + "args.top_k = -1\n")if "grpo" in trainer_file and trl_version >= Version("0.18.0"):