Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Give vLLM a valid top_k when fast_inference enables it in the GRPO trainer (#11675)

Daniel Han4c30b0d7c90c
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
index 33d93278642..8ef1e928281 100644
--- a/.github/workflows/version-compat-ci.yml
+++ b/.github/workflows/version-compat-ci.yml
@@ -315,6 +315,7 @@ jobs:
tests/version_compat/test_trl_fake_train_cpu.py \
tests/version_compat/test_trl_padding_free_max_length.py \
tests/version_compat/test_trl_loss_normalization_contract.py \
+ tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \
-v --tb=short
# `main` is scheduled/dispatch-only so PR jobs stay fast and a bleeding-edge
# TRL break does not red every PR. github.event_name is valid in a step if.
@@ -336,6 +337,7 @@ jobs:
tests/version_compat/test_trl_fake_train_cpu.py \
tests/version_compat/test_trl_padding_free_max_length.py \
tests/version_compat/test_trl_loss_normalization_contract.py \
+ tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \
-v --tb=short
- name: Save the pip cache
diff --git a/tests/fast_inference/test_fast_inference.py b/tests/fast_inference/test_fast_inference.py
index b0a9585f248..d482dbd6c25 100644
--- a/tests/fast_inference/test_fast_inference.py
+++ b/tests/fast_inference/test_fast_inference.py
@@ -141,7 +141,11 @@ def test_fast_inference():
# The trainer must actually route rollouts through vLLM, otherwise it would
# fall back to HF generation and never exercise WorkerLoRAManager.
assert trainer.args.use_vllm, "GRPO is not configured to use vLLM"
- assert getattr(trainer, "llm", None) is not None, "GRPO did not bind a vLLM engine"
+ # TRL >= 0.28 keeps the engine on trainer.vllm_generation.llm instead of trainer.llm.
+ engine = getattr(trainer, "llm", None) or getattr(
+ getattr(trainer, "vllm_generation", None), "llm", None
+ )
+ assert engine is not None, "GRPO did not bind a vLLM engine"
with header_footer_context("GRPO train (vLLM LoRA rollout)"):
trainer_stats = trainer.train()
diff --git a/tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py b/tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py
new file mode 100644
index 00000000000..6463cf59b7e
--- /dev/null
+++ b/tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py
@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""A fast_inference model must not hand vLLM top_k=None.
+
+Unsloth's GRPOConfig defaults top_k to None and maps it to -1 only when use_vllm is set on the
+config. A notebook that gets vLLM from FastLanguageModel(fast_inference=True) never sets it, so
+the trainer turns use_vllm on in __init__, after that guard ran, and TRL >= 0.27 passes the None
+straight to vllm.SamplingParams, which raises TypeError. Runs the generated __init__ block.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import inspect
+import os
+import re
+import sys
+import textwrap
+import types
+from pathlib import Path
+
+import pytest
+
+
+os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
+os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
+
+if importlib.util.find_spec("torch") is None:
+ pytest.skip("torch not installed", allow_module_level = True)
+
+_SPOOF_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(_SPOOF_DIR))
+import _zoo_aggressive_cuda_spoof as _spoof # noqa: E402
+
+_spoof.apply()
+
+
+def _vllm_setter_block():
+ pytest.importorskip("trl")
+ import unsloth # noqa: F401
+ from trl import GRPOTrainer
+
+ assert GRPOTrainer.__name__ == "UnslothGRPOTrainer", "GRPO patch did not apply"
+ # __init__ is wrapped after generation, so read the generated module rather than the method.
+ source = inspect.getsource(sys.modules[GRPOTrainer.__module__])
+ match = re.search(
+ r"^( +)if hasattr\(model, 'vllm_engine'\) and hasattr\(args, 'use_vllm'\):\n(?:\1 {4}.*\n)+",
+ source,
+ flags = re.MULTILINE,
+ )
+ assert match is not None, "vLLM setter block not found in the generated trainer"
+ return textwrap.dedent(match.group(0))
+
+
+def _run(block, model, **args):
+ args = types.SimpleNamespace(**args)
+ exec(block, {"os": os}, {"model": model, "args": args})
+ return args
+
+
[email protected]("top_k", [None, 0])
+def test_fast_inference_model_gets_a_vllm_safe_top_k(top_k):
+ engine_model = types.SimpleNamespace(vllm_engine = object())
+ args = _run(_vllm_setter_block(), engine_model, use_vllm = False, top_k = top_k)
+ assert args.use_vllm is True
+ assert args.top_k == -1
+
+
[email protected]("top_k", [1, 20, -1])
+def test_a_user_top_k_is_kept(top_k):
+ engine_model = types.SimpleNamespace(vllm_engine = object())
+ assert _run(_vllm_setter_block(), engine_model, use_vllm = False, top_k = top_k).top_k == top_k
+
+
+def test_a_model_without_vllm_keeps_none():
+ # HF generate reads None as "use the model's generation_config", so it must survive.
+ args = _run(_vllm_setter_block(), types.SimpleNamespace(), use_vllm = False, top_k = None)
+ assert args.top_k is None and args.use_vllm is False
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index d6be24f4feb..b95b7ebcdd4 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -3270,6 +3270,12 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
+ "if (getattr(args, 'use_vllm', False) == False):\n"
+ " " * 16
+ "args.use_vllm = True\n"
+ # TRL >= 0.27 hands args.top_k to vLLM's SamplingParams unchanged, and it rejects None;
+ # the config-time guard misses this when vLLM comes only from fast_inference.
+ + " " * 12
+ + "if getattr(args, 'top_k', -1) is None or getattr(args, 'top_k', -1) == 0:\n"
+ + " " * 16
+ + "args.top_k = -1\n"
)
if "grpo" in trainer_file and trl_version >= Version("0.18.0"):