Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO: keep TRL vLLM weight sync on TRL < 0.28 unless fast_inference shares weights (#13911)

Daniel Han36878d65de15
diff --git a/tests/test_grpo_move_model_to_vllm_sync.py b/tests/test_grpo_move_model_to_vllm_sync.py
new file mode 100644
index 00000000000..11f99dae159
--- /dev/null
+++ b/tests/test_grpo_move_model_to_vllm_sync.py
@@ -0,0 +1,204 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""TRL < 0.28 GRPO: _move_model_to_vllm is a no-op only for fast_inference, TRL's own vLLM still gets weights."""
+
+import types
+from contextlib import nullcontext
+
+import pytest
+import torch
+
+
+# trl 0.27.2 grpo_trainer.py GRPOTrainer._move_model_to_vllm, FSDP branches dropped.
+_TRL_SOURCE = """
+ def _move_model_to_vllm(self):
+ deepspeed_plugin = self.accelerator.state.deepspeed_plugin
+ zero_stage_3 = deepspeed_plugin is not None and deepspeed_plugin.zero_stage == 3
+ gather_if_zero3 = nullcontext
+ if is_peft_model(self.model):
+ with gather_if_zero3(list(self.model.parameters())):
+ self.model.merge_adapter()
+ for name, param in self.model.named_parameters():
+ name = name.removeprefix("base_model.model.").replace(".base_layer", "")
+ if self.model.prefix in name:
+ continue
+ if "original_module" in name:
+ continue
+ name = self._fix_param_name_to_vllm(name)
+ if self.vllm_mode == "server" and self.accelerator.is_main_process:
+ self.vllm_client.update_named_param(name, param.data)
+ elif self.vllm_mode == "colocate":
+ llm_model = self.llm.llm_engine.model_executor.driver_worker.model_runner.model
+ llm_model.load_weights([(name, param.data)])
+ self.model.unmerge_adapter()
+ elif self.is_fsdp_enabled:
+ pass
+ else:
+ for name, param in self.model.named_parameters():
+ name = self._fix_param_name_to_vllm(name)
+ with gather_if_zero3([param]):
+ if self.vllm_mode == "server" and self.accelerator.is_main_process:
+ self.vllm_client.update_named_param(name, param.data)
+ elif self.vllm_mode == "colocate":
+ llm_model = self.llm.llm_engine.model_executor.driver_worker.model_runner.model
+ llm_model.load_weights([(name, param.data)])
+
+ if self.vllm_mode == "server" and self.accelerator.is_main_process:
+ self.vllm_client.reset_prefix_cache()
+ elif self.vllm_mode == "colocate":
+ self.llm.reset_prefix_cache()
+"""
+
+
+class _Recorder:
+ def __init__(self):
+ self.loaded = []
+ self.resets = 0
+
+ def load_weights(self, weights):
+ self.loaded.extend(name for name, _ in weights)
+
+ def update_named_param(self, name, data):
+ self.loaded.append(name)
+
+ def reset_prefix_cache(self):
+ self.resets += 1
+
+
+def _trainer(vllm_mode, shared_weights = False):
+ rec = _Recorder()
+ llm = types.SimpleNamespace(
+ llm_engine = types.SimpleNamespace(
+ model_executor = types.SimpleNamespace(
+ driver_worker = types.SimpleNamespace(model_runner = types.SimpleNamespace(model = rec))
+ )
+ ),
+ reset_prefix_cache = rec.reset_prefix_cache,
+ )
+ if shared_weights:
+ llm.shared_weights = True
+ trainer = types.SimpleNamespace(
+ accelerator = types.SimpleNamespace(
+ state = types.SimpleNamespace(deepspeed_plugin = None), is_main_process = True
+ ),
+ is_fsdp_enabled = False,
+ model = torch.nn.Linear(2, 2),
+ vllm_mode = vllm_mode,
+ vllm_client = rec,
+ _fix_param_name_to_vllm = lambda name: name,
+ )
+ if vllm_mode == "colocate":
+ trainer.llm = llm
+ return trainer, rec
+
+
+def _patched_method():
+ import textwrap
+
+ from unsloth.models.rl_replacements import grpo_trainer__move_model_to_vllm
+
+ source = grpo_trainer__move_model_to_vllm("_move_model_to_vllm", _TRL_SOURCE)
+ try:
+ from unsloth.models.rl import _strip_vllm_weight_loads
+ except ImportError:
+ _strip_vllm_weight_loads = None
+ if _strip_vllm_weight_loads is not None:
+ source = _strip_vllm_weight_loads("_move_model_to_vllm", source, "grpo_trainer")
+ namespace = {
+ "nullcontext": nullcontext,
+ "is_peft_model": lambda model: hasattr(model, "merge_adapter"),
+ }
+ exec(textwrap.dedent(source), namespace)
+ return namespace["_move_model_to_vllm"]
+
+
[email protected]("vllm_mode", ["colocate", "server"])
+def test_trl_vllm_receives_trainer_weights(vllm_mode):
+ trainer, rec = _trainer(vllm_mode)
+ _patched_method()(trainer)
+ assert rec.loaded == ["weight", "bias"]
+ assert rec.resets == 1
+
+
+def test_fast_inference_shared_weights_skips_sync():
+ trainer, rec = _trainer("colocate", shared_weights = True)
+ assert _patched_method()(trainer) is None
+ assert rec.loaded == [] and rec.resets == 0
+
+
+def test_other_functions_and_trainers_still_stripped():
+ from unsloth.models.rl import _strip_vllm_weight_loads
+
+ line = "\n llm_model.load_weights(state_dict.items())\n"
+ for function, trainer_file in (
+ ("_generate", "grpo_trainer"),
+ ("_move_model_to_vllm", "rloo_trainer"),
+ ):
+ out = _strip_vllm_weight_loads(function, line, trainer_file)
+ assert "load_weights" not in out and "pass" in out
+ for function in (
+ "_move_model_to_vllm",
+ "_sync_fsdp_params_to_vllm",
+ "_sync_fsdp2_params_to_vllm",
+ ):
+ assert _strip_vllm_weight_loads(function, line, "grpo_trainer") == line
+
+
+class Params4bit(torch.nn.Parameter):
+ pass
+
+
+def test_bnb_quantized_base_keeps_no_op():
+ # merge_adapter on a bitsandbytes base requantizes it from the fp32 LoRA delta and the next
+ # fast_lora backward fails on dtype, so a quantized model must not enter TRL's sync body.
+ trainer, rec = _trainer("colocate")
+ trainer.model.weight = Params4bit(trainer.model.weight.data, requires_grad = False)
+ method = _patched_method()
+ assert method(trainer) is None and method(trainer) is None
+ assert rec.loaded == [] and rec.resets == 0
+
+
+def test_quantized_load_flag_keeps_no_op():
+ # FSDP2 wraps Params4bit into a DTensor; the model's load flag still marks it quantized.
+ trainer, rec = _trainer("colocate")
+ trainer.model.is_loaded_in_4bit = True
+ method = _patched_method()
+ assert method(trainer) is None
+ assert rec.loaded == [] and rec.resets == 0
+
+
+def _lora_model(dtype):
+ peft = pytest.importorskip("peft")
+ torch.manual_seed(1)
+ base = torch.nn.Sequential(torch.nn.Linear(64, 64, bias = False)).to(dtype)
+ config = peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["0"], init_lora_weights = False)
+ return peft.get_peft_model(base, config)
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
+def test_lora_sync_never_touches_frozen_base(dtype):
+ # merge_adapter + unmerge_adapter is a lossy round trip in fp16 / bf16, so syncing a LoRA
+ # model through TRL's PEFT branch would drift the frozen base weights every step.
+ model = _lora_model(dtype)
+ weight = model.base_model.model[0].base_layer.weight
+ before = weight.detach().clone()
+ trainer, rec = _trainer("colocate")
+ trainer.model = model
+ method = _patched_method()
+ for _ in range(3):
+ assert method(trainer) is None
+ assert torch.equal(weight, before)
+ assert rec.loaded == [] and rec.resets == 0
+
+
+def test_fp8_base_keeps_no_op():
+ # merge_adapter refuses FP8 grouped-linear bases, and a 1-byte float base is not a dense sync.
+ if not hasattr(torch, "float8_e4m3fn"):
+ pytest.skip("no float8 dtype")
+ trainer, rec = _trainer("colocate")
+ trainer.model.weight = torch.nn.Parameter(
+ trainer.model.weight.data.to(torch.float8_e4m3fn), requires_grad = False
+ )
+ assert _patched_method()(trainer) is None
+ assert rec.loaded == [] and rec.resets == 0
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index fcf0bf3a70e..2b6baa5c853 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -99,6 +99,38 @@ except Exception:
# TrainingArguments falls back to this FSDP version when fsdp_config names none (1 -> 2 in transformers 5.10.0).
_FSDP_DEFAULT_VERSION = 2 if transformers_version >= Version("5.10.0") else 1
+_GRPO_VLLM_SYNC_FUNCTIONS = frozenset(
+ (
+ "_move_model_to_vllm",
+ "_sync_fsdp_params_to_vllm", # TRL < 0.20
+ "_sync_fsdp1_params_to_vllm",
+ "_sync_fsdp2_params_to_vllm",
+ )
+)
+
+
+def _strip_vllm_weight_loads(function, source, trainer_file):
+ # GRPO's _move_model_to_vllm returns early under fast_inference (grpo_trainer__move_model_to_vllm),
+ # and TRL's own vLLM (colocate or server) needs these lines to receive the trainer weights.
+ if trainer_file == "grpo_trainer" and function in _GRPO_VLLM_SYNC_FUNCTIONS:
+ return source
+ source = re.sub(
+ r"(\n[\s]{4,}).+?model_executor\.driver_worker.+?\n",
+ r"\n\1pass\n",
+ source,
+ )
+ source = re.sub(
+ r"(\n[\s]{4,}).+?load_weights\(.+?\n",
+ r"\n\1pass\n",
+ source,
+ )
+ source = re.sub(
+ r"\.state_dict\(\)",
+ r"",
+ source,
+ )
+ return source
+
def vLLMSamplingParams(**kwargs):
from vllm import SamplingParams
@@ -4194,23 +4226,7 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
source,
)
- source = re.sub(
- r"(\n[\s]{4,}).+?model_executor\.driver_worker.+?\n",
- r"\n\1pass\n",
- source,
- )
-
- source = re.sub(
- r"(\n[\s]{4,}).+?load_weights\(.+?\n",
- r"\n\1pass\n",
- source,
- )
-
- source = re.sub(
- r"\.state_dict\(\)",
- r"",
- source,
- )
+ source = _strip_vllm_weight_loads(function, source, trainer_file)
if "CUDA_VISIBLE_DEVICES" in os.environ:
lora_name = (
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 679f27e1c07..fdd7f05a70c 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -2349,15 +2349,62 @@ def grpo_trainer_fix_maybe_apply_chat_template(function_name, function):
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_fix_maybe_apply_chat_template)
+def _unsloth_vllm_sync_unsafe(model):
+ # FSDP2 turns Params4bit into a DTensor, so also read the load flags the model keeps.
+ if getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False):
+ return True
+ if getattr(model, "peft_config", None):
+ return True
+ for module in model.modules():
+ if hasattr(module, "lora_A") or hasattr(module, "base_layer"):
+ return True
+ for param in model.parameters():
+ if type(param).__name__ in ("Params4bit", "Int8Params"):
+ return True
+ if param.is_floating_point() and param.element_size() == 1:
+ return True
+ return False
+
+
def grpo_trainer__move_model_to_vllm(function_name, function):
if function_name != "_move_model_to_vllm":
return function
- def _move_model_to_vllm(self, *args, **kwargs):
- return None
+ # Only fast_inference shares weights with vLLM. TRL's own vLLM (colocate or server, e.g. full
+ # finetuning, which fast_inference refuses) must still receive the trainer weights every step.
+ match = re.search(
+ r"^[ \t]*def _move_model_to_vllm\(.*?\).*?:[ \t]*\n",
+ function,
+ re.MULTILINE | re.DOTALL,
+ )
+ body_indent = (
+ re.search(r"^([ \t]+)\S", function[match.end() :], re.MULTILINE) if match else None
+ )
+ if match is None or body_indent is None:
- function = inspect.getsource(_move_model_to_vllm)
- return function
+ def _move_model_to_vllm(self, *args, **kwargs):
+ return None
+
+ return inspect.getsource(_move_model_to_vllm)
+ indent = body_indent.group(1)
+ # Keep the old no-op where TRL's sync is unsafe:
+ # - LoRA / PEFT: merge_adapter + unmerge_adapter on the live model is a lossy round trip in
+ # fp16 / bf16, so frozen base weights drift every sync; it also requantizes bitsandbytes bases
+ # from the fp32 delta (fast_lora backward then fails) and refuses FP8 grouped-linear bases.
+ # - Quantized bases (bitsandbytes, 1-byte floats such as FP8) cannot be loaded into vLLM as is.
+ # So the sync only runs for dense full finetuning.
+ guard = (
+ f"{indent}if getattr(getattr(self, 'llm', None), 'shared_weights', False):\n"
+ f"{indent} return None\n"
+ f"{indent}if getattr(self, '_unsloth_vllm_sync_skip', None) is None:\n"
+ f"{indent} from unsloth.models.rl_replacements import _unsloth_vllm_sync_unsafe\n"
+ f"{indent} self._unsloth_vllm_sync_skip = _unsloth_vllm_sync_unsafe(self.model)\n"
+ f"{indent} if self._unsloth_vllm_sync_skip:\n"
+ f"{indent} print('Unsloth: vLLM weight sync is skipped for a LoRA / PEFT or quantized model without fast_inference, so rollouts use the initial weights. Load with fast_inference = True to sample from the trained model.')\n"
+ f"{indent}if self._unsloth_vllm_sync_skip:\n"
+ f"{indent} return None\n"
+ )
+ return function[: match.end()] + guard + function[match.end() :]
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__move_model_to_vllm)