Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO: keep TRL vLLM weight sync on TRL < 0.28 unless fast_inference shares weights (#13911)
Daniel Han36878d65de15
diff --git a/tests/test_grpo_move_model_to_vllm_sync.py b/tests/test_grpo_move_model_to_vllm_sync.pynew file mode 100644index 00000000000..11f99dae159--- /dev/null+++ b/tests/test_grpo_move_model_to_vllm_sync.py@@ -0,0 +1,204 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""TRL < 0.28 GRPO: _move_model_to_vllm is a no-op only for fast_inference, TRL's own vLLM still gets weights."""++import types+from contextlib import nullcontext++import pytest+import torch+++# trl 0.27.2 grpo_trainer.py GRPOTrainer._move_model_to_vllm, FSDP branches dropped.+_TRL_SOURCE = """+ def _move_model_to_vllm(self):+ deepspeed_plugin = self.accelerator.state.deepspeed_plugin+ zero_stage_3 = deepspeed_plugin is not None and deepspeed_plugin.zero_stage == 3+ gather_if_zero3 = nullcontext+ if is_peft_model(self.model):+ with gather_if_zero3(list(self.model.parameters())):+ self.model.merge_adapter()+ for name, param in self.model.named_parameters():+ name = name.removeprefix("base_model.model.").replace(".base_layer", "")+ if self.model.prefix in name:+ continue+ if "original_module" in name:+ continue+ name = self._fix_param_name_to_vllm(name)+ if self.vllm_mode == "server" and self.accelerator.is_main_process:+ self.vllm_client.update_named_param(name, param.data)+ elif self.vllm_mode == "colocate":+ llm_model = self.llm.llm_engine.model_executor.driver_worker.model_runner.model+ llm_model.load_weights([(name, param.data)])+ self.model.unmerge_adapter()+ elif self.is_fsdp_enabled:+ pass+ else:+ for name, param in self.model.named_parameters():+ name = self._fix_param_name_to_vllm(name)+ with gather_if_zero3([param]):+ if self.vllm_mode == "server" and self.accelerator.is_main_process:+ self.vllm_client.update_named_param(name, param.data)+ elif self.vllm_mode == "colocate":+ llm_model = self.llm.llm_engine.model_executor.driver_worker.model_runner.model+ llm_model.load_weights([(name, param.data)])++ if self.vllm_mode == "server" and self.accelerator.is_main_process:+ self.vllm_client.reset_prefix_cache()+ elif self.vllm_mode == "colocate":+ self.llm.reset_prefix_cache()+"""+++class _Recorder:+ def __init__(self):+ self.loaded = []+ self.resets = 0++ def load_weights(self, weights):+ self.loaded.extend(name for name, _ in weights)++ def update_named_param(self, name, data):+ self.loaded.append(name)++ def reset_prefix_cache(self):+ self.resets += 1+++def _trainer(vllm_mode, shared_weights = False):+ rec = _Recorder()+ llm = types.SimpleNamespace(+ llm_engine = types.SimpleNamespace(+ model_executor = types.SimpleNamespace(+ driver_worker = types.SimpleNamespace(model_runner = types.SimpleNamespace(model = rec))+ )+ ),+ reset_prefix_cache = rec.reset_prefix_cache,+ )+ if shared_weights:+ llm.shared_weights = True+ trainer = types.SimpleNamespace(+ accelerator = types.SimpleNamespace(+ state = types.SimpleNamespace(deepspeed_plugin = None), is_main_process = True+ ),+ is_fsdp_enabled = False,+ model = torch.nn.Linear(2, 2),+ vllm_mode = vllm_mode,+ vllm_client = rec,+ _fix_param_name_to_vllm = lambda name: name,+ )+ if vllm_mode == "colocate":+ trainer.llm = llm+ return trainer, rec+++def _patched_method():+ import textwrap++ from unsloth.models.rl_replacements import grpo_trainer__move_model_to_vllm++ source = grpo_trainer__move_model_to_vllm("_move_model_to_vllm", _TRL_SOURCE)+ try:+ from unsloth.models.rl import _strip_vllm_weight_loads+ except ImportError:+ _strip_vllm_weight_loads = None+ if _strip_vllm_weight_loads is not None:+ source = _strip_vllm_weight_loads("_move_model_to_vllm", source, "grpo_trainer")+ namespace = {+ "nullcontext": nullcontext,+ "is_peft_model": lambda model: hasattr(model, "merge_adapter"),+ }+ exec(textwrap.dedent(source), namespace)+ return namespace["_move_model_to_vllm"]++[email protected]("vllm_mode", ["colocate", "server"])+def test_trl_vllm_receives_trainer_weights(vllm_mode):+ trainer, rec = _trainer(vllm_mode)+ _patched_method()(trainer)+ assert rec.loaded == ["weight", "bias"]+ assert rec.resets == 1+++def test_fast_inference_shared_weights_skips_sync():+ trainer, rec = _trainer("colocate", shared_weights = True)+ assert _patched_method()(trainer) is None+ assert rec.loaded == [] and rec.resets == 0+++def test_other_functions_and_trainers_still_stripped():+ from unsloth.models.rl import _strip_vllm_weight_loads++ line = "\n llm_model.load_weights(state_dict.items())\n"+ for function, trainer_file in (+ ("_generate", "grpo_trainer"),+ ("_move_model_to_vllm", "rloo_trainer"),+ ):+ out = _strip_vllm_weight_loads(function, line, trainer_file)+ assert "load_weights" not in out and "pass" in out+ for function in (+ "_move_model_to_vllm",+ "_sync_fsdp_params_to_vllm",+ "_sync_fsdp2_params_to_vllm",+ ):+ assert _strip_vllm_weight_loads(function, line, "grpo_trainer") == line+++class Params4bit(torch.nn.Parameter):+ pass+++def test_bnb_quantized_base_keeps_no_op():+ # merge_adapter on a bitsandbytes base requantizes it from the fp32 LoRA delta and the next+ # fast_lora backward fails on dtype, so a quantized model must not enter TRL's sync body.+ trainer, rec = _trainer("colocate")+ trainer.model.weight = Params4bit(trainer.model.weight.data, requires_grad = False)+ method = _patched_method()+ assert method(trainer) is None and method(trainer) is None+ assert rec.loaded == [] and rec.resets == 0+++def test_quantized_load_flag_keeps_no_op():+ # FSDP2 wraps Params4bit into a DTensor; the model's load flag still marks it quantized.+ trainer, rec = _trainer("colocate")+ trainer.model.is_loaded_in_4bit = True+ method = _patched_method()+ assert method(trainer) is None+ assert rec.loaded == [] and rec.resets == 0+++def _lora_model(dtype):+ peft = pytest.importorskip("peft")+ torch.manual_seed(1)+ base = torch.nn.Sequential(torch.nn.Linear(64, 64, bias = False)).to(dtype)+ config = peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["0"], init_lora_weights = False)+ return peft.get_peft_model(base, config)++[email protected]("dtype", [torch.float16, torch.bfloat16])+def test_lora_sync_never_touches_frozen_base(dtype):+ # merge_adapter + unmerge_adapter is a lossy round trip in fp16 / bf16, so syncing a LoRA+ # model through TRL's PEFT branch would drift the frozen base weights every step.+ model = _lora_model(dtype)+ weight = model.base_model.model[0].base_layer.weight+ before = weight.detach().clone()+ trainer, rec = _trainer("colocate")+ trainer.model = model+ method = _patched_method()+ for _ in range(3):+ assert method(trainer) is None+ assert torch.equal(weight, before)+ assert rec.loaded == [] and rec.resets == 0+++def test_fp8_base_keeps_no_op():+ # merge_adapter refuses FP8 grouped-linear bases, and a 1-byte float base is not a dense sync.+ if not hasattr(torch, "float8_e4m3fn"):+ pytest.skip("no float8 dtype")+ trainer, rec = _trainer("colocate")+ trainer.model.weight = torch.nn.Parameter(+ trainer.model.weight.data.to(torch.float8_e4m3fn), requires_grad = False+ )+ assert _patched_method()(trainer) is None+ assert rec.loaded == [] and rec.resets == 0diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex fcf0bf3a70e..2b6baa5c853 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -99,6 +99,38 @@ except Exception:# TrainingArguments falls back to this FSDP version when fsdp_config names none (1 -> 2 in transformers 5.10.0)._FSDP_DEFAULT_VERSION = 2 if transformers_version >= Version("5.10.0") else 1+_GRPO_VLLM_SYNC_FUNCTIONS = frozenset(+ (+ "_move_model_to_vllm",+ "_sync_fsdp_params_to_vllm", # TRL < 0.20+ "_sync_fsdp1_params_to_vllm",+ "_sync_fsdp2_params_to_vllm",+ )+)+++def _strip_vllm_weight_loads(function, source, trainer_file):+ # GRPO's _move_model_to_vllm returns early under fast_inference (grpo_trainer__move_model_to_vllm),+ # and TRL's own vLLM (colocate or server) needs these lines to receive the trainer weights.+ if trainer_file == "grpo_trainer" and function in _GRPO_VLLM_SYNC_FUNCTIONS:+ return source+ source = re.sub(+ r"(\n[\s]{4,}).+?model_executor\.driver_worker.+?\n",+ r"\n\1pass\n",+ source,+ )+ source = re.sub(+ r"(\n[\s]{4,}).+?load_weights\(.+?\n",+ r"\n\1pass\n",+ source,+ )+ source = re.sub(+ r"\.state_dict\(\)",+ r"",+ source,+ )+ return source+def vLLMSamplingParams(**kwargs):from vllm import SamplingParams@@ -4194,23 +4226,7 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importsource,)- source = re.sub(- r"(\n[\s]{4,}).+?model_executor\.driver_worker.+?\n",- r"\n\1pass\n",- source,- )-- source = re.sub(- r"(\n[\s]{4,}).+?load_weights\(.+?\n",- r"\n\1pass\n",- source,- )-- source = re.sub(- r"\.state_dict\(\)",- r"",- source,- )+ source = _strip_vllm_weight_loads(function, source, trainer_file)if "CUDA_VISIBLE_DEVICES" in os.environ:lora_name = (diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 679f27e1c07..fdd7f05a70c 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -2349,15 +2349,62 @@ def grpo_trainer_fix_maybe_apply_chat_template(function_name, function):RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_fix_maybe_apply_chat_template)+def _unsloth_vllm_sync_unsafe(model):+ # FSDP2 turns Params4bit into a DTensor, so also read the load flags the model keeps.+ if getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False):+ return True+ if getattr(model, "peft_config", None):+ return True+ for module in model.modules():+ if hasattr(module, "lora_A") or hasattr(module, "base_layer"):+ return True+ for param in model.parameters():+ if type(param).__name__ in ("Params4bit", "Int8Params"):+ return True+ if param.is_floating_point() and param.element_size() == 1:+ return True+ return False++def grpo_trainer__move_model_to_vllm(function_name, function):if function_name != "_move_model_to_vllm":return function- def _move_model_to_vllm(self, *args, **kwargs):- return None+ # Only fast_inference shares weights with vLLM. TRL's own vLLM (colocate or server, e.g. full+ # finetuning, which fast_inference refuses) must still receive the trainer weights every step.+ match = re.search(+ r"^[ \t]*def _move_model_to_vllm\(.*?\).*?:[ \t]*\n",+ function,+ re.MULTILINE | re.DOTALL,+ )+ body_indent = (+ re.search(r"^([ \t]+)\S", function[match.end() :], re.MULTILINE) if match else None+ )+ if match is None or body_indent is None:- function = inspect.getsource(_move_model_to_vllm)- return function+ def _move_model_to_vllm(self, *args, **kwargs):+ return None++ return inspect.getsource(_move_model_to_vllm)+ indent = body_indent.group(1)+ # Keep the old no-op where TRL's sync is unsafe:+ # - LoRA / PEFT: merge_adapter + unmerge_adapter on the live model is a lossy round trip in+ # fp16 / bf16, so frozen base weights drift every sync; it also requantizes bitsandbytes bases+ # from the fp32 delta (fast_lora backward then fails) and refuses FP8 grouped-linear bases.+ # - Quantized bases (bitsandbytes, 1-byte floats such as FP8) cannot be loaded into vLLM as is.+ # So the sync only runs for dense full finetuning.+ guard = (+ f"{indent}if getattr(getattr(self, 'llm', None), 'shared_weights', False):\n"+ f"{indent} return None\n"+ f"{indent}if getattr(self, '_unsloth_vllm_sync_skip', None) is None:\n"+ f"{indent} from unsloth.models.rl_replacements import _unsloth_vllm_sync_unsafe\n"+ f"{indent} self._unsloth_vllm_sync_skip = _unsloth_vllm_sync_unsafe(self.model)\n"+ f"{indent} if self._unsloth_vllm_sync_skip:\n"+ f"{indent} print('Unsloth: vLLM weight sync is skipped for a LoRA / PEFT or quantized model without fast_inference, so rollouts use the initial weights. Load with fast_inference = True to sample from the trained model.')\n"+ f"{indent}if self._unsloth_vllm_sync_skip:\n"+ f"{indent} return None\n"+ )+ return function[: match.end()] + guard + function[match.end() :]RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__move_model_to_vllm)