Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO: sync LoRA weights to TRL's own vLLM engine on TRL < 0.28, refuse quantized bases (#14831)
Daniel Han4dad471914c8
diff --git a/tests/test_grpo_move_model_to_vllm_sync.py b/tests/test_grpo_move_model_to_vllm_sync.pyindex 11f99dae159..f1215b02f64 100644--- a/tests/test_grpo_move_model_to_vllm_sync.py+++ b/tests/test_grpo_move_model_to_vllm_sync.py@@ -1,7 +1,8 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""TRL < 0.28 GRPO: _move_model_to_vllm is a no-op only for fast_inference, TRL's own vLLM still gets weights."""+"""TRL < 0.28 GRPO: _move_model_to_vllm is a no-op only for fast_inference, TRL's own vLLM still gets weights+(dense as is, LoRA merged out of place); a quantized base raises instead of training against frozen rollouts."""import typesfrom contextlib import nullcontext@@ -25,7 +26,7 @@ _TRL_SOURCE = """continueif "original_module" in name:continue- name = self._fix_param_name_to_vllm(name)+ name = self._fix_param_name_to_vllm(name, extra_prefixes=["modules_to_save.default."])if self.vllm_mode == "server" and self.accelerator.is_main_process:self.vllm_client.update_named_param(name, param.data)elif self.vllm_mode == "colocate":@@ -51,6 +52,13 @@ _TRL_SOURCE = """"""+def _trl_fix_param_name_to_vllm(name, extra_prefixes = None):+ # trl 0.27.2 grpo_trainer.py GRPOTrainer._fix_param_name_to_vllm+ for prefix in ["_checkpoint_wrapped_module."] + (extra_prefixes or []):+ name = name.replace(prefix, "")+ return name++class _Recorder:def __init__(self):self.loaded = []@@ -86,7 +94,7 @@ def _trainer(vllm_mode, shared_weights = False):model = torch.nn.Linear(2, 2),vllm_mode = vllm_mode,vllm_client = rec,- _fix_param_name_to_vllm = lambda name: name,+ _fix_param_name_to_vllm = _trl_fix_param_name_to_vllm,)if vllm_mode == "colocate":trainer.llm = llm@@ -149,56 +157,720 @@ class Params4bit(torch.nn.Parameter):pass-def test_bnb_quantized_base_keeps_no_op():+_STALE_ENV = "UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS"+++def test_bnb_quantized_base_raises(monkeypatch):# merge_adapter on a bitsandbytes base requantizes it from the fp32 LoRA delta and the next- # fast_lora backward fails on dtype, so a quantized model must not enter TRL's sync body.+ # fast_lora backward fails on dtype, and packed bytes are not vLLM's layout: rollouts would stay+ # at the initial weights, so refuse before the first rollout instead of training against them.+ monkeypatch.delenv(_STALE_ENV, raising = False)+ trainer, rec = _trainer("colocate")+ trainer.model.weight = Params4bit(trainer.model.weight.data, requires_grad = False)+ with pytest.raises(RuntimeError, match = "fast_inference = True"):+ _patched_method()(trainer)+ assert rec.loaded == [] and rec.resets == 0+++def test_bnb_quantized_base_opt_in_keeps_no_op(monkeypatch):+ monkeypatch.setenv(_STALE_ENV, "1")trainer, rec = _trainer("colocate")trainer.model.weight = Params4bit(trainer.model.weight.data, requires_grad = False)method = _patched_method()assert method(trainer) is None and method(trainer) is Noneassert rec.loaded == [] and rec.resets == 0+ assert trainer._unsloth_vllm_sync_skip is True-def test_quantized_load_flag_keeps_no_op():[email protected]("vllm_mode", ["colocate", "server"])+def test_quantized_load_flag_raises(vllm_mode, monkeypatch):# FSDP2 wraps Params4bit into a DTensor; the model's load flag still marks it quantized.- trainer, rec = _trainer("colocate")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ trainer, rec = _trainer(vllm_mode)trainer.model.is_loaded_in_4bit = True- method = _patched_method()- assert method(trainer) is None+ with pytest.raises(RuntimeError, match = "quantized"):+ _patched_method()(trainer)assert rec.loaded == [] and rec.resets == 0-def _lora_model(dtype):+class _WeightRecorder(_Recorder):+ def __init__(self):+ super().__init__()+ self.tensors = []++ def load_weights(self, weights):+ for name, tensor in weights:+ self.loaded.append(name)+ self.tensors.append((name, tensor.detach().clone()))++ def update_named_param(self, name, data):+ self.loaded.append(name)+ self.tensors.append((name, data.detach().clone()))+++def _lora_model(dtype, **kwargs):peft = pytest.importorskip("peft")torch.manual_seed(1)- base = torch.nn.Sequential(torch.nn.Linear(64, 64, bias = False)).to(dtype)- config = peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["0"], init_lora_weights = False)+ base = torch.nn.Sequential(torch.nn.Linear(64, 64, bias = False), torch.nn.Linear(64, 8)).to(+ dtype+ )+ config = peft.LoraConfig(+ r = 4, lora_alpha = 8, target_modules = ["0"], init_lora_weights = False, **kwargs+ )return peft.get_peft_model(base, config)+def _lora_trainer(vllm_mode, dtype, **kwargs):+ model = _lora_model(dtype, **kwargs)+ trainer, _ = _trainer(vllm_mode)+ rec = _WeightRecorder()+ trainer.model = model+ trainer.vllm_client = rec+ if vllm_mode == "colocate":+ trainer.llm.llm_engine.model_executor.driver_worker.model_runner.model = rec+ trainer.llm.reset_prefix_cache = rec.reset_prefix_cache+ return trainer, rec, model+++def _expected_merged(model):+ layer = model.base_model.model[0]+ weight = layer.base_layer.weight+ delta = layer.lora_B["default"].weight.float() @ layer.lora_A["default"].weight.float()+ return (weight.float() + delta * layer.scaling["default"]).to(weight.dtype)++[email protected]("vllm_mode", ["colocate", "server"])[email protected]("dtype", [torch.float16, torch.bfloat16])+def test_lora_sync_pushes_merged_weights_every_step(vllm_mode, dtype):+ # TRL's own vLLM engine (fast_inference = False) must sample from the trained adapter: push+ # the merged weight under the base-layer name each step, as TRL's PEFT branch does.+ trainer, rec, model = _lora_trainer(vllm_mode, dtype)+ method = _patched_method()+ for step in range(2):+ with torch.no_grad():+ model.base_model.model[0].lora_B["default"].weight.add_(0.5)+ before = len(rec.tensors)+ assert method(trainer) is None+ pushed = dict(rec.tensors[before:])+ assert sorted(pushed) == ["0.weight", "1.bias", "1.weight"]+ assert torch.equal(pushed["0.weight"], _expected_merged(model))+ assert torch.equal(pushed["1.weight"], model.base_model.model[1].weight)+ assert rec.resets == 2+ assert trainer._unsloth_vllm_sync_skip is False++@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])def test_lora_sync_never_touches_frozen_base(dtype):- # merge_adapter + unmerge_adapter is a lossy round trip in fp16 / bf16, so syncing a LoRA- # model through TRL's PEFT branch would drift the frozen base weights every step.- model = _lora_model(dtype)- weight = model.base_model.model[0].base_layer.weight- before = weight.detach().clone()- trainer, rec = _trainer("colocate")- trainer.model = model+ # merge_adapter + unmerge_adapter is a lossy round trip in fp16 / bf16, so the merged weights+ # are built out of place and the frozen base stays bit-identical across syncs.+ trainer, rec, model = _lora_trainer("colocate", dtype)+ layer = model.base_model.model[0]+ before = layer.base_layer.weight.detach().clone()method = _patched_method()for _ in range(3):assert method(trainer) is None- assert torch.equal(weight, before)- assert rec.loaded == [] and rec.resets == 0+ assert torch.equal(layer.base_layer.weight, before)+ assert not layer.merged+ assert len(rec.tensors) == 9+++def test_dora_syncs_out_of_place():+ # DoRA needs PEFT's own merge (magnitude vector): it runs on a copy of the base weight.+ trainer, rec, model = _lora_trainer("colocate", torch.float32, use_dora = True)+ assert _patched_method()(trainer) is None+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"+ assert "0.weight" in rec.loaded and not model.base_model.model[0].merged+ assert rec.resets == 1-def test_fp8_base_keeps_no_op():+def test_fp8_base_raises(monkeypatch):# merge_adapter refuses FP8 grouped-linear bases, and a 1-byte float base is not a dense sync.if not hasattr(torch, "float8_e4m3fn"):pytest.skip("no float8 dtype")+ monkeypatch.delenv(_STALE_ENV, raising = False)trainer, rec = _trainer("colocate")trainer.model.weight = torch.nn.Parameter(trainer.model.weight.data.to(torch.float8_e4m3fn), requires_grad = False)+ with pytest.raises(RuntimeError, match = "fast_inference = True"):+ _patched_method()(trainer)+ assert rec.loaded == [] and rec.resets == 0+++# Codex review round 2: quantizers beyond bitsandbytes, non-LoRA PEFT, unmergeable LoRA variants,+# target_parameters, custom modules_to_save adapter names, trainable tokens.+++class _PackedLinear(torch.nn.Module):+ # GPTQ QuantLinear / AWQ WQLinear keep int32 packed weights in a qweight buffer.+ def __init__(self):+ super().__init__()+ self.register_buffer("qweight", torch.zeros(2, 2, dtype = torch.int32))+ self.bias = torch.nn.Parameter(torch.zeros(2))+++def _mark_hf_quantizer(model):+ model.hf_quantizer = object()+++def _mark_quantization_method(model):+ model.quantization_method = "hqq"+++def _mark_quantization_config(model):+ model.config = types.SimpleNamespace(quantization_config = {"quant_method": "awq", "bits": 4})+++def _mark_packed_weight(model):+ model.add_module("packed", _PackedLinear())+++ "mark",+ [_mark_hf_quantizer, _mark_quantization_method, _mark_quantization_config, _mark_packed_weight],+)+def test_gptq_awq_hqq_style_quantized_base_raises(mark, monkeypatch):+ monkeypatch.delenv(_STALE_ENV, raising = False)+ trainer, rec = _trainer("colocate")+ mark(trainer.model)+ with pytest.raises(RuntimeError, match = "quantized"):+ _patched_method()(trainer)+ assert rec.loaded == [] and rec.resets == 0+++def test_dequantized_config_still_syncs():+ # A dequantize = True load leaves dense weights: it is a dense sync, not a quantized base.+ trainer, rec = _trainer("colocate")+ trainer.model.config = types.SimpleNamespace(+ quantization_config = {"quant_method": "fp8", "dequantize": True}+ )+ _patched_method()(trainer)+ assert rec.loaded == ["weight", "bias"]+++def _peft_trainer(model, vllm_mode = "colocate"):+ trainer, _ = _trainer(vllm_mode)+ rec = _WeightRecorder()+ trainer.model = model+ trainer.vllm_client = rec+ if vllm_mode == "colocate":+ trainer.llm.llm_engine.model_executor.driver_worker.model_runner.model = rec+ trainer.llm.reset_prefix_cache = rec.reset_prefix_cache+ return trainer, rec+++def _perturb_trainable(model):+ with torch.no_grad():+ for name, param in model.named_parameters():+ if param.requires_grad:+ param.add_(torch.randn_like(param) * 0.3 + 0.2)+++def _merged_state(model):+ import copy+ return {k: v.clone() for k, v in copy.deepcopy(model).merge_and_unload().state_dict().items()}+++def _base_mlp():+ torch.manual_seed(1)+ return torch.nn.Sequential(torch.nn.Linear(16, 16, bias = False), torch.nn.Linear(16, 8))+++def _non_lora_config(kind):+ peft = pytest.importorskip("peft")+ if kind == "loha":+ return peft.LoHaConfig(r = 2, target_modules = ["0"], init_weights = False)+ if kind == "lokr":+ return peft.LoKrConfig(r = 2, target_modules = ["0"], init_weights = False)+ if kind == "ia3":+ return peft.IA3Config(target_modules = ["0"], feedforward_modules = [])+ return peft.VeraConfig(r = 2, target_modules = ["0"])++[email protected]("kind", ["loha", "lokr", "ia3", "vera"])+def test_non_lora_peft_syncs_merged_weights(kind):+ # VeRA / LoHa / LoKr / IA3 have no lora_A: the plain LoRA merge would push the frozen base weight.+ # PEFT's own merge must build what is pushed (on a copy of the base, see the drift test below).+ peft = pytest.importorskip("peft")+ model = peft.get_peft_model(_base_mlp(), _non_lora_config(kind))+ _perturb_trainable(model)+ expected = _merged_state(model)+ trainer, rec = _peft_trainer(model)+ assert _patched_method()(trainer) is None+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"+ pushed = dict(rec.tensors)+ assert sorted(pushed) == ["0.weight", "1.bias", "1.weight"]+ assert not torch.equal(pushed["0.weight"], model.base_model.model[0].base_layer.weight)+ torch.testing.assert_close(pushed["0.weight"], expected["0.weight"])+ assert rec.resets == 1+++def test_alora_raises_instead_of_flattening(monkeypatch):+ # aLoRA only applies after its invocation tokens; PEFT refuses to merge it, and a static merge+ # would serve a different model than the one being trained.+ peft = pytest.importorskip("peft")+ if "alora_invocation_tokens" not in peft.LoraConfig.__dataclass_fields__:+ pytest.skip("this PEFT has no aLoRA")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ config = peft.LoraConfig(+ r = 2, target_modules = ["0"], alora_invocation_tokens = [5], init_lora_weights = False+ )+ model = peft.get_peft_model(_base_mlp(), config)+ trainer, rec = _peft_trainer(model)+ with pytest.raises(RuntimeError, match = "aLoRA"):+ _patched_method()(trainer)+ assert rec.loaded == [] and rec.resets == 0+++class _Experts(torch.nn.Module):+ # Fused MoE experts as in transformers 5 (gate_up_proj is a bare 3D parameter).+ def __init__(self):+ super().__init__()+ self.gate_up_proj = torch.nn.Parameter(torch.randn(4, 16, 8))++ def forward(self, x):+ return torch.einsum("bi,eio->bo", x, self.gate_up_proj)+++def test_target_parameters_param_wrapper_syncs_merged_experts():+ # ParamWrapper has no base_layer.weight, so the out-of-place merge pushed the untrained experts.+ peft = pytest.importorskip("peft")+ if "target_parameters" not in peft.LoraConfig.__dataclass_fields__:+ pytest.skip("this PEFT has no target_parameters")+ torch.manual_seed(1)+ base = torch.nn.Sequential(torch.nn.Linear(16, 16, bias = False))+ base.add_module("experts", _Experts())+ config = peft.LoraConfig(+ r = 2,+ target_modules = ["0"],+ target_parameters = ["experts.gate_up_proj"],+ init_lora_weights = False,+ )+ model = peft.get_peft_model(base, config)+ _perturb_trainable(model)+ expected = _merged_state(model)+ trainer, rec = _peft_trainer(model)+ assert _patched_method()(trainer) is None+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"+ pushed = dict(rec.tensors)+ assert sorted(pushed) == ["0.weight", "experts.gate_up_proj"]+ torch.testing.assert_close(pushed["experts.gate_up_proj"], expected["experts.gate_up_proj"])+ torch.testing.assert_close(pushed["0.weight"], expected["0.weight"])++[email protected]("second_adapter", [False, True])+def test_modules_to_save_custom_adapter_name(second_adapter):+ # TRL strips only "modules_to_save.default.": a custom name reached vLLM as is and the trained+ # head was never synced. Push the active adapter's copy under the base name, and nothing else.+ peft = pytest.importorskip("peft")+ config = peft.LoraConfig(+ r = 2, target_modules = ["0"], modules_to_save = ["1"], init_lora_weights = False+ )+ model = peft.get_peft_model(_base_mlp(), config, adapter_name = "mine")+ if second_adapter:+ model.add_adapter("other", config)+ model.set_adapter("mine")+ _perturb_trainable(model)+ head = model.base_model.model[1]+ trainer, rec = _peft_trainer(model)assert _patched_method()(trainer) is None+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"+ assert sorted(rec.loaded) == ["0.weight", "1.bias", "1.weight"]+ pushed = dict(rec.tensors)+ assert torch.equal(pushed["1.weight"], head.modules_to_save["mine"].weight)+ assert not torch.equal(pushed["1.weight"], head.original_module.weight)+ layer = model.base_model.model[0]+ delta = layer.lora_B["mine"].weight.float() @ layer.lora_A["mine"].weight.float()+ expected = (layer.base_layer.weight.float() + delta * layer.scaling["mine"]).to(torch.float32)+ assert torch.equal(pushed["0.weight"], expected)+++def test_trainable_token_indices_raises(monkeypatch):+ # The trained rows live in token_adapter.trainable_tokens_delta.<adapter>; neither TRL's names+ # nor a LoRA merge map that onto vLLM's embedding, so refuse instead of dropping the rows.+ peft = pytest.importorskip("peft")+ if "trainable_token_indices" not in peft.LoraConfig.__dataclass_fields__:+ pytest.skip("this PEFT has no trainable_token_indices")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ torch.manual_seed(1)+ base = torch.nn.Sequential(torch.nn.Embedding(32, 16), torch.nn.Linear(16, 16))+ config = peft.LoraConfig(r = 2, target_modules = ["1"], trainable_token_indices = {"0": [1, 3]})+ model = peft.get_peft_model(base, config)+ trainer, rec = _peft_trainer(model)+ with pytest.raises(RuntimeError, match = "trainable"):+ _patched_method()(trainer)+ assert rec.loaded == [] and rec.resets == 0+++# Codex review round 3: layer_replication, and the in-place merge / unmerge drift of the frozen base.+++def _tiny_llama(dtype = torch.float32):+ transformers = pytest.importorskip("transformers")+ torch.manual_seed(1)+ config = transformers.LlamaConfig(+ vocab_size = 32,+ hidden_size = 16,+ intermediate_size = 32,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ attention_bias = True,+ mlp_bias = True,+ )+ return transformers.LlamaForCausalLM(config).to(dtype)+++def test_layer_replication_raises(monkeypatch):+ # Replicas share one base tensor (named_parameters yields it once) but train separate adapters,+ # and vLLM loads the checkpoint's layer count: no name can carry each replica's merged weight.+ peft = pytest.importorskip("peft")+ if "layer_replication" not in peft.LoraConfig.__dataclass_fields__:+ pytest.skip("this PEFT has no layer_replication")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ config = peft.LoraConfig(+ r = 2,+ target_modules = ["q_proj"],+ layer_replication = [[0, 2], [1, 2]],+ init_lora_weights = False,+ )+ model = peft.get_peft_model(_tiny_llama(), config)+ assert len(model.base_model.model.model.layers) == 3+ trainer, rec = _peft_trainer(model)+ with pytest.raises(RuntimeError, match = "layer_replication"):+ _patched_method()(trainer)+ assert rec.loaded == [] and rec.resets == 0+++def _drift_config(kind):+ peft = pytest.importorskip("peft")+ targets = ["q_proj", "v_proj", "down_proj"]+ if kind == "dora":+ return peft.LoraConfig(r = 2, target_modules = targets, use_dora = True, init_lora_weights = False)+ if kind == "lora_bias":+ return peft.LoraConfig(r = 2, target_modules = targets, lora_bias = True, init_lora_weights = False)+ if kind == "loha":+ return peft.LoHaConfig(r = 2, target_modules = targets, init_weights = False)+ if kind == "lokr":+ return peft.LoKrConfig(r = 2, target_modules = targets, init_weights = False)+ if kind == "ia3":+ return peft.IA3Config(target_modules = targets, feedforward_modules = ["down_proj"])+ return peft.VeraConfig(r = 2, target_modules = ["q_proj", "v_proj"])++[email protected]("kind", ["dora", "lora_bias", "loha", "lokr", "ia3", "vera"])+def test_bf16_non_plain_peft_never_drifts_frozen_base(kind):+ # TRL's body merge_adapter()s and unmerge_adapter()s the live bf16 model every sync; those rounded+ # in-place edits are not lossless, so the frozen base drifted outside the optimizer. The merge now+ # runs on a copy: after 10 syncs every frozen tensor keeps its exact bytes, and each sync pushes+ # exactly what PEFT's own merge of the current adapter gives.+ peft = pytest.importorskip("peft")+ model = peft.get_peft_model(_tiny_llama(torch.bfloat16), _drift_config(kind))+ frozen = {n: p.detach().clone() for n, p in model.named_parameters() if not p.requires_grad}+ trainer, rec = _peft_trainer(model)+ method = _patched_method()+ for _ in range(10):+ _perturb_trainable(model)+ expected = _merged_state(model)+ before = len(rec.tensors)+ assert method(trainer) is None+ pushed = dict(rec.tensors[before:])+ assert pushed["model.layers.0.self_attn.q_proj.weight"].dtype == torch.bfloat16+ for name in (+ "model.layers.0.self_attn.q_proj.weight",+ "model.layers.1.self_attn.v_proj.weight",+ ):+ assert torch.equal(pushed[name], expected[name]), name+ drift = max(+ (p.detach().float() - frozen[n].float()).abs().max().item()+ for n, p in model.named_parameters()+ if n in frozen+ )+ assert drift == 0.0+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"+ for module in model.modules():+ assert not getattr(module, "merged", False)+ assert not getattr(module, "_caches", {})+++def test_bf16_target_parameters_never_drifts_frozen_base():+ peft = pytest.importorskip("peft")+ if "target_parameters" not in peft.LoraConfig.__dataclass_fields__:+ pytest.skip("this PEFT has no target_parameters")+ torch.manual_seed(1)+ base = torch.nn.Sequential(torch.nn.Linear(16, 16, bias = False))+ base.add_module("experts", _Experts())+ base = base.to(torch.bfloat16)+ config = peft.LoraConfig(+ r = 2,+ target_modules = ["0"],+ target_parameters = ["experts.gate_up_proj"],+ init_lora_weights = False,+ )+ model = peft.get_peft_model(base, config)+ gate_up = model.base_model.model.experts.base_layer.gate_up_proj+ before = gate_up.detach().clone()+ trainer, rec = _peft_trainer(model)+ method = _patched_method()+ for _ in range(10):+ _perturb_trainable(model)+ expected = _merged_state(model)+ start = len(rec.tensors)+ assert method(trainer) is None+ pushed = dict(rec.tensors[start:])+ assert torch.equal(pushed["experts.gate_up_proj"], expected["experts.gate_up_proj"])+ assert torch.equal(gate_up, before)+++def _sharded(+ trainer,+ fsdp = False,+ zero3 = False,+):+ trainer.is_fsdp_enabled = fsdp+ if zero3:+ trainer.accelerator.state.deepspeed_plugin = types.SimpleNamespace(zero_stage = 3)+ return trainer++[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])+def test_fsdp_peft_raises(dtype, monkeypatch):+ # TRL calls merge_adapter before it gathers the FSDP shards, so the full-size deltas meet sharded+ # base weights (and the in-place round trip drifts a 16-bit base): refuse instead.+ monkeypatch.delenv(_STALE_ENV, raising = False)+ trainer, rec, model = _lora_trainer("colocate", dtype)+ _sharded(trainer, fsdp = True)+ with pytest.raises(RuntimeError, match = "FSDP"):+ _patched_method()(trainer)assert rec.loaded == [] and rec.resets == 0+++def test_zero3_keeps_trl_body():+ # ZeRO-3: TRL merges inside GatheredParameters(modifier_rank = None), whose in-place edits are+ # discarded on exit.+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)+ _sharded(trainer, zero3 = True)+ assert _patched_method()(trainer) is None+ assert trainer._unsloth_vllm_sync_kind == "trl" and "0.weight" in rec.loaded+++def test_lora_on_a_shared_base_weight_raises(monkeypatch):+ # Tied input / output weights both adapted: one vLLM name cannot carry two different deltas.+ peft = pytest.importorskip("peft")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ base = _base_mlp()+ base.append(torch.nn.Linear(16, 16, bias = False))+ base[2].weight = base[0].weight+ model = peft.get_peft_model(+ base, peft.LoraConfig(r = 2, target_modules = ["0", "2"], init_lora_weights = False)+ )+ trainer, rec = _peft_trainer(model)+ with pytest.raises(RuntimeError, match = "one base weight shared by"):+ _patched_method()(trainer)+ assert rec.loaded == []+++def test_lora_on_a_weight_tied_to_an_unadapted_module_raises(monkeypatch):+ # Only one site of a tied weight adapted: vLLM still gets one tensor for both names.+ peft = pytest.importorskip("peft")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ base = _base_mlp()+ base.append(torch.nn.Linear(16, 16, bias = False))+ base[2].weight = base[0].weight+ model = peft.get_peft_model(+ base, peft.LoraConfig(r = 2, target_modules = ["2"], init_lora_weights = False)+ )+ trainer, rec = _peft_trainer(model)+ with pytest.raises(RuntimeError, match = "one base weight shared by"):+ _patched_method()(trainer)+ assert rec.loaded == []+++def test_accelerate_offloaded_model_raises(monkeypatch):+ monkeypatch.delenv(_STALE_ENV, raising = False)+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)+ model.base_model.model[1]._hf_hook = types.SimpleNamespace(offload = True)+ with pytest.raises(RuntimeError, match = "Accelerate device_map"):+ _patched_method()(trainer)+ assert rec.loaded == []+++def test_switching_to_an_unsupported_adapter_is_refused_before_the_next_sync(monkeypatch):+ peft = pytest.importorskip("peft")+ if "alora_invocation_tokens" not in peft.LoraConfig.__dataclass_fields__:+ pytest.skip("this PEFT has no aLoRA")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ model = peft.get_peft_model(+ _base_mlp(), peft.LoraConfig(r = 2, target_modules = ["0"], init_lora_weights = False)+ )+ model.add_adapter(+ "other",+ peft.LoraConfig(+ r = 2, target_modules = ["0"], alora_invocation_tokens = [5], init_lora_weights = False+ ),+ )+ model.set_adapter("default")+ trainer, rec = _peft_trainer(model)+ method = _patched_method()+ assert method(trainer) is None and "0.weight" in rec.loaded+ model.set_adapter("other")+ before = len(rec.loaded)+ with pytest.raises(RuntimeError, match = "cannot sync this PEFT model.*aLoRA"):+ method(trainer)+ assert len(rec.loaded) == before+++def test_inactive_unsupported_adapter_does_not_block_sync():+ peft = pytest.importorskip("peft")+ model = peft.get_peft_model(+ _base_mlp(), peft.LoraConfig(r = 2, target_modules = ["0"], init_lora_weights = False)+ )+ if "alora_invocation_tokens" not in peft.LoraConfig.__dataclass_fields__:+ pytest.skip("this PEFT has no aLoRA")+ other = peft.LoraConfig(+ r = 2, target_modules = ["0"], alora_invocation_tokens = [5], init_lora_weights = False+ )+ model.add_adapter("other", other)+ model.set_adapter("default")+ trainer, rec = _peft_trainer(model)+ assert _patched_method()(trainer) is None+ assert trainer._unsloth_vllm_sync_kind == "peft_merged" and "0.weight" in rec.loaded++[email protected]("kind", ["ln_tuning", "poly"])+def test_peft_without_a_static_merge_raises(kind, monkeypatch):+ # LN tuning swaps the module on merge, POLY routes per task: neither has one merged weight.+ peft = pytest.importorskip("peft")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ if kind == "ln_tuning":+ base = torch.nn.Sequential(torch.nn.Linear(16, 16), torch.nn.LayerNorm(16))+ config = peft.LNTuningConfig(target_modules = ["1"])+ else:+ base = _base_mlp()+ config = peft.PolyConfig(r = 2, target_modules = ["0"], n_tasks = 2)+ model = peft.get_peft_model(base, config)+ trainer, rec = _peft_trainer(model)+ with pytest.raises(RuntimeError, match = "no static merge"):+ _patched_method()(trainer)+ assert rec.loaded == []+++def test_block_swapped_model_raises(monkeypatch):+ monkeypatch.delenv(_STALE_ENV, raising = False)+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)+ model.base_model.model._unsloth_block_swap = object()+ with pytest.raises(RuntimeError, match = "offload_layers"):+ _patched_method()(trainer)+ assert rec.loaded == []+++def test_legacy_trl_without_name_fixer_still_syncs():+ # TRL < 0.20 has no _fix_param_name_to_vllm and sends the names as they are.+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)+ del trainer._fix_param_name_to_vllm+ assert _patched_method()(trainer) is None+ assert sorted(dict(rec.tensors)) == ["0.weight", "1.bias", "1.weight"]+++def test_lora_on_multihead_attention_raises(monkeypatch):+ # PEFT's MultiheadAttention merge swaps in new parameters instead of editing the base ones.+ peft = pytest.importorskip("peft")+ monkeypatch.delenv(_STALE_ENV, raising = False)++ class _Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.attn = torch.nn.MultiheadAttention(8, 2)+ self.proj = torch.nn.Linear(8, 8)++ model = peft.get_peft_model(+ _Block(), peft.LoraConfig(r = 2, target_modules = ["attn"], init_lora_weights = False)+ )+ trainer, rec = _peft_trainer(model)+ with pytest.raises(RuntimeError, match = "MultiheadAttention"):+ _patched_method()(trainer)+ assert rec.loaded == []+++def test_mixed_peft_model_raises(monkeypatch):+ peft = pytest.importorskip("peft")+ monkeypatch.delenv(_STALE_ENV, raising = False)+ model = peft.get_peft_model(_base_mlp(), peft.LoraConfig(r = 2, target_modules = ["0"]), mixed = True)+ model.add_adapter("loha", peft.LoHaConfig(r = 2, target_modules = ["0"]))+ model.set_adapter(["default", "loha"])+ trainer, rec = _peft_trainer(model)+ with pytest.raises(RuntimeError, match = "mixed PEFT model"):+ _patched_method()(trainer)+ assert rec.loaded == []+++def test_trainer_without_vllm_mode_uses_its_colocated_engine():+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)+ del trainer.vllm_mode+ trainer.vllm_client = None+ assert _patched_method()(trainer) is None+ assert sorted(dict(rec.tensors)) == ["0.weight", "1.bias", "1.weight"]+++def test_loha_rank_dropout_is_off_while_merging():+ # Training mode with rank dropout: the pushed weight is the full inference-time merge.+ peft = pytest.importorskip("peft")+ model = peft.get_peft_model(+ _base_mlp(),+ peft.LoHaConfig(r = 4, target_modules = ["0"], rank_dropout = 0.5, init_weights = False),+ )+ _perturb_trainable(model)+ model.eval()+ expected = _merged_state(model)+ model.train()+ trainer, rec = _peft_trainer(model)+ method = _patched_method()+ assert method(trainer) is None and method(trainer) is None+ pushed = [tensor for name, tensor in rec.tensors if name == "0.weight"]+ assert len(pushed) == 2 and torch.equal(pushed[0], pushed[1])+ torch.testing.assert_close(pushed[0], expected["0.weight"])+ assert all(m.training for m in model.modules())+++def test_disabled_non_plain_adapter_pushes_the_base():+ peft = pytest.importorskip("peft")+ model = peft.get_peft_model(_base_mlp(), _non_lora_config("loha"))+ _perturb_trainable(model)+ trainer, rec = _peft_trainer(model)+ with model.disable_adapter():+ assert _patched_method()(trainer) is None+ pushed = dict(rec.tensors)+ assert torch.equal(pushed["0.weight"], model.base_model.model[0].base_layer.weight)+++def test_base_module_named_like_the_adapter_prefix_is_still_pushed():+ peft = pytest.importorskip("peft")++ class _Named(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q_lora_proj = torch.nn.Linear(16, 16, bias = False)++ model = peft.get_peft_model(+ _Named(), peft.LoraConfig(r = 2, target_modules = ["q_lora_proj"], init_lora_weights = False)+ )+ trainer, rec = _peft_trainer(model)+ assert _patched_method()(trainer) is None+ assert sorted(dict(rec.tensors)) == ["q_lora_proj.weight"]+++def test_adapter_on_a_composite_base_is_refused():+ from unsloth.models.rl_replacements import _unsloth_vllm_peft_unsupported++ class _Custom(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.base_layer = torch.nn.Sequential(torch.nn.Linear(4, 4))+ self.lora_A = torch.nn.ModuleDict()++ def get_base_layer(self):+ return self.base_layer++ model = torch.nn.Sequential(_Custom())+ model.peft_config = {}+ assert any("composite" in p for p in _unsloth_vllm_peft_unsupported(model))diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 02b4920c583..84f63a3f400 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -2476,14 +2476,30 @@ def grpo_trainer_fix_maybe_apply_chat_template(function_name, function):RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_fix_maybe_apply_chat_template)-def _unsloth_vllm_sync_unsafe(model):+def _unsloth_vllm_quantized(model):# FSDP2 turns Params4bit into a DTensor, so also read the load flags the model keeps.if getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False):return True- if getattr(model, "peft_config", None):+ # transformers marks every quantizer it loads (GPTQ, AWQ, HQQ, torchao, compressed-tensors, ...).+ # A dequantize = True load removes all of these, so they only remain on a quantized model.+ if getattr(model, "is_quantized", False) is True or getattr(model, "quantization_method", None):+ return True+ if getattr(model, "hf_quantizer", None) is not None:return True+ quantization_config = getattr(getattr(model, "config", None), "quantization_config", None)+ if quantization_config:+ dequantize = (+ quantization_config.get("dequantize", False)+ if isinstance(quantization_config, dict)+ else getattr(quantization_config, "dequantize", False)+ )+ if not dequantize:+ return Truefor module in model.modules():- if hasattr(module, "lora_A") or hasattr(module, "base_layer"):+ # Packed integer weights (GPTQ / AWQ qweight, HQQ W_q) are not vLLM's layout.+ if isinstance(getattr(module, "qweight", None), torch.Tensor) or isinstance(+ getattr(module, "W_q", None), torch.Tensor+ ):return Truefor param in model.parameters():if type(param).__name__ in ("Params4bit", "Int8Params"):@@ -2493,6 +2509,438 @@ def _unsloth_vllm_sync_unsafe(model):return False+def _unsloth_vllm_lora_layers(model):+ return [+ module+ for module in model.modules()+ if hasattr(module, "lora_A") and getattr(module, "base_layer", None) is not None+ ]+++def _unsloth_vllm_sync_unsafe(model):+ if _unsloth_vllm_quantized(model):+ return True+ if getattr(model, "peft_config", None):+ return True+ return len(_unsloth_vllm_lora_layers(model)) != 0+++# LoRA layer classes whose merged weight _unsloth_vllm_merged_weight computes: one dense base weight.+_UNSLOTH_VLLM_PLAIN_LORA = ("Linear", "Embedding", "Conv1d", "Conv2d", "Conv3d")+# LoRA variants PEFT itself cannot merge into the base weight (they condition on the input).+_UNSLOTH_VLLM_UNMERGEABLE_VARIANTS = ("ALora", "Arrow", "QALora")+++def _unsloth_vllm_peft_type(config):+ peft_type = getattr(config, "peft_type", None)+ return str(getattr(peft_type, "value", peft_type))+++# PEFT types whose merge writes the merged weight into the base tensors (no module swap, no per-input routing).+_UNSLOTH_VLLM_MERGEABLE_TYPES = ("LORA", "LOHA", "LOKR", "IA3", "VERA")+++def _unsloth_vllm_active_adapters(model):+ configs = getattr(model, "peft_config", None) or {}+ active = getattr(model, "active_adapters", None)+ if callable(active):+ active = active()+ if isinstance(active, str):+ active = [active]+ return set(active) if active else set(configs)+++def _unsloth_vllm_shared_targets(model):+ """Adapted base parameters reachable under more than one name (tied embed_tokens / lm_head, adapted+ at one or both sites): vLLM gets one tensor for them, so some call site would get the wrong delta."""+ names = {}+ for name, param in model.named_parameters(remove_duplicate = False):+ names.setdefault(id(param), []).append(name)+ shared = {}+ for module in model.modules():+ if getattr(module, "base_layer", None) is None or not hasattr(module, "get_base_layer"):+ continue+ for param in module.get_base_layer().parameters(recurse = False):+ if len(names.get(id(param), ())) > 1:+ shared[id(param)] = " and ".join(names[id(param)])+ return list(shared.values())+++def _unsloth_vllm_offloaded(model):+ """How the model keeps weights off the device between forwards, or None: a sync would read the+ empty or meta placeholders."""+ for module in model.modules():+ if getattr(module, "_unsloth_block_swap", None) is not None:+ return "block-swapped decoder layers (offload_layers)"+ hook = getattr(module, "_hf_hook", None)+ hooks = getattr(hook, "hooks", None) or ([hook] if hook is not None else [])+ if any(getattr(h, "offload", False) for h in hooks):+ return "CPU / disk offloaded layers (Accelerate device_map)"+ if any(param.device.type == "meta" for param in model.parameters()):+ return "meta (offloaded) parameters"+ return None+++def _unsloth_vllm_peft_unsupported(model):+ """PEFT setups neither TRL's PEFT branch nor the out-of-place merge can sync: names of the culprits.+ Only active adapters count; an inactive one does not shape the policy being synced."""+ problems = []+ active = _unsloth_vllm_active_adapters(model)+ if type(model).__name__ == "PeftMixedModel":+ # Adapter parameters carry per-type prefixes (lora_, hada_, lokr_, ...) the sync cannot filter.+ problems.append("a mixed PEFT model (get_peft_model(..., mixed = True))")+ for name, config in (getattr(model, "peft_config", None) or {}).items():+ # Replicas share one base tensor but train separate adapters, and vLLM loads the checkpoint's+ # layer count: there is no per-replica weight to send (PEFT also refuses to merge these).+ if getattr(config, "layer_replication", None):+ problems.append(f"layer_replication on adapter '{name}'")+ if name not in active:+ continue+ peft_type = _unsloth_vllm_peft_type(config)+ if getattr(config, "is_prompt_learning", False) or getattr(+ config, "is_adaption_prompt", False+ ):+ problems.append(f"prompt learning adapter '{name}' ({_unsloth_vllm_peft_type(config)})")+ if getattr(config, "trainable_token_indices", None):+ problems.append(f"trainable_token_indices on adapter '{name}'")+ if getattr(config, "alora_invocation_tokens", None):+ problems.append(f"aLoRA adapter '{name}'")+ if getattr(config, "arrow_config", None) is not None:+ problems.append(f"Arrow adapter '{name}'")+ elif not getattr(config, "is_prompt_learning", False) and (+ peft_type not in _UNSLOTH_VLLM_MERGEABLE_TYPES+ ):+ # LN tuning swaps modules, POLY / X-LoRA route per input: no single merged weight.+ problems.append(f"{peft_type} adapter '{name}' (no static merge into the base weights)")+ for module_name, module in model.named_modules():+ # trainable_token_indices, even on an inactive adapter: its token_adapter.* parameters and+ # names do not exist in vLLM and are pushed all the same.+ if type(module).__name__ in ("TrainableTokensWrapper", "TrainableTokensLayer"):+ problems.append(f"trainable tokens at {module_name}")+ break+ for module_name, module in model.named_modules():+ # PEFT's MultiheadAttention merge replaces in_proj_weight / out_proj.weight instead of+ # editing them, so a copy merge cannot leave the live module untouched.+ if type(module).__name__ == "MultiheadAttention" and hasattr(module, "lora_A"):+ problems.append(f"LoRA on MultiheadAttention at {module_name}")+ break+ for module_name, module in model.named_modules():+ # A custom tuner layer on a composite base: its weights live in child modules the copy+ # merge does not see, so the merged weights would never be sent.+ if getattr(module, "base_layer", None) is None or not hasattr(module, "get_base_layer"):+ continue+ base = module.get_base_layer()+ if any(any(True for _ in child.parameters()) for child in base.children()):+ problems.append(f"adapter on the composite module {module_name}")+ break+ for module in _unsloth_vllm_lora_layers(model):+ for adapter, variant in (getattr(module, "lora_variant", None) or {}).items():+ if adapter not in active:+ continue+ variant_name = type(variant).__name__+ if any(variant_name.startswith(x) for x in _UNSLOTH_VLLM_UNMERGEABLE_VARIANTS):+ problems.append(f"LoRA variant {variant_name}")+ return list(dict.fromkeys(problems))+++def _unsloth_vllm_modules_to_save(model):+ """(wrapper name, adapter it serves or None for original_module, all its adapter names)."""+ wrappers = []+ for module_name, module in model.named_modules():+ if type(module).__name__ != "ModulesToSaveWrapper":+ continue+ copies = getattr(module, "modules_to_save", None) or {}+ served = None+ if not getattr(module, "disable_adapters", False):+ for adapter in getattr(module, "active_adapters", None) or []:+ if adapter in copies:+ served = adapter+ break+ wrappers.append((module_name, served, list(copies.keys())))+ return wrappers+++def _unsloth_vllm_plain_lora(model):+ """True when every PEFT piece is plain LoRA that _unsloth_vllm_sync_peft_merged fully syncs."""+ configs = getattr(model, "peft_config", None) or {}+ active = _unsloth_vllm_active_adapters(model)+ if any(+ _unsloth_vllm_peft_type(config) != "LORA"+ for name, config in configs.items()+ if name in active+ ):+ return False+ try:+ from peft.tuners.tuners_utils import BaseTunerLayer+ except Exception:+ BaseTunerLayer = ()+ lora_layers = {id(module) for module in _unsloth_vllm_lora_layers(model)}+ for module in model.modules():+ is_tuner = (BaseTunerLayer != () and isinstance(module, BaseTunerLayer)) or (+ id(module) in lora_layers+ )+ if not is_tuner:+ continue+ # ParamWrapper (target_parameters), TrainableTokensLayer, ...: not a single dense weight.+ if id(module) not in lora_layers or type(module).__name__ not in _UNSLOTH_VLLM_PLAIN_LORA:+ return False+ base_layer = module.base_layer+ if not isinstance(getattr(base_layer, "weight", None), torch.Tensor):+ return False+ if getattr(module, "lora_A", None) is not None and hasattr(base_layer, "base_layer"):+ return False+ for flags in ("lora_variant", "use_dora", "lora_bias"):+ if any(v for a, v in (getattr(module, flags, None) or {}).items() if a in active):+ return False+ return True+++def _unsloth_vllm_sync_kind(trainer):+ """How _move_model_to_vllm syncs TRL's own vLLM engine (fast_inference = False):+ "trl" = TRL's body (dense, or sharded PEFT where its in-place merge cannot drift the base),+ "peft_merged" = PEFT merged out of place, "skip" = no sync (explicit opt-in only). Anything that cannot be synced raises: rollouts+ would come from the initial weights all run."""+ model = trainer.model+ allow_stale = os.environ.get("UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS", "0") == "1"+ if _unsloth_vllm_quantized(model):+ if allow_stale:+ print(+ "Unsloth: UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1, so vLLM weight sync is skipped for this "+ "quantized model and every rollout uses the initial weights."+ )+ return "skip"+ raise RuntimeError(+ "Unsloth: GRPO with use_vllm = True but fast_inference = False cannot sync a quantized "+ "(bitsandbytes, GPTQ, AWQ, HQQ, torchao, FP8, ...) model into TRL's own vLLM engine, so every "+ "rollout would come from the initial weights while the policy trains.\n"+ "Load the model with fast_inference = True, or load a 16-bit base (load_in_4bit = False, "+ "load_in_8bit = False). Set UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1 to train anyway."+ )+ offloaded = _unsloth_vllm_offloaded(model)+ if offloaded is not None:+ if allow_stale:+ print(+ "Unsloth: UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1, so vLLM weight sync is skipped for this "+ f"model with {offloaded} and every rollout uses the initial weights."+ )+ return "skip"+ raise RuntimeError(+ "Unsloth: GRPO with use_vllm = True but fast_inference = False cannot sync a model with "+ f"{offloaded}: those weights are empty or on the meta device between forwards.\n"+ "Keep every layer on the GPU (no offload_layers, no CPU / disk device_map), or load the "+ "model with fast_inference = True. Set UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1 to train anyway."+ )+ if not _unsloth_vllm_sync_unsafe(model):+ return "trl"+ state = getattr(getattr(trainer, "accelerator", None), "state", None)+ deepspeed_plugin = getattr(state, "deepspeed_plugin", None)+ sharded = getattr(trainer, "is_fsdp_enabled", False) or (+ deepspeed_plugin is not None and getattr(deepspeed_plugin, "zero_stage", 0) == 3+ )+ problems = _unsloth_vllm_peft_unsupported(model)+ problems.extend(+ f"one base weight shared by {names}" for names in _unsloth_vllm_shared_targets(model)+ )+ # Unsharded: every mergeable PEFT setup is merged out of place, so the frozen base never moves.+ # ZeRO-3: TRL merges inside GatheredParameters(modifier_rank = None), which discards in-place edits+ # of the gathered copy on exit, so its merge / unmerge cannot drift the stored base either.+ # FSDP: TRL calls merge_adapter before it gathers the sharded parameters, so PEFT's full-size+ # deltas meet sharded base weights (and the in-place round trip would drift a 16-bit base).+ kind = "peft_merged" if not sharded else "trl"+ if getattr(trainer, "is_fsdp_enabled", False):+ problems.append(+ "FSDP (TRL merges the adapters before it gathers the sharded weights; use DeepSpeed "+ "ZeRO-3 or a single process)"+ )+ if kind == "trl":+ # TRL strips only "modules_to_save.default.", so other adapter names reach vLLM as is.+ for wrapper, _, adapters in _unsloth_vllm_modules_to_save(model):+ if any(adapter != "default" for adapter in adapters):+ problems.append(f"modules_to_save at {wrapper} for adapters {adapters}")+ if problems:+ if allow_stale:+ print(+ "Unsloth: UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1, so vLLM weight sync is skipped for this "+ f"PEFT model ({'; '.join(problems)}) and every rollout uses the initial weights."+ )+ return "skip"+ raise RuntimeError(+ "Unsloth: GRPO with use_vllm = True but fast_inference = False cannot sync this PEFT model "+ "into TRL's own vLLM engine: "+ + "; ".join(problems)+ + ". These cannot be merged into the "+ "base weights under vLLM's parameter names, so every rollout would come from the initial "+ "weights while the policy trains.\n"+ "Use a plain LoRA adapter, or set UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1 to train anyway."+ )+ return kind+++def _unsloth_vllm_sync_kind_cached(trainer):+ """_unsloth_vllm_sync_kind, decided again whenever the active adapters change (set_adapter)."""+ key = tuple(sorted(_unsloth_vllm_active_adapters(trainer.model)))+ if getattr(trainer, "_unsloth_vllm_sync_kind", None) is None or (+ getattr(trainer, "_unsloth_vllm_sync_key", None) != key+ ):+ trainer._unsloth_vllm_sync_kind = _unsloth_vllm_sync_kind(trainer)+ trainer._unsloth_vllm_sync_skip = trainer._unsloth_vllm_sync_kind == "skip"+ trainer._unsloth_vllm_sync_key = key+ return trainer._unsloth_vllm_sync_kind+++def _unsloth_vllm_merged_weight(module, weight):+ # W + sum of active deltas, computed out of place in fp32 and rounded once. merge_adapter ++ # unmerge_adapter would round twice in place and drift the frozen base every sync.+ merged = None+ if not getattr(module, "disable_adapters", False) and not getattr(module, "merged", False):+ for adapter in module.active_adapters:+ lora_A = module.lora_A[adapter] if adapter in module.lora_A else None+ if isinstance(lora_A, torch.nn.Linear) and adapter in module.lora_B:+ delta = module.lora_B[adapter].weight.float() @ lora_A.weight.float()+ delta = delta * module.scaling[adapter]+ if getattr(module, "fan_in_fan_out", False):+ delta = delta.T+ elif adapter in module.lora_A or adapter in getattr(module, "lora_embedding_A", {}):+ delta = module.get_delta_weight(adapter).float()+ else:+ continue+ delta = delta.to(weight.device).reshape(weight.shape)+ merged = delta if merged is None else merged + delta+ if merged is None:+ return weight.data+ return (weight.float() + merged).to(weight.dtype)+++def _unsloth_vllm_tuner_groups(model):+ """{id(base param): (real base module, [tuner layers wrapping it])} for PEFT's own merge."""+ from peft.tuners.tuners_utils import BaseTunerLayer++ groups = {}+ for module in model.modules():+ if isinstance(module, BaseTunerLayer):+ base = module.get_base_layer()+ groups.setdefault(id(base), (base, []))[1].append(module)+ return {+ id(param): group+ for group in groups.values()+ for param in group[0].parameters(recurse = False)+ }+++def _unsloth_vllm_peft_merge_out_of_place(base, layers):+ """Run PEFT's own merge (DoRA, lora_bias, LoHa, LoKr, IA3, VeRA, ParamWrapper, ...) on copies of+ the base tensors and return {id(param): merged tensor}; the live parameters keep their exact bytes."""+ params = list(base.parameters(recurse = False))+ originals = [param.data for param in params]+ states = [+ (layer, list(getattr(layer, "merged_adapters", [])), dict(getattr(layer, "_caches", {})))+ for layer in layers+ ]+ # Eval mode while merging: LoHa / LoKr rank dropout would otherwise thin the delta at random.+ modes = [(module, module.training) for layer in layers for module in layer.modules()]+ try:+ for param, data in zip(params, originals):+ param.data = data.clone()+ for layer in layers:+ layer.train(False)+ # model.modules() order, as PeftModel.merge_adapter merges them. A disabled layer runs only+ # its base in training, so it adds nothing (PEFT's merge ignores disable_adapters).+ for layer in layers:+ if not getattr(layer, "disable_adapters", False):+ layer.merge()+ return {id(param): param.data for param in params}+ finally:+ for module, training in modes:+ module.training = training+ for param, data in zip(params, originals):+ param.data = data+ for layer, merged_adapters, caches in states:+ if hasattr(layer, "merged_adapters"):+ layer.merged_adapters[:] = merged_adapters+ if isinstance(getattr(layer, "_caches", None), dict):+ layer._caches.clear()+ layer._caches.update(caches)+++def _unsloth_vllm_adapter_param_ids(model):+ """Adapter tensors: every parameter under a tuner layer except its base layer's own."""+ ids = set()+ for module in model.modules():+ if getattr(module, "base_layer", None) is None or not hasattr(module, "get_base_layer"):+ continue+ base_ids = {id(param) for param in module.get_base_layer().parameters()}+ ids.update(id(param) for param in module.parameters() if id(param) not in base_ids)+ return ids+++_UNSLOTH_VLLM_MODULES_TO_SAVE = re.compile(r"\.(?:modules_to_save\.[^.]+|original_module)\.")++[email protected]_grad()+def _unsloth_vllm_sync_peft_merged(trainer):+ """TRL's PEFT sync (merged weights under the base-layer names), without mutating the model."""+ model = trainer.model+ # Plain LoRA: W + sum of deltas in fp32, rounded once. Anything else: PEFT's merge on a copy.+ plain = _unsloth_vllm_plain_lora(model)+ lora_of = {}+ groups = {}+ if plain:+ for module in _unsloth_vllm_lora_layers(model):+ weight = getattr(module.base_layer, "weight", None)+ if weight is not None:+ lora_of[id(weight)] = module+ else:+ groups = _unsloth_vllm_tuner_groups(model)+ merged_cache = {}+ # modules_to_save: push the copy the active adapter trains (any adapter name), never the rest.+ unserved = []+ for wrapper, served, adapters in _unsloth_vllm_modules_to_save(model):+ unserved.extend(f"{wrapper}.modules_to_save.{a}." for a in adapters if a != served)+ if served is not None:+ unserved.append(f"{wrapper}.original_module.")+ unserved = tuple(unserved)+ # Structural, not a name match: a base module named q_lora_proj is still a base weight.+ adapter_ids = _unsloth_vllm_adapter_param_ids(model)+ # TRL 0.18 has no vllm_mode: its in-process engine is the colocated one.+ mode = getattr(trainer, "vllm_mode", None) or (+ "server" if getattr(trainer, "vllm_client", None) is not None else "colocate"+ )+ is_main = getattr(trainer.accelerator, "is_main_process", True)+ llm_model = None+ if mode == "colocate":+ llm_model = trainer.llm.llm_engine.model_executor.driver_worker.model_runner.model+ # TRL < 0.20 sends the names as they are and has no fixer.+ fix_name = getattr(trainer, "_fix_param_name_to_vllm", None) or (lambda name, **_: name)+ for full_name, param in model.named_parameters():+ if unserved and full_name.startswith(unserved):+ continue+ if id(param) in adapter_ids:+ continue+ name = full_name.removeprefix("base_model.model.").replace(".base_layer", "")+ name = fix_name(_UNSLOTH_VLLM_MODULES_TO_SAVE.sub(".", name))+ if plain:+ module = lora_of.get(id(param))+ tensor = param.data if module is None else _unsloth_vllm_merged_weight(module, param)+ elif id(param) in groups:+ # One base module at a time: its weight and bias are adjacent in named_parameters.+ if id(param) not in merged_cache:+ merged_cache.clear()+ merged_cache.update(_unsloth_vllm_peft_merge_out_of_place(*groups[id(param)]))+ tensor = merged_cache.pop(id(param))+ else:+ tensor = param.data+ if mode == "server":+ if is_main:+ trainer.vllm_client.update_named_param(name, tensor)+ elif mode == "colocate":+ llm_model.load_weights([(name, tensor)])+ if mode == "server" and is_main:+ trainer.vllm_client.reset_prefix_cache()+ elif mode == "colocate":+ trainer.llm.reset_prefix_cache()+ return None++def grpo_trainer__move_model_to_vllm(function_name, function):if function_name != "_move_model_to_vllm":return function@@ -2514,22 +2962,26 @@ def grpo_trainer__move_model_to_vllm(function_name, function):return inspect.getsource(_move_model_to_vllm)indent = body_indent.group(1)- # Keep the old no-op where TRL's sync is unsafe:- # - LoRA / PEFT: merge_adapter + unmerge_adapter on the live model is a lossy round trip in- # fp16 / bf16, so frozen base weights drift every sync; it also requantizes bitsandbytes bases- # from the fp32 delta (fast_lora backward then fails) and refuses FP8 grouped-linear bases.- # - Quantized bases (bitsandbytes, 1-byte floats such as FP8) cannot be loaded into vLLM as is.- # So the sync only runs for dense full finetuning.+ # Only the fast_inference engine shares weights; every other engine must get the trained weights:+ # - Dense: TRL's own body.+ # - Plain LoRA on a 16-bit base: merged weights under the base-layer names, as TRL does, but+ # computed out of place. merge_adapter + unmerge_adapter on the live model is a lossy round+ # trip in fp16 / bf16, so the frozen base would drift every sync.+ # - Other mergeable PEFT (LoHa, LoKr, IA3, VeRA, DoRA, lora_bias, target_parameters): PEFT's own+ # merge run on copies of the base tensors, so the live base keeps its exact bytes.+ # - ZeRO-3: TRL's body (its gathered copy is discarded on exit). FSDP with PEFT: raise.+ # - Quantized bases (bitsandbytes, GPTQ, AWQ, HQQ, FP8, ...), unmergeable adapters (aLoRA, Arrow,+ # trainable tokens, prompt learning): raise instead of sampling from the initial weights all run.guard = (f"{indent}if getattr(getattr(self, 'llm', None), 'shared_weights', False):\n"f"{indent} return None\n"- f"{indent}if getattr(self, '_unsloth_vllm_sync_skip', None) is None:\n"- f"{indent} from unsloth.models.rl_replacements import _unsloth_vllm_sync_unsafe\n"- f"{indent} self._unsloth_vllm_sync_skip = _unsloth_vllm_sync_unsafe(self.model)\n"- f"{indent} if self._unsloth_vllm_sync_skip:\n"- f"{indent} print('Unsloth: vLLM weight sync is skipped for a LoRA / PEFT or quantized model without fast_inference, so rollouts use the initial weights. Load with fast_inference = True to sample from the trained model.')\n"- f"{indent}if self._unsloth_vllm_sync_skip:\n"+ f"{indent}from unsloth.models.rl_replacements import _unsloth_vllm_sync_kind_cached\n"+ f"{indent}_unsloth_vllm_sync_kind_cached(self)\n"+ f"{indent}if self._unsloth_vllm_sync_kind == 'skip':\n"f"{indent} return None\n"+ f"{indent}if self._unsloth_vllm_sync_kind == 'peft_merged':\n"+ f"{indent} from unsloth.models.rl_replacements import _unsloth_vllm_sync_peft_merged\n"+ f"{indent} return _unsloth_vllm_sync_peft_merged(self)\n")return function[: match.end()] + guard + function[match.end() :]