Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO: sync LoRA weights to TRL's own vLLM engine on TRL < 0.28, refuse quantized bases (#14831)

Daniel Han4dad471914c8
diff --git a/tests/test_grpo_move_model_to_vllm_sync.py b/tests/test_grpo_move_model_to_vllm_sync.py
index 11f99dae159..f1215b02f64 100644
--- a/tests/test_grpo_move_model_to_vllm_sync.py
+++ b/tests/test_grpo_move_model_to_vllm_sync.py
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""TRL < 0.28 GRPO: _move_model_to_vllm is a no-op only for fast_inference, TRL's own vLLM still gets weights."""
+"""TRL < 0.28 GRPO: _move_model_to_vllm is a no-op only for fast_inference, TRL's own vLLM still gets weights
+(dense as is, LoRA merged out of place); a quantized base raises instead of training against frozen rollouts."""
import types
from contextlib import nullcontext
@@ -25,7 +26,7 @@ _TRL_SOURCE = """
continue
if "original_module" in name:
continue
- name = self._fix_param_name_to_vllm(name)
+ name = self._fix_param_name_to_vllm(name, extra_prefixes=["modules_to_save.default."])
if self.vllm_mode == "server" and self.accelerator.is_main_process:
self.vllm_client.update_named_param(name, param.data)
elif self.vllm_mode == "colocate":
@@ -51,6 +52,13 @@ _TRL_SOURCE = """
"""
+def _trl_fix_param_name_to_vllm(name, extra_prefixes = None):
+ # trl 0.27.2 grpo_trainer.py GRPOTrainer._fix_param_name_to_vllm
+ for prefix in ["_checkpoint_wrapped_module."] + (extra_prefixes or []):
+ name = name.replace(prefix, "")
+ return name
+
+
class _Recorder:
def __init__(self):
self.loaded = []
@@ -86,7 +94,7 @@ def _trainer(vllm_mode, shared_weights = False):
model = torch.nn.Linear(2, 2),
vllm_mode = vllm_mode,
vllm_client = rec,
- _fix_param_name_to_vllm = lambda name: name,
+ _fix_param_name_to_vllm = _trl_fix_param_name_to_vllm,
)
if vllm_mode == "colocate":
trainer.llm = llm
@@ -149,56 +157,720 @@ class Params4bit(torch.nn.Parameter):
pass
-def test_bnb_quantized_base_keeps_no_op():
+_STALE_ENV = "UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS"
+
+
+def test_bnb_quantized_base_raises(monkeypatch):
# merge_adapter on a bitsandbytes base requantizes it from the fp32 LoRA delta and the next
- # fast_lora backward fails on dtype, so a quantized model must not enter TRL's sync body.
+ # fast_lora backward fails on dtype, and packed bytes are not vLLM's layout: rollouts would stay
+ # at the initial weights, so refuse before the first rollout instead of training against them.
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ trainer, rec = _trainer("colocate")
+ trainer.model.weight = Params4bit(trainer.model.weight.data, requires_grad = False)
+ with pytest.raises(RuntimeError, match = "fast_inference = True"):
+ _patched_method()(trainer)
+ assert rec.loaded == [] and rec.resets == 0
+
+
+def test_bnb_quantized_base_opt_in_keeps_no_op(monkeypatch):
+ monkeypatch.setenv(_STALE_ENV, "1")
trainer, rec = _trainer("colocate")
trainer.model.weight = Params4bit(trainer.model.weight.data, requires_grad = False)
method = _patched_method()
assert method(trainer) is None and method(trainer) is None
assert rec.loaded == [] and rec.resets == 0
+ assert trainer._unsloth_vllm_sync_skip is True
-def test_quantized_load_flag_keeps_no_op():
[email protected]("vllm_mode", ["colocate", "server"])
+def test_quantized_load_flag_raises(vllm_mode, monkeypatch):
# FSDP2 wraps Params4bit into a DTensor; the model's load flag still marks it quantized.
- trainer, rec = _trainer("colocate")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ trainer, rec = _trainer(vllm_mode)
trainer.model.is_loaded_in_4bit = True
- method = _patched_method()
- assert method(trainer) is None
+ with pytest.raises(RuntimeError, match = "quantized"):
+ _patched_method()(trainer)
assert rec.loaded == [] and rec.resets == 0
-def _lora_model(dtype):
+class _WeightRecorder(_Recorder):
+ def __init__(self):
+ super().__init__()
+ self.tensors = []
+
+ def load_weights(self, weights):
+ for name, tensor in weights:
+ self.loaded.append(name)
+ self.tensors.append((name, tensor.detach().clone()))
+
+ def update_named_param(self, name, data):
+ self.loaded.append(name)
+ self.tensors.append((name, data.detach().clone()))
+
+
+def _lora_model(dtype, **kwargs):
peft = pytest.importorskip("peft")
torch.manual_seed(1)
- base = torch.nn.Sequential(torch.nn.Linear(64, 64, bias = False)).to(dtype)
- config = peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["0"], init_lora_weights = False)
+ base = torch.nn.Sequential(torch.nn.Linear(64, 64, bias = False), torch.nn.Linear(64, 8)).to(
+ dtype
+ )
+ config = peft.LoraConfig(
+ r = 4, lora_alpha = 8, target_modules = ["0"], init_lora_weights = False, **kwargs
+ )
return peft.get_peft_model(base, config)
+def _lora_trainer(vllm_mode, dtype, **kwargs):
+ model = _lora_model(dtype, **kwargs)
+ trainer, _ = _trainer(vllm_mode)
+ rec = _WeightRecorder()
+ trainer.model = model
+ trainer.vllm_client = rec
+ if vllm_mode == "colocate":
+ trainer.llm.llm_engine.model_executor.driver_worker.model_runner.model = rec
+ trainer.llm.reset_prefix_cache = rec.reset_prefix_cache
+ return trainer, rec, model
+
+
+def _expected_merged(model):
+ layer = model.base_model.model[0]
+ weight = layer.base_layer.weight
+ delta = layer.lora_B["default"].weight.float() @ layer.lora_A["default"].weight.float()
+ return (weight.float() + delta * layer.scaling["default"]).to(weight.dtype)
+
+
[email protected]("vllm_mode", ["colocate", "server"])
[email protected]("dtype", [torch.float16, torch.bfloat16])
+def test_lora_sync_pushes_merged_weights_every_step(vllm_mode, dtype):
+ # TRL's own vLLM engine (fast_inference = False) must sample from the trained adapter: push
+ # the merged weight under the base-layer name each step, as TRL's PEFT branch does.
+ trainer, rec, model = _lora_trainer(vllm_mode, dtype)
+ method = _patched_method()
+ for step in range(2):
+ with torch.no_grad():
+ model.base_model.model[0].lora_B["default"].weight.add_(0.5)
+ before = len(rec.tensors)
+ assert method(trainer) is None
+ pushed = dict(rec.tensors[before:])
+ assert sorted(pushed) == ["0.weight", "1.bias", "1.weight"]
+ assert torch.equal(pushed["0.weight"], _expected_merged(model))
+ assert torch.equal(pushed["1.weight"], model.base_model.model[1].weight)
+ assert rec.resets == 2
+ assert trainer._unsloth_vllm_sync_skip is False
+
+
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_lora_sync_never_touches_frozen_base(dtype):
- # merge_adapter + unmerge_adapter is a lossy round trip in fp16 / bf16, so syncing a LoRA
- # model through TRL's PEFT branch would drift the frozen base weights every step.
- model = _lora_model(dtype)
- weight = model.base_model.model[0].base_layer.weight
- before = weight.detach().clone()
- trainer, rec = _trainer("colocate")
- trainer.model = model
+ # merge_adapter + unmerge_adapter is a lossy round trip in fp16 / bf16, so the merged weights
+ # are built out of place and the frozen base stays bit-identical across syncs.
+ trainer, rec, model = _lora_trainer("colocate", dtype)
+ layer = model.base_model.model[0]
+ before = layer.base_layer.weight.detach().clone()
method = _patched_method()
for _ in range(3):
assert method(trainer) is None
- assert torch.equal(weight, before)
- assert rec.loaded == [] and rec.resets == 0
+ assert torch.equal(layer.base_layer.weight, before)
+ assert not layer.merged
+ assert len(rec.tensors) == 9
+
+
+def test_dora_syncs_out_of_place():
+ # DoRA needs PEFT's own merge (magnitude vector): it runs on a copy of the base weight.
+ trainer, rec, model = _lora_trainer("colocate", torch.float32, use_dora = True)
+ assert _patched_method()(trainer) is None
+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"
+ assert "0.weight" in rec.loaded and not model.base_model.model[0].merged
+ assert rec.resets == 1
-def test_fp8_base_keeps_no_op():
+def test_fp8_base_raises(monkeypatch):
# merge_adapter refuses FP8 grouped-linear bases, and a 1-byte float base is not a dense sync.
if not hasattr(torch, "float8_e4m3fn"):
pytest.skip("no float8 dtype")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
trainer, rec = _trainer("colocate")
trainer.model.weight = torch.nn.Parameter(
trainer.model.weight.data.to(torch.float8_e4m3fn), requires_grad = False
)
+ with pytest.raises(RuntimeError, match = "fast_inference = True"):
+ _patched_method()(trainer)
+ assert rec.loaded == [] and rec.resets == 0
+
+
+# Codex review round 2: quantizers beyond bitsandbytes, non-LoRA PEFT, unmergeable LoRA variants,
+# target_parameters, custom modules_to_save adapter names, trainable tokens.
+
+
+class _PackedLinear(torch.nn.Module):
+ # GPTQ QuantLinear / AWQ WQLinear keep int32 packed weights in a qweight buffer.
+ def __init__(self):
+ super().__init__()
+ self.register_buffer("qweight", torch.zeros(2, 2, dtype = torch.int32))
+ self.bias = torch.nn.Parameter(torch.zeros(2))
+
+
+def _mark_hf_quantizer(model):
+ model.hf_quantizer = object()
+
+
+def _mark_quantization_method(model):
+ model.quantization_method = "hqq"
+
+
+def _mark_quantization_config(model):
+ model.config = types.SimpleNamespace(quantization_config = {"quant_method": "awq", "bits": 4})
+
+
+def _mark_packed_weight(model):
+ model.add_module("packed", _PackedLinear())
+
+
+ "mark",
+ [_mark_hf_quantizer, _mark_quantization_method, _mark_quantization_config, _mark_packed_weight],
+)
+def test_gptq_awq_hqq_style_quantized_base_raises(mark, monkeypatch):
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ trainer, rec = _trainer("colocate")
+ mark(trainer.model)
+ with pytest.raises(RuntimeError, match = "quantized"):
+ _patched_method()(trainer)
+ assert rec.loaded == [] and rec.resets == 0
+
+
+def test_dequantized_config_still_syncs():
+ # A dequantize = True load leaves dense weights: it is a dense sync, not a quantized base.
+ trainer, rec = _trainer("colocate")
+ trainer.model.config = types.SimpleNamespace(
+ quantization_config = {"quant_method": "fp8", "dequantize": True}
+ )
+ _patched_method()(trainer)
+ assert rec.loaded == ["weight", "bias"]
+
+
+def _peft_trainer(model, vllm_mode = "colocate"):
+ trainer, _ = _trainer(vllm_mode)
+ rec = _WeightRecorder()
+ trainer.model = model
+ trainer.vllm_client = rec
+ if vllm_mode == "colocate":
+ trainer.llm.llm_engine.model_executor.driver_worker.model_runner.model = rec
+ trainer.llm.reset_prefix_cache = rec.reset_prefix_cache
+ return trainer, rec
+
+
+def _perturb_trainable(model):
+ with torch.no_grad():
+ for name, param in model.named_parameters():
+ if param.requires_grad:
+ param.add_(torch.randn_like(param) * 0.3 + 0.2)
+
+
+def _merged_state(model):
+ import copy
+ return {k: v.clone() for k, v in copy.deepcopy(model).merge_and_unload().state_dict().items()}
+
+
+def _base_mlp():
+ torch.manual_seed(1)
+ return torch.nn.Sequential(torch.nn.Linear(16, 16, bias = False), torch.nn.Linear(16, 8))
+
+
+def _non_lora_config(kind):
+ peft = pytest.importorskip("peft")
+ if kind == "loha":
+ return peft.LoHaConfig(r = 2, target_modules = ["0"], init_weights = False)
+ if kind == "lokr":
+ return peft.LoKrConfig(r = 2, target_modules = ["0"], init_weights = False)
+ if kind == "ia3":
+ return peft.IA3Config(target_modules = ["0"], feedforward_modules = [])
+ return peft.VeraConfig(r = 2, target_modules = ["0"])
+
+
[email protected]("kind", ["loha", "lokr", "ia3", "vera"])
+def test_non_lora_peft_syncs_merged_weights(kind):
+ # VeRA / LoHa / LoKr / IA3 have no lora_A: the plain LoRA merge would push the frozen base weight.
+ # PEFT's own merge must build what is pushed (on a copy of the base, see the drift test below).
+ peft = pytest.importorskip("peft")
+ model = peft.get_peft_model(_base_mlp(), _non_lora_config(kind))
+ _perturb_trainable(model)
+ expected = _merged_state(model)
+ trainer, rec = _peft_trainer(model)
+ assert _patched_method()(trainer) is None
+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"
+ pushed = dict(rec.tensors)
+ assert sorted(pushed) == ["0.weight", "1.bias", "1.weight"]
+ assert not torch.equal(pushed["0.weight"], model.base_model.model[0].base_layer.weight)
+ torch.testing.assert_close(pushed["0.weight"], expected["0.weight"])
+ assert rec.resets == 1
+
+
+def test_alora_raises_instead_of_flattening(monkeypatch):
+ # aLoRA only applies after its invocation tokens; PEFT refuses to merge it, and a static merge
+ # would serve a different model than the one being trained.
+ peft = pytest.importorskip("peft")
+ if "alora_invocation_tokens" not in peft.LoraConfig.__dataclass_fields__:
+ pytest.skip("this PEFT has no aLoRA")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ config = peft.LoraConfig(
+ r = 2, target_modules = ["0"], alora_invocation_tokens = [5], init_lora_weights = False
+ )
+ model = peft.get_peft_model(_base_mlp(), config)
+ trainer, rec = _peft_trainer(model)
+ with pytest.raises(RuntimeError, match = "aLoRA"):
+ _patched_method()(trainer)
+ assert rec.loaded == [] and rec.resets == 0
+
+
+class _Experts(torch.nn.Module):
+ # Fused MoE experts as in transformers 5 (gate_up_proj is a bare 3D parameter).
+ def __init__(self):
+ super().__init__()
+ self.gate_up_proj = torch.nn.Parameter(torch.randn(4, 16, 8))
+
+ def forward(self, x):
+ return torch.einsum("bi,eio->bo", x, self.gate_up_proj)
+
+
+def test_target_parameters_param_wrapper_syncs_merged_experts():
+ # ParamWrapper has no base_layer.weight, so the out-of-place merge pushed the untrained experts.
+ peft = pytest.importorskip("peft")
+ if "target_parameters" not in peft.LoraConfig.__dataclass_fields__:
+ pytest.skip("this PEFT has no target_parameters")
+ torch.manual_seed(1)
+ base = torch.nn.Sequential(torch.nn.Linear(16, 16, bias = False))
+ base.add_module("experts", _Experts())
+ config = peft.LoraConfig(
+ r = 2,
+ target_modules = ["0"],
+ target_parameters = ["experts.gate_up_proj"],
+ init_lora_weights = False,
+ )
+ model = peft.get_peft_model(base, config)
+ _perturb_trainable(model)
+ expected = _merged_state(model)
+ trainer, rec = _peft_trainer(model)
+ assert _patched_method()(trainer) is None
+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"
+ pushed = dict(rec.tensors)
+ assert sorted(pushed) == ["0.weight", "experts.gate_up_proj"]
+ torch.testing.assert_close(pushed["experts.gate_up_proj"], expected["experts.gate_up_proj"])
+ torch.testing.assert_close(pushed["0.weight"], expected["0.weight"])
+
+
[email protected]("second_adapter", [False, True])
+def test_modules_to_save_custom_adapter_name(second_adapter):
+ # TRL strips only "modules_to_save.default.": a custom name reached vLLM as is and the trained
+ # head was never synced. Push the active adapter's copy under the base name, and nothing else.
+ peft = pytest.importorskip("peft")
+ config = peft.LoraConfig(
+ r = 2, target_modules = ["0"], modules_to_save = ["1"], init_lora_weights = False
+ )
+ model = peft.get_peft_model(_base_mlp(), config, adapter_name = "mine")
+ if second_adapter:
+ model.add_adapter("other", config)
+ model.set_adapter("mine")
+ _perturb_trainable(model)
+ head = model.base_model.model[1]
+ trainer, rec = _peft_trainer(model)
assert _patched_method()(trainer) is None
+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"
+ assert sorted(rec.loaded) == ["0.weight", "1.bias", "1.weight"]
+ pushed = dict(rec.tensors)
+ assert torch.equal(pushed["1.weight"], head.modules_to_save["mine"].weight)
+ assert not torch.equal(pushed["1.weight"], head.original_module.weight)
+ layer = model.base_model.model[0]
+ delta = layer.lora_B["mine"].weight.float() @ layer.lora_A["mine"].weight.float()
+ expected = (layer.base_layer.weight.float() + delta * layer.scaling["mine"]).to(torch.float32)
+ assert torch.equal(pushed["0.weight"], expected)
+
+
+def test_trainable_token_indices_raises(monkeypatch):
+ # The trained rows live in token_adapter.trainable_tokens_delta.<adapter>; neither TRL's names
+ # nor a LoRA merge map that onto vLLM's embedding, so refuse instead of dropping the rows.
+ peft = pytest.importorskip("peft")
+ if "trainable_token_indices" not in peft.LoraConfig.__dataclass_fields__:
+ pytest.skip("this PEFT has no trainable_token_indices")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ torch.manual_seed(1)
+ base = torch.nn.Sequential(torch.nn.Embedding(32, 16), torch.nn.Linear(16, 16))
+ config = peft.LoraConfig(r = 2, target_modules = ["1"], trainable_token_indices = {"0": [1, 3]})
+ model = peft.get_peft_model(base, config)
+ trainer, rec = _peft_trainer(model)
+ with pytest.raises(RuntimeError, match = "trainable"):
+ _patched_method()(trainer)
+ assert rec.loaded == [] and rec.resets == 0
+
+
+# Codex review round 3: layer_replication, and the in-place merge / unmerge drift of the frozen base.
+
+
+def _tiny_llama(dtype = torch.float32):
+ transformers = pytest.importorskip("transformers")
+ torch.manual_seed(1)
+ config = transformers.LlamaConfig(
+ vocab_size = 32,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ attention_bias = True,
+ mlp_bias = True,
+ )
+ return transformers.LlamaForCausalLM(config).to(dtype)
+
+
+def test_layer_replication_raises(monkeypatch):
+ # Replicas share one base tensor (named_parameters yields it once) but train separate adapters,
+ # and vLLM loads the checkpoint's layer count: no name can carry each replica's merged weight.
+ peft = pytest.importorskip("peft")
+ if "layer_replication" not in peft.LoraConfig.__dataclass_fields__:
+ pytest.skip("this PEFT has no layer_replication")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ config = peft.LoraConfig(
+ r = 2,
+ target_modules = ["q_proj"],
+ layer_replication = [[0, 2], [1, 2]],
+ init_lora_weights = False,
+ )
+ model = peft.get_peft_model(_tiny_llama(), config)
+ assert len(model.base_model.model.model.layers) == 3
+ trainer, rec = _peft_trainer(model)
+ with pytest.raises(RuntimeError, match = "layer_replication"):
+ _patched_method()(trainer)
+ assert rec.loaded == [] and rec.resets == 0
+
+
+def _drift_config(kind):
+ peft = pytest.importorskip("peft")
+ targets = ["q_proj", "v_proj", "down_proj"]
+ if kind == "dora":
+ return peft.LoraConfig(r = 2, target_modules = targets, use_dora = True, init_lora_weights = False)
+ if kind == "lora_bias":
+ return peft.LoraConfig(r = 2, target_modules = targets, lora_bias = True, init_lora_weights = False)
+ if kind == "loha":
+ return peft.LoHaConfig(r = 2, target_modules = targets, init_weights = False)
+ if kind == "lokr":
+ return peft.LoKrConfig(r = 2, target_modules = targets, init_weights = False)
+ if kind == "ia3":
+ return peft.IA3Config(target_modules = targets, feedforward_modules = ["down_proj"])
+ return peft.VeraConfig(r = 2, target_modules = ["q_proj", "v_proj"])
+
+
[email protected]("kind", ["dora", "lora_bias", "loha", "lokr", "ia3", "vera"])
+def test_bf16_non_plain_peft_never_drifts_frozen_base(kind):
+ # TRL's body merge_adapter()s and unmerge_adapter()s the live bf16 model every sync; those rounded
+ # in-place edits are not lossless, so the frozen base drifted outside the optimizer. The merge now
+ # runs on a copy: after 10 syncs every frozen tensor keeps its exact bytes, and each sync pushes
+ # exactly what PEFT's own merge of the current adapter gives.
+ peft = pytest.importorskip("peft")
+ model = peft.get_peft_model(_tiny_llama(torch.bfloat16), _drift_config(kind))
+ frozen = {n: p.detach().clone() for n, p in model.named_parameters() if not p.requires_grad}
+ trainer, rec = _peft_trainer(model)
+ method = _patched_method()
+ for _ in range(10):
+ _perturb_trainable(model)
+ expected = _merged_state(model)
+ before = len(rec.tensors)
+ assert method(trainer) is None
+ pushed = dict(rec.tensors[before:])
+ assert pushed["model.layers.0.self_attn.q_proj.weight"].dtype == torch.bfloat16
+ for name in (
+ "model.layers.0.self_attn.q_proj.weight",
+ "model.layers.1.self_attn.v_proj.weight",
+ ):
+ assert torch.equal(pushed[name], expected[name]), name
+ drift = max(
+ (p.detach().float() - frozen[n].float()).abs().max().item()
+ for n, p in model.named_parameters()
+ if n in frozen
+ )
+ assert drift == 0.0
+ assert trainer._unsloth_vllm_sync_kind == "peft_merged"
+ for module in model.modules():
+ assert not getattr(module, "merged", False)
+ assert not getattr(module, "_caches", {})
+
+
+def test_bf16_target_parameters_never_drifts_frozen_base():
+ peft = pytest.importorskip("peft")
+ if "target_parameters" not in peft.LoraConfig.__dataclass_fields__:
+ pytest.skip("this PEFT has no target_parameters")
+ torch.manual_seed(1)
+ base = torch.nn.Sequential(torch.nn.Linear(16, 16, bias = False))
+ base.add_module("experts", _Experts())
+ base = base.to(torch.bfloat16)
+ config = peft.LoraConfig(
+ r = 2,
+ target_modules = ["0"],
+ target_parameters = ["experts.gate_up_proj"],
+ init_lora_weights = False,
+ )
+ model = peft.get_peft_model(base, config)
+ gate_up = model.base_model.model.experts.base_layer.gate_up_proj
+ before = gate_up.detach().clone()
+ trainer, rec = _peft_trainer(model)
+ method = _patched_method()
+ for _ in range(10):
+ _perturb_trainable(model)
+ expected = _merged_state(model)
+ start = len(rec.tensors)
+ assert method(trainer) is None
+ pushed = dict(rec.tensors[start:])
+ assert torch.equal(pushed["experts.gate_up_proj"], expected["experts.gate_up_proj"])
+ assert torch.equal(gate_up, before)
+
+
+def _sharded(
+ trainer,
+ fsdp = False,
+ zero3 = False,
+):
+ trainer.is_fsdp_enabled = fsdp
+ if zero3:
+ trainer.accelerator.state.deepspeed_plugin = types.SimpleNamespace(zero_stage = 3)
+ return trainer
+
+
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_fsdp_peft_raises(dtype, monkeypatch):
+ # TRL calls merge_adapter before it gathers the FSDP shards, so the full-size deltas meet sharded
+ # base weights (and the in-place round trip drifts a 16-bit base): refuse instead.
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ trainer, rec, model = _lora_trainer("colocate", dtype)
+ _sharded(trainer, fsdp = True)
+ with pytest.raises(RuntimeError, match = "FSDP"):
+ _patched_method()(trainer)
assert rec.loaded == [] and rec.resets == 0
+
+
+def test_zero3_keeps_trl_body():
+ # ZeRO-3: TRL merges inside GatheredParameters(modifier_rank = None), whose in-place edits are
+ # discarded on exit.
+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)
+ _sharded(trainer, zero3 = True)
+ assert _patched_method()(trainer) is None
+ assert trainer._unsloth_vllm_sync_kind == "trl" and "0.weight" in rec.loaded
+
+
+def test_lora_on_a_shared_base_weight_raises(monkeypatch):
+ # Tied input / output weights both adapted: one vLLM name cannot carry two different deltas.
+ peft = pytest.importorskip("peft")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ base = _base_mlp()
+ base.append(torch.nn.Linear(16, 16, bias = False))
+ base[2].weight = base[0].weight
+ model = peft.get_peft_model(
+ base, peft.LoraConfig(r = 2, target_modules = ["0", "2"], init_lora_weights = False)
+ )
+ trainer, rec = _peft_trainer(model)
+ with pytest.raises(RuntimeError, match = "one base weight shared by"):
+ _patched_method()(trainer)
+ assert rec.loaded == []
+
+
+def test_lora_on_a_weight_tied_to_an_unadapted_module_raises(monkeypatch):
+ # Only one site of a tied weight adapted: vLLM still gets one tensor for both names.
+ peft = pytest.importorskip("peft")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ base = _base_mlp()
+ base.append(torch.nn.Linear(16, 16, bias = False))
+ base[2].weight = base[0].weight
+ model = peft.get_peft_model(
+ base, peft.LoraConfig(r = 2, target_modules = ["2"], init_lora_weights = False)
+ )
+ trainer, rec = _peft_trainer(model)
+ with pytest.raises(RuntimeError, match = "one base weight shared by"):
+ _patched_method()(trainer)
+ assert rec.loaded == []
+
+
+def test_accelerate_offloaded_model_raises(monkeypatch):
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)
+ model.base_model.model[1]._hf_hook = types.SimpleNamespace(offload = True)
+ with pytest.raises(RuntimeError, match = "Accelerate device_map"):
+ _patched_method()(trainer)
+ assert rec.loaded == []
+
+
+def test_switching_to_an_unsupported_adapter_is_refused_before_the_next_sync(monkeypatch):
+ peft = pytest.importorskip("peft")
+ if "alora_invocation_tokens" not in peft.LoraConfig.__dataclass_fields__:
+ pytest.skip("this PEFT has no aLoRA")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ model = peft.get_peft_model(
+ _base_mlp(), peft.LoraConfig(r = 2, target_modules = ["0"], init_lora_weights = False)
+ )
+ model.add_adapter(
+ "other",
+ peft.LoraConfig(
+ r = 2, target_modules = ["0"], alora_invocation_tokens = [5], init_lora_weights = False
+ ),
+ )
+ model.set_adapter("default")
+ trainer, rec = _peft_trainer(model)
+ method = _patched_method()
+ assert method(trainer) is None and "0.weight" in rec.loaded
+ model.set_adapter("other")
+ before = len(rec.loaded)
+ with pytest.raises(RuntimeError, match = "cannot sync this PEFT model.*aLoRA"):
+ method(trainer)
+ assert len(rec.loaded) == before
+
+
+def test_inactive_unsupported_adapter_does_not_block_sync():
+ peft = pytest.importorskip("peft")
+ model = peft.get_peft_model(
+ _base_mlp(), peft.LoraConfig(r = 2, target_modules = ["0"], init_lora_weights = False)
+ )
+ if "alora_invocation_tokens" not in peft.LoraConfig.__dataclass_fields__:
+ pytest.skip("this PEFT has no aLoRA")
+ other = peft.LoraConfig(
+ r = 2, target_modules = ["0"], alora_invocation_tokens = [5], init_lora_weights = False
+ )
+ model.add_adapter("other", other)
+ model.set_adapter("default")
+ trainer, rec = _peft_trainer(model)
+ assert _patched_method()(trainer) is None
+ assert trainer._unsloth_vllm_sync_kind == "peft_merged" and "0.weight" in rec.loaded
+
+
[email protected]("kind", ["ln_tuning", "poly"])
+def test_peft_without_a_static_merge_raises(kind, monkeypatch):
+ # LN tuning swaps the module on merge, POLY routes per task: neither has one merged weight.
+ peft = pytest.importorskip("peft")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ if kind == "ln_tuning":
+ base = torch.nn.Sequential(torch.nn.Linear(16, 16), torch.nn.LayerNorm(16))
+ config = peft.LNTuningConfig(target_modules = ["1"])
+ else:
+ base = _base_mlp()
+ config = peft.PolyConfig(r = 2, target_modules = ["0"], n_tasks = 2)
+ model = peft.get_peft_model(base, config)
+ trainer, rec = _peft_trainer(model)
+ with pytest.raises(RuntimeError, match = "no static merge"):
+ _patched_method()(trainer)
+ assert rec.loaded == []
+
+
+def test_block_swapped_model_raises(monkeypatch):
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)
+ model.base_model.model._unsloth_block_swap = object()
+ with pytest.raises(RuntimeError, match = "offload_layers"):
+ _patched_method()(trainer)
+ assert rec.loaded == []
+
+
+def test_legacy_trl_without_name_fixer_still_syncs():
+ # TRL < 0.20 has no _fix_param_name_to_vllm and sends the names as they are.
+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)
+ del trainer._fix_param_name_to_vllm
+ assert _patched_method()(trainer) is None
+ assert sorted(dict(rec.tensors)) == ["0.weight", "1.bias", "1.weight"]
+
+
+def test_lora_on_multihead_attention_raises(monkeypatch):
+ # PEFT's MultiheadAttention merge swaps in new parameters instead of editing the base ones.
+ peft = pytest.importorskip("peft")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+
+ class _Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.attn = torch.nn.MultiheadAttention(8, 2)
+ self.proj = torch.nn.Linear(8, 8)
+
+ model = peft.get_peft_model(
+ _Block(), peft.LoraConfig(r = 2, target_modules = ["attn"], init_lora_weights = False)
+ )
+ trainer, rec = _peft_trainer(model)
+ with pytest.raises(RuntimeError, match = "MultiheadAttention"):
+ _patched_method()(trainer)
+ assert rec.loaded == []
+
+
+def test_mixed_peft_model_raises(monkeypatch):
+ peft = pytest.importorskip("peft")
+ monkeypatch.delenv(_STALE_ENV, raising = False)
+ model = peft.get_peft_model(_base_mlp(), peft.LoraConfig(r = 2, target_modules = ["0"]), mixed = True)
+ model.add_adapter("loha", peft.LoHaConfig(r = 2, target_modules = ["0"]))
+ model.set_adapter(["default", "loha"])
+ trainer, rec = _peft_trainer(model)
+ with pytest.raises(RuntimeError, match = "mixed PEFT model"):
+ _patched_method()(trainer)
+ assert rec.loaded == []
+
+
+def test_trainer_without_vllm_mode_uses_its_colocated_engine():
+ trainer, rec, model = _lora_trainer("colocate", torch.bfloat16)
+ del trainer.vllm_mode
+ trainer.vllm_client = None
+ assert _patched_method()(trainer) is None
+ assert sorted(dict(rec.tensors)) == ["0.weight", "1.bias", "1.weight"]
+
+
+def test_loha_rank_dropout_is_off_while_merging():
+ # Training mode with rank dropout: the pushed weight is the full inference-time merge.
+ peft = pytest.importorskip("peft")
+ model = peft.get_peft_model(
+ _base_mlp(),
+ peft.LoHaConfig(r = 4, target_modules = ["0"], rank_dropout = 0.5, init_weights = False),
+ )
+ _perturb_trainable(model)
+ model.eval()
+ expected = _merged_state(model)
+ model.train()
+ trainer, rec = _peft_trainer(model)
+ method = _patched_method()
+ assert method(trainer) is None and method(trainer) is None
+ pushed = [tensor for name, tensor in rec.tensors if name == "0.weight"]
+ assert len(pushed) == 2 and torch.equal(pushed[0], pushed[1])
+ torch.testing.assert_close(pushed[0], expected["0.weight"])
+ assert all(m.training for m in model.modules())
+
+
+def test_disabled_non_plain_adapter_pushes_the_base():
+ peft = pytest.importorskip("peft")
+ model = peft.get_peft_model(_base_mlp(), _non_lora_config("loha"))
+ _perturb_trainable(model)
+ trainer, rec = _peft_trainer(model)
+ with model.disable_adapter():
+ assert _patched_method()(trainer) is None
+ pushed = dict(rec.tensors)
+ assert torch.equal(pushed["0.weight"], model.base_model.model[0].base_layer.weight)
+
+
+def test_base_module_named_like_the_adapter_prefix_is_still_pushed():
+ peft = pytest.importorskip("peft")
+
+ class _Named(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q_lora_proj = torch.nn.Linear(16, 16, bias = False)
+
+ model = peft.get_peft_model(
+ _Named(), peft.LoraConfig(r = 2, target_modules = ["q_lora_proj"], init_lora_weights = False)
+ )
+ trainer, rec = _peft_trainer(model)
+ assert _patched_method()(trainer) is None
+ assert sorted(dict(rec.tensors)) == ["q_lora_proj.weight"]
+
+
+def test_adapter_on_a_composite_base_is_refused():
+ from unsloth.models.rl_replacements import _unsloth_vllm_peft_unsupported
+
+ class _Custom(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.base_layer = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ self.lora_A = torch.nn.ModuleDict()
+
+ def get_base_layer(self):
+ return self.base_layer
+
+ model = torch.nn.Sequential(_Custom())
+ model.peft_config = {}
+ assert any("composite" in p for p in _unsloth_vllm_peft_unsupported(model))
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 02b4920c583..84f63a3f400 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -2476,14 +2476,30 @@ def grpo_trainer_fix_maybe_apply_chat_template(function_name, function):
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_fix_maybe_apply_chat_template)
-def _unsloth_vllm_sync_unsafe(model):
+def _unsloth_vllm_quantized(model):
# FSDP2 turns Params4bit into a DTensor, so also read the load flags the model keeps.
if getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False):
return True
- if getattr(model, "peft_config", None):
+ # transformers marks every quantizer it loads (GPTQ, AWQ, HQQ, torchao, compressed-tensors, ...).
+ # A dequantize = True load removes all of these, so they only remain on a quantized model.
+ if getattr(model, "is_quantized", False) is True or getattr(model, "quantization_method", None):
+ return True
+ if getattr(model, "hf_quantizer", None) is not None:
return True
+ quantization_config = getattr(getattr(model, "config", None), "quantization_config", None)
+ if quantization_config:
+ dequantize = (
+ quantization_config.get("dequantize", False)
+ if isinstance(quantization_config, dict)
+ else getattr(quantization_config, "dequantize", False)
+ )
+ if not dequantize:
+ return True
for module in model.modules():
- if hasattr(module, "lora_A") or hasattr(module, "base_layer"):
+ # Packed integer weights (GPTQ / AWQ qweight, HQQ W_q) are not vLLM's layout.
+ if isinstance(getattr(module, "qweight", None), torch.Tensor) or isinstance(
+ getattr(module, "W_q", None), torch.Tensor
+ ):
return True
for param in model.parameters():
if type(param).__name__ in ("Params4bit", "Int8Params"):
@@ -2493,6 +2509,438 @@ def _unsloth_vllm_sync_unsafe(model):
return False
+def _unsloth_vllm_lora_layers(model):
+ return [
+ module
+ for module in model.modules()
+ if hasattr(module, "lora_A") and getattr(module, "base_layer", None) is not None
+ ]
+
+
+def _unsloth_vllm_sync_unsafe(model):
+ if _unsloth_vllm_quantized(model):
+ return True
+ if getattr(model, "peft_config", None):
+ return True
+ return len(_unsloth_vllm_lora_layers(model)) != 0
+
+
+# LoRA layer classes whose merged weight _unsloth_vllm_merged_weight computes: one dense base weight.
+_UNSLOTH_VLLM_PLAIN_LORA = ("Linear", "Embedding", "Conv1d", "Conv2d", "Conv3d")
+# LoRA variants PEFT itself cannot merge into the base weight (they condition on the input).
+_UNSLOTH_VLLM_UNMERGEABLE_VARIANTS = ("ALora", "Arrow", "QALora")
+
+
+def _unsloth_vllm_peft_type(config):
+ peft_type = getattr(config, "peft_type", None)
+ return str(getattr(peft_type, "value", peft_type))
+
+
+# PEFT types whose merge writes the merged weight into the base tensors (no module swap, no per-input routing).
+_UNSLOTH_VLLM_MERGEABLE_TYPES = ("LORA", "LOHA", "LOKR", "IA3", "VERA")
+
+
+def _unsloth_vllm_active_adapters(model):
+ configs = getattr(model, "peft_config", None) or {}
+ active = getattr(model, "active_adapters", None)
+ if callable(active):
+ active = active()
+ if isinstance(active, str):
+ active = [active]
+ return set(active) if active else set(configs)
+
+
+def _unsloth_vllm_shared_targets(model):
+ """Adapted base parameters reachable under more than one name (tied embed_tokens / lm_head, adapted
+ at one or both sites): vLLM gets one tensor for them, so some call site would get the wrong delta."""
+ names = {}
+ for name, param in model.named_parameters(remove_duplicate = False):
+ names.setdefault(id(param), []).append(name)
+ shared = {}
+ for module in model.modules():
+ if getattr(module, "base_layer", None) is None or not hasattr(module, "get_base_layer"):
+ continue
+ for param in module.get_base_layer().parameters(recurse = False):
+ if len(names.get(id(param), ())) > 1:
+ shared[id(param)] = " and ".join(names[id(param)])
+ return list(shared.values())
+
+
+def _unsloth_vllm_offloaded(model):
+ """How the model keeps weights off the device between forwards, or None: a sync would read the
+ empty or meta placeholders."""
+ for module in model.modules():
+ if getattr(module, "_unsloth_block_swap", None) is not None:
+ return "block-swapped decoder layers (offload_layers)"
+ hook = getattr(module, "_hf_hook", None)
+ hooks = getattr(hook, "hooks", None) or ([hook] if hook is not None else [])
+ if any(getattr(h, "offload", False) for h in hooks):
+ return "CPU / disk offloaded layers (Accelerate device_map)"
+ if any(param.device.type == "meta" for param in model.parameters()):
+ return "meta (offloaded) parameters"
+ return None
+
+
+def _unsloth_vllm_peft_unsupported(model):
+ """PEFT setups neither TRL's PEFT branch nor the out-of-place merge can sync: names of the culprits.
+ Only active adapters count; an inactive one does not shape the policy being synced."""
+ problems = []
+ active = _unsloth_vllm_active_adapters(model)
+ if type(model).__name__ == "PeftMixedModel":
+ # Adapter parameters carry per-type prefixes (lora_, hada_, lokr_, ...) the sync cannot filter.
+ problems.append("a mixed PEFT model (get_peft_model(..., mixed = True))")
+ for name, config in (getattr(model, "peft_config", None) or {}).items():
+ # Replicas share one base tensor but train separate adapters, and vLLM loads the checkpoint's
+ # layer count: there is no per-replica weight to send (PEFT also refuses to merge these).
+ if getattr(config, "layer_replication", None):
+ problems.append(f"layer_replication on adapter '{name}'")
+ if name not in active:
+ continue
+ peft_type = _unsloth_vllm_peft_type(config)
+ if getattr(config, "is_prompt_learning", False) or getattr(
+ config, "is_adaption_prompt", False
+ ):
+ problems.append(f"prompt learning adapter '{name}' ({_unsloth_vllm_peft_type(config)})")
+ if getattr(config, "trainable_token_indices", None):
+ problems.append(f"trainable_token_indices on adapter '{name}'")
+ if getattr(config, "alora_invocation_tokens", None):
+ problems.append(f"aLoRA adapter '{name}'")
+ if getattr(config, "arrow_config", None) is not None:
+ problems.append(f"Arrow adapter '{name}'")
+ elif not getattr(config, "is_prompt_learning", False) and (
+ peft_type not in _UNSLOTH_VLLM_MERGEABLE_TYPES
+ ):
+ # LN tuning swaps modules, POLY / X-LoRA route per input: no single merged weight.
+ problems.append(f"{peft_type} adapter '{name}' (no static merge into the base weights)")
+ for module_name, module in model.named_modules():
+ # trainable_token_indices, even on an inactive adapter: its token_adapter.* parameters and
+ # names do not exist in vLLM and are pushed all the same.
+ if type(module).__name__ in ("TrainableTokensWrapper", "TrainableTokensLayer"):
+ problems.append(f"trainable tokens at {module_name}")
+ break
+ for module_name, module in model.named_modules():
+ # PEFT's MultiheadAttention merge replaces in_proj_weight / out_proj.weight instead of
+ # editing them, so a copy merge cannot leave the live module untouched.
+ if type(module).__name__ == "MultiheadAttention" and hasattr(module, "lora_A"):
+ problems.append(f"LoRA on MultiheadAttention at {module_name}")
+ break
+ for module_name, module in model.named_modules():
+ # A custom tuner layer on a composite base: its weights live in child modules the copy
+ # merge does not see, so the merged weights would never be sent.
+ if getattr(module, "base_layer", None) is None or not hasattr(module, "get_base_layer"):
+ continue
+ base = module.get_base_layer()
+ if any(any(True for _ in child.parameters()) for child in base.children()):
+ problems.append(f"adapter on the composite module {module_name}")
+ break
+ for module in _unsloth_vllm_lora_layers(model):
+ for adapter, variant in (getattr(module, "lora_variant", None) or {}).items():
+ if adapter not in active:
+ continue
+ variant_name = type(variant).__name__
+ if any(variant_name.startswith(x) for x in _UNSLOTH_VLLM_UNMERGEABLE_VARIANTS):
+ problems.append(f"LoRA variant {variant_name}")
+ return list(dict.fromkeys(problems))
+
+
+def _unsloth_vllm_modules_to_save(model):
+ """(wrapper name, adapter it serves or None for original_module, all its adapter names)."""
+ wrappers = []
+ for module_name, module in model.named_modules():
+ if type(module).__name__ != "ModulesToSaveWrapper":
+ continue
+ copies = getattr(module, "modules_to_save", None) or {}
+ served = None
+ if not getattr(module, "disable_adapters", False):
+ for adapter in getattr(module, "active_adapters", None) or []:
+ if adapter in copies:
+ served = adapter
+ break
+ wrappers.append((module_name, served, list(copies.keys())))
+ return wrappers
+
+
+def _unsloth_vllm_plain_lora(model):
+ """True when every PEFT piece is plain LoRA that _unsloth_vllm_sync_peft_merged fully syncs."""
+ configs = getattr(model, "peft_config", None) or {}
+ active = _unsloth_vllm_active_adapters(model)
+ if any(
+ _unsloth_vllm_peft_type(config) != "LORA"
+ for name, config in configs.items()
+ if name in active
+ ):
+ return False
+ try:
+ from peft.tuners.tuners_utils import BaseTunerLayer
+ except Exception:
+ BaseTunerLayer = ()
+ lora_layers = {id(module) for module in _unsloth_vllm_lora_layers(model)}
+ for module in model.modules():
+ is_tuner = (BaseTunerLayer != () and isinstance(module, BaseTunerLayer)) or (
+ id(module) in lora_layers
+ )
+ if not is_tuner:
+ continue
+ # ParamWrapper (target_parameters), TrainableTokensLayer, ...: not a single dense weight.
+ if id(module) not in lora_layers or type(module).__name__ not in _UNSLOTH_VLLM_PLAIN_LORA:
+ return False
+ base_layer = module.base_layer
+ if not isinstance(getattr(base_layer, "weight", None), torch.Tensor):
+ return False
+ if getattr(module, "lora_A", None) is not None and hasattr(base_layer, "base_layer"):
+ return False
+ for flags in ("lora_variant", "use_dora", "lora_bias"):
+ if any(v for a, v in (getattr(module, flags, None) or {}).items() if a in active):
+ return False
+ return True
+
+
+def _unsloth_vllm_sync_kind(trainer):
+ """How _move_model_to_vllm syncs TRL's own vLLM engine (fast_inference = False):
+ "trl" = TRL's body (dense, or sharded PEFT where its in-place merge cannot drift the base),
+ "peft_merged" = PEFT merged out of place, "skip" = no sync (explicit opt-in only). Anything that cannot be synced raises: rollouts
+ would come from the initial weights all run."""
+ model = trainer.model
+ allow_stale = os.environ.get("UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS", "0") == "1"
+ if _unsloth_vllm_quantized(model):
+ if allow_stale:
+ print(
+ "Unsloth: UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1, so vLLM weight sync is skipped for this "
+ "quantized model and every rollout uses the initial weights."
+ )
+ return "skip"
+ raise RuntimeError(
+ "Unsloth: GRPO with use_vllm = True but fast_inference = False cannot sync a quantized "
+ "(bitsandbytes, GPTQ, AWQ, HQQ, torchao, FP8, ...) model into TRL's own vLLM engine, so every "
+ "rollout would come from the initial weights while the policy trains.\n"
+ "Load the model with fast_inference = True, or load a 16-bit base (load_in_4bit = False, "
+ "load_in_8bit = False). Set UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1 to train anyway."
+ )
+ offloaded = _unsloth_vllm_offloaded(model)
+ if offloaded is not None:
+ if allow_stale:
+ print(
+ "Unsloth: UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1, so vLLM weight sync is skipped for this "
+ f"model with {offloaded} and every rollout uses the initial weights."
+ )
+ return "skip"
+ raise RuntimeError(
+ "Unsloth: GRPO with use_vllm = True but fast_inference = False cannot sync a model with "
+ f"{offloaded}: those weights are empty or on the meta device between forwards.\n"
+ "Keep every layer on the GPU (no offload_layers, no CPU / disk device_map), or load the "
+ "model with fast_inference = True. Set UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1 to train anyway."
+ )
+ if not _unsloth_vllm_sync_unsafe(model):
+ return "trl"
+ state = getattr(getattr(trainer, "accelerator", None), "state", None)
+ deepspeed_plugin = getattr(state, "deepspeed_plugin", None)
+ sharded = getattr(trainer, "is_fsdp_enabled", False) or (
+ deepspeed_plugin is not None and getattr(deepspeed_plugin, "zero_stage", 0) == 3
+ )
+ problems = _unsloth_vllm_peft_unsupported(model)
+ problems.extend(
+ f"one base weight shared by {names}" for names in _unsloth_vllm_shared_targets(model)
+ )
+ # Unsharded: every mergeable PEFT setup is merged out of place, so the frozen base never moves.
+ # ZeRO-3: TRL merges inside GatheredParameters(modifier_rank = None), which discards in-place edits
+ # of the gathered copy on exit, so its merge / unmerge cannot drift the stored base either.
+ # FSDP: TRL calls merge_adapter before it gathers the sharded parameters, so PEFT's full-size
+ # deltas meet sharded base weights (and the in-place round trip would drift a 16-bit base).
+ kind = "peft_merged" if not sharded else "trl"
+ if getattr(trainer, "is_fsdp_enabled", False):
+ problems.append(
+ "FSDP (TRL merges the adapters before it gathers the sharded weights; use DeepSpeed "
+ "ZeRO-3 or a single process)"
+ )
+ if kind == "trl":
+ # TRL strips only "modules_to_save.default.", so other adapter names reach vLLM as is.
+ for wrapper, _, adapters in _unsloth_vllm_modules_to_save(model):
+ if any(adapter != "default" for adapter in adapters):
+ problems.append(f"modules_to_save at {wrapper} for adapters {adapters}")
+ if problems:
+ if allow_stale:
+ print(
+ "Unsloth: UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1, so vLLM weight sync is skipped for this "
+ f"PEFT model ({'; '.join(problems)}) and every rollout uses the initial weights."
+ )
+ return "skip"
+ raise RuntimeError(
+ "Unsloth: GRPO with use_vllm = True but fast_inference = False cannot sync this PEFT model "
+ "into TRL's own vLLM engine: "
+ + "; ".join(problems)
+ + ". These cannot be merged into the "
+ "base weights under vLLM's parameter names, so every rollout would come from the initial "
+ "weights while the policy trains.\n"
+ "Use a plain LoRA adapter, or set UNSLOTH_VLLM_ALLOW_STALE_ROLLOUTS=1 to train anyway."
+ )
+ return kind
+
+
+def _unsloth_vllm_sync_kind_cached(trainer):
+ """_unsloth_vllm_sync_kind, decided again whenever the active adapters change (set_adapter)."""
+ key = tuple(sorted(_unsloth_vllm_active_adapters(trainer.model)))
+ if getattr(trainer, "_unsloth_vllm_sync_kind", None) is None or (
+ getattr(trainer, "_unsloth_vllm_sync_key", None) != key
+ ):
+ trainer._unsloth_vllm_sync_kind = _unsloth_vllm_sync_kind(trainer)
+ trainer._unsloth_vllm_sync_skip = trainer._unsloth_vllm_sync_kind == "skip"
+ trainer._unsloth_vllm_sync_key = key
+ return trainer._unsloth_vllm_sync_kind
+
+
+def _unsloth_vllm_merged_weight(module, weight):
+ # W + sum of active deltas, computed out of place in fp32 and rounded once. merge_adapter +
+ # unmerge_adapter would round twice in place and drift the frozen base every sync.
+ merged = None
+ if not getattr(module, "disable_adapters", False) and not getattr(module, "merged", False):
+ for adapter in module.active_adapters:
+ lora_A = module.lora_A[adapter] if adapter in module.lora_A else None
+ if isinstance(lora_A, torch.nn.Linear) and adapter in module.lora_B:
+ delta = module.lora_B[adapter].weight.float() @ lora_A.weight.float()
+ delta = delta * module.scaling[adapter]
+ if getattr(module, "fan_in_fan_out", False):
+ delta = delta.T
+ elif adapter in module.lora_A or adapter in getattr(module, "lora_embedding_A", {}):
+ delta = module.get_delta_weight(adapter).float()
+ else:
+ continue
+ delta = delta.to(weight.device).reshape(weight.shape)
+ merged = delta if merged is None else merged + delta
+ if merged is None:
+ return weight.data
+ return (weight.float() + merged).to(weight.dtype)
+
+
+def _unsloth_vllm_tuner_groups(model):
+ """{id(base param): (real base module, [tuner layers wrapping it])} for PEFT's own merge."""
+ from peft.tuners.tuners_utils import BaseTunerLayer
+
+ groups = {}
+ for module in model.modules():
+ if isinstance(module, BaseTunerLayer):
+ base = module.get_base_layer()
+ groups.setdefault(id(base), (base, []))[1].append(module)
+ return {
+ id(param): group
+ for group in groups.values()
+ for param in group[0].parameters(recurse = False)
+ }
+
+
+def _unsloth_vllm_peft_merge_out_of_place(base, layers):
+ """Run PEFT's own merge (DoRA, lora_bias, LoHa, LoKr, IA3, VeRA, ParamWrapper, ...) on copies of
+ the base tensors and return {id(param): merged tensor}; the live parameters keep their exact bytes."""
+ params = list(base.parameters(recurse = False))
+ originals = [param.data for param in params]
+ states = [
+ (layer, list(getattr(layer, "merged_adapters", [])), dict(getattr(layer, "_caches", {})))
+ for layer in layers
+ ]
+ # Eval mode while merging: LoHa / LoKr rank dropout would otherwise thin the delta at random.
+ modes = [(module, module.training) for layer in layers for module in layer.modules()]
+ try:
+ for param, data in zip(params, originals):
+ param.data = data.clone()
+ for layer in layers:
+ layer.train(False)
+ # model.modules() order, as PeftModel.merge_adapter merges them. A disabled layer runs only
+ # its base in training, so it adds nothing (PEFT's merge ignores disable_adapters).
+ for layer in layers:
+ if not getattr(layer, "disable_adapters", False):
+ layer.merge()
+ return {id(param): param.data for param in params}
+ finally:
+ for module, training in modes:
+ module.training = training
+ for param, data in zip(params, originals):
+ param.data = data
+ for layer, merged_adapters, caches in states:
+ if hasattr(layer, "merged_adapters"):
+ layer.merged_adapters[:] = merged_adapters
+ if isinstance(getattr(layer, "_caches", None), dict):
+ layer._caches.clear()
+ layer._caches.update(caches)
+
+
+def _unsloth_vllm_adapter_param_ids(model):
+ """Adapter tensors: every parameter under a tuner layer except its base layer's own."""
+ ids = set()
+ for module in model.modules():
+ if getattr(module, "base_layer", None) is None or not hasattr(module, "get_base_layer"):
+ continue
+ base_ids = {id(param) for param in module.get_base_layer().parameters()}
+ ids.update(id(param) for param in module.parameters() if id(param) not in base_ids)
+ return ids
+
+
+_UNSLOTH_VLLM_MODULES_TO_SAVE = re.compile(r"\.(?:modules_to_save\.[^.]+|original_module)\.")
+
+
+def _unsloth_vllm_sync_peft_merged(trainer):
+ """TRL's PEFT sync (merged weights under the base-layer names), without mutating the model."""
+ model = trainer.model
+ # Plain LoRA: W + sum of deltas in fp32, rounded once. Anything else: PEFT's merge on a copy.
+ plain = _unsloth_vllm_plain_lora(model)
+ lora_of = {}
+ groups = {}
+ if plain:
+ for module in _unsloth_vllm_lora_layers(model):
+ weight = getattr(module.base_layer, "weight", None)
+ if weight is not None:
+ lora_of[id(weight)] = module
+ else:
+ groups = _unsloth_vllm_tuner_groups(model)
+ merged_cache = {}
+ # modules_to_save: push the copy the active adapter trains (any adapter name), never the rest.
+ unserved = []
+ for wrapper, served, adapters in _unsloth_vllm_modules_to_save(model):
+ unserved.extend(f"{wrapper}.modules_to_save.{a}." for a in adapters if a != served)
+ if served is not None:
+ unserved.append(f"{wrapper}.original_module.")
+ unserved = tuple(unserved)
+ # Structural, not a name match: a base module named q_lora_proj is still a base weight.
+ adapter_ids = _unsloth_vllm_adapter_param_ids(model)
+ # TRL 0.18 has no vllm_mode: its in-process engine is the colocated one.
+ mode = getattr(trainer, "vllm_mode", None) or (
+ "server" if getattr(trainer, "vllm_client", None) is not None else "colocate"
+ )
+ is_main = getattr(trainer.accelerator, "is_main_process", True)
+ llm_model = None
+ if mode == "colocate":
+ llm_model = trainer.llm.llm_engine.model_executor.driver_worker.model_runner.model
+ # TRL < 0.20 sends the names as they are and has no fixer.
+ fix_name = getattr(trainer, "_fix_param_name_to_vllm", None) or (lambda name, **_: name)
+ for full_name, param in model.named_parameters():
+ if unserved and full_name.startswith(unserved):
+ continue
+ if id(param) in adapter_ids:
+ continue
+ name = full_name.removeprefix("base_model.model.").replace(".base_layer", "")
+ name = fix_name(_UNSLOTH_VLLM_MODULES_TO_SAVE.sub(".", name))
+ if plain:
+ module = lora_of.get(id(param))
+ tensor = param.data if module is None else _unsloth_vllm_merged_weight(module, param)
+ elif id(param) in groups:
+ # One base module at a time: its weight and bias are adjacent in named_parameters.
+ if id(param) not in merged_cache:
+ merged_cache.clear()
+ merged_cache.update(_unsloth_vllm_peft_merge_out_of_place(*groups[id(param)]))
+ tensor = merged_cache.pop(id(param))
+ else:
+ tensor = param.data
+ if mode == "server":
+ if is_main:
+ trainer.vllm_client.update_named_param(name, tensor)
+ elif mode == "colocate":
+ llm_model.load_weights([(name, tensor)])
+ if mode == "server" and is_main:
+ trainer.vllm_client.reset_prefix_cache()
+ elif mode == "colocate":
+ trainer.llm.reset_prefix_cache()
+ return None
+
+
def grpo_trainer__move_model_to_vllm(function_name, function):
if function_name != "_move_model_to_vllm":
return function
@@ -2514,22 +2962,26 @@ def grpo_trainer__move_model_to_vllm(function_name, function):
return inspect.getsource(_move_model_to_vllm)
indent = body_indent.group(1)
- # Keep the old no-op where TRL's sync is unsafe:
- # - LoRA / PEFT: merge_adapter + unmerge_adapter on the live model is a lossy round trip in
- # fp16 / bf16, so frozen base weights drift every sync; it also requantizes bitsandbytes bases
- # from the fp32 delta (fast_lora backward then fails) and refuses FP8 grouped-linear bases.
- # - Quantized bases (bitsandbytes, 1-byte floats such as FP8) cannot be loaded into vLLM as is.
- # So the sync only runs for dense full finetuning.
+ # Only the fast_inference engine shares weights; every other engine must get the trained weights:
+ # - Dense: TRL's own body.
+ # - Plain LoRA on a 16-bit base: merged weights under the base-layer names, as TRL does, but
+ # computed out of place. merge_adapter + unmerge_adapter on the live model is a lossy round
+ # trip in fp16 / bf16, so the frozen base would drift every sync.
+ # - Other mergeable PEFT (LoHa, LoKr, IA3, VeRA, DoRA, lora_bias, target_parameters): PEFT's own
+ # merge run on copies of the base tensors, so the live base keeps its exact bytes.
+ # - ZeRO-3: TRL's body (its gathered copy is discarded on exit). FSDP with PEFT: raise.
+ # - Quantized bases (bitsandbytes, GPTQ, AWQ, HQQ, FP8, ...), unmergeable adapters (aLoRA, Arrow,
+ # trainable tokens, prompt learning): raise instead of sampling from the initial weights all run.
guard = (
f"{indent}if getattr(getattr(self, 'llm', None), 'shared_weights', False):\n"
f"{indent} return None\n"
- f"{indent}if getattr(self, '_unsloth_vllm_sync_skip', None) is None:\n"
- f"{indent} from unsloth.models.rl_replacements import _unsloth_vllm_sync_unsafe\n"
- f"{indent} self._unsloth_vllm_sync_skip = _unsloth_vllm_sync_unsafe(self.model)\n"
- f"{indent} if self._unsloth_vllm_sync_skip:\n"
- f"{indent} print('Unsloth: vLLM weight sync is skipped for a LoRA / PEFT or quantized model without fast_inference, so rollouts use the initial weights. Load with fast_inference = True to sample from the trained model.')\n"
- f"{indent}if self._unsloth_vllm_sync_skip:\n"
+ f"{indent}from unsloth.models.rl_replacements import _unsloth_vllm_sync_kind_cached\n"
+ f"{indent}_unsloth_vllm_sync_kind_cached(self)\n"
+ f"{indent}if self._unsloth_vllm_sync_kind == 'skip':\n"
f"{indent} return None\n"
+ f"{indent}if self._unsloth_vllm_sync_kind == 'peft_merged':\n"
+ f"{indent} from unsloth.models.rl_replacements import _unsloth_vllm_sync_peft_merged\n"
+ f"{indent} return _unsloth_vllm_sync_peft_merged(self)\n"
)
return function[: match.end()] + guard + function[match.end() :]