Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Sync full finetune weights into colocated vLLM in a few load_weights calls (#13901)
Daniel Han6c229ce09992
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.ymlindex 5bc577c26dc..c5c5278eb78 100644--- a/.github/workflows/version-compat-ci.yml+++ b/.github/workflows/version-compat-ci.yml@@ -99,6 +99,7 @@ jobs:tests/version_compat/test_bitsandbytes_pinned_symbols.py \tests/version_compat/test_transformers_pinned_symbols.py \tests/version_compat/test_no_dtype_kwarg_at_load.py \+ tests/version_compat/test_trl_vllm_colocate_weight_sync.py \-n 4 --dist loadfile -v --tb=shortcap-site-consistency:diff --git a/tests/version_compat/test_trl_vllm_colocate_weight_sync.py b/tests/version_compat/test_trl_vllm_colocate_weight_sync.pynew file mode 100644index 00000000000..19ed9aaf18f--- /dev/null+++ b/tests/version_compat/test_trl_vllm_colocate_weight_sync.py@@ -0,0 +1,229 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""Full-finetune colocate weight sync: few load_weights calls, same tensors, same order."""++from __future__ import annotations++import importlib.machinery+import importlib.util+import linecache+import sys+import textwrap+import types++import pytest+++if importlib.util.find_spec("torch") is None:+ pytest.skip("torch not installed; this test drives the real patch", allow_module_level = True)++import torch+++_INIT_VLLM = """+def _init_vllm(self, model):+ if self.mode == "colocate":+ self.llm = LLM(+ model = model.name_or_path,+ )+"""++# TRL >= 1.10 VLLMGeneration.sync_weights, trimmed to the parts the patch reads.+_SYNC_WEIGHTS = """+def sync_weights(self):+ if self.mode == "colocate":+ for name, param in self._iter_named_params():+ self.llm.llm_engine.model_executor.driver_worker.model_runner.model.load_weights([(name, param)])+ self.llm.reset_prefix_cache()+"""++_GENERATE = """+def generate(self, prompts, **kwargs):+ self.sync_weights()+ return self.llm.generate(prompts)+"""+++class FakeVllmModel:+ def __init__(self):+ self.calls = []++ def load_weights(self, weights):+ self.calls.append(list(weights))+++class FakeEngine:+ shared_weights = False++ def __init__(self, *args, **kwargs):+ self.model = FakeVllmModel()+ runner = types.SimpleNamespace(model = self.model)+ worker = types.SimpleNamespace(model_runner = runner)+ self.llm_engine = types.SimpleNamespace(+ model_executor = types.SimpleNamespace(driver_worker = worker)+ )+ self.resets = 0++ def reset_prefix_cache(self):+ self.resets += 1++ def generate(self, *args, **kwargs):+ return ["generated"]+++def _rl_replacements():+ from unsloth.models import rl_replacements+ return rl_replacements+++def _build_fake_trl(+ monkeypatch,+ sync_src = _SYNC_WEIGHTS,+ version = "1.15.0",+):+ class_src = (+ "class VLLMGeneration:\n"+ + "\n".join(+ textwrap.indent(textwrap.dedent(src), " ")+ for src in (_INIT_VLLM, sync_src, _GENERATE)+ )+ + "\n"+ )+ filename = f"<fake_trl_{version}_vllm_generation_sync>"+ monkeypatch.setitem(+ linecache.cache,+ filename,+ (len(class_src), None, [line + "\n" for line in class_src.splitlines()], filename),+ )+ vllm_generation = types.ModuleType("trl.generation.vllm_generation")+ vllm_generation.__file__ = filename+ vllm_generation.__spec__ = importlib.machinery.ModuleSpec(+ name = "trl.generation.vllm_generation", loader = None, origin = filename+ )+ vllm_generation.LLM = FakeEngine+ exec(compile(class_src, filename, "exec"), vllm_generation.__dict__)++ generation = types.ModuleType("trl.generation")+ generation.__spec__ = importlib.machinery.ModuleSpec(+ name = "trl.generation", loader = None, origin = "<fake trl>"+ )+ generation.vllm_generation = vllm_generation+ trl = types.ModuleType("trl")+ trl.__spec__ = importlib.machinery.ModuleSpec(name = "trl", loader = None, origin = "<fake trl>")+ trl.__spec__.submodule_search_locations = []+ trl.__path__ = []+ trl.__version__ = version+ trl.generation = generation+ monkeypatch.setitem(sys.modules, "trl", trl)+ monkeypatch.setitem(sys.modules, "trl.generation", generation)+ monkeypatch.setitem(sys.modules, "trl.generation.vllm_generation", vllm_generation)+ monkeypatch.setattr(+ _rl_replacements(), "importlib_version", lambda name: version if name == "trl" else "0"+ )+ return vllm_generation.VLLMGeneration+++def _params():+ # 4 MiB + 4 MiB + 1 MiB + 4 MiB of float32.+ return [+ ("a.weight", torch.zeros(1024, 1024)),+ ("b.weight", torch.zeros(1024, 1024)),+ ("c.bias", torch.zeros(256, 1024)),+ ("d.weight", torch.zeros(1024, 1024)),+ ]+++def _make_generation(+ cls,+ params,+ deepspeed = None,+ fsdp = None,+ peft = False,+):+ self = cls.__new__(cls)+ self.mode = "colocate"+ self.llm = FakeEngine()+ self.accelerator = types.SimpleNamespace(+ state = types.SimpleNamespace(deepspeed_plugin = deepspeed, fsdp_plugin = fsdp)+ )+ self.model = types.SimpleNamespace(peft_config = {"default": object()} if peft else None)+ self._iter_named_params = lambda: iter(params)+ return self+++def _sent(self):+ return [pair for call in self.llm.model.calls for pair in call]+++def _assert_same_tensors(sent, params):+ assert [n for n, _ in sent] == [n for n, _ in params]+ assert all(+ s is p for (_, s), (_, p) in zip(sent, params)+ ), "tensors must be passed by reference"+++def test_full_finetune_sync_is_one_load_weights_call(monkeypatch):+ monkeypatch.delenv("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", raising = False)+ cls = _build_fake_trl(monkeypatch)+ _rl_replacements().vllm_generation_init_patch()+ params = _params()+ self = _make_generation(cls, params)+ cls.sync_weights(self)+ assert (+ len(self.llm.model.calls) == 1+ ), f"expected one load_weights call, got {len(self.llm.model.calls)}"+ _assert_same_tensors(_sent(self), params)+ assert self.llm.resets == 1+ assert self._unsloth_bucketed_weight_syncs == 1+++def test_bucket_size_caps_each_call(monkeypatch):+ monkeypatch.setenv("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", "8")+ cls = _build_fake_trl(monkeypatch)+ _rl_replacements().vllm_generation_init_patch()+ params = _params()+ self = _make_generation(cls, params)+ cls.sync_weights(self)+ assert [len(c) for c in self.llm.model.calls] == [2, 2]+ _assert_same_tensors(_sent(self), params)+++ "kwargs",+ [+ dict(env = "0"),+ dict(deepspeed = object()),+ dict(fsdp = object()),+ dict(peft = True),+ ],+ ids = ["kill_switch", "deepspeed", "fsdp", "peft"],+)+def test_per_tensor_when_disabled_or_unsafe(monkeypatch, kwargs):+ env = kwargs.pop("env", None)+ if env is None:+ monkeypatch.delenv("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", env)+ cls = _build_fake_trl(monkeypatch)+ _rl_replacements().vllm_generation_init_patch()+ params = _params()+ self = _make_generation(cls, params, **kwargs)+ cls.sync_weights(self)+ assert [len(c) for c in self.llm.model.calls] == [1, 1, 1, 1]+ _assert_same_tensors(_sent(self), params)+ assert not getattr(self, "_unsloth_bucketed_weight_syncs", 0)+++def test_other_sync_layouts_are_left_alone(monkeypatch):+ sync_src = """+def sync_weights(self):+ for name, param in self.model.named_parameters():+ self.llm.llm_engine.model_executor.driver_worker.model_runner.model.load_weights([(name, param.data)])+"""+ cls = _build_fake_trl(monkeypatch, sync_src = sync_src)+ _rl_replacements().vllm_generation_init_patch()+ params = _params()+ self = _make_generation(cls, params)+ self.model = types.SimpleNamespace(named_parameters = lambda: iter(params), peft_config = None)+ cls.sync_weights(self)+ assert [len(c) for c in self.llm.model.calls] == [1, 1, 1, 1]diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 1f99f27516f..679f27e1c07 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -4160,6 +4160,49 @@ def openenv_vllm_reload_weights():RL_ADDITIONAL_FUNCTIONS["openenv"].append(openenv_vllm_reload_weights)+def _vllm_weight_sync_bucket_bytes():+ raw = os.environ.get("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", "1024")+ try:+ mb = float(raw)+ except ValueError:+ mb = 1024.0+ return int(mb * 2**20) if mb > 0 else 0+++def _unsloth_vllm_sync_colocate(generation, vllm_model):+ """TRL's colocate sync calls load_weights once per tensor, and every call rebuilds vLLM's+ parameter lookup (~0.4 ms each). Hand over the same (name, tensor) references in a few calls:+ no copies, and each weight_loader runs on the same data, so vLLM's weights are bit-identical.+ UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB = 0 restores the per-tensor loop."""+ weights = generation._iter_named_params()+ cap = _vllm_weight_sync_bucket_bytes()+ state = getattr(getattr(generation, "accelerator", None), "state", None)+ # ZeRO-3 / FSDP yield gathered tensors that are only valid until the next step of the+ # iterator, and PEFT unmerges in place once it is exhausted: holding references is wrong there.+ unsafe = (+ state is None+ or getattr(state, "deepspeed_plugin", None) is not None+ or getattr(state, "fsdp_plugin", None) is not None+ or getattr(generation.model, "peft_config", None) is not None+ )+ if cap == 0 or unsafe:+ for name, param in weights:+ vllm_model.load_weights([(name, param)])+ return+ bucket, size = [], 0+ for name, param in weights:+ bucket.append((name, param))+ size += param.numel() * param.element_size()+ if size >= cap:+ vllm_model.load_weights(bucket)+ bucket, size = [], 0+ if bucket:+ vllm_model.load_weights(bucket)+ generation._unsloth_bucketed_weight_syncs = (+ getattr(generation, "_unsloth_bucketed_weight_syncs", 0) + 1+ )++def vllm_generation_init_patch():# trl moved vllm code to trl/generation/vllm_generation.py (commit 0eb66d8, 0.28.0+), which must be patched so it does not build a second vLLM instance when fast_inference has one.@@ -4263,6 +4306,23 @@ def vllm_generation_init_patch():raise RuntimeError("Unsloth: Warning - regex did not match, VLLMGeneration.sync_weights patch may have failed")+ # TRL >= 1.10 colocate loop; older layouts stay per-tensor.+ colocate_loop = re.compile(+ r"^(?P<indent>[ \t]*)for name, param in self\._iter_named_params\(\):\n"+ r"(?P=indent)[ \t]+self\.llm\.llm_engine\.model_executor\.driver_worker\.model_runner\.model"+ r"\.load_weights\(\[\(name, param\)\]\)\n",+ re.MULTILINE,+ )+ patched_src, n_loops = colocate_loop.subn(+ lambda m: (+ f"{m.group('indent')}_unsloth_vllm_sync_colocate(self, "+ "self.llm.llm_engine.model_executor.driver_worker.model_runner.model)\n"+ ),+ patched_src,+ count = 1,+ )+ if n_loops:+ vllm_generation._unsloth_vllm_sync_colocate = _unsloth_vllm_sync_colocatereturn patched_src# `generate` is deliberately NOT source-patched: its two anchors drifted (TRL >= 1.x deleted the collective_rpc("reload_weights") call for self.sync_weights(), so the anchor matched 0 times, raised, and took the lora injection with it, making GRPO rollouts come from the BASE model), and the generate regex is not paren-balanced and mis-edits multi-line calls. Intercept on the vLLM engine instead of TRL's method body: self.llm is the vLLM LLM object in colocate mode in every release with VLLMGeneration, and generate / chat / collective_rpc are public stable APIs checked across vLLM 0.11.0-0.27.1. The override is scoped to one VLLMGeneration.generate call and undone in a finally.