Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Sync full finetune weights into colocated vLLM in a few load_weights calls (#13901)

Daniel Han6c229ce09992
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
index 5bc577c26dc..c5c5278eb78 100644
--- a/.github/workflows/version-compat-ci.yml
+++ b/.github/workflows/version-compat-ci.yml
@@ -99,6 +99,7 @@ jobs:
tests/version_compat/test_bitsandbytes_pinned_symbols.py \
tests/version_compat/test_transformers_pinned_symbols.py \
tests/version_compat/test_no_dtype_kwarg_at_load.py \
+ tests/version_compat/test_trl_vllm_colocate_weight_sync.py \
-n 4 --dist loadfile -v --tb=short
cap-site-consistency:
diff --git a/tests/version_compat/test_trl_vllm_colocate_weight_sync.py b/tests/version_compat/test_trl_vllm_colocate_weight_sync.py
new file mode 100644
index 00000000000..19ed9aaf18f
--- /dev/null
+++ b/tests/version_compat/test_trl_vllm_colocate_weight_sync.py
@@ -0,0 +1,229 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""Full-finetune colocate weight sync: few load_weights calls, same tensors, same order."""
+
+from __future__ import annotations
+
+import importlib.machinery
+import importlib.util
+import linecache
+import sys
+import textwrap
+import types
+
+import pytest
+
+
+if importlib.util.find_spec("torch") is None:
+ pytest.skip("torch not installed; this test drives the real patch", allow_module_level = True)
+
+import torch
+
+
+_INIT_VLLM = """
+def _init_vllm(self, model):
+ if self.mode == "colocate":
+ self.llm = LLM(
+ model = model.name_or_path,
+ )
+"""
+
+# TRL >= 1.10 VLLMGeneration.sync_weights, trimmed to the parts the patch reads.
+_SYNC_WEIGHTS = """
+def sync_weights(self):
+ if self.mode == "colocate":
+ for name, param in self._iter_named_params():
+ self.llm.llm_engine.model_executor.driver_worker.model_runner.model.load_weights([(name, param)])
+ self.llm.reset_prefix_cache()
+"""
+
+_GENERATE = """
+def generate(self, prompts, **kwargs):
+ self.sync_weights()
+ return self.llm.generate(prompts)
+"""
+
+
+class FakeVllmModel:
+ def __init__(self):
+ self.calls = []
+
+ def load_weights(self, weights):
+ self.calls.append(list(weights))
+
+
+class FakeEngine:
+ shared_weights = False
+
+ def __init__(self, *args, **kwargs):
+ self.model = FakeVllmModel()
+ runner = types.SimpleNamespace(model = self.model)
+ worker = types.SimpleNamespace(model_runner = runner)
+ self.llm_engine = types.SimpleNamespace(
+ model_executor = types.SimpleNamespace(driver_worker = worker)
+ )
+ self.resets = 0
+
+ def reset_prefix_cache(self):
+ self.resets += 1
+
+ def generate(self, *args, **kwargs):
+ return ["generated"]
+
+
+def _rl_replacements():
+ from unsloth.models import rl_replacements
+ return rl_replacements
+
+
+def _build_fake_trl(
+ monkeypatch,
+ sync_src = _SYNC_WEIGHTS,
+ version = "1.15.0",
+):
+ class_src = (
+ "class VLLMGeneration:\n"
+ + "\n".join(
+ textwrap.indent(textwrap.dedent(src), " ")
+ for src in (_INIT_VLLM, sync_src, _GENERATE)
+ )
+ + "\n"
+ )
+ filename = f"<fake_trl_{version}_vllm_generation_sync>"
+ monkeypatch.setitem(
+ linecache.cache,
+ filename,
+ (len(class_src), None, [line + "\n" for line in class_src.splitlines()], filename),
+ )
+ vllm_generation = types.ModuleType("trl.generation.vllm_generation")
+ vllm_generation.__file__ = filename
+ vllm_generation.__spec__ = importlib.machinery.ModuleSpec(
+ name = "trl.generation.vllm_generation", loader = None, origin = filename
+ )
+ vllm_generation.LLM = FakeEngine
+ exec(compile(class_src, filename, "exec"), vllm_generation.__dict__)
+
+ generation = types.ModuleType("trl.generation")
+ generation.__spec__ = importlib.machinery.ModuleSpec(
+ name = "trl.generation", loader = None, origin = "<fake trl>"
+ )
+ generation.vllm_generation = vllm_generation
+ trl = types.ModuleType("trl")
+ trl.__spec__ = importlib.machinery.ModuleSpec(name = "trl", loader = None, origin = "<fake trl>")
+ trl.__spec__.submodule_search_locations = []
+ trl.__path__ = []
+ trl.__version__ = version
+ trl.generation = generation
+ monkeypatch.setitem(sys.modules, "trl", trl)
+ monkeypatch.setitem(sys.modules, "trl.generation", generation)
+ monkeypatch.setitem(sys.modules, "trl.generation.vllm_generation", vllm_generation)
+ monkeypatch.setattr(
+ _rl_replacements(), "importlib_version", lambda name: version if name == "trl" else "0"
+ )
+ return vllm_generation.VLLMGeneration
+
+
+def _params():
+ # 4 MiB + 4 MiB + 1 MiB + 4 MiB of float32.
+ return [
+ ("a.weight", torch.zeros(1024, 1024)),
+ ("b.weight", torch.zeros(1024, 1024)),
+ ("c.bias", torch.zeros(256, 1024)),
+ ("d.weight", torch.zeros(1024, 1024)),
+ ]
+
+
+def _make_generation(
+ cls,
+ params,
+ deepspeed = None,
+ fsdp = None,
+ peft = False,
+):
+ self = cls.__new__(cls)
+ self.mode = "colocate"
+ self.llm = FakeEngine()
+ self.accelerator = types.SimpleNamespace(
+ state = types.SimpleNamespace(deepspeed_plugin = deepspeed, fsdp_plugin = fsdp)
+ )
+ self.model = types.SimpleNamespace(peft_config = {"default": object()} if peft else None)
+ self._iter_named_params = lambda: iter(params)
+ return self
+
+
+def _sent(self):
+ return [pair for call in self.llm.model.calls for pair in call]
+
+
+def _assert_same_tensors(sent, params):
+ assert [n for n, _ in sent] == [n for n, _ in params]
+ assert all(
+ s is p for (_, s), (_, p) in zip(sent, params)
+ ), "tensors must be passed by reference"
+
+
+def test_full_finetune_sync_is_one_load_weights_call(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", raising = False)
+ cls = _build_fake_trl(monkeypatch)
+ _rl_replacements().vllm_generation_init_patch()
+ params = _params()
+ self = _make_generation(cls, params)
+ cls.sync_weights(self)
+ assert (
+ len(self.llm.model.calls) == 1
+ ), f"expected one load_weights call, got {len(self.llm.model.calls)}"
+ _assert_same_tensors(_sent(self), params)
+ assert self.llm.resets == 1
+ assert self._unsloth_bucketed_weight_syncs == 1
+
+
+def test_bucket_size_caps_each_call(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", "8")
+ cls = _build_fake_trl(monkeypatch)
+ _rl_replacements().vllm_generation_init_patch()
+ params = _params()
+ self = _make_generation(cls, params)
+ cls.sync_weights(self)
+ assert [len(c) for c in self.llm.model.calls] == [2, 2]
+ _assert_same_tensors(_sent(self), params)
+
+
+ "kwargs",
+ [
+ dict(env = "0"),
+ dict(deepspeed = object()),
+ dict(fsdp = object()),
+ dict(peft = True),
+ ],
+ ids = ["kill_switch", "deepspeed", "fsdp", "peft"],
+)
+def test_per_tensor_when_disabled_or_unsafe(monkeypatch, kwargs):
+ env = kwargs.pop("env", None)
+ if env is None:
+ monkeypatch.delenv("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", env)
+ cls = _build_fake_trl(monkeypatch)
+ _rl_replacements().vllm_generation_init_patch()
+ params = _params()
+ self = _make_generation(cls, params, **kwargs)
+ cls.sync_weights(self)
+ assert [len(c) for c in self.llm.model.calls] == [1, 1, 1, 1]
+ _assert_same_tensors(_sent(self), params)
+ assert not getattr(self, "_unsloth_bucketed_weight_syncs", 0)
+
+
+def test_other_sync_layouts_are_left_alone(monkeypatch):
+ sync_src = """
+def sync_weights(self):
+ for name, param in self.model.named_parameters():
+ self.llm.llm_engine.model_executor.driver_worker.model_runner.model.load_weights([(name, param.data)])
+"""
+ cls = _build_fake_trl(monkeypatch, sync_src = sync_src)
+ _rl_replacements().vllm_generation_init_patch()
+ params = _params()
+ self = _make_generation(cls, params)
+ self.model = types.SimpleNamespace(named_parameters = lambda: iter(params), peft_config = None)
+ cls.sync_weights(self)
+ assert [len(c) for c in self.llm.model.calls] == [1, 1, 1, 1]
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 1f99f27516f..679f27e1c07 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -4160,6 +4160,49 @@ def openenv_vllm_reload_weights():
RL_ADDITIONAL_FUNCTIONS["openenv"].append(openenv_vllm_reload_weights)
+def _vllm_weight_sync_bucket_bytes():
+ raw = os.environ.get("UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB", "1024")
+ try:
+ mb = float(raw)
+ except ValueError:
+ mb = 1024.0
+ return int(mb * 2**20) if mb > 0 else 0
+
+
+def _unsloth_vllm_sync_colocate(generation, vllm_model):
+ """TRL's colocate sync calls load_weights once per tensor, and every call rebuilds vLLM's
+ parameter lookup (~0.4 ms each). Hand over the same (name, tensor) references in a few calls:
+ no copies, and each weight_loader runs on the same data, so vLLM's weights are bit-identical.
+ UNSLOTH_VLLM_WEIGHT_SYNC_BUCKET_MB = 0 restores the per-tensor loop."""
+ weights = generation._iter_named_params()
+ cap = _vllm_weight_sync_bucket_bytes()
+ state = getattr(getattr(generation, "accelerator", None), "state", None)
+ # ZeRO-3 / FSDP yield gathered tensors that are only valid until the next step of the
+ # iterator, and PEFT unmerges in place once it is exhausted: holding references is wrong there.
+ unsafe = (
+ state is None
+ or getattr(state, "deepspeed_plugin", None) is not None
+ or getattr(state, "fsdp_plugin", None) is not None
+ or getattr(generation.model, "peft_config", None) is not None
+ )
+ if cap == 0 or unsafe:
+ for name, param in weights:
+ vllm_model.load_weights([(name, param)])
+ return
+ bucket, size = [], 0
+ for name, param in weights:
+ bucket.append((name, param))
+ size += param.numel() * param.element_size()
+ if size >= cap:
+ vllm_model.load_weights(bucket)
+ bucket, size = [], 0
+ if bucket:
+ vllm_model.load_weights(bucket)
+ generation._unsloth_bucketed_weight_syncs = (
+ getattr(generation, "_unsloth_bucketed_weight_syncs", 0) + 1
+ )
+
+
def vllm_generation_init_patch():
# trl moved vllm code to trl/generation/vllm_generation.py (commit 0eb66d8, 0.28.0+), which must be patched so it does not build a second vLLM instance when fast_inference has one.
@@ -4263,6 +4306,23 @@ def vllm_generation_init_patch():
raise RuntimeError(
"Unsloth: Warning - regex did not match, VLLMGeneration.sync_weights patch may have failed"
)
+ # TRL >= 1.10 colocate loop; older layouts stay per-tensor.
+ colocate_loop = re.compile(
+ r"^(?P<indent>[ \t]*)for name, param in self\._iter_named_params\(\):\n"
+ r"(?P=indent)[ \t]+self\.llm\.llm_engine\.model_executor\.driver_worker\.model_runner\.model"
+ r"\.load_weights\(\[\(name, param\)\]\)\n",
+ re.MULTILINE,
+ )
+ patched_src, n_loops = colocate_loop.subn(
+ lambda m: (
+ f"{m.group('indent')}_unsloth_vllm_sync_colocate(self, "
+ "self.llm.llm_engine.model_executor.driver_worker.model_runner.model)\n"
+ ),
+ patched_src,
+ count = 1,
+ )
+ if n_loops:
+ vllm_generation._unsloth_vllm_sync_colocate = _unsloth_vllm_sync_colocate
return patched_src
# `generate` is deliberately NOT source-patched: its two anchors drifted (TRL >= 1.x deleted the collective_rpc("reload_weights") call for self.sync_weights(), so the anchor matched 0 times, raised, and took the lora injection with it, making GRPO rollouts come from the BASE model), and the generate regex is not paren-balanced and mis-edits multi-line calls. Intercept on the vLLM engine instead of TRL's method body: self.llm is the vLLM LLM object in colocate mode in every release with VLLMGeneration, and generate / chat / collective_rpc are public stable APIs checked across vLLM 0.11.0-0.27.1. The override is scoped to one VLLMGeneration.generate call and undone in a finally.