Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO routing replay: prime router formulas before compile and score the PEFT reference without replay (#14756)
Daniel Han956c4ef762dc
diff --git a/tests/test_grpo_routing_replay_trainer_hooks.py b/tests/test_grpo_routing_replay_trainer_hooks.pynew file mode 100644index 00000000000..640075842c1--- /dev/null+++ b/tests/test_grpo_routing_replay_trainer_hooks.py@@ -0,0 +1,277 @@+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""GRPO trainer installation of MoE routing replay (UNSLOTH_GRPO_ROUTING_REPLAY=1), CPU only.++Runs the exact post-init code rl.py writes into the compiled GRPO trainer: a cold router must+compile with fullgraph=True, and PEFT reference scoring (disable_adapter) must route naturally."""++import ast+import os+import pathlib+from types import SimpleNamespace++import numpy as np+import pytest+import torch++transformers = pytest.importorskip("transformers")+try:+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM+ from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeTopKRouter # noqa: F401+except Exception:+ pytest.skip("needs transformers >= 5 router modules", allow_module_level = True)+rr = pytest.importorskip("unsloth_zoo.routing_replay")++RL_PY = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl.py"+LAYERS, EXPERTS, TOP_K, PAD = 2, 8, 2, 0+++def _post_init_code():+ # The string constant rl.py appends to RLTrainer_post for grpo_trainer.+ for node in ast.walk(ast.parse(RL_PY.read_text(encoding = "utf-8"))):+ if isinstance(node, ast.Constant) and isinstance(node.value, str) \+ and "UNSLOTH_GRPO_ROUTING_REPLAY', '0') == '1' and hasattr(model, 'vllm_engine')" in node.value:+ return node.value+ raise AssertionError("routing replay post-init code not found in rl.py")+++def _run_post_init(model, trainer_model = None):+ trainer = SimpleNamespace(model = trainer_model if trainer_model is not None else model,+ processing_class = SimpleNamespace(pad_token_id = PAD))+ exec(_post_init_code(), {"os": os, "self": trainer, "model": model})+++def replay(monkeypatch):+ monkeypatch.setenv(rr.ROUTING_REPLAY_ENV, "1")+ rr.reset_routing_replay()+ torch._dynamo.reset()+ yield+ rr.reset_routing_replay()+ torch._dynamo.reset()+++def _moe(seed = 0):+ torch.manual_seed(seed)+ config = Qwen3MoeConfig(+ vocab_size = 64, hidden_size = 32, intermediate_size = 64, moe_intermediate_size = 16,+ num_hidden_layers = LAYERS, num_attention_heads = 4, num_key_value_heads = 2, head_dim = 8,+ num_experts = EXPERTS, num_experts_per_tok = TOP_K, norm_topk_prob = True, pad_token_id = PAD,+ )+ model = Qwen3MoeForCausalLM(config).float().eval()+ for layer in model.model.layers:+ layer.mlp.gate.weight.data.normal_(0, 0.5)+ model.vllm_engine = None+ return model+++def _natural(model, ids):+ captured = {}+ handles = [+ layer.mlp.gate.register_forward_hook(lambda m, a, out, i = i: captured.__setitem__(i, out[2].clone()))+ for i, layer in enumerate(model.model.layers)+ ]+ with torch.no_grad():+ logits = model(input_ids = torch.tensor([ids])).logits+ for h in handles:+ h.remove()+ return logits, torch.stack([captured[i] for i in range(LAYERS)], dim = 1)+++def test_cold_router_compiles_fullgraph(replay):+ model = _moe()+ _run_post_init(model)+ router = model.model.layers[0].mlp.gate+ assert router._forward_hooks, "routing replay hooks were not installed"+ x = torch.randn(5, 32, generator = torch.Generator().manual_seed(1))+ with torch.no_grad():+ natural = router.forward(x)[2]+ forced = (natural + 3) % EXPERTS+ rr._STATE.in_forward, rr._STATE.record = True, None+ rr._STATE.current = forced[:, None, :].expand(-1, LAYERS, -1).contiguous()+ try:+ compiled = torch.compile(router, backend = "eager", fullgraph = True)+ logits, weights, ids = compiled(x.requires_grad_(True)) # first call: no eager warm-up+ weights.square().sum().backward()+ finally:+ rr._STATE.in_forward, rr._STATE.current = False, None+ assert torch.equal(ids.long(), forced.long()) # the replay ran inside the compiled graph+ expected = torch.softmax(logits.detach().float().gather(-1, forced.long()), -1)+ torch.testing.assert_close(weights.detach().float(), expected, rtol = 1e-5, atol = 1e-6)+ assert x.grad is not None and bool(x.grad.abs().sum() > 0)+++def test_peft_reference_scoring_routes_naturally(replay):+ peft = pytest.importorskip("peft")+ base = _moe()+ ids = [5, 6, 7, 30, 31, 32]+ natural_logits, natural_ids = _natural(base, ids)+ model = peft.get_peft_model(base, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))+ model.vllm_engine = None+ _run_post_init(model)+ routers = [layer.mlp.gate for layer in model.base_model.model.model.layers]+ hooks_before = [(dict(r._forward_hooks), dict(r._forward_pre_hooks)) for r in routers]+ assert all(h for h, _ in hooks_before)++ forced = ((natural_ids + 3) % EXPERTS).numpy().astype(np.int16)+ rr.record_routed_experts([SimpleNamespace(+ prompt_token_ids = ids[:3], outputs = [SimpleNamespace(token_ids = ids[3:], routed_experts = forced[:-1])],+ )], pad_token_id = PAD)++ with torch.no_grad():+ policy = model(input_ids = torch.tensor([ids])).logits+ # LoRA B starts at zero: any difference from the base comes from the replayed routes.+ assert not torch.allclose(policy, natural_logits, atol = 1e-4)+ with model.disable_adapter():+ reference = model(input_ids = torch.tensor([ids])).logits+ assert all(not r._forward_hooks for r in routers)+ torch.testing.assert_close(reference, natural_logits, rtol = 1e-5, atol = 1e-6)+ again = model(input_ids = torch.tensor([ids])).logits+ torch.testing.assert_close(again, policy, rtol = 1e-5, atol = 1e-6)+ assert [(dict(r._forward_hooks), dict(r._forward_pre_hooks)) for r in routers] == hooks_before+++def test_no_replay_installs_nothing(monkeypatch):+ monkeypatch.delenv(rr.ROUTING_REPLAY_ENV, raising = False)+ peft = pytest.importorskip("peft")+ model = peft.get_peft_model(_moe(), peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))+ model.vllm_engine = None+ disable_adapter = model.disable_adapter+ _run_post_init(model)+ assert model.disable_adapter == disable_adapter+ assert all(not m._forward_hooks and not m._forward_pre_hooks for m in model.modules())+++def test_reference_pass_restores_hook_order_with_prepended_hooks(replay):+ # A hook registered after installation with prepend=True has a newer id but runs first: the+ # reference pass must leave every hook table in exactly the order it found it.+ peft = pytest.importorskip("peft")+ model = peft.get_peft_model(_moe(), peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))+ model.vllm_engine = None+ _run_post_init(model)+ decoder = model.base_model.model.model+ router = decoder.layers[0].mlp.gate+ calls = []+ router.register_forward_hook(lambda m, a, out: calls.append("prepended"), prepend = True)+ router.register_forward_hook(lambda m, a, out: calls.append("appended"))+ decoder.register_forward_pre_hook(lambda m, a, k: None, prepend = True, with_kwargs = True)+ decoder.layers[0].register_forward_pre_hook(lambda m, a: None, prepend = True)+ tables = ("_forward_hooks", "_forward_hooks_with_kwargs", "_forward_hooks_always_called",+ "_forward_pre_hooks", "_forward_pre_hooks_with_kwargs")++ def orders():+ return [[list(getattr(m, t, {}).items()) for t in tables] for m in model.modules()]++ before = orders()+ ids = torch.tensor([[5, 6, 7, 30]])+ with torch.no_grad():+ with model.disable_adapter():+ model(input_ids = ids)+ assert orders() == before+ calls.clear()+ model(input_ids = ids)+ assert calls == ["prepended", "appended"]+++def _bias_router_model(model_type):+ from transformers import AutoConfig, AutoModelForCausalLM++ extra = {+ "minimax_m2": dict(num_local_experts = EXPERTS),+ "hy_v3": dict(num_local_experts = EXPERTS),+ "lfm2_moe": dict(num_experts = EXPERTS, use_expert_bias = True, num_dense_layers = 0,+ layer_types = ["full_attention"] * LAYERS),+ }[model_type]+ try:+ config = AutoConfig.for_model(+ model_type, vocab_size = 64, hidden_size = 32, intermediate_size = 64, moe_intermediate_size = 16,+ num_hidden_layers = LAYERS, num_attention_heads = 4, num_key_value_heads = 2, head_dim = 8,+ num_experts_per_tok = TOP_K, pad_token_id = PAD, **extra,+ )+ except Exception as error:+ pytest.skip(f"this transformers has no {model_type}: {error}")+ torch.manual_seed(0)+ model = AutoModelForCausalLM.from_config(config).float().eval()+ model.vllm_engine = None+ return model++[email protected]("model_type", ["minimax_m2", "hy_v3", "lfm2_moe"])+def test_sigmoid_correction_bias_routers_route_naturally_with_a_warning(replay, model_type):+ # MiniMax-M2 / HY-V3: TopKRouter.forward(hidden_states, e_score_correction_bias), the bias a+ # buffer of the parent MoE block; LFM2-MoE: forward(hidden_states, expert_bias = None). All three+ # take sigmoid(logits) top-k weights, which no zoo replay formula reproduces: priming must reach+ # the router (no TypeError), then take the hooks off and say so.+ model = _bias_router_model(model_type)+ routers = [m for m in model.modules() if "TopKRouter" in type(m).__name__]+ if not routers:+ pytest.skip(f"{model_type} has no TopKRouter module in this transformers (gate inside the MoE block)")+ ids = [5, 6, 7, 30, 31, 32]+ with torch.no_grad():+ natural = model(input_ids = torch.tensor([ids])).logits+ name = type(routers[0]).__name__+ with pytest.warns(UserWarning, match = rf"routing replay is off for {len(routers)} router.*{name} \(routing replay has no weight formula"):+ _run_post_init(model)+ assert all(not r._forward_hooks for r in routers)+ # A recorded rollout with other experts no longer changes anything.+ layers = model.config.num_hidden_layers+ forced = np.full((len(ids) - 1, layers, TOP_K), 0, dtype = np.int16)+ forced[..., 1] = 1+ rr.record_routed_experts([SimpleNamespace(+ prompt_token_ids = ids[:3], outputs = [SimpleNamespace(token_ids = ids[3:], routed_experts = forced)],+ )], pad_token_id = PAD)+ with torch.no_grad():+ torch.testing.assert_close(model(input_ids = torch.tensor([ids])).logits, natural)+++def test_formula_that_only_fits_the_priming_input_is_rejected(replay):+ # A sigmoid router with tiny logits: on the priming input every weight is ~1 / k, so the zoo's+ # softmax_renorm fits there; on spread logits it does not reproduce the router's weights.+ import torch.nn.functional as F++ model = _moe()+ original = type(model.model.layers[0].mlp.gate)++ class SigmoidTopKRouter(original):+ def forward(self, hidden_states):+ hidden_states = hidden_states.reshape(-1, self.hidden_dim)+ logits = F.linear(hidden_states, self.weight)+ weights, ids = logits.float().sigmoid().topk(self.top_k, dim = -1)+ return logits, (weights / weights.sum(-1, keepdim = True)).to(logits.dtype), ids++ gate = model.model.layers[0].mlp.gate+ gate.__class__ = SigmoidTopKRouter+ gate.weight.data.normal_(0, 1e-4)+ with pytest.warns(UserWarning, match = r"SigmoidTopKRouter \(the softmax_renorm weight formula does not reproduce"):+ _run_post_init(model)+ assert not gate._forward_hooks+ assert model.model.layers[1].mlp.gate._forward_hooks # the real softmax router passes the check+++def test_router_that_cannot_be_primed_loses_its_hook_with_a_warning(replay):+ model = _moe()+ original = type(model.model.layers[0].mlp.gate)++ class OddTopKRouter(original):+ # A required argument the parent block does not hold: priming cannot build the call.+ def forward(self, hidden_states, routing_temperature):+ return super().forward(hidden_states / routing_temperature)++ model.model.layers[0].mlp.gate.__class__ = OddTopKRouter+ with pytest.warns(UserWarning, match = r"routing replay is off for 1 router.*OddTopKRouter"):+ _run_post_init(model)+ assert not model.model.layers[0].mlp.gate._forward_hooks+ assert model.model.layers[1].mlp.gate._forward_hooks+ assert getattr(model.model.layers[1].mlp.gate, "_unsloth_replay_formula", None) is not Nonediff --git a/unsloth/models/_routing_replay.py b/unsloth/models/_routing_replay.pynew file mode 100644index 00000000000..03574176813--- /dev/null+++ b/unsloth/models/_routing_replay.py@@ -0,0 +1,247 @@+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""GRPO trainer side of unsloth_zoo's MoE routing replay (UNSLOTH_GRPO_ROUTING_REPLAY=1).++Only used when routing replay is requested. Two things the zoo hooks need from the trainer:+- their routing weight formula is picked from the first replayed router call, which reads tensor+ values; done inside a compiled region that is a graph break (fullgraph fails), so it is picked+ here once, eagerly, right after the hooks are installed;+- PEFT reference scoring (`disable_adapter()`) must see the base model's own routing, as a separate+ reference model does, so the hooks are detached inside that context.+"""++import contextlib+import inspect+import warnings++import torch++__all__ = ["routing_replay_hook_snapshot", "prepare_routing_replay"]++_HOOK_DICTS = (+ ("_forward_hooks", ("_forward_hooks_with_kwargs", "_forward_hooks_always_called")),+ ("_forward_pre_hooks", ("_forward_pre_hooks_with_kwargs",)),+)+++def routing_replay_hook_snapshot(model):+ """Hook ids on every module of `model`, taken before install_routing_replay."""+ return {+ id(module): tuple(set(getattr(module, name, {})) for name, _ in _HOOK_DICTS)+ for module in model.modules()+ }+++def _new_hooks(model, before):+ hooks = []+ for module in model.modules():+ old = before.get(id(module), (set(), set()))+ for (name, extras), seen in zip(_HOOK_DICTS, old):+ for hook_id in getattr(module, name, {}):+ if hook_id not in seen:+ hooks.append((module, name, extras, hook_id))+ return hooks+++def _detached(hooks):+ # Hook order is call order, and ids do not give it (register_*_hook(prepend = True) puts a newer+ # id first): keep each table's own key order and put it back exactly.+ orders = {}+ removed = []+ for module, name, extras, hook_id in hooks:+ for table in [getattr(module, name, None)] + [getattr(module, e, None) for e in extras]:+ if table is None or hook_id not in table:+ continue+ orders.setdefault(id(table), (table, list(table.keys())))+ removed.append((table, hook_id, table.pop(hook_id)))+ try:+ yield+ finally:+ for table, hook_id, value in removed:+ table[hook_id] = value+ for table, order in orders.values():+ known = set(order)+ added = [k for k in table if k not in known] # registered inside the context: keep last+ items = dict(table)+ table.clear()+ for k in order + added:+ if k in items:+ table[k] = items[k]+++def _router_extra_args(module, parent):+ # Routers such as MiniMax-M2 / HY-V3 (`forward(hidden_states, e_score_correction_bias)`) or+ # LFM2-MoE (`forward(hidden_states, expert_bias = None)`) take the parent MoE block's buffer:+ # the block calls `self.gate(hidden_states, self.e_score_correction_bias)`. Pass the parent's+ # same-named tensor; None when a required argument has none.+ try:+ params = list(inspect.signature(module.forward).parameters.values())[1:]+ except (TypeError, ValueError):+ return {}+ extra = {}+ for param in params:+ if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):+ continue+ value = getattr(parent, param.name, None) if parent is not None else None+ if torch.is_tensor(value):+ extra[param.name] = value+ elif param.default is param.empty:+ return None+ return extra+++def _prime_formula(rr, module, hook, layers, parent = None):+ """Run the zoo hook once, eagerly, on a synthetic router call whose replay is the natural+ routing: it picks and caches module._unsloth_replay_formula exactly as the first real call+ would. Returns None when primed, else why it could not be."""+ weight = getattr(module, "weight", None)+ hidden = getattr(module, "hidden_dim", None) or getattr(module, "in_features", None)+ if hidden is None and torch.is_tensor(weight) and weight.dim() == 2:+ hidden = weight.shape[-1]+ if not isinstance(hidden, int) or not torch.is_tensor(weight) or not weight.is_floating_point():+ return "its hidden size or weight is unknown"+ extra = _router_extra_args(module, parent)+ if extra is None:+ return "its forward needs an argument the parent MoE block does not hold"+ generator = torch.Generator().manual_seed(0)+ x = torch.randn(8, hidden, generator = generator).to(weight.device, weight.dtype)+ state = rr._STATE+ fields = ("current", "in_forward", "record", "pending", "recomputed", "changed")+ old = {f: getattr(state, f) for f in fields}+ old_stats = dict(state.stats)+ if getattr(module, "_unsloth_replay_formula", None) is not None:+ return _check_formula(rr, module, x, extra) # already picked (an earlier install)+ try:+ with torch.no_grad():+ output = module.forward(x, **extra) # no hooks: the zoo hook is called by hand below+ positions = rr._split_router_output(output)+ if positions is None:+ return "its outputs are not (logits, weights, ids)"+ ids = output[positions[2]].reshape(-1, output[positions[2]].shape[-1]).long()+ state.current = ids[:, None, :].expand(-1, layers, -1).contiguous()+ state.in_forward, state.record, state.pending, state.recomputed = True, None, [], False+ hook(module, (x, *extra.values()), output)+ except Exception as error:+ return f"priming raised {type(error).__name__}: {error}"+ finally:+ for f, v in old.items():+ setattr(state, f, v)+ state.stats.clear()+ state.stats.update(old_stats)+ if getattr(module, "_unsloth_replay_formula", None) is None:+ return "the routing replay hook did not pick a weight formula"+ return _check_formula(rr, module, x, extra)+++# The check also runs with the input scaled so the router logits spread to these standard+# deviations: near-zero logits make every formula give ~1 / k, so the priming input alone can let a+# wrong formula through (sigmoid routing matched by softmax).+_CHECK_LOGIT_STDS = (1.0, 4.0)+++def _check_formula(rr, module, x, extra):+ """None when the cached zoo formula reproduces this router's own top-k weights, else why not.+ The zoo stores False when it found none (sigmoid / correction bias routers, MiniMax-M2, HY-V3,+ LFM2-MoE): replay would silently keep natural routing there, so it counts as a failure too."""+ formula = getattr(module, "_unsloth_replay_formula", None)+ if formula is False or formula is None:+ return "routing replay has no weight formula for this router (e.g. sigmoid routing)"+ fn = rr._formulas().get(formula) if hasattr(rr, "_formulas") else None+ if fn is None:+ return f"unknown routing replay weight formula {formula!r}"+ with torch.no_grad():+ output = module.forward(x, **extra)+ positions = rr._split_router_output(output)+ if positions is None:+ return "its outputs are not (logits, weights, ids)"+ spread = float(output[positions[0]].float().std())+ scales = [1.0] + ([min(t / spread, 1e6) for t in _CHECK_LOGIT_STDS] if spread > 0 else [])+ for scale in scales:+ output = module.forward(x * scale, **extra)+ positions = rr._split_router_output(output)+ if positions is None:+ return "its outputs are not (logits, weights, ids)"+ li, wi, ii, dense = positions+ logits = output[li].reshape(-1, output[li].shape[-1])+ ids = output[ii].reshape(-1, output[ii].shape[-1]).long()+ weights = output[wi].reshape(logits.shape).gather(-1, ids) if dense else output[wi].reshape(ids.shape)+ candidate = fn(logits, ids).to(weights.dtype).float()+ if not torch.allclose(candidate, weights.float(), rtol = 2e-2, atol = 2e-3):+ return f"the {formula} weight formula does not reproduce its routing weights"+ return None+++def _remove_hook(module, name, extras, hook_id):+ for table in [getattr(module, name, None)] + [getattr(module, e, None) for e in extras]:+ if table is not None:+ table.pop(hook_id, None)+++def prepare_routing_replay(model, before):+ """After install_routing_replay(model, ...): prime the router formulas outside compilation and+ detach the replay hooks during `model.disable_adapter()` (PEFT reference scoring). A router that+ cannot be primed loses its replay hook (it keeps its natural routing) with a warning, rather+ than reaching the data dependent formula pick inside a compiled graph. Returns the number of+ replay hooks left (0 when routing replay installed nothing)."""+ hooks = _new_hooks(model, before)+ if not hooks:+ return 0+ try:+ import unsloth_zoo.routing_replay as rr+ except Exception:+ rr = None+ if rr is not None and all(hasattr(rr, n) for n in ("_STATE", "_is_router", "_split_router_output", "_LAYER_INDEX_RE")):+ layer_of, parent_of = {}, {}+ for name, module in model.named_modules():+ match = rr._LAYER_INDEX_RE.search(f"{name}.")+ if match is not None:+ layer_of[id(module)] = int(match.group(1))+ for child in module.children():+ parent_of[id(child)] = module+ routers = [+ entry for entry in hooks+ if entry[1] == "_forward_hooks" and rr._is_router(entry[0]) and id(entry[0]) in layer_of+ ]+ # One replay row per decoder layer index, every row the natural routing.+ layers = 1 + max((layer_of[id(m)] for m, *_ in routers), default = 0)+ failed = []+ for entry in routers:+ module, name, extras, hook_id = entry+ why = _prime_formula(rr, module, module._forward_hooks[hook_id], layers, parent_of.get(id(module)))+ if why is not None:+ _remove_hook(*entry)+ failed.append((type(module).__name__, why))+ if failed:+ removed = {id(e) for e in routers if e[3] not in getattr(e[0], e[1], {})}+ hooks = [e for e in hooks if id(e) not in removed]+ names = sorted({f"{cls} ({why})" for cls, why in failed})+ warnings.warn(+ f"Unsloth: routing replay is off for {len(failed)} router(s) it could not prepare, "+ f"which keep their natural routing: {'; '.join(names)}.",+ stacklevel = 2,+ )++ disable_adapter = getattr(model, "disable_adapter", None)+ if callable(disable_adapter) and not getattr(disable_adapter, "_unsloth_routing_replay", False):++ @contextlib.contextmanager+ def disable_adapter_without_replay(*args, **kwargs):+ with disable_adapter(*args, **kwargs), _detached(hooks):+ yield++ disable_adapter_without_replay._unsloth_routing_replay = True+ model.disable_adapter = disable_adapter_without_replay+ return len(hooks)diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 2f724174e41..98b5aa8413e 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -3221,7 +3221,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" except ImportError:\n"" print('Unsloth: UNSLOTH_GRPO_ROUTING_REPLAY=1 needs a newer unsloth_zoo: pip install -U unsloth_zoo')\n"" else:\n"+ " from unsloth.models._routing_replay import routing_replay_hook_snapshot, prepare_routing_replay\n"+ " # TRL scores the PEFT reference through self.model.disable_adapter(), so prepare that object.\n"+ " _unsloth_replay_model = getattr(self, 'model', None)\n"+ " _unsloth_replay_model = model if _unsloth_replay_model is None else _unsloth_replay_model\n"+ " _unsloth_replay_hooks_before = routing_replay_hook_snapshot(_unsloth_replay_model)\n"" install_routing_replay(model, model.vllm_engine, getattr(self, 'processing_class', None))\n"+ " # Pick the router weight formulas outside compilation; PEFT reference scoring routes naturally.\n"+ " prepare_routing_replay(_unsloth_replay_model, _unsloth_replay_hooks_before)\n""pass\n")