Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO routing replay: prime router formulas before compile and score the PEFT reference without replay (#14756)

Daniel Han956c4ef762dc
diff --git a/tests/test_grpo_routing_replay_trainer_hooks.py b/tests/test_grpo_routing_replay_trainer_hooks.py
new file mode 100644
index 00000000000..640075842c1
--- /dev/null
+++ b/tests/test_grpo_routing_replay_trainer_hooks.py
@@ -0,0 +1,277 @@
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""GRPO trainer installation of MoE routing replay (UNSLOTH_GRPO_ROUTING_REPLAY=1), CPU only.
+
+Runs the exact post-init code rl.py writes into the compiled GRPO trainer: a cold router must
+compile with fullgraph=True, and PEFT reference scoring (disable_adapter) must route naturally."""
+
+import ast
+import os
+import pathlib
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+import torch
+
+transformers = pytest.importorskip("transformers")
+try:
+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM
+ from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeTopKRouter # noqa: F401
+except Exception:
+ pytest.skip("needs transformers >= 5 router modules", allow_module_level = True)
+rr = pytest.importorskip("unsloth_zoo.routing_replay")
+
+RL_PY = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl.py"
+LAYERS, EXPERTS, TOP_K, PAD = 2, 8, 2, 0
+
+
+def _post_init_code():
+ # The string constant rl.py appends to RLTrainer_post for grpo_trainer.
+ for node in ast.walk(ast.parse(RL_PY.read_text(encoding = "utf-8"))):
+ if isinstance(node, ast.Constant) and isinstance(node.value, str) \
+ and "UNSLOTH_GRPO_ROUTING_REPLAY', '0') == '1' and hasattr(model, 'vllm_engine')" in node.value:
+ return node.value
+ raise AssertionError("routing replay post-init code not found in rl.py")
+
+
+def _run_post_init(model, trainer_model = None):
+ trainer = SimpleNamespace(model = trainer_model if trainer_model is not None else model,
+ processing_class = SimpleNamespace(pad_token_id = PAD))
+ exec(_post_init_code(), {"os": os, "self": trainer, "model": model})
+
+
+def replay(monkeypatch):
+ monkeypatch.setenv(rr.ROUTING_REPLAY_ENV, "1")
+ rr.reset_routing_replay()
+ torch._dynamo.reset()
+ yield
+ rr.reset_routing_replay()
+ torch._dynamo.reset()
+
+
+def _moe(seed = 0):
+ torch.manual_seed(seed)
+ config = Qwen3MoeConfig(
+ vocab_size = 64, hidden_size = 32, intermediate_size = 64, moe_intermediate_size = 16,
+ num_hidden_layers = LAYERS, num_attention_heads = 4, num_key_value_heads = 2, head_dim = 8,
+ num_experts = EXPERTS, num_experts_per_tok = TOP_K, norm_topk_prob = True, pad_token_id = PAD,
+ )
+ model = Qwen3MoeForCausalLM(config).float().eval()
+ for layer in model.model.layers:
+ layer.mlp.gate.weight.data.normal_(0, 0.5)
+ model.vllm_engine = None
+ return model
+
+
+def _natural(model, ids):
+ captured = {}
+ handles = [
+ layer.mlp.gate.register_forward_hook(lambda m, a, out, i = i: captured.__setitem__(i, out[2].clone()))
+ for i, layer in enumerate(model.model.layers)
+ ]
+ with torch.no_grad():
+ logits = model(input_ids = torch.tensor([ids])).logits
+ for h in handles:
+ h.remove()
+ return logits, torch.stack([captured[i] for i in range(LAYERS)], dim = 1)
+
+
+def test_cold_router_compiles_fullgraph(replay):
+ model = _moe()
+ _run_post_init(model)
+ router = model.model.layers[0].mlp.gate
+ assert router._forward_hooks, "routing replay hooks were not installed"
+ x = torch.randn(5, 32, generator = torch.Generator().manual_seed(1))
+ with torch.no_grad():
+ natural = router.forward(x)[2]
+ forced = (natural + 3) % EXPERTS
+ rr._STATE.in_forward, rr._STATE.record = True, None
+ rr._STATE.current = forced[:, None, :].expand(-1, LAYERS, -1).contiguous()
+ try:
+ compiled = torch.compile(router, backend = "eager", fullgraph = True)
+ logits, weights, ids = compiled(x.requires_grad_(True)) # first call: no eager warm-up
+ weights.square().sum().backward()
+ finally:
+ rr._STATE.in_forward, rr._STATE.current = False, None
+ assert torch.equal(ids.long(), forced.long()) # the replay ran inside the compiled graph
+ expected = torch.softmax(logits.detach().float().gather(-1, forced.long()), -1)
+ torch.testing.assert_close(weights.detach().float(), expected, rtol = 1e-5, atol = 1e-6)
+ assert x.grad is not None and bool(x.grad.abs().sum() > 0)
+
+
+def test_peft_reference_scoring_routes_naturally(replay):
+ peft = pytest.importorskip("peft")
+ base = _moe()
+ ids = [5, 6, 7, 30, 31, 32]
+ natural_logits, natural_ids = _natural(base, ids)
+ model = peft.get_peft_model(base, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))
+ model.vllm_engine = None
+ _run_post_init(model)
+ routers = [layer.mlp.gate for layer in model.base_model.model.model.layers]
+ hooks_before = [(dict(r._forward_hooks), dict(r._forward_pre_hooks)) for r in routers]
+ assert all(h for h, _ in hooks_before)
+
+ forced = ((natural_ids + 3) % EXPERTS).numpy().astype(np.int16)
+ rr.record_routed_experts([SimpleNamespace(
+ prompt_token_ids = ids[:3], outputs = [SimpleNamespace(token_ids = ids[3:], routed_experts = forced[:-1])],
+ )], pad_token_id = PAD)
+
+ with torch.no_grad():
+ policy = model(input_ids = torch.tensor([ids])).logits
+ # LoRA B starts at zero: any difference from the base comes from the replayed routes.
+ assert not torch.allclose(policy, natural_logits, atol = 1e-4)
+ with model.disable_adapter():
+ reference = model(input_ids = torch.tensor([ids])).logits
+ assert all(not r._forward_hooks for r in routers)
+ torch.testing.assert_close(reference, natural_logits, rtol = 1e-5, atol = 1e-6)
+ again = model(input_ids = torch.tensor([ids])).logits
+ torch.testing.assert_close(again, policy, rtol = 1e-5, atol = 1e-6)
+ assert [(dict(r._forward_hooks), dict(r._forward_pre_hooks)) for r in routers] == hooks_before
+
+
+def test_no_replay_installs_nothing(monkeypatch):
+ monkeypatch.delenv(rr.ROUTING_REPLAY_ENV, raising = False)
+ peft = pytest.importorskip("peft")
+ model = peft.get_peft_model(_moe(), peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))
+ model.vllm_engine = None
+ disable_adapter = model.disable_adapter
+ _run_post_init(model)
+ assert model.disable_adapter == disable_adapter
+ assert all(not m._forward_hooks and not m._forward_pre_hooks for m in model.modules())
+
+
+def test_reference_pass_restores_hook_order_with_prepended_hooks(replay):
+ # A hook registered after installation with prepend=True has a newer id but runs first: the
+ # reference pass must leave every hook table in exactly the order it found it.
+ peft = pytest.importorskip("peft")
+ model = peft.get_peft_model(_moe(), peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))
+ model.vllm_engine = None
+ _run_post_init(model)
+ decoder = model.base_model.model.model
+ router = decoder.layers[0].mlp.gate
+ calls = []
+ router.register_forward_hook(lambda m, a, out: calls.append("prepended"), prepend = True)
+ router.register_forward_hook(lambda m, a, out: calls.append("appended"))
+ decoder.register_forward_pre_hook(lambda m, a, k: None, prepend = True, with_kwargs = True)
+ decoder.layers[0].register_forward_pre_hook(lambda m, a: None, prepend = True)
+ tables = ("_forward_hooks", "_forward_hooks_with_kwargs", "_forward_hooks_always_called",
+ "_forward_pre_hooks", "_forward_pre_hooks_with_kwargs")
+
+ def orders():
+ return [[list(getattr(m, t, {}).items()) for t in tables] for m in model.modules()]
+
+ before = orders()
+ ids = torch.tensor([[5, 6, 7, 30]])
+ with torch.no_grad():
+ with model.disable_adapter():
+ model(input_ids = ids)
+ assert orders() == before
+ calls.clear()
+ model(input_ids = ids)
+ assert calls == ["prepended", "appended"]
+
+
+def _bias_router_model(model_type):
+ from transformers import AutoConfig, AutoModelForCausalLM
+
+ extra = {
+ "minimax_m2": dict(num_local_experts = EXPERTS),
+ "hy_v3": dict(num_local_experts = EXPERTS),
+ "lfm2_moe": dict(num_experts = EXPERTS, use_expert_bias = True, num_dense_layers = 0,
+ layer_types = ["full_attention"] * LAYERS),
+ }[model_type]
+ try:
+ config = AutoConfig.for_model(
+ model_type, vocab_size = 64, hidden_size = 32, intermediate_size = 64, moe_intermediate_size = 16,
+ num_hidden_layers = LAYERS, num_attention_heads = 4, num_key_value_heads = 2, head_dim = 8,
+ num_experts_per_tok = TOP_K, pad_token_id = PAD, **extra,
+ )
+ except Exception as error:
+ pytest.skip(f"this transformers has no {model_type}: {error}")
+ torch.manual_seed(0)
+ model = AutoModelForCausalLM.from_config(config).float().eval()
+ model.vllm_engine = None
+ return model
+
+
[email protected]("model_type", ["minimax_m2", "hy_v3", "lfm2_moe"])
+def test_sigmoid_correction_bias_routers_route_naturally_with_a_warning(replay, model_type):
+ # MiniMax-M2 / HY-V3: TopKRouter.forward(hidden_states, e_score_correction_bias), the bias a
+ # buffer of the parent MoE block; LFM2-MoE: forward(hidden_states, expert_bias = None). All three
+ # take sigmoid(logits) top-k weights, which no zoo replay formula reproduces: priming must reach
+ # the router (no TypeError), then take the hooks off and say so.
+ model = _bias_router_model(model_type)
+ routers = [m for m in model.modules() if "TopKRouter" in type(m).__name__]
+ if not routers:
+ pytest.skip(f"{model_type} has no TopKRouter module in this transformers (gate inside the MoE block)")
+ ids = [5, 6, 7, 30, 31, 32]
+ with torch.no_grad():
+ natural = model(input_ids = torch.tensor([ids])).logits
+ name = type(routers[0]).__name__
+ with pytest.warns(UserWarning, match = rf"routing replay is off for {len(routers)} router.*{name} \(routing replay has no weight formula"):
+ _run_post_init(model)
+ assert all(not r._forward_hooks for r in routers)
+ # A recorded rollout with other experts no longer changes anything.
+ layers = model.config.num_hidden_layers
+ forced = np.full((len(ids) - 1, layers, TOP_K), 0, dtype = np.int16)
+ forced[..., 1] = 1
+ rr.record_routed_experts([SimpleNamespace(
+ prompt_token_ids = ids[:3], outputs = [SimpleNamespace(token_ids = ids[3:], routed_experts = forced)],
+ )], pad_token_id = PAD)
+ with torch.no_grad():
+ torch.testing.assert_close(model(input_ids = torch.tensor([ids])).logits, natural)
+
+
+def test_formula_that_only_fits_the_priming_input_is_rejected(replay):
+ # A sigmoid router with tiny logits: on the priming input every weight is ~1 / k, so the zoo's
+ # softmax_renorm fits there; on spread logits it does not reproduce the router's weights.
+ import torch.nn.functional as F
+
+ model = _moe()
+ original = type(model.model.layers[0].mlp.gate)
+
+ class SigmoidTopKRouter(original):
+ def forward(self, hidden_states):
+ hidden_states = hidden_states.reshape(-1, self.hidden_dim)
+ logits = F.linear(hidden_states, self.weight)
+ weights, ids = logits.float().sigmoid().topk(self.top_k, dim = -1)
+ return logits, (weights / weights.sum(-1, keepdim = True)).to(logits.dtype), ids
+
+ gate = model.model.layers[0].mlp.gate
+ gate.__class__ = SigmoidTopKRouter
+ gate.weight.data.normal_(0, 1e-4)
+ with pytest.warns(UserWarning, match = r"SigmoidTopKRouter \(the softmax_renorm weight formula does not reproduce"):
+ _run_post_init(model)
+ assert not gate._forward_hooks
+ assert model.model.layers[1].mlp.gate._forward_hooks # the real softmax router passes the check
+
+
+def test_router_that_cannot_be_primed_loses_its_hook_with_a_warning(replay):
+ model = _moe()
+ original = type(model.model.layers[0].mlp.gate)
+
+ class OddTopKRouter(original):
+ # A required argument the parent block does not hold: priming cannot build the call.
+ def forward(self, hidden_states, routing_temperature):
+ return super().forward(hidden_states / routing_temperature)
+
+ model.model.layers[0].mlp.gate.__class__ = OddTopKRouter
+ with pytest.warns(UserWarning, match = r"routing replay is off for 1 router.*OddTopKRouter"):
+ _run_post_init(model)
+ assert not model.model.layers[0].mlp.gate._forward_hooks
+ assert model.model.layers[1].mlp.gate._forward_hooks
+ assert getattr(model.model.layers[1].mlp.gate, "_unsloth_replay_formula", None) is not None
diff --git a/unsloth/models/_routing_replay.py b/unsloth/models/_routing_replay.py
new file mode 100644
index 00000000000..03574176813
--- /dev/null
+++ b/unsloth/models/_routing_replay.py
@@ -0,0 +1,247 @@
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""GRPO trainer side of unsloth_zoo's MoE routing replay (UNSLOTH_GRPO_ROUTING_REPLAY=1).
+
+Only used when routing replay is requested. Two things the zoo hooks need from the trainer:
+- their routing weight formula is picked from the first replayed router call, which reads tensor
+ values; done inside a compiled region that is a graph break (fullgraph fails), so it is picked
+ here once, eagerly, right after the hooks are installed;
+- PEFT reference scoring (`disable_adapter()`) must see the base model's own routing, as a separate
+ reference model does, so the hooks are detached inside that context.
+"""
+
+import contextlib
+import inspect
+import warnings
+
+import torch
+
+__all__ = ["routing_replay_hook_snapshot", "prepare_routing_replay"]
+
+_HOOK_DICTS = (
+ ("_forward_hooks", ("_forward_hooks_with_kwargs", "_forward_hooks_always_called")),
+ ("_forward_pre_hooks", ("_forward_pre_hooks_with_kwargs",)),
+)
+
+
+def routing_replay_hook_snapshot(model):
+ """Hook ids on every module of `model`, taken before install_routing_replay."""
+ return {
+ id(module): tuple(set(getattr(module, name, {})) for name, _ in _HOOK_DICTS)
+ for module in model.modules()
+ }
+
+
+def _new_hooks(model, before):
+ hooks = []
+ for module in model.modules():
+ old = before.get(id(module), (set(), set()))
+ for (name, extras), seen in zip(_HOOK_DICTS, old):
+ for hook_id in getattr(module, name, {}):
+ if hook_id not in seen:
+ hooks.append((module, name, extras, hook_id))
+ return hooks
+
+
+def _detached(hooks):
+ # Hook order is call order, and ids do not give it (register_*_hook(prepend = True) puts a newer
+ # id first): keep each table's own key order and put it back exactly.
+ orders = {}
+ removed = []
+ for module, name, extras, hook_id in hooks:
+ for table in [getattr(module, name, None)] + [getattr(module, e, None) for e in extras]:
+ if table is None or hook_id not in table:
+ continue
+ orders.setdefault(id(table), (table, list(table.keys())))
+ removed.append((table, hook_id, table.pop(hook_id)))
+ try:
+ yield
+ finally:
+ for table, hook_id, value in removed:
+ table[hook_id] = value
+ for table, order in orders.values():
+ known = set(order)
+ added = [k for k in table if k not in known] # registered inside the context: keep last
+ items = dict(table)
+ table.clear()
+ for k in order + added:
+ if k in items:
+ table[k] = items[k]
+
+
+def _router_extra_args(module, parent):
+ # Routers such as MiniMax-M2 / HY-V3 (`forward(hidden_states, e_score_correction_bias)`) or
+ # LFM2-MoE (`forward(hidden_states, expert_bias = None)`) take the parent MoE block's buffer:
+ # the block calls `self.gate(hidden_states, self.e_score_correction_bias)`. Pass the parent's
+ # same-named tensor; None when a required argument has none.
+ try:
+ params = list(inspect.signature(module.forward).parameters.values())[1:]
+ except (TypeError, ValueError):
+ return {}
+ extra = {}
+ for param in params:
+ if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):
+ continue
+ value = getattr(parent, param.name, None) if parent is not None else None
+ if torch.is_tensor(value):
+ extra[param.name] = value
+ elif param.default is param.empty:
+ return None
+ return extra
+
+
+def _prime_formula(rr, module, hook, layers, parent = None):
+ """Run the zoo hook once, eagerly, on a synthetic router call whose replay is the natural
+ routing: it picks and caches module._unsloth_replay_formula exactly as the first real call
+ would. Returns None when primed, else why it could not be."""
+ weight = getattr(module, "weight", None)
+ hidden = getattr(module, "hidden_dim", None) or getattr(module, "in_features", None)
+ if hidden is None and torch.is_tensor(weight) and weight.dim() == 2:
+ hidden = weight.shape[-1]
+ if not isinstance(hidden, int) or not torch.is_tensor(weight) or not weight.is_floating_point():
+ return "its hidden size or weight is unknown"
+ extra = _router_extra_args(module, parent)
+ if extra is None:
+ return "its forward needs an argument the parent MoE block does not hold"
+ generator = torch.Generator().manual_seed(0)
+ x = torch.randn(8, hidden, generator = generator).to(weight.device, weight.dtype)
+ state = rr._STATE
+ fields = ("current", "in_forward", "record", "pending", "recomputed", "changed")
+ old = {f: getattr(state, f) for f in fields}
+ old_stats = dict(state.stats)
+ if getattr(module, "_unsloth_replay_formula", None) is not None:
+ return _check_formula(rr, module, x, extra) # already picked (an earlier install)
+ try:
+ with torch.no_grad():
+ output = module.forward(x, **extra) # no hooks: the zoo hook is called by hand below
+ positions = rr._split_router_output(output)
+ if positions is None:
+ return "its outputs are not (logits, weights, ids)"
+ ids = output[positions[2]].reshape(-1, output[positions[2]].shape[-1]).long()
+ state.current = ids[:, None, :].expand(-1, layers, -1).contiguous()
+ state.in_forward, state.record, state.pending, state.recomputed = True, None, [], False
+ hook(module, (x, *extra.values()), output)
+ except Exception as error:
+ return f"priming raised {type(error).__name__}: {error}"
+ finally:
+ for f, v in old.items():
+ setattr(state, f, v)
+ state.stats.clear()
+ state.stats.update(old_stats)
+ if getattr(module, "_unsloth_replay_formula", None) is None:
+ return "the routing replay hook did not pick a weight formula"
+ return _check_formula(rr, module, x, extra)
+
+
+# The check also runs with the input scaled so the router logits spread to these standard
+# deviations: near-zero logits make every formula give ~1 / k, so the priming input alone can let a
+# wrong formula through (sigmoid routing matched by softmax).
+_CHECK_LOGIT_STDS = (1.0, 4.0)
+
+
+def _check_formula(rr, module, x, extra):
+ """None when the cached zoo formula reproduces this router's own top-k weights, else why not.
+ The zoo stores False when it found none (sigmoid / correction bias routers, MiniMax-M2, HY-V3,
+ LFM2-MoE): replay would silently keep natural routing there, so it counts as a failure too."""
+ formula = getattr(module, "_unsloth_replay_formula", None)
+ if formula is False or formula is None:
+ return "routing replay has no weight formula for this router (e.g. sigmoid routing)"
+ fn = rr._formulas().get(formula) if hasattr(rr, "_formulas") else None
+ if fn is None:
+ return f"unknown routing replay weight formula {formula!r}"
+ with torch.no_grad():
+ output = module.forward(x, **extra)
+ positions = rr._split_router_output(output)
+ if positions is None:
+ return "its outputs are not (logits, weights, ids)"
+ spread = float(output[positions[0]].float().std())
+ scales = [1.0] + ([min(t / spread, 1e6) for t in _CHECK_LOGIT_STDS] if spread > 0 else [])
+ for scale in scales:
+ output = module.forward(x * scale, **extra)
+ positions = rr._split_router_output(output)
+ if positions is None:
+ return "its outputs are not (logits, weights, ids)"
+ li, wi, ii, dense = positions
+ logits = output[li].reshape(-1, output[li].shape[-1])
+ ids = output[ii].reshape(-1, output[ii].shape[-1]).long()
+ weights = output[wi].reshape(logits.shape).gather(-1, ids) if dense else output[wi].reshape(ids.shape)
+ candidate = fn(logits, ids).to(weights.dtype).float()
+ if not torch.allclose(candidate, weights.float(), rtol = 2e-2, atol = 2e-3):
+ return f"the {formula} weight formula does not reproduce its routing weights"
+ return None
+
+
+def _remove_hook(module, name, extras, hook_id):
+ for table in [getattr(module, name, None)] + [getattr(module, e, None) for e in extras]:
+ if table is not None:
+ table.pop(hook_id, None)
+
+
+def prepare_routing_replay(model, before):
+ """After install_routing_replay(model, ...): prime the router formulas outside compilation and
+ detach the replay hooks during `model.disable_adapter()` (PEFT reference scoring). A router that
+ cannot be primed loses its replay hook (it keeps its natural routing) with a warning, rather
+ than reaching the data dependent formula pick inside a compiled graph. Returns the number of
+ replay hooks left (0 when routing replay installed nothing)."""
+ hooks = _new_hooks(model, before)
+ if not hooks:
+ return 0
+ try:
+ import unsloth_zoo.routing_replay as rr
+ except Exception:
+ rr = None
+ if rr is not None and all(hasattr(rr, n) for n in ("_STATE", "_is_router", "_split_router_output", "_LAYER_INDEX_RE")):
+ layer_of, parent_of = {}, {}
+ for name, module in model.named_modules():
+ match = rr._LAYER_INDEX_RE.search(f"{name}.")
+ if match is not None:
+ layer_of[id(module)] = int(match.group(1))
+ for child in module.children():
+ parent_of[id(child)] = module
+ routers = [
+ entry for entry in hooks
+ if entry[1] == "_forward_hooks" and rr._is_router(entry[0]) and id(entry[0]) in layer_of
+ ]
+ # One replay row per decoder layer index, every row the natural routing.
+ layers = 1 + max((layer_of[id(m)] for m, *_ in routers), default = 0)
+ failed = []
+ for entry in routers:
+ module, name, extras, hook_id = entry
+ why = _prime_formula(rr, module, module._forward_hooks[hook_id], layers, parent_of.get(id(module)))
+ if why is not None:
+ _remove_hook(*entry)
+ failed.append((type(module).__name__, why))
+ if failed:
+ removed = {id(e) for e in routers if e[3] not in getattr(e[0], e[1], {})}
+ hooks = [e for e in hooks if id(e) not in removed]
+ names = sorted({f"{cls} ({why})" for cls, why in failed})
+ warnings.warn(
+ f"Unsloth: routing replay is off for {len(failed)} router(s) it could not prepare, "
+ f"which keep their natural routing: {'; '.join(names)}.",
+ stacklevel = 2,
+ )
+
+ disable_adapter = getattr(model, "disable_adapter", None)
+ if callable(disable_adapter) and not getattr(disable_adapter, "_unsloth_routing_replay", False):
+
+ @contextlib.contextmanager
+ def disable_adapter_without_replay(*args, **kwargs):
+ with disable_adapter(*args, **kwargs), _detached(hooks):
+ yield
+
+ disable_adapter_without_replay._unsloth_routing_replay = True
+ model.disable_adapter = disable_adapter_without_replay
+ return len(hooks)
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 2f724174e41..98b5aa8413e 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -3221,7 +3221,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" except ImportError:\n"
" print('Unsloth: UNSLOTH_GRPO_ROUTING_REPLAY=1 needs a newer unsloth_zoo: pip install -U unsloth_zoo')\n"
" else:\n"
+ " from unsloth.models._routing_replay import routing_replay_hook_snapshot, prepare_routing_replay\n"
+ " # TRL scores the PEFT reference through self.model.disable_adapter(), so prepare that object.\n"
+ " _unsloth_replay_model = getattr(self, 'model', None)\n"
+ " _unsloth_replay_model = model if _unsloth_replay_model is None else _unsloth_replay_model\n"
+ " _unsloth_replay_hooks_before = routing_replay_hook_snapshot(_unsloth_replay_model)\n"
" install_routing_replay(model, model.vllm_engine, getattr(self, 'processing_class', None))\n"
+ " # Pick the router weight formulas outside compilation; PEFT reference scoring routes naturally.\n"
+ " prepare_routing_replay(_unsloth_replay_model, _unsloth_replay_hooks_before)\n"
"pass\n"
)