Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Anchor the GRPO MoE aux-loss fail-fast on the assignment, not its expression (#11747)

Daniel Han04793f0ede71
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
index 83abd62ad29..33d93278642 100644
--- a/.github/workflows/version-compat-ci.yml
+++ b/.github/workflows/version-compat-ci.yml
@@ -113,6 +113,7 @@ jobs:
PYTHONPATH=. python -m pytest \
tests/vllm_compat/test_vllm_pinned_symbols.py \
tests/version_compat/test_trl_grpo_pinned_symbols.py \
+ tests/version_compat/test_grpo_aux_loss_fail_fast.py \
tests/version_compat/test_peft_pinned_symbols.py \
tests/version_compat/test_unsloth_zoo_save_merged_pinned_symbols.py \
tests/version_compat/test_peft_conversion_symbol_backfill.py \
diff --git a/tests/version_compat/_rl_anchors.py b/tests/version_compat/_rl_anchors.py
new file mode 100644
index 00000000000..5234c449660
--- /dev/null
+++ b/tests/version_compat/_rl_anchors.py
@@ -0,0 +1,31 @@
+"""Anchors of unsloth/models/rl.py, read without importing it (that needs a GPU stack)."""
+
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+
+RL_PY = Path(__file__).resolve().parents[2] / "unsloth" / "models" / "rl.py"
+
+
+def _rl_py_constant(name: str) -> str:
+ """A module-level string constant of unsloth/models/rl.py."""
+ tree = ast.parse(RL_PY.read_text("utf-8"))
+ for node in tree.body:
+ if isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id == name for t in node.targets
+ ):
+ return ast.literal_eval(node.value)
+ raise AssertionError(f"unsloth/models/rl.py no longer defines {name}")
+
+
+def _reject_aux_loss_opt_in(src: str) -> str:
+ """rl.py's substitution, applied exactly as rl.py applies it."""
+ return re.sub(
+ _rl_py_constant("_GRPO_AUX_LOSS_ENABLED_LINE"),
+ lambda m: m.group(0) + "\n" + m.group(1) + _rl_py_constant("_GRPO_AUX_LOSS_REJECT"),
+ src,
+ count = 1,
+ flags = re.MULTILINE,
+ )
diff --git a/tests/version_compat/test_grpo_aux_loss_fail_fast.py b/tests/version_compat/test_grpo_aux_loss_fail_fast.py
new file mode 100644
index 00000000000..e814ea7a300
--- /dev/null
+++ b/tests/version_compat/test_grpo_aux_loss_fail_fast.py
@@ -0,0 +1,51 @@
+"""GRPO's MoE router-aux fail-fast lands on every known spelling of its anchor, offline."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.version_compat._rl_anchors import RL_PY, _reject_aux_loss_opt_in
+
+
+ "assignment",
+ [
+ # TRL 1.7.0 through the last release.
+ "self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0",
+ # TRL main after #7248: the coefficient falls back to the model config's when None.
+ 'self.aux_loss_enabled = hasattr(text_config, "output_router_logits") and self.router_aux_loss_coef != 0.0',
+ ],
+)
+def test_the_aux_loss_fail_fast_lands_on_every_known_spelling(assignment: str):
+ """Offline: both spellings get the fail-fast at the assignment's indentation, and it runs."""
+ src = (
+ "class GRPOTrainer:\n"
+ " def __init__(self, args, is_moe, text_config):\n"
+ " self.router_aux_loss_coef = args.router_aux_loss_coef\n"
+ f" {assignment}\n"
+ " self.done = True\n"
+ )
+ patched = _reject_aux_loss_opt_in(src)
+ assert patched.count("raise NotImplementedError") == 1, patched
+ ns: dict = {}
+ exec(compile(patched, "<grpo>", "exec"), ns)
+
+ class Args:
+ router_aux_loss_coef = 0.001
+
+ class MoEConfig:
+ output_router_logits = False
+
+ with pytest.raises(NotImplementedError, match = "router_aux_loss_coef = 0"):
+ ns["GRPOTrainer"](Args, True, MoEConfig())
+ Args.router_aux_loss_coef = 0.0
+ assert ns["GRPOTrainer"](Args, True, MoEConfig()).done
+
+
+def test_rl_py_applies_the_anchor_it_defines():
+ """The contract tests exercise the constants; this holds rl.py to actually substituting with them."""
+ src = RL_PY.read_text("utf-8")
+ use = src.index("_GRPO_AUX_LOSS_ENABLED_LINE,", src.index("_GRPO_AUX_LOSS_ENABLED_LINE =") + 1)
+ call = src[src.rindex("re.sub(", 0, use) : use + 400]
+ assert "_GRPO_AUX_LOSS_REJECT" in call and "re.MULTILINE" in call, call
+ assert 'RLTrainer_name == "GRPOTrainer"' in src[:use]
diff --git a/tests/version_compat/test_trl_grpo_pinned_symbols.py b/tests/version_compat/test_trl_grpo_pinned_symbols.py
index bea7822d06e..b34da374f4c 100644
--- a/tests/version_compat/test_trl_grpo_pinned_symbols.py
+++ b/tests/version_compat/test_trl_grpo_pinned_symbols.py
@@ -17,6 +17,7 @@ import re
import pytest
from tests.version_compat._fetch import fetch_text, first_match, has_def
+from tests.version_compat._rl_anchors import _reject_aux_loss_opt_in, _rl_py_constant
# Every stable TRL release from 0.18.2 (pyproject floor) onwards, plus `main`.
@@ -533,19 +534,28 @@ def test_trl_grpo_quantized_model_cast_contract(tag: str):
def test_trl_grpo_aux_loss_enabled_contract(tag: str):
- """rl.py (trl>=1.7.0) appends a fail-fast after
- `self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0` so an
- explicit MoE router-aux opt-in errors instead of silently training without
- the penalty (the optimized forward cannot compute it). A change to this
- line drops the guard silently (PR #6904)."""
+ """rl.py (trl>=1.7.0) appends a fail-fast after GRPOTrainer's one
+ `self.aux_loss_enabled = ...` assignment, so an MoE router-aux opt-in errors
+ instead of silently training without the penalty (the optimized forward
+ cannot compute it; PR #6904). It anchors on the assignment, not its
+ expression: TRL #7248 rewrote the expression to read the coefficient from
+ the model config, and the exact-text anchor then no-oped on TRL main."""
if not _tag_ge(tag, "1.7.0"):
pytest.skip(f"{tag}: aux_loss_enabled / router_aux_loss_coef added in TRL 1.7.0")
src = fetch_text("huggingface/trl", tag, "trl/trainer/grpo_trainer.py")
assert src is not None
- assert "self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0" in src, (
- f"{tag}: `aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0` "
- f"changed; unsloth/models/rl.py's fail-fast .replace() anchor no-ops"
+ hits = re.findall(_rl_py_constant("_GRPO_AUX_LOSS_ENABLED_LINE"), src, flags = re.MULTILINE)
+ assert len(hits) == 1, (
+ f"{tag}: expected one `self.aux_loss_enabled = ...` assignment in GRPOTrainer, found "
+ f"{len(hits)}; unsloth/models/rl.py's fail-fast anchor is ambiguous or gone"
)
+ patched = _reject_aux_loss_opt_in(src)
+ line = next(l for l in patched.splitlines() if "self.aux_loss_enabled = " in l)
+ after = patched.splitlines()[patched.splitlines().index(line) + 1]
+ assert after.strip() == _rl_py_constant(
+ "_GRPO_AUX_LOSS_REJECT"
+ ), f"{tag}: the fail-fast did not land directly after the assignment: {after!r}"
+ assert after[: len(after) - len(after.lstrip())] == line[: len(line) - len(line.lstrip())]
def test_trl_grpo_per_token_logps_aux_arity_contract(tag: str):
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index d797282ad4c..f5d1138fa44 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -1008,6 +1008,16 @@ def _pin_pristine_sft_loss_type(config_cls):
_UNSLOTH_KBIT_PREP_GUARD_FLAG = "_unsloth_skips_kbit_prep_for_peft_models"
+# The one assignment of `self.aux_loss_enabled` in TRL's GRPOTrainer.__init__, whatever its right-hand
+# side. TRL 1.7.0 wrote `is_moe and args.router_aux_loss_coef != 0.0`; TRL main (#7248) reads the
+# coefficient from the model config when it is None. Anchoring on the exact expression lost the
+# fail-fast below on the first rewrite. Read without importing by tests/version_compat.
+_GRPO_AUX_LOSS_ENABLED_LINE = r"^([ \t]*)self\.aux_loss_enabled = [^\n]+$"
+_GRPO_AUX_LOSS_REJECT = (
+ 'if self.aux_loss_enabled: raise NotImplementedError("Unsloth GRPO does not compute the MoE router '
+ 'auxiliary loss; set router_aux_loss_coef = 0 (the Unsloth default).")'
+)
+
def _guard_kbit_prep_against_peft_models():
"""Stop TRL below 0.24.0 re-preparing a model Unsloth already prepared.
@@ -2957,10 +2967,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
if trl_version >= Version("1.7.0"):
# router_aux_loss_coef / aux_loss_enabled arrived in TRL 1.7.0, and the optimized GRPO forward cannot compute the MoE router aux loss, so reject an explicit opt-in at init.
- RLTrainer_source = RLTrainer_source.replace(
- "self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0",
- "self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0\n"
- ' if self.aux_loss_enabled: raise NotImplementedError("Unsloth GRPO does not compute the MoE router auxiliary loss; set router_aux_loss_coef = 0 (the Unsloth default).")',
+ RLTrainer_source = re.sub(
+ _GRPO_AUX_LOSS_ENABLED_LINE,
+ lambda m: m.group(0) + "\n" + m.group(1) + _GRPO_AUX_LOSS_REJECT,
+ RLTrainer_source,
+ count = 1,
+ flags = re.MULTILINE,
)
elif trl_version >= Version("0.27.0"):