Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Anchor the GRPO MoE aux-loss fail-fast on the assignment, not its expression (#11747)
Daniel Han04793f0ede71
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.ymlindex 83abd62ad29..33d93278642 100644--- a/.github/workflows/version-compat-ci.yml+++ b/.github/workflows/version-compat-ci.yml@@ -113,6 +113,7 @@ jobs:PYTHONPATH=. python -m pytest \tests/vllm_compat/test_vllm_pinned_symbols.py \tests/version_compat/test_trl_grpo_pinned_symbols.py \+ tests/version_compat/test_grpo_aux_loss_fail_fast.py \tests/version_compat/test_peft_pinned_symbols.py \tests/version_compat/test_unsloth_zoo_save_merged_pinned_symbols.py \tests/version_compat/test_peft_conversion_symbol_backfill.py \diff --git a/tests/version_compat/_rl_anchors.py b/tests/version_compat/_rl_anchors.pynew file mode 100644index 00000000000..5234c449660--- /dev/null+++ b/tests/version_compat/_rl_anchors.py@@ -0,0 +1,31 @@+"""Anchors of unsloth/models/rl.py, read without importing it (that needs a GPU stack)."""++from __future__ import annotations++import ast+import re+from pathlib import Path++RL_PY = Path(__file__).resolve().parents[2] / "unsloth" / "models" / "rl.py"+++def _rl_py_constant(name: str) -> str:+ """A module-level string constant of unsloth/models/rl.py."""+ tree = ast.parse(RL_PY.read_text("utf-8"))+ for node in tree.body:+ if isinstance(node, ast.Assign) and any(+ isinstance(t, ast.Name) and t.id == name for t in node.targets+ ):+ return ast.literal_eval(node.value)+ raise AssertionError(f"unsloth/models/rl.py no longer defines {name}")+++def _reject_aux_loss_opt_in(src: str) -> str:+ """rl.py's substitution, applied exactly as rl.py applies it."""+ return re.sub(+ _rl_py_constant("_GRPO_AUX_LOSS_ENABLED_LINE"),+ lambda m: m.group(0) + "\n" + m.group(1) + _rl_py_constant("_GRPO_AUX_LOSS_REJECT"),+ src,+ count = 1,+ flags = re.MULTILINE,+ )diff --git a/tests/version_compat/test_grpo_aux_loss_fail_fast.py b/tests/version_compat/test_grpo_aux_loss_fail_fast.pynew file mode 100644index 00000000000..e814ea7a300--- /dev/null+++ b/tests/version_compat/test_grpo_aux_loss_fail_fast.py@@ -0,0 +1,51 @@+"""GRPO's MoE router-aux fail-fast lands on every known spelling of its anchor, offline."""++from __future__ import annotations++import pytest++from tests.version_compat._rl_anchors import RL_PY, _reject_aux_loss_opt_in+++ "assignment",+ [+ # TRL 1.7.0 through the last release.+ "self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0",+ # TRL main after #7248: the coefficient falls back to the model config's when None.+ 'self.aux_loss_enabled = hasattr(text_config, "output_router_logits") and self.router_aux_loss_coef != 0.0',+ ],+)+def test_the_aux_loss_fail_fast_lands_on_every_known_spelling(assignment: str):+ """Offline: both spellings get the fail-fast at the assignment's indentation, and it runs."""+ src = (+ "class GRPOTrainer:\n"+ " def __init__(self, args, is_moe, text_config):\n"+ " self.router_aux_loss_coef = args.router_aux_loss_coef\n"+ f" {assignment}\n"+ " self.done = True\n"+ )+ patched = _reject_aux_loss_opt_in(src)+ assert patched.count("raise NotImplementedError") == 1, patched+ ns: dict = {}+ exec(compile(patched, "<grpo>", "exec"), ns)++ class Args:+ router_aux_loss_coef = 0.001++ class MoEConfig:+ output_router_logits = False++ with pytest.raises(NotImplementedError, match = "router_aux_loss_coef = 0"):+ ns["GRPOTrainer"](Args, True, MoEConfig())+ Args.router_aux_loss_coef = 0.0+ assert ns["GRPOTrainer"](Args, True, MoEConfig()).done+++def test_rl_py_applies_the_anchor_it_defines():+ """The contract tests exercise the constants; this holds rl.py to actually substituting with them."""+ src = RL_PY.read_text("utf-8")+ use = src.index("_GRPO_AUX_LOSS_ENABLED_LINE,", src.index("_GRPO_AUX_LOSS_ENABLED_LINE =") + 1)+ call = src[src.rindex("re.sub(", 0, use) : use + 400]+ assert "_GRPO_AUX_LOSS_REJECT" in call and "re.MULTILINE" in call, call+ assert 'RLTrainer_name == "GRPOTrainer"' in src[:use]diff --git a/tests/version_compat/test_trl_grpo_pinned_symbols.py b/tests/version_compat/test_trl_grpo_pinned_symbols.pyindex bea7822d06e..b34da374f4c 100644--- a/tests/version_compat/test_trl_grpo_pinned_symbols.py+++ b/tests/version_compat/test_trl_grpo_pinned_symbols.py@@ -17,6 +17,7 @@ import reimport pytestfrom tests.version_compat._fetch import fetch_text, first_match, has_def+from tests.version_compat._rl_anchors import _reject_aux_loss_opt_in, _rl_py_constant# Every stable TRL release from 0.18.2 (pyproject floor) onwards, plus `main`.@@ -533,19 +534,28 @@ def test_trl_grpo_quantized_model_cast_contract(tag: str):def test_trl_grpo_aux_loss_enabled_contract(tag: str):- """rl.py (trl>=1.7.0) appends a fail-fast after- `self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0` so an- explicit MoE router-aux opt-in errors instead of silently training without- the penalty (the optimized forward cannot compute it). A change to this- line drops the guard silently (PR #6904)."""+ """rl.py (trl>=1.7.0) appends a fail-fast after GRPOTrainer's one+ `self.aux_loss_enabled = ...` assignment, so an MoE router-aux opt-in errors+ instead of silently training without the penalty (the optimized forward+ cannot compute it; PR #6904). It anchors on the assignment, not its+ expression: TRL #7248 rewrote the expression to read the coefficient from+ the model config, and the exact-text anchor then no-oped on TRL main."""if not _tag_ge(tag, "1.7.0"):pytest.skip(f"{tag}: aux_loss_enabled / router_aux_loss_coef added in TRL 1.7.0")src = fetch_text("huggingface/trl", tag, "trl/trainer/grpo_trainer.py")assert src is not None- assert "self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0" in src, (- f"{tag}: `aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0` "- f"changed; unsloth/models/rl.py's fail-fast .replace() anchor no-ops"+ hits = re.findall(_rl_py_constant("_GRPO_AUX_LOSS_ENABLED_LINE"), src, flags = re.MULTILINE)+ assert len(hits) == 1, (+ f"{tag}: expected one `self.aux_loss_enabled = ...` assignment in GRPOTrainer, found "+ f"{len(hits)}; unsloth/models/rl.py's fail-fast anchor is ambiguous or gone")+ patched = _reject_aux_loss_opt_in(src)+ line = next(l for l in patched.splitlines() if "self.aux_loss_enabled = " in l)+ after = patched.splitlines()[patched.splitlines().index(line) + 1]+ assert after.strip() == _rl_py_constant(+ "_GRPO_AUX_LOSS_REJECT"+ ), f"{tag}: the fail-fast did not land directly after the assignment: {after!r}"+ assert after[: len(after) - len(after.lstrip())] == line[: len(line) - len(line.lstrip())]def test_trl_grpo_per_token_logps_aux_arity_contract(tag: str):diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex d797282ad4c..f5d1138fa44 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -1008,6 +1008,16 @@ def _pin_pristine_sft_loss_type(config_cls):_UNSLOTH_KBIT_PREP_GUARD_FLAG = "_unsloth_skips_kbit_prep_for_peft_models"+# The one assignment of `self.aux_loss_enabled` in TRL's GRPOTrainer.__init__, whatever its right-hand+# side. TRL 1.7.0 wrote `is_moe and args.router_aux_loss_coef != 0.0`; TRL main (#7248) reads the+# coefficient from the model config when it is None. Anchoring on the exact expression lost the+# fail-fast below on the first rewrite. Read without importing by tests/version_compat.+_GRPO_AUX_LOSS_ENABLED_LINE = r"^([ \t]*)self\.aux_loss_enabled = [^\n]+$"+_GRPO_AUX_LOSS_REJECT = (+ 'if self.aux_loss_enabled: raise NotImplementedError("Unsloth GRPO does not compute the MoE router '+ 'auxiliary loss; set router_aux_loss_coef = 0 (the Unsloth default).")'+)+def _guard_kbit_prep_against_peft_models():"""Stop TRL below 0.24.0 re-preparing a model Unsloth already prepared.@@ -2957,10 +2967,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):if trl_version >= Version("1.7.0"):# router_aux_loss_coef / aux_loss_enabled arrived in TRL 1.7.0, and the optimized GRPO forward cannot compute the MoE router aux loss, so reject an explicit opt-in at init.- RLTrainer_source = RLTrainer_source.replace(- "self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0",- "self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0\n"- ' if self.aux_loss_enabled: raise NotImplementedError("Unsloth GRPO does not compute the MoE router auxiliary loss; set router_aux_loss_coef = 0 (the Unsloth default).")',+ RLTrainer_source = re.sub(+ _GRPO_AUX_LOSS_ENABLED_LINE,+ lambda m: m.group(0) + "\n" + m.group(1) + _GRPO_AUX_LOSS_REJECT,+ RLTrainer_source,+ count = 1,+ flags = re.MULTILINE,)elif trl_version >= Version("0.27.0"):