Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Do not inject the GRPO prompt-trim block into a TRL that cannot host it (#11169)
Daniel Hanabc305650643
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.ymlindex 7bb66504e79..83abd62ad29 100644--- a/.github/workflows/version-compat-ci.yml+++ b/.github/workflows/version-compat-ci.yml@@ -309,6 +309,7 @@ jobs:cd unslothpython -c "import trl; print('Resolved TRL', trl.__version__)"PYTHONPATH=. python -m pytest \+ tests/version_compat/test_grpo_protected_tokens_guard.py \tests/version_compat/test_trl_grpo_fake_run.py \tests/version_compat/test_trl_fake_train_cpu.py \tests/version_compat/test_trl_padding_free_max_length.py \@@ -329,6 +330,7 @@ jobs:cd unslothpython -c "import trl; print('Resolved TRL', trl.__version__)"PYTHONPATH=. python -m pytest \+ tests/version_compat/test_grpo_protected_tokens_guard.py \tests/version_compat/test_trl_grpo_fake_run.py \tests/version_compat/test_trl_fake_train_cpu.py \tests/version_compat/test_trl_padding_free_max_length.py \diff --git a/tests/version_compat/test_grpo_protected_tokens_guard.py b/tests/version_compat/test_grpo_protected_tokens_guard.pynew file mode 100644index 00000000000..5323c87df36--- /dev/null+++ b/tests/version_compat/test_grpo_protected_tokens_guard.py@@ -0,0 +1,179 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""The injected GRPO prompt-trim block must only replace a block that can host it.++`grpo_trainer__generate_and_score_completions` in `unsloth/models/rl_replacements.py`+substitutes a TRL 0.20.0+ block -- it calls `truncate_with_protected_tokens` and reads+`self.pad_token`, `self.image_token` and three vision token ids -- into TRL's+`_generate_and_score_completions`. Driving the real substitution against each release's+actual source, the selecting regex matches 0.18.2 through 0.23.1 and stops matching at+0.24.0, while all four symbols exist only from 0.20.0, so on 0.18.2 and 0.19.1 (both inside+the declared window `trl>=0.18.2,!=0.19.0,<=0.24.0`) it replaced working native slicing with+names that do not resolve. TRL's own call to the helper inside the block being replaced is+the discriminator, so the substitution is gated on it; `getattr(..., None)` for the three ids+covers a fork or subclass that does not set one.++Reads the substitution's OUTPUT and executes the statements it emits, so it needs neither the+affected TRL nor a GPU.+"""++from __future__ import annotations++import importlib.util+import os+import re+import sys+from pathlib import Path++import pytest+++os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")+os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")++# daily-fresh-fetch collects this directory with only pytest installed.+if importlib.util.find_spec("torch") is None:+ pytest.skip("torch not installed", allow_module_level = True)++_SPOOF_DIR = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(_SPOOF_DIR))+import _zoo_aggressive_cuda_spoof as _spoof # noqa: E402++_spoof.apply()+++# Fixtures, not the installed TRL, so both groups are covered on a runner that has neither.+# Both satisfy the selecting regex and the two-statement rule; they differ only in the helper.+_TRL_0_18_BLOCK = """+ def _generate_and_score_completions(self, inputs):+ device = self.accelerator.device+ prompts = [x["prompt"] for x in inputs]+ if self.max_prompt_length is not None:+ prompt_ids = prompt_ids[:, -self.max_prompt_length :]+ prompt_mask = prompt_mask[:, -self.max_prompt_length :]+ if self.use_vllm:+ pass+ return inputs+"""++_TRL_0_22_BLOCK = """+ def _generate_and_score_completions(self, inputs):+ device = self.accelerator.device+ prompts = [x["prompt"] for x in inputs]+ if self.max_prompt_length is not None:+ protected = [self.image_token_id, self.vision_start_token_id, self.vision_end_token_id]+ protected = [token for token in protected if token is not None]+ prompt_ids, prompt_mask = truncate_with_protected_tokens(+ prompt_ids, prompt_mask, self.max_prompt_length, protected+ )+ prompts_text = [re.sub(rf"^({re.escape(self.pad_token)})+", "", text) for text in prompts_text]+ if self.use_vllm:+ pass+ return inputs+"""+++def _injected(source: str) -> str:+ from unsloth.models.rl_replacements import grpo_trainer__generate_and_score_completions+ return grpo_trainer__generate_and_score_completions("_generate_and_score_completions", source)+++class _TrainerWithoutVisionIds:+ """A stand-in for GRPOTrainer on a TRL that never sets the three ids."""++ max_prompt_length = 8+++def _protected_lines(emitted: str) -> str:+ """The emitted `protected = [...]` statements, dedented so they can be executed."""+ match = re.search(+ r"^(\s*)protected = \[.*?^\1protected = \[token for token in protected if token is not None\]",+ emitted,+ re.DOTALL | re.MULTILINE,+ )+ assert match, (+ "the substitution no longer emits the two `protected` statements this test drives. "+ "If the block was rewritten, retarget the test; do not delete it."+ )+ indent = match.group(1)+ return "\n".join(line[len(indent) :] for line in match.group(0).split("\n"))+++def test_the_pre_0_20_block_is_left_alone() -> None:+ """The regression: 0.18.2 and 0.19.1 have no truncate_with_protected_tokens, no+ self.pad_token and no self.image_token, so the injected block cannot run there. TRL's+ own slicing has to survive untouched."""+ source = _TRL_0_18_BLOCK+ emitted = _injected(source)+ assert "truncate_with_protected_tokens" not in emitted, (+ "the substitution fired on a block with no truncate_with_protected_tokens in it, so "+ "the emitted call has no definition to reach and training raises NameError"+ )+ assert (+ "prompt_ids = prompt_ids[:, -self.max_prompt_length :]" in emitted+ ), "TRL's own prompt slicing was removed on a release that cannot host the replacement"+ for absent in ("self.pad_token", "self.image_token", "protected"):+ assert absent not in emitted, f"{absent!r} was injected into a release that lacks it"+++def test_the_substitution_still_fires_on_the_0_20_shape() -> None:+ """Vacuity guard: everything below is about the emitted block, so a guard that stopped+ matching the supported shape would make all of it pass while testing nothing."""+ emitted = _injected(_TRL_0_22_BLOCK)+ assert 'getattr(self, "image_token_id", None)' in emitted, (+ "the prompt-trim substitution did not fire on the trl 0.20.0+ block shape, so the "+ "rest of this file proves nothing about it"+ )+++def test_the_emitted_block_runs_without_the_vision_token_ids() -> None:+ """Belt and braces: a trainer that sets none of the three must not raise."""+ lines = _protected_lines(_injected(_TRL_0_22_BLOCK))+ scope = {"self": _TrainerWithoutVisionIds()}+ exec(lines, scope) # noqa: S102 - executing our own generated source is the point+ assert scope["protected"] == [], (+ f"with none of the three ids set, nothing should be protected, got "+ f"{scope['protected']!r}"+ )+++def test_the_ids_are_still_protected_when_trl_does_set_them() -> None:+ """The guard must not quietly stop protecting vision tokens where they exist."""++ class _TrainerWithVisionIds:+ max_prompt_length = 8+ image_token_id = 151655+ vision_start_token_id = 151652+ vision_end_token_id = 151653++ lines = _protected_lines(_injected(_TRL_0_22_BLOCK))+ scope = {"self": _TrainerWithVisionIds()}+ exec(lines, scope) # noqa: S102+ assert scope["protected"] == [151655, 151652, 151653], (+ f"the vision token ids must still reach truncate_with_protected_tokens, got "+ f"{scope['protected']!r}"+ )+++def test_one_missing_id_does_not_discard_the_others() -> None:+ """Partial support is the realistic case, and the None filter has to survive it."""++ class _TrainerWithOneId:+ max_prompt_length = 8+ image_token_id = 151655++ lines = _protected_lines(_injected(_TRL_0_22_BLOCK))+ scope = {"self": _TrainerWithOneId()}+ exec(lines, scope) # noqa: S102+ assert scope["protected"] == [151655], f"got {scope['protected']!r}"+++def test_bare_attribute_access_would_still_have_failed() -> None:+ """Negative control: the three tests above pass trivially if the emitted lines stopped+ reading the ids at all, so prove the pre-fix spelling really does raise here."""+ pre_fix = (+ "protected = [self.image_token_id, self.vision_start_token_id, self.vision_end_token_id]\n"+ "protected = [token for token in protected if token is not None]\n"+ )+ with pytest.raises(AttributeError, match = "image_token_id"):+ exec(pre_fix, {"self": _TrainerWithoutVisionIds()}) # noqa: S102diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex d57473b18d2..73ee531c422 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -1095,12 +1095,24 @@ def grpo_trainer__generate_and_score_completions(function_name, function):if (sum(re.match(rf"{spacing}[^\s]", x) is not None for x in splits) == 2and len(spacing) >= 8+ # The regex above also matches trl 0.18.2 and 0.19.1, which the declared window+ # admits and which have none of the symbols the replacement below emits; TRL's own+ # call to the helper is the discriminator (0.20.0-0.23.1 have it, those two do not),+ # and without it TRL's native slicing must stay. tests/version_compat/+ # test_grpo_protected_tokens_guard.py measures this per version.+ and "truncate_with_protected_tokens" in replace_part):+ # getattr: the next emitted line drops None, so an id a fork or subclass leaves+ # unset means "nothing to protect" rather than AttributeError in generated code.new_replacement = f"""\n{spacing}if self.max_prompt_length is not None:# If max_prompt_length is set, we trim the prompt to keep only the last `max_prompt_length` tokens.# Then we decode those tokens back into text. We manually remove leading pad tokens from the decoded text,# because we can't use `skip_special_tokens=True` (some special tokens are still needed for generation).- protected = [self.image_token_id, self.vision_start_token_id, self.vision_end_token_id]+ protected = [+ getattr(self, "image_token_id", None),+ getattr(self, "vision_start_token_id", None),+ getattr(self, "vision_end_token_id", None),+ ]protected = [token for token in protected if token is not None]prompt_ids, prompt_mask = truncate_with_protected_tokens(prompt_ids, prompt_mask, self.max_prompt_length, protected