Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Support TRL 1.14 and 1.15: ORPO/CPO row cap, masked GRPO sampling metrics, cap to 1.15.0 (#13203)
Daniel Han3b865bade353
diff --git a/.github/workflows/notebooks-ci.yml b/.github/workflows/notebooks-ci.ymlindex e9f46bb2321..bd7e996d3e8 100644--- a/.github/workflows/notebooks-ci.yml+++ b/.github/workflows/notebooks-ci.yml@@ -354,7 +354,7 @@ jobs:# Keep in lockstep with PIN_TRL / PIN_TRANSFORMERS in unslothai/notebooks/update_all_notebooks.py# and pyproject's caps. triton: `import unsloth` needs it at module scope.# datasets must exclude 4.4.0-4.5.0 (patch_datasets raises NotImplementedError there).- pip install 'transformers>=4.56,<=5.17.0' 'trl>=0.22,<=1.13.0' 'accelerate>=1.0' \+ pip install 'transformers>=4.56,<=5.17.0' 'trl>=0.22,<=1.15.0' 'accelerate>=1.0' \'datasets>=3.4.1,!=4.0.*,!=4.1.0,!=4.4.*,!=4.5.0,<5.0.0' 'peft>=0.15,<0.20' \'bitsandbytes>=0.43' 'sentencepiece' 'protobuf' triton \Pillow safetensors tqdm packaging psutildiff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.ymlindex 2751c26aaef..803e8c43d90 100644--- a/.github/workflows/version-compat-ci.yml+++ b/.github/workflows/version-compat-ci.yml@@ -156,7 +156,7 @@ jobs:torch_pins: >-torch torchvision 'torchcodec>=0.12'pkg_pins: >-- 'transformers==5.17.0' 'trl==1.13.0'+ 'transformers==5.17.0' 'trl==1.15.0'peft sentence-transformers accelerate 'datasets>=3.4.1,!=4.0.*,!=4.1.0,!=4.4.*,!=4.5.0,<5.0.0'bitsandbytes triton hf_transfer- slug: latestdiff --git a/pyproject.toml b/pyproject.tomlindex 8048c430517..a8f519631d8 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -259,7 +259,7 @@ huggingfacenotorch = [# Ceiling = newest release the version matrix ran. Each `!=` was tested and rejected: keep them.# Apple Silicon stays at 5.5.0 via unsloth_zoo's marker-split requirement."transformers>=4.52.4,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.17.0",- "trl>=0.18.2,!=0.19.0,<=1.13.0",+ "trl>=0.18.2,!=0.19.0,<=1.15.0","sentence-transformers",]# torchcodec backend for Gemma audio / datasets>=4 (#7225).@@ -988,7 +988,7 @@ colab-new = []colab-no-deps = ["accelerate>=0.34.1",- "trl>=0.18.2,!=0.19.0,<=1.13.0",+ "trl>=0.18.2,!=0.19.0,<=1.15.0","peft>=0.18.0","xformers ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')","bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0",diff --git a/studio/backend/requirements/no-torch-runtime.txt b/studio/backend/requirements/no-torch-runtime.txtindex b3de86b0479..ebec0404ffe 100644--- a/studio/backend/requirements/no-torch-runtime.txt+++ b/studio/backend/requirements/no-torch-runtime.txt@@ -88,7 +88,7 @@ h11==0.16.0tokenizers>=0.22.0,<=0.23.0transformers==5.5.0; python_version >= "3.10"transformers==4.57.6; python_version < "3.10"-trl>=0.18.2,!=0.19.0,<=1.13.0+trl>=0.18.2,!=0.19.0,<=1.15.0# Matches extras-no-deps.txt rather than the newest release: that file installs# --no-deps too and runs after this one, so whatever it names wins. Pinning a# higher version here would record a number no install ever ends up with. Bumpdiff --git a/tests/test_grpo_is_metrics_mask.py b/tests/test_grpo_is_metrics_mask.pynew file mode 100644index 00000000000..0a5939d4c25--- /dev/null+++ b/tests/test_grpo_is_metrics_mask.py@@ -0,0 +1,85 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""GRPO `sampling/*` metrics must reduce over the loss mask, as TRL's _generate_and_score_completions does."""++from __future__ import annotations++import ast+import importlib.util+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")++RL_REPLACEMENTS = (+ Path(importlib.util.find_spec("unsloth").origin).parent / "models" / "rl_replacements.py"+)+++def _helper():+ tree = ast.parse(RL_REPLACEMENTS.read_text(encoding = "utf-8"))+ nodes = [+ n+ for n in tree.body+ if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_grpo_is_metric_values"+ ]+ assert len(nodes) == 1, "no _unsloth_grpo_is_metric_values in rl_replacements.py"+ ns = {"torch": torch}+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(RL_REPLACEMENTS), "exec"), ns)+ return ns["_unsloth_grpo_is_metric_values"]+++def _batch(sequence_level):+ torch.manual_seed(0)+ mask = torch.tensor(+ [[1, 1, 1, 0, 0], [1, 1, 1, 1, 1], [1, 0, 0, 0, 0], [0, 0, 0, 0, 0]], dtype = torch.float32+ )+ old = torch.randn(4, 5) * 0.1 - 2.0+ sampling = old + torch.randn(4, 5) * 0.05+ log_ratio = (old - sampling) * mask+ if sequence_level:+ ratio = torch.exp(log_ratio.sum(-1, keepdim = True))+ else:+ ratio = torch.exp(log_ratio)+ return mask, old, sampling, ratio+++def _trl_reference(mask, old, sampling, ratio, sequence_level):+ # TRL 1.15 grpo_trainer.py `if self.use_vllm and self.vllm_importance_sampling_correction:` block.+ keep = mask.bool()+ delta = torch.abs(old - sampling)+ delta = delta[keep & ~torch.isnan(delta)]+ flat = ratio.flatten() if sequence_level else ratio[keep]+ return delta, flat+++def _unsloth_inputs(mask, old, sampling, ratio):+ # What grpo_compute_loss returns: both zero-filled outside the mask, ratio broadcast to (B, T).+ return torch.abs(old - sampling) * mask, ratio * mask++[email protected]("sequence_level", [False, True])+def test_metric_values_match_trl(sequence_level):+ mask, old, sampling, ratio = _batch(sequence_level)+ delta, flat = _unsloth_inputs(mask, old, sampling, ratio)+ got_delta, got_ratio = _helper()(delta, flat, mask, sequence_level)+ want_delta, want_ratio = _trl_reference(mask, old, sampling, ratio, sequence_level)+ for reduce in (torch.mean, torch.min, torch.max):+ assert torch.allclose(reduce(got_delta), reduce(want_delta))+ assert torch.allclose(reduce(got_ratio), reduce(want_ratio))+++def test_unfiltered_values_read_zero_min_ratio():+ # Control: reducing the zero-filled tensors directly is what the metrics did before.+ mask, old, sampling, ratio = _batch(False)+ delta, flat = _unsloth_inputs(mask, old, sampling, ratio)+ assert flat.min().item() == 0.0+ assert _helper()(delta, flat, mask, False)[1].min().item() > 0.5+++def test_shape_mismatch_keeps_the_old_reduction():+ mask, old, sampling, ratio = _batch(False)+ delta, flat = _unsloth_inputs(mask, old, sampling, ratio)+ got_delta, got_ratio = _helper()(delta, flat, mask[:, :3], False)+ assert got_delta.numel() == delta.numel() and got_ratio.numel() == flat.numel()diff --git a/tests/test_trl_cap_sites.py b/tests/test_trl_cap_sites.pyindex 58edf7932e8..baccf549650 100644--- a/tests/test_trl_cap_sites.py+++ b/tests/test_trl_cap_sites.py@@ -21,16 +21,22 @@ RUNTIME_MIRROR = REPO / "studio" / "backend" / "requirements" / "no-torch-runtimRL_REPLACEMENTS = REPO / "unsloth" / "models" / "rl_replacements.py"# Newest TRL the matrix was run against; moving it means re-running the sweep first.-TESTED_CEILING = Version("1.13.0")+TESTED_CEILING = Version("1.15.0")REJECTED = ("0.19.0",)-NEWLY_ADMITTED = ("0.29.1", "1.0.0", "1.6.0", "1.7.0", "1.13.0")+NEWLY_ADMITTED = ("0.29.1", "1.0.0", "1.6.0", "1.7.0", "1.13.0", "1.14.2", "1.15.0")ZOO_TRL_CEILING_BEFORE_THE_LIFT = Version("0.24.0")ZOO_FLOOR_WITH_LIFTED_TRL_CAP = Version("2026.9.5")+# unsloth_zoo 2026.9.5+ caps trl here.+ZOO_TRL_CEILING_OF_LIFTED_FLOOR = Version("1.13.0")++# First published unsloth_zoo admitting TESTED_CEILING; None defers the gate until it ships.+ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING = None+# Every unsloth_zoo up to 2026.9.7 caps datasets here; pip intersects, so users stay under it.ZOO_DATASETS_CEILING_BEFORE_THE_LIFT = SpecifierSet("<4.4.0")@@ -806,6 +812,15 @@ def test_the_recorded_trl_datasets_floors_still_match_pypi() -> None:def _newest_published_zoo_datasets_windows(timeout: float = 10.0):"""(zoo version, datasets windows) for the newest unsloth_zoo on PyPI, else None; never raises."""+ return _newest_published_zoo_windows("datasets", timeout)+++def _newest_published_zoo_trl_windows(timeout: float = 10.0):+ """(zoo version, trl windows) for the newest unsloth_zoo on PyPI, else None; never raises."""+ return _newest_published_zoo_windows("trl", timeout)+++def _newest_published_zoo_windows(name: str, timeout: float = 10.0):import jsonimport urllib.errorimport urllib.request@@ -829,7 +844,7 @@ def _newest_published_zoo_datasets_windows(timeout: float = 10.0):req = Requirement(raw)except InvalidRequirement:continue- if req.name.lower() != "datasets" or not req.specifier:+ if req.name.lower() != name or not req.specifier:continuewindows.append(req.specifier)if not windows:@@ -908,3 +923,78 @@ def test_the_zoo_datasets_expiry_can_fail(monkeypatch) -> None:monkeypatch.setattr(module, "_newest_published_zoo_datasets_windows", lambda timeout = 10.0: None)with pytest.raises(pytest.skip.Exception):test_the_zoo_datasets_deferral_expires_when_the_zoo_release_ships()+++def test_the_declared_zoo_floor_admits_the_tested_trl_ceiling() -> None:+ """pip gives the user the LOWER of our and the zoo's trl ceilings; deferred while the zoo lift is unreleased."""+ if TESTED_CEILING <= ZOO_TRL_CEILING_OF_LIFTED_FLOOR:+ pytest.skip("the zoo floor already admits the tested ceiling")+ if ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING is None:+ pytest.skip(+ f"deferred: no published unsloth_zoo admits trl {TESTED_CEILING} yet, so pip keeps "+ f"users at {ZOO_TRL_CEILING_OF_LIFTED_FLOOR}. Re-enable by setting "+ f"ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING to the release that carries it."+ )+ reqs = _pyproject_zoo()+ assert reqs, "pyproject.toml names no versioned unsloth_zoo requirement at all"+ stale = {}+ for req in reqs:+ lower = [+ Version(str(spec.version))+ for spec in req.specifier+ if spec.operator in (">=", "==", "~=")+ ]+ if not lower or max(lower) < ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING:+ stale[str(req)] = str(max(lower)) if lower else "none"+ assert not stale, (+ f"pyproject.toml admits trl up to {TESTED_CEILING} while still accepting unsloth_zoo "+ f"{stale}, whose own cap is {ZOO_TRL_CEILING_OF_LIFTED_FLOOR}. Raise the floor to "+ f"{ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING} in the same commit."+ )+++def test_the_zoo_trl_deferral_expires_when_the_zoo_release_ships() -> None:+ """Self-expiring deferral: fails only on positive PyPI evidence the zoo now admits TESTED_CEILING."""+ if TESTED_CEILING <= ZOO_TRL_CEILING_OF_LIFTED_FLOOR:+ pytest.skip("nothing deferred")+ if ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING is not None:+ pytest.skip(+ "the floor already names a zoo admitting the ceiling, so the gate above is live"+ )+ published = _newest_published_zoo_trl_windows()+ if published is None:+ pytest.skip("PyPI could not be asked for unsloth_zoo, so the deferral stands")+ zoo_version, zoo_windows = published+ covering = [str(w) for w in zoo_windows if w.contains(str(TESTED_CEILING))]+ assert not covering, (+ f"unsloth_zoo {zoo_version} is published and admits trl {TESTED_CEILING} "+ f"({', '.join(covering)}), so the deferral is over. Set "+ f"ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING to {zoo_version} and raise the unsloth_zoo floor "+ f"in pyproject.toml to it in the same commit."+ )+++def test_the_zoo_trl_expiry_can_fail(monkeypatch) -> None:+ """NEGATIVE CONTROL: expiry fires on a covering zoo and stays silent on a capped one or no PyPI."""+ module = sys.modules[__name__]+ monkeypatch.setattr(module, "ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING", None)+ monkeypatch.setattr(module, "TESTED_CEILING", Version("1.15.0"))+ monkeypatch.setattr(+ module,+ "_newest_published_zoo_trl_windows",+ lambda timeout = 10.0: (Version("2026.10.9"), [SpecifierSet(">=0.18.2,!=0.19.0,<=1.15.0")]),+ )+ with pytest.raises(AssertionError) as raised:+ test_the_zoo_trl_deferral_expires_when_the_zoo_release_ships()+ assert "2026.10.9" in str(raised.value)++ monkeypatch.setattr(+ module,+ "_newest_published_zoo_trl_windows",+ lambda timeout = 10.0: (Version("2026.10.3"), [SpecifierSet(">=0.18.2,!=0.19.0,<=1.13.0")]),+ )+ test_the_zoo_trl_deferral_expires_when_the_zoo_release_ships()++ monkeypatch.setattr(module, "_newest_published_zoo_trl_windows", lambda timeout = 10.0: None)+ with pytest.raises(pytest.skip.Exception):+ test_the_zoo_trl_deferral_expires_when_the_zoo_release_ships()diff --git a/tests/version_compat/test_orpo_cpo_long_rows.py b/tests/version_compat/test_orpo_cpo_long_rows.pyindex 19f4180be1d..2dc2b721f17 100644--- a/tests/version_compat/test_orpo_cpo_long_rows.py+++ b/tests/version_compat/test_orpo_cpo_long_rows.py@@ -88,6 +88,8 @@ def _tokenize_row(trainer, patched):assert new != src, "row cap anchor not found in this TRL tokenize_row"src = newns = dict(vars(module))+ # TRL 1.15+ warns through accelerate's logger, which needs a PartialState this harness never builds.+ ns["logger"] = SimpleNamespace(warning_once = lambda *a, **k: None, warning = lambda *a, **k: None)exec(src, ns)exec(textwrap.dedent(methods["build_tokenized_answer"]), ns)fake = SimpleNamespace(@@ -154,3 +156,11 @@ def test_unpatched_trl_row_overflows(name):prompt, chosen, rejected = LONG_ROWS["long_prompt"]out = _tokenize_row(trainer, False)({"prompt": prompt, "chosen": chosen, "rejected": rejected})assert len(out["chosen_input_ids"]) > 32++[email protected]("name", ["orpo", "cpo"])+def test_the_row_cap_anchors_on_this_trl(name):+ src = textwrap.dedent(_trainer(name)[1]["tokenize_row"])+ if "max_prompt_length" in src:+ pytest.skip("this TRL truncates the prompt itself")+ assert "_unsloth_ul" in _row_cap()("tokenize_row", src)diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex bdb76720637..50fcf7bc8b6 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -960,6 +960,13 @@ def orpo_trainer_row_cap(function_name, function):return function# Before TRL's own response cut: its negative slice end can empty the shorter answer.match = re.search(r"(?m)^([ \t]*)longer_response_length = max\(", function)+ if match is None:+ # TRL 1.15+ cuts answers but never the prompt, so a long prompt still overflows max_length.+ match = re.search(+ r"(?m)^([ \t]*)for answer_tokens in \[chosen_tokens, rejected_tokens\]:\n"+ r"\1[ \t]+if len\(answer_tokens\[\"prompt_input_ids\"\]\) \+ len\(answer_tokens\[\"input_ids\"\]\) > self\.max_length:",+ function,+ )if match is None:return functionindent = match.group(1)@@ -3159,6 +3166,23 @@ def grpo_update_SamplingParams(return generation_kwargs+def _unsloth_grpo_is_metric_values(delta, flat_is_ratio, mask, sequence_level):+ # Inputs are zero-filled outside the mask; TRL reduces over masked tokens only, else min ratio reads 0.+ if mask is None or delta.shape != mask.shape or flat_is_ratio.shape != mask.shape:+ return delta.reshape(-1), flat_is_ratio.reshape(-1)+ keep = mask.to(torch.bool)+ if not sequence_level:+ return delta[keep], flat_is_ratio[keep]+ # A row with no kept tokens is exp(0) = 1, as in TRL.+ counts = keep.sum(dim = -1)+ per_row = torch.where(+ counts > 0,+ (flat_is_ratio * keep).sum(dim = -1) / counts.clamp(min = 1),+ torch.ones_like(counts, dtype = flat_is_ratio.dtype),+ )+ return delta[keep], per_row++grpo_compute_loss = RL_REPLACEMENTS["grpo_compute_loss"]grpo_compute_loss_slow = RL_REPLACEMENTS["grpo_compute_loss_slow"]UnslothEfficientGRPO = RL_REPLACEMENTS["UnslothEfficientGRPO"]@@ -3174,6 +3198,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_hidden_stateRL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_mm_token_id))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_fix_mm_token_type_ids))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_accumulation_steps))+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_is_metric_values))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_vision_inputs))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_split_vision_by_sample))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_unsplit_vision))@@ -3541,6 +3566,13 @@ def grpo_trainer_compute_loss(function_name, function):and delta is not Noneand getattr(self, "vllm_importance_sampling_correction", False)):+ delta, flat_is_ratio = _unsloth_grpo_is_metric_values(+ delta,+ flat_is_ratio,+ completion_mask,+ (getattr(self.args, "vllm_importance_sampling_mode", None) or "token_truncate")+ in ("sequence_mask", "sequence_truncate"),+ )mean_delta = (torch.mean(delta)if delta.numel() > 0@@ -3766,7 +3798,6 @@ def openenv_vllm_reload_weights():returnif Version(importlib_version("trl")) < Version("0.26.0"):return-try:import trl.experimental.openenv.utils as openenv_utilsimport trl.experimental.openenv as openenv