Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Support TRL 1.14 and 1.15: ORPO/CPO row cap, masked GRPO sampling metrics, cap to 1.15.0 (#13203)

Daniel Han3b865bade353
diff --git a/.github/workflows/notebooks-ci.yml b/.github/workflows/notebooks-ci.yml
index e9f46bb2321..bd7e996d3e8 100644
--- a/.github/workflows/notebooks-ci.yml
+++ b/.github/workflows/notebooks-ci.yml
@@ -354,7 +354,7 @@ jobs:
# Keep in lockstep with PIN_TRL / PIN_TRANSFORMERS in unslothai/notebooks/update_all_notebooks.py
# and pyproject's caps. triton: `import unsloth` needs it at module scope.
# datasets must exclude 4.4.0-4.5.0 (patch_datasets raises NotImplementedError there).
- pip install 'transformers>=4.56,<=5.17.0' 'trl>=0.22,<=1.13.0' 'accelerate>=1.0' \
+ pip install 'transformers>=4.56,<=5.17.0' 'trl>=0.22,<=1.15.0' 'accelerate>=1.0' \
'datasets>=3.4.1,!=4.0.*,!=4.1.0,!=4.4.*,!=4.5.0,<5.0.0' 'peft>=0.15,<0.20' \
'bitsandbytes>=0.43' 'sentencepiece' 'protobuf' triton \
Pillow safetensors tqdm packaging psutil
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
index 2751c26aaef..803e8c43d90 100644
--- a/.github/workflows/version-compat-ci.yml
+++ b/.github/workflows/version-compat-ci.yml
@@ -156,7 +156,7 @@ jobs:
torch_pins: >-
torch torchvision 'torchcodec>=0.12'
pkg_pins: >-
- 'transformers==5.17.0' 'trl==1.13.0'
+ 'transformers==5.17.0' 'trl==1.15.0'
peft sentence-transformers accelerate 'datasets>=3.4.1,!=4.0.*,!=4.1.0,!=4.4.*,!=4.5.0,<5.0.0'
bitsandbytes triton hf_transfer
- slug: latest
diff --git a/pyproject.toml b/pyproject.toml
index 8048c430517..a8f519631d8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -259,7 +259,7 @@ huggingfacenotorch = [
# Ceiling = newest release the version matrix ran. Each `!=` was tested and rejected: keep them.
# Apple Silicon stays at 5.5.0 via unsloth_zoo's marker-split requirement.
"transformers>=4.52.4,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.17.0",
- "trl>=0.18.2,!=0.19.0,<=1.13.0",
+ "trl>=0.18.2,!=0.19.0,<=1.15.0",
"sentence-transformers",
]
# torchcodec backend for Gemma audio / datasets>=4 (#7225).
@@ -988,7 +988,7 @@ colab-new = [
]
colab-no-deps = [
"accelerate>=0.34.1",
- "trl>=0.18.2,!=0.19.0,<=1.13.0",
+ "trl>=0.18.2,!=0.19.0,<=1.15.0",
"peft>=0.18.0",
"xformers ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')",
"bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0",
diff --git a/studio/backend/requirements/no-torch-runtime.txt b/studio/backend/requirements/no-torch-runtime.txt
index b3de86b0479..ebec0404ffe 100644
--- a/studio/backend/requirements/no-torch-runtime.txt
+++ b/studio/backend/requirements/no-torch-runtime.txt
@@ -88,7 +88,7 @@ h11==0.16.0
tokenizers>=0.22.0,<=0.23.0
transformers==5.5.0; python_version >= "3.10"
transformers==4.57.6; python_version < "3.10"
-trl>=0.18.2,!=0.19.0,<=1.13.0
+trl>=0.18.2,!=0.19.0,<=1.15.0
# Matches extras-no-deps.txt rather than the newest release: that file installs
# --no-deps too and runs after this one, so whatever it names wins. Pinning a
# higher version here would record a number no install ever ends up with. Bump
diff --git a/tests/test_grpo_is_metrics_mask.py b/tests/test_grpo_is_metrics_mask.py
new file mode 100644
index 00000000000..0a5939d4c25
--- /dev/null
+++ b/tests/test_grpo_is_metrics_mask.py
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""GRPO `sampling/*` metrics must reduce over the loss mask, as TRL's _generate_and_score_completions does."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+RL_REPLACEMENTS = (
+ Path(importlib.util.find_spec("unsloth").origin).parent / "models" / "rl_replacements.py"
+)
+
+
+def _helper():
+ tree = ast.parse(RL_REPLACEMENTS.read_text(encoding = "utf-8"))
+ nodes = [
+ n
+ for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_grpo_is_metric_values"
+ ]
+ assert len(nodes) == 1, "no _unsloth_grpo_is_metric_values in rl_replacements.py"
+ ns = {"torch": torch}
+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(RL_REPLACEMENTS), "exec"), ns)
+ return ns["_unsloth_grpo_is_metric_values"]
+
+
+def _batch(sequence_level):
+ torch.manual_seed(0)
+ mask = torch.tensor(
+ [[1, 1, 1, 0, 0], [1, 1, 1, 1, 1], [1, 0, 0, 0, 0], [0, 0, 0, 0, 0]], dtype = torch.float32
+ )
+ old = torch.randn(4, 5) * 0.1 - 2.0
+ sampling = old + torch.randn(4, 5) * 0.05
+ log_ratio = (old - sampling) * mask
+ if sequence_level:
+ ratio = torch.exp(log_ratio.sum(-1, keepdim = True))
+ else:
+ ratio = torch.exp(log_ratio)
+ return mask, old, sampling, ratio
+
+
+def _trl_reference(mask, old, sampling, ratio, sequence_level):
+ # TRL 1.15 grpo_trainer.py `if self.use_vllm and self.vllm_importance_sampling_correction:` block.
+ keep = mask.bool()
+ delta = torch.abs(old - sampling)
+ delta = delta[keep & ~torch.isnan(delta)]
+ flat = ratio.flatten() if sequence_level else ratio[keep]
+ return delta, flat
+
+
+def _unsloth_inputs(mask, old, sampling, ratio):
+ # What grpo_compute_loss returns: both zero-filled outside the mask, ratio broadcast to (B, T).
+ return torch.abs(old - sampling) * mask, ratio * mask
+
+
[email protected]("sequence_level", [False, True])
+def test_metric_values_match_trl(sequence_level):
+ mask, old, sampling, ratio = _batch(sequence_level)
+ delta, flat = _unsloth_inputs(mask, old, sampling, ratio)
+ got_delta, got_ratio = _helper()(delta, flat, mask, sequence_level)
+ want_delta, want_ratio = _trl_reference(mask, old, sampling, ratio, sequence_level)
+ for reduce in (torch.mean, torch.min, torch.max):
+ assert torch.allclose(reduce(got_delta), reduce(want_delta))
+ assert torch.allclose(reduce(got_ratio), reduce(want_ratio))
+
+
+def test_unfiltered_values_read_zero_min_ratio():
+ # Control: reducing the zero-filled tensors directly is what the metrics did before.
+ mask, old, sampling, ratio = _batch(False)
+ delta, flat = _unsloth_inputs(mask, old, sampling, ratio)
+ assert flat.min().item() == 0.0
+ assert _helper()(delta, flat, mask, False)[1].min().item() > 0.5
+
+
+def test_shape_mismatch_keeps_the_old_reduction():
+ mask, old, sampling, ratio = _batch(False)
+ delta, flat = _unsloth_inputs(mask, old, sampling, ratio)
+ got_delta, got_ratio = _helper()(delta, flat, mask[:, :3], False)
+ assert got_delta.numel() == delta.numel() and got_ratio.numel() == flat.numel()
diff --git a/tests/test_trl_cap_sites.py b/tests/test_trl_cap_sites.py
index 58edf7932e8..baccf549650 100644
--- a/tests/test_trl_cap_sites.py
+++ b/tests/test_trl_cap_sites.py
@@ -21,16 +21,22 @@ RUNTIME_MIRROR = REPO / "studio" / "backend" / "requirements" / "no-torch-runtim
RL_REPLACEMENTS = REPO / "unsloth" / "models" / "rl_replacements.py"
# Newest TRL the matrix was run against; moving it means re-running the sweep first.
-TESTED_CEILING = Version("1.13.0")
+TESTED_CEILING = Version("1.15.0")
REJECTED = ("0.19.0",)
-NEWLY_ADMITTED = ("0.29.1", "1.0.0", "1.6.0", "1.7.0", "1.13.0")
+NEWLY_ADMITTED = ("0.29.1", "1.0.0", "1.6.0", "1.7.0", "1.13.0", "1.14.2", "1.15.0")
ZOO_TRL_CEILING_BEFORE_THE_LIFT = Version("0.24.0")
ZOO_FLOOR_WITH_LIFTED_TRL_CAP = Version("2026.9.5")
+# unsloth_zoo 2026.9.5+ caps trl here.
+ZOO_TRL_CEILING_OF_LIFTED_FLOOR = Version("1.13.0")
+
+# First published unsloth_zoo admitting TESTED_CEILING; None defers the gate until it ships.
+ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING = None
+
# Every unsloth_zoo up to 2026.9.7 caps datasets here; pip intersects, so users stay under it.
ZOO_DATASETS_CEILING_BEFORE_THE_LIFT = SpecifierSet("<4.4.0")
@@ -806,6 +812,15 @@ def test_the_recorded_trl_datasets_floors_still_match_pypi() -> None:
def _newest_published_zoo_datasets_windows(timeout: float = 10.0):
"""(zoo version, datasets windows) for the newest unsloth_zoo on PyPI, else None; never raises."""
+ return _newest_published_zoo_windows("datasets", timeout)
+
+
+def _newest_published_zoo_trl_windows(timeout: float = 10.0):
+ """(zoo version, trl windows) for the newest unsloth_zoo on PyPI, else None; never raises."""
+ return _newest_published_zoo_windows("trl", timeout)
+
+
+def _newest_published_zoo_windows(name: str, timeout: float = 10.0):
import json
import urllib.error
import urllib.request
@@ -829,7 +844,7 @@ def _newest_published_zoo_datasets_windows(timeout: float = 10.0):
req = Requirement(raw)
except InvalidRequirement:
continue
- if req.name.lower() != "datasets" or not req.specifier:
+ if req.name.lower() != name or not req.specifier:
continue
windows.append(req.specifier)
if not windows:
@@ -908,3 +923,78 @@ def test_the_zoo_datasets_expiry_can_fail(monkeypatch) -> None:
monkeypatch.setattr(module, "_newest_published_zoo_datasets_windows", lambda timeout = 10.0: None)
with pytest.raises(pytest.skip.Exception):
test_the_zoo_datasets_deferral_expires_when_the_zoo_release_ships()
+
+
+def test_the_declared_zoo_floor_admits_the_tested_trl_ceiling() -> None:
+ """pip gives the user the LOWER of our and the zoo's trl ceilings; deferred while the zoo lift is unreleased."""
+ if TESTED_CEILING <= ZOO_TRL_CEILING_OF_LIFTED_FLOOR:
+ pytest.skip("the zoo floor already admits the tested ceiling")
+ if ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING is None:
+ pytest.skip(
+ f"deferred: no published unsloth_zoo admits trl {TESTED_CEILING} yet, so pip keeps "
+ f"users at {ZOO_TRL_CEILING_OF_LIFTED_FLOOR}. Re-enable by setting "
+ f"ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING to the release that carries it."
+ )
+ reqs = _pyproject_zoo()
+ assert reqs, "pyproject.toml names no versioned unsloth_zoo requirement at all"
+ stale = {}
+ for req in reqs:
+ lower = [
+ Version(str(spec.version))
+ for spec in req.specifier
+ if spec.operator in (">=", "==", "~=")
+ ]
+ if not lower or max(lower) < ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING:
+ stale[str(req)] = str(max(lower)) if lower else "none"
+ assert not stale, (
+ f"pyproject.toml admits trl up to {TESTED_CEILING} while still accepting unsloth_zoo "
+ f"{stale}, whose own cap is {ZOO_TRL_CEILING_OF_LIFTED_FLOOR}. Raise the floor to "
+ f"{ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING} in the same commit."
+ )
+
+
+def test_the_zoo_trl_deferral_expires_when_the_zoo_release_ships() -> None:
+ """Self-expiring deferral: fails only on positive PyPI evidence the zoo now admits TESTED_CEILING."""
+ if TESTED_CEILING <= ZOO_TRL_CEILING_OF_LIFTED_FLOOR:
+ pytest.skip("nothing deferred")
+ if ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING is not None:
+ pytest.skip(
+ "the floor already names a zoo admitting the ceiling, so the gate above is live"
+ )
+ published = _newest_published_zoo_trl_windows()
+ if published is None:
+ pytest.skip("PyPI could not be asked for unsloth_zoo, so the deferral stands")
+ zoo_version, zoo_windows = published
+ covering = [str(w) for w in zoo_windows if w.contains(str(TESTED_CEILING))]
+ assert not covering, (
+ f"unsloth_zoo {zoo_version} is published and admits trl {TESTED_CEILING} "
+ f"({', '.join(covering)}), so the deferral is over. Set "
+ f"ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING to {zoo_version} and raise the unsloth_zoo floor "
+ f"in pyproject.toml to it in the same commit."
+ )
+
+
+def test_the_zoo_trl_expiry_can_fail(monkeypatch) -> None:
+ """NEGATIVE CONTROL: expiry fires on a covering zoo and stays silent on a capped one or no PyPI."""
+ module = sys.modules[__name__]
+ monkeypatch.setattr(module, "ZOO_FLOOR_ADMITTING_TESTED_TRL_CEILING", None)
+ monkeypatch.setattr(module, "TESTED_CEILING", Version("1.15.0"))
+ monkeypatch.setattr(
+ module,
+ "_newest_published_zoo_trl_windows",
+ lambda timeout = 10.0: (Version("2026.10.9"), [SpecifierSet(">=0.18.2,!=0.19.0,<=1.15.0")]),
+ )
+ with pytest.raises(AssertionError) as raised:
+ test_the_zoo_trl_deferral_expires_when_the_zoo_release_ships()
+ assert "2026.10.9" in str(raised.value)
+
+ monkeypatch.setattr(
+ module,
+ "_newest_published_zoo_trl_windows",
+ lambda timeout = 10.0: (Version("2026.10.3"), [SpecifierSet(">=0.18.2,!=0.19.0,<=1.13.0")]),
+ )
+ test_the_zoo_trl_deferral_expires_when_the_zoo_release_ships()
+
+ monkeypatch.setattr(module, "_newest_published_zoo_trl_windows", lambda timeout = 10.0: None)
+ with pytest.raises(pytest.skip.Exception):
+ test_the_zoo_trl_deferral_expires_when_the_zoo_release_ships()
diff --git a/tests/version_compat/test_orpo_cpo_long_rows.py b/tests/version_compat/test_orpo_cpo_long_rows.py
index 19f4180be1d..2dc2b721f17 100644
--- a/tests/version_compat/test_orpo_cpo_long_rows.py
+++ b/tests/version_compat/test_orpo_cpo_long_rows.py
@@ -88,6 +88,8 @@ def _tokenize_row(trainer, patched):
assert new != src, "row cap anchor not found in this TRL tokenize_row"
src = new
ns = dict(vars(module))
+ # TRL 1.15+ warns through accelerate's logger, which needs a PartialState this harness never builds.
+ ns["logger"] = SimpleNamespace(warning_once = lambda *a, **k: None, warning = lambda *a, **k: None)
exec(src, ns)
exec(textwrap.dedent(methods["build_tokenized_answer"]), ns)
fake = SimpleNamespace(
@@ -154,3 +156,11 @@ def test_unpatched_trl_row_overflows(name):
prompt, chosen, rejected = LONG_ROWS["long_prompt"]
out = _tokenize_row(trainer, False)({"prompt": prompt, "chosen": chosen, "rejected": rejected})
assert len(out["chosen_input_ids"]) > 32
+
+
[email protected]("name", ["orpo", "cpo"])
+def test_the_row_cap_anchors_on_this_trl(name):
+ src = textwrap.dedent(_trainer(name)[1]["tokenize_row"])
+ if "max_prompt_length" in src:
+ pytest.skip("this TRL truncates the prompt itself")
+ assert "_unsloth_ul" in _row_cap()("tokenize_row", src)
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index bdb76720637..50fcf7bc8b6 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -960,6 +960,13 @@ def orpo_trainer_row_cap(function_name, function):
return function
# Before TRL's own response cut: its negative slice end can empty the shorter answer.
match = re.search(r"(?m)^([ \t]*)longer_response_length = max\(", function)
+ if match is None:
+ # TRL 1.15+ cuts answers but never the prompt, so a long prompt still overflows max_length.
+ match = re.search(
+ r"(?m)^([ \t]*)for answer_tokens in \[chosen_tokens, rejected_tokens\]:\n"
+ r"\1[ \t]+if len\(answer_tokens\[\"prompt_input_ids\"\]\) \+ len\(answer_tokens\[\"input_ids\"\]\) > self\.max_length:",
+ function,
+ )
if match is None:
return function
indent = match.group(1)
@@ -3159,6 +3166,23 @@ def grpo_update_SamplingParams(
return generation_kwargs
+def _unsloth_grpo_is_metric_values(delta, flat_is_ratio, mask, sequence_level):
+ # Inputs are zero-filled outside the mask; TRL reduces over masked tokens only, else min ratio reads 0.
+ if mask is None or delta.shape != mask.shape or flat_is_ratio.shape != mask.shape:
+ return delta.reshape(-1), flat_is_ratio.reshape(-1)
+ keep = mask.to(torch.bool)
+ if not sequence_level:
+ return delta[keep], flat_is_ratio[keep]
+ # A row with no kept tokens is exp(0) = 1, as in TRL.
+ counts = keep.sum(dim = -1)
+ per_row = torch.where(
+ counts > 0,
+ (flat_is_ratio * keep).sum(dim = -1) / counts.clamp(min = 1),
+ torch.ones_like(counts, dtype = flat_is_ratio.dtype),
+ )
+ return delta[keep], per_row
+
+
grpo_compute_loss = RL_REPLACEMENTS["grpo_compute_loss"]
grpo_compute_loss_slow = RL_REPLACEMENTS["grpo_compute_loss_slow"]
UnslothEfficientGRPO = RL_REPLACEMENTS["UnslothEfficientGRPO"]
@@ -3174,6 +3198,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_hidden_state
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_mm_token_id))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_fix_mm_token_type_ids))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_accumulation_steps))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_is_metric_values))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_vision_inputs))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_split_vision_by_sample))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_unsplit_vision))
@@ -3541,6 +3566,13 @@ def grpo_trainer_compute_loss(function_name, function):
and delta is not None
and getattr(self, "vllm_importance_sampling_correction", False)
):
+ delta, flat_is_ratio = _unsloth_grpo_is_metric_values(
+ delta,
+ flat_is_ratio,
+ completion_mask,
+ (getattr(self.args, "vllm_importance_sampling_mode", None) or "token_truncate")
+ in ("sequence_mask", "sequence_truncate"),
+ )
mean_delta = (
torch.mean(delta)
if delta.numel() > 0
@@ -3766,7 +3798,6 @@ def openenv_vllm_reload_weights():
return
if Version(importlib_version("trl")) < Version("0.26.0"):
return
-
try:
import trl.experimental.openenv.utils as openenv_utils
import trl.experimental.openenv as openenv