Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO: opt-in zero-variance group exclusion and prompt-level loss aggregation (#13891)
Daniel Hanfe3370a046e5
diff --git a/tests/test_grpo_sample_weight.py b/tests/test_grpo_sample_weight.pynew file mode 100644index 00000000000..b270ad31662--- /dev/null+++ b/tests/test_grpo_sample_weight.py@@ -0,0 +1,285 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Opt-in ScaleRL loss weights for GRPO: zero-variance group exclusion and prompt-level aggregation.++`_unsloth_grpo_sample_weight` runs inside TRL's `_generate_and_score_completions`, where the+group layout still exists, and returns per-completion weights normalised over the whole+generation batch. unsloth_zoo's `grpo_compute_loss` sums `loss * weight` per micro-batch with the+dapo process / accumulation rescale. Summed over micro-batches and averaged over processes (DDP)+this must equal the ScaleRL objective (arXiv 2510.13786):+ (1 / kept groups) * sum_g (1 / tokens in g) * sum_{i in g, t} l_it (prompt level)+ (1 / tokens in kept groups) * sum_{i in kept groups, t} l_it (exclusion only)+The helpers are lifted from the shipped source with `ast`, so nothing imports unsloth.+"""++from __future__ import annotations++import ast+import inspect+import warnings+from collections import defaultdict+from pathlib import Path+from types import SimpleNamespace++import pytest+import torch++SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"+++def _load():+ tree = ast.parse(SOURCE.read_text(encoding = "utf-8"))+ nodes = [+ n+ for n in tree.body+ if isinstance(n, ast.FunctionDef)+ and n.name in ("_unsloth_grpo_sample_weight_mode", "_unsloth_grpo_sample_weight")+ ]+ assert len(nodes) == 2+ ns = {"torch": torch}+ exec(compile(ast.Module(nodes, []), str(SOURCE), "exec"), ns)+ return ns["_unsloth_grpo_sample_weight"]+++sample_weight = _load()+G = 4+++class _Acc:+ def __init__(self, everything):+ self.everything = everything++ def gather(self, tensor):+ return self.everything+++def _trainer(loss_type, lengths_all):+ return SimpleNamespace(+ args = SimpleNamespace(loss_type = loss_type),+ num_generations = G,+ reward_weights = torch.tensor([1.0, 0.5]),+ accelerator = _Acc(lengths_all),+ _metrics = {"train": defaultdict(list), "eval": defaultdict(list)},+ )+++def _batch(+ seed,+ groups = 4,+ cols = 7,+):+ gen = torch.Generator().manual_seed(seed)+ rows = groups * G+ rpf = torch.randint(0, 3, (rows, 2), generator = gen).float()+ rpf[0:G] = torch.tensor([1.0, 2.0]) # group 0: zero variance+ rpf[2 * G : 3 * G, 0] = 0.0+ rpf[2 * G : 3 * G, 1] = torch.nan # group 2: one reward func never applies, still zero variance+ rpf[3 * G, :] = torch.nan # an unscorable completion in a varied group+ mask = (torch.rand(rows, cols, generator = gen) > 0.3).long()+ mask[5] = 0+ new = (-torch.rand(rows, cols, generator = gen, dtype = torch.float64) * 2).requires_grad_(True)+ old = new.detach() + torch.randn(rows, cols, generator = gen, dtype = torch.float64) * 0.1+ adv = torch.randn(rows, generator = gen, dtype = torch.float64)+ return rpf, mask, new, old, adv+++def _per_token(new, old, adv):+ coef = (new - old).exp()+ a = adv.unsqueeze(1)+ return -torch.min(coef * a, coef.clamp(0.8, 1.28) * a)+++def _oracle(rpf, mask, new, old, adv, exclude, prompt):+ rewards = (rpf * torch.tensor([1.0, 0.5])).nansum(1)+ rewards[torch.isnan(rpf).all(1)] = torch.nan+ groups = rewards.view(-1, G)+ keep = []+ for g in groups:+ g = g[~torch.isnan(g)]+ keep.append(not exclude or len(g) < 2 or float(g.std()) > 1e-6)+ per_row = (_per_token(new, old, adv) * mask).sum(-1).view(-1, G)+ tokens = mask.sum(-1).double().view(-1, G).sum(-1)+ kept = [i for i, k in enumerate(keep) if k]+ if prompt:+ return sum(per_row[i].sum() / tokens[i].clamp(min = 1) for i in kept) / max(len(kept), 1)+ return sum(per_row[i].sum() for i in kept) / tokens[kept].sum().clamp(min = 1)+++rr = pytest.importorskip("unsloth_zoo.rl_replacements")+HAS_ZOO = "sample_weight" in inspect.getsource(rr.grpo_compute_loss)+++def _unsloth(rpf, mask, new, old, adv, loss_type, processes, spg, monkeypatch, exclude, prompt):+ monkeypatch.setenv("UNSLOTH_GRPO_ZERO_VARIANCE_EXCLUDE", "1" if exclude else "0")+ monkeypatch.setenv("UNSLOTH_GRPO_LOSS_AGGREGATION", "prompt" if prompt else "")+ rows = rpf.shape[0]+ local = rows // processes+ lengths_all = mask.sum(1).float()+ total = 0+ for p in range(processes):+ ps = slice(p * local, (p + 1) * local)+ trainer = _trainer(loss_type, lengths_all)+ w = sample_weight(trainer, "train", rpf, mask[ps], None, ps)+ assert w is not None and w.shape == (local,)+ # TRL shuffles the local batch, then splits it into steps_per_generation micro-batches.+ perm = torch.randperm(local, generator = torch.Generator().manual_seed(p))+ idx = torch.arange(rows)[ps][perm]+ micro = local // spg+ for m in range(spg):+ sel = idx[m * micro : (m + 1) * micro]+ loss = rr.grpo_compute_loss(+ None,+ new[sel],+ old[sel],+ None,+ torch.zeros_like(mask[sel]),+ mask[sel],+ 0.0,+ adv[sel],+ loss_type = loss_type,+ epsilon_low = 0.2,+ epsilon_high = 0.28,+ max_completion_length = mask.shape[1],+ num_items_in_batch = mask.sum(),+ current_gradient_accumulation_steps = spg,+ steps_per_generation = spg,+ num_processes = processes,+ importance_sampling_level = "token",+ sample_weight = w[perm][m * micro : (m + 1) * micro],+ )[0]+ total = total + loss+ return total / processes # DDP averages the processes' gradients++[email protected](not HAS_ZOO, reason = "unsloth_zoo without sample_weight support")[email protected]("exclude,prompt", [(True, False), (False, True), (True, True)])[email protected]("processes,spg", [(1, 1), (1, 4), (2, 2), (4, 1)])[email protected]("seed", [0, 1])+def test_weights_reproduce_scalerl_objective(exclude, prompt, processes, spg, seed, monkeypatch):+ rpf, mask, new, old, adv = _batch(seed)+ got = _unsloth(rpf, mask, new, old, adv, "dapo", processes, spg, monkeypatch, exclude, prompt)+ want = _oracle(rpf, mask, new, old, adv, exclude, prompt)+ torch.testing.assert_close(got, want, rtol = 1e-6, atol = 1e-9)+ (g1,) = torch.autograd.grad(got, new)+ (g2,) = torch.autograd.grad(want, new)+ torch.testing.assert_close(g1, g2, rtol = 1e-6, atol = 1e-9)++[email protected](not HAS_ZOO, reason = "unsloth_zoo without sample_weight support")+def test_no_zero_variance_group_matches_dapo(monkeypatch):+ rpf, mask, new, old, adv = _batch(3)+ rpf[0:G, 0] = torch.arange(G).float()+ rpf[2 * G : 3 * G, 0] = torch.arange(G).float()+ got = _unsloth(rpf, mask, new, old, adv, "dapo", 2, 2, monkeypatch, True, False)+ total = 0+ for sl in (slice(0, 4), slice(4, 8), slice(8, 12), slice(12, 16)):+ total = (+ total+ + rr.grpo_compute_loss(+ None,+ new[sl],+ old[sl],+ None,+ torch.zeros_like(mask[sl]),+ mask[sl],+ 0.0,+ adv[sl],+ loss_type = "dapo",+ epsilon_low = 0.2,+ epsilon_high = 0.28,+ num_items_in_batch = mask.sum(),+ current_gradient_accumulation_steps = 2,+ steps_per_generation = 2,+ num_processes = 2,+ )[0]+ )+ torch.testing.assert_close(got, total / 2, rtol = 1e-6, atol = 1e-9)+++def test_off_eval_and_unsupported_loss_types_return_none(monkeypatch):+ rpf, mask, *_ = _batch(0)+ lengths = mask.sum(1).float()+ monkeypatch.delenv("UNSLOTH_GRPO_ZERO_VARIANCE_EXCLUDE", raising = False)+ monkeypatch.delenv("UNSLOTH_GRPO_LOSS_AGGREGATION", raising = False)+ assert sample_weight(_trainer("dapo", lengths), "train", rpf, mask, None, slice(0, 16)) is None+ monkeypatch.setenv("UNSLOTH_GRPO_ZERO_VARIANCE_EXCLUDE", "1")+ assert sample_weight(_trainer("dapo", lengths), "eval", rpf, mask, None, slice(0, 16)) is None+ for loss_type, prompt in (("grpo", ""), ("dr_grpo", ""), ("sapo", ""), ("luspo", "prompt")):+ monkeypatch.setenv("UNSLOTH_GRPO_LOSS_AGGREGATION", prompt)+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ assert (+ sample_weight(_trainer(loss_type, lengths), "train", rpf, mask, None, slice(0, 16))+ is None+ )+ assert caught+ monkeypatch.setenv("UNSLOTH_GRPO_LOSS_AGGREGATION", "sequence")+ with pytest.raises(ValueError):+ sample_weight(_trainer("dapo", lengths), "train", rpf, mask, None, slice(0, 16))+++def test_tool_mask_and_metric(monkeypatch):+ monkeypatch.setenv("UNSLOTH_GRPO_ZERO_VARIANCE_EXCLUDE", "1")+ monkeypatch.setenv("UNSLOTH_GRPO_LOSS_AGGREGATION", "")+ rpf, mask, *_ = _batch(1)+ tool = (torch.arange(mask.shape[1]) % 2 == 0).long().expand_as(mask)+ trainer = _trainer("dapo", (mask * tool).sum(1).float())+ w = sample_weight(trainer, "train", rpf, mask, tool, slice(0, 16))+ kept = torch.tensor([0.0] * 4 + [1.0] * 4 + [0.0] * 4 + [1.0] * 4)+ torch.testing.assert_close(w, kept / ((mask * tool).sum(1).float() * kept).sum())+ assert trainer._metrics["train"]["frac_reward_zero_std_excluded"] == [0.5]+++def test_trl_generation_anchors_present():+ trl_grpo = pytest.importorskip("trl.trainer.grpo_trainer")+ # From the file: the imported method can be a runtime wrapper.+ text = Path(trl_grpo.__file__).read_text(encoding = "utf-8")+ node = next(+ n+ for n in ast.walk(ast.parse(text))+ if isinstance(n, ast.FunctionDef) and n.name == "_generate_and_score_completions"+ )+ src = ast.get_source_segment(text, node, padded = True)+ assert (+ src.count(+ 'self._metrics[mode]["frac_reward_zero_std"].append(is_std_zero.float().mean().item())'+ )+ == 1+ )+ assert src.count("\n return output") == 1+ for name in ("rewards_per_func", "process_slice", "completion_mask"):+ assert name in src+++def _kept_groups(trainer, rpf, monkeypatch):+ monkeypatch.setenv("UNSLOTH_GRPO_ZERO_VARIANCE_EXCLUDE", "1")+ monkeypatch.setenv("UNSLOTH_GRPO_LOSS_AGGREGATION", "")+ mask = torch.ones(rpf.shape[0], 3, dtype = torch.long)+ trainer.accelerator = _Acc(mask.sum(1).float())+ w = sample_weight(trainer, "train", rpf, mask, None, slice(None))+ return (w.view(-1, G) > 0).any(-1).tolist()+++def test_exclusion_follows_normalize_then_sum(monkeypatch):+ # Raw weighted sum (weights 1, 0.5) is constant, the per-function normalised sum is not: learning signal.+ signal = torch.tensor([[0.0, 2.0], [1.0, 0.0], [0.0, 2.0], [1.0, 0.0]])+ trainer = _trainer("dapo", None)+ trainer.multi_objective_aggregation = "normalize_then_sum"+ assert _kept_groups(trainer, signal, monkeypatch) == [True]+ trainer.multi_objective_aggregation = "sum_then_normalize"+ assert _kept_groups(trainer, signal, monkeypatch) == [False]+ # Every function constant within the group: the normalised sum is zero, no signal either way.+ flat = torch.tensor([[1.0, 2.0]] * G)+ trainer.multi_objective_aggregation = "normalize_then_sum"+ assert _kept_groups(trainer, flat, monkeypatch) == [False]+++def test_group_with_one_scored_completion_is_kept(monkeypatch):+ rpf = torch.full((2 * G, 2), torch.nan)+ rpf[0] = torch.tensor([1.0, 0.0])+ rpf[G:] = torch.tensor([[0.0, 1.0], [1.0, 1.0], [2.0, 1.0], [0.0, 0.0]])+ for aggregation in ("sum_then_normalize", "normalize_then_sum"):+ trainer = _trainer("dapo", None)+ trainer.multi_objective_aggregation = aggregation+ assert _kept_groups(trainer, rpf, monkeypatch) == [True, True]diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 569fbaa6aa2..1f99f27516f 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -2263,6 +2263,32 @@ def grpo_trainer__generate_and_score_completions(function_name, function):+ function[_signature.end() :])+ # Opt-in ScaleRL loss weights (zero-variance group exclusion, prompt-level aggregation). They need+ # the group layout, which only exists here: the loss sees shuffled micro-batches.+ _zero_std_anchor = (+ 'self._metrics[mode]["frac_reward_zero_std"].append(is_std_zero.float().mean().item())'+ )+ _return_anchor = "\n return output"+ if function.count(_zero_std_anchor) == 1 and function.count(_return_anchor) == 1:+ function = function.replace(+ _zero_std_anchor,+ _zero_std_anchor + "\n"+ " _unsloth_sample_weight = _unsloth_grpo_sample_weight(\n"+ " self, mode, rewards_per_func, completion_mask, locals().get('tool_mask'), process_slice,\n"+ " )",+ )+ function = function.replace(+ _return_anchor,+ "\n if _unsloth_sample_weight is not None:\n"+ ' output["unsloth_sample_weight"] = _unsloth_sample_weight' + _return_anchor,+ )+ elif _unsloth_grpo_sample_weight_mode() is not None:+ _warn_once(+ "grpo_sample_weight",+ "Unsloth: UNSLOTH_GRPO_ZERO_VARIANCE_EXCLUDE / UNSLOTH_GRPO_LOSS_AGGREGATION could "+ "not be installed for this TRL version and are ignored.",+ )+return function@@ -3333,6 +3359,85 @@ def grpo_update_SamplingParams(return generation_kwargs+def _unsloth_grpo_sample_weight_mode():+ import os++ exclude = os.environ.get("UNSLOTH_GRPO_ZERO_VARIANCE_EXCLUDE", "0") == "1"+ aggregation = os.environ.get("UNSLOTH_GRPO_LOSS_AGGREGATION", "").strip().lower()+ if aggregation not in ("", "prompt"):+ raise ValueError(+ f"Unsloth: UNSLOTH_GRPO_LOSS_AGGREGATION must be 'prompt' or unset, got {aggregation}."+ )+ if not exclude and aggregation != "prompt":+ return None+ return exclude, aggregation == "prompt"+++def _unsloth_grpo_sample_weight(+ trainer, mode, rewards_per_func, completion_mask, tool_mask, process_slice+):+ """Per-completion loss weights normalised over the whole generation batch (all processes),+ or None when off. ScaleRL (arXiv 2510.13786): zero-variance filtering drops groups whose+ rewards are all equal from the loss and its token count; prompt-level aggregation averages+ tokens within each group, then over groups. Training batches only."""+ settings = _unsloth_grpo_sample_weight_mode()+ if settings is None or mode != "train":+ return None+ exclude, prompt_level = settings+ loss_type = getattr(trainer.args, "loss_type", None)+ if (prompt_level and loss_type == "luspo") or (+ not prompt_level and loss_type not in ("bnpo", "dapo", "cispo", "vespo")+ ):+ if not getattr(trainer, "_unsloth_sample_weight_warned", False):+ import warnings+ warnings.warn(+ f"Unsloth: zero-variance exclusion alone needs a token-normalised loss_type "+ f"(bnpo, dapo, cispo, vespo) and prompt-level aggregation does not support "+ f"luspo; ignored for loss_type={loss_type}.",+ stacklevel = 2,+ )+ trainer._unsloth_sample_weight_warned = True+ return None+ num_generations = trainer.num_generations+ loss_mask = completion_mask if tool_mask is None else completion_mask * tool_mask+ lengths = trainer.accelerator.gather(loss_mask.sum(dim = 1).to(torch.float32))++ def nan_variance(x, dim):+ # TRL's nanstd squared: Bessel corrected, nan (never "zero") with fewer than two values.+ centred = x - torch.nanmean(x, dim = dim, keepdim = True)+ count = (~torch.isnan(x)).sum(dim = dim, keepdim = True)+ variance = torch.nansum(centred * centred, dim = dim, keepdim = True) / (count - 1).clamp(min = 1)+ return torch.where(count > 1, variance, torch.full_like(variance, torch.nan))++ weights = trainer.reward_weights.to(rewards_per_func.device).unsqueeze(0)+ if getattr(trainer, "multi_objective_aggregation", None) == "normalize_then_sum":+ # TRL normalises each reward function within its group before the weighted sum.+ per_func = rewards_per_func.view(-1, num_generations, rewards_per_func.shape[1])+ per_func = (per_func - torch.nanmean(per_func, dim = 1, keepdim = True)) / (+ nan_variance(per_func, 1).sqrt() + 1e-4+ )+ rewards = (per_func.view_as(rewards_per_func) * weights).nansum(dim = 1)+ else:+ rewards = (rewards_per_func * weights).nansum(dim = 1)+ rewards[torch.isnan(rewards_per_func).all(dim = 1)] = torch.nan+ variance = nan_variance(rewards.view(-1, num_generations), 1).squeeze(1)+ group_keep = torch.ones_like(variance, dtype = torch.bool)+ if exclude:+ # TRL's frac_reward_zero_std test, per group.+ group_keep = ~torch.isclose(variance.sqrt(), torch.zeros_like(variance))+ keep = group_keep.repeat_interleave(num_generations).to(torch.float32).to(lengths.device)+ if prompt_level:+ group_tokens = (lengths * keep).view(-1, num_generations).sum(dim = 1)+ groups = group_keep.sum().to(torch.float32).clamp(min = 1.0)+ weight = keep / (group_tokens.clamp(min = 1.0).repeat_interleave(num_generations) * groups)+ else:+ weight = keep / (lengths * keep).sum().clamp(min = 1.0)+ trainer._metrics[mode]["frac_reward_zero_std_excluded"].append(+ float(1.0 - group_keep.float().mean()) if exclude else 0.0+ )+ return weight[process_slice].to(completion_mask.device)++def _unsloth_grpo_is_metric_values(delta, flat_is_ratio, mask, sequence_level):# Inputs are zero-filled outside the mask; TRL reduces over masked tokens only, else min ratio reads 0.if mask is None or delta.shape != mask.shape or flat_is_ratio.shape != mask.shape:@@ -3366,6 +3471,8 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_mm_token_id))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_fix_mm_token_type_ids))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_accumulation_steps))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_is_metric_values))+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_sample_weight_mode))+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_sample_weight))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_vision_inputs))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_split_vision_by_sample))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_unsplit_vision))@@ -3670,6 +3777,14 @@ def grpo_trainer_compute_loss(function_name, function):)self._unsloth_grpo_tool_mask_zoo_checked = True_grpo_accumulated_loss_kwargs = {}+ _sample_weight = inputs.get("unsloth_sample_weight", None)+ if _sample_weight is not None:+ if "sample_weight" not in inspect.getsource(grpo_compute_loss):+ raise RuntimeError(+ "UNSLOTH_GRPO_ZERO_VARIANCE_EXCLUDE / UNSLOTH_GRPO_LOSS_AGGREGATION need a "+ "newer unsloth_zoo. Please upgrade unsloth_zoo."+ )+ _grpo_accumulated_loss_kwargs["sample_weight"] = _sample_weightif tool_mask is not None:_grpo_accumulated_loss_kwargs["tool_mask"] = tool_maskif hasattr(self.args, "loss_type"):