Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""GRPO training-step host syncs: deferred loss metrics, sync-free nanmin / nanmax, one read in the
PrefixGrouper layout build and host lengths carried on packed_seq_lengths. Values must match the old reads."""
import math
import pytest
import torch
import unsloth.models.rl_replacements as rr
import unsloth.utils.packing as packing
from unsloth.utils import prefix_grouper as pg
def _trl_nanmin(tensor):
# trl.trainer.utils.nanmin
if torch.isnan(tensor).all():
return torch.tensor(float("nan"), dtype = tensor.dtype, device = tensor.device)
return torch.min(tensor[~torch.isnan(tensor)])
def _trl_nanmax(tensor):
if torch.isnan(tensor).all():
return torch.tensor(float("nan"), dtype = tensor.dtype, device = tensor.device)
return torch.max(tensor[~torch.isnan(tensor)])
def _same_float(a, b):
return (math.isnan(a) and math.isnan(b)) or a == b
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16])
def test_nanmin_nanmax_match_trl(dtype):
g = torch.Generator().manual_seed(0)
for i in range(400):
x = torch.randn(int(torch.randint(1, 9, (1,), generator = g)), generator = g).to(dtype)
if i % 3 == 0:
x[torch.rand(x.shape, generator = g) < 0.5] = float("nan")
if i % 11 == 0:
x[:] = float("nan")
if i % 13 == 0:
x[0] = float("inf") if i % 2 else float("-inf")
for new, old in (
(rr._unsloth_grpo_nanmin, _trl_nanmin),
(rr._unsloth_grpo_nanmax, _trl_nanmax),
):
a, b = new(x), old(x)
assert a.dtype == b.dtype and a.shape == b.shape
assert _same_float(a.item(), b.item())
class _Trainer:
def __init__(self):
self.logged = []
def log(
self,
logs,
start_time = None,
):
self.logged.append(dict(logs))
class _ItemCounter:
def __init__(self, monkeypatch):
self.n = 0
orig_item, orig_tolist = torch.Tensor.item, torch.Tensor.tolist
def item(t):
self.n += 1
return orig_item(t)
def tolist(t):
self.n += 1
return orig_tolist(t)
monkeypatch.setattr(torch.Tensor, "item", item)
monkeypatch.setattr(torch.Tensor, "tolist", tolist)
def test_deferred_metrics_match_item_and_read_once_per_log(monkeypatch):
g = torch.Generator().manual_seed(1)
values = [
torch.randn((), generator = g).to(dt)
for dt in (torch.float32, torch.bfloat16, torch.float16) * 5
]
values.append(torch.tensor(7)) # an integer tensor stays an int, like .item()
expected = {"a": [], "b": []}
for i, v in enumerate(values):
expected["ab"[i % 2]].append(v.item())
t = _Trainer()
metrics = {"a": [], "b": []}
counter = _ItemCounter(monkeypatch)
for i, v in enumerate(values):
rr._unsloth_grpo_defer_metric(t, metrics["ab"[i % 2]], v)
assert counter.n == 0 # nothing read during the step
assert metrics == {"a": [], "b": []}
t.log({"loss": 1.0})
assert (
counter.n == 2
) # one batched read per (device, float / int) group: the floats, then the int
assert metrics == expected
assert [type(x) for x in metrics["a"] + metrics["b"]] == [
type(x) for x in expected["a"] + expected["b"]
]
assert t.logged == [{"loss": 1.0}]
t.log({"loss": 2.0}) # nothing pending: no read, original log still runs
assert counter.n == 2 and len(t.logged) == 2
def test_deferred_metrics_keep_order_and_bound(monkeypatch):
t = _Trainer()
target = []
vals = [torch.tensor(float(i)) for i in range(4096 + 10)]
for v in vals:
rr._unsloth_grpo_defer_metric(t, target, v)
assert target == [float(i) for i in range(4096)] # bounded queue flushed once
t.log({})
assert target == [float(i) for i in range(4096 + 10)]
rr._unsloth_grpo_defer_metric(t, target, 3.5) # plain floats append directly
assert target[-1] == 3.5
def test_compute_loss_has_no_item_calls():
src = rr.grpo_trainer_compute_loss("compute_loss", "def compute_loss(self): pass")
assert ".item()" not in src
assert "nanmin(" not in src.replace("_unsloth_grpo_nanmin(", "")
assert "nanmax(" not in src.replace("_unsloth_grpo_nanmax(", "")
assert src.count("_unsloth_grpo_defer_metric(") >= 11
def test_helpers_are_emitted_into_generated_trainer():
pre = "\n".join(rr.RL_PRE_ITEMS["grpo_trainer"])
for name in (
"_unsloth_grpo_nanmin",
"_unsloth_grpo_nanmax",
"_unsloth_grpo_flush_metrics",
"_unsloth_grpo_defer_metric",
):
assert f"def {name}(" in pre
def _old_layout_inputs(input_ids, logits_to_keep, pad_id, left_pad):
total_rows, L = input_ids.shape
keep = input_ids != pad_id
cstart = ((L - logits_to_keep) - left_pad).to(torch.long)
keep_cpu = keep.tolist()
real_cols = [[c for c in range(L) if keep_cpu[r][c]] for r in range(total_rows)]
return input_ids.tolist(), real_cols, cstart.tolist(), int(left_pad.max().item())
def _same_layout(ref, got):
if ref is None or got is None:
return ref is None and got is None
names = ("flat_ids", "position_ids", "tgt_rows", "tgt_cols", "tgt_pred", "tgt_flat")
if not all(torch.equal(getattr(ref, n), getattr(got, n)) for n in names):
return False
seg_a, seg_b = ref.prefix_seg_info, got.prefix_seg_info
for n in ("group_of_kv", "is_prefix", "suffix_of_kv"):
if not torch.equal(getattr(seg_a, n), getattr(seg_b, n)):
return False
return all(
getattr(ref, n) == getattr(got, n) for n in ("total_rows", "L", "W", "tok_r", "signature")
)
def test_build_group_layout_single_read_same_inputs(monkeypatch):
g = torch.Generator().manual_seed(2)
seen = []
orig = pg._build_groups
def spy(ids_cpu, real_cols_cpu, cstart_cpu, num_generations, total_rows):
seen.append((ids_cpu, real_cols_cpu, cstart_cpu))
return orig(ids_cpu, real_cols_cpu, cstart_cpu, num_generations, total_rows)
fast = fallback = 0
for trial in range(200):
G = 2 + trial % 3
groups = 1 + trial % 3
P = int(torch.randint(2, 7, (1,), generator = g))
C = int(torch.randint(2, 7, (1,), generator = g))
rows = []
for _ in range(groups):
prompt = torch.randint(5, 40, (P,), generator = g)
lp = int(torch.randint(0, P - 1, (1,), generator = g))
for _ in range(G):
row = torch.cat([prompt, torch.randint(5, 40, (C,), generator = g)])
row[:lp] = 0
cl = int(torch.randint(1, C + 1, (1,), generator = g))
row[P + cl :] = 0
if trial % 4 == 1:
row[P + cl // 2] = 0 # non-contiguous row -> the per-column fallback
rows.append(row)
raw = torch.stack(rows)
left_pad = (raw[:, :P] == 0).sum(dim = 1)
ref = pg._build_group_layout_reference(raw, C, 0, G, left_pad, apply_tokr_gate = False)
seen.clear()
monkeypatch.setattr(pg, "_build_groups", spy)
copies = []
orig_cpu = torch.Tensor.cpu
monkeypatch.setattr(
torch.Tensor, "cpu", lambda t, *a, **k: copies.append(1) or orig_cpu(t, *a, **k)
)
layout = pg.build_group_layout(raw, C, 0, G, left_pad, apply_tokr_gate = False)
monkeypatch.undo()
assert _same_layout(ref, layout)
ids_cpu, real_cols, cstart, max_lp = _old_layout_inputs(raw, C, 0, left_pad)
if any(cols != list(range(cols[0], cols[0] + len(cols))) for cols in real_cols):
# A row with a gap goes through the reference builder on the same host inputs.
assert len(seen) == 1
assert seen[0] == (ids_cpu, real_cols, cstart)
fallback += 1
else:
# Gap-free rows build the groups vectorized from a single host copy.
assert seen == []
assert len(copies) == 1
fast += 1
if layout is not None:
assert layout.W == C + max_lp
assert fast > 0 and fallback > 0
def test_attached_host_lengths_skip_the_read(monkeypatch):
packing.clear_packed_caches()
lengths = torch.tensor([3, 5, 2], dtype = torch.int32)
lengths._unsloth_lengths_tuple = (3, 5, 2)
counter = _ItemCounter(monkeypatch)
lens, cu, max_seqlen = packing.get_packed_info_from_kwargs(
{"packed_seq_lengths": lengths}, torch.device("cpu")
)
assert counter.n == 0
assert max_seqlen == 5 and cu.tolist() == [0, 3, 8, 10] and torch.equal(lens, lengths)
assert packing._host_segment_lengths(lengths) == (3, 5, 2)
packing.clear_packed_caches()
# a tuple that does not match the tensor is ignored (read as before)
bad = torch.tensor([4, 4], dtype = torch.int32)
bad._unsloth_lengths_tuple = (4,)
assert (
packing.get_packed_info_from_kwargs({"packed_seq_lengths": bad}, torch.device("cpu"))[2]
== 4
)
packing.clear_packed_caches()
def test_flush_never_converts_to_float64_on_device(monkeypatch):
# No float64 on FP64-less XPUs.
class _Trainer:
pass
trainer = _Trainer()
targets = {dt: [] for dt in (torch.float32, torch.bfloat16, torch.float16, torch.int64)}
expected = {dt: [] for dt in targets}
g = torch.Generator().manual_seed(5)
trainer._unsloth_pending_metrics = []
for dt, target in targets.items():
for _ in range(7):
v = (torch.randn((), generator = g) * 100).to(dt)
expected[dt].append(v.item())
trainer._unsloth_pending_metrics.append((target, v))
requested = []
orig_to = torch.Tensor.to
def spy_to(self, *args, **kwargs):
dtypes = [a for a in args if isinstance(a, torch.dtype)] + [kwargs.get("dtype")]
requested.extend(d for d in dtypes if d is not None)
return orig_to(self, *args, **kwargs)
monkeypatch.setattr(torch.Tensor, "to", spy_to)
rr._unsloth_grpo_flush_metrics(trainer)
assert torch.float64 not in requested
for dt in targets:
assert targets[dt] == expected[dt]
assert all(type(x) is type(y) for x, y in zip(targets[dt], expected[dt]))
def test_deferred_metrics_do_not_keep_the_trainer_alive():
# `del trainer` must free it without gc.collect (no reference cycle).
import gc
import weakref
class _Owner(_Trainer):
pass
gc.disable()
try:
t = _Owner()
target = []
rr._unsloth_grpo_defer_metric(t, target, torch.tensor(1.5))
t.log({})
assert target == [1.5]
rr._unsloth_grpo_defer_metric(t, target, torch.tensor(2.5)) # pending again
ref = weakref.ref(t)
del t
assert ref() is None
finally:
gc.enable()