Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO: take the per-step host syncs out of the loss path (#15855)

Daniel Hanb80df648bf88
diff --git a/tests/test_grpo_metrics_bucket.py b/tests/test_grpo_metrics_bucket.py
index 297358af188..f43462e4542 100644
--- a/tests/test_grpo_metrics_bucket.py
+++ b/tests/test_grpo_metrics_bucket.py
@@ -25,6 +25,10 @@ HELPERS = {
"_unsloth_text_configs",
"_unsloth_get_final_logit_softcapping",
"_unsloth_resolve_logit_scales",
+ "_unsloth_grpo_nanmin",
+ "_unsloth_grpo_nanmax",
+ "_unsloth_grpo_flush_metrics",
+ "_unsloth_grpo_defer_metric",
}
BATCH, PROMPT, COMPLETION = 2, 3, 4
@@ -83,6 +87,7 @@ def _trainer(training, should_evaluate):
_metrics = {"train": defaultdict(list), "eval": defaultdict(list)},
_get_per_token_logps = lambda *args, **kwargs: torch.zeros(BATCH, COMPLETION),
processing_class = None,
+ log = lambda logs, start_time = None: None,
)
@@ -111,6 +116,7 @@ def _inputs():
def test_metrics_follow_model_training(training, should_evaluate, bucket):
trainer = _trainer(training, should_evaluate)
_compute_loss(4.0)(trainer, trainer.model, _inputs())
+ trainer.log({})
other = "eval" if bucket == "train" else "train"
assert trainer._metrics[bucket]["completion_length"] == [4.0]
assert trainer._metrics[bucket]["kl"] == [0.25]
@@ -122,5 +128,6 @@ def test_eval_pass_does_not_change_the_logged_train_length():
_compute_loss(4.0)(trainer, trainer.model, _inputs())
trainer.model.training = True
_compute_loss(8.0)(trainer, trainer.model, _inputs())
+ trainer.log({})
lengths = trainer._metrics["train"]["completion_length"]
assert sum(lengths) / len(lengths) == 8.0
diff --git a/tests/utils/test_grpo_host_syncs.py b/tests/utils/test_grpo_host_syncs.py
new file mode 100644
index 00000000000..12ecded590b
--- /dev/null
+++ b/tests/utils/test_grpo_host_syncs.py
@@ -0,0 +1,271 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""GRPO training-step host syncs: deferred loss metrics, sync-free nanmin / nanmax, one read in the
+PrefixGrouper layout build and host lengths carried on packed_seq_lengths. Values must match the old reads."""
+
+import math
+
+import pytest
+import torch
+
+import unsloth.models.rl_replacements as rr
+import unsloth.utils.packing as packing
+from unsloth.utils import prefix_grouper as pg
+
+
+def _trl_nanmin(tensor):
+ # trl.trainer.utils.nanmin
+ if torch.isnan(tensor).all():
+ return torch.tensor(float("nan"), dtype = tensor.dtype, device = tensor.device)
+ return torch.min(tensor[~torch.isnan(tensor)])
+
+
+def _trl_nanmax(tensor):
+ if torch.isnan(tensor).all():
+ return torch.tensor(float("nan"), dtype = tensor.dtype, device = tensor.device)
+ return torch.max(tensor[~torch.isnan(tensor)])
+
+
+def _same_float(a, b):
+ return (math.isnan(a) and math.isnan(b)) or a == b
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16, torch.float16])
+def test_nanmin_nanmax_match_trl(dtype):
+ g = torch.Generator().manual_seed(0)
+ for i in range(400):
+ x = torch.randn(int(torch.randint(1, 9, (1,), generator = g)), generator = g).to(dtype)
+ if i % 3 == 0:
+ x[torch.rand(x.shape, generator = g) < 0.5] = float("nan")
+ if i % 11 == 0:
+ x[:] = float("nan")
+ if i % 13 == 0:
+ x[0] = float("inf") if i % 2 else float("-inf")
+ for new, old in (
+ (rr._unsloth_grpo_nanmin, _trl_nanmin),
+ (rr._unsloth_grpo_nanmax, _trl_nanmax),
+ ):
+ a, b = new(x), old(x)
+ assert a.dtype == b.dtype and a.shape == b.shape
+ assert _same_float(a.item(), b.item())
+
+
+class _Trainer:
+ def __init__(self):
+ self.logged = []
+
+ def log(
+ self,
+ logs,
+ start_time = None,
+ ):
+ self.logged.append(dict(logs))
+
+
+class _ItemCounter:
+ def __init__(self, monkeypatch):
+ self.n = 0
+ orig_item, orig_tolist = torch.Tensor.item, torch.Tensor.tolist
+
+ def item(t):
+ self.n += 1
+ return orig_item(t)
+
+ def tolist(t):
+ self.n += 1
+ return orig_tolist(t)
+
+ monkeypatch.setattr(torch.Tensor, "item", item)
+ monkeypatch.setattr(torch.Tensor, "tolist", tolist)
+
+
+def test_deferred_metrics_match_item_and_read_once_per_log(monkeypatch):
+ g = torch.Generator().manual_seed(1)
+ values = [
+ torch.randn((), generator = g).to(dt)
+ for dt in (torch.float32, torch.bfloat16, torch.float16) * 5
+ ]
+ values.append(torch.tensor(7)) # an integer tensor stays an int, like .item()
+ expected = {"a": [], "b": []}
+ for i, v in enumerate(values):
+ expected["ab"[i % 2]].append(v.item())
+
+ t = _Trainer()
+ metrics = {"a": [], "b": []}
+ counter = _ItemCounter(monkeypatch)
+ for i, v in enumerate(values):
+ rr._unsloth_grpo_defer_metric(t, metrics["ab"[i % 2]], v)
+ assert counter.n == 0 # nothing read during the step
+ assert metrics == {"a": [], "b": []}
+ t.log({"loss": 1.0})
+ assert (
+ counter.n == 2
+ ) # one batched read per (device, float / int) group: the floats, then the int
+ assert metrics == expected
+ assert [type(x) for x in metrics["a"] + metrics["b"]] == [
+ type(x) for x in expected["a"] + expected["b"]
+ ]
+ assert t.logged == [{"loss": 1.0}]
+ t.log({"loss": 2.0}) # nothing pending: no read, original log still runs
+ assert counter.n == 2 and len(t.logged) == 2
+
+
+def test_deferred_metrics_keep_order_and_bound(monkeypatch):
+ t = _Trainer()
+ target = []
+ vals = [torch.tensor(float(i)) for i in range(4096 + 10)]
+ for v in vals:
+ rr._unsloth_grpo_defer_metric(t, target, v)
+ assert target == [float(i) for i in range(4096)] # bounded queue flushed once
+ t.log({})
+ assert target == [float(i) for i in range(4096 + 10)]
+ rr._unsloth_grpo_defer_metric(t, target, 3.5) # plain floats append directly
+ assert target[-1] == 3.5
+
+
+def test_compute_loss_has_no_item_calls():
+ src = rr.grpo_trainer_compute_loss("compute_loss", "def compute_loss(self): pass")
+ assert ".item()" not in src
+ assert "nanmin(" not in src.replace("_unsloth_grpo_nanmin(", "")
+ assert "nanmax(" not in src.replace("_unsloth_grpo_nanmax(", "")
+ assert src.count("_unsloth_grpo_defer_metric(") >= 11
+
+
+def test_helpers_are_emitted_into_generated_trainer():
+ pre = "\n".join(rr.RL_PRE_ITEMS["grpo_trainer"])
+ for name in (
+ "_unsloth_grpo_nanmin",
+ "_unsloth_grpo_nanmax",
+ "_unsloth_grpo_flush_metrics",
+ "_unsloth_grpo_defer_metric",
+ ):
+ assert f"def {name}(" in pre
+
+
+def _old_layout_inputs(input_ids, logits_to_keep, pad_id, left_pad):
+ total_rows, L = input_ids.shape
+ keep = input_ids != pad_id
+ cstart = ((L - logits_to_keep) - left_pad).to(torch.long)
+ keep_cpu = keep.tolist()
+ real_cols = [[c for c in range(L) if keep_cpu[r][c]] for r in range(total_rows)]
+ return input_ids.tolist(), real_cols, cstart.tolist(), int(left_pad.max().item())
+
+
+def test_build_group_layout_single_read_same_inputs(monkeypatch):
+ g = torch.Generator().manual_seed(2)
+ seen = []
+ orig = pg._build_groups
+
+ def spy(ids_cpu, real_cols_cpu, cstart_cpu, num_generations, total_rows):
+ seen.append((ids_cpu, real_cols_cpu, cstart_cpu))
+ return orig(ids_cpu, real_cols_cpu, cstart_cpu, num_generations, total_rows)
+
+ monkeypatch.setattr(pg, "_build_groups", spy)
+ for trial in range(200):
+ G = 2 + trial % 3
+ groups = 1 + trial % 3
+ P = int(torch.randint(2, 7, (1,), generator = g))
+ C = int(torch.randint(2, 7, (1,), generator = g))
+ rows = []
+ for _ in range(groups):
+ prompt = torch.randint(5, 40, (P,), generator = g)
+ lp = int(torch.randint(0, P - 1, (1,), generator = g))
+ for _ in range(G):
+ row = torch.cat([prompt, torch.randint(5, 40, (C,), generator = g)])
+ row[:lp] = 0
+ cl = int(torch.randint(1, C + 1, (1,), generator = g))
+ row[P + cl :] = 0
+ if trial % 4 == 1:
+ row[P + cl // 2] = 0 # non-contiguous row -> the per-column fallback
+ rows.append(row)
+ raw = torch.stack(rows)
+ left_pad = (raw[:, :P] == 0).sum(dim = 1)
+ seen.clear()
+ counter = _ItemCounter(monkeypatch)
+ layout = pg.build_group_layout(raw, C, 0, G, left_pad, apply_tokr_gate = False)
+ reads = counter.n
+ monkeypatch.undo()
+ monkeypatch.setattr(pg, "_build_groups", spy)
+ ids_cpu, real_cols, cstart, max_lp = _old_layout_inputs(raw, C, 0, left_pad)
+ assert len(seen) == 1
+ assert seen[0] == (ids_cpu, real_cols, cstart)
+ assert reads == 1
+ if layout is not None:
+ assert layout.W == C + max_lp
+
+
+def test_attached_host_lengths_skip_the_read(monkeypatch):
+ packing.clear_packed_caches()
+ lengths = torch.tensor([3, 5, 2], dtype = torch.int32)
+ lengths._unsloth_lengths_tuple = (3, 5, 2)
+ counter = _ItemCounter(monkeypatch)
+ lens, cu, max_seqlen = packing.get_packed_info_from_kwargs(
+ {"packed_seq_lengths": lengths}, torch.device("cpu")
+ )
+ assert counter.n == 0
+ assert max_seqlen == 5 and cu.tolist() == [0, 3, 8, 10] and torch.equal(lens, lengths)
+ assert packing._host_segment_lengths(lengths) == (3, 5, 2)
+ packing.clear_packed_caches()
+ # a tuple that does not match the tensor is ignored (read as before)
+ bad = torch.tensor([4, 4], dtype = torch.int32)
+ bad._unsloth_lengths_tuple = (4,)
+ assert (
+ packing.get_packed_info_from_kwargs({"packed_seq_lengths": bad}, torch.device("cpu"))[2]
+ == 4
+ )
+ packing.clear_packed_caches()
+
+
+def test_flush_never_converts_to_float64_on_device(monkeypatch):
+ # No float64 on FP64-less XPUs.
+ class _Trainer:
+ pass
+
+ trainer = _Trainer()
+ targets = {dt: [] for dt in (torch.float32, torch.bfloat16, torch.float16, torch.int64)}
+ expected = {dt: [] for dt in targets}
+ g = torch.Generator().manual_seed(5)
+ trainer._unsloth_pending_metrics = []
+ for dt, target in targets.items():
+ for _ in range(7):
+ v = (torch.randn((), generator = g) * 100).to(dt)
+ expected[dt].append(v.item())
+ trainer._unsloth_pending_metrics.append((target, v))
+ requested = []
+ orig_to = torch.Tensor.to
+
+ def spy_to(self, *args, **kwargs):
+ dtypes = [a for a in args if isinstance(a, torch.dtype)] + [kwargs.get("dtype")]
+ requested.extend(d for d in dtypes if d is not None)
+ return orig_to(self, *args, **kwargs)
+
+ monkeypatch.setattr(torch.Tensor, "to", spy_to)
+ rr._unsloth_grpo_flush_metrics(trainer)
+ assert torch.float64 not in requested
+ for dt in targets:
+ assert targets[dt] == expected[dt]
+ assert all(type(x) is type(y) for x, y in zip(targets[dt], expected[dt]))
+
+
+def test_deferred_metrics_do_not_keep_the_trainer_alive():
+ # `del trainer` must free it without gc.collect (no reference cycle).
+ import gc
+ import weakref
+
+ class _Owner(_Trainer):
+ pass
+
+ gc.disable()
+ try:
+ t = _Owner()
+ target = []
+ rr._unsloth_grpo_defer_metric(t, target, torch.tensor(1.5))
+ t.log({})
+ assert target == [1.5]
+ rr._unsloth_grpo_defer_metric(t, target, torch.tensor(2.5)) # pending again
+ ref = weakref.ref(t)
+ del t
+ assert ref() is None
+ finally:
+ gc.enable()
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 95ac73a8020..7da7f3a2b7c 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -4824,6 +4824,71 @@ def _unsloth_grpo_is_metric_values(delta, flat_is_ratio, mask, sequence_level):
return delta[keep], per_row
+def _unsloth_grpo_nanmin(tensor):
+ # Sync-free trl nanmin: same value, all-NaN still returns NaN.
+ is_nan = torch.isnan(tensor)
+ smallest = torch.where(is_nan, float("inf"), tensor).min()
+ return torch.where(is_nan.all(), float("nan"), smallest)
+
+
+def _unsloth_grpo_nanmax(tensor):
+ is_nan = torch.isnan(tensor)
+ largest = torch.where(is_nan, float("-inf"), tensor).max()
+ return torch.where(is_nan.all(), float("nan"), largest)
+
+
+def _unsloth_grpo_flush_metrics(trainer):
+ # float32 holds fp16 / bf16 / fp32 exactly (same as .item()) and avoids float64 on FP64-less XPUs.
+ pending = trainer.__dict__.get("_unsloth_pending_metrics")
+ if not pending:
+ return
+ trainer._unsloth_pending_metrics = []
+ by_key = {}
+ for i, (_, value) in enumerate(pending):
+ if value.dtype == torch.float64 or not value.is_floating_point():
+ dtype = value.dtype if value.dtype == torch.float64 else torch.int64
+ else:
+ dtype = torch.float32
+ by_key.setdefault((value.device, dtype), []).append(i)
+ values = [None] * len(pending)
+ for (_, dtype), idx in by_key.items():
+ host = torch.stack([pending[i][1].reshape(()).to(dtype) for i in idx]).tolist()
+ for i, v in zip(idx, host):
+ values[i] = v
+ for (target, _), v in zip(pending, values):
+ target.append(v)
+
+
+def _unsloth_grpo_defer_metric(trainer, target, value):
+ # target.append(value.item()), but read once per log instead of one sync per metric.
+ if not torch.is_tensor(value):
+ target.append(value)
+ return
+ pending = trainer.__dict__.get("_unsloth_pending_metrics")
+ if pending is None:
+ pending = trainer._unsloth_pending_metrics = []
+ _instance_log = trainer.__dict__.get("log")
+ # weakref: a strong ref from trainer.__dict__ would be a reference cycle keeping the trainer alive.
+ import weakref
+
+ try:
+ _trainer_ref = weakref.ref(trainer)
+ except TypeError:
+ _trainer_ref = lambda: trainer # noqa: E731 objects without weakref support
+
+ def log(*args, **kwargs):
+ owner = _trainer_ref()
+ _unsloth_grpo_flush_metrics(owner)
+ if _instance_log is not None:
+ return _instance_log(*args, **kwargs)
+ return type(owner).log(owner, *args, **kwargs)
+
+ trainer.log = log
+ pending.append((target, value.detach()))
+ if len(pending) >= 4096:
+ _unsloth_grpo_flush_metrics(trainer)
+
+
def _unsloth_grpo_is_sequence_level(args_mode):
# One IS ratio per sequence: TRL's sequence_* and unsloth_zoo's geometric_* (verl seq_mean_k1),
# with the UNSLOTH_GRPO_VLLM_IS_MODE override unsloth_zoo's grpo_accumulated_loss applies.
@@ -4853,6 +4918,10 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_fix_mm_token_type
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_accumulation_steps))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_is_metric_values))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_is_sequence_level))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_nanmin))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_nanmax))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_flush_metrics))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_defer_metric))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_sample_weight_mode))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_rollout_logps_as_old))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_trl_masks_unscorable))
@@ -5231,11 +5300,13 @@ def grpo_trainer_compute_loss(function_name, function):
if "train" in self._metrics:
# model.training like TRL; should_evaluate is False under eval_on_start and evaluate().
mode = "train" if self.model.training else "eval"
- self._metrics[mode]["completion_length"].append(completion_length.item())
- self._metrics[mode]["kl"].append(mean_kl.item())
+ _unsloth_grpo_defer_metric(
+ self, self._metrics[mode]["completion_length"], completion_length
+ )
+ _unsloth_grpo_defer_metric(self, self._metrics[mode]["kl"], mean_kl)
else:
- self._metrics["completion_length"].append(completion_length.item())
- self._metrics["kl"].append(mean_kl.item())
+ _unsloth_grpo_defer_metric(self, self._metrics["completion_length"], completion_length)
+ _unsloth_grpo_defer_metric(self, self._metrics["kl"], mean_kl)
if delta is not None and (
(self.use_vllm and getattr(self, "vllm_importance_sampling_correction", False))
@@ -5260,11 +5331,15 @@ def grpo_trainer_compute_loss(function_name, function):
if delta.numel() > 0
else torch.tensor(0.0, device = self.model.device)
)
- self._metrics[mode]["sampling/sampling_logp_difference/mean"].append(
- self.accelerator.gather(mean_delta).mean().item()
+ _unsloth_grpo_defer_metric(
+ self,
+ self._metrics[mode]["sampling/sampling_logp_difference/mean"],
+ self.accelerator.gather(mean_delta).mean(),
)
- self._metrics[mode]["sampling/sampling_logp_difference/max"].append(
- self.accelerator.gather(max_delta).max().item()
+ _unsloth_grpo_defer_metric(
+ self,
+ self._metrics[mode]["sampling/sampling_logp_difference/max"],
+ self.accelerator.gather(max_delta).max(),
)
min_importance_sampling_ratio = (
@@ -5282,20 +5357,24 @@ def grpo_trainer_compute_loss(function_name, function):
if flat_is_ratio.numel() > 0
else torch.tensor(0.0, device = self.model.device)
)
- self._metrics[mode]["sampling/importance_sampling_ratio/min"].append(
+ _unsloth_grpo_defer_metric(
+ self,
+ self._metrics[mode]["sampling/importance_sampling_ratio/min"],
self.accelerator.gather(min_importance_sampling_ratio)
.nan_to_num(nan = float("inf"))
- .min()
- .item()
+ .min(),
)
- self._metrics[mode]["sampling/importance_sampling_ratio/mean"].append(
- self.accelerator.gather(mean_importance_sampling_ratio).nanmean().item()
+ _unsloth_grpo_defer_metric(
+ self,
+ self._metrics[mode]["sampling/importance_sampling_ratio/mean"],
+ self.accelerator.gather(mean_importance_sampling_ratio).nanmean(),
)
- self._metrics[mode]["sampling/importance_sampling_ratio/max"].append(
+ _unsloth_grpo_defer_metric(
+ self,
+ self._metrics[mode]["sampling/importance_sampling_ratio/max"],
self.accelerator.gather(max_importance_sampling_ratio)
.nan_to_num(nan = float("-inf"))
- .max()
- .item()
+ .max(),
)
completion_token_count = completion_mask.sum().clamp(min = 1.0)
@@ -5320,21 +5399,33 @@ def grpo_trainer_compute_loss(function_name, function):
clip_ratio = masked_batch_mean(is_region_clipped.float())
gathered_low_clip = self.accelerator.gather(low_clip)
- self._metrics[mode]["clip_ratio/low_mean"].append(gathered_low_clip.nanmean().item())
- self._metrics[mode]["clip_ratio/low_min"].append(nanmin(gathered_low_clip).item())
+ _unsloth_grpo_defer_metric(
+ self, self._metrics[mode]["clip_ratio/low_mean"], gathered_low_clip.nanmean()
+ )
+ _unsloth_grpo_defer_metric(
+ self,
+ self._metrics[mode]["clip_ratio/low_min"],
+ _unsloth_grpo_nanmin(gathered_low_clip),
+ )
gathered_high_clip = self.accelerator.gather(high_clip)
- self._metrics[mode]["clip_ratio/high_mean"].append(gathered_high_clip.nanmean().item())
- self._metrics[mode]["clip_ratio/high_max"].append(nanmax(gathered_high_clip).item())
+ _unsloth_grpo_defer_metric(
+ self, self._metrics[mode]["clip_ratio/high_mean"], gathered_high_clip.nanmean()
+ )
+ _unsloth_grpo_defer_metric(
+ self,
+ self._metrics[mode]["clip_ratio/high_max"],
+ _unsloth_grpo_nanmax(gathered_high_clip),
+ )
gathered_clip_ratio = self.accelerator.gather(clip_ratio)
- self._metrics[mode]["clip_ratio/region_mean"].append(
- gathered_clip_ratio.nanmean().item()
+ _unsloth_grpo_defer_metric(
+ self, self._metrics[mode]["clip_ratio/region_mean"], gathered_clip_ratio.nanmean()
)
elif self.loss_type == "cispo":
is_cispo_clipped = (coef_1 > self.epsilon_high) & (advantages > 0)
cispo_clip_ratio = masked_batch_mean(is_cispo_clipped.float())
gathered_cispo_clip_ratio = self.accelerator.gather(cispo_clip_ratio)
- self._metrics[mode]["cispo_clip_ratio"].append(
- gathered_cispo_clip_ratio.nanmean().item()
+ _unsloth_grpo_defer_metric(
+ self, self._metrics[mode]["cispo_clip_ratio"], gathered_cispo_clip_ratio.nanmean()
)
return loss
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index 004e3b311cc..75065fa4ea3 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -1817,8 +1817,9 @@ def get_packed_info_from_kwargs(
lengths = seq_lengths.to(device = device, dtype = torch.int32, non_blocking = True)
cu_seqlens = torch.zeros(lengths.numel() + 1, dtype = torch.int32, device = device)
torch.cumsum(lengths, dim = 0, dtype = torch.int32, out = cu_seqlens[1:])
- # The one D2H read per batch.
- host = tuple(lengths.tolist())
+ host = _attached_host_lengths(seq_lengths)
+ if host is None:
+ host = tuple(lengths.tolist())
max_seqlen = max(host)
result = (lengths, cu_seqlens, max_seqlen)
@@ -1831,6 +1832,14 @@ def get_packed_info_from_kwargs(
return result
+def _attached_host_lengths(seq_lengths) -> Optional[Tuple[int, ...]]:
+ """Host `_unsloth_lengths_tuple` attached by the producer (unsloth_zoo GRPO), used if the count matches."""
+ host = getattr(seq_lengths, "_unsloth_lengths_tuple", None)
+ if not isinstance(host, tuple) or len(host) != seq_lengths.numel():
+ return None
+ return host
+
+
def _cached_host_lengths(seq_lengths, cu_seqlens = None) -> Optional[Tuple[int, ...]]:
"""Host copy of the lengths get_packed_info_from_kwargs built (no device read), else None.
With cu_seqlens, also requires that it is the cumsum built beside those lengths."""
@@ -1844,6 +1853,8 @@ def _cached_host_lengths(seq_lengths, cu_seqlens = None) -> Optional[Tuple[int,
def _host_segment_lengths(seq_lengths) -> Tuple[int, ...]:
host = _cached_host_lengths(seq_lengths)
+ if host is None:
+ host = _attached_host_lengths(seq_lengths)
if host is not None:
return host
return tuple(int(length) for length in seq_lengths.tolist())
diff --git a/unsloth/utils/prefix_grouper.py b/unsloth/utils/prefix_grouper.py
index 9efdcc95b9b..9a67dbeef83 100644
--- a/unsloth/utils/prefix_grouper.py
+++ b/unsloth/utils/prefix_grouper.py
@@ -229,21 +229,33 @@ def build_group_layout(
total_rows, L = input_ids.shape
keep = input_ids != pad_id
# Completion start column per row, matching create_completion_attention_mask / _pk_cstart.
- cstart = ((L - logits_to_keep) - left_pad_tokens_per_prompt).to(torch.long)
- cstart_cpu = cstart.tolist()
- ids_cpu = input_ids.tolist()
+ cstart = ((L - logits_to_keep) - left_pad_tokens_per_prompt).to(device = device, dtype = torch.long)
# GRPO rows are one contiguous real run; the O(B*L) scan is only a fallback.
n_real = keep.sum(dim = 1)
first = torch.argmax(keep.to(torch.int8), dim = 1)
ar = torch.arange(L, device = device)
- contiguous = bool(
- (keep == ((ar >= first.unsqueeze(1)) & (ar < (first + n_real).unsqueeze(1)))).all()
- )
- if contiguous:
- real_cols_cpu = [list(range(f, f + n)) for f, n in zip(first.tolist(), n_real.tolist())]
+ contiguous = (keep == ((ar >= first.unsqueeze(1)) & (ar < (first + n_real).unsqueeze(1)))).all()
+ # One device-to-host read for everything below.
+ host = torch.cat(
+ [
+ input_ids.reshape(-1).to(torch.long),
+ cstart.reshape(-1),
+ first.to(torch.long),
+ n_real.to(torch.long),
+ contiguous.to(torch.long).reshape(1),
+ ]
+ ).tolist()
+ n_ids = total_rows * L
+ ids_cpu = [host[r * L : (r + 1) * L] for r in range(total_rows)]
+ cstart_cpu = host[n_ids : n_ids + total_rows]
+ first_cpu = host[n_ids + total_rows : n_ids + 2 * total_rows]
+ n_real_cpu = host[n_ids + 2 * total_rows : n_ids + 3 * total_rows]
+ if host[-1]:
+ real_cols_cpu = [list(range(f, f + n)) for f, n in zip(first_cpu, n_real_cpu)]
else:
- keep_cpu = keep.tolist()
- real_cols_cpu = [[c for c in range(L) if keep_cpu[r][c]] for r in range(total_rows)]
+ real_cols_cpu = [
+ [c for c in range(L) if ids_cpu[r][c] != pad_id] for r in range(total_rows)
+ ]
groups = _build_groups(ids_cpu, real_cols_cpu, cstart_cpu, num_generations, total_rows)
if groups is None:
@@ -304,7 +316,8 @@ def build_group_layout(
flat_ids = input_ids[fr, fc].unsqueeze(0) # [1, T] (grad-safe gather)
position_ids = torch.tensor(pos_list, device = device, dtype = torch.long).unsqueeze(0)
- max_left_pad = int(left_pad_tokens_per_prompt.max().item()) if total_rows else 0
+ # cstart = (L - logits_to_keep) - left_pad, so max left pad comes from the host copy.
+ max_left_pad = ((L - logits_to_keep) - min(cstart_cpu)) if total_rows else 0
W = logits_to_keep + max_left_pad
# The logic is structural, so key on (num_groups, group_sizes), not exact lengths.