Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO packed logprobs: an OOM caps the packed length instead of disabling packing (#13877)

Daniel Hanb2ea078a520d
diff --git a/tests/test_grpo_packed_nograd_oom_recovery.py b/tests/test_grpo_packed_nograd_oom_recovery.py
new file mode 100644
index 00000000000..d2bde8af23c
--- /dev/null
+++ b/tests/test_grpo_packed_nograd_oom_recovery.py
@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""An OOM in the GRPO packed (no-grad) logprob forward caps the packed length instead of
+disabling packing for the run. Runs the real block via test_grpo_packed_raw_logits_nograd."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+
+torch = pytest.importorskip("torch")
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import test_grpo_packed_raw_logits_nograd as harness # noqa: E402
+
+
+class _OOMOnceModel(harness._Model):
+ def __init__(self, error):
+ super().__init__(hidden_states = False)
+ self.error = error
+
+ def forward(
+ self,
+ input_ids = None,
+ packed_seq_lengths = None,
+ **kwargs,
+ ):
+ if packed_seq_lengths is not None and self.error is not None:
+ error, self.error = self.error, None
+ raise error
+ return super().forward(input_ids = input_ids, packed_seq_lengths = packed_seq_lengths, **kwargs)
+
+
+class _OOMInVerifyModel(harness._Model):
+ """The packed forward succeeds; the first per-row verifier forward runs out of memory."""
+
+ def __init__(self):
+ super().__init__(hidden_states = False)
+ self.armed = True
+
+ def forward(
+ self,
+ input_ids = None,
+ packed_seq_lengths = None,
+ **kwargs,
+ ):
+ if packed_seq_lengths is None and self.armed:
+ self.armed = False
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")
+ return super().forward(input_ids = input_ids, packed_seq_lengths = packed_seq_lengths, **kwargs)
+
+
+def _shorter_batch():
+ pad = harness.PAD_ID
+ return torch.tensor(
+ [
+ [pad, pad, pad, 5, 7, 9, 11, 13],
+ [pad, pad, 4, 6, 8, 10, 12, 14],
+ ]
+ )
+
+
+def test_oom_keeps_packing_for_batches_that_fit(monkeypatch):
+ model = _OOMOnceModel(torch.cuda.OutOfMemoryError("CUDA out of memory"))
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
+ ns, *_ = harness._run_packed_block(model = model)
+ assert ns["_pk_use"] is False
+ assert getattr(model, "_unsloth_seq_packing_nograd_ok", None) is not False
+ oom_T = ns["_pk_T"]
+ assert model._unsloth_seq_packing_nograd_unsafe_T == oom_T
+
+ model.calls.clear()
+ ns, *_ = harness._run_packed_block(model = model)
+ assert ns["_pk_use"] is False
+ assert not any(call.packed for call in model.calls)
+
+ monkeypatch.setattr(harness, "_batch", _shorter_batch)
+ model.calls.clear()
+ ns, _, input_ids, max_left_pad = harness._run_packed_block(model = model)
+ assert ns["_pk_T"] < oom_T
+ assert ns["_pk_use"] is True
+ assert any(call.packed for call in model.calls)
+ mask = harness._completion_mask_of(
+ input_ids[:, -(harness.KEEP + max_left_pad) :],
+ harness._left_pad_of(input_ids, harness.KEEP, harness.PAD_ID),
+ max_left_pad,
+ harness.PAD_ID,
+ ).float()
+ expected = harness._reference_logprobs(model, input_ids, max_left_pad)
+ assert torch.allclose(ns["_pk_result"].float() * mask, expected * mask, atol = 1e-5)
+
+
+def test_non_oom_failure_still_disables_packing():
+ model = _OOMOnceModel(RuntimeError("kernel rejected the packed layout"))
+ ns, *_ = harness._run_packed_block(model = model)
+ assert ns["_pk_use"] is False
+ assert model._unsloth_seq_packing_nograd_ok is False
+
+
+def test_oom_in_the_per_row_verifier_disables_packing(monkeypatch):
+ # The packed forward completed, so the OOM says nothing about the packed length.
+ model = _OOMInVerifyModel()
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
+ ns, *_ = harness._run_packed_block(model = model)
+ assert any(call.packed for call in model.calls)
+ assert ns["_pk_use"] is False
+ assert model._unsloth_seq_packing_nograd_ok is False
+ assert getattr(model, "_unsloth_seq_packing_nograd_unsafe_T", None) is None
+
+
+def test_oom_log_reports_the_cap_not_disabled(monkeypatch, capsys):
+ model = _OOMOnceModel(torch.cuda.OutOfMemoryError("CUDA out of memory"))
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
+ ns, *_ = harness._run_packed_block(model = model, UNSLOTH_ENABLE_LOGGING = True)
+ out = capsys.readouterr().out
+ assert f"capped below T={ns['_pk_T']}" in out
+ assert "disabled" not in out
+
+ model = _OOMOnceModel(RuntimeError("kernel rejected the packed layout"))
+ harness._run_packed_block(model = model, UNSLOTH_ENABLE_LOGGING = True)
+ assert "disabled (fell back to padded)" in capsys.readouterr().out
diff --git a/tests/test_grpo_packed_raw_logits_nograd.py b/tests/test_grpo_packed_raw_logits_nograd.py
index ce9d29b0607..b90c6aba7a1 100644
--- a/tests/test_grpo_packed_raw_logits_nograd.py
+++ b/tests/test_grpo_packed_raw_logits_nograd.py
@@ -248,7 +248,11 @@ def _batch():
)
-def _run_packed_block(hidden_states = False, model = None):
+def _run_packed_block(
+ hidden_states = False,
+ model = None,
+ **overrides,
+):
"""Exec the real packed + verify block and hand back its locals."""
if model is None:
model = _Model(hidden_states = hidden_states)
@@ -298,6 +302,7 @@ def _run_packed_block(hidden_states = False, model = None):
"logit_softcapping": 0.0,
"temperature": 1.0,
}
+ namespace.update(overrides)
exec(_packed_block_source(), namespace)
return namespace, model, input_ids, max_left_pad
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 063edc66b82..a270be3e3a7 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -2614,6 +2614,9 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
# Sequence packing (default on; UNSLOTH_GRPO_SEQ_PACKING=0 disables): one varlen block-diagonal forward replaces the padded loop exactly and fixes its left-pad RoPE error. Self-verified, re-checked as T grows, falls back if a backend ignores packed_seq_lengths, and lm_head runs on completion positions only.
_pk_result = None
_pk_use = False
+ _pk_T = None
+ # True only while the packed forward and its scatter run, so an OOM elsewhere (setup, per-row verifier) is not read as a length limit.
+ _pk_in_packed = False
_pk_enabled = UNSLOTH_GRPO_SEQ_PACKING_ON
# Without zoo#840's masked-column guard, zeroed prompt/pad columns turn NaN in exp().
_pk_enabled = _pk_enabled and UNSLOTH_ZOO_HAS_MASKED_COL_GUARD
@@ -2670,6 +2673,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
# Per-row completion start after left-packing, matching create_completion_attention_mask.
_pk_cstart = (_pk_L - logits_to_keep) - left_pad_tokens_per_prompt # [rows]
_pk_ctgt = (_pk_nz_idx[1:, 1] >= _pk_cstart[_pk_nz_idx[1:, 0]]) & _pk_within
+ _pk_in_packed = True
with _get_inference_mode_context_manager(model):
with torch.amp.autocast(
device_type = DEVICE_TYPE_TORCH,
@@ -2722,6 +2726,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
.index_put((_pk_tgt,), _pk_sel.to(torch.float32))
.view(total_rows, _pk_L)[:, -_pk_W:]
)
+ _pk_in_packed = False
# Re-verify when T or the longest segment grows past the verified envelope; a LongRoPE cache switch can change the result.
_pk_vT = int(
getattr(unwrapped_model, "_unsloth_seq_packing_nograd_verified_T", 0)
@@ -2823,19 +2828,36 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
flush = True,
)
except Exception as _pk_err:
- # Any failure: drop intermediates, use the padded loop, do not retry.
+ # Any failure: drop intermediates and use the padded loop for this batch.
_pk_hidden = None
_pk_sel = None
_pk_result = None
_pk_use = False
if isinstance(_pk_err, torch.cuda.OutOfMemoryError):
torch.cuda.empty_cache()
- unwrapped_model._unsloth_seq_packing_nograd_ok = False
- if UNSLOTH_ENABLE_LOGGING:
- print(
- f"[Unsloth] GRPO sequence-packing (no-grad) disabled (fell back to padded): {_pk_err!r}",
- flush = True,
+ if (
+ isinstance(_pk_err, torch.cuda.OutOfMemoryError)
+ and _pk_in_packed
+ and _pk_T is not None
+ ):
+ # An OOM in the packed forward says this flattened length does not fit, not that packing is wrong: keep packing smaller batches.
+ _pk_unsafe = getattr(
+ unwrapped_model, "_unsloth_seq_packing_nograd_unsafe_T", None
)
+ _pk_unsafe = _pk_T if _pk_unsafe is None else min(_pk_unsafe, _pk_T)
+ unwrapped_model._unsloth_seq_packing_nograd_unsafe_T = _pk_unsafe
+ if UNSLOTH_ENABLE_LOGGING:
+ print(
+ f"[Unsloth] GRPO sequence-packing (no-grad) OOM at T={_pk_T}: packing capped below T={_pk_unsafe}, this batch uses the padded loop: {_pk_err!r}",
+ flush = True,
+ )
+ else:
+ unwrapped_model._unsloth_seq_packing_nograd_ok = False
+ if UNSLOTH_ENABLE_LOGGING:
+ print(
+ f"[Unsloth] GRPO sequence-packing (no-grad) disabled (fell back to padded): {_pk_err!r}",
+ flush = True,
+ )
# PrefixGrouper first-use self-verify (no-grad): compare the untrusted PG result to the packed result over the completion mask. Below tol_ok trust the structure, at or above TOL_KILL mark it unsafe forever, borderline falls back for this shape.
if _pg_forward_fn is not None and not _pg_use:
if _pk_use and _pk_result is not None: