Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep the padding mask for left-padded label-less training forwards (#13154)

Daniel Han62354edbf4fd
diff --git a/tests/test_llama_left_padding_training_mask.py b/tests/test_llama_left_padding_training_mask.py
new file mode 100644
index 00000000000..31aaa95d2ac
--- /dev/null
+++ b/tests/test_llama_left_padding_training_mask.py
@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Left-padded, label-less training forwards keep their padding mask (ms-swift rerankers, #3705)."""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+import pytest
+from real_accelerator import has_real_cuda
+
+torch = pytest.importorskip("torch")
+
+LLAMA_PY = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "llama.py"
+
+
+def _lift(name):
+ tree = ast.parse(LLAMA_PY.read_text(encoding = "utf-8"))
+ node = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == name)
+ namespace = {"torch": torch}
+ exec(compile(ast.Module(body = [node], type_ignores = []), str(LLAMA_PY), "exec"), namespace)
+ return namespace[name]
+
+
[email protected]("dtype", [torch.long, torch.int32, torch.bool, torch.float32])
+def test_is_left_padded(dtype):
+ is_left_padded = _lift("_is_left_padded")
+
+ def check(rows):
+ return is_left_padded(torch.tensor(rows).to(dtype))
+
+ assert check([[1, 1, 1], [1, 1, 1]]) is False
+ assert check([[1, 1, 0], [1, 0, 0]]) is False
+ assert check([[0, 1, 1], [1, 1, 1]]) is True
+ assert check([[1, 1, 1], [0, 0, 1]]) is True
+ # Zeros only inside a row are not left padding; no collator builds them.
+ assert check([[1, 0, 1]]) is False
+ assert check([[1]]) is False
+ assert is_left_padded(torch.ones(2, 1, 3, 3)) is False
+
+
+TINY = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+
+
[email protected](not has_real_cuda(), reason = "Unsloth fast forward needs a GPU")
+def test_left_padded_label_less_training_forward_matches_unpadded():
+ from unsloth import FastLanguageModel
+
+ model, _ = FastLanguageModel.from_pretrained(
+ TINY,
+ max_seq_length = 64,
+ dtype = torch.bfloat16,
+ load_in_4bit = False,
+ )
+ model = FastLanguageModel.get_peft_model(
+ model,
+ r = 8,
+ lora_alpha = 16,
+ target_modules = ["q_proj", "v_proj"],
+ use_gradient_checkpointing = "unsloth",
+ )
+ model.train()
+ torch.manual_seed(0)
+ vocab = model.config.vocab_size
+ long_row = torch.randint(3, vocab, (1, 12), device = "cuda")
+ short_row = torch.randint(3, vocab, (1, 7), device = "cuda")
+ pad = 5
+ input_ids = torch.cat(
+ [long_row, torch.cat([torch.full((1, pad), 0, device = "cuda"), short_row], 1)]
+ )
+ attention_mask = torch.ones_like(input_ids)
+ attention_mask[1, :pad] = 0
+
+ padded = model(input_ids = input_ids, attention_mask = attention_mask).logits
+ alone = model(input_ids = short_row, attention_mask = torch.ones_like(short_row)).logits
+ # Measured on a B200: 3.7e-4 with the mask kept, 5.5e-2 when the pads are attended (logits ~4e-2).
+ torch.testing.assert_close(padded[1, pad:].float(), alone[0].float(), atol = 5e-3, rtol = 0)
+
+ model.zero_grad()
+ score = padded[1, -1, :2]
+ (score[0] - score[1]).backward()
+ grads = [p.grad for p in model.parameters() if p.requires_grad and p.grad is not None]
+ assert grads and all(torch.isfinite(g).all() for g in grads)
diff --git a/tests/test_ppo_trainer_patches.py b/tests/test_ppo_trainer_patches.py
index 77b1f70ff7b..1cd50a8c29d 100644
--- a/tests/test_ppo_trainer_patches.py
+++ b/tests/test_ppo_trainer_patches.py
@@ -349,9 +349,14 @@ def test_ppo_training_keeps_padding_masks_then_restores():
def test_llama_training_forward_mask_is_opt_in():
- # Every other trainer keeps today's behaviour: training mode drops the mask unless the flag is set.
+ # Every other trainer keeps today's behaviour: training mode drops the mask unless the flag is
+ # set or a label-less batch is left padded (test_llama_left_padding_training_mask.py).
source = (RL_PY.parent / "llama.py").read_text(encoding = "utf-8")
assert (
- 'elif self.training and not getattr(self, "_unsloth_keep_padding_mask", False):\n'
+ " elif (\n"
+ " self.training\n"
+ ' and not getattr(self, "_unsloth_keep_padding_mask", False)\n'
+ ' and not (getattr(self, "_has_no_labels", False) and _is_left_padded(attention_mask))\n'
+ " ):\n"
" attention_mask = None\n"
) in source
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index aa95aa9d703..601477952d5 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -984,6 +984,16 @@ __DTYPE_MAP = {
}
+def _is_left_padded(attention_mask):
+ # Dropping the training mask is exact only for right padding, where causal attention never
+ # reaches the trailing pads. A label-less forward (custom losses, rerankers) zeroes pad embeddings,
+ # so attending to left pads sends gradients to NaN (#3705). Checked only then: it syncs, so read
+ # one column; a left-padded row always starts with a pad.
+ if attention_mask.dim() != 2 or attention_mask.shape[-1] < 2:
+ return False
+ return not bool(attention_mask[:, 0].all())
+
+
# Ported from transformers models/llama/modeling_llama.py#L825
def LlamaModel_fast_forward(
self,
@@ -1127,7 +1137,11 @@ def LlamaModel_fast_forward(
if attention_mask is None:
padding_mask = None
- elif self.training and not getattr(self, "_unsloth_keep_padding_mask", False):
+ elif (
+ self.training
+ and not getattr(self, "_unsloth_keep_padding_mask", False)
+ and not (getattr(self, "_has_no_labels", False) and _is_left_padded(attention_mask))
+ ):
attention_mask = None
padding_mask = None
else: