Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep the padding mask for left-padded label-less training forwards (#13154)
Daniel Han62354edbf4fd
diff --git a/tests/test_llama_left_padding_training_mask.py b/tests/test_llama_left_padding_training_mask.pynew file mode 100644index 00000000000..31aaa95d2ac--- /dev/null+++ b/tests/test_llama_left_padding_training_mask.py@@ -0,0 +1,84 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Left-padded, label-less training forwards keep their padding mask (ms-swift rerankers, #3705)."""++from __future__ import annotations++import ast+from pathlib import Path++import pytest+from real_accelerator import has_real_cuda++torch = pytest.importorskip("torch")++LLAMA_PY = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "llama.py"+++def _lift(name):+ tree = ast.parse(LLAMA_PY.read_text(encoding = "utf-8"))+ node = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == name)+ namespace = {"torch": torch}+ exec(compile(ast.Module(body = [node], type_ignores = []), str(LLAMA_PY), "exec"), namespace)+ return namespace[name]++[email protected]("dtype", [torch.long, torch.int32, torch.bool, torch.float32])+def test_is_left_padded(dtype):+ is_left_padded = _lift("_is_left_padded")++ def check(rows):+ return is_left_padded(torch.tensor(rows).to(dtype))++ assert check([[1, 1, 1], [1, 1, 1]]) is False+ assert check([[1, 1, 0], [1, 0, 0]]) is False+ assert check([[0, 1, 1], [1, 1, 1]]) is True+ assert check([[1, 1, 1], [0, 0, 1]]) is True+ # Zeros only inside a row are not left padding; no collator builds them.+ assert check([[1, 0, 1]]) is False+ assert check([[1]]) is False+ assert is_left_padded(torch.ones(2, 1, 3, 3)) is False+++TINY = "trl-internal-testing/tiny-Qwen3ForCausalLM"++[email protected](not has_real_cuda(), reason = "Unsloth fast forward needs a GPU")+def test_left_padded_label_less_training_forward_matches_unpadded():+ from unsloth import FastLanguageModel++ model, _ = FastLanguageModel.from_pretrained(+ TINY,+ max_seq_length = 64,+ dtype = torch.bfloat16,+ load_in_4bit = False,+ )+ model = FastLanguageModel.get_peft_model(+ model,+ r = 8,+ lora_alpha = 16,+ target_modules = ["q_proj", "v_proj"],+ use_gradient_checkpointing = "unsloth",+ )+ model.train()+ torch.manual_seed(0)+ vocab = model.config.vocab_size+ long_row = torch.randint(3, vocab, (1, 12), device = "cuda")+ short_row = torch.randint(3, vocab, (1, 7), device = "cuda")+ pad = 5+ input_ids = torch.cat(+ [long_row, torch.cat([torch.full((1, pad), 0, device = "cuda"), short_row], 1)]+ )+ attention_mask = torch.ones_like(input_ids)+ attention_mask[1, :pad] = 0++ padded = model(input_ids = input_ids, attention_mask = attention_mask).logits+ alone = model(input_ids = short_row, attention_mask = torch.ones_like(short_row)).logits+ # Measured on a B200: 3.7e-4 with the mask kept, 5.5e-2 when the pads are attended (logits ~4e-2).+ torch.testing.assert_close(padded[1, pad:].float(), alone[0].float(), atol = 5e-3, rtol = 0)++ model.zero_grad()+ score = padded[1, -1, :2]+ (score[0] - score[1]).backward()+ grads = [p.grad for p in model.parameters() if p.requires_grad and p.grad is not None]+ assert grads and all(torch.isfinite(g).all() for g in grads)diff --git a/tests/test_ppo_trainer_patches.py b/tests/test_ppo_trainer_patches.pyindex 77b1f70ff7b..1cd50a8c29d 100644--- a/tests/test_ppo_trainer_patches.py+++ b/tests/test_ppo_trainer_patches.py@@ -349,9 +349,14 @@ def test_ppo_training_keeps_padding_masks_then_restores():def test_llama_training_forward_mask_is_opt_in():- # Every other trainer keeps today's behaviour: training mode drops the mask unless the flag is set.+ # Every other trainer keeps today's behaviour: training mode drops the mask unless the flag is+ # set or a label-less batch is left padded (test_llama_left_padding_training_mask.py).source = (RL_PY.parent / "llama.py").read_text(encoding = "utf-8")assert (- 'elif self.training and not getattr(self, "_unsloth_keep_padding_mask", False):\n'+ " elif (\n"+ " self.training\n"+ ' and not getattr(self, "_unsloth_keep_padding_mask", False)\n'+ ' and not (getattr(self, "_has_no_labels", False) and _is_left_padded(attention_mask))\n'+ " ):\n"" attention_mask = None\n") in sourcediff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex aa95aa9d703..601477952d5 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -984,6 +984,16 @@ __DTYPE_MAP = {}+def _is_left_padded(attention_mask):+ # Dropping the training mask is exact only for right padding, where causal attention never+ # reaches the trailing pads. A label-less forward (custom losses, rerankers) zeroes pad embeddings,+ # so attending to left pads sends gradients to NaN (#3705). Checked only then: it syncs, so read+ # one column; a left-padded row always starts with a pad.+ if attention_mask.dim() != 2 or attention_mask.shape[-1] < 2:+ return False+ return not bool(attention_mask[:, 0].all())++# Ported from transformers models/llama/modeling_llama.py#L825def LlamaModel_fast_forward(self,@@ -1127,7 +1137,11 @@ def LlamaModel_fast_forward(if attention_mask is None:padding_mask = None- elif self.training and not getattr(self, "_unsloth_keep_padding_mask", False):+ elif (+ self.training+ and not getattr(self, "_unsloth_keep_padding_mask", False)+ and not (getattr(self, "_has_no_labels", False) and _is_left_padded(attention_mask))+ ):attention_mask = Nonepadding_mask = Noneelse: