Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep the pad embedding row when the tokenizer emits the pad token, never zero an EOS row (#13628)
Daniel Han4ec4b4c00be7
diff --git a/tests/python/test_pad_row_kept_when_emitted.py b/tests/python/test_pad_row_kept_when_emitted.pynew file mode 100644index 00000000000..3491b48aae9--- /dev/null+++ b/tests/python/test_pad_row_kept_when_emitted.py@@ -0,0 +1,213 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Pad-row zeroing at load must spare EOS ids and tokens the tokenizer adds itself (#4984)."""++import json++import pytest++import unsloth.tokenizer_utils as tu+++class _Tok:+ def __init__(+ self,+ prefix = (),+ suffix = (),+ fail = False,+ ):+ self.prefix, self.suffix, self.fail = list(prefix), list(suffix), fail++ def __call__(+ self,+ text,+ add_special_tokens = True,+ ):+ if self.fail:+ raise ValueError("needs images")+ ids = [65]+ if add_special_tokens:+ ids = self.prefix + ids + self.suffix+ return {"input_ids": ids}+++class _Proc:+ def __init__(self, tokenizer):+ self.tokenizer = tokenizer++ def __call__(self, *args, **kwargs):+ raise AssertionError("the inner tokenizer must be asked, not the processor")+++def test_appended_eos_is_reported():+ assert tu._special_ids_added_on_encode(_Tok(suffix = [151643])) == {151643}+++def test_prepended_bos_and_appended_eos():+ assert tu._special_ids_added_on_encode(_Tok(prefix = [2], suffix = [1])) == {1, 2}+++def test_plain_tokenizer_adds_nothing():+ assert tu._special_ids_added_on_encode(_Tok()) == frozenset()+++def test_processor_asks_inner_tokenizer():+ assert tu._special_ids_added_on_encode(_Proc(_Tok(suffix = [7]))) == {7}+++def test_batched_nested_ids():+ class _Batched(_Tok):+ def __call__(+ self,+ text,+ add_special_tokens = True,+ ):+ return {"input_ids": [super().__call__(text, add_special_tokens)["input_ids"]]}++ assert tu._special_ids_added_on_encode(_Batched(prefix = [2])) == {2}+++def test_failing_tokenizer_keeps_old_behaviour():+ assert tu._special_ids_added_on_encode(_Tok(fail = True)) == frozenset()+++def _tokenizer_from_json(post_processor, special):+ tokenizers = pytest.importorskip("tokenizers")+ from transformers import PreTrainedTokenizerFast++ vocab = {"A": 0, **{tok: i + 1 for i, tok in enumerate(special)}}+ spec = {+ "version": "1.0",+ "added_tokens": [+ {+ "id": i,+ "content": t,+ "single_word": False,+ "lstrip": False,+ "rstrip": False,+ "normalized": False,+ "special": True,+ }+ for t, i in vocab.items()+ if t != "A"+ ],+ "normalizer": None,+ "pre_tokenizer": {"type": "Whitespace"},+ "post_processor": post_processor,+ "decoder": None,+ "model": {"type": "WordLevel", "vocab": vocab, "unk_token": "A"},+ }+ return PreTrainedTokenizerFast(tokenizer_object = tokenizers.Tokenizer.from_str(json.dumps(spec)))+++def test_real_template_processing_appending_pad():+ # The harrier-oss-v1-0.6b post_processor: single = $A <|endoftext|>.+ tok = _tokenizer_from_json(+ {+ "type": "TemplateProcessing",+ "single": [+ {"Sequence": {"id": "A", "type_id": 0}},+ {"SpecialToken": {"id": "<|endoftext|>", "type_id": 0}},+ ],+ "pair": [+ {"Sequence": {"id": "A", "type_id": 0}},+ {"Sequence": {"id": "B", "type_id": 0}},+ ],+ "special_tokens": {+ "<|endoftext|>": {"id": "<|endoftext|>", "ids": [1], "tokens": ["<|endoftext|>"]}+ },+ },+ ["<|endoftext|>"],+ )+ tok.pad_token = "<|endoftext|>"+ assert tok.pad_token_id in tu._special_ids_added_on_encode(tok)+++def test_real_tokenizer_without_post_processor():+ tok = _tokenizer_from_json(None, ["<pad>"])+ tok.pad_token = "<pad>"+ assert tok.pad_token_id not in tu._special_ids_added_on_encode(tok)+++class _PadTok(_Tok):+ def __init__(+ self,+ pad_token_id,+ eos_token_id,+ suffix = (),+ ):+ super().__init__(suffix = suffix)+ self.pad_token_id, self.eos_token_id = pad_token_id, eos_token_id+++def _model(+ padding_idx,+ tied = False,+ config_eos = None,+ generation_eos = None,+):+ import types++ torch = pytest.importorskip("torch")+ model = torch.nn.Module()+ model.embed = torch.nn.Embedding(8, 4, padding_idx = padding_idx)+ with torch.no_grad():+ model.embed.weight.fill_(1.0)+ model.lm_head = torch.nn.Linear(4, 8, bias = False)+ if tied:+ model.lm_head.weight = model.embed.weight+ model.config = types.SimpleNamespace(eos_token_id = config_eos)+ model.generation_config = types.SimpleNamespace(eos_token_id = generation_eos)+ return model+++def _zeroed(model):+ return [i for i in range(8) if not model.embed.weight[i].any()]+++def test_plain_pad_row_is_zeroed():+ model = _model(5)+ tu.zero_pad_embedding_rows(model, _PadTok(5, 2))+ assert _zeroed(model) == [5]+++def test_pad_equal_to_eos_is_never_zeroed():+ model = _model(2)+ tu.zero_pad_embedding_rows(model, _PadTok(2, 2))+ assert _zeroed(model) == []++[email protected]("where", ["config", "generation"])+def test_pad_in_model_eos_list_is_never_zeroed(where):+ # Llama 3 style: several EOS ids in config / generation_config, the tokenizer names one.+ kwargs = {"config_eos": [2, 5]} if where == "config" else {"generation_eos": [2, 5]}+ model = _model(5, **kwargs)+ tu.zero_pad_embedding_rows(model, _PadTok(5, 2))+ assert _zeroed(model) == []+++def test_config_padding_idx_on_eos_row_is_not_zeroed():+ # Phi-3.5-mini: config pad_token_id 32000 is EOS, the tokenizer pads with 32009.+ model = _model(2)+ tu.zero_pad_embedding_rows(model, _PadTok(6, 2))+ assert _zeroed(model) == []+++def test_emitted_pad_row_is_kept():+ model = _model(5)+ tu.zero_pad_embedding_rows(model, _PadTok(5, 2, suffix = [5]))+ assert _zeroed(model) == []+++def test_tied_embedding_is_not_zeroed():+ model = _model(5, tied = True)+ tu.zero_pad_embedding_rows(model, _PadTok(5, 2))+ assert _zeroed(model) == []+++def test_no_tokenizer_or_pad_is_a_no_op():+ model = _model(5)+ tu.zero_pad_embedding_rows(model, None)+ tu.zero_pad_embedding_rows(model, _PadTok(None, 2))+ assert _zeroed(model) == []diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex f0405d48af2..c4f5efe83ca 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -3846,28 +3846,9 @@ class FastLlamaModel:model._old_generate = model.generateunsloth_fast_generate.__doc__ = model._old_generate.__doc__model.generate = types.MethodType(unsloth_fast_generate, model)- # Zero weight[padding_idx] only for embeddings NOT tied to lm_head: when tied, zeroing the row- # forces pad logit = 0, which beats the negative logits of real tokens (Gemma) and makes the- # decoder emit <pad>. Skipped when eos_token == pad_token.- eos_token_id = getattr(tokenizer, "eos_token_id", None) if tokenizer is not None else None- pad_token_id = getattr(tokenizer, "pad_token_id", None) if tokenizer is not None else None- if tokenizer is not None and eos_token_id != pad_token_id:- lm_head = getattr(model, "lm_head", None)- lm_head_weight = getattr(lm_head, "weight", None) if lm_head is not None else None- with torch.no_grad():- for name, module in model.named_modules():- if type(module) is torch.nn.Embedding:- if (- getattr(module, "weight", None) is not None- and getattr(module, "padding_idx", None) is not None- ):- if module.padding_idx < module.weight.shape[0]:- if (- lm_head_weight is not None- and module.weight.data_ptr() == lm_head_weight.data_ptr()- ):- continue- module.weight[module.padding_idx] = 0+ from ..tokenizer_utils import zero_pad_embedding_rows++ zero_pad_embedding_rows(model, tokenizer)# LAST: post_patch replaces the embedding modules and the QKV/MLP patching below replaces the# forwards a hook wraps, so an earlier attach is lost. Skipped under vLLM, which owns thediff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 0465f33b91e..524c29f690e 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -5047,28 +5047,9 @@ class FastBaseModel:m.for_training = functools.partial(FastBaseModel.for_training, m)m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model- # Set weight[padding_idx] = 0 only for embeddings NOT tied with the lm_head: when tied, zeroing the padding row zeros the lm_head row and forces logit = 0 for the pad token. Only when the tokenizer is defined, since eos_token == pad_token sometimes.- pad_token_id = getattr(tokenizer, "pad_token_id", None)- lm_head = getattr(model, "lm_head", None)- lm_head_weight = getattr(lm_head, "weight", None) if lm_head is not None else None- if tokenizer is not None and getattr(tokenizer, "eos_token_id", None) != pad_token_id:- with torch.no_grad():- for name, module in model.named_modules():- if type(module) is torch.nn.Embedding:- if (- getattr(module, "weight", None) is not None- and getattr(module, "padding_idx", None) is not None- ):- if (- module.padding_idx == pad_token_id- and module.padding_idx < module.weight.shape[0]- ):- if (- lm_head_weight is not None- and module.weight.data_ptr() == lm_head_weight.data_ptr()- ):- continue- module.weight[module.padding_idx] = 0+ from ..tokenizer_utils import zero_pad_embedding_rows++ zero_pad_embedding_rows(model, tokenizer)return model@staticmethoddiff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex f16ea6d6605..9d130be12d5 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -145,6 +145,56 @@ def _tokenizer_auto_adds_bos(tokenizer):return bool(input_ids) and input_ids[0] == bos_token_id+def _special_ids_added_on_encode(tokenizer):+ """Ids the tokenizer itself adds to plain text (post-processor BOS / EOS); empty if it cannot be asked."""+ tokenizer = getattr(tokenizer, "tokenizer", tokenizer)+ try:+ with_special = tokenizer("A")["input_ids"]+ without_special = tokenizer("A", add_special_tokens = False)["input_ids"]+ except Exception:+ return frozenset()++ def flat(ids):+ while isinstance(ids, (list, tuple)) and ids and isinstance(ids[0], (list, tuple)):+ ids = ids[0]+ return ids if isinstance(ids, (list, tuple)) else []++ return frozenset(flat(with_special)) - frozenset(flat(without_special))+++def _as_id_set(ids):+ if ids is None:+ return set()+ if isinstance(ids, int):+ return {ids}+ return {i for i in ids if isinstance(i, int)}+++def zero_pad_embedding_rows(model, tokenizer):+ """Zero the tokenizer's pad row, except for an EOS id (stopping), an id the tokenizer adds to every+ input (#4984, harrier pools it) or an lm_head-tied embedding (pad logit 0 beats Gemma's negatives)."""+ pad_token_id = getattr(tokenizer, "pad_token_id", None) if tokenizer is not None else None+ if not isinstance(pad_token_id, int):+ return+ eos_ids = _as_id_set(getattr(tokenizer, "eos_token_id", None))+ for config in (getattr(model, "config", None), getattr(model, "generation_config", None)):+ eos_ids |= _as_id_set(getattr(config, "eos_token_id", None))+ if pad_token_id in eos_ids or pad_token_id in _special_ids_added_on_encode(tokenizer):+ return+ lm_head_weight = getattr(getattr(model, "lm_head", None), "weight", None)+ with torch.no_grad():+ for module in model.modules():+ if (+ type(module) is torch.nn.Embedding+ and module.padding_idx == pad_token_id+ and pad_token_id < module.weight.shape[0]+ and (+ lm_head_weight is None or module.weight.data_ptr() != lm_head_weight.data_ptr()+ )+ ):+ module.weight[pad_token_id] = 0++def _strip_bos_from_chat_template_text(chat_template):if not isinstance(chat_template, str) or not chat_template:return chat_template