Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep the pad embedding row when the tokenizer emits the pad token, never zero an EOS row (#13628)

Daniel Han4ec4b4c00be7
diff --git a/tests/python/test_pad_row_kept_when_emitted.py b/tests/python/test_pad_row_kept_when_emitted.py
new file mode 100644
index 00000000000..3491b48aae9
--- /dev/null
+++ b/tests/python/test_pad_row_kept_when_emitted.py
@@ -0,0 +1,213 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Pad-row zeroing at load must spare EOS ids and tokens the tokenizer adds itself (#4984)."""
+
+import json
+
+import pytest
+
+import unsloth.tokenizer_utils as tu
+
+
+class _Tok:
+ def __init__(
+ self,
+ prefix = (),
+ suffix = (),
+ fail = False,
+ ):
+ self.prefix, self.suffix, self.fail = list(prefix), list(suffix), fail
+
+ def __call__(
+ self,
+ text,
+ add_special_tokens = True,
+ ):
+ if self.fail:
+ raise ValueError("needs images")
+ ids = [65]
+ if add_special_tokens:
+ ids = self.prefix + ids + self.suffix
+ return {"input_ids": ids}
+
+
+class _Proc:
+ def __init__(self, tokenizer):
+ self.tokenizer = tokenizer
+
+ def __call__(self, *args, **kwargs):
+ raise AssertionError("the inner tokenizer must be asked, not the processor")
+
+
+def test_appended_eos_is_reported():
+ assert tu._special_ids_added_on_encode(_Tok(suffix = [151643])) == {151643}
+
+
+def test_prepended_bos_and_appended_eos():
+ assert tu._special_ids_added_on_encode(_Tok(prefix = [2], suffix = [1])) == {1, 2}
+
+
+def test_plain_tokenizer_adds_nothing():
+ assert tu._special_ids_added_on_encode(_Tok()) == frozenset()
+
+
+def test_processor_asks_inner_tokenizer():
+ assert tu._special_ids_added_on_encode(_Proc(_Tok(suffix = [7]))) == {7}
+
+
+def test_batched_nested_ids():
+ class _Batched(_Tok):
+ def __call__(
+ self,
+ text,
+ add_special_tokens = True,
+ ):
+ return {"input_ids": [super().__call__(text, add_special_tokens)["input_ids"]]}
+
+ assert tu._special_ids_added_on_encode(_Batched(prefix = [2])) == {2}
+
+
+def test_failing_tokenizer_keeps_old_behaviour():
+ assert tu._special_ids_added_on_encode(_Tok(fail = True)) == frozenset()
+
+
+def _tokenizer_from_json(post_processor, special):
+ tokenizers = pytest.importorskip("tokenizers")
+ from transformers import PreTrainedTokenizerFast
+
+ vocab = {"A": 0, **{tok: i + 1 for i, tok in enumerate(special)}}
+ spec = {
+ "version": "1.0",
+ "added_tokens": [
+ {
+ "id": i,
+ "content": t,
+ "single_word": False,
+ "lstrip": False,
+ "rstrip": False,
+ "normalized": False,
+ "special": True,
+ }
+ for t, i in vocab.items()
+ if t != "A"
+ ],
+ "normalizer": None,
+ "pre_tokenizer": {"type": "Whitespace"},
+ "post_processor": post_processor,
+ "decoder": None,
+ "model": {"type": "WordLevel", "vocab": vocab, "unk_token": "A"},
+ }
+ return PreTrainedTokenizerFast(tokenizer_object = tokenizers.Tokenizer.from_str(json.dumps(spec)))
+
+
+def test_real_template_processing_appending_pad():
+ # The harrier-oss-v1-0.6b post_processor: single = $A <|endoftext|>.
+ tok = _tokenizer_from_json(
+ {
+ "type": "TemplateProcessing",
+ "single": [
+ {"Sequence": {"id": "A", "type_id": 0}},
+ {"SpecialToken": {"id": "<|endoftext|>", "type_id": 0}},
+ ],
+ "pair": [
+ {"Sequence": {"id": "A", "type_id": 0}},
+ {"Sequence": {"id": "B", "type_id": 0}},
+ ],
+ "special_tokens": {
+ "<|endoftext|>": {"id": "<|endoftext|>", "ids": [1], "tokens": ["<|endoftext|>"]}
+ },
+ },
+ ["<|endoftext|>"],
+ )
+ tok.pad_token = "<|endoftext|>"
+ assert tok.pad_token_id in tu._special_ids_added_on_encode(tok)
+
+
+def test_real_tokenizer_without_post_processor():
+ tok = _tokenizer_from_json(None, ["<pad>"])
+ tok.pad_token = "<pad>"
+ assert tok.pad_token_id not in tu._special_ids_added_on_encode(tok)
+
+
+class _PadTok(_Tok):
+ def __init__(
+ self,
+ pad_token_id,
+ eos_token_id,
+ suffix = (),
+ ):
+ super().__init__(suffix = suffix)
+ self.pad_token_id, self.eos_token_id = pad_token_id, eos_token_id
+
+
+def _model(
+ padding_idx,
+ tied = False,
+ config_eos = None,
+ generation_eos = None,
+):
+ import types
+
+ torch = pytest.importorskip("torch")
+ model = torch.nn.Module()
+ model.embed = torch.nn.Embedding(8, 4, padding_idx = padding_idx)
+ with torch.no_grad():
+ model.embed.weight.fill_(1.0)
+ model.lm_head = torch.nn.Linear(4, 8, bias = False)
+ if tied:
+ model.lm_head.weight = model.embed.weight
+ model.config = types.SimpleNamespace(eos_token_id = config_eos)
+ model.generation_config = types.SimpleNamespace(eos_token_id = generation_eos)
+ return model
+
+
+def _zeroed(model):
+ return [i for i in range(8) if not model.embed.weight[i].any()]
+
+
+def test_plain_pad_row_is_zeroed():
+ model = _model(5)
+ tu.zero_pad_embedding_rows(model, _PadTok(5, 2))
+ assert _zeroed(model) == [5]
+
+
+def test_pad_equal_to_eos_is_never_zeroed():
+ model = _model(2)
+ tu.zero_pad_embedding_rows(model, _PadTok(2, 2))
+ assert _zeroed(model) == []
+
+
[email protected]("where", ["config", "generation"])
+def test_pad_in_model_eos_list_is_never_zeroed(where):
+ # Llama 3 style: several EOS ids in config / generation_config, the tokenizer names one.
+ kwargs = {"config_eos": [2, 5]} if where == "config" else {"generation_eos": [2, 5]}
+ model = _model(5, **kwargs)
+ tu.zero_pad_embedding_rows(model, _PadTok(5, 2))
+ assert _zeroed(model) == []
+
+
+def test_config_padding_idx_on_eos_row_is_not_zeroed():
+ # Phi-3.5-mini: config pad_token_id 32000 is EOS, the tokenizer pads with 32009.
+ model = _model(2)
+ tu.zero_pad_embedding_rows(model, _PadTok(6, 2))
+ assert _zeroed(model) == []
+
+
+def test_emitted_pad_row_is_kept():
+ model = _model(5)
+ tu.zero_pad_embedding_rows(model, _PadTok(5, 2, suffix = [5]))
+ assert _zeroed(model) == []
+
+
+def test_tied_embedding_is_not_zeroed():
+ model = _model(5, tied = True)
+ tu.zero_pad_embedding_rows(model, _PadTok(5, 2))
+ assert _zeroed(model) == []
+
+
+def test_no_tokenizer_or_pad_is_a_no_op():
+ model = _model(5)
+ tu.zero_pad_embedding_rows(model, None)
+ tu.zero_pad_embedding_rows(model, _PadTok(None, 2))
+ assert _zeroed(model) == []
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index f0405d48af2..c4f5efe83ca 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -3846,28 +3846,9 @@ class FastLlamaModel:
model._old_generate = model.generate
unsloth_fast_generate.__doc__ = model._old_generate.__doc__
model.generate = types.MethodType(unsloth_fast_generate, model)
- # Zero weight[padding_idx] only for embeddings NOT tied to lm_head: when tied, zeroing the row
- # forces pad logit = 0, which beats the negative logits of real tokens (Gemma) and makes the
- # decoder emit <pad>. Skipped when eos_token == pad_token.
- eos_token_id = getattr(tokenizer, "eos_token_id", None) if tokenizer is not None else None
- pad_token_id = getattr(tokenizer, "pad_token_id", None) if tokenizer is not None else None
- if tokenizer is not None and eos_token_id != pad_token_id:
- lm_head = getattr(model, "lm_head", None)
- lm_head_weight = getattr(lm_head, "weight", None) if lm_head is not None else None
- with torch.no_grad():
- for name, module in model.named_modules():
- if type(module) is torch.nn.Embedding:
- if (
- getattr(module, "weight", None) is not None
- and getattr(module, "padding_idx", None) is not None
- ):
- if module.padding_idx < module.weight.shape[0]:
- if (
- lm_head_weight is not None
- and module.weight.data_ptr() == lm_head_weight.data_ptr()
- ):
- continue
- module.weight[module.padding_idx] = 0
+ from ..tokenizer_utils import zero_pad_embedding_rows
+
+ zero_pad_embedding_rows(model, tokenizer)
# LAST: post_patch replaces the embedding modules and the QKV/MLP patching below replaces the
# forwards a hook wraps, so an earlier attach is lost. Skipped under vLLM, which owns the
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 0465f33b91e..524c29f690e 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -5047,28 +5047,9 @@ class FastBaseModel:
m.for_training = functools.partial(FastBaseModel.for_training, m)
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
- # Set weight[padding_idx] = 0 only for embeddings NOT tied with the lm_head: when tied, zeroing the padding row zeros the lm_head row and forces logit = 0 for the pad token. Only when the tokenizer is defined, since eos_token == pad_token sometimes.
- pad_token_id = getattr(tokenizer, "pad_token_id", None)
- lm_head = getattr(model, "lm_head", None)
- lm_head_weight = getattr(lm_head, "weight", None) if lm_head is not None else None
- if tokenizer is not None and getattr(tokenizer, "eos_token_id", None) != pad_token_id:
- with torch.no_grad():
- for name, module in model.named_modules():
- if type(module) is torch.nn.Embedding:
- if (
- getattr(module, "weight", None) is not None
- and getattr(module, "padding_idx", None) is not None
- ):
- if (
- module.padding_idx == pad_token_id
- and module.padding_idx < module.weight.shape[0]
- ):
- if (
- lm_head_weight is not None
- and module.weight.data_ptr() == lm_head_weight.data_ptr()
- ):
- continue
- module.weight[module.padding_idx] = 0
+ from ..tokenizer_utils import zero_pad_embedding_rows
+
+ zero_pad_embedding_rows(model, tokenizer)
return model
@staticmethod
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index f16ea6d6605..9d130be12d5 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -145,6 +145,56 @@ def _tokenizer_auto_adds_bos(tokenizer):
return bool(input_ids) and input_ids[0] == bos_token_id
+def _special_ids_added_on_encode(tokenizer):
+ """Ids the tokenizer itself adds to plain text (post-processor BOS / EOS); empty if it cannot be asked."""
+ tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
+ try:
+ with_special = tokenizer("A")["input_ids"]
+ without_special = tokenizer("A", add_special_tokens = False)["input_ids"]
+ except Exception:
+ return frozenset()
+
+ def flat(ids):
+ while isinstance(ids, (list, tuple)) and ids and isinstance(ids[0], (list, tuple)):
+ ids = ids[0]
+ return ids if isinstance(ids, (list, tuple)) else []
+
+ return frozenset(flat(with_special)) - frozenset(flat(without_special))
+
+
+def _as_id_set(ids):
+ if ids is None:
+ return set()
+ if isinstance(ids, int):
+ return {ids}
+ return {i for i in ids if isinstance(i, int)}
+
+
+def zero_pad_embedding_rows(model, tokenizer):
+ """Zero the tokenizer's pad row, except for an EOS id (stopping), an id the tokenizer adds to every
+ input (#4984, harrier pools it) or an lm_head-tied embedding (pad logit 0 beats Gemma's negatives)."""
+ pad_token_id = getattr(tokenizer, "pad_token_id", None) if tokenizer is not None else None
+ if not isinstance(pad_token_id, int):
+ return
+ eos_ids = _as_id_set(getattr(tokenizer, "eos_token_id", None))
+ for config in (getattr(model, "config", None), getattr(model, "generation_config", None)):
+ eos_ids |= _as_id_set(getattr(config, "eos_token_id", None))
+ if pad_token_id in eos_ids or pad_token_id in _special_ids_added_on_encode(tokenizer):
+ return
+ lm_head_weight = getattr(getattr(model, "lm_head", None), "weight", None)
+ with torch.no_grad():
+ for module in model.modules():
+ if (
+ type(module) is torch.nn.Embedding
+ and module.padding_idx == pad_token_id
+ and pad_token_id < module.weight.shape[0]
+ and (
+ lm_head_weight is None or module.weight.data_ptr() != lm_head_weight.data_ptr()
+ )
+ ):
+ module.weight[pad_token_id] = 0
+
+
def _strip_bos_from_chat_template_text(chat_template):
if not isinstance(chat_template, str) or not chat_template:
return chat_template