Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Train embed_tokens and lm_head in FastModel after add_new_tokens (#13057)

Daniel Han0ff48e8cb99a
diff --git a/tests/test_fastmodel_new_tokens_reload.py b/tests/test_fastmodel_new_tokens_reload.py
new file mode 100644
index 00000000000..a2cca089dbf
--- /dev/null
+++ b/tests/test_fastmodel_new_tokens_reload.py
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""FastModel trains and saves the rows add_new_tokens adds, so a reload keeps them (#1343)."""
+
+import pytest
+
+# Padded vocab (151936 rows, 151669 tokens): new tokens fit without a resize, so PEFT saves no embedding by itself.
+TINY = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+NEW_TOKENS = ["<|tok_a|>", "<|tok_b|>", "<|tok_c|>"]
+
+
+def test_fastmodel_keeps_added_token_rows_across_reload(tmp_path):
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ from unsloth import FastModel, add_new_tokens
+
+ model, tokenizer = FastModel.from_pretrained(TINY, max_seq_length = 128, load_in_4bit = False)
+ tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
+ old_len = len(tokenizer)
+ add_new_tokens(model, tokenizer, NEW_TOKENS)
+ model = FastModel.get_peft_model(model, r = 8, lora_alpha = 8)
+
+ assert {"embed_tokens", "lm_head"} <= {
+ m.rsplit(".", 1)[-1] for m in (model.peft_config["default"].modules_to_save or ())
+ }
+ rows = slice(old_len, old_len + len(NEW_TOKENS))
+ with torch.no_grad():
+ for emb in (model.get_input_embeddings(), model.get_output_embeddings()):
+ emb.weight[rows] += 0.5
+ saved_in = model.get_input_embeddings().weight[rows].detach().float().cpu().clone()
+ saved_out = model.get_output_embeddings().weight[rows].detach().float().cpu().clone()
+ model.save_pretrained(tmp_path)
+ tokenizer.save_pretrained(tmp_path)
+ del model
+ torch.cuda.empty_cache()
+
+ model2, _ = FastModel.from_pretrained(str(tmp_path), max_seq_length = 128, load_in_4bit = False)
+ torch.testing.assert_close(
+ model2.get_input_embeddings().weight[rows].detach().float().cpu(), saved_in
+ )
+ torch.testing.assert_close(
+ model2.get_output_embeddings().weight[rows].detach().float().cpu(), saved_out
+ )
+
+
+def test_fastmodel_leaves_trainable_token_indices_alone():
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ from unsloth import FastModel, add_new_tokens
+
+ model, tokenizer = FastModel.from_pretrained(TINY, max_seq_length = 128, load_in_4bit = False)
+ tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
+ old_len = len(tokenizer)
+ add_new_tokens(model, tokenizer, NEW_TOKENS)
+ indices = list(range(old_len, old_len + len(NEW_TOKENS)))
+ model = FastModel.get_peft_model(model, r = 8, lora_alpha = 8, trainable_token_indices = indices)
+
+ assert not model.peft_config["default"].modules_to_save
+ assert not any(type(m).__name__ == "ModulesToSaveWrapper" for m in model.modules())
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index c085f5acf48..bf6b2ffba58 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -4482,6 +4482,30 @@ class FastBaseModel:
skip = _vllm_unmovable_embedding_modules(model, target_modules),
)
_raise_if_no_lora_targets_left(target_modules, _moved, target_parameters)
+ # add_new_tokens rows stay frozen at their mean init otherwise, and on a padded vocab PEFT saves neither matrix, so reload drops them (#1343). Matches FastLanguageModel; skipped under vLLM, which cannot train them, and when trainable_token_indices already trains the rows.
+ if (
+ getattr(model, "_need_to_train_embeddings", False)
+ and finetune_language_layers
+ and getattr(model, "vllm_engine", None) is None
+ and not kwargs.get("trainable_token_indices")
+ ):
+ if modules_to_save is None:
+ modules_to_save = []
+ elif isinstance(modules_to_save, str):
+ modules_to_save = [modules_to_save]
+ else:
+ modules_to_save = list(modules_to_save)
+ _missing = [
+ m
+ for m in ("embed_tokens", "lm_head")
+ if m not in _embedding_leaves(modules_to_save)
+ ]
+ if _missing:
+ print(
+ "Unsloth: You added new tokens but did not specify if you wanted to "
+ "train the lm_head and embed_tokens.\nWe must turn it on for you."
+ )
+ modules_to_save.extend(_missing)
ensure_weight_tying = _effective_weight_tying(
model,
modules_to_save,