Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Train embed_tokens and lm_head in FastModel after add_new_tokens (#13057)
Daniel Han0ff48e8cb99a
diff --git a/tests/test_fastmodel_new_tokens_reload.py b/tests/test_fastmodel_new_tokens_reload.pynew file mode 100644index 00000000000..a2cca089dbf--- /dev/null+++ b/tests/test_fastmodel_new_tokens_reload.py@@ -0,0 +1,60 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""FastModel trains and saves the rows add_new_tokens adds, so a reload keeps them (#1343)."""++import pytest++# Padded vocab (151936 rows, 151669 tokens): new tokens fit without a resize, so PEFT saves no embedding by itself.+TINY = "trl-internal-testing/tiny-Qwen3ForCausalLM"+NEW_TOKENS = ["<|tok_a|>", "<|tok_b|>", "<|tok_c|>"]+++def test_fastmodel_keeps_added_token_rows_across_reload(tmp_path):+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA")+ from unsloth import FastModel, add_new_tokens++ model, tokenizer = FastModel.from_pretrained(TINY, max_seq_length = 128, load_in_4bit = False)+ tokenizer = getattr(tokenizer, "tokenizer", tokenizer)+ old_len = len(tokenizer)+ add_new_tokens(model, tokenizer, NEW_TOKENS)+ model = FastModel.get_peft_model(model, r = 8, lora_alpha = 8)++ assert {"embed_tokens", "lm_head"} <= {+ m.rsplit(".", 1)[-1] for m in (model.peft_config["default"].modules_to_save or ())+ }+ rows = slice(old_len, old_len + len(NEW_TOKENS))+ with torch.no_grad():+ for emb in (model.get_input_embeddings(), model.get_output_embeddings()):+ emb.weight[rows] += 0.5+ saved_in = model.get_input_embeddings().weight[rows].detach().float().cpu().clone()+ saved_out = model.get_output_embeddings().weight[rows].detach().float().cpu().clone()+ model.save_pretrained(tmp_path)+ tokenizer.save_pretrained(tmp_path)+ del model+ torch.cuda.empty_cache()++ model2, _ = FastModel.from_pretrained(str(tmp_path), max_seq_length = 128, load_in_4bit = False)+ torch.testing.assert_close(+ model2.get_input_embeddings().weight[rows].detach().float().cpu(), saved_in+ )+ torch.testing.assert_close(+ model2.get_output_embeddings().weight[rows].detach().float().cpu(), saved_out+ )+++def test_fastmodel_leaves_trainable_token_indices_alone():+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA")+ from unsloth import FastModel, add_new_tokens++ model, tokenizer = FastModel.from_pretrained(TINY, max_seq_length = 128, load_in_4bit = False)+ tokenizer = getattr(tokenizer, "tokenizer", tokenizer)+ old_len = len(tokenizer)+ add_new_tokens(model, tokenizer, NEW_TOKENS)+ indices = list(range(old_len, old_len + len(NEW_TOKENS)))+ model = FastModel.get_peft_model(model, r = 8, lora_alpha = 8, trainable_token_indices = indices)++ assert not model.peft_config["default"].modules_to_save+ assert not any(type(m).__name__ == "ModulesToSaveWrapper" for m in model.modules())diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex c085f5acf48..bf6b2ffba58 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -4482,6 +4482,30 @@ class FastBaseModel:skip = _vllm_unmovable_embedding_modules(model, target_modules),)_raise_if_no_lora_targets_left(target_modules, _moved, target_parameters)+ # add_new_tokens rows stay frozen at their mean init otherwise, and on a padded vocab PEFT saves neither matrix, so reload drops them (#1343). Matches FastLanguageModel; skipped under vLLM, which cannot train them, and when trainable_token_indices already trains the rows.+ if (+ getattr(model, "_need_to_train_embeddings", False)+ and finetune_language_layers+ and getattr(model, "vllm_engine", None) is None+ and not kwargs.get("trainable_token_indices")+ ):+ if modules_to_save is None:+ modules_to_save = []+ elif isinstance(modules_to_save, str):+ modules_to_save = [modules_to_save]+ else:+ modules_to_save = list(modules_to_save)+ _missing = [+ m+ for m in ("embed_tokens", "lm_head")+ if m not in _embedding_leaves(modules_to_save)+ ]+ if _missing:+ print(+ "Unsloth: You added new tokens but did not specify if you wanted to "+ "train the lm_head and embed_tokens.\nWe must turn it on for you."+ )+ modules_to_save.extend(_missing)ensure_weight_tying = _effective_weight_tying(model,modules_to_save,