Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Reload LoRA adapters trained with added tokens (#4219)
Daniel Han20aa77836b89
diff --git a/tests/test_adapter_vocab_growth.py b/tests/test_adapter_vocab_growth.pynew file mode 100644index 00000000000..3e2bccf3b24--- /dev/null+++ b/tests/test_adapter_vocab_growth.py@@ -0,0 +1,134 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""A LoRA adapter trained after adding tokens reloads onto its smaller base (#1215)."""++import pytest+import torch++safetensors_torch = pytest.importorskip("safetensors.torch")+transformers = pytest.importorskip("transformers")+++def _save_adapter(+ path,+ rows,+ hidden = 8,+ with_lm_head = True,+):+ tensors = {+ "base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight": torch.zeros(4, hidden),+ "base_model.model.model.embed_tokens.weight": torch.zeros(rows, hidden),+ }+ if with_lm_head:+ tensors["base_model.model.lm_head.weight"] = torch.zeros(rows, hidden)+ path.mkdir(parents = True, exist_ok = True)+ safetensors_torch.save_file(tensors, str(path / "adapter_model.safetensors"))+ return str(path)+++def _tiny_model(vocab = 32, hidden = 8):+ config = transformers.LlamaConfig(+ vocab_size = vocab,+ hidden_size = hidden,+ intermediate_size = 16,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ tie_word_embeddings = False,+ )+ return transformers.LlamaForCausalLM(config)+++def test_vocab_rows_read_from_saved_embeddings(tmp_path):+ from unsloth.models.loader import _adapter_vocab_rows++ assert _adapter_vocab_rows(_save_adapter(tmp_path / "a", 35)) == 35+ lora_embed = tmp_path / "b"+ lora_embed.mkdir()+ safetensors_torch.save_file(+ {"base_model.model.model.embed_tokens.lora_embedding_A": torch.zeros(4, 40)},+ str(lora_embed / "adapter_model.safetensors"),+ )+ assert _adapter_vocab_rows(str(lora_embed)) == 40+ plain = tmp_path / "c"+ plain.mkdir()+ safetensors_torch.save_file(+ {"base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight": torch.zeros(4, 8)},+ str(plain / "adapter_model.safetensors"),+ )+ assert _adapter_vocab_rows(str(plain)) is None+ assert _adapter_vocab_rows(str(tmp_path / "missing")) is None+++def test_base_grows_to_fit_adapter(tmp_path):+ from unsloth.models.loader import _grow_vocab_for_adapter++ model = _tiny_model()+ assert _grow_vocab_for_adapter(model, _save_adapter(tmp_path / "a", 35))+ assert model.get_input_embeddings().weight.shape[0] == 35+ assert model.get_output_embeddings().weight.shape[0] == 35+++def test_short_lm_head_alone_is_grown(tmp_path):+ from unsloth.models.loader import _grow_vocab_for_adapter++ model = _tiny_model()+ embed = model.get_input_embeddings()+ model.set_input_embeddings(torch.nn.Embedding(35, embed.weight.shape[1]))+ assert _grow_vocab_for_adapter(model, _save_adapter(tmp_path / "a", 35))+ assert model.get_output_embeddings().weight.shape[0] == 35+++def test_never_shrinks_or_touches_matching_base(tmp_path):+ from unsloth.models.loader import _grow_vocab_for_adapter++ model = _tiny_model(vocab = 40)+ weight = model.get_input_embeddings().weight+ assert not _grow_vocab_for_adapter(model, _save_adapter(tmp_path / "small", 35))+ assert not _grow_vocab_for_adapter(model, _save_adapter(tmp_path / "same", 40))+ assert model.get_input_embeddings().weight is weight+++def test_uncached_hub_adapter_is_read_from_its_header(monkeypatch):+ huggingface_hub = pytest.importorskip("huggingface_hub")+ from types import SimpleNamespace+ from unsloth.models.loader import _adapter_vocab_rows++ downloads, headers = [], []++ def fake_download(repo_id, filename, **kwargs):+ downloads.append(kwargs.get("local_files_only"))+ raise FileNotFoundError(filename)++ def fake_header(repo_id, filename, **kwargs):+ headers.append((repo_id, filename, kwargs.get("revision")))+ return SimpleNamespace(+ tensors = {"base_model.model.lm_head.weight": SimpleNamespace(shape = [35, 8])}+ )++ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)+ monkeypatch.setattr(huggingface_hub, "parse_safetensors_file_metadata", fake_header)+ assert _adapter_vocab_rows("someone/adapter", revision = "r", header_only = True) == 35+ assert downloads and all(downloads)+ assert headers == [("someone/adapter", "adapter_model.safetensors", "r")]+ assert _adapter_vocab_rows("someone/adapter", local_files_only = True, header_only = True) is None+ assert len(headers) == 1+++def test_lora_wrapped_head_rows(tmp_path):+ from unsloth.models.loader import _adapter_vocab_rows++ path = tmp_path / "a"+ path.mkdir()+ safetensors_torch.save_file(+ {+ "base_model.model.lm_head.lora_A.weight": torch.zeros(4, 8),+ "base_model.model.lm_head.lora_B.weight": torch.zeros(35, 4),+ },+ str(path / "adapter_model.safetensors"),+ )+ assert _adapter_vocab_rows(str(path)) == 35+ safetensors_torch.save_file(+ {"base_model.model.lm_head.base_layer.weight": torch.zeros(36, 8)},+ str(path / "adapter_model.safetensors"),+ )+ assert _adapter_vocab_rows(str(path)) == 36diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex bf105bc739a..221347b67ec 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -389,22 +389,24 @@ _OMNI_AUTO_CLASS_NAMES = ()-def _adapter_file_keys(path):+def _adapter_file_shapes(path):if path.endswith(".safetensors"):from safetensors import safe_openwith safe_open(path, framework = "pt") as handle:- return list(handle.keys())- return list(torch.load(path, map_location = "meta", weights_only = True).keys())+ return {key: tuple(handle.get_slice(key).get_shape()) for key in handle.keys()}+ state_dict = torch.load(path, map_location = "meta", weights_only = True)+ return {key: tuple(value.shape) for key, value in state_dict.items()}-def _adapter_weight_keys(+def _adapter_weight_shapes(adapter_name,token = None,revision = None,local_files_only = False,cache_dir = None,+ header_only = False,):- """Tensor names of a saved adapter without loading weights, or None."""+ """{tensor name: shape} of a saved adapter without loading weights, or None. header_only range-reads an uncached hub header."""filenames = ("adapter_model.safetensors", "adapter_model.bin")try:local = os.path.expanduser(adapter_name)@@ -412,7 +414,7 @@ def _adapter_weight_keys(for filename in filenames:path = os.path.join(local, filename)if os.path.exists(path):- return _adapter_file_keys(path)+ return _adapter_file_shapes(path)return Nonefrom huggingface_hub import hf_hub_download@@ -424,16 +426,99 @@ def _adapter_weight_keys(revision = revision,token = token,cache_dir = cache_dir,- local_files_only = local_files_only,+ local_files_only = local_files_only or header_only,)except Exception:continue- return _adapter_file_keys(path)+ return _adapter_file_shapes(path)+ if header_only and not local_files_only:+ from huggingface_hub import parse_safetensors_file_metadata+ metadata = parse_safetensors_file_metadata(+ adapter_name, "adapter_model.safetensors", revision = revision, token = token+ )+ return {key: tuple(info.shape) for key, info in metadata.tensors.items()}except Exception:passreturn None+def _adapter_weight_keys(+ adapter_name,+ token = None,+ revision = None,+ local_files_only = False,+ cache_dir = None,+):+ """Tensor names of a saved adapter without loading weights, or None."""+ shapes = _adapter_weight_shapes(+ adapter_name,+ token = token,+ revision = revision,+ local_files_only = local_files_only,+ cache_dir = cache_dir,+ )+ return None if shapes is None else list(shapes)+++_ADAPTER_VOCAB_SUFFIXES = (+ "embed_tokens.weight",+ "lm_head.weight",+ "wte.weight",+ "word_embeddings.weight",+ "embed_in.weight",+ "embed_out.weight",+)+++def _adapter_vocab_rows(adapter_name, **hub_kwargs):+ rows = []+ for key, shape in (_adapter_weight_shapes(adapter_name, **hub_kwargs) or {}).items():+ if len(shape) != 2:+ continue+ key = key.replace(".base_layer.", ".").replace(".lora_B.", ".")+ if key.endswith(_ADAPTER_VOCAB_SUFFIXES):+ rows.append(shape[0])+ elif key.endswith("lora_embedding_A") or ".lora_embedding_A." in key:+ rows.append(shape[1])+ return max(rows, default = None)+++def _resize_vocab(model, new_size):+ model.resize_token_embeddings(new_size)+ # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module swap of all, after every repair above.+ try:+ from unsloth.models.vision import _repair_dispatch_hooks+ _repaired = _repair_dispatch_hooks(model)+ if _repaired:+ logger.info(+ f"Unsloth: re-attached dispatch hooks to {_repaired} module(s) "+ "left unhooked by the vocabulary resize."+ )+ except Exception as _exc:+ logger.warning(+ f"Unsloth: could not check the dispatch hooks after resizing "+ f"the vocabulary ({type(_exc).__name__}: {_exc})."+ )+++def _grow_vocab_for_adapter(model, adapter_name, **hub_kwargs):+ """An adapter trained after adding tokens saves more embedding rows than its base, which PeftModel refuses to load (#1215)."""+ rows = _adapter_vocab_rows(adapter_name, **hub_kwargs)+ try:+ current = model.get_input_embeddings().weight.shape[0]+ # patch_model_and_tokenizer grows only the input embedding to len(tokenizer), so lm_head can still be short.+ output = model.get_output_embeddings()+ if output is not None:+ current = min(current, output.weight.shape[0])+ except Exception:+ return False+ if rows is None or rows <= current:+ return False+ print(f"Unsloth: the adapter adds tokens, so the vocabulary grows from {current} to {rows}.")+ _resize_vocab(model, rows)+ return True++def _composition_children(model_config):names = [name[: -len("_config")]@@ -1411,21 +1496,7 @@ class FastLanguageModel(FastLlamaModel):)if resize_model_vocab is not None:- model.resize_token_embeddings(resize_model_vocab)- # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module swap of all, after every repair above.- try:- from unsloth.models.vision import _repair_dispatch_hooks- _repaired = _repair_dispatch_hooks(model)- if _repaired:- logger.info(- f"Unsloth: re-attached dispatch hooks to {_repaired} module(s) "- "left unhooked by the vocabulary resize."- )- except Exception as _exc:- logger.warning(- f"Unsloth: could not check the dispatch hooks after resizing "- f"the vocabulary ({type(_exc).__name__}: {_exc})."- )+ _resize_vocab(model, resize_model_vocab)if hasattr(model, "add_model_tags"):model.add_model_tags(@@ -1504,6 +1575,15 @@ class FastLanguageModel(FastLlamaModel):)if _grouped_config is not None:peft_load_kwargs["config"] = _grouped_config+ if not fast_inference:+ _grow_vocab_for_adapter(+ model,+ old_model_name,+ token = token,+ revision = revision,+ local_files_only = local_files_only,+ cache_dir = kwargs.get("cache_dir"),+ )model = PeftModel.from_pretrained(model,old_model_name,@@ -2526,6 +2606,29 @@ class FastModel(FastBaseModel):# A PEFT load resolved model_name to the base, which the caller's ref is not for.model_revision = base_revision if not is_peft else None+ # _grow_vocab_for_adapter resizes the embedding after the load, which the offload hooks do not survive.+ _adapter_grows_vocab = False+ if is_peft and not fast_inference and offload_embedding == OFFLOAD_EMBEDDING_AUTO:+ _adapter_rows = _adapter_vocab_rows(+ old_model_name,+ token = token,+ revision = adapter_revision,+ local_files_only = local_files_only,+ cache_dir = kwargs.get("cache_dir"),+ header_only = True,+ )+ _text_config = (+ model_config.get_text_config()+ if hasattr(model_config, "get_text_config")+ else model_config+ )+ _base_vocab = _config_get(_text_config, "vocab_size")+ _adapter_grows_vocab = (+ _adapter_rows is not None+ and _base_vocab is not None+ and _adapter_rows > _base_vocab+ )+model, tokenizer = FastBaseModel.from_pretrained(model_name = model_name,max_seq_length = max_seq_length,@@ -2555,7 +2658,8 @@ class FastModel(FastBaseModel):# resize_token_embeddings below replaces the embedding module and hooks do not follow, so an offload installed during the load would leave a CPU embedding feeding a GPU decoder. An explicit request is left alone.offload_embedding = (False- if resize_model_vocab is not None and offload_embedding == OFFLOAD_EMBEDDING_AUTO+ if (resize_model_vocab is not None or _adapter_grows_vocab)+ and offload_embedding == OFFLOAD_EMBEDDING_AUTOelse offload_embedding),float32_mixed_precision = float32_mixed_precision,@@ -2575,21 +2679,7 @@ class FastModel(FastBaseModel):_drop_text_only_key_mapping(model, _text_key_mapping)if resize_model_vocab is not None:- model.resize_token_embeddings(resize_model_vocab)- # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module swap of all, after every repair above.- try:- from unsloth.models.vision import _repair_dispatch_hooks- _repaired = _repair_dispatch_hooks(model)- if _repaired:- logger.info(- f"Unsloth: re-attached dispatch hooks to {_repaired} module(s) "- "left unhooked by the vocabulary resize."- )- except Exception as _exc:- logger.warning(- f"Unsloth: could not check the dispatch hooks after resizing "- f"the vocabulary ({type(_exc).__name__}: {_exc})."- )+ _resize_vocab(model, resize_model_vocab)if hasattr(model, "add_model_tags"):model.add_model_tags(@@ -2716,6 +2806,15 @@ class FastModel(FastBaseModel):)if _grouped_config is not None:peft_load_kwargs["config"] = _grouped_config+ if not fast_inference:+ _grow_vocab_for_adapter(+ model,+ old_model_name,+ token = token,+ revision = revision,+ local_files_only = local_files_only,+ cache_dir = kwargs.get("cache_dir"),+ )try:model = PeftModel.from_pretrained(model,