Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Reload LoRA adapters trained with added tokens (#4219)

Daniel Han20aa77836b89
diff --git a/tests/test_adapter_vocab_growth.py b/tests/test_adapter_vocab_growth.py
new file mode 100644
index 00000000000..3e2bccf3b24
--- /dev/null
+++ b/tests/test_adapter_vocab_growth.py
@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""A LoRA adapter trained after adding tokens reloads onto its smaller base (#1215)."""
+
+import pytest
+import torch
+
+safetensors_torch = pytest.importorskip("safetensors.torch")
+transformers = pytest.importorskip("transformers")
+
+
+def _save_adapter(
+ path,
+ rows,
+ hidden = 8,
+ with_lm_head = True,
+):
+ tensors = {
+ "base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight": torch.zeros(4, hidden),
+ "base_model.model.model.embed_tokens.weight": torch.zeros(rows, hidden),
+ }
+ if with_lm_head:
+ tensors["base_model.model.lm_head.weight"] = torch.zeros(rows, hidden)
+ path.mkdir(parents = True, exist_ok = True)
+ safetensors_torch.save_file(tensors, str(path / "adapter_model.safetensors"))
+ return str(path)
+
+
+def _tiny_model(vocab = 32, hidden = 8):
+ config = transformers.LlamaConfig(
+ vocab_size = vocab,
+ hidden_size = hidden,
+ intermediate_size = 16,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ tie_word_embeddings = False,
+ )
+ return transformers.LlamaForCausalLM(config)
+
+
+def test_vocab_rows_read_from_saved_embeddings(tmp_path):
+ from unsloth.models.loader import _adapter_vocab_rows
+
+ assert _adapter_vocab_rows(_save_adapter(tmp_path / "a", 35)) == 35
+ lora_embed = tmp_path / "b"
+ lora_embed.mkdir()
+ safetensors_torch.save_file(
+ {"base_model.model.model.embed_tokens.lora_embedding_A": torch.zeros(4, 40)},
+ str(lora_embed / "adapter_model.safetensors"),
+ )
+ assert _adapter_vocab_rows(str(lora_embed)) == 40
+ plain = tmp_path / "c"
+ plain.mkdir()
+ safetensors_torch.save_file(
+ {"base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight": torch.zeros(4, 8)},
+ str(plain / "adapter_model.safetensors"),
+ )
+ assert _adapter_vocab_rows(str(plain)) is None
+ assert _adapter_vocab_rows(str(tmp_path / "missing")) is None
+
+
+def test_base_grows_to_fit_adapter(tmp_path):
+ from unsloth.models.loader import _grow_vocab_for_adapter
+
+ model = _tiny_model()
+ assert _grow_vocab_for_adapter(model, _save_adapter(tmp_path / "a", 35))
+ assert model.get_input_embeddings().weight.shape[0] == 35
+ assert model.get_output_embeddings().weight.shape[0] == 35
+
+
+def test_short_lm_head_alone_is_grown(tmp_path):
+ from unsloth.models.loader import _grow_vocab_for_adapter
+
+ model = _tiny_model()
+ embed = model.get_input_embeddings()
+ model.set_input_embeddings(torch.nn.Embedding(35, embed.weight.shape[1]))
+ assert _grow_vocab_for_adapter(model, _save_adapter(tmp_path / "a", 35))
+ assert model.get_output_embeddings().weight.shape[0] == 35
+
+
+def test_never_shrinks_or_touches_matching_base(tmp_path):
+ from unsloth.models.loader import _grow_vocab_for_adapter
+
+ model = _tiny_model(vocab = 40)
+ weight = model.get_input_embeddings().weight
+ assert not _grow_vocab_for_adapter(model, _save_adapter(tmp_path / "small", 35))
+ assert not _grow_vocab_for_adapter(model, _save_adapter(tmp_path / "same", 40))
+ assert model.get_input_embeddings().weight is weight
+
+
+def test_uncached_hub_adapter_is_read_from_its_header(monkeypatch):
+ huggingface_hub = pytest.importorskip("huggingface_hub")
+ from types import SimpleNamespace
+ from unsloth.models.loader import _adapter_vocab_rows
+
+ downloads, headers = [], []
+
+ def fake_download(repo_id, filename, **kwargs):
+ downloads.append(kwargs.get("local_files_only"))
+ raise FileNotFoundError(filename)
+
+ def fake_header(repo_id, filename, **kwargs):
+ headers.append((repo_id, filename, kwargs.get("revision")))
+ return SimpleNamespace(
+ tensors = {"base_model.model.lm_head.weight": SimpleNamespace(shape = [35, 8])}
+ )
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
+ monkeypatch.setattr(huggingface_hub, "parse_safetensors_file_metadata", fake_header)
+ assert _adapter_vocab_rows("someone/adapter", revision = "r", header_only = True) == 35
+ assert downloads and all(downloads)
+ assert headers == [("someone/adapter", "adapter_model.safetensors", "r")]
+ assert _adapter_vocab_rows("someone/adapter", local_files_only = True, header_only = True) is None
+ assert len(headers) == 1
+
+
+def test_lora_wrapped_head_rows(tmp_path):
+ from unsloth.models.loader import _adapter_vocab_rows
+
+ path = tmp_path / "a"
+ path.mkdir()
+ safetensors_torch.save_file(
+ {
+ "base_model.model.lm_head.lora_A.weight": torch.zeros(4, 8),
+ "base_model.model.lm_head.lora_B.weight": torch.zeros(35, 4),
+ },
+ str(path / "adapter_model.safetensors"),
+ )
+ assert _adapter_vocab_rows(str(path)) == 35
+ safetensors_torch.save_file(
+ {"base_model.model.lm_head.base_layer.weight": torch.zeros(36, 8)},
+ str(path / "adapter_model.safetensors"),
+ )
+ assert _adapter_vocab_rows(str(path)) == 36
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index bf105bc739a..221347b67ec 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -389,22 +389,24 @@ _OMNI_AUTO_CLASS_NAMES = (
)
-def _adapter_file_keys(path):
+def _adapter_file_shapes(path):
if path.endswith(".safetensors"):
from safetensors import safe_open
with safe_open(path, framework = "pt") as handle:
- return list(handle.keys())
- return list(torch.load(path, map_location = "meta", weights_only = True).keys())
+ return {key: tuple(handle.get_slice(key).get_shape()) for key in handle.keys()}
+ state_dict = torch.load(path, map_location = "meta", weights_only = True)
+ return {key: tuple(value.shape) for key, value in state_dict.items()}
-def _adapter_weight_keys(
+def _adapter_weight_shapes(
adapter_name,
token = None,
revision = None,
local_files_only = False,
cache_dir = None,
+ header_only = False,
):
- """Tensor names of a saved adapter without loading weights, or None."""
+ """{tensor name: shape} of a saved adapter without loading weights, or None. header_only range-reads an uncached hub header."""
filenames = ("adapter_model.safetensors", "adapter_model.bin")
try:
local = os.path.expanduser(adapter_name)
@@ -412,7 +414,7 @@ def _adapter_weight_keys(
for filename in filenames:
path = os.path.join(local, filename)
if os.path.exists(path):
- return _adapter_file_keys(path)
+ return _adapter_file_shapes(path)
return None
from huggingface_hub import hf_hub_download
@@ -424,16 +426,99 @@ def _adapter_weight_keys(
revision = revision,
token = token,
cache_dir = cache_dir,
- local_files_only = local_files_only,
+ local_files_only = local_files_only or header_only,
)
except Exception:
continue
- return _adapter_file_keys(path)
+ return _adapter_file_shapes(path)
+ if header_only and not local_files_only:
+ from huggingface_hub import parse_safetensors_file_metadata
+ metadata = parse_safetensors_file_metadata(
+ adapter_name, "adapter_model.safetensors", revision = revision, token = token
+ )
+ return {key: tuple(info.shape) for key, info in metadata.tensors.items()}
except Exception:
pass
return None
+def _adapter_weight_keys(
+ adapter_name,
+ token = None,
+ revision = None,
+ local_files_only = False,
+ cache_dir = None,
+):
+ """Tensor names of a saved adapter without loading weights, or None."""
+ shapes = _adapter_weight_shapes(
+ adapter_name,
+ token = token,
+ revision = revision,
+ local_files_only = local_files_only,
+ cache_dir = cache_dir,
+ )
+ return None if shapes is None else list(shapes)
+
+
+_ADAPTER_VOCAB_SUFFIXES = (
+ "embed_tokens.weight",
+ "lm_head.weight",
+ "wte.weight",
+ "word_embeddings.weight",
+ "embed_in.weight",
+ "embed_out.weight",
+)
+
+
+def _adapter_vocab_rows(adapter_name, **hub_kwargs):
+ rows = []
+ for key, shape in (_adapter_weight_shapes(adapter_name, **hub_kwargs) or {}).items():
+ if len(shape) != 2:
+ continue
+ key = key.replace(".base_layer.", ".").replace(".lora_B.", ".")
+ if key.endswith(_ADAPTER_VOCAB_SUFFIXES):
+ rows.append(shape[0])
+ elif key.endswith("lora_embedding_A") or ".lora_embedding_A." in key:
+ rows.append(shape[1])
+ return max(rows, default = None)
+
+
+def _resize_vocab(model, new_size):
+ model.resize_token_embeddings(new_size)
+ # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module swap of all, after every repair above.
+ try:
+ from unsloth.models.vision import _repair_dispatch_hooks
+ _repaired = _repair_dispatch_hooks(model)
+ if _repaired:
+ logger.info(
+ f"Unsloth: re-attached dispatch hooks to {_repaired} module(s) "
+ "left unhooked by the vocabulary resize."
+ )
+ except Exception as _exc:
+ logger.warning(
+ f"Unsloth: could not check the dispatch hooks after resizing "
+ f"the vocabulary ({type(_exc).__name__}: {_exc})."
+ )
+
+
+def _grow_vocab_for_adapter(model, adapter_name, **hub_kwargs):
+ """An adapter trained after adding tokens saves more embedding rows than its base, which PeftModel refuses to load (#1215)."""
+ rows = _adapter_vocab_rows(adapter_name, **hub_kwargs)
+ try:
+ current = model.get_input_embeddings().weight.shape[0]
+ # patch_model_and_tokenizer grows only the input embedding to len(tokenizer), so lm_head can still be short.
+ output = model.get_output_embeddings()
+ if output is not None:
+ current = min(current, output.weight.shape[0])
+ except Exception:
+ return False
+ if rows is None or rows <= current:
+ return False
+ print(f"Unsloth: the adapter adds tokens, so the vocabulary grows from {current} to {rows}.")
+ _resize_vocab(model, rows)
+ return True
+
+
def _composition_children(model_config):
names = [
name[: -len("_config")]
@@ -1411,21 +1496,7 @@ class FastLanguageModel(FastLlamaModel):
)
if resize_model_vocab is not None:
- model.resize_token_embeddings(resize_model_vocab)
- # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module swap of all, after every repair above.
- try:
- from unsloth.models.vision import _repair_dispatch_hooks
- _repaired = _repair_dispatch_hooks(model)
- if _repaired:
- logger.info(
- f"Unsloth: re-attached dispatch hooks to {_repaired} module(s) "
- "left unhooked by the vocabulary resize."
- )
- except Exception as _exc:
- logger.warning(
- f"Unsloth: could not check the dispatch hooks after resizing "
- f"the vocabulary ({type(_exc).__name__}: {_exc})."
- )
+ _resize_vocab(model, resize_model_vocab)
if hasattr(model, "add_model_tags"):
model.add_model_tags(
@@ -1504,6 +1575,15 @@ class FastLanguageModel(FastLlamaModel):
)
if _grouped_config is not None:
peft_load_kwargs["config"] = _grouped_config
+ if not fast_inference:
+ _grow_vocab_for_adapter(
+ model,
+ old_model_name,
+ token = token,
+ revision = revision,
+ local_files_only = local_files_only,
+ cache_dir = kwargs.get("cache_dir"),
+ )
model = PeftModel.from_pretrained(
model,
old_model_name,
@@ -2526,6 +2606,29 @@ class FastModel(FastBaseModel):
# A PEFT load resolved model_name to the base, which the caller's ref is not for.
model_revision = base_revision if not is_peft else None
+ # _grow_vocab_for_adapter resizes the embedding after the load, which the offload hooks do not survive.
+ _adapter_grows_vocab = False
+ if is_peft and not fast_inference and offload_embedding == OFFLOAD_EMBEDDING_AUTO:
+ _adapter_rows = _adapter_vocab_rows(
+ old_model_name,
+ token = token,
+ revision = adapter_revision,
+ local_files_only = local_files_only,
+ cache_dir = kwargs.get("cache_dir"),
+ header_only = True,
+ )
+ _text_config = (
+ model_config.get_text_config()
+ if hasattr(model_config, "get_text_config")
+ else model_config
+ )
+ _base_vocab = _config_get(_text_config, "vocab_size")
+ _adapter_grows_vocab = (
+ _adapter_rows is not None
+ and _base_vocab is not None
+ and _adapter_rows > _base_vocab
+ )
+
model, tokenizer = FastBaseModel.from_pretrained(
model_name = model_name,
max_seq_length = max_seq_length,
@@ -2555,7 +2658,8 @@ class FastModel(FastBaseModel):
# resize_token_embeddings below replaces the embedding module and hooks do not follow, so an offload installed during the load would leave a CPU embedding feeding a GPU decoder. An explicit request is left alone.
offload_embedding = (
False
- if resize_model_vocab is not None and offload_embedding == OFFLOAD_EMBEDDING_AUTO
+ if (resize_model_vocab is not None or _adapter_grows_vocab)
+ and offload_embedding == OFFLOAD_EMBEDDING_AUTO
else offload_embedding
),
float32_mixed_precision = float32_mixed_precision,
@@ -2575,21 +2679,7 @@ class FastModel(FastBaseModel):
_drop_text_only_key_mapping(model, _text_key_mapping)
if resize_model_vocab is not None:
- model.resize_token_embeddings(resize_model_vocab)
- # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module swap of all, after every repair above.
- try:
- from unsloth.models.vision import _repair_dispatch_hooks
- _repaired = _repair_dispatch_hooks(model)
- if _repaired:
- logger.info(
- f"Unsloth: re-attached dispatch hooks to {_repaired} module(s) "
- "left unhooked by the vocabulary resize."
- )
- except Exception as _exc:
- logger.warning(
- f"Unsloth: could not check the dispatch hooks after resizing "
- f"the vocabulary ({type(_exc).__name__}: {_exc})."
- )
+ _resize_vocab(model, resize_model_vocab)
if hasattr(model, "add_model_tags"):
model.add_model_tags(
@@ -2716,6 +2806,15 @@ class FastModel(FastBaseModel):
)
if _grouped_config is not None:
peft_load_kwargs["config"] = _grouped_config
+ if not fast_inference:
+ _grow_vocab_for_adapter(
+ model,
+ old_model_name,
+ token = token,
+ revision = revision,
+ local_files_only = local_files_only,
+ cache_dir = kwargs.get("cache_dir"),
+ )
try:
model = PeftModel.from_pretrained(
model,