Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Apply embedding_learning_rate to embeddings under full finetuning (#13171)
Daniel Han0cad5cd0e9eb
diff --git a/tests/utils/test_unsloth_optimizer_full_finetune_embedding_lr.py b/tests/utils/test_unsloth_optimizer_full_finetune_embedding_lr.pynew file mode 100644index 00000000000..e4440778486--- /dev/null+++ b/tests/utils/test_unsloth_optimizer_full_finetune_embedding_lr.py@@ -0,0 +1,248 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+#+# Under full finetuning the embedding and lm_head are plain parameters with no PEFT+# `modules_to_save` copy, so the name match found nothing and `embedding_learning_rate`+# was dropped without a word: they trained at the base learning rate.++from types import MethodType, SimpleNamespace++import pytest++import unsloth # noqa: F401 (must precede transformers/trl)+from unsloth.trainer import UnslothTrainer+++LR, EMBEDDING_LR = 2e-4, 5e-5+++def _model(nn, tied):+ class Tiny(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Module()+ self.model.embed_tokens = nn.Embedding(10, 4)+ self.model.proj = nn.Linear(4, 4)+ self.lm_head = nn.Linear(4, 10, bias = False)+ if tied:+ self.lm_head.weight = self.model.embed_tokens.weight++ def get_input_embeddings(self):+ return self.model.embed_tokens++ def get_output_embeddings(self):+ return self.lm_head++ return Tiny()+++def _optimizer(model):+ from transformers import Trainer+ from trl import SFTConfig++ args = SFTConfig(+ output_dir = "/tmp/unsloth-full-ft-embedding-lr-test",+ learning_rate = LR,+ weight_decay = 0.01,+ optim = "adamw_torch",+ report_to = [],+ )+ args.embedding_learning_rate = EMBEDDING_LR+ trainer = SimpleNamespace(args = args, model = model, optimizer = None)+ trainer.get_decay_parameter_names = MethodType(Trainer.get_decay_parameter_names, trainer)+ return UnslothTrainer.create_optimizer(trainer), trainer.get_decay_parameter_names(model)+++def _lr_by_name(torch, model):+ optimizer, _ = _optimizer(model)+ named = {id(p): n for n, p in model.named_parameters()}+ return {named[id(p)]: g["lr"] for g in optimizer.param_groups for p in g["params"]}++[email protected]("tied", [False, True])+def test_full_finetune_embeddings_get_embedding_learning_rate(tied):+ torch = pytest.importorskip("torch")+ lrs = _lr_by_name(torch, _model(torch.nn, tied))++ assert lrs["model.embed_tokens.weight"] == EMBEDDING_LR, lrs+ if not tied:+ assert lrs["lm_head.weight"] == EMBEDDING_LR, lrs+ assert lrs["model.proj.weight"] == LR, lrs+ assert lrs["model.proj.bias"] == LR, lrs+++def test_frozen_embeddings_stay_out_of_the_optimizer():+ torch = pytest.importorskip("torch")+ model = _model(torch.nn, tied = False)+ model.model.embed_tokens.weight.requires_grad_(False)+ lrs = _lr_by_name(torch, model)++ assert "model.embed_tokens.weight" not in lrs, lrs+ assert lrs["lm_head.weight"] == EMBEDDING_LR, lrs+++def test_a_checkpoint_from_before_resumes_with_its_own_moments():+ # The previous layout put the embeddings in the ordinary decay groups, so the new+ # groups differ; every parameter must get its own moments back, and the embeddings+ # their own learning rate at the saved point of the schedule.+ torch = pytest.importorskip("torch")+ from torch.optim.lr_scheduler import LambdaLR+ from unsloth.trainer import _install_legacy_scheduler_resume++ schedule = lambda step: 0.5**step+ model = _model(torch.nn, tied = False)+ named = {id(p): n for n, p in model.named_parameters()}+ _, decay = _optimizer(model)+ params = list(model.named_parameters())+ old = torch.optim.AdamW(+ [+ {"params": [p for n, p in params if n in decay], "weight_decay": 0.01},+ {"params": [p for n, p in params if n not in decay], "weight_decay": 0.0},+ ],+ lr = LR,+ )+ old_scheduler = LambdaLR(old, schedule)+ for index, (_, param) in enumerate(params):+ param.grad = torch.full_like(param, float(index + 1))+ old.step()+ old_scheduler.step()+ saved, saved_scheduler = old.state_dict(), old_scheduler.state_dict()+ flat = [p for group in old.param_groups for p in group["params"]]+ want = {named[id(p)]: float(saved["state"][i]["exp_avg"].sum()) for i, p in enumerate(flat)}++ new, _ = _optimizer(model)+ assert len(new.param_groups) != len(saved["param_groups"]), "no migration exercised"+ scheduler = _install_legacy_scheduler_resume(LambdaLR(new, schedule), new)+ new.load_state_dict(saved)+ scheduler.load_state_dict(saved_scheduler)++ state = new.state_dict()["state"]+ flat = [p for group in new.param_groups for p in group["params"]]+ for index, param in enumerate(flat):+ assert float(state[index]["exp_avg"].sum()) == pytest.approx(want[named[id(param)]])+ lr_of = {named[id(p)]: g["lr"] for g in new.param_groups for p in g["params"]}+ assert lr_of["model.embed_tokens.weight"] == pytest.approx(EMBEDDING_LR * 0.5), lr_of+ assert lr_of["model.proj.weight"] == pytest.approx(LR * 0.5), lr_of++ new.step()+ scheduler.step()+ lr_of = {named[id(p)]: g["lr"] for g in new.param_groups for p in g["params"]}+ assert lr_of["lm_head.weight"] == pytest.approx(EMBEDDING_LR * 0.25), lr_of+ assert lr_of["model.proj.bias"] == pytest.approx(LR * 0.25), lr_of+++def test_a_plateau_reduction_survives_the_same_resume():+ # ReduceLROnPlateau sets no initial_lr, so the saved reduction is read against the+ # lr the old group was built with.+ torch = pytest.importorskip("torch")+ from torch.optim.lr_scheduler import ReduceLROnPlateau+ from unsloth.trainer import _install_legacy_scheduler_resume++ model = _model(torch.nn, tied = False)+ named = {id(p): n for n, p in model.named_parameters()}+ _, decay = _optimizer(model)+ params = list(model.named_parameters())+ old = torch.optim.AdamW(+ [+ {"params": [p for n, p in params if n in decay], "weight_decay": 0.01},+ {"params": [p for n, p in params if n not in decay], "weight_decay": 0.0},+ ],+ lr = LR,+ )+ old_scheduler = ReduceLROnPlateau(old, factor = 0.5, patience = 0)+ for metric in (1.0, 2.0):+ old_scheduler.step(metric)+ assert old.param_groups[0]["lr"] == pytest.approx(LR * 0.5)++ new, _ = _optimizer(model)+ scheduler = _install_legacy_scheduler_resume(+ ReduceLROnPlateau(new, factor = 0.5, patience = 0), new+ )+ new.load_state_dict(old.state_dict())+ scheduler.load_state_dict(old_scheduler.state_dict())++ lr_of = {named[id(p)]: g["lr"] for g in new.param_groups for p in g["params"]}+ assert lr_of["model.embed_tokens.weight"] == pytest.approx(EMBEDDING_LR * 0.5), lr_of+ assert lr_of["model.proj.bias"] == pytest.approx(LR * 0.5), lr_of+ assert len(scheduler.min_lrs) == len(new.param_groups)+ scheduler.step(3.0)+ lr_of = {named[id(p)]: g["lr"] for g in new.param_groups for p in g["params"]}+ assert lr_of["lm_head.weight"] == pytest.approx(EMBEDDING_LR * 0.25), lr_of+++def test_a_checkpoint_from_this_layout_is_not_migrated_again():+ # Only the embeddings train: both layouts are one group of the same size, so the+ # marker, not the sizes, keeps a current checkpoint from being rescaled on resume.+ torch = pytest.importorskip("torch")++ model = _model(torch.nn, tied = True)+ model.model.proj.requires_grad_(False)+ first, _ = _optimizer(model)+ first.param_groups[0]["lr"] = EMBEDDING_LR * 0.5+ saved = first.state_dict()++ second, _ = _optimizer(model)+ second.load_state_dict(saved)+ assert second.param_groups[0]["lr"] == pytest.approx(EMBEDDING_LR * 0.5)+ assert second.state_dict()["param_groups"][0].get("unsloth_embedding_split")+++def test_greedy_lr_keeps_its_per_group_arrays_on_the_same_resume():+ torch = pytest.importorskip("torch")+ GreedyLR = getattr(pytest.importorskip("transformers.optimization"), "GreedyLR", None)+ if GreedyLR is None:+ pytest.skip("transformers without GreedyLR")+ from unsloth.trainer import _install_legacy_scheduler_resume++ model = _model(torch.nn, tied = False)+ _, decay = _optimizer(model)+ params = list(model.named_parameters())+ old = torch.optim.AdamW(+ [+ {"params": [p for n, p in params if n in decay], "weight_decay": 0.01},+ {"params": [p for n, p in params if n not in decay], "weight_decay": 0.0},+ ],+ lr = LR,+ )+ saved_scheduler = GreedyLR(old, min_lr = 0.0, max_lr = 1.0).state_dict()++ new, _ = _optimizer(model)+ scheduler = _install_legacy_scheduler_resume(GreedyLR(new, min_lr = 0.0, max_lr = 1.0), new)+ new.load_state_dict(old.state_dict())+ scheduler.load_state_dict(saved_scheduler)+ for key in ("max_lrs", "_init_lrs", "min_lrs"):+ assert len(getattr(scheduler, key)) == len(new.param_groups), key+++def test_q_galore_embeddings_get_embedding_learning_rate():+ torch = pytest.importorskip("torch")+ pytest.importorskip("bitsandbytes")+ from unsloth.trainer import QGaloreConfig++ model = _model(torch.nn, tied = False)+ args = SimpleNamespace(+ learning_rate = LR,+ weight_decay = 0.0,+ adam_beta1 = 0.9,+ adam_beta2 = 0.999,+ adam_epsilon = 1e-8,+ )+ trainer = SimpleNamespace(args = args, model = model, optimizer = None)+ config = QGaloreConfig(rank = 2, weight_quant = False, target_modules = ["proj"])+ optimizer = UnslothTrainer._create_q_galore_optimizer(trainer, config, EMBEDDING_LR, model)+ named = {id(p): n for n, p in model.named_parameters()}+ lrs = {named[id(p)]: g["lr"] for g in optimizer.param_groups for p in g["params"]}++ assert lrs["model.embed_tokens.weight"] == EMBEDDING_LR, lrs+ assert lrs["lm_head.weight"] == EMBEDDING_LR, lrs+ assert lrs["model.proj.weight"] == LR, lrs+++def test_lm_head_without_an_output_getter_still_gets_embedding_learning_rate():+ torch = pytest.importorskip("torch")+ model = _model(torch.nn, tied = False)+ model.get_output_embeddings = lambda: None # transformers 4.x default+ lrs = _lr_by_name(torch, model)++ assert lrs["lm_head.weight"] == EMBEDDING_LR, lrs+ assert lrs["model.proj.weight"] == LR, lrsdiff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 44480a3df56..c5dd49445d1 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -711,6 +711,25 @@ class UnslothTrainingArguments(TrainingArguments):)+def _embedding_weight_ids(model):+ """Full finetuning has no modules_to_save copy, so the trainable embedding and lm_head+ are found by identity; matching on the PEFT name alone trained them at the base lr."""+ embedding_ids = set()+ for owner in (model, getattr(model, "module", None)):+ for getter in ("get_input_embeddings", "get_output_embeddings"):+ try:+ weight = getattr(getattr(owner, getter)(), "weight", None)+ except Exception:+ continue+ if weight is not None:+ embedding_ids.add(id(weight))+ # transformers 4.x's default get_output_embeddings returns None even with an lm_head.+ weight = getattr(getattr(owner, "lm_head", None), "weight", None)+ if weight is not None:+ embedding_ids.add(id(weight))+ return embedding_ids++def _create_unsloth_optimizer(model,optimizer_cls,@@ -740,11 +759,18 @@ def _create_unsloth_optimizer(# A subset of non_embeddings, not its own dict: legacy resume replays the old two-group order.lora_plus_names = set()+ embedding_ids = _embedding_weight_ids(model)++ # Name-only classification of releases before the identity match, kept to resume their checkpoints.+ previous_groups = {"non_embeddings": {}, "embeddings": {}}for name, param in model.named_parameters():if not param.requires_grad:continue- if name.endswith("modules_to_save.default.weight"):- partial_name = name[: -len(".modules_to_save.default.weight")]+ by_name = name.endswith("modules_to_save.default.weight")+ previous_groups["embeddings" if by_name else "non_embeddings"][name] = param+ if by_name or id(param) in embedding_ids:+ partial_name = name.removesuffix(".modules_to_save.default.weight")+ partial_name = partial_name.removesuffix(".weight")partial_name = partial_name[partial_name.rfind(".") + 1 :]if embedding_lr != lr:print(@@ -784,30 +810,44 @@ def _create_unsloth_optimizer(# Empty groups are dropped (a LoRA run trains no bias and no norm, so both no-decay ones are# empty): AdafactorSchedule.get_lr reads group["params"][0] unguarded, and load_state_dict# rejects a checkpoint whose group count differs, which would break resume.- optimizer_grouped_parameters = []- group_roles = []- for group, source, group_lr in (- ("non_embeddings", "non_embeddings", lr),- ("lora_plus", "non_embeddings", lr * (loraplus_lr_ratio or 1.0)),- ("embeddings", "embeddings", embedding_lr),- ):- for decays in (True, False):- params = [- param- for name, param in param_groups[source].items()- if (name in decay_parameter_names) is decays- and (group == "lora_plus") is (name in lora_plus_names)- ]- if not params:- continue- optimizer_grouped_parameters.append(- {- "params": params,- "weight_decay": weight_decay if decays else 0.0,- "lr": group_lr,- }- )- group_roles.append(group)+ def _layout(groups):+ grouped_parameters, roles = [], []+ for group, source, group_lr in (+ ("non_embeddings", "non_embeddings", lr),+ ("lora_plus", "non_embeddings", lr * (loraplus_lr_ratio or 1.0)),+ ("embeddings", "embeddings", embedding_lr),+ ):+ for decays in (True, False):+ params = [+ param+ for name, param in groups[source].items()+ if (name in decay_parameter_names) is decays+ and (group == "lora_plus") is (name in lora_plus_names)+ ]+ if not params:+ continue+ grouped_parameters.append(+ {+ "params": params,+ "weight_decay": weight_decay if decays else 0.0,+ "lr": group_lr,+ }+ )+ roles.append(group)+ return grouped_parameters, roles++ optimizer_grouped_parameters, group_roles = _layout(param_groups)+ previous_layout = None+ if len(previous_groups["embeddings"]) != len(param_groups["embeddings"]):+ previous_grouped, _ = _layout(previous_groups)+ previous_layout = (+ [param for group in previous_grouped for param in group["params"]],+ [len(group["params"]) for group in previous_grouped],+ [group["lr"] for group in previous_grouped],+ )+ # Sizes alone cannot tell the layouts apart when only embeddings train: mark this one.+ for group in optimizer_grouped_parameters:+ group[_EMBEDDING_SPLIT_MARKER] = Trueoptimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)# Same as Trainer.create_optimizer: keep embedding optimizer state in 32 bits under 8-bit bnb.if "bitsandbytes" in str(optimizer_cls) and optimizer_kwargs.get("optim_bits", None) == 8:@@ -820,10 +860,11 @@ def _create_unsloth_optimizer(manager.register_module_override(module, "weight", {"optim_bits": 32})_install_legacy_resume(optimizer,- legacy_params = list(param_groups["non_embeddings"].values())- + list(param_groups["embeddings"].values()),- legacy_sizes = (len(param_groups["non_embeddings"]), len(param_groups["embeddings"])),+ legacy_params = list(previous_groups["non_embeddings"].values())+ + list(previous_groups["embeddings"].values()),+ legacy_sizes = (len(previous_groups["non_embeddings"]), len(previous_groups["embeddings"])),group_roles = group_roles,+ previous_layout = previous_layout,)return optimizer@@ -886,6 +927,59 @@ def _migrate_legacy_optimizer_state(return {"state": remapped_state, "param_groups": migrated_groups}+_EMBEDDING_SPLIT_MARKER = "unsloth_embedding_split"+++def _migrate_previous_layout_state(+ state_dict, optimizer, previous_params, previous_sizes, previous_lrs+):+ """A checkpoint from before full finetuning embeddings got their own groups, or None.++ Progress is read against the old group's built lr, not initial_lr, which ReduceLROnPlateau never sets.+ """+ saved_groups = state_dict.get("param_groups") or []+ if any(g.get(_EMBEDDING_SPLIT_MARKER) for g in saved_groups):+ return None+ if [len(g["params"]) for g in saved_groups] != list(previous_sizes):+ return None+ saved_ids = [i for g in saved_groups for i in g["params"]]+ new_index_of_param = {+ id(param): index+ for index, param in enumerate(p for g in optimizer.param_groups for p in g["params"])+ }+ if not (len(set(saved_ids)) == len(new_index_of_param) == len(previous_params)):+ return None+ param_of_saved_id = dict(zip(saved_ids, previous_params))+ saved_index_of_param = {+ id(param_of_saved_id[i]): index+ for index, group in enumerate(saved_groups)+ for i in group["params"]+ }+ try:+ remapped_state = {+ new_index_of_param[id(param_of_saved_id[saved_id])]: value+ for saved_id, value in state_dict["state"].items()+ }+ except KeyError:+ return None++ cursor, migrated_groups = 0, []+ for group in optimizer.param_groups:+ index = saved_index_of_param[id(group["params"][0])]+ saved = saved_groups[index]+ base_lr = group.get("initial_lr", group["lr"])+ progress = saved["lr"] / previous_lrs[index] if previous_lrs[index] else 1.0+ migrated = {key: value for key, value in saved.items() if key != "params"}+ migrated.update(lr = base_lr * progress, weight_decay = group["weight_decay"])+ if "initial_lr" in migrated:+ migrated["initial_lr"] = base_lr+ migrated[_EMBEDDING_SPLIT_MARKER] = True+ migrated["params"] = list(range(cursor, cursor + len(group["params"])))+ migrated_groups.append(migrated)+ cursor += len(group["params"])+ return {"state": remapped_state, "param_groups": migrated_groups}++def _unsloth_base_optimizer(optimizer):"""The optimizer we built, through any wrapper, or None.@@ -920,6 +1014,13 @@ def _install_legacy_scheduler_resume(scheduler, optimizer):# torch.save cannot pickle it, failing the first save even with no resume.def load_state_dict(self, state_dict):state_dict = dict(state_dict)+ if getattr(built, "_unsloth_loaded_previous", False):+ # Per-group lists of the old layout: keep the ones built from this optimizer.+ for key in ("base_lrs", "_last_lr", "min_lrs", "max_lrs", "_init_lrs"):+ state_dict.pop(key, None)+ base.load_state_dict(self, state_dict)+ self._last_lr = [group["lr"] for group in self.optimizer.param_groups]+ return# Gated on the optimizer having just migrated: length alone cannot tell a legacy# [ordinary, embedding] pair from two current non-embedding groups, and remapping# the latter collapses distinct per-group min_lr floors onto the first.@@ -940,7 +1041,13 @@ def _install_legacy_scheduler_resume(scheduler, optimizer):return scheduler-def _install_legacy_resume(optimizer, legacy_params, legacy_sizes, group_roles):+def _install_legacy_resume(+ optimizer,+ legacy_params,+ legacy_sizes,+ group_roles,+ previous_layout = None,+):"""Let `load_state_dict` accept a checkpoint written before the decay split."""optimizer._unsloth_group_roles = list(group_roles)original = optimizer.load_state_dict@@ -948,6 +1055,17 @@ def _install_legacy_resume(optimizer, legacy_params, legacy_sizes, group_roles):@wraps(original)def load_state_dict(state_dict):saved = state_dict.get("param_groups") or []+ previous = None+ if previous_layout is not None:+ previous = _migrate_previous_layout_state(state_dict, optimizer, *previous_layout)+ optimizer._unsloth_loaded_previous = previous is not None+ if previous is not None:+ print(+ "Unsloth: moving the embeddings of a checkpoint saved before full finetuning "+ "used embedding_learning_rate into their own optimizer group."+ )+ optimizer._unsloth_loaded_legacy = False+ return original(previous)# Every recognisably legacy checkpoint, not only reshaped ones: torch takes group# hyperparameters from the saved dict, so a LoRA resume, where the layouts coincide,# would otherwise reload the 0.0 this change exists to correct.@@ -1087,6 +1205,7 @@ class UnslothTrainer(SFTTrainer):if embedding_lr is not None:# Fast param -> name lookup, O(N) instead of O(N*M).param_to_name = {id(p): name for name, p in model.named_parameters()}+ embedding_ids = _embedding_weight_ids(model)new_groups = []for group in param_groups:@@ -1099,8 +1218,11 @@ class UnslothTrainer(SFTTrainer):other_params = []for p in group["params"]:name = param_to_name.get(id(p))- if name and name.endswith("modules_to_save.default.weight"):- partial_name = name[: -len(".modules_to_save.default.weight")]+ if name and (+ name.endswith("modules_to_save.default.weight") or id(p) in embedding_ids+ ):+ partial_name = name.removesuffix(".modules_to_save.default.weight")+ partial_name = partial_name.removesuffix(".weight")partial_name = partial_name[partial_name.rfind(".") + 1 :]print(f"Unsloth: Setting lr = {embedding_lr:.2e} instead of {lr:.2e} for {partial_name}."