Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Apply embedding_learning_rate to embeddings under full finetuning (#13171)

Daniel Han0cad5cd0e9eb
diff --git a/tests/utils/test_unsloth_optimizer_full_finetune_embedding_lr.py b/tests/utils/test_unsloth_optimizer_full_finetune_embedding_lr.py
new file mode 100644
index 00000000000..e4440778486
--- /dev/null
+++ b/tests/utils/test_unsloth_optimizer_full_finetune_embedding_lr.py
@@ -0,0 +1,248 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+#
+# Under full finetuning the embedding and lm_head are plain parameters with no PEFT
+# `modules_to_save` copy, so the name match found nothing and `embedding_learning_rate`
+# was dropped without a word: they trained at the base learning rate.
+
+from types import MethodType, SimpleNamespace
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers/trl)
+from unsloth.trainer import UnslothTrainer
+
+
+LR, EMBEDDING_LR = 2e-4, 5e-5
+
+
+def _model(nn, tied):
+ class Tiny(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Module()
+ self.model.embed_tokens = nn.Embedding(10, 4)
+ self.model.proj = nn.Linear(4, 4)
+ self.lm_head = nn.Linear(4, 10, bias = False)
+ if tied:
+ self.lm_head.weight = self.model.embed_tokens.weight
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+ return Tiny()
+
+
+def _optimizer(model):
+ from transformers import Trainer
+ from trl import SFTConfig
+
+ args = SFTConfig(
+ output_dir = "/tmp/unsloth-full-ft-embedding-lr-test",
+ learning_rate = LR,
+ weight_decay = 0.01,
+ optim = "adamw_torch",
+ report_to = [],
+ )
+ args.embedding_learning_rate = EMBEDDING_LR
+ trainer = SimpleNamespace(args = args, model = model, optimizer = None)
+ trainer.get_decay_parameter_names = MethodType(Trainer.get_decay_parameter_names, trainer)
+ return UnslothTrainer.create_optimizer(trainer), trainer.get_decay_parameter_names(model)
+
+
+def _lr_by_name(torch, model):
+ optimizer, _ = _optimizer(model)
+ named = {id(p): n for n, p in model.named_parameters()}
+ return {named[id(p)]: g["lr"] for g in optimizer.param_groups for p in g["params"]}
+
+
[email protected]("tied", [False, True])
+def test_full_finetune_embeddings_get_embedding_learning_rate(tied):
+ torch = pytest.importorskip("torch")
+ lrs = _lr_by_name(torch, _model(torch.nn, tied))
+
+ assert lrs["model.embed_tokens.weight"] == EMBEDDING_LR, lrs
+ if not tied:
+ assert lrs["lm_head.weight"] == EMBEDDING_LR, lrs
+ assert lrs["model.proj.weight"] == LR, lrs
+ assert lrs["model.proj.bias"] == LR, lrs
+
+
+def test_frozen_embeddings_stay_out_of_the_optimizer():
+ torch = pytest.importorskip("torch")
+ model = _model(torch.nn, tied = False)
+ model.model.embed_tokens.weight.requires_grad_(False)
+ lrs = _lr_by_name(torch, model)
+
+ assert "model.embed_tokens.weight" not in lrs, lrs
+ assert lrs["lm_head.weight"] == EMBEDDING_LR, lrs
+
+
+def test_a_checkpoint_from_before_resumes_with_its_own_moments():
+ # The previous layout put the embeddings in the ordinary decay groups, so the new
+ # groups differ; every parameter must get its own moments back, and the embeddings
+ # their own learning rate at the saved point of the schedule.
+ torch = pytest.importorskip("torch")
+ from torch.optim.lr_scheduler import LambdaLR
+ from unsloth.trainer import _install_legacy_scheduler_resume
+
+ schedule = lambda step: 0.5**step
+ model = _model(torch.nn, tied = False)
+ named = {id(p): n for n, p in model.named_parameters()}
+ _, decay = _optimizer(model)
+ params = list(model.named_parameters())
+ old = torch.optim.AdamW(
+ [
+ {"params": [p for n, p in params if n in decay], "weight_decay": 0.01},
+ {"params": [p for n, p in params if n not in decay], "weight_decay": 0.0},
+ ],
+ lr = LR,
+ )
+ old_scheduler = LambdaLR(old, schedule)
+ for index, (_, param) in enumerate(params):
+ param.grad = torch.full_like(param, float(index + 1))
+ old.step()
+ old_scheduler.step()
+ saved, saved_scheduler = old.state_dict(), old_scheduler.state_dict()
+ flat = [p for group in old.param_groups for p in group["params"]]
+ want = {named[id(p)]: float(saved["state"][i]["exp_avg"].sum()) for i, p in enumerate(flat)}
+
+ new, _ = _optimizer(model)
+ assert len(new.param_groups) != len(saved["param_groups"]), "no migration exercised"
+ scheduler = _install_legacy_scheduler_resume(LambdaLR(new, schedule), new)
+ new.load_state_dict(saved)
+ scheduler.load_state_dict(saved_scheduler)
+
+ state = new.state_dict()["state"]
+ flat = [p for group in new.param_groups for p in group["params"]]
+ for index, param in enumerate(flat):
+ assert float(state[index]["exp_avg"].sum()) == pytest.approx(want[named[id(param)]])
+ lr_of = {named[id(p)]: g["lr"] for g in new.param_groups for p in g["params"]}
+ assert lr_of["model.embed_tokens.weight"] == pytest.approx(EMBEDDING_LR * 0.5), lr_of
+ assert lr_of["model.proj.weight"] == pytest.approx(LR * 0.5), lr_of
+
+ new.step()
+ scheduler.step()
+ lr_of = {named[id(p)]: g["lr"] for g in new.param_groups for p in g["params"]}
+ assert lr_of["lm_head.weight"] == pytest.approx(EMBEDDING_LR * 0.25), lr_of
+ assert lr_of["model.proj.bias"] == pytest.approx(LR * 0.25), lr_of
+
+
+def test_a_plateau_reduction_survives_the_same_resume():
+ # ReduceLROnPlateau sets no initial_lr, so the saved reduction is read against the
+ # lr the old group was built with.
+ torch = pytest.importorskip("torch")
+ from torch.optim.lr_scheduler import ReduceLROnPlateau
+ from unsloth.trainer import _install_legacy_scheduler_resume
+
+ model = _model(torch.nn, tied = False)
+ named = {id(p): n for n, p in model.named_parameters()}
+ _, decay = _optimizer(model)
+ params = list(model.named_parameters())
+ old = torch.optim.AdamW(
+ [
+ {"params": [p for n, p in params if n in decay], "weight_decay": 0.01},
+ {"params": [p for n, p in params if n not in decay], "weight_decay": 0.0},
+ ],
+ lr = LR,
+ )
+ old_scheduler = ReduceLROnPlateau(old, factor = 0.5, patience = 0)
+ for metric in (1.0, 2.0):
+ old_scheduler.step(metric)
+ assert old.param_groups[0]["lr"] == pytest.approx(LR * 0.5)
+
+ new, _ = _optimizer(model)
+ scheduler = _install_legacy_scheduler_resume(
+ ReduceLROnPlateau(new, factor = 0.5, patience = 0), new
+ )
+ new.load_state_dict(old.state_dict())
+ scheduler.load_state_dict(old_scheduler.state_dict())
+
+ lr_of = {named[id(p)]: g["lr"] for g in new.param_groups for p in g["params"]}
+ assert lr_of["model.embed_tokens.weight"] == pytest.approx(EMBEDDING_LR * 0.5), lr_of
+ assert lr_of["model.proj.bias"] == pytest.approx(LR * 0.5), lr_of
+ assert len(scheduler.min_lrs) == len(new.param_groups)
+ scheduler.step(3.0)
+ lr_of = {named[id(p)]: g["lr"] for g in new.param_groups for p in g["params"]}
+ assert lr_of["lm_head.weight"] == pytest.approx(EMBEDDING_LR * 0.25), lr_of
+
+
+def test_a_checkpoint_from_this_layout_is_not_migrated_again():
+ # Only the embeddings train: both layouts are one group of the same size, so the
+ # marker, not the sizes, keeps a current checkpoint from being rescaled on resume.
+ torch = pytest.importorskip("torch")
+
+ model = _model(torch.nn, tied = True)
+ model.model.proj.requires_grad_(False)
+ first, _ = _optimizer(model)
+ first.param_groups[0]["lr"] = EMBEDDING_LR * 0.5
+ saved = first.state_dict()
+
+ second, _ = _optimizer(model)
+ second.load_state_dict(saved)
+ assert second.param_groups[0]["lr"] == pytest.approx(EMBEDDING_LR * 0.5)
+ assert second.state_dict()["param_groups"][0].get("unsloth_embedding_split")
+
+
+def test_greedy_lr_keeps_its_per_group_arrays_on_the_same_resume():
+ torch = pytest.importorskip("torch")
+ GreedyLR = getattr(pytest.importorskip("transformers.optimization"), "GreedyLR", None)
+ if GreedyLR is None:
+ pytest.skip("transformers without GreedyLR")
+ from unsloth.trainer import _install_legacy_scheduler_resume
+
+ model = _model(torch.nn, tied = False)
+ _, decay = _optimizer(model)
+ params = list(model.named_parameters())
+ old = torch.optim.AdamW(
+ [
+ {"params": [p for n, p in params if n in decay], "weight_decay": 0.01},
+ {"params": [p for n, p in params if n not in decay], "weight_decay": 0.0},
+ ],
+ lr = LR,
+ )
+ saved_scheduler = GreedyLR(old, min_lr = 0.0, max_lr = 1.0).state_dict()
+
+ new, _ = _optimizer(model)
+ scheduler = _install_legacy_scheduler_resume(GreedyLR(new, min_lr = 0.0, max_lr = 1.0), new)
+ new.load_state_dict(old.state_dict())
+ scheduler.load_state_dict(saved_scheduler)
+ for key in ("max_lrs", "_init_lrs", "min_lrs"):
+ assert len(getattr(scheduler, key)) == len(new.param_groups), key
+
+
+def test_q_galore_embeddings_get_embedding_learning_rate():
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("bitsandbytes")
+ from unsloth.trainer import QGaloreConfig
+
+ model = _model(torch.nn, tied = False)
+ args = SimpleNamespace(
+ learning_rate = LR,
+ weight_decay = 0.0,
+ adam_beta1 = 0.9,
+ adam_beta2 = 0.999,
+ adam_epsilon = 1e-8,
+ )
+ trainer = SimpleNamespace(args = args, model = model, optimizer = None)
+ config = QGaloreConfig(rank = 2, weight_quant = False, target_modules = ["proj"])
+ optimizer = UnslothTrainer._create_q_galore_optimizer(trainer, config, EMBEDDING_LR, model)
+ named = {id(p): n for n, p in model.named_parameters()}
+ lrs = {named[id(p)]: g["lr"] for g in optimizer.param_groups for p in g["params"]}
+
+ assert lrs["model.embed_tokens.weight"] == EMBEDDING_LR, lrs
+ assert lrs["lm_head.weight"] == EMBEDDING_LR, lrs
+ assert lrs["model.proj.weight"] == LR, lrs
+
+
+def test_lm_head_without_an_output_getter_still_gets_embedding_learning_rate():
+ torch = pytest.importorskip("torch")
+ model = _model(torch.nn, tied = False)
+ model.get_output_embeddings = lambda: None # transformers 4.x default
+ lrs = _lr_by_name(torch, model)
+
+ assert lrs["lm_head.weight"] == EMBEDDING_LR, lrs
+ assert lrs["model.proj.weight"] == LR, lrs
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 44480a3df56..c5dd49445d1 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -711,6 +711,25 @@ class UnslothTrainingArguments(TrainingArguments):
)
+def _embedding_weight_ids(model):
+ """Full finetuning has no modules_to_save copy, so the trainable embedding and lm_head
+ are found by identity; matching on the PEFT name alone trained them at the base lr."""
+ embedding_ids = set()
+ for owner in (model, getattr(model, "module", None)):
+ for getter in ("get_input_embeddings", "get_output_embeddings"):
+ try:
+ weight = getattr(getattr(owner, getter)(), "weight", None)
+ except Exception:
+ continue
+ if weight is not None:
+ embedding_ids.add(id(weight))
+ # transformers 4.x's default get_output_embeddings returns None even with an lm_head.
+ weight = getattr(getattr(owner, "lm_head", None), "weight", None)
+ if weight is not None:
+ embedding_ids.add(id(weight))
+ return embedding_ids
+
+
def _create_unsloth_optimizer(
model,
optimizer_cls,
@@ -740,11 +759,18 @@ def _create_unsloth_optimizer(
# A subset of non_embeddings, not its own dict: legacy resume replays the old two-group order.
lora_plus_names = set()
+ embedding_ids = _embedding_weight_ids(model)
+
+ # Name-only classification of releases before the identity match, kept to resume their checkpoints.
+ previous_groups = {"non_embeddings": {}, "embeddings": {}}
for name, param in model.named_parameters():
if not param.requires_grad:
continue
- if name.endswith("modules_to_save.default.weight"):
- partial_name = name[: -len(".modules_to_save.default.weight")]
+ by_name = name.endswith("modules_to_save.default.weight")
+ previous_groups["embeddings" if by_name else "non_embeddings"][name] = param
+ if by_name or id(param) in embedding_ids:
+ partial_name = name.removesuffix(".modules_to_save.default.weight")
+ partial_name = partial_name.removesuffix(".weight")
partial_name = partial_name[partial_name.rfind(".") + 1 :]
if embedding_lr != lr:
print(
@@ -784,30 +810,44 @@ def _create_unsloth_optimizer(
# Empty groups are dropped (a LoRA run trains no bias and no norm, so both no-decay ones are
# empty): AdafactorSchedule.get_lr reads group["params"][0] unguarded, and load_state_dict
# rejects a checkpoint whose group count differs, which would break resume.
- optimizer_grouped_parameters = []
- group_roles = []
- for group, source, group_lr in (
- ("non_embeddings", "non_embeddings", lr),
- ("lora_plus", "non_embeddings", lr * (loraplus_lr_ratio or 1.0)),
- ("embeddings", "embeddings", embedding_lr),
- ):
- for decays in (True, False):
- params = [
- param
- for name, param in param_groups[source].items()
- if (name in decay_parameter_names) is decays
- and (group == "lora_plus") is (name in lora_plus_names)
- ]
- if not params:
- continue
- optimizer_grouped_parameters.append(
- {
- "params": params,
- "weight_decay": weight_decay if decays else 0.0,
- "lr": group_lr,
- }
- )
- group_roles.append(group)
+ def _layout(groups):
+ grouped_parameters, roles = [], []
+ for group, source, group_lr in (
+ ("non_embeddings", "non_embeddings", lr),
+ ("lora_plus", "non_embeddings", lr * (loraplus_lr_ratio or 1.0)),
+ ("embeddings", "embeddings", embedding_lr),
+ ):
+ for decays in (True, False):
+ params = [
+ param
+ for name, param in groups[source].items()
+ if (name in decay_parameter_names) is decays
+ and (group == "lora_plus") is (name in lora_plus_names)
+ ]
+ if not params:
+ continue
+ grouped_parameters.append(
+ {
+ "params": params,
+ "weight_decay": weight_decay if decays else 0.0,
+ "lr": group_lr,
+ }
+ )
+ roles.append(group)
+ return grouped_parameters, roles
+
+ optimizer_grouped_parameters, group_roles = _layout(param_groups)
+ previous_layout = None
+ if len(previous_groups["embeddings"]) != len(param_groups["embeddings"]):
+ previous_grouped, _ = _layout(previous_groups)
+ previous_layout = (
+ [param for group in previous_grouped for param in group["params"]],
+ [len(group["params"]) for group in previous_grouped],
+ [group["lr"] for group in previous_grouped],
+ )
+ # Sizes alone cannot tell the layouts apart when only embeddings train: mark this one.
+ for group in optimizer_grouped_parameters:
+ group[_EMBEDDING_SPLIT_MARKER] = True
optimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)
# Same as Trainer.create_optimizer: keep embedding optimizer state in 32 bits under 8-bit bnb.
if "bitsandbytes" in str(optimizer_cls) and optimizer_kwargs.get("optim_bits", None) == 8:
@@ -820,10 +860,11 @@ def _create_unsloth_optimizer(
manager.register_module_override(module, "weight", {"optim_bits": 32})
_install_legacy_resume(
optimizer,
- legacy_params = list(param_groups["non_embeddings"].values())
- + list(param_groups["embeddings"].values()),
- legacy_sizes = (len(param_groups["non_embeddings"]), len(param_groups["embeddings"])),
+ legacy_params = list(previous_groups["non_embeddings"].values())
+ + list(previous_groups["embeddings"].values()),
+ legacy_sizes = (len(previous_groups["non_embeddings"]), len(previous_groups["embeddings"])),
group_roles = group_roles,
+ previous_layout = previous_layout,
)
return optimizer
@@ -886,6 +927,59 @@ def _migrate_legacy_optimizer_state(
return {"state": remapped_state, "param_groups": migrated_groups}
+_EMBEDDING_SPLIT_MARKER = "unsloth_embedding_split"
+
+
+def _migrate_previous_layout_state(
+ state_dict, optimizer, previous_params, previous_sizes, previous_lrs
+):
+ """A checkpoint from before full finetuning embeddings got their own groups, or None.
+
+ Progress is read against the old group's built lr, not initial_lr, which ReduceLROnPlateau never sets.
+ """
+ saved_groups = state_dict.get("param_groups") or []
+ if any(g.get(_EMBEDDING_SPLIT_MARKER) for g in saved_groups):
+ return None
+ if [len(g["params"]) for g in saved_groups] != list(previous_sizes):
+ return None
+ saved_ids = [i for g in saved_groups for i in g["params"]]
+ new_index_of_param = {
+ id(param): index
+ for index, param in enumerate(p for g in optimizer.param_groups for p in g["params"])
+ }
+ if not (len(set(saved_ids)) == len(new_index_of_param) == len(previous_params)):
+ return None
+ param_of_saved_id = dict(zip(saved_ids, previous_params))
+ saved_index_of_param = {
+ id(param_of_saved_id[i]): index
+ for index, group in enumerate(saved_groups)
+ for i in group["params"]
+ }
+ try:
+ remapped_state = {
+ new_index_of_param[id(param_of_saved_id[saved_id])]: value
+ for saved_id, value in state_dict["state"].items()
+ }
+ except KeyError:
+ return None
+
+ cursor, migrated_groups = 0, []
+ for group in optimizer.param_groups:
+ index = saved_index_of_param[id(group["params"][0])]
+ saved = saved_groups[index]
+ base_lr = group.get("initial_lr", group["lr"])
+ progress = saved["lr"] / previous_lrs[index] if previous_lrs[index] else 1.0
+ migrated = {key: value for key, value in saved.items() if key != "params"}
+ migrated.update(lr = base_lr * progress, weight_decay = group["weight_decay"])
+ if "initial_lr" in migrated:
+ migrated["initial_lr"] = base_lr
+ migrated[_EMBEDDING_SPLIT_MARKER] = True
+ migrated["params"] = list(range(cursor, cursor + len(group["params"])))
+ migrated_groups.append(migrated)
+ cursor += len(group["params"])
+ return {"state": remapped_state, "param_groups": migrated_groups}
+
+
def _unsloth_base_optimizer(optimizer):
"""The optimizer we built, through any wrapper, or None.
@@ -920,6 +1014,13 @@ def _install_legacy_scheduler_resume(scheduler, optimizer):
# torch.save cannot pickle it, failing the first save even with no resume.
def load_state_dict(self, state_dict):
state_dict = dict(state_dict)
+ if getattr(built, "_unsloth_loaded_previous", False):
+ # Per-group lists of the old layout: keep the ones built from this optimizer.
+ for key in ("base_lrs", "_last_lr", "min_lrs", "max_lrs", "_init_lrs"):
+ state_dict.pop(key, None)
+ base.load_state_dict(self, state_dict)
+ self._last_lr = [group["lr"] for group in self.optimizer.param_groups]
+ return
# Gated on the optimizer having just migrated: length alone cannot tell a legacy
# [ordinary, embedding] pair from two current non-embedding groups, and remapping
# the latter collapses distinct per-group min_lr floors onto the first.
@@ -940,7 +1041,13 @@ def _install_legacy_scheduler_resume(scheduler, optimizer):
return scheduler
-def _install_legacy_resume(optimizer, legacy_params, legacy_sizes, group_roles):
+def _install_legacy_resume(
+ optimizer,
+ legacy_params,
+ legacy_sizes,
+ group_roles,
+ previous_layout = None,
+):
"""Let `load_state_dict` accept a checkpoint written before the decay split."""
optimizer._unsloth_group_roles = list(group_roles)
original = optimizer.load_state_dict
@@ -948,6 +1055,17 @@ def _install_legacy_resume(optimizer, legacy_params, legacy_sizes, group_roles):
@wraps(original)
def load_state_dict(state_dict):
saved = state_dict.get("param_groups") or []
+ previous = None
+ if previous_layout is not None:
+ previous = _migrate_previous_layout_state(state_dict, optimizer, *previous_layout)
+ optimizer._unsloth_loaded_previous = previous is not None
+ if previous is not None:
+ print(
+ "Unsloth: moving the embeddings of a checkpoint saved before full finetuning "
+ "used embedding_learning_rate into their own optimizer group."
+ )
+ optimizer._unsloth_loaded_legacy = False
+ return original(previous)
# Every recognisably legacy checkpoint, not only reshaped ones: torch takes group
# hyperparameters from the saved dict, so a LoRA resume, where the layouts coincide,
# would otherwise reload the 0.0 this change exists to correct.
@@ -1087,6 +1205,7 @@ class UnslothTrainer(SFTTrainer):
if embedding_lr is not None:
# Fast param -> name lookup, O(N) instead of O(N*M).
param_to_name = {id(p): name for name, p in model.named_parameters()}
+ embedding_ids = _embedding_weight_ids(model)
new_groups = []
for group in param_groups:
@@ -1099,8 +1218,11 @@ class UnslothTrainer(SFTTrainer):
other_params = []
for p in group["params"]:
name = param_to_name.get(id(p))
- if name and name.endswith("modules_to_save.default.weight"):
- partial_name = name[: -len(".modules_to_save.default.weight")]
+ if name and (
+ name.endswith("modules_to_save.default.weight") or id(p) in embedding_ids
+ ):
+ partial_name = name.removesuffix(".modules_to_save.default.weight")
+ partial_name = partial_name.removesuffix(".weight")
partial_name = partial_name[partial_name.rfind(".") + 1 :]
print(
f"Unsloth: Setting lr = {embedding_lr:.2e} instead of {lr:.2e} for {partial_name}."