Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep weight_decay when embedding_learning_rate is set (#11470)
Vineeth Sai Varikuntlaed004ac705b9
diff --git a/tests/utils/test_create_optimizer_model_arg.py b/tests/utils/test_create_optimizer_model_arg.pyindex 43bf8f3e1a3..eda1b94af49 100644--- a/tests/utils/test_create_optimizer_model_arg.py+++ b/tests/utils/test_create_optimizer_model_arg.py@@ -132,4 +132,8 @@ def test_embedding_lr_without_embeddings_is_still_fine_off_the_delayed_path():plain = nn.Linear(8, 8, bias = False)optimizer = _create_unsloth_optimizer(plain, torch.optim.AdamW, {"lr": 1e-3}, 5e-5)- assert optimizer.param_groups[1]["params"] == []+ # Asserted by meaning rather than by group index: empty groups are dropped, so+ # there is no embedding group at all, and the one weight trains at the ordinary lr.+ groups = optimizer.param_groups+ assert [p for group in groups for p in group["params"]] == [plain.weight]+ assert all(group["lr"] == 1e-3 for group in groups), groupsdiff --git a/tests/utils/test_unsloth_optimizer_weight_decay.py b/tests/utils/test_unsloth_optimizer_weight_decay.pynew file mode 100644index 00000000000..de283b0ba51--- /dev/null+++ b/tests/utils/test_unsloth_optimizer_weight_decay.py@@ -0,0 +1,523 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+#+# `embedding_learning_rate` routes optimizer creation through+# `_create_unsloth_optimizer`, which read the decay from `optimizer_kwargs`.+# transformers keeps the decay on the param groups and only puts it in+# optimizer_kwargs for schedule-free and stable_adamw, so the `.get(..., 0.0)`+# default always won and the whole run trained with no weight decay, silently,+# with the value the user set still on the config.++from types import MethodType, SimpleNamespace++import pytest++import unsloth # noqa: F401 (must precede transformers/trl)+from unsloth.trainer import UnslothTrainer+++EMBEDDING = "model.embed_tokens.modules_to_save.default.weight"+++def _model(torch, nn):+ model = nn.Module()+ model.proj = nn.Linear(4, 4) # .weight decays, .bias does not+ model.norm = nn.LayerNorm(4) # neither weight nor bias decays+ embed = nn.Module()+ embed.modules_to_save = nn.ModuleDict({"default": nn.Linear(4, 4, bias = False)})+ inner = nn.Module()+ inner.embed_tokens = embed+ model.model = inner+ return model+++def _trainer(torch, weight_decay):+ from transformers import Trainer+ from trl import SFTConfig++ args = SFTConfig(+ output_dir = "/tmp/unsloth-weight-decay-test",+ weight_decay = weight_decay,+ learning_rate = 2e-4,+ optim = "adamw_torch",+ report_to = [],+ )+ args.embedding_learning_rate = 5e-5+ trainer = SimpleNamespace(args = args, model = None, optimizer = None)+ trainer.get_decay_parameter_names = MethodType(Trainer.get_decay_parameter_names, trainer)+ return trainer+++def _groups(torch, nn, weight_decay):+ model = _model(torch, nn)+ trainer = _trainer(torch, weight_decay)+ trainer.model = model+ optimizer = UnslothTrainer.create_optimizer(trainer, model)+ named = {id(p): n for n, p in model.named_parameters()}+ return [+ {+ "names": sorted(named[id(p)] for p in group["params"]),+ "weight_decay": group["weight_decay"],+ "lr": group["lr"],+ }+ for group in optimizer.param_groups+ ]+++def test_weight_decay_from_the_config_reaches_the_param_groups():+ torch = pytest.importorskip("torch")+ nn = torch.nn++ groups = _groups(torch, nn, weight_decay = 0.1)+ decayed = {name for g in groups if g["weight_decay"] == 0.1 for name in g["names"]}++ assert "proj.weight" in decayed, f"weight_decay=0.1 never reached the optimizer: {groups}"+ assert EMBEDDING in decayed, "the embedding group must carry the decay too"+++def test_biases_and_norms_stay_out_of_the_decay():+ # Trainer.get_decay_parameter_names excludes them on the path this replaces.+ torch = pytest.importorskip("torch")+ nn = torch.nn++ groups = _groups(torch, nn, weight_decay = 0.1)+ undecayed = {name for g in groups if g["weight_decay"] == 0.0 for name in g["names"]}++ assert {"proj.bias", "norm.weight", "norm.bias"} <= undecayed, groups+++def test_the_embedding_group_keeps_its_own_learning_rate():+ torch = pytest.importorskip("torch")+ nn = torch.nn++ groups = _groups(torch, nn, weight_decay = 0.1)+ for group in groups:+ if EMBEDDING in group["names"]:+ assert group["lr"] == 5e-5, groups+ elif group["names"]:+ assert group["lr"] == 2e-4, groups+++def test_zero_weight_decay_is_still_zero():+ torch = pytest.importorskip("torch")+ nn = torch.nn++ groups = _groups(torch, nn, weight_decay = 0.0)+ assert all(group["weight_decay"] == 0.0 for group in groups), groups+++def _lora_shaped(torch, nn):+ """A LoRA run trains no bias and no norm, so both no-decay groups come back empty."""+ model = _model(torch, nn)+ model.proj.bias.requires_grad_(False)+ model.norm.weight.requires_grad_(False)+ model.norm.bias.requires_grad_(False)+ return model+++def _optimizer(+ torch,+ model,+ weight_decay,+ optim = "adamw_torch",+):+ trainer = _trainer(torch, weight_decay)+ trainer.args.optim = optim+ trainer.model = model+ return UnslothTrainer.create_optimizer(trainer, model)+++def test_no_empty_param_groups():+ # An empty group is not free. AdafactorSchedule reads group["params"][0] with no+ # guard, and torch's load_state_dict insists the checkpoint have the same number+ # of groups, so an empty one is a crash and a broken resume for nothing.+ torch = pytest.importorskip("torch")+ nn = torch.nn++ for model in (_model(torch, nn), _lora_shaped(torch, nn)):+ optimizer = _optimizer(torch, model, 0.1)+ assert all(group["params"] for group in optimizer.param_groups), optimizer.param_groups+++def test_adafactor_schedule_survives_the_split():+ torch = pytest.importorskip("torch")+ nn = torch.nn+ from transformers.optimization import AdafactorSchedule++ model = _lora_shaped(torch, nn)+ optimizer = _optimizer(torch, model, 0.1, optim = "adafactor")+ for param in model.parameters():+ if param.requires_grad:+ param.grad = torch.randn_like(param)+ optimizer.step()++ assert AdafactorSchedule(optimizer).get_lr() # IndexError if any group is empty+++def test_a_lora_run_started_before_this_can_still_resume():+ # The old two-group build is what every checkpoint out there was written with.+ torch = pytest.importorskip("torch")+ nn = torch.nn++ model = _lora_shaped(torch, nn)+ trainable = [(name, p) for name, p in model.named_parameters() if p.requires_grad]+ old = torch.optim.AdamW(+ [+ {+ "params": [p for name, p in trainable if name != EMBEDDING],+ "weight_decay": 0.0,+ "lr": 2e-4,+ },+ {+ "params": [p for name, p in trainable if name == EMBEDDING],+ "weight_decay": 0.0,+ "lr": 5e-5,+ },+ ],+ lr = 2e-4,+ )+ for _, param in trainable:+ param.grad = torch.randn_like(param)+ old.step()++ new = _optimizer(torch, model, 0.1)+ new.load_state_dict(old.state_dict()) # ValueError if the group count moved+++def _decay_names(torch, model):+ from transformers import Trainer+ return Trainer.get_decay_parameter_names(None, model)+++def _legacy_optimizer(+ torch,+ model,+ lr = 2e-4,+ embedding_lr = 5e-5,+):+ """The two groups every checkpoint written before the decay split carries."""+ trainable = [(name, p) for name, p in model.named_parameters() if p.requires_grad]+ return torch.optim.AdamW(+ [+ {+ "params": [p for name, p in trainable if name != EMBEDDING],+ "weight_decay": 0.0,+ "lr": lr,+ },+ {+ "params": [p for name, p in trainable if name == EMBEDDING],+ "weight_decay": 0.0,+ "lr": embedding_lr,+ },+ ],+ lr = lr,+ )+++def test_a_full_finetune_checkpoint_from_before_the_split_still_resumes():+ # Trainable norms give the new layout three groups where the checkpoint has two.+ torch = pytest.importorskip("torch")+ nn = torch.nn++ model = _model(torch, nn)+ old = _legacy_optimizer(torch, model)+ for index, (_, param) in enumerate(model.named_parameters()):+ param.grad = torch.full_like(param, float(index + 1))+ old.step()+ saved = old.state_dict()+ flat = [p for group in old.param_groups for p in group["params"]]+ named = {id(p): n for n, p in model.named_parameters()}+ want = {named[id(p)]: float(saved["state"][i]["exp_avg"].sum()) for i, p in enumerate(flat)}++ new = _optimizer(torch, model, 0.1)+ assert len(new.param_groups) != len(saved["param_groups"]), "no migration exercised"+ new.load_state_dict(saved)++ # Not just "it loaded": every parameter must get ITS OWN moments back.+ state = new.state_dict()["state"]+ flat = [p for group in new.param_groups for p in group["params"]]+ for index, param in enumerate(flat):+ assert float(state[index]["exp_avg"].sum()) == pytest.approx(want[named[id(param)]])+ # and the decay the run was configured with, not the 0.0 the checkpoint carries.+ assert any(group["weight_decay"] == 0.1 for group in new.param_groups), new.param_groups+++def test_a_checkpoint_that_is_not_the_old_shape_is_refused_not_guessed():+ # Fails closed: pairing a parameter with another parameter's moments would corrupt+ # the run silently, which is worse than the error this keeps.+ torch = pytest.importorskip("torch")+ nn = torch.nn++ model = _model(torch, nn)+ new = _optimizer(torch, model, 0.1)+ bogus = {+ "state": {},+ "param_groups": [{"params": [0], "lr": 2e-4, "weight_decay": 0.0}],+ }+ with pytest.raises(ValueError):+ new.load_state_dict(bogus)+++def test_the_scheduler_survives_the_same_resume():+ # Fixing only the optimizer moves the failure one line later, into base_lrs.+ torch = pytest.importorskip("torch")+ nn = torch.nn+ from torch.optim.lr_scheduler import LambdaLR+ from unsloth.trainer import _install_legacy_scheduler_resume++ model = _model(torch, nn)+ old_scheduler = LambdaLR(_legacy_optimizer(torch, model), lambda step: 1.0)+ old_scheduler.step()+ saved = old_scheduler.state_dict()++ new_optimizer = _optimizer(torch, model, 0.1)+ new_optimizer.load_state_dict(_legacy_optimizer(torch, model).state_dict())+ new_scheduler = _install_legacy_scheduler_resume(+ LambdaLR(new_optimizer, lambda step: 1.0), new_optimizer+ )+ new_scheduler.load_state_dict(saved)+ new_scheduler.step() # ValueError from zip(strict=True) if base_lrs was not expanded+ assert len(new_scheduler.base_lrs) == len(new_optimizer.param_groups)+++def test_a_checkpoint_with_the_same_group_count_but_a_different_shape_migrates():+ # No trainable embedding: legacy is [all non-embeddings, empty], new is+ # [decayed, non-decayed]. Two groups either way, so a count check misses it.+ torch = pytest.importorskip("torch")+ nn = torch.nn+ from unsloth.trainer import _create_unsloth_optimizer++ model = nn.Module()+ model.proj = nn.Linear(4, 4) # .weight decays, .bias does not+ trainable = [p for p in model.parameters() if p.requires_grad]+ old = torch.optim.AdamW(+ [+ {"params": trainable, "lr": 2e-4, "weight_decay": 0.0},+ {"params": [], "lr": 5e-5, "weight_decay": 0.0},+ ],+ lr = 2e-4,+ )+ for index, param in enumerate(model.parameters()):+ param.grad = torch.full_like(param, float(index + 1))+ old.step()+ saved = old.state_dict()++ new = _create_unsloth_optimizer(+ model,+ torch.optim.AdamW,+ {"lr": 2e-4},+ 5e-5,+ weight_decay = 0.1,+ decay_parameter_names = _decay_names(torch, model),+ )+ assert len(new.param_groups) == len(saved["param_groups"]), "count check would have caught it"+ assert [len(g["params"]) for g in new.param_groups] != [+ len(g["params"]) for g in saved["param_groups"]+ ]+ new.load_state_dict(saved)+ assert sum(len(g["params"]) for g in new.param_groups) == len(trainable)+++def test_the_plateau_scheduler_min_lrs_are_remapped_too():+ torch = pytest.importorskip("torch")+ nn = torch.nn+ from torch.optim.lr_scheduler import ReduceLROnPlateau+ from unsloth.trainer import _install_legacy_scheduler_resume++ model = _model(torch, nn)+ old = ReduceLROnPlateau(_legacy_optimizer(torch, model), min_lr = [1e-7, 2e-7])+ saved = old.state_dict()++ new_optimizer = _optimizer(torch, model, 0.1)+ # transformers loads the optimizer before the scheduler; that is what marks the+ # checkpoint legacy, so the sequence matters and is reproduced here.+ new_optimizer.load_state_dict(_legacy_optimizer(torch, model).state_dict())+ new_scheduler = _install_legacy_scheduler_resume(+ ReduceLROnPlateau(new_optimizer, min_lr = 1e-7), new_optimizer+ )+ new_scheduler.load_state_dict(saved)+ assert len(new_scheduler.min_lrs) == len(new_optimizer.param_groups)+ for _ in range(14):+ new_scheduler.step(1.0) # RuntimeError if min_lrs is still the legacy length+++def test_resuming_a_lora_checkpoint_keeps_the_corrected_decay():+ # Old and new layouts coincide here, so nothing needs reshaping, but torch takes the+ # hyperparameters from the saved dict: without the migration the run reloads the 0.0+ # this change exists to correct and trains undecayed again, silently.+ torch = pytest.importorskip("torch")+ nn = torch.nn++ model = _lora_shaped(torch, nn)+ old = _legacy_optimizer(torch, model)+ for _, param in [(n, p) for n, p in model.named_parameters() if p.requires_grad]:+ param.grad = torch.randn_like(param)+ old.step()+ saved = old.state_dict()++ new = _optimizer(torch, model, 0.1)+ assert [len(g["params"]) for g in new.param_groups] == [+ len(g["params"]) for g in saved["param_groups"]+ ], "shapes must coincide or this is testing the reshaping path instead"+ new.load_state_dict(saved)+ assert all(group["weight_decay"] == 0.1 for group in new.param_groups), new.param_groups+++def test_scheduler_state_of_equal_length_is_remapped_by_role_not_position():+ # No trainable embedding: both current groups are non-embeddings, so the saved+ # [ordinary, embedding] pair is the same length but means something else.+ torch = pytest.importorskip("torch")+ nn = torch.nn+ from torch.optim.lr_scheduler import LambdaLR+ from unsloth.trainer import _create_unsloth_optimizer, _install_legacy_scheduler_resume++ model = nn.Module()+ model.proj = nn.Linear(4, 4) # .weight decays, .bias does not+ trainable = [p for p in model.parameters() if p.requires_grad]+ old_optimizer = torch.optim.AdamW(+ [{"params": trainable, "lr": 2e-4}, {"params": [], "lr": 5e-5}], lr = 2e-4+ )+ saved = LambdaLR(old_optimizer, lambda step: 1.0).state_dict()+ assert saved["base_lrs"] == [2e-4, 5e-5]++ new_optimizer = _create_unsloth_optimizer(+ model,+ torch.optim.AdamW,+ {"lr": 2e-4},+ 5e-5,+ weight_decay = 0.1,+ decay_parameter_names = _decay_names(torch, model),+ )+ assert new_optimizer._unsloth_group_roles == ["non_embeddings", "non_embeddings"]+ new_optimizer.load_state_dict(old_optimizer.state_dict())+ scheduler = _install_legacy_scheduler_resume(+ LambdaLR(new_optimizer, lambda step: 1.0), new_optimizer+ )+ scheduler.load_state_dict(saved)+ assert scheduler.base_lrs == [2e-4, 2e-4], scheduler.base_lrs+++def test_the_scheduler_hook_stays_out_of_the_checkpoint():+ # A scheduler's state_dict is its __dict__ minus the optimizer, so hooking by+ # instance attribute puts an unpicklable closure into every checkpoint and the+ # FIRST save fails, resume or no resume.+ torch = pytest.importorskip("torch")+ nn = torch.nn+ import io+ from torch.optim.lr_scheduler import LambdaLR+ from unsloth.trainer import _install_legacy_scheduler_resume++ model = _model(torch, nn)+ optimizer = _optimizer(torch, model, 0.1)+ scheduler = _install_legacy_scheduler_resume(LambdaLR(optimizer, lambda step: 1.0), optimizer)++ state = scheduler.state_dict()+ assert not [key for key, value in state.items() if callable(value)], state.keys()+ torch.save(state, io.BytesIO()) # PicklingError if the hook leaked into the state+ torch.save(optimizer.state_dict(), io.BytesIO())+ # and it is still an ordinary scheduler to everyone else+ assert isinstance(scheduler, LambdaLR)+++def test_the_scheduler_hook_survives_an_optimizer_wrapper():+ # accelerator.prepare swaps in an AcceleratedOptimizer before create_scheduler runs.+ # It defines no __getattr__, so asking it directly finds no roles and the hook would+ # quietly do nothing on the ordinary training path. That property of the real class is+ # asserted here too, so this stops passing if accelerate ever changes it.+ torch = pytest.importorskip("torch")+ nn = torch.nn+ from torch.optim.lr_scheduler import LambdaLR+ from unsloth.trainer import _install_legacy_scheduler_resume++ accelerate_optimizer = pytest.importorskip("accelerate.optimizer")+ real = accelerate_optimizer.AcceleratedOptimizer+ assert not any(+ "__getattr__" in cls.__dict__ for cls in real.__mro__+ ), "AcceleratedOptimizer now delegates attributes; the unwrap may be unnecessary"++ class Wrapper(torch.optim.Optimizer):+ """The same shape as the real one: holds the optimizer, forwards nothing else."""++ def __init__(self, optimizer):+ self.optimizer = optimizer++ @property+ def param_groups(self):+ return self.optimizer.param_groups++ model = _model(torch, nn)+ optimizer = _optimizer(torch, model, 0.1)+ wrapped = Wrapper(optimizer)+ assert "_unsloth_group_roles" not in wrapped.__dict__++ scheduler = _install_legacy_scheduler_resume(LambdaLR(wrapped, lambda step: 1.0), wrapped)+ assert getattr(type(scheduler), "_unsloth_legacy_resume", False), "hook silently skipped"++ optimizer.load_state_dict(_legacy_optimizer(torch, model).state_dict())+ saved = LambdaLR(_legacy_optimizer(torch, model), lambda step: 1.0).state_dict()+ scheduler.load_state_dict(saved)+ assert len(scheduler.base_lrs) == len(optimizer.param_groups)+++def test_migration_keeps_optimizer_specific_group_state():+ # Schedule-free and friends keep algorithm progress in the param group itself;+ # rebuilding from the fresh group would reset it while keeping the per-parameter+ # state, leaving the resumed optimizer inconsistent with itself.+ torch = pytest.importorskip("torch")+ nn = torch.nn++ model = _lora_shaped(torch, nn)+ old = _legacy_optimizer(torch, model)+ for group in old.param_groups:+ group["k"] = 17+ group["weight_sum"] = 3.5+ for _, param in [(n, p) for n, p in model.named_parameters() if p.requires_grad]:+ param.grad = torch.randn_like(param)+ old.step()++ new = _optimizer(torch, model, 0.1)+ new.load_state_dict(old.state_dict())+ assert all(group["k"] == 17 for group in new.param_groups), new.param_groups+ assert all(group["weight_sum"] == 3.5 for group in new.param_groups), new.param_groups+ # but the decay is still the corrected one, not the checkpoint's 0.0+ assert all(group["weight_decay"] == 0.1 for group in new.param_groups), new.param_groups+++def test_a_current_two_group_checkpoint_is_not_mistaken_for_a_legacy_one():+ # With no trainable embedding both current groups are non_embeddings, so a checkpoint+ # this code wrote also has two scheduler entries. Remapping it by role would copy the+ # first group's value over the second, and distinct per-group min_lr floors would come+ # back collapsed. Length cannot tell the two apart; only the optimizer load can.+ torch = pytest.importorskip("torch")+ nn = torch.nn+ from torch.optim.lr_scheduler import ReduceLROnPlateau+ from unsloth.trainer import _create_unsloth_optimizer, _install_legacy_scheduler_resume++ model = nn.Module()+ model.proj = nn.Linear(4, 4) # .weight decays, .bias does not++ def build():+ optimizer = _create_unsloth_optimizer(+ model,+ torch.optim.AdamW,+ {"lr": 2e-4},+ 5e-5,+ weight_decay = 0.1,+ decay_parameter_names = _decay_names(torch, model),+ )+ assert optimizer._unsloth_group_roles == ["non_embeddings", "non_embeddings"]+ return optimizer++ written_by_this_code = build()+ saved = _install_legacy_scheduler_resume(+ ReduceLROnPlateau(written_by_this_code, min_lr = [1e-7, 2e-7]), written_by_this_code+ ).state_dict()+ assert saved["min_lrs"] == [1e-7, 2e-7]++ resumed = build()+ resumed.load_state_dict(written_by_this_code.state_dict()) # not a legacy shape+ scheduler = _install_legacy_scheduler_resume(+ ReduceLROnPlateau(resumed, min_lr = [1e-7, 2e-7]), resumed+ )+ scheduler.load_state_dict(saved)+ assert scheduler.min_lrs == [1e-7, 2e-7], scheduler.min_lrsdiff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 6bb5dbe0acc..86d92a04af8 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -679,9 +679,17 @@ def _create_unsloth_optimizer(optimizer_kwargs,embedding_lr = 5e-5,require_embedding_match = False,+ weight_decay = 0.0,+ decay_parameter_names = None,):lr = optimizer_kwargs["lr"]- weight_decay = optimizer_kwargs.get("weight_decay", 0.0)+ # transformers puts weight_decay in optimizer_kwargs only for schedule-free and stable_adamw,+ # so reading it from there alone meant the 0.0 default always won (Trainer.create_optimizer).+ weight_decay = optimizer_kwargs.get("weight_decay", weight_decay)+ # Trainer.get_decay_parameter_names excludes biases and norms; the default here decays all.+ if decay_parameter_names is None:+ decay_parameter_names = [name for name, _ in model.named_parameters()]+ decay_parameter_names = set(decay_parameter_names)param_groups = {"non_embeddings": {},@@ -714,22 +722,179 @@ def _create_unsloth_optimizer("without FSDP, or drop embedding_learning_rate.")- optimizer_grouped_parameters = [- {- "params": list(param_groups["non_embeddings"].values()),- "weight_decay": weight_decay,- "lr": lr,- },- {- "params": list(param_groups["embeddings"].values()),- "weight_decay": weight_decay,- "lr": embedding_lr,- },- ]+ # Empty groups are dropped (a LoRA run trains no bias and no norm, so both no-decay ones are+ # empty): AdafactorSchedule.get_lr reads group["params"][0] unguarded, and load_state_dict+ # rejects a checkpoint whose group count differs, which would break resume.+ optimizer_grouped_parameters = []+ group_roles = []+ for group, group_lr in (("non_embeddings", lr), ("embeddings", embedding_lr)):+ for decays in (True, False):+ params = [+ param+ for name, param in param_groups[group].items()+ if (name in decay_parameter_names) is decays+ ]+ if not params:+ continue+ optimizer_grouped_parameters.append(+ {+ "params": params,+ "weight_decay": weight_decay if decays else 0.0,+ "lr": group_lr,+ }+ )+ group_roles.append(group)optimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)+ _install_legacy_resume(+ optimizer,+ legacy_params = list(param_groups["non_embeddings"].values())+ + list(param_groups["embeddings"].values()),+ legacy_sizes = (len(param_groups["non_embeddings"]), len(param_groups["embeddings"])),+ group_roles = group_roles,+ )return optimizer+_LEGACY_ROLE_ORDER = ("non_embeddings", "embeddings")+++def _migrate_legacy_optimizer_state(+ state_dict, optimizer, legacy_params, legacy_sizes, group_roles+):+ """A pre-split checkpoint rewritten into this optimizer's layout, or None to refuse.++ Exact, not inferred: torch keys saved state by a parameter's position in the flat+ concatenation of its groups, and the old layout (non-embeddings then embeddings, each+ in `named_parameters` order) is reproducible from the model. Anything else returns+ None and lets torch raise, since pairing a parameter with another parameter's moments+ would corrupt the run in silence where today it stops.+ """+ saved_groups = state_dict.get("param_groups") or []+ if len(saved_groups) != len(_LEGACY_ROLE_ORDER):+ return None+ if tuple(len(g["params"]) for g in saved_groups) != tuple(legacy_sizes):+ return None+ saved_ids = [i for g in saved_groups for i in g["params"]]+ if len(saved_ids) != len(legacy_params) or len(set(saved_ids)) != len(saved_ids):+ return None++ param_of_saved_id = dict(zip(saved_ids, legacy_params))+ new_index_of_param = {+ id(param): index+ for index, param in enumerate(p for g in optimizer.param_groups for p in g["params"])+ }+ if len(new_index_of_param) != len(legacy_params):+ return None+ try:+ remapped_state = {+ new_index_of_param[id(param_of_saved_id[saved_id])]: value+ for saved_id, value in state_dict["state"].items()+ }+ except KeyError:+ return None++ saved_of_role = dict(zip(_LEGACY_ROLE_ORDER, saved_groups))+ cursor, migrated_groups = 0, []+ for group, role in zip(optimizer.param_groups, group_roles):+ size = len(group["params"])+ # Saved group as the base so per-group optimizer state survives (schedule-free+ # keeps k, weight_sum, lr_max there); only params and weight_decay are overridden.+ migrated = {key: value for key, value in group.items() if key != "params"}+ migrated.update(+ {key: value for key, value in saved_of_role[role].items() if key != "params"}+ )+ migrated["weight_decay"] = group["weight_decay"]+ migrated["params"] = list(range(cursor, cursor + size))+ migrated_groups.append(migrated)+ cursor += size+ return {"state": remapped_state, "param_groups": migrated_groups}+++def _unsloth_base_optimizer(optimizer):+ """The optimizer we built, through any wrapper, or None.++ `accelerator.prepare` swaps in an `AcceleratedOptimizer`, which defines no+ `__getattr__`, so asking it directly finds nothing and the hook silently no-ops.+ """+ seen = 0+ while optimizer is not None and seen < 8:+ if "_unsloth_group_roles" in optimizer.__dict__:+ return optimizer+ optimizer = getattr(optimizer, "optimizer", None)+ seen += 1+ return None+++def _install_legacy_scheduler_resume(scheduler, optimizer):+ """The same remap for the scheduler, which keys `base_lrs` by group too.++ Fixing only the optimizer moves the failure one line later: `load_state_dict` overwrites+ `base_lrs` wholesale, and the next step raises from `zip(..., strict=True)`.+ """+ built = _unsloth_base_optimizer(optimizer)+ if built is None:+ return scheduler+ roles = built._unsloth_group_roles+ base = type(scheduler)+ if getattr(base, "_unsloth_legacy_resume", False):+ return scheduler++ # On a per-instance subclass, NOT the instance: a scheduler's state_dict is its __dict__+ # minus the optimizer, so an instance attribute puts this closure in every checkpoint and+ # torch.save cannot pickle it, failing the first save even with no resume.+ def load_state_dict(self, state_dict):+ state_dict = dict(state_dict)+ # Gated on the optimizer having just migrated: length alone cannot tell a legacy+ # [ordinary, embedding] pair from two current non-embedding groups, and remapping+ # the latter collapses distinct per-group min_lr floors onto the first.+ if not getattr(built, "_unsloth_loaded_legacy", False):+ return base.load_state_dict(self, state_dict)+ for key in ("base_lrs", "_last_lr", "min_lrs"):+ saved = state_dict.get(key)+ if isinstance(saved, list) and len(saved) == len(_LEGACY_ROLE_ORDER):+ by_role = dict(zip(_LEGACY_ROLE_ORDER, saved))+ state_dict[key] = [by_role[role] for role in roles]+ return base.load_state_dict(self, state_dict)++ scheduler.__class__ = type(+ base.__name__,+ (base,),+ {"load_state_dict": load_state_dict, "_unsloth_legacy_resume": True},+ )+ return scheduler+++def _install_legacy_resume(optimizer, legacy_params, legacy_sizes, group_roles):+ """Let `load_state_dict` accept a checkpoint written before the decay split."""+ optimizer._unsloth_group_roles = list(group_roles)+ original = optimizer.load_state_dict++ @wraps(original)+ def load_state_dict(state_dict):+ saved = state_dict.get("param_groups") or []+ # Every recognisably legacy checkpoint, not only reshaped ones: torch takes group+ # hyperparameters from the saved dict, so a LoRA resume, where the layouts coincide,+ # would otherwise reload the 0.0 this change exists to correct.+ migrated = _migrate_legacy_optimizer_state(+ state_dict, optimizer, legacy_params, legacy_sizes, group_roles+ )+ # transformers loads the optimizer first, so this is what the scheduler hook reads.+ optimizer._unsloth_loaded_legacy = migrated is not None+ if migrated is not None:+ if [len(group["params"]) for group in saved] != [+ len(group["params"]) for group in optimizer.param_groups+ ]:+ print(+ f"Unsloth: remapping {len(saved)} optimizer parameter group(s) from a "+ f"checkpoint saved before weight decay was split out onto "+ f"{len(optimizer.param_groups)}."+ )+ state_dict = migrated+ return original(state_dict)++ optimizer.load_state_dict = load_state_dict++_SUPER_CREATE_OPTIMIZER_TAKES_MODEL = None@@ -773,9 +938,19 @@ class UnslothTrainer(SFTTrainer):optimizer_kwargs,embedding_learning_rate,require_embedding_match = model is not None,+ weight_decay = self.args.weight_decay,+ decay_parameter_names = self.get_decay_parameter_names(target_model),)return self.optimizer+ def create_scheduler(+ self,+ num_training_steps: int,+ optimizer = None,+ ):+ scheduler = super().create_scheduler(num_training_steps, optimizer)+ return _install_legacy_scheduler_resume(scheduler, optimizer or self.optimizer)+def _create_q_galore_optimizer(self,config: "QGaloreConfig",