Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Apply Q-GaLore weight decay before the projected update (#10931)
Vineeth Sai Varikuntla45124cc77e6a
diff --git a/.github/scripts/repack_sqlite_vec.py b/.github/scripts/repack_sqlite_vec.pyindex 142bd3b5e95..b69b684df85 100644--- a/.github/scripts/repack_sqlite_vec.py+++ b/.github/scripts/repack_sqlite_vec.py@@ -5,6 +5,7 @@The wheel is pure Python plus one DLL, generated upstream by sqlite-dist: replacing the DLL,the WHEEL tag and RECORD is the whole port. Usage: repack_sqlite_vec.py <amd64 wheel> <vec0.dll> <out dir>"""+import base64, hashlib, struct, sys, zipfilefrom pathlib import Path@@ -18,10 +19,12 @@ assert data[e_lfanew : e_lfanew + 4] == b"PE\0\0", "not a PE file"machine = struct.unpack_from("<H", data, e_lfanew + 4)[0]assert machine == 0xAA64, f"vec0.dll machine is {machine:#x}, not ARM64 (0xaa64)"+def record_line(name: str, blob: bytes) -> str:digest = base64.urlsafe_b64encode(hashlib.sha256(blob).digest()).rstrip(b"=").decode()return f"{name},sha256={digest},{len(blob)}"+out.mkdir(parents = True, exist_ok = True)with zipfile.ZipFile(src) as zin, zipfile.ZipFile(dst, "w", zipfile.ZIP_DEFLATED) as zout:names = zin.namelist()diff --git a/tests/utils/test_q_galore.py b/tests/utils/test_q_galore.pyindex 69fbd64d8a7..e9f20181040 100644--- a/tests/utils/test_q_galore.py+++ b/tests/utils/test_q_galore.py@@ -120,8 +120,9 @@ def test_optimizer_constructs_against_the_installed_bitsandbytes():@pytest.mark.skipif(not _adamw_mod._HAS_BNB, reason = "bitsandbytes is required")@pytest.mark.parametrize("projected", [True, False])@pytest.mark.parametrize("initial_value", [0.0, 1.0])-def test_default_optimizer_updates_match_adamw(projected, initial_value):- pytest.importorskip("bitsandbytes")[email protected]("weight_decay", [0.0, 0.1])+def test_default_optimizer_updates_match_adamw(projected, initial_value, weight_decay):+ bnb = pytest.importorskip("bitsandbytes")device = "cuda" if torch.cuda.is_available() else "cpu"requires_bnb_optimizer(device)param = nn.Parameter(torch.full((2, 2), initial_value, device = device))@@ -129,8 +130,10 @@ def test_default_optimizer_updates_match_adamw(projected, initial_value):group = {"params": [param]}if projected:group.update(rank = 2, scale = 1.0, quant = False)- optimizer = _adamw_mod.QGaLoreAdamW8bit([group], lr = 0.1, weight_decay = 0.0)- reference_optimizer = torch.optim.AdamW([reference], lr = 0.1, weight_decay = 0.0)+ optimizer = _adamw_mod.QGaLoreAdamW8bit([group], lr = 0.1, weight_decay = weight_decay)+ # Unprojected steps delegate to bitsandbytes, whose CUDA kernel decays AFTER the update.+ reference_cls = torch.optim.AdamW if projected else bnb.optim.AdamW8bit+ reference_optimizer = reference_cls([reference], lr = 0.1, weight_decay = weight_decay)# A diagonal gradient keeps full-rank projection aligned with the AdamW reference.gradient = torch.diag(torch.tensor([2.0, 1.0], device = device))for weight, opt in [(param, optimizer), (reference, reference_optimizer)]:@@ -573,30 +576,6 @@ class TestQGaLoreIntegration:assert "betas" in param_names, "betas not in QGaLoreAdamW8bit.__init__ params"assert "eps" in param_names, "eps not in QGaLoreAdamW8bit.__init__ params"- def test_weight_decay_uses_saved_data(self):- """Weight decay should apply standard decoupled AdamW decay on current weights."""- _adamw_mod_local = sys.modules["unsloth.optimizers.q_galore_adamw"]-- p = torch.nn.Parameter(torch.ones(4, 4))- p._saved_data = torch.ones(4, 4) * 2.0 # Pre-update weights- # Simulate project-back: p.data = p._saved_data + projected update.- p.data = p._saved_data.add_(torch.ones(4, 4) * 1.0) # p.data is now 3.0-- group = {"weight_decay": 0.1, "lr": 1.0, "_wd_saved": 0.1}-- # Decoupled weight decay must use p.data, not p._saved_data.- p.data.add_(- p.data,- alpha = -group["lr"] * group["_wd_saved"],- )-- del p._saved_data # Clean up after all uses, matching fixed code-- # 3.0 - (1.0 * 0.1 * 3.0) = 2.7- assert torch.allclose(- p.data, torch.tensor(2.7)- ), "Weight decay didn't use p.data for decoupled decay!"-def test_params_float_after_weight_quant_step(self):"""After a step with weight_quant=True, parameters must remain floating point."""_adamw_mod_local = sys.modules["unsloth.optimizers.q_galore_adamw"]diff --git a/unsloth/optimizers/q_galore_adamw.py b/unsloth/optimizers/q_galore_adamw.pyindex b705a3d6d20..1aa4df7a50e 100644--- a/unsloth/optimizers/q_galore_adamw.py+++ b/unsloth/optimizers/q_galore_adamw.py@@ -185,18 +185,18 @@ class QGaLoreAdamW8bit(Optimizer2State):self.update_step(group, p, gindex, pindex)if "rank" in group:- # p.data now holds the weight update in low-rank space.- p.data = p._saved_data.add_(state["projector"].project_back(p.data))-- # Re-apply decoupled weight decay using pre-update weights.+ # Decay the pre-update weight BEFORE adding the update; it touches only+ # p._saved_data, so p.data still holds the low-rank update below.if "_wd_saved" in group:- p.data.add_(- p.data,+ p._saved_data.add_(+ p._saved_data,alpha = -group["lr"] * group["_wd_saved"],)group["weight_decay"] = group["_wd_saved"]del group["_wd_saved"]+ # project_back stays inline: a loop-local name outlives the iteration (+64 MiB).+ p.data = p._saved_data.add_(state["projector"].project_back(p.data))del p._saved_dataif has_weight_quant: