Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Apply Q-GaLore weight decay before the projected update (#10931)

Vineeth Sai Varikuntla45124cc77e6a
diff --git a/.github/scripts/repack_sqlite_vec.py b/.github/scripts/repack_sqlite_vec.py
index 142bd3b5e95..b69b684df85 100644
--- a/.github/scripts/repack_sqlite_vec.py
+++ b/.github/scripts/repack_sqlite_vec.py
@@ -5,6 +5,7 @@
The wheel is pure Python plus one DLL, generated upstream by sqlite-dist: replacing the DLL,
the WHEEL tag and RECORD is the whole port. Usage: repack_sqlite_vec.py <amd64 wheel> <vec0.dll> <out dir>
"""
+
import base64, hashlib, struct, sys, zipfile
from pathlib import Path
@@ -18,10 +19,12 @@ assert data[e_lfanew : e_lfanew + 4] == b"PE\0\0", "not a PE file"
machine = struct.unpack_from("<H", data, e_lfanew + 4)[0]
assert machine == 0xAA64, f"vec0.dll machine is {machine:#x}, not ARM64 (0xaa64)"
+
def record_line(name: str, blob: bytes) -> str:
digest = base64.urlsafe_b64encode(hashlib.sha256(blob).digest()).rstrip(b"=").decode()
return f"{name},sha256={digest},{len(blob)}"
+
out.mkdir(parents = True, exist_ok = True)
with zipfile.ZipFile(src) as zin, zipfile.ZipFile(dst, "w", zipfile.ZIP_DEFLATED) as zout:
names = zin.namelist()
diff --git a/tests/utils/test_q_galore.py b/tests/utils/test_q_galore.py
index 69fbd64d8a7..e9f20181040 100644
--- a/tests/utils/test_q_galore.py
+++ b/tests/utils/test_q_galore.py
@@ -120,8 +120,9 @@ def test_optimizer_constructs_against_the_installed_bitsandbytes():
@pytest.mark.skipif(not _adamw_mod._HAS_BNB, reason = "bitsandbytes is required")
@pytest.mark.parametrize("projected", [True, False])
@pytest.mark.parametrize("initial_value", [0.0, 1.0])
-def test_default_optimizer_updates_match_adamw(projected, initial_value):
- pytest.importorskip("bitsandbytes")
[email protected]("weight_decay", [0.0, 0.1])
+def test_default_optimizer_updates_match_adamw(projected, initial_value, weight_decay):
+ bnb = pytest.importorskip("bitsandbytes")
device = "cuda" if torch.cuda.is_available() else "cpu"
requires_bnb_optimizer(device)
param = nn.Parameter(torch.full((2, 2), initial_value, device = device))
@@ -129,8 +130,10 @@ def test_default_optimizer_updates_match_adamw(projected, initial_value):
group = {"params": [param]}
if projected:
group.update(rank = 2, scale = 1.0, quant = False)
- optimizer = _adamw_mod.QGaLoreAdamW8bit([group], lr = 0.1, weight_decay = 0.0)
- reference_optimizer = torch.optim.AdamW([reference], lr = 0.1, weight_decay = 0.0)
+ optimizer = _adamw_mod.QGaLoreAdamW8bit([group], lr = 0.1, weight_decay = weight_decay)
+ # Unprojected steps delegate to bitsandbytes, whose CUDA kernel decays AFTER the update.
+ reference_cls = torch.optim.AdamW if projected else bnb.optim.AdamW8bit
+ reference_optimizer = reference_cls([reference], lr = 0.1, weight_decay = weight_decay)
# A diagonal gradient keeps full-rank projection aligned with the AdamW reference.
gradient = torch.diag(torch.tensor([2.0, 1.0], device = device))
for weight, opt in [(param, optimizer), (reference, reference_optimizer)]:
@@ -573,30 +576,6 @@ class TestQGaLoreIntegration:
assert "betas" in param_names, "betas not in QGaLoreAdamW8bit.__init__ params"
assert "eps" in param_names, "eps not in QGaLoreAdamW8bit.__init__ params"
- def test_weight_decay_uses_saved_data(self):
- """Weight decay should apply standard decoupled AdamW decay on current weights."""
- _adamw_mod_local = sys.modules["unsloth.optimizers.q_galore_adamw"]
-
- p = torch.nn.Parameter(torch.ones(4, 4))
- p._saved_data = torch.ones(4, 4) * 2.0 # Pre-update weights
- # Simulate project-back: p.data = p._saved_data + projected update.
- p.data = p._saved_data.add_(torch.ones(4, 4) * 1.0) # p.data is now 3.0
-
- group = {"weight_decay": 0.1, "lr": 1.0, "_wd_saved": 0.1}
-
- # Decoupled weight decay must use p.data, not p._saved_data.
- p.data.add_(
- p.data,
- alpha = -group["lr"] * group["_wd_saved"],
- )
-
- del p._saved_data # Clean up after all uses, matching fixed code
-
- # 3.0 - (1.0 * 0.1 * 3.0) = 2.7
- assert torch.allclose(
- p.data, torch.tensor(2.7)
- ), "Weight decay didn't use p.data for decoupled decay!"
-
def test_params_float_after_weight_quant_step(self):
"""After a step with weight_quant=True, parameters must remain floating point."""
_adamw_mod_local = sys.modules["unsloth.optimizers.q_galore_adamw"]
diff --git a/unsloth/optimizers/q_galore_adamw.py b/unsloth/optimizers/q_galore_adamw.py
index b705a3d6d20..1aa4df7a50e 100644
--- a/unsloth/optimizers/q_galore_adamw.py
+++ b/unsloth/optimizers/q_galore_adamw.py
@@ -185,18 +185,18 @@ class QGaLoreAdamW8bit(Optimizer2State):
self.update_step(group, p, gindex, pindex)
if "rank" in group:
- # p.data now holds the weight update in low-rank space.
- p.data = p._saved_data.add_(state["projector"].project_back(p.data))
-
- # Re-apply decoupled weight decay using pre-update weights.
+ # Decay the pre-update weight BEFORE adding the update; it touches only
+ # p._saved_data, so p.data still holds the low-rank update below.
if "_wd_saved" in group:
- p.data.add_(
- p.data,
+ p._saved_data.add_(
+ p._saved_data,
alpha = -group["lr"] * group["_wd_saved"],
)
group["weight_decay"] = group["_wd_saved"]
del group["_wd_saved"]
+ # project_back stays inline: a loop-local name outlives the iteration (+64 MiB).
+ p.data = p._saved_data.add_(state["projector"].project_back(p.data))
del p._saved_data
if has_weight_quant: