Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Restore full-rank gradients after Q-GaLore updates (#10878)

Vineeth Sai Varikuntla48ab67fe561d
diff --git a/tests/utils/test_q_galore.py b/tests/utils/test_q_galore.py
index 91bf397b3a9..9f1ba174bdc 100644
--- a/tests/utils/test_q_galore.py
+++ b/tests/utils/test_q_galore.py
@@ -176,6 +176,32 @@ def test_removed_bitsandbytes_options_are_rejected(name, value):
_adamw_mod.QGaLoreAdamW8bit([nn.Parameter(torch.ones(2))], **{name: value})
[email protected]("out_features,in_features", [(4, 3), (3, 4)])
+def test_projected_step_preserves_full_rank_gradients(out_features, in_features):
+ bnb = pytest.importorskip("bitsandbytes")
+ try:
+ probe = nn.Parameter(torch.ones(2))
+ probe.grad = torch.zeros_like(probe)
+ bnb.optim.AdamW32bit([probe], lr = 0.0).step()
+ except Exception as exc:
+ pytest.skip(
+ reason = f"bitsandbytes build lacks CPU optimizer kernels (older releases): {exc}"
+ )
+ model = nn.Linear(in_features, out_features, bias = False)
+ optimizer = _adamw_mod.QGaLoreAdamW8bit(
+ [{"params": list(model.parameters()), "rank": 1, "quant": False}],
+ lr = 0.01,
+ weight_decay = 0.0,
+ )
+ inputs = torch.arange(2 * in_features, dtype = torch.float32).reshape(2, in_features)
+ for _ in range(2):
+ optimizer.zero_grad(set_to_none = False)
+ model(inputs).square().mean().backward()
+ expected_grad = model.weight.grad.clone()
+ optimizer.step()
+ torch.testing.assert_close(model.weight.grad, expected_grad)
+
+
# ======================================================================
# Projector tests
# ======================================================================
diff --git a/unsloth/optimizers/q_galore_adamw.py b/unsloth/optimizers/q_galore_adamw.py
index d8a900d99d2..fe085db3a3e 100644
--- a/unsloth/optimizers/q_galore_adamw.py
+++ b/unsloth/optimizers/q_galore_adamw.py
@@ -172,7 +172,8 @@ class QGaLoreAdamW8bit(Optimizer2State):
group["_wd_saved"] = group["weight_decay"]
group["weight_decay"] = 0
- grad = state["projector"].project(p.grad, state["step"])
+ full_rank_grad = p.grad
+ grad = state["projector"].project(full_rank_grad, state["step"])
# Zero p.data so the 8-bit update writes the pure delta.
p._saved_data = p.data.clone()
@@ -198,6 +199,7 @@ class QGaLoreAdamW8bit(Optimizer2State):
# project_back stays inline: a loop-local name outlives the iteration (+64 MiB).
p.data = p._saved_data.add_(state["projector"].project_back(p.data))
+ p.grad = full_rank_grad
del p._saved_data
if has_weight_quant: