Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Restore full-rank gradients after Q-GaLore updates (#10878)
Vineeth Sai Varikuntla48ab67fe561d
diff --git a/tests/utils/test_q_galore.py b/tests/utils/test_q_galore.pyindex 91bf397b3a9..9f1ba174bdc 100644--- a/tests/utils/test_q_galore.py+++ b/tests/utils/test_q_galore.py@@ -176,6 +176,32 @@ def test_removed_bitsandbytes_options_are_rejected(name, value):_adamw_mod.QGaLoreAdamW8bit([nn.Parameter(torch.ones(2))], **{name: value})[email protected]("out_features,in_features", [(4, 3), (3, 4)])+def test_projected_step_preserves_full_rank_gradients(out_features, in_features):+ bnb = pytest.importorskip("bitsandbytes")+ try:+ probe = nn.Parameter(torch.ones(2))+ probe.grad = torch.zeros_like(probe)+ bnb.optim.AdamW32bit([probe], lr = 0.0).step()+ except Exception as exc:+ pytest.skip(+ reason = f"bitsandbytes build lacks CPU optimizer kernels (older releases): {exc}"+ )+ model = nn.Linear(in_features, out_features, bias = False)+ optimizer = _adamw_mod.QGaLoreAdamW8bit(+ [{"params": list(model.parameters()), "rank": 1, "quant": False}],+ lr = 0.01,+ weight_decay = 0.0,+ )+ inputs = torch.arange(2 * in_features, dtype = torch.float32).reshape(2, in_features)+ for _ in range(2):+ optimizer.zero_grad(set_to_none = False)+ model(inputs).square().mean().backward()+ expected_grad = model.weight.grad.clone()+ optimizer.step()+ torch.testing.assert_close(model.weight.grad, expected_grad)++# ======================================================================# Projector tests# ======================================================================diff --git a/unsloth/optimizers/q_galore_adamw.py b/unsloth/optimizers/q_galore_adamw.pyindex d8a900d99d2..fe085db3a3e 100644--- a/unsloth/optimizers/q_galore_adamw.py+++ b/unsloth/optimizers/q_galore_adamw.py@@ -172,7 +172,8 @@ class QGaLoreAdamW8bit(Optimizer2State):group["_wd_saved"] = group["weight_decay"]group["weight_decay"] = 0- grad = state["projector"].project(p.grad, state["step"])+ full_rank_grad = p.grad+ grad = state["projector"].project(full_rank_grad, state["step"])# Zero p.data so the 8-bit update writes the pure delta.p._saved_data = p.data.clone()@@ -198,6 +199,7 @@ class QGaLoreAdamW8bit(Optimizer2State):# project_back stays inline: a loop-local name outlives the iteration (+64 MiB).p.data = p._saved_data.add_(state["projector"].project_back(p.data))+ p.grad = full_rank_graddel p._saved_dataif has_weight_quant: