Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix Q-GaLore quantization of single-sign groups (#10841)

Vineeth Sai Varikuntla3bdb5a4214dc
diff --git a/.github/workflows/q-galore-tests.yml b/.github/workflows/q-galore-tests.yml
new file mode 100644
index 00000000000..567b8051180
--- /dev/null
+++ b/.github/workflows/q-galore-tests.yml
@@ -0,0 +1,52 @@
+name: Q-GaLore CPU tests
+
+on:
+ push:
+ branches: [main]
+ paths:
+ - "unsloth/optimizers/**"
+ - "tests/utils/test_q_galore.py"
+ - ".github/workflows/q-galore-tests.yml"
+ pull_request:
+ paths:
+ - "unsloth/optimizers/**"
+ - "tests/utils/test_q_galore.py"
+ - ".github/workflows/q-galore-tests.yml"
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}-${{ github.ref == 'refs/heads/main' && github.sha || '' }}
+ cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+permissions:
+ contents: read
+
+jobs:
+ q-galore:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ strategy:
+ fail-fast: false
+ matrix:
+ # bitsandbytes is not API-stable across the range pyproject allows, so pinning one
+ # version tests one point of it. 0.45.5 is the floor, 0.49.2 the last with
+ # percentile_clipping / block_wise, 0.50.2 the first without them and with CPU
+ # optimizer kernels. This PR's tests are quantizer-only and touch no bitsandbytes
+ # entry point; the constructor coverage that uses these pins arrives with #10874.
+ bitsandbytes: ["0.45.5", "0.49.2", "0.50.2"]
+ name: q-galore (bitsandbytes ${{ matrix.bitsandbytes }})
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.12"
+ - name: Install CPU optimizer test dependencies
+ run: |
+ python -m pip install --index-url https://download.pytorch.org/whl/cpu 'torch==2.10.0'
+ python -m pip install 'pytest==9.0.3' 'bitsandbytes==${{ matrix.bitsandbytes }}'
+ - name: Run Q-GaLore regressions
+ # No --confcutdir: this file loads the optimizers through importlib specs, so the
+ # GPU-spoof conftest neither helps nor hinders it. Verified both ways.
+ run: python -m pytest tests/utils/test_q_galore.py -q
diff --git a/tests/utils/test_q_galore.py b/tests/utils/test_q_galore.py
index 5190d01b124..ccf4910bf2c 100644
--- a/tests/utils/test_q_galore.py
+++ b/tests/utils/test_q_galore.py
@@ -113,6 +113,13 @@ class TestGaLoreProjector:
# INT4 values should be in range [0, 15]
assert proj.ortho_matrix.max() <= 15
+ @pytest.mark.parametrize("n_bit", [4, 8])
+ def test_quantized_projection_preserves_constant_rank_one_gradient(self, n_bit):
+ grad = torch.ones(8, 4)
+ proj = GaLoreProjector(rank = 1, quant = True, n_bit = n_bit)
+ restored = proj.project_back(proj.project(grad, step = 0))
+ torch.testing.assert_close(restored, grad)
+
def test_adaptive_scheduling(self):
"""update_proj_gap increases when cosine similarity exceeds threshold."""
proj = GaLoreProjector(
@@ -155,6 +162,27 @@ class TestGaLoreProjector:
class TestQuantizationUtils:
"""Tests for _quantize, _dequantize, _quantize_stochastic."""
+ @pytest.mark.parametrize("quantize", [_quantize, _quantize_stochastic])
+ @pytest.mark.parametrize("n_bit", [4, 8])
+ @pytest.mark.parametrize("group_size", [-1, 2])
+ def test_roundtrip_single_sign_groups(self, quantize, n_bit, group_size):
+ weights = torch.tensor(
+ [
+ [1.0, 1.5, 2.0, 2.5],
+ [-1.0, -1.5, -2.0, -2.5],
+ [0.5, 0.5, 0.5, 0.5],
+ [-0.5, -0.5, -0.5, -0.5],
+ [0.0, 0.0, 0.0, 0.0],
+ [-1.0, -0.5, 0.5, 1.0],
+ ]
+ )
+ quantized = quantize(weights, q_group_size = group_size, n_bit = n_bit)
+ restored = _dequantize(*quantized)
+ scales = quantized[1]
+ error = (restored - weights).abs().reshape(scales.shape[0], -1)
+ # Stochastic rounding may move by one quantization step, but must not clip a group.
+ assert torch.all(error <= scales + 1e-6)
+
def test_quantize_dequantize_roundtrip(self):
"""Quantize → dequantize has bounded error."""
w = torch.randn(32, 64)
diff --git a/unsloth/optimizers/q_galore_projector.py b/unsloth/optimizers/q_galore_projector.py
index e17295a9e46..088461406e4 100644
--- a/unsloth/optimizers/q_galore_projector.py
+++ b/unsloth/optimizers/q_galore_projector.py
@@ -275,8 +275,8 @@ def _quantize(
w = w.reshape(-1, q_group_size)
assert w.dim() == 2
- max_val = w.amax(dim = 1, keepdim = True)
- min_val = w.amin(dim = 1, keepdim = True)
+ max_val = w.amax(dim = 1, keepdim = True).clamp(min = 0)
+ min_val = w.amin(dim = 1, keepdim = True).clamp(max = 0)
max_int = 2**n_bit - 1
min_int = 0
scales = (max_val - min_val).clamp(min = 1e-5) / max_int
@@ -324,8 +324,8 @@ def _quantize_stochastic(
w = w.reshape(-1, q_group_size)
assert w.dim() == 2
- max_val = w.amax(dim = 1, keepdim = True)
- min_val = w.amin(dim = 1, keepdim = True)
+ max_val = w.amax(dim = 1, keepdim = True).clamp(min = 0)
+ min_val = w.amin(dim = 1, keepdim = True).clamp(max = 0)
max_int = 2**n_bit - 1
min_int = 0
scales = (max_val - min_val).clamp(min = 1e-5) / max_int