Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix Q-GaLore quantization of single-sign groups (#10841)
Vineeth Sai Varikuntla3bdb5a4214dc
diff --git a/.github/workflows/q-galore-tests.yml b/.github/workflows/q-galore-tests.ymlnew file mode 100644index 00000000000..567b8051180--- /dev/null+++ b/.github/workflows/q-galore-tests.yml@@ -0,0 +1,52 @@+name: Q-GaLore CPU tests++on:+ push:+ branches: [main]+ paths:+ - "unsloth/optimizers/**"+ - "tests/utils/test_q_galore.py"+ - ".github/workflows/q-galore-tests.yml"+ pull_request:+ paths:+ - "unsloth/optimizers/**"+ - "tests/utils/test_q_galore.py"+ - ".github/workflows/q-galore-tests.yml"+ workflow_dispatch:++concurrency:+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}-${{ github.ref == 'refs/heads/main' && github.sha || '' }}+ cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}++permissions:+ contents: read++jobs:+ q-galore:+ runs-on: ubuntu-latest+ timeout-minutes: 10+ strategy:+ fail-fast: false+ matrix:+ # bitsandbytes is not API-stable across the range pyproject allows, so pinning one+ # version tests one point of it. 0.45.5 is the floor, 0.49.2 the last with+ # percentile_clipping / block_wise, 0.50.2 the first without them and with CPU+ # optimizer kernels. This PR's tests are quantizer-only and touch no bitsandbytes+ # entry point; the constructor coverage that uses these pins arrives with #10874.+ bitsandbytes: ["0.45.5", "0.49.2", "0.50.2"]+ name: q-galore (bitsandbytes ${{ matrix.bitsandbytes }})+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.12"+ - name: Install CPU optimizer test dependencies+ run: |+ python -m pip install --index-url https://download.pytorch.org/whl/cpu 'torch==2.10.0'+ python -m pip install 'pytest==9.0.3' 'bitsandbytes==${{ matrix.bitsandbytes }}'+ - name: Run Q-GaLore regressions+ # No --confcutdir: this file loads the optimizers through importlib specs, so the+ # GPU-spoof conftest neither helps nor hinders it. Verified both ways.+ run: python -m pytest tests/utils/test_q_galore.py -qdiff --git a/tests/utils/test_q_galore.py b/tests/utils/test_q_galore.pyindex 5190d01b124..ccf4910bf2c 100644--- a/tests/utils/test_q_galore.py+++ b/tests/utils/test_q_galore.py@@ -113,6 +113,13 @@ class TestGaLoreProjector:# INT4 values should be in range [0, 15]assert proj.ortho_matrix.max() <= 15+ @pytest.mark.parametrize("n_bit", [4, 8])+ def test_quantized_projection_preserves_constant_rank_one_gradient(self, n_bit):+ grad = torch.ones(8, 4)+ proj = GaLoreProjector(rank = 1, quant = True, n_bit = n_bit)+ restored = proj.project_back(proj.project(grad, step = 0))+ torch.testing.assert_close(restored, grad)+def test_adaptive_scheduling(self):"""update_proj_gap increases when cosine similarity exceeds threshold."""proj = GaLoreProjector(@@ -155,6 +162,27 @@ class TestGaLoreProjector:class TestQuantizationUtils:"""Tests for _quantize, _dequantize, _quantize_stochastic."""+ @pytest.mark.parametrize("quantize", [_quantize, _quantize_stochastic])+ @pytest.mark.parametrize("n_bit", [4, 8])+ @pytest.mark.parametrize("group_size", [-1, 2])+ def test_roundtrip_single_sign_groups(self, quantize, n_bit, group_size):+ weights = torch.tensor(+ [+ [1.0, 1.5, 2.0, 2.5],+ [-1.0, -1.5, -2.0, -2.5],+ [0.5, 0.5, 0.5, 0.5],+ [-0.5, -0.5, -0.5, -0.5],+ [0.0, 0.0, 0.0, 0.0],+ [-1.0, -0.5, 0.5, 1.0],+ ]+ )+ quantized = quantize(weights, q_group_size = group_size, n_bit = n_bit)+ restored = _dequantize(*quantized)+ scales = quantized[1]+ error = (restored - weights).abs().reshape(scales.shape[0], -1)+ # Stochastic rounding may move by one quantization step, but must not clip a group.+ assert torch.all(error <= scales + 1e-6)+def test_quantize_dequantize_roundtrip(self):"""Quantize → dequantize has bounded error."""w = torch.randn(32, 64)diff --git a/unsloth/optimizers/q_galore_projector.py b/unsloth/optimizers/q_galore_projector.pyindex e17295a9e46..088461406e4 100644--- a/unsloth/optimizers/q_galore_projector.py+++ b/unsloth/optimizers/q_galore_projector.py@@ -275,8 +275,8 @@ def _quantize(w = w.reshape(-1, q_group_size)assert w.dim() == 2- max_val = w.amax(dim = 1, keepdim = True)- min_val = w.amin(dim = 1, keepdim = True)+ max_val = w.amax(dim = 1, keepdim = True).clamp(min = 0)+ min_val = w.amin(dim = 1, keepdim = True).clamp(max = 0)max_int = 2**n_bit - 1min_int = 0scales = (max_val - min_val).clamp(min = 1e-5) / max_int@@ -324,8 +324,8 @@ def _quantize_stochastic(w = w.reshape(-1, q_group_size)assert w.dim() == 2- max_val = w.amax(dim = 1, keepdim = True)- min_val = w.amin(dim = 1, keepdim = True)+ max_val = w.amax(dim = 1, keepdim = True).clamp(min = 0)+ min_val = w.amin(dim = 1, keepdim = True).clamp(max = 0)max_int = 2**n_bit - 1min_int = 0scales = (max_val - min_val).clamp(min = 1e-5) / max_int