Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Let GPTQ checkpoints train and generate with LoRA (#13068)
Daniel Han8605d549bc68
diff --git a/tests/test_gptq_packed_lora.py b/tests/test_gptq_packed_lora.pynew file mode 100644index 00000000000..362441e13ff--- /dev/null+++ b/tests/test_gptq_packed_lora.py@@ -0,0 +1,108 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""GPTQ bases (packed qweight, no dense .weight) train and decode through their own forward (#1573)."""++from __future__ import annotations++import types++import pytest+import torch+import unsloth # noqa: F401+import transformers.utils++from unsloth.kernels.utils import fast_linear_forward+from unsloth.models.llama import _base_weight_dtype, _has_packed_base+from unsloth.models.loader_utils import gptq_trainable_quantization_config+++class _PackedLinear(torch.nn.Module):+ """GPTQ-shaped linear: int32 qweight buffer, no dense weight, optional metadata-only `weight`."""++ def __init__(+ self,+ in_features = 16,+ out_features = 8,+ weight_metadata = False,+ ):+ super().__init__()+ self.in_features, self.out_features = in_features, out_features+ self.register_buffer(+ "qweight", torch.zeros(in_features // 8, out_features, dtype = torch.int32)+ )+ self.register_buffer("dense", torch.randn(out_features, in_features))+ self.bias = None+ if weight_metadata:+ self.weight = types.SimpleNamespace(+ dtype = torch.float16, shape = (out_features, in_features)+ )++ def forward(self, X):+ return X @ self.dense.t()++[email protected]("weight_metadata", [False, True])+def test_packed_base_detected(weight_metadata):+ proj = _PackedLinear(weight_metadata = weight_metadata)+ assert _base_weight_dtype(proj) is None+ assert _has_packed_base(proj)+ assert _has_packed_base(torch.nn.Linear(4, 4), proj)+++def test_dense_and_mxfp4_bases_not_packed():+ assert not _has_packed_base(torch.nn.Linear(4, 4, dtype = torch.bfloat16))++ class _Mxfp4(torch.nn.Module):+ _unsloth_mxfp4_packed_linear = True++ assert not _has_packed_base(_Mxfp4())++[email protected]("weight_metadata", [False, True])+def test_fast_linear_forward_runs_packed_module(weight_metadata):+ proj = _PackedLinear(weight_metadata = weight_metadata)+ X = torch.randn(1, 1, 16)+ torch.testing.assert_close(fast_linear_forward(proj, X), proj(X))+ out = torch.empty(1, 1, 8)+ assert fast_linear_forward(proj, X, out = out) is out+ torch.testing.assert_close(out, proj(X))+++def test_fast_linear_forward_dense_unchanged():+ proj = torch.nn.Linear(16, 8, bias = False)+ X = torch.randn(1, 1, 16)+ torch.testing.assert_close(fast_linear_forward(proj, X), proj(X))+++def _config(quantization_config):+ return types.SimpleNamespace(quantization_config = quantization_config)+++_GPTQ8 = {"quant_method": "gptq", "bits": 8, "group_size": 128, "desc_act": False, "sym": True}+++def test_gptq_checkpoint_gets_trainable_backend(monkeypatch):+ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: True)+ cfg = gptq_trainable_quantization_config(_config(dict(_GPTQ8)), None)+ assert cfg.backend == "auto_trainable" and cfg.bits == 8+ assert cfg.get_loading_attributes()["backend"] == "auto_trainable"+++def test_gptq_backend_left_alone(monkeypatch):+ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: True)+ assert gptq_trainable_quantization_config(_config(dict(_GPTQ8)), object()) is None+ assert (+ gptq_trainable_quantization_config(_config({**_GPTQ8, "backend": "gptq_torch"}), None)+ is None+ )+ assert (+ gptq_trainable_quantization_config(_config({"quant_method": "awq", "bits": 4}), None)+ is None+ )+ assert (+ gptq_trainable_quantization_config(_config({"quant_method": "bitsandbytes"}), None) is None+ )+ assert gptq_trainable_quantization_config(_config(None), None) is None+ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: False)+ assert gptq_trainable_quantization_config(_config(dict(_GPTQ8)), None) is Nonediff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 78c1e5ee8b3..dbf8fb29d80 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -397,6 +397,15 @@ def _packed_base(base_layer):return base_layer.weight_packed, base_layer.mxfp4_quant_state()+def _has_packed_qweight(proj):+ # GPTQ / AWQ keep a packed qweight and no dense .weight tensor: only their own forward can run them.+ base_layer = proj._modules.get("base_layer", proj)+ return (+ not isinstance(getattr(base_layer, "weight", None), torch_Tensor)+ and _packed_base(base_layer) is None+ )++def _is_packed_state(quant_state):return getattr(type(quant_state), "_unsloth_packed_weight_state", False)@@ -1406,6 +1415,7 @@ def fast_linear_forward(_has_multiple_active_adapters(proj)or _has_active_dora_adapter(proj)or _has_active_lora_bias(proj)+ or _has_packed_qweight(proj)):result = proj(X)return result if out is None else out.copy_(result)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 21ec7b524d2..dac5d1bc72e 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2532,11 +2532,23 @@ def restore_transformers_family(model_types):def _base_weight_dtype(proj):- weight = getattr(proj, "base_layer", proj).weight+ # None: no dense .weight (GPTQ / AWQ qweight), which the fused LoRA kernels cannot read.+ weight = getattr(getattr(proj, "base_layer", proj), "weight", None)+ if not isinstance(weight, torch.Tensor):+ return Nonequant_state = getattr(weight, "quant_state", None)return quant_state.dtype if quant_state is not None else weight.dtype+def _has_packed_base(*projs):+ # GPTQ / AWQ bases hold packed qweight and no dense .weight, so PEFT's own forward runs instead of the fused kernels.+ return any(+ _base_weight_dtype(p) is None+ and not getattr(type(getattr(p, "base_layer", p)), "_unsloth_mxfp4_packed_linear", False)+ for p in projs+ )++_FUSED_LORA_MLPS = (apply_lora_mlp_swiglu, apply_lora_mlp_geglu_exact, apply_lora_mlp_geglu_approx)@@ -2907,6 +2919,7 @@ class FastLlamaModel:from .loader_utils import (check_and_disable_bitsandbytes_loading,+ gptq_trainable_quantization_config,quantization_config_selects_bnb_4bit,sync_unsloth_model_name_bnb_flags,)@@ -2944,6 +2957,13 @@ class FastLlamaModel:# checkpoint left unarmed (vLLM reads it itself) must not reach vLLM as a bitsandbytes load.if not (_explicit_bnb_4bit and _checked_4bit):load_in_4bit, load_in_8bit = _checked_4bit, _checked_8bit+ # vLLM reads the checkpoint itself and picks its own GPTQ kernel.+ if not _vllm_will_load_weights(fast_inference, num_labels):+ _gptq_config = gptq_trainable_quantization_config(+ model_config, _user_quantization_config+ )+ if _gptq_config is not None:+ kwargs["quantization_config"] = _gptq_configif offload_layers and load_in_8bit:offload_layers = refuse_block_swap_load(offload_layers, "supports 16-bit and 4-bit loads, not load_in_8bit."@@ -4373,6 +4393,7 @@ class FastLlamaModel:and (len(getattr(up_proj, "lora_magnitude_vector", []) or []) == 0)and (len(getattr(down_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(gate_proj, up_proj, down_proj)+ and not _has_packed_base(gate_proj, up_proj, down_proj)and not _has_active_lora_bias(gate_proj)and not _has_active_lora_bias(up_proj)and not _has_active_lora_bias(down_proj)@@ -4406,6 +4427,7 @@ class FastLlamaModel:and (len(getattr(k_proj, "lora_magnitude_vector", []) or []) == 0)and (len(getattr(v_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(q_proj, k_proj, v_proj)+ and not _has_packed_base(q_proj, k_proj, v_proj)and not _has_active_lora_bias(q_proj)and not _has_active_lora_bias(k_proj)and not _has_active_lora_bias(v_proj)@@ -4427,6 +4449,7 @@ class FastLlamaModel:and (getattr(o_proj, "base_layer", o_proj).bias is None)and (len(getattr(o_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(o_proj)+ and not _has_packed_base(o_proj)and not _has_active_lora_bias(o_proj)):layer.self_attn.apply_o = apply_lora_odiff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex d9ba756b4ef..e94e612847a 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -3172,6 +3172,31 @@ def sync_load_when_quantizing(quantization_config, model_config):os.environ.pop(_ASYNC_LOAD_ENV, None)+def gptq_trainable_quantization_config(model_config, user_quantization_config):+ """GPTQConfig asking gptqmodel for a trainable kernel on a GPTQ checkpoint, else None.++ gptqmodel's default kernels (Marlin / ExLlama) raise NotImplementedError on model.train(); only+ `backend` is a loading attribute, so the checkpoint's own bits / group_size still apply.+ """+ if user_quantization_config is not None:+ return None+ qc = getattr(model_config, "quantization_config", None)+ if qc is not None and not isinstance(qc, dict):+ qc = qc.to_dict()+ if not qc or str(qc.get("quant_method", "")).lower() != "gptq":+ return None+ if qc.get("backend") not in (None, "auto"):+ return None+ try:+ from transformers import GPTQConfig+ from transformers.utils import is_gptqmodel_available+ except ImportError:+ return None+ if not is_gptqmodel_available() or "backend" not in inspect.signature(GPTQConfig).parameters:+ return None+ return GPTQConfig(bits = qc["bits"], backend = "auto_trainable")++def warn_if_bitsandbytes_quantized_nothing(model,quantization_config,diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 612aa602a7b..308642bcee5 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -3132,6 +3132,7 @@ class FastBaseModel:from .loader_utils import (check_and_disable_bitsandbytes_loading,+ gptq_trainable_quantization_config,quantization_config_selects_bnb_4bit,sync_unsloth_model_name_bnb_flags,)@@ -3487,6 +3488,12 @@ class FastBaseModel:if user_quantization_config is None:kwargs["quantization_config"] = quantization_config+ # Replaces the checkpoint's own GPTQ config built above; vLLM reads the checkpoint itself and picks its own kernel.+ if not (fast_inference and is_vLLM_available()):+ _gptq_config = gptq_trainable_quantization_config(auto_config, user_quantization_config)+ if _gptq_config is not None:+ kwargs["quantization_config"] = _gptq_config+# torch_dtype is resolved above, where the device-map planner also needs it.kwargs = add_dtype_kwargs(torch_dtype, kwargs)