Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Let GPTQ checkpoints train and generate with LoRA (#13068)

Daniel Han8605d549bc68
diff --git a/tests/test_gptq_packed_lora.py b/tests/test_gptq_packed_lora.py
new file mode 100644
index 00000000000..362441e13ff
--- /dev/null
+++ b/tests/test_gptq_packed_lora.py
@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""GPTQ bases (packed qweight, no dense .weight) train and decode through their own forward (#1573)."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+import torch
+import unsloth # noqa: F401
+import transformers.utils
+
+from unsloth.kernels.utils import fast_linear_forward
+from unsloth.models.llama import _base_weight_dtype, _has_packed_base
+from unsloth.models.loader_utils import gptq_trainable_quantization_config
+
+
+class _PackedLinear(torch.nn.Module):
+ """GPTQ-shaped linear: int32 qweight buffer, no dense weight, optional metadata-only `weight`."""
+
+ def __init__(
+ self,
+ in_features = 16,
+ out_features = 8,
+ weight_metadata = False,
+ ):
+ super().__init__()
+ self.in_features, self.out_features = in_features, out_features
+ self.register_buffer(
+ "qweight", torch.zeros(in_features // 8, out_features, dtype = torch.int32)
+ )
+ self.register_buffer("dense", torch.randn(out_features, in_features))
+ self.bias = None
+ if weight_metadata:
+ self.weight = types.SimpleNamespace(
+ dtype = torch.float16, shape = (out_features, in_features)
+ )
+
+ def forward(self, X):
+ return X @ self.dense.t()
+
+
[email protected]("weight_metadata", [False, True])
+def test_packed_base_detected(weight_metadata):
+ proj = _PackedLinear(weight_metadata = weight_metadata)
+ assert _base_weight_dtype(proj) is None
+ assert _has_packed_base(proj)
+ assert _has_packed_base(torch.nn.Linear(4, 4), proj)
+
+
+def test_dense_and_mxfp4_bases_not_packed():
+ assert not _has_packed_base(torch.nn.Linear(4, 4, dtype = torch.bfloat16))
+
+ class _Mxfp4(torch.nn.Module):
+ _unsloth_mxfp4_packed_linear = True
+
+ assert not _has_packed_base(_Mxfp4())
+
+
[email protected]("weight_metadata", [False, True])
+def test_fast_linear_forward_runs_packed_module(weight_metadata):
+ proj = _PackedLinear(weight_metadata = weight_metadata)
+ X = torch.randn(1, 1, 16)
+ torch.testing.assert_close(fast_linear_forward(proj, X), proj(X))
+ out = torch.empty(1, 1, 8)
+ assert fast_linear_forward(proj, X, out = out) is out
+ torch.testing.assert_close(out, proj(X))
+
+
+def test_fast_linear_forward_dense_unchanged():
+ proj = torch.nn.Linear(16, 8, bias = False)
+ X = torch.randn(1, 1, 16)
+ torch.testing.assert_close(fast_linear_forward(proj, X), proj(X))
+
+
+def _config(quantization_config):
+ return types.SimpleNamespace(quantization_config = quantization_config)
+
+
+_GPTQ8 = {"quant_method": "gptq", "bits": 8, "group_size": 128, "desc_act": False, "sym": True}
+
+
+def test_gptq_checkpoint_gets_trainable_backend(monkeypatch):
+ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: True)
+ cfg = gptq_trainable_quantization_config(_config(dict(_GPTQ8)), None)
+ assert cfg.backend == "auto_trainable" and cfg.bits == 8
+ assert cfg.get_loading_attributes()["backend"] == "auto_trainable"
+
+
+def test_gptq_backend_left_alone(monkeypatch):
+ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: True)
+ assert gptq_trainable_quantization_config(_config(dict(_GPTQ8)), object()) is None
+ assert (
+ gptq_trainable_quantization_config(_config({**_GPTQ8, "backend": "gptq_torch"}), None)
+ is None
+ )
+ assert (
+ gptq_trainable_quantization_config(_config({"quant_method": "awq", "bits": 4}), None)
+ is None
+ )
+ assert (
+ gptq_trainable_quantization_config(_config({"quant_method": "bitsandbytes"}), None) is None
+ )
+ assert gptq_trainable_quantization_config(_config(None), None) is None
+ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: False)
+ assert gptq_trainable_quantization_config(_config(dict(_GPTQ8)), None) is None
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 78c1e5ee8b3..dbf8fb29d80 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -397,6 +397,15 @@ def _packed_base(base_layer):
return base_layer.weight_packed, base_layer.mxfp4_quant_state()
+def _has_packed_qweight(proj):
+ # GPTQ / AWQ keep a packed qweight and no dense .weight tensor: only their own forward can run them.
+ base_layer = proj._modules.get("base_layer", proj)
+ return (
+ not isinstance(getattr(base_layer, "weight", None), torch_Tensor)
+ and _packed_base(base_layer) is None
+ )
+
+
def _is_packed_state(quant_state):
return getattr(type(quant_state), "_unsloth_packed_weight_state", False)
@@ -1406,6 +1415,7 @@ def fast_linear_forward(
_has_multiple_active_adapters(proj)
or _has_active_dora_adapter(proj)
or _has_active_lora_bias(proj)
+ or _has_packed_qweight(proj)
):
result = proj(X)
return result if out is None else out.copy_(result)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 21ec7b524d2..dac5d1bc72e 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2532,11 +2532,23 @@ def restore_transformers_family(model_types):
def _base_weight_dtype(proj):
- weight = getattr(proj, "base_layer", proj).weight
+ # None: no dense .weight (GPTQ / AWQ qweight), which the fused LoRA kernels cannot read.
+ weight = getattr(getattr(proj, "base_layer", proj), "weight", None)
+ if not isinstance(weight, torch.Tensor):
+ return None
quant_state = getattr(weight, "quant_state", None)
return quant_state.dtype if quant_state is not None else weight.dtype
+def _has_packed_base(*projs):
+ # GPTQ / AWQ bases hold packed qweight and no dense .weight, so PEFT's own forward runs instead of the fused kernels.
+ return any(
+ _base_weight_dtype(p) is None
+ and not getattr(type(getattr(p, "base_layer", p)), "_unsloth_mxfp4_packed_linear", False)
+ for p in projs
+ )
+
+
_FUSED_LORA_MLPS = (apply_lora_mlp_swiglu, apply_lora_mlp_geglu_exact, apply_lora_mlp_geglu_approx)
@@ -2907,6 +2919,7 @@ class FastLlamaModel:
from .loader_utils import (
check_and_disable_bitsandbytes_loading,
+ gptq_trainable_quantization_config,
quantization_config_selects_bnb_4bit,
sync_unsloth_model_name_bnb_flags,
)
@@ -2944,6 +2957,13 @@ class FastLlamaModel:
# checkpoint left unarmed (vLLM reads it itself) must not reach vLLM as a bitsandbytes load.
if not (_explicit_bnb_4bit and _checked_4bit):
load_in_4bit, load_in_8bit = _checked_4bit, _checked_8bit
+ # vLLM reads the checkpoint itself and picks its own GPTQ kernel.
+ if not _vllm_will_load_weights(fast_inference, num_labels):
+ _gptq_config = gptq_trainable_quantization_config(
+ model_config, _user_quantization_config
+ )
+ if _gptq_config is not None:
+ kwargs["quantization_config"] = _gptq_config
if offload_layers and load_in_8bit:
offload_layers = refuse_block_swap_load(
offload_layers, "supports 16-bit and 4-bit loads, not load_in_8bit."
@@ -4373,6 +4393,7 @@ class FastLlamaModel:
and (len(getattr(up_proj, "lora_magnitude_vector", []) or []) == 0)
and (len(getattr(down_proj, "lora_magnitude_vector", []) or []) == 0)
and not has_mxfp4_base(gate_proj, up_proj, down_proj)
+ and not _has_packed_base(gate_proj, up_proj, down_proj)
and not _has_active_lora_bias(gate_proj)
and not _has_active_lora_bias(up_proj)
and not _has_active_lora_bias(down_proj)
@@ -4406,6 +4427,7 @@ class FastLlamaModel:
and (len(getattr(k_proj, "lora_magnitude_vector", []) or []) == 0)
and (len(getattr(v_proj, "lora_magnitude_vector", []) or []) == 0)
and not has_mxfp4_base(q_proj, k_proj, v_proj)
+ and not _has_packed_base(q_proj, k_proj, v_proj)
and not _has_active_lora_bias(q_proj)
and not _has_active_lora_bias(k_proj)
and not _has_active_lora_bias(v_proj)
@@ -4427,6 +4449,7 @@ class FastLlamaModel:
and (getattr(o_proj, "base_layer", o_proj).bias is None)
and (len(getattr(o_proj, "lora_magnitude_vector", []) or []) == 0)
and not has_mxfp4_base(o_proj)
+ and not _has_packed_base(o_proj)
and not _has_active_lora_bias(o_proj)
):
layer.self_attn.apply_o = apply_lora_o
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index d9ba756b4ef..e94e612847a 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -3172,6 +3172,31 @@ def sync_load_when_quantizing(quantization_config, model_config):
os.environ.pop(_ASYNC_LOAD_ENV, None)
+def gptq_trainable_quantization_config(model_config, user_quantization_config):
+ """GPTQConfig asking gptqmodel for a trainable kernel on a GPTQ checkpoint, else None.
+
+ gptqmodel's default kernels (Marlin / ExLlama) raise NotImplementedError on model.train(); only
+ `backend` is a loading attribute, so the checkpoint's own bits / group_size still apply.
+ """
+ if user_quantization_config is not None:
+ return None
+ qc = getattr(model_config, "quantization_config", None)
+ if qc is not None and not isinstance(qc, dict):
+ qc = qc.to_dict()
+ if not qc or str(qc.get("quant_method", "")).lower() != "gptq":
+ return None
+ if qc.get("backend") not in (None, "auto"):
+ return None
+ try:
+ from transformers import GPTQConfig
+ from transformers.utils import is_gptqmodel_available
+ except ImportError:
+ return None
+ if not is_gptqmodel_available() or "backend" not in inspect.signature(GPTQConfig).parameters:
+ return None
+ return GPTQConfig(bits = qc["bits"], backend = "auto_trainable")
+
+
def warn_if_bitsandbytes_quantized_nothing(
model,
quantization_config,
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 612aa602a7b..308642bcee5 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -3132,6 +3132,7 @@ class FastBaseModel:
from .loader_utils import (
check_and_disable_bitsandbytes_loading,
+ gptq_trainable_quantization_config,
quantization_config_selects_bnb_4bit,
sync_unsloth_model_name_bnb_flags,
)
@@ -3487,6 +3488,12 @@ class FastBaseModel:
if user_quantization_config is None:
kwargs["quantization_config"] = quantization_config
+ # Replaces the checkpoint's own GPTQ config built above; vLLM reads the checkpoint itself and picks its own kernel.
+ if not (fast_inference and is_vLLM_available()):
+ _gptq_config = gptq_trainable_quantization_config(auto_config, user_quantization_config)
+ if _gptq_config is not None:
+ kwargs["quantization_config"] = _gptq_config
+
# torch_dtype is resolved above, where the device-map planner also needs it.
kwargs = add_dtype_kwargs(torch_dtype, kwargs)