Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Let AWQ checkpoints train with LoRA (#13584)
Daniel Han8e68a99263f6
diff --git a/tests/test_gptq_packed_lora.py b/tests/test_gptq_packed_lora.pyindex 362441e13ff..b531f8f07b6 100644--- a/tests/test_gptq_packed_lora.py+++ b/tests/test_gptq_packed_lora.py@@ -96,13 +96,35 @@ def test_gptq_backend_left_alone(monkeypatch):gptq_trainable_quantization_config(_config({**_GPTQ8, "backend": "gptq_torch"}), None)is None)- assert (- gptq_trainable_quantization_config(_config({"quant_method": "awq", "bits": 4}), None)- is None- )assert (gptq_trainable_quantization_config(_config({"quant_method": "bitsandbytes"}), None) is None)assert gptq_trainable_quantization_config(_config(None), None) is Nonemonkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: False)assert gptq_trainable_quantization_config(_config(dict(_GPTQ8)), None) is None+++_AWQ4 = {"quant_method": "awq", "bits": 4, "group_size": 64, "zero_point": True, "version": "gemm"}+++def test_awq_checkpoint_gets_trainable_backend(monkeypatch):+ from transformers.quantizers.auto import AutoHfQuantizer++ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: True)+ cfg = gptq_trainable_quantization_config(_config(dict(_AWQ4)), None)+ assert type(cfg).__name__ == "AwqConfig" and cfg.backend == "auto_trainable"+ merged = AutoHfQuantizer.merge_quantization_configs(dict(_AWQ4), cfg)+ assert (merged.bits, merged.group_size, merged.zero_point) == (4, 64, True)+ assert merged.format == "gemm" and merged.backend == "auto_trainable"+ # Configs saved by transformers 4.x (Qwen/Qwen3-8B-AWQ) carry the legacy default backend.+ for legacy in ("autoawq", "llm-awq"):+ legacy_cfg = gptq_trainable_quantization_config(_config({**_AWQ4, "backend": legacy}), None)+ assert legacy_cfg is not None and legacy_cfg.backend == "auto_trainable"+++def test_awq_backend_left_alone(monkeypatch):+ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: True)+ assert gptq_trainable_quantization_config(_config(dict(_AWQ4)), object()) is None+ assert gptq_trainable_quantization_config(_config({**_AWQ4, "backend": "gemm"}), None) is None+ monkeypatch.setattr(transformers.utils, "is_gptqmodel_available", lambda: False)+ assert gptq_trainable_quantization_config(_config(dict(_AWQ4)), None) is Nonediff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex e69da4a678a..e89486c203a 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -3364,7 +3364,7 @@ def sync_load_when_quantizing(quantization_config, model_config):def gptq_trainable_quantization_config(model_config, user_quantization_config):- """GPTQConfig asking gptqmodel for a trainable kernel on a GPTQ checkpoint, else None.+ """GPTQConfig / AwqConfig asking gptqmodel for a trainable kernel on a GPTQ or AWQ checkpoint, else None.gptqmodel's default kernels (Marlin / ExLlama) raise NotImplementedError on model.train(); only`backend` is a loading attribute, so the checkpoint's own bits / group_size still apply.@@ -3374,17 +3374,31 @@ def gptq_trainable_quantization_config(model_config, user_quantization_config):qc = getattr(model_config, "quantization_config", None)if qc is not None and not isinstance(qc, dict):qc = qc.to_dict()- if not qc or str(qc.get("quant_method", "")).lower() != "gptq":+ quant_method = str((qc or {}).get("quant_method", "")).lower()+ if quant_method not in ("gptq", "awq"):return None- if qc.get("backend") not in (None, "auto"):+ # transformers 4.x wrote "autoawq" / "llm-awq" into AWQ configs (Qwen3-8B-AWQ); AwqConfig reads both as auto.+ default_backends = (+ (None, "auto", "autoawq", "llm-awq") if quant_method == "awq" else (None, "auto")+ )+ if qc.get("backend") not in default_backends:return Nonetry:- from transformers import GPTQConfig+ from transformers import AwqConfig, GPTQConfigfrom transformers.utils import is_gptqmodel_availableexcept ImportError:return Noneif not is_gptqmodel_available() or "backend" not in inspect.signature(GPTQConfig).parameters:return None+ if quant_method == "awq":+ # transformers < 5 AwqConfig has no auto_trainable backend (autoawq era).+ try:+ from transformers.utils.quantization_config import AwqBackend+ except ImportError:+ return None+ if not hasattr(AwqBackend, "AUTO_TRAINABLE"):+ return None+ return AwqConfig(bits = qc.get("bits", 4), backend = "auto_trainable")return GPTQConfig(bits = qc["bits"], backend = "auto_trainable")