Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix int8-int4 QAT LoRA at low ranks and on tied-embedding models (#13552)
Daniel Han0943697bf88c
diff --git a/tests/saving/test_qat_torchao_save.py b/tests/saving/test_qat_torchao_save.pyindex a47b94c354b..c33c3dcd227 100644--- a/tests/saving/test_qat_torchao_save.py+++ b/tests/saving/test_qat_torchao_save.py@@ -188,3 +188,81 @@ def test_peft_qat_save_reloads_with_merged_names(monkeypatch, tmp_path):for module in peft_model.modules():assert not hasattr(module, "weight_fake_quantizer")assert not _is_quantized(getattr(module, "weight", None))+++class _TiedModel(_Model):+ def __init__(self):+ super().__init__()+ self.config = type("cfg", (), {"model_type": "llama", "tie_word_embeddings": True})()+ self.lm_head = torch.nn.Linear(256, 32, bias = False)+ self.lm_head.weight = self.embed_tokens.weight++ def get_output_embeddings(self):+ return self.lm_head+++def test_int8_int4_lora_rank_below_group_size():+ # lora_B at rank 16 has in_features 16, which PerGroup(32) rejects.+ from peft import LoraConfig, get_peft_model++ from unsloth.models._utils import _peft_untied_lm_head++ peft_model = get_peft_model(+ _TiedModel(), LoraConfig(r = 16, target_modules = ["q_proj", "down_proj"])+ )+ _prepare_model_for_qat(peft_model, "int8-int4")+ block = peft_model.base_model.model.layers[0]+ assert hasattr(block.q_proj.base_layer, "weight_fake_quantizer")+ assert hasattr(block.q_proj.lora_A["default"], "weight_fake_quantizer")+ assert not hasattr(block.q_proj.lora_B["default"], "weight_fake_quantizer")+ lm_head = peft_model.base_model.model.lm_head+ assert _peft_untied_lm_head(peft_model) == ("lm_head.weight", lm_head.weight)+ assert lm_head.weight.data_ptr() != peft_model.base_model.model.embed_tokens.weight.data_ptr()+ assert not lm_head.weight.requires_grad++[email protected]("layout", ["single", "consolidated", "sharded"])+def test_untie_merged_checkpoint(tmp_path, layout):+ import json++ from safetensors import safe_open+ from safetensors.torch import save_file++ tensors = {+ "thinker.model.embed_tokens.weight": torch.randn(32, 8),+ "thinker.model.norm.weight": torch.ones(8),+ }+ if layout == "sharded":+ # A local checkpoint can already use the name the head would get.+ save_file(tensors, tmp_path / "model-lm_head.safetensors")+ weight_map = {k: "model-lm_head.safetensors" for k in tensors}+ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps({"metadata": {}, "weight_map": weight_map}), encoding = "utf-8"+ )+ else:+ name = "model.safetensors" if layout == "single" else "consolidated.safetensors"+ save_file(tensors, tmp_path / name)+ # A reused directory can still hold an adapter save.+ save_file({"x": torch.zeros(1)}, tmp_path / "adapter_model.safetensors")+ config = {+ "thinker_config": {"text_config": {"tie_word_embeddings": True}},+ "talker_config": {"tie_word_embeddings": True},+ }+ (tmp_path / "config.json").write_text(json.dumps(config), encoding = "utf-8")+ # The model's own head, which can differ from the checkpoint's (trained) embedding.+ lm_head = torch.randn(32, 8)++ save_mod._untie_merged_checkpoint(str(tmp_path), "thinker.lm_head.weight", lm_head)++ config = json.loads((tmp_path / "config.json").read_text(encoding = "utf-8"))+ assert config["thinker_config"]["text_config"]["tie_word_embeddings"] is False+ assert config["talker_config"]["tie_word_embeddings"] is True+ if layout == "sharded":+ index = json.loads((tmp_path / "model.safetensors.index.json").read_text(encoding = "utf-8"))+ shard = tmp_path / index["weight_map"]["thinker.lm_head.weight"]+ with safe_open(str(tmp_path / "model-lm_head.safetensors"), "pt") as h:+ assert set(h.keys()) == set(tensors)+ else:+ shard = tmp_path / name+ with safe_open(str(shard), "pt") as h:+ assert torch.equal(h.get_tensor("thinker.lm_head.weight"), lm_head)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex e1e776c372d..f31cb706dce 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -7257,7 +7257,8 @@ def _untie_input_output_embeddings(model: torch.nn.Module) -> None:if out_proj.weight.data_ptr() == in_emb.weight.data_ptr():with torch.no_grad():W = in_emb.weight.detach().clone()- out_proj.weight = torch.nn.Parameter(W) # new storage, keeps dtype/device+ # Same trainability as the embedding, so a LoRA model keeps the head frozen.+ out_proj.weight = torch.nn.Parameter(W, requires_grad = in_emb.weight.requires_grad)def _no_tie(self):return@@ -7303,23 +7304,39 @@ def _strip_torchao_fake_quantization(model):)+def _peft_merged_module_name(fqn):+ """A PEFT module name as the merged 16-bit checkpoint names it, or None for adapter-only modules."""+ parts = fqn.split(".")+ if parts[:2] == ["base_model", "model"]:+ parts = parts[2:]+ # Adapter weights are merged into their base layer; a modules_to_save copy is saved under its wrapper's name.+ if any(p.startswith("lora_") or p == "original_module" for p in parts):+ return None+ if "modules_to_save" in parts:+ i = parts.index("modules_to_save")+ parts = parts[:i] + parts[i + 2 :]+ return ".".join(p for p in parts if p != "base_layer")+++def _peft_untied_lm_head(model):+ """(merged checkpoint key, weight) of the lm_head when QAT untied it from the embedding, else None."""+ config = getattr(model, "_torchao_config", None)+ if getattr(config, "prequantization_transform", None) is not _untie_input_output_embeddings:+ return None+ lm_head = model.get_output_embeddings()+ names = [_peft_merged_module_name(n) for n, m in model.named_modules() if m is lm_head]+ if not names or not names[0]:+ return None+ return names[0] + ".weight", lm_head.weight++def _merged_torchao_config_for_peft_qat(model):"""TorchAoConfig for the merged 16-bit reload of a QAT LoRA model, naming modules as the reload does (PEFT wrappers removed)."""from transformers import TorchAoConfigfrom torchao.quantization import ModuleFqnToConfigfrom torchao.quantization.quant_api import _is_linear- def _merged_name(fqn):- parts = fqn.split(".")- if parts[:2] == ["base_model", "model"]:- parts = parts[2:]- # Adapter weights are merged into their base layer; a modules_to_save copy is saved under its wrapper's name.- if any(p.startswith("lora_") or p == "original_module" for p in parts):- return None- if "modules_to_save" in parts:- i = parts.index("modules_to_save")- parts = parts[:i] + parts[i + 2 :]- return ".".join(p for p in parts if p != "base_layer")+ _merged_name = _peft_merged_module_nameconfigs = model._torchao_config.base_config_and_filter_fnsif len(configs) == 1:@@ -7474,6 +7491,26 @@ def _prepare_model_for_qat()except ImportError:raise ImportError(TORCHAO_MSG)+ from torchao.quantization.quant_api import _is_linear++ group_size = 32+ # PerGroup(32) rejects in_features not divisible by 32, e.g. lora_B at rank 16.+ linear_filter = lambda m, fqn: _is_linear(m, fqn) and m.in_features % group_size == 0+ skipped = [+ name+ for name, module in model.named_modules()+ if _is_linear(module, name)+ and module.in_features % group_size != 0+ and ".lora_" not in name+ ]+ if skipped:+ warnings.warn(+ f"Unsloth: qat_scheme='int8-int4' uses PerGroup({group_size}); these Linear layers "+ f"have in_features not divisible by {group_size} and stay in full precision: "+ + ", ".join(skipped[:8])+ + (f", ... ({len(skipped) - 8} more)" if len(skipped) > 8 else ""),+ stacklevel = 2,+ )torchao_config = TorchAOConfig(qat_scheme = qat_scheme,base_config_and_filter_fns = [@@ -7483,9 +7520,9 @@ def _prepare_model_for_qat(),(Int8DynamicActivationIntxWeightConfig(- weight_dtype = torch.int4, weight_granularity = PerGroup(32)+ weight_dtype = torch.int4, weight_granularity = PerGroup(group_size)),- None,+ linear_filter,),],prequantization_transform = _untie_input_output_embeddings,diff --git a/unsloth/save.py b/unsloth/save.pyindex 79768d63530..4eca801fa53 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -71,6 +71,7 @@ from .models.loader_utils import (from .models._utils import (_convert_torchao_model,_merged_torchao_config_for_peft_qat,+ _peft_untied_lm_head,_strip_torchao_fake_quantization,lora_relative_to_original_base,)@@ -6044,6 +6045,7 @@ def _unsloth_save_torchao_with_attached_config(if isinstance(model, (PeftModelForCausalLM, PeftModel)):# Quantizing in place leaves int4 base layers the LoRA merge cannot read, so merge in 16-bit and quantize the reload with the QAT configs.torchao_config = _merged_torchao_config_for_peft_qat(model)+ untied_lm_head = _peft_untied_lm_head(model)_strip_torchao_fake_quantization(model)_unsloth_save_torchao_with_given_config(model = model,@@ -6052,6 +6054,7 @@ def _unsloth_save_torchao_with_attached_config(torchao_config = torchao_config,push_to_hub = push_to_hub,token = token,+ untied_lm_head = untied_lm_head,)return@@ -6068,6 +6071,66 @@ def _unsloth_save_torchao_with_attached_config(tokenizer.save_pretrained(save_directory)+def _untie_merged_checkpoint(save_directory, lm_head_key, lm_head_weight):+ """Give a tied merged 16-bit checkpoint the model's own lm_head, so its torchao reload quantizes it apart from the embedding as QAT trained them."""+ from safetensors.torch import load_file, save_file++ config_path = os.path.join(save_directory, "config.json")+ with open(config_path, encoding = "utf-8") as f:+ config = json.load(f)+ # Only the configs on lm_head's path (e.g. thinker_config.text_config for thinker.lm_head), not sibling decoders.+ candidates, node = [config, config.get("text_config")], config+ for part in lm_head_key.split(".")[:-2]:+ node = node.get(f"{part}_config") if isinstance(node, dict) else None+ if not isinstance(node, dict):+ break+ candidates += [node, node.get("text_config")]+ tied = [c for c in candidates if isinstance(c, dict) and c.get("tie_word_embeddings")]+ if not tied:+ return++ weight = lm_head_weight.detach().to("cpu").contiguous()+ index_path = os.path.join(save_directory, "model.safetensors.index.json")+ if os.path.exists(index_path):+ with open(index_path, encoding = "utf-8") as f:+ index = json.load(f)+ if lm_head_key not in index["weight_map"]:+ used, shard, i = set(index["weight_map"].values()), "model-lm_head.safetensors", 0+ while shard in used:+ i += 1+ shard = f"model-lm_head-{i}.safetensors"+ save_file(+ {lm_head_key: weight},+ os.path.join(save_directory, shard),+ metadata = {"format": "pt"},+ )+ index["weight_map"][lm_head_key] = shard+ with open(index_path, "w", encoding = "utf-8") as f:+ json.dump(index, f, indent = 2)+ else:+ # The merge keeps the base repo's file name (model.safetensors, consolidated.safetensors, ...).+ files = [+ f+ for f in os.listdir(save_directory)+ if f.endswith(".safetensors") and not f.startswith("adapter_model")+ ]+ if "model.safetensors" in files:+ files = ["model.safetensors"]+ if len(files) != 1:+ return+ path = os.path.join(save_directory, files[0])+ tensors = load_file(path)+ if lm_head_key not in tensors:+ tensors[lm_head_key] = weight+ save_file(tensors, path, metadata = {"format": "pt"})+ del tensors++ for c in tied:+ c["tie_word_embeddings"] = False+ with open(config_path, "w", encoding = "utf-8") as f:+ json.dump(config, f, indent = 2)++def _unsloth_save_torchao_with_given_config(model,save_directory: Union[str, os.PathLike],@@ -6075,6 +6138,7 @@ def _unsloth_save_torchao_with_given_config(torchao_config,push_to_hub: bool = False,token: Optional[Union[str, bool]] = None,+ untied_lm_head = None,):"""Quantize the model with torchao and save a torchao quantized checkpoint.@@ -6093,6 +6157,7 @@ def _unsloth_save_torchao_with_given_config(arguments["push_to_hub"] = Falsearguments["save_method"] = "merged_16bit"del arguments["torchao_config"]+ del arguments["untied_lm_head"]if not isinstance(model, PeftModelForCausalLM) and not isinstance(model, PeftModel):model.save_pretrained(save_directory)@@ -6100,6 +6165,9 @@ def _unsloth_save_torchao_with_given_config(else:unsloth_generic_save(**arguments)+ if untied_lm_head is not None:+ _untie_merged_checkpoint(save_directory, *untied_lm_head)+for _ in range(3):gc.collect()