Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix int8-int4 QAT LoRA at low ranks and on tied-embedding models (#13552)

Daniel Han0943697bf88c
diff --git a/tests/saving/test_qat_torchao_save.py b/tests/saving/test_qat_torchao_save.py
index a47b94c354b..c33c3dcd227 100644
--- a/tests/saving/test_qat_torchao_save.py
+++ b/tests/saving/test_qat_torchao_save.py
@@ -188,3 +188,81 @@ def test_peft_qat_save_reloads_with_merged_names(monkeypatch, tmp_path):
for module in peft_model.modules():
assert not hasattr(module, "weight_fake_quantizer")
assert not _is_quantized(getattr(module, "weight", None))
+
+
+class _TiedModel(_Model):
+ def __init__(self):
+ super().__init__()
+ self.config = type("cfg", (), {"model_type": "llama", "tie_word_embeddings": True})()
+ self.lm_head = torch.nn.Linear(256, 32, bias = False)
+ self.lm_head.weight = self.embed_tokens.weight
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+
+def test_int8_int4_lora_rank_below_group_size():
+ # lora_B at rank 16 has in_features 16, which PerGroup(32) rejects.
+ from peft import LoraConfig, get_peft_model
+
+ from unsloth.models._utils import _peft_untied_lm_head
+
+ peft_model = get_peft_model(
+ _TiedModel(), LoraConfig(r = 16, target_modules = ["q_proj", "down_proj"])
+ )
+ _prepare_model_for_qat(peft_model, "int8-int4")
+ block = peft_model.base_model.model.layers[0]
+ assert hasattr(block.q_proj.base_layer, "weight_fake_quantizer")
+ assert hasattr(block.q_proj.lora_A["default"], "weight_fake_quantizer")
+ assert not hasattr(block.q_proj.lora_B["default"], "weight_fake_quantizer")
+ lm_head = peft_model.base_model.model.lm_head
+ assert _peft_untied_lm_head(peft_model) == ("lm_head.weight", lm_head.weight)
+ assert lm_head.weight.data_ptr() != peft_model.base_model.model.embed_tokens.weight.data_ptr()
+ assert not lm_head.weight.requires_grad
+
+
[email protected]("layout", ["single", "consolidated", "sharded"])
+def test_untie_merged_checkpoint(tmp_path, layout):
+ import json
+
+ from safetensors import safe_open
+ from safetensors.torch import save_file
+
+ tensors = {
+ "thinker.model.embed_tokens.weight": torch.randn(32, 8),
+ "thinker.model.norm.weight": torch.ones(8),
+ }
+ if layout == "sharded":
+ # A local checkpoint can already use the name the head would get.
+ save_file(tensors, tmp_path / "model-lm_head.safetensors")
+ weight_map = {k: "model-lm_head.safetensors" for k in tensors}
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps({"metadata": {}, "weight_map": weight_map}), encoding = "utf-8"
+ )
+ else:
+ name = "model.safetensors" if layout == "single" else "consolidated.safetensors"
+ save_file(tensors, tmp_path / name)
+ # A reused directory can still hold an adapter save.
+ save_file({"x": torch.zeros(1)}, tmp_path / "adapter_model.safetensors")
+ config = {
+ "thinker_config": {"text_config": {"tie_word_embeddings": True}},
+ "talker_config": {"tie_word_embeddings": True},
+ }
+ (tmp_path / "config.json").write_text(json.dumps(config), encoding = "utf-8")
+ # The model's own head, which can differ from the checkpoint's (trained) embedding.
+ lm_head = torch.randn(32, 8)
+
+ save_mod._untie_merged_checkpoint(str(tmp_path), "thinker.lm_head.weight", lm_head)
+
+ config = json.loads((tmp_path / "config.json").read_text(encoding = "utf-8"))
+ assert config["thinker_config"]["text_config"]["tie_word_embeddings"] is False
+ assert config["talker_config"]["tie_word_embeddings"] is True
+ if layout == "sharded":
+ index = json.loads((tmp_path / "model.safetensors.index.json").read_text(encoding = "utf-8"))
+ shard = tmp_path / index["weight_map"]["thinker.lm_head.weight"]
+ with safe_open(str(tmp_path / "model-lm_head.safetensors"), "pt") as h:
+ assert set(h.keys()) == set(tensors)
+ else:
+ shard = tmp_path / name
+ with safe_open(str(shard), "pt") as h:
+ assert torch.equal(h.get_tensor("thinker.lm_head.weight"), lm_head)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index e1e776c372d..f31cb706dce 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -7257,7 +7257,8 @@ def _untie_input_output_embeddings(model: torch.nn.Module) -> None:
if out_proj.weight.data_ptr() == in_emb.weight.data_ptr():
with torch.no_grad():
W = in_emb.weight.detach().clone()
- out_proj.weight = torch.nn.Parameter(W) # new storage, keeps dtype/device
+ # Same trainability as the embedding, so a LoRA model keeps the head frozen.
+ out_proj.weight = torch.nn.Parameter(W, requires_grad = in_emb.weight.requires_grad)
def _no_tie(self):
return
@@ -7303,23 +7304,39 @@ def _strip_torchao_fake_quantization(model):
)
+def _peft_merged_module_name(fqn):
+ """A PEFT module name as the merged 16-bit checkpoint names it, or None for adapter-only modules."""
+ parts = fqn.split(".")
+ if parts[:2] == ["base_model", "model"]:
+ parts = parts[2:]
+ # Adapter weights are merged into their base layer; a modules_to_save copy is saved under its wrapper's name.
+ if any(p.startswith("lora_") or p == "original_module" for p in parts):
+ return None
+ if "modules_to_save" in parts:
+ i = parts.index("modules_to_save")
+ parts = parts[:i] + parts[i + 2 :]
+ return ".".join(p for p in parts if p != "base_layer")
+
+
+def _peft_untied_lm_head(model):
+ """(merged checkpoint key, weight) of the lm_head when QAT untied it from the embedding, else None."""
+ config = getattr(model, "_torchao_config", None)
+ if getattr(config, "prequantization_transform", None) is not _untie_input_output_embeddings:
+ return None
+ lm_head = model.get_output_embeddings()
+ names = [_peft_merged_module_name(n) for n, m in model.named_modules() if m is lm_head]
+ if not names or not names[0]:
+ return None
+ return names[0] + ".weight", lm_head.weight
+
+
def _merged_torchao_config_for_peft_qat(model):
"""TorchAoConfig for the merged 16-bit reload of a QAT LoRA model, naming modules as the reload does (PEFT wrappers removed)."""
from transformers import TorchAoConfig
from torchao.quantization import ModuleFqnToConfig
from torchao.quantization.quant_api import _is_linear
- def _merged_name(fqn):
- parts = fqn.split(".")
- if parts[:2] == ["base_model", "model"]:
- parts = parts[2:]
- # Adapter weights are merged into their base layer; a modules_to_save copy is saved under its wrapper's name.
- if any(p.startswith("lora_") or p == "original_module" for p in parts):
- return None
- if "modules_to_save" in parts:
- i = parts.index("modules_to_save")
- parts = parts[:i] + parts[i + 2 :]
- return ".".join(p for p in parts if p != "base_layer")
+ _merged_name = _peft_merged_module_name
configs = model._torchao_config.base_config_and_filter_fns
if len(configs) == 1:
@@ -7474,6 +7491,26 @@ def _prepare_model_for_qat(
)
except ImportError:
raise ImportError(TORCHAO_MSG)
+ from torchao.quantization.quant_api import _is_linear
+
+ group_size = 32
+ # PerGroup(32) rejects in_features not divisible by 32, e.g. lora_B at rank 16.
+ linear_filter = lambda m, fqn: _is_linear(m, fqn) and m.in_features % group_size == 0
+ skipped = [
+ name
+ for name, module in model.named_modules()
+ if _is_linear(module, name)
+ and module.in_features % group_size != 0
+ and ".lora_" not in name
+ ]
+ if skipped:
+ warnings.warn(
+ f"Unsloth: qat_scheme='int8-int4' uses PerGroup({group_size}); these Linear layers "
+ f"have in_features not divisible by {group_size} and stay in full precision: "
+ + ", ".join(skipped[:8])
+ + (f", ... ({len(skipped) - 8} more)" if len(skipped) > 8 else ""),
+ stacklevel = 2,
+ )
torchao_config = TorchAOConfig(
qat_scheme = qat_scheme,
base_config_and_filter_fns = [
@@ -7483,9 +7520,9 @@ def _prepare_model_for_qat(
),
(
Int8DynamicActivationIntxWeightConfig(
- weight_dtype = torch.int4, weight_granularity = PerGroup(32)
+ weight_dtype = torch.int4, weight_granularity = PerGroup(group_size)
),
- None,
+ linear_filter,
),
],
prequantization_transform = _untie_input_output_embeddings,
diff --git a/unsloth/save.py b/unsloth/save.py
index 79768d63530..4eca801fa53 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -71,6 +71,7 @@ from .models.loader_utils import (
from .models._utils import (
_convert_torchao_model,
_merged_torchao_config_for_peft_qat,
+ _peft_untied_lm_head,
_strip_torchao_fake_quantization,
lora_relative_to_original_base,
)
@@ -6044,6 +6045,7 @@ def _unsloth_save_torchao_with_attached_config(
if isinstance(model, (PeftModelForCausalLM, PeftModel)):
# Quantizing in place leaves int4 base layers the LoRA merge cannot read, so merge in 16-bit and quantize the reload with the QAT configs.
torchao_config = _merged_torchao_config_for_peft_qat(model)
+ untied_lm_head = _peft_untied_lm_head(model)
_strip_torchao_fake_quantization(model)
_unsloth_save_torchao_with_given_config(
model = model,
@@ -6052,6 +6054,7 @@ def _unsloth_save_torchao_with_attached_config(
torchao_config = torchao_config,
push_to_hub = push_to_hub,
token = token,
+ untied_lm_head = untied_lm_head,
)
return
@@ -6068,6 +6071,66 @@ def _unsloth_save_torchao_with_attached_config(
tokenizer.save_pretrained(save_directory)
+def _untie_merged_checkpoint(save_directory, lm_head_key, lm_head_weight):
+ """Give a tied merged 16-bit checkpoint the model's own lm_head, so its torchao reload quantizes it apart from the embedding as QAT trained them."""
+ from safetensors.torch import load_file, save_file
+
+ config_path = os.path.join(save_directory, "config.json")
+ with open(config_path, encoding = "utf-8") as f:
+ config = json.load(f)
+ # Only the configs on lm_head's path (e.g. thinker_config.text_config for thinker.lm_head), not sibling decoders.
+ candidates, node = [config, config.get("text_config")], config
+ for part in lm_head_key.split(".")[:-2]:
+ node = node.get(f"{part}_config") if isinstance(node, dict) else None
+ if not isinstance(node, dict):
+ break
+ candidates += [node, node.get("text_config")]
+ tied = [c for c in candidates if isinstance(c, dict) and c.get("tie_word_embeddings")]
+ if not tied:
+ return
+
+ weight = lm_head_weight.detach().to("cpu").contiguous()
+ index_path = os.path.join(save_directory, "model.safetensors.index.json")
+ if os.path.exists(index_path):
+ with open(index_path, encoding = "utf-8") as f:
+ index = json.load(f)
+ if lm_head_key not in index["weight_map"]:
+ used, shard, i = set(index["weight_map"].values()), "model-lm_head.safetensors", 0
+ while shard in used:
+ i += 1
+ shard = f"model-lm_head-{i}.safetensors"
+ save_file(
+ {lm_head_key: weight},
+ os.path.join(save_directory, shard),
+ metadata = {"format": "pt"},
+ )
+ index["weight_map"][lm_head_key] = shard
+ with open(index_path, "w", encoding = "utf-8") as f:
+ json.dump(index, f, indent = 2)
+ else:
+ # The merge keeps the base repo's file name (model.safetensors, consolidated.safetensors, ...).
+ files = [
+ f
+ for f in os.listdir(save_directory)
+ if f.endswith(".safetensors") and not f.startswith("adapter_model")
+ ]
+ if "model.safetensors" in files:
+ files = ["model.safetensors"]
+ if len(files) != 1:
+ return
+ path = os.path.join(save_directory, files[0])
+ tensors = load_file(path)
+ if lm_head_key not in tensors:
+ tensors[lm_head_key] = weight
+ save_file(tensors, path, metadata = {"format": "pt"})
+ del tensors
+
+ for c in tied:
+ c["tie_word_embeddings"] = False
+ with open(config_path, "w", encoding = "utf-8") as f:
+ json.dump(config, f, indent = 2)
+
+
def _unsloth_save_torchao_with_given_config(
model,
save_directory: Union[str, os.PathLike],
@@ -6075,6 +6138,7 @@ def _unsloth_save_torchao_with_given_config(
torchao_config,
push_to_hub: bool = False,
token: Optional[Union[str, bool]] = None,
+ untied_lm_head = None,
):
"""Quantize the model with torchao and save a torchao quantized checkpoint.
@@ -6093,6 +6157,7 @@ def _unsloth_save_torchao_with_given_config(
arguments["push_to_hub"] = False
arguments["save_method"] = "merged_16bit"
del arguments["torchao_config"]
+ del arguments["untied_lm_head"]
if not isinstance(model, PeftModelForCausalLM) and not isinstance(model, PeftModel):
model.save_pretrained(save_directory)
@@ -6100,6 +6165,9 @@ def _unsloth_save_torchao_with_given_config(
else:
unsloth_generic_save(**arguments)
+ if untied_lm_head is not None:
+ _untie_merged_checkpoint(save_directory, *untied_lm_head)
+
for _ in range(3):
gc.collect()