Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix save_pretrained_torchao for full finetuning QAT on transformers 5 (#14639)

Daniel Hana81cb774da44
diff --git a/tests/saving/test_qat_torchao_save.py b/tests/saving/test_qat_torchao_save.py
index 712e0c9eb60..951471af7d2 100644
--- a/tests/saving/test_qat_torchao_save.py
+++ b/tests/saving/test_qat_torchao_save.py
@@ -335,3 +335,44 @@ def test_phone_deployment_config_only_where_it_applies(monkeypatch, tmp_path, ca
assert "optimum-executorch" in printed and "group size 32" in printed
else:
assert "group_size=32" in printed
+
+
+class _TokenizerStub:
+ def save_pretrained(self, save_directory, **kwargs):
+ os.makedirs(save_directory, exist_ok = True)
+
+
+def test_full_finetune_qat_save_reloads_quantized(tmp_path):
+ # Full finetuning converts in place; transformers 5 refused the torchao subclasses without an hf_quantizer.
+ from transformers import AutoModelForCausalLM, LlamaConfig
+
+ config = LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 128,
+ intermediate_size = 256,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ tie_word_embeddings = False,
+ )
+ torch.manual_seed(0)
+ model = AutoModelForCausalLM.from_config(config).to(torch.bfloat16)
+ model = _prepare_model_for_qat(model, _int8_qat_config())
+ save_mod._unsloth_save_torchao_with_attached_config(model, str(tmp_path), _TokenizerStub())
+ assert getattr(model, "hf_quantizer", None) is None
+
+ reloaded = AutoModelForCausalLM.from_pretrained(str(tmp_path), dtype = torch.bfloat16)
+ assert reloaded.config.quantization_config is not None
+ ours = dict(model.named_modules())
+ n_quantized = 0
+ for name, module in reloaded.named_modules():
+ if not isinstance(module, torch.nn.Linear):
+ continue
+ before, after = ours[name].weight, module.weight
+ assert type(after) is type(before), name
+ if _is_quantized(before):
+ n_quantized += 1
+ assert torch.equal(after.dequantize(), before.dequantize()), name
+ else:
+ assert torch.equal(after, before), name
+ assert n_quantized == 8
diff --git a/unsloth/save.py b/unsloth/save.py
index 5a39918ee5d..830de413e8e 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -6822,12 +6822,35 @@ def _unsloth_save_torchao_with_attached_config(
# TorchAO does not support safe_serialization reliably.
safe_serialization = False
- if push_to_hub:
- model.push_to_hub(save_directory, safe_serialization = safe_serialization, token = token)
- tokenizer.push_to_hub(save_directory, token = token)
- else:
- model.save_pretrained(save_directory, safe_serialization = safe_serialization)
- tokenizer.save_pretrained(save_directory)
+ with _torchao_hf_quantizer_attached(model):
+ if push_to_hub:
+ model.push_to_hub(save_directory, safe_serialization = safe_serialization, token = token)
+ tokenizer.push_to_hub(save_directory, token = token)
+ else:
+ model.save_pretrained(save_directory, safe_serialization = safe_serialization)
+ tokenizer.save_pretrained(save_directory)
+
+
+@contextmanager
+def _torchao_hf_quantizer_attached(model):
+ """Attach a torchao HfQuantizer for the save of a model quantized in place.
+
+ transformers 5 otherwise treats the torchao tensor subclasses as plain tensors and fails its tied-weight
+ storage check; 4.x pickles them with safe_serialization = False and needs nothing."""
+ if getattr(model, "hf_quantizer", None) is not None or Version(
+ importlib_version("transformers")
+ ) < Version("5.0.0"):
+ yield
+ return
+ from transformers.quantizers.auto import AutoHfQuantizer
+
+ model.hf_quantizer = AutoHfQuantizer.from_config(
+ model.config.quantization_config, pre_quantized = False
+ )
+ try:
+ yield
+ finally:
+ del model.hf_quantizer
def _untie_merged_checkpoint(save_directory, lm_head_key, lm_head_weight):