Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix save_pretrained_torchao for full finetuning QAT on transformers 5 (#14639)
Daniel Hana81cb774da44
diff --git a/tests/saving/test_qat_torchao_save.py b/tests/saving/test_qat_torchao_save.pyindex 712e0c9eb60..951471af7d2 100644--- a/tests/saving/test_qat_torchao_save.py+++ b/tests/saving/test_qat_torchao_save.py@@ -335,3 +335,44 @@ def test_phone_deployment_config_only_where_it_applies(monkeypatch, tmp_path, caassert "optimum-executorch" in printed and "group size 32" in printedelse:assert "group_size=32" in printed+++class _TokenizerStub:+ def save_pretrained(self, save_directory, **kwargs):+ os.makedirs(save_directory, exist_ok = True)+++def test_full_finetune_qat_save_reloads_quantized(tmp_path):+ # Full finetuning converts in place; transformers 5 refused the torchao subclasses without an hf_quantizer.+ from transformers import AutoModelForCausalLM, LlamaConfig++ config = LlamaConfig(+ vocab_size = 64,+ hidden_size = 128,+ intermediate_size = 256,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ tie_word_embeddings = False,+ )+ torch.manual_seed(0)+ model = AutoModelForCausalLM.from_config(config).to(torch.bfloat16)+ model = _prepare_model_for_qat(model, _int8_qat_config())+ save_mod._unsloth_save_torchao_with_attached_config(model, str(tmp_path), _TokenizerStub())+ assert getattr(model, "hf_quantizer", None) is None++ reloaded = AutoModelForCausalLM.from_pretrained(str(tmp_path), dtype = torch.bfloat16)+ assert reloaded.config.quantization_config is not None+ ours = dict(model.named_modules())+ n_quantized = 0+ for name, module in reloaded.named_modules():+ if not isinstance(module, torch.nn.Linear):+ continue+ before, after = ours[name].weight, module.weight+ assert type(after) is type(before), name+ if _is_quantized(before):+ n_quantized += 1+ assert torch.equal(after.dequantize(), before.dequantize()), name+ else:+ assert torch.equal(after, before), name+ assert n_quantized == 8diff --git a/unsloth/save.py b/unsloth/save.pyindex 5a39918ee5d..830de413e8e 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -6822,12 +6822,35 @@ def _unsloth_save_torchao_with_attached_config(# TorchAO does not support safe_serialization reliably.safe_serialization = False- if push_to_hub:- model.push_to_hub(save_directory, safe_serialization = safe_serialization, token = token)- tokenizer.push_to_hub(save_directory, token = token)- else:- model.save_pretrained(save_directory, safe_serialization = safe_serialization)- tokenizer.save_pretrained(save_directory)+ with _torchao_hf_quantizer_attached(model):+ if push_to_hub:+ model.push_to_hub(save_directory, safe_serialization = safe_serialization, token = token)+ tokenizer.push_to_hub(save_directory, token = token)+ else:+ model.save_pretrained(save_directory, safe_serialization = safe_serialization)+ tokenizer.save_pretrained(save_directory)+++@contextmanager+def _torchao_hf_quantizer_attached(model):+ """Attach a torchao HfQuantizer for the save of a model quantized in place.++ transformers 5 otherwise treats the torchao tensor subclasses as plain tensors and fails its tied-weight+ storage check; 4.x pickles them with safe_serialization = False and needs nothing."""+ if getattr(model, "hf_quantizer", None) is not None or Version(+ importlib_version("transformers")+ ) < Version("5.0.0"):+ yield+ return+ from transformers.quantizers.auto import AutoHfQuantizer++ model.hf_quantizer = AutoHfQuantizer.from_config(+ model.config.quantization_config, pre_quantized = False+ )+ try:+ yield+ finally:+ del model.hf_quantizerdef _untie_merged_checkpoint(save_directory, lm_head_key, lm_head_weight):