Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix phone-deployment QAT for Gemma3 and LoRA, and write the matching ExecuTorch export config (#14376)
Daniel Han7c0e7d2d3d09
diff --git a/tests/saving/test_qat_torchao_save.py b/tests/saving/test_qat_torchao_save.pyindex c33c3dcd227..712e0c9eb60 100644--- a/tests/saving/test_qat_torchao_save.py+++ b/tests/saving/test_qat_torchao_save.py@@ -6,6 +6,7 @@from __future__ import annotationsimport importlib.util+import osimport pytestimport torch@@ -213,7 +214,8 @@ def test_int8_int4_lora_rank_below_group_size():_prepare_model_for_qat(peft_model, "int8-int4")block = peft_model.base_model.model.layers[0]assert hasattr(block.q_proj.base_layer, "weight_fake_quantizer")- assert hasattr(block.q_proj.lora_A["default"], "weight_fake_quantizer")+ # Only the frozen base weight is fake-quantized (torchtune QATLoRALinear); adapters merge before quantization.+ assert not hasattr(block.q_proj.lora_A["default"], "weight_fake_quantizer")assert not hasattr(block.q_proj.lora_B["default"], "weight_fake_quantizer")lm_head = peft_model.base_model.model.lm_headassert _peft_untied_lm_head(peft_model) == ("lm_head.weight", lm_head.weight)@@ -266,3 +268,70 @@ def test_untie_merged_checkpoint(tmp_path, layout):shard = tmp_path / namewith safe_open(str(shard), "pt") as h:assert torch.equal(h.get_tensor("thinker.lm_head.weight"), lm_head)++[email protected]("peft", [True, False], ids = ["peft", "full"])[email protected]("qat_scheme", ["int8-int4", "int8"])+def test_phone_deployment_save_writes_executorch_config(monkeypatch, tmp_path, qat_scheme, peft):+ target = tmp_path / "out"+ # A "-torchao" sibling left by an earlier LoRA export must not attract a full model's config.+ os.makedirs(str(target) + "-torchao")++ class _PeftStub(_Model):+ pass++ def fake_save(model, save_directory, **kw):+ os.makedirs(save_directory + "-torchao" if peft else save_directory, exist_ok = True)++ monkeypatch.setattr(save_mod, "_unsloth_save_torchao_with_attached_config", fake_save)+ monkeypatch.setattr(save_mod, "PeftModel", _PeftStub)+ monkeypatch.setattr(save_mod, "PeftModelForCausalLM", _PeftStub)+ model = _PeftStub() if peft else _Model()+ model._torchao_config = _int8_qat_config()+ model._torchao_config.qat_scheme = qat_scheme+ model._torchao_config._unsloth_builtin_recipe = True+ save_mod.unsloth_save_pretrained_torchao(model, str(target), tokenizer = object())+ path = tmp_path / ("out-torchao" if peft else "out") / "executorch_export_config.yaml"+ stale = tmp_path / ("out" if peft else "out-torchao") / "executorch_export_config.yaml"+ assert not stale.exists()+ if qat_scheme != "int8-int4":+ assert not path.exists()+ return+ import yaml++ config = yaml.safe_load(path.read_text())+ assert config["quantization"] == {+ "qmode": "8da4w",+ "group_size": 32,+ "use_hqq": False,+ "embedding_quantize": "8,0",+ }+ assert config["backend"]["xnnpack"]["enabled"] is True++[email protected]("case", ["custom_recipe", "gemma3", "hub"])+def test_phone_deployment_config_only_where_it_applies(monkeypatch, tmp_path, capsys, case):+ target = tmp_path / "out"++ def fake_save(model, save_directory, **kw):+ os.makedirs(save_directory, exist_ok = True)++ monkeypatch.setattr(save_mod, "_unsloth_save_torchao_with_attached_config", fake_save)+ model = _Model()+ model._torchao_config = _int8_qat_config()+ model._torchao_config.qat_scheme = "int8-int4"+ if case != "custom_recipe":+ # A recipe Unsloth built from the scheme name; a custom TorchAOConfig keeps only the label.+ model._torchao_config._unsloth_builtin_recipe = True+ if case == "gemma3":+ model.config.model_type = "gemma3_text"+ kwargs = dict(push_to_hub = True, token = "x") if case == "hub" else {}+ save_mod.unsloth_save_pretrained_torchao(model, str(target), tokenizer = object(), **kwargs)+ assert not (target / "executorch_export_config.yaml").exists()+ printed = capsys.readouterr().out+ if case == "custom_recipe":+ assert "ExecuTorch" not in printed+ elif case == "gemma3":+ assert "optimum-executorch" in printed and "group size 32" in printed+ else:+ assert "group_size=32" in printeddiff --git a/tests/test_qat_phone_deployment.py b/tests/test_qat_phone_deployment.pynew file mode 100644index 00000000000..b339eaff7ed--- /dev/null+++ b/tests/test_qat_phone_deployment.py@@ -0,0 +1,265 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""QAT scheme construction on tiny CPU models."""++from __future__ import annotations++import importlib.util+import warnings++import pytest+import torch++pytestmark = pytest.mark.skipif(+ importlib.util.find_spec("torchao") is None, reason = "require torchao"+)++from unsloth.models._utils import (+ TorchAOConfig,+ _convert_torchao_model,+ _prepare_model_for_qat,+)+++def _tiny_gemma3():+ from transformers import Gemma3ForCausalLM, Gemma3TextConfig++ torch.manual_seed(0)+ config = Gemma3TextConfig(+ vocab_size = 256,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 32,+ tie_word_embeddings = True,+ )+ return Gemma3ForCausalLM(config).eval()+++def _tiny_qwen3():+ from transformers import Qwen3Config, Qwen3ForCausalLM++ torch.manual_seed(0)+ config = Qwen3Config(+ vocab_size = 256,+ hidden_size = 128,+ intermediate_size = 256,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ num_key_value_heads = 1,+ head_dim = 32,+ )+ return Qwen3ForCausalLM(config).eval()+++def _is_fake_quantized(module):+ return hasattr(module, "weight_fake_quantizer")+++def test_phone_deployment_alias_in_prepare():+ model = _prepare_model_for_qat(_tiny_qwen3(), "phone-deployment")+ assert model._torchao_config.qat_scheme == "int8-int4"+ assert _is_fake_quantized(model.model.layers[0].self_attn.q_proj)+ assert _is_fake_quantized(model.model.embed_tokens)+++def test_gemma3_int8_int4_keeps_phone_scheme():+ from torchao.quantization import Int8DynamicActivationIntxWeightConfig++ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")+ assert model._torchao_config.qat_scheme == "int8-int4"+ configs = [c for c, _ in model._torchao_config.base_config_and_filter_fns]+ assert any(isinstance(c, Int8DynamicActivationIntxWeightConfig) for c in configs)+ assert _is_fake_quantized(model.model.layers[0].self_attn.q_proj)+++def _int8_embedding_config():+ from torchao.quantization import IntxWeightOnlyConfig+ from torchao.quantization.granularity import PerAxis+ return TorchAOConfig(+ qat_scheme = None,+ base_config_and_filter_fns = [+ (+ IntxWeightOnlyConfig(weight_dtype = torch.int8, granularity = PerAxis(0)),+ lambda m, _: isinstance(m, torch.nn.Embedding),+ )+ ],+ )++[email protected]("scheme", ["int8-int4", "custom"])[email protected]("convert", ["none", "unsloth", "manual"])+def test_gemma3_fake_quantized_embedding_keeps_embed_scale(scheme, convert):+ from torchao.quantization import quantize_+ from torchao.quantization.qat import QATConfig+ from transformers.models.gemma3.modeling_gemma3 import Gemma3TextScaledWordEmbedding++ model = _tiny_gemma3()+ ids = torch.randint(0, model.config.vocab_size, (2, 12))+ with torch.no_grad():+ ref = model.get_input_embeddings()(ids)+ assert float(model.get_input_embeddings().embed_scale) == 8.0++ model = _prepare_model_for_qat(+ model, "int8-int4" if scheme == "int8-int4" else _int8_embedding_config()+ )+ emb = model.get_input_embeddings()+ assert _is_fake_quantized(emb)+ if convert == "unsloth":+ _convert_torchao_model(model)+ elif convert == "manual":+ quantize_(model, QATConfig(step = "convert"))+ emb = model.get_input_embeddings()+ assert isinstance(emb, Gemma3TextScaledWordEmbedding)+ with torch.no_grad():+ out = emb(ids)+ # int8 per-row error is ~0.4% of the row max; a lost sqrt(hidden_size) = 8x scale is not.+ rel = (out - ref).abs().max() / ref.abs().max()+ assert rel < 0.02, float(rel)+++def test_int4_skips_linears_not_divisible_by_group_size():+ class _M(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.ok = torch.nn.Linear(256, 64, bias = False)+ self.odd = torch.nn.Linear(192, 64, bias = False)++ def forward(self, x):+ return self.ok(torch.nn.functional.pad(x, (0, 64))) + self.odd(x)++ model = _M().to(torch.bfloat16)+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ model = _prepare_model_for_qat(model, "int4")+ assert _is_fake_quantized(model.ok)+ assert not _is_fake_quantized(model.odd)+ assert any("odd" in str(w.message) for w in caught)+ model(torch.randn(2, 192, dtype = torch.bfloat16))++[email protected]("qat_scheme", ["int8-int4", "int4", "int8", "fp8-fp8"])+def test_lora_adapters_are_not_fake_quantized(qat_scheme, monkeypatch):+ from peft import LoraConfig, get_peft_model++ monkeypatch.delenv("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", raising = False)+ model = get_peft_model(+ _tiny_qwen3(),+ LoraConfig(r = 32, lora_alpha = 32, target_modules = ["q_proj", "o_proj"]),+ )+ model = _prepare_model_for_qat(model, qat_scheme)+ q_proj = model.base_model.model.model.layers[0].self_attn.q_proj+ assert _is_fake_quantized(q_proj.base_layer)+ assert not _is_fake_quantized(q_proj.lora_A["default"])+ assert not _is_fake_quantized(q_proj.lora_B["default"])+++def test_lora_fake_quant_opt_out(monkeypatch):+ from peft import LoraConfig, get_peft_model++ monkeypatch.setenv("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", "1")+ model = get_peft_model(+ _tiny_qwen3(), LoraConfig(r = 32, lora_alpha = 32, target_modules = ["q_proj"])+ )+ model = _prepare_model_for_qat(model, "int8-int4")+ q_proj = model.base_model.model.model.layers[0].self_attn.q_proj+ assert _is_fake_quantized(q_proj.lora_A["default"])+++class _EoiEmbedding(torch.nn.Embedding):+ # T5Gemma2TextScaledWordEmbedding pattern: an extra registered parameter read in forward.+ def __init__(self, num, dim):+ super().__init__(num, dim)+ self.eoi_embedding = torch.nn.Parameter(torch.ones(dim))++ def forward(self, input_ids):+ out = super().forward(input_ids)+ out[input_ids == 0] = self.eoi_embedding.to(out.dtype)+ return out++[email protected]("convert", [False, True])+def test_embedding_subclass_keeps_extra_parameters(convert):+ model = _tiny_gemma3()+ model.model.embed_tokens = _EoiEmbedding(model.config.vocab_size, model.config.hidden_size)+ eoi = model.model.embed_tokens.eoi_embedding+ model = _prepare_model_for_qat(model, "int8-int4")+ if convert:+ _convert_torchao_model(model)+ emb = model.get_input_embeddings()+ assert isinstance(emb, _EoiEmbedding)+ assert emb.eoi_embedding is eoi+ with torch.no_grad():+ out = emb(torch.tensor([[0, 1, 2]]))+ assert torch.equal(out[0, 0], eoi)+++def test_compiled_fake_quantized_embedding_pickles():+ import pickle++ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")+ emb = model.get_input_embeddings()+ emb.compile()+ restored = pickle.loads(pickle.dumps(emb))+ assert type(restored) is type(emb)+ assert "_compiled_call_impl" not in vars(restored) or restored._compiled_call_impl is None+++def test_fake_quantized_embedding_subclass_pickles():+ import io+ import pickle++ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")+ emb = model.get_input_embeddings()+ assert _is_fake_quantized(emb)+ restored = pickle.loads(pickle.dumps(emb))+ assert type(restored) is type(emb)+ buf = io.BytesIO()+ torch.save(emb, buf)+ buf.seek(0)+ assert type(torch.load(buf, weights_only = False)) is type(emb)+++def test_fake_quantized_embedding_unpickles_in_fresh_process(tmp_path):+ import os+ import pickle+ import subprocess+ import sys++ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")+ emb = model.get_input_embeddings()+ path = tmp_path / "emb.pkl"+ path.write_bytes(pickle.dumps(emb))+ ids = torch.tensor([[1, 2, 3]])+ expected = emb(ids)+ code = (+ "import pickle, sys, torch\n"+ f"emb = pickle.loads(open({str(path)!r}, 'rb').read())\n"+ "print(type(emb).__name__, float(emb.embed_scale))\n"+ "torch.save(emb(torch.tensor([[1, 2, 3]])), sys.argv[1])\n"+ )+ out_path = tmp_path / "out.pt"+ env = dict(os.environ)+ env["PYTHONPATH"] = os.pathsep.join([os.getcwd(), env.get("PYTHONPATH", "")])+ res = subprocess.run(+ [sys.executable, "-c", code, str(out_path)],+ capture_output = True,+ text = True,+ timeout = 600,+ env = env,+ )+ assert res.returncode == 0, res.stderr[-3000:]+ assert res.stdout.split()[-2] == type(emb).__name__+ assert float(res.stdout.split()[-1]) == float(emb.embed_scale)+ assert torch.equal(torch.load(out_path), expected)+++def test_builtin_recipe_flag_only_for_scheme_names():+ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")+ assert model._torchao_config._unsloth_builtin_recipe is True+ custom = _int8_embedding_config()+ model = _prepare_model_for_qat(_tiny_gemma3(), custom)+ assert not getattr(model._torchao_config, "_unsloth_builtin_recipe", False)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex d611b52fce8..5ccab86768b 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -142,7 +142,7 @@ __all__ = [import torch-from typing import Union, Optional, List, Any, Callable, Tuple, Iterator+from typing import Union, Optional, List, Any, Callable, Dict, Tuple, Iteratorfrom platform import system as platform_systemplatform_system = platform_system()@@ -7524,7 +7524,9 @@ class TorchAOConfig:default_factory = lambda: [(Int4WeightOnlyConfig(group_size = 128),- lambda m, _: isinstance(m, torch.nn.Linear) and getattr(m, "in_features", 0) >= 128,+ lambda m, _: isinstance(m, torch.nn.Linear)+ and getattr(m, "in_features", 0) >= 128+ and m.in_features % 128 == 0,),])@@ -7562,6 +7564,104 @@ def _untie_input_output_embeddings(model: torch.nn.Module) -> None:assert out_proj.weight.data_ptr() != in_emb.weight.data_ptr(), "Embeddings still tied!"+def _embedding_subclasses(model: torch.nn.Module) -> Dict[str, torch.nn.Module]:+ """nn.Embedding subclasses with their own forward (Gemma3TextScaledWordEmbedding scales by embed_scale)."""+ try:+ from torchao.quantization.qat.embedding import FakeQuantizedEmbedding+ except ImportError:+ return {}+ return {+ fqn: module+ for fqn, module in model.named_modules()+ if isinstance(module, torch.nn.Embedding)+ and type(module) is not torch.nn.Embedding+ and not isinstance(module, FakeQuantizedEmbedding)+ }+++def _copy_embedding_subclass_state(dst: torch.nn.Module, src: torch.nn.Module) -> None:+ """Copy the parameters, submodules, buffers and plain attributes an Embedding subclass adds (e.g.+ Gemma3's embed_scale, T5Gemma2's eoi_embedding) onto dst."""+ for name, param in src._parameters.items():+ if name not in dst._parameters:+ dst.register_parameter(name, param)+ for name, module in src._modules.items():+ if name not in dst._modules:+ dst.add_module(name, module)+ for name, buffer in src._buffers.items():+ if name not in dst._buffers:+ dst.register_buffer(+ name, buffer, persistent = name not in src._non_persistent_buffers_set+ )+ for name, value in vars(src).items():+ if name.startswith("_") or name in vars(dst) or isinstance(value, torch.nn.Module):+ continue+ setattr(dst, name, value)+++_FAKE_QUANTIZED_EMBEDDING_CLASSES = {}+++def _fake_quantized_embedding_class(original_cls):+ """FakeQuantizedEmbedding that keeps original_cls.forward: its super().forward() is the fake-quantized lookup."""+ from torchao.quantization.qat.embedding import FakeQuantizedEmbedding++ cls = _FAKE_QUANTIZED_EMBEDDING_CLASSES.get(original_cls)+ if cls is not None:+ return cls++ def to_embedding(self):+ # QATConfig(step = "convert") swaps back to a plain nn.Embedding; restore the subclass and its scale.+ embedding = FakeQuantizedEmbedding.to_embedding(self)+ embedding.__class__ = original_cls+ _copy_embedding_subclass_state(embedding, self)+ return embedding++ def __reduce_ex__(self, protocol):+ # Pickle by the original class, so a fresh process (torch.load, spawn workers) rebuilds this one.+ return (_rebuild_fake_quantized_embedding, (original_cls,), self.__getstate__())++ name = "_FakeQuantized_" + re.sub(+ r"\W", "_", f"{original_cls.__module__}.{original_cls.__qualname__}"+ )+ cls = type(+ name,+ (original_cls, FakeQuantizedEmbedding),+ {+ "to_embedding": to_embedding,+ "__reduce_ex__": __reduce_ex__,+ "__module__": __name__,+ "__qualname__": name,+ },+ )+ _FAKE_QUANTIZED_EMBEDDING_CLASSES[original_cls] = cls+ return cls+++def _rebuild_fake_quantized_embedding(original_cls):+ cls = _fake_quantized_embedding_class(original_cls)+ return cls.__new__(cls)+++def _restore_fake_quantized_embedding_subclasses(+ model: torch.nn.Module, originals: Dict[str, torch.nn.Module]+) -> None:+ """QAT prepare swaps in a plain FakeQuantizedEmbedding, dropping e.g. Gemma3's embed_scale; restore it."""+ if not originals:+ return+ from torchao.quantization.qat.embedding import FakeQuantizedEmbedding++ for fqn, original in originals.items():+ try:+ module = model.get_submodule(fqn)+ except AttributeError:+ continue+ if type(module) is not FakeQuantizedEmbedding:+ continue+ module.__class__ = _fake_quantized_embedding_class(type(original))+ _copy_embedding_subclass_state(module, original)++def _filter_fn_to_fqns(model: torch.nn.Module, filter_fn: Callable[[torch.nn.Module, str], bool]) -> Iterator[str]:@@ -7730,6 +7830,54 @@ def _convert_torchao_model(model):model.config.quantization_config = quantization_config+def _group_size_linear_filter(model, group_size, qat_scheme):+ """PerGroup filter: in_features not divisible by group_size breaks the fake quantizer's reshape."""+ filter_fn = lambda m, _: (+ isinstance(m, torch.nn.Linear)+ and m.in_features >= group_size+ and m.in_features % group_size == 0+ )+ skipped = [+ name+ for name, module in model.named_modules()+ if isinstance(module, torch.nn.Linear)+ and module.in_features >= group_size+ and module.in_features % group_size != 0+ and not _is_lora_adapter_name(name)+ ]+ if skipped:+ warnings.warn(+ f"Unsloth: qat_scheme='{qat_scheme}' uses group_size {group_size}; these Linear layers have "+ f"in_features not divisible by {group_size} and stay in full precision: "+ + ", ".join(skipped[:8])+ + (f", ... ({len(skipped) - 8} more)" if len(skipped) > 8 else ""),+ stacklevel = 3,+ )+ return filter_fn+++def _is_lora_adapter_name(fqn: str) -> bool:+ return any(part.startswith("lora_") for part in fqn.split("."))+++def _exclude_lora_adapters_from_qat(model, torchao_config) -> None:+ """Adapters are merged before quantization, so only base weights are fake-quantized."""+ if os.environ.get("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", "0") == "1":+ return+ if not any(_is_lora_adapter_name(name) for name, _ in model.named_modules()):+ return+ from torchao.quantization.quant_api import _is_linear++ def _skip_lora(filter_fn):+ filter_fn = filter_fn or _is_linear+ return lambda m, fqn: not _is_lora_adapter_name(fqn) and filter_fn(m, fqn)++ torchao_config.base_config_and_filter_fns = [+ (base_config, _skip_lora(filter_fn))+ for base_config, filter_fn in torchao_config.base_config_and_filter_fns+ ]++def _prepare_model_for_qat(model: torch.nn.Module, qat_scheme: Union[str, TorchAOConfig]) -> torch.nn.Module:@@ -7741,16 +7889,9 @@ def _prepare_model_for_qat(except ImportError:raise ImportError(TORCHAO_MSG)- # Gemma3 int8 embedding quantization struggles with its 262144-token vocabulary, so auto-switch to int4 weight-only.- if qat_scheme == "int8-int4":- model_types = get_transformers_model_type(model.config)- is_gemma3 = any("gemma3" in mt or "gemma_3" in mt for mt in model_types)- if is_gemma3:- print(- "Unsloth: Gemma3 has a large vocabulary causing int8 embedding issues. "- "Switching to int4 weight-only QAT for training stability."- )- qat_scheme = "int4"+ # get_peft_model passes the scheme straight here, unaliased.+ if qat_scheme == "phone-deployment":+ qat_scheme = "int8-int4"if not isinstance(qat_scheme, TorchAOConfig):torchao_config: Optional[TorchAOConfig] = None@@ -7761,7 +7902,7 @@ def _prepare_model_for_qat(raise ImportError(TORCHAO_MSG)group_size = 128base_config = Float8DynamicActivationInt4WeightConfig()- filter_fn = lambda m, _: isinstance(m, torch.nn.Linear) and m.in_features >= group_size+ filter_fn = _group_size_linear_filter(model, group_size, qat_scheme)torchao_config = TorchAOConfig(qat_scheme = qat_scheme,base_config_and_filter_fns = [(base_config, filter_fn)],@@ -7828,7 +7969,7 @@ def _prepare_model_for_qat(raise ImportError(TORCHAO_MSG)group_size = 128base_config = Int4WeightOnlyConfig(group_size = group_size)- filter_fn = lambda m, _: isinstance(m, torch.nn.Linear) and m.in_features >= group_size+ filter_fn = _group_size_linear_filter(model, group_size, qat_scheme)torchao_config = TorchAOConfig(qat_scheme = qat_scheme,base_config_and_filter_fns = [(base_config, filter_fn)],@@ -7892,6 +8033,9 @@ def _prepare_model_for_qat(else:raise ValueError(f"Unexpected QAT scheme {qat_scheme}")assert torchao_config is not None, f"TorchAOConfig was not set for {qat_scheme}"+ # Built here from the scheme name, so save-time export hints may assume its exact recipe.+ torchao_config._unsloth_builtin_recipe = True+ _exclude_lora_adapters_from_qat(model, torchao_config)else:torchao_config = qat_scheme@@ -7903,8 +8047,10 @@ def _prepare_model_for_qat(if torchao_config.prequantization_transform is not None:torchao_config.prequantization_transform(model)+ embedding_subclasses = _embedding_subclasses(model)for base_config, filter_fn in torchao_config.base_config_and_filter_fns:quantize_(model, QATConfig(base_config, step = "prepare"), filter_fn = filter_fn)+ _restore_fake_quantized_embedding_subclasses(model, embedding_subclasses)return modeldiff --git a/unsloth/save.py b/unsloth/save.pyindex c1ba0e0aa37..6145797cb31 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -7692,6 +7692,66 @@ def _unsloth_save_torchao(clean_gpu_cache()+# Must match qat_scheme="int8-int4"; export_llm defaults (group_size 128, use_hqq true) do not.+EXECUTORCH_PHONE_DEPLOYMENT_CONFIG = """\+# ExecuTorch export_llm config matching Unsloth qat_scheme="phone-deployment" (int8-int4).+# Export from a merged 16-bit checkpoint (model.save_pretrained_merged(dir, tokenizer)), converted to+# ExecuTorch's format, e.g. for Qwen3: python -m examples.models.qwen3.convert_weights <dir> model.pth+# Then: python -m extension.llm.export.export_llm --config this_file.yaml \\+# +base.model_class=<e.g. qwen3_0_6b> +base.params=<model params json> +base.checkpoint=model.pth+model:+ use_kv_cache: True+ use_sdpa_with_kv_cache: True+ dtype_override: fp32+quantization:+ qmode: 8da4w+ group_size: 32+ use_hqq: False+ embedding_quantize: "8,0"+backend:+ xnnpack:+ enabled: True+ extended_ops: True+"""+++# HF model_types whose architecture ExecuTorch's export_llm ModelType covers (llama2/3.x, smollm2, qwen2.5, qwen3,+# qwen3.5, phi-4-mini, lfm2); anything else (Gemma3, ...) goes through optimum-executorch instead.+_EXPORT_LLM_MODEL_TYPES = {"llama", "qwen2", "qwen3", "qwen3_5", "qwen3_5_text", "phi3", "lfm2"}+++def _write_executorch_export_config(+ qat_scheme,+ directory,+ model_type = None,+):+ """Write and print the ExecuTorch export settings matching a phone-deployment QAT model."""+ if qat_scheme != "int8-int4":+ return None+ settings = (+ "8da4w linear (dynamic int8 activations, int4 weights, group size 32, affine, not HQQ) and 8-bit "+ "per-channel embeddings"+ )+ if model_type not in _EXPORT_LLM_MODEL_TYPES:+ print(+ f"Unsloth: ExecuTorch's export_llm does not support model type {model_type!r}; export the merged "+ f"16-bit model with optimum-executorch (XNNPACK recipe) using {settings}, to match this QAT recipe."+ )+ return None+ path = None+ if directory is not None and os.path.isdir(directory):+ path = os.path.join(directory, "executorch_export_config.yaml")+ with open(path, "w", encoding = "utf-8") as f:+ f.write(EXECUTORCH_PHONE_DEPLOYMENT_CONFIG)+ print(+ "Unsloth: For ExecuTorch, export the merged 16-bit model with export_llm using "+ "quantization.qmode=8da4w quantization.group_size=32 quantization.use_hqq=false "+ 'quantization.embedding_quantize="8,0" (the defaults, group_size 128 with HQQ, do not match '+ "this QAT recipe)." + (f" Config written to {path}" if path else "")+ )+ return path++@_fsdp2_refuses_save("save_pretrained_torchao")def unsloth_save_pretrained_torchao(self,@@ -7714,6 +7774,8 @@ def unsloth_save_pretrained_torchao(token = get_token()has_qat_config = hasattr(self, "_torchao_config") and self._torchao_config is not None+ qat_scheme = getattr(self._torchao_config, "qat_scheme", None) if has_qat_config else None+ is_peft = isinstance(self, (PeftModelForCausalLM, PeftModel))if torchao_config is not None:if has_qat_config:@@ -7741,6 +7803,16 @@ def unsloth_save_pretrained_torchao(token = token,)+ if torchao_config is None and getattr(self._torchao_config, "_unsloth_builtin_recipe", False):+ # The PEFT path merges and writes the quantized copy to the "-torchao" sibling; the rest write in place.+ # Hub saves get the printed flags only.+ save_directory = str(save_directory)+ _write_executorch_export_config(+ qat_scheme,+ None if push_to_hub else (save_directory + "-torchao" if is_peft else save_directory),+ model_type = getattr(getattr(self, "config", None), "model_type", None),+ )+for _ in range(3):gc.collect()