Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix phone-deployment QAT for Gemma3 and LoRA, and write the matching ExecuTorch export config (#14376)

Daniel Han7c0e7d2d3d09
diff --git a/tests/saving/test_qat_torchao_save.py b/tests/saving/test_qat_torchao_save.py
index c33c3dcd227..712e0c9eb60 100644
--- a/tests/saving/test_qat_torchao_save.py
+++ b/tests/saving/test_qat_torchao_save.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import importlib.util
+import os
import pytest
import torch
@@ -213,7 +214,8 @@ def test_int8_int4_lora_rank_below_group_size():
_prepare_model_for_qat(peft_model, "int8-int4")
block = peft_model.base_model.model.layers[0]
assert hasattr(block.q_proj.base_layer, "weight_fake_quantizer")
- assert hasattr(block.q_proj.lora_A["default"], "weight_fake_quantizer")
+ # Only the frozen base weight is fake-quantized (torchtune QATLoRALinear); adapters merge before quantization.
+ assert not hasattr(block.q_proj.lora_A["default"], "weight_fake_quantizer")
assert not hasattr(block.q_proj.lora_B["default"], "weight_fake_quantizer")
lm_head = peft_model.base_model.model.lm_head
assert _peft_untied_lm_head(peft_model) == ("lm_head.weight", lm_head.weight)
@@ -266,3 +268,70 @@ def test_untie_merged_checkpoint(tmp_path, layout):
shard = tmp_path / name
with safe_open(str(shard), "pt") as h:
assert torch.equal(h.get_tensor("thinker.lm_head.weight"), lm_head)
+
+
[email protected]("peft", [True, False], ids = ["peft", "full"])
[email protected]("qat_scheme", ["int8-int4", "int8"])
+def test_phone_deployment_save_writes_executorch_config(monkeypatch, tmp_path, qat_scheme, peft):
+ target = tmp_path / "out"
+ # A "-torchao" sibling left by an earlier LoRA export must not attract a full model's config.
+ os.makedirs(str(target) + "-torchao")
+
+ class _PeftStub(_Model):
+ pass
+
+ def fake_save(model, save_directory, **kw):
+ os.makedirs(save_directory + "-torchao" if peft else save_directory, exist_ok = True)
+
+ monkeypatch.setattr(save_mod, "_unsloth_save_torchao_with_attached_config", fake_save)
+ monkeypatch.setattr(save_mod, "PeftModel", _PeftStub)
+ monkeypatch.setattr(save_mod, "PeftModelForCausalLM", _PeftStub)
+ model = _PeftStub() if peft else _Model()
+ model._torchao_config = _int8_qat_config()
+ model._torchao_config.qat_scheme = qat_scheme
+ model._torchao_config._unsloth_builtin_recipe = True
+ save_mod.unsloth_save_pretrained_torchao(model, str(target), tokenizer = object())
+ path = tmp_path / ("out-torchao" if peft else "out") / "executorch_export_config.yaml"
+ stale = tmp_path / ("out" if peft else "out-torchao") / "executorch_export_config.yaml"
+ assert not stale.exists()
+ if qat_scheme != "int8-int4":
+ assert not path.exists()
+ return
+ import yaml
+
+ config = yaml.safe_load(path.read_text())
+ assert config["quantization"] == {
+ "qmode": "8da4w",
+ "group_size": 32,
+ "use_hqq": False,
+ "embedding_quantize": "8,0",
+ }
+ assert config["backend"]["xnnpack"]["enabled"] is True
+
+
[email protected]("case", ["custom_recipe", "gemma3", "hub"])
+def test_phone_deployment_config_only_where_it_applies(monkeypatch, tmp_path, capsys, case):
+ target = tmp_path / "out"
+
+ def fake_save(model, save_directory, **kw):
+ os.makedirs(save_directory, exist_ok = True)
+
+ monkeypatch.setattr(save_mod, "_unsloth_save_torchao_with_attached_config", fake_save)
+ model = _Model()
+ model._torchao_config = _int8_qat_config()
+ model._torchao_config.qat_scheme = "int8-int4"
+ if case != "custom_recipe":
+ # A recipe Unsloth built from the scheme name; a custom TorchAOConfig keeps only the label.
+ model._torchao_config._unsloth_builtin_recipe = True
+ if case == "gemma3":
+ model.config.model_type = "gemma3_text"
+ kwargs = dict(push_to_hub = True, token = "x") if case == "hub" else {}
+ save_mod.unsloth_save_pretrained_torchao(model, str(target), tokenizer = object(), **kwargs)
+ assert not (target / "executorch_export_config.yaml").exists()
+ printed = capsys.readouterr().out
+ if case == "custom_recipe":
+ assert "ExecuTorch" not in printed
+ elif case == "gemma3":
+ assert "optimum-executorch" in printed and "group size 32" in printed
+ else:
+ assert "group_size=32" in printed
diff --git a/tests/test_qat_phone_deployment.py b/tests/test_qat_phone_deployment.py
new file mode 100644
index 00000000000..b339eaff7ed
--- /dev/null
+++ b/tests/test_qat_phone_deployment.py
@@ -0,0 +1,265 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""QAT scheme construction on tiny CPU models."""
+
+from __future__ import annotations
+
+import importlib.util
+import warnings
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(
+ importlib.util.find_spec("torchao") is None, reason = "require torchao"
+)
+
+from unsloth.models._utils import (
+ TorchAOConfig,
+ _convert_torchao_model,
+ _prepare_model_for_qat,
+)
+
+
+def _tiny_gemma3():
+ from transformers import Gemma3ForCausalLM, Gemma3TextConfig
+
+ torch.manual_seed(0)
+ config = Gemma3TextConfig(
+ vocab_size = 256,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ tie_word_embeddings = True,
+ )
+ return Gemma3ForCausalLM(config).eval()
+
+
+def _tiny_qwen3():
+ from transformers import Qwen3Config, Qwen3ForCausalLM
+
+ torch.manual_seed(0)
+ config = Qwen3Config(
+ vocab_size = 256,
+ hidden_size = 128,
+ intermediate_size = 256,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ )
+ return Qwen3ForCausalLM(config).eval()
+
+
+def _is_fake_quantized(module):
+ return hasattr(module, "weight_fake_quantizer")
+
+
+def test_phone_deployment_alias_in_prepare():
+ model = _prepare_model_for_qat(_tiny_qwen3(), "phone-deployment")
+ assert model._torchao_config.qat_scheme == "int8-int4"
+ assert _is_fake_quantized(model.model.layers[0].self_attn.q_proj)
+ assert _is_fake_quantized(model.model.embed_tokens)
+
+
+def test_gemma3_int8_int4_keeps_phone_scheme():
+ from torchao.quantization import Int8DynamicActivationIntxWeightConfig
+
+ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ assert model._torchao_config.qat_scheme == "int8-int4"
+ configs = [c for c, _ in model._torchao_config.base_config_and_filter_fns]
+ assert any(isinstance(c, Int8DynamicActivationIntxWeightConfig) for c in configs)
+ assert _is_fake_quantized(model.model.layers[0].self_attn.q_proj)
+
+
+def _int8_embedding_config():
+ from torchao.quantization import IntxWeightOnlyConfig
+ from torchao.quantization.granularity import PerAxis
+ return TorchAOConfig(
+ qat_scheme = None,
+ base_config_and_filter_fns = [
+ (
+ IntxWeightOnlyConfig(weight_dtype = torch.int8, granularity = PerAxis(0)),
+ lambda m, _: isinstance(m, torch.nn.Embedding),
+ )
+ ],
+ )
+
+
[email protected]("scheme", ["int8-int4", "custom"])
[email protected]("convert", ["none", "unsloth", "manual"])
+def test_gemma3_fake_quantized_embedding_keeps_embed_scale(scheme, convert):
+ from torchao.quantization import quantize_
+ from torchao.quantization.qat import QATConfig
+ from transformers.models.gemma3.modeling_gemma3 import Gemma3TextScaledWordEmbedding
+
+ model = _tiny_gemma3()
+ ids = torch.randint(0, model.config.vocab_size, (2, 12))
+ with torch.no_grad():
+ ref = model.get_input_embeddings()(ids)
+ assert float(model.get_input_embeddings().embed_scale) == 8.0
+
+ model = _prepare_model_for_qat(
+ model, "int8-int4" if scheme == "int8-int4" else _int8_embedding_config()
+ )
+ emb = model.get_input_embeddings()
+ assert _is_fake_quantized(emb)
+ if convert == "unsloth":
+ _convert_torchao_model(model)
+ elif convert == "manual":
+ quantize_(model, QATConfig(step = "convert"))
+ emb = model.get_input_embeddings()
+ assert isinstance(emb, Gemma3TextScaledWordEmbedding)
+ with torch.no_grad():
+ out = emb(ids)
+ # int8 per-row error is ~0.4% of the row max; a lost sqrt(hidden_size) = 8x scale is not.
+ rel = (out - ref).abs().max() / ref.abs().max()
+ assert rel < 0.02, float(rel)
+
+
+def test_int4_skips_linears_not_divisible_by_group_size():
+ class _M(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.ok = torch.nn.Linear(256, 64, bias = False)
+ self.odd = torch.nn.Linear(192, 64, bias = False)
+
+ def forward(self, x):
+ return self.ok(torch.nn.functional.pad(x, (0, 64))) + self.odd(x)
+
+ model = _M().to(torch.bfloat16)
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ model = _prepare_model_for_qat(model, "int4")
+ assert _is_fake_quantized(model.ok)
+ assert not _is_fake_quantized(model.odd)
+ assert any("odd" in str(w.message) for w in caught)
+ model(torch.randn(2, 192, dtype = torch.bfloat16))
+
+
[email protected]("qat_scheme", ["int8-int4", "int4", "int8", "fp8-fp8"])
+def test_lora_adapters_are_not_fake_quantized(qat_scheme, monkeypatch):
+ from peft import LoraConfig, get_peft_model
+
+ monkeypatch.delenv("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", raising = False)
+ model = get_peft_model(
+ _tiny_qwen3(),
+ LoraConfig(r = 32, lora_alpha = 32, target_modules = ["q_proj", "o_proj"]),
+ )
+ model = _prepare_model_for_qat(model, qat_scheme)
+ q_proj = model.base_model.model.model.layers[0].self_attn.q_proj
+ assert _is_fake_quantized(q_proj.base_layer)
+ assert not _is_fake_quantized(q_proj.lora_A["default"])
+ assert not _is_fake_quantized(q_proj.lora_B["default"])
+
+
+def test_lora_fake_quant_opt_out(monkeypatch):
+ from peft import LoraConfig, get_peft_model
+
+ monkeypatch.setenv("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", "1")
+ model = get_peft_model(
+ _tiny_qwen3(), LoraConfig(r = 32, lora_alpha = 32, target_modules = ["q_proj"])
+ )
+ model = _prepare_model_for_qat(model, "int8-int4")
+ q_proj = model.base_model.model.model.layers[0].self_attn.q_proj
+ assert _is_fake_quantized(q_proj.lora_A["default"])
+
+
+class _EoiEmbedding(torch.nn.Embedding):
+ # T5Gemma2TextScaledWordEmbedding pattern: an extra registered parameter read in forward.
+ def __init__(self, num, dim):
+ super().__init__(num, dim)
+ self.eoi_embedding = torch.nn.Parameter(torch.ones(dim))
+
+ def forward(self, input_ids):
+ out = super().forward(input_ids)
+ out[input_ids == 0] = self.eoi_embedding.to(out.dtype)
+ return out
+
+
[email protected]("convert", [False, True])
+def test_embedding_subclass_keeps_extra_parameters(convert):
+ model = _tiny_gemma3()
+ model.model.embed_tokens = _EoiEmbedding(model.config.vocab_size, model.config.hidden_size)
+ eoi = model.model.embed_tokens.eoi_embedding
+ model = _prepare_model_for_qat(model, "int8-int4")
+ if convert:
+ _convert_torchao_model(model)
+ emb = model.get_input_embeddings()
+ assert isinstance(emb, _EoiEmbedding)
+ assert emb.eoi_embedding is eoi
+ with torch.no_grad():
+ out = emb(torch.tensor([[0, 1, 2]]))
+ assert torch.equal(out[0, 0], eoi)
+
+
+def test_compiled_fake_quantized_embedding_pickles():
+ import pickle
+
+ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ emb = model.get_input_embeddings()
+ emb.compile()
+ restored = pickle.loads(pickle.dumps(emb))
+ assert type(restored) is type(emb)
+ assert "_compiled_call_impl" not in vars(restored) or restored._compiled_call_impl is None
+
+
+def test_fake_quantized_embedding_subclass_pickles():
+ import io
+ import pickle
+
+ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ emb = model.get_input_embeddings()
+ assert _is_fake_quantized(emb)
+ restored = pickle.loads(pickle.dumps(emb))
+ assert type(restored) is type(emb)
+ buf = io.BytesIO()
+ torch.save(emb, buf)
+ buf.seek(0)
+ assert type(torch.load(buf, weights_only = False)) is type(emb)
+
+
+def test_fake_quantized_embedding_unpickles_in_fresh_process(tmp_path):
+ import os
+ import pickle
+ import subprocess
+ import sys
+
+ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ emb = model.get_input_embeddings()
+ path = tmp_path / "emb.pkl"
+ path.write_bytes(pickle.dumps(emb))
+ ids = torch.tensor([[1, 2, 3]])
+ expected = emb(ids)
+ code = (
+ "import pickle, sys, torch\n"
+ f"emb = pickle.loads(open({str(path)!r}, 'rb').read())\n"
+ "print(type(emb).__name__, float(emb.embed_scale))\n"
+ "torch.save(emb(torch.tensor([[1, 2, 3]])), sys.argv[1])\n"
+ )
+ out_path = tmp_path / "out.pt"
+ env = dict(os.environ)
+ env["PYTHONPATH"] = os.pathsep.join([os.getcwd(), env.get("PYTHONPATH", "")])
+ res = subprocess.run(
+ [sys.executable, "-c", code, str(out_path)],
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ env = env,
+ )
+ assert res.returncode == 0, res.stderr[-3000:]
+ assert res.stdout.split()[-2] == type(emb).__name__
+ assert float(res.stdout.split()[-1]) == float(emb.embed_scale)
+ assert torch.equal(torch.load(out_path), expected)
+
+
+def test_builtin_recipe_flag_only_for_scheme_names():
+ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ assert model._torchao_config._unsloth_builtin_recipe is True
+ custom = _int8_embedding_config()
+ model = _prepare_model_for_qat(_tiny_gemma3(), custom)
+ assert not getattr(model._torchao_config, "_unsloth_builtin_recipe", False)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index d611b52fce8..5ccab86768b 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -142,7 +142,7 @@ __all__ = [
import torch
-from typing import Union, Optional, List, Any, Callable, Tuple, Iterator
+from typing import Union, Optional, List, Any, Callable, Dict, Tuple, Iterator
from platform import system as platform_system
platform_system = platform_system()
@@ -7524,7 +7524,9 @@ class TorchAOConfig:
default_factory = lambda: [
(
Int4WeightOnlyConfig(group_size = 128),
- lambda m, _: isinstance(m, torch.nn.Linear) and getattr(m, "in_features", 0) >= 128,
+ lambda m, _: isinstance(m, torch.nn.Linear)
+ and getattr(m, "in_features", 0) >= 128
+ and m.in_features % 128 == 0,
),
]
)
@@ -7562,6 +7564,104 @@ def _untie_input_output_embeddings(model: torch.nn.Module) -> None:
assert out_proj.weight.data_ptr() != in_emb.weight.data_ptr(), "Embeddings still tied!"
+def _embedding_subclasses(model: torch.nn.Module) -> Dict[str, torch.nn.Module]:
+ """nn.Embedding subclasses with their own forward (Gemma3TextScaledWordEmbedding scales by embed_scale)."""
+ try:
+ from torchao.quantization.qat.embedding import FakeQuantizedEmbedding
+ except ImportError:
+ return {}
+ return {
+ fqn: module
+ for fqn, module in model.named_modules()
+ if isinstance(module, torch.nn.Embedding)
+ and type(module) is not torch.nn.Embedding
+ and not isinstance(module, FakeQuantizedEmbedding)
+ }
+
+
+def _copy_embedding_subclass_state(dst: torch.nn.Module, src: torch.nn.Module) -> None:
+ """Copy the parameters, submodules, buffers and plain attributes an Embedding subclass adds (e.g.
+ Gemma3's embed_scale, T5Gemma2's eoi_embedding) onto dst."""
+ for name, param in src._parameters.items():
+ if name not in dst._parameters:
+ dst.register_parameter(name, param)
+ for name, module in src._modules.items():
+ if name not in dst._modules:
+ dst.add_module(name, module)
+ for name, buffer in src._buffers.items():
+ if name not in dst._buffers:
+ dst.register_buffer(
+ name, buffer, persistent = name not in src._non_persistent_buffers_set
+ )
+ for name, value in vars(src).items():
+ if name.startswith("_") or name in vars(dst) or isinstance(value, torch.nn.Module):
+ continue
+ setattr(dst, name, value)
+
+
+_FAKE_QUANTIZED_EMBEDDING_CLASSES = {}
+
+
+def _fake_quantized_embedding_class(original_cls):
+ """FakeQuantizedEmbedding that keeps original_cls.forward: its super().forward() is the fake-quantized lookup."""
+ from torchao.quantization.qat.embedding import FakeQuantizedEmbedding
+
+ cls = _FAKE_QUANTIZED_EMBEDDING_CLASSES.get(original_cls)
+ if cls is not None:
+ return cls
+
+ def to_embedding(self):
+ # QATConfig(step = "convert") swaps back to a plain nn.Embedding; restore the subclass and its scale.
+ embedding = FakeQuantizedEmbedding.to_embedding(self)
+ embedding.__class__ = original_cls
+ _copy_embedding_subclass_state(embedding, self)
+ return embedding
+
+ def __reduce_ex__(self, protocol):
+ # Pickle by the original class, so a fresh process (torch.load, spawn workers) rebuilds this one.
+ return (_rebuild_fake_quantized_embedding, (original_cls,), self.__getstate__())
+
+ name = "_FakeQuantized_" + re.sub(
+ r"\W", "_", f"{original_cls.__module__}.{original_cls.__qualname__}"
+ )
+ cls = type(
+ name,
+ (original_cls, FakeQuantizedEmbedding),
+ {
+ "to_embedding": to_embedding,
+ "__reduce_ex__": __reduce_ex__,
+ "__module__": __name__,
+ "__qualname__": name,
+ },
+ )
+ _FAKE_QUANTIZED_EMBEDDING_CLASSES[original_cls] = cls
+ return cls
+
+
+def _rebuild_fake_quantized_embedding(original_cls):
+ cls = _fake_quantized_embedding_class(original_cls)
+ return cls.__new__(cls)
+
+
+def _restore_fake_quantized_embedding_subclasses(
+ model: torch.nn.Module, originals: Dict[str, torch.nn.Module]
+) -> None:
+ """QAT prepare swaps in a plain FakeQuantizedEmbedding, dropping e.g. Gemma3's embed_scale; restore it."""
+ if not originals:
+ return
+ from torchao.quantization.qat.embedding import FakeQuantizedEmbedding
+
+ for fqn, original in originals.items():
+ try:
+ module = model.get_submodule(fqn)
+ except AttributeError:
+ continue
+ if type(module) is not FakeQuantizedEmbedding:
+ continue
+ module.__class__ = _fake_quantized_embedding_class(type(original))
+ _copy_embedding_subclass_state(module, original)
+
+
def _filter_fn_to_fqns(
model: torch.nn.Module, filter_fn: Callable[[torch.nn.Module, str], bool]
) -> Iterator[str]:
@@ -7730,6 +7830,54 @@ def _convert_torchao_model(model):
model.config.quantization_config = quantization_config
+def _group_size_linear_filter(model, group_size, qat_scheme):
+ """PerGroup filter: in_features not divisible by group_size breaks the fake quantizer's reshape."""
+ filter_fn = lambda m, _: (
+ isinstance(m, torch.nn.Linear)
+ and m.in_features >= group_size
+ and m.in_features % group_size == 0
+ )
+ skipped = [
+ name
+ for name, module in model.named_modules()
+ if isinstance(module, torch.nn.Linear)
+ and module.in_features >= group_size
+ and module.in_features % group_size != 0
+ and not _is_lora_adapter_name(name)
+ ]
+ if skipped:
+ warnings.warn(
+ f"Unsloth: qat_scheme='{qat_scheme}' uses group_size {group_size}; these Linear layers have "
+ f"in_features not divisible by {group_size} and stay in full precision: "
+ + ", ".join(skipped[:8])
+ + (f", ... ({len(skipped) - 8} more)" if len(skipped) > 8 else ""),
+ stacklevel = 3,
+ )
+ return filter_fn
+
+
+def _is_lora_adapter_name(fqn: str) -> bool:
+ return any(part.startswith("lora_") for part in fqn.split("."))
+
+
+def _exclude_lora_adapters_from_qat(model, torchao_config) -> None:
+ """Adapters are merged before quantization, so only base weights are fake-quantized."""
+ if os.environ.get("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", "0") == "1":
+ return
+ if not any(_is_lora_adapter_name(name) for name, _ in model.named_modules()):
+ return
+ from torchao.quantization.quant_api import _is_linear
+
+ def _skip_lora(filter_fn):
+ filter_fn = filter_fn or _is_linear
+ return lambda m, fqn: not _is_lora_adapter_name(fqn) and filter_fn(m, fqn)
+
+ torchao_config.base_config_and_filter_fns = [
+ (base_config, _skip_lora(filter_fn))
+ for base_config, filter_fn in torchao_config.base_config_and_filter_fns
+ ]
+
+
def _prepare_model_for_qat(
model: torch.nn.Module, qat_scheme: Union[str, TorchAOConfig]
) -> torch.nn.Module:
@@ -7741,16 +7889,9 @@ def _prepare_model_for_qat(
except ImportError:
raise ImportError(TORCHAO_MSG)
- # Gemma3 int8 embedding quantization struggles with its 262144-token vocabulary, so auto-switch to int4 weight-only.
- if qat_scheme == "int8-int4":
- model_types = get_transformers_model_type(model.config)
- is_gemma3 = any("gemma3" in mt or "gemma_3" in mt for mt in model_types)
- if is_gemma3:
- print(
- "Unsloth: Gemma3 has a large vocabulary causing int8 embedding issues. "
- "Switching to int4 weight-only QAT for training stability."
- )
- qat_scheme = "int4"
+ # get_peft_model passes the scheme straight here, unaliased.
+ if qat_scheme == "phone-deployment":
+ qat_scheme = "int8-int4"
if not isinstance(qat_scheme, TorchAOConfig):
torchao_config: Optional[TorchAOConfig] = None
@@ -7761,7 +7902,7 @@ def _prepare_model_for_qat(
raise ImportError(TORCHAO_MSG)
group_size = 128
base_config = Float8DynamicActivationInt4WeightConfig()
- filter_fn = lambda m, _: isinstance(m, torch.nn.Linear) and m.in_features >= group_size
+ filter_fn = _group_size_linear_filter(model, group_size, qat_scheme)
torchao_config = TorchAOConfig(
qat_scheme = qat_scheme,
base_config_and_filter_fns = [(base_config, filter_fn)],
@@ -7828,7 +7969,7 @@ def _prepare_model_for_qat(
raise ImportError(TORCHAO_MSG)
group_size = 128
base_config = Int4WeightOnlyConfig(group_size = group_size)
- filter_fn = lambda m, _: isinstance(m, torch.nn.Linear) and m.in_features >= group_size
+ filter_fn = _group_size_linear_filter(model, group_size, qat_scheme)
torchao_config = TorchAOConfig(
qat_scheme = qat_scheme,
base_config_and_filter_fns = [(base_config, filter_fn)],
@@ -7892,6 +8033,9 @@ def _prepare_model_for_qat(
else:
raise ValueError(f"Unexpected QAT scheme {qat_scheme}")
assert torchao_config is not None, f"TorchAOConfig was not set for {qat_scheme}"
+ # Built here from the scheme name, so save-time export hints may assume its exact recipe.
+ torchao_config._unsloth_builtin_recipe = True
+ _exclude_lora_adapters_from_qat(model, torchao_config)
else:
torchao_config = qat_scheme
@@ -7903,8 +8047,10 @@ def _prepare_model_for_qat(
if torchao_config.prequantization_transform is not None:
torchao_config.prequantization_transform(model)
+ embedding_subclasses = _embedding_subclasses(model)
for base_config, filter_fn in torchao_config.base_config_and_filter_fns:
quantize_(model, QATConfig(base_config, step = "prepare"), filter_fn = filter_fn)
+ _restore_fake_quantized_embedding_subclasses(model, embedding_subclasses)
return model
diff --git a/unsloth/save.py b/unsloth/save.py
index c1ba0e0aa37..6145797cb31 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -7692,6 +7692,66 @@ def _unsloth_save_torchao(
clean_gpu_cache()
+# Must match qat_scheme="int8-int4"; export_llm defaults (group_size 128, use_hqq true) do not.
+EXECUTORCH_PHONE_DEPLOYMENT_CONFIG = """\
+# ExecuTorch export_llm config matching Unsloth qat_scheme="phone-deployment" (int8-int4).
+# Export from a merged 16-bit checkpoint (model.save_pretrained_merged(dir, tokenizer)), converted to
+# ExecuTorch's format, e.g. for Qwen3: python -m examples.models.qwen3.convert_weights <dir> model.pth
+# Then: python -m extension.llm.export.export_llm --config this_file.yaml \\
+# +base.model_class=<e.g. qwen3_0_6b> +base.params=<model params json> +base.checkpoint=model.pth
+model:
+ use_kv_cache: True
+ use_sdpa_with_kv_cache: True
+ dtype_override: fp32
+quantization:
+ qmode: 8da4w
+ group_size: 32
+ use_hqq: False
+ embedding_quantize: "8,0"
+backend:
+ xnnpack:
+ enabled: True
+ extended_ops: True
+"""
+
+
+# HF model_types whose architecture ExecuTorch's export_llm ModelType covers (llama2/3.x, smollm2, qwen2.5, qwen3,
+# qwen3.5, phi-4-mini, lfm2); anything else (Gemma3, ...) goes through optimum-executorch instead.
+_EXPORT_LLM_MODEL_TYPES = {"llama", "qwen2", "qwen3", "qwen3_5", "qwen3_5_text", "phi3", "lfm2"}
+
+
+def _write_executorch_export_config(
+ qat_scheme,
+ directory,
+ model_type = None,
+):
+ """Write and print the ExecuTorch export settings matching a phone-deployment QAT model."""
+ if qat_scheme != "int8-int4":
+ return None
+ settings = (
+ "8da4w linear (dynamic int8 activations, int4 weights, group size 32, affine, not HQQ) and 8-bit "
+ "per-channel embeddings"
+ )
+ if model_type not in _EXPORT_LLM_MODEL_TYPES:
+ print(
+ f"Unsloth: ExecuTorch's export_llm does not support model type {model_type!r}; export the merged "
+ f"16-bit model with optimum-executorch (XNNPACK recipe) using {settings}, to match this QAT recipe."
+ )
+ return None
+ path = None
+ if directory is not None and os.path.isdir(directory):
+ path = os.path.join(directory, "executorch_export_config.yaml")
+ with open(path, "w", encoding = "utf-8") as f:
+ f.write(EXECUTORCH_PHONE_DEPLOYMENT_CONFIG)
+ print(
+ "Unsloth: For ExecuTorch, export the merged 16-bit model with export_llm using "
+ "quantization.qmode=8da4w quantization.group_size=32 quantization.use_hqq=false "
+ 'quantization.embedding_quantize="8,0" (the defaults, group_size 128 with HQQ, do not match '
+ "this QAT recipe)." + (f" Config written to {path}" if path else "")
+ )
+ return path
+
+
@_fsdp2_refuses_save("save_pretrained_torchao")
def unsloth_save_pretrained_torchao(
self,
@@ -7714,6 +7774,8 @@ def unsloth_save_pretrained_torchao(
token = get_token()
has_qat_config = hasattr(self, "_torchao_config") and self._torchao_config is not None
+ qat_scheme = getattr(self._torchao_config, "qat_scheme", None) if has_qat_config else None
+ is_peft = isinstance(self, (PeftModelForCausalLM, PeftModel))
if torchao_config is not None:
if has_qat_config:
@@ -7741,6 +7803,16 @@ def unsloth_save_pretrained_torchao(
token = token,
)
+ if torchao_config is None and getattr(self._torchao_config, "_unsloth_builtin_recipe", False):
+ # The PEFT path merges and writes the quantized copy to the "-torchao" sibling; the rest write in place.
+ # Hub saves get the printed flags only.
+ save_directory = str(save_directory)
+ _write_executorch_export_config(
+ qat_scheme,
+ None if push_to_hub else (save_directory + "-torchao" if is_peft else save_directory),
+ model_type = getattr(getattr(self, "config", None), "model_type", None),
+ )
+
for _ in range(3):
gc.collect()