Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

QAT: fake quantize Gemma embeddings on the float32 and Accelerate-hooked paths (#14702)

Daniel Han8e8eb3d5d04c
diff --git a/tests/test_qat_phone_deployment.py b/tests/test_qat_phone_deployment.py
index 635bb463a0f..b1b61eb62aa 100644
--- a/tests/test_qat_phone_deployment.py
+++ b/tests/test_qat_phone_deployment.py
@@ -270,3 +270,402 @@ def test_builtin_recipe_flag_only_for_scheme_names():
custom = _int8_embedding_config()
model = _prepare_model_for_qat(_tiny_gemma3(), custom)
assert not getattr(model._torchao_config, "_unsloth_builtin_recipe", False)
+
+
+def _count_fake_quant_calls(emb):
+ calls = []
+ emb.weight_fake_quantizer.register_forward_hook(lambda *args: calls.append(1))
+ return calls
+
+
+def _expected_fake_quantized_lookup(emb, ids):
+ # FakeQuantizedEmbedding's own lookup, then Gemma3's float32 embed_scale multiply.
+ with torch.no_grad():
+ w = emb.weight_fake_quantizer(emb.weight)
+ return torch.nn.functional.embedding(ids, w).float() * emb.embed_scale
+
+
+def _direct_weight_forward(self, input_ids):
+ # unsloth_zoo's UNSLOTH_FORCE_FLOAT32 Gemma3 / Gemma4 embedding forward: reads self.weight, no super().
+ embeds = torch.nn.functional.embedding(
+ input_ids, weight = self.weight, padding_idx = self.padding_idx
+ )
+ return embeds.to(torch.float32) * self.embed_scale
+
+
[email protected]("forward", ["upstream", "direct_weight"])
+def test_gemma3_embedding_forward_is_fake_quantized_once(forward, monkeypatch):
+ from transformers.models.gemma3.modeling_gemma3 import Gemma3TextScaledWordEmbedding
+
+ if forward == "direct_weight":
+ monkeypatch.setattr(Gemma3TextScaledWordEmbedding, "forward", _direct_weight_forward)
+ model = _tiny_gemma3()
+ ids = torch.randint(0, model.config.vocab_size, (2, 12))
+ with torch.no_grad():
+ unquantized = model.get_input_embeddings()(ids).float()
+ model = _prepare_model_for_qat(model, "int8-int4")
+ emb = model.get_input_embeddings()
+ calls = _count_fake_quant_calls(emb)
+ with torch.no_grad():
+ out = emb(ids).float()
+ assert len(calls) == 1
+ assert not torch.equal(out, unquantized)
+ torch.testing.assert_close(out, _expected_fake_quantized_lookup(emb, ids))
+ # The swapped-in weight is scoped to the call.
+ assert isinstance(emb.weight, torch.nn.Parameter)
+
+
+def test_accelerate_hooked_embedding_runs_fake_quantized_forward():
+ accelerate_hooks = pytest.importorskip("accelerate.hooks")
+
+ model = _tiny_gemma3()
+ original = model.get_input_embeddings()
+ ids = torch.randint(0, model.config.vocab_size, (2, 12))
+ with torch.no_grad():
+ unquantized = original(ids).float()
+ accelerate_hooks.add_hook_to_module(
+ original, accelerate_hooks.AlignDevicesHook(execution_device = "cpu", io_same_device = True)
+ )
+ model = _prepare_model_for_qat(model, "int8-int4")
+ emb = model.get_input_embeddings()
+ assert emb is not original
+ # The hook moved onto the replacement and wraps its forward, not the old module's.
+ assert hasattr(emb, "_hf_hook")
+ assert getattr(emb.forward, "args", ()) != (original,)
+ calls = _count_fake_quant_calls(emb)
+ with torch.no_grad():
+ out = emb(ids).float()
+ assert len(calls) == 1
+ assert not torch.equal(out, unquantized)
+ torch.testing.assert_close(out, _expected_fake_quantized_lookup(emb, ids))
+
+
+def test_fake_quantized_embedding_forward_compiles_fullgraph():
+ model = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ emb = model.get_input_embeddings()
+ ids = torch.randint(0, model.config.vocab_size, (2, 12))
+ torch._dynamo.reset()
+ compiled = torch.compile(emb, backend = "eager", fullgraph = True)
+ with torch.no_grad():
+ torch.testing.assert_close(compiled(ids), emb(ids))
+
+
+def _offload_embedding(model, **hook_kwargs):
+ # What cpu_offload / dispatch_model install: the real weight in weights_map, the parameter on meta between calls.
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+ from accelerate.utils import PrefixedDataset
+
+ prefix = "model.embed_tokens."
+ weights = {k: v.clone() for k, v in model.state_dict().items() if k.startswith(prefix)}
+ add_hook_to_module(
+ model.get_input_embeddings(),
+ AlignDevicesHook(
+ execution_device = "cpu",
+ offload = True,
+ weights_map = PrefixedDataset(weights, prefix),
+ **hook_kwargs,
+ ),
+ )
+ assert model.get_input_embeddings().weight.device.type == "meta"
+ return model
+
+
+def test_offloaded_embedding_is_fake_quantized():
+ pytest.importorskip("accelerate.hooks")
+ ids = torch.randint(0, 256, (2, 12))
+ reference = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4").get_input_embeddings()
+ with torch.no_grad():
+ expected = reference(ids).float()
+
+ model = _prepare_model_for_qat(_offload_embedding(_tiny_gemma3()), "int8-int4")
+ emb = model.get_input_embeddings()
+ assert getattr(emb._hf_hook, "offload", False)
+ calls = _count_fake_quant_calls(emb)
+ with torch.no_grad():
+ out = emb(ids).float()
+ assert out.device.type == "cpu"
+ assert len(calls) == 1
+ torch.testing.assert_close(out, expected)
+ assert emb.weight.device.type == "meta" # offloaded again after the call
+
+
+def test_offloaded_embedding_hook_that_cannot_move_is_refused():
+ pytest.importorskip("accelerate.hooks")
+ model = _offload_embedding(_tiny_gemma3(), place_submodules = True)
+ original = model.get_input_embeddings()
+ with pytest.raises(RuntimeError, match = "offloaded embedding"):
+ _prepare_model_for_qat(model, "int8-int4")
+ assert model.get_input_embeddings() is original
+
+
[email protected]("convert", ["unsloth", "strip"])
+def test_offloaded_embedding_converts_to_a_real_embedding(convert):
+ pytest.importorskip("accelerate.hooks")
+ from torchao.utils import TorchAOBaseTensor
+ from unsloth.models._utils import _strip_torchao_fake_quantization
+
+ ids = torch.randint(0, 256, (2, 12))
+ reference = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ model = _prepare_model_for_qat(_offload_embedding(_tiny_gemma3()), "int8-int4")
+ if convert == "unsloth":
+ _convert_torchao_model(reference)
+ _convert_torchao_model(model)
+ else:
+ _strip_torchao_fake_quantization(reference)
+ _strip_torchao_fake_quantization(model)
+ emb = model.get_input_embeddings()
+ assert emb.weight.device.type == "cpu"
+ assert not hasattr(emb, "_hf_hook")
+ assert isinstance(emb.weight, TorchAOBaseTensor) == (convert == "unsloth")
+ with torch.no_grad():
+ torch.testing.assert_close(emb(ids), reference.get_input_embeddings()(ids))
+
+
+def test_offloaded_embedding_converts_on_cpu_not_the_execution_device():
+ pytest.importorskip("accelerate.hooks")
+ model = _prepare_model_for_qat(_offload_embedding(_tiny_gemma3()), "int8-int4")
+ emb = model.get_input_embeddings()
+ # A real offload runs on a GPU; conversion must not pull the whole table onto it.
+ emb._hf_hook.execution_device = torch.device("cuda", 7)
+ _convert_torchao_model(model)
+ emb = model.get_input_embeddings()
+ assert emb.weight.device.type == "cpu"
+ assert all(b.device.type == "cpu" for b in emb.buffers())
+
+
[email protected]("qat_scheme", ["int4", "int8"])
+def test_offload_hook_on_unselected_embedding_is_not_refused(qat_scheme):
+ pytest.importorskip("accelerate.hooks")
+ model = _offload_embedding(_tiny_gemma3(), place_submodules = True)
+ original = model.get_input_embeddings()
+ model = _prepare_model_for_qat(model, qat_scheme) # Linear-only recipes never touch the embedding
+ assert model.get_input_embeddings() is original
+ assert _is_fake_quantized(model.model.layers[0].mlp.down_proj) # in_features 128 fits int4's group
+
+
[email protected]("convert", ["unsloth", "strip"])
+def test_offloaded_embedding_with_offloaded_buffers_converts(convert):
+ # offload_buffers = True on a meta-initialized module: weight AND embed_scale live only in weights_map.
+ pytest.importorskip("accelerate.hooks")
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+ from accelerate.utils import PrefixedDataset
+ from unsloth.models._utils import _strip_torchao_fake_quantization
+
+ ids = torch.randint(0, 256, (2, 12))
+ reference = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ model = _tiny_gemma3()
+ emb = model.get_input_embeddings()
+ scale = emb.embed_scale.detach().clone()
+ emb.register_buffer("embed_scale", scale.clone(), persistent = True)
+ prefix = "model.embed_tokens."
+ weights = {prefix + "weight": emb.weight.detach().clone(), prefix + "embed_scale": scale.clone()}
+ emb.to("meta")
+ add_hook_to_module(
+ emb,
+ AlignDevicesHook(
+ execution_device = "cpu",
+ offload = True,
+ offload_buffers = True,
+ weights_map = PrefixedDataset(weights, prefix),
+ ),
+ )
+ assert emb.embed_scale.device.type == "meta"
+ model = _prepare_model_for_qat(model, "int8-int4")
+ with torch.no_grad(): # the hook still serves forward from weights_map
+ torch.testing.assert_close(model.get_input_embeddings()(ids), reference.get_input_embeddings()(ids))
+ if convert == "unsloth":
+ _convert_torchao_model(reference)
+ _convert_torchao_model(model)
+ else:
+ _strip_torchao_fake_quantization(reference)
+ _strip_torchao_fake_quantization(model)
+ emb = model.get_input_embeddings()
+ assert not hasattr(emb, "_hf_hook")
+ assert emb.weight.device.type == "cpu"
+ assert emb.embed_scale.device.type == "cpu"
+ assert float(emb.embed_scale) == float(scale)
+ with torch.no_grad():
+ torch.testing.assert_close(emb(ids), reference.get_input_embeddings()(ids))
+
+
+def _direct_weight_override(self, input_ids):
+ return torch.nn.functional.embedding(input_ids, self.weight).float() * self.embed_scale
+
+
+def _delegating_override(self, input_ids):
+ return type(self).forward(self, input_ids)
+
+
+def _install_override(emb, kind):
+ import types
+
+ if kind == "direct_weight":
+ emb.forward = types.MethodType(_direct_weight_override, emb)
+ elif kind == "delegating":
+ emb.forward = types.MethodType(_delegating_override, emb)
+ elif kind == "accelerate_over_override":
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+
+ emb.forward = types.MethodType(_delegating_override, emb)
+ add_hook_to_module(emb, AlignDevicesHook(execution_device = "cpu", io_same_device = True))
+ elif kind == "forward_hook":
+ return emb.register_forward_hook(lambda m, a, out: out * 2)
+ elif kind == "unsloth_offload_hooks":
+ from unsloth.models.vision import _install_offload_embedding_hooks
+
+ assert _install_offload_embedding_hooks(emb, None, torch.device("cpu"))
+ return None
+
+
+ "kind",
+ ["direct_weight", "delegating", "accelerate_over_override", "forward_hook", "unsloth_offload_hooks"],
+)
[email protected]("convert", ["unsloth", "strip"])
+def test_embedding_with_instance_forward_or_hooks_is_left_out_of_qat(kind, convert):
+ # Nothing is rebound: the embedding keeps its module, override and hooks, and stays full precision throughout.
+ pytest.importorskip("accelerate.hooks")
+ from torchao.utils import TorchAOBaseTensor
+ from unsloth.models._utils import _strip_torchao_fake_quantization
+
+ model = _tiny_gemma3()
+ original = model.get_input_embeddings()
+ handle = _install_override(original, kind)
+ ids = torch.randint(0, model.config.vocab_size, (2, 12))
+ with torch.no_grad():
+ before = original(ids)
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ model = _prepare_model_for_qat(model, "int8-int4")
+ emb = model.get_input_embeddings()
+ assert emb is original and not _is_fake_quantized(emb)
+ assert sum("leaves these embeddings in full precision" in str(w.message) for w in caught) == 1
+ assert _is_fake_quantized(model.model.layers[0].self_attn.q_proj) # the rest of the recipe still applies
+ with torch.no_grad():
+ torch.testing.assert_close(emb(ids), before)
+ if handle is not None:
+ handle.remove() # the caller's handle still controls the live module
+ with torch.no_grad():
+ torch.testing.assert_close(emb(ids) * 2, before)
+ if convert == "unsloth":
+ _convert_torchao_model(model)
+ else:
+ _strip_torchao_fake_quantization(model)
+ assert model.get_input_embeddings() is original
+ assert not isinstance(original.weight, TorchAOBaseTensor) # saved as trained: unquantized
+
+
[email protected]("offload", [False, True])
+def test_sequential_hook_with_align_devices_hook_is_supported(offload):
+ accelerate_hooks = pytest.importorskip("accelerate.hooks")
+ from accelerate.utils import PrefixedDataset
+ from torchao.utils import TorchAOBaseTensor
+
+ ids = torch.randint(0, 256, (2, 12))
+ reference = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4")
+ with torch.no_grad():
+ expected = reference.get_input_embeddings()(ids)
+ model = _tiny_gemma3()
+ emb = model.get_input_embeddings()
+ kwargs = {"execution_device": "cpu", "offload": offload}
+ if offload:
+ prefix = "model.embed_tokens."
+ kwargs["weights_map"] = PrefixedDataset({prefix + "weight": emb.weight.detach().clone()}, prefix)
+ accelerate_hooks.add_hook_to_module(emb, accelerate_hooks.AlignDevicesHook(**kwargs))
+ accelerate_hooks.add_hook_to_module(emb, accelerate_hooks.ModelHook(), append = True)
+ assert type(emb._hf_hook).__name__ == "SequentialHook"
+ model = _prepare_model_for_qat(model, "int8-int4")
+ emb = model.get_input_embeddings()
+ assert _is_fake_quantized(emb) and type(emb._hf_hook).__name__ == "SequentialHook"
+ calls = _count_fake_quant_calls(emb)
+ with torch.no_grad():
+ torch.testing.assert_close(emb(ids), expected)
+ assert len(calls) == 1
+ _convert_torchao_model(reference)
+ _convert_torchao_model(model)
+ emb = model.get_input_embeddings()
+ assert emb.weight.device.type == "cpu" and isinstance(emb.weight, TorchAOBaseTensor)
+ with torch.no_grad():
+ torch.testing.assert_close(emb(ids), reference.get_input_embeddings()(ids))
+
+
+def test_two_offload_hooks_are_left_out_of_qat():
+ accelerate_hooks = pytest.importorskip("accelerate.hooks")
+ model = _tiny_gemma3()
+ emb = model.get_input_embeddings()
+ from accelerate.utils import PrefixedDataset
+
+ weights = {"w.weight": emb.weight.detach().clone()}
+ for append in (False, True):
+ hook = accelerate_hooks.AlignDevicesHook(
+ execution_device = "cpu", offload = True, weights_map = PrefixedDataset(weights, "w.")
+ )
+ accelerate_hooks.add_hook_to_module(emb, hook, append = append)
+ with pytest.warns(UserWarning, match = "2 AlignDevicesHooks"):
+ model = _prepare_model_for_qat(model, "int8-int4")
+ assert model.get_input_embeddings() is emb and not _is_fake_quantized(emb)
+
+
[email protected]("offload", [False, True])
+def test_plain_embedding_keeps_its_accelerate_hook(offload):
+ accelerate_hooks = pytest.importorskip("accelerate.hooks")
+ from torchao.quantization.qat.embedding import FakeQuantizedEmbedding
+ from torchao.utils import TorchAOBaseTensor
+
+ ids = torch.randint(0, 256, (2, 12))
+ reference = _prepare_model_for_qat(_tiny_qwen3(), "int8-int4")
+ with torch.no_grad():
+ expected = reference.get_input_embeddings()(ids)
+ model = _tiny_qwen3()
+ emb = model.get_input_embeddings()
+ assert type(emb) is torch.nn.Embedding
+ accelerate_hooks.add_hook_to_module(
+ emb, accelerate_hooks.AlignDevicesHook(execution_device = "cpu", offload = offload, io_same_device = True)
+ )
+ model = _prepare_model_for_qat(model, "int8-int4")
+ emb = model.get_input_embeddings()
+ assert type(emb) is FakeQuantizedEmbedding and hasattr(emb, "_hf_hook")
+ calls = _count_fake_quant_calls(emb)
+ with torch.no_grad():
+ torch.testing.assert_close(emb(ids), expected)
+ assert len(calls) == 1
+ _convert_torchao_model(reference)
+ _convert_torchao_model(model)
+ emb = model.get_input_embeddings()
+ assert emb.weight.device.type == "cpu" and isinstance(emb.weight, TorchAOBaseTensor)
+
+
+def test_input_require_grads_hook_follows_the_qat_embedding():
+ # transformers' enable_input_require_grads hook is not a reason to skip QAT; it is re-registered on the swap.
+ model = _tiny_gemma3()
+ model.enable_input_require_grads()
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ model = _prepare_model_for_qat(model, "int8-int4")
+ assert not any("leaves these embeddings" in str(w.message) for w in caught)
+ emb = model.get_input_embeddings()
+ assert _is_fake_quantized(emb)
+ ids = torch.randint(0, model.config.vocab_size, (2, 12))
+ for p in emb.parameters():
+ p.requires_grad_(False)
+ assert emb(ids).requires_grad
+ model.disable_input_require_grads() # the owner's handles control the live embedding
+ assert not emb(ids).requires_grad
+
+
+def test_peft_require_grad_hook_follows_the_qat_embedding():
+ # PEFT's fallback hook (no handle kept) on a model without enable_input_require_grads.
+ model = _tiny_gemma3()
+ emb = model.get_input_embeddings()
+
+ def make_inputs_require_grad(module, input, output):
+ output.requires_grad_(True)
+
+ emb.register_forward_hook(make_inputs_require_grad)
+ model = _prepare_model_for_qat(model, "int8-int4")
+ emb = model.get_input_embeddings()
+ assert _is_fake_quantized(emb)
+ assert list(emb._forward_hooks.values()) == [make_inputs_require_grad]
+ for p in emb.parameters():
+ p.requires_grad_(False)
+ assert emb(torch.tensor([[1, 2, 3]])).requires_grad
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 4e3af86f9ea..b232a2d999d 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -7572,7 +7572,8 @@ def _untie_input_output_embeddings(model: torch.nn.Module) -> None:
def _embedding_subclasses(model: torch.nn.Module) -> Dict[str, torch.nn.Module]:
- """nn.Embedding subclasses with their own forward (Gemma3TextScaledWordEmbedding scales by embed_scale)."""
+ """Embeddings QAT must restore after torchao's swap: nn.Embedding subclasses with their own forward
+ (Gemma3TextScaledWordEmbedding scales by embed_scale), and plain embeddings carrying an Accelerate hook."""
try:
from torchao.quantization.qat.embedding import FakeQuantizedEmbedding
except ImportError:
@@ -7581,19 +7582,22 @@ def _embedding_subclasses(model: torch.nn.Module) -> Dict[str, torch.nn.Module]:
fqn: module
for fqn, module in model.named_modules()
if isinstance(module, torch.nn.Embedding)
- and type(module) is not torch.nn.Embedding
and not isinstance(module, FakeQuantizedEmbedding)
+ and (type(module) is not torch.nn.Embedding or hasattr(module, "_hf_hook"))
}
def _copy_embedding_subclass_state(dst: torch.nn.Module, src: torch.nn.Module) -> None:
"""Copy the parameters, submodules, buffers and plain attributes an Embedding subclass adds (e.g.
- Gemma3's embed_scale, T5Gemma2's eoi_embedding) onto dst."""
+ Gemma3's embed_scale, T5Gemma2's eoi_embedding) onto dst, then move src's Accelerate hook.
+ The instance `forward` is never copied: the only one QAT admits is Accelerate's wrapper, which is bound to src
+ and is re-created on dst by _move_dispatch_hook (see _qat_embedding_unsupported)."""
for name, param in src._parameters.items():
if name not in dst._parameters:
dst.register_parameter(name, param)
for name, module in src._modules.items():
- if name not in dst._modules:
+ # weight_fake_quantizer is FakeQuantizedEmbedding's own; a converted plain embedding must not carry it.
+ if name not in dst._modules and name != "weight_fake_quantizer":
dst.add_module(name, module)
for name, buffer in src._buffers.items():
if name not in dst._buffers:
@@ -7601,16 +7605,301 @@ def _copy_embedding_subclass_state(dst: torch.nn.Module, src: torch.nn.Module) -
name, buffer, persistent = name not in src._non_persistent_buffers_set
)
for name, value in vars(src).items():
- if name.startswith("_") or name in vars(dst) or isinstance(value, torch.nn.Module):
+ if (
+ name.startswith("_")
+ or name == "forward"
+ or name in vars(dst)
+ or isinstance(value, torch.nn.Module)
+ ):
continue
setattr(dst, name, value)
+ _move_dispatch_hook(dst, src)
+
+
+def _call_with_fake_quantized_weight(module: torch.nn.Module, fn, *args, **kwargs):
+ """Run fn with module.weight swapped for its fake quantized value, so a class forward that reads self.weight
+ directly (unsloth_zoo's float32 Gemma forward) trains against the quantized weight.
+ Re-entrant: inside the scope weight is a plain tensor, not the Parameter, so it is quantized once."""
+ fake_quantizer = getattr(module, "weight_fake_quantizer", None)
+ weight = module.weight
+ if fake_quantizer is None or not isinstance(weight, torch.nn.Parameter):
+ return fn(*args, **kwargs)
+ from torch.nn.utils.stateless import _reparametrize_module
+
+ with _reparametrize_module(module, {"weight": fake_quantizer(weight)}):
+ return fn(*args, **kwargs)
+
+
+def _is_accelerate_forward(module: torch.nn.Module, forward) -> bool:
+ """Accelerate's add_hook_to_module wrapper: a partial of new_forward over module, the hooked forward in _old_forward."""
+ return (
+ isinstance(forward, functools.partial)
+ and bool(forward.args)
+ and forward.args[0] is module
+ and "_old_forward" in vars(module)
+ )
+
+
+def _is_class_forward(module: torch.nn.Module, forward) -> bool:
+ import types
+
+ return (
+ isinstance(forward, types.MethodType)
+ and forward.__self__ is module
+ and forward.__func__ is getattr(type(module).forward, "__func__", type(module).forward)
+ )
+
+
+def _is_require_grads_hook(hook) -> bool:
+ """transformers' enable_input_require_grads hook (make_inputs_require_grads, handles kept on the model) or
+ PEFT's fallback (make_inputs_require_grad, no handle kept). Both are re-registered after the QAT swap."""
+ return getattr(hook, "__name__", None) in ("make_inputs_require_grads", "make_inputs_require_grad")
+
+
+def _unowned_require_grads_hooks(originals: Dict[str, torch.nn.Module]):
+ """PEFT's make_inputs_require_grad hooks per embedding: nobody holds their handle, so they can be re-registered."""
+ return {
+ fqn: [h for h in module._forward_hooks.values() if getattr(h, "__name__", None) == "make_inputs_require_grad"]
+ for fqn, module in originals.items()
+ }
+
+
+def _reregister_unowned_require_grads_hooks(model: torch.nn.Module, originals, hooks_by_fqn) -> None:
+ for fqn, hooks in hooks_by_fqn.items():
+ if not hooks:
+ continue
+ try:
+ module = model.get_submodule(fqn)
+ except AttributeError:
+ continue
+ if module is originals[fqn]:
+ continue # not swapped: its hooks are still there
+ for hook in hooks:
+ module.register_forward_hook(hook)
+
+
+def _require_grads_owners(model: torch.nn.Module):
+ return [
+ m
+ for m in model.modules()
+ if getattr(m, "_require_grads_hooks", None) or hasattr(m, "_require_grads_hook")
+ ]
+
+
+def _reinstall_input_require_grads(owners) -> None:
+ """Re-run enable_input_require_grads on the models that had it, so the hook lands on the QAT embedding
+ and the owner's handles point at it (disable first: no duplicate on an embedding QAT left alone)."""
+ for owner in owners:
+ try:
+ owner.disable_input_require_grads()
+ owner.enable_input_require_grads()
+ except Exception as exc:
+ warnings.warn(
+ f"Unsloth: could not re-enable input require grads after QAT ({type(exc).__name__}: {exc}).",
+ RuntimeWarning,
+ stacklevel = 3,
+ )
+
+
+def _flat_accelerate_hooks(hook):
+ """The hooks inside an Accelerate _hf_hook: itself, or a SequentialHook's children (nested ones flattened)."""
+ if hook is None:
+ return []
+ children = getattr(hook, "hooks", None)
+ if type(hook).__name__ == "SequentialHook" and children is not None:
+ return [h for child in children for h in _flat_accelerate_hooks(child)]
+ return [hook]
+
+
+def _offload_hook(module: torch.nn.Module):
+ """The offloading AlignDevicesHook on module, also inside a SequentialHook, else None."""
+ hooks = [
+ h for h in _flat_accelerate_hooks(getattr(module, "_hf_hook", None)) if getattr(h, "offload", False)
+ ]
+ return hooks[0] if hooks else None
+
+
+def _qat_embedding_unsupported(module: torch.nn.Module):
+ """Why QAT cannot swap this embedding without losing behaviour, else None. Supported: no instance forward, or
+ only Accelerate's wrapper over the class forward, with at most one AlignDevicesHook (also in a SequentialHook)
+ and no forward pre / post hooks. Anything else would have to be rebound onto the replacement, so it is not."""
+ if len(module._forward_pre_hooks) or any(
+ not _is_require_grads_hook(hook) for hook in module._forward_hooks.values()
+ ):
+ return "it has forward hooks"
+ forward = vars(module).get("forward")
+ if forward is not None:
+ if not _is_accelerate_forward(module, forward):
+ return "it has an instance-level forward override"
+ if not _is_class_forward(module, vars(module)["_old_forward"]):
+ return "Accelerate's hook wraps an instance-level forward override"
+ hooks = _flat_accelerate_hooks(getattr(module, "_hf_hook", None))
+ align = [h for h in hooks if type(h).__name__ == "AlignDevicesHook"]
+ if len(align) > 1:
+ return f"it has {len(align)} AlignDevicesHooks"
+ offloading = [h for h in hooks if getattr(h, "offload", False)]
+ if offloading and type(offloading[0]).__name__ != "AlignDevicesHook":
+ return f"its offload hook is a {type(offloading[0]).__name__}, not an AlignDevicesHook"
+ return None
+
+
+class _SkipExcludedEmbeddings:
+ """A QAT filter that leaves embeddings marked _unsloth_qat_excluded out (prepare, convert and save alike)."""
+
+ def __init__(self, filter_fn):
+ self.filter_fn = filter_fn
+
+ def __call__(self, module, fqn):
+ return not getattr(module, "_unsloth_qat_excluded", False) and self.filter_fn(module, fqn)
+
+
+def _exclude_unsupported_qat_embeddings(model: torch.nn.Module, torchao_config) -> None:
+ """Leave out of QAT, with one warning, each selected embedding _qat_embedding_unsupported rejects."""
+ configs = torchao_config.base_config_and_filter_fns
+ filters = [f for _, f in configs if f is not None]
+ if not filters:
+ return # a None filter selects Linears only
+ excluded = []
+ for fqn, module in model.named_modules():
+ if not isinstance(module, torch.nn.Embedding) or not any(f(module, fqn) for f in filters):
+ continue
+ reason = _qat_embedding_unsupported(module)
+ if reason is not None:
+ module._unsloth_qat_excluded = True
+ excluded.append(f"{fqn} ({reason})")
+ if not excluded:
+ return
+ torchao_config.base_config_and_filter_fns = [
+ (c, f if f is None or isinstance(f, _SkipExcludedEmbeddings) else _SkipExcludedEmbeddings(f))
+ for c, f in configs
+ ]
+ warnings.warn(
+ "Unsloth: QAT leaves these embeddings in full precision (they are trained and saved unquantized), "
+ "since swapping them would drop behaviour QAT cannot carry over: " + ", ".join(excluded),
+ stacklevel = 3,
+ )
+
+
+def _materialize_offloaded_fake_quantized_embeddings(model: torch.nn.Module) -> None:
+ """QAT convert / strip read embedding weights directly, not through forward: give an offloaded fake quantized
+ embedding its real weight on the CPU (where offload keeps it; the execution GPU may not fit it) and drop the
+ offload hook, so conversion sees real data."""
+ for module in list(model.modules()):
+ hook = _offload_hook(module)
+ if not (
+ isinstance(module, torch.nn.Embedding)
+ and hasattr(module, "weight_fake_quantizer")
+ and hook is not None
+ and getattr(hook, "weights_map", None) is not None
+ ):
+ continue
+ from accelerate.hooks import remove_hook_from_module
+ from accelerate.utils import set_module_tensor_to_device
+
+ weights_map = hook.weights_map
+ values = {n: weights_map[n] for n, _ in module.named_parameters(recurse = False)}
+ # offload_buffers = True keeps buffers (Gemma's embed_scale) in weights_map too, meta on the module.
+ buffers = {}
+ for name, buffer in module.named_buffers(recurse = False):
+ if name in weights_map:
+ buffers[name] = weights_map[name]
+ elif buffer.device.type == "meta":
+ raise RuntimeError(
+ f"Unsloth: cannot convert the offloaded embedding: its buffer {name!r} is on the meta "
+ "device and its offload hook's weights_map has no value for it."
+ )
+ remove_hook_from_module(module) # the whole chain, a SequentialHook included
+ for name, value in values.items():
+ set_module_tensor_to_device(module, name, "cpu", value = value)
+ # Keep the module on one device: offloaded buffers come from weights_map, the rest sat on the GPU.
+ for name, buffer in list(module.named_buffers(recurse = False)):
+ if name in buffers:
+ set_module_tensor_to_device(module, name, "cpu", value = buffers[name])
+ elif buffer.device.type != "cpu":
+ set_module_tensor_to_device(module, name, "cpu")
+
+
+def _offload_hook_problem(module: torch.nn.Module):
+ """Why an Accelerate offload hook on module cannot be moved onto its fake quantized replacement, else None."""
+ hook = _offload_hook(module)
+ if hook is None:
+ return None
+ if type(hook).__name__ != "AlignDevicesHook":
+ return f"its offload hook is a {type(hook).__name__}, not an AlignDevicesHook"
+ if getattr(hook, "weights_map", None) is None:
+ return "its offload hook has no weights_map"
+ if getattr(hook, "place_submodules", False):
+ return "its offload hook also places submodules"
+ missing = [n for n, _ in module.named_parameters(recurse = False) if n not in hook.weights_map]
+ if missing:
+ return f"its offload weights_map has no {missing}"
+ return None
+
+
+def _check_offloaded_embeddings(originals: Dict[str, torch.nn.Module], base_config_and_filter_fns) -> None:
+ """Refuse, before QAT replaces anything, an offloaded embedding selected for QAT whose real weight could not
+ follow it. A filter of None selects Linears only, so it never selects an embedding."""
+ filters = [f for _, f in base_config_and_filter_fns if f is not None]
+ for fqn, module in originals.items():
+ if not any(f(module, fqn) for f in filters):
+ continue
+ problem = _offload_hook_problem(module)
+ if problem is not None:
+ raise RuntimeError(
+ f"Unsloth: QAT cannot fake quantize the CPU / disk offloaded embedding {fqn!r}: {problem}. "
+ "Load the model without offloading the embedding (e.g. device_map = 'auto' with enough GPU "
+ "memory, or keep the embedding on a GPU in device_map)."
+ )
+
+
+def _move_dispatch_hook(dst: torch.nn.Module, src: torch.nn.Module) -> None:
+ """Re-attach src's Accelerate hook (an AlignDevicesHook, or a SequentialHook holding one) to dst, so a split
+ model still aligns devices at dst's own forward. An offload hook moves too: its weights_map holds the real
+ weight, which the hook loads onto dst (meta between calls) before dst's forward fake quantizes it."""
+ hook = getattr(src, "_hf_hook", None)
+ if hook is None or hasattr(dst, "_hf_hook"):
+ return
+ offload_hook = _offload_hook(src)
+ if offload_hook is not None:
+ problem = _offload_hook_problem(src)
+ if problem is not None:
+ raise RuntimeError(
+ f"Unsloth: cannot move the offload hook onto the fake quantized embedding: {problem}."
+ )
+ # init_hook re-reads devices from dst, already meta; keep src's so detaching still restores the weight.
+ original_devices = dict(getattr(offload_hook, "original_devices", {}) or {})
+ try:
+ from accelerate.hooks import add_hook_to_module
+
+ add_hook_to_module(dst, hook)
+ if offload_hook is not None and original_devices:
+ offload_hook.original_devices = original_devices
+ except Exception as exc:
+ if offload_hook is not None:
+ raise RuntimeError(
+ "Unsloth: could not move the offload hook onto the fake quantized embedding "
+ f"({type(exc).__name__}: {exc})."
+ ) from exc
+ warnings.warn(
+ f"Unsloth: could not re-attach a dispatch hook to the fake quantized embedding "
+ f"({type(exc).__name__}: {exc}).",
+ RuntimeWarning,
+ stacklevel = 3,
+ )
+
+
+class _PlainEmbeddingLookup(torch.nn.Embedding):
+ """Sits between an Embedding subclass and FakeQuantizedEmbedding: super().forward() is a plain lookup of the
+ (already fake quantized) weight."""
+
+ forward = torch.nn.Embedding.forward
_FAKE_QUANTIZED_EMBEDDING_CLASSES = {}
def _fake_quantized_embedding_class(original_cls):
- """FakeQuantizedEmbedding that keeps original_cls.forward: its super().forward() is the fake-quantized lookup."""
+ """FakeQuantizedEmbedding that keeps original_cls.forward, run on the fake quantized weight."""
from torchao.quantization.qat.embedding import FakeQuantizedEmbedding
cls = _FAKE_QUANTIZED_EMBEDDING_CLASSES.get(original_cls)
@@ -7628,13 +7917,19 @@ def _fake_quantized_embedding_class(original_cls):
# Pickle by the original class, so a fresh process (torch.load, spawn workers) rebuilds this one.
return (_rebuild_fake_quantized_embedding, (original_cls,), self.__getstate__())
+ def forward(self, *args, **kwargs):
+ # Fake quantize the weight the original forward reads, whether it calls super().forward() (transformers'
+ # Gemma3) or F.embedding(self.weight) directly (unsloth_zoo's float32 Gemma3 / Gemma4 patch).
+ return _call_with_fake_quantized_weight(self, original_cls.forward, self, *args, **kwargs)
+
name = "_FakeQuantized_" + re.sub(
r"\W", "_", f"{original_cls.__module__}.{original_cls.__qualname__}"
)
cls = type(
name,
- (original_cls, FakeQuantizedEmbedding),
+ (original_cls, _PlainEmbeddingLookup, FakeQuantizedEmbedding),
{
+ "forward": forward,
"to_embedding": to_embedding,
"__reduce_ex__": __reduce_ex__,
"__module__": __name__,
@@ -7665,6 +7960,9 @@ def _restore_fake_quantized_embedding_subclasses(
continue
if type(module) is not FakeQuantizedEmbedding:
continue
+ if type(original) is torch.nn.Embedding:
+ _move_dispatch_hook(module, original) # torchao's own class already fake quantizes the lookup
+ continue
module.__class__ = _fake_quantized_embedding_class(type(original))
_copy_embedding_subclass_state(module, original)
@@ -7698,6 +7996,7 @@ def _strip_torchao_fake_quantization(model):
from torchao.quantization import quantize_
from torchao.quantization.qat import QATConfig
+ _materialize_offloaded_fake_quantized_embeddings(model)
# torchao's NVFP4 / MX QAT linears fake-quantize inside forward() and carry no weight_fake_quantizer.
try:
from torchao.prototype import qat as prototype_qat
@@ -7821,6 +8120,7 @@ def _convert_torchao_model(model):
from torchao.quantization.qat import QATConfig
from torchao.utils import TorchAOBaseTensor
+ _materialize_offloaded_fake_quantized_embeddings(model)
module_to_fqn_dict = {}
for base_config, filter_fn in model._torchao_config.base_config_and_filter_fns:
quantize_(model, QATConfig(base_config, step = "convert"), filter_fn = filter_fn)
@@ -8346,10 +8646,19 @@ def _prepare_model_for_qat(
if torchao_config.prequantization_transform is not None:
torchao_config.prequantization_transform(model)
+ _exclude_unsupported_qat_embeddings(model, torchao_config)
+ require_grads_owners = _require_grads_owners(model)
+ all_embeddings = {
+ fqn: m for fqn, m in model.named_modules() if isinstance(m, torch.nn.Embedding)
+ }
+ unowned_require_grads = _unowned_require_grads_hooks(all_embeddings)
embedding_subclasses = _embedding_subclasses(model)
+ _check_offloaded_embeddings(embedding_subclasses, torchao_config.base_config_and_filter_fns)
for base_config, filter_fn in torchao_config.base_config_and_filter_fns:
quantize_(model, QATConfig(base_config, step = "prepare"), filter_fn = filter_fn)
_restore_fake_quantized_embedding_subclasses(model, embedding_subclasses)
+ _reinstall_input_require_grads(require_grads_owners)
+ _reregister_unowned_require_grads_hooks(model, all_embeddings, unowned_require_grads)
_patch_microscaling_qat_forwards(model)
return model