Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

QAT: fake quantize embeddings under FSDP1 and torch.func.functional_call (#15385)

Daniel Han4bd1155a72cc
diff --git a/tests/test_qat_phone_deployment.py b/tests/test_qat_phone_deployment.py
index 42d7105be64..8e7f08863bd 100644
--- a/tests/test_qat_phone_deployment.py
+++ b/tests/test_qat_phone_deployment.py
@@ -350,6 +350,77 @@ def test_fake_quantized_embedding_forward_compiles_fullgraph():
torch.testing.assert_close(compiled(ids), emb(ids))
+def _expected_lookup_with(emb, weight, ids):
+ with torch.no_grad():
+ w = emb.weight_fake_quantizer(weight)
+ return torch.nn.functional.embedding(ids, w).float() * emb.embed_scale
+
+
+def test_functional_call_plain_tensor_weight_is_fake_quantized():
+ emb = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4").get_input_embeddings()
+ ids = torch.randint(0, 256, (2, 12))
+ weight = emb.weight.detach().clone().requires_grad_()
+ calls = _count_fake_quant_calls(emb)
+ out = torch.func.functional_call(emb, {"weight": weight}, (ids,)).float()
+ assert len(calls) == 1
+ torch.testing.assert_close(out.detach(), _expected_lookup_with(emb, weight, ids))
+ out.sum().backward()
+ assert weight.grad is not None and torch.isfinite(weight.grad).all()
+ assert isinstance(emb.weight, torch.nn.Parameter)
+
+
+def test_nested_fake_quantized_weight_call_quantizes_once():
+ from unsloth.models._utils import _call_with_fake_quantized_weight
+
+ emb = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4").get_input_embeddings()
+ ids = torch.randint(0, 256, (2, 12))
+ calls = _count_fake_quant_calls(emb)
+
+ def lookup():
+ return torch.nn.functional.embedding(ids, emb.weight).float() * emb.embed_scale
+
+ with torch.no_grad():
+ out = _call_with_fake_quantized_weight(emb, _call_with_fake_quantized_weight, emb, lookup)
+ assert len(calls) == 1
+ torch.testing.assert_close(out, _expected_fake_quantized_lookup(emb, ids))
+ assert "_unsloth_weight_fake_quantized" not in vars(emb)
+
+
[email protected]("use_orig_params", [False, True])
+def test_fsdp1_embedding_is_fake_quantized(use_orig_params, tmp_path):
+ import torch.distributed as dist
+
+ if not dist.is_available() or not dist.is_gloo_available():
+ pytest.skip("requires torch.distributed with gloo")
+ if dist.is_initialized():
+ pytest.skip("a process group is already initialized")
+ from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+
+ emb = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4").get_input_embeddings()
+ ids = torch.randint(0, 256, (2, 12))
+ with torch.no_grad():
+ expected = emb(ids).float()
+ dist.init_process_group(
+ "gloo", store = dist.FileStore(str(tmp_path / "store"), 1), rank = 0, world_size = 1
+ )
+ try:
+ wrapped = FSDP(emb, device_id = torch.device("cpu"), use_orig_params = use_orig_params)
+ calls = _count_fake_quant_calls(emb)
+ seen = []
+ emb.register_forward_pre_hook(
+ lambda m, a: seen.append(isinstance(m.weight, torch.nn.Parameter))
+ )
+ out = wrapped(ids).float()
+ out.sum().backward()
+ grads = [p.grad for p in wrapped.parameters() if p.requires_grad]
+ finally:
+ dist.destroy_process_group()
+ assert seen == [False]
+ assert len(calls) == 1
+ torch.testing.assert_close(out.detach(), expected)
+ assert grads and all(g is not None and torch.isfinite(g).all() for g in grads)
+
+
def _offload_embedding(model, **hook_kwargs):
# What cpu_offload / dispatch_model install: the real weight in weights_map, the parameter on meta between calls.
from accelerate.hooks import AlignDevicesHook, add_hook_to_module
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 18939cd37ab..59c2f3c22d2 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -8227,15 +8227,27 @@ def _copy_embedding_subclass_state(dst: torch.nn.Module, src: torch.nn.Module) -
def _call_with_fake_quantized_weight(module: torch.nn.Module, fn, *args, **kwargs):
"""Run fn with module.weight swapped for its fake quantized value, so a class forward that reads self.weight
directly (unsloth_zoo's float32 Gemma forward) trains against the quantized weight.
- Re-entrant: inside the scope weight is a plain tensor, not the Parameter, so it is quantized once."""
+ Re-entrant via a flag, not the weight's type: FSDP1 / functional_call install plain tensors that still need quantizing."""
fake_quantizer = getattr(module, "weight_fake_quantizer", None)
- weight = module.weight
- if fake_quantizer is None or not isinstance(weight, torch.nn.Parameter):
+ if fake_quantizer is None or vars(module).get("_unsloth_weight_fake_quantized", False):
return fn(*args, **kwargs)
from torch.nn.utils.stateless import _reparametrize_module
- with _reparametrize_module(module, {"weight": fake_quantizer(weight)}):
- return fn(*args, **kwargs)
+ weight = module.weight
+ fake_quantized = fake_quantizer(weight)
+ module._unsloth_weight_fake_quantized = True
+ try:
+ if "weight" in vars(module):
+ # FSDP1's view in the instance dict shadows the registered Parameter; swap the view itself.
+ vars(module)["weight"] = fake_quantized
+ try:
+ return fn(*args, **kwargs)
+ finally:
+ vars(module)["weight"] = weight
+ with _reparametrize_module(module, {"weight": fake_quantized}):
+ return fn(*args, **kwargs)
+ finally:
+ del module._unsloth_weight_fake_quantized
def _is_accelerate_forward(module: torch.nn.Module, forward) -> bool: