Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
QAT: fake quantize embeddings under FSDP1 and torch.func.functional_call (#15385)
Daniel Han4bd1155a72cc
diff --git a/tests/test_qat_phone_deployment.py b/tests/test_qat_phone_deployment.pyindex 42d7105be64..8e7f08863bd 100644--- a/tests/test_qat_phone_deployment.py+++ b/tests/test_qat_phone_deployment.py@@ -350,6 +350,77 @@ def test_fake_quantized_embedding_forward_compiles_fullgraph():torch.testing.assert_close(compiled(ids), emb(ids))+def _expected_lookup_with(emb, weight, ids):+ with torch.no_grad():+ w = emb.weight_fake_quantizer(weight)+ return torch.nn.functional.embedding(ids, w).float() * emb.embed_scale+++def test_functional_call_plain_tensor_weight_is_fake_quantized():+ emb = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4").get_input_embeddings()+ ids = torch.randint(0, 256, (2, 12))+ weight = emb.weight.detach().clone().requires_grad_()+ calls = _count_fake_quant_calls(emb)+ out = torch.func.functional_call(emb, {"weight": weight}, (ids,)).float()+ assert len(calls) == 1+ torch.testing.assert_close(out.detach(), _expected_lookup_with(emb, weight, ids))+ out.sum().backward()+ assert weight.grad is not None and torch.isfinite(weight.grad).all()+ assert isinstance(emb.weight, torch.nn.Parameter)+++def test_nested_fake_quantized_weight_call_quantizes_once():+ from unsloth.models._utils import _call_with_fake_quantized_weight++ emb = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4").get_input_embeddings()+ ids = torch.randint(0, 256, (2, 12))+ calls = _count_fake_quant_calls(emb)++ def lookup():+ return torch.nn.functional.embedding(ids, emb.weight).float() * emb.embed_scale++ with torch.no_grad():+ out = _call_with_fake_quantized_weight(emb, _call_with_fake_quantized_weight, emb, lookup)+ assert len(calls) == 1+ torch.testing.assert_close(out, _expected_fake_quantized_lookup(emb, ids))+ assert "_unsloth_weight_fake_quantized" not in vars(emb)++[email protected]("use_orig_params", [False, True])+def test_fsdp1_embedding_is_fake_quantized(use_orig_params, tmp_path):+ import torch.distributed as dist++ if not dist.is_available() or not dist.is_gloo_available():+ pytest.skip("requires torch.distributed with gloo")+ if dist.is_initialized():+ pytest.skip("a process group is already initialized")+ from torch.distributed.fsdp import FullyShardedDataParallel as FSDP++ emb = _prepare_model_for_qat(_tiny_gemma3(), "int8-int4").get_input_embeddings()+ ids = torch.randint(0, 256, (2, 12))+ with torch.no_grad():+ expected = emb(ids).float()+ dist.init_process_group(+ "gloo", store = dist.FileStore(str(tmp_path / "store"), 1), rank = 0, world_size = 1+ )+ try:+ wrapped = FSDP(emb, device_id = torch.device("cpu"), use_orig_params = use_orig_params)+ calls = _count_fake_quant_calls(emb)+ seen = []+ emb.register_forward_pre_hook(+ lambda m, a: seen.append(isinstance(m.weight, torch.nn.Parameter))+ )+ out = wrapped(ids).float()+ out.sum().backward()+ grads = [p.grad for p in wrapped.parameters() if p.requires_grad]+ finally:+ dist.destroy_process_group()+ assert seen == [False]+ assert len(calls) == 1+ torch.testing.assert_close(out.detach(), expected)+ assert grads and all(g is not None and torch.isfinite(g).all() for g in grads)++def _offload_embedding(model, **hook_kwargs):# What cpu_offload / dispatch_model install: the real weight in weights_map, the parameter on meta between calls.from accelerate.hooks import AlignDevicesHook, add_hook_to_modulediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 18939cd37ab..59c2f3c22d2 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -8227,15 +8227,27 @@ def _copy_embedding_subclass_state(dst: torch.nn.Module, src: torch.nn.Module) -def _call_with_fake_quantized_weight(module: torch.nn.Module, fn, *args, **kwargs):"""Run fn with module.weight swapped for its fake quantized value, so a class forward that reads self.weightdirectly (unsloth_zoo's float32 Gemma forward) trains against the quantized weight.- Re-entrant: inside the scope weight is a plain tensor, not the Parameter, so it is quantized once."""+ Re-entrant via a flag, not the weight's type: FSDP1 / functional_call install plain tensors that still need quantizing."""fake_quantizer = getattr(module, "weight_fake_quantizer", None)- weight = module.weight- if fake_quantizer is None or not isinstance(weight, torch.nn.Parameter):+ if fake_quantizer is None or vars(module).get("_unsloth_weight_fake_quantized", False):return fn(*args, **kwargs)from torch.nn.utils.stateless import _reparametrize_module- with _reparametrize_module(module, {"weight": fake_quantizer(weight)}):- return fn(*args, **kwargs)+ weight = module.weight+ fake_quantized = fake_quantizer(weight)+ module._unsloth_weight_fake_quantized = True+ try:+ if "weight" in vars(module):+ # FSDP1's view in the instance dict shadows the registered Parameter; swap the view itself.+ vars(module)["weight"] = fake_quantized+ try:+ return fn(*args, **kwargs)+ finally:+ vars(module)["weight"] = weight+ with _reparametrize_module(module, {"weight": fake_quantized}):+ return fn(*args, **kwargs)+ finally:+ del module._unsloth_weight_fake_quantizeddef _is_accelerate_forward(module: torch.nn.Module, forward) -> bool: