Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
NVFP4 QAT: accept fp16 autocast and non-contiguous activations (#14780)
Daniel Hand08356c07d34
diff --git a/tests/test_qat_microscaling_schemes.py b/tests/test_qat_microscaling_schemes.pyindex 9ab7885965c..17ab0943bd1 100644--- a/tests/test_qat_microscaling_schemes.py+++ b/tests/test_qat_microscaling_schemes.py@@ -358,3 +358,56 @@ def test_nvfp4_accepts_rank4_inputs(monkeypatch):for shape in [(6, 64), (2, 3, 64), (2, 3, 5, 64)]:out = model(torch.randn(*shape, dtype = torch.bfloat16))assert out.shape == (*shape[:-1], 128)+++def _torchao_nvfp4_contract_forward(self, x):+ # What torchao's NVFP4 QAT forward accepts: nvfp4_quantize asserts these, then the output follows x.+ assert x.dim() == 2, x.shape+ assert x.dtype in (torch.bfloat16, torch.float32), f"{x.dtype} not supported"+ assert x.is_contiguous(), "Only support contiguous data for now"+ return (x.float() @ self.weight.float().t()).to(x.dtype)+++def test_nvfp4_accepts_strided_inputs(monkeypatch):+ _probe_ok(monkeypatch)+ model = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = False)).to(torch.bfloat16)+ ref = model[0].weight.detach().clone()+ model = _prepare_model_for_qat(model, "nvfp4")+ monkeypatch.setattr(_nvfp4_linear(), "forward", _torchao_nvfp4_contract_forward)+ base = torch.randn(4, 6, 128, dtype = torch.bfloat16)+ for x in [base[0, :, ::2], base[:, :, :64], base.transpose(0, 1)[..., 64:].unsqueeze(0)]:+ assert not x.is_contiguous()+ out = model(x)+ assert out.shape == (*x.shape[:-1], 128) and out.dtype == torch.bfloat16+ expected = (x.float() @ ref.float().t()).to(torch.bfloat16)+ torch.testing.assert_close(out, expected)++[email protected]("weight_dtype", [torch.bfloat16, torch.float32])+def test_nvfp4_takes_float16_autocast_activations(monkeypatch, weight_dtype):+ # Linear(63, 64) stays plain (63 is not a multiple of 16) and emits float16 under autocast.+ _probe_ok(monkeypatch)+ torch.manual_seed(0)+ model = torch.nn.Sequential(+ torch.nn.Linear(63, 64, bias = False), torch.nn.Linear(64, 128, bias = False)+ ).to(weight_dtype)+ model = _prepare_model_for_qat(model, "nvfp4")+ assert not isinstance(model[0], _nvfp4_linear()) and isinstance(model[1], _nvfp4_linear())+ monkeypatch.setattr(_nvfp4_linear(), "forward", _torchao_nvfp4_contract_forward)+ x = torch.randn(2, 3, 63, dtype = weight_dtype, requires_grad = True)+ with torch.autocast("cpu", dtype = torch.float16):+ out = model(x)+ assert out.dtype == torch.float16 and out.shape == (2, 3, 128)+ out.float().sum().backward()+ assert x.grad is not None and model[1].weight.grad is not None+++def test_nvfp4_float16_activations_under_bfloat16_autocast_return_bfloat16(monkeypatch):+ _probe_ok(monkeypatch)+ model = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = False)).to(torch.bfloat16)+ model = _prepare_model_for_qat(model, "nvfp4")+ monkeypatch.setattr(_nvfp4_linear(), "forward", _torchao_nvfp4_contract_forward)+ x = torch.randn(2, 3, 64, dtype = torch.float16)+ with torch.autocast("cpu", dtype = torch.bfloat16):+ out = model(x)+ assert out.dtype == torch.bfloat16 and out.shape == (2, 3, 128)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex aa56b77e1da..3bfdf35175a 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -8744,13 +8744,24 @@ def _mx_autocast_forward(self, x):def _nvfp4_any_rank_forward(self, x):- # torchao's NVFP4FakeQuantizedLinear flattens only rank-3 inputs; vision layers see rank 4+.+ # torchao's NVFP4FakeQuantizedLinear flattens only rank-3 inputs (vision layers see rank 4+), and+ # its quantizer asserts contiguous bfloat16 / float32 activations (nvfp4_quantize).from torchao.prototype.qat import NVFP4FakeQuantizedLinear- if x.dim() == 2:- return NVFP4FakeQuantizedLinear.forward(self, x)- out = NVFP4FakeQuantizedLinear.forward(self, x.reshape(-1, x.shape[-1]))- return out.view(*x.shape[:-1], out.shape[-1])+ out_dtype = None+ # float16 autocast hands over float16 activations (an earlier Linear / SDPA); prepare only+ # admits bfloat16 / float32 weights, so quantize from the weight dtype and return what an+ # autocast Linear would: the autocast dtype, not the input's.+ if x.dtype == torch.float16 and torch.is_autocast_enabled(x.device.type):+ out_dtype = torch.get_autocast_dtype(x.device.type)+ x = x.to(self.weight.dtype)+ x2 = x if x.dim() == 2 else x.reshape(-1, x.shape[-1])+ if not x2.is_contiguous():+ x2 = x2.contiguous()+ out = NVFP4FakeQuantizedLinear.forward(self, x2)+ if x.dim() != 2:+ out = out.view(*x.shape[:-1], out.shape[-1])+ return out if out_dtype is None or out.dtype == out_dtype else out.to(out_dtype)def _patch_microscaling_qat_forwards(model) -> None: