Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

NVFP4 QAT: accept fp16 autocast and non-contiguous activations (#14780)

Daniel Hand08356c07d34
diff --git a/tests/test_qat_microscaling_schemes.py b/tests/test_qat_microscaling_schemes.py
index 9ab7885965c..17ab0943bd1 100644
--- a/tests/test_qat_microscaling_schemes.py
+++ b/tests/test_qat_microscaling_schemes.py
@@ -358,3 +358,56 @@ def test_nvfp4_accepts_rank4_inputs(monkeypatch):
for shape in [(6, 64), (2, 3, 64), (2, 3, 5, 64)]:
out = model(torch.randn(*shape, dtype = torch.bfloat16))
assert out.shape == (*shape[:-1], 128)
+
+
+def _torchao_nvfp4_contract_forward(self, x):
+ # What torchao's NVFP4 QAT forward accepts: nvfp4_quantize asserts these, then the output follows x.
+ assert x.dim() == 2, x.shape
+ assert x.dtype in (torch.bfloat16, torch.float32), f"{x.dtype} not supported"
+ assert x.is_contiguous(), "Only support contiguous data for now"
+ return (x.float() @ self.weight.float().t()).to(x.dtype)
+
+
+def test_nvfp4_accepts_strided_inputs(monkeypatch):
+ _probe_ok(monkeypatch)
+ model = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = False)).to(torch.bfloat16)
+ ref = model[0].weight.detach().clone()
+ model = _prepare_model_for_qat(model, "nvfp4")
+ monkeypatch.setattr(_nvfp4_linear(), "forward", _torchao_nvfp4_contract_forward)
+ base = torch.randn(4, 6, 128, dtype = torch.bfloat16)
+ for x in [base[0, :, ::2], base[:, :, :64], base.transpose(0, 1)[..., 64:].unsqueeze(0)]:
+ assert not x.is_contiguous()
+ out = model(x)
+ assert out.shape == (*x.shape[:-1], 128) and out.dtype == torch.bfloat16
+ expected = (x.float() @ ref.float().t()).to(torch.bfloat16)
+ torch.testing.assert_close(out, expected)
+
+
[email protected]("weight_dtype", [torch.bfloat16, torch.float32])
+def test_nvfp4_takes_float16_autocast_activations(monkeypatch, weight_dtype):
+ # Linear(63, 64) stays plain (63 is not a multiple of 16) and emits float16 under autocast.
+ _probe_ok(monkeypatch)
+ torch.manual_seed(0)
+ model = torch.nn.Sequential(
+ torch.nn.Linear(63, 64, bias = False), torch.nn.Linear(64, 128, bias = False)
+ ).to(weight_dtype)
+ model = _prepare_model_for_qat(model, "nvfp4")
+ assert not isinstance(model[0], _nvfp4_linear()) and isinstance(model[1], _nvfp4_linear())
+ monkeypatch.setattr(_nvfp4_linear(), "forward", _torchao_nvfp4_contract_forward)
+ x = torch.randn(2, 3, 63, dtype = weight_dtype, requires_grad = True)
+ with torch.autocast("cpu", dtype = torch.float16):
+ out = model(x)
+ assert out.dtype == torch.float16 and out.shape == (2, 3, 128)
+ out.float().sum().backward()
+ assert x.grad is not None and model[1].weight.grad is not None
+
+
+def test_nvfp4_float16_activations_under_bfloat16_autocast_return_bfloat16(monkeypatch):
+ _probe_ok(monkeypatch)
+ model = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = False)).to(torch.bfloat16)
+ model = _prepare_model_for_qat(model, "nvfp4")
+ monkeypatch.setattr(_nvfp4_linear(), "forward", _torchao_nvfp4_contract_forward)
+ x = torch.randn(2, 3, 64, dtype = torch.float16)
+ with torch.autocast("cpu", dtype = torch.bfloat16):
+ out = model(x)
+ assert out.dtype == torch.bfloat16 and out.shape == (2, 3, 128)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index aa56b77e1da..3bfdf35175a 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -8744,13 +8744,24 @@ def _mx_autocast_forward(self, x):
def _nvfp4_any_rank_forward(self, x):
- # torchao's NVFP4FakeQuantizedLinear flattens only rank-3 inputs; vision layers see rank 4+.
+ # torchao's NVFP4FakeQuantizedLinear flattens only rank-3 inputs (vision layers see rank 4+), and
+ # its quantizer asserts contiguous bfloat16 / float32 activations (nvfp4_quantize).
from torchao.prototype.qat import NVFP4FakeQuantizedLinear
- if x.dim() == 2:
- return NVFP4FakeQuantizedLinear.forward(self, x)
- out = NVFP4FakeQuantizedLinear.forward(self, x.reshape(-1, x.shape[-1]))
- return out.view(*x.shape[:-1], out.shape[-1])
+ out_dtype = None
+ # float16 autocast hands over float16 activations (an earlier Linear / SDPA); prepare only
+ # admits bfloat16 / float32 weights, so quantize from the weight dtype and return what an
+ # autocast Linear would: the autocast dtype, not the input's.
+ if x.dtype == torch.float16 and torch.is_autocast_enabled(x.device.type):
+ out_dtype = torch.get_autocast_dtype(x.device.type)
+ x = x.to(self.weight.dtype)
+ x2 = x if x.dim() == 2 else x.reshape(-1, x.shape[-1])
+ if not x2.is_contiguous():
+ x2 = x2.contiguous()
+ out = NVFP4FakeQuantizedLinear.forward(self, x2)
+ if x.dim() != 2:
+ out = out.view(*x.shape[:-1], out.shape[-1])
+ return out if out_dtype is None or out.dtype == out_dtype else out.to(out_dtype)
def _patch_microscaling_qat_forwards(model) -> None: