Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Honour FP8Linear.block_size in the patched FP8 forward (32x32 block checkpoints) (#12317)
Daniel Hand167b8901604
diff --git a/tests/test_fp8linear_module_block_size.py b/tests/test_fp8linear_module_block_size.pynew file mode 100644index 00000000000..58016fc7cff--- /dev/null+++ b/tests/test_fp8linear_module_block_size.py@@ -0,0 +1,69 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""Patched FP8Linear forward must honour the module's block_size, not assume 128x128."""++import pytest+import torch++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),+ reason = "block-FP8 kernels need CUDA sm89+",+)+++def _make_layer(out_features, in_features, block, scale_fmt):+ from unsloth.kernels import fp8++ if fp8.FP8Linear is None:+ pytest.skip("this transformers has no FP8Linear")+ if scale_fmt == "ue8m0" and not hasattr(torch, "float8_e8m0fnu"):+ pytest.skip("this torch has no float8_e8m0fnu")+ # transformers <= 5.5 FP8Linear has block_size but no scale_fmt (float scales only).+ extra = {} if scale_fmt == "float" else {"scale_fmt": scale_fmt}+ try:+ layer = fp8.FP8Linear(in_features, out_features, block_size = block, **extra)+ except TypeError:+ pytest.skip("this transformers FP8Linear takes no scale_fmt")+ layer = layer.cuda()+ torch.manual_seed(0)+ weight = (torch.randn(out_features, in_features, device = "cuda") * 0.5).to(torch.float8_e4m3fn)+ # Power-of-two scales so float and ue8m0 formats hold the same values.+ grid = (-(-out_features // block[0]), -(-in_features // block[1]))+ scale = torch.exp2(torch.randint(-9, -4, grid, device = "cuda").float())+ layer.weight = torch.nn.Parameter(weight, requires_grad = False)+ layer.weight_scale_inv = torch.nn.Parameter(+ scale.to(layer.weight_scale_inv.dtype), requires_grad = False+ )+ assert getattr(layer.weight, "block_size", None) is None+ assert getattr(layer.weight_scale_inv, "block_size", None) is None+ return layer, weight, scale+++def _reference(X, weight, scale, block):+ out_features, in_features = weight.shape+ s = scale.repeat_interleave(block[0], 0)[:out_features]+ s = s.repeat_interleave(block[1], 1)[:, :in_features]+ W = weight.float() * s+ return X.float() @ W.T, W++[email protected]("scale_fmt", ["float", "ue8m0"])[email protected]("block", [(32, 32), (128, 128)], ids = ["32x32", "128x128"])+def test_fp8linear_uses_module_block_size(block, scale_fmt):+ out_features, in_features = 512, 1024+ layer, weight, scale = _make_layer(out_features, in_features, block, scale_fmt)+ torch.manual_seed(1)+ X = torch.randn(3, 64, in_features, device = "cuda", dtype = torch.bfloat16, requires_grad = True)++ out = layer(X)+ assert out.shape == (3, 64, out_features) and out.dtype == torch.bfloat16+ ref, W = _reference(X.detach(), weight, scale, block)+ # Tolerates fp8 activation rounding; a wrong block size errors at order 1.+ rel = (out.float() - ref).norm() / ref.norm()+ assert rel < 0.05, rel.item()++ grad_out = torch.randn_like(out)+ out.backward(grad_out)+ grad_ref = grad_out.float() @ W+ rel_grad = (X.grad.float() - grad_ref).norm() / grad_ref.norm()+ assert rel_grad < 0.02, rel_grad.item()diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 0a5834cbe1d..e2689a549e1 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -1023,7 +1023,17 @@ def module_forward_patch(forward_function, scale_attr = "weight_scale"):return torch.nn.functional.linear(X, weight.to(X.dtype), None if bias is None else bias.to(X.dtype))- out = forward_function(X, weight, getattr(self, scale_attr))+ weight_scale = getattr(self, scale_attr)+ # Kernels read block_size off weight/scale, else assume 128x128; tagging 128x128 would only cost a recompile.+ module_block_size = getattr(self, "block_size", None)+ if (+ module_block_size is not None+ and list(module_block_size) != [128, 128]+ and getattr(weight, "block_size", None) is None+ and getattr(weight_scale, "block_size", None) is None+ ):+ weight.block_size = list(module_block_size)+ out = forward_function(X, weight, weight_scale)# The kernels take no bias, so a biased Linear (Qwen2-style q/k/v) adds it here; fbgemm keeps it fp32.bias = self._parameters.get("bias")return out if bias is None else out + bias.to(out.dtype)