Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 GEMV on Blackwell + Triton 3.3: no graph break under torch 2.7, keep the exact GEMV shapes that are faster (#14026)
Daniel Han15eaf67caa8f
diff --git a/tests/test_compressed_tensors_fp8_kernel_route.py b/tests/test_compressed_tensors_fp8_kernel_route.pyindex 179ac31af81..37473ba7f70 100644--- a/tests/test_compressed_tensors_fp8_kernel_route.py+++ b/tests/test_compressed_tensors_fp8_kernel_route.py@@ -261,12 +261,13 @@ def test_decode_gemv_matches_the_dequantized_matmul(shape, rows):W, s = model.lin.weight, model.lin.weight_scaleX = torch.randn(*rows, shape[1], device = dev, dtype = torch.bfloat16)with torch.no_grad():- if _fp8_gemv_miscompiles(W.device):- assert not can_use_fp8_rowwise_gemv(X, W, s)+ if not can_use_fp8_rowwise_gemv(X, W, s):+ # Only Blackwell + Triton < 3.4 refuses a decode shape, and only past BLOCK_N 1 or for small N, long K.+ assert _fp8_gemv_miscompiles(W.device)+ assert shape[0] > 4096 or (shape[0] < 256 and shape[1] > 4096)pytest.skip("Triton < 3.4 miscompiles this GEMV on Blackwell; decode takes the dequant path")- assert can_use_fp8_rowwise_gemv(X, W, s)y = fp8_rowwise_gemv(X, W, s)y_again = fp8_rowwise_gemv(X, W, s)y_ref = X.float() @ ref.t()diff --git a/tests/test_fp8_rowwise_gemv_blackwell_gate.py b/tests/test_fp8_rowwise_gemv_blackwell_gate.pyindex 5eb1eaa52bb..844f29974ed 100644--- a/tests/test_fp8_rowwise_gemv_blackwell_gate.py+++ b/tests/test_fp8_rowwise_gemv_blackwell_gate.py@@ -46,7 +46,9 @@ def test_gate(F, monkeypatch, capability, version, hip, expected):@pytest.mark.skipif(not has_real_cuda(), reason = "needs a GPU")[email protected]("N, K", [(5000, 1536), (40000, 1024), (4096, 4096)])+ "N, K", [(5000, 1536), (40000, 1024), (4096, 4096), (256, 16384), (2, 4096)]+)def test_chosen_gemv_is_exact(F, N, K):if F._fp8_kernel_unsupported(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn)):pytest.skip("no fp8e4nv Triton kernel on this device")@@ -60,3 +62,90 @@ def test_chosen_gemv_is_exact(F, N, K):out = F.fp8_rowwise_gemv(X, W, S).float().reshape(-1)ref = (X.float() @ (W.float() * S).T).reshape(-1)assert (out - ref).abs().max() <= 1e-2 * ref.abs().max()+++ "version, release",+ [("3.3.1", (3, 3)), ("3.4.0rc1", (3, 4)), ("3.10.0", (3, 10)), ("3.3.1+git9a8b7c6", (3, 3))],+)+def test_triton_release_parse(F, monkeypatch, version, release):+ monkeypatch.setattr(F.triton, "__version__", version)+ assert F._triton_release() == release++[email protected]("version", ["3.3.1", "3.4.0"])+def test_gate_traces_without_graph_break(F, monkeypatch, version):+ # Compiled decode steps call the gate; torch 2.7 dynamo broke inside packaging's Version.+ monkeypatch.setattr(F.torch.cuda, "get_device_capability", lambda device = None: (10, 0))+ monkeypatch.setattr(F.triton, "__version__", version)+ monkeypatch.setattr(F.torch.version, "hip", None)++ def fn(x):+ return x + int(F._fp8_gemv_miscompiles(0))++ F._fp8_gemv_miscompiles.cache_clear()+ torch._dynamo.reset()+ try:+ out = torch.compile(fn, backend = "eager", fullgraph = True)(torch.zeros(2))+ assert out.tolist() == [float(version.startswith("3.3"))] * 2+ finally:+ torch._dynamo.reset()+ F._fp8_gemv_miscompiles.cache_clear()++[email protected](not has_real_cuda(), reason = "needs a GPU")+ "capability, version, N, K, expected",+ [+ # Blackwell + Triton 3.3: BLOCK_N 1 only, and not few rows over a long K.+ ((10, 0), "3.3.1", 256, 65536, True),+ ((12, 0), "3.3.1", 4096, 8192, True),+ ((10, 0), "3.3.1", 255, 4096, True),+ ((10, 0), "3.3.1", 2, 4096, True),+ ((10, 0), "3.3.1", 255, 4097, False),+ ((12, 0), "3.3.1", 2, 65536, False),+ ((10, 0), "3.3.1", 4097, 1024, False),+ ((10, 0), "3.2.0", 256, 4096, False),+ ((10, 0), "3.4.0", 4097, 8192, True),+ ((10, 0), "3.4.0", 2, 65536, True),+ ((9, 0), "3.3.1", 2, 65536, True),+ ],+)+def test_gate_boundary(F, monkeypatch, capability, version, N, K, expected):+ monkeypatch.setattr(F.torch.cuda, "get_device_capability", lambda device = None: capability)+ monkeypatch.setattr(F.triton, "__version__", version)+ monkeypatch.setattr(F.torch.version, "hip", None)+ caches = (F._fp8_gemv_miscompiles, F._triton_builds_blackwell, F._fp8_device_lacks_kernel)+ for cache in caches:+ cache.cache_clear()+ try:+ X = torch.zeros(1, K, device = "cuda", dtype = torch.bfloat16)+ W = torch.zeros(N, K, device = "cuda", dtype = torch.float8_e4m3fn)+ S = torch.ones(N, 1, device = "cuda")+ with torch.no_grad():+ assert F.can_use_fp8_rowwise_gemv(X, W, S) is expected+ finally:+ for cache in caches:+ cache.cache_clear()++[email protected](not has_real_cuda(), reason = "needs a GPU")[email protected]("N, K", [(1024, 512), (2, 65536)])+def test_gemv_route_compiles_fullgraph(F, N, K):+ if F._fp8_kernel_unsupported(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn)):+ pytest.skip("no fp8e4nv Triton kernel on this device")+ X = torch.randn(1, 1, K, device = "cuda", dtype = torch.bfloat16)+ W = (torch.randn(N, K, device = "cuda") * 0.1).to(torch.float8_e4m3fn)+ S = torch.rand(N, 1, device = "cuda") * 0.01 + 0.001++ def fn(X):+ return X.sum() * 2 if F.can_use_fp8_rowwise_gemv(X, W, S) else X.sum() * 3++ F._fp8_gemv_miscompiles.cache_clear()+ torch._dynamo.reset()+ try:+ with torch.no_grad():+ assert torch.equal(torch.compile(fn, backend = "eager", fullgraph = True)(X), fn(X))+ finally:+ torch._dynamo.reset()+ F._fp8_gemv_miscompiles.cache_clear()diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 1e2c5c63e96..b71263da86b 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -485,15 +485,22 @@ def _rocm_fp8e4nv_kernel_supported(device):return arch in ("gfx940", "gfx941", "gfx942", "gfx950")+def _triton_release():+ # Compiled decode steps trace the gates below; torch 2.7 dynamo graph breaks inside packaging's Version.+ try:+ major, minor = triton.__version__.split(".")[:2]+ return int(major), int(minor)+ except Exception:+ return Version(triton.__version__).release[:2]++@functools.lru_cache(maxsize = None)def _fp8_gemv_miscompiles(device):# Triton 3.3 miscompiles this GEMV on Blackwell (garbage rows at BLOCK_N 4/8); 3.4 is exact.if torch.version.hip is not None:return Falsetry:- return torch.cuda.get_device_capability(device) >= (10, 0) and Version(- triton.__version__- ).release[:2] < (3, 4)+ return torch.cuda.get_device_capability(device) >= (10, 0) and _triton_release() < (3, 4)except Exception:return True@@ -573,7 +580,22 @@ def can_use_fp8_rowwise_gemv(X, weight, weight_scale):return Falseif torch.is_grad_enabled() and (X.requires_grad or weight.requires_grad):return False- return not _fp8_kernel_unsupported(weight) and not _fp8_gemv_miscompiles(weight.device)+ if _fp8_kernel_unsupported(weight):+ return False+ if not _fp8_gemv_miscompiles(weight.device):+ return True+ # Triton 3.3 is exact at BLOCK_N 1 (N <= 4096); beats dequant + matmul when N >= 256 or K <= 4096 (RTX PRO 6000).+ N, K = weight.shape+ return N <= 4096 and (N >= 256 or K <= 4096) and _triton_builds_blackwell()++[email protected]_cache(maxsize = None)+def _triton_builds_blackwell():+ # Triton 3.2 bundles a ptxas without sm_100 / sm_120.+ try:+ return _triton_release() >= (3, 3)+ except Exception:+ return Falsedef fp8_rowwise_gemv(X, weight, weight_scale):