Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 GEMV on Blackwell + Triton 3.3: no graph break under torch 2.7, keep the exact GEMV shapes that are faster (#14026)

Daniel Han15eaf67caa8f
diff --git a/tests/test_compressed_tensors_fp8_kernel_route.py b/tests/test_compressed_tensors_fp8_kernel_route.py
index 179ac31af81..37473ba7f70 100644
--- a/tests/test_compressed_tensors_fp8_kernel_route.py
+++ b/tests/test_compressed_tensors_fp8_kernel_route.py
@@ -261,12 +261,13 @@ def test_decode_gemv_matches_the_dequantized_matmul(shape, rows):
W, s = model.lin.weight, model.lin.weight_scale
X = torch.randn(*rows, shape[1], device = dev, dtype = torch.bfloat16)
with torch.no_grad():
- if _fp8_gemv_miscompiles(W.device):
- assert not can_use_fp8_rowwise_gemv(X, W, s)
+ if not can_use_fp8_rowwise_gemv(X, W, s):
+ # Only Blackwell + Triton < 3.4 refuses a decode shape, and only past BLOCK_N 1 or for small N, long K.
+ assert _fp8_gemv_miscompiles(W.device)
+ assert shape[0] > 4096 or (shape[0] < 256 and shape[1] > 4096)
pytest.skip(
"Triton < 3.4 miscompiles this GEMV on Blackwell; decode takes the dequant path"
)
- assert can_use_fp8_rowwise_gemv(X, W, s)
y = fp8_rowwise_gemv(X, W, s)
y_again = fp8_rowwise_gemv(X, W, s)
y_ref = X.float() @ ref.t()
diff --git a/tests/test_fp8_rowwise_gemv_blackwell_gate.py b/tests/test_fp8_rowwise_gemv_blackwell_gate.py
index 5eb1eaa52bb..844f29974ed 100644
--- a/tests/test_fp8_rowwise_gemv_blackwell_gate.py
+++ b/tests/test_fp8_rowwise_gemv_blackwell_gate.py
@@ -46,7 +46,9 @@ def test_gate(F, monkeypatch, capability, version, hip, expected):
@pytest.mark.skipif(not has_real_cuda(), reason = "needs a GPU")
[email protected]("N, K", [(5000, 1536), (40000, 1024), (4096, 4096)])
+ "N, K", [(5000, 1536), (40000, 1024), (4096, 4096), (256, 16384), (2, 4096)]
+)
def test_chosen_gemv_is_exact(F, N, K):
if F._fp8_kernel_unsupported(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn)):
pytest.skip("no fp8e4nv Triton kernel on this device")
@@ -60,3 +62,90 @@ def test_chosen_gemv_is_exact(F, N, K):
out = F.fp8_rowwise_gemv(X, W, S).float().reshape(-1)
ref = (X.float() @ (W.float() * S).T).reshape(-1)
assert (out - ref).abs().max() <= 1e-2 * ref.abs().max()
+
+
+ "version, release",
+ [("3.3.1", (3, 3)), ("3.4.0rc1", (3, 4)), ("3.10.0", (3, 10)), ("3.3.1+git9a8b7c6", (3, 3))],
+)
+def test_triton_release_parse(F, monkeypatch, version, release):
+ monkeypatch.setattr(F.triton, "__version__", version)
+ assert F._triton_release() == release
+
+
[email protected]("version", ["3.3.1", "3.4.0"])
+def test_gate_traces_without_graph_break(F, monkeypatch, version):
+ # Compiled decode steps call the gate; torch 2.7 dynamo broke inside packaging's Version.
+ monkeypatch.setattr(F.torch.cuda, "get_device_capability", lambda device = None: (10, 0))
+ monkeypatch.setattr(F.triton, "__version__", version)
+ monkeypatch.setattr(F.torch.version, "hip", None)
+
+ def fn(x):
+ return x + int(F._fp8_gemv_miscompiles(0))
+
+ F._fp8_gemv_miscompiles.cache_clear()
+ torch._dynamo.reset()
+ try:
+ out = torch.compile(fn, backend = "eager", fullgraph = True)(torch.zeros(2))
+ assert out.tolist() == [float(version.startswith("3.3"))] * 2
+ finally:
+ torch._dynamo.reset()
+ F._fp8_gemv_miscompiles.cache_clear()
+
+
[email protected](not has_real_cuda(), reason = "needs a GPU")
+ "capability, version, N, K, expected",
+ [
+ # Blackwell + Triton 3.3: BLOCK_N 1 only, and not few rows over a long K.
+ ((10, 0), "3.3.1", 256, 65536, True),
+ ((12, 0), "3.3.1", 4096, 8192, True),
+ ((10, 0), "3.3.1", 255, 4096, True),
+ ((10, 0), "3.3.1", 2, 4096, True),
+ ((10, 0), "3.3.1", 255, 4097, False),
+ ((12, 0), "3.3.1", 2, 65536, False),
+ ((10, 0), "3.3.1", 4097, 1024, False),
+ ((10, 0), "3.2.0", 256, 4096, False),
+ ((10, 0), "3.4.0", 4097, 8192, True),
+ ((10, 0), "3.4.0", 2, 65536, True),
+ ((9, 0), "3.3.1", 2, 65536, True),
+ ],
+)
+def test_gate_boundary(F, monkeypatch, capability, version, N, K, expected):
+ monkeypatch.setattr(F.torch.cuda, "get_device_capability", lambda device = None: capability)
+ monkeypatch.setattr(F.triton, "__version__", version)
+ monkeypatch.setattr(F.torch.version, "hip", None)
+ caches = (F._fp8_gemv_miscompiles, F._triton_builds_blackwell, F._fp8_device_lacks_kernel)
+ for cache in caches:
+ cache.cache_clear()
+ try:
+ X = torch.zeros(1, K, device = "cuda", dtype = torch.bfloat16)
+ W = torch.zeros(N, K, device = "cuda", dtype = torch.float8_e4m3fn)
+ S = torch.ones(N, 1, device = "cuda")
+ with torch.no_grad():
+ assert F.can_use_fp8_rowwise_gemv(X, W, S) is expected
+ finally:
+ for cache in caches:
+ cache.cache_clear()
+
+
[email protected](not has_real_cuda(), reason = "needs a GPU")
[email protected]("N, K", [(1024, 512), (2, 65536)])
+def test_gemv_route_compiles_fullgraph(F, N, K):
+ if F._fp8_kernel_unsupported(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn)):
+ pytest.skip("no fp8e4nv Triton kernel on this device")
+ X = torch.randn(1, 1, K, device = "cuda", dtype = torch.bfloat16)
+ W = (torch.randn(N, K, device = "cuda") * 0.1).to(torch.float8_e4m3fn)
+ S = torch.rand(N, 1, device = "cuda") * 0.01 + 0.001
+
+ def fn(X):
+ return X.sum() * 2 if F.can_use_fp8_rowwise_gemv(X, W, S) else X.sum() * 3
+
+ F._fp8_gemv_miscompiles.cache_clear()
+ torch._dynamo.reset()
+ try:
+ with torch.no_grad():
+ assert torch.equal(torch.compile(fn, backend = "eager", fullgraph = True)(X), fn(X))
+ finally:
+ torch._dynamo.reset()
+ F._fp8_gemv_miscompiles.cache_clear()
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 1e2c5c63e96..b71263da86b 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -485,15 +485,22 @@ def _rocm_fp8e4nv_kernel_supported(device):
return arch in ("gfx940", "gfx941", "gfx942", "gfx950")
+def _triton_release():
+ # Compiled decode steps trace the gates below; torch 2.7 dynamo graph breaks inside packaging's Version.
+ try:
+ major, minor = triton.__version__.split(".")[:2]
+ return int(major), int(minor)
+ except Exception:
+ return Version(triton.__version__).release[:2]
+
+
@functools.lru_cache(maxsize = None)
def _fp8_gemv_miscompiles(device):
# Triton 3.3 miscompiles this GEMV on Blackwell (garbage rows at BLOCK_N 4/8); 3.4 is exact.
if torch.version.hip is not None:
return False
try:
- return torch.cuda.get_device_capability(device) >= (10, 0) and Version(
- triton.__version__
- ).release[:2] < (3, 4)
+ return torch.cuda.get_device_capability(device) >= (10, 0) and _triton_release() < (3, 4)
except Exception:
return True
@@ -573,7 +580,22 @@ def can_use_fp8_rowwise_gemv(X, weight, weight_scale):
return False
if torch.is_grad_enabled() and (X.requires_grad or weight.requires_grad):
return False
- return not _fp8_kernel_unsupported(weight) and not _fp8_gemv_miscompiles(weight.device)
+ if _fp8_kernel_unsupported(weight):
+ return False
+ if not _fp8_gemv_miscompiles(weight.device):
+ return True
+ # Triton 3.3 is exact at BLOCK_N 1 (N <= 4096); beats dequant + matmul when N >= 256 or K <= 4096 (RTX PRO 6000).
+ N, K = weight.shape
+ return N <= 4096 and (N >= 256 or K <= 4096) and _triton_builds_blackwell()
+
+
[email protected]_cache(maxsize = None)
+def _triton_builds_blackwell():
+ # Triton 3.2 bundles a ptxas without sm_100 / sm_120.
+ try:
+ return _triton_release() >= (3, 3)
+ except Exception:
+ return False
def fp8_rowwise_gemv(X, weight, weight_scale):