Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kernels: request enable_fp_fusion=True explicitly, ahead of Triton 3.9's False default (#13828)

Daniel Hane1cdbf435000
diff --git a/tests/test_triton_fp_fusion.py b/tests/test_triton_fp_fusion.py
new file mode 100644
index 00000000000..80d2e7fd802
--- /dev/null
+++ b/tests/test_triton_fp_fusion.py
@@ -0,0 +1,286 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Every Triton launch passes enable_fp_fusion=True: Triton 3.9 defaults it to False.
+The GPU check emulates that via TRITON_DEFAULT_FP_FUSION=0 in a fresh process."""
+
+import ast
+import json
+import os
+import pathlib
+import subprocess
+import sys
+
+import pytest
+
+KERNELS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "kernels"
+
+# Subscript calls that are not launches, or forward options already carrying enable_fp_fusion.
+_NOT_LAUNCHES = {
+ "bnb_override.py": {"_FALLBACK"},
+ "int4_packed.py": {
+ "compiled",
+ "call",
+ }, # a CompiledKernel built by warmup(enable_fp_fusion=True)
+ "utils.py": {"plan"},
+ "nf4.py": {"kernel"},
+ "triton_launch.py": {"kernel"},
+}
+
+
+def _launch_sites():
+ for path in sorted(KERNELS.rglob("*.py")):
+ rel = path.relative_to(KERNELS)
+ if rel.parts[0] == "moe" and rel.parts[1] in ("tests", "benchmark"):
+ continue
+ src = path.read_text()
+ if "triton" not in src:
+ continue
+ for node in ast.walk(ast.parse(src)):
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Subscript):
+ name = ast.get_source_segment(src, node.func.value)
+ if name in _NOT_LAUNCHES.get(path.name, ()):
+ continue
+ yield f"{rel}:{node.lineno}:{name}", node
+
+
+def test_every_launch_requests_fp_fusion():
+ sites = list(_launch_sites())
+ assert len(sites) >= 25, [s for s, _ in sites]
+ missing = [
+ site for site, node in sites if not any(k.arg == "enable_fp_fusion" for k in node.keywords)
+ ]
+ assert not missing, f"Triton launches without enable_fp_fusion: {missing}"
+
+
+def test_nf4_options_and_warmup_carry_fp_fusion():
+ nf4 = (KERNELS / "nf4.py").read_text()
+ assert '"enable_fp_fusion": bool(fp_fusion)' in nf4
+ gemv = (KERNELS / "nf4_gemv.py").read_text()
+ assert gemv.count("enable_fp_fusion = True") == 2
+ int4 = ast.parse((KERNELS / "int4_packed.py").read_text())
+ warmups = [
+ node
+ for node in ast.walk(int4)
+ if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "warmup"
+ ]
+ assert warmups and all(
+ any(k.arg == "enable_fp_fusion" and getattr(k.value, "value", None) is True for k in node.keywords)
+ for node in warmups
+ )
+
+
+def _compiled_options(kernel):
+ """enable_fp_fusion of every compiled entry of a Triton JITFunction (3.0 - 3.9 layouts)."""
+ from triton.runtime.jit import JITFunction
+
+ seen = []
+ while not isinstance(kernel, JITFunction) and hasattr(kernel, "fn"): # autotune / heuristics
+ kernel = kernel.fn
+ if not isinstance(kernel, JITFunction):
+ return seen
+ cache = getattr(kernel, "cache", None)
+ if isinstance(cache, dict): # Triton <= 3.2: cache[device][key]
+ tables = list(cache.values())
+ else: # Triton 3.3+: device_caches[device] = (kernel_cache, ...)
+ tables = [entry[0] for entry in kernel.device_caches.values()]
+ for table in tables:
+ for compiled in table.values():
+ meta = compiled.metadata
+ value = (
+ meta.get("enable_fp_fusion")
+ if isinstance(meta, dict)
+ else getattr(meta, "enable_fp_fusion", None)
+ )
+ ptx = compiled.asm.get("ptx", "") if hasattr(compiled, "asm") else ""
+ seen.append((value, "fma.rn" in ptx))
+ return seen
+
+
+def exercise(device = "cuda"):
+ """Run the representative kernels once each; returns {name: output tensors}."""
+ import torch
+ import triton # noqa: F401
+
+ from unsloth.kernels import (
+ cross_entropy_loss,
+ fp8,
+ geglu,
+ nf4,
+ nf4_gemv,
+ rms_layernorm,
+ rope_embedding,
+ swiglu,
+ )
+
+ out = {}
+ g = torch.Generator(device = device).manual_seed(0)
+ dt = torch.bfloat16 if torch.cuda.get_device_capability(device) >= (8, 0) else torch.float16
+ rnd = lambda *s, dtype = dt: torch.randn(*s, generator = g, device = device, dtype = torch.float32).to(
+ dtype
+ )
+
+ norm = torch.nn.Module()
+ norm.weight = torch.nn.Parameter(rnd(256))
+ norm.variance_epsilon = 1e-6
+ X = rnd(64, 256).requires_grad_()
+ Y = rms_layernorm.fast_rms_layernorm(norm, X)
+ Y.backward(rnd(64, 256))
+ out["rmsnorm"] = (Y.detach(), X.grad, norm.weight.grad)
+
+ Q = rnd(2, 4, 16, 64).requires_grad_()
+ K = rnd(2, 2, 16, 64).requires_grad_()
+ cos, sin = rnd(16, 64), rnd(16, 64)
+ q, k = rope_embedding.fast_rope_embedding(Q, K, cos, sin)
+ (q.float().sum() + k.float().square().sum()).backward()
+ out["rope"] = (q.detach(), k.detach(), Q.grad, K.grad)
+ idx = torch.arange(16, device = device).repeat(2, 1)
+ q2, k2 = rope_embedding.fast_rope_embedding(Q.detach(), K.detach(), cos, sin, idx)
+ out["rope_qk"] = (q2, k2)
+
+ # Chunked path needs vocab > 65536.
+ for name, vocab in (("ce", 1000), ("ce_chunked", 70000)):
+ logits = rnd(2, 8, vocab).requires_grad_()
+ labels = torch.randint(0, vocab, (2, 8), generator = g, device = device)
+ loss = cross_entropy_loss.fast_cross_entropy_loss(logits, labels)
+ loss.backward()
+ out[name] = (loss.detach(), logits.grad)
+
+ e, gg, dw = rnd(32, 512), rnd(32, 512), rnd(32, 512)
+ e3, g3 = e.view(2, 16, 512), gg.view(2, 16, 512)
+ out["swiglu_fwd"] = (swiglu.swiglu_fg_kernel(e3, g3),)
+ out["swiglu_bwd"] = tuple(
+ t.clone() for t in swiglu.swiglu_DWf_DW_dfg_kernel(dw.clone(), e.clone(), gg.clone())
+ )
+ out["geglu_exact_fwd"] = (geglu.geglu_exact_forward_kernel(e3, g3),)
+ out["geglu_exact_bwd"] = tuple(
+ t.clone() for t in geglu.geglu_exact_backward_kernel(dw.clone(), e.clone(), gg.clone())
+ )
+ out["geglu_approx_fwd"] = (geglu.geglu_approx_forward_kernel(e3, g3),)
+ out["geglu_approx_bwd"] = tuple(
+ t.clone() for t in geglu.geglu_approx_backward_kernel(dw.clone(), e.clone(), gg.clone())
+ )
+
+ # Hand-built NF4 state so bitsandbytes is not needed.
+ n, kk, bs, bs2 = 256, 512, 64, 256
+ W_u8 = torch.randint(
+ 0, 256, (n * kk // 2, 1), generator = g, device = device, dtype = torch.int32
+ ).to(torch.uint8)
+ n_blocks = n * kk // bs
+ code = torch.linspace(-1, 1, 16, device = device)
+ code2 = torch.linspace(-1, 1, 256, device = device)
+ absmax_q = torch.randint(0, 256, (n_blocks,), generator = g, device = device, dtype = torch.int32).to(
+ torch.uint8
+ )
+ absmax2 = torch.rand(n_blocks // bs2, generator = g, device = device) + 0.5
+ offset = torch.tensor(0.1, device = device)
+ args = (W_u8, absmax_q, code2, absmax2, offset, code, bs, bs2)
+ out["nf4_dequant_eager"] = (nf4.dequantize_nf4(*args, (n, kk), dt),)
+ out["nf4_dequant_op"] = (torch.ops.unsloth.dequantize_nf4(*args, [n, kk], dt),)
+ Xv = rnd(1, 1, kk)
+ out["nf4_gemv"] = (nf4_gemv.gemv_nf4(Xv, *args, (n, kk), dt),)
+
+ Wf = (rnd(512, 1024, dtype = torch.float32) / 4).to(torch.float8_e4m3fn)
+ if not fp8._fp8_kernel_unsupported(Wf):
+ scale = torch.rand(512, 1, generator = g, device = device) + 0.5
+ out["fp8_gemv"] = (fp8.fp8_rowwise_gemv(rnd(1, 1, 1024), Wf, scale),)
+ torch.cuda.synchronize()
+
+ modules = (cross_entropy_loss, fp8, geglu, nf4, nf4_gemv, rms_layernorm, rope_embedding, swiglu)
+ meta = {}
+ for module in modules:
+ for attr, value in vars(module).items():
+ options = (
+ _compiled_options(value) if hasattr(value, "run") or hasattr(value, "fn") else []
+ )
+ if options:
+ meta[f"{module.__name__.rsplit('.', 1)[-1]}.{attr}"] = options
+ return out, meta
+
+
+def _child():
+ import torch
+
+ import unsloth # noqa: F401
+
+ from unsloth.kernels import nf4
+
+ _, meta = exercise()
+ print(
+ "A1META "
+ + json.dumps(
+ {"meta": meta, "hip": bool(torch.version.hip), "mul_rn": bool(nf4._HAS_MUL_RN)}
+ )
+ )
+
+
+def _run_child(tmp_path, default):
+ env = dict(os.environ)
+ env["TRITON_DEFAULT_FP_FUSION"] = default
+ env["TRITON_CACHE_DIR"] = str(tmp_path / f"triton_{default}")
+ env.pop("TRITON_FORCE_DISABLE_FP_FUSION", None)
+ code = f"import sys; sys.path.insert(0, {str(pathlib.Path(__file__).parent)!r}); import test_triton_fp_fusion as t; t._child()"
+ proc = subprocess.run(
+ [sys.executable, "-c", code], env = env, capture_output = True, text = True, timeout = 900
+ )
+ line = [l for l in proc.stdout.splitlines() if l.startswith("A1META ")]
+ assert proc.returncode == 0 and line, proc.stdout[-3000:] + proc.stderr[-6000:]
+ return json.loads(line[-1][len("A1META ") :])
+
+
+def _needs_gpu():
+ try:
+ sys.path.insert(0, str(pathlib.Path(__file__).parent / "_shared"))
+ from real_accelerator import has_real_cuda
+ real = has_real_cuda()
+ except Exception:
+ import torch
+ real = torch.cuda.is_available()
+ if not real:
+ pytest.skip("needs a CUDA / ROCm GPU")
+ pytest.importorskip("triton")
+
+
[email protected]("default", ["0", "1"])
+def test_compiled_kernels_keep_fp_fusion(tmp_path, default):
+ _needs_gpu()
+ result = _run_child(tmp_path, default)
+ meta = result["meta"]
+ expected = {
+ "rms_layernorm._rms_layernorm_forward",
+ "rms_layernorm._rms_layernorm_backward",
+ "rope_embedding._rope_embedding",
+ "rope_embedding._rope_embedding_QK",
+ "cross_entropy_loss._cross_entropy_forward",
+ "cross_entropy_loss._chunked_cross_entropy_forward",
+ "cross_entropy_loss._cross_entropy_backward",
+ "swiglu._fg_kernel",
+ "swiglu._DWf_DW_dfg_kernel",
+ "geglu._exact_forward_kernel",
+ "geglu._exact_backward_kernel",
+ "geglu._approx_forward_kernel",
+ "geglu._approx_backward_kernel",
+ "nf4._nf4_dequant_kernel",
+ }
+ # Multi-row RMSNorm kernels replace the one-row ones on CUDA for narrow rows.
+ if "rms_layernorm._rms_layernorm_forward_rows" in meta:
+ expected.discard("rms_layernorm._rms_layernorm_forward")
+ if "rms_layernorm._rms_layernorm_backward_rows" in meta:
+ expected.discard("rms_layernorm._rms_layernorm_backward")
+ assert expected <= set(meta), sorted(expected - set(meta))
+ assert any(name.startswith("nf4_gemv.") for name in meta), sorted(meta)
+
+ nf4_values = {value for value, _ in meta.pop("nf4._nf4_dequant_kernel")}
+ # Eager dequant stays deliberately unfused; the traced op fuses where mul.rn exists.
+ assert False in nf4_values
+ assert (True in nf4_values) == result["mul_rn"], nf4_values
+ bad = {name: opts for name, opts in meta.items() if any(value is not True for value, _ in opts)}
+ assert not bad, f"compiled without enable_fp_fusion=True: {bad}"
+ if not result["hip"]:
+ # Contraction really happened: fma.rn in PTX.
+ for name in (
+ "rope_embedding._rope_embedding",
+ "geglu._exact_forward_kernel",
+ "swiglu._DWf_DW_dfg_kernel",
+ ):
+ assert all(fma for _, fma in meta[name]), (name, meta[name])
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 038109083ca..38fc705dddc 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -333,6 +333,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
DO_LOGIT_SCALING = DO_LOGIT_SCALING,
LOGIT_SCALE = logit_scaling,
num_warps = num_warps,
+ enable_fp_fusion = True,
)
else:
# For large vocabs > 65336 like Gemma 256K
@@ -365,6 +366,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
DO_LOGIT_SCALING = DO_LOGIT_SCALING,
LOGIT_SCALE = logit_scaling,
num_warps = 32 if not is_cdna() else 16,
+ enable_fp_fusion = True,
)
# logsumexp(chunked_logsumexp) - x, with the -x done separately.
logsumexp = torch.logsumexp(logsumexp, dim = 1)
@@ -414,6 +416,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
DO_LOGIT_SCALING = ctx.DO_LOGIT_SCALING,
LOGIT_SCALE = ctx.logit_scaling,
num_warps = 8,
+ enable_fp_fusion = True,
)
return (
dlogits,
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 24b73445261..be3858e4eb4 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -165,7 +165,7 @@ def weight_dequant_block(
triton.cdiv(N, meta["BLOCK_SIZE"]),
)
with _fp8_triton_device_context(x):
- weight_dequant_kernel[grid](x, s, y, M, N, BLOCK_SIZE = block_size)
+ weight_dequant_kernel[grid](x, s, y, M, N, BLOCK_SIZE = block_size, enable_fp_fusion = True)
return y
@@ -257,7 +257,9 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, tor
return (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)
with _fp8_triton_device_context(x):
- act_quant_kernel[grid](x, y, s, BLOCK_SIZE = block_size, LONG_INDEXING = long_indexing(x))
+ act_quant_kernel[grid](
+ x, y, s, BLOCK_SIZE = block_size, LONG_INDEXING = long_indexing(x), enable_fp_fusion = True
+ )
return y, s
@@ -420,6 +422,7 @@ def w8a8_block_fp8_matmul_triton(
LONG_INDEXING = long_indexing(A, B, C),
# Default 4 warps starve 128x128 tiles (8 is 1.3-2.1x faster from M >= 128); smaller decode tiles gain nothing.
num_warps = 8 if BLOCK_SIZE_M == 128 else 4,
+ enable_fp_fusion = True,
)
return C
@@ -557,7 +560,17 @@ def fp8_rowwise_gemv(X, weight, weight_scale):
block_n = 1 if N <= 4096 else (4 if N <= 32768 else 8)
with _fp8_triton_device_context(X, static_device_count = True):
_fp8_rowwise_gemv_kernel[(triton.cdiv(N, block_n),)](
- X, weight, scale, out, N, K, weight.stride(0), BLOCK_N = block_n, BLOCK_K = 512, num_warps = 2
+ X,
+ weight,
+ scale,
+ out,
+ N,
+ K,
+ weight.stride(0),
+ BLOCK_N = block_n,
+ BLOCK_K = 512,
+ num_warps = 2,
+ enable_fp_fusion = True,
)
return out
diff --git a/unsloth/kernels/geglu.py b/unsloth/kernels/geglu.py
index de6ab070c77..e6884506ac2 100644
--- a/unsloth/kernels/geglu.py
+++ b/unsloth/kernels/geglu.py
@@ -62,6 +62,7 @@ def geglu_exact_forward_kernel(gate, up):
n_elements,
BLOCK_SIZE = BLOCK_SIZE,
LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,
+ enable_fp_fusion = True,
)
return out
@@ -128,6 +129,7 @@ def geglu_exact_backward_kernel(DW, e, g):
n_elements,
BLOCK_SIZE = BLOCK_SIZE,
LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,
+ enable_fp_fusion = True,
)
return DW, e, g
@@ -171,6 +173,7 @@ def geglu_approx_forward_kernel(gate, up):
n_elements,
BLOCK_SIZE = BLOCK_SIZE,
LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,
+ enable_fp_fusion = True,
)
return out
@@ -245,5 +248,6 @@ def geglu_approx_backward_kernel(DW, e, g):
n_elements,
BLOCK_SIZE = BLOCK_SIZE,
LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,
+ enable_fp_fusion = True,
)
return DW, e, g
diff --git a/unsloth/kernels/int4_packed.py b/unsloth/kernels/int4_packed.py
index 33165ca3b3b..19de18012a7 100644
--- a/unsloth/kernels/int4_packed.py
+++ b/unsloth/kernels/int4_packed.py
@@ -315,7 +315,9 @@ def _launch(kernel, launchers, key, grid, args, **constexprs):
if _FAST_LAUNCH is not False:
compiled = launchers.get(key)
if compiled is None:
- compiled = launchers[key] = kernel.warmup(*args, grid = grid, **constexprs)
+ compiled = launchers[key] = kernel.warmup(
+ *args, grid = grid, **constexprs, enable_fp_fusion = True
+ )
try:
compiled[grid](*args)
_FAST_LAUNCH = True
@@ -324,7 +326,7 @@ def _launch(kernel, launchers, key, grid, args, **constexprs):
if _FAST_LAUNCH:
raise
_FAST_LAUNCH = False
- kernel[grid](*args, **constexprs)
+ kernel[grid](*args, **constexprs, enable_fp_fusion = True)
def int4_dequantize(
@@ -619,6 +621,7 @@ def _dequant_repacked(
words, qs.scale, qs.scale if zp is None else zp, out, NT_N, NT_K, qs.scale.stride(0),
0 if zp is None else zp.stride(0), out.stride(0), MARLIN = qs.layout == "marlin", TN = TN, TK = TK,
R = R, GROUP = group, HAS_Z = zp is not None, RAW = raw, num_warps = 4,
+ enable_fp_fusion = True,
) # fmt: skip
return out
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index a9a33c0e32f..d693493c2e6 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -236,6 +236,7 @@ class Fast_Layernorm(torch.autograd.Function):
BLOCK_SIZE = BLOCK_SIZE,
HAS_BIAS = has_bias,
num_warps = num_warps,
+ enable_fp_fusion = True,
)
ctx.eps = eps
ctx.BLOCK_SIZE = BLOCK_SIZE
@@ -269,6 +270,7 @@ class Fast_Layernorm(torch.autograd.Function):
LONG_INDEXING = LONG_INDEXING,
BLOCK_SIZE = ctx.BLOCK_SIZE,
num_warps = ctx.num_warps,
+ enable_fp_fusion = True,
)
return dY.view(*shape), None, None, None
@@ -301,6 +303,7 @@ class Fast_Layernorm(torch.autograd.Function):
TRAIN_W = train_w,
TRAIN_B = train_b,
num_warps = ctx.num_warps,
+ enable_fp_fusion = True,
)
dW = torch.empty(n_cols, dtype = W.dtype, device = dY.device) if train_w else None
dB = torch.empty(n_cols, dtype = b.dtype, device = dY.device) if train_b else None
@@ -314,6 +317,7 @@ class Fast_Layernorm(torch.autograd.Function):
n_cols,
BLOCK_ROWS = 64,
BLOCK_COLS = 64,
+ enable_fp_fusion = True,
)
return dY.view(*shape), dW, dB, None
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index 62d76ddbe3a..3b935e5e5c3 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -264,7 +264,9 @@ def _grouped_gemm_forward_impl(
is_fake = _is_tracing(X, W)
if not is_fake:
- compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](**kernel_args)
+ compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](
+ **kernel_args, enable_fp_fusion = True
+ )
if autotune:
log_kernel_info(compiled_kernel, kernel.best_config)
else:
@@ -404,7 +406,9 @@ def _grouped_gemm_dX_impl(
is_fake = _is_tracing(dY, W)
if not is_fake:
- compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](**kernel_args)
+ compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](
+ **kernel_args, enable_fp_fusion = True
+ )
if autotune:
log_kernel_info(compiled_kernel, kernel.best_config)
@@ -550,7 +554,9 @@ def _grouped_gemm_dW_impl(
is_fake = _is_tracing(X, dY)
if not is_fake:
- compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](**kernel_args)
+ compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](
+ **kernel_args, enable_fp_fusion = True
+ )
if autotune:
log_kernel_info(compiled_kernel, kernel.best_config)
diff --git a/unsloth/kernels/nf4.py b/unsloth/kernels/nf4.py
index b2d75ea1a06..e1a0a91b9ed 100644
--- a/unsloth/kernels/nf4.py
+++ b/unsloth/kernels/nf4.py
@@ -262,11 +262,8 @@ def _launch(
LUT_MODE = lut_mode,
)
grid = (-(-n_blocks // rows),)
- options = (
- {"num_warps": num_warps}
- if fp_fusion
- else {"num_warps": num_warps, "enable_fp_fusion": False}
- )
+ # Explicit both ways: Triton 3.9 defaults enable_fp_fusion to False.
+ options = {"num_warps": num_warps, "enable_fp_fusion": bool(fp_fusion)}
if kernel is _nf4_dequant_kernel:
entry = launch(kernel, grid, args, 7, constexprs, W.device.index, **options)
if plan:
diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.py
index 52de268b6b8..82ba41b5662 100644
--- a/unsloth/kernels/nf4_gemv.py
+++ b/unsloth/kernels/nf4_gemv.py
@@ -260,11 +260,20 @@ def _launch(
BLOCK_K = block_k,
)
if kernel is _gemv_nf4_kernel or kernel is _gemv_nf4_words_kernel:
- entry = launch(kernel, grid, args, 8, constexprs, X.device.index, num_warps = num_warps)
+ entry = launch(
+ kernel,
+ grid,
+ args,
+ 8,
+ constexprs,
+ X.device.index,
+ num_warps = num_warps,
+ enable_fp_fusion = True,
+ )
if plan:
return _rerun(entry, grid, args, X.device.index)
else:
- kernel[grid](*args, **constexprs, num_warps = num_warps)
+ kernel[grid](*args, **constexprs, num_warps = num_warps, enable_fp_fusion = True)
return out
diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.py
index 42d88a079e4..98d3ebfcd78 100644
--- a/unsloth/kernels/nvfp4.py
+++ b/unsloth/kernels/nvfp4.py
@@ -145,6 +145,7 @@ def _nvfp4_dequantize_triton(
# int32 offsets wrap once the output passes 2^31 elements; int64 only then, as it costs ~2%.
INT64_OFFSETS = rows * half * 2 >= 2**31,
num_warps = 4,
+ enable_fp_fusion = True,
)
return out
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 81409172a34..038cfd5f21c 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -590,6 +590,7 @@ def _rms_forward(
# int32 rows measured slower only for the Gemma multi-row forward: keep main's int64.
LONG_INDEXING = long or gemma,
num_warps = num_warps,
+ enable_fp_fusion = True,
)
return Y, r
except Exception as error:
@@ -612,6 +613,7 @@ def _rms_forward(
LONG_INDEXING = long,
BLOCK_SIZE = BLOCK_SIZE,
num_warps = num_warps,
+ enable_fp_fusion = True,
)
return Y, r
@@ -706,6 +708,7 @@ def _rms_backward(
LONG_INDEXING = long,
TRAIN_W = train_w,
num_warps = num_warps,
+ enable_fp_fusion = True,
)
except Exception as error:
if _multirow_must_raise(error, wrap):
@@ -742,6 +745,7 @@ def _rms_backward(
TRAIN_W = train_w,
num_warps = num_warps,
**launch,
+ enable_fp_fusion = True,
)
return _rms_sum_partials(partials, W, wrap)
@@ -752,7 +756,13 @@ def _rms_sum_partials(partials, W, wrap):
programs, n_cols = partials.shape
dW = torch.empty(n_cols, dtype = W.dtype, device = partials.device)
wrap(_rms_sum_dw_partials)[(triton.cdiv(n_cols, 64),)](
- partials, dW, programs, n_cols, BLOCK_ROWS = 64, BLOCK_COLS = 64
+ partials,
+ dW,
+ programs,
+ n_cols,
+ BLOCK_ROWS = 64,
+ BLOCK_COLS = 64,
+ enable_fp_fusion = True,
)
return dW
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 9dfe2a5d05c..181367a6d3c 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -215,6 +215,7 @@ def _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, backward, wrap):
LONG_INDEXING = long_indexing(Q, block = BLOCK_SIZE),
BLOCK_SIZE = BLOCK_SIZE,
num_warps = num_warps,
+ enable_fp_fusion = True,
)
@@ -297,6 +298,7 @@ def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):
BLOCK_SIZE = BLOCK_SIZE,
EVICT_INDICES = _eviction_hints_ok(Q.device),
num_warps = num_warps,
+ enable_fp_fusion = True,
)
diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py
index 2078f3266e0..13b2ea33ebc 100644
--- a/unsloth/kernels/swiglu.py
+++ b/unsloth/kernels/swiglu.py
@@ -56,6 +56,7 @@ def swiglu_fg_kernel(e, g):
n_elements,
BLOCK_SIZE = BLOCK_SIZE,
LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,
+ enable_fp_fusion = True,
)
return h
@@ -129,5 +130,6 @@ def swiglu_DWf_DW_dfg_kernel(DW, e, g):
n_elements,
BLOCK_SIZE = BLOCK_SIZE,
LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,
+ enable_fp_fusion = True,
)
return DW, e, g