Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kernels: request enable_fp_fusion=True explicitly, ahead of Triton 3.9's False default (#13828)
Daniel Hane1cdbf435000
diff --git a/tests/test_triton_fp_fusion.py b/tests/test_triton_fp_fusion.pynew file mode 100644index 00000000000..80d2e7fd802--- /dev/null+++ b/tests/test_triton_fp_fusion.py@@ -0,0 +1,286 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Every Triton launch passes enable_fp_fusion=True: Triton 3.9 defaults it to False.+The GPU check emulates that via TRITON_DEFAULT_FP_FUSION=0 in a fresh process."""++import ast+import json+import os+import pathlib+import subprocess+import sys++import pytest++KERNELS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "kernels"++# Subscript calls that are not launches, or forward options already carrying enable_fp_fusion.+_NOT_LAUNCHES = {+ "bnb_override.py": {"_FALLBACK"},+ "int4_packed.py": {+ "compiled",+ "call",+ }, # a CompiledKernel built by warmup(enable_fp_fusion=True)+ "utils.py": {"plan"},+ "nf4.py": {"kernel"},+ "triton_launch.py": {"kernel"},+}+++def _launch_sites():+ for path in sorted(KERNELS.rglob("*.py")):+ rel = path.relative_to(KERNELS)+ if rel.parts[0] == "moe" and rel.parts[1] in ("tests", "benchmark"):+ continue+ src = path.read_text()+ if "triton" not in src:+ continue+ for node in ast.walk(ast.parse(src)):+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Subscript):+ name = ast.get_source_segment(src, node.func.value)+ if name in _NOT_LAUNCHES.get(path.name, ()):+ continue+ yield f"{rel}:{node.lineno}:{name}", node+++def test_every_launch_requests_fp_fusion():+ sites = list(_launch_sites())+ assert len(sites) >= 25, [s for s, _ in sites]+ missing = [+ site for site, node in sites if not any(k.arg == "enable_fp_fusion" for k in node.keywords)+ ]+ assert not missing, f"Triton launches without enable_fp_fusion: {missing}"+++def test_nf4_options_and_warmup_carry_fp_fusion():+ nf4 = (KERNELS / "nf4.py").read_text()+ assert '"enable_fp_fusion": bool(fp_fusion)' in nf4+ gemv = (KERNELS / "nf4_gemv.py").read_text()+ assert gemv.count("enable_fp_fusion = True") == 2+ int4 = ast.parse((KERNELS / "int4_packed.py").read_text())+ warmups = [+ node+ for node in ast.walk(int4)+ if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "warmup"+ ]+ assert warmups and all(+ any(k.arg == "enable_fp_fusion" and getattr(k.value, "value", None) is True for k in node.keywords)+ for node in warmups+ )+++def _compiled_options(kernel):+ """enable_fp_fusion of every compiled entry of a Triton JITFunction (3.0 - 3.9 layouts)."""+ from triton.runtime.jit import JITFunction++ seen = []+ while not isinstance(kernel, JITFunction) and hasattr(kernel, "fn"): # autotune / heuristics+ kernel = kernel.fn+ if not isinstance(kernel, JITFunction):+ return seen+ cache = getattr(kernel, "cache", None)+ if isinstance(cache, dict): # Triton <= 3.2: cache[device][key]+ tables = list(cache.values())+ else: # Triton 3.3+: device_caches[device] = (kernel_cache, ...)+ tables = [entry[0] for entry in kernel.device_caches.values()]+ for table in tables:+ for compiled in table.values():+ meta = compiled.metadata+ value = (+ meta.get("enable_fp_fusion")+ if isinstance(meta, dict)+ else getattr(meta, "enable_fp_fusion", None)+ )+ ptx = compiled.asm.get("ptx", "") if hasattr(compiled, "asm") else ""+ seen.append((value, "fma.rn" in ptx))+ return seen+++def exercise(device = "cuda"):+ """Run the representative kernels once each; returns {name: output tensors}."""+ import torch+ import triton # noqa: F401++ from unsloth.kernels import (+ cross_entropy_loss,+ fp8,+ geglu,+ nf4,+ nf4_gemv,+ rms_layernorm,+ rope_embedding,+ swiglu,+ )++ out = {}+ g = torch.Generator(device = device).manual_seed(0)+ dt = torch.bfloat16 if torch.cuda.get_device_capability(device) >= (8, 0) else torch.float16+ rnd = lambda *s, dtype = dt: torch.randn(*s, generator = g, device = device, dtype = torch.float32).to(+ dtype+ )++ norm = torch.nn.Module()+ norm.weight = torch.nn.Parameter(rnd(256))+ norm.variance_epsilon = 1e-6+ X = rnd(64, 256).requires_grad_()+ Y = rms_layernorm.fast_rms_layernorm(norm, X)+ Y.backward(rnd(64, 256))+ out["rmsnorm"] = (Y.detach(), X.grad, norm.weight.grad)++ Q = rnd(2, 4, 16, 64).requires_grad_()+ K = rnd(2, 2, 16, 64).requires_grad_()+ cos, sin = rnd(16, 64), rnd(16, 64)+ q, k = rope_embedding.fast_rope_embedding(Q, K, cos, sin)+ (q.float().sum() + k.float().square().sum()).backward()+ out["rope"] = (q.detach(), k.detach(), Q.grad, K.grad)+ idx = torch.arange(16, device = device).repeat(2, 1)+ q2, k2 = rope_embedding.fast_rope_embedding(Q.detach(), K.detach(), cos, sin, idx)+ out["rope_qk"] = (q2, k2)++ # Chunked path needs vocab > 65536.+ for name, vocab in (("ce", 1000), ("ce_chunked", 70000)):+ logits = rnd(2, 8, vocab).requires_grad_()+ labels = torch.randint(0, vocab, (2, 8), generator = g, device = device)+ loss = cross_entropy_loss.fast_cross_entropy_loss(logits, labels)+ loss.backward()+ out[name] = (loss.detach(), logits.grad)++ e, gg, dw = rnd(32, 512), rnd(32, 512), rnd(32, 512)+ e3, g3 = e.view(2, 16, 512), gg.view(2, 16, 512)+ out["swiglu_fwd"] = (swiglu.swiglu_fg_kernel(e3, g3),)+ out["swiglu_bwd"] = tuple(+ t.clone() for t in swiglu.swiglu_DWf_DW_dfg_kernel(dw.clone(), e.clone(), gg.clone())+ )+ out["geglu_exact_fwd"] = (geglu.geglu_exact_forward_kernel(e3, g3),)+ out["geglu_exact_bwd"] = tuple(+ t.clone() for t in geglu.geglu_exact_backward_kernel(dw.clone(), e.clone(), gg.clone())+ )+ out["geglu_approx_fwd"] = (geglu.geglu_approx_forward_kernel(e3, g3),)+ out["geglu_approx_bwd"] = tuple(+ t.clone() for t in geglu.geglu_approx_backward_kernel(dw.clone(), e.clone(), gg.clone())+ )++ # Hand-built NF4 state so bitsandbytes is not needed.+ n, kk, bs, bs2 = 256, 512, 64, 256+ W_u8 = torch.randint(+ 0, 256, (n * kk // 2, 1), generator = g, device = device, dtype = torch.int32+ ).to(torch.uint8)+ n_blocks = n * kk // bs+ code = torch.linspace(-1, 1, 16, device = device)+ code2 = torch.linspace(-1, 1, 256, device = device)+ absmax_q = torch.randint(0, 256, (n_blocks,), generator = g, device = device, dtype = torch.int32).to(+ torch.uint8+ )+ absmax2 = torch.rand(n_blocks // bs2, generator = g, device = device) + 0.5+ offset = torch.tensor(0.1, device = device)+ args = (W_u8, absmax_q, code2, absmax2, offset, code, bs, bs2)+ out["nf4_dequant_eager"] = (nf4.dequantize_nf4(*args, (n, kk), dt),)+ out["nf4_dequant_op"] = (torch.ops.unsloth.dequantize_nf4(*args, [n, kk], dt),)+ Xv = rnd(1, 1, kk)+ out["nf4_gemv"] = (nf4_gemv.gemv_nf4(Xv, *args, (n, kk), dt),)++ Wf = (rnd(512, 1024, dtype = torch.float32) / 4).to(torch.float8_e4m3fn)+ if not fp8._fp8_kernel_unsupported(Wf):+ scale = torch.rand(512, 1, generator = g, device = device) + 0.5+ out["fp8_gemv"] = (fp8.fp8_rowwise_gemv(rnd(1, 1, 1024), Wf, scale),)+ torch.cuda.synchronize()++ modules = (cross_entropy_loss, fp8, geglu, nf4, nf4_gemv, rms_layernorm, rope_embedding, swiglu)+ meta = {}+ for module in modules:+ for attr, value in vars(module).items():+ options = (+ _compiled_options(value) if hasattr(value, "run") or hasattr(value, "fn") else []+ )+ if options:+ meta[f"{module.__name__.rsplit('.', 1)[-1]}.{attr}"] = options+ return out, meta+++def _child():+ import torch++ import unsloth # noqa: F401++ from unsloth.kernels import nf4++ _, meta = exercise()+ print(+ "A1META "+ + json.dumps(+ {"meta": meta, "hip": bool(torch.version.hip), "mul_rn": bool(nf4._HAS_MUL_RN)}+ )+ )+++def _run_child(tmp_path, default):+ env = dict(os.environ)+ env["TRITON_DEFAULT_FP_FUSION"] = default+ env["TRITON_CACHE_DIR"] = str(tmp_path / f"triton_{default}")+ env.pop("TRITON_FORCE_DISABLE_FP_FUSION", None)+ code = f"import sys; sys.path.insert(0, {str(pathlib.Path(__file__).parent)!r}); import test_triton_fp_fusion as t; t._child()"+ proc = subprocess.run(+ [sys.executable, "-c", code], env = env, capture_output = True, text = True, timeout = 900+ )+ line = [l for l in proc.stdout.splitlines() if l.startswith("A1META ")]+ assert proc.returncode == 0 and line, proc.stdout[-3000:] + proc.stderr[-6000:]+ return json.loads(line[-1][len("A1META ") :])+++def _needs_gpu():+ try:+ sys.path.insert(0, str(pathlib.Path(__file__).parent / "_shared"))+ from real_accelerator import has_real_cuda+ real = has_real_cuda()+ except Exception:+ import torch+ real = torch.cuda.is_available()+ if not real:+ pytest.skip("needs a CUDA / ROCm GPU")+ pytest.importorskip("triton")++[email protected]("default", ["0", "1"])+def test_compiled_kernels_keep_fp_fusion(tmp_path, default):+ _needs_gpu()+ result = _run_child(tmp_path, default)+ meta = result["meta"]+ expected = {+ "rms_layernorm._rms_layernorm_forward",+ "rms_layernorm._rms_layernorm_backward",+ "rope_embedding._rope_embedding",+ "rope_embedding._rope_embedding_QK",+ "cross_entropy_loss._cross_entropy_forward",+ "cross_entropy_loss._chunked_cross_entropy_forward",+ "cross_entropy_loss._cross_entropy_backward",+ "swiglu._fg_kernel",+ "swiglu._DWf_DW_dfg_kernel",+ "geglu._exact_forward_kernel",+ "geglu._exact_backward_kernel",+ "geglu._approx_forward_kernel",+ "geglu._approx_backward_kernel",+ "nf4._nf4_dequant_kernel",+ }+ # Multi-row RMSNorm kernels replace the one-row ones on CUDA for narrow rows.+ if "rms_layernorm._rms_layernorm_forward_rows" in meta:+ expected.discard("rms_layernorm._rms_layernorm_forward")+ if "rms_layernorm._rms_layernorm_backward_rows" in meta:+ expected.discard("rms_layernorm._rms_layernorm_backward")+ assert expected <= set(meta), sorted(expected - set(meta))+ assert any(name.startswith("nf4_gemv.") for name in meta), sorted(meta)++ nf4_values = {value for value, _ in meta.pop("nf4._nf4_dequant_kernel")}+ # Eager dequant stays deliberately unfused; the traced op fuses where mul.rn exists.+ assert False in nf4_values+ assert (True in nf4_values) == result["mul_rn"], nf4_values+ bad = {name: opts for name, opts in meta.items() if any(value is not True for value, _ in opts)}+ assert not bad, f"compiled without enable_fp_fusion=True: {bad}"+ if not result["hip"]:+ # Contraction really happened: fma.rn in PTX.+ for name in (+ "rope_embedding._rope_embedding",+ "geglu._exact_forward_kernel",+ "swiglu._DWf_DW_dfg_kernel",+ ):+ assert all(fma for _, fma in meta[name]), (name, meta[name])diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 038109083ca..38fc705dddc 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -333,6 +333,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):DO_LOGIT_SCALING = DO_LOGIT_SCALING,LOGIT_SCALE = logit_scaling,num_warps = num_warps,+ enable_fp_fusion = True,)else:# For large vocabs > 65336 like Gemma 256K@@ -365,6 +366,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):DO_LOGIT_SCALING = DO_LOGIT_SCALING,LOGIT_SCALE = logit_scaling,num_warps = 32 if not is_cdna() else 16,+ enable_fp_fusion = True,)# logsumexp(chunked_logsumexp) - x, with the -x done separately.logsumexp = torch.logsumexp(logsumexp, dim = 1)@@ -414,6 +416,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):DO_LOGIT_SCALING = ctx.DO_LOGIT_SCALING,LOGIT_SCALE = ctx.logit_scaling,num_warps = 8,+ enable_fp_fusion = True,)return (dlogits,diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 24b73445261..be3858e4eb4 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -165,7 +165,7 @@ def weight_dequant_block(triton.cdiv(N, meta["BLOCK_SIZE"]),)with _fp8_triton_device_context(x):- weight_dequant_kernel[grid](x, s, y, M, N, BLOCK_SIZE = block_size)+ weight_dequant_kernel[grid](x, s, y, M, N, BLOCK_SIZE = block_size, enable_fp_fusion = True)return y@@ -257,7 +257,9 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, torreturn (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)with _fp8_triton_device_context(x):- act_quant_kernel[grid](x, y, s, BLOCK_SIZE = block_size, LONG_INDEXING = long_indexing(x))+ act_quant_kernel[grid](+ x, y, s, BLOCK_SIZE = block_size, LONG_INDEXING = long_indexing(x), enable_fp_fusion = True+ )return y, s@@ -420,6 +422,7 @@ def w8a8_block_fp8_matmul_triton(LONG_INDEXING = long_indexing(A, B, C),# Default 4 warps starve 128x128 tiles (8 is 1.3-2.1x faster from M >= 128); smaller decode tiles gain nothing.num_warps = 8 if BLOCK_SIZE_M == 128 else 4,+ enable_fp_fusion = True,)return C@@ -557,7 +560,17 @@ def fp8_rowwise_gemv(X, weight, weight_scale):block_n = 1 if N <= 4096 else (4 if N <= 32768 else 8)with _fp8_triton_device_context(X, static_device_count = True):_fp8_rowwise_gemv_kernel[(triton.cdiv(N, block_n),)](- X, weight, scale, out, N, K, weight.stride(0), BLOCK_N = block_n, BLOCK_K = 512, num_warps = 2+ X,+ weight,+ scale,+ out,+ N,+ K,+ weight.stride(0),+ BLOCK_N = block_n,+ BLOCK_K = 512,+ num_warps = 2,+ enable_fp_fusion = True,)return outdiff --git a/unsloth/kernels/geglu.py b/unsloth/kernels/geglu.pyindex de6ab070c77..e6884506ac2 100644--- a/unsloth/kernels/geglu.py+++ b/unsloth/kernels/geglu.py@@ -62,6 +62,7 @@ def geglu_exact_forward_kernel(gate, up):n_elements,BLOCK_SIZE = BLOCK_SIZE,LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,+ enable_fp_fusion = True,)return out@@ -128,6 +129,7 @@ def geglu_exact_backward_kernel(DW, e, g):n_elements,BLOCK_SIZE = BLOCK_SIZE,LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,+ enable_fp_fusion = True,)return DW, e, g@@ -171,6 +173,7 @@ def geglu_approx_forward_kernel(gate, up):n_elements,BLOCK_SIZE = BLOCK_SIZE,LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,+ enable_fp_fusion = True,)return out@@ -245,5 +248,6 @@ def geglu_approx_backward_kernel(DW, e, g):n_elements,BLOCK_SIZE = BLOCK_SIZE,LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,+ enable_fp_fusion = True,)return DW, e, gdiff --git a/unsloth/kernels/int4_packed.py b/unsloth/kernels/int4_packed.pyindex 33165ca3b3b..19de18012a7 100644--- a/unsloth/kernels/int4_packed.py+++ b/unsloth/kernels/int4_packed.py@@ -315,7 +315,9 @@ def _launch(kernel, launchers, key, grid, args, **constexprs):if _FAST_LAUNCH is not False:compiled = launchers.get(key)if compiled is None:- compiled = launchers[key] = kernel.warmup(*args, grid = grid, **constexprs)+ compiled = launchers[key] = kernel.warmup(+ *args, grid = grid, **constexprs, enable_fp_fusion = True+ )try:compiled[grid](*args)_FAST_LAUNCH = True@@ -324,7 +326,7 @@ def _launch(kernel, launchers, key, grid, args, **constexprs):if _FAST_LAUNCH:raise_FAST_LAUNCH = False- kernel[grid](*args, **constexprs)+ kernel[grid](*args, **constexprs, enable_fp_fusion = True)def int4_dequantize(@@ -619,6 +621,7 @@ def _dequant_repacked(words, qs.scale, qs.scale if zp is None else zp, out, NT_N, NT_K, qs.scale.stride(0),0 if zp is None else zp.stride(0), out.stride(0), MARLIN = qs.layout == "marlin", TN = TN, TK = TK,R = R, GROUP = group, HAS_Z = zp is not None, RAW = raw, num_warps = 4,+ enable_fp_fusion = True,) # fmt: skipreturn outdiff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex a9a33c0e32f..d693493c2e6 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -236,6 +236,7 @@ class Fast_Layernorm(torch.autograd.Function):BLOCK_SIZE = BLOCK_SIZE,HAS_BIAS = has_bias,num_warps = num_warps,+ enable_fp_fusion = True,)ctx.eps = epsctx.BLOCK_SIZE = BLOCK_SIZE@@ -269,6 +270,7 @@ class Fast_Layernorm(torch.autograd.Function):LONG_INDEXING = LONG_INDEXING,BLOCK_SIZE = ctx.BLOCK_SIZE,num_warps = ctx.num_warps,+ enable_fp_fusion = True,)return dY.view(*shape), None, None, None@@ -301,6 +303,7 @@ class Fast_Layernorm(torch.autograd.Function):TRAIN_W = train_w,TRAIN_B = train_b,num_warps = ctx.num_warps,+ enable_fp_fusion = True,)dW = torch.empty(n_cols, dtype = W.dtype, device = dY.device) if train_w else NonedB = torch.empty(n_cols, dtype = b.dtype, device = dY.device) if train_b else None@@ -314,6 +317,7 @@ class Fast_Layernorm(torch.autograd.Function):n_cols,BLOCK_ROWS = 64,BLOCK_COLS = 64,+ enable_fp_fusion = True,)return dY.view(*shape), dW, dB, Nonediff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex 62d76ddbe3a..3b935e5e5c3 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -264,7 +264,9 @@ def _grouped_gemm_forward_impl(is_fake = _is_tracing(X, W)if not is_fake:- compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](**kernel_args)+ compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](+ **kernel_args, enable_fp_fusion = True+ )if autotune:log_kernel_info(compiled_kernel, kernel.best_config)else:@@ -404,7 +406,9 @@ def _grouped_gemm_dX_impl(is_fake = _is_tracing(dY, W)if not is_fake:- compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](**kernel_args)+ compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](+ **kernel_args, enable_fp_fusion = True+ )if autotune:log_kernel_info(compiled_kernel, kernel.best_config)@@ -550,7 +554,9 @@ def _grouped_gemm_dW_impl(is_fake = _is_tracing(X, dY)if not is_fake:- compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](**kernel_args)+ compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](+ **kernel_args, enable_fp_fusion = True+ )if autotune:log_kernel_info(compiled_kernel, kernel.best_config)diff --git a/unsloth/kernels/nf4.py b/unsloth/kernels/nf4.pyindex b2d75ea1a06..e1a0a91b9ed 100644--- a/unsloth/kernels/nf4.py+++ b/unsloth/kernels/nf4.py@@ -262,11 +262,8 @@ def _launch(LUT_MODE = lut_mode,)grid = (-(-n_blocks // rows),)- options = (- {"num_warps": num_warps}- if fp_fusion- else {"num_warps": num_warps, "enable_fp_fusion": False}- )+ # Explicit both ways: Triton 3.9 defaults enable_fp_fusion to False.+ options = {"num_warps": num_warps, "enable_fp_fusion": bool(fp_fusion)}if kernel is _nf4_dequant_kernel:entry = launch(kernel, grid, args, 7, constexprs, W.device.index, **options)if plan:diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.pyindex 52de268b6b8..82ba41b5662 100644--- a/unsloth/kernels/nf4_gemv.py+++ b/unsloth/kernels/nf4_gemv.py@@ -260,11 +260,20 @@ def _launch(BLOCK_K = block_k,)if kernel is _gemv_nf4_kernel or kernel is _gemv_nf4_words_kernel:- entry = launch(kernel, grid, args, 8, constexprs, X.device.index, num_warps = num_warps)+ entry = launch(+ kernel,+ grid,+ args,+ 8,+ constexprs,+ X.device.index,+ num_warps = num_warps,+ enable_fp_fusion = True,+ )if plan:return _rerun(entry, grid, args, X.device.index)else:- kernel[grid](*args, **constexprs, num_warps = num_warps)+ kernel[grid](*args, **constexprs, num_warps = num_warps, enable_fp_fusion = True)return outdiff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.pyindex 42d88a079e4..98d3ebfcd78 100644--- a/unsloth/kernels/nvfp4.py+++ b/unsloth/kernels/nvfp4.py@@ -145,6 +145,7 @@ def _nvfp4_dequantize_triton(# int32 offsets wrap once the output passes 2^31 elements; int64 only then, as it costs ~2%.INT64_OFFSETS = rows * half * 2 >= 2**31,num_warps = 4,+ enable_fp_fusion = True,)return outdiff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 81409172a34..038cfd5f21c 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -590,6 +590,7 @@ def _rms_forward(# int32 rows measured slower only for the Gemma multi-row forward: keep main's int64.LONG_INDEXING = long or gemma,num_warps = num_warps,+ enable_fp_fusion = True,)return Y, rexcept Exception as error:@@ -612,6 +613,7 @@ def _rms_forward(LONG_INDEXING = long,BLOCK_SIZE = BLOCK_SIZE,num_warps = num_warps,+ enable_fp_fusion = True,)return Y, r@@ -706,6 +708,7 @@ def _rms_backward(LONG_INDEXING = long,TRAIN_W = train_w,num_warps = num_warps,+ enable_fp_fusion = True,)except Exception as error:if _multirow_must_raise(error, wrap):@@ -742,6 +745,7 @@ def _rms_backward(TRAIN_W = train_w,num_warps = num_warps,**launch,+ enable_fp_fusion = True,)return _rms_sum_partials(partials, W, wrap)@@ -752,7 +756,13 @@ def _rms_sum_partials(partials, W, wrap):programs, n_cols = partials.shapedW = torch.empty(n_cols, dtype = W.dtype, device = partials.device)wrap(_rms_sum_dw_partials)[(triton.cdiv(n_cols, 64),)](- partials, dW, programs, n_cols, BLOCK_ROWS = 64, BLOCK_COLS = 64+ partials,+ dW,+ programs,+ n_cols,+ BLOCK_ROWS = 64,+ BLOCK_COLS = 64,+ enable_fp_fusion = True,)return dWdiff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 9dfe2a5d05c..181367a6d3c 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -215,6 +215,7 @@ def _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, backward, wrap):LONG_INDEXING = long_indexing(Q, block = BLOCK_SIZE),BLOCK_SIZE = BLOCK_SIZE,num_warps = num_warps,+ enable_fp_fusion = True,)@@ -297,6 +298,7 @@ def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):BLOCK_SIZE = BLOCK_SIZE,EVICT_INDICES = _eviction_hints_ok(Q.device),num_warps = num_warps,+ enable_fp_fusion = True,)diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.pyindex 2078f3266e0..13b2ea33ebc 100644--- a/unsloth/kernels/swiglu.py+++ b/unsloth/kernels/swiglu.py@@ -56,6 +56,7 @@ def swiglu_fg_kernel(e, g):n_elements,BLOCK_SIZE = BLOCK_SIZE,LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,+ enable_fp_fusion = True,)return h@@ -129,5 +130,6 @@ def swiglu_DWf_DW_dfg_kernel(DW, e, g):n_elements,BLOCK_SIZE = BLOCK_SIZE,LONG_INDEXING = 0 if n_elements <= INT32_SAFETY_BUFFER else 1,+ enable_fp_fusion = True,)return DW, e, g