Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Speed up block-FP8 LoRA training: run FP8 linears eagerly, 8 warps for 128-row GEMM tiles (#12027)
Daniel Han7c8e50a573f0
diff --git a/tests/test_fp8_compile_fullgraph.py b/tests/test_fp8_compile_fullgraph.pynew file mode 100644index 00000000000..c3f98d73529--- /dev/null+++ b/tests/test_fp8_compile_fullgraph.py@@ -0,0 +1,40 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Block-FP8 linears sit inside compiled modules (e.g. Gemma 3 attention / MLP), so they must trace without graph breaks.+import os++import pytest++torch = pytest.importorskip("torch")++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),+ reason = "block-FP8 Triton kernels need CUDA sm89+",+)+++def test_block_fp8_linear_compiles_fullgraph_and_matches_eager():+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")+ import torch._dynamo+ from unsloth.kernels import fp8 as F++ def step(X, W, s):+ return F.fp8_linear(X, W, s).float().square().mean()++ torch._dynamo.reset()+ compiled = torch.compile(step, fullgraph = True)+ # Several weight shapes (the second triggers automatic dynamic shapes); K = 200 takes the dequant fallback.+ for N, K in [(512, 256), (256, 256), (256, 768), (768, 256), (512, 200)]:+ torch.manual_seed(0)+ X = torch.randn(256, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ W = (torch.randn(N, K, device = "cuda") * 0.02).to(torch.float8_e4m3fn)+ s = torch.rand(-(-N // 128), -(-K // 128), device = "cuda") + 0.5+ s.block_size = [128, 128]++ compiled(X, W, s).backward()+ grad_compiled, X.grad = X.grad, None+ step(X, W, s).backward()+ assert X.grad.norm() > 0+ torch.testing.assert_close(grad_compiled, X.grad, rtol = 0, atol = 0)++ explain = torch._dynamo.explain(step)(X, W, s)+ assert explain.graph_break_count == 0, explain.break_reasonsdiff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 92973106d4a..e881cc1e6d5 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -46,6 +46,41 @@ def _fp8_triton_device_context(tensor: torch.Tensor, static_device_count = Falsereturn nullcontext()+# Module level: a torch.library.Library deregisters its ops once collected.+_fp8_library = None+++def _opaque_under_compile(name, fake):+ # torch.compile swaps the launcher for one registered op: tracing it breaks the graph (device_count) and recompiles+ # the Triton kernel under dynamic shapes 40-80x slower. Eager calls still run the plain function, at no cost.+ # torch.library.Library, not custom_op: compiled graphs dispatch it with ~3 us per call instead of ~23 us.+ def decorator(fn):+ global _fp8_library+ substitute_in_graph = getattr(torch._dynamo, "substitute_in_graph", None)+ if substitute_in_graph is None or not hasattr(torch.library, "infer_schema"):+ return fn+ try:+ if _fp8_library is None:+ _fp8_library = torch.library.Library("unsloth", "FRAGMENT")+ _fp8_library.define(name + torch.library.infer_schema(fn, mutates_args = ()))+ _fp8_library.impl(name, fn, "CompositeExplicitAutograd")+ torch.library.register_fake(f"unsloth::{name}", fake, lib = _fp8_library)+ substitute_in_graph(fn)(+ functools.wraps(fn)(+ lambda *args, **kwargs: getattr(torch.ops.unsloth, name)(*args, **kwargs)+ )+ )+ except (+ RuntimeError,+ TypeError,+ ValueError,+ ): # Already registered by an earlier import of this module.+ pass+ return fn++ return decorator++try:from transformers.integrations.finegrained_fp8 import FP8Linearexcept:@@ -104,11 +139,17 @@ def weight_dequant_kernel(x_ptr, s_ptr, y_ptr, M, N, BLOCK_SIZE: tl.constexpr):tl.store(y_ptr + offs, y, mask = mask)+@_opaque_under_compile(+ "fp8_weight_dequant_block",+ lambda x, s, block_size = 128, dtype = torch.bfloat16: torch.empty_like(+ x, dtype = dtype, memory_format = torch.contiguous_format+ ),+)def weight_dequant_block(x: torch.Tensor,s: torch.Tensor,block_size: int = 128,- dtype = torch.bfloat16,+ dtype: torch.dtype = torch.bfloat16,) -> torch.Tensor:if not x.is_contiguous():x = x.contiguous()@@ -187,6 +228,13 @@ def act_quant_kernel(x_ptr, y_ptr, s_ptr, BLOCK_SIZE: tl.constexpr):tl.store(s_ptr + pid, s)+@_opaque_under_compile(+ "fp8_act_quant",+ lambda x, block_size = 128: (+ x.new_empty(x.shape, dtype = torch.float8_e4m3fn),+ x.new_empty(*x.shape[:-1], x.shape[-1] // block_size, dtype = torch.float32),+ ),+)def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, torch.Tensor]:if not x.is_contiguous():x = x.contiguous()@@ -284,6 +332,12 @@ def _w8a8_block_fp8_matmul(tl.store(c_ptrs, c, mask = c_mask)+@_opaque_under_compile(+ "fp8_block_matmul_triton",+ lambda A, B, As, Bs, block_size, output_dtype = torch.float32: A.new_empty(+ A.shape[:-1] + (B.shape[0],), dtype = output_dtype+ ),+)def w8a8_block_fp8_matmul_triton(A: torch.Tensor,B: torch.Tensor,@@ -345,18 +399,26 @@ def w8a8_block_fp8_matmul_triton(BLOCK_SIZE_N = BLOCK_SIZE_N,BLOCK_SIZE_K = BLOCK_SIZE_K,GROUP_SIZE_M = 8,+ # Default 4 warps starve 128x128 tiles (8 is 1.3-2.1x faster from M >= 128); smaller decode tiles gain nothing.+ num_warps = 8 if BLOCK_SIZE_M == 128 else 4,)return C+@_opaque_under_compile(+ "fp8_block_matmul_torchao",+ lambda A, B, As, Bs, block_size, output_dtype = torch.bfloat16: A.new_empty(+ A.shape[:-1] + (B.shape[0],), dtype = output_dtype+ ),+)def torchao_block_matmul(act_q: torch.Tensor,weight_q: torch.Tensor,act_scale: torch.Tensor,weight_scale: torch.Tensor,- block_size: tuple[int, int],+ block_size: list[int],output_dtype: torch.dtype = torch.bfloat16,-):+) -> torch.Tensor:with _fp8_triton_device_context(act_q):out = torchao_blockwise_gemm(act_q.contiguous(),@@ -556,7 +618,8 @@ class FP8BlockQuantLinear(torch.autograd.Function):ctx.weight = weightctx.weight_scale = original_weight_scalectx.block_size = block_size- return torch_matmul(X, W_deq.T).to(X.dtype)+ output = torch_matmul(X, W_deq.T)+ return output if output.dtype == X.dtype else output.to(X.dtype)qinput, scale = act_quant(X, block_size[1])output = fp8_block_matmul(@@ -570,7 +633,8 @@ class FP8BlockQuantLinear(torch.autograd.Function):ctx.weight = weightctx.weight_scale = original_weight_scalectx.block_size = block_size- return output.to(X.dtype)+ # No no-op .to(): torch 2.11's compiled autograd.Function returns zero dX when the output aliases an intermediate.+ return output if output.dtype == X.dtype else output.to(X.dtype)@staticmethoddef backward(ctx, grad_output):@@ -582,7 +646,8 @@ class FP8BlockQuantLinear(torch.autograd.Function):return grad_X, None, None-@torch_compile+# Not torch.compiled: after a model's second weight shape, automatic dynamic shapes recompile the+# user Triton kernel into a version 40-80x slower than eager (the only work here is that kernel).def fp8_torch_block_quant_forward(X, weight, weight_scale):return FP8BlockQuantLinear.apply(X, weight, weight_scale)@@ -831,7 +896,7 @@ except:pass-@torch_compile+# Not torch.compiled, for the same reason as fp8_torch_block_quant_forward.def fp8_linear(X,weight,