Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Triton RoPE (auto on ROCm and NVIDIA CUDA Linux) and AdaLN kernels for FLUX; inference only,
never while compiling; AdaLN auto on NVIDIA CUDA Linux off the ``off`` speed tier (one rounding instead of three, not
bit-identical); UNSLOTH_DIFFUSION_FUSED_ROPE, UNSLOTH_DIFFUSION_FUSED_ADALN"""
from __future__ import annotations
import functools
import logging
import os
import sys
import threading
from typing import Any, Callable, Optional
from .rocm_bf16 import is_rocm_torch
FUSED_ROPE_ENV = "UNSLOTH_DIFFUSION_FUSED_ROPE"
FUSED_ADALN_ENV = "UNSLOTH_DIFFUSION_FUSED_ADALN"
_MODULES = (
"diffusers.models.transformers.transformer_flux",
"diffusers.models.transformers.transformer_flux2",
)
_ROPE_ATTR = "apply_rotary_emb"
_ADALN_CLASSES = ("AdaLayerNormZero", "AdaLayerNormZeroSingle", "AdaLayerNormContinuous")
_MAX_D = 16384
_LOCK = threading.Lock()
_ROPE_STOCK: dict = {}
_ADALN_PREV: dict = {}
# Survives uninstall, so a fused forward restored later by another layer still reaches a real forward.
_ADALN_ORIGINAL: dict = {}
COUNTS = {"rope_fused": 0, "rope_stock": 0, "adaln_fused": 0, "adaln_stock": 0}
# Set by the first failed RoPE launch (JIT / driver / PTX): the rest of the load runs stock, not one failing JIT per call.
_ROPE_FAILED = [False]
# Same latch for the fused AdaLN modulation launch.
_ADALN_FAILED = [False]
def _log_failed(path: str, exc: BaseException) -> None:
try:
logging.getLogger(__name__).warning(
"diffusion.rocm_fused: fused %s launch failed (%s: %s); stock %s for the rest of this load",
path,
type(exc).__name__,
exc,
path,
)
except Exception: # noqa: BLE001
pass
def _mode(env: str) -> str:
v = (os.environ.get(env) or "auto").strip().lower()
if v in ("0", "off", "false", "no"):
return "off"
if v in ("1", "on", "true", "yes", "force"):
return "on"
return "auto"
def _is_rocm() -> bool:
try:
import torch
except Exception: # noqa: BLE001
return False
return is_rocm_torch(torch)
def _is_nvidia_linux() -> bool:
"""NVIDIA CUDA torch on Linux (WSL included). Windows keeps stock: a Triton JIT failure would retry per call."""
if not sys.platform.startswith("linux"):
return False
try:
import torch
except Exception: # noqa: BLE001
return False
return not is_rocm_torch(torch) and bool(getattr(torch.version, "cuda", None))
# gfx1151 klein 1024^2: RoPE 8.76 -> 8.09 s pixel-identical; AdaLN adds no speed and moves pixels (LPIPS 0.004).
_AUTO_ON = {FUSED_ROPE_ENV: True, FUSED_ADALN_ENV: False}
_AUTO_ON_NVIDIA = {FUSED_ROPE_ENV: True, FUSED_ADALN_ENV: True}
def _switch_on(env: str) -> bool:
mode = _mode(env)
if mode == "on":
return True
if mode != "auto":
return False
if _AUTO_ON.get(env, False) and _is_rocm():
return True
return _AUTO_ON_NVIDIA.get(env, False) and _is_nvidia_linux()
def wanted(
env: str,
dtype: Any,
device: Any = "cuda",
) -> bool:
if not _switch_on(env):
return False
try:
import torch
except Exception: # noqa: BLE001
return False
if dtype not in (torch.bfloat16, torch.float16) or not str(device).startswith("cuda"):
return False
if env == FUSED_ROPE_ENV:
return _rope_kernel() is not None
return _adaln_kernel() is not None
@functools.lru_cache(maxsize = 1)
def _rope_kernel() -> Optional[Callable]:
from .diffusion_flux2_rope import _kernel
return _kernel()
def _rope_eligible(
x: Any, freqs_cis: Any, use_real: bool, use_real_unbind_dim: int, sequence_dim: int
) -> bool:
import torch
if not use_real or use_real_unbind_dim != -1 or sequence_dim != 1:
return False
if not isinstance(freqs_cis, (tuple, list)) or len(freqs_cis) != 2:
return False
if (
not torch.is_tensor(x)
or x.dtype not in (torch.bfloat16, torch.float16)
or not x.is_cuda
or x.dim() != 4
):
return False
if x.requires_grad and torch.is_grad_enabled():
return False
B, S, H, D = x.shape
if D % 2 or x.stride(3) != 1 or x.numel() == 0:
return False
# Kernel offsets are int32.
if x.storage_offset() + sum((n - 1) * st for n, st in zip(x.shape, x.stride())) >= 2**31 - 1:
return False
for f in freqs_cis:
if not torch.is_tensor(f) or f.dim() != 2 or tuple(f.shape) != (S, D):
return False
if f.device != x.device or f.stride(1) != 1 or f.dtype is not torch.float32:
return False
return True
def _make_rope(module_name: str) -> Callable:
def _fused_apply_rotary_emb(
x,
freqs_cis,
use_real: bool = True,
use_real_unbind_dim: int = -1,
sequence_dim: int = 2,
):
import torch
stock = _ROPE_STOCK.get(module_name) or _ROPE_ORIGINAL[module_name]
# Checked first: Dynamo graph-breaks on Triton's import path, so a compiled block stays whole on stock.
if not torch.compiler.is_compiling():
launch = None if _ROPE_FAILED[0] else _rope_kernel()
if launch is not None and _rope_eligible(
x, freqs_cis, use_real, use_real_unbind_dim, sequence_dim
):
try:
out = launch(x, freqs_cis[0], freqs_cis[1])
COUNTS["rope_fused"] += 1
return out
except Exception as exc: # noqa: BLE001 -- never fail a render over the fast path
_ROPE_FAILED[0] = True
_log_failed("RoPE", exc)
COUNTS["rope_stock"] += 1
return stock(
x,
freqs_cis,
use_real = use_real,
use_real_unbind_dim = use_real_unbind_dim,
sequence_dim = sequence_dim,
)
_fused_apply_rotary_emb.__unsloth_fused_rope__ = module_name
return _fused_apply_rotary_emb
# Survives uninstall so a forward that looked up the patch just before teardown still reaches stock.
_ROPE_ORIGINAL: dict = {}
_ROPE_FNS: dict = {m: _make_rope(m) for m in _MODULES}
def install_rope(
module_name: str,
dtype: Any,
device: Any = "cuda",
) -> bool:
if module_name not in _ROPE_FNS or not wanted(FUSED_ROPE_ENV, dtype, device):
uninstall_rope()
return False
import importlib
with _LOCK:
if module_name in _ROPE_STOCK:
return True
try:
mod = importlib.import_module(module_name)
except Exception: # noqa: BLE001
return False
stock = getattr(mod, _ROPE_ATTR, None)
# Anything but diffusers' own function (another fused patch, a changed symbol) is left alone.
if stock is None or getattr(stock, "__name__", "") != "apply_rotary_emb":
return False
_ROPE_STOCK[module_name] = stock
_ROPE_ORIGINAL[module_name] = stock
setattr(mod, _ROPE_ATTR, _ROPE_FNS[module_name])
return True
def uninstall_rope() -> None:
with _LOCK:
_ROPE_FAILED[0] = False
for name in list(_ROPE_STOCK):
stock = _ROPE_STOCK.pop(name)
mod = sys.modules.get(name)
if mod is not None and getattr(mod, _ROPE_ATTR, None) is _ROPE_FNS[name]:
setattr(mod, _ROPE_ATTR, stock)
@functools.lru_cache(maxsize = 1)
def _adaln_kernel() -> Optional[Callable]:
try:
import triton
import triton.language as tl
except Exception: # noqa: BLE001
return None
@triton.jit(do_not_specialize = ["S"])
def _adaln_fwd(
X,
SCALE,
SHIFT,
OUT,
S,
D,
sxb,
sxs,
scb,
shb,
sob,
sos,
eps,
BLOCK: tl.constexpr, # block size along hidden
):
row = tl.program_id(0)
b = row // S
s = row - b * S
cols = tl.arange(0, BLOCK)
mask = cols < D
x = tl.load(X + b * sxb + s * sxs + cols, mask = mask, other = 0.0).to(tl.float32)
mean = tl.sum(x, axis = 0) / D
xc = tl.where(mask, x - mean, 0.0)
var = tl.sum(xc * xc, axis = 0) / D
rstd = 1.0 / tl.sqrt(var + eps)
dt = OUT.dtype.element_ty
# stock: layer_norm (rounded) * (1 + scale) (rounded) -> product (rounded) + shift (rounded)
n = (xc * rstd).to(dt)
sc = tl.load(SCALE + b * scb + cols, mask = mask, other = 0.0).to(tl.float32)
m = (1.0 + sc).to(dt)
p = (n.to(tl.float32) * m.to(tl.float32)).to(dt)
sh = tl.load(SHIFT + b * shb + cols, mask = mask, other = 0.0).to(tl.float32)
o = (p.to(tl.float32) + sh).to(dt)
tl.store(OUT + b * sob + s * sos + cols, o, mask = mask)
def launch(x: Any, scale: Any, shift: Any, eps: float) -> Any:
import torch
B, S, D = x.shape
out = torch.empty((B, S, D), dtype = x.dtype, device = x.device)
block = triton.next_power_of_2(D)
with torch.cuda.device(x.device):
_adaln_fwd[(B * S,)](
x,
scale,
shift,
out,
S,
D,
x.stride(0),
x.stride(1),
scale.stride(0),
shift.stride(0),
out.stride(0),
out.stride(1),
float(eps),
BLOCK = block,
num_warps = 4 if block <= 1024 else (8 if block <= 4096 else 16),
enable_fp_fusion = False,
)
return out
return launch
def _plain_layer_norm(norm: Any) -> Optional[float]:
"""eps when ``norm`` is a plain ``nn.LayerNorm`` with no affine over the last dim, else None."""
import torch
if type(norm) is not torch.nn.LayerNorm or norm.weight is not None or norm.bias is not None:
return None
if len(tuple(norm.normalized_shape)) != 1:
return None
return float(norm.eps)
def _adaln_eligible(norm: Any, x: Any, scale: Any, shift: Any) -> Optional[float]:
"""The LayerNorm eps when the fused kernel can take this call, else None."""
import torch
eps = _plain_layer_norm(norm)
if eps is None or not torch.is_tensor(x) or not x.is_cuda or x.dim() != 3:
return None
if torch.is_grad_enabled() and (x.requires_grad or scale.requires_grad or shift.requires_grad):
return None
B, S, D = x.shape
if (
x.dtype not in (torch.bfloat16, torch.float16)
or D > _MAX_D
or D != norm.normalized_shape[0]
or x.numel() == 0
):
return None
if x.stride(2) != 1 or x.numel() >= 2**31 - 1:
return None
for t in (scale, shift):
if (
not torch.is_tensor(t)
or t.dtype is not x.dtype
or t.device != x.device
or tuple(t.shape) != (B, D)
):
return None
if t.stride(1) != 1:
return None
return eps
def _count(key: str) -> None:
"""Eager census only: a counter bumped under Dynamo becomes a guard on its value, so every call recompiles."""
import torch
if not torch.compiler.is_compiling():
COUNTS[key] += 1
def _fused_modulate(norm: Any, x: Any, scale: Any, shift: Any) -> Optional[Any]:
"""``norm(x) * (1 + scale[:, None]) + shift[:, None]`` in one kernel, or None when not eligible."""
import torch
if torch.compiler.is_compiling() or _ADALN_FAILED[0]:
return None
eps = _adaln_eligible(norm, x, scale, shift)
if eps is None:
return None
launch = _adaln_kernel()
if launch is None:
return None
try:
return launch(x, scale, shift, eps)
except Exception as exc: # noqa: BLE001 -- never fail a render over the fast path
_ADALN_FAILED[0] = True
_log_failed("AdaLN", exc)
return None
def _prev(cls: type) -> Callable:
prev = _ADALN_PREV.get(cls)
return prev if prev is not None else _ADALN_ORIGINAL[cls]
def _adaln_zero_forward(
self,
x,
timestep = None,
class_labels = None,
hidden_dtype = None,
emb = None,
):
cls = _CLASSES["AdaLayerNormZero"]
if self.emb is not None or timestep is not None or class_labels is not None:
_count("adaln_stock")
return _prev(cls)(self, x, timestep, class_labels, hidden_dtype, emb)
e = self.linear(self.silu(emb))
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = e.chunk(6, dim = 1)
out = _fused_modulate(self.norm, x, scale_msa, shift_msa)
if out is None:
_count("adaln_stock")
out = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
else:
_count("adaln_fused")
return out, gate_msa, shift_mlp, scale_mlp, gate_mlp
def _adaln_zero_single_forward(
self,
x,
emb = None,
):
e = self.linear(self.silu(emb))
shift_msa, scale_msa, gate_msa = e.chunk(3, dim = 1)
out = _fused_modulate(self.norm, x, scale_msa, shift_msa)
if out is None:
_count("adaln_stock")
out = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
else:
_count("adaln_fused")
return out, gate_msa
def _adaln_continuous_forward(self, x, conditioning_embedding):
e = self.linear(self.silu(conditioning_embedding).to(x.dtype))
scale, shift = e.chunk(2, dim = 1)
out = _fused_modulate(self.norm, x, scale, shift)
if out is None:
_count("adaln_stock")
out = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
else:
_count("adaln_fused")
return out
_FORWARDS = {
"AdaLayerNormZero": _adaln_zero_forward,
"AdaLayerNormZeroSingle": _adaln_zero_single_forward,
"AdaLayerNormContinuous": _adaln_continuous_forward,
}
# Source lines the fused forwards reimplement: a diffusers release that changes them keeps the stock class.
_STOCK_LINES = {
"AdaLayerNormZero": (
"shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim=1)",
"x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]",
),
"AdaLayerNormZeroSingle": (
"shift_msa, scale_msa, gate_msa = emb.chunk(3, dim=1)",
"x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]",
),
"AdaLayerNormContinuous": (
"emb = self.linear(self.silu(conditioning_embedding).to(x.dtype))",
"scale, shift = torch.chunk(emb, 2, dim=1)",
"x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]",
),
}
_CLASSES: dict = {}
def _stock_classes() -> dict:
if not _CLASSES:
try:
from diffusers.models import normalization as nm
except Exception: # noqa: BLE001
return {}
for name in _ADALN_CLASSES:
cls = getattr(nm, name, None)
if cls is not None:
_CLASSES[name] = cls
return _CLASSES
def _stock_body_ok(cls: type, name: str) -> bool:
import inspect
# The stock body is checked on the ORIGINAL forward (the Studio eager patch may be live on top of it).
fn = cls.__dict__.get("forward")
for cand in (getattr(fn, "__wrapped__", None), fn):
if cand is None:
continue
try:
src = inspect.getsource(cand)
except (OSError, TypeError):
continue
if all(line in src for line in _STOCK_LINES[name]):
return True
return False
def install_adaln(dtype: Any, device: Any = "cuda") -> int:
"""Wrap the live AdaLN forwards (stock or Studio's eager addcmul patch). Returns the number wrapped."""
uninstall_adaln()
if not wanted(FUSED_ADALN_ENV, dtype, device):
return 0
classes = _stock_classes()
with _LOCK:
for name, cls in classes.items():
live = cls.forward
# Studio's eager addcmul patch was fingerprinted against the same stock body at its own install.
ours = (getattr(live, "__module__", "") or "").endswith("diffusion_eager_patches")
if not ours and not _stock_body_ok(cls, name):
continue
_ADALN_PREV[cls] = live
_ADALN_ORIGINAL.setdefault(cls, live)
cls.forward = _FORWARDS[name]
return len(_ADALN_PREV)
def uninstall_adaln() -> None:
with _LOCK:
_ADALN_FAILED[0] = False
for cls, prev in list(_ADALN_PREV.items()):
# Not ours on top (deferred eager patch): keep the entry for the uninstall after that layer's.
if cls.__dict__.get("forward") in _FORWARDS.values():
cls.forward = prev
del _ADALN_PREV[cls]
def install_for_pipe(
pipe: Any,
dtype: Any,
device: Any = "cuda",
logger: Any = None,
*,
adaln: bool = True,
) -> dict:
"""Install for a FLUX.1 / FLUX.2 denoiser, otherwise restore stock; ``adaln=False`` (speed off) keeps stock AdaLN."""
transformer = getattr(pipe, "transformer", None)
module_name = type(transformer).__module__ if transformer is not None else ""
if module_name not in _MODULES:
uninstall()
return {"rope": False, "adaln": 0}
uninstall()
rope = install_rope(module_name, dtype, device)
adaln = install_adaln(dtype, device) if adaln else 0
if logger is not None and (rope or adaln):
try:
logger.info(
"diffusion.rocm_fused: fused RoPE %s, fused AdaLN on %d classes",
"on" if rope else "off",
adaln,
)
except Exception: # noqa: BLE001
pass
return {"rope": rope, "adaln": adaln}
def uninstall() -> None:
uninstall_rope()
uninstall_adaln()
def is_installed() -> bool:
return bool(_ROPE_STOCK) or any(
cls.__dict__.get("forward") in _FORWARDS.values() for cls in _ADALN_PREV
)