Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

8.2 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Bit-identical FLUX.2 RoPE Triton kernel for fp16 GPUs; off via UNSLOTH_DIFFUSION_FLUX2_FUSED_ROPE=0."""

from __future__ import annotations

import functools
import logging
import os
import threading
from typing import Any, Callable, Optional
from .rocm_bf16 import is_rocm_torch

FLUX2_FUSED_ROPE_ENV = "UNSLOTH_DIFFUSION_FLUX2_FUSED_ROPE"
_MODULE = "diffusers.models.transformers.transformer_flux2"
_ATTR = "apply_rotary_emb"
_LOCK = threading.Lock()
_STOCK: dict = {}
# Survives uninstall so a forward that looked up the patch just before teardown still reaches stock.
_ORIGINAL: list = [None]
# Set by the first failed launch (JIT / driver / PTX): the rest of the load runs stock, not one failing JIT per call.
_FAILED = [False]


def disabled() -> bool:
    return (os.environ.get(FLUX2_FUSED_ROPE_ENV) or "").strip().lower() in (
        "0",
        "off",
        "false",
        "no",
    )


def wanted(dtype: Any, device: Any = "cuda") -> bool:
    if disabled():
        return False
    try:
        import torch
    except Exception:  # noqa: BLE001
        return False
    if dtype is not torch.float16 or not str(device).startswith("cuda"):
        return False
    if is_rocm_torch(torch):
        return False
    return _kernel() is not None


@functools.lru_cache(maxsize = 1)
def _kernel() -> Optional[Callable]:
    """None when Triton is unavailable."""
    try:
        import triton
        import triton.language as tl
    except Exception:  # noqa: BLE001
        return None

    @triton.jit(do_not_specialize = ["S"])
    def _rope_fwd(
        X,
        COS,
        SIN,
        OUT,
        S,
        H,
        sxb,
        sxs,
        sxh,
        scs,
        sss,
        sob,
        sos,
        soh,
        HALF: tl.constexpr,  # hd // 2
        BLOCK: tl.constexpr,  # block size over heads x pairs
    ):
        row = tl.program_id(0)
        b = row // S
        s = row - b * S
        idx = tl.arange(0, BLOCK)
        mask = idx < H * HALF
        h = idx // HALF
        i = idx - h * HALF
        xo = X + b * sxb + s * sxs + h * sxh + 2 * i
        xe = tl.load(xo, mask = mask, other = 0.0).to(tl.float32)
        xd = tl.load(xo + 1, mask = mask, other = 0.0).to(tl.float32)
        ce = tl.load(COS + s * scs + 2 * i, mask = mask, other = 0.0).to(tl.float32)
        cd = tl.load(COS + s * scs + 2 * i + 1, mask = mask, other = 0.0).to(tl.float32)
        se = tl.load(SIN + s * sss + 2 * i, mask = mask, other = 0.0).to(tl.float32)
        sd = tl.load(SIN + s * sss + 2 * i + 1, mask = mask, other = 0.0).to(tl.float32)
        # stock: x.float() * cos + stack([-x_odd, x_even]).float() * sin, each product rounded on its own
        oe = xe * ce + (-xd) * se
        od = xd * cd + xe * sd
        oo = OUT + b * sob + s * sos + h * soh + 2 * i
        tl.store(oo, oe.to(OUT.dtype.element_ty), mask = mask)
        tl.store(oo + 1, od.to(OUT.dtype.element_ty), mask = mask)

    def launch(x: Any, cos: Any, sin: Any) -> Any:
        import torch

        B, S, H, D = x.shape
        # Same layout stock returns: a dense input keeps its strides, anything else comes back contiguous.
        out = torch.empty_like(x)
        if out.stride(3) != 1:
            out = torch.empty((B, S, H, D), dtype = x.dtype, device = x.device)
        half = D // 2
        block = triton.next_power_of_2(H * half)
        with torch.cuda.device(x.device):
            _rope_fwd[(B * S,)](
                x,
                cos,
                sin,
                out,
                S,
                H,
                x.stride(0),
                x.stride(1),
                x.stride(2),
                cos.stride(0),
                sin.stride(0),
                out.stride(0),
                out.stride(1),
                out.stride(2),
                HALF = half,
                BLOCK = block,
                num_warps = 4 if block <= 2048 else 8,
                enable_fp_fusion = False,
            )
        return out

    return launch


def _eligible(
    x: Any, freqs_cis: Any, use_real: bool, use_real_unbind_dim: int, sequence_dim: int
) -> bool:
    import torch

    if not use_real or use_real_unbind_dim != -1 or sequence_dim != 1:
        return False
    if not isinstance(freqs_cis, (tuple, list)) or len(freqs_cis) != 2:
        return False
    if not torch.is_tensor(x) or x.dtype is not torch.float16 or not x.is_cuda or x.dim() != 4:
        return False
    if x.requires_grad and torch.is_grad_enabled():
        return False
    B, S, H, D = x.shape
    if D % 2 or x.stride(3) != 1 or x.numel() == 0:
        return False
    # Kernel offsets are int32.
    if x.storage_offset() + sum((n - 1) * st for n, st in zip(x.shape, x.stride())) >= 2**31 - 1:
        return False
    cos, sin = freqs_cis
    for f in (cos, sin):
        if not torch.is_tensor(f) or f.dim() != 2 or tuple(f.shape) != (S, D):
            return False
        if f.device != x.device or f.stride(1) != 1:
            return False
        if f.dtype not in (torch.float32, torch.float16):
            return False
    return True


def _log_failed(exc: BaseException) -> None:
    try:
        logging.getLogger(__name__).warning(
            "diffusion.flux2_rope: fused fp16 RoPE launch failed (%s: %s); stock RoPE for the rest of this load",
            type(exc).__name__,
            exc,
        )
    except Exception:  # noqa: BLE001
        pass


def _fused_apply_rotary_emb(
    x,
    freqs_cis,
    use_real: bool = True,
    use_real_unbind_dim: int = -1,
    sequence_dim: int = 2,
):
    import torch

    stock = _STOCK.get(_ATTR) or _ORIGINAL[0]
    # Checked before _kernel(): Dynamo graph-breaks on Triton's import path, so a compiled block stays whole on stock.
    if torch.compiler.is_compiling():
        return stock(
            x,
            freqs_cis,
            use_real = use_real,
            use_real_unbind_dim = use_real_unbind_dim,
            sequence_dim = sequence_dim,
        )
    launch = None if _FAILED[0] else _kernel()
    if launch is not None and _eligible(x, freqs_cis, use_real, use_real_unbind_dim, sequence_dim):
        try:
            return launch(x, freqs_cis[0], freqs_cis[1])
        except Exception as exc:  # noqa: BLE001 -- never fail a render over the fast path
            _FAILED[0] = True
            _log_failed(exc)
    return stock(
        x,
        freqs_cis,
        use_real = use_real,
        use_real_unbind_dim = use_real_unbind_dim,
        sequence_dim = sequence_dim,
    )


def install(
    dtype: Any,
    device: Any = "cuda",
    logger: Any = None,
) -> bool:
    """Idempotent; False (nothing patched) when not wanted or diffusers' symbol is not the expected one."""
    if not wanted(dtype, device):
        uninstall()
        return False
    import importlib

    with _LOCK:
        _FAILED[0] = False
        if _ATTR in _STOCK:
            return True
        try:
            mod = importlib.import_module(_MODULE)
        except Exception:  # noqa: BLE001
            return False
        stock = getattr(mod, _ATTR, None)
        if stock is None or getattr(stock, "__name__", "") != "apply_rotary_emb":
            return False
        _STOCK[_ATTR] = stock
        _ORIGINAL[0] = stock
        setattr(mod, _ATTR, _fused_apply_rotary_emb)
    if logger is not None:
        try:
            logger.info("diffusion.flux2_rope: fused fp16 RoPE kernel installed")
        except Exception:  # noqa: BLE001
            pass
    return True


def install_for_pipe(
    pipe: Any,
    dtype: Any,
    device: Any = "cuda",
    logger: Any = None,
) -> bool:
    """Install for an fp16 FLUX.2 denoiser, otherwise restore stock."""
    transformer = getattr(pipe, "transformer", None)
    if transformer is None or type(transformer).__module__ != _MODULE:
        uninstall()
        return False
    return install(dtype, device, logger)


def uninstall() -> None:
    import sys
    with _LOCK:
        _FAILED[0] = False
        stock = _STOCK.pop(_ATTR, None)
        if stock is None:
            return
        mod = sys.modules.get(_MODULE)
        if mod is not None and getattr(mod, _ATTR, None) is _fused_apply_rotary_emb:
            setattr(mod, _ATTR, stock)


def is_installed() -> bool:
    return _ATTR in _STOCK