Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

19.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""FlashInfer NVFP4 kernels as custom ops (for ``fullgraph``); fake impls must match its allocation EXACTLY."""

from __future__ import annotations

import os
import threading
from typing import Any, Optional

NVFP4_BACKEND_ENV = "UNSLOTH_NVFP4_BACKEND"
NVFP4_BACKENDS = ("auto", "torchao", "flashinfer")
BACKEND_TORCHAO = "torchao"
BACKEND_FLASHINFER = "flashinfer"

# Restores the full M x N memset in the GEMM op. Strictly slower and no safer; see ``_mm_impl``.
NVFP4_ZERO_BUFFER_ENV = "UNSLOTH_NVFP4_ZERO_BUFFER"

# Necessary condition only: sm_120 is listed but routinely fails the preflight on CUDA 12.8.
NVFP4_FLASHINFER_CAPS = frozenset({(10, 0), (10, 3), (12, 0)})

DEFAULT_MM_BACKEND = "cutlass"

FP4_MAX = 6.0
FP8_MAX = 448.0

_TRUE_TOKENS = ("1", "true", "yes", "on")

_REGISTER_LOCK = threading.Lock()
_REGISTERED = False
_PREFLIGHT_LOCK = threading.Lock()
_PREFLIGHT: dict[int, dict] = {}
# Last transient preflight failure per device index: reporting only, never used to pick a backend.
_PREFLIGHT_TRANSIENT: dict[int, dict] = {}
_WARNED: set = set()

_BARRIER_LOCK = threading.Lock()
_BARRIERS: dict[int, Any] = {}


def _swizzled_sf_numel(
    rows: int,
    cols: int,
    row_size: int = 128,
) -> int:
    """Swizzled scale-factor element count; copied so fake impls never touch the FlashInfer JIT."""
    return ((rows + row_size - 1) // row_size * row_size) * ((cols + 3) // 4 * 4)


def _device_guard(t: Any):
    """``torch.cuda.device`` for the tensor's device; EVERY flashinfer call needs one (a foreign device bricks the card)."""
    import torch
    return torch.cuda.device(t.device)


def _zero_buffer_enabled() -> bool:
    return os.environ.get(NVFP4_ZERO_BUFFER_ENV, "").strip().lower() in _TRUE_TOKENS


def _device_index(device: Any) -> int:
    import torch
    index = getattr(device, "index", None)
    return torch.cuda.current_device() if index is None else int(index)


def _is_capturing() -> bool:
    try:
        import torch
        return bool(torch.cuda.is_current_stream_capturing())
    except Exception:  # noqa: BLE001 - a torch without the query cannot be capturing
        return False


def _barrier(device: Any):
    """Per-device 1-element bf16 barrier buffer, uncached while capturing (it would die with the graph)."""
    import torch

    index = _device_index(device)
    buf = _BARRIERS.get(index)
    if buf is not None:
        return buf
    fresh = torch.empty(1, device = device, dtype = torch.bfloat16)
    if _is_capturing():
        return fresh
    with _BARRIER_LOCK:
        return _BARRIERS.setdefault(index, fresh)


def _fire_barrier(device: Any):
    buf = _barrier(device)
    buf.zero_()
    return buf


def reset_barriers() -> None:
    """One allocated under a model's allocator state must not reach the next model's graph pool."""
    with _BARRIER_LOCK:
        _BARRIERS.clear()


def global_scale(t: Any):
    return (FP4_MAX * FP8_MAX / t.float().abs().amax().clamp(min = 1e-8)).reshape(1).to(t.device)


def _quantize_impl(x: Any, global_sf: Any):
    import flashinfer

    from . import diffusion_nvfp4_dispatch as dispatch
    with _device_guard(x):
        xq, sf = dispatch._fast_quantize(x, global_sf)
        if xq is not None:
            return xq, sf
        return flashinfer.nvfp4_quantize(x, global_sf, do_shuffle = False)


def _mm_impl(xq: Any, wq: Any, x_sf: Any, w_sf: Any, alpha: Any, n: int, backend: str):
    """NVFP4 GEMM on a row-major ``[N, K/2]`` weight. A kernel (any, not ``torch.empty``) MUST run between
    the quantiser and this GEMM: cutlass launches with PDL but ``griddepcontrol`` is compiled out."""
    import flashinfer
    import torch

    from . import diffusion_nvfp4_dispatch as dispatch
    from . import diffusion_nvfp4_tactics as tactics

    with _device_guard(xq):
        m = xq.shape[0]
        if _zero_buffer_enabled():
            out = torch.zeros(m, n, device = xq.device, dtype = torch.bfloat16)
        else:
            out = torch.empty(m, n, device = xq.device, dtype = torch.bfloat16)
            _fire_barrier(xq.device)
        fast = dispatch.enabled(xq.device)
        # Bucketed fast path: the plan tunes its own M bucket once (and reads it back from disk).
        if not (fast and tactics.bucketed_enabled()) and _claim_first_call_tune(
            m, xq.shape[1] * 2, n
        ):
            # Before the dispatch plan, which caches whatever tactic FlashInfer holds on its first build.
            try:
                with flashinfer.autotune(True):
                    return flashinfer.mm_fp4(
                        xq, wq.T, x_sf, w_sf.T, alpha, torch.bfloat16, out = out, backend = backend
                    )
            except Exception:  # noqa: BLE001 - claimed, so it is not retried; run it untuned
                pass
        if fast:
            wq_t, w_sf_t = dispatch.transposed(wq), dispatch.transposed(w_sf)
            plan = dispatch.gemm_plan(xq, wq_t, x_sf, w_sf_t, alpha, out, n, backend)
            if plan is not None:
                runner, tactic, workspace = plan
                runner(
                    inputs = [
                        xq,
                        wq_t,
                        x_sf,
                        w_sf_t,
                        alpha,
                        torch.bfloat16,
                        out,
                        16,
                        True,
                        workspace,
                    ],
                    tactic = tactic,
                )
                return out
        return flashinfer.mm_fp4(
            xq, wq.T, x_sf, w_sf.T, alpha, torch.bfloat16, out = out, backend = backend
        )


def _flashinfer_bucket(m: int) -> int:
    """The M FlashInfer's own mm_fp4 tuning keys ``m`` under (exact ``m`` when it cannot be read)."""
    try:
        from flashinfer.gemm.gemm_base import _MM_FP4_TUNING_CONFIG_128x4
        return int(
            _MM_FP4_TUNING_CONFIG_128x4.dynamic_tensor_specs[0].map_to_tuning_buckets(int(m))
        )
    except Exception:  # noqa: BLE001 - an unreadable mapper only costs a retune per length
        return int(m)


def _first_tune_key(m: int, k: int, n: int) -> tuple:
    """Public-API tuning covers FlashInfer's whole bucket, so a new length in a tuned bucket is no
    new tune; ``UNSLOTH_NVFP4_TACTIC_CACHE=0`` restores one claim per exact length."""
    from .diffusion_nvfp4_tactics import bucketed_enabled

    if bucketed_enabled():
        return ("flashinfer_bucket", _flashinfer_bucket(m), int(k), int(n))
    return (int(m), int(k), int(n))


def _claim_first_call_tune(m: int, k: int, n: int) -> bool:
    """Claims the one autotune for a shape, never inside a CUDA graph capture, where profiling is
    illegal."""
    from .diffusion_nvfp4_linear import _TUNED_SHAPES

    key = _first_tune_key(m, k, n)
    if key in _TUNED_SHAPES or _is_capturing():
        return False
    _TUNED_SHAPES.add(key)
    return True


def _quantize_fake(x: Any, global_sf: Any):
    import torch

    m, k = x.shape
    cols = k // 16
    # The leading dim is NOT m whenever the padding is non-trivial.
    total = _swizzled_sf_numel(m, cols, 128)
    return (
        x.new_empty((m, k // 2), dtype = torch.uint8),
        x.new_empty((total // cols, cols), dtype = torch.uint8),
    )


def _mm_fake(xq: Any, wq: Any, x_sf: Any, w_sf: Any, alpha: Any, n: int, backend: str):
    import torch
    return xq.new_empty((xq.shape[0], n), dtype = torch.bfloat16)


def register_ops() -> None:
    global _REGISTERED
    if _REGISTERED:
        return
    with _REGISTER_LOCK:
        if _REGISTERED:
            return
        import torch

        if "unsloth_nvfp4::quantize" not in torch.library._defs:
            quantize = torch.library.custom_op(
                "unsloth_nvfp4::quantize",
                _quantize_impl,
                mutates_args = (),
                schema = "(Tensor x, Tensor global_sf) -> (Tensor, Tensor)",
            )
            quantize.register_fake(_quantize_fake)
        if "unsloth_nvfp4::mm" not in torch.library._defs:
            mm = torch.library.custom_op(
                "unsloth_nvfp4::mm",
                _mm_impl,
                mutates_args = (),
                schema = (
                    "(Tensor xq, Tensor wq, Tensor x_sf, Tensor w_sf, Tensor alpha, "
                    "int n, str backend) -> Tensor"
                ),
            )
            mm.register_fake(_mm_fake)
        _REGISTERED = True


def unswizzle_sf(
    sf: Any,
    m: int,
    k: int,
    vec: int = 16,
):
    """cutlass 128x4 swizzled block scales -> the plain ``[m, k // vec]`` e4m3 matrix."""
    import torch

    kb = k // vec
    m_pad = (m + 127) // 128 * 128
    k_pad = (kb + 3) // 4 * 4
    flat = sf.reshape(-1).view(torch.float8_e4m3fn)
    v = (
        flat.reshape(m_pad // 128, k_pad // 4, 32, 4, 4)
        .permute(0, 3, 2, 1, 4)
        .reshape(m_pad, k_pad)
    )
    return v[:m, :kb].contiguous()


def swizzle_sf(sf_lin: Any, m: int, k: int):
    """Plain ``[m, k // 16]`` e4m3 block scales -> the cutlass 128x4 swizzled flat buffer."""
    import torch

    kb = k // 16
    m_pad = (m + 127) // 128 * 128
    k_pad = (kb + 3) // 4 * 4
    full = torch.zeros(m_pad, k_pad, device = sf_lin.device, dtype = torch.uint8)
    full[:m, :kb] = sf_lin.reshape(m, kb).view(torch.uint8)
    v = full.reshape(m_pad, k_pad // 4, 4).reshape(m_pad // 128, 4, 32, k_pad // 4, 4)
    return v.permute(0, 3, 2, 1, 4).reshape(-1).contiguous()


_E2M1_MAGNITUDES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
_E2M1_LUT = _E2M1_MAGNITUDES + tuple(-v for v in _E2M1_MAGNITUDES)

_LUT_CACHE: dict = {}


def e2m1_lut(device: Any):
    """The signed e2m1 decode table on ``device``, memoised so it is never first allocated during capture."""
    import torch

    # Traced: a graph constant. Reading the memo would guard on it, and its first insert would recompile the caller.
    if torch.compiler.is_compiling():
        return torch.tensor(_E2M1_LUT, device = device, dtype = torch.float32)
    key = (str(device.type), device.index)
    table = _LUT_CACHE.get(key)
    if table is None:
        table = torch.tensor(_E2M1_LUT, device = device, dtype = torch.float32)
        _LUT_CACHE[key] = table
    return table


def reset_lut_cache() -> None:
    _LUT_CACHE.clear()


def dequantize_nvfp4_weight(
    wq: Any,
    w_sf: Any,
    per_tensor_scale: Any,
    *,
    dtype: Any = None,
):
    """The packed NVFP4 operand as a TRANSIENT dense ``[N, K]`` weight, in torchao's exact arithmetic order."""
    import torch

    if dtype is None:
        dtype = torch.bfloat16
    q = wq if wq.dtype == torch.uint8 else wq.view(torch.uint8)
    rows, half_k = int(q.shape[-2]), int(q.shape[-1])
    cols = half_k * 2
    blocks = cols // 16
    lut = e2m1_lut(q.device)
    codes = q.to(torch.int32)
    values = torch.stack((lut[codes & 0x0F], lut[codes >> 4]), dim = -1).reshape(rows, blocks, 16)
    block_scale = unswizzle_sf(w_sf, rows, cols).to(torch.float32)
    if not isinstance(per_tensor_scale, torch.Tensor):
        per_tensor_scale = torch.tensor(
            [float(per_tensor_scale)], device = q.device, dtype = torch.float32
        )
    step = (block_scale * per_tensor_scale.to(torch.float32).reshape(1)).reshape(rows, blocks, 1)
    return (values * step).reshape(rows, cols).to(dtype)


def sf_matrix_shape(rows: int, cols: int) -> tuple[int, int]:
    """FlashInfer's 2D view of a swizzled scale buffer (leading dim is padded, not ``rows``)."""
    padded_cols = (cols + 3) // 4 * 4
    return (_swizzled_sf_numel(rows, cols, 128) // padded_cols, padded_cols)


def nvfp4_preflight(device: Any = None, *, refresh: bool = False) -> dict:
    """A tiny GUARDED 128x256 quantise + GEMM on ``device``, memoised per device. Never raises."""
    from .diffusion_nvfp4_flag import nvfp4_diffusion_enabled

    if not nvfp4_diffusion_enabled():
        # Not memoised: turning the switch on must not inherit a verdict nobody measured.
        return {
            "ok": False,
            "reason": "NVFP4 is disabled in this build",
            "capability": None,
            "name": "",
        }
    import torch

    if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
        return {"ok": False, "reason": "cuda unavailable", "capability": None, "name": ""}
    if device is None:
        dev = torch.device("cuda", torch.cuda.current_device())
    elif isinstance(device, int):
        # torch.device(int) asks the accelerator API, which raises on a host without one.
        dev = torch.device("cuda", device)
    else:
        dev = torch.device(device)
    if dev.type != "cuda":
        return {"ok": False, "reason": f"device {dev} is not cuda", "capability": None, "name": ""}
    index = dev.index if dev.index is not None else torch.cuda.current_device()
    dev = torch.device("cuda", index)
    if not refresh:
        cached = _PREFLIGHT.get(index)
        if cached is not None:
            return dict(cached)

    try:
        capability = tuple(torch.cuda.get_device_capability(dev))
        name = str(torch.cuda.get_device_name(dev))
    except Exception as exc:  # noqa: BLE001 - a device that cannot be described cannot be used
        capability, name = None, ""
        rec = {
            "ok": False,
            "reason": f"{type(exc).__name__}: {str(exc)[:200]}",
            "capability": capability,
            "name": name,
        }
        with _PREFLIGHT_LOCK:
            _PREFLIGHT[index] = rec
            _PREFLIGHT_TRANSIENT.pop(index, None)
        return dict(rec)

    rec = {"ok": False, "reason": "", "capability": capability, "name": name}
    try:
        finite = _preflight_probe(dev)
        rec["ok"] = finite
        rec["reason"] = "ok" if finite else "mm_fp4 produced a non-finite result"
        if finite:
            from . import diffusion_nvfp4_dispatch as dispatch

            fast_ok, fast_reason = dispatch.verify(dev)
            rec["fast_dispatch"] = fast_ok
            rec["fast_dispatch_reason"] = fast_reason
    except Exception as exc:  # noqa: BLE001 - every failure mode here means "use torchao"
        rec["reason"] = f"{type(exc).__name__}: {str(exc)[:200]}"
        if _transient_preflight_failure(exc):
            # Not memoised: a model about to be evicted may own the card, so OOM means "not now".
            with _PREFLIGHT_LOCK:
                _PREFLIGHT_TRANSIENT[index] = dict(rec)
            return dict(rec)

    with _PREFLIGHT_LOCK:
        _PREFLIGHT[index] = rec
        _PREFLIGHT_TRANSIENT.pop(index, None)
    return dict(rec)


def _preflight_probe(dev: Any) -> bool:
    import flashinfer
    import torch
    with torch.cuda.device(dev):
        # An explicit CPU generator: up to torch 2.11 a default-generator draw raises while another thread
        # records, and a CPU draw moved to the device needs no CUDA generator (absent on a spoofed / CPU-only torch).
        gen = torch.Generator().manual_seed(0)
        x = torch.randn(128, 256, dtype = torch.bfloat16, generator = gen).to(dev) * 0.05
        w = torch.randn(128, 256, dtype = torch.bfloat16, generator = gen).to(dev) * 0.02
        a_gsf, w_gsf = global_scale(x), global_scale(w)
        xq, x_sf = flashinfer.nvfp4_quantize(x, a_gsf, do_shuffle = False)
        wq, w_sf = flashinfer.nvfp4_quantize(w, w_gsf, do_shuffle = False)
        out = torch.zeros(128, 128, device = dev, dtype = torch.bfloat16)
        y = flashinfer.mm_fp4(
            xq,
            wq.T,
            x_sf,
            w_sf.T,
            (1.0 / (a_gsf * w_gsf)).float(),
            torch.bfloat16,
            out = out,
            backend = DEFAULT_MM_BACKEND,
        )
        from .diffusion_cuda_graph import synchronize_outside_capture

        synchronize_outside_capture(dev)
        return bool(torch.isfinite(y).all().item())


def _transient_preflight_failure(exc: BaseException) -> bool:
    """Allocation failures only; an import or JIT failure stays cached."""
    if isinstance(exc, MemoryError):
        return True
    try:
        import torch
        if isinstance(exc, torch.cuda.OutOfMemoryError):
            return True
    except Exception:  # noqa: BLE001 - no torch to ask means fall through to the text
        pass
    text = str(exc).lower()
    return "out of memory" in text or "cudaerrormemoryallocation" in text


def reset_preflight_cache() -> None:
    with _PREFLIGHT_LOCK:
        _PREFLIGHT.clear()
        _PREFLIGHT_TRANSIENT.clear()
    _WARNED.clear()


def nvfp4_backend_env() -> str:
    raw = os.environ.get(NVFP4_BACKEND_ENV, "").strip().lower()
    return raw if raw in NVFP4_BACKENDS else "auto"


def _flashinfer_available() -> tuple[bool, str]:
    try:
        import flashinfer
        return True, str(getattr(flashinfer, "__version__", "unknown"))
    except Exception as exc:  # noqa: BLE001 - any import failure means torchao
        return False, f"{type(exc).__name__}: {str(exc)[:120]}"


def _device_capability(device: Any = None) -> Optional[tuple]:
    try:
        import torch
        if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
            return None
        return tuple(torch.cuda.get_device_capability(device))
    except Exception:  # noqa: BLE001 - an unreadable capability is not a flashinfer device
        return None


def _resolve_backend(device: Any = None) -> tuple[str, str]:
    from .diffusion_nvfp4_flag import NVFP4_DIFFUSION_ENV, nvfp4_diffusion_enabled

    if not nvfp4_diffusion_enabled():
        return BACKEND_TORCHAO, (
            f"NVFP4 is disabled in this build (set {NVFP4_DIFFUSION_ENV}=1 to enable it)"
        )
    requested = nvfp4_backend_env()
    if requested == BACKEND_TORCHAO:
        return BACKEND_TORCHAO, f"{NVFP4_BACKEND_ENV}=torchao"

    available, detail = _flashinfer_available()
    if not available:
        return BACKEND_TORCHAO, f"flashinfer unavailable ({detail})"
    capability = _device_capability(device)
    if capability is None:
        return BACKEND_TORCHAO, "no CUDA device capability to check"
    if tuple(capability) not in NVFP4_FLASHINFER_CAPS:
        return BACKEND_TORCHAO, (
            "sm_%d%d is not in the flashinfer NVFP4 set %s"
            % (
                capability[0],
                capability[1],
                ", ".join(sorted("sm_%d%d" % c for c in NVFP4_FLASHINFER_CAPS)),
            )
        )
    probe = nvfp4_preflight(device)
    if not probe.get("ok"):
        return BACKEND_TORCHAO, f"preflight failed: {probe.get('reason', 'unknown')}"
    return BACKEND_FLASHINFER, f"flashinfer {detail} preflight ok on sm_%d%d" % tuple(capability)


def nvfp4_backend_reason(device: Any = None) -> str:
    return _resolve_backend(device)[1]


def select_nvfp4_backend(device: Any = None) -> str:
    """``"torchao"`` or ``"flashinfer"`` for ``device``; a failing explicit ``flashinfer`` falls back to torchao."""
    backend, reason = _resolve_backend(device)
    if backend != BACKEND_FLASHINFER and nvfp4_backend_env() == BACKEND_FLASHINFER:
        key = (str(device), reason)
        if key not in _WARNED:
            _WARNED.add(key)
            import warnings
            warnings.warn(
                f"{NVFP4_BACKEND_ENV}=flashinfer was requested but {reason}; using torchao",
                RuntimeWarning,
                stacklevel = 2,
            )
    return backend