Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

21.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""ComfyUI nvfp4/mxfp8 layer decode; nvfp4 stores the EVEN column in the HIGH nibble."""

from __future__ import annotations

import os
import threading
from typing import Any, Optional

NVFP4 = "nvfp4"
MXFP8 = "mxfp8"
BLOCK_SIZES = {NVFP4: 16, MXFP8: 32}
COMFY_NVFP4_ENV = "UNSLOTH_DIFFUSION_COMFY_NVFP4"
COMFY_MXFP8_ENV = "UNSLOTH_DIFFUSION_COMFY_MXFP8"
NVFP4_RUNTIME = "flashinfer"
MXFP8_RUNTIME = "scaled_mm"

_OFF = ("0", "off", "false", "no")
_E2M1 = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
_FP8_MAX = 448.0
_E8M0_BIAS = 127


def _roundup(x: int, m: int) -> int:
    return (int(x) + m - 1) // m * m


def tiled_scale_numel(rows: int, blocks: int) -> int:
    """Elements of a cuBLAS 128x4-tiled scale buffer for ``rows`` x ``blocks`` scales (padding included)."""
    return _roundup(rows, 128) * _roundup(blocks, 4)


def untile_scales(tiled: Any, rows: int, blocks: int) -> Any:
    """The plain ``[rows, blocks]`` scale matrix from a 128x4-tiled buffer (any shape, same dtype)."""
    rp, bp = _roundup(rows, 128), _roundup(blocks, 4)
    flat = tiled.reshape(-1)
    if flat.numel() != rp * bp:
        raise ValueError(
            f"tiled scales hold {flat.numel()} values, {rp * bp} expected for {rows}x{blocks}"
        )
    # tile (row // 128, block // 4) is 512 values: inner (row % 32, (row % 128) // 32, block % 4)
    from .diffusion_single_file_converters import charge_host_ram

    # the permuted copy, and while it is still live the trimmed copy when the block count is padded
    trimmed = rows * blocks if blocks != bp else 0
    charge_host_ram((rp * bp + trimmed) * flat.element_size())
    v = flat.reshape(rp // 128, bp // 4, 32, 4, 4).permute(0, 3, 2, 1, 4).reshape(rp, bp)
    return v[:rows, :blocks].contiguous()


def tile_scales(plain: Any) -> Any:
    """``untile_scales`` inverted: a flat 128x4-tiled buffer, zero padded."""
    import torch

    rows, blocks = plain.shape
    rp, bp = _roundup(rows, 128), _roundup(blocks, 4)
    raw = plain.view(torch.uint8) if plain.element_size() == 1 else plain
    from .diffusion_single_file_converters import charge_host_ram

    charge_host_ram(2 * rp * bp * raw.element_size())  # the padded buffer and its permuted copy
    full = torch.zeros(rp, bp, dtype = raw.dtype, device = raw.device)
    full[:rows, :blocks] = raw
    out = full.reshape(rp // 128, 4, 32, bp // 4, 4).permute(0, 3, 2, 1, 4).reshape(-1).contiguous()
    return out.view(plain.dtype) if plain.element_size() == 1 else out


def swap_nibbles(packed: Any) -> Any:
    """High-first fp4 packing <-> low-first (``0xAB`` -> ``0xBA``)."""
    import torch

    from .diffusion_single_file_converters import charge_host_ram

    u = packed.view(torch.uint8) if packed.dtype != torch.uint8 else packed
    # One output (the source stays memory-mapped), filled in bounded row chunks: the expression form held
    # the left shift, the right shift and their OR at once, three packed copies.
    flat = u.reshape(u.shape[0], -1) if u.dim() > 1 else u.reshape(-1, 1)
    row = max(1, int(flat.shape[1]))
    chunk = max(1, min(int(flat.shape[0]), _DEQUANT_CHUNK_BYTES // row))
    charge_host_ram(u.numel() + chunk * row)
    out = torch.empty_like(flat)
    for start in range(0, int(flat.shape[0]), chunk):
        part = flat[start : start + chunk]
        dst = out[start : start + chunk]
        torch.bitwise_left_shift(part, 4, out = dst)
        dst |= part >> 4
    return out.reshape(u.shape)


def decode_layer(fmt: str, weight: Any, weight_scale: Any) -> tuple[Any, Any]:
    """Row-major ``(codes, plain scales)``; nvfp4 codes come out low-nibble-first (FlashInfer's packing)."""
    import torch

    rows = int(weight.shape[0])
    if fmt == NVFP4:
        cols = int(weight.shape[1]) * 2
        codes = swap_nibbles(weight)
        scale = untile_scales(weight_scale.view(torch.uint8), rows, cols // 16).view(
            torch.float8_e4m3fn
        )
        return codes, scale
    if fmt == MXFP8:
        codes = weight if weight.dtype == torch.float8_e4m3fn else weight.view(torch.float8_e4m3fn)
        scale = untile_scales(weight_scale.view(torch.uint8), rows, int(weight.shape[1]) // 32)
        return codes, scale
    raise ValueError(f"not a block-scaled ComfyUI format: {fmt!r}")


def logical_cols(fmt: str, codes: Any) -> int:
    return int(codes.shape[1]) * (2 if fmt == NVFP4 else 1)


# Row chunks of a fallback block dequantization keep its float32 / int32 intermediates under this many bytes.
_DEQUANT_CHUNK_BYTES = 16 << 20


def _dequant_temp_bytes_per_row(fmt: str, cols: int, pre_quant: bool) -> int:
    """Peak temp bytes per row: 14 per column for nvfp4, 8 for mxfp8, plus 4 with a pre-quant scale."""
    peak = 14 * cols if fmt == NVFP4 else 8 * cols
    return peak + (4 * cols if pre_quant else 0)


def _dequant_block_rows(
    fmt,
    codes,
    scale,
    tensor_scale,
    pre_quant_scale,
    lut = None,
):
    import torch

    rows = int(codes.shape[0])
    if fmt == NVFP4:
        if lut is None:
            lut = torch.tensor(
                _E2M1 + tuple(-v for v in _E2M1), dtype = torch.float32, device = codes.device
            )
        c = codes.to(torch.int32)
        values = torch.stack((lut[c & 0x0F], lut[c >> 4]), dim = -1).reshape(rows, -1, 16)
        del c
        step = scale.to(torch.float32)
        if tensor_scale is not None:
            step = step * tensor_scale
        weight = (values * step.unsqueeze(-1)).reshape(rows, -1)
    elif fmt == MXFP8:
        exp = scale.view(torch.uint8).to(torch.float32) - _E8M0_BIAS
        weight = (
            codes.to(torch.float32).reshape(rows, -1, 32) * torch.exp2(exp).unsqueeze(-1)
        ).reshape(rows, -1)
    else:
        raise ValueError(f"not a block-scaled ComfyUI format: {fmt!r}")
    if pre_quant_scale is not None:
        weight = weight * pre_quant_scale
    return weight


def dequant_block(
    fmt: str,
    codes: Any,
    scale: Any,
    *,
    tensor_scale: Any = None,
    pre_quant_scale: Any = None,
    dtype: Any = None,
    chunk_bytes: Optional[int] = None,
    lut: Any = None,
) -> Any:
    """Dense ``[N, K]`` weight; ``pre_quant_scale`` folds into the columns: ``(x * s) @ W.T == x @ (W * s).T``.

    Dequantized in row chunks straight into the output, so the float32 intermediates stay bounded by
    ``chunk_bytes`` (default ``_DEQUANT_CHUNK_BYTES``) instead of several times the layer; every op is per row, so
    the result is bit-identical to one pass. ``tensor_scale``: one value or one per row. ``lut``: the e2m1 table
    already on the device (a forward under CUDA graph capture must not upload one)."""
    import torch

    from .diffusion_single_file_converters import charge_host_ram

    if fmt not in BLOCK_SIZES:
        raise ValueError(f"not a block-scaled ComfyUI format: {fmt!r}")
    dtype = dtype or torch.bfloat16
    rows = int(codes.shape[0])
    cols = logical_cols(fmt, codes)
    per_row = _dequant_temp_bytes_per_row(fmt, cols, pre_quant_scale is not None)
    chunk = max(1, min(rows, int(chunk_bytes or _DEQUANT_CHUNK_BYTES) // max(per_row, 1)))
    charge_host_ram(rows * cols * torch.empty((), dtype = dtype).element_size() + chunk * per_row)
    ts = None
    if tensor_scale is not None:
        ts = torch.as_tensor(tensor_scale, dtype = torch.float32, device = codes.device).reshape(-1, 1)
    pqs = None
    if pre_quant_scale is not None:
        pqs = pre_quant_scale.to(torch.float32).reshape(1, -1)
    out = torch.empty((rows, cols), dtype = dtype, device = codes.device)
    for start in range(0, rows, chunk):
        stop = min(rows, start + chunk)
        part_ts = ts if ts is None or ts.shape[0] == 1 else ts[start:stop]
        out[start:stop] = _dequant_block_rows(
            fmt, codes[start:stop], scale[start:stop], part_ts, pqs, lut
        ).to(dtype)
    return out


def mx_quantize_activation(x: Any) -> tuple[Any, Any]:
    """``[M, K]`` -> (e4m3 codes, tiled e8m0 scales), exponent ``ceil(log2(amax / 448))`` per 32 columns (ComfyUI's rule)."""
    import torch

    m, k = x.shape
    blocks = x.float().reshape(m, k // 32, 32)
    amax = blocks.abs().amax(dim = -1)
    exp = torch.ceil(torch.log2(torch.clamp(amax / _FP8_MAX, min = 2.0**-127)))
    exp = torch.clamp(exp, -_E8M0_BIAS, _E8M0_BIAS)
    q = torch.clamp(blocks * torch.exp2(-exp).unsqueeze(-1), -_FP8_MAX, _FP8_MAX)
    codes = q.to(torch.float8_e4m3fn).reshape(m, k)
    biased = (exp + _E8M0_BIAS).to(torch.uint8)
    rows_p, blocks_p = _roundup(m, 128), _roundup(k // 32, 4)
    full = torch.full((rows_p, blocks_p), _E8M0_BIAS, dtype = torch.uint8, device = x.device)
    full[:m, : k // 32] = biased
    tiled = full.reshape(rows_p // 128, 4, 32, blocks_p // 4, 4).permute(0, 3, 2, 1, 4).reshape(-1)
    return codes, tiled.view(torch.float8_e8m0fnu)


def mxfp8_linear_class():
    import torch
    from torch import nn

    global _MX_CLASS
    if _MX_CLASS is not None:
        return _MX_CLASS

    class ComfyMXFP8Linear(nn.Module):
        """Plain buffers so offload hooks can move it; forward stays capture-safe (no host sync)."""

        def __init__(
            self,
            in_features: int,
            out_features: int,
            *,
            codes,
            tiled_scale,
            bias = None,
        ):
            super().__init__()
            self.in_features = int(in_features)
            self.out_features = int(out_features)
            self.register_buffer("weight_q", codes)
            self.register_buffer("weight_sf", tiled_scale.view(torch.uint8))
            self.register_buffer("bias", bias)

        def forward(self, x):
            shape = x.shape
            out_dtype = x.dtype if x.dtype in (torch.bfloat16, torch.float16) else torch.bfloat16
            flat = x.reshape(-1, self.in_features)
            if flat.shape[0] == 0:
                return flat.new_zeros((0, self.out_features), dtype = out_dtype).reshape(
                    *shape[:-1], self.out_features
                )
            xq, x_sf = mx_quantize_activation(flat)
            out = torch._scaled_mm(
                xq,
                self.weight_q.t(),
                scale_a = x_sf,
                scale_b = self.weight_sf.view(torch.float8_e8m0fnu),
                out_dtype = torch.bfloat16,
            )
            if self.bias is not None:
                out = out + self.bias.to(out.dtype)
            return out.to(out_dtype).reshape(*shape[:-1], self.out_features)

        def extra_repr(self) -> str:  # pragma: no cover - debug aid
            return (
                f"in_features={self.in_features}, out_features={self.out_features}, mxfp8=scaled_mm"
            )

    _MX_CLASS = ComfyMXFP8Linear
    return _MX_CLASS


_MX_CLASS = None
_NVFP4_DYNAMIC_CLASS = None


def nvfp4_dynamic_linear_class():
    """FlashInfer NVFP4 Linear scaling activations from each call's amax, for layers without ``input_scale``."""
    global _NVFP4_DYNAMIC_CLASS
    if _NVFP4_DYNAMIC_CLASS is not None:
        return _NVFP4_DYNAMIC_CLASS
    import torch
    from torch.nn import functional as F

    from .diffusion_nvfp4_bias import fused_bias_add_
    from .diffusion_nvfp4_linear import nvfp4_linear_class
    from .diffusion_nvfp4_ops import _device_guard, dequantize_nvfp4_weight, global_scale

    base = nvfp4_linear_class()

    class ComfyNVFP4DynamicLinear(base):
        def forward(self, x):
            shape = x.shape
            flat = x.reshape(-1, self.in_features)
            out_dtype = flat.dtype
            if out_dtype not in (torch.bfloat16, torch.float16):
                flat = flat.to(torch.bfloat16)
            if flat.shape[0] == 0:
                return flat.new_zeros((0, self.out_features), dtype = out_dtype).reshape(
                    *shape[:-1], self.out_features
                )
            if self.protect.armed and self.protect.protected:
                out = F.linear(
                    flat,
                    dequantize_nvfp4_weight(self.wq, self.w_sf, self.w_scale, dtype = flat.dtype),
                )
            else:
                with _device_guard(flat):
                    a_gsf = global_scale(flat)
                    xq, x_sf = torch.ops.unsloth_nvfp4.quantize(flat, a_gsf)
                    out = torch.ops.unsloth_nvfp4.mm(
                        xq,
                        self.wq,
                        x_sf,
                        self.w_sf,
                        self.w_scale / a_gsf,
                        self.out_features,
                        self.backend,
                    )
            out = out.to(out_dtype)
            if self.bias is not None:
                fused_bias_add_(out, self.bias)
            return out.reshape(*shape[:-1], self.out_features)

    _NVFP4_DYNAMIC_CLASS = ComfyNVFP4DynamicLinear
    return _NVFP4_DYNAMIC_CLASS


_PROBE_LOCK = threading.Lock()
_MX_PROBE: dict = {}


def _device_of(target: Any) -> Any:
    import torch

    device = getattr(target, "torch_device", None) or getattr(target, "device", None)
    if device is None:
        return None
    try:
        return torch.device(device)
    except Exception:  # noqa: BLE001 -- not a device we can name
        return None


def mxfp8_runtime_reason(target: Any) -> Optional[str]:
    """None when ``torch._scaled_mm`` runs e8m0 block-scaled fp8 on ``target``'s device (cached probe), else why not."""
    try:
        import torch
    except Exception:  # noqa: BLE001
        return "torch unavailable"
    device = _device_of(target)
    if device is None or device.type != "cuda" or not torch.cuda.is_available():
        return f"device {device} is not CUDA"
    if not hasattr(torch, "float8_e8m0fnu"):
        return f"torch {torch.__version__} has no float8_e8m0fnu"
    index = device.index if device.index is not None else torch.cuda.current_device()
    capability = tuple(torch.cuda.get_device_capability(index))
    if capability < (10, 0):
        return "sm_%d%d has no MXFP8 tensor cores (Blackwell sm_100+ needed)" % capability
    with _PROBE_LOCK:
        if index not in _MX_PROBE:
            try:
                with torch.cuda.device(index):
                    # An explicit CPU generator: up to torch 2.11 a default-generator draw raises while another thread
                    # records, and a CPU draw moved to the device needs no CUDA generator (absent on a spoofed / CPU-only torch).
                    gen = torch.Generator().manual_seed(0)
                    x = torch.randn(128, 64, dtype = torch.bfloat16, generator = gen).to("cuda")
                    w = torch.randn(32, 64, dtype = torch.bfloat16, generator = gen).to("cuda")
                    wq, wsf = mx_quantize_activation(w)
                    linear = mxfp8_linear_class()(64, 32, codes = wq, tiled_scale = wsf)
                    y = linear(x).float()
                    ref = x.float() @ w.float().T
                    err = float((y - ref).norm() / ref.norm())
                    _MX_PROBE[index] = None if err < 0.1 else f"probe error {err:.3f}"
            except Exception as exc:  # noqa: BLE001 -- any failure means dequantize
                # not cached: a transient failure (an OOM on a busy GPU) must not pin this process to dequant
                return f"{type(exc).__name__}: {str(exc)[:160]}"
        return _MX_PROBE[index]


def reset_probe_cache() -> None:
    with _PROBE_LOCK:
        _MX_PROBE.clear()


def _env_off(name: str) -> bool:
    return (os.environ.get(name) or "").strip().lower() in _OFF


def comfy_block_runtime_layers(model: Any) -> int:
    """How many nvfp4 / mxfp8 layers of a ComfyUI load kept their codes (they cannot carry LoRA adapters)."""
    info = getattr(model, "_unsloth_comfy_quant", None) or {}
    try:
        return int(info.get(NVFP4) or 0) + int(info.get(MXFP8) or 0)
    except (TypeError, ValueError):
        return 0


def comfy_nvfp4_runtime_possible(family: Optional[str]) -> bool:
    """Whether the switch and the family let a ComfyUI nvfp4 file keep its codes (the device is the load's call)."""
    try:
        from .diffusion_transformer_quant import TQ_NVFP4, _family_denied
        return not _env_off(COMFY_NVFP4_ENV) and not _family_denied(family, TQ_NVFP4)
    except Exception:  # noqa: BLE001 -- no answer, no install
        return False


def comfy_block_backend(
    fmt: str,
    target: Any,
    family: Optional[str],
    *,
    dtype: Any = None,
) -> tuple[Optional[str], str]:
    """``(runtime or None, reason)`` for ``fmt`` layers on ``target``; None = dequantize (always correct).
    nvfp4 follows Studio's own NVFP4 gating (flag, FlashInfer backend choice, family); mxfp8 needs the probe + bf16."""
    try:
        from .diffusion_transformer_quant import TQ_MXFP8, TQ_NVFP4, _family_denied
        if fmt == NVFP4:
            if _env_off(COMFY_NVFP4_ENV):
                return None, f"{COMFY_NVFP4_ENV}=0"
            from .diffusion_nvfp4_flag import NVFP4_DIFFUSION_ENV, nvfp4_diffusion_enabled

            if not nvfp4_diffusion_enabled():
                return None, f"Unsloth's NVFP4 runtime is off ({NVFP4_DIFFUSION_ENV}=1 enables it)"
            if _family_denied(family, TQ_NVFP4):
                return None, f"nvfp4 is denied for {family}"
            device = _device_of(target)
            if device is None or device.type != "cuda":
                return None, f"device {device} is not CUDA"
            from .diffusion_nvfp4_ops import BACKEND_FLASHINFER, _resolve_backend

            backend, reason = _resolve_backend(device)
            if backend != BACKEND_FLASHINFER:
                return None, reason
            return NVFP4_RUNTIME, reason
        if fmt == MXFP8:
            if _env_off(COMFY_MXFP8_ENV):
                return None, f"{COMFY_MXFP8_ENV}=0"
            import torch

            if dtype is not None and dtype is not torch.bfloat16:
                return (
                    None,
                    f"the pipeline runs {str(dtype).replace('torch.', '')}, the mxfp8 GEMM needs bfloat16",
                )
            if _family_denied(family, TQ_MXFP8):
                return None, f"mxfp8 is denied for {family}"
            reason = mxfp8_runtime_reason(target)
            if reason:
                return None, reason
            return MXFP8_RUNTIME, "torch._scaled_mm e8m0 block scales"
    except Exception as exc:  # noqa: BLE001 -- an unanswerable probe takes the exact dense path
        return None, f"{type(exc).__name__}: {str(exc)[:160]}"
    return None, f"unknown format {fmt!r}"


def build_runtime_linear(
    fmt: str,
    linear: Any,
    codes: Any,
    scale: Any,
    *,
    tensor_scale: Any = None,
    input_scale: Any = None,
    bias: Any = None,
) -> Any:
    import torch

    from .diffusion_single_file_converters import charge_host_ram

    rows = int(codes.shape[0])
    if fmt == NVFP4:
        from .diffusion_nvfp4_linear import nvfp4_linear_class
        from .diffusion_nvfp4_ops import register_ops, sf_matrix_shape

        register_ops()
        cols = logical_cols(fmt, codes)
        w_sf = (
            tile_scales(scale)
            .view(torch.uint8)
            .reshape(sf_matrix_shape(rows, cols // 16))
            .contiguous()
        )
        w_scale = torch.as_tensor(tensor_scale, dtype = torch.float32).reshape(1).clone()
        if input_scale is None:
            # no calibrated scale: per-call amax, as ComfyUI does
            a_gsf = torch.ones(1, dtype = torch.float32)
            cls = nvfp4_dynamic_linear_class()
        else:
            in_scale = torch.as_tensor(input_scale, dtype = torch.float32).reshape(1)
            # FlashInfer's activation global scale is the reciprocal of ComfyUI's decode-side input_scale.
            a_gsf = (1.0 / in_scale).clone()
            cls = nvfp4_linear_class()
        if not codes.is_contiguous():
            charge_host_ram(int(codes.numel()) * int(codes.element_size()))
        return cls(
            cols,
            rows,
            wq = codes.contiguous(),
            w_sf = w_sf,
            alpha = (w_scale / a_gsf).clone(),
            a_gsf = a_gsf,
            bias = bias,
            w_scale = w_scale,
            activation_scales_baked = True,
        )
    if fmt == MXFP8:
        if not codes.is_contiguous():
            charge_host_ram(int(codes.numel()) * int(codes.element_size()))
        return mxfp8_linear_class()(
            int(codes.shape[1]),
            rows,
            codes = codes.contiguous(),
            tiled_scale = tile_scales(scale),
            bias = bias,
        )
    raise ValueError(f"not a block-scaled ComfyUI format: {fmt!r}")


def comfy_block_backends(
    scan: Any,
    target: Any,
    family: Optional[str],
    *,
    dtype: Any = None,
    logger: Any = None,
    lora: bool = False,
) -> dict:
    """``load_comfy_quant_transformer`` backend kwargs, logging per format which runtime keeps it or why not."""
    out = {"nvfp4_backend": None, "mxfp8_backend": None}
    counts = scan.counts() if scan is not None else {}
    for fmt in (NVFP4, MXFP8):
        if not counts.get(fmt):
            continue
        backend, reason = comfy_block_backend(fmt, target, family, dtype = dtype)
        if lora and backend:
            backend, reason = None, "a LoRA is selected: adapters need dense Linears"
        out[f"{fmt}_backend"] = backend
        if logger is not None:
            if backend:
                logger.info(
                    "diffusion.comfy_quant: %d %s layer(s) stay %s on the %s runtime (%s)",
                    counts[fmt],
                    fmt,
                    fmt,
                    backend,
                    reason,
                )
            else:
                logger.info(
                    "diffusion.comfy_quant: %d %s layer(s) have no native runtime here (%s): widened to the compute "
                    "dtype on load, or kept packed and widened per layer when the load stores its weights compactly",
                    counts[fmt],
                    fmt,
                    reason,
                )
    return out