Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

13.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""MOSS-Image fused q/k/v GEMM and int8 Linears; falls back to stock under LoRA on proj.q/k/v."""

from __future__ import annotations

import functools
import os
import threading
from typing import Any

MOSS_FUSED_ENV = "UNSLOTH_DIFFUSION_MOSS_FUSED"
MOSS_FUSED_QKV_ENV = "UNSLOTH_DIFFUSION_MOSS_FUSED_QKV"
_LOCK = threading.Lock()


def _off(name: str) -> bool:
    return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")


def _on(name: str) -> bool:
    return (os.environ.get(name) or "").strip().lower() in ("1", "on", "true", "yes")


def moss_fused_disabled() -> bool:
    return _off(MOSS_FUSED_ENV)


def _blocks(root: Any) -> list:
    from .diffusion_moss_image import _Block
    return [m for m in root.modules() if type(m) is _Block]


def _int8_runnable(linear: Any) -> bool:
    from torch import nn

    from .diffusion_convrot import is_rotated_linear
    from .diffusion_int8_fused import _I8_GEMM_MARK, _plain_int8_weight

    d = getattr(linear, "__dict__", {})
    return (
        (type(linear) is nn.Linear or is_rotated_linear(linear))
        and "forward" not in d
        and _I8_GEMM_MARK not in d
        and _plain_int8_weight(linear.weight)
    )


def install(
    transformer: Any,
    logger: Any = None,
    offload_active: bool = False,
) -> dict:
    """Before the first compile; deferred to the first forward when the weights are not on the GPU yet. A streamed
    (offloaded) denoiser keeps the stock projections: its hooks own each Linear's weights."""
    result = {"fused_qkv": 0, "int8_runner": 0, "qk_rope": 0}
    if (
        moss_fused_disabled()
        or offload_active
        or type(transformer).__name__ != "MOSSImageTransformer"
    ):
        uninstall(transformer)
        return result
    from .diffusion_int8_fused import resident_cuda_device, run_on_first_call

    if resident_cuda_device(transformer) is not None:
        return install_modules(transformer, logger)
    run_on_first_call(transformer, "moss_fused", lambda t: install_modules(t, logger))
    return {"fused_qkv": None, "int8_runner": None}


def install_modules(root: Any, logger: Any = None) -> dict:
    from .diffusion_int8_fused import arm_linear, int8_linear, resident_cuda_device
    import torch

    from .diffusion_moss_image import FUSED_QKV_ATTR, LINEAR_RUNNER_ATTR, QK_ROPE_ATTR
    from .diffusion_zimage_fused import _fuse_linears, _share_storage

    result = {"fused_qkv": 0, "int8_runner": 0, "qk_rope": 0}
    dev = resident_cuda_device(root)
    if dev is None or moss_fused_disabled():
        return result
    blocks = _blocks(root)
    int8 = any(_int8_runnable(b.proj["q"]) for b in blocks)
    armed = arm_linear(dev) if int8 else False
    fuse = not _off(MOSS_FUSED_QKV_ENV)
    index = dev.index if dev.index is not None else torch.cuda.current_device()
    rope_op = qk_rope_op() if _on(MOSS_QK_ROPE_ENV) and qk_rope_device_ok(index) else None
    with _LOCK:
        for block in blocks:
            norms_ok = all(
                n.weight.dtype == torch.bfloat16 and n.weight.numel() == 128
                for n in (block.q_norm, block.k_norm)
            )
            if rope_op is not None and norms_ok:
                block.__dict__[QK_ROPE_ATTR] = rope_op
                result["qk_rope"] += 1
            else:
                block.__dict__.pop(QK_ROPE_ATTR, None)
            runner = int8_linear if int8 and _int8_runnable(block.proj["q"]) else None
            if runner is not None:
                block.__dict__[LINEAR_RUNNER_ATTR] = runner
                result["int8_runner"] += 1
            if not fuse or runner is None:
                # bf16: one q/k/v GEMM measured neutral (B200, 0.0879 vs 0.0881 s/step, inside the A/A spread)
                block.__dict__.pop(FUSED_QKV_ATTR, None)
                continue
            parts = [block.proj[n] for n in ("q", "k", "v")]
            rec = block.__dict__.get(FUSED_QKV_ATTR)
            if rec is None or any(a is not b for a, b in zip(rec[1:4], parts)):
                fused = _fuse_linears(parts)
                if fused is None or not _share_storage(fused, parts):
                    continue
                block.__dict__[FUSED_QKV_ATTR] = (fused, *parts, runner)
            result["fused_qkv"] += 1
    if logger is not None:
        logger.info(
            "diffusion.moss_fused: one q/k/v GEMM on %d of %d block(s); int8 runner on %d (act quant kernel %s); "
            "fused q/k norm + RoPE on %d",
            result["fused_qkv"],
            len(blocks),
            result["int8_runner"],
            "probed" if armed else "torchao's",
            result["qk_rope"],
        )
    return result


def uninstall(transformer: Any = None) -> None:
    """Drop the fused Linears (the per-row views stay valid) and any pending install."""
    if transformer is None:
        return
    from .diffusion_int8_fused import cancel_first_call
    from .diffusion_moss_image import FUSED_QKV_ATTR, LINEAR_RUNNER_ATTR, QK_ROPE_ATTR

    cancel_first_call(transformer, "moss_fused")
    try:
        blocks = _blocks(transformer)
    except Exception:  # noqa: BLE001
        return
    with _LOCK:
        for block in blocks:
            block.__dict__.pop(FUSED_QKV_ATTR, None)
            block.__dict__.pop(LINEAR_RUNNER_ATTR, None)
            block.__dict__.pop(QK_ROPE_ATTR, None)


# Fuses q/k RMSNorm + RoPE + layout copy into one pass; keeps every eager bf16 rounding of the port.
MOSS_QK_ROPE_ENV = "UNSLOTH_DIFFUSION_MOSS_QK_ROPE"
_QK_ROPE_OP = "moss_qk_norm_rope"
_HEADS_PER_PROGRAM = 8


@functools.lru_cache(maxsize = 1)
def _qk_rope_kernel() -> Any:
    try:
        import triton
        import triton.language as tl
    except Exception:  # noqa: BLE001
        return None

    @triton.jit
    def _norm_rope(
        x_ptr,
        w_ptr,
        o_ptr,
        base,
        out,
        h,
        j,
        c1,
        s1,
        c2,
        s2,
        mask,
        EPS: tl.constexpr,  # norm epsilon, per model
        D: tl.constexpr,  # hd
    ):
        # rotate(x) = [-x2, x1]; each product and sum rounded to bf16 like eager
        x1 = tl.load(x_ptr + base + h * D + j, mask = mask, other = 0.0).to(tl.float32)
        x2 = tl.load(x_ptr + base + h * D + D // 2 + j, mask = mask, other = 0.0).to(tl.float32)
        r = 1.0 / tl.sqrt_rn((tl.sum(x1 * x1, axis = 1) + tl.sum(x2 * x2, axis = 1)) / D + EPS)
        w1 = tl.load(w_ptr + j).to(tl.float32)
        w2 = tl.load(w_ptr + D // 2 + j).to(tl.float32)
        n1 = ((x1 * r[:, None]).to(tl.bfloat16).to(tl.float32) * w1).to(tl.bfloat16).to(tl.float32)
        n2 = ((x2 * r[:, None]).to(tl.bfloat16).to(tl.float32) * w2).to(tl.bfloat16).to(tl.float32)
        o1 = (n1 * c1).to(tl.bfloat16).to(tl.float32) + (-n2 * s1).to(tl.bfloat16).to(tl.float32)
        o2 = (n2 * c2).to(tl.bfloat16).to(tl.float32) + (n1 * s2).to(tl.bfloat16).to(tl.float32)
        tl.store(o_ptr + out + j, o1.to(tl.bfloat16), mask = mask)
        tl.store(o_ptr + out + D // 2 + j, o2.to(tl.bfloat16), mask = mask)

    @triton.jit(do_not_specialize = ["S"])
    def kernel(
        q_ptr,
        k_ptr,
        v_ptr,
        qw_ptr,
        kw_ptr,
        cos_ptr,
        sin_ptr,
        oq_ptr,
        ok_ptr,
        ov_ptr,
        S,
        H,
        sqb,
        sqs,
        skb,
        sks,
        svb,
        svs,
        scs,
        EPS: tl.constexpr,  # norm epsilon, per model
        D: tl.constexpr,  # hd
        HB: tl.constexpr,  # heads per program
    ):
        row = tl.program_id(0)
        b = row // S
        s = row % S
        h = tl.program_id(1) * HB + tl.arange(0, HB)[:, None]
        j = tl.arange(0, D // 2)[None, :]
        mask = h < H
        c1 = tl.load(cos_ptr + s * scs + j).to(tl.float32)
        s1 = tl.load(sin_ptr + s * scs + j).to(tl.float32)
        c2 = tl.load(cos_ptr + s * scs + D // 2 + j).to(tl.float32)
        s2 = tl.load(sin_ptr + s * scs + D // 2 + j).to(tl.float32)
        out = ((b * H + h) * S + s) * D
        _norm_rope(
            q_ptr, qw_ptr, oq_ptr, b * sqb + s * sqs, out, h, j, c1, s1, c2, s2, mask, EPS, D
        )
        _norm_rope(
            k_ptr, kw_ptr, ok_ptr, b * skb + s * sks, out, h, j, c1, s1, c2, s2, mask, EPS, D
        )
        jj = tl.arange(0, D)[None, :]
        v = tl.load(v_ptr + b * svb + s * svs + h * D + jj, mask = mask, other = 0.0)
        tl.store(ov_ptr + out + jj, v, mask = mask)

    return kernel


def _launch_qk_rope(q, k, v, qw, kw, cos, sin, heads):
    """q / k / v: [B, S, H*D] bf16 (last dim contiguous); cos / sin: [1, S, 1, D]; returns [B, H, S, D] contiguous x3."""
    import triton

    batch, seq, width = q.shape
    dim = width // heads
    oq = q.new_empty((batch, heads, seq, dim))
    ok = torch_empty_like(oq)
    ov = torch_empty_like(oq)
    grid = (batch * seq, triton.cdiv(heads, _HEADS_PER_PROGRAM))
    with _on_card(q):
        _qk_rope_kernel()[grid](
            q,
            k,
            v,
            qw,
            kw,
            cos,
            sin,
            oq,
            ok,
            ov,
            seq,
            heads,
            q.stride(0),
            q.stride(1),
            k.stride(0),
            k.stride(1),
            v.stride(0),
            v.stride(1),
            cos.stride(1),
            EPS = 1e-6,
            D = dim,
            HB = _HEADS_PER_PROGRAM,
            # ptxas otherwise FMA-fuses the rounded products into the sum on sm_100
            enable_fp_fusion = False,
        )
    return oq, ok, ov


def _on_card(t):
    from .diffusion_vae_fused import _on_card as on_card
    return on_card(t)


def torch_empty_like(t):
    import torch
    return torch.empty_like(t)


def reference_qk_rope(q, k, v, qw, kw, cos, sin, heads):
    """The port's eager ops, in [B, H, S, D]."""
    import torch

    def norm(x, w):
        y = x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim = True) + 1e-6)
        return y.to(x.dtype) * w

    def rot(x):
        a, b = x.chunk(2, -1)
        return x * cos + torch.cat((-b, a), -1) * sin

    shape = (*q.shape[:2], heads, q.shape[-1] // heads)
    qh = rot(norm(q.unflatten(-1, (heads, -1)), qw).reshape(shape))
    kh = rot(norm(k.unflatten(-1, (heads, -1)), kw).reshape(shape))
    return qh.transpose(1, 2), kh.transpose(1, 2), v.reshape(shape).transpose(1, 2)


@functools.lru_cache(maxsize = 1)
def qk_rope_op() -> Any:
    """``unsloth_studio::moss_qk_norm_rope`` (opaque to Inductor, allocation only, CUDA-graph safe), or None."""
    try:
        import torch
    except Exception:  # noqa: BLE001
        return None
    ns = getattr(torch.ops, "unsloth_studio", None)
    if ns is not None and hasattr(ns, _QK_ROPE_OP):
        return getattr(ns, _QK_ROPE_OP)
    if _qk_rope_kernel() is None:
        return None
    try:

        @torch.library.custom_op(
            f"unsloth_studio::{_QK_ROPE_OP}",
            mutates_args = (),
            schema = "(Tensor q, Tensor k, Tensor v, Tensor qw, Tensor kw, Tensor cos, Tensor sin, int heads)"
            " -> (Tensor, Tensor, Tensor)",
        )
        def _op(q, k, v, qw, kw, cos, sin, heads):
            return _launch_qk_rope(q, k, v, qw, kw, cos, sin, heads)

        @_op.register_fake
        def _(q, k, v, qw, kw, cos, sin, heads):
            batch, seq, width = q.shape
            out = q.new_empty((batch, heads, seq, width // heads))
            return out, torch.empty_like(out), torch.empty_like(out)
    except Exception:  # noqa: BLE001
        return None
    return getattr(torch.ops.unsloth_studio, _QK_ROPE_OP)


@functools.lru_cache(maxsize = 8)
def qk_rope_device_ok(index: int) -> bool:
    """Kernel matches eager ops: every element within two bf16 ulps of its rotated pair magnitude,
    at most 0.5% off at all, over ragged lengths and big values."""
    try:
        import torch

        op = qk_rope_op()
        if op is None:
            return False
        dev = torch.device("cuda", index)
        g = torch.Generator(device = "cpu").manual_seed(7)
        for seq, heads in ((37, 4), (1029, 24), (5, 3)):
            width = heads * 128
            qkv = (torch.randn(2, seq, 3 * width, generator = g) * 3).to(torch.bfloat16).to(dev)
            qkv[:, :, :7] *= 300
            q, k, v = qkv.chunk(3, -1)
            qw = (torch.rand(128, generator = g) + 0.5).to(torch.bfloat16).to(dev)
            kw = (torch.rand(128, generator = g) + 0.5).to(torch.bfloat16).to(dev)
            ang = torch.rand(seq, 64, generator = g) * 50
            cos = torch.cat((ang.cos(), ang.cos()), -1)[None, :, None].to(torch.bfloat16).to(dev)
            sin = torch.cat((ang.sin(), ang.sin()), -1)[None, :, None].to(torch.bfloat16).to(dev)
            got = op(q, k, v, qw, kw, cos, sin, heads)
            ref = reference_qk_rope(q, k, v, qw, kw, cos, sin, heads)
            if not torch.equal(got[2], ref[2]):
                return False
            for a, r in zip(got[:2], ref[:2]):
                diff = (a.float() - r.float()).abs()
                # A bf16 tie moves an output a few ulps of the pair magnitude.
                rf = r.float()
                pair = (
                    rf[..., : rf.shape[-1] // 2].square() + rf[..., rf.shape[-1] // 2 :].square()
                ).sqrt()
                ulp = (torch.cat((pair, pair), -1) * 2.0**-6).clamp_min(2.0**-14)
                if bool((diff > ulp).any()) or float((diff > 0).float().mean()) > 5e-3:
                    return False
        from .diffusion_cuda_graph import synchronize_outside_capture

        synchronize_outside_capture(dev)
        return True
    except Exception:  # noqa: BLE001
        return False