Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

72.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Fused int8 FFN activation kernel, bit-identical to eager; only a plain dynamic symmetric per-row
Int8Tensor on CUDA with Triton 3.2 or newer takes it, else stock path;
UNSLOTH_DIFFUSION_INT8_FUSED=0 disables."""

from __future__ import annotations

import os
import sys
import threading
import types
from functools import lru_cache
from typing import Any, Optional
from .rocm_bf16 import is_rocm_torch

INT8_FUSED_ENV = "UNSLOTH_DIFFUSION_INT8_FUSED"
MLP_FAST_ACT_QUANT_ENV = "UNSLOTH_DIFFUSION_INT8_MLP_FAST_ACT_QUANT"
_MIN_TRITON = (3, 2)
# torch._int_mm needs more than 16 rows; torchao's own safe_int_mm handles the rest, so small M keeps the stock path.
_MIN_ROWS = 17
_OP_NAMESPACE = "unsloth_studio"
_OP_NAME = "int8_dq_gelu_quant"
_OP_NAME_SWIGLU = "int8_dq_swiglu_quant"
_OP_NAME_ACTQ = "int8_act_quant_rows"
_SWIGLU_ATTR = "_unsloth_i8_swiglu"

_LOCK = threading.Lock()
# Read by the traced forwards: dynamo must not trace into the lru_cache'd registration.
_OP_HANDLE: Any = None
# set once the act quant kernel matched torchao bit for bit on a device; None = torchao's quant
_ACTQ_HANDLE: Any = None
# device indices whose own probe passed; the kernel never runs on a device that failed or was never probed
_ACTQ_DEVICES: frozenset = frozenset()
_RQ_HANDLE: Any = None
_RQ_DEVICES: frozenset = frozenset()
# UNSLOTH_DIFFUSION_INT8_FUSED=0 at the last install: int8_linear is then module(x)
_LINEAR_OFF = False
# MLP_FAST_ACT_QUANT_ENV at the last install: a global, so traced forwards never read os.environ
_MLP_FAST_ACTQ = True
# torchao's safe_int_mm home, resolved at install (outside any trace) for the same reason.
_INTMM_MODULE: Any = None
# Marker on each patched module (no global registry: it would pin an unloaded transformer).
_MARK = "_unsloth_i8_fused_prev"
_NO_PREV = object()
# diffusion_int8_gemm's marker: int8_linear leaves those Linears to their fused-dequant GEMM forward
_I8_GEMM_MARK = "_unsloth_i8_gemm_prev"


def int8_fused_disabled() -> bool:
    return (os.environ.get(INT8_FUSED_ENV) or "").strip().lower() in ("0", "off", "false", "no")


def _triton_version_ok(version: Optional[str] = None) -> bool:
    if version is None:
        try:
            import triton
            version = str(triton.__version__)
        except Exception:  # noqa: BLE001
            return False
    import re

    match = re.match(r"(\d+)\.(\d+)", version)
    return bool(match) and (int(match.group(1)), int(match.group(2))) >= _MIN_TRITON


@lru_cache(maxsize = 1)
def _triton_jit_toolchain_ok() -> bool:
    """Windows: Triton needs the MSVC CRT headers. Elsewhere, or if the probe fails, True."""
    if sys.platform != "win32":
        return True
    try:
        from .._msvc_env import crt_headers_reachable
        return bool(crt_headers_reachable())
    except Exception:  # noqa: BLE001
        return True


@lru_cache(maxsize = 1)
def _kernels() -> Optional[types.SimpleNamespace]:
    """Compile-on-first-use Triton kernel, or None when Triton is unavailable."""
    try:
        import triton
        import triton.language as tl
    except Exception:  # noqa: BLE001 - no Triton means the stock path
        return None
    # Triton <= 3.2 resolves string annotations against module globals, not this function's locals
    globals().update(triton = triton, tl = tl)

    @triton.jit
    def _rbf16(x):
        # Round-to-nearest-even to bf16, kept in fp32 (one cvt; faster here than the integer-op form).
        return x.to(tl.bfloat16).to(tl.float32)

    @triton.jit
    def _gelu_tanh(y):
        # ATen's GELU(tanh) op for op (y / (1 + exp(-2u)) is NOT bf16-equal); *_rn: ptxas FMA-fuses f32x2 on sm_100.
        cube = tl.extra.cuda.libdevice.mul_rn(tl.extra.cuda.libdevice.mul_rn(y, y), y)
        inner = 0.7978845608028654 * tl.extra.cuda.libdevice.add_rn(
            y, tl.extra.cuda.libdevice.mul_rn(0.044715, cube)
        )
        return 0.5 * y * (1.0 + tl.extra.cuda.libdevice.tanh(inner))

    @triton.jit
    def _pre(
        c_ptr,
        xs,
        ws_ptr,
        b_ptr,
        offs,
        mask,
        HAS_BIAS: tl.constexpr,  # bias present
        WS_FP32: tl.constexpr,  # weight scale is fp32
        EVICT: tl.constexpr,  # load eviction policy
        FINAL_ROUND: tl.constexpr,  # round to bf16 at the end
        FP32_SCALE: tl.constexpr,  # scale kept in fp32 (torchao >= 0.18)
        PIN_RN: tl.constexpr,  # pin round-to-nearest mul/add (sm100+)
    ):
        # Epilogue as torchao: (int32 * x_scale) to bf16, * w_scale, + bias, then bf16; FINAL_ROUND=False defers the last round (it commutes with row max).
        c = tl.load(c_ptr + offs, mask = mask, other = 0, eviction_policy = EVICT)
        if FP32_SCALE:
            c = c.to(tl.float32)
        else:
            # int32 -> fp32 -> bf16 rounds twice; Triton folds a plain cast chain into one rounding.
            c = _rbf16(tl.extra.cuda.libdevice.int2float_rn(c))
        y = _rbf16(c * xs)
        w = tl.load(ws_ptr + offs, mask = mask, other = 0.0, eviction_policy = "evict_last").to(
            tl.float32
        )
        # PIN_RN (sm_100+): ptxas fuses packed f32x2 mul + add into an FMA even with fp fusion off; elsewhere it costs.
        if PIN_RN:
            y = tl.extra.cuda.libdevice.mul_rn(y, w)
        else:
            y = y * w
        if not WS_FP32:
            y = _rbf16(y)
        if HAS_BIAS:
            b = tl.load(b_ptr + offs, mask = mask, other = 0.0, eviction_policy = "evict_last").to(
                tl.float32
            )
            if PIN_RN:
                y = tl.extra.cuda.libdevice.add_rn(y, b)
            else:
                y = y + b
        if FINAL_ROUND:
            y = _rbf16(y)
        return y

    @triton.jit
    def dq_gelu_quant(
        c_ptr,
        xs_ptr,
        ws_ptr,
        b_ptr,
        p_ptr,
        q_ptr,
        s_ptr,
        N,
        NA,
        S,
        stride_c,
        stride_q,
        p_sb,
        p_ss,
        p_sh,
        HAS_BIAS: tl.constexpr,  # bias present
        WS_FP32: tl.constexpr,  # weight scale is fp32
        FP32_SCALE: tl.constexpr,  # scale kept in fp32 (torchao >= 0.18)
        PIN_RN: tl.constexpr,  # pin round-to-nearest mul/add (sm100+)
        HAS_PREFIX: tl.constexpr,  # fused prefix (attn output) columns
        HEAD_DIM: tl.constexpr,  # hd of the prefix
        CHUNK: tl.constexpr,  # columns per loop step
        CHUNK_A: tl.constexpr,  # prefix columns per loop step
    ):
        row = tl.program_id(0).to(tl.int64)
        xs = tl.load(xs_ptr + row).to(tl.float32)
        base = tl.arange(0, CHUNK)
        crow = c_ptr + row * stride_c
        qrow = q_ptr + row * stride_q
        # pass 1a: max of the pre-activation. GELU is increasing for y > -0.75 and |gelu(y)| < 0.16997 for y < 0,
        # so max|bf16(gelu)| over the row is bf16(gelu(max y)) whenever that is >= 0.171875; otherwise recompute.
        vmax = tl.full((CHUNK,), float("-inf"), tl.float32)
        for k in range(0, N, CHUNK):
            offs = k + base
            m = offs < N
            y = _pre(
                crow,
                xs,
                ws_ptr,
                b_ptr,
                offs,
                m,
                HAS_BIAS,
                WS_FP32,
                "evict_last",
                False,
                FP32_SCALE,
                PIN_RN,
            )
            vmax = tl.maximum(vmax, tl.where(m, y, float("-inf")))
        # bf16 rounding is monotone, so max(round(v)) == round(max(v)): round once, after the reduction.
        amax = _rbf16(_gelu_tanh(_rbf16(tl.max(vmax, axis = 0))))
        if amax < 0.171875:
            acc = tl.zeros((CHUNK,), dtype = tl.float32)
            for k in range(0, N, CHUNK):
                offs = k + base
                m = offs < N
                g = _rbf16(
                    _gelu_tanh(
                        _pre(
                            crow,
                            xs,
                            ws_ptr,
                            b_ptr,
                            offs,
                            m,
                            HAS_BIAS,
                            WS_FP32,
                            "evict_last",
                            True,
                            FP32_SCALE,
                            PIN_RN,
                        )
                    )
                )
                acc = tl.maximum(acc, tl.where(m, tl.abs(g), 0.0))
            amax = tl.max(acc, axis = 0)
        if HAS_PREFIX:
            # FluxSingleTransformerBlock: the bf16 attention output sits in front of the GELU branch in the row.
            b_idx = row // S
            s_idx = row % S
            pbase = p_ptr + b_idx * p_sb + s_idx * p_ss
            abase = tl.arange(0, CHUNK_A)
            pacc = tl.zeros((CHUNK_A,), dtype = tl.float32)
            for k in range(0, NA, CHUNK_A):
                j = k + abase
                m = j < NA
                v = tl.load(pbase + (j // HEAD_DIM) * p_sh + (j % HEAD_DIM), mask = m, other = 0.0).to(
                    tl.float32
                )
                pacc = tl.maximum(pacc, tl.abs(v))
            amax = tl.maximum(amax, tl.max(pacc, axis = 0))
        scale = _rbf16(tl.math.div_rn(amax, 127.5))
        scale = tl.maximum(scale, 1.1920928955078125e-07)
        inv = tl.math.div_rn(1.0, scale)
        if not FP32_SCALE:
            inv = _rbf16(inv)
        tl.store(s_ptr + row, scale.to(s_ptr.dtype.element_ty))
        if HAS_PREFIX:
            for k in range(0, NA, CHUNK_A):
                j = k + abase
                m = j < NA
                v = tl.load(pbase + (j // HEAD_DIM) * p_sh + (j % HEAD_DIM), mask = m, other = 0.0).to(
                    tl.float32
                )
                p = v * inv
                if not FP32_SCALE:
                    p = _rbf16(p)
                qi = tl.extra.cuda.libdevice.nearbyint(p)
                qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)
                tl.store(qrow + j, qi.to(tl.int8), mask = m)
        # pass 2: rebuild (the int32 row is mostly still in L2) and quantize.
        for k in range(0, N, CHUNK):
            offs = k + base
            m = offs < N
            g = _rbf16(
                _gelu_tanh(
                    _pre(
                        crow,
                        xs,
                        ws_ptr,
                        b_ptr,
                        offs,
                        m,
                        HAS_BIAS,
                        WS_FP32,
                        "evict_first",
                        True,
                        FP32_SCALE,
                        PIN_RN,
                    )
                )
            )
            p = g * inv
            if not FP32_SCALE:
                p = _rbf16(p)
            qi = tl.extra.cuda.libdevice.nearbyint(p)
            qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)
            tl.store(qrow + NA + offs, qi.to(tl.int8), mask = m)

    @triton.jit
    def _silu(y):
        # ATen: x / (1 + exp(-x)) in fp32
        return y / (1.0 + tl.exp(-y))

    @triton.jit
    def _swiglu_val(
        crow,
        xs,
        ws_ptr,
        b_ptr,
        g0,
        v0,
        offs,
        m,
        HAS_BIAS: tl.constexpr,  # bias present
        WS_FP32: tl.constexpr,  # weight scale is fp32
        EVICT: tl.constexpr,  # load eviction policy
        FP32_SCALE: tl.constexpr,  # scale kept in fp32 (torchao >= 0.18)
        PIN_RN: tl.constexpr,  # pin round-to-nearest mul/add (sm100+)
    ):
        g = _pre(
            crow + g0,
            xs,
            ws_ptr + g0,
            b_ptr + g0,
            offs,
            m,
            HAS_BIAS,
            WS_FP32,
            EVICT,
            True,
            FP32_SCALE,
            PIN_RN,
        )
        v = _pre(
            crow + v0,
            xs,
            ws_ptr + v0,
            b_ptr + v0,
            offs,
            m,
            HAS_BIAS,
            WS_FP32,
            EVICT,
            True,
            FP32_SCALE,
            PIN_RN,
        )
        return _rbf16(_rbf16(_silu(g)) * v)

    @triton.jit
    def dq_swiglu_quant(
        c_ptr,
        xs_ptr,
        ws_ptr,
        b_ptr,
        h_ptr,
        q_ptr,
        s_ptr,
        N,
        G0,
        V0,
        stride_c,
        stride_q,
        HAS_BIAS: tl.constexpr,  # bias present
        WS_FP32: tl.constexpr,  # weight scale is fp32
        FP32_SCALE: tl.constexpr,  # scale kept in fp32 (torchao >= 0.18)
        PIN_RN: tl.constexpr,  # pin round-to-nearest mul/add (sm100+)
        CHUNK: tl.constexpr,  # columns per loop step
    ):
        # Pass 1 parks the bf16 product in a scratch row: recomputing two int32 rows is ALU-bound.
        row = tl.program_id(0).to(tl.int64)
        xs = tl.load(xs_ptr + row).to(tl.float32)
        base = tl.arange(0, CHUNK)
        crow = c_ptr + row * stride_c
        hrow = h_ptr + row * N
        acc = tl.zeros((CHUNK,), dtype = tl.float32)
        for k in range(0, N, CHUNK):
            offs = k + base
            m = offs < N
            h = _swiglu_val(
                crow,
                xs,
                ws_ptr,
                b_ptr,
                G0,
                V0,
                offs,
                m,
                HAS_BIAS,
                WS_FP32,
                "evict_first",
                FP32_SCALE,
                PIN_RN,
            )
            tl.store(hrow + offs, h.to(tl.bfloat16), mask = m, eviction_policy = "evict_last")
            acc = tl.maximum(acc, tl.where(m, tl.abs(h), 0.0))
        amax = tl.max(acc, axis = 0)
        scale = _rbf16(tl.math.div_rn(amax, 127.5))
        scale = tl.maximum(scale, 1.1920928955078125e-07)
        inv = tl.math.div_rn(1.0, scale)
        if not FP32_SCALE:
            inv = _rbf16(inv)
        tl.store(s_ptr + row, scale.to(s_ptr.dtype.element_ty))
        qrow = q_ptr + row * stride_q
        for k in range(0, N, CHUNK):
            offs = k + base
            m = offs < N
            h = tl.load(hrow + offs, mask = m, other = 0.0, eviction_policy = "evict_first").to(
                tl.float32
            )
            p = h * inv
            if not FP32_SCALE:
                p = _rbf16(p)
            qi = tl.extra.cuda.libdevice.nearbyint(p)
            qi = tl.minimum(tl.maximum(qi, -128.0), 127.0)
            tl.store(qrow + offs, qi.to(tl.int8), mask = m)

    @triton.jit
    def act_quant_rows(
        x_ptr,
        q_ptr,
        s_ptr,
        K,
        stride_x,
        stride_q,
        BLOCK: tl.constexpr,  # block size along the row
        ONE_PASS: tl.constexpr,  # row fits one block
        FP32_SCALE: tl.constexpr,  # scale kept in fp32 (torchao >= 0.18)
    ):
        # Matches torchao from_hp PerRow: torchao <=0.17 rounds s, 1/s and x*(1/s) to bf16; >=0.18 keeps them fp32; q = clamp(rint, -128, 127).
        row = tl.program_id(0).to(tl.int64)
        offs = tl.arange(0, BLOCK)
        xrow = x_ptr + row * stride_x
        qrow = q_ptr + row * stride_q
        if ONE_PASS:
            m = offs < K
            x = tl.load(xrow + offs, mask = m, other = 0.0).to(tl.float32)
            amax = tl.max(tl.abs(x), axis = 0)
        else:
            acc = tl.zeros((BLOCK,), dtype = tl.float32)
            for k in range(0, K, BLOCK):
                m = k + offs < K
                x = tl.load(xrow + k + offs, mask = m, other = 0.0).to(tl.float32)
                acc = tl.maximum(acc, tl.abs(x))
            amax = tl.max(acc, axis = 0)
        scale = _rbf16(tl.maximum(_rbf16(tl.math.div_rn(amax, 127.5)), 1.1920928955078125e-07))
        inv = tl.math.div_rn(1.0, scale)
        if not FP32_SCALE:
            inv = _rbf16(inv)
        tl.store(s_ptr + row, scale.to(s_ptr.dtype.element_ty))
        if ONE_PASS:
            p = x * inv
            if not FP32_SCALE:
                p = _rbf16(p)
            p = tl.minimum(tl.maximum(tl.extra.cuda.libdevice.rint(p), -128.0), 127.0)
            tl.store(qrow + offs, p.to(tl.int8), mask = m)
        else:
            for k in range(0, K, BLOCK):
                m = k + offs < K
                x = tl.load(xrow + k + offs, mask = m, other = 0.0).to(tl.float32)
                p = x * inv
                if not FP32_SCALE:
                    p = _rbf16(p)
                p = tl.minimum(tl.maximum(tl.extra.cuda.libdevice.rint(p), -128.0), 127.0)
                tl.store(qrow + k + offs, p.to(tl.int8), mask = m)

    @triton.jit
    def gelu_bf16(x_ptr, o_ptr, n, BLOCK: tl.constexpr):  # block size along the row
        offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
        m = offs < n
        y = tl.load(x_ptr + offs, mask = m, other = 0.0).to(tl.float32)
        tl.store(o_ptr + offs, _gelu_tanh(y).to(tl.bfloat16), mask = m)

    @triton.jit
    def dq_bf16(
        c_ptr,
        xs_ptr,
        ws_ptr,
        b_ptr,
        o_ptr,
        N,
        HAS_BIAS: tl.constexpr,  # bias present
        WS_FP32: tl.constexpr,  # weight scale is fp32
        FP32_SCALE: tl.constexpr,  # scale kept in fp32 (torchao >= 0.18)
        PIN_RN: tl.constexpr,  # pin round-to-nearest mul/add (sm100+)
        CHUNK: tl.constexpr,  # columns per loop step
    ):
        row = tl.program_id(0).to(tl.int64)
        xs = tl.load(xs_ptr + row).to(tl.float32)
        base = tl.arange(0, CHUNK)
        for k in range(0, N, CHUNK):
            offs = k + base
            m = offs < N
            y = _pre(
                c_ptr + row * N,
                xs,
                ws_ptr,
                b_ptr,
                offs,
                m,
                HAS_BIAS,
                WS_FP32,
                "evict_first",
                True,
                FP32_SCALE,
                PIN_RN,
            )
            tl.store(o_ptr + row * N + offs, y.to(tl.bfloat16), mask = m)

    return types.SimpleNamespace(
        dq_gelu_quant = dq_gelu_quant,
        dq_swiglu_quant = dq_swiglu_quant,
        act_quant_rows = act_quant_rows,
        gelu_bf16 = gelu_bf16,
        dq_bf16 = dq_bf16,
        triton = triton,
    )


@lru_cache(maxsize = 8)
def _pin_rn_index(index: int) -> bool:
    import torch
    return torch.cuda.get_device_capability(index)[0] >= 10


def _pin_rn(device: Any) -> bool:
    """Packed f32x2 mul / add (which ptxas contracts) exist from sm_100 on."""
    import torch
    return _pin_rn_index(device.index if device.index is not None else torch.cuda.current_device())


def _scale_dtype(xs: Any) -> Any:
    """The output act-quant scale dtype: the incoming one (torchao's contract: fp32 on >= 0.18, bf16 on <= 0.17)."""
    import torch
    return torch.float32 if xs.dtype == torch.float32 else torch.bfloat16


def _launch(c: Any, xs: Any, ws: Any, bias: Any, prefix: Any) -> tuple:
    """Run the kernel. ``prefix`` is None or a [B, S, H, D] view (any strides, last dim contiguous)."""
    import torch

    k = _kernels()
    m_rows, n = c.shape
    if prefix is not None:
        b, s, h, d = prefix.shape
        na = h * d
        p_sb, p_ss, p_sh = prefix.stride(0), prefix.stride(1), prefix.stride(2)
    else:
        s, d, na = 1, 1, 0
        p_sb = p_ss = p_sh = 0
    q = torch.empty((m_rows, na + n), device = c.device, dtype = torch.int8)
    scale = torch.empty((m_rows,), device = c.device, dtype = _scale_dtype(xs))
    with torch.cuda.device(c.device):
        k.dq_gelu_quant[(m_rows,)](
            c,
            xs,
            ws,
            bias if bias is not None else ws,
            prefix if prefix is not None else ws,
            q,
            scale,
            n,
            na,
            s,
            c.stride(0),
            q.stride(0),
            p_sb,
            p_ss,
            p_sh,
            HAS_BIAS = bias is not None,
            WS_FP32 = ws.dtype == torch.float32,
            FP32_SCALE = xs.dtype == torch.float32,
            PIN_RN = _pin_rn(xs.device),
            HAS_PREFIX = prefix is not None,
            HEAD_DIM = d,
            CHUNK = 2048,
            CHUNK_A = 1024,
            num_warps = 8,
            enable_fp_fusion = False,
        )
    return q, scale


def _launch_swiglu(
    c: Any, xs: Any, ws: Any, bias: Any, gate_col: int, value_col: int, n: int
) -> tuple:
    """SwiGLU on the int32 output ``c`` of one fused GEMM: gate columns [gate_col, +n), value columns [value_col, +n)."""
    import torch

    k = _kernels()
    m_rows = c.shape[0]
    q = torch.empty((m_rows, n), device = c.device, dtype = torch.int8)
    scale = torch.empty((m_rows,), device = c.device, dtype = _scale_dtype(xs))
    scratch = torch.empty((m_rows, n), device = c.device, dtype = torch.bfloat16)
    with torch.cuda.device(c.device):
        k.dq_swiglu_quant[(m_rows,)](
            c,
            xs,
            ws,
            bias if bias is not None else ws,
            scratch,
            q,
            scale,
            n,
            gate_col,
            value_col,
            c.stride(0),
            q.stride(0),
            HAS_BIAS = bias is not None,
            WS_FP32 = ws.dtype == torch.float32,
            FP32_SCALE = xs.dtype == torch.float32,
            PIN_RN = _pin_rn(xs.device),
            CHUNK = 2048,
            num_warps = 8,
            enable_fp_fusion = False,
        )
    return q, scale


def reference_dq_swiglu_quant(
    c: Any, xs: Any, ws: Any, bias: Any, gate_col: int, value_col: int, n: int
) -> tuple:
    """Eager mirror: two torchao epilogues (gate, value), ATen SiLU, bf16 product, torchao per-row act quant."""
    import torch
    import torch.nn.functional as F

    def part(col):
        y = (c[:, col : col + n] * xs.reshape(-1, 1)).to(torch.bfloat16) * ws[col : col + n]
        if bias is not None:
            y = y + bias[col : col + n]
        return y.to(torch.bfloat16)

    h = F.silu(part(gate_col)) * part(value_col)
    return _reference_act_quant(h, _scale_dtype(xs))


def _reference_act_quant(h: Any, scale_dtype: Any) -> tuple:
    """torchao ``Int8Tensor.from_hp(h, PerRow())`` for a bf16 ``h``."""
    import torch

    amax = torch.maximum(-h.amin(dim = 1).clamp(max = 0), h.amax(dim = 1).clamp(min = 0))
    scale = (amax / 127.5).clamp(min = torch.finfo(torch.float32).eps).to(scale_dtype)
    q = torch.clamp(torch.round(h * (1.0 / scale).reshape(-1, 1)), -128, 127).to(torch.int8)
    return q, scale


@lru_cache(maxsize = 1)
def _op() -> Any:
    """The torch.library op (opaque to dynamo, CUDA-graph safe: no host sync, allocations only), or None."""
    try:
        import torch
    except Exception:  # noqa: BLE001
        return None
    qualname = f"{_OP_NAMESPACE}::{_OP_NAME}"
    ns = getattr(torch.ops, _OP_NAMESPACE, None)
    if ns is not None and hasattr(ns, _OP_NAME) and hasattr(ns, _OP_NAME_SWIGLU):
        return types.SimpleNamespace(
            gelu = getattr(ns, _OP_NAME), swiglu = getattr(ns, _OP_NAME_SWIGLU)
        )
    custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
    if custom_op is None:  # torch < 2.4
        return None
    try:
        # Explicit schema: string annotations + lazy torch import break schema inference.
        @custom_op(
            qualname,
            mutates_args = (),
            schema = "(Tensor c, Tensor xs, Tensor ws, Tensor? bias, Tensor? prefix) -> (Tensor, Tensor)",
        )
        def _dq_gelu_quant(c, xs, ws, bias, prefix):
            return _launch(c, xs, ws, bias, prefix)

        @_dq_gelu_quant.register_fake
        def _(c, xs, ws, bias, prefix):
            na = 0 if prefix is None else prefix.shape[-2] * prefix.shape[-1]
            return (
                c.new_empty((c.shape[0], na + c.shape[1]), dtype = torch.int8),
                c.new_empty((c.shape[0],), dtype = _scale_dtype(xs)),
            )

        @custom_op(
            f"{_OP_NAMESPACE}::{_OP_NAME_SWIGLU}",
            mutates_args = (),
            schema = "(Tensor c, Tensor xs, Tensor ws, Tensor? bias, int gate_col, int value_col, int n) -> (Tensor, Tensor)",
        )
        def _dq_swiglu_quant(c, xs, ws, bias, gate_col, value_col, n):
            return _launch_swiglu(c, xs, ws, bias, gate_col, value_col, n)

        @_dq_swiglu_quant.register_fake
        def _(c, xs, ws, bias, gate_col, value_col, n):
            return (
                c.new_empty((c.shape[0], n), dtype = torch.int8),
                c.new_empty((c.shape[0],), dtype = _scale_dtype(xs)),
            )
    except Exception:  # noqa: BLE001 - a registration failure keeps the stock path
        return None
    ns = getattr(torch.ops, _OP_NAMESPACE)
    return types.SimpleNamespace(gelu = getattr(ns, _OP_NAME), swiglu = getattr(ns, _OP_NAME_SWIGLU))


@lru_cache(maxsize = 1)
def _act_scale_fp32() -> bool:
    """Whether this torchao's ``Int8Tensor.from_hp`` returns an fp32 activation scale (0.18+) or the input's bf16."""
    try:
        import torch
        from torchao.quantization.granularity import PerRow
        from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor

        x = torch.ones(1, 16, dtype = torch.bfloat16)
        return Int8Tensor.from_hp(x, PerRow()).scale.dtype == torch.float32
    except Exception:  # noqa: BLE001 - the device probe still has to match torchao bit for bit
        return False


def _launch_act_quant(x2d: Any) -> tuple:
    """(int8 [M, K], scale [M] in torchao's dtype) for a contiguous-row bf16 ``x2d``."""
    import torch

    k = _kernels()
    m, kk = x2d.shape
    if x2d.stride(-1) != 1:
        x2d = x2d.contiguous()
    fp32 = _act_scale_fp32()
    q = torch.empty((m, kk), device = x2d.device, dtype = torch.int8)
    s = torch.empty((m,), device = x2d.device, dtype = torch.float32 if fp32 else x2d.dtype)
    if m == 0:
        return q, s
    one_pass = kk <= 4096
    block = k.triton.next_power_of_2(kk) if one_pass else 2048
    with torch.cuda.device(x2d.device):
        k.act_quant_rows[(m,)](
            x2d,
            q,
            s,
            kk,
            x2d.stride(0),
            q.stride(0),
            BLOCK = block,
            ONE_PASS = one_pass,
            FP32_SCALE = fp32,
            num_warps = 8,
            enable_fp_fusion = False,
        )
    return q, s


def reference_act_quant(x2d: Any) -> tuple:
    """torchao's own dynamic symmetric per-row quant of ``x2d``: (int8, scale [M])."""
    from torchao.quantization.granularity import PerRow
    from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor

    t = Int8Tensor.from_hp(x2d, PerRow())
    return t.qdata, t.scale.reshape(-1)


@lru_cache(maxsize = 1)
def _act_quant_op() -> Any:
    """``unsloth_studio::int8_act_quant_rows`` (opaque to Inductor, allocation only), or None."""
    try:
        import torch
    except Exception:  # noqa: BLE001
        return None
    ns = getattr(torch.ops, _OP_NAMESPACE, None)
    if ns is not None and hasattr(ns, _OP_NAME_ACTQ):
        return getattr(ns, _OP_NAME_ACTQ)
    custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
    if custom_op is None:
        return None
    try:

        @custom_op(
            f"{_OP_NAMESPACE}::{_OP_NAME_ACTQ}",
            mutates_args = (),
            schema = "(Tensor x) -> (Tensor, Tensor)",
        )
        def _int8_act_quant_rows(x):
            return _launch_act_quant(x)

        @_int8_act_quant_rows.register_fake
        def _(x):
            return (
                x.new_empty((x.shape[0], x.shape[1]), dtype = torch.int8),
                x.new_empty((x.shape[0],), dtype = torch.float32 if _act_scale_fp32() else x.dtype),
            )
    except Exception:  # noqa: BLE001
        return None
    return getattr(getattr(torch.ops, _OP_NAMESPACE), _OP_NAME_ACTQ)


@lru_cache(maxsize = 8)
def _act_quant_device_ok(index: int) -> bool:
    """The per-row act quant kernel must equal torchao's quant bit for bit on this device and torchao: outlier
    channels, a zero row, huge values, one-pass and looped widths, ragged M."""
    try:
        import torch

        if _kernels() is None or _act_quant_op() is None:
            return False
        dev = torch.device("cuda", index)
        g = torch.Generator(device = "cpu").manual_seed(4)
        for m, k in ((33, 3840), (257, 10240), (17, 2560), (130, 256 * 7), (19, 4096), (5, 4097)):
            x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)
            x[:, :5] *= 60
            x[1] = 0
            x[2, 3] = 3e4
            x = x.to(torch.bfloat16).to(dev)
            q, s = _launch_act_quant(x)
            rq, rs = reference_act_quant(x)
            if not (torch.equal(q, rq) and s.dtype == rs.dtype and torch.equal(s, rs)):
                return False
        from .diffusion_cuda_graph import synchronize_outside_capture

        synchronize_outside_capture(dev)
        return True
    except Exception:  # noqa: BLE001 - any failure keeps torchao's quant
        return False


def _fast_act_quant(x2d: Any, weight: Any) -> tuple:
    """``_act_quant`` for a plain per-row Int8Tensor weight: the probed kernel when it matched torchao on this
    device (one pass over the row at bandwidth, where Inductor's codegen of the same quant ran 2-3x slower on B200),
    else torchao's own."""
    import torch

    if (
        _ACTQ_HANDLE is not None
        and x2d.is_cuda
        and x2d.device.index in _ACTQ_DEVICES
        and x2d.dtype == torch.bfloat16
        and x2d.dim() == 2
        and x2d.shape[0] > 0
    ):
        q, s = _ACTQ_HANDLE(x2d)
        return q, s.reshape(-1, 1)
    return _act_quant(x2d, weight)


def _mlp_act_quant(x2d: Any, weight: Any) -> tuple:
    """``_fast_act_quant`` for ``_plain_int8_weight`` weights, torchao's quant under the kill switch."""
    if _MLP_FAST_ACTQ:
        return _fast_act_quant(x2d, weight)
    return _act_quant(x2d, weight)


@lru_cache(maxsize = 8)
def _device_ok(index: int) -> bool:
    """CUDA NVIDIA, Triton new enough, toolchain present, kernel compiles and matches the reference once."""
    try:
        import torch
    except Exception:  # noqa: BLE001
        return False
    if not torch.cuda.is_available() or is_rocm_torch(torch):
        return False
    if (
        not (_triton_version_ok() and _triton_jit_toolchain_ok())
        or _kernels() is None
        or _op() is None
    ):
        return False
    try:
        dev = torch.device("cuda", index)
        g = torch.Generator(device = "cpu").manual_seed(0)
        c = torch.randint(-(2**20), 2**20, (33, 200), generator = g, dtype = torch.int32)
        c[1:5] = _bf16_tie_ints((4, 200), g)
        c = c.to(dev)
        xs = (torch.rand(33, generator = g) * 1e-3 + 1e-5).to(torch.bfloat16)
        xs[1:5] *= 2**-10
        ws = (torch.rand(200, generator = g) * 1e-4 + 1e-6).to(dev)
        bias = (torch.randn(200, generator = g) * 0.1).to(torch.bfloat16).to(dev)
        for x_scale in (xs.float().to(dev), xs.to(dev)):
            for w_scale in (ws, ws.to(torch.bfloat16)):
                q, s = _launch(c, x_scale, w_scale, bias, None)
                q_ref, s_ref = reference_dq_gelu_quant(c, x_scale, w_scale, bias, None)
                if not (s.dtype == s_ref.dtype and torch.equal(q, q_ref) and torch.equal(s, s_ref)):
                    return False
                q, s = _launch_swiglu(c, x_scale, w_scale, bias, 104, 0, 96)
                q_ref, s_ref = reference_dq_swiglu_quant(c, x_scale, w_scale, bias, 104, 0, 96)
                if not (s.dtype == s_ref.dtype and torch.equal(q, q_ref) and torch.equal(s, s_ref)):
                    return False
        return (
            _act_quant_contract_ok(dev)
            and _gelu_matches_aten(dev)
            and _epilogue_matches_torchao(dev)
        )
    except Exception:  # noqa: BLE001 - any build / launch failure keeps the stock path
        return False


def _bf16_tie_ints(shape: tuple, generator: Any) -> Any:
    """int32 values in [2^24, 2^30) within one fp32 ulp of a bf16 midpoint: one vs two roundings disagree there."""
    import torch

    n = 1
    for d in shape:
        n *= d
    e = torch.randint(24, 30, (n,), generator = generator, dtype = torch.int64)
    mant = torch.randint(0, 128, (n,), generator = generator, dtype = torch.int64)
    ulp = torch.bitwise_left_shift(torch.ones_like(e), e - 23)
    off = torch.randint(-1, 2, (n,), generator = generator, dtype = torch.int64) * (ulp // 2).clamp(
        min = 1
    ) + torch.randint(-1, 2, (n,), generator = generator, dtype = torch.int64)
    mid = (
        torch.bitwise_left_shift(torch.ones_like(e), e)
        + mant * torch.bitwise_left_shift(torch.ones_like(e), e - 7)
        + torch.bitwise_left_shift(torch.ones_like(e), e - 8)
    )
    sign = torch.randint(0, 2, (n,), generator = generator, dtype = torch.int64) * 2 - 1
    return ((mid + off) * sign).to(torch.int32).reshape(shape)


def _epilogue_matches_torchao(
    dev: Any,
    m: int = 1031,
    n: int = 2056,
) -> bool:
    """The epilogue equals torchao's ``bf16(bf16(c * xs) * ws + bias)``; fp32 ``ws`` + bias exposes an FMA."""
    import torch

    g = torch.Generator(device = "cpu").manual_seed(2)
    c = torch.randint(-(2**20), 2**20, (m, n), generator = g, dtype = torch.int32).to(dev)
    ws = (torch.rand(n, generator = g) * 1e-4 + 1e-6).to(dev)
    bias = (torch.randn(n, generator = g) * 0.1).to(torch.bfloat16).to(dev)
    xs = (torch.rand(m, generator = g) * 1e-3 + 1e-5).to(torch.bfloat16)
    k = _kernels()
    for x_scale in (xs.float().to(dev), xs.to(dev)):
        out = torch.empty((m, n), device = dev, dtype = torch.bfloat16)
        with torch.cuda.device(dev):
            k.dq_bf16[(m,)](
                c,
                x_scale,
                ws,
                bias,
                out,
                n,
                HAS_BIAS = True,
                WS_FP32 = True,
                FP32_SCALE = x_scale.dtype == torch.float32,
                PIN_RN = _pin_rn(x_scale.device),
                CHUNK = 2048,
                num_warps = 8,
                enable_fp_fusion = False,
            )
        ref = ((c * x_scale.reshape(-1, 1)).to(torch.bfloat16) * ws + bias).to(torch.bfloat16)
        if not torch.equal(out, ref):
            return False
    return True


def _gelu_matches_aten(dev: Any) -> bool:
    """The kernels' GELU(tanh) equals ``F.gelu(x, approximate="tanh")`` on every finite bf16 ``x``, bit for bit."""
    import torch
    import torch.nn.functional as F

    x = torch.arange(-(2**15), 2**15, dtype = torch.int32).to(torch.int16).view(torch.bfloat16)
    x = x[torch.isfinite(x)].to(dev)
    out = torch.empty_like(x)
    k = _kernels()
    with torch.cuda.device(dev):
        k.gelu_bf16[(k.triton.cdiv(x.numel(), 1024),)](
            x, out, x.numel(), BLOCK = 1024, num_warps = 4, enable_fp_fusion = False
        )
    return bool(torch.equal(out.view(torch.int16), F.gelu(x, approximate = "tanh").view(torch.int16)))


def _act_quant_contract_ok(dev: Any) -> bool:
    """``_reference_act_quant`` in this torchao's scale dtype equals ``Int8Tensor.from_hp(h, PerRow())`` bit for bit."""
    import torch
    from torchao.quantization.granularity import PerRow
    from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor

    g = torch.Generator(device = "cpu").manual_seed(1)
    h = torch.randn(37, 520, generator = g) * (torch.rand(1, 520, generator = g) * 4)
    h[:, :3] *= 50
    h[2] = 0
    h = h.to(torch.bfloat16).to(dev)
    t = Int8Tensor.from_hp(h, PerRow())
    s_ref = t.scale.reshape(-1)
    q, s = _reference_act_quant(h, s_ref.dtype)
    return bool(torch.equal(q, t.qdata) and torch.equal(s, s_ref))


def reference_dq_gelu_quant(c: Any, xs: Any, ws: Any, bias: Any, prefix: Any) -> tuple:
    """Eager torch mirror of the stock chain (torchao epilogue, ATen GELU, torchao per-row act quant)."""
    import torch
    import torch.nn.functional as F

    y = (c * xs.reshape(-1, 1)).to(torch.bfloat16)
    y = y * ws.flatten()
    if bias is not None:
        y = y + bias
    g = F.gelu(y.to(torch.bfloat16), approximate = "tanh")
    if prefix is not None:
        g = torch.cat(
            [prefix.reshape(prefix.shape[0] * prefix.shape[1], -1).to(g.dtype), g], dim = -1
        )
    return _reference_act_quant(g, _scale_dtype(xs))


def _plain_int8_weight(w: Any) -> bool:
    """A torchao Int8Tensor with dynamic, symmetric, per-row activation quant and nothing else attached."""
    try:
        if type(w).__name__ != "Int8Tensor":
            return False
        kw = getattr(w, "act_quant_kwargs", None)
        if kw is None or getattr(w, "act_pre_scale", None) is not None:
            return False
        if (
            getattr(w, "act_quant_scale", None) is not None
            or getattr(w, "act_quant_zero_point", None) is not None
        ):
            return False
        if getattr(kw, "reduce_range", False) or getattr(w, "reduce_range", False):
            return False
        if "SYMMETRIC" not in str(getattr(kw, "mapping_type", "")) or "ASYMMETRIC" in str(
            getattr(kw, "mapping_type", "")
        ):
            return False
        if type(getattr(kw, "granularity", None)).__name__ != "PerRow":
            return False
        qd, sc = w.qdata, w.scale
        if qd.dim() != 2 or list(getattr(w, "block_size", [])) != [1, qd.shape[1]]:
            return False
        if sc.dim() != 2 or tuple(sc.shape) != (qd.shape[0], 1):
            return False
        return qd.shape[0] % 8 == 0 and qd.shape[1] % 8 == 0
    except Exception:  # noqa: BLE001
        return False


def _act_quant(x2d: Any, weight: Any) -> tuple:
    """torchao's own dynamic activation quant for ``weight`` (the stock path's call), as (int8 rows, fp32 scale)."""
    from torchao.quantization.quantize_.common.quantize_tensor_kwargs import (
        _choose_quant_func_and_quantize_tensor,
    )

    kwargs = {"scale": weight.act_quant_scale}
    # torchao <= 0.16 has no activation zero point (neither the attribute nor the kwarg).
    if hasattr(weight, "act_quant_zero_point"):
        kwargs["zero_point"] = weight.act_quant_zero_point
    act = _choose_quant_func_and_quantize_tensor(x2d, weight.act_quant_kwargs, **kwargs)
    return act.qdata, act.scale


def _resolve_intmm() -> Any:
    """The module holding torchao's ``safe_int_mm``, or None. torchao <= 0.18 ships it in ``torchao.kernel.intmm``;
    main after pytorch/ao#4718 moved it to the int8 workflow and deleted ``torchao.kernel``. Read off the module at
    call time, so Studio's capture-safe rebinding (diffusion_torchao_patches) is the one that runs."""
    import importlib

    from .diffusion_torchao_patches import _TORCHAO_INTMM_MODULES

    for name in _TORCHAO_INTMM_MODULES:
        try:
            module = importlib.import_module(name)
        except ImportError:
            continue
        if callable(getattr(module, "safe_int_mm", None)):
            return module
    return None


def ensure_intmm_module() -> Any:
    """Resolve and cache ``_INTMM_MODULE`` (idempotent). Must run eagerly, outside any trace: ``int8_linear`` runs on
    every int8 Linear of a fused attention processor whether or not ``_finalize`` installed anything (an offloaded or
    not yet resident denoiser), and an ``importlib.import_module`` reached from ``_int_mm`` under ``torch.compile`` is
    a graph break in every one of them."""
    global _INTMM_MODULE
    module = _INTMM_MODULE
    if module is None:
        module = _resolve_intmm()
        if module is not None:
            _INTMM_MODULE = module
    return module


def _int_mm(a: Any, weight: Any) -> Any:
    module = _INTMM_MODULE
    if module is None:  # a direct call before any install (tests): resolve eagerly
        module = _resolve_intmm()
        if module is None:
            raise ImportError("torchao ships no safe_int_mm in any known module")
    return module.safe_int_mm(a, weight.qdata.contiguous().t())


def _linear_from_q(q: Any, xs: Any, weight: Any, bias: Any, out_dtype: Any) -> Any:
    """torchao Int8Tensor linear on an already-quantized activation: same ops, same order, as the stock epilogue."""
    import torch

    if out_dtype == torch.bfloat16:
        from .diffusion_int8_gemm import linear_from_q
        fused = linear_from_q(q, xs.reshape(-1), weight, bias)
        if fused is not None:
            return fused
    inter = torch.float32 if xs.dtype == torch.float16 else xs.dtype
    y = (_int_mm(q, weight) * xs.reshape(-1, 1).to(inter)).to(out_dtype)
    y = y * weight.scale.flatten()
    if bias is not None:
        y = y + bias
    return y.to(out_dtype)


def _ff_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:
    """``FeedForward.forward`` for GELU(tanh) -> Linear with plain int8 weights."""
    import torch

    proj = self.net[0].proj
    down = self.net[2]
    lead = hidden_states.shape[:-1]
    x2d = hidden_states.reshape(-1, hidden_states.shape[-1])
    if (
        args
        or kwargs
        or x2d.shape[0] < _MIN_ROWS
        or not hidden_states.is_cuda
        or hidden_states.dtype != torch.bfloat16
    ):
        return type(self).forward(self, hidden_states, *args, **kwargs)
    xq, xs = _mlp_act_quant(x2d, proj.weight)
    c = _int_mm(xq.reshape(-1, xq.shape[-1]), proj.weight)
    q, s = _OP_HANDLE.gelu(c, xs.reshape(-1), proj.weight.scale.flatten(), proj.bias, None)
    y = _linear_from_q(q, s, down.weight, down.bias, hidden_states.dtype)
    for extra in self.net[3:]:
        y = extra(y)
    return y.reshape(*lead, y.shape[-1])


def _flux_single_forward(
    self: Any,
    hidden_states: Any,
    encoder_hidden_states: Any,
    temb: Any,
    image_rotary_emb: Any = None,
    joint_attention_kwargs: Any = None,
) -> Any:
    """``FluxSingleTransformerBlock.forward`` with the GELU branch, the concat and ``proj_out``'s activation
    quant fused into one kernel. Mirrors the installed class forward (stock or Studio's addcmul arch patch)."""
    import torch

    text_seq_len = encoder_hidden_states.shape[1]
    rows = hidden_states.shape[0] * (hidden_states.shape[1] + text_seq_len)
    if rows < _MIN_ROWS or hidden_states.dtype != torch.bfloat16 or not hidden_states.is_cuda:
        return type(self).forward(
            self,
            hidden_states,
            encoder_hidden_states,
            temb,
            image_rotary_emb,
            joint_attention_kwargs,
        )
    hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim = 1)
    residual = hidden_states
    norm_hidden_states, gate = self.norm(hidden_states, emb = temb)
    x2d = norm_hidden_states.reshape(rows, -1)
    xq, xs = _mlp_act_quant(x2d, self.proj_mlp.weight)
    c = _int_mm(xq.reshape(rows, -1), self.proj_mlp.weight)
    joint_attention_kwargs = joint_attention_kwargs or {}
    attn_output = self.attn(
        hidden_states = norm_hidden_states,
        image_rotary_emb = image_rotary_emb,
        **joint_attention_kwargs,
    )
    heads = self.attn.heads
    prefix = attn_output.unflatten(-1, (heads, attn_output.shape[-1] // heads))
    if prefix.stride(-1) != 1:
        prefix = prefix.contiguous()
    q, s = _OP_HANDLE.gelu(
        c, xs.reshape(-1), self.proj_mlp.weight.scale.flatten(), self.proj_mlp.bias, prefix
    )
    out = _linear_from_q(q, s, self.proj_out.weight, self.proj_out.bias, hidden_states.dtype)
    out = out.reshape(*hidden_states.shape[:-1], out.shape[-1])
    gate = gate.unsqueeze(1)
    if getattr(self, "_unsloth_i8_addcmul", False):
        hidden_states = torch.addcmul(residual, gate, out)
    else:
        hidden_states = residual + gate * out
    if hidden_states.dtype == torch.float16:
        hidden_states = hidden_states.clip(-65504, 65504)
    return hidden_states[:, :text_seq_len], hidden_states[:, text_seq_len:]


def _swiglu_spec(module: Any) -> Optional[tuple]:
    """(fused in-proj name, None, down name, width, gate column, value column) for the SwiGLU MLPs whose two halves
    already come from ONE Linear, else None."""
    name = type(module).__name__
    try:
        if name == "FeedForward" and type(module).__module__ == "diffusers.models.attention":
            from diffusers.models.activations import SwiGLU

            net = getattr(module, "net", None)
            if net is None or len(net) < 3 or type(net[0]) is not SwiGLU:
                return None
            if any(type(extra).__name__ != "Dropout" for extra in (net[1], *net[3:])):
                return None
            n = net[0].proj.out_features // 2
            # SwiGLU.forward: hidden, gate = proj(x).chunk(2) -> value first, gate second
            return ("net.0.proj", None, "net.2", n, n, 0)
        if (
            name == "Flux2FeedForward"
            and type(getattr(module, "act_fn", None)).__name__ == "Flux2SwiGLU"
        ):
            n = module.linear_in.out_features // 2
            return ("linear_in", None, "linear_out", n, 0, n)  # gate first, value second (FLUX.2)
    except Exception:  # noqa: BLE001
        return None
    return None


def _split_spec(module: Any) -> Optional[tuple]:
    """Two separate projections (gate, value) + down: Z-Image FeedForward (w1, w3, w2), Qwen-Image-2.1 SwiGLU."""
    name = type(module).__name__
    mod = type(module).__module__
    if (
        name == "FeedForward"
        and mod.endswith("transformer_z_image")
        and all(hasattr(module, n) for n in ("w1", "w2", "w3"))
    ):
        return ("w1", "w3", "w2")
    if name == "QwenImage21SwiGLUFeedForward":
        return ("gate_layer", "proj", "out")
    return None


# Off: bit-exact, but moved FLUX.2 / Qwen-Image compiled renders further from eager (LPIPS).
_SWIGLU_ALL_LAYOUTS = False


def _is_zimage_ff(module: Any) -> bool:
    return _split_spec(module) == ("w1", "w3", "w2") and type(module).__module__.endswith(
        "transformer_z_image"
    )


def _swiglu_layout_allowed(module: Any) -> bool:
    if _SWIGLU_ALL_LAYOUTS or _is_zimage_ff(module):
        return True
    # rotated down projection: the SwiGLU kernel never runs
    from .diffusion_qwenimage21_fused import rotated_ff
    return rotated_ff(module)


def _get(module: Any, dotted: str) -> Any:
    for part in dotted.split("."):
        module = module[int(part)] if part.isdigit() else getattr(module, part)
    return module


def _torchao_epilogue(c: Any, xs: Any, ws: Any, bias: Any, out_dtype: Any) -> Any:
    """torchao's Int8Tensor linear after ``_int_mm``, op for op: ``int_scaled_matmul``'s ``c * x_scale`` in the scale's
    dtype (fp32 for an fp16 scale), cast, ``* w_scale``, ``+ bias``, cast."""
    import torch

    inter = torch.float if xs.dtype == torch.half else xs.dtype
    y = (c * xs.reshape(-1, 1).to(inter)).to(out_dtype)
    y = y * ws.flatten()
    if bias is not None:
        y = y + bias
    return y.to(out_dtype)


def int8_linear(module: Any, x: Any) -> Any:
    """Skips the torchao 0.17 zero-point correction (a no-op for symmetric int8 activations); LoRA, legacy, small-M, CPU and other cases stay module(x)."""
    from torch import nn

    from .diffusion_convrot import is_rotated_linear

    # an instance-level forward is an offload hook (accelerate / diffusers) or another wrapper: only it may run
    if (
        _LINEAR_OFF
        or not (type(module) is nn.Linear or is_rotated_linear(module))
        or _I8_GEMM_MARK in module.__dict__
        or "forward" in module.__dict__
    ):
        return module(x)
    return int8_linear_core(module, x)


def int8_linear_core(module: Any, x: Any) -> Any:
    """``int8_linear`` minus its wrapper checks, for a forward installed on ``module``: fallbacks run the class
    forward, never ``module(x)`` (would recurse)."""
    import torch

    from .diffusion_convrot import is_rotated_linear

    rotated = is_rotated_linear(module)
    if _LINEAR_OFF:
        return _stock_call(module, x)
    weight = module.weight
    lead = x.shape[:-1]
    x2d = x.reshape(-1, x.shape[-1])
    if (
        x2d.shape[0] < _MIN_ROWS
        or not x.is_cuda
        or x.dtype != torch.bfloat16
        or not _plain_int8_weight(weight)
        or getattr(weight, "act_pre_scale", None) is not None
        or weight.device != x.device
    ):
        return _stock_call(module, x)
    if rotated:
        xq, xs = _rotated_act_quant(x2d, module.convrot_groupsize, weight)
    else:
        xq, xs = _fast_act_quant(x2d, weight)
    y = _torchao_epilogue(
        _int_mm(xq.reshape(-1, xq.shape[-1]), weight), xs, weight.scale, module.bias, x.dtype
    )
    return y.reshape(*lead, y.shape[-1])


def _stock_call(module: Any, x: Any) -> Any:
    if "forward" in module.__dict__:
        return type(module).forward(module, x)
    return module(x)


def _rotate(x2d: Any, group: int) -> Any:
    """``ConvRotLinear.forward``'s input rotation, same ops and dtype."""
    from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
    return rotate_convrot_activation(
        x2d, build_convrot_hadamard(group, device = x2d.device, dtype = x2d.dtype), group
    )


def _rotated_act_quant(x2d: Any, group: int, weight: Any) -> tuple:
    """``_fast_act_quant(_rotate(x2d, group), weight)``: one fused rotation + quant kernel (``rotq_h16``) on a device
    it was armed for (an arch the tiled ``rotq_i8`` does not cover), else the rotation GEMM + act quant."""
    import torch

    if (
        _RQ_HANDLE is not None
        and group == 256
        and x2d.is_cuda
        and x2d.device.index in _RQ_DEVICES
        and x2d.dtype == torch.bfloat16
        and x2d.dim() == 2
        and x2d.shape[0] > 0
        and x2d.shape[1] % 256 == 0
    ):
        q, s = _RQ_HANDLE(x2d, group, True)
        return q, s.reshape(-1, 1)
    return _fast_act_quant(_rotate(x2d, group), weight)


def arm_rotated(index: int) -> bool:
    """Arm ``rotq_h16`` for card ``index`` (outside any trace); False keeps the rotation GEMM + act quant."""
    global _RQ_HANDLE, _RQ_DEVICES
    if int8_fused_disabled():
        return False
    try:
        from .diffusion_convrot_quant import _rotq_op, unlisted_device_ok

        actq = _act_quant_op() if _act_quant_device_ok(index) else None
        ok = unlisted_device_ok(index, actq)
        handle = _rotq_op() if ok else None
    except Exception:  # noqa: BLE001 - optimisation only
        ok, handle = False, None
    with _LOCK:
        if ok and handle is not None:
            _RQ_HANDLE = handle
            _RQ_DEVICES = _RQ_DEVICES | {index}
            return True
        _RQ_DEVICES = _RQ_DEVICES - {index}
    return False


def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:
    """SwiGLU MLP with plain int8 weights: one act quant, one GEMM over [gate | value], fused SiLU-product quant.
    ConvRot parts: the shared input rotates once; a rotated down projection gets the bf16 product (see module doc)."""
    import torch

    rec = self.__dict__.get(_SWIGLU_ATTR)
    lead = hidden_states.shape[:-1]
    x2d = hidden_states.reshape(-1, hidden_states.shape[-1])
    ok = (
        rec is not None
        and not args
        and not kwargs
        and x2d.shape[0] >= _MIN_ROWS
        and hidden_states.is_cuda
    )
    ok = ok and hidden_states.dtype == torch.bfloat16
    if ok:
        fused_in, parts, down_name, n, gate_col, value_col, in_group, down_group = rec
        ok = (
            all(_get(self, nm) is mod for nm, mod in parts)
            and fused_in.weight.device == hidden_states.device
        )
    if not ok:
        return type(self).forward(self, hidden_states, *args, **kwargs)
    down = _get(self, down_name)
    if in_group:
        xq, xs = _rotated_act_quant(x2d, in_group, fused_in.weight)
    else:
        xq, xs = _mlp_act_quant(x2d, fused_in.weight)
    c = _int_mm(xq.reshape(-1, xq.shape[-1]), fused_in.weight)
    if down_group:
        ws = fused_in.weight.scale.flatten()
        bias = fused_in.bias
        dtype = hidden_states.dtype

        def half(col):
            return _torchao_epilogue(
                c[:, col : col + n],
                xs,
                ws[col : col + n],
                None if bias is None else bias[col : col + n],
                dtype,
            )

        gating = getattr(self, "_forward_silu_gating", None)
        g, v = half(gate_col), half(value_col)
        h = gating(g, v) if callable(gating) else torch.nn.functional.silu(g) * v
        y = int8_linear(down, h)
        return y.reshape(*lead, y.shape[-1])
    q, s = _OP_HANDLE.swiglu(
        c, xs.reshape(-1), fused_in.weight.scale.flatten(), fused_in.bias, gate_col, value_col, n
    )
    y = _linear_from_q(q, s, down.weight, down.bias, hidden_states.dtype)
    return y.reshape(*lead, y.shape[-1])


def _prepare_swiglu(module: Any) -> bool:
    """Attach the (off-tree) record the SwiGLU forward reads; False keeps the stock forward."""
    from torch import nn

    if not _swiglu_layout_allowed(module):
        return False
    spec = _swiglu_spec(module)
    if spec is not None:
        in_name, _unused, down_name, n, gate_col, value_col = spec
        lin, down = _get(module, in_name), _get(module, down_name)
        if type(lin) is not nn.Linear or type(down) is not nn.Linear:
            return False
        if not (_plain_int8_weight(lin.weight) and _plain_int8_weight(down.weight)):
            return False
        module.__dict__[_SWIGLU_ATTR] = (
            lin,
            ((in_name, lin), (down_name, down)),
            down_name,
            n,
            gate_col,
            value_col,
            0,
            0,
        )
        return True
    split = _split_spec(module)
    if split is None:
        return False
    gate_name, value_name, down_name = split
    gate, value, down = _get(module, gate_name), _get(module, value_name), _get(module, down_name)
    groups = _split_rotation_groups(gate, value, down)
    if groups is None:
        return False
    if not all(_plain_int8_weight(m.weight) for m in (gate, value, down)):
        return False
    from .diffusion_zimage_fused import _fuse_linears, _share_storage

    fused = _fuse_linears([gate, value])
    if fused is None or not _share_storage(fused, [gate, value]):
        return False
    n = gate.weight.shape[0]
    parts = ((gate_name, gate), (value_name, value), (down_name, down))
    module.__dict__[_SWIGLU_ATTR] = (fused, parts, down_name, n, 0, n, *groups)
    return True


def _split_rotation_groups(gate: Any, value: Any, down: Any) -> Optional[tuple]:
    """(in-projection ConvRot group, down ConvRot group), 0 = plain, for a split SwiGLU; None when the parts are
    neither plain Linears nor ConvRotLinears the forward reproduces (gate and value must agree: one input)."""
    from .diffusion_zimage_fused import _rotation_group

    in_group = _rotation_group([gate, value])
    down_group = _rotation_group([down])
    if in_group is None or down_group is None:
        return None
    return in_group, down_group


def _exact_linear(*modules: Any) -> bool:
    """Plain nn.Linear only: a subclass (ConvRotLinear) may transform the input the fused _int_mm would skip."""
    from torch import nn
    return all(type(m) is nn.Linear for m in modules)


def _ff_eligible(module: Any) -> bool:
    try:
        from diffusers.models.activations import GELU
        from diffusers.models.attention import FeedForward
    except Exception:  # noqa: BLE001
        return False
    if type(module) is not FeedForward:
        return False
    net = getattr(module, "net", None)
    if net is None or len(net) < 3:
        return False
    act, drop, down = net[0], net[1], net[2]
    if type(act) is not GELU or getattr(act, "approximate", None) != "tanh":
        return False
    if type(drop).__name__ != "Dropout" or not _exact_linear(act.proj, down):
        return False
    if any(type(extra).__name__ != "Dropout" for extra in net[3:]):
        return False
    return _plain_int8_weight(act.proj.weight) and _plain_int8_weight(down.weight)


def _flux_single_eligible(module: Any) -> bool:
    if type(module).__name__ != "FluxSingleTransformerBlock":
        return False
    try:
        act = module.act_mlp
        if type(act).__name__ != "GELU" or getattr(act, "approximate", None) != "tanh":
            return False
        if getattr(module.attn, "heads", None) is None or not getattr(
            module.attn, "pre_only", False
        ):
            return False
        if not _exact_linear(module.proj_mlp, module.proj_out):
            return False
        return _plain_int8_weight(module.proj_mlp.weight) and _plain_int8_weight(
            module.proj_out.weight
        )
    except Exception:  # noqa: BLE001
        return False


def _flux_single_class_is_arch_patched(module: Any) -> bool:
    try:
        from . import diffusion_arch_patches as ap
        return getattr(type(module), "forward", None) is getattr(
            ap, "_flux_single_forward", object()
        )
    except Exception:  # noqa: BLE001
        return False


def resident_cuda_device(module: Any) -> Any:
    """The one CUDA device every parameter of ``module`` lives on (NVIDIA, not ROCm), else None."""
    try:
        import torch
        if is_rocm_torch(torch):
            return None
        devices = {p.device for p in module.parameters()}
    except Exception:  # noqa: BLE001
        return None
    if len(devices) != 1:
        return None
    dev = next(iter(devices))
    return dev if dev.type == "cuda" else None


def resident_cuda_cards(module: Any) -> tuple:
    """The CUDA ordinals ``module``'s weights sit on: its one resident card, or every card of a layer split (each
    block on one card). Empty when any weight is off the GPU, or on ROCm."""
    dev = resident_cuda_device(module)
    if dev is not None:
        import torch
        return (dev.index if dev.index is not None else torch.cuda.current_device(),)
    from .diffusion_int8_gemm import _split_cards

    return _split_cards(module)


def run_on_first_call(module: Any, key: str, fn: Any) -> None:
    """Run ``fn(module)`` once, eagerly, right before ``module``'s first forward (after load-time placement moved
    the weights, before the regional compile traces the blocks and before any CUDA-graph capture). Idempotent per key."""
    hooks = module.__dict__.setdefault("_unsloth_first_call_hooks", {})
    if key in hooks:
        return

    def _pre_hook(mod, args, kwargs):
        handle = hooks.pop(key, None)
        if handle is not None:
            handle.remove()
        try:
            import torch

            # Under inference_mode, detaching torchao weight subclasses raises and fusion silently stays off.
            with torch.inference_mode(False), torch.no_grad():
                fn(mod)
        except Exception:  # noqa: BLE001 - an optimisation: the stock path stays
            pass
        return None

    hooks[key] = module.register_forward_pre_hook(_pre_hook, with_kwargs = True)


def cancel_first_call(module: Any, key: str) -> None:
    """Drop a ``run_on_first_call`` hook that has not fired yet (no-op if it has, or was never registered)."""
    hooks = getattr(module, "__dict__", {}).get("_unsloth_first_call_hooks")
    handle = hooks.pop(key, None) if hooks else None
    if handle is not None:
        handle.remove()


def install(
    transformer: Any,
    logger: Any = None,
    offload_active: bool = False,
) -> int:
    """Idempotent; returns the (candidate) count. Must run before the first compiled forward, which traces ``forward``."""
    global _ACTQ_HANDLE, _ACTQ_DEVICES, _RQ_DEVICES, _LINEAR_OFF, _MLP_FAST_ACTQ
    _MLP_FAST_ACTQ = (os.environ.get(MLP_FAST_ACT_QUANT_ENV) or "").strip().lower() not in (
        "0",
        "off",
        "false",
        "no",
    )
    if int8_fused_disabled():
        # process-global: a previous load's probe must not outlive the kill switch
        _ACTQ_HANDLE = None
        _ACTQ_DEVICES = frozenset()
        _RQ_DEVICES = frozenset()
        _LINEAR_OFF = True
        return 0
    _LINEAR_OFF = False
    # Before the offload / residency returns: int8_linear (fused attention processors) runs either way.
    try:
        ensure_intmm_module()
    except Exception:  # noqa: BLE001 - _int_mm resolves on its own
        pass
    if transformer is None or offload_active:
        return 0
    if resident_cuda_device(transformer) is not None:
        return _finalize(transformer, logger)
    candidates = sum(
        1
        for m in transformer.modules()
        if _ff_eligible(m) or _flux_single_eligible(m) or _swiglu_candidate(m)
    )
    if candidates:
        run_on_first_call(transformer, "int8_fused", lambda t: _finalize(t, logger))
    return candidates


def arm_linear(device: Any) -> bool:
    """Arm ``int8_linear`` on ``device`` for a denoiser with no fused-MLP candidate (MOSS-Image calls it from its own
    block forward): resolve torchao's ``safe_int_mm`` home and probe the act quant kernel. True when the probed kernel
    serves this device; False keeps torchao's quant (``int8_linear`` still works, at the stock quant's speed)."""
    global _ACTQ_HANDLE, _ACTQ_DEVICES, _INTMM_MODULE, _LINEAR_OFF
    if int8_fused_disabled():
        _LINEAR_OFF = True
        return False
    _LINEAR_OFF = False
    try:
        import torch

        intmm = _resolve_intmm()
        if intmm is None:
            return False
        _INTMM_MODULE = intmm
        index = device.index if device.index is not None else torch.cuda.current_device()
        if not _device_ok(index):
            return False
        actq = _act_quant_op() if _act_quant_device_ok(index) else None
    except Exception:  # noqa: BLE001 - optimisation only
        return False
    with _LOCK:
        if actq is None:
            _ACTQ_DEVICES = _ACTQ_DEVICES - {index}
            return False
        _ACTQ_HANDLE = actq
        _ACTQ_DEVICES = _ACTQ_DEVICES | {index}
    return True


def _swiglu_candidate(module: Any) -> bool:
    return (
        _swiglu_spec(module) is not None or _split_spec(module) is not None
    ) and _swiglu_layout_allowed(module)


def _swiglu_eligible(module: Any) -> bool:
    """The checks ``_prepare_swiglu`` makes, without fusing anything."""
    from torch import nn

    if not _swiglu_layout_allowed(module):
        return False
    try:
        spec = _swiglu_spec(module)
        if spec is not None:
            parts = (_get(module, spec[0]), _get(module, spec[2]))
            return all(type(m) is nn.Linear and _plain_int8_weight(m.weight) for m in parts)
        split = _split_spec(module)
        if split is None:
            return False
        parts = tuple(_get(module, name) for name in split)
        if _split_rotation_groups(*parts) is None:
            return False
        return all(_plain_int8_weight(m.weight) for m in parts)
    except Exception:  # noqa: BLE001
        return False


def _has_rotated(transformer: Any) -> bool:
    try:
        from .diffusion_convrot import is_rotated_linear
        return any(is_rotated_linear(m) for m in transformer.modules())
    except Exception:  # noqa: BLE001
        return False


def _has_eligible(transformer: Any) -> bool:
    return any(
        _MARK in m.__dict__ or _ff_eligible(m) or _swiglu_eligible(m) or _flux_single_eligible(m)
        for m in transformer.modules()
    )


def _finalize(transformer: Any, logger: Any = None) -> int:
    global _OP_HANDLE, _ACTQ_HANDLE, _ACTQ_DEVICES
    # Before every early return below: an offloaded denoiser keeps int8_linear in its attention processors.
    intmm = ensure_intmm_module()
    dev = resident_cuda_device(transformer)
    cards: tuple = ()
    if dev is None:
        from .diffusion_int8_gemm import _split_cards
        cards = _split_cards(transformer)
        if not cards:
            return 0
    # Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.
    if not _has_eligible(transformer):
        return 0
    if (
        intmm is None
    ):  # an int8 GEMM home this file does not know: keep the stock path rather than fail the render
        return 0
    import torch

    if cards:
        usable = {index for index in cards if _device_ok(index)}
        if not usable:
            return 0
        indices = sorted(usable)
    else:
        if not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):
            return 0
        indices = [dev.index if dev.index is not None else torch.cuda.current_device()]
    count = 0
    rearm = False
    if _has_rotated(transformer):
        for index in indices:
            arm_rotated(index)
    actq_ok = {index: _act_quant_device_ok(index) for index in indices}
    actq = _act_quant_op() if any(actq_ok.values()) else None
    with _LOCK:
        _OP_HANDLE = _op()
        for index, ok in actq_ok.items():
            if ok and actq is not None:
                _ACTQ_HANDLE = actq
                _ACTQ_DEVICES = _ACTQ_DEVICES | {index}
            else:
                _ACTQ_DEVICES = _ACTQ_DEVICES - {index}
        for _name, module in transformer.named_modules():
            if _MARK in module.__dict__:
                count += 1
                continue
            if cards and _module_card(module) not in usable:
                continue  # its card failed the kernel probe (or holds no weights of it): stock path
            if _ff_eligible(module):
                fn = _ff_forward
            elif _prepare_swiglu(module):
                fn = _swiglu_forward
            elif _flux_single_eligible(module):
                fn = _flux_single_forward
                module._unsloth_i8_addcmul = _flux_single_class_is_arch_patched(module)
            else:
                continue
            bound = types.MethodType(fn, module)
            slot = _hooked_forward_slot(module)
            if slot is not None:
                # A step cache hook owns the instance forward: swap under it so its skip logic survives.
                ref, attr = slot
                stock_inner = _drop_compiled_inner(module)
                module.__dict__[_MARK] = (
                    stock_inner if stock_inner is not None else getattr(ref, attr)
                )
                setattr(ref, attr, bound)
                rearm = rearm or stock_inner is not None
            else:
                module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
                module.forward = bound
            count += 1
    if rearm:
        try:
            from .diffusion_cache import _compile_hooked_block_inners
            _compile_hooked_block_inners(transformer, logger)
        except Exception:  # noqa: BLE001 - the fused inner still runs, eager
            pass
    if logger is not None and count:
        logger.info(
            "diffusion.int8_fused: %d int8 MLP region(s) run the fused dequant/activation/quant kernel",
            count,
        )
    return count


def _module_card(module: Any) -> Optional[int]:
    for t in module.parameters():
        dev = t.device
        return dev.index if dev.type == "cuda" else None
    return None


_OFF_TREE_RECORDS = (
    (_SWIGLU_ATTR, "swiglu"),
    ("_unsloth_fused_qkv", "qkv"),
    ("_unsloth_q21_fused_qkv", "qkv"),
)


def relocate_after_layer_split(transformer: Any, logger: Any = None) -> dict:
    """After a layer split, off-tree fused Linears are rebuilt on their parts' new card or dropped
    to the stock path, and cards failing the probe get stock forwards back; idempotent, must run
    before the first forward."""
    from .diffusion_zimage_fused import _fuse_linears, _share_storage

    out = {"rebuilt": 0, "dropped": 0, "restored": 0}
    global _ACTQ_DEVICES
    cards = {}
    with _LOCK:
        for module in transformer.modules():
            for attr, kind in _OFF_TREE_RECORDS:
                rec = module.__dict__.get(attr)
                if rec is None:
                    continue
                if kind == "swiglu":
                    parts = [m for _n, m in rec[1][:2]] if len(rec[1]) == 3 else None
                else:
                    parts = list(rec[1:4])
                if parts is None:
                    continue  # the in-projection is the module's own Linear: it moved with the block
                fused = rec[0]
                here = parts[0].weight.device
                if fused.weight.device == here:
                    continue
                rebuilt = _fuse_linears(parts)
                if rebuilt is not None and _share_storage(rebuilt, parts):
                    module.__dict__[attr] = (rebuilt, *rec[1:])
                    out["rebuilt"] += 1
                else:
                    module.__dict__.pop(attr, None)
                    out["dropped"] += 1
            if _MARK in module.__dict__:
                index = _module_card(module)
                if index is not None:
                    cards.setdefault(index, []).append(module)
    rotated = _has_rotated(transformer)
    for index, modules in cards.items():
        if _device_ok(index):
            if _act_quant_device_ok(index) and _ACTQ_HANDLE is not None:
                _ACTQ_DEVICES = _ACTQ_DEVICES | {index}
            if rotated:
                arm_rotated(index)
            continue
        for module in modules:
            _restore_module(module)
            out["restored"] += 1
    if logger is not None and any(out.values()):
        logger.info(
            "diffusion.int8_fused: layer split re-homed fused records (%d rebuilt, %d dropped, %d regions stock)",
            out["rebuilt"],
            out["dropped"],
            out["restored"],
        )
    return out


def drop_stranded_fused_records(transformer: Any) -> int:
    """Drop fused records whose parts left their card after a failed split (``pipe.to("cpu")`` cannot free them).
    Returns how many were dropped."""
    dropped = 0
    with _LOCK:
        for module in transformer.modules():
            for attr, kind in _OFF_TREE_RECORDS:
                rec = module.__dict__.get(attr)
                if rec is None:
                    continue
                if kind == "swiglu":
                    parts = [m for _n, m in rec[1][:2]] if len(rec[1]) == 3 else None
                else:
                    parts = list(rec[1:4])
                if parts is None:
                    continue  # the in-projection is the module's own Linear: it moved with the block
                try:
                    stranded = any(p.weight.device != rec[0].weight.device for p in parts)
                except Exception:  # noqa: BLE001 - an unreadable record cannot be trusted either
                    stranded = True
                if stranded:
                    module.__dict__.pop(attr, None)
                    dropped += 1
    return dropped


def _hooked_forward_slot(module: Any) -> Optional[tuple]:
    """(fn_ref, attribute) holding the module's own forward in the innermost ``HookFunctionReference``, else None."""
    registry = getattr(module, "_diffusers_hook", None)
    fn_refs = getattr(registry, "_fn_refs", None)
    if not fn_refs or "forward" not in getattr(module, "__dict__", {}):
        return None
    ref = fn_refs[0]
    if getattr(ref, "original_forward", None) is not None:
        return ref, "original_forward"
    if getattr(ref, "forward", None) is not None:
        return ref, "forward"
    return None


def _is_fused_forward(fn: Any, module: Any) -> bool:
    return getattr(fn, "__self__", None) is module and getattr(fn, "__func__", None) in (
        _ff_forward,
        _swiglu_forward,
        _flux_single_forward,
    )


def _class_forward(module: Any) -> Any:
    return types.MethodType(type(module).forward, module)


def _cache_hooks(module: Any) -> list:
    try:
        from .diffusion_cache import _CACHE_HOOK_NAMES
    except Exception:  # noqa: BLE001
        return []
    hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}
    return [hooks[name] for name in _CACHE_HOOK_NAMES if name in hooks]


def _drop_compiled_inner(module: Any) -> Any:
    """Drop a cache hook's compiled wrapper of the STOCK inner forward (deferred install); return that inner, else None."""
    stock = None
    for hook in _cache_hooks(module):
        inner = getattr(hook, "_unsloth_orig_inner", None)
        if inner is not None:
            hook._unsloth_orig_inner = None
            stock = inner
    return stock


def _disarm_fused_inner(module: Any, prev: Any) -> None:
    """Uninstall under an armed cache hook: its compiled inner wraps the fused forward, point it back at stock."""
    for hook in _cache_hooks(module):
        inner = getattr(hook, "_unsloth_orig_inner", None)
        if inner is not None and _is_fused_forward(inner, module):
            stock = prev if prev is not _NO_PREV else _class_forward(module)
            hook._unsloth_orig_inner = None
            try:
                hook.fn_ref.original_forward = stock
            except Exception:  # noqa: BLE001
                pass


def uninstall(transformer: Any = None) -> None:
    """Restore the stock forwards under ``transformer`` (a dropped transformer needs nothing: the patch is per instance)."""
    if transformer is None:
        return
    # A deferred install still pending (weights not yet on the GPU) must not fire at the next forward.
    cancel_first_call(transformer, "int8_fused")
    with _LOCK:
        for module in transformer.modules():
            _restore_module(module)


def _restore_module(module: Any) -> None:
    """One region's stock forward back (caller holds ``_LOCK`` or owns the module)."""
    prev = module.__dict__.pop(_MARK, None)
    if prev is None:
        return
    slot = _hooked_forward_slot(module)
    if slot is not None and _is_fused_forward(getattr(*slot), module):
        setattr(*slot, prev if prev is not _NO_PREV else _class_forward(module))
    elif _is_fused_forward(module.__dict__.get("forward"), module):
        if prev is _NO_PREV:
            module.__dict__.pop("forward", None)
        else:
            module.forward = prev
    _disarm_fused_inner(module, prev)
    module.__dict__.pop("_unsloth_i8_addcmul", None)
    module.__dict__.pop(_SWIGLU_ATTR, None)


def is_installed(module: Any) -> bool:
    return _MARK in getattr(module, "__dict__", {})