Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

42.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Tests for ``diffusion_int8_fused.py``. The kernel / model tests need CUDA, Triton and torchao's Int8Tensor."""

from __future__ import annotations

import textwrap

import pytest

from core.inference import diffusion_int8_fused as fused

torch = pytest.importorskip("torch")


def _cuda_int8_ready() -> bool:
    if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
        return False
    try:
        from torchao.quantization import Int8DynamicActivationInt8WeightConfig  # noqa: F401
        from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor  # noqa: F401
    except Exception:  # noqa: BLE001
        return False
    return fused._device_ok(torch.cuda.current_device())


needs_cuda = pytest.mark.skipif(
    not _cuda_int8_ready(), reason = "needs CUDA (not ROCm), Triton and torchao Int8Tensor"
)


def _cuda_int8_toolchain() -> bool:
    """The probe's prerequisites without the probe: on such a host a refused probe is a failure, not a skip."""
    if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
        return False
    try:
        from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor  # noqa: F401
    except Exception:  # noqa: BLE001
        return False
    return fused._triton_version_ok() and fused._triton_jit_toolchain_ok()


@pytest.mark.skipif(
    not _cuda_int8_toolchain(), reason = "needs CUDA (not ROCm), Triton >= 3.2, torchao"
)
def test_device_probe_accepts_this_torchao():
    fused._device_ok.cache_clear()
    assert fused._device_ok(torch.cuda.current_device())


def _int8_config():
    """Studio's int8 config as an ``Int8Tensor`` (0.17 defaults to the legacy tensor), no process-wide setter."""
    from torchao.quantization import Int8DynamicActivationInt8WeightConfig

    cfg = Int8DynamicActivationInt8WeightConfig(set_inductor_config = False)
    if hasattr(cfg, "version"):
        cfg.version = 2
    return cfg


def _require_int8tensor(module):
    """Skip when the torchao (<= 0.17) int8 config builds the legacy tensor: the fused path then keeps stock."""
    for m in module.modules():
        if isinstance(m, torch.nn.Linear) and type(m.weight).__name__ not in (
            "Parameter",
            "Tensor",
        ):
            if type(m.weight).__name__ != "Int8Tensor":
                pytest.skip(f"torchao int8 config builds {type(m.weight).__name__}, not Int8Tensor")
            return module
    return module


@pytest.fixture(autouse = True)
def _clean_env(monkeypatch):
    monkeypatch.delenv(fused.INT8_FUSED_ENV, raising = False)
    # pins the stock rotation; rotq_h16 dispatch: test_diffusion_convrot_unlisted.py
    monkeypatch.setenv("UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED", "0")
    monkeypatch.setattr(fused, "_RQ_DEVICES", frozenset())
    yield


def test_int_mm_finds_safe_int_mm_after_torchao_kernel_was_removed(monkeypatch):
    """torchao main moved ``safe_int_mm`` out of ``torchao.kernel.intmm`` (and deleted ``torchao.kernel``):
    the fused forwards must find it in its new home, not fail every int8 render with ModuleNotFoundError."""
    import sys
    import types

    calls = []
    new_home = types.ModuleType("torchao.quantization.quantize_.workflows.int8.kernels")
    new_home.safe_int_mm = lambda a, b: calls.append((a, b)) or "out"
    monkeypatch.setitem(sys.modules, "torchao.kernel", None)
    monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", None)
    monkeypatch.setitem(sys.modules, new_home.__name__, new_home)
    monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)
    weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))
    a = torch.ones(4, 2, dtype = torch.int8)
    assert fused._int_mm(a, weight) == "out"
    assert len(calls) == 1 and calls[0][0] is a and tuple(calls[0][1].shape) == (2, 3)


def test_int_mm_prefers_the_released_home(monkeypatch):
    """torchao <= 0.18 keeps ``torchao.kernel.intmm``: it wins over any other copy, so the capture-safe rebinding
    Studio installs there is the one that runs."""
    import sys
    import types

    old_home = types.ModuleType("torchao.kernel.intmm")
    old_home.safe_int_mm = lambda a, b: "old"
    new_home = types.ModuleType("torchao.quantization.quantize_.workflows.int8.kernels")
    new_home.safe_int_mm = lambda a, b: "new"
    monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", old_home)
    monkeypatch.setitem(sys.modules, new_home.__name__, new_home)
    monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)
    weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))
    assert fused._int_mm(torch.ones(4, 2, dtype = torch.int8), weight) == "old"


def _intmm_graph_breaks(monkeypatch, prepare) -> int:
    """Graph breaks dynamo counts in ``_int_mm`` after ``prepare`` ran with no int8 GEMM home cached yet."""
    import sys
    import types

    home = types.ModuleType("torchao.kernel.intmm")
    home.safe_int_mm = lambda a, b: a.float() @ b.float()
    monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", home)
    monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)
    prepare()
    weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))

    def fn(a):
        return fused._int_mm(a, weight) * 2

    torch._dynamo.reset()
    try:
        explained = torch._dynamo.explain(fn)(torch.ones(4, 2, dtype = torch.int8))
    finally:
        torch._dynamo.reset()
    return explained.graph_break_count


def test_int_mm_no_graph_break_after_offloaded_install(monkeypatch):
    """An offloaded denoiser: install returns before _finalize, yet the fused attention processors still run
    int8_linear on every int8 Linear. The GEMM home must already be cached, or each ``_int_mm`` under compile
    imports it (``importlib.import_module`` is a dynamo graph break per Linear)."""
    model = torch.nn.Linear(2, 3)
    monkeypatch.setattr(fused, "int8_fused_disabled", lambda: False)
    assert _intmm_graph_breaks(monkeypatch, lambda: fused.install(model, offload_active = True)) == 0


def test_int_mm_no_graph_break_after_finalize_early_return(monkeypatch):
    """_finalize returns early off CUDA (group offload, CPU), so cache the GEMM home first."""
    model = torch.nn.Linear(2, 3)
    assert fused.resident_cuda_device(model) is None
    assert _intmm_graph_breaks(monkeypatch, lambda: fused._finalize(model)) == 0


@pytest.mark.parametrize(
    "version, ok",
    [
        ("3.7.1", True),
        ("3.2.0", True),
        ("3.1.0", False),
        ("2.3.1", False),
        ("garbage", False),
        ("4.0", True),
    ],
)
def test_triton_version_gate(version, ok):
    assert fused._triton_version_ok(version) is ok


@pytest.mark.parametrize(
    "value, off", [("0", True), ("off", True), ("false", True), ("1", False), ("", False)]
)
def test_kill_switch(monkeypatch, value, off):
    monkeypatch.setenv(fused.INT8_FUSED_ENV, value)
    assert fused.int8_fused_disabled() is off


def test_install_noop_when_disabled(monkeypatch):
    monkeypatch.setenv(fused.INT8_FUSED_ENV, "0")
    mod = torch.nn.Sequential(torch.nn.Linear(8, 8))
    assert fused.install(mod) == 0
    assert "forward" not in mod[0].__dict__


def test_plain_tensor_is_not_eligible():
    assert fused._plain_int8_weight(torch.zeros(8, 8)) is False
    assert fused._plain_int8_weight(None) is False


def test_install_noop_on_cpu_module():
    FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward

    ff = FeedForward(64, 64, activation_fn = "gelu-approximate")
    assert fused.install(ff) == 0
    assert "forward" not in ff.__dict__


def _rand_inputs(
    m,
    n,
    *,
    ws_dtype = torch.float32,
    bias = True,
    seed = 0,
    xs_dtype = torch.float32,
):
    g = torch.Generator(device = "cpu").manual_seed(seed)
    c = torch.randint(-(2**20), 2**20, (m, n), generator = g, dtype = torch.int32).cuda()
    xs = (torch.rand(m, generator = g) * 1e-3 + 1e-5).to(torch.bfloat16).to(xs_dtype).cuda()
    ws = (torch.rand(n, generator = g) * 1e-4 + 1e-6).to(ws_dtype).cuda()
    b = (torch.randn(n, generator = g) * 0.1).to(torch.bfloat16).cuda() if bias else None
    return c, xs, ws, b


_XS_DTYPES = pytest.mark.parametrize(
    "xs_dtype", [torch.float32, torch.bfloat16], ids = ["xs_fp32", "xs_bf16"]
)


@needs_cuda
@_XS_DTYPES
@pytest.mark.parametrize(
    "m, n, ws_dtype, bias",
    [
        (4096, 12288, torch.float32, True),  # Qwen-Image / FLUX.1 MLP
        (4101, 3000, torch.float32, True),  # odd rows, N not a multiple of the chunk
        (512, 12288, torch.bfloat16, False),
        (17, 8, torch.float32, True),
    ],
)
def test_kernel_bit_exact_vs_eager_reference(m, n, ws_dtype, bias, xs_dtype):
    c, xs, ws, b = _rand_inputs(m, n, ws_dtype = ws_dtype, bias = bias, xs_dtype = xs_dtype)
    q, s = fused._launch(c, xs, ws, b, None)
    q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, None)
    assert s.dtype == s_ref.dtype == xs_dtype
    assert torch.equal(s, s_ref)
    assert torch.equal(q, q_ref)


@needs_cuda
@_XS_DTYPES
def test_kernel_small_activations_take_the_exact_amax_path(xs_dtype):
    # All pre-activations negative: max|gelu| comes from the negative lobe, not gelu(max y).
    c, xs, ws, b = _rand_inputs(256, 1024, bias = False, xs_dtype = xs_dtype)
    c = -c.abs() - 1
    q, s = fused._launch(c, xs, ws, b, None)
    q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, None)
    assert torch.equal(s, s_ref) and torch.equal(q, q_ref)


@needs_cuda
@_XS_DTYPES
@pytest.mark.parametrize("transposed", [False, True])
def test_kernel_prefix_segment(transposed, xs_dtype):
    bsz, seq, heads, hd, n = 2, 300, 4, 64, 1024
    c, xs, ws, b = _rand_inputs(bsz * seq, n, xs_dtype = xs_dtype)
    if transposed:  # SDPA output layout [B, H, S, D] viewed as [B, S, H, D]
        prefix = (
            (torch.randn(bsz, heads, seq, hd, device = "cuda") * 3).to(torch.bfloat16).transpose(1, 2)
        )
    else:
        prefix = (torch.randn(bsz, seq, heads, hd, device = "cuda") * 3).to(torch.bfloat16)
    q, s = fused._launch(c, xs, ws, b, prefix)
    q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, prefix)
    assert q.shape == (bsz * seq, heads * hd + n)
    assert torch.equal(s, s_ref) and torch.equal(q, q_ref)


def _quantized_ff(
    dim = 256,
    inner = 1024,
    seed = 0,
):
    FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward
    from torchao.quantization import quantize_

    torch.manual_seed(seed)
    ff = (
        FeedForward(dim, dim, mult = inner // dim, activation_fn = "gelu-approximate")
        .cuda()
        .to(torch.bfloat16)
        .eval()
    )
    for p in ff.parameters():
        p.data.normal_(0, 0.05)
    quantize_(ff, _int8_config())
    return _require_int8tensor(ff)


@needs_cuda
def test_feedforward_bit_identical_to_stock_eager():
    ff = _quantized_ff()
    x = torch.randn(2, 300, 256, device = "cuda", dtype = torch.bfloat16)
    with torch.no_grad():
        ref = ff(x)
        assert fused.install(ff) == 1
        out = ff(x)
    assert torch.equal(out, ref)


@needs_cuda
def test_feedforward_small_m_keeps_stock_path():
    ff = _quantized_ff()
    x = torch.randn(1, 8, 256, device = "cuda", dtype = torch.bfloat16)
    with torch.no_grad():
        ref = ff(x)
        fused.install(ff)
        out = ff(x)
    assert torch.equal(out, ref)


@needs_cuda
def test_feedforward_compiles_without_graph_break():
    ff = _quantized_ff()
    x = torch.randn(1, 512, 256, device = "cuda", dtype = torch.bfloat16)
    with torch.no_grad():
        eager = ff(x)
        torch._dynamo.reset()
        stock_compiled = torch.compile(ff, fullgraph = True)(x)
        fused.install(ff)
        torch._dynamo.reset()
        out = torch.compile(ff, fullgraph = True)(x)
    _assert_within_compile_floor(out, stock_compiled, eager)


@needs_cuda
def test_install_is_idempotent_and_holds_no_global_reference():
    import gc
    import weakref

    ff = _quantized_ff()
    assert fused.install(ff) == 1
    assert fused.install(ff) == 1
    ref = weakref.ref(ff)
    del ff
    gc.collect()
    assert ref() is None


@needs_cuda
def test_uninstall_restores_stock_forward():
    ff = _quantized_ff()
    fused.install(ff)
    assert "forward" in ff.__dict__
    fused.uninstall(ff)
    assert "forward" not in ff.__dict__
    assert not fused.is_installed(ff)


@needs_cuda
def test_flux_single_block_bit_identical_to_stock_eager():
    tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")
    from torchao.quantization import quantize_

    torch.manual_seed(0)
    blk = (
        tf.FluxSingleTransformerBlock(dim = 256, num_attention_heads = 4, attention_head_dim = 64)
        .cuda()
        .to(torch.bfloat16)
        .eval()
    )
    for p in blk.parameters():
        p.data.normal_(0, 0.05)
    quantize_(
        blk,
        _int8_config(),
        filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear) and "norm" not in fqn,
    )
    _require_int8tensor(blk)
    hid = torch.randn(1, 200, 256, device = "cuda", dtype = torch.bfloat16)
    enc = torch.randn(1, 40, 256, device = "cuda", dtype = torch.bfloat16)
    temb = torch.randn(1, 256, device = "cuda", dtype = torch.bfloat16)
    with torch.no_grad():
        ref = blk(hid, enc, temb)
        assert fused.install(blk) == 1
        out = blk(hid, enc, temb)
    assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])


@needs_cuda
@_XS_DTYPES
@pytest.mark.parametrize(
    "m, n, gate_col, value_col", [(4352, 10240, 0, 10240), (300, 1024, 1024, 0), (17, 64, 0, 64)]
)
def test_swiglu_kernel_bit_exact_vs_eager_reference(m, n, gate_col, value_col, xs_dtype):
    c, xs, ws, b = _rand_inputs(m, 2 * n, bias = False, xs_dtype = xs_dtype)
    q, s = fused._launch_swiglu(c, xs, ws, None, gate_col, value_col, n)
    q_ref, s_ref = fused.reference_dq_swiglu_quant(c, xs, ws, None, gate_col, value_col, n)
    assert s.dtype == s_ref.dtype == xs_dtype
    assert torch.equal(s, s_ref) and torch.equal(q, q_ref)


def _bf16_tie_ints(m, n, seed):
    """int32 accumulators in [2^24, 2^30) within one fp32 ulp of a bf16 rounding midpoint, both signs."""
    g = torch.Generator(device = "cpu").manual_seed(seed)
    e = torch.randint(24, 30, (m, n), generator = g, dtype = torch.int64)
    one = torch.ones_like(e)
    mid = (
        torch.bitwise_left_shift(one, e)
        + torch.randint(0, 128, (m, n), generator = g, dtype = torch.int64)
        * torch.bitwise_left_shift(one, e - 7)
        + torch.bitwise_left_shift(one, e - 8)
    )
    half_ulp = torch.bitwise_left_shift(one, e - 24)
    off = torch.randint(-1, 2, (m, n), generator = g, dtype = torch.int64) * half_ulp
    off = off + torch.randint(-1, 2, (m, n), generator = g, dtype = torch.int64)
    sign = torch.randint(0, 2, (m, n), generator = g, dtype = torch.int64) * 2 - 1
    return ((mid + off) * sign).to(torch.int32).cuda()


def _double_rounding_hits(c):
    """How many int32 values round differently int32 -> bf16 directly (exact RNE in int64) than via fp32 (torch)."""
    v = c.cpu().to(torch.int64)
    a = v.abs()
    e = torch.floor(torch.log2(a.double().clamp(min = 1))).to(torch.int64)
    shift = (e - 7).clamp(min = 1)
    one = torch.ones_like(a)
    q = torch.bitwise_right_shift(a, shift)
    r = a - torch.bitwise_left_shift(q, shift)
    half = torch.bitwise_left_shift(one, shift - 1)
    q = q + ((r > half) | ((r == half) & (q % 2 == 1))).to(torch.int64)
    single = torch.bitwise_left_shift(q, shift) * v.sign()
    double = c.cpu().float().to(torch.bfloat16).double().to(torch.int64)
    return int((single != double)[a >= 2**24].sum())


@needs_cuda
@_XS_DTYPES
def test_kernels_round_large_accumulators_like_torchao(xs_dtype):
    m, n = 64, 2048
    c = _bf16_tie_ints(m, n, 3)
    assert _double_rounding_hits(c) > 100
    _, xs, ws, b = _rand_inputs(m, n, xs_dtype = xs_dtype)
    xs = (xs * 2**-10).to(xs_dtype)  # keeps |c * xs| in the curved range of GELU / SiLU
    q, s = fused._launch(c, xs, ws, b, None)
    q_ref, s_ref = fused.reference_dq_gelu_quant(c, xs, ws, b, None)
    assert torch.equal(s, s_ref) and torch.equal(q, q_ref)
    q, s = fused._launch_swiglu(c, xs, ws, b, n // 2, 0, n // 2)
    q_ref, s_ref = fused.reference_dq_swiglu_quant(c, xs, ws, b, n // 2, 0, n // 2)
    assert torch.equal(s, s_ref) and torch.equal(q, q_ref)


@needs_cuda
def test_gelu_rounds_every_bf16_input_like_aten():
    assert fused._gelu_matches_aten(torch.device("cuda", torch.cuda.current_device()))


@needs_cuda
def test_epilogue_rounds_scale_product_and_bias_separately():
    assert fused._epilogue_matches_torchao(
        torch.device("cuda", torch.cuda.current_device()), 4101, 3000
    )


@needs_cuda
def test_reference_act_quant_is_torchao_own():
    from torchao.quantization.granularity import PerRow
    from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor

    g = torch.Generator(device = "cpu").manual_seed(5)
    h = torch.randn(300, 3072, generator = g) * (torch.rand(1, 3072, generator = g) * 6)
    h[:, :4] *= 80
    h[7] = 0
    h = h.to(torch.bfloat16).cuda()
    t = Int8Tensor.from_hp(h, PerRow())
    q, s = fused._reference_act_quant(h, t.scale.dtype)
    assert torch.equal(q, t.qdata) and torch.equal(s, t.scale.reshape(-1))


@needs_cuda
@_XS_DTYPES
def test_fake_ops_report_the_real_scale_dtype(xs_dtype):
    from torch._subclasses.fake_tensor import FakeTensorMode

    c, xs, ws, b = _rand_inputs(40, 256, xs_dtype = xs_dtype)
    op = fused._op()
    real = (op.gelu(c, xs, ws, b, None)[1], op.swiglu(c, xs, ws, b, 128, 0, 128)[1])
    with FakeTensorMode() as mode:
        fc, fxs, fws, fb = (mode.from_tensor(t) for t in (c, xs, ws, b))
        fake = (op.gelu(fc, fxs, fws, fb, None)[1], op.swiglu(fc, fxs, fws, fb, 128, 0, 128)[1])
    assert [t.dtype for t in fake] == [t.dtype for t in real] == [xs_dtype, xs_dtype]


@needs_cuda
def test_quantizing_leaves_fp32_matmul_precision_alone():
    # The default handler's config setter enables TF32 process-wide; Studio opts out.
    before = torch.get_float32_matmul_precision()
    _quantized_ff()
    assert torch.get_float32_matmul_precision() == before


def _quantize(module):
    from torchao.quantization import quantize_

    for p in module.parameters():
        p.data.normal_(0, 0.05)
    quantize_(module, _int8_config())
    return _require_int8tensor(module)


@needs_cuda
@pytest.mark.parametrize("kind", ["diffusers_swiglu", "zimage", "flux2", "qwenimage21"])
def test_swiglu_mlps_bit_identical_to_stock_eager(kind, monkeypatch):
    monkeypatch.setattr(fused, "_SWIGLU_ALL_LAYOUTS", True)
    torch.manual_seed(0)
    if kind == "diffusers_swiglu":
        FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward
        ff = FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)
    elif kind == "zimage":
        zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")
        ff = zmod.FeedForward(256, 512)
    elif kind == "flux2":
        f2 = pytest.importorskip("diffusers.models.transformers.transformer_flux2")
        ff = f2.Flux2FeedForward(256, inner_dim = 512)
    else:
        q21 = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")
        ff = q21.QwenImage21SwiGLUFeedForward(256, 512)
    ff = _quantize(ff.cuda().to(torch.bfloat16).eval())
    x = torch.randn(2, 150, 256, device = "cuda", dtype = torch.bfloat16)
    with torch.no_grad():
        ref = ff(x)
        torch._dynamo.reset()
        stock_compiled = torch.compile(ff, fullgraph = True)(x)
        assert fused.install(ff) == 1
        out = ff(x)
        torch._dynamo.reset()
        compiled = torch.compile(ff, fullgraph = True)(x)
    assert torch.equal(out, ref)
    _assert_within_compile_floor(compiled, stock_compiled, ref)


def _assert_within_compile_floor(compiled, stock_compiled, eager):
    # Inductor's act quant is not eager-exact, so the bar is stock compile's distance from eager.
    floor = (stock_compiled.float() - eager.float()).abs()
    ours = (compiled.float() - eager.float()).abs()
    assert ours.max().item() <= 1.5 * floor.max().item() + 1e-6
    assert ours.mean().item() <= 1.1 * floor.mean().item() + 1e-6


@needs_cuda
def test_cpu_placed_model_is_swapped_at_the_first_forward():
    ff = _quantized_ff().cpu()
    assert fused.install(ff) == 1
    assert not fused.is_installed(ff)
    ff = ff.cuda()
    x = torch.randn(1, 64, 256, device = "cuda", dtype = torch.bfloat16)
    with torch.no_grad():
        ff(x)
    assert fused.is_installed(ff)
    assert (
        "_unsloth_first_call_hooks" in ff.__dict__ and not ff.__dict__["_unsloth_first_call_hooks"]
    )


@needs_cuda
def test_zimage_swiglu_deferred_swap_engages_under_inference_mode():
    zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")
    torch.manual_seed(0)
    ff = _quantize(zmod.FeedForward(256, 512).to(torch.bfloat16).eval())
    assert fused.install(ff) == 1
    ff = ff.cuda()
    x = torch.randn(1, 64, 256, device = "cuda", dtype = torch.bfloat16)
    with torch.inference_mode():
        ref = type(ff).forward(ff, x)
        out = ff(x)
    assert fused.is_installed(ff)
    assert torch.equal(out, ref)


def test_offload_skips_install():
    assert fused.install(torch.nn.Linear(8, 8), offload_active = True) == 0


@needs_cuda
@pytest.mark.parametrize("kind", ["gelu", "swiglu"])
def test_convrot_linears_keep_the_stock_forward(kind, monkeypatch):
    # ConvRotLinear rotates the input before the GEMM; the fused _int_mm would skip it.
    from core.inference.diffusion_convrot import _install_rotation

    if kind == "gelu":
        ff = _quantized_ff()
        lins = (ff.net[0].proj, ff.net[2])
    else:
        FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward

        torch.manual_seed(0)
        ff = _quantize(
            FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)
            .cuda()
            .to(torch.bfloat16)
            .eval()
        )
        lins = (ff.net[0].proj, ff.net[2])
    monkeypatch.setattr(fused, "_SWIGLU_ALL_LAYOUTS", True)
    for lin in lins:
        _install_rotation(lin, 16)
    assert fused.install(ff) == 0
    assert not fused.is_installed(ff)


def _swiglu_module(kind):
    if kind == "diffusers_swiglu":
        FeedForward = pytest.importorskip("diffusers.models.attention").FeedForward
        return FeedForward(256, inner_dim = 512, activation_fn = "swiglu", bias = False)
    if kind == "zimage":
        return pytest.importorskip("diffusers.models.transformers.transformer_z_image").FeedForward(
            256, 512
        )
    if kind == "flux2":
        return pytest.importorskip(
            "diffusers.models.transformers.transformer_flux2"
        ).Flux2FeedForward(256, inner_dim = 512)
    return pytest.importorskip(
        "diffusers.models.transformers.transformer_qwenimage21"
    ).QwenImage21SwiGLUFeedForward(256, 512)


@pytest.mark.parametrize("kind", ["diffusers_swiglu", "zimage", "flux2", "qwenimage21"])
def test_swiglu_quality_gate_allows_zimage_only(kind):
    ff = _swiglu_module(kind)
    assert fused._swiglu_candidate(ff) is (kind == "zimage")
    assert fused._swiglu_layout_allowed(ff) is (kind == "zimage")


@needs_cuda
@pytest.mark.parametrize("kind", ["zimage", "flux2", "qwenimage21"])
def test_swiglu_install_count_follows_the_quality_gate(kind):
    torch.manual_seed(0)
    ff = _quantize(_swiglu_module(kind).cuda().to(torch.bfloat16).eval())
    assert fused.install(ff) == (1 if kind == "zimage" else 0)
    assert fused.is_installed(ff) is (kind == "zimage")


def test_ineligible_resident_model_never_probes_the_kernels(monkeypatch):
    probed = []
    monkeypatch.setattr(fused, "resident_cuda_device", lambda m: torch.device("cuda", 0))
    monkeypatch.setattr(fused, "_device_ok", lambda index: probed.append(index) or True)
    mod = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.GELU(), torch.nn.Linear(8, 8))
    assert fused.install(mod) == 0
    assert probed == []
    assert "forward" not in mod[0].__dict__


def test_uninstall_cancels_the_deferred_install(monkeypatch):
    mod = torch.nn.Sequential(torch.nn.Linear(8, 8))
    target = mod[0]
    monkeypatch.setattr(fused, "_ff_eligible", lambda m: m is target)
    finalized = []
    monkeypatch.setattr(fused, "_finalize", lambda t, logger = None: finalized.append(t) or 1)
    assert fused.install(mod) == 1
    assert "int8_fused" in mod.__dict__["_unsloth_first_call_hooks"]
    fused.uninstall(mod)
    mod(torch.randn(2, 8))
    assert finalized == []
    assert "int8_fused" not in mod.__dict__.get("_unsloth_first_call_hooks", {})
    assert not mod._forward_pre_hooks


def _tiny_flux():
    tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")
    torch.manual_seed(0)
    return tf.FluxTransformer2DModel(
        patch_size = 1,
        in_channels = 4,
        num_layers = 1,
        num_single_layers = 3,
        attention_head_dim = 16,
        num_attention_heads = 2,
        joint_attention_dim = 32,
        pooled_projection_dim = 32,
        axes_dims_rope = (4, 6, 6),
    ).eval()


def _flux_inputs():
    g = torch.Generator().manual_seed(1)
    return dict(
        hidden_states = torch.randn(1, 16, 4, generator = g),
        encoder_hidden_states = torch.randn(1, 8, 32, generator = g),
        pooled_projections = torch.randn(1, 32, generator = g),
        timestep = torch.tensor([1.0]),
        img_ids = torch.zeros(16, 3),
        txt_ids = torch.zeros(8, 3),
        return_dict = False,
    )


def _fake_cuda_install(monkeypatch, model):
    """Real _finalize swap on CPU (fused forwards fall back off CUDA); returns the list the swapped forward appends to."""
    calls = []
    real = fused._flux_single_forward

    def spy(self, *args, **kwargs):
        calls.append(self)
        return real(self, *args, **kwargs)

    monkeypatch.setattr(fused, "_flux_single_forward", spy)
    monkeypatch.setattr(fused, "resident_cuda_device", lambda m: torch.device("cuda", 0))
    monkeypatch.setattr(fused, "_device_ok", lambda index: True)
    monkeypatch.setattr(fused, "_op", lambda: None)
    monkeypatch.setattr(fused, "_ff_eligible", lambda m: False)
    monkeypatch.setattr(fused, "_prepare_swiglu", lambda m: False)
    monkeypatch.setattr(fused, "_has_eligible", lambda t: True)
    monkeypatch.setattr(
        fused, "_flux_single_eligible", lambda m: type(m).__name__ == "FluxSingleTransformerBlock"
    )
    assert fused.install(model) == len(model.single_transformer_blocks)
    return calls


def _fbcache(model):
    hooks = pytest.importorskip("diffusers.hooks")
    model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))


def _two_steps(model):
    outs = []
    with torch.no_grad(), model.cache_context("cond"):
        for _ in range(2):
            outs.append(model(**_flux_inputs())[0])
    return outs


def test_fused_flux_single_keeps_fbcache_hooks_installed_before(monkeypatch):
    # Swap must go under the FBCache hooks, else the tail hook never records residuals.
    import copy

    model = _tiny_flux()
    ref_model = copy.deepcopy(model)
    _fbcache(ref_model)
    ref = _two_steps(ref_model)

    _fbcache(model)
    wrappers = [b.__dict__.get("forward") for b in model.single_transformer_blocks]
    calls = _fake_cuda_install(monkeypatch, model)
    out = _two_steps(model)
    assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
    assert [b.__dict__.get("forward") for b in model.single_transformer_blocks] == wrappers
    assert len(calls) == len(model.single_transformer_blocks)

    model.disable_cache()
    for b in model.single_transformer_blocks:
        assert b.forward.__func__ is fused._flux_single_forward
    fused.uninstall(model)
    for b in model.single_transformer_blocks:
        assert not fused.is_installed(b)
        assert b.forward.__func__ is type(b).forward


def test_fused_flux_single_then_fbcache_and_uninstall_keeps_hooks(monkeypatch):
    import copy

    model = _tiny_flux()
    ref_model = copy.deepcopy(model)
    _fbcache(ref_model)
    ref = _two_steps(ref_model)

    calls = _fake_cuda_install(monkeypatch, model)
    _fbcache(model)
    out = _two_steps(model)
    assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
    assert len(calls) == len(model.single_transformer_blocks)
    wrappers = [b.__dict__.get("forward") for b in model.single_transformer_blocks]
    fused.uninstall(model)
    assert [b.__dict__.get("forward") for b in model.single_transformer_blocks] == wrappers
    calls.clear()
    out = _two_steps(model)
    # Both caches hold run 1's residuals; a fresh reference rounds step 0 differently on some CPUs.
    ref = _two_steps(ref_model)
    assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1]) and not calls


def test_fused_flux_single_rearms_a_compiled_cache_inner(monkeypatch):
    import copy

    from core.inference import diffusion_cache

    model = _tiny_flux()
    ref_model = copy.deepcopy(model)
    _fbcache(ref_model)
    ref = _two_steps(ref_model)

    compiled = []

    def fake_compile(fn, **kwargs):
        def wrapper(*args, **kw):
            return fn(*args, **kw)

        wrapper.inner = fn
        compiled.append(fn)
        return wrapper

    monkeypatch.setattr(torch, "compile", fake_compile)
    _fbcache(model)
    for b in model.single_transformer_blocks:
        b._compiled_call_impl = b._call_impl  # stands in for compile_repeated_blocks
    armed = diffusion_cache._compile_hooked_block_inners(model)
    assert armed == len(model.single_transformer_blocks)
    calls = _fake_cuda_install(monkeypatch, model)
    for b in model.single_transformer_blocks:
        inner = b._diffusers_hook.hooks["fbc_block_hook"].fn_ref.original_forward
        assert inner.inner.__func__ is fused._flux_single_forward
    out = _two_steps(model)
    assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
    assert len(calls) == len(model.single_transformer_blocks)

    fused.uninstall(model)
    for b in model.single_transformer_blocks:
        inner = b._diffusers_hook.hooks["fbc_block_hook"].fn_ref.original_forward
        assert inner.__func__ is type(b).forward


@needs_cuda
def test_int8_flux_under_fbcache_renders_through_the_fused_kernel():
    import copy

    tf = pytest.importorskip("diffusers.models.transformers.transformer_flux")
    hooks = pytest.importorskip("diffusers.hooks")
    from torchao.quantization import quantize_

    torch.manual_seed(0)
    model = (
        tf.FluxTransformer2DModel(
            patch_size = 1,
            in_channels = 16,
            num_layers = 1,
            num_single_layers = 2,
            attention_head_dim = 64,
            num_attention_heads = 4,
            joint_attention_dim = 64,
            pooled_projection_dim = 64,
            axes_dims_rope = (16, 24, 24),
        )
        .cuda()
        .to(torch.bfloat16)
        .eval()
    )
    for p in model.parameters():
        p.data.normal_(0, 0.05)
    quantize_(
        model,
        _int8_config(),
        filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear)
        and "single_transformer_blocks" in fqn
        and "norm" not in fqn,
    )
    _require_int8tensor(model)
    ref_model = copy.deepcopy(model)
    g = torch.Generator(device = "cuda").manual_seed(1)
    kwargs = dict(
        hidden_states = torch.randn(1, 256, 16, device = "cuda", dtype = torch.bfloat16, generator = g),
        encoder_hidden_states = torch.randn(
            1, 64, 64, device = "cuda", dtype = torch.bfloat16, generator = g
        ),
        pooled_projections = torch.randn(1, 64, device = "cuda", dtype = torch.bfloat16, generator = g),
        timestep = torch.tensor([1.0], device = "cuda"),
        img_ids = torch.zeros(256, 3, device = "cuda"),
        txt_ids = torch.zeros(64, 3, device = "cuda"),
        return_dict = False,
    )

    def run(m):
        outs = []
        with torch.no_grad(), m.cache_context("cond"):
            for _ in range(2):
                outs.append(m(**kwargs)[0])
        return outs

    ref_model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))
    ref = run(ref_model)
    model.enable_cache(hooks.FirstBlockCacheConfig(threshold = 1e9))
    assert fused.install(model) == 2
    out = run(model)
    assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1])
    assert all(fused.is_installed(b) for b in model.single_transformer_blocks)


def _zimage_convrot_ff(rotate):
    """Z-Image FeedForward(256, 512) with ``rotate`` of w1 / w2 / w3 ConvRot-rotated at group 256, then int8."""
    from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_

    from core.inference.diffusion_convrot import rotate_linears_, warm_rotation_cache

    zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")
    torch.manual_seed(0)
    ff = zmod.FeedForward(256, 512).cuda().to(torch.bfloat16).eval()
    for p in ff.parameters():
        p.data.normal_(0, 0.05)
    rotate_linears_(ff, list(rotate), 256)
    warm_rotation_cache(ff, "cuda", torch.bfloat16)
    quantize_(ff, _int8tensor_config(Int8DynamicActivationInt8WeightConfig))
    return _require_int8tensor(ff)


def _int8tensor_config(cls):
    """The Int8Tensor (v2) config the hosted int8 checkpoints carry; torchao <= 0.17 defaults to the legacy tensor."""
    try:
        return cls(version = 2)
    except TypeError:
        return cls()


def _outlier_input():
    g = torch.Generator(device = "cpu").manual_seed(3)
    x = torch.randn(2, 150, 256, generator = g)
    x[..., :4] *= 40
    return x.to(torch.bfloat16).cuda()


@needs_cuda
@pytest.mark.parametrize(
    "rotate, groups",
    [(("w1", "w2", "w3"), (256, 256)), (("w1", "w3"), (256, 0)), (("w2",), (0, 256))],
)
def test_zimage_convrot_swiglu_fuses_and_matches_stock_eager(rotate, groups):
    if not groups[1] and not _act_scale_is_fp32():
        # A plain w2 uses the kernel's own quant, matching torchao's fp32-scale act quant (>= 0.18) only.
        pytest.skip(
            "the fused SwiGLU kernel's act quant matches torchao >= 0.18 (fp32 activation scale) only"
        )
    ff = _zimage_convrot_ff(rotate)
    x = _outlier_input()
    with torch.no_grad():
        ref = ff(x)
        torch._dynamo.reset()
        stock_compiled = torch.compile(ff, fullgraph = True)(x)
        assert fused.install(ff) == 1
        assert ff.__dict__[fused._SWIGLU_ATTR][6:] == groups
        out = ff(x)
        torch._dynamo.reset()
        compiled = torch.compile(ff, fullgraph = True)(x)
    assert torch.equal(out, ref)
    _assert_within_compile_floor(compiled, stock_compiled, ref)


@needs_cuda
def test_zimage_convrot_swiglu_with_disagreeing_gate_and_value_keeps_stock():
    ff = _zimage_convrot_ff(("w1",))
    assert fused.install(ff) == 0
    assert not fused.is_installed(ff)


def _act_scale_is_fp32() -> bool:
    from torchao.quantization.granularity import PerRow
    from torchao.quantization.quantize_.workflows.int8.int8_tensor import Int8Tensor

    x = torch.ones(32, 64, dtype = torch.bfloat16)
    return Int8Tensor.from_hp(x, PerRow()).scale.dtype == torch.float32


@needs_cuda
@pytest.mark.parametrize(
    "m, k", [(4224, 3840), (4224, 10240), (77, 2560), (17, 4096), (40, 4352), (3, 256)]
)
def test_act_quant_kernel_is_bit_exact_vs_torchao(m, k):
    assert fused._act_quant_device_ok(torch.cuda.current_device())
    g = torch.Generator(device = "cpu").manual_seed(m + k)
    x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)
    x[:, :5] *= 80
    x[min(1, m - 1)] = 0
    x = x.to(torch.bfloat16).cuda()
    q, s = fused._act_quant_op()(x)
    rq, rs = fused.reference_act_quant(x)
    # torchao scale dtype: bf16 up to 0.17, fp32 from 0.18.
    assert s.dtype == rs.dtype == (torch.float32 if fused._act_scale_fp32() else torch.bfloat16)
    assert torch.equal(q, rq) and torch.equal(s, rs)
    with torch.no_grad():
        torch._dynamo.reset()
        cq, cs = torch.compile(lambda t: fused._act_quant_op()(t), fullgraph = True)(x)
    assert torch.equal(cq, rq) and torch.equal(cs, rs)


@needs_cuda
@pytest.mark.parametrize("fast_quant", [False, True])
@pytest.mark.parametrize("rotated", [False, True])
@pytest.mark.parametrize("bias", [False, True])
def test_int8_linear_equals_the_module_without_the_zero_point_pass(
    rotated, bias, fast_quant, monkeypatch
):
    monkeypatch.setattr(fused, "_ACTQ_HANDLE", fused._act_quant_op() if fast_quant else None)
    monkeypatch.setattr(fused, "_ACTQ_DEVICES", frozenset({torch.cuda.current_device()}))
    from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_

    from core.inference.diffusion_convrot import rotate_linears_, warm_rotation_cache

    torch.manual_seed(0)
    holder = torch.nn.Sequential(torch.nn.Linear(512, 384, bias = bias)).cuda().to(torch.bfloat16)
    if rotated:
        rotate_linears_(holder, ["0"], 256)
        warm_rotation_cache(holder, "cuda", torch.bfloat16)
    quantize_(holder, _int8tensor_config(Int8DynamicActivationInt8WeightConfig))
    lin = _require_int8tensor(holder)[0]
    x = _outlier_input().repeat(1, 1, 2)
    with torch.no_grad():
        ref = lin(x)
        out = fused.int8_linear(lin, x)
        torch._dynamo.reset()
        compiled = torch.compile(lambda t: fused.int8_linear(lin, t), fullgraph = True)(x)
        stock_compiled = torch.compile(lin, fullgraph = True)(x)
    assert torch.equal(out, ref)
    _assert_within_compile_floor(compiled, stock_compiled, ref)
    small = x[:, :4]
    assert torch.equal(fused.int8_linear(lin, small), lin(small))


def test_kill_switch_also_drops_the_act_quant_kernel(monkeypatch):
    monkeypatch.setattr(fused, "_ACTQ_HANDLE", object())
    monkeypatch.setattr(fused, "_ACTQ_DEVICES", frozenset({0}))
    monkeypatch.setenv(fused.INT8_FUSED_ENV, "0")
    assert fused.install(torch.nn.Linear(8, 8)) == 0
    assert fused._ACTQ_HANDLE is None and not fused._ACTQ_DEVICES


@needs_cuda
def test_kill_switch_makes_int8_linear_the_module(monkeypatch):
    class Reached(Exception):
        pass

    def reached(*a, **k):
        raise Reached

    # Otherwise int8_linear would run a plain Linear itself.
    monkeypatch.setattr(fused, "_plain_int8_weight", lambda w: True)
    monkeypatch.setattr(fused, "_fast_act_quant", reached)
    lin = torch.nn.Linear(64, 32).cuda().to(torch.bfloat16)
    x = torch.randn(32, 64, device = "cuda", dtype = torch.bfloat16)
    monkeypatch.delenv(fused.INT8_FUSED_ENV, raising = False)
    fused.install(None)
    with pytest.raises(Reached):
        fused.int8_linear(lin, x)
    monkeypatch.setenv(fused.INT8_FUSED_ENV, "0")
    fused.install(None)
    assert torch.equal(fused.int8_linear(lin, x), lin(x))
    monkeypatch.delenv(fused.INT8_FUSED_ENV)
    fused.install(None)
    assert fused._LINEAR_OFF is False


@needs_cuda
def test_int8_linear_leaves_offload_hooks_and_off_device_weights_to_the_module(monkeypatch):
    from accelerate.hooks import ModelHook, add_hook_to_module

    class Reached(Exception):
        pass

    def reached(*a, **k):
        raise Reached

    monkeypatch.setattr(fused, "_LINEAR_OFF", False)
    monkeypatch.setattr(fused, "_plain_int8_weight", lambda w: True)
    monkeypatch.setattr(fused, "_fast_act_quant", reached)
    x = torch.randn(32, 64, device = "cuda", dtype = torch.bfloat16)
    lin = torch.nn.Linear(64, 32).cuda().to(torch.bfloat16)
    with pytest.raises(Reached):
        fused.int8_linear(lin, x)

    calls = []

    class Onload(ModelHook):
        def pre_forward(self, module, *args, **kwargs):
            calls.append(module.weight.device.type)
            module.to("cuda")
            return args, kwargs

    hooked = torch.nn.Linear(64, 32).cuda().to(torch.bfloat16)
    add_hook_to_module(hooked, Onload())
    assert torch.equal(fused.int8_linear(hooked, x), hooked(x)) and calls == ["cuda", "cuda"]
    off_device = torch.nn.Linear(64, 32).to(torch.bfloat16)
    with pytest.raises(RuntimeError):
        fused.int8_linear(off_device, x)


@needs_cuda
def test_act_quant_kernel_only_runs_on_a_device_that_passed_its_probe(monkeypatch):
    calls = []
    monkeypatch.setattr(fused, "_ACTQ_HANDLE", lambda x: calls.append(x) or (x, x[:, 0]))
    monkeypatch.setattr(fused, "_act_quant", lambda x, w: ("stock", None))
    x = torch.zeros(32, 256, device = "cuda", dtype = torch.bfloat16)
    monkeypatch.setattr(fused, "_ACTQ_DEVICES", frozenset({x.device.index + 1}))
    assert fused._fast_act_quant(x, None) == ("stock", None) and not calls
    monkeypatch.setattr(fused, "_ACTQ_DEVICES", frozenset({x.device.index}))
    fused._fast_act_quant(x, None)
    assert len(calls) == 1


def test_int8_linear_leaves_an_installed_int8_gemm_forward_alone(monkeypatch):
    from core.inference import diffusion_int8_gemm as g8

    assert fused._I8_GEMM_MARK == g8._MARK
    lin = torch.nn.Linear(8, 8)
    lin.__dict__[g8._MARK] = object()  # what diffusion_int8_gemm.install leaves on a swapped Linear
    lin.forward = lambda x: "int8_gemm forward"
    assert fused.int8_linear(lin, torch.randn(32, 8)) == "int8_gemm forward"


def test_fused_mlps_and_v2_linears_quantize_through_the_probed_kernel():
    """The three fused MLP forwards and diffusion_int8_gemm's v2 Linears use the probed one-pass act quant (bit
    identical to torchao's where it engages), not torchao's own quant."""
    import ast
    import inspect

    from core.inference import diffusion_int8_gemm as g8

    for fn in (
        fused._ff_forward,
        fused._flux_single_forward,
        fused._swiglu_forward,
        g8._act_quant_v2,
    ):
        calls = {
            n.func.id
            for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(fn))))
            if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
        }
        assert "_act_quant" not in calls, fn.__name__
        assert "_mlp_act_quant" in calls, fn.__name__


def test_mlp_act_quant_kill_switch(monkeypatch):
    monkeypatch.setattr(fused, "_fast_act_quant", lambda x, w: "fast")
    monkeypatch.setattr(fused, "_act_quant", lambda x, w: "torchao")
    monkeypatch.delenv(fused.MLP_FAST_ACT_QUANT_ENV, raising = False)
    monkeypatch.setenv(
        fused.INT8_FUSED_ENV, "0"
    )  # install returns right after reading the switches
    fused.install(None)
    assert fused._mlp_act_quant(None, None) == "fast"
    monkeypatch.setenv(fused.MLP_FAST_ACT_QUANT_ENV, "0")
    fused.install(None)
    assert fused._mlp_act_quant(None, None) == "torchao"
    monkeypatch.delenv(fused.MLP_FAST_ACT_QUANT_ENV)
    monkeypatch.delenv(fused.INT8_FUSED_ENV)
    fused.install(None)
    assert fused._MLP_FAST_ACTQ is True