Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

17.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Cached FlashInfer dispatch: allowlist, all-or-nothing private import, per-device bit-identity."""

from __future__ import annotations

import sys
import types

import pytest

from core.inference import diffusion_nvfp4_dispatch as dispatch
from core.inference import diffusion_nvfp4_ops as ops

_PRIVATE = {
    "flashinfer.autotuner": ("AutoTuner",),
    "flashinfer.fp4_quantization": ("get_fp4_quantization_module",),
    "flashinfer.gemm.gemm_base": (
        "DEFAULT_WORKSPACE_SIZE",
        "_MM_FP4_TUNING_CONFIG_128x4",
        "_get_cache_buf",
        "get_cutlass_fp4_gemm_module",
    ),
    "flashinfer.utils": ("device_support_pdl", "get_compute_capability"),
}


@pytest.fixture(autouse = True)
def _clean_dispatch():
    dispatch.reset()
    yield
    dispatch.reset()


def _fake_flashinfer(
    monkeypatch,
    *,
    version = "0.6.6",
    drop = (),
):
    root = types.ModuleType("flashinfer")
    root.__version__ = version
    root.__path__ = []
    monkeypatch.setitem(sys.modules, "flashinfer", root)
    gemm = types.ModuleType("flashinfer.gemm")
    gemm.__path__ = []
    monkeypatch.setitem(sys.modules, "flashinfer.gemm", gemm)
    for name, symbols in _PRIVATE.items():
        module = types.ModuleType(name)
        for symbol in symbols:
            if symbol not in drop:
                setattr(module, symbol, object())
        monkeypatch.setitem(sys.modules, name, module)
    return root


def test_the_allowlisted_version_with_every_symbol_is_available(monkeypatch):
    _fake_flashinfer(monkeypatch)
    ok, reason = dispatch.available()
    assert ok is True, reason
    assert "0.6.6" in reason


@pytest.mark.parametrize("version", ["0.6.5", "0.6.7", "0.7.0", "unknown"])
def test_an_unlisted_version_refuses_even_when_every_symbol_is_there(monkeypatch, version):
    """Exact, not a minimum: a private symbol that moves in 0.6.7 is not a bug in 0.6.7."""
    _fake_flashinfer(monkeypatch, version = version)
    ok, reason = dispatch.available()
    assert ok is False
    assert version in reason and "public API" in reason


@pytest.mark.parametrize(
    "missing",
    ["AutoTuner", "_get_cache_buf", "_MM_FP4_TUNING_CONFIG_128x4", "get_cutlass_fp4_gemm_module"],
)
def test_one_missing_private_symbol_takes_the_whole_fast_path_down(monkeypatch, missing):
    _fake_flashinfer(monkeypatch, drop = (missing,))
    ok, reason = dispatch.available()
    assert ok is False
    assert "ImportError" in reason and missing in reason


def test_the_env_switch_refuses_before_it_imports_anything(monkeypatch):
    def _boom(*_a, **_kw):  # pragma: no cover - reached only on a regression
        raise AssertionError("the probe imported flashinfer under FAST_DISPATCH=0")

    monkeypatch.setitem(sys.modules, "flashinfer", property(_boom))
    monkeypatch.setenv(dispatch.NVFP4_FAST_DISPATCH_ENV, "0")
    ok, reason = dispatch.available()
    assert ok is False and reason.endswith("=0")


def test_env_one_skips_the_version_check_and_nothing_else(monkeypatch):
    monkeypatch.setenv(dispatch.NVFP4_FAST_DISPATCH_ENV, "1")
    _fake_flashinfer(monkeypatch, version = "0.7.0")
    assert dispatch.available()[0] is True

    dispatch.reset()
    _fake_flashinfer(monkeypatch, version = "0.7.0", drop = ("AutoTuner",))
    ok, reason = dispatch.available()
    assert ok is False and "AutoTuner" in reason


@pytest.mark.parametrize("value", ["", "auto", "AUTO", "yes", "2"])
def test_an_unrecognised_env_value_reads_as_auto(monkeypatch, value):
    monkeypatch.setenv(dispatch.NVFP4_FAST_DISPATCH_ENV, value)
    assert dispatch.fast_dispatch_env() == "auto"


def test_nothing_is_enabled_until_verify_has_passed_on_that_device(monkeypatch):
    _fake_flashinfer(monkeypatch)
    monkeypatch.setattr(ops, "_device_index", lambda device: int(device))
    assert dispatch.available()[0] is True
    assert dispatch.enabled(0) is False
    assert dispatch.quant_fn(0) is None


def test_a_failed_verify_is_remembered_and_never_retried(monkeypatch):
    _fake_flashinfer(monkeypatch)
    monkeypatch.setattr(ops, "_device_index", lambda device: int(device))
    calls = []

    def _fail(device):
        calls.append(device)
        return False, "stub failure"

    monkeypatch.setattr(dispatch, "_run_verify", _fail)
    assert dispatch.verify(0) == (False, "stub failure")
    assert dispatch.verify(0) == (False, "stub failure")
    assert len(calls) == 1
    assert dispatch.enabled(0) is False


def test_verify_does_not_run_the_gemm_when_the_library_is_wrong(monkeypatch):
    _fake_flashinfer(monkeypatch, version = "0.7.0")
    monkeypatch.setattr(ops, "_device_index", lambda device: int(device))

    def _boom(_device):  # pragma: no cover - reached only on a regression
        raise AssertionError("verify ran the GEMM on an unlisted version")

    monkeypatch.setattr(dispatch, "_run_verify", _boom)
    ok, reason = dispatch.verify(0)
    assert ok is False and "0.7.0" in reason


class _Ptr:
    def __init__(
        self,
        pointer,
        shape = (8, 4),
    ):
        self._pointer = pointer
        self.shape = shape

    def data_ptr(self):
        return self._pointer

    def detach(self):
        return self

    @property
    def T(self):
        return ("view", self._pointer, self.shape)


def test_a_cold_plan_is_never_built_during_a_capture(monkeypatch):
    """``choose_one`` may PROFILE, and a profiling launch inside a capture is in the graph forever."""
    _fake_flashinfer(monkeypatch)
    monkeypatch.setattr(ops, "_device_index", lambda device: 0)
    monkeypatch.setattr(dispatch, "enabled", lambda device: True)
    monkeypatch.setattr(ops, "_is_capturing", lambda: True)

    def _boom(*_a, **_kw):  # pragma: no cover - reached only on a regression
        raise AssertionError("a plan was built inside a capture")

    monkeypatch.setattr(dispatch, "_build_plan", _boom)
    xq = types.SimpleNamespace(device = 0, shape = (512, 1536))
    assert dispatch.gemm_plan(xq, None, None, None, None, None, 3072, "cutlass") is None

    dispatch._GEMM_PLAN[(512, 1536, 3072, "cutlass", 0)] = ("runner", 7, "ws")
    assert dispatch.gemm_plan(xq, None, None, None, None, None, 3072, "cutlass") == (
        "runner",
        7,
        "ws",
    )


def test_a_backend_other_than_cutlass_has_no_cached_plan(monkeypatch):
    monkeypatch.setattr(ops, "_device_index", lambda device: 0)
    monkeypatch.setattr(dispatch, "enabled", lambda device: True)
    monkeypatch.setattr(ops, "_is_capturing", lambda: False)
    xq = types.SimpleNamespace(device = 0, shape = (512, 1536))
    assert dispatch.gemm_plan(xq, None, None, None, None, None, 3072, "trtllm") is None


def test_the_transpose_cache_holds_the_view_and_revalidates_pointer_and_shape():
    weight = _Ptr(1024)
    view = dispatch.transposed(weight)
    assert dispatch.transposed(weight) is view
    other = _Ptr(1024, shape = (4, 8))
    assert dispatch.transposed(other) is not view
    weight._pointer = 2048
    assert dispatch.transposed(weight) is not view


def test_the_transpose_cache_is_bounded():
    held = [_Ptr(pointer) for pointer in range(dispatch._TRANSPOSE_CACHE_MAX)]
    for weight in held:
        dispatch.transposed(weight)
    assert len(dispatch._TRANSPOSED) == dispatch._TRANSPOSE_CACHE_MAX
    last = _Ptr(10**9)
    dispatch.transposed(last)
    assert len(dispatch._TRANSPOSED) == 1


def test_the_transpose_cache_releases_a_weight_that_was_collected():
    """A superseded load never resets, so only the cache letting go can free its weights."""
    import gc

    weight = _Ptr(4096)
    dispatch.transposed(weight)
    assert len(dispatch._TRANSPOSED) == 1
    del weight
    gc.collect()
    assert dispatch._TRANSPOSED == {}


def test_a_live_weight_survives_a_superseded_loads_collection():
    import gc

    live = _Ptr(64)
    live_view = dispatch.transposed(live)
    stale = _Ptr(128)
    dispatch.transposed(stale)
    del stale
    gc.collect()
    assert len(dispatch._TRANSPOSED) == 1
    assert dispatch.transposed(live) is live_view


def test_reset_clears_every_cache(monkeypatch):
    _fake_flashinfer(monkeypatch)
    monkeypatch.setattr(ops, "_device_index", lambda device: int(device))
    dispatch.available()
    held = _Ptr(7)
    dispatch.transposed(held)
    dispatch._GEMM_PLAN[("k",)] = ("runner", 0, "ws")
    dispatch._QUANT_FN[0] = ("fn", True)
    dispatch._VERIFIED[0] = (True, "ok")
    assert dispatch.enabled(0) is True

    dispatch.reset()
    assert dispatch._AVAILABLE is None
    assert not dispatch._GEMM_PLAN and not dispatch._QUANT_FN
    assert not dispatch._TRANSPOSED and not dispatch._VERIFIED
    assert dispatch.enabled(0) is False


def test_describe_reports_what_is_cached(monkeypatch):
    _fake_flashinfer(monkeypatch)
    monkeypatch.setattr(ops, "_device_index", lambda device: int(device))
    dispatch._VERIFIED[1] = (True, "ok")
    dispatch._VERIFIED[2] = (False, "nope")
    held = _Ptr(3)
    dispatch.transposed(held)
    record = dispatch.describe()
    assert record["available"] is True
    assert record["verified_devices"] == [1]
    assert record["transposed"] == 1


def test_a_quantiser_built_for_a_failed_verify_is_never_handed_out(monkeypatch):
    """A cache read ahead of the gate would hand a failed device the quantiser its verify rejected."""
    _fake_flashinfer(monkeypatch)
    monkeypatch.setattr(ops, "_device_index", lambda device: int(device))
    dispatch._QUANT_FN[0] = ("fn", True)
    dispatch._VERIFIED[0] = (False, "the cached quantiser is not bit-identical to nvfp4_quantize")
    assert dispatch.enabled(0) is False
    assert dispatch.quant_fn(0) is None
    x = types.SimpleNamespace(device = 0, is_contiguous = lambda: True)
    assert dispatch._fast_quantize(x, None) == (None, None)


def test_the_unload_reset_also_forgets_the_preflight_that_ran_verify():
    from core.inference import diffusion_nvfp4_linear as nl
    try:
        ops._PREFLIGHT[0] = {"ok": True, "fast_dispatch": True}
        dispatch._VERIFIED[0] = (True, "ok")
        nl.reset_nvfp4_state()
        assert not ops._PREFLIGHT
        assert not dispatch._VERIFIED
    finally:
        ops.reset_preflight_cache()


def _block_after(source: str, marker: str, lines: int) -> str:
    return "\n".join(source[source.index(marker) :].splitlines()[:lines])


def test_an_aborted_load_drops_the_caches_that_pin_the_failed_model():
    import inspect

    from core.inference.diffusion import DiffusionBackend
    from core.inference.video import VideoBackend

    load = inspect.getsource(DiffusionBackend.load_pipeline)
    assert "reset_nvfp4_state()" in _block_after(load, "diffusion.transformer_quant_fallback", 30)
    assert "reset_nvfp4_state()" in _block_after(load, "if not state_committed:", 20)
    assert "reset_nvfp4_state()" in _block_after(
        inspect.getsource(VideoBackend._run_load), "video.load_failed", 15
    )


CUDA_SHAPES = (
    (32, 2560, 3840),
    (1056, 3840, 3840),
    (1056, 3840, 10240),
    (4128, 3840, 10240),
    (27280, 3072, 14336),
)


def _cuda_or_skip():
    torch = pytest.importorskip("torch")
    if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
        pytest.skip("needs CUDA")
    if tuple(torch.cuda.get_device_capability(0)) not in ops.NVFP4_FLASHINFER_CAPS:
        pytest.skip("this device has no flashinfer NVFP4 kernels")
    pytest.importorskip("flashinfer")
    return torch


def test_the_real_flashinfer_verifies_bit_identical():
    torch = _cuda_or_skip()
    import flashinfer

    if flashinfer.__version__ not in dispatch._SUPPORTED:
        pytest.skip(f"flashinfer {flashinfer.__version__} is outside the allowlist by design")
    ok, reason = dispatch.verify(torch.device("cuda", 0))
    assert ok is True, reason
    assert dispatch.enabled(torch.device("cuda", 0)) is True


@pytest.mark.parametrize("m,k,n", CUDA_SHAPES)
def test_the_cached_dispatch_matches_the_public_api_exactly(m, k, n, monkeypatch):
    # Exact-length mode: both arms then read the tactic from FlashInfer's own AutoTuner, so the bits
    # must agree. The bucketed default picks its own (equally valid) tactic per bucket; it is covered
    # in test_diffusion_nvfp4_tactics.py.
    monkeypatch.setenv("UNSLOTH_NVFP4_TACTIC_CACHE", "0")
    torch = _cuda_or_skip()
    import flashinfer

    if flashinfer.__version__ not in dispatch._SUPPORTED:
        pytest.skip(f"flashinfer {flashinfer.__version__} is outside the allowlist by design")

    device = torch.device("cuda", 0)
    torch.manual_seed(m + k + n)
    with torch.cuda.device(device), torch.inference_mode():
        x = torch.randn(m, k, device = device, dtype = torch.bfloat16) * 0.05
        w = torch.randn(n, k, device = device, dtype = torch.bfloat16) * 0.02
        a_gsf, w_gsf = ops.global_scale(x), ops.global_scale(w)
        wq, w_sf = flashinfer.nvfp4_quantize(w, w_gsf, do_shuffle = False)
        alpha = (1.0 / (a_gsf * w_gsf)).float()

        dispatch.reset()
        assert dispatch.enabled(device) is False
        want_q, want_sf = ops._quantize_impl(x, a_gsf)
        want = ops._mm_impl(want_q, wq, want_sf, w_sf, alpha, n, ops.DEFAULT_MM_BACKEND)

        assert dispatch.verify(device)[0] is True
        got_q, got_sf = ops._quantize_impl(x, a_gsf)
        got = ops._mm_impl(got_q, wq, got_sf, w_sf, alpha, n, ops.DEFAULT_MM_BACKEND)
        torch.cuda.synchronize(device)

    assert torch.equal(want_q, got_q) and torch.equal(want_sf, got_sf)
    assert torch.equal(want, got), float((want.float() - got.float()).abs().max())
    assert (m, k // 2, n, ops.DEFAULT_MM_BACKEND, 0) in dispatch._GEMM_PLAN


def test_the_preflight_unlocks_the_fast_dispatch_and_says_so():
    torch = _cuda_or_skip()
    import flashinfer

    if flashinfer.__version__ not in dispatch._SUPPORTED:
        pytest.skip(f"flashinfer {flashinfer.__version__} is outside the allowlist by design")

    ops.reset_preflight_cache()
    dispatch.reset()
    record = ops.nvfp4_preflight(0, refresh = True)
    assert record["ok"] is True, record["reason"]
    assert record["fast_dispatch"] is True, record["fast_dispatch_reason"]
    assert dispatch.enabled(torch.device("cuda", 0)) is True
    ops.reset_preflight_cache()


def test_a_collected_layer_takes_its_cached_weights_and_their_vram_with_it():
    torch = _cuda_or_skip()
    import gc

    import flashinfer

    from core.inference import diffusion_nvfp4_linear as nl

    if flashinfer.__version__ not in dispatch._SUPPORTED:
        pytest.skip(f"flashinfer {flashinfer.__version__} is outside the allowlist by design")

    device = torch.device("cuda", 0)
    dispatch.reset()
    assert dispatch.verify(device)[0] is True
    ops.register_ops()

    in_features, out_features = 3072, 18432
    torch.manual_seed(0)
    with torch.cuda.device(device), torch.inference_mode():
        w = torch.randn(out_features, in_features, device = device, dtype = torch.bfloat16) * 0.02
        x = torch.randn(64, in_features, device = device, dtype = torch.bfloat16) * 0.05
        a_gsf, w_gsf = ops.global_scale(x), ops.global_scale(w)
        wq, w_sf = flashinfer.nvfp4_quantize(w, w_gsf, do_shuffle = False)
        layer = nl.nvfp4_linear_class()(
            in_features,
            out_features,
            wq = wq.clone(),
            w_sf = w_sf.clone(),
            alpha = (1.0 / (a_gsf * w_gsf)).float(),
            a_gsf = a_gsf,
        )
        del wq, w_sf, w
        gc.collect()
        torch.cuda.empty_cache()
        before = torch.cuda.memory_allocated(device)
        nl._TUNED_SHAPES.add((x.shape[0], in_features, out_features))
        layer(x)
        torch.cuda.synchronize(device)

    assert len(dispatch._TRANSPOSED) == 2
    assert {id(layer.wq), id(layer.w_sf)} == set(dispatch._TRANSPOSED)
    weight_bytes = layer.wq.numel() + layer.w_sf.numel()

    del layer
    gc.collect()
    assert dispatch._TRANSPOSED == {}
    torch.cuda.empty_cache()
    freed = before - torch.cuda.memory_allocated(device)
    assert freed >= weight_bytes, f"{freed} bytes released, expected at least {weight_bytes}"


def _sig_066(
    input,
    global_scale = None,
    sf_vec_size = 16,
    sf_use_ue8m0 = False,
    is_sf_swizzled_layout = True,
    is_sf_8x4_layout = False,
    enable_pdl = None,
):
    return {
        "inversed": False,
        "pdl": enable_pdl,
        "vec": sf_vec_size,
        "swizzled": is_sf_swizzled_layout,
    }


def _sig_073(
    input,
    global_scale = None,
    sf_vec_size = 16,
    sf_use_ue8m0 = False,
    is_sf_swizzled_layout = True,
    is_sf_8x4_layout = False,
    is_global_scale_inversed = False,
    enable_pdl = None,
    nvfp4_4over6_code = -1,
):
    return {
        "inversed": is_global_scale_inversed,
        "pdl": enable_pdl,
        "vec": sf_vec_size,
        "swizzled": is_sf_swizzled_layout,
    }


@pytest.mark.parametrize("fn", [_sig_066, _sig_073], ids = ["0.6.6", "0.7.3"])
def test_fast_quantize_passes_pdl_by_keyword_on_both_layouts(monkeypatch, fn):
    seen = {}

    def _quant(x, gsf, *args, **kw):
        seen.update(fn(x, gsf, *args, **kw))
        return x, types.SimpleNamespace(reshape = lambda shape: "sf")

    monkeypatch.setattr(dispatch, "quant_fn", lambda device, force = False: (_quant, True))
    x = types.SimpleNamespace(is_contiguous = lambda: True, device = "cuda:0", shape = (4, 32))
    dispatch._fast_quantize(x, None)
    assert seen == {"inversed": False, "pdl": True, "vec": 16, "swizzled": True}


def test_the_dispatch_allowlist_covers_every_release_studio_installs():
    from core.inference.diffusion_nvfp4_install import FLASHINFER_VERSIONS
    assert set(FLASHINFER_VERSIONS) <= set(dispatch._SUPPORTED)