Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

18.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0


from __future__ import annotations

import ast
import pathlib

import pytest

from core.inference import diffusion_nvfp4_ops as ops

_INFERENCE_DIR = pathlib.Path(ops.__file__).resolve().parent


@pytest.fixture(autouse = True)
def _clean_backend_state(monkeypatch):
    monkeypatch.delenv(ops.NVFP4_BACKEND_ENV, raising = False)
    ops.reset_preflight_cache()
    yield
    ops.reset_preflight_cache()


def _stub(
    monkeypatch,
    *,
    available = True,
    capability = (10, 0),
    preflight_ok = True,
):
    monkeypatch.setattr(
        ops,
        "_flashinfer_available",
        lambda: (available, "0.6.6" if available else "ImportError: no flashinfer"),
    )
    monkeypatch.setattr(ops, "_device_capability", lambda device = None: capability)
    monkeypatch.setattr(
        ops,
        "nvfp4_preflight",
        lambda device = None, **kw: {
            "ok": preflight_ok,
            "capability": capability,
            "name": "stub",
            "reason": "ok" if preflight_ok else "JIT build failed",
        },
    )


def test_auto_selects_flashinfer_when_import_capability_and_preflight_all_pass(monkeypatch):
    _stub(monkeypatch)
    assert ops.select_nvfp4_backend(0) == "flashinfer"
    assert "preflight ok" in ops.nvfp4_backend_reason(0)


@pytest.mark.parametrize("capability", [(10, 0), (10, 3), (12, 0)])
def test_every_capability_in_the_flashinfer_set_selects_it(monkeypatch, capability):
    _stub(monkeypatch, capability = capability)
    assert ops.select_nvfp4_backend(0) == "flashinfer"


@pytest.mark.parametrize("capability", [(8, 9), (9, 0), (12, 1)])
def test_a_capability_outside_the_set_falls_to_torchao_and_names_it(monkeypatch, capability):
    _stub(monkeypatch, capability = capability)
    assert ops.select_nvfp4_backend(0) == "torchao"
    assert "sm_%d%d" % capability in ops.nvfp4_backend_reason(0)


def test_no_cuda_capability_is_torchao(monkeypatch):
    _stub(monkeypatch, capability = None)
    assert ops.select_nvfp4_backend() == "torchao"
    assert "capability" in ops.nvfp4_backend_reason()


def test_import_failure_is_torchao_which_is_how_windows_resolves(monkeypatch):
    _stub(monkeypatch, available = False)
    assert ops.select_nvfp4_backend(0) == "torchao"
    assert "flashinfer unavailable" in ops.nvfp4_backend_reason(0)


def test_a_failed_preflight_is_torchao_even_on_a_supported_capability(monkeypatch):
    _stub(monkeypatch, preflight_ok = False)
    assert ops.select_nvfp4_backend(0) == "torchao"
    assert "preflight failed: JIT build failed" in ops.nvfp4_backend_reason(0)


def test_explicit_torchao_never_probes(monkeypatch):
    def _boom(*a, **kw):  # pragma: no cover - reached only on a regression
        raise AssertionError("the probe ran under UNSLOTH_NVFP4_BACKEND=torchao")

    monkeypatch.setattr(ops, "_flashinfer_available", _boom)
    monkeypatch.setattr(ops, "nvfp4_preflight", _boom)
    monkeypatch.setenv(ops.NVFP4_BACKEND_ENV, "torchao")
    assert ops.select_nvfp4_backend(0) == "torchao"
    assert ops.nvfp4_backend_reason(0).endswith("=torchao")


def test_explicit_flashinfer_that_fails_a_check_warns_and_uses_torchao(monkeypatch):
    _stub(monkeypatch, preflight_ok = False)
    monkeypatch.setenv(ops.NVFP4_BACKEND_ENV, "flashinfer")
    with pytest.warns(RuntimeWarning, match = "preflight failed"):
        assert ops.select_nvfp4_backend(0) == "torchao"


def test_explicit_flashinfer_that_passes_warns_about_nothing(monkeypatch):
    _stub(monkeypatch)
    monkeypatch.setenv(ops.NVFP4_BACKEND_ENV, "flashinfer")
    import warnings

    with warnings.catch_warnings():
        warnings.simplefilter("error")
        assert ops.select_nvfp4_backend(0) == "flashinfer"


@pytest.mark.parametrize("value", ["", "  ", "cutlass", "TorchAO nope"])
def test_an_unrecognised_env_value_reads_as_auto(monkeypatch, value):
    _stub(monkeypatch)
    monkeypatch.setenv(ops.NVFP4_BACKEND_ENV, value)
    assert ops.nvfp4_backend_env() == "auto"
    assert ops.select_nvfp4_backend(0) == "flashinfer"


@pytest.mark.parametrize("value", ["FlashInfer", " flashinfer "])
def test_the_env_value_is_case_and_space_insensitive(monkeypatch, value):
    _stub(monkeypatch, preflight_ok = False)
    monkeypatch.setenv(ops.NVFP4_BACKEND_ENV, value)
    with pytest.warns(RuntimeWarning):
        assert ops.select_nvfp4_backend(0) == "torchao"


def _dotted(node: ast.AST) -> str:
    parts: list[str] = []
    while isinstance(node, ast.Attribute):
        parts.append(node.attr)
        node = node.value
    if isinstance(node, ast.Name):
        parts.append(node.id)
    return ".".join(reversed(parts))


def _is_guard(node: ast.AST) -> bool:
    """``torch.cuda.device(...)`` or the module's own ``_device_guard(...)`` alias."""
    if not isinstance(node, ast.Call):
        return False
    name = _dotted(node.func)
    return name.endswith("torch.cuda.device") or name.endswith("_device_guard")


# FlashInfer private entry points are imported by name, so a prefix check cannot see them.
_PRIVATE_LAUNCHES = frozenset(
    {
        "choose_one",
        "cutlass_fp4_gemm_runner",
        "get_cutlass_fp4_gemm_module",
        "_get_cache_buf",
        "fp4_quantize_sm100",
        "get_fp4_quantization_module",
    }
)


def _is_launch(node: ast.Call) -> str:
    # A Triton launch is a Call on a Subscript (kernel[grid](...)).
    if isinstance(node.func, ast.Subscript):
        name = _dotted(node.func.value)
        return name if name.endswith("_kernel") else ""
    name = _dotted(node.func)
    if name.startswith("flashinfer.") or name.startswith("_fi."):
        return name
    if "torch.ops.unsloth_nvfp4" in name:
        return name
    if name.rsplit(".", 1)[-1] in _PRIVATE_LAUNCHES:
        return name
    return ""


class _LaunchVisitor(ast.NodeVisitor):
    """Records every launch whose lexical ancestry contains no device guard."""

    def __init__(self) -> None:
        self.depth = 0
        self.unguarded: list[tuple[int, str]] = []

    def _visit_with(self, node) -> None:
        guarded = 0
        for item in node.items:
            self.visit(item.context_expr)
            if _is_guard(item.context_expr):
                guarded = 1
        self.depth += guarded
        for stmt in node.body:
            self.visit(stmt)
        self.depth -= guarded

    visit_With = _visit_with
    visit_AsyncWith = _visit_with

    def visit_Call(self, node: ast.Call) -> None:
        launch = _is_launch(node)
        if launch and self.depth == 0:
            self.unguarded.append((node.lineno, launch))
        self.generic_visit(node)


def _nvfp4_sources() -> list[pathlib.Path]:
    return sorted(_INFERENCE_DIR.glob("diffusion_nvfp4_*.py"))


def test_the_guard_visitor_catches_an_unguarded_launch():
    tree = ast.parse(
        "import flashinfer\n"
        "def f(x):\n"
        "    with torch.cuda.device(x.device):\n"
        "        flashinfer.mm_fp4(x)\n"
        "    flashinfer.nvfp4_quantize(x)\n"
        "    torch.ops.unsloth_nvfp4.mm(x)\n"
        "    _bias_add_kernel[grid](x)\n"
        "    with torch.cuda.device(x.device):\n"
        "        _bias_add_kernel[grid](x)\n"
        "    _get_cache_buf('ws', 1, x.device)\n"
    )
    visitor = _LaunchVisitor()
    visitor.visit(tree)
    assert [name for _, name in visitor.unguarded] == [
        "flashinfer.nvfp4_quantize",
        "torch.ops.unsloth_nvfp4.mm",
        "_bias_add_kernel",
        "_get_cache_buf",
    ]


def test_every_flashinfer_launch_in_the_nvfp4_modules_sits_inside_a_device_guard():
    sources = _nvfp4_sources()
    assert sources, "no diffusion_nvfp4_*.py modules found to guard"
    offences: list[str] = []
    for path in sources:
        visitor = _LaunchVisitor()
        visitor.visit(ast.parse(path.read_text(encoding = "utf-8")))
        offences += [
            f"{path.name}:{line}: {name} outside torch.cuda.device"
            for line, name in visitor.unguarded
        ]
    assert not offences, "\n".join(offences)


_STREAM_BANNED = ("set_stream", "set_device", "setDevice")


def _banned_stream_calls(source: str) -> list[tuple[int, str]]:
    """``set_stream`` silently sets the current DEVICE too, and ``set_device`` moves what the guard restores."""
    found: list[tuple[int, str]] = []
    for node in ast.walk(ast.parse(source)):
        if isinstance(node, ast.Attribute) and node.attr in _STREAM_BANNED:
            found.append((node.lineno, _dotted(node) or node.attr))
        elif isinstance(node, ast.Name) and node.id in _STREAM_BANNED:
            found.append((node.lineno, node.id))
    return found


def test_the_banned_call_detector_sees_an_aliased_set_stream():
    assert _banned_stream_calls("import torch\ncuda = torch.cuda\ncuda.set_stream(s)\n")
    assert _banned_stream_calls("from torch.cuda import set_device\nset_device(1)\n")
    assert not _banned_stream_calls("def reset_stream_cache():\n    return None\n")


def test_the_modules_never_set_the_current_stream_or_device():
    offences: list[str] = []
    for path in _nvfp4_sources():
        offences += [
            f"{path.name}:{line}: {name} switches the current device behind the guard"
            for line, name in _banned_stream_calls(path.read_text(encoding = "utf-8"))
        ]
    assert not offences, "\n".join(offences)


@pytest.mark.parametrize(
    "m,k,n",
    [
        (1, 3072, 18432),
        (1, 256, 15360),
        (77, 3072, 3072),
        (129, 256, 15360),
        (512, 3072, 12288),
        (4096, 3072, 3072),
        (16384, 3072, 18432),
    ],
)
def test_the_fake_impls_reproduce_the_flashinfer_allocation(m, k, n):
    torch = pytest.importorskip("torch")

    x = torch.empty((m, k), dtype = torch.bfloat16, device = "meta")
    gsf = torch.empty((1,), dtype = torch.float32, device = "meta")
    xq, x_sf = ops._quantize_fake(x, gsf)
    cols = k // 16
    total = ops._swizzled_sf_numel(m, cols, 128)

    assert tuple(xq.shape) == (m, k // 2) and xq.dtype == torch.uint8
    assert tuple(x_sf.shape) == (total // cols, cols) and x_sf.dtype == torch.uint8
    assert x_sf.numel() == total
    assert x_sf.shape[0] >= m

    w = torch.empty((n, k // 2), dtype = torch.uint8, device = "meta")
    w_sf = torch.empty(ops.sf_matrix_shape(n, cols), dtype = torch.uint8, device = "meta")
    alpha = torch.empty((1,), dtype = torch.float32, device = "meta")
    out = ops._mm_fake(xq, w, x_sf, w_sf, alpha, n, "cutlass")
    assert tuple(out.shape) == (m, n) and out.dtype == torch.bfloat16


def test_swizzled_numel_pads_rows_to_128_and_columns_to_4():
    assert ops._swizzled_sf_numel(1, 16, 128) == 128 * 16
    assert ops._swizzled_sf_numel(129, 16, 128) == 256 * 16
    assert ops._swizzled_sf_numel(128, 17, 128) == 128 * 20
    assert ops._swizzled_sf_numel(512, 192, 128) == 512 * 192


def test_sf_matrix_shape_is_the_flat_buffer_reshaped():
    assert ops.sf_matrix_shape(3072, 192) == (3072, 192)
    assert ops.sf_matrix_shape(1, 16) == (128, 16)
    assert ops.sf_matrix_shape(15360, 16) == (15360, 16)


def test_register_ops_is_idempotent():
    pytest.importorskip("torch")
    import torch

    ops.register_ops()
    ops.register_ops()
    assert hasattr(torch.ops.unsloth_nvfp4, "quantize")
    assert hasattr(torch.ops.unsloth_nvfp4, "mm")


def test_real_preflight_reports_ok_on_a_blackwell_card_with_flashinfer():
    torch = pytest.importorskip("torch")
    if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
        pytest.skip("needs CUDA")
    capability = tuple(torch.cuda.get_device_capability(0))
    if capability not in ops.NVFP4_FLASHINFER_CAPS:
        pytest.skip("sm_%d%d has no flashinfer NVFP4 kernels" % capability)
    pytest.importorskip("flashinfer")

    ops.reset_preflight_cache()
    record = ops.nvfp4_preflight(0)
    assert record["capability"] == capability
    assert record["name"]
    assert record["ok"] is True, record["reason"]
    assert record["reason"] == "ok"
    ops.nvfp4_preflight(0)
    assert ops.nvfp4_preflight(0) == record
    assert ops.select_nvfp4_backend(0) == "flashinfer"


def _fake_blackwell(monkeypatch):
    """Enough of ``torch.cuda`` for ``nvfp4_preflight`` to reach its probe on a masked-off host."""
    torch = pytest.importorskip("torch")
    monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
    monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device = None: (10, 0))
    monkeypatch.setattr(torch.cuda, "get_device_name", lambda device = None: "stub B200")
    return torch


def test_an_allocation_failure_is_answered_but_not_cached(monkeypatch):
    """An OOM during AUTO planning means 'not now'; cached, it would drop nvfp4 for the process."""
    torch = _fake_blackwell(monkeypatch)
    calls: list = []

    def _oom(dev):
        calls.append(dev)
        raise torch.cuda.OutOfMemoryError("CUDA out of memory. Tried to allocate 64.00 MiB")

    monkeypatch.setattr(ops, "_preflight_probe", _oom)
    first = ops.nvfp4_preflight(0)
    assert first["ok"] is False
    assert "out of memory" in first["reason"].lower()

    monkeypatch.setattr(ops, "_preflight_probe", lambda dev: True)
    second = ops.nvfp4_preflight(0)
    assert second["ok"] is True
    assert len(calls) == 1


def test_a_host_property_failure_stays_cached(monkeypatch):
    """A JIT build that cannot run here never will, so the failure is cached."""
    _fake_blackwell(monkeypatch)
    calls: list = []

    def _boom(dev):
        calls.append(dev)
        raise RuntimeError("JIT build failed")

    monkeypatch.setattr(ops, "_preflight_probe", _boom)
    assert ops.nvfp4_preflight(0)["ok"] is False

    monkeypatch.setattr(ops, "_preflight_probe", lambda dev: True)
    assert ops.nvfp4_preflight(0)["ok"] is False
    assert len(calls) == 1


def test_a_successful_probe_is_cached(monkeypatch):
    _fake_blackwell(monkeypatch)
    calls: list = []

    def _ok(dev):
        calls.append(dev)
        return True

    monkeypatch.setattr(ops, "_preflight_probe", _ok)
    assert ops.nvfp4_preflight(0)["ok"] is True
    assert ops.nvfp4_preflight(0)["ok"] is True
    assert len(calls) == 1


@pytest.mark.parametrize(
    "exc,transient",
    [
        (MemoryError(), True),
        (RuntimeError("CUDA out of memory. Tried to allocate 64.00 MiB"), True),
        (RuntimeError("cudaErrorMemoryAllocation"), True),
        (RuntimeError("JIT build failed"), False),
        (ImportError("no flashinfer"), False),
    ],
)
def test_only_allocation_failures_read_as_transient(exc, transient):
    assert ops._transient_preflight_failure(exc) is transient


def test_an_unprewarmed_gemm_shape_autotunes_on_its_first_eager_call_only(monkeypatch):
    import contextlib
    import sys
    import types

    # The per-length legacy mode; the bucketed default is covered in test_diffusion_nvfp4_tactics.py.
    monkeypatch.setenv("UNSLOTH_NVFP4_TACTIC_CACHE", "0")

    torch = pytest.importorskip("torch")
    from core.inference import diffusion_nvfp4_linear as nl

    state = {"tuning": False, "capturing": False}
    calls = []

    @contextlib.contextmanager
    def _autotune(flag = True):
        state["tuning"] = flag
        try:
            yield
        finally:
            state["tuning"] = False

    def _mm_fp4(
        a,
        b,
        a_sf,
        b_sf,
        alpha,
        dtype,
        out = None,
        backend = None,
    ):
        calls.append((a.shape[0], state["tuning"]))
        return out

    fake = types.ModuleType("flashinfer")
    fake.autotune = _autotune
    fake.mm_fp4 = _mm_fp4
    monkeypatch.setitem(sys.modules, "flashinfer", fake)
    monkeypatch.setattr(ops, "_device_guard", lambda t: contextlib.nullcontext())
    monkeypatch.setattr(ops, "_fire_barrier", lambda device: None)
    from core.inference import diffusion_nvfp4_dispatch as dispatch

    monkeypatch.setattr(dispatch, "enabled", lambda device: False)
    monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: state["capturing"])
    nl.reset_tuned_shapes()
    try:
        wq = torch.zeros(64, 16, dtype = torch.uint8)  # N = 64, K = 32

        def _mm(m):
            xq = torch.zeros(m, 16, dtype = torch.uint8)
            ops._mm_impl(xq, wq, None, wq, None, 64, "cutlass")

        nl._TUNED_SHAPES.add((1, 32, 64))
        _mm(1)
        _mm(4352)
        _mm(4352)
        state["capturing"] = True
        _mm(8192)
        assert calls == [(1, False), (4352, True), (4352, False), (8192, False)]
        assert (4352, 32, 64) in nl._TUNED_SHAPES
        assert (8192, 32, 64) not in nl._TUNED_SHAPES
    finally:
        nl.reset_tuned_shapes()


def test_the_first_call_tune_runs_before_the_cached_dispatch_plan_is_built(monkeypatch):
    """``gemm_plan`` snapshots the tactic on its first build, so a plan before the tune pins the fallback."""
    import contextlib
    import sys
    import types

    # Legacy mode only: the bucketed plan tunes its own bucket (test_diffusion_nvfp4_tactics.py).
    monkeypatch.setenv("UNSLOTH_NVFP4_TACTIC_CACHE", "0")

    torch = pytest.importorskip("torch")
    from core.inference import diffusion_nvfp4_dispatch as dispatch
    from core.inference import diffusion_nvfp4_linear as nl

    events = []
    tuning = {"on": False}

    @contextlib.contextmanager
    def _autotune(flag = True):
        tuning["on"] = flag
        try:
            yield
        finally:
            tuning["on"] = False

    def _mm_fp4(
        *args,
        out = None,
        **kwargs,
    ):
        events.append(("mm_fp4", tuning["on"]))
        return out

    def _plan(xq, *args, **kwargs):
        events.append(("plan_built", tuning["on"]))
        return (lambda inputs, tactic: events.append(("plan_run", tactic)), 7, None)

    fake = types.ModuleType("flashinfer")
    fake.autotune = _autotune
    fake.mm_fp4 = _mm_fp4
    monkeypatch.setitem(sys.modules, "flashinfer", fake)
    monkeypatch.setattr(ops, "_device_guard", lambda t: contextlib.nullcontext())
    monkeypatch.setattr(ops, "_fire_barrier", lambda device: None)
    monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
    monkeypatch.setattr(dispatch, "enabled", lambda device: True)
    monkeypatch.setattr(dispatch, "transposed", lambda t: t)
    monkeypatch.setattr(dispatch, "gemm_plan", _plan)
    nl.reset_tuned_shapes()
    try:
        wq = torch.zeros(64, 16, dtype = torch.uint8)
        xq = torch.zeros(4352, 16, dtype = torch.uint8)
        ops._mm_impl(xq, wq, None, wq, None, 64, "cutlass")
        ops._mm_impl(xq, wq, None, wq, None, 64, "cutlass")
        assert events == [("mm_fp4", True), ("plan_built", False), ("plan_run", 7)]
    finally:
        nl.reset_tuned_shapes()