Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Kernel-side tests for the NVFP4 flashinfer path: the device guard and the ordering barrier."""
from __future__ import annotations
import contextlib
import sys
import types
import pytest
from core.inference import diffusion_nvfp4_dispatch as dispatch
from core.inference import diffusion_nvfp4_linear as nl
from core.inference import diffusion_nvfp4_ops as ops
class _FakeDevice:
def __init__(
self,
index,
kind = "cuda",
):
self.type = kind
self.index = index
def __eq__(self, other):
return isinstance(other, _FakeDevice) and (self.type, self.index) == (
other.type,
other.index,
)
def __hash__(self):
return hash((self.type, self.index))
def __repr__(self): # pragma: no cover - debug aid
return f"cuda:{self.index}"
class _FakeTensor:
def __init__(
self,
shape = (1,),
device = None,
dtype = None,
):
self.shape = tuple(shape)
self.device = device if device is not None else _FakeDevice(0)
self.dtype = dtype
def _same(self, shape = None):
return _FakeTensor(self.shape if shape is None else shape, self.device, self.dtype)
def float(self):
return self._same()
def abs(self):
return self._same()
def amax(self):
return self._same((1,))
def clamp(self, **_kw):
return self._same((1,))
def all(self):
return self._same((1,))
def item(self):
return True
def reshape(self, *shape):
flat = shape[0] if len(shape) == 1 and isinstance(shape[0], (tuple, list)) else shape
return self._same(tuple(flat) if not isinstance(flat, int) else (flat,))
def contiguous(self):
return self
def is_contiguous(self):
return True
def to(
self,
device = None,
**_kw,
):
out = self._same()
if isinstance(device, _FakeDevice):
out.device = device
return out
def zero_(self):
_RECORDER.launches.append(("zero_", _RECORDER.current))
return self
def add_(self, _other):
return self
@property
def T(self):
return self._same(tuple(reversed(self.shape)))
def __mul__(self, _other):
return self._same()
__rmul__ = __mul__
def __truediv__(self, _other):
return self._same()
def __rtruediv__(self, _other):
return self._same()
class _Recorder:
def __init__(self):
self.current = 0
self.launches: list[tuple] = []
self.stack: list[int] = []
def reset(self):
self.current = 0
self.launches.clear()
self.stack.clear()
_RECORDER = _Recorder()
@contextlib.contextmanager
def _fake_device_guard(device):
index = device.index if isinstance(device, _FakeDevice) else int(device)
_RECORDER.stack.append(_RECORDER.current)
_RECORDER.current = index
try:
yield
finally:
_RECORDER.current = _RECORDER.stack.pop()
def _record(name):
def _call(*args, **kwargs):
_RECORDER.launches.append((name, _RECORDER.current))
if name == "nvfp4_quantize":
x = args[0]
return (_FakeTensor(x.shape, x.device), _FakeTensor(x.shape, x.device))
out = kwargs.get("out")
return out if out is not None else _FakeTensor((1,), _FakeDevice(_RECORDER.current))
return _call
def _fake_alloc(name):
def _call(
*shape,
device = None,
dtype = None,
**_kw,
):
flat = shape[0] if len(shape) == 1 and isinstance(shape[0], (tuple, list)) else shape
_RECORDER.launches.append((name, _RECORDER.current))
return _FakeTensor(tuple(flat), device, dtype)
return _call
def _fake_torch():
torch = types.ModuleType("torch")
torch.bfloat16 = "bfloat16"
torch.float32 = "float32"
torch.uint8 = "uint8"
torch.zeros = _fake_alloc("zeros")
torch.empty = _fake_alloc("empty")
torch.randn = _fake_alloc("randn")
torch.isfinite = lambda t: t
class _Generator:
def manual_seed(self, _seed):
return self
torch.Generator = _Generator
def _device(kind, index = None):
if isinstance(kind, _FakeDevice):
return kind
if isinstance(kind, int):
return _FakeDevice(kind, "cuda")
return _FakeDevice(0 if index is None else index, str(kind))
torch.device = _device
torch.inference_mode = contextlib.nullcontext
cuda = types.SimpleNamespace(
is_available = lambda: True,
device = _fake_device_guard,
current_device = lambda: _RECORDER.current,
device_count = lambda: 2,
get_device_capability = lambda dev = None: (10, 0),
get_device_name = lambda dev = None: "stub B200",
synchronize = lambda dev = None: _RECORDER.launches.append(("synchronize", _RECORDER.current)),
is_current_stream_capturing = lambda: False,
)
torch.cuda = cuda
return torch
def _fake_flashinfer():
fi = types.ModuleType("flashinfer")
fi.__version__ = "0.6.6"
fi.nvfp4_quantize = _record("nvfp4_quantize")
fi.mm_fp4 = _record("mm_fp4")
fi.autotune = lambda _on: contextlib.nullcontext()
return fi
@pytest.fixture
def stub_kernels(monkeypatch):
_RECORDER.reset()
dispatch.reset()
monkeypatch.setitem(sys.modules, "torch", _fake_torch())
monkeypatch.setitem(sys.modules, "flashinfer", _fake_flashinfer())
monkeypatch.setattr(ops, "register_ops", lambda: None)
monkeypatch.setattr(nl, "register_ops", lambda: None)
ops.reset_preflight_cache()
yield _RECORDER
_RECORDER.reset()
ops.reset_preflight_cache()
dispatch.reset()
def _launch_devices(recorder, *names) -> list[int]:
return [device for name, device in recorder.launches if name in names]
def test_the_stub_records_the_device_a_launch_actually_sees(stub_kernels):
"""The detector's own control: without a guard the launch reads the CURRENT device."""
import flashinfer
flashinfer.mm_fp4()
assert _launch_devices(stub_kernels, "mm_fp4") == [0]
def test_the_quantize_body_enters_the_tensors_device(stub_kernels):
x = _FakeTensor((128, 256), _FakeDevice(1))
ops._quantize_impl(x, _FakeTensor((1,), _FakeDevice(1)))
assert _launch_devices(stub_kernels, "nvfp4_quantize") == [1]
assert stub_kernels.current == 0
def test_the_mm_body_enters_the_tensors_device_for_the_barrier_and_the_gemm(stub_kernels):
device = _FakeDevice(1)
ops._mm_impl(
_FakeTensor((4096, 1536), device),
_FakeTensor((12288, 1536), device),
_FakeTensor((4096, 192), device),
_FakeTensor((12288, 192), device),
_FakeTensor((1,), device),
12288,
"cutlass",
)
assert set(_launch_devices(stub_kernels, "zeros", "empty", "mm_fp4")) == {1}
assert stub_kernels.current == 0
def test_the_preflight_enters_the_probed_device(stub_kernels):
record = ops.nvfp4_preflight(_FakeDevice(1))
assert record["ok"] is True, record["reason"]
assert set(_launch_devices(stub_kernels, "nvfp4_quantize", "mm_fp4", "synchronize")) == {1}
assert stub_kernels.current == 0
def test_the_prewarm_enters_each_layers_device(stub_kernels):
class NVFP4FlashInferLinear:
def __init__(self):
self.in_features = 1536
self.out_features = 12288
self.a_gsf = _FakeTensor((1,), _FakeDevice(1))
self.wq = _FakeTensor((12288, 768), _FakeDevice(1))
self._tuned = False
def __call__(self, _x):
_RECORDER.launches.append(("layer", _RECORDER.current))
return _FakeTensor((1,), self.wq.device)
layer = NVFP4FlashInferLinear()
tree = types.SimpleNamespace(named_modules = lambda: [("blocks.0.ff", layer)])
nl.reset_tuned_shapes()
try:
assert nl.nvfp4_prewarm(tree, (512,)) == 1
finally:
nl.reset_tuned_shapes()
assert set(_launch_devices(stub_kernels, "layer", "zeros", "synchronize")) == {1}
assert stub_kernels.current == 0
def test_no_guard_is_left_open_when_a_launch_raises(stub_kernels, monkeypatch):
"""A leaked guard is worse than no guard: it moves the whole process onto another card."""
import flashinfer
def _boom(*_a, **_kw):
raise RuntimeError("kernel refused")
monkeypatch.setattr(flashinfer, "nvfp4_quantize", _boom)
with pytest.raises(RuntimeError):
ops._quantize_impl(_FakeTensor((8, 16), _FakeDevice(1)), _FakeTensor((1,), _FakeDevice(1)))
assert stub_kernels.current == 0
assert stub_kernels.stack == []
def test_a_layer_on_card_one_runs_correctly_while_the_current_device_is_card_zero():
torch = pytest.importorskip("torch")
if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
pytest.skip("needs CUDA")
if torch.cuda.device_count() < 2:
pytest.skip("needs two visible CUDA devices")
if tuple(torch.cuda.get_device_capability(1)) not in ops.NVFP4_FLASHINFER_CAPS:
pytest.skip("device 1 has no flashinfer NVFP4 kernels")
pytest.importorskip("flashinfer")
pytest.importorskip("torchao")
import torch.nn as nn
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
from torchao.quantization import quantize_
torch.manual_seed(0)
with torch.cuda.device(1):
linear = nn.Linear(3072, 3072, bias = True).to("cuda:1", torch.bfloat16)
quantize_(linear, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
x = torch.randn(512, 3072, device = "cuda:1", dtype = torch.bfloat16) * 0.05
converted = nl.nvfp4_linear_from_torchao(linear, ops.global_scale(x))
with torch.inference_mode():
want = converted(x)
torch.cuda.synchronize(1)
assert torch.cuda.current_device() == 0
with torch.inference_mode():
got = converted(x)
torch.cuda.synchronize(1)
assert bool(torch.isfinite(got).all())
assert torch.equal(got, want)
def _mm_once(
device_index = 1,
m = 4096,
n = 12288,
k = 3072,
):
device = _FakeDevice(device_index)
return ops._mm_impl(
_FakeTensor((m, k // 2), device),
_FakeTensor((n, k // 2), device),
_FakeTensor((m, k // 16), device),
_FakeTensor((n, k // 16), device),
_FakeTensor((1,), device),
n,
"cutlass",
)
@pytest.fixture(autouse = True)
def _clean_barriers():
ops.reset_barriers()
yield
ops.reset_barriers()
def test_the_barrier_is_allocated_once_per_device(stub_kernels):
for _ in range(5):
_mm_once(1)
for _ in range(5):
_mm_once(0)
barrier_allocs = [1 for name, _ in stub_kernels.launches if name == "empty"]
assert len(barrier_allocs) == 12, stub_kernels.launches
assert sorted(ops._BARRIERS) == [0, 1]
assert ops._BARRIERS[0] is not ops._BARRIERS[1]
def test_the_barrier_fill_precedes_every_gemm(stub_kernels):
"""A kernel MUST exist between the activation quantiser and the GEMM (PDL, no griddepcontrol)."""
for _ in range(3):
_mm_once(1)
order = [name for name, _ in stub_kernels.launches if name in ("zero_", "mm_fp4")]
assert order == ["zero_", "mm_fp4"] * 3
def test_the_barrier_is_not_cached_when_the_stream_is_capturing(stub_kernels, monkeypatch):
monkeypatch.setattr(ops, "_is_capturing", lambda: True)
_mm_once(1)
assert ops._BARRIERS == {}
order = [name for name, _ in stub_kernels.launches if name in ("zero_", "mm_fp4")]
assert order == ["zero_", "mm_fp4"]
monkeypatch.setattr(ops, "_is_capturing", lambda: False)
_mm_once(1)
warm = ops._BARRIERS[1]
monkeypatch.setattr(ops, "_is_capturing", lambda: True)
_mm_once(1)
assert ops._BARRIERS[1] is warm
def test_reset_barriers_drops_them(stub_kernels):
_mm_once(1)
assert list(ops._BARRIERS) == [1]
ops.reset_barriers()
assert ops._BARRIERS == {}
_mm_once(1)
assert list(ops._BARRIERS) == [1]
def test_the_zero_buffer_env_restores_the_full_memset(stub_kernels, monkeypatch):
monkeypatch.setenv(ops.NVFP4_ZERO_BUFFER_ENV, "1")
_mm_once(1)
names = [name for name, _ in stub_kernels.launches]
assert "zeros" in names and "zero_" not in names
assert ops._BARRIERS == {}
def test_the_barrier_is_not_an_op_argument():
"""A mutable tensor input would go through ``auto_functionalized`` and be CLONED per call."""
schema = "(Tensor xq, Tensor wq, Tensor x_sf, Tensor w_sf, Tensor alpha, int n, str backend)"
import inspect
assert schema.count("Tensor") == 5
params = list(inspect.signature(ops._mm_impl).parameters)
assert params == ["xq", "wq", "x_sf", "w_sf", "alpha", "n", "backend"]
def _nvfp4_cuda_or_skip():
torch = pytest.importorskip("torch")
if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
pytest.skip("needs CUDA")
if tuple(torch.cuda.get_device_capability(0)) not in ops.NVFP4_FLASHINFER_CAPS:
pytest.skip("this device has no flashinfer NVFP4 kernels")
pytest.importorskip("flashinfer")
pytest.importorskip("torchao")
return torch
def _real_operands(
torch,
m,
k,
n,
seed = 0,
):
import flashinfer
torch.manual_seed(seed)
with torch.cuda.device(0):
x = torch.randn(m, k, device = "cuda", dtype = torch.bfloat16) * 0.05
w = torch.randn(n, k, device = "cuda", dtype = torch.bfloat16) * 0.02
a_gsf, w_gsf = ops.global_scale(x), ops.global_scale(w)
xq, x_sf = flashinfer.nvfp4_quantize(x, a_gsf, do_shuffle = False)
wq, w_sf = flashinfer.nvfp4_quantize(w, w_gsf, do_shuffle = False)
alpha = (1.0 / (a_gsf * w_gsf)).float()
return xq, wq, x_sf, w_sf, alpha
def test_the_persistent_barrier_is_bit_identical_to_the_per_call_one():
torch = _nvfp4_cuda_or_skip()
ops.reset_barriers()
xq, wq, x_sf, w_sf, alpha = _real_operands(torch, 4096, 3072, 12288)
with torch.inference_mode():
reference = None
for _ in range(50):
out = ops._mm_impl(xq, wq, x_sf, w_sf, alpha, 12288, ops.DEFAULT_MM_BACKEND)
assert bool(torch.isfinite(out).all())
if reference is None:
reference = out.clone()
else:
assert torch.equal(out, reference)
pointer = ops._BARRIERS[0].data_ptr()
for _ in range(10):
ops._mm_impl(xq, wq, x_sf, w_sf, alpha, 12288, ops.DEFAULT_MM_BACKEND)
assert ops._BARRIERS[0].data_ptr() == pointer
ops.reset_barriers()
def test_the_barrier_pointer_is_stable_across_a_capture_and_replay():
"""The barrier is warmed OUTSIDE the capture and must be the one the replay fires."""
torch = _nvfp4_cuda_or_skip()
ops.reset_barriers()
xq, wq, x_sf, w_sf, alpha = _real_operands(torch, 512, 3072, 3072, seed = 5)
with torch.cuda.device(0), torch.inference_mode():
eager = ops._mm_impl(xq, wq, x_sf, w_sf, alpha, 3072, ops.DEFAULT_MM_BACKEND).clone()
before = ops._BARRIERS[0].data_ptr()
torch.cuda.synchronize(0)
pool = torch.cuda.graph_pool_handle()
graph = torch.cuda.CUDAGraph()
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
ops._mm_impl(xq, wq, x_sf, w_sf, alpha, 3072, ops.DEFAULT_MM_BACKEND)
torch.cuda.current_stream().wait_stream(stream)
with torch.cuda.graph(graph, pool = pool):
captured = ops._mm_impl(xq, wq, x_sf, w_sf, alpha, 3072, ops.DEFAULT_MM_BACKEND)
assert ops._BARRIERS[0].data_ptr() == before
for _ in range(5):
graph.replay()
torch.cuda.synchronize(0)
assert torch.equal(captured, eager)
assert ops._BARRIERS[0].data_ptr() == before
del graph
ops.reset_barriers()
BIAS_SHAPES = (
(16384, 12288),
(4096, 3072),
(1024, 12288),
(333, 4096),
(1, 512),
(9304, 3072),
(7, 1024),
)
def _bias_pair(
torch,
m,
n,
dtype = None,
device = "cuda",
):
dtype = torch.bfloat16 if dtype is None else dtype
torch.manual_seed(m * 31 + n)
out = torch.randn(m, n, device = device, dtype = dtype)
bias = torch.randn(n, device = device, dtype = dtype)
return out, bias
def test_the_fast_bias_falls_back_without_triton(monkeypatch):
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
monkeypatch.setattr(fb, "_HAVE_TRITON", False)
out = torch.zeros(4, 8)
bias = torch.arange(8, dtype = torch.float32)
assert fb.fused_bias_add_(out, bias) is out
assert torch.equal(out[0], bias)
@pytest.mark.parametrize("value", ["0", "off", "FALSE", " no "])
def test_the_env_switch_takes_the_kernel_out_of_the_path(monkeypatch, value):
pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
monkeypatch.setenv(fb.NVFP4_FAST_BIAS_ENV, value)
assert fb.fast_bias_enabled() is False
@pytest.mark.parametrize("value", ["", "auto", "1", "AUTO"])
def test_auto_and_one_both_leave_it_on(monkeypatch, value):
pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
monkeypatch.setenv(fb.NVFP4_FAST_BIAS_ENV, value)
assert fb.fast_bias_enabled() is True
def test_the_kernel_declines_a_shape_or_dtype_it_does_not_cover(monkeypatch):
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
if not fb._HAVE_TRITON:
pytest.skip("needs triton")
if not torch.cuda.is_available():
pytest.skip("needs CUDA")
monkeypatch.setattr(fb, "_FAST_BIAS_MIN_NUMEL", 0)
out, bias = _bias_pair(torch, 8, 16)
assert fb._eligible(out, bias)
assert not fb._eligible(*_bias_pair(torch, 8, 16, dtype = torch.float32))
assert not fb._eligible(out.T.contiguous().T, bias)
assert not fb._eligible(out.cpu(), bias.cpu())
assert not fb._eligible(out, bias[:8])
assert not fb._eligible(out, bias.reshape(1, 16))
def test_the_kernel_declines_below_the_size_floor_and_above_int32(monkeypatch):
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
if not fb._HAVE_TRITON:
pytest.skip("needs triton")
if not torch.cuda.is_available():
pytest.skip("needs CUDA")
out, bias = _bias_pair(torch, 1024, 3840)
assert not fb._eligible(out, bias)
monkeypatch.setattr(fb, "_FAST_BIAS_MIN_NUMEL", out.numel())
assert fb._eligible(out, bias)
monkeypatch.setattr(fb, "_FAST_BIAS_MAX_NUMEL", out.numel() - 1)
assert not fb._eligible(out, bias)
def test_the_kernel_declines_while_tracing(monkeypatch):
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
calls = []
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
monkeypatch.setattr(fb, "_eligible", lambda *a: calls.append("eligible") or True)
out = torch.zeros(4, 8)
fb.fused_bias_add_(out, torch.ones(8))
assert calls == []
assert float(out[0, 0]) == 1.0
@pytest.mark.parametrize("m,n", BIAS_SHAPES)
def test_the_fused_bias_is_bit_identical_to_add_(m, n, monkeypatch):
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
if not torch.cuda.is_available():
pytest.skip("needs CUDA")
if not fb._HAVE_TRITON:
pytest.skip("needs triton")
monkeypatch.setattr(fb, "_FAST_BIAS_MIN_NUMEL", 0)
out, bias = _bias_pair(torch, m, n)
want = out.clone().add_(bias)
got = fb.fused_bias_add_(out, bias)
assert got is out
# Bit-identical, not close: the accuracy gates compare renders against stored references.
assert torch.equal(got, want), float((got.float() - want.float()).abs().max())
def test_an_empty_output_is_left_alone():
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
if not torch.cuda.is_available():
pytest.skip("needs CUDA")
out = torch.zeros(0, 32, device = "cuda", dtype = torch.bfloat16)
assert fb.fused_bias_add_(out, torch.ones(32, device = "cuda", dtype = torch.bfloat16)) is out
def test_m3_the_fused_bias_against_add_at_the_bench_shapes(capsys, monkeypatch):
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_bias as fb
monkeypatch.setattr(fb, "_FAST_BIAS_MIN_NUMEL", 0)
if not torch.cuda.is_available():
pytest.skip("needs CUDA")
if not fb._HAVE_TRITON:
pytest.skip("needs triton")
import time
lines = []
for m, n in ((4096, 12288), (4096, 3072), (4096, 4096), (16384, 12288)):
out, bias = _bias_pair(torch, m, n)
row = {}
for name, fn in (
("add_", lambda: out.add_(bias)),
("fused", lambda: fb.fused_bias_add_(out, bias)),
):
for _ in range(5):
fn()
torch.cuda.synchronize()
best = float("inf")
for _ in range(5):
start = time.perf_counter()
for _ in range(20):
fn()
torch.cuda.synchronize()
best = min(best, (time.perf_counter() - start) / 20 * 1e3)
row[name] = best
lines.append(
f" bias {m}x{n}: add_ {row['add_']:.4f} ms, fused {row['fused']:.4f} ms, "
f"{row['add_'] / row['fused']:.2f}x"
)
with capsys.disabled():
print("\n" + "\n".join(lines))