Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
from __future__ import annotations
import types
import os
import pytest
from core.inference import diffusion_nvfp4_linear as nl
from core.inference import diffusion_nvfp4_ops as ops
REAL_SHAPES = ((3072, 3072), (12288, 3072), (18432, 3072), (15360, 256))
FORWARD_REL_BOUND = 0.05
DENSE_GAP_BOUND = 0.002
def _cuda_or_skip():
torch = pytest.importorskip("torch")
if os.environ.get("CUDA_VISIBLE_DEVICES", None) == "":
pytest.skip("CUDA devices are masked off for this process")
if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
pytest.skip("needs CUDA")
capability = tuple(torch.cuda.get_device_capability(0))
if capability not in ops.NVFP4_FLASHINFER_CAPS:
pytest.skip("sm_%d%d has no flashinfer NVFP4 kernels" % capability)
pytest.importorskip("flashinfer")
pytest.importorskip("torchao")
return torch
def _torchao_linear(
torch,
out_features: int,
in_features: int,
*,
bias: bool = True,
seed: int = 0,
):
import torch.nn as nn
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
from torchao.quantization import quantize_
torch.manual_seed(seed)
linear = nn.Linear(in_features, out_features, bias = bias).to("cuda", torch.bfloat16)
dense = nn.Linear(in_features, out_features, bias = bias).to("cuda", torch.bfloat16)
dense.load_state_dict(linear.state_dict())
quantize_(linear, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
return linear, dense
def _rel(a, b) -> float:
return float((a.float() - b.float()).norm() / b.float().norm())
@pytest.mark.parametrize("out_features,in_features", REAL_SHAPES)
def test_torchao_and_flashinfer_pack_the_same_payload(out_features, in_features):
torch = _cuda_or_skip()
import flashinfer
from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
torch.manual_seed(0)
with torch.cuda.device(0):
w = torch.randn(out_features, in_features, device = "cuda", dtype = torch.bfloat16) * 0.02
amax = w.float().abs().amax().clamp(min = 1e-8)
w_gsf = (6.0 * 448.0 / amax).reshape(1)
per_tensor_scale = (amax / (6.0 * 448.0)).reshape(1)
tensor = NVFP4Tensor.to_nvfp4(w, per_tensor_scale = per_tensor_scale, is_swizzled_scales = True)
fi_q, fi_sf = flashinfer.nvfp4_quantize(w, w_gsf, do_shuffle = False)
ao_q = tensor.qdata.view(torch.uint8)
assert tuple(ao_q.shape) == tuple(fi_q.shape) == (out_features, in_features // 2)
assert torch.equal(tensor.scale.reshape(-1).view(torch.uint8), fi_sf.reshape(-1))
assert tensor.scale.numel() == ops._swizzled_sf_numel(out_features, in_features // 16, 128)
codes = lambda t: torch.stack(((t & 0xF).long(), (t >> 4).long()), -1).reshape(t.shape[0], -1)
ca, cb = codes(ao_q), codes(fi_q)
differing = ca != cb
fraction = float(differing.float().mean())
assert fraction < 0.005, f"{fraction:.4%} of nibbles differ, which is more than tie noise"
if fraction:
assert torch.equal((ca >= 8)[differing], (cb >= 8)[differing])
assert int(((ca & 7).int() - (cb & 7).int()).abs()[differing].max()) == 1
@pytest.mark.parametrize("m", [1, 512, 4096, 16384])
def test_converted_layer_matches_the_torchao_module(m):
torch = _cuda_or_skip()
linear, dense = _torchao_linear(torch, 18432, 3072)
x = torch.randn(m, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
converted = nl.nvfp4_linear_from_torchao(linear, ops.global_scale(x))
with torch.inference_mode():
want = linear(x)
got = converted(x)
reference = dense(x)
assert bool(torch.isfinite(got).all())
assert tuple(got.shape) == (m, 18432)
against_torchao = _rel(got, want)
assert against_torchao < FORWARD_REL_BOUND, f"M={m} rel={against_torchao:.5f}"
assert abs(_rel(got, reference) - _rel(want, reference)) < DENSE_GAP_BOUND
def test_converted_layer_keeps_the_leading_dimensions_and_the_bias():
torch = _cuda_or_skip()
linear, _ = _torchao_linear(torch, 3072, 3072)
converted = nl.nvfp4_linear_from_torchao(
linear, torch.tensor([448.0 * 6.0 / 0.2], device = "cuda")
)
assert converted.bias is not None
x = torch.randn(2, 77, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
with torch.inference_mode():
assert tuple(converted(x).shape) == (2, 77, 3072)
empty = converted(torch.zeros(0, 3072, device = "cuda", dtype = torch.bfloat16))
assert tuple(empty.shape) == (0, 3072)
@pytest.mark.parametrize("out_features,in_features", [(18432, 3072), (15360, 256)])
def test_m1_gemm_is_finite_on_both_backends(out_features, in_features, capsys):
torch = _cuda_or_skip()
import time
linear, _ = _torchao_linear(torch, out_features, in_features)
x = torch.randn(1, in_features, device = "cuda", dtype = torch.bfloat16) * 0.05
converted = nl.nvfp4_linear_from_torchao(linear, ops.global_scale(x))
timings = {}
for name, module in (("torchao", linear), ("flashinfer", converted)):
with torch.inference_mode():
for _ in range(3):
y = module(x)
torch.cuda.synchronize()
start = time.perf_counter()
for _ in range(20):
y = module(x)
torch.cuda.synchronize()
timings[name] = (time.perf_counter() - start) / 20 * 1e3
assert bool(torch.isfinite(y).all()), f"{name} produced a non-finite M=1 result"
assert tuple(y.shape) == (1, out_features)
with capsys.disabled():
print(
f"\n M=1 ({in_features} -> {out_features}) per call: "
+ ", ".join(f"{k} {v:.3f} ms" for k, v in timings.items())
)
def test_a_two_layer_block_compiles_fullgraph():
torch = _cuda_or_skip()
pytest.importorskip("triton") # inductor cannot build a GPU kernel without it (TritonMissing)
import torch.nn as nn
class Block(nn.Module):
def __init__(self, first, second):
super().__init__()
self.first, self.second = first, second
def forward(self, x):
return self.second(torch.nn.functional.silu(self.first(x)))
linear_a, _ = _torchao_linear(torch, 3072, 3072, seed = 1)
linear_b, _ = _torchao_linear(torch, 3072, 3072, seed = 2)
x = torch.randn(512, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
gsf = ops.global_scale(x)
block = Block(
nl.nvfp4_linear_from_torchao(linear_a, gsf), nl.nvfp4_linear_from_torchao(linear_b, gsf)
).eval()
from torch._dynamo.utils import counters
torch._dynamo.reset()
counters.clear()
compiled = torch.compile(block, fullgraph = True)
with torch.inference_mode():
got = compiled(x)
want = block(x)
assert not counters["graph_break"], dict(counters["graph_break"])
assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])
assert bool(torch.isfinite(got).all())
assert _rel(got, want) < 0.03
torch._dynamo.reset()
def test_graphed_forward_captures_and_replays_a_converted_block():
torch = _cuda_or_skip()
import torch.nn as nn
from core.inference import diffusion_cuda_graph as cg
class TinyDiT(nn.Module):
def __init__(self, layer):
super().__init__()
self.layer = layer
def forward(
self,
hidden_states,
timestep,
return_dict = True,
):
out = self.layer(hidden_states) + timestep
if return_dict:
return types.SimpleNamespace(sample = out)
return (out,)
linear, _ = _torchao_linear(torch, 3072, 3072, seed = 3)
x = torch.randn(512, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
module = TinyDiT(nl.nvfp4_linear_from_torchao(linear, ops.global_scale(x))).eval()
handle = cg.GraphedForward(module).enable()
try:
timestep = torch.zeros(1, 1, device = "cuda", dtype = torch.bfloat16)
with torch.inference_mode():
eager = handle.orig(hidden_states = x, timestep = timestep, return_dict = False)[0].clone()
first = None
for _ in range(20):
replayed = module(hidden_states = x, timestep = timestep, return_dict = False)[0]
assert _rel(replayed, eager) < FORWARD_REL_BOUND
first = replayed.clone() if first is None else first
assert torch.equal(replayed, first)
assert handle.stats["captures"] == 1
assert handle.stats["replays"] == 20
assert bool(torch.isfinite(replayed).all())
finally:
cg.uninstall_all([handle])
def test_convert_nvfp4_backend_moves_a_real_quantized_tree_and_prewarms_it():
torch = _cuda_or_skip()
import torch.nn as nn
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
from torchao.quantization import quantize_
torch.manual_seed(7)
tree = nn.Sequential(nn.Linear(3072, 3072), nn.SiLU(), nn.Linear(3072, 3072))
tree = tree.to("cuda", torch.bfloat16).eval()
quantize_(tree, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
x = torch.randn(512, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
with torch.inference_mode():
hidden = tree[1](tree[0](x))
before = tree(x)
metadata = {
"nvfp4_policy": {"activation_scales_baked": True},
"act_global_scales": {
"0": float(ops.global_scale(x)),
"2": float(ops.global_scale(hidden)),
},
}
assert nl.convert_nvfp4_backend(tree, metadata, "flashinfer") == 2
assert nl.is_nvfp4_flashinfer_linear(tree[0]) and nl.is_nvfp4_flashinfer_linear(tree[2])
with torch.inference_mode():
after = tree(x)
assert bool(torch.isfinite(after).all())
assert _rel(after, before) < FORWARD_REL_BOUND
nl.reset_tuned_shapes()
assert nl.nvfp4_prewarm(tree, (512,)) == 1
assert tree[0]._tuned is True and tree[2]._tuned is True
assert nl.nvfp4_prewarm(tree, (512,)) == 0
with torch.inference_mode():
tuned = tree(x)
assert bool(torch.isfinite(tuned).all())
nl.reset_tuned_shapes()
class _FakeNVFP4Tensor:
__name__ = "NVFP4Tensor"
def __init__(self):
self.qdata = object()
self.scale = object()
self.per_tensor_scale = None
_FakeNVFP4Tensor.__name__ = "NVFP4Tensor"
_FakeNVFP4Tensor.__qualname__ = "NVFP4Tensor"
def _quantized_tree():
import torch.nn as nn
class Fake(nn.Module):
def __init__(self):
super().__init__()
self.in_features = 64
self.out_features = 64
self.weight = _FakeNVFP4Tensor()
class Tree(nn.Module):
def __init__(self):
super().__init__()
self.attn = nn.Module()
self.attn.to_q = Fake()
self.attn.to_k = Fake()
self.plain = nn.Linear(8, 8)
return Tree()
def test_the_walk_finds_the_quantized_linears_and_leaves_the_plain_one_alone():
pytest.importorskip("torch")
tree = _quantized_tree()
names = [
name
for name, mod in nl._iter_linears(tree)
if nl.is_nvfp4_tensor(getattr(mod, "weight", None))
]
assert names == ["attn.to_q", "attn.to_k"]
class _RecordingLogger:
def __init__(self):
self.lines: list[str] = []
def __getattr__(self, _level):
return self.lines.append
@property
def text(self) -> str:
return "\n".join(self.lines)
def test_conversion_refuses_without_baked_activation_scales():
pytest.importorskip("torch")
tree = _quantized_tree()
logger = _RecordingLogger()
assert nl.convert_nvfp4_backend(tree, {"scheme": "nvfp4"}, "flashinfer", logger = logger) == 0
assert "bakes no activation scales" in logger.text
assert type(tree.attn.to_q).__name__ != "NVFP4FlashInferLinear"
def test_conversion_refuses_when_one_layer_has_no_scale():
pytest.importorskip("torch")
tree = _quantized_tree()
logger = _RecordingLogger()
metadata = {
"nvfp4_policy": {"activation_scales_baked": True},
"act_global_scales": {"attn.to_q": 1234.0},
}
assert nl.convert_nvfp4_backend(tree, metadata, "flashinfer", logger = logger) == 0
assert "attn.to_k" in logger.text
assert "1 of 2" in logger.text
assert type(tree.attn.to_q).__name__ != "NVFP4FlashInferLinear"
def test_a_layer_that_fails_half_way_leaves_the_whole_tree_on_torchao(monkeypatch):
torch = pytest.importorskip("torch")
import torch.nn as nn
tree = _quantized_tree()
originals = dict(nl._iter_linears(tree))
logger = _RecordingLogger()
metadata = {"act_global_scales": {"attn.to_q": 1.0, "attn.to_k": 2.0}}
def _convert(linear, a_gsf, **kwargs):
if linear is originals["attn.to_k"]:
raise ValueError("this NVFP4 weight carries no per_tensor_scale")
return nn.Identity()
monkeypatch.setattr(nl, "nvfp4_linear_from_torchao", _convert)
assert nl.convert_nvfp4_backend(tree, metadata, "flashinfer", logger = logger) == 0
assert "attn.to_k could not move" in logger.text
assert tree.attn.to_q is originals["attn.to_q"]
assert tree.attn.to_k is originals["attn.to_k"]
assert torch is not None
def test_a_declared_but_empty_scale_block_is_refused_as_loudly_as_a_missing_one():
pytest.importorskip("torch")
logger = _RecordingLogger()
metadata = {"nvfp4_policy": {"activation_scales_baked": True}, "act_global_scales": {}}
assert nl.convert_nvfp4_backend(_quantized_tree(), metadata, "flashinfer", logger = logger) == 0
assert "flag set, scales missing" in logger.text
def test_conversion_is_a_no_op_for_the_torchao_backend():
pytest.importorskip("torch")
tree = _quantized_tree()
metadata = {"act_global_scales": {"attn.to_q": 1.0, "attn.to_k": 1.0}}
assert nl.convert_nvfp4_backend(tree, metadata, "torchao") == 0
def test_conversion_skips_a_model_with_no_nvfp4_weights():
torch = pytest.importorskip("torch")
import torch.nn as nn
tree = nn.Sequential(nn.Linear(8, 8), nn.SiLU(), nn.Linear(8, 8))
metadata = {"act_global_scales": {"0": 1.0}}
assert nl.convert_nvfp4_backend(tree, metadata, "flashinfer") == 0
assert isinstance(tree[0], nn.Linear)
assert torch is not None
def test_state_dict_round_trips_with_and_without_a_bias():
torch = pytest.importorskip("torch")
cls = nl.nvfp4_linear_class()
assert cls is nl.nvfp4_linear_class()
def _build(bias: bool):
return cls(
64,
32,
wq = torch.arange(32 * 32, dtype = torch.uint8).reshape(32, 32),
w_sf = torch.ones(128, 4, dtype = torch.uint8),
alpha = torch.tensor([0.25]),
a_gsf = torch.tensor([1344.0]),
bias = torch.arange(32, dtype = torch.bfloat16) if bias else None,
)
with_bias = _build(True)
assert sorted(with_bias.state_dict()) == ["a_gsf", "alpha", "bias", "w_scale", "w_sf", "wq"]
assert float(with_bias.w_scale) == 0.25 * 1344.0
reloaded = _build(True)
reloaded.load_state_dict(with_bias.state_dict(), strict = True)
for key, value in with_bias.state_dict().items():
assert torch.equal(reloaded.state_dict()[key], value)
without_bias = _build(False)
assert "bias" not in without_bias.state_dict()
assert nl.is_nvfp4_flashinfer_linear(without_bias)
assert without_bias._tuned is False
def test_prewarm_without_flashinfer_is_a_no_op(monkeypatch):
pytest.importorskip("torch")
import builtins
real_import = builtins.__import__
def _no_flashinfer(name, *args, **kwargs):
if name == "flashinfer":
raise ImportError("no flashinfer here")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", _no_flashinfer)
assert nl.nvfp4_prewarm(_quantized_tree(), (1, 512)) == 0
def test_a_whole_model_artifact_converts_without_a_policy_block():
"""Video artifacts quantise EVERY admitted linear and declare no policy."""
torch = _cuda_or_skip()
import torch.nn as nn
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
from torchao.quantization import quantize_
torch.manual_seed(11)
tree = nn.Sequential(nn.Linear(3072, 3072), nn.SiLU(), nn.Linear(3072, 3072))
tree = tree.to("cuda", torch.bfloat16).eval()
quantize_(tree, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
x = torch.randn(512, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
with torch.inference_mode():
hidden = tree[1](tree[0](x))
before = tree(x)
metadata = {
"scheme": "nvfp4",
"family": "wan2.2-ti2v-5b",
"activation_scales_baked": True,
"act_global_scales": {
"0": float(ops.global_scale(x)),
"2": float(ops.global_scale(hidden)),
},
}
assert nl.convert_nvfp4_backend(tree, metadata, "flashinfer") == 2
assert nl.is_nvfp4_flashinfer_linear(tree[0]) and nl.is_nvfp4_flashinfer_linear(tree[2])
with torch.inference_mode():
after = tree(x)
assert bool(torch.isfinite(after).all())
assert _rel(after, before) < FORWARD_REL_BOUND
def test_a_whole_model_artifact_that_baked_nothing_says_the_flag_is_set():
pytest.importorskip("torch")
logger = _RecordingLogger()
metadata = {"scheme": "nvfp4", "activation_scales_baked": True, "act_global_scales": {}}
assert nl.convert_nvfp4_backend(_quantized_tree(), metadata, "flashinfer", logger = logger) == 0
assert "flag set, scales missing" in logger.text
def test_an_fp32_layer_runs_and_answers_in_fp32():
"""Wan2.2's fp32 time embedder feeds FP32 activations, which ``fp4_quantize`` rejects."""
torch = _cuda_or_skip()
import torch.nn as nn
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
from torchao.quantization import quantize_
torch.manual_seed(3)
layer = nn.Linear(3072, 3072).to("cuda", torch.float32).eval()
reference = nn.Linear(3072, 3072).to("cuda", torch.float32).eval()
reference.load_state_dict(layer.state_dict())
quantize_(layer, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
quantize_(reference, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
x = torch.randn(512, 3072, device = "cuda", dtype = torch.float32) * 0.05
with torch.inference_mode():
before = reference(x)
converted = nl.nvfp4_linear_from_torchao(layer, float(ops.global_scale(x)))
with torch.inference_mode():
after = converted(x)
assert after.dtype == torch.float32
assert bool(torch.isfinite(after).all())
assert _rel(after.float(), before.float()) < FORWARD_REL_BOUND
with torch.inference_mode():
empty = converted(x[:0])
assert empty.shape == (0, 3072) and empty.dtype == torch.float32