Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""ComfyUI nvfp4 and mxfp8 single-file tests; the encoder shares no code with the module under test."""
from __future__ import annotations
import json
import types
import pytest
torch = pytest.importorskip("torch")
safetensors_torch = pytest.importorskip("safetensors.torch")
nn = torch.nn
import core.inference.diffusion_comfy_block as cb # noqa: E402
import core.inference.diffusion_comfy_quant as cq # noqa: E402
DIM = 512
E2M1 = torch.tensor(
[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0]
)
def _conf(**conf) -> torch.Tensor:
return torch.tensor(list(json.dumps(conf).encode("utf-8")), dtype = torch.uint8)
def _save(
path,
tensors,
metadata = None,
) -> str:
safetensors_torch.save_file(
{k: v.contiguous().clone() for k, v in tensors.items()}, str(path), metadata = metadata
)
return str(path)
def _ceil(x: int, m: int) -> int:
return -(-x // m) * m
def _tile_offsets(rows: int, blocks: int) -> torch.Tensor:
"""Flat offset of scale (r, b) in the cuBLAS 1D block-scaling-factor layout, element by element."""
r = torch.arange(rows).view(-1, 1)
b = torch.arange(blocks).view(1, -1)
tiles_per_row = _ceil(blocks, 4) // 4
return (
(r // 128) * tiles_per_row * 512
+ (b // 4) * 512
+ (r % 32) * 16
+ ((r % 128) // 32) * 4
+ (b % 4)
)
def _tile(plain_u8: torch.Tensor) -> torch.Tensor:
rows, blocks = plain_u8.shape
flat = torch.zeros(_ceil(rows, 128) * _ceil(blocks, 4), dtype = torch.uint8)
flat[_tile_offsets(rows, blocks).reshape(-1)] = plain_u8.reshape(-1)
return flat.reshape(_ceil(rows, 128), _ceil(blocks, 4))
def _untile(tiled: torch.Tensor, rows: int, blocks: int) -> torch.Tensor:
return tiled.reshape(-1)[_tile_offsets(rows, blocks)]
def nvfp4_encode(w: torch.Tensor) -> dict:
"""ComfyUI nvfp4 layout of ``w`` (round-to-nearest e2m1; exactness of the encoder is not under test)."""
rows, cols = w.shape
w = w.float()
tensor_scale = w.abs().amax() / (448.0 * 6.0)
blocks = w.reshape(rows, cols // 16, 16)
block_scale = (blocks.abs().amax(-1) / 6.0 / tensor_scale).to(torch.float8_e4m3fn)
step = block_scale.float() * tensor_scale
x = blocks / torch.where(step == 0, torch.ones_like(step), step).unsqueeze(-1)
nibble = (x.unsqueeze(-1) - E2M1[:8].abs().mul(x.sign().unsqueeze(-1))).abs().argmin(-1)
nibble = torch.where(x < 0, nibble + 8, nibble).reshape(rows, cols).to(torch.uint8)
packed = (nibble[:, 0::2] << 4) | nibble[:, 1::2] # even column -> high nibble
return {
"weight": packed,
"weight_scale": _tile(block_scale.view(torch.uint8)).view(torch.float8_e4m3fn),
"weight_scale_2": tensor_scale.reshape(()),
"input_scale": torch.tensor(0.01),
}
def nvfp4_reference(t: dict) -> torch.Tensor:
q = t["weight"]
rows, cols = q.shape[0], q.shape[1] * 2
j = torch.arange(cols)
byte = q[:, j // 2].to(torch.int64)
nibble = torch.where(j % 2 == 0, byte >> 4, byte & 15)
scales = (
_untile(t["weight_scale"].view(torch.uint8), rows, cols // 16)
.view(torch.float8_e4m3fn)
.float()
)
step = scales * t["weight_scale_2"].float()
return E2M1[nibble] * step[:, j // 16]
def mxfp8_encode(w: torch.Tensor) -> dict:
rows, cols = w.shape
blocks = w.float().reshape(rows, cols // 32, 32)
exp = torch.ceil(torch.log2((blocks.abs().amax(-1) / 448.0).clamp(min = 2.0**-127)))
codes = (
(blocks / torch.exp2(exp).unsqueeze(-1))
.clamp(-448, 448)
.to(torch.float8_e4m3fn)
.reshape(rows, cols)
)
return {
"weight": codes,
"weight_scale": _tile((exp + 127).to(torch.uint8)).view(torch.float8_e8m0fnu),
}
def mxfp8_reference(t: dict) -> torch.Tensor:
q = t["weight"]
rows, cols = q.shape
exp = _untile(t["weight_scale"].view(torch.uint8), rows, cols // 32).float() - 127
return q.float() * torch.exp2(exp)[:, torch.arange(cols) // 32]
ENCODERS = {"nvfp4": (nvfp4_encode, nvfp4_reference), "mxfp8": (mxfp8_encode, mxfp8_reference)}
@pytest.mark.parametrize("rows,blocks", [(128, 4), (200, 6), (3, 1), (384, 120)])
def test_tiling_matches_the_cublas_formula_and_round_trips(rows, blocks):
plain = torch.randint(0, 255, (rows, blocks), dtype = torch.uint8)
tiled = cb.tile_scales(plain)
assert torch.equal(tiled, _tile(plain).reshape(-1))
assert torch.equal(cb.untile_scales(tiled, rows, blocks), plain)
assert cb.tiled_scale_numel(rows, blocks) == tiled.numel()
@pytest.mark.parametrize("fmt", ["nvfp4", "mxfp8"])
@pytest.mark.parametrize("rows", [256, 200])
def test_dequant_is_bit_exact_against_the_reference(fmt, rows):
torch.manual_seed(0)
encode, reference = ENCODERS[fmt]
t = encode(torch.randn(rows, 96) * 0.05)
codes, scale = cb.decode_layer(fmt, t["weight"], t["weight_scale"])
got = cb.dequant_block(
fmt, codes, scale, tensor_scale = t.get("weight_scale_2"), dtype = torch.float32
)
assert torch.equal(got, reference(t))
def test_nvfp4_codes_are_repacked_low_nibble_first():
t = nvfp4_encode(torch.randn(128, 64))
codes, _scale = cb.decode_layer("nvfp4", t["weight"], t["weight_scale"])
assert torch.equal(codes & 0x0F, t["weight"] >> 4)
assert torch.equal(codes >> 4, t["weight"] & 0x0F)
def test_pre_quant_scale_is_folded_into_the_columns():
t = nvfp4_encode(torch.randn(128, 64))
codes, scale = cb.decode_layer("nvfp4", t["weight"], t["weight_scale"])
smooth = torch.rand(64) + 0.5
got = cb.dequant_block(
"nvfp4",
codes,
scale,
tensor_scale = t["weight_scale_2"],
pre_quant_scale = smooth,
dtype = torch.float32,
)
assert torch.equal(got, nvfp4_reference(t) * smooth)
def _layer_file(
tmp_path,
fmt,
rows = 128,
cols = 64,
drop = (),
extra = None,
conf = None,
name = "m",
):
encode, _ref = ENCODERS[fmt]
t = encode(torch.randn(rows, cols))
tensors = {f"a.{k}": v for k, v in t.items() if k not in drop}
tensors.update(extra or {})
tensors["a.comfy_quant"] = _conf(**(conf or {"format": fmt}))
return _save(tmp_path / f"{name}.safetensors", tensors)
@pytest.mark.parametrize("fmt", ["nvfp4", "mxfp8"])
def test_block_formats_are_detected(tmp_path, fmt):
scan = cq.refuse_comfy_quant(_layer_file(tmp_path, fmt))
assert scan.counts() == {fmt: 1}
assert not scan.problems
def test_block_formats_in_the_header_table_are_detected(tmp_path):
t = nvfp4_encode(torch.randn(128, 64))
meta = {
"_quantization_metadata": json.dumps(
{"format_version": "1.0", "layers": {"a": {"format": "nvfp4"}}}
)
}
path = _save(tmp_path / "h.safetensors", {f"a.{k}": v for k, v in t.items()}, metadata = meta)
assert cq.refuse_comfy_quant(path).counts() == {"nvfp4": 1}
@pytest.mark.parametrize(
"fmt,kwargs,why",
[
("nvfp4", {"drop": ("weight_scale_2",)}, "weight_scale_2"),
("nvfp4", {"drop": ("weight_scale",)}, "weight_scale is missing"),
("nvfp4", {"extra": {"a.weight_codebook": torch.zeros(4)}}, "weight_codebook"),
("nvfp4", {"extra": {"a.pre_quant_scale": torch.ones(3)}}, "pre_quant_scale"),
("nvfp4", {"conf": {"format": "nvfp4", "convrot": True}}, "ConvRot on nvfp4"),
("mxfp8", {"extra": {"a.weight_scale_2": torch.ones(())}}, "weight_scale_2"),
],
)
def test_malformed_block_layers_are_refused(tmp_path, fmt, kwargs, why):
with pytest.raises(ValueError, match = why):
cq.refuse_comfy_quant(_layer_file(tmp_path, fmt, **kwargs))
def test_untiled_block_scales_are_refused(tmp_path):
t = nvfp4_encode(torch.randn(256, 64))
t["weight_scale"] = t["weight_scale"][:200] # a plain [N, K/16] matrix is not ComfyUI's layout
path = _save(
tmp_path / "u.safetensors",
{**{f"a.{k}": v for k, v in t.items()}, "a.comfy_quant": _conf(format = "nvfp4")},
)
with pytest.raises(ValueError, match = "tiled layout"):
cq.refuse_comfy_quant(path)
@pytest.mark.parametrize("fmt", ["asym_w4a8_int8", "w6a8_int8", "brand_new"])
def test_remaining_formats_keep_the_refusal(tmp_path, fmt):
path = _save(
tmp_path / "r.safetensors",
{
"a.weight": torch.zeros(8, 8, dtype = torch.uint8),
"a.weight_scale": torch.ones(8, 1),
"a.comfy_quant": _conf(format = fmt),
},
)
with pytest.raises(ValueError, match = rf"1 layer\(s\) in ComfyUI format '{fmt}'"):
cq.refuse_comfy_quant(path)
def test_resident_size_prices_kept_and_dequantized_nvfp4(tmp_path):
path = _layer_file(tmp_path, "nvfp4", rows = 2048, cols = 2048)
kept = cq.comfy_resident_mib(path, keep_int8 = False, keep_fp8 = False, keep_nvfp4 = True)
dense = cq.comfy_resident_mib(path, keep_int8 = False, keep_fp8 = False)
assert dense == 8 + 1 # 2048 x 2048 bf16, plus the scales rounded up
assert kept == 3 # 2 MiB of codes + 256 KiB of block scales
kw = dict(keep_int8 = False, keep_fp8 = False, keep_nvfp4 = True, block_divisible = {"nvfp4": 16})
assert cq.comfy_resident_mib(path, min_features = 2048, **kw) == 3
assert cq.comfy_resident_mib(path, min_features = 2049, **kw) == dense
smoothed = _layer_file(
tmp_path,
"nvfp4",
rows = 2048,
cols = 2048,
name = "s",
extra = {"a.pre_quant_scale": torch.ones(2048)},
)
assert cq.comfy_resident_mib(smoothed, **kw) > kept
def test_lora_and_block_runtime_layers(monkeypatch, tmp_path):
scan = cq.scan_comfy_quant(_layer_file(tmp_path, "mxfp8"))
monkeypatch.setattr(cb, "mxfp8_runtime_reason", lambda target: None)
assert (
cb.comfy_block_backends(scan, _target("cuda:0"), "krea-2")["mxfp8_backend"] == "scaled_mm"
)
assert (
cb.comfy_block_backends(scan, _target("cuda:0"), "krea-2", lora = True)["mxfp8_backend"]
is None
)
model = types.SimpleNamespace(_unsloth_comfy_quant = {"mxfp8": 3, "nvfp4": 0})
assert cb.comfy_block_runtime_layers(model) == 3
assert (
cb.comfy_block_runtime_layers(types.SimpleNamespace(_unsloth_comfy_quant = {"mxfp8": 0})) == 0
)
assert cb.comfy_block_runtime_layers(None) == 0
def test_install_hint_reads_the_default_hub_cache_too(monkeypatch, tmp_path):
import huggingface_hub
from core.inference.diffusion import DiffusionBackend
path = str(_layer_file(tmp_path, "nvfp4"))
seen = []
def fake(
repo_id,
filename,
cache_dir = None,
):
seen.append(cache_dir)
return path if cache_dir is None else None
monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", fake)
assert DiffusionBackend._comfy_single_file_holds_nvfp4("Comfy-Org/x", "m.safetensors")
assert seen[-1] is None and len(seen) == 2
class _Block(nn.Module):
def __init__(self, dim: int) -> None:
super().__init__()
self.to_q = nn.Linear(dim, dim)
self.to_k = nn.Linear(dim, dim)
self.to_v = nn.Linear(dim, dim)
self.to_out = nn.Linear(dim, dim)
class _Tiny(nn.Module):
_keep_in_fp32_modules = None
_keys_to_ignore_on_load_unexpected = None
def __init__(
self,
dim: int = DIM,
blocks: int = 2,
) -> None:
super().__init__()
self.blocks = nn.ModuleList(_Block(dim) for _ in range(blocks))
self.norm = nn.LayerNorm(dim)
@classmethod
def load_config(cls, repo, **_kwargs):
return {"dim": DIM, "blocks": 2}
@staticmethod
def _get_signature_keys(_cls):
return {"dim", "blocks"}, set()
@classmethod
def from_config(cls, config):
return cls(**config)
def _convert(checkpoint = None, **_kwargs):
"""What the diffusers converters do to a ComfyUI DiT: split the fused qkv by rows, rename the rest."""
out = {}
for key, value in checkpoint.items():
if ".qkv." in key:
stem, leaf = key.rsplit(".qkv.", 1)
for part, chunk in zip(("to_q", "to_k", "to_v"), value.chunk(3, dim = 0)):
out[f"{stem}.{part}.{leaf}"] = chunk
else:
out[key.replace(".out.", ".to_out.")] = value
return out
@pytest.fixture
def block_file(tmp_path, monkeypatch):
def make(fmt, drop_input_scale = False):
monkeypatch.setattr(
cq,
"_mapping",
lambda cls: (
_convert,
types.SimpleNamespace(
_should_convert_state_dict_to_diffusers = lambda a, b: set(a) != set(b),
_get_mapping_function_kwargs = lambda fn, **kw: {},
),
),
)
torch.manual_seed(0)
dense = _Tiny()
encode, reference = ENCODERS[fmt]
tensors, expected = {}, {}
for b, block in enumerate(dense.blocks):
layers = {
f"blocks.{b}.qkv": torch.cat(
[block.to_q.weight, block.to_k.weight, block.to_v.weight]
),
f"blocks.{b}.out": block.to_out.weight,
}
for name, weight in layers.items():
t = encode(weight.detach())
if drop_input_scale:
t.pop("input_scale", None)
tensors.update({f"{name}.{k}": v for k, v in t.items()})
tensors[f"{name}.comfy_quant"] = _conf(format = fmt)
ref = reference(t)
if name.endswith(".qkv"):
for part, chunk in zip(("to_q", "to_k", "to_v"), ref.chunk(3)):
expected[f"blocks.{b}.{part}.weight"] = chunk
else:
expected[f"blocks.{b}.to_out.weight"] = ref
bias = getattr(block, "to_out" if name.endswith("out") else "to_q").bias
tensors[f"{name}.bias"] = (
torch.cat([block.to_q.bias, block.to_k.bias, block.to_v.bias])
if name.endswith(".qkv")
else bias
).detach()
tensors["norm.weight"] = dense.norm.weight.detach()
tensors["norm.bias"] = dense.norm.bias.detach()
return _save(tmp_path / f"tiny_{fmt}.safetensors", tensors), tensors, expected
return make
def _load(
path,
dtype = torch.float32,
**kwargs,
):
return cq.load_comfy_quant_transformer(
_Tiny,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": dtype, "config": "base/repo", "subfolder": "transformer"},
int8_backend = None,
family = "z-image",
**kwargs,
)
@pytest.mark.parametrize("fmt", ["nvfp4", "mxfp8"])
def test_dequant_load_is_bit_exact_through_the_row_split(block_file, fmt):
path, tensors, expected = block_file(fmt)
model = _load(path)
info = model._unsloth_comfy_quant
assert info["dequantized"] == 4 and info[fmt] == 0
state = model.state_dict()
for key, want in expected.items():
assert torch.equal(state[key], want), key
assert torch.equal(state["blocks.0.to_k.bias"], tensors["blocks.0.qkv.bias"][DIM : 2 * DIM])
@pytest.mark.parametrize("fmt", ["nvfp4", "mxfp8"])
def test_runtime_linears_keep_codes_and_tiled_scales_unchanged(block_file, fmt):
if fmt == "nvfp4":
pytest.importorskip("flashinfer")
path, tensors, _expected = block_file(fmt)
model = _load(path, dtype = torch.bfloat16, **{f"{fmt}_backend": "kept"})
assert model._unsloth_comfy_quant[fmt] == 8 and model._unsloth_comfy_quant["dequantized"] == 0
out = model.blocks[1].to_out
file_codes = tensors["blocks.1.out.weight"]
file_scale = tensors["blocks.1.out.weight_scale"].view(torch.uint8).reshape(-1)
if fmt == "nvfp4":
assert type(out).__name__ == "NVFP4FlashInferLinear"
assert torch.equal(out.wq, cb.swap_nibbles(file_codes))
assert torch.equal(out.w_sf.reshape(-1), file_scale)
assert out.w_scale.item() == tensors["blocks.1.out.weight_scale_2"].item()
assert out.a_gsf.item() == pytest.approx(1.0 / tensors["blocks.1.out.input_scale"].item())
q = model.blocks[
0
].to_q # a row split of the fused qkv: scales re-tiled from the plain matrix
plain = cb.untile_scales(
tensors["blocks.0.qkv.weight_scale"].view(torch.uint8), 3 * DIM, DIM // 16
)
assert torch.equal(cb.untile_scales(q.w_sf, DIM, DIM // 16), plain[:DIM])
else:
assert type(out).__name__ == "ComfyMXFP8Linear"
assert torch.equal(out.weight_q.view(torch.uint8), file_codes.view(torch.uint8))
assert torch.equal(out.weight_sf, file_scale)
assert torch.equal(out.bias.float(), tensors["blocks.1.out.bias"].bfloat16().float())
def _blackwell() -> bool:
return torch.cuda.is_available() and torch.cuda.get_device_capability() >= (10, 0)
def test_nvfp4_without_an_input_scale_scales_activations_per_call(block_file):
pytest.importorskip("flashinfer")
path, _tensors, _expected = block_file("nvfp4", drop_input_scale = True)
model = _load(path, dtype = torch.bfloat16, nvfp4_backend = "kept")
assert model._unsloth_comfy_quant["nvfp4"] == 8
assert {type(m).__name__ for m in model.modules() if hasattr(m, "wq")} == {
"ComfyNVFP4DynamicLinear"
}
from core.inference.diffusion_nvfp4_linear import is_nvfp4_flashinfer_linear
from core.inference.diffusion_nvfp4_protect import protect_controller, protect_layers
layers = [m for _n, m in protect_layers(model)]
assert len(layers) == 8 and all(is_nvfp4_flashinfer_linear(m) for m in layers)
# its own step controller, not the process-wide one
assert (
all(m.protect is layers[0].protect for m in layers)
and layers[0].protect is not protect_controller()
)
@pytest.mark.skipif(not _blackwell(), reason = "needs a Blackwell GPU")
@pytest.mark.parametrize("fmt,dynamic", [("nvfp4", False), ("nvfp4", True), ("mxfp8", False)])
def test_runtime_forward_matches_the_dequantized_product(block_file, fmt, dynamic, monkeypatch):
if fmt == "nvfp4":
pytest.importorskip("flashinfer")
monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "1")
path, _tensors, expected = block_file(fmt, drop_input_scale = dynamic)
model = _load(path, dtype = torch.bfloat16, **{f"{fmt}_backend": "kept"}).cuda()
x = torch.randn(64, DIM, device = "cuda", dtype = torch.bfloat16)
ref = (
x.float() @ expected["blocks.1.to_out.weight"].cuda().T
+ model.blocks[1].to_out.bias.float()
)
got = model.blocks[1].to_out(x).float()
rel = ((got - ref).norm() / ref.norm()).item()
assert rel < (0.15 if fmt == "nvfp4" else 0.05), rel # activation quantization error only
def _target(device):
return types.SimpleNamespace(torch_device = device, dtype = torch.bfloat16)
def test_nvfp4_on_cpu_or_mps_dequantizes(monkeypatch):
monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "1")
for device in ("cpu", "mps"):
backend, reason = cb.comfy_block_backend("nvfp4", _target(device), "z-image")
assert backend is None and "not CUDA" in reason
def test_nvfp4_follows_studios_nvfp4_switch(monkeypatch):
import core.inference.diffusion_nvfp4_ops as ops
monkeypatch.setattr(ops, "_resolve_backend", lambda device = None: ("flashinfer", "preflight ok"))
monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "0")
backend, reason = cb.comfy_block_backend("nvfp4", _target("cuda:0"), "z-image")
assert backend is None and "UNSLOTH_NVFP4_DIFFUSION=1" in reason
monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "1")
assert cb.comfy_block_backend("nvfp4", _target("cuda:0"), "z-image")[0] == "flashinfer"
monkeypatch.setenv("UNSLOTH_DIFFUSION_COMFY_NVFP4", "0")
assert cb.comfy_block_backend("nvfp4", _target("cuda:0"), "z-image")[0] is None
def test_nvfp4_without_flashinfer_or_on_a_denied_family_dequantizes(monkeypatch):
import core.inference.diffusion_nvfp4_ops as ops
monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "1")
monkeypatch.setattr(
ops,
"_resolve_backend",
lambda device = None: ("torchao", "sm_90 is not in the flashinfer NVFP4 set"),
)
backend, reason = cb.comfy_block_backend("nvfp4", _target("cuda:0"), "z-image")
assert backend is None and "sm_90" in reason
monkeypatch.setattr(ops, "_resolve_backend", lambda device = None: ("flashinfer", "ok"))
assert cb.comfy_block_backend("nvfp4", _target("cuda:0"), "qwen-image")[0] is None
def test_flashinfer_install_is_only_wanted_when_the_codes_can_stay_nvfp4(monkeypatch):
assert cb.comfy_nvfp4_runtime_possible("z-image")
assert not cb.comfy_nvfp4_runtime_possible("qwen-image")
monkeypatch.setenv("UNSLOTH_DIFFUSION_COMFY_NVFP4", "0")
assert not cb.comfy_nvfp4_runtime_possible("z-image")
def test_mxfp8_needs_blackwell_bf16_and_an_allowed_family(monkeypatch):
monkeypatch.setattr(cb, "mxfp8_runtime_reason", lambda target: None)
assert (
cb.comfy_block_backend("mxfp8", _target("cuda:0"), "krea-2", dtype = torch.bfloat16)[0]
== "scaled_mm"
)
assert (
cb.comfy_block_backend("mxfp8", _target("cuda:0"), "krea-2", dtype = torch.float16)[0] is None
)
assert cb.comfy_block_backend("mxfp8", _target("cuda:0"), "qwen-image")[0] is None
monkeypatch.setenv("UNSLOTH_DIFFUSION_COMFY_MXFP8", "0")
assert cb.comfy_block_backend("mxfp8", _target("cuda:0"), "krea-2")[0] is None
def test_mxfp8_probe_refuses_pre_blackwell_and_non_cuda(monkeypatch):
assert "not CUDA" in cb.mxfp8_runtime_reason(_target("cpu"))
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (9, 0))
assert "sm_90" in cb.mxfp8_runtime_reason(_target("cuda:0"))
def test_backends_are_logged_per_format_present(caplog):
import logging
scan = types.SimpleNamespace(counts = lambda: {"nvfp4": 3})
logger = logging.getLogger("comfy_block_test")
with caplog.at_level(logging.INFO, logger = "comfy_block_test"):
out = cb.comfy_block_backends(scan, _target("cpu"), "z-image", logger = logger)
assert out == {"nvfp4_backend": None, "mxfp8_backend": None}
assert "3 nvfp4 layer(s) have no native runtime" in caplog.text and "mxfp8" not in caplog.text
def test_a_failed_mxfp8_probe_is_not_cached(monkeypatch):
import contextlib
cb.reset_probe_cache()
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (10, 0))
monkeypatch.setattr(torch.cuda, "device", lambda index: contextlib.nullcontext())
def boom(*_a, **_k):
raise torch.OutOfMemoryError("CUDA out of memory")
# the probe's first allocation fails, as it would on a full GPU (and before CPU-only torch can object)
monkeypatch.setattr(torch, "randn", boom)
assert "OutOfMemoryError" in cb.mxfp8_runtime_reason(_target("cuda:0"))
assert cb._MX_PROBE == {}
def test_a_lora_load_prices_block_layers_dequantized(monkeypatch, tmp_path):
import core.inference.diffusion as d
path = str(_layer_file(tmp_path, "mxfp8", rows = 2048, cols = 2048))
monkeypatch.setattr(d, "comfy_block_backend", lambda fmt, *a, **k: ("scaled_mm", "ok"))
monkeypatch.setattr(d, "comfy_int8_backend", lambda *a, **k: None)
monkeypatch.setattr(d, "comfy_fp8_backend", lambda *a, **k: None)
fam = types.SimpleNamespace(name = "krea-2")
kept = d.DiffusionBackend._comfy_single_file_resident_mib(path, fam, _target("cuda:0"), None)
lora = d.DiffusionBackend._comfy_single_file_resident_mib(
path, fam, _target("cuda:0"), None, lora = True
)
assert kept is not None and lora > kept
def test_mxfp8_e8m0_scales_are_priced_one_byte(tmp_path):
path = _layer_file(tmp_path, "mxfp8", rows = 4096, cols = 8192)
# 32 MiB of fp8 codes + 1 MiB of e8m0 block scales + the comfy_quant bytes, rounded up (35 at 2 bytes)
assert cq.comfy_resident_mib(path, keep_int8 = False, keep_fp8 = False, keep_mxfp8 = True) == 34
def test_dequantized_nvfp4_is_priced_at_its_unpacked_width(block_file, monkeypatch):
"""nvfp4 packs two columns per byte, so the header's numel is half the dense N x K the dequantization
allocates; the host RAM check must price the dense weight."""
import core.inference.diffusion_single_file_converters as conv
path, _tensors, expected = block_file("nvfp4")
seen = []
monkeypatch.setattr(conv.HostRamBudget, "check", lambda self, need: seen.append(need))
_load(path)
dense = sum(t.numel() for t in expected.values()) * 4 # float32 compute
assert seen and seen[0] >= dense
def _tight_host(monkeypatch, headroom_mib):
"""A host whose free RAM leaves ``headroom_mib`` above the small-host reserve; every byte priced."""
import core.inference.diffusion_single_file_converters as conv
import core.inference.diffusion_small_host as small_host
total = 32768
monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", 0)
monkeypatch.delenv(small_host.HOST_RAM_CHECK_ENV, raising = False)
monkeypatch.setattr(
small_host,
"host_ram_mib",
lambda: (total, small_host.host_reserve_mib(total) + headroom_mib),
)
return conv
def test_layers_the_runtime_filter_rejects_are_charged_as_dequantized(block_file, monkeypatch):
"""A runtime for the format keeps only layers passing its filter (size, name exclusions, ...); the
rest are expanded to the compute dtype, each checked before its expansion."""
conv = _tight_host(monkeypatch, headroom_mib = 1 << 20)
seen = []
monkeypatch.setattr(conv.HostRamBudget, "charge", lambda self, n: seen.append(int(n)))
path, _tensors, _expected = block_file("mxfp8")
model = _load(path, dtype = torch.bfloat16, mxfp8_backend = "kept", min_features = 1 << 20)
assert model._unsloth_comfy_quant["dequantized"] == 8
# every bf16 to_q / to_k / to_v / to_out, checked for at least its dense size
assert sum(1 for n in seen if n >= DIM * DIM * 2) >= 8
def test_kept_nvfp4_codes_are_charged_for_their_nibble_swapped_copy(block_file, monkeypatch):
"""With an nvfp4 runtime the codes are kept, but decode_layer writes a nibble-swapped uint8 copy of
the packed weight (plus untiled scales): new host RAM, checked even though nothing is dequantized."""
conv = _tight_host(monkeypatch, headroom_mib = 1 << 20)
seen = []
monkeypatch.setattr(conv.HostRamBudget, "charge", lambda self, n: seen.append(int(n)))
path, tensors, _expected = block_file("nvfp4")
try:
_load(path, dtype = torch.bfloat16, nvfp4_backend = "kept")
except ImportError:
pass # no flashinfer here: the checks ran while decoding, before any runtime Linear is built
packed = tensors["blocks.0.out.weight"]
assert any(n >= packed.numel() * packed.element_size() for n in seen), seen
def test_a_runtime_linear_checks_its_tiled_scale_buffer(block_file, monkeypatch):
"""A kept mxfp8 layer re-tiles its scales for the runtime Linear (a zero-padded buffer and the
permuted copy it keeps): both are new host memory, checked before they are allocated."""
import core.inference.diffusion_single_file_converters as conv
import core.inference.diffusion_small_host as small_host
monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", 0)
monkeypatch.delenv(small_host.HOST_RAM_CHECK_ENV, raising = False)
monkeypatch.setattr(small_host, "host_ram_mib", lambda: (1 << 20, 1 << 20))
seen = []
monkeypatch.setattr(conv.HostRamBudget, "charge", lambda self, n: seen.append(int(n)))
path, _tensors, _expected = block_file("mxfp8")
_load(path, dtype = torch.bfloat16, mxfp8_backend = "kept")
assert 2 * _ceil(DIM, 128) * _ceil(DIM // 32, 4) in seen
class _Many(nn.Module):
"""Many bias-free square Linears: enough layers that per-layer transients would add up."""
_keep_in_fp32_modules = None
_keys_to_ignore_on_load_unexpected = None
LAYERS, WIDTH = 48, 2048
def __init__(
self,
layers: int = LAYERS,
width: int = WIDTH,
) -> None:
super().__init__()
self.layers = nn.ModuleList(nn.Linear(width, width, bias = False) for _ in range(layers))
@classmethod
def load_config(cls, repo, **_kwargs):
return {"layers": cls.LAYERS, "width": cls.WIDTH}
@staticmethod
def _get_signature_keys(_cls):
return {"layers", "width"}, set()
@classmethod
def from_config(cls, config):
return cls(**config)
def _rss_anon_bytes():
import ctypes
try:
ctypes.CDLL("libc.so.6").malloc_trim(0) # hand freed heap back, so RssAnon is what is live
except OSError:
pass
try:
status = open("/proc/self/status")
except OSError: # no procfs (Windows, macOS): callers skip on None
return None
with status:
for line in status:
if line.startswith("RssAnon:"):
return int(line.split()[1]) * 1024
return None
def test_a_many_layer_nvfp4_fallback_load_that_fits_is_not_refused(tmp_path, monkeypatch):
"""Without an nvfp4 runtime, each layer is dequantized then freed; free RAM is this process's
RssAnon, and the charge is dense plus 10% and one chunk, so a fitting load is not refused."""
import core.inference.diffusion_single_file_converters as conv
import core.inference.diffusion_small_host as small_host
if not torch.__config__.show() or _rss_anon_bytes() is None:
pytest.skip("needs /proc/self/status RssAnon")
layers, width = _Many.LAYERS, _Many.WIDTH
gen = torch.Generator().manual_seed(0)
tensors = {}
for i in range(layers):
stem = f"layers.{i}"
tensors[f"{stem}.weight"] = torch.randint(
0, 256, (width, width // 2), dtype = torch.uint8, generator = gen
)
plain = torch.randint(56, 64, (width, width // 16), dtype = torch.uint8, generator = gen)
tensors[f"{stem}.weight_scale"] = _tile(plain).view(torch.float8_e4m3fn)
tensors[f"{stem}.weight_scale_2"] = torch.tensor(0.01)
tensors[f"{stem}.comfy_quant"] = _conf(format = "nvfp4")
path = _save(tmp_path / "many_nvfp4.safetensors", tensors)
del tensors
monkeypatch.setattr(
cq,
"_mapping",
lambda cls: (
lambda checkpoint = None, **_kw: checkpoint,
types.SimpleNamespace(
_should_convert_state_dict_to_diffusers = lambda a, b: False,
_get_mapping_function_kwargs = lambda fn, **kw: {},
),
),
)
dense = layers * width * width * 2 # bf16
total = 1 << 20
reserve = small_host.host_reserve_mib(total)
start = _rss_anon_bytes()
headroom = int(dense * 1.10) + getattr(cb, "_DEQUANT_CHUNK_BYTES", 16 << 20)
monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", 0)
monkeypatch.delenv(small_host.HOST_RAM_CHECK_ENV, raising = False)
monkeypatch.setattr(
small_host,
"host_ram_mib",
lambda: (total, reserve + (headroom - (_rss_anon_bytes() - start)) // (1 << 20)),
)
model = cq.load_comfy_quant_transformer(
_Many,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": torch.bfloat16, "config": "base/repo"},
int8_backend = None,
nvfp4_backend = None,
family = "z-image",
)
assert model._unsloth_comfy_quant["dequantized"] == layers
assert model.layers[0].weight.dtype == torch.bfloat16
def test_a_small_file_that_widens_past_the_floor_is_still_checked(block_file, monkeypatch):
"""The check floor applies to what the load writes, not to the file: nvfp4 packs 8 dense float32
bytes into half a byte, so a file under the floor can still widen far past it."""
import os
conv = _tight_host(monkeypatch, headroom_mib = 1)
path, _tensors, _expected = block_file("nvfp4")
monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", os.path.getsize(path) // (1 << 20) + 1)
with pytest.raises(conv.HostRamShortfallError, match = "needs about"):
_load(path) # float32: 8 MiB dense from a ~1 MiB file
_PEAK_PROBE = r"""
import ctypes, json, sys, threading, time
sys.path.insert(0, sys.argv[2])
import torch
import core.inference.diffusion_comfy_block as cb
import core.inference.diffusion_single_file_converters as conv
fmt = sys.argv[1]
def rss_anon(trim):
if trim:
ctypes.CDLL("libc.so.6").malloc_trim(0) # hand freed heap back, so RssAnon is what is live
with open("/proc/self/status") as status:
for line in status:
if line.startswith("RssAnon:"):
return int(line.split()[1]) * 1024
rows, cols = 4096, 4096
gen = torch.Generator().manual_seed(1)
if fmt == "nvfp4":
codes = torch.randint(0, 256, (rows, cols // 2), dtype = torch.uint8, generator = gen)
scale = torch.randint(56, 64, (rows, cols // 16), dtype = torch.uint8, generator = gen)
scale = scale.view(torch.float8_e4m3fn)
else:
codes = torch.randint(0, 120, (rows, cols), dtype = torch.uint8, generator = gen)
codes = codes.view(torch.float8_e4m3fn)
scale = torch.randint(120, 130, (rows, cols // 32), dtype = torch.uint8, generator = gen)
seen = []
conv.charge_host_ram = lambda n: seen.append(int(n))
def measure():
peak, stop = [0], [False]
def sample():
while not stop[0]:
peak[0] = max(peak[0], rss_anon(False))
time.sleep(0.0005)
start = rss_anon(True)
thread = threading.Thread(target = sample, daemon = True)
thread.start()
out = cb.dequant_block(fmt, codes, scale, tensor_scale = 0.01, dtype = torch.bfloat16)
stop[0] = True
thread.join()
assert out.shape == (rows, cols)
return peak[0] - start
growth = min(measure() for _ in range(3))
print("PEAK " + json.dumps({"seen": seen, "growth": growth}))
"""
@pytest.mark.parametrize("fmt", ["nvfp4", "mxfp8"])
def test_block_dequant_is_checked_for_its_peak_intermediates(fmt):
"""Fallback dequant builds int32/float32 intermediates several times the bf16 output, so check
the peak, not the output; measure in a fresh process, since earlier tests in one pytest
worker add ~7 MB of RssAnon."""
import subprocess
import sys
from pathlib import Path
if _rss_anon_bytes() is None:
pytest.skip("needs /proc/self/status RssAnon")
backend = str(Path(__file__).resolve().parents[1])
proc = subprocess.run(
[sys.executable, "-c", _PEAK_PROBE, fmt, backend],
capture_output = True,
text = True,
timeout = 600,
)
line = [l for l in proc.stdout.splitlines() if l.startswith("PEAK ")]
assert proc.returncode == 0 and line, proc.stdout[-2000:] + proc.stderr[-4000:]
result = json.loads(line[-1][len("PEAK ") :])
seen, growth = result["seen"], result["growth"]
assert seen and max(seen) >= 0.9 * growth, (seen, growth)
@pytest.mark.parametrize("fmt", ["nvfp4", "mxfp8"])
def test_chunked_block_dequant_is_bit_identical_to_one_pass(monkeypatch, fmt):
rows, cols = 300, 256
gen = torch.Generator().manual_seed(2)
if fmt == "nvfp4":
codes = torch.randint(0, 256, (rows, cols // 2), dtype = torch.uint8, generator = gen)
scale = torch.randint(40, 70, (rows, cols // 16), dtype = torch.uint8, generator = gen)
scale = scale.view(torch.float8_e4m3fn)
tensor_scale = torch.rand(rows, generator = gen) + 0.01 # per row, as a joined layer carries
else:
codes = torch.randint(0, 120, (rows, cols), dtype = torch.uint8, generator = gen)
codes = codes.view(torch.float8_e4m3fn)
scale = torch.randint(110, 140, (rows, cols // 32), dtype = torch.uint8, generator = gen)
tensor_scale = None
pqs = torch.rand(cols, generator = gen) + 0.5
# the pre-chunking formula, in one pass
if fmt == "nvfp4":
lut = torch.tensor(cb._E2M1 + tuple(-v for v in cb._E2M1), dtype = torch.float32)
c = codes.to(torch.int32)
values = torch.stack((lut[c & 0x0F], lut[c >> 4]), dim = -1).reshape(rows, -1, 16)
step = scale.to(torch.float32) * tensor_scale.to(torch.float32).reshape(-1, 1)
weight = (values * step.unsqueeze(-1)).reshape(rows, -1)
else:
exp = scale.view(torch.uint8).to(torch.float32) - cb._E8M0_BIAS
weight = (
codes.to(torch.float32).reshape(rows, -1, 32) * torch.exp2(exp).unsqueeze(-1)
).reshape(rows, -1)
want = (weight * pqs.reshape(1, -1)).to(torch.bfloat16)
for chunk_bytes in (1, 7 * 4096, 1 << 30): # one row per chunk, uneven chunks, one pass
monkeypatch.setattr(cb, "_DEQUANT_CHUNK_BYTES", chunk_bytes, raising = False)
got = cb.dequant_block(
fmt, codes, scale, tensor_scale = tensor_scale, pre_quant_scale = pqs, dtype = torch.bfloat16
)
assert torch.equal(got, want), chunk_bytes
_HOST_PEAK_PROBE = r"""
import ctypes, json, sys, threading, time
sys.path.insert(0, sys.argv[2])
import torch
import core.inference.diffusion_comfy_block as cb
import core.inference.diffusion_comfy_quant as cq
import core.inference.diffusion_single_file_converters as conv
kind = sys.argv[1]
def rss_anon(trim):
if trim:
ctypes.CDLL("libc.so.6").malloc_trim(0) # hand freed heap back, so RssAnon is what is live
with open("/proc/self/status") as status:
for line in status:
if line.startswith("RssAnon:"):
return int(line.split()[1]) * 1024
rows, cols = 4096, 4096
gen = torch.Generator().manual_seed(3)
if kind == "swap_nibbles":
packed = torch.randint(0, 256, (8192, 8192), dtype = torch.uint8, generator = gen)
elif kind == "int8_convrot":
codes = torch.randint(-127, 128, (rows, cols), dtype = torch.int8, generator = gen)
scale, group = torch.rand(rows, 1, generator = gen) / 127.0, 256
else:
codes = (torch.randn(rows, cols, generator = gen) * 50).to(torch.float8_e4m3fn)
scale, group = torch.tensor([0.01]), 0
seen = []
conv.charge_host_ram = lambda n: seen.append(int(n))
def measure():
peak, stop = [0], [False]
def sample():
while not stop[0]:
peak[0] = max(peak[0], rss_anon(False))
time.sleep(0.0005)
start = rss_anon(True)
thread = threading.Thread(target = sample, daemon = True)
thread.start()
if kind == "swap_nibbles":
out = cb.swap_nibbles(packed)
assert out.shape == packed.shape
else:
out = cq._dequant(codes, scale, group, torch.bfloat16)
assert out.shape == (rows, cols) and out.dtype == torch.bfloat16
stop[0] = True
thread.join()
return peak[0] - start
growth = min(measure() for _ in range(3))
print("PEAK " + json.dumps({"seen": seen, "growth": growth}))
"""
def _host_peak_in_a_fresh_process(kind):
"""(charges, growth) for one host-side op, measured with nothing else in the process."""
import subprocess
import sys
from pathlib import Path
backend = str(Path(__file__).resolve().parents[1])
proc = subprocess.run(
[sys.executable, "-c", _HOST_PEAK_PROBE, kind, backend],
capture_output = True,
text = True,
timeout = 600,
)
line = [l for l in proc.stdout.splitlines() if l.startswith("PEAK ")]
assert proc.returncode == 0 and line, proc.stdout[-2000:] + proc.stderr[-4000:]
result = json.loads(line[-1][len("PEAK ") :])
return result["seen"], result["growth"]
@pytest.mark.parametrize("kind", ["int8_convrot", "fp8"])
def test_int8_fp8_dequant_is_checked_for_its_float32_peak(kind):
"""int8/fp8 dequant holds float32 codes and their product (plus ConvRot's float32 matmul), so
the check must cover that peak; run in a fresh process, since worker readings carry earlier
tests' memory."""
if _rss_anon_bytes() is None:
pytest.skip("needs /proc/self/status RssAnon")
seen, growth = _host_peak_in_a_fresh_process(kind)
assert seen and max(seen) >= 0.9 * growth, (seen, growth)
def test_a_load_under_the_floor_is_checked_once_its_allocations_reach_it(block_file, monkeypatch):
"""File and estimate sit under the floor, but the mxfp8 runtime filter rejects every layer, so
they are dequantized; the check must still run once allocations reach it."""
import os
conv = _tight_host(monkeypatch, headroom_mib = 0)
path, _tensors, _expected = block_file("mxfp8")
monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", os.path.getsize(path) // (1 << 20) + 1)
with pytest.raises(conv.HostRamShortfallError, match = "needs at least"):
_load(path, dtype = torch.bfloat16, mxfp8_backend = "kept", min_features = 1 << 20)
def test_swap_nibbles_is_bit_identical_and_checked_for_its_peak(monkeypatch):
gen = torch.Generator().manual_seed(4)
small = torch.randint(0, 256, (37, 19), dtype = torch.uint8, generator = gen)
for chunk_bytes in (1, 64, 1 << 30):
monkeypatch.setattr(cb, "_DEQUANT_CHUNK_BYTES", chunk_bytes, raising = False)
assert torch.equal(cb.swap_nibbles(small), ((small << 4) | (small >> 4)).to(torch.uint8))
monkeypatch.undo()
if _rss_anon_bytes() is None:
pytest.skip("needs /proc/self/status RssAnon")
# Peak in a fresh process, like the dequant checks above.
seen, growth = _host_peak_in_a_fresh_process("swap_nibbles")
assert seen and max(seen) >= 0.9 * growth, (seen, growth)
def test_untiling_padded_blocks_checks_both_live_copies(monkeypatch):
"""With a block count that is not a multiple of 4 the permuted padded buffer and the trimmed copy are
live together; both are checked before either is made."""
import core.inference.diffusion_single_file_converters as conv
rows, blocks = 200, 6
plain = torch.randint(0, 255, (rows, blocks), dtype = torch.uint8)
tiled = cb.tile_scales(plain)
seen = []
monkeypatch.setattr(conv, "charge_host_ram", lambda n: seen.append(int(n)))
assert torch.equal(cb.untile_scales(tiled, rows, blocks), plain)
assert seen == [_ceil(rows, 128) * _ceil(blocks, 4) + rows * blocks]
@pytest.mark.parametrize("fmt", ["nvfp4", "mxfp8"])
def test_a_compact_storage_load_holds_block_layers_at_their_file_size(block_file, fmt):
"""The fp8-storage pick (widening would offload) keeps nvfp4 / mxfp8 layers no runtime takes near their file
size, not as a bf16 copy, and their weights stay the dense load's."""
path, _tensors, _expected = block_file(fmt)
dense = _load(path, dtype = torch.bfloat16)
compact = _load(path, dtype = torch.bfloat16, fp8_storage = True)
names = [f"blocks.{b}.{n}" for b in range(2) for n in ("to_q", "to_k", "to_v", "to_out")]
def held(model, name):
mod = model.get_submodule(name)
tensors = [
t for n, t in list(mod.named_parameters()) + list(mod.named_buffers()) if n != "bias"
]
return sum(t.numel() * t.element_size() for t in tensors)
bf16 = sum(held(dense, n) for n in names)
kept = sum(held(compact, n) for n in names)
assert bf16 == 8 * DIM * DIM * 2
assert kept < (0.3 if fmt == "nvfp4" else 0.6) * bf16, (kept, bf16)
for name in names:
assert torch.equal(
compact.get_submodule(name).weight.view(torch.int16),
dense.get_submodule(name).weight.view(torch.int16),
), name