Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""INT8 and INT8-ConvRot share ``transformer_quant="int8"``; a bundle from one reported a hit for the other, then
recompiled in full. The key now carries ``graph_variant`` for quantized loads; dense and GGUF keys are unchanged."""
from __future__ import annotations
import json
import types
import pytest
torch = pytest.importorskip("torch")
from torch import nn # noqa: E402
from core.inference import diffusion_compile_cache as cc # noqa: E402
from core.inference.diffusion_convrot import ( # noqa: E402
apply_activation_rotation,
rotation_metadata,
)
_FP_KW = dict(
family = "z-image",
dtype = "torch.bfloat16",
attention_backend = "_native_cudnn",
compile_kwargs = {"fullgraph": True, "dynamic": True, "mode": "default"},
)
def _model():
torch.manual_seed(0)
m = nn.Sequential(nn.Linear(256, 256, bias = False), nn.LayerNorm(256), nn.Linear(256, 64))
m._repeated_blocks = ["Linear"]
return m.to(torch.bfloat16)
def _int8(act = True):
tq = pytest.importorskip("torchao.quantization")
m = _model()
cfg = (
tq.Int8DynamicActivationInt8WeightConfig(version = 2)
if act
else tq.Int8WeightOnlyConfig(version = 2)
)
tq.quantize_(m, cfg)
return m
def _convrot(m):
apply_activation_rotation(m, rotation_metadata(256, ["0"]))
return m
def _key(transformer, quant = "int8"):
return cc.cache_key(
cc.environment_fingerprint(),
cc.model_fingerprint(transformer = transformer, quant = quant, **_FP_KW),
)
@pytest.fixture(autouse = True)
def _clean_switches(monkeypatch):
for name in cc._GRAPH_SWITCHES:
monkeypatch.delenv(name, raising = False)
def test_identical_config_keys_the_same_across_builds():
a, b = _int8(), _int8()
assert _key(a) == _key(b)
# Survives a JSON round trip (the manifest's exact-match guard compares the parsed dict).
fp = cc.model_fingerprint(transformer = a, quant = "int8", **_FP_KW)
assert json.loads(json.dumps(fp, sort_keys = True, default = str)) == fp
def test_plain_and_convrot_int8_key_apart():
plain = _int8()
rotated = _convrot(_int8())
assert _key(plain) != _key(rotated)
variant = cc.graph_variant(rotated)
assert variant["rotation"] == {"group": 256, "kind": "convrot_hadamard_v1", "linears": 1}
assert any(sig.startswith("ConvRotLinear/") for sig in variant["weights"])
assert "rotation" not in cc.graph_variant(plain)
def test_w8a8_and_weight_only_key_apart():
assert _key(_int8(act = True)) != _key(_int8(act = False))
def test_runtime_quantise_and_prequant_scale_dtype_key_apart():
# v2 hosted stores fp32 scales; runtime quantise or v1 bf16 scales need a different key.
a, b = _int8(), _int8()
w = b[0].weight
if type(w).__name__ != "Int8Tensor":
pytest.skip("this torchao predates Int8Tensor")
other = torch.float32 if w.scale.dtype == torch.bfloat16 else torch.bfloat16
w.scale = w.scale.to(other)
if b[0].weight.scale.dtype != other:
pytest.skip("this torchao does not take a scale reassignment")
assert _key(a) != _key(b)
def test_a_native_twin_keys_apart_from_torchao():
# A ComfyUI file served by native layers instead of torchao: different module class, different key.
a = _int8()
b = _int8()
native_cls = type("NativeInt8Linear", (nn.Linear,), {})
b[0].__class__ = native_cls
assert _key(a) != _key(b)
def test_an_instance_forward_swap_keys_apart():
a, b = _int8(), _int8()
def fused_forward(self, x): # stands in for a fused-kernel swap installed at load
return x
b[0].forward = types.MethodType(fused_forward, b[0])
assert _key(a) != _key(b)
@pytest.mark.parametrize("name,default", sorted(cc._GRAPH_SWITCHES.items()))
@pytest.mark.parametrize("quant", ["int8", None])
def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, default, quant):
if default == "protect":
from core.inference import diffusion_nvfp4_protect as pr
# an sm_100-class card, whose unset default is auto
monkeypatch.setattr(pr, "default_model_spec", lambda device = None: "auto")
m = _int8() if quant else _model()
k0 = _key(m, quant)
rows = default.partition(":")[2] if default.startswith(("rows:", "count:")) else None
for same in (
("",)
+ (() if default == "raw" else ("auto",))
+ ((default,) if default in ("on", "off") else ())
+ ((rows,) if rows else ())
):
monkeypatch.setenv(name, same)
assert _key(m, quant) == k0, same
if default.startswith("count:"):
# 0 is a real value here (no dense channels), not an off word.
flipped = "0"
elif rows:
flipped = "1"
else:
flipped = {
"on": "0",
"off": "1",
"auto": "0",
"raw": "torchao",
"raw:auto": "torchao",
"protect": "all",
}[default]
monkeypatch.setenv(name, flipped)
assert _key(m, quant) != k0
assert name in cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)["switches"]
def test_a_default_environment_adds_no_switches():
assert cc.graph_switches() == {}
assert "switches" not in cc.model_fingerprint(transformer = _model(), quant = None, **_FP_KW)
def test_a_comfy_single_file_with_no_engaged_scheme_still_keys_its_layout():
# ComfyUI single-file loads engage no scheme (quant reads none), so the tree itself has to key them apart.
dense = _model()
comfy = _int8()
fp_dense = cc.model_fingerprint(transformer = dense, quant = None, **_FP_KW)
fp_comfy = cc.model_fingerprint(transformer = comfy, quant = None, **_FP_KW)
assert "variant" not in fp_dense
assert "variant" in fp_comfy
assert _key(dense, None) != _key(comfy, None)
assert _key(comfy, None) != _key(_convrot(_int8()), None)
def test_an_offload_hook_forward_alone_keeps_a_dense_key():
m = _model()
k0 = _key(m, None)
m[0].forward = types.MethodType(lambda self, x: x, m[0])
assert "variant" not in cc.model_fingerprint(transformer = m, quant = None, **_FP_KW)
assert _key(m, None) == k0
def test_dense_and_gguf_keys_are_unchanged():
# No variant on a dense or GGUF load, so their existing bundles keep hitting.
m = _model()
for quant in (None, "gguf"):
fp = cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)
assert "variant" not in fp
def test_unreadable_tree_never_raises():
variant = cc.graph_variant(types.SimpleNamespace())
assert variant["weights"].startswith("unavailable")
def test_a_bundle_saved_for_one_variant_is_a_miss_for_the_other(monkeypatch, tmp_path):
store: dict = {}
monkeypatch.setattr(
torch.compiler, "save_cache_artifacts", lambda: (b"artifacts", object()), raising = False
)
def fake_load(data):
store["loaded"] = data
return object()
monkeypatch.setattr(torch.compiler, "load_cache_artifacts", fake_load, raising = False)
monkeypatch.setenv(cc._ENV_DIR, str(tmp_path))
monkeypatch.setenv(cc._ENV_MODE, "1")
rotated = cc.begin(transformer = _convrot(_int8()), quant = "int8", **_FP_KW)
assert rotated is not None and not rotated.hit
assert cc.save(rotated) is True
cc.restore(rotated)
plain = cc.begin(transformer = _int8(), quant = "int8", **_FP_KW)
assert plain is not None and not plain.hit and plain.key != rotated.key
assert "loaded" not in store
cc.restore(plain)
again = cc.begin(transformer = _convrot(_int8()), quant = "int8", **_FP_KW)
assert again is not None and again.hit and again.key == rotated.key
cc.restore(again)
@pytest.mark.parametrize("default", ["auto", ""])
def test_protect_steps_key_apart_only_off_the_cards_default(monkeypatch, default):
"""sm_100-class cards default to auto, others to off: the unset run shares its card's default bundle."""
from core.inference import diffusion_nvfp4_protect as pr
monkeypatch.setattr(pr, "default_model_spec", lambda device = None: default)
m = _int8()
k0 = _key(m)
monkeypatch.setenv("UNSLOTH_NVFP4_BACKEND", "auto")
keys = {}
for value in ("auto", "off", "0"):
monkeypatch.setenv("UNSLOTH_NVFP4_PROTECT_STEPS", value)
keys[value] = _key(m)
same = value == (default or "off")
assert (keys[value] == k0) is same, value
assert cc.graph_switches() == ({} if same else {"UNSLOTH_NVFP4_PROTECT_STEPS": value})
assert len(set(keys.values())) == 3
def test_a_dequantizing_weight_property_is_never_read():
reads = []
class PackedLinear(nn.Module):
def __init__(self, scheme):
super().__init__()
self.register_buffer("weight_q", torch.zeros(4, 4, dtype = scheme))
@property
def weight(self):
reads.append(1)
return self.weight_q.float()
m = _model()
m[0] = PackedLinear(torch.int8)
other = _model()
other[0] = PackedLinear(torch.uint8)
fp = cc.model_fingerprint(transformer = m, quant = None, **_FP_KW)
assert not reads
assert "variant" in fp and any(sig.endswith("/packed") for sig in fp["variant"]["weights"])
assert _key(m, None) != _key(other, None)
assert not reads
def test_per_row_and_per_tensor_scales_key_apart():
class FakeFp8:
tensor_data_names = ("qdata", "scale")
tensor_attribute_names = ("block_size",)
dtype = torch.bfloat16
def __init__(self, block, scale_shape):
self.shape = (64, 256)
self.block_size = block
self.qdata = torch.zeros(64, 256, dtype = torch.float8_e4m3fn)
self.scale = torch.ones(scale_shape)
per_row = cc._tensor_signature(FakeFp8([1, 256], (64, 1)))
per_tensor = cc._tensor_signature(FakeFp8([64, 256], (1,)))
assert per_row != per_tensor
assert "block_size=1,full" in per_row and "block_size=full,full" in per_tensor
def _stored_fp8(fmt):
from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class
m = _model()
lin = m[0]
fp8 = getattr(torch, fmt)
w = lin.weight.detach().float()
scale = w.abs().amax(dim = 1, keepdim = True) / torch.finfo(fp8).max
m[0] = native_linear_class()(lin, NATIVE_FP8, codes = (w / scale).to(fp8), scale = scale)
return m
def test_stored_fp8_formats_key_apart_and_each_keys_stable():
"""E4M3 and E5M2 codes decode through different integer graphs, so they never share a bundle."""
e4, e5 = _stored_fp8("float8_e4m3fn"), _stored_fp8("float8_e5m2")
assert e4[0].fp8_format == "float8_e4m3fn" and e5[0].fp8_format == "float8_e5m2"
assert _key(e4, "fp8") != _key(e5, "fp8")
assert _key(e4, "fp8") == _key(_stored_fp8("float8_e4m3fn"), "fp8")
assert _key(e5, "fp8") == _key(_stored_fp8("float8_e5m2"), "fp8")