Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

10.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""INT8 and INT8-ConvRot share ``transformer_quant="int8"``; a bundle from one reported a hit for the other, then
recompiled in full. The key now carries ``graph_variant`` for quantized loads; dense and GGUF keys are unchanged."""

from __future__ import annotations

import json
import types

import pytest

torch = pytest.importorskip("torch")
from torch import nn  # noqa: E402

from core.inference import diffusion_compile_cache as cc  # noqa: E402
from core.inference.diffusion_convrot import (  # noqa: E402
    apply_activation_rotation,
    rotation_metadata,
)

_FP_KW = dict(
    family = "z-image",
    dtype = "torch.bfloat16",
    attention_backend = "_native_cudnn",
    compile_kwargs = {"fullgraph": True, "dynamic": True, "mode": "default"},
)


def _model():
    torch.manual_seed(0)
    m = nn.Sequential(nn.Linear(256, 256, bias = False), nn.LayerNorm(256), nn.Linear(256, 64))
    m._repeated_blocks = ["Linear"]
    return m.to(torch.bfloat16)


def _int8(act = True):
    tq = pytest.importorskip("torchao.quantization")
    m = _model()
    cfg = (
        tq.Int8DynamicActivationInt8WeightConfig(version = 2)
        if act
        else tq.Int8WeightOnlyConfig(version = 2)
    )
    tq.quantize_(m, cfg)
    return m


def _convrot(m):
    apply_activation_rotation(m, rotation_metadata(256, ["0"]))
    return m


def _key(transformer, quant = "int8"):
    return cc.cache_key(
        cc.environment_fingerprint(),
        cc.model_fingerprint(transformer = transformer, quant = quant, **_FP_KW),
    )


@pytest.fixture(autouse = True)
def _clean_switches(monkeypatch):
    for name in cc._GRAPH_SWITCHES:
        monkeypatch.delenv(name, raising = False)


def test_identical_config_keys_the_same_across_builds():
    a, b = _int8(), _int8()
    assert _key(a) == _key(b)
    # Survives a JSON round trip (the manifest's exact-match guard compares the parsed dict).
    fp = cc.model_fingerprint(transformer = a, quant = "int8", **_FP_KW)
    assert json.loads(json.dumps(fp, sort_keys = True, default = str)) == fp


def test_plain_and_convrot_int8_key_apart():
    plain = _int8()
    rotated = _convrot(_int8())
    assert _key(plain) != _key(rotated)
    variant = cc.graph_variant(rotated)
    assert variant["rotation"] == {"group": 256, "kind": "convrot_hadamard_v1", "linears": 1}
    assert any(sig.startswith("ConvRotLinear/") for sig in variant["weights"])
    assert "rotation" not in cc.graph_variant(plain)


def test_w8a8_and_weight_only_key_apart():
    assert _key(_int8(act = True)) != _key(_int8(act = False))


def test_runtime_quantise_and_prequant_scale_dtype_key_apart():
    # v2 hosted stores fp32 scales; runtime quantise or v1 bf16 scales need a different key.
    a, b = _int8(), _int8()
    w = b[0].weight
    if type(w).__name__ != "Int8Tensor":
        pytest.skip("this torchao predates Int8Tensor")
    other = torch.float32 if w.scale.dtype == torch.bfloat16 else torch.bfloat16
    w.scale = w.scale.to(other)
    if b[0].weight.scale.dtype != other:
        pytest.skip("this torchao does not take a scale reassignment")
    assert _key(a) != _key(b)


def test_a_native_twin_keys_apart_from_torchao():
    # A ComfyUI file served by native layers instead of torchao: different module class, different key.
    a = _int8()
    b = _int8()
    native_cls = type("NativeInt8Linear", (nn.Linear,), {})
    b[0].__class__ = native_cls
    assert _key(a) != _key(b)


def test_an_instance_forward_swap_keys_apart():
    a, b = _int8(), _int8()

    def fused_forward(self, x):  # stands in for a fused-kernel swap installed at load
        return x

    b[0].forward = types.MethodType(fused_forward, b[0])
    assert _key(a) != _key(b)


@pytest.mark.parametrize("name,default", sorted(cc._GRAPH_SWITCHES.items()))
@pytest.mark.parametrize("quant", ["int8", None])
def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, default, quant):
    if default == "protect":
        from core.inference import diffusion_nvfp4_protect as pr

        # an sm_100-class card, whose unset default is auto
        monkeypatch.setattr(pr, "default_model_spec", lambda device = None: "auto")
    m = _int8() if quant else _model()
    k0 = _key(m, quant)
    rows = default.partition(":")[2] if default.startswith(("rows:", "count:")) else None
    for same in (
        ("",)
        + (() if default == "raw" else ("auto",))
        + ((default,) if default in ("on", "off") else ())
        + ((rows,) if rows else ())
    ):
        monkeypatch.setenv(name, same)
        assert _key(m, quant) == k0, same
    if default.startswith("count:"):
        # 0 is a real value here (no dense channels), not an off word.
        flipped = "0"
    elif rows:
        flipped = "1"
    else:
        flipped = {
            "on": "0",
            "off": "1",
            "auto": "0",
            "raw": "torchao",
            "raw:auto": "torchao",
            "protect": "all",
        }[default]
    monkeypatch.setenv(name, flipped)
    assert _key(m, quant) != k0
    assert name in cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)["switches"]


def test_a_default_environment_adds_no_switches():
    assert cc.graph_switches() == {}
    assert "switches" not in cc.model_fingerprint(transformer = _model(), quant = None, **_FP_KW)


def test_a_comfy_single_file_with_no_engaged_scheme_still_keys_its_layout():
    # ComfyUI single-file loads engage no scheme (quant reads none), so the tree itself has to key them apart.
    dense = _model()
    comfy = _int8()
    fp_dense = cc.model_fingerprint(transformer = dense, quant = None, **_FP_KW)
    fp_comfy = cc.model_fingerprint(transformer = comfy, quant = None, **_FP_KW)
    assert "variant" not in fp_dense
    assert "variant" in fp_comfy
    assert _key(dense, None) != _key(comfy, None)
    assert _key(comfy, None) != _key(_convrot(_int8()), None)


def test_an_offload_hook_forward_alone_keeps_a_dense_key():
    m = _model()
    k0 = _key(m, None)
    m[0].forward = types.MethodType(lambda self, x: x, m[0])
    assert "variant" not in cc.model_fingerprint(transformer = m, quant = None, **_FP_KW)
    assert _key(m, None) == k0


def test_dense_and_gguf_keys_are_unchanged():
    # No variant on a dense or GGUF load, so their existing bundles keep hitting.
    m = _model()
    for quant in (None, "gguf"):
        fp = cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)
        assert "variant" not in fp


def test_unreadable_tree_never_raises():
    variant = cc.graph_variant(types.SimpleNamespace())
    assert variant["weights"].startswith("unavailable")


def test_a_bundle_saved_for_one_variant_is_a_miss_for_the_other(monkeypatch, tmp_path):
    store: dict = {}
    monkeypatch.setattr(
        torch.compiler, "save_cache_artifacts", lambda: (b"artifacts", object()), raising = False
    )

    def fake_load(data):
        store["loaded"] = data
        return object()

    monkeypatch.setattr(torch.compiler, "load_cache_artifacts", fake_load, raising = False)
    monkeypatch.setenv(cc._ENV_DIR, str(tmp_path))
    monkeypatch.setenv(cc._ENV_MODE, "1")
    rotated = cc.begin(transformer = _convrot(_int8()), quant = "int8", **_FP_KW)
    assert rotated is not None and not rotated.hit
    assert cc.save(rotated) is True
    cc.restore(rotated)
    plain = cc.begin(transformer = _int8(), quant = "int8", **_FP_KW)
    assert plain is not None and not plain.hit and plain.key != rotated.key
    assert "loaded" not in store
    cc.restore(plain)
    again = cc.begin(transformer = _convrot(_int8()), quant = "int8", **_FP_KW)
    assert again is not None and again.hit and again.key == rotated.key
    cc.restore(again)


@pytest.mark.parametrize("default", ["auto", ""])
def test_protect_steps_key_apart_only_off_the_cards_default(monkeypatch, default):
    """sm_100-class cards default to auto, others to off: the unset run shares its card's default bundle."""
    from core.inference import diffusion_nvfp4_protect as pr

    monkeypatch.setattr(pr, "default_model_spec", lambda device = None: default)
    m = _int8()
    k0 = _key(m)
    monkeypatch.setenv("UNSLOTH_NVFP4_BACKEND", "auto")
    keys = {}
    for value in ("auto", "off", "0"):
        monkeypatch.setenv("UNSLOTH_NVFP4_PROTECT_STEPS", value)
        keys[value] = _key(m)
        same = value == (default or "off")
        assert (keys[value] == k0) is same, value
        assert cc.graph_switches() == ({} if same else {"UNSLOTH_NVFP4_PROTECT_STEPS": value})
    assert len(set(keys.values())) == 3


def test_a_dequantizing_weight_property_is_never_read():
    reads = []

    class PackedLinear(nn.Module):
        def __init__(self, scheme):
            super().__init__()
            self.register_buffer("weight_q", torch.zeros(4, 4, dtype = scheme))

        @property
        def weight(self):
            reads.append(1)
            return self.weight_q.float()

    m = _model()
    m[0] = PackedLinear(torch.int8)
    other = _model()
    other[0] = PackedLinear(torch.uint8)
    fp = cc.model_fingerprint(transformer = m, quant = None, **_FP_KW)
    assert not reads
    assert "variant" in fp and any(sig.endswith("/packed") for sig in fp["variant"]["weights"])
    assert _key(m, None) != _key(other, None)
    assert not reads


def test_per_row_and_per_tensor_scales_key_apart():
    class FakeFp8:
        tensor_data_names = ("qdata", "scale")
        tensor_attribute_names = ("block_size",)
        dtype = torch.bfloat16

        def __init__(self, block, scale_shape):
            self.shape = (64, 256)
            self.block_size = block
            self.qdata = torch.zeros(64, 256, dtype = torch.float8_e4m3fn)
            self.scale = torch.ones(scale_shape)

    per_row = cc._tensor_signature(FakeFp8([1, 256], (64, 1)))
    per_tensor = cc._tensor_signature(FakeFp8([64, 256], (1,)))
    assert per_row != per_tensor
    assert "block_size=1,full" in per_row and "block_size=full,full" in per_tensor


def _stored_fp8(fmt):
    from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class

    m = _model()
    lin = m[0]
    fp8 = getattr(torch, fmt)
    w = lin.weight.detach().float()
    scale = w.abs().amax(dim = 1, keepdim = True) / torch.finfo(fp8).max
    m[0] = native_linear_class()(lin, NATIVE_FP8, codes = (w / scale).to(fp8), scale = scale)
    return m


def test_stored_fp8_formats_key_apart_and_each_keys_stable():
    """E4M3 and E5M2 codes decode through different integer graphs, so they never share a bundle."""
    e4, e5 = _stored_fp8("float8_e4m3fn"), _stored_fp8("float8_e5m2")
    assert e4[0].fp8_format == "float8_e4m3fn" and e5[0].fp8_format == "float8_e5m2"
    assert _key(e4, "fp8") != _key(e5, "fp8")
    assert _key(e4, "fp8") == _key(_stored_fp8("float8_e4m3fn"), "fp8")
    assert _key(e5, "fp8") == _key(_stored_fp8("float8_e5m2"), "fp8")