Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

20.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Qwen-Image-2.1 defaults to the hosted int8 ConvRot weight-only text encoder and falls back to the fp8 one."""

from __future__ import annotations

import json
import types

import pytest

torch = pytest.importorskip("torch")
pytest.importorskip("transformers.models.qwen3_vl")
pytest.importorskip("accelerate")

import core.inference.diffusion_te_prequant as tpq  # noqa: E402
import core.inference.prequant_safetensors as ps  # noqa: E402
from core.inference.diffusion_families import detect_family  # noqa: E402
from core.inference.diffusion_precision import (  # noqa: E402
    RESOLVED_APPLIED,
    RESOLVED_FELL_BACK,
    effective_te_quant,
    quantize_text_encoders,
    resolve_te_quant_request,
    te_quant_needs_resident_weights,
)
from core.inference.diffusion_prequant import ALLOW_LOCAL_PREQUANT_PATH_ENV  # noqa: E402
from core.inference.diffusion_text_encoder_trim import is_trimmed  # noqa: E402

GROUP = 256
HIDDEN, VOCAB = 256, 64
BASE = "Qwen/Qwen-Image-2.1"
REPO, INT8_NAME = (
    "unsloth/Qwen-Image-2.1-FP8",
    "Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors",
)
FP8_NAME = "Qwen-Image-2.1-text_encoder-FP8.safetensors"
CUDA_BF16 = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)


def get_family(name):
    return detect_family("", override = name)


def _config():
    from transformers.models.qwen3_vl.configuration_qwen3_vl import (
        Qwen3VLConfig,
        Qwen3VLTextConfig,
        Qwen3VLVisionConfig,
    )

    config = Qwen3VLConfig(
        vision_config = Qwen3VLVisionConfig(
            depth = 1,
            hidden_size = 32,
            intermediate_size = 64,
            num_heads = 4,
            out_hidden_size = HIDDEN,
            deepstack_visual_indexes = [],
        ).to_dict(),
        text_config = Qwen3VLTextConfig(
            hidden_size = HIDDEN,
            intermediate_size = 512,
            num_hidden_layers = 2,
            num_attention_heads = 4,
            num_key_value_heads = 2,
            head_dim = 64,
            vocab_size = VOCAB,
            tie_word_embeddings = False,
            rope_scaling = {
                "rope_type": "default",
                "mrope_section": [16, 8, 8],
                "mrope_interleaved": True,
            },
        ).to_dict(),
        tie_word_embeddings = False,
    )
    config._attn_implementation = "sdpa"
    return config


def _encoder(seed = 0):
    from transformers import Qwen3VLForConditionalGeneration
    torch.manual_seed(seed)
    return Qwen3VLForConditionalGeneration(_config()).eval()


def _hidden(encoder, ids):
    lm = encoder.model.language_model
    handle = lm.norm.register_forward_hook(lambda m, a, o: a[0])
    try:
        with torch.no_grad():
            out = encoder(
                input_ids = ids, attention_mask = torch.ones_like(ids), output_hidden_states = True
            )
    finally:
        handle.remove()
    return out.hidden_states


def _container(tmp_path, name, state, fmt, scheme):
    from safetensors.torch import save_file

    meta = {
        ps.UNSLOTH_FORMAT_KEY: fmt,
        ps.UNSLOTH_METADATA_KEY: json.dumps(
            {
                "scheme": scheme,
                "component": "text_encoder",
                "base_model_id": BASE,
                "te_class": "Qwen3VLForConditionalGeneration",
            }
        ),
        "tensor_names": json.dumps(sorted(state)),
        **{n: json.dumps({"_type": "Tensor"}) for n in state},
    }
    path = tmp_path / name
    save_file({k: v.contiguous() for k, v in state.items()}, str(path), metadata = meta)
    return path


def _int8_state(encoder):
    """The builder's layout: decoder projections int8 + scale + comfy_quant, lm_head dropped, the rest dense."""
    import re

    lin = re.compile(
        r"^model\.language_model\.layers\.\d+\.(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)\.weight$"
    )
    blob = torch.tensor(
        list(
            json.dumps(
                {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": GROUP}
            ).encode()
        ),
        dtype = torch.uint8,
    )
    state = {}
    for k, v in encoder.state_dict().items():
        if k == "lm_head.weight":
            continue
        if lin.match(k):
            q, s = tpq.quantize_int8_convrot_weight(v, GROUP)
            state[k] = q
            state[k[:-7] + ".weight_scale"] = s
            state[k[:-7] + ".comfy_quant"] = blob.clone()
        else:
            state[k] = v.detach().clone()
    return state


def _dequantized_reference(encoder, state):
    """Dense encoder whose projections are the int8 weights mapped back: W = (q * s) @ blockdiag(H)."""
    from core.inference.diffusion_convrot import build_convrot_hadamard

    h = build_convrot_hadamard(GROUP, dtype = torch.float32)
    sd = encoder.state_dict()
    for k in list(sd):
        if k + "_scale" in state:
            q, s = state[k].float(), state[k + "_scale"]
            o, i = q.shape
            sd[k] = ((q * s).reshape(o, i // GROUP, GROUP) @ h).reshape(o, i)
    encoder.load_state_dict(sd)
    return encoder


@pytest.fixture
def int8_file(tmp_path, monkeypatch):
    monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
    monkeypatch.setenv(ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
    import transformers

    monkeypatch.setattr(transformers.AutoConfig, "from_pretrained", lambda *_a, **_k: _config())
    encoder = _encoder(seed = 5)
    state = _int8_state(encoder)
    path = _container(tmp_path, INT8_NAME, state, tpq.TE_PREQUANT_FORMAT_INT8_CONVROT, "int8")
    return encoder, state, path


def _load(
    path,
    scheme = "int8",
    trim = True,
):
    return tpq.load_prequant_text_encoder(
        BASE,
        "text_encoder",
        tpq.TePrequantSource(kind = "path", location = str(path)),
        dtype = torch.float32,
        scheme = scheme,
        local_files_only = True,
        trim_lm_head = trim,
    )


def test_family_default_is_hosted_int8_and_needs_no_torchao():
    fam = get_family("qwen-image-2.1")
    mode, auto = resolve_te_quant_request(None, fam.te_quant_auto)
    assert (mode, auto) == ("int8", True)
    # No keep-bf16 schedule, so pre-load gates see the fp8 storage cast: no torchao, offload allowed.
    assert effective_te_quant(mode, fam.name) == "fp8"
    assert not te_quant_needs_resident_weights(effective_te_quant(mode, fam.name))


def test_int8_source_prefers_the_convrot_file_then_the_fp8_one():
    fam = get_family("qwen-image-2.1")
    sources = tpq.te_prequant_sources_for_base(fam, BASE, te_quant_mode = "int8", target = CUDA_BF16)
    src = sources["text_encoder"]
    assert (src.kind, src.location) == ("repo", REPO)
    assert tpq.te_candidate_filenames(src) == (
        INT8_NAME,
        FP8_NAME,
        "Qwen-Image-2.1-text_encoder-FP8.pt",
    )
    fp8 = tpq.te_prequant_sources(fam, te_quant_mode = "fp8", target = CUDA_BF16)["text_encoder"]
    assert INT8_NAME not in tpq.te_candidate_filenames(fp8)
    # Families without a hosted int8 encoder resolve nothing for int8, as before.
    assert tpq.te_prequant_sources(get_family("sdxl"), te_quant_mode = "int8", target = CUDA_BF16) == {}


def test_keeping_the_lm_head_resolves_the_fp8_file(monkeypatch):
    from core.inference.diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV

    monkeypatch.setenv(KEEP_LM_HEAD_ENV, "1")
    src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
    assert src.filename == FP8_NAME


def test_int8_file_loads_as_convrot_linears_and_matches_its_dequantized_weights(int8_file):
    from core.inference.video_minimax_h3_te import _int8_convrot_linear_class

    reference, state, path = int8_file
    encoder = _load(path)
    assert encoder is not None
    assert getattr(encoder, tpq.TE_PREQUANT_SCHEME_ATTR) == "int8"
    assert is_trimmed(encoder)
    cls = _int8_convrot_linear_class()
    layers = encoder.model.language_model.layers
    assert sum(isinstance(m, cls) for m in layers.modules()) == 2 * 7
    assert not any(isinstance(m, torch.nn.Linear) for m in layers.modules())
    assert all(m.weight.dtype == torch.int8 for m in layers.modules() if isinstance(m, cls))
    assert any(isinstance(m, torch.nn.Linear) for m in encoder.model.visual.modules())
    # Plain tensors only, so Module.to() and group offloading move it like any module.
    assert all(
        type(t) in (torch.Tensor, torch.nn.Parameter)
        for t in [*encoder.parameters(), *encoder.buffers()]
    )
    ids = torch.randint(0, VOCAB, (1, 9))
    expected = _hidden(_dequantized_reference(_encoder(seed = 5), state), ids)
    got = _hidden(encoder, ids)
    for a, b in zip(expected, got):
        torch.testing.assert_close(b, a, rtol = 1e-4, atol = 1e-4)
    dense = _hidden(reference, ids)[-1]
    cos = torch.nn.functional.cosine_similarity(dense.flatten(), got[-1].flatten(), dim = 0)
    assert cos > 0.99


def test_int8_file_is_refused_for_an_fp8_request_and_without_the_trim(int8_file):
    _, _, path = int8_file
    assert _load(path, scheme = "fp8") is None
    assert _load(path, trim = False) is None


def test_int8_request_falls_back_to_the_fp8_file_when_the_hub_404s(tmp_path, monkeypatch):
    from huggingface_hub.errors import EntryNotFoundError

    monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
    monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
    fp8 = tmp_path / FP8_NAME
    fp8.write_bytes(b"x")
    asked: list = []

    def fake_download(repo_id, filename, **_k):
        asked.append(filename)
        if filename == INT8_NAME:
            raise EntryNotFoundError(f"404 {filename}")
        return str(fp8)

    import huggingface_hub

    monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
    src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
    assert tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path)) == str(fp8)
    assert asked == [INT8_NAME, FP8_NAME]


def test_mirror_serves_the_int8_file_before_the_hub(tmp_path, monkeypatch):
    mirror = tmp_path / "mirror"
    target = mirror / REPO / INT8_NAME
    target.parent.mkdir(parents = True)
    target.write_bytes(b"x" * 7)
    monkeypatch.setenv(tpq.TE_PREQUANT_MIRROR_ENV, str(mirror))
    monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
    import huggingface_hub

    monkeypatch.setattr(
        huggingface_hub, "hf_hub_download", lambda *a, **k: pytest.fail("the Hub was asked")
    )
    src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
    assert tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path)) == str(target.resolve())

    class NoApi:
        def model_info(self, *a, **k):
            raise AssertionError("planning asked the Hub")

    assert tpq.te_prequant_hub_files({"text_encoder": src}, NoApi()) == {
        "text_encoder": [(INT8_NAME, 7)]
    }
    assert tpq.te_prequant_mirror_path(REPO, "../x") is None


def test_hub_plan_counts_the_file_the_repo_holds(monkeypatch):
    monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
    src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")

    def api(names):
        sib = [types.SimpleNamespace(rfilename = n, size = s) for n, s in names]
        return types.SimpleNamespace(model_info = lambda *a, **k: types.SimpleNamespace(siblings = sib))

    assert tpq.te_prequant_hub_files({"te": src}, api([(FP8_NAME, 9)])) == {"te": [(FP8_NAME, 9)]}
    assert tpq.te_prequant_hub_files({"te": src}, api([(FP8_NAME, 9), (INT8_NAME, 8)])) == {
        "te": [(INT8_NAME, 8)]
    }


def _pipe(encoder):
    return types.SimpleNamespace(text_encoder = encoder)


def test_hosted_int8_encoder_is_reported_and_never_recast(int8_file, monkeypatch):
    import core.inference.diffusion_precision as prec

    _, _, path = int8_file
    encoder = _load(path)
    monkeypatch.setattr(
        prec, "_cast_fp8", lambda *a, **k: pytest.fail("int8 encoder re-cast to fp8")
    )
    for offload in (False, True):
        out = quantize_text_encoders(
            _pipe(encoder), CUDA_BF16, mode = "int8", family = "qwen-image-2.1", offload_active = offload
        )
        assert (out.mode, out.status) == ("int8", RESOLVED_APPLIED)


def test_fp8_fallback_encoder_reports_why_int8_did_not_engage(monkeypatch):
    import core.inference.diffusion_precision as prec

    encoder = torch.nn.Linear(2, 2)
    setattr(encoder, tpq.TE_PREQUANT_SCHEME_ATTR, "fp8")
    cast: list = []
    monkeypatch.setattr(prec, "_cast_fp8", lambda enc, tgt: cast.append(enc))
    monkeypatch.setattr(prec, "te_quant_supported", lambda *a: True)
    out = quantize_text_encoders(_pipe(encoder), CUDA_BF16, mode = "int8", family = "qwen-image-2.1")
    assert (out.mode, out.status) == ("fp8", RESOLVED_FELL_BACK)
    assert "int8 ConvRot" in out.reason
    assert cast == [encoder]


def test_pipe_kwargs_loads_with_the_requested_scheme(monkeypatch):
    seen: list = []
    monkeypatch.setattr(
        tpq,
        "te_prequant_sources_for_base",
        lambda *a, **k: {"text_encoder": tpq.TePrequantSource(kind = "repo", location = REPO)},
    )
    monkeypatch.setattr(
        tpq, "load_prequant_text_encoder", lambda *a, **k: seen.append(k["scheme"]) or object()
    )
    for mode in ("int8", "fp8"):
        tpq.te_prequant_pipe_kwargs(
            get_family("qwen-image-2.1"), BASE, te_quant_mode = mode, target = CUDA_BF16, dtype = None
        )
    assert seen == ["int8", "fp8"]


def test_an_unreachable_hub_still_takes_the_cached_fp8_file(tmp_path, monkeypatch):
    """Online, int8 never cached, Hub down: the cached fp8 file loads instead of a dense pull."""
    from huggingface_hub.errors import LocalEntryNotFoundError

    monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
    monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
    fp8 = tmp_path / FP8_NAME
    fp8.write_bytes(b"x")
    asked: list = []

    def fake_download(
        repo_id,
        filename,
        local_files_only = False,
        **_k,
    ):
        asked.append((filename, local_files_only))
        if filename == FP8_NAME and local_files_only:
            return str(fp8)
        raise LocalEntryNotFoundError("connection error")

    import huggingface_hub

    monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
    src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
    assert tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path)) == str(fp8)
    assert asked == [(INT8_NAME, False), (FP8_NAME, True)]
    monkeypatch.setattr(
        huggingface_hub,
        "hf_hub_download",
        lambda *a, **k: (_ for _ in ()).throw(LocalEntryNotFoundError("connection error")),
    )
    with pytest.raises(LocalEntryNotFoundError):
        tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path))


def test_an_unreachable_hub_takes_the_fp8_file_from_the_other_cache_root(tmp_path, monkeypatch):
    """Outage with the fp8 file cached only under the default root (a moved cache setting)."""
    from huggingface_hub.errors import LocalEntryNotFoundError

    monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
    monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
    fp8 = tmp_path / "other" / FP8_NAME
    fp8.parent.mkdir()
    fp8.write_bytes(b"x")
    import core.inference.diffusion_prequant as dpq

    monkeypatch.setattr(
        dpq,
        "_cached_in_root",
        lambda _src, root, name = None: str(fp8) if root is None and name == FP8_NAME else None,
    )

    def fake_download(
        repo_id,
        filename,
        cache_dir = None,
        local_files_only = False,
        **_k,
    ):
        if filename == FP8_NAME and local_files_only and cache_dir is None:
            return str(fp8)
        raise LocalEntryNotFoundError("connection error")

    import huggingface_hub

    monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
    src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
    assert tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path / "live")) == str(fp8)


def test_an_int8_file_that_will_not_build_falls_back_to_the_fp8_names(monkeypatch):
    calls: list = []
    sentinel = object()
    src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
    monkeypatch.setattr(tpq, "te_prequant_sources_for_base", lambda *a, **k: {"text_encoder": src})

    def fake_load(base, component, source, **k):
        calls.append((source.filename, k["scheme"]))
        return None if source.filename == INT8_NAME else sentinel

    monkeypatch.setattr(tpq, "load_prequant_text_encoder", fake_load)
    for held, expected in ((True, sentinel), (False, None)):
        calls.clear()
        monkeypatch.setattr(tpq, "_held_locally", lambda *a, held = held: held)
        out = tpq.te_prequant_pipe_kwargs(
            get_family("qwen-image-2.1"), BASE, te_quant_mode = "int8", target = CUDA_BF16, dtype = None
        )
        assert out.get("text_encoder") is expected
        # Only a present-but-unbuildable int8 file earns an fp8 retry; a 404 already fell through.
        assert calls == (
            [(INT8_NAME, "int8"), (FP8_NAME, "fp8")] if held else [(INT8_NAME, "int8")]
        )


def test_keep_lm_head_reason_names_the_switch(monkeypatch):
    import core.inference.diffusion_precision as prec
    from core.inference.diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV

    monkeypatch.setenv(KEEP_LM_HEAD_ENV, "1")
    monkeypatch.setattr(prec, "_cast_fp8", lambda enc, tgt: None)
    monkeypatch.setattr(prec, "te_quant_supported", lambda *a: True)
    out = quantize_text_encoders(
        _pipe(torch.nn.Linear(2, 2)), CUDA_BF16, mode = "int8", family = "qwen-image-2.1"
    )
    assert out.mode == "fp8" and KEEP_LM_HEAD_ENV in out.reason


def _te_gate_mps(monkeypatch, family, te_mode):
    import core.inference.diffusion as d
    from core.inference.diffusion_families import _FAMILIES

    monkeypatch.delenv("UNSLOTH_TE_KEEP_LM_HEAD", raising = False)
    fam = next(f for f in _FAMILIES if f.name == family)
    d.DiffusionBackend._assert_precision_for_target(
        None,
        fam,
        types.SimpleNamespace(device = "mps", dtype = torch.bfloat16),
        model_kind = "pipeline",
        pinned = None,
        te_mode = te_mode,
        memory_mode = None,
        cpu_offload = False,
    )


@pytest.mark.parametrize("family", ["flux.3-klein", "flux.3-klein-base", "qwen-image-2.1"])
def test_preflight_admits_the_hosted_int8_encoder_on_mps(monkeypatch, family):
    # effective_te_quant rewrites int8 to fp8 on a family with no keep-bf16 schedule, and MPS has no fp8, but these
    # families host a plain-tensor int8 ConvRot file that int8_convrot_te_runs_on enables there. The preflight
    # confirms the file is published (the Hub listing, stubbed here) before it admits the load.
    import types

    import huggingface_hub

    import core.inference.diffusion_te_prequant as tpq

    names = [e[1] for comps in tpq.TE_INT8_CONVROT_FILES.values() for e in comps.values()]

    class _Api:
        def model_info(
            self,
            repo,
            files_metadata = False,
        ):
            return types.SimpleNamespace(
                siblings = [types.SimpleNamespace(rfilename = n, size = 1) for n in names]
            )

    monkeypatch.setattr(huggingface_hub, "HfApi", lambda *a, **k: _Api())
    monkeypatch.setattr(tpq, "_held_locally", lambda *a, **k: False)
    monkeypatch.setattr(tpq, "_hub_offline", lambda: False)
    tpq._confirmed_cache.clear()
    _te_gate_mps(monkeypatch, family, "int8")


def test_preflight_still_refuses_int8_on_mps_without_a_hosted_file(monkeypatch):
    import core.inference.diffusion_te_prequant as tpq

    # Every family with a hosted encoder now has an int8 file too, so take one away.
    monkeypatch.delitem(tpq.TE_INT8_CONVROT_FILES, "flux.2-klein")
    with pytest.raises(RuntimeError, match = "text_encoder_quant"):
        _te_gate_mps(monkeypatch, "flux.2-klein", "int8")
    with pytest.raises(RuntimeError, match = "text_encoder_quant"):
        _te_gate_mps(monkeypatch, "flux.3-klein", "fp8")


def test_preflight_refuses_int8_on_mps_when_the_lm_head_is_kept(monkeypatch):
    # UNSLOTH_TE_KEEP_LM_HEAD resolves int8 to the fp8 file, which MPS cannot run.
    import core.inference.diffusion as d
    from core.inference.diffusion_families import _FAMILIES
    from core.inference.diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV

    monkeypatch.setenv(KEEP_LM_HEAD_ENV, "1")
    fam = next(f for f in _FAMILIES if f.name == "flux.3-klein")
    with pytest.raises(RuntimeError, match = "text_encoder_quant"):
        d.DiffusionBackend._assert_precision_for_target(
            None,
            fam,
            types.SimpleNamespace(device = "mps", dtype = torch.bfloat16),
            model_kind = "pipeline",
            pinned = None,
            te_mode = "int8",
            memory_mode = None,
            cpu_offload = False,
        )