Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import sys
import types
import pytest
torch = pytest.importorskip("torch")
from torch import nn
import core.inference.diffusion_transformer_quant as tq
from core.inference.diffusion_convrot import CONVROT_ATTR, is_rotated_linear
def candidate_filenames_of(source):
"""Studio's own containers in resolver order; the ComfyUI-format twins have their own tests."""
from core.inference.diffusion_prequant import candidate_filenames_of as names_of
from core.inference.diffusion_prequant import is_comfy_prequant_filename
return tuple(n for n in names_of(source) if not is_comfy_prequant_filename(n))
@pytest.fixture(autouse = True)
def _convrot_opted_in(monkeypatch):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, "1")
class _Attn(nn.Module):
def __init__(self, dim):
super().__init__()
self.to_q = nn.Linear(dim, dim, bias = False)
self.to_k = nn.Linear(dim, dim, bias = False)
self.to_v = nn.Linear(dim, dim, bias = False)
self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias = False)])
def forward(self, x):
return self.to_out[0](self.to_q(x) * self.to_k(x).sigmoid() + self.to_v(x))
class _Mlp(nn.Module):
def __init__(self, dim):
super().__init__()
self.gate_layer = nn.Linear(dim, 3 * dim, bias = False)
self.out = nn.Linear(3 * dim, dim, bias = False)
def forward(self, x):
return self.out(torch.nn.functional.silu(self.gate_layer(x)))
class _Block(nn.Module):
def __init__(self, dim):
super().__init__()
self.attn = _Attn(dim)
self.img_mlp = _Mlp(dim)
self.extra = nn.Linear(dim, dim, bias = False)
def forward(self, x):
x = x + self.attn(x)
return self.extra(x + self.img_mlp(x))
class _Tiny(nn.Module):
def __init__(self, dim = 512):
super().__init__()
self.txt_in = nn.Linear(dim, dim, bias = False)
self.small = nn.Linear(dim, 64, bias = False)
self.odd = nn.Linear(640, dim, bias = False)
self.transformer_blocks = nn.ModuleList([_Block(dim) for _ in range(2)])
def forward(self, x, y):
h = self.txt_in(x) + self.odd(y)
for b in self.transformer_blocks:
h = b(h)
return h, self.small(h)
_ROTATED = {
f"transformer_blocks.{i}.{n}"
for i in range(2)
for n in (
"attn.to_q",
"attn.to_k",
"attn.to_v",
"attn.to_out.0",
"img_mlp.gate_layer",
"img_mlp.out",
)
}
def _filter(family):
return tq.make_filter_fn(
512, exclude_name_tokens = tq.exclude_tokens_for_scheme(tq.TQ_INT8, family) + ("lora_",)
)
def test_convrot_spec_only_for_int8_on_declared_families():
group, suffixes = tq.convrot_spec_for_scheme(tq.TQ_INT8, "qwen-image-2.1")
assert group == 256 and set(suffixes) == {
"attn.to_q",
"attn.to_k",
"attn.to_v",
"attn.to_out.0",
"img_mlp.gate_layer",
"img_mlp.proj",
"img_mlp.out",
}
assert tq.convrot_spec_for_scheme(tq.TQ_INT8, " Qwen-Image-2.1 ")[0] == 256
for scheme in (tq.TQ_FP8, tq.TQ_NVFP4, tq.TQ_MXFP8):
assert tq.convrot_spec_for_scheme(scheme, "qwen-image-2.1") == (0, ())
for family in (None, "qwen-image", "qwen-image-edit", "flux.1", "minimax-h3"):
assert tq.convrot_spec_for_scheme(tq.TQ_INT8, family) == (0, ())
def test_int8_artifact_names_the_rotated_build_first_and_keeps_the_plain_one():
from core.inference.diffusion_families import detect_family
from core.inference.diffusion_prequant import resolve_prequant_source
fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")
names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))
assert names[:2] == (
"Qwen-Image-2.1-INT8-ConvRot.safetensors",
"Qwen-Image-2.1-INT8.safetensors",
)
assert (
candidate_filenames_of(resolve_prequant_source(fam, "fp8"))[0]
== "Qwen-Image-2.1-FP8.safetensors"
)
def test_with_the_kill_switch_int8_is_plain_everywhere(monkeypatch):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, "0")
from core.inference.diffusion_families import detect_family
from core.inference.diffusion_prequant import resolve_prequant_source
fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")
names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))
assert names[0] == "Qwen-Image-2.1-INT8.safetensors"
assert "Qwen-Image-2.1-INT8-ConvRot.safetensors" not in names
model = _Tiny()
assert (
tq.apply_runtime_convrot(model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"))
== ()
)
assert not any(is_rotated_linear(m) for m in model.modules())
assert not hasattr(model, CONVROT_ATTR)
for off in ("0", "", "off", "false"):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, off)
assert not tq.int8_convrot_enabled()
def test_quantize_transformer_with_the_kill_switch_rotates_nothing(monkeypatch):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, "0")
seen = []
_stub_torchao(monkeypatch, seen)
monkeypatch.setattr(
tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_INT8
)
pipe = types.SimpleNamespace(transformer = _Tiny())
assert (
tq.quantize_transformer(pipe, object(), mode = "int8", family = "qwen-image-2.1") == tq.TQ_INT8
)
assert seen == [[]]
def test_convrot_does_not_touch_the_exclusion_set():
assert tq.exclude_tokens_for_scheme(
tq.TQ_INT8, "qwen-image-2.1"
) == tq._INT8_EXCLUDE_NAME_TOKENS + ("txt_in",)
def test_runtime_convrot_rotates_the_quantized_set_and_keeps_the_model_exact():
torch.manual_seed(0)
model = _Tiny().float()
x, y = torch.randn(4, 512), torch.randn(4, 640)
with torch.no_grad():
ref = model(x, y)
rotated = tq.apply_runtime_convrot(
model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1")
)
got = model(x, y)
assert set(rotated) == _ROTATED
assert not is_rotated_linear(model.txt_in)
assert not is_rotated_linear(model.small)
assert not is_rotated_linear(model.odd)
assert not any(is_rotated_linear(blk.extra) for blk in model.transformer_blocks)
assert getattr(model, CONVROT_ATTR)["linears"] == 12
for a, b in zip(ref, got):
torch.testing.assert_close(a, b, rtol = 1e-4, atol = 1e-4)
def test_runtime_convrot_is_inert_elsewhere():
model = _Tiny()
assert tq.apply_runtime_convrot(model, tq.TQ_INT8, "qwen-image", _filter("qwen-image")) == ()
assert (
tq.apply_runtime_convrot(model, tq.TQ_FP8, "qwen-image-2.1", _filter("qwen-image-2.1"))
== ()
)
assert not any(is_rotated_linear(m) for m in model.modules())
assert not hasattr(model, CONVROT_ATTR)
def _stub_torchao(monkeypatch, seen):
tqz = types.ModuleType("torchao.quantization")
def quantize_(
module,
config,
filter_fn = None,
):
seen.append(
sorted(n for n, m in module.named_modules() if is_rotated_linear(m) and filter_fn(m, n))
)
tqz.quantize_ = quantize_
tqz.Int8DynamicActivationInt8WeightConfig = lambda **kw: "int8-cfg"
tqz.Float8DynamicActivationFloat8WeightConfig = lambda **kw: "fp8-cfg"
tqz.PerRow = lambda: "per-row"
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
monkeypatch.setattr(tq, "_make_quant_config", lambda scheme, fast_accum = None: f"{scheme}-cfg")
@pytest.mark.parametrize(
"family, expect_rotated", [("qwen-image-2.1", 12), ("qwen-image", 0), (None, 0)]
)
def test_quantize_transformer_rotates_before_quantize(monkeypatch, family, expect_rotated):
seen = []
_stub_torchao(monkeypatch, seen)
monkeypatch.setattr(
tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_INT8
)
pipe = types.SimpleNamespace(transformer = _Tiny())
assert tq.quantize_transformer(pipe, object(), mode = "int8", family = family) == tq.TQ_INT8
assert len(seen) == 1 and len(seen[0]) == expect_rotated
def test_quantize_transformer_leaves_fp8_unrotated(monkeypatch):
seen = []
_stub_torchao(monkeypatch, seen)
monkeypatch.setattr(
tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_FP8
)
pipe = types.SimpleNamespace(transformer = _Tiny())
assert tq.quantize_transformer(pipe, object(), mode = "fp8", family = "qwen-image-2.1") == tq.TQ_FP8
assert seen == [[]]
def test_runtime_convrot_warms_the_hadamard_for_the_target_device():
from core.inference import diffusion_convrot as cr
cr._HADAMARD_CACHE.clear()
model = _Tiny()
target = types.SimpleNamespace(device = "cuda", torch_device = "cpu", dtype = torch.bfloat16)
tq.apply_runtime_convrot(
model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"), target = target
)
assert (256, "cpu", torch.bfloat16) in cr._HADAMARD_CACHE
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "compiles a CUDA graph")
def test_runtime_convrot_adds_no_recompile():
from torch._dynamo.utils import counters
from core.inference import diffusion_convrot as cr
cr._HADAMARD_CACHE.clear()
torch._dynamo.reset()
counters.clear()
model = _Tiny().to(torch.bfloat16)
target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
tq.apply_runtime_convrot(
model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"), target = target
)
block = torch.compile(model.cuda().transformer_blocks[0], fullgraph = True)
x = torch.randn(4, 512, device = "cuda", dtype = torch.bfloat16)
for _ in range(3):
block(x)
assert counters["stats"]["unique_graphs"] == 1
def test_unreachable_hub_still_loads_the_plain_artifact_already_cached(monkeypatch, tmp_path):
from huggingface_hub.errors import LocalEntryNotFoundError
from core.inference import diffusion_prequant as pq
from core.inference.diffusion_families import detect_family
plain = tmp_path / "Qwen-Image-2.1-INT8.safetensors"
plain.write_bytes(b"weights")
monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)
monkeypatch.setattr(
"huggingface_hub.try_to_load_from_cache",
lambda repo_id, filename, cache_dir = None, **k: str(tmp_path / filename)
if (tmp_path / filename).is_file()
else None,
)
asked = []
def _dl(
repo_id,
filename,
token = None,
cache_dir = None,
local_files_only = False,
):
asked.append(filename)
raise LocalEntryNotFoundError("connection error")
monkeypatch.setattr("huggingface_hub.hf_hub_download", _dl)
source = pq.resolve_prequant_source(
detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1"), "int8"
)
assert pq._resolve_checkpoint_path(source, None, None) == str(plain)
# Only the first name, the ComfyUI twin, is asked; the unreachable Hub answers from cache.
assert asked == ["Qwen-Image-2.1-INT8-ConvRot-ComfyUI.safetensors"]
plain.unlink()
with pytest.raises(LocalEntryNotFoundError):
pq._resolve_checkpoint_path(source, None, None)
def test_builder_publishes_rotated_and_plain_int8_under_different_names():
import importlib.util
from pathlib import Path
from core.inference.diffusion_families import detect_family
script = Path(__file__).resolve().parents[3] / "scripts" / "build_prequant_checkpoint.py"
spec = importlib.util.spec_from_file_location("_build_prequant_for_convrot_test", script)
build = importlib.util.module_from_spec(spec)
spec.loader.exec_module(build)
fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")
repo = "unsloth/Qwen-Image-2.1-FP8"
dest = lambda rotated: build.upload_destination(
fam, "int8", rotated = rotated, safetensors = True, upload_repo = repo
)
assert dest(True) == "Qwen-Image-2.1-INT8-ConvRot.safetensors"
assert dest(False) == "Qwen-Image-2.1-INT8.safetensors"
spec_build = build.upload_destination(
fam, "int8", rotated = True, safetensors = True, upload_repo = repo, convrot_group = 256
)
assert spec_build == "Qwen-Image-2.1-INT8-ConvRot.safetensors"
with pytest.raises(ValueError, match = "reserved"):
build.upload_destination(
fam, "int8", rotated = True, safetensors = True, upload_repo = repo, convrot_group = 64
)
assert (
build.upload_destination(fam, "fp8", rotated = False, safetensors = True, upload_repo = repo)
== "Qwen-Image-2.1-FP8.safetensors"
)
class _LoraLike(nn.Module):
def __init__(self, base):
super().__init__()
self.base_layer = base
self.lora_A = nn.Linear(base.in_features, 4, bias = False)
self.lora_B = nn.Linear(4, base.out_features, bias = False)
def forward(self, x):
return self.base_layer(x) + self.lora_B(self.lora_A(x))
def _check_base_layers_rotated(model):
torch.manual_seed(1)
x, y = torch.randn(4, 512), torch.randn(4, 640)
with torch.no_grad():
ref = model(x, y)
rotated = tq.apply_runtime_convrot(
model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1")
)
got = model(x, y)
for i in range(2):
for n in ("attn.to_q", "attn.to_v", "img_mlp.out"):
assert f"transformer_blocks.{i}.{n}.base_layer" in rotated
assert len(rotated) == 12 and not any("lora_" in f for f in rotated)
for a, b in zip(ref, got):
torch.testing.assert_close(a, b, rtol = 1e-4, atol = 1e-4)
def test_lora_baked_targets_rotate_their_base_layer_and_stay_exact():
# Adapters attach before quantize_, so a LoRA target is '<suffix>.base_layer'.
torch.manual_seed(0)
model = _Tiny().float()
for blk in model.transformer_blocks:
blk.attn.to_q = _LoraLike(blk.attn.to_q)
blk.attn.to_v = _LoraLike(blk.attn.to_v)
blk.img_mlp.out = _LoraLike(blk.img_mlp.out)
_check_base_layers_rotated(model)
@pytest.fixture
def _fresh_peft_torchao_probe():
# peft caches is_torchao_available(); a cached value leaks into later tests that block torchao.
is_torchao_available = pytest.importorskip("peft.import_utils").is_torchao_available
is_torchao_available.cache_clear()
yield
is_torchao_available.cache_clear()
def test_real_peft_adapters_rotate_their_base_layer_and_stay_exact(_fresh_peft_torchao_probe):
peft = pytest.importorskip("peft")
torch.manual_seed(0)
cfg = peft.LoraConfig(
r = 4, lora_alpha = 8, target_modules = ["to_q", "to_v", "out"], init_lora_weights = False
)
try:
model = peft.inject_adapter_in_model(cfg, _Tiny().float())
except (
ImportError
) as exc: # peft < 0.19 with torchao >= 0.18, patched only once unsloth is imported
pytest.skip(f"peft cannot dispatch LoRA on this torchao: {exc}")
_check_base_layers_rotated(model)
_ZIMAGE_SUFFIXES = {
"attention.to_q",
"attention.to_k",
"attention.to_v",
"attention.to_out.0",
"feed_forward.w1",
"feed_forward.w2",
"feed_forward.w3",
"cap_embedder.1",
}
def test_zimage_int8_convrot_spec_and_artifact_name():
group, suffixes = tq.convrot_spec_for_scheme(tq.TQ_INT8, "z-image")
assert group == 256 and set(suffixes) == _ZIMAGE_SUFFIXES
assert (
tq.convrot_prequant_filename(tq.TQ_INT8, "z-image")
== "Z-Image-Turbo-INT8-ConvRot.safetensors"
)
for scheme in (tq.TQ_FP8, tq.TQ_NVFP4, tq.TQ_MXFP8):
assert tq.convrot_spec_for_scheme(scheme, "z-image") == (0, ())
assert tq.convrot_prequant_filename(scheme, "z-image") is None
def test_zimage_convrot_is_default_on_and_the_env_is_its_kill_switch(monkeypatch):
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
assert tq.int8_convrot_enabled("z-image") and tq.int8_convrot_enabled(" Z-Image ")
# every other family keeps the opt-in it had
for family in (None, "qwen-image", "flux.1", "minimax-h3"):
assert not tq.int8_convrot_enabled(family)
for off in ("0", "off", "false", "no", " OFF "):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, off)
assert not tq.int8_convrot_enabled("z-image")
assert not tq.int8_convrot_enabled("qwen-image-2.1")
monkeypatch.setenv(tq.INT8_CONVROT_ENV, "1")
assert tq.int8_convrot_enabled("qwen-image-2.1") and tq.int8_convrot_enabled("z-image")
@pytest.mark.parametrize("env, rotated_first", [(None, True), ("0", False), ("1", True)])
def test_zimage_int8_resolves_the_rotated_artifact_first_unless_killed(
monkeypatch, env, rotated_first
):
from core.inference.diffusion_families import detect_family
from core.inference.diffusion_prequant import resolve_prequant_source
if env is None:
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
else:
monkeypatch.setenv(tq.INT8_CONVROT_ENV, env)
fam = detect_family("Tongyi-MAI/Z-Image-Turbo")
src = resolve_prequant_source(fam, "int8")
names = candidate_filenames_of(src)
assert src.location == "unsloth/Z-Image-Turbo-FP8"
assert "Z-Image-Turbo-INT8.pt" in names
assert (names[0] == "Z-Image-Turbo-INT8-ConvRot.safetensors") is rotated_first
assert ("Z-Image-Turbo-INT8-ConvRot.safetensors" in names) is rotated_first
assert "Z-Image-Turbo-INT8-ConvRot.safetensors" not in candidate_filenames_of(
resolve_prequant_source(fam, "fp8")
)
def test_zimage_rotated_set_is_exactly_the_int8_quantized_set(monkeypatch):
# ConvRot must cover exactly the Linears int8 quantizes; an unrotated one is a silent accuracy hole.
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")
from accelerate import init_empty_weights
with init_empty_weights():
model = zmod.ZImageTransformer2DModel(
all_patch_size = (2,),
all_f_patch_size = (1,),
in_channels = 16,
dim = 3840,
n_layers = 30,
n_refiner_layers = 2,
n_heads = 30,
n_kv_heads = 30,
norm_eps = 1e-5,
qk_norm = True,
cap_feat_dim = 2560,
rope_theta = 256.0,
t_scale = 1000.0,
axes_dims = [32, 48, 48],
axes_lens = [1024, 512, 512],
)
filt = _filter("z-image")
quantized = {n for n, m in model.named_modules() if isinstance(m, nn.Linear) and filt(m, n)}
group, suffixes = tq.convrot_spec_for_scheme(tq.TQ_INT8, "z-image")
rotated = set(tq.convrot_fqns(model, filt, group, suffixes))
assert len(quantized) == 239
assert rotated == quantized
def test_zimage_rotated_artifact_is_only_named_in_its_own_repo(monkeypatch):
import dataclasses
from core.inference.diffusion_families import detect_family
from core.inference.diffusion_prequant import resolve_prequant_source
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
fam = dataclasses.replace(
detect_family("Tongyi-MAI/Z-Image-Turbo"), prequant_repos = (("int8", "org/other-int8"),)
)
names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))
assert "Z-Image-Turbo-INT8-ConvRot.safetensors" not in names
def test_qwen_image_21_convrot_is_default_on_and_the_env_is_its_kill_switch(monkeypatch):
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
assert tq.int8_convrot_enabled("qwen-image-2.1") and tq.int8_convrot_enabled(" Qwen-Image-2.1 ")
for off in ("0", "off", "false", "no"):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, off)
assert not tq.int8_convrot_enabled("qwen-image-2.1")
@pytest.mark.parametrize("env, rotated_first", [(None, True), ("0", False), ("1", True)])
def test_qwen_image_21_int8_resolves_the_rotated_artifact_first_unless_killed(
monkeypatch, env, rotated_first
):
from core.inference.diffusion_families import detect_family
from core.inference.diffusion_prequant import (
candidate_filenames_of,
comfy_prequant_filename,
is_comfy_prequant_filename,
resolve_prequant_source,
)
if env is None:
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
else:
monkeypatch.setenv(tq.INT8_CONVROT_ENV, env)
for repo in ("Qwen/Qwen-Image-2.1", "unsloth/Qwen-Image-2.1"):
fam = detect_family(repo)
assert fam is not None and fam.name == "qwen-image-2.1"
src = resolve_prequant_source(fam, "int8")
# -ComfyUI twins ride ahead of each artifact; compare Studio's own names
everything = candidate_filenames_of(src)
names = tuple(n for n in everything if not is_comfy_prequant_filename(n))
assert everything[0] == comfy_prequant_filename(names[0])
assert src.location == "unsloth/Qwen-Image-2.1-FP8"
assert "Qwen-Image-2.1-INT8.safetensors" in names
assert (names[0] == "Qwen-Image-2.1-INT8-ConvRot.safetensors") is rotated_first
assert ("Qwen-Image-2.1-INT8-ConvRot.safetensors" in names) is rotated_first
assert "Qwen-Image-2.1-INT8-ConvRot.safetensors" not in candidate_filenames_of(
resolve_prequant_source(fam, "fp8")
)
def test_qwen_image_21_runtime_int8_rotates_by_default(monkeypatch):
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
seen = []
_stub_torchao(monkeypatch, seen)
monkeypatch.setattr(
tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_INT8
)
pipe = types.SimpleNamespace(transformer = _Tiny())
assert (
tq.quantize_transformer(pipe, object(), mode = "int8", family = "qwen-image-2.1") == tq.TQ_INT8
)
assert len(seen) == 1 and set(seen[0]) == _ROTATED
def test_qwen_image_21_offline_default_still_loads_the_cached_plain_artifact(monkeypatch, tmp_path):
from huggingface_hub.errors import LocalEntryNotFoundError
from core.inference import diffusion_prequant as pq
from core.inference.diffusion_families import detect_family
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
plain = tmp_path / "Qwen-Image-2.1-INT8.safetensors"
plain.write_bytes(b"weights")
monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)
monkeypatch.setattr(
"huggingface_hub.try_to_load_from_cache",
lambda repo_id, filename, cache_dir = None, **k: str(tmp_path / filename)
if (tmp_path / filename).is_file()
else None,
)
asked = []
def _dl(
repo_id,
filename,
token = None,
cache_dir = None,
local_files_only = False,
):
asked.append(filename)
raise LocalEntryNotFoundError("connection error")
monkeypatch.setattr("huggingface_hub.hf_hub_download", _dl)
source = pq.resolve_prequant_source(detect_family("Qwen/Qwen-Image-2.1"), "int8")
assert pq._resolve_checkpoint_path(source, None, None) == str(plain)
assert asked == ["Qwen-Image-2.1-INT8-ConvRot-ComfyUI.safetensors"]
def test_qwen_image_21_rotated_set_is_exactly_the_int8_quantized_set(monkeypatch):
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
diffusers = pytest.importorskip("diffusers")
cls = getattr(diffusers, "QwenImage21Transformer2DModel", None)
if cls is None:
pytest.skip(
reason = "installed diffusers has no QwenImage21Transformer2DModel to build the skeleton from"
)
from accelerate import init_empty_weights
with init_empty_weights():
model = cls()
filt = _filter("qwen-image-2.1")
quantized = {n for n, m in model.named_modules() if isinstance(m, nn.Linear) and filt(m, n)}
group, suffixes = tq.convrot_spec_for_scheme(tq.TQ_INT8, "qwen-image-2.1")
rotated = set(tq.convrot_fqns(model, filt, group, suffixes))
assert quantized and rotated == quantized
# FLUX.2 klein (4B) and FLUX.3 klein Linear layouts (fqn, in, out), from the published diffusers checkpoints.
_FLUX2_KLEIN_LINEARS = (
[("context_embedder", 7680, 3072), ("x_embedder", 128, 3072), ("proj_out", 3072, 128)]
+ [("norm_out.linear", 3072, 6144), ("double_stream_modulation_img.linear", 3072, 18432)]
+ [
("double_stream_modulation_txt.linear", 3072, 18432),
("single_stream_modulation.linear", 3072, 9216),
]
+ [("time_guidance_embed.timestep_embedder.linear_1", 256, 3072)]
+ [("time_guidance_embed.timestep_embedder.linear_2", 3072, 3072)]
+ [
(f"transformer_blocks.{i}.{n}", k, o)
for i in range(5)
for n, k, o in (
("attn.to_q", 3072, 3072),
("attn.to_k", 3072, 3072),
("attn.to_v", 3072, 3072),
("attn.to_out.0", 3072, 3072),
("attn.add_q_proj", 3072, 3072),
("attn.add_k_proj", 3072, 3072),
("attn.add_v_proj", 3072, 3072),
("attn.to_add_out", 3072, 3072),
("ff.linear_in", 3072, 18432),
("ff.linear_out", 9216, 3072),
("ff_context.linear_in", 3072, 18432),
("ff_context.linear_out", 9216, 3072),
)
]
+ [
(f"single_transformer_blocks.{i}.{n}", k, o)
for i in range(20)
for n, k, o in (("attn.to_qkv_mlp_proj", 3072, 27648), ("attn.to_out", 12288, 3072))
]
)
_FLUX3_BLOCK = (
("attn.to_q", 3072, 3072),
("attn.to_k", 3072, 3072),
("attn.to_v", 3072, 3072),
("attn.to_out.0", 3072, 3072),
("ff.linear_in", 3072, 18432),
("ff.linear_out", 9216, 3072),
)
_FLUX3_LINEARS = (
[("context_embedder", 20480, 3072), ("proj_out", 3072, 128), ("norm_out.linear", 3072, 6144)]
+ [("x_embedders.image", 128, 3072), ("x_embedders.image_cond", 128, 3072)]
+ [("timestep_embedder.linear_1", 256, 3072), ("timestep_embedder.linear_2", 3072, 3072)]
+ [
(f"{s}_stream_modulations.{m}.linear", 3072, 9216)
for s in ("early", "single")
for m in ("image", "image_cond", "txt")
]
+ [
(f"mode_blocks.{m}.{i}.{n}", k, o)
for m in ("image", "image_cond")
for i in range(5)
for n, k, o in _FLUX3_BLOCK
]
+ [(f"txt_mode_blocks.{i}.{n}", k, o) for i in range(5) for n, k, o in _FLUX3_BLOCK]
+ [(f"single_transformer_blocks.{i}.{n}", k, o) for i in range(28) for n, k, o in _FLUX3_BLOCK]
)
def _skeleton(linears):
"""Meta-device module tree with exactly these Linear fqns and shapes."""
root = nn.Module()
for fqn, k, o in linears:
parent = root
*path, leaf = fqn.split(".")
for part in path:
if not hasattr(parent, part):
parent.add_module(part, nn.Module())
parent = getattr(parent, part)
parent.add_module(leaf, nn.Linear(k, o, bias = True, device = "meta"))
return root
@pytest.mark.parametrize(
"family, linears, expect",
[
("flux.2-klein", _FLUX2_KLEIN_LINEARS, 101),
("flux.3-klein", _FLUX3_LINEARS, 228),
("flux.3-klein-base", _FLUX3_LINEARS, 228),
],
)
def test_flux_rotated_set_is_exactly_the_int8_quantized_set(monkeypatch, family, linears, expect):
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
model = _skeleton(linears)
filt = _filter(family)
quantized = {n for n, m in model.named_modules() if isinstance(m, nn.Linear) and filt(m, n)}
group, suffixes = tq.convrot_spec_for_scheme(tq.TQ_INT8, family)
assert group == 256
rotated = set(tq.convrot_fqns(model, filt, group, suffixes))
assert len(quantized) == expect
assert rotated == quantized
# FLUX.3's keep-list (text-stream blocks, modulations, context_embedder) stays dense and unrotated
assert (
not any(n.startswith(("txt_mode_blocks.", "context_embedder")) for n in rotated)
or family == "flux.2-klein"
)
@pytest.mark.parametrize("family", ["flux.2-klein", "flux.3-klein", "flux.3-klein-base"])
def test_flux_convrot_is_default_on_and_the_env_is_its_kill_switch(monkeypatch, family):
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
assert tq.int8_convrot_enabled(family)
for off in ("0", "off", "false", "no"):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, off)
assert not tq.int8_convrot_enabled(family)
model = _skeleton(_FLUX2_KLEIN_LINEARS if family == "flux.2-klein" else _FLUX3_LINEARS)
assert tq.apply_runtime_convrot(model, tq.TQ_INT8, family, _filter(family)) == ()
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
for other in ("flux.2-dev", "flux.1", "flux.1-kontext"):
assert tq.convrot_spec_for_scheme(tq.TQ_INT8, other) == (0, ())
assert not tq.int8_convrot_enabled(other)
@pytest.mark.parametrize(
"repo, hosted, rotated",
[
(
"black-forest-labs/FLUX.2-klein-4B",
"unsloth/FLUX.2-klein-4B-FP8",
"FLUX.2-klein-4B-INT8-ConvRot.safetensors",
),
(
"black-forest-labs/flux-3-image-klein",
"unsloth/flux-3-image-klein-FP8",
"flux-3-image-klein-INT8-ConvRot.safetensors",
),
],
)
def test_flux_int8_reads_only_the_rotated_hosted_file_unless_killed(
monkeypatch, repo, hosted, rotated
):
# Default names only the rotated file; the kill switch names only the unrotated one.
from core.inference import diffusion_prequant as pq
from core.inference import diffusion_prequant_cache as pqc
from core.inference.diffusion_families import detect_family
monkeypatch.setattr(pqc, "lookup", lambda *a, **k: None)
fam = detect_family(repo)
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
assert tq.convrot_drops_plain_hosted(tq.TQ_INT8, fam.name)
assert not tq.convrot_drops_plain_hosted(tq.TQ_FP8, fam.name)
source = pq.resolve_prequant_source(fam, "int8")
assert source.kind == "repo" and source.location == hosted
assert candidate_filenames_of(source) == (rotated,)
monkeypatch.setenv(tq.INT8_CONVROT_ENV, "0")
plain = pq.resolve_prequant_source(fam, "int8")
assert plain.kind == "repo" and plain.location == hosted
names = candidate_filenames_of(plain)
assert names and not any("ConvRot" in n for n in names)
for env in ("0", "1"):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, env)
fp8 = pq.resolve_prequant_source(fam, "fp8")
assert fp8.kind == "repo" and not any("ConvRot" in n for n in candidate_filenames_of(fp8))
def test_flux_missing_rotated_file_falls_back_to_the_rotated_load_time_build(monkeypatch, tmp_path):
# Before the rotated file is published: the hub 404s, the plain file is never fetched, the load quantizes.
from huggingface_hub.errors import EntryNotFoundError
from core.inference import diffusion_prequant as pq
from core.inference.diffusion_families import detect_family
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)
monkeypatch.setattr("huggingface_hub.try_to_load_from_cache", lambda *a, **k: None)
asked = []
def _dl(
repo_id,
filename,
token = None,
cache_dir = None,
local_files_only = False,
):
asked.append(filename)
raise EntryNotFoundError("404")
monkeypatch.setattr("huggingface_hub.hf_hub_download", _dl)
source = pq.resolve_prequant_source(detect_family("black-forest-labs/FLUX.2-klein-4B"), "int8")
with pytest.raises(EntryNotFoundError):
pq._resolve_checkpoint_path(source, None, None)
assert asked and all("ConvRot" in n for n in asked)
def test_flux3_base_keeps_its_hosted_rotated_int8(monkeypatch):
# BFL's hosted int8_convrot denoiser is already rotated: still read by default
from core.inference import diffusion_prequant as pq
from core.inference.diffusion_families import detect_family
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
fam = detect_family("black-forest-labs/flux-3-image-klein-base")
assert fam.name == "flux.3-klein-base"
assert not tq.convrot_drops_plain_hosted(tq.TQ_INT8, fam.name)
source = pq.resolve_prequant_source(fam, "int8")
assert source is not None and source.kind == "repo"
assert "flux-3-image-klein-base-INT8.safetensors" in candidate_filenames_of(source)
def test_hosted_plain_skip_is_only_on_the_flux_klein_families():
assert tq._INT8_FAMILY_CONVROT_HOSTED_PLAIN == frozenset({"flux.2-klein", "flux.3-klein"})
def test_flux_klein_runtime_int8_rotates_by_default(monkeypatch):
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
seen = []
_stub_torchao(monkeypatch, seen)
monkeypatch.setattr(
tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_INT8
)
small = [
(n, 512 if k >= 512 else k, 512 if o >= 512 else o) for n, k, o in _FLUX2_KLEIN_LINEARS
]
model = _skeleton(small).to_empty(device = "cpu")
pipe = types.SimpleNamespace(transformer = model)
assert tq.quantize_transformer(pipe, object(), mode = "int8", family = "flux.2-klein") == tq.TQ_INT8
assert len(seen) == 1 and len(seen[0]) == 101
assert getattr(model, CONVROT_ATTR)["linears"] == 101