Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: opt-in Hadamard rotation for Qwen-Image-2.1's int8 transformer so it matches bf16 (#11835)
Daniel Hanb33abbcc964e
diff --git a/scripts/build_prequant_checkpoint.py b/scripts/build_prequant_checkpoint.pyindex bbdb1cc7122..3906130f3a3 100644--- a/scripts/build_prequant_checkpoint.py+++ b/scripts/build_prequant_checkpoint.py@@ -67,6 +67,7 @@ def upload_destination(override: Optional[str] = None,upload_repo: Optional[str] = None,component: str = DEFAULT_COMPONENT,+ convrot_group: Optional[int] = None,) -> str:"""The repo-root filename this build should publish under.@@ -124,8 +125,21 @@ def upload_destination(if not rotated and not safetensors:return prequant_filename(scheme)from core.inference.diffusion_families import family_prequant_filename+ from core.inference.diffusion_transformer_quant import (+ convrot_prequant_filename,+ convrot_spec_for_scheme,+ )- preferred = family_prequant_filename(fam, scheme)+ family = getattr(fam, "name", None)+ rotated_name = convrot_prequant_filename(scheme, family) if rotated else None+ if rotated_name:+ spec_group = convrot_spec_for_scheme(scheme, family)[0]+ if convrot_group is not None and int(convrot_group) != spec_group:+ raise ValueError(+ f"{rotated_name} is reserved for the ConvRot group {spec_group} build that matches "+ f"the runtime path; a group {convrot_group} build needs --upload-filename."+ )+ preferred = rotated_name or family_prequant_filename(fam, scheme)why = "a rotated checkpoint" if rotated else "a safetensors checkpoint"if not preferred:# A PLAIN safetensors build now has a derived name, and only because@@ -208,7 +222,9 @@ def main(argv = None) -> int:type = int,default = 0,help = "bake a ConvRot block-Hadamard activation rotation at this group size (a power of "- "4; 0 = off). Every quantized Linear whose in_features the group divides has its "+ "4; 0 = off). A family with an int8 ConvRot spec at this group (Qwen-Image-2.1: "+ "256) rotates exactly the Linears that spec names, so the artifact matches the opt-in "+ "runtime path (UNSLOTH_DIFFUSION_INT8_CONVROT=1); otherwise every quantized Linear whose in_features the group divides has its ""weight rotated before quantize_ so the quantizer sees a flatter distribution; the ""exact fqn list is recorded in the checkpoint and the loader rotates the ""activations of that list and nothing else. Writes the v2 format tag.",@@ -240,6 +256,8 @@ def main(argv = None) -> int:TQ_SCHEMES,_make_quant_config,_resolve_fast_accum,+ convrot_fqns,+ convrot_spec_for_scheme,make_filter_fn,)from torchao.quantization import quantize_@@ -253,6 +271,9 @@ def main(argv = None) -> int:if fam is None:print(f"error: unknown family '{args.family}'", flush = True)return 2+ convrot_group = int(args.convrot_groupsize)+ spec_group, spec_suffixes = convrot_spec_for_scheme(scheme, fam.name)+ convrot_suffixes: tuple = spec_suffixes if convrot_group and convrot_group == spec_group else ()# What the artifact RECORDS as its base, which is not always what this build READ. Weights staged into a local# directory keep that directory's name, and the loader's ``_same_base_model`` compares final path segments: a# checkpoint built from ./temp/qwen_image_21 records a base whose tail is "qwen_image_21", the load asks for@@ -314,7 +335,8 @@ def main(argv = None) -> int:upload_dest = upload_destination(fam,scheme,- rotated = bool(args.convrot_groupsize),+ rotated = bool(convrot_group),+ convrot_group = convrot_group,safetensors = is_safetensors_out,override = args.upload_filename,upload_repo = args.upload_repo,@@ -346,15 +368,17 @@ def main(argv = None) -> int:# ConvRot, BEFORE quantize_: rotating the weights is only worth anything if the quantizer then sees the rotated# distribution. The fqn list is recorded, never re-derived at load time.rotation: dict = {}- if args.convrot_groupsize:+ if convrot_group:from core.inference.diffusion_convrot import (rotatable_fqns,rotate_linears_,rotation_metadata,)- group = int(args.convrot_groupsize)+ group = int(convrot_group)rotatable, not_divisible = rotatable_fqns(transformer, filter_fn, group)+ if convrot_suffixes:+ rotatable = convrot_fqns(transformer, filter_fn, group, convrot_suffixes)refusal = convrot_refusal(group, rotatable, not_divisible)if refusal:print(f"error: {refusal}", flush = True)diff --git a/studio/backend/core/inference/diffusion_convrot.py b/studio/backend/core/inference/diffusion_convrot.pyindex e6a5af8d8fc..a1962f0fed3 100644--- a/studio/backend/core/inference/diffusion_convrot.py+++ b/studio/backend/core/inference/diffusion_convrot.py@@ -213,6 +213,19 @@ def _install_rotation(module: Any, group_size: int) -> None:module.__class__ = convrot_linear_class()+def warm_rotation_cache(transformer: Any, device: Any, dtype: Any) -> int:+ """Prefill ``_HADAMARD_CACHE``: dynamo guards on key absence, so a cold first compile recompiles on call 2."""+ import torch++ device = torch.device(device)+ if device.type == "cuda" and device.index is None and torch.cuda.is_available():+ device = torch.device("cuda", torch.cuda.current_device())+ groups = {int(m.convrot_groupsize) for m in transformer.modules() if is_rotated_linear(m)}+ for group in groups:+ build_convrot_hadamard(group, device = device, dtype = dtype)+ return len(groups)++def declares_rotation(metadata: Any) -> bool:"""True when ``metadata`` claims its weights were rotated offline.diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex d6862b562ba..9af61da7e8d 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -558,8 +558,15 @@ def resolve_prequant_source(# Family-declared name first when there is one, then the derived chain, which puts the# safetensors spelling ahead of the pickle. Order-preserving dedup so a family that declares# exactly what the chain would derive does not make the downloader ask twice for it.+ declared = (preferred,) if preferred else ()+ # opt-in rotated artifact goes first; the plain chain stays behind it (not yet hosted, or offline)+ from .diffusion_transformer_quant import convrot_prequant_filename, int8_convrot_enabled++ rotated = convrot_prequant_filename(scheme, getattr(fam, "name", None))+ if rotated and int8_convrot_enabled():+ declared = (rotated,) + declarednames: list[str] = []- for name in ((preferred,) if preferred else ()) + derived:+ for name in declared + derived:if name and name not in names:names.append(name)return PrequantSource(@@ -567,7 +574,7 @@ def resolve_prequant_source(location = repo_id,filename = names[0],fallback_filenames = tuple(names[1:]),- declared_filenames = (preferred,) if preferred else (),+ declared_filenames = declared,)return None@@ -1203,7 +1210,11 @@ def load_prequantized_transformer(# dense fallback) for one this build cannot honour exactly. After load_state_dict because the meta retry above# rebuilds the module; before apply_small_m_padding because padding reparents the Linears and the recorded# fqns name the unwrapped tree.- from .diffusion_convrot import apply_activation_rotation+ from .diffusion_convrot import (+ apply_activation_rotation,+ declares_rotation,+ warm_rotation_cache,+ )apply_activation_rotation(transformer, metadata, logger = logger)@@ -1218,6 +1229,17 @@ def load_prequantized_transformer(del ckpttransformer = transformer.to(device)+ if declares_rotation(metadata):+ try:+ import torch++ on = next(iter(transformer.parameters()), None)+ dtype = getattr(+ torch, str(metadata.get("torch_dtype") or "bfloat16"), torch.bfloat16+ )+ warm_rotation_cache(transformer, on.device if on is not None else device, dtype)+ except Exception: # noqa: BLE001+ pass# Same small-M row padding the runtime quantise path applies, and for the same reason: a checkpoint built# under the current exclusion set QUANTISES the family's small-M linears, so without the wrappers they would# raise inside _int_mm the moment the compiled scope reaches them. After load_state_dict, since wrapping@@ -1393,7 +1415,16 @@ def _resolve_checkpoint_path(# happened. Offline, a cache miss is the only verdict there is, so the chain is# walked exactly as for a 404.if not local_files_only or last:- raise+ cached = (+ None+ if last+ else cached_checkpoint_path(+ source, cache_dir = cache_dir, names = names[index + 1 :]+ )+ )+ if cached is None:+ raise+ return cachedexcept EntryNotFoundError:if last:raisediff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 0b7364bd47e..db486cad475 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -204,6 +204,110 @@ def apply_small_m_padding(return wrapped+# Per-family int8 ConvRot (group, rotated fqn suffixes); not an exclusion, so plain artifacts still validate.+_INT8_FAMILY_CONVROT: dict[str, tuple[int, tuple[str, ...]]] = {+ "qwen-image-2.1": (+ 256,+ (+ "attn.to_q",+ "attn.to_k",+ "attn.to_v",+ "attn.to_out.0",+ "img_mlp.gate_layer",+ "img_mlp.proj",+ "img_mlp.out",+ ),+ ),+}+++_INT8_FAMILY_CONVROT_FILENAME: dict[str, str] = {+ "qwen-image-2.1": "Qwen-Image-2.1-INT8-ConvRot.safetensors",+}++INT8_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_CONVROT"+++def int8_convrot_enabled() -> bool:+ return (_os.environ.get(INT8_CONVROT_ENV) or "").strip().lower() in ("1", "on", "true", "yes")+++def convrot_spec_for_scheme(+ scheme: str, family: Optional[str] = None+) -> tuple[int, tuple[str, ...]]:+ """The family's int8 ConvRot table entry, regardless of the opt-in flag."""+ if scheme != TQ_INT8:+ return 0, ()+ return _INT8_FAMILY_CONVROT.get(str(family or "").strip().lower(), (0, ()))+++def convrot_prequant_filename(scheme: str, family: Optional[str] = None) -> Optional[str]:+ if scheme != TQ_INT8:+ return None+ return _INT8_FAMILY_CONVROT_FILENAME.get(str(family or "").strip().lower())+++def convrot_fqns(+ transformer: Any, filter_fn: Any, group: int, suffixes: tuple[str, ...]+) -> tuple[str, ...]:+ """Quantized Linears matching ``suffixes`` with width divisible by ``group`` (LoRA ``base_layer`` stays exact)."""+ from .diffusion_convrot import rotatable_fqns++ rotatable, _ = rotatable_fqns(transformer, filter_fn, group)+ names = tuple(n for s in suffixes for n in (s, s + ".base_layer"))+ return tuple(f for f in rotatable if any(f == n or f.endswith("." + n) for n in names))+++def apply_runtime_convrot(+ transformer: Any,+ scheme: str,+ family: Optional[str],+ filter_fn: Any,+ *,+ target: Any = None,+ logger: Any = None,+) -> tuple[str, ...]:+ """Rotate BEFORE quantize_; a later failure leaves an exact dense model, so fallback stays correct."""+ group, suffixes = convrot_spec_for_scheme(scheme, family)+ if not group or not int8_convrot_enabled():+ return ()+ from .diffusion_convrot import CONVROT_ATTR, CONVROT_KIND, rotate_linears_, warm_rotation_cache++ rotated = rotate_linears_(+ transformer, convrot_fqns(transformer, filter_fn, group, suffixes), group+ )+ if rotated:+ # forward device, not the weights' (CPU-first quantize)+ weight = transformer.get_submodule(rotated[0]).weight+ device = getattr(target, "torch_device", None) or getattr(target, "device", None)+ dtype = getattr(target, "dtype", None)+ try:+ warm_rotation_cache(+ transformer,+ device or weight.device,+ dtype if dtype is not None and not isinstance(dtype, str) else weight.dtype,+ )+ except Exception: # noqa: BLE001 - only saves one recompile+ pass+ try:+ setattr(+ transformer,+ CONVROT_ATTR,+ {"kind": CONVROT_KIND, "group": group, "linears": len(rotated)},+ )+ except Exception: # noqa: BLE001 - diagnostic marker only+ pass+ if logger is not None:+ logger.info(+ "diffusion.transformer_quant: ConvRot group %d on %d %s linears (%s)",+ group,+ len(rotated),+ scheme,+ family,+ )+ return rotated++# nvfp4 raises on an empty activation; HunyuanVideo-1.5's attention trim hits it every t2v render._HUNYUAN15_NVFP4_ZERO_ROW_TOKENS = ("image_embedder", "context_embedder_2")_NVFP4_FAMILY_ZERO_ROW_NAME_TOKENS: dict[str, tuple[str, ...]] = {@@ -1733,15 +1837,17 @@ def quantize_transformer(# exclude_tokens_for_scheme, whose list is baked into prequant metadata.exclude = exclude_tokens_for_scheme(scheme, family) + ("lora_",)divisible = divisible_for_scheme(scheme)+ filter_fn = make_filter_fn(+ min_features,+ exclude_name_tokens = exclude,+ require_bf16 = scheme in _REQUIRE_BF16_SCHEMES,+ require_divisible = divisible,+ )+ apply_runtime_convrot(transformer, scheme, family, filter_fn, target = target, logger = logger)quantize_(transformer,_make_quant_config(scheme, fast_accum = fast_accum),- filter_fn = make_filter_fn(- min_features,- exclude_name_tokens = exclude,- require_bf16 = scheme in _REQUIRE_BF16_SCHEMES,- require_divisible = divisible,- ),+ filter_fn = filter_fn,)# Pad this family's small-M linears now that the weights are quantized and in place. Not best-effort: a raise# here means the transformer is quantized but not safely compilable, so it falls into the except below and thediff --git a/studio/backend/tests/test_diffusion_int8_runtime_convrot.py b/studio/backend/tests/test_diffusion_int8_runtime_convrot.pynew file mode 100644index 00000000000..f9fde1b62da--- /dev/null+++ b/studio/backend/tests/test_diffusion_int8_runtime_convrot.py@@ -0,0 +1,404 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+++import sys+import types++import pytest++torch = pytest.importorskip("torch")+from torch import nn++import core.inference.diffusion_transformer_quant as tq+from core.inference.diffusion_convrot import CONVROT_ATTR, is_rotated_linear++[email protected](autouse = True)+def _convrot_opted_in(monkeypatch):+ monkeypatch.setenv(tq.INT8_CONVROT_ENV, "1")+++class _Attn(nn.Module):+ def __init__(self, dim):+ super().__init__()+ self.to_q = nn.Linear(dim, dim, bias = False)+ self.to_k = nn.Linear(dim, dim, bias = False)+ self.to_v = nn.Linear(dim, dim, bias = False)+ self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias = False)])++ def forward(self, x):+ return self.to_out[0](self.to_q(x) * self.to_k(x).sigmoid() + self.to_v(x))+++class _Mlp(nn.Module):+ def __init__(self, dim):+ super().__init__()+ self.gate_layer = nn.Linear(dim, 3 * dim, bias = False)+ self.out = nn.Linear(3 * dim, dim, bias = False)++ def forward(self, x):+ return self.out(torch.nn.functional.silu(self.gate_layer(x)))+++class _Block(nn.Module):+ def __init__(self, dim):+ super().__init__()+ self.attn = _Attn(dim)+ self.img_mlp = _Mlp(dim)+ self.extra = nn.Linear(dim, dim, bias = False)++ def forward(self, x):+ x = x + self.attn(x)+ return self.extra(x + self.img_mlp(x))+++class _Tiny(nn.Module):+ def __init__(self, dim = 512):+ super().__init__()+ self.txt_in = nn.Linear(dim, dim, bias = False)+ self.small = nn.Linear(dim, 64, bias = False)+ self.odd = nn.Linear(640, dim, bias = False)+ self.transformer_blocks = nn.ModuleList([_Block(dim) for _ in range(2)])++ def forward(self, x, y):+ h = self.txt_in(x) + self.odd(y)+ for b in self.transformer_blocks:+ h = b(h)+ return h, self.small(h)+++_ROTATED = {+ f"transformer_blocks.{i}.{n}"+ for i in range(2)+ for n in (+ "attn.to_q",+ "attn.to_k",+ "attn.to_v",+ "attn.to_out.0",+ "img_mlp.gate_layer",+ "img_mlp.out",+ )+}+++def _filter(family):+ return tq.make_filter_fn(+ 512, exclude_name_tokens = tq.exclude_tokens_for_scheme(tq.TQ_INT8, family) + ("lora_",)+ )+++def test_convrot_spec_only_for_int8_on_declared_families():+ group, suffixes = tq.convrot_spec_for_scheme(tq.TQ_INT8, "qwen-image-2.1")+ assert group == 256 and set(suffixes) == {+ "attn.to_q",+ "attn.to_k",+ "attn.to_v",+ "attn.to_out.0",+ "img_mlp.gate_layer",+ "img_mlp.proj",+ "img_mlp.out",+ }+ assert tq.convrot_spec_for_scheme(tq.TQ_INT8, " Qwen-Image-2.1 ")[0] == 256+ for scheme in (tq.TQ_FP8, tq.TQ_NVFP4, tq.TQ_MXFP8):+ assert tq.convrot_spec_for_scheme(scheme, "qwen-image-2.1") == (0, ())+ for family in (None, "qwen-image", "qwen-image-edit", "flux.1", "minimax-h3"):+ assert tq.convrot_spec_for_scheme(tq.TQ_INT8, family) == (0, ())+++def test_int8_artifact_names_the_rotated_build_first_and_keeps_the_plain_one():+ from core.inference.diffusion_families import detect_family+ from core.inference.diffusion_prequant import candidate_filenames_of, resolve_prequant_source++ fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")+ names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))+ assert names[:2] == (+ "Qwen-Image-2.1-INT8-ConvRot.safetensors",+ "Qwen-Image-2.1-INT8.safetensors",+ )+ assert (+ candidate_filenames_of(resolve_prequant_source(fam, "fp8"))[0]+ == "Qwen-Image-2.1-FP8.safetensors"+ )+++def test_without_the_opt_in_int8_is_plain_everywhere(monkeypatch):+ monkeypatch.delenv(tq.INT8_CONVROT_ENV)+ from core.inference.diffusion_families import detect_family+ from core.inference.diffusion_prequant import candidate_filenames_of, resolve_prequant_source++ fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")+ names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))+ assert names[0] == "Qwen-Image-2.1-INT8.safetensors"+ assert "Qwen-Image-2.1-INT8-ConvRot.safetensors" not in names+ model = _Tiny()+ assert (+ tq.apply_runtime_convrot(model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"))+ == ()+ )+ assert not any(is_rotated_linear(m) for m in model.modules())+ assert not hasattr(model, CONVROT_ATTR)+ for off in ("0", "", "off", "false"):+ monkeypatch.setenv(tq.INT8_CONVROT_ENV, off)+ assert not tq.int8_convrot_enabled()+++def test_quantize_transformer_without_the_opt_in_rotates_nothing(monkeypatch):+ monkeypatch.delenv(tq.INT8_CONVROT_ENV)+ seen = []+ _stub_torchao(monkeypatch, seen)+ monkeypatch.setattr(+ tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_INT8+ )+ pipe = types.SimpleNamespace(transformer = _Tiny())+ assert (+ tq.quantize_transformer(pipe, object(), mode = "int8", family = "qwen-image-2.1") == tq.TQ_INT8+ )+ assert seen == [[]]+++def test_convrot_does_not_touch_the_exclusion_set():+ assert tq.exclude_tokens_for_scheme(+ tq.TQ_INT8, "qwen-image-2.1"+ ) == tq._INT8_EXCLUDE_NAME_TOKENS + ("txt_in",)+++def test_runtime_convrot_rotates_the_quantized_set_and_keeps_the_model_exact():+ torch.manual_seed(0)+ model = _Tiny().float()+ x, y = torch.randn(4, 512), torch.randn(4, 640)+ with torch.no_grad():+ ref = model(x, y)+ rotated = tq.apply_runtime_convrot(+ model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1")+ )+ got = model(x, y)+ assert set(rotated) == _ROTATED+ assert not is_rotated_linear(model.txt_in)+ assert not is_rotated_linear(model.small)+ assert not is_rotated_linear(model.odd)+ assert not any(is_rotated_linear(blk.extra) for blk in model.transformer_blocks)+ assert getattr(model, CONVROT_ATTR)["linears"] == 12+ for a, b in zip(ref, got):+ torch.testing.assert_close(a, b, rtol = 1e-4, atol = 1e-4)+++def test_runtime_convrot_is_inert_elsewhere():+ model = _Tiny()+ assert tq.apply_runtime_convrot(model, tq.TQ_INT8, "qwen-image", _filter("qwen-image")) == ()+ assert (+ tq.apply_runtime_convrot(model, tq.TQ_FP8, "qwen-image-2.1", _filter("qwen-image-2.1"))+ == ()+ )+ assert not any(is_rotated_linear(m) for m in model.modules())+ assert not hasattr(model, CONVROT_ATTR)+++def _stub_torchao(monkeypatch, seen):+ tqz = types.ModuleType("torchao.quantization")++ def quantize_(+ module,+ config,+ filter_fn = None,+ ):+ seen.append(+ sorted(n for n, m in module.named_modules() if is_rotated_linear(m) and filter_fn(m, n))+ )++ tqz.quantize_ = quantize_+ tqz.Int8DynamicActivationInt8WeightConfig = lambda **kw: "int8-cfg"+ tqz.Float8DynamicActivationFloat8WeightConfig = lambda **kw: "fp8-cfg"+ tqz.PerRow = lambda: "per-row"+ monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)+ monkeypatch.setattr(tq, "_make_quant_config", lambda scheme, fast_accum = None: f"{scheme}-cfg")+++ "family, expect_rotated", [("qwen-image-2.1", 12), ("qwen-image", 0), (None, 0)]+)+def test_quantize_transformer_rotates_before_quantize(monkeypatch, family, expect_rotated):+ seen = []+ _stub_torchao(monkeypatch, seen)+ monkeypatch.setattr(+ tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_INT8+ )+ pipe = types.SimpleNamespace(transformer = _Tiny())+ assert tq.quantize_transformer(pipe, object(), mode = "int8", family = family) == tq.TQ_INT8+ assert len(seen) == 1 and len(seen[0]) == expect_rotated+++def test_quantize_transformer_leaves_fp8_unrotated(monkeypatch):+ seen = []+ _stub_torchao(monkeypatch, seen)+ monkeypatch.setattr(+ tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_FP8+ )+ pipe = types.SimpleNamespace(transformer = _Tiny())+ assert tq.quantize_transformer(pipe, object(), mode = "fp8", family = "qwen-image-2.1") == tq.TQ_FP8+ assert seen == [[]]+++def test_runtime_convrot_warms_the_hadamard_for_the_target_device():+ from core.inference import diffusion_convrot as cr++ cr._HADAMARD_CACHE.clear()+ model = _Tiny()+ target = types.SimpleNamespace(device = "cuda", torch_device = "cpu", dtype = torch.bfloat16)+ tq.apply_runtime_convrot(+ model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"), target = target+ )+ assert (256, "cpu", torch.bfloat16) in cr._HADAMARD_CACHE++[email protected](not torch.cuda.is_available(), reason = "compiles a CUDA graph")+def test_runtime_convrot_adds_no_recompile():+ from torch._dynamo.utils import counters++ from core.inference import diffusion_convrot as cr++ cr._HADAMARD_CACHE.clear()+ torch._dynamo.reset()+ counters.clear()+ model = _Tiny().to(torch.bfloat16)+ target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+ tq.apply_runtime_convrot(+ model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"), target = target+ )+ block = torch.compile(model.cuda().transformer_blocks[0], fullgraph = True)+ x = torch.randn(4, 512, device = "cuda", dtype = torch.bfloat16)+ for _ in range(3):+ block(x)+ assert counters["stats"]["unique_graphs"] == 1+++def test_unreachable_hub_still_loads_the_plain_artifact_already_cached(monkeypatch, tmp_path):+ from huggingface_hub.errors import LocalEntryNotFoundError++ from core.inference import diffusion_prequant as pq+ from core.inference.diffusion_families import detect_family++ plain = tmp_path / "Qwen-Image-2.1-INT8.safetensors"+ plain.write_bytes(b"weights")+ monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)+ monkeypatch.setattr(+ "huggingface_hub.try_to_load_from_cache",+ lambda repo_id, filename, cache_dir = None, **k: str(tmp_path / filename)+ if (tmp_path / filename).is_file()+ else None,+ )+ asked = []++ def _dl(+ repo_id,+ filename,+ token = None,+ cache_dir = None,+ local_files_only = False,+ ):+ asked.append(filename)+ raise LocalEntryNotFoundError("connection error")++ monkeypatch.setattr("huggingface_hub.hf_hub_download", _dl)+ source = pq.resolve_prequant_source(+ detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1"), "int8"+ )+ assert pq._resolve_checkpoint_path(source, None, None) == str(plain)+ assert asked == ["Qwen-Image-2.1-INT8-ConvRot.safetensors"]+ plain.unlink()+ with pytest.raises(LocalEntryNotFoundError):+ pq._resolve_checkpoint_path(source, None, None)+++def test_builder_publishes_rotated_and_plain_int8_under_different_names():+ import importlib.util+ from pathlib import Path++ from core.inference.diffusion_families import detect_family++ script = Path(__file__).resolve().parents[3] / "scripts" / "build_prequant_checkpoint.py"+ spec = importlib.util.spec_from_file_location("_build_prequant_for_convrot_test", script)+ build = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(build)+ fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")+ repo = "unsloth/Qwen-Image-2.1-FP8"+ dest = lambda rotated: build.upload_destination(+ fam, "int8", rotated = rotated, safetensors = True, upload_repo = repo+ )+ assert dest(True) == "Qwen-Image-2.1-INT8-ConvRot.safetensors"+ assert dest(False) == "Qwen-Image-2.1-INT8.safetensors"+ spec_build = build.upload_destination(+ fam, "int8", rotated = True, safetensors = True, upload_repo = repo, convrot_group = 256+ )+ assert spec_build == "Qwen-Image-2.1-INT8-ConvRot.safetensors"+ with pytest.raises(ValueError, match = "reserved"):+ build.upload_destination(+ fam, "int8", rotated = True, safetensors = True, upload_repo = repo, convrot_group = 64+ )+ assert (+ build.upload_destination(fam, "fp8", rotated = False, safetensors = True, upload_repo = repo)+ == "Qwen-Image-2.1-FP8.safetensors"+ )+++class _LoraLike(nn.Module):+ def __init__(self, base):+ super().__init__()+ self.base_layer = base+ self.lora_A = nn.Linear(base.in_features, 4, bias = False)+ self.lora_B = nn.Linear(4, base.out_features, bias = False)++ def forward(self, x):+ return self.base_layer(x) + self.lora_B(self.lora_A(x))+++def _check_base_layers_rotated(model):+ torch.manual_seed(1)+ x, y = torch.randn(4, 512), torch.randn(4, 640)+ with torch.no_grad():+ ref = model(x, y)+ rotated = tq.apply_runtime_convrot(+ model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1")+ )+ got = model(x, y)+ for i in range(2):+ for n in ("attn.to_q", "attn.to_v", "img_mlp.out"):+ assert f"transformer_blocks.{i}.{n}.base_layer" in rotated+ assert len(rotated) == 12 and not any("lora_" in f for f in rotated)+ for a, b in zip(ref, got):+ torch.testing.assert_close(a, b, rtol = 1e-4, atol = 1e-4)+++def test_lora_baked_targets_rotate_their_base_layer_and_stay_exact():+ # Studio attaches adapters before quantize_, so a LoRA target is "<suffix>.base_layer"+ torch.manual_seed(0)+ model = _Tiny().float()+ for blk in model.transformer_blocks:+ blk.attn.to_q = _LoraLike(blk.attn.to_q)+ blk.attn.to_v = _LoraLike(blk.attn.to_v)+ blk.img_mlp.out = _LoraLike(blk.img_mlp.out)+ _check_base_layers_rotated(model)+++def _fresh_peft_torchao_probe():+ # peft caches is_torchao_available(); a value cached here leaks into later tests that block torchao+ is_torchao_available = pytest.importorskip("peft.import_utils").is_torchao_available+ is_torchao_available.cache_clear()+ yield+ is_torchao_available.cache_clear()+++def test_real_peft_adapters_rotate_their_base_layer_and_stay_exact(_fresh_peft_torchao_probe):+ peft = pytest.importorskip("peft")+ torch.manual_seed(0)+ cfg = peft.LoraConfig(+ r = 4, lora_alpha = 8, target_modules = ["to_q", "to_v", "out"], init_lora_weights = False+ )+ try:+ model = peft.inject_adapter_in_model(cfg, _Tiny().float())+ except (+ ImportError+ ) as exc: # peft < 0.19 with torchao >= 0.18, patched only once unsloth is imported+ pytest.skip(f"peft cannot dispatch LoRA on this torchao: {exc}")+ _check_base_layers_rotated(model)