Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: opt-in Hadamard rotation for Qwen-Image-2.1's int8 transformer so it matches bf16 (#11835)

Daniel Hanb33abbcc964e
diff --git a/scripts/build_prequant_checkpoint.py b/scripts/build_prequant_checkpoint.py
index bbdb1cc7122..3906130f3a3 100644
--- a/scripts/build_prequant_checkpoint.py
+++ b/scripts/build_prequant_checkpoint.py
@@ -67,6 +67,7 @@ def upload_destination(
override: Optional[str] = None,
upload_repo: Optional[str] = None,
component: str = DEFAULT_COMPONENT,
+ convrot_group: Optional[int] = None,
) -> str:
"""The repo-root filename this build should publish under.
@@ -124,8 +125,21 @@ def upload_destination(
if not rotated and not safetensors:
return prequant_filename(scheme)
from core.inference.diffusion_families import family_prequant_filename
+ from core.inference.diffusion_transformer_quant import (
+ convrot_prequant_filename,
+ convrot_spec_for_scheme,
+ )
- preferred = family_prequant_filename(fam, scheme)
+ family = getattr(fam, "name", None)
+ rotated_name = convrot_prequant_filename(scheme, family) if rotated else None
+ if rotated_name:
+ spec_group = convrot_spec_for_scheme(scheme, family)[0]
+ if convrot_group is not None and int(convrot_group) != spec_group:
+ raise ValueError(
+ f"{rotated_name} is reserved for the ConvRot group {spec_group} build that matches "
+ f"the runtime path; a group {convrot_group} build needs --upload-filename."
+ )
+ preferred = rotated_name or family_prequant_filename(fam, scheme)
why = "a rotated checkpoint" if rotated else "a safetensors checkpoint"
if not preferred:
# A PLAIN safetensors build now has a derived name, and only because
@@ -208,7 +222,9 @@ def main(argv = None) -> int:
type = int,
default = 0,
help = "bake a ConvRot block-Hadamard activation rotation at this group size (a power of "
- "4; 0 = off). Every quantized Linear whose in_features the group divides has its "
+ "4; 0 = off). A family with an int8 ConvRot spec at this group (Qwen-Image-2.1: "
+ "256) rotates exactly the Linears that spec names, so the artifact matches the opt-in "
+ "runtime path (UNSLOTH_DIFFUSION_INT8_CONVROT=1); otherwise every quantized Linear whose in_features the group divides has its "
"weight rotated before quantize_ so the quantizer sees a flatter distribution; the "
"exact fqn list is recorded in the checkpoint and the loader rotates the "
"activations of that list and nothing else. Writes the v2 format tag.",
@@ -240,6 +256,8 @@ def main(argv = None) -> int:
TQ_SCHEMES,
_make_quant_config,
_resolve_fast_accum,
+ convrot_fqns,
+ convrot_spec_for_scheme,
make_filter_fn,
)
from torchao.quantization import quantize_
@@ -253,6 +271,9 @@ def main(argv = None) -> int:
if fam is None:
print(f"error: unknown family '{args.family}'", flush = True)
return 2
+ convrot_group = int(args.convrot_groupsize)
+ spec_group, spec_suffixes = convrot_spec_for_scheme(scheme, fam.name)
+ convrot_suffixes: tuple = spec_suffixes if convrot_group and convrot_group == spec_group else ()
# What the artifact RECORDS as its base, which is not always what this build READ. Weights staged into a local
# directory keep that directory's name, and the loader's ``_same_base_model`` compares final path segments: a
# checkpoint built from ./temp/qwen_image_21 records a base whose tail is "qwen_image_21", the load asks for
@@ -314,7 +335,8 @@ def main(argv = None) -> int:
upload_dest = upload_destination(
fam,
scheme,
- rotated = bool(args.convrot_groupsize),
+ rotated = bool(convrot_group),
+ convrot_group = convrot_group,
safetensors = is_safetensors_out,
override = args.upload_filename,
upload_repo = args.upload_repo,
@@ -346,15 +368,17 @@ def main(argv = None) -> int:
# ConvRot, BEFORE quantize_: rotating the weights is only worth anything if the quantizer then sees the rotated
# distribution. The fqn list is recorded, never re-derived at load time.
rotation: dict = {}
- if args.convrot_groupsize:
+ if convrot_group:
from core.inference.diffusion_convrot import (
rotatable_fqns,
rotate_linears_,
rotation_metadata,
)
- group = int(args.convrot_groupsize)
+ group = int(convrot_group)
rotatable, not_divisible = rotatable_fqns(transformer, filter_fn, group)
+ if convrot_suffixes:
+ rotatable = convrot_fqns(transformer, filter_fn, group, convrot_suffixes)
refusal = convrot_refusal(group, rotatable, not_divisible)
if refusal:
print(f"error: {refusal}", flush = True)
diff --git a/studio/backend/core/inference/diffusion_convrot.py b/studio/backend/core/inference/diffusion_convrot.py
index e6a5af8d8fc..a1962f0fed3 100644
--- a/studio/backend/core/inference/diffusion_convrot.py
+++ b/studio/backend/core/inference/diffusion_convrot.py
@@ -213,6 +213,19 @@ def _install_rotation(module: Any, group_size: int) -> None:
module.__class__ = convrot_linear_class()
+def warm_rotation_cache(transformer: Any, device: Any, dtype: Any) -> int:
+ """Prefill ``_HADAMARD_CACHE``: dynamo guards on key absence, so a cold first compile recompiles on call 2."""
+ import torch
+
+ device = torch.device(device)
+ if device.type == "cuda" and device.index is None and torch.cuda.is_available():
+ device = torch.device("cuda", torch.cuda.current_device())
+ groups = {int(m.convrot_groupsize) for m in transformer.modules() if is_rotated_linear(m)}
+ for group in groups:
+ build_convrot_hadamard(group, device = device, dtype = dtype)
+ return len(groups)
+
+
def declares_rotation(metadata: Any) -> bool:
"""True when ``metadata`` claims its weights were rotated offline.
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index d6862b562ba..9af61da7e8d 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -558,8 +558,15 @@ def resolve_prequant_source(
# Family-declared name first when there is one, then the derived chain, which puts the
# safetensors spelling ahead of the pickle. Order-preserving dedup so a family that declares
# exactly what the chain would derive does not make the downloader ask twice for it.
+ declared = (preferred,) if preferred else ()
+ # opt-in rotated artifact goes first; the plain chain stays behind it (not yet hosted, or offline)
+ from .diffusion_transformer_quant import convrot_prequant_filename, int8_convrot_enabled
+
+ rotated = convrot_prequant_filename(scheme, getattr(fam, "name", None))
+ if rotated and int8_convrot_enabled():
+ declared = (rotated,) + declared
names: list[str] = []
- for name in ((preferred,) if preferred else ()) + derived:
+ for name in declared + derived:
if name and name not in names:
names.append(name)
return PrequantSource(
@@ -567,7 +574,7 @@ def resolve_prequant_source(
location = repo_id,
filename = names[0],
fallback_filenames = tuple(names[1:]),
- declared_filenames = (preferred,) if preferred else (),
+ declared_filenames = declared,
)
return None
@@ -1203,7 +1210,11 @@ def load_prequantized_transformer(
# dense fallback) for one this build cannot honour exactly. After load_state_dict because the meta retry above
# rebuilds the module; before apply_small_m_padding because padding reparents the Linears and the recorded
# fqns name the unwrapped tree.
- from .diffusion_convrot import apply_activation_rotation
+ from .diffusion_convrot import (
+ apply_activation_rotation,
+ declares_rotation,
+ warm_rotation_cache,
+ )
apply_activation_rotation(transformer, metadata, logger = logger)
@@ -1218,6 +1229,17 @@ def load_prequantized_transformer(
del ckpt
transformer = transformer.to(device)
+ if declares_rotation(metadata):
+ try:
+ import torch
+
+ on = next(iter(transformer.parameters()), None)
+ dtype = getattr(
+ torch, str(metadata.get("torch_dtype") or "bfloat16"), torch.bfloat16
+ )
+ warm_rotation_cache(transformer, on.device if on is not None else device, dtype)
+ except Exception: # noqa: BLE001
+ pass
# Same small-M row padding the runtime quantise path applies, and for the same reason: a checkpoint built
# under the current exclusion set QUANTISES the family's small-M linears, so without the wrappers they would
# raise inside _int_mm the moment the compiled scope reaches them. After load_state_dict, since wrapping
@@ -1393,7 +1415,16 @@ def _resolve_checkpoint_path(
# happened. Offline, a cache miss is the only verdict there is, so the chain is
# walked exactly as for a 404.
if not local_files_only or last:
- raise
+ cached = (
+ None
+ if last
+ else cached_checkpoint_path(
+ source, cache_dir = cache_dir, names = names[index + 1 :]
+ )
+ )
+ if cached is None:
+ raise
+ return cached
except EntryNotFoundError:
if last:
raise
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 0b7364bd47e..db486cad475 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -204,6 +204,110 @@ def apply_small_m_padding(
return wrapped
+# Per-family int8 ConvRot (group, rotated fqn suffixes); not an exclusion, so plain artifacts still validate.
+_INT8_FAMILY_CONVROT: dict[str, tuple[int, tuple[str, ...]]] = {
+ "qwen-image-2.1": (
+ 256,
+ (
+ "attn.to_q",
+ "attn.to_k",
+ "attn.to_v",
+ "attn.to_out.0",
+ "img_mlp.gate_layer",
+ "img_mlp.proj",
+ "img_mlp.out",
+ ),
+ ),
+}
+
+
+_INT8_FAMILY_CONVROT_FILENAME: dict[str, str] = {
+ "qwen-image-2.1": "Qwen-Image-2.1-INT8-ConvRot.safetensors",
+}
+
+INT8_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_CONVROT"
+
+
+def int8_convrot_enabled() -> bool:
+ return (_os.environ.get(INT8_CONVROT_ENV) or "").strip().lower() in ("1", "on", "true", "yes")
+
+
+def convrot_spec_for_scheme(
+ scheme: str, family: Optional[str] = None
+) -> tuple[int, tuple[str, ...]]:
+ """The family's int8 ConvRot table entry, regardless of the opt-in flag."""
+ if scheme != TQ_INT8:
+ return 0, ()
+ return _INT8_FAMILY_CONVROT.get(str(family or "").strip().lower(), (0, ()))
+
+
+def convrot_prequant_filename(scheme: str, family: Optional[str] = None) -> Optional[str]:
+ if scheme != TQ_INT8:
+ return None
+ return _INT8_FAMILY_CONVROT_FILENAME.get(str(family or "").strip().lower())
+
+
+def convrot_fqns(
+ transformer: Any, filter_fn: Any, group: int, suffixes: tuple[str, ...]
+) -> tuple[str, ...]:
+ """Quantized Linears matching ``suffixes`` with width divisible by ``group`` (LoRA ``base_layer`` stays exact)."""
+ from .diffusion_convrot import rotatable_fqns
+
+ rotatable, _ = rotatable_fqns(transformer, filter_fn, group)
+ names = tuple(n for s in suffixes for n in (s, s + ".base_layer"))
+ return tuple(f for f in rotatable if any(f == n or f.endswith("." + n) for n in names))
+
+
+def apply_runtime_convrot(
+ transformer: Any,
+ scheme: str,
+ family: Optional[str],
+ filter_fn: Any,
+ *,
+ target: Any = None,
+ logger: Any = None,
+) -> tuple[str, ...]:
+ """Rotate BEFORE quantize_; a later failure leaves an exact dense model, so fallback stays correct."""
+ group, suffixes = convrot_spec_for_scheme(scheme, family)
+ if not group or not int8_convrot_enabled():
+ return ()
+ from .diffusion_convrot import CONVROT_ATTR, CONVROT_KIND, rotate_linears_, warm_rotation_cache
+
+ rotated = rotate_linears_(
+ transformer, convrot_fqns(transformer, filter_fn, group, suffixes), group
+ )
+ if rotated:
+ # forward device, not the weights' (CPU-first quantize)
+ weight = transformer.get_submodule(rotated[0]).weight
+ device = getattr(target, "torch_device", None) or getattr(target, "device", None)
+ dtype = getattr(target, "dtype", None)
+ try:
+ warm_rotation_cache(
+ transformer,
+ device or weight.device,
+ dtype if dtype is not None and not isinstance(dtype, str) else weight.dtype,
+ )
+ except Exception: # noqa: BLE001 - only saves one recompile
+ pass
+ try:
+ setattr(
+ transformer,
+ CONVROT_ATTR,
+ {"kind": CONVROT_KIND, "group": group, "linears": len(rotated)},
+ )
+ except Exception: # noqa: BLE001 - diagnostic marker only
+ pass
+ if logger is not None:
+ logger.info(
+ "diffusion.transformer_quant: ConvRot group %d on %d %s linears (%s)",
+ group,
+ len(rotated),
+ scheme,
+ family,
+ )
+ return rotated
+
+
# nvfp4 raises on an empty activation; HunyuanVideo-1.5's attention trim hits it every t2v render.
_HUNYUAN15_NVFP4_ZERO_ROW_TOKENS = ("image_embedder", "context_embedder_2")
_NVFP4_FAMILY_ZERO_ROW_NAME_TOKENS: dict[str, tuple[str, ...]] = {
@@ -1733,15 +1837,17 @@ def quantize_transformer(
# exclude_tokens_for_scheme, whose list is baked into prequant metadata.
exclude = exclude_tokens_for_scheme(scheme, family) + ("lora_",)
divisible = divisible_for_scheme(scheme)
+ filter_fn = make_filter_fn(
+ min_features,
+ exclude_name_tokens = exclude,
+ require_bf16 = scheme in _REQUIRE_BF16_SCHEMES,
+ require_divisible = divisible,
+ )
+ apply_runtime_convrot(transformer, scheme, family, filter_fn, target = target, logger = logger)
quantize_(
transformer,
_make_quant_config(scheme, fast_accum = fast_accum),
- filter_fn = make_filter_fn(
- min_features,
- exclude_name_tokens = exclude,
- require_bf16 = scheme in _REQUIRE_BF16_SCHEMES,
- require_divisible = divisible,
- ),
+ filter_fn = filter_fn,
)
# Pad this family's small-M linears now that the weights are quantized and in place. Not best-effort: a raise
# here means the transformer is quantized but not safely compilable, so it falls into the except below and the
diff --git a/studio/backend/tests/test_diffusion_int8_runtime_convrot.py b/studio/backend/tests/test_diffusion_int8_runtime_convrot.py
new file mode 100644
index 00000000000..f9fde1b62da
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_int8_runtime_convrot.py
@@ -0,0 +1,404 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+import sys
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+from torch import nn
+
+import core.inference.diffusion_transformer_quant as tq
+from core.inference.diffusion_convrot import CONVROT_ATTR, is_rotated_linear
+
+
[email protected](autouse = True)
+def _convrot_opted_in(monkeypatch):
+ monkeypatch.setenv(tq.INT8_CONVROT_ENV, "1")
+
+
+class _Attn(nn.Module):
+ def __init__(self, dim):
+ super().__init__()
+ self.to_q = nn.Linear(dim, dim, bias = False)
+ self.to_k = nn.Linear(dim, dim, bias = False)
+ self.to_v = nn.Linear(dim, dim, bias = False)
+ self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias = False)])
+
+ def forward(self, x):
+ return self.to_out[0](self.to_q(x) * self.to_k(x).sigmoid() + self.to_v(x))
+
+
+class _Mlp(nn.Module):
+ def __init__(self, dim):
+ super().__init__()
+ self.gate_layer = nn.Linear(dim, 3 * dim, bias = False)
+ self.out = nn.Linear(3 * dim, dim, bias = False)
+
+ def forward(self, x):
+ return self.out(torch.nn.functional.silu(self.gate_layer(x)))
+
+
+class _Block(nn.Module):
+ def __init__(self, dim):
+ super().__init__()
+ self.attn = _Attn(dim)
+ self.img_mlp = _Mlp(dim)
+ self.extra = nn.Linear(dim, dim, bias = False)
+
+ def forward(self, x):
+ x = x + self.attn(x)
+ return self.extra(x + self.img_mlp(x))
+
+
+class _Tiny(nn.Module):
+ def __init__(self, dim = 512):
+ super().__init__()
+ self.txt_in = nn.Linear(dim, dim, bias = False)
+ self.small = nn.Linear(dim, 64, bias = False)
+ self.odd = nn.Linear(640, dim, bias = False)
+ self.transformer_blocks = nn.ModuleList([_Block(dim) for _ in range(2)])
+
+ def forward(self, x, y):
+ h = self.txt_in(x) + self.odd(y)
+ for b in self.transformer_blocks:
+ h = b(h)
+ return h, self.small(h)
+
+
+_ROTATED = {
+ f"transformer_blocks.{i}.{n}"
+ for i in range(2)
+ for n in (
+ "attn.to_q",
+ "attn.to_k",
+ "attn.to_v",
+ "attn.to_out.0",
+ "img_mlp.gate_layer",
+ "img_mlp.out",
+ )
+}
+
+
+def _filter(family):
+ return tq.make_filter_fn(
+ 512, exclude_name_tokens = tq.exclude_tokens_for_scheme(tq.TQ_INT8, family) + ("lora_",)
+ )
+
+
+def test_convrot_spec_only_for_int8_on_declared_families():
+ group, suffixes = tq.convrot_spec_for_scheme(tq.TQ_INT8, "qwen-image-2.1")
+ assert group == 256 and set(suffixes) == {
+ "attn.to_q",
+ "attn.to_k",
+ "attn.to_v",
+ "attn.to_out.0",
+ "img_mlp.gate_layer",
+ "img_mlp.proj",
+ "img_mlp.out",
+ }
+ assert tq.convrot_spec_for_scheme(tq.TQ_INT8, " Qwen-Image-2.1 ")[0] == 256
+ for scheme in (tq.TQ_FP8, tq.TQ_NVFP4, tq.TQ_MXFP8):
+ assert tq.convrot_spec_for_scheme(scheme, "qwen-image-2.1") == (0, ())
+ for family in (None, "qwen-image", "qwen-image-edit", "flux.1", "minimax-h3"):
+ assert tq.convrot_spec_for_scheme(tq.TQ_INT8, family) == (0, ())
+
+
+def test_int8_artifact_names_the_rotated_build_first_and_keeps_the_plain_one():
+ from core.inference.diffusion_families import detect_family
+ from core.inference.diffusion_prequant import candidate_filenames_of, resolve_prequant_source
+
+ fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")
+ names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))
+ assert names[:2] == (
+ "Qwen-Image-2.1-INT8-ConvRot.safetensors",
+ "Qwen-Image-2.1-INT8.safetensors",
+ )
+ assert (
+ candidate_filenames_of(resolve_prequant_source(fam, "fp8"))[0]
+ == "Qwen-Image-2.1-FP8.safetensors"
+ )
+
+
+def test_without_the_opt_in_int8_is_plain_everywhere(monkeypatch):
+ monkeypatch.delenv(tq.INT8_CONVROT_ENV)
+ from core.inference.diffusion_families import detect_family
+ from core.inference.diffusion_prequant import candidate_filenames_of, resolve_prequant_source
+
+ fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")
+ names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))
+ assert names[0] == "Qwen-Image-2.1-INT8.safetensors"
+ assert "Qwen-Image-2.1-INT8-ConvRot.safetensors" not in names
+ model = _Tiny()
+ assert (
+ tq.apply_runtime_convrot(model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"))
+ == ()
+ )
+ assert not any(is_rotated_linear(m) for m in model.modules())
+ assert not hasattr(model, CONVROT_ATTR)
+ for off in ("0", "", "off", "false"):
+ monkeypatch.setenv(tq.INT8_CONVROT_ENV, off)
+ assert not tq.int8_convrot_enabled()
+
+
+def test_quantize_transformer_without_the_opt_in_rotates_nothing(monkeypatch):
+ monkeypatch.delenv(tq.INT8_CONVROT_ENV)
+ seen = []
+ _stub_torchao(monkeypatch, seen)
+ monkeypatch.setattr(
+ tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_INT8
+ )
+ pipe = types.SimpleNamespace(transformer = _Tiny())
+ assert (
+ tq.quantize_transformer(pipe, object(), mode = "int8", family = "qwen-image-2.1") == tq.TQ_INT8
+ )
+ assert seen == [[]]
+
+
+def test_convrot_does_not_touch_the_exclusion_set():
+ assert tq.exclude_tokens_for_scheme(
+ tq.TQ_INT8, "qwen-image-2.1"
+ ) == tq._INT8_EXCLUDE_NAME_TOKENS + ("txt_in",)
+
+
+def test_runtime_convrot_rotates_the_quantized_set_and_keeps_the_model_exact():
+ torch.manual_seed(0)
+ model = _Tiny().float()
+ x, y = torch.randn(4, 512), torch.randn(4, 640)
+ with torch.no_grad():
+ ref = model(x, y)
+ rotated = tq.apply_runtime_convrot(
+ model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1")
+ )
+ got = model(x, y)
+ assert set(rotated) == _ROTATED
+ assert not is_rotated_linear(model.txt_in)
+ assert not is_rotated_linear(model.small)
+ assert not is_rotated_linear(model.odd)
+ assert not any(is_rotated_linear(blk.extra) for blk in model.transformer_blocks)
+ assert getattr(model, CONVROT_ATTR)["linears"] == 12
+ for a, b in zip(ref, got):
+ torch.testing.assert_close(a, b, rtol = 1e-4, atol = 1e-4)
+
+
+def test_runtime_convrot_is_inert_elsewhere():
+ model = _Tiny()
+ assert tq.apply_runtime_convrot(model, tq.TQ_INT8, "qwen-image", _filter("qwen-image")) == ()
+ assert (
+ tq.apply_runtime_convrot(model, tq.TQ_FP8, "qwen-image-2.1", _filter("qwen-image-2.1"))
+ == ()
+ )
+ assert not any(is_rotated_linear(m) for m in model.modules())
+ assert not hasattr(model, CONVROT_ATTR)
+
+
+def _stub_torchao(monkeypatch, seen):
+ tqz = types.ModuleType("torchao.quantization")
+
+ def quantize_(
+ module,
+ config,
+ filter_fn = None,
+ ):
+ seen.append(
+ sorted(n for n, m in module.named_modules() if is_rotated_linear(m) and filter_fn(m, n))
+ )
+
+ tqz.quantize_ = quantize_
+ tqz.Int8DynamicActivationInt8WeightConfig = lambda **kw: "int8-cfg"
+ tqz.Float8DynamicActivationFloat8WeightConfig = lambda **kw: "fp8-cfg"
+ tqz.PerRow = lambda: "per-row"
+ monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
+ monkeypatch.setattr(tq, "_make_quant_config", lambda scheme, fast_accum = None: f"{scheme}-cfg")
+
+
+ "family, expect_rotated", [("qwen-image-2.1", 12), ("qwen-image", 0), (None, 0)]
+)
+def test_quantize_transformer_rotates_before_quantize(monkeypatch, family, expect_rotated):
+ seen = []
+ _stub_torchao(monkeypatch, seen)
+ monkeypatch.setattr(
+ tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_INT8
+ )
+ pipe = types.SimpleNamespace(transformer = _Tiny())
+ assert tq.quantize_transformer(pipe, object(), mode = "int8", family = family) == tq.TQ_INT8
+ assert len(seen) == 1 and len(seen[0]) == expect_rotated
+
+
+def test_quantize_transformer_leaves_fp8_unrotated(monkeypatch):
+ seen = []
+ _stub_torchao(monkeypatch, seen)
+ monkeypatch.setattr(
+ tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_: tq.TQ_FP8
+ )
+ pipe = types.SimpleNamespace(transformer = _Tiny())
+ assert tq.quantize_transformer(pipe, object(), mode = "fp8", family = "qwen-image-2.1") == tq.TQ_FP8
+ assert seen == [[]]
+
+
+def test_runtime_convrot_warms_the_hadamard_for_the_target_device():
+ from core.inference import diffusion_convrot as cr
+
+ cr._HADAMARD_CACHE.clear()
+ model = _Tiny()
+ target = types.SimpleNamespace(device = "cuda", torch_device = "cpu", dtype = torch.bfloat16)
+ tq.apply_runtime_convrot(
+ model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"), target = target
+ )
+ assert (256, "cpu", torch.bfloat16) in cr._HADAMARD_CACHE
+
+
[email protected](not torch.cuda.is_available(), reason = "compiles a CUDA graph")
+def test_runtime_convrot_adds_no_recompile():
+ from torch._dynamo.utils import counters
+
+ from core.inference import diffusion_convrot as cr
+
+ cr._HADAMARD_CACHE.clear()
+ torch._dynamo.reset()
+ counters.clear()
+ model = _Tiny().to(torch.bfloat16)
+ target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+ tq.apply_runtime_convrot(
+ model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1"), target = target
+ )
+ block = torch.compile(model.cuda().transformer_blocks[0], fullgraph = True)
+ x = torch.randn(4, 512, device = "cuda", dtype = torch.bfloat16)
+ for _ in range(3):
+ block(x)
+ assert counters["stats"]["unique_graphs"] == 1
+
+
+def test_unreachable_hub_still_loads_the_plain_artifact_already_cached(monkeypatch, tmp_path):
+ from huggingface_hub.errors import LocalEntryNotFoundError
+
+ from core.inference import diffusion_prequant as pq
+ from core.inference.diffusion_families import detect_family
+
+ plain = tmp_path / "Qwen-Image-2.1-INT8.safetensors"
+ plain.write_bytes(b"weights")
+ monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)
+ monkeypatch.setattr(
+ "huggingface_hub.try_to_load_from_cache",
+ lambda repo_id, filename, cache_dir = None, **k: str(tmp_path / filename)
+ if (tmp_path / filename).is_file()
+ else None,
+ )
+ asked = []
+
+ def _dl(
+ repo_id,
+ filename,
+ token = None,
+ cache_dir = None,
+ local_files_only = False,
+ ):
+ asked.append(filename)
+ raise LocalEntryNotFoundError("connection error")
+
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", _dl)
+ source = pq.resolve_prequant_source(
+ detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1"), "int8"
+ )
+ assert pq._resolve_checkpoint_path(source, None, None) == str(plain)
+ assert asked == ["Qwen-Image-2.1-INT8-ConvRot.safetensors"]
+ plain.unlink()
+ with pytest.raises(LocalEntryNotFoundError):
+ pq._resolve_checkpoint_path(source, None, None)
+
+
+def test_builder_publishes_rotated_and_plain_int8_under_different_names():
+ import importlib.util
+ from pathlib import Path
+
+ from core.inference.diffusion_families import detect_family
+
+ script = Path(__file__).resolve().parents[3] / "scripts" / "build_prequant_checkpoint.py"
+ spec = importlib.util.spec_from_file_location("_build_prequant_for_convrot_test", script)
+ build = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(build)
+ fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")
+ repo = "unsloth/Qwen-Image-2.1-FP8"
+ dest = lambda rotated: build.upload_destination(
+ fam, "int8", rotated = rotated, safetensors = True, upload_repo = repo
+ )
+ assert dest(True) == "Qwen-Image-2.1-INT8-ConvRot.safetensors"
+ assert dest(False) == "Qwen-Image-2.1-INT8.safetensors"
+ spec_build = build.upload_destination(
+ fam, "int8", rotated = True, safetensors = True, upload_repo = repo, convrot_group = 256
+ )
+ assert spec_build == "Qwen-Image-2.1-INT8-ConvRot.safetensors"
+ with pytest.raises(ValueError, match = "reserved"):
+ build.upload_destination(
+ fam, "int8", rotated = True, safetensors = True, upload_repo = repo, convrot_group = 64
+ )
+ assert (
+ build.upload_destination(fam, "fp8", rotated = False, safetensors = True, upload_repo = repo)
+ == "Qwen-Image-2.1-FP8.safetensors"
+ )
+
+
+class _LoraLike(nn.Module):
+ def __init__(self, base):
+ super().__init__()
+ self.base_layer = base
+ self.lora_A = nn.Linear(base.in_features, 4, bias = False)
+ self.lora_B = nn.Linear(4, base.out_features, bias = False)
+
+ def forward(self, x):
+ return self.base_layer(x) + self.lora_B(self.lora_A(x))
+
+
+def _check_base_layers_rotated(model):
+ torch.manual_seed(1)
+ x, y = torch.randn(4, 512), torch.randn(4, 640)
+ with torch.no_grad():
+ ref = model(x, y)
+ rotated = tq.apply_runtime_convrot(
+ model, tq.TQ_INT8, "qwen-image-2.1", _filter("qwen-image-2.1")
+ )
+ got = model(x, y)
+ for i in range(2):
+ for n in ("attn.to_q", "attn.to_v", "img_mlp.out"):
+ assert f"transformer_blocks.{i}.{n}.base_layer" in rotated
+ assert len(rotated) == 12 and not any("lora_" in f for f in rotated)
+ for a, b in zip(ref, got):
+ torch.testing.assert_close(a, b, rtol = 1e-4, atol = 1e-4)
+
+
+def test_lora_baked_targets_rotate_their_base_layer_and_stay_exact():
+ # Studio attaches adapters before quantize_, so a LoRA target is "<suffix>.base_layer"
+ torch.manual_seed(0)
+ model = _Tiny().float()
+ for blk in model.transformer_blocks:
+ blk.attn.to_q = _LoraLike(blk.attn.to_q)
+ blk.attn.to_v = _LoraLike(blk.attn.to_v)
+ blk.img_mlp.out = _LoraLike(blk.img_mlp.out)
+ _check_base_layers_rotated(model)
+
+
+def _fresh_peft_torchao_probe():
+ # peft caches is_torchao_available(); a value cached here leaks into later tests that block torchao
+ is_torchao_available = pytest.importorskip("peft.import_utils").is_torchao_available
+ is_torchao_available.cache_clear()
+ yield
+ is_torchao_available.cache_clear()
+
+
+def test_real_peft_adapters_rotate_their_base_layer_and_stay_exact(_fresh_peft_torchao_probe):
+ peft = pytest.importorskip("peft")
+ torch.manual_seed(0)
+ cfg = peft.LoraConfig(
+ r = 4, lora_alpha = 8, target_modules = ["to_q", "to_v", "out"], init_lora_weights = False
+ )
+ try:
+ model = peft.inject_adapter_in_model(cfg, _Tiny().float())
+ except (
+ ImportError
+ ) as exc: # peft < 0.19 with torchao >= 0.18, patched only once unsloth is imported
+ pytest.skip(f"peft cannot dispatch LoRA on this torchao: {exc}")
+ _check_base_layers_rotated(model)