Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: run the fused-dequant int8 GEMM on untuned sm80+ archs behind a probe and latency check (#15654)
Daniel Han36bd1af5964b
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex ac823469d82..66277e50c47 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -221,6 +221,7 @@ _GRAPH_SWITCHES = {"UNSLOTH_DIFFUSION_INT8_GEMM": "auto","UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT": "on","UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED": "on",+ "UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED": "auto","UNSLOTH_DIFFUSION_INT8_ROTQUANT": "auto","UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED": "auto","UNSLOTH_DIFFUSION_INT8_FUSED": "on",diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex 109cdb29b33..af744793d03 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -1,7 +1,8 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""int8 GEMM with torchao dequant epilogue in Triton, bf16 out; sm80/sm89/sm120 only, else stock+"""int8 GEMM with torchao dequant epilogue in Triton, bf16 out; sm80/sm89/sm120 measured, other sm80+ behind a+bit-exact probe and a latency check (UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED=0 keeps them stock), sm100 / sm103 stock(ROCm and CPU never reach it); Linear output bit-identical, compiled block not end to end;UNSLOTH_DIFFUSION_INT8_GEMM=0 disables."""@@ -19,6 +20,7 @@ INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"INT8_GEMM_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_TILES"INT8_SWIGLU_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED"+INT8_GEMM_UNLISTED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED"_MIN_TRITON = (3, 2)# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge._MIN_ROWS = 17@@ -50,8 +52,10 @@ _ARCH_CONFIG = {(8, 9): (256, 128, 128, 8, 8, 3), # L4(12, 0): (128, 128, 64, 8, 4, 4), # RTX PRO 6000}-# When the arch tile does not fit this part's shared memory.+# When the arch tile does not fit this part's shared memory, and the unlisted archs' tile (64 KB of stages: fits 99 KB)._FALLBACK_CONFIG = (128, 128, 64, 8, 4, 4)+# Measured slower than cuBLAS: not probed unless forced.+_UNLISTED_SKIP = ((10, 0), (10, 3))# First match wins, bounds inclusive. No L4 rule on purpose: power-capped clocks lose end to end.# (major, minor) -> ((N_lo, N_hi, K_lo, K_hi, tile), ...), M >= _SHAPE_MIN_M only.@@ -109,6 +113,33 @@ def _triton_version_ok(version: Optional[str] = None) -> bool:return bool(match) and (int(match.group(1)), int(match.group(2))) >= _MIN_TRITON+def unlisted_mode() -> str:+ """'off', 'force' (no latency check) or 'auto' for the sm80+ archs without an ``_ARCH_CONFIG`` tile."""+ gemm = int8_gemm_mode()+ if gemm != "auto":+ return gemm+ raw = (os.environ.get(INT8_GEMM_UNLISTED_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no"):+ return "off"+ if raw in ("1", "on", "true", "yes", "force"):+ return "force"+ return "auto"+++def unlisted_arch(capability: Optional[tuple]) -> bool:+ if capability is None:+ return False+ cap = (int(capability[0]), int(capability[1]))+ return cap >= (8, 0) and cap not in _ARCH_CONFIG+++def unlisted_wanted(capability: Optional[tuple], mode: Optional[str] = None) -> bool:+ mode = unlisted_mode() if mode is None else mode+ if mode == "off" or not unlisted_arch(capability):+ return False+ return mode == "force" or (int(capability[0]), int(capability[1])) not in _UNLISTED_SKIP++def arch_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:"""The tile for this compute capability, or None where the lever stays off."""mode = int8_gemm_mode() if mode is None else mode@@ -308,6 +339,8 @@ def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:_DEVICE_TILES[index] = () # drop the shape tiles, keep the probed defaultreturn _run(a, w, xs, ws, bias)_DEVICE_CFG[index] = None+ if index in _UNLISTED_CARDS:+ _UNLISTED_PROBE[index] = Falsereturn reference(a, w, xs, ws, bias)@@ -352,7 +385,10 @@ def shape_tiles(capability: Optional[tuple]) -> tuple:def device_config(index: int) -> Optional[tuple]:"""Probe once per device: arch gate, Triton, op registration, then a launch that must match the eager torchaoepilogue bit for bit (bf16 and fp32 activation scales, bias, ragged M / N / K). None keeps the stock path. Each- per-shape tile is probed the same way; one that fails leaves its shapes on the default tile."""+ per-shape tile is probed the same way; one that fails leaves its shapes on the default tile. Unlisted archs:+ ``unlisted_config``, re-read per call so its switch acts between loads."""+ if index in _UNLISTED_CARDS:+ return unlisted_config(index)if index in _DEVICE_CFG:return _DEVICE_CFG[index]cfg = None@@ -367,6 +403,9 @@ def device_config(index: int) -> Optional[tuple]:and _op() is not None):cap = torch.cuda.get_device_capability(index)+ if unlisted_arch(cap):+ _UNLISTED_CARDS.add(index)+ return unlisted_config(index)want = arch_config(cap)for cand in (want, _FALLBACK_CONFIG) if want is not None else ():if cand is not None and _probe(index, cand):@@ -386,6 +425,99 @@ def device_config(index: int) -> Optional[tuple]:return cfg+_UNLISTED_CARDS: set = set()+_UNLISTED_PROBE: dict = {}+_UNLISTED_OK: dict = {}+_UNLISTED_TIMING: dict = {}+# (M, N, K): Qwen-Image-2.1 attention projection, MLP in, MLP down+_UNLISTED_LATENCY_SHAPES = ((2048, 4096, 4096), (2048, 12288, 4096), (2048, 4096, 12288))+++def _unlisted_latency(index: int, cfg: tuple) -> dict:+ """{"fused": us, "stock": us} summed over ``_UNLISTED_LATENCY_SHAPES``, each arm captured in a CUDA graph and+ replayed, best of seven interleaved rounds. Stock = cuBLAS ``_int_mm``; its epilogue fuses into the consumer, so+ each arm ends in one read of its output (int32 for stock, bf16 here) standing in for that consumer."""+ import torch+ from .diffusion_cuda_graph import CAPTURE_MODE, _capturing, synchronize_outside_capture++ dev = torch.device("cuda", index)+ g = torch.Generator(device = "cpu").manual_seed(5)+ ops = []+ for m, n, k in _UNLISTED_LATENCY_SHAPES:+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)+ xs = torch.full((m,), 1e-3, device = dev, dtype = torch.bfloat16)+ ws = torch.full((n,), 1e-3, device = dev, dtype = torch.bfloat16)+ ops.append((a, w, xs, ws))+ fns = {+ "fused": lambda a, w, xs, ws: _launch(a, w, xs, ws, None, cfg).amax(),+ "stock": lambda a, w, xs, ws: torch._int_mm(a, w.t()).amax(),+ }+ reps = 2+ graphs = {}+ with torch.cuda.device(dev):+ side = torch.cuda.Stream(dev)+ side.wait_stream(torch.cuda.current_stream(dev))+ with torch.cuda.stream(side):+ for fn in fns.values():+ for op in ops:+ fn(*op) # compile / autotune outside the capture+ torch.cuda.current_stream(dev).wait_stream(side)+ synchronize_outside_capture(dev)+ for name, fn in fns.items():+ graph = torch.cuda.CUDAGraph()+ with _capturing(), torch.cuda.graph(graph, capture_error_mode = CAPTURE_MODE):+ for _ in range(reps):+ for op in ops:+ fn(*op)+ graphs[name] = graph+ for graph in graphs.values():+ graph.replay()+ synchronize_outside_capture(dev)+ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)+ best = {name: float("inf") for name in graphs}+ for _ in range(7):+ for name, graph in graphs.items():+ start.record()+ graph.replay()+ end.record()+ end.synchronize()+ best[name] = min(best[name], start.elapsed_time(end) * 1000.0 / reps)+ del graphs+ return best+++def unlisted_config(index: int) -> Optional[tuple]:+ """Unlisted sm80+ card: ``_FALLBACK_CONFIG`` when it passed the bit-exact probe (once per device) and the latency+ check (once per (device, mode); skipped under force), else None. Outside any trace / capture."""+ import torch++ mode = unlisted_mode()+ cfg = None+ try:+ if unlisted_wanted(torch.cuda.get_device_capability(index), mode):+ if index not in _UNLISTED_PROBE:+ _UNLISTED_PROBE[index] = _probe(index, _FALLBACK_CONFIG)+ if _UNLISTED_PROBE[index]:+ key = (index, mode)+ if key not in _UNLISTED_OK:+ ok = True+ if mode != "force":+ try:+ timing = _unlisted_latency(index, _FALLBACK_CONFIG)+ _UNLISTED_TIMING[index] = {k: round(v, 1) for k, v in timing.items()}+ ok = timing["fused"] <= timing["stock"]+ except Exception: # noqa: BLE001 - main's path stays+ ok = False+ _UNLISTED_OK[key] = ok+ cfg = _FALLBACK_CONFIG if _UNLISTED_OK[key] else None+ except Exception: # noqa: BLE001+ cfg = None+ _DEVICE_TILES[index] = ()+ _DEVICE_CFG[index] = cfg+ return cfg++# (M, N, K, bias, fp32 scales). Ragged N / K stay on 16: an off-16 probe compiles the spilling variant (see ``_aligned``)._PROBE_SHAPES = ((257, 384, 512, False, False),@@ -847,6 +979,35 @@ def _gates(capability: Optional[tuple]) -> tuple:return gemm, rotq+def _unlisted_rotated(index: int) -> bool:+ """Unlisted card: ``rotq_h16`` armed for it (probe + its own latency check). Takes no ``diffusion_int8_fused``+ lock: its SwiGLU fusion reaches this through ``rotated_ff`` while holding it."""+ if not convrot_enabled():+ return False+ try:+ from . import diffusion_convrot_quant as cq+ from .diffusion_int8_fused import _act_quant_device_ok, _act_quant_op++ actq = None+ if (index, cq.unlisted_mode()) not in cq._UNLISTED_DEVICE:+ actq = _act_quant_op() if _act_quant_device_ok(index) else None+ return bool(cq.unlisted_device_ok(index, actq))+ except Exception: # noqa: BLE001+ return False+++def rotated_swapped(index: int) -> Optional[bool]:+ """On an unlisted card (outside any trace): True when rotated Linears go through this swap (fused GEMM and the+ ``rotq_h16`` rotation + act quant both on), so the Qwen-Image-2.1 / Z-Image stock fusions stand down. False keeps+ them, exactly as on main. None on a listed arch (``_gates`` decides)."""+ import torch++ cap = torch.cuda.get_device_capability(index)+ if not unlisted_arch(cap) or _gates(cap)[1] or int8_gemm_mode() == "off":+ return None+ return _unlisted_rotated(index) and device_config(index) is not None++def install(transformer: Any,logger: Any = None,@@ -874,7 +1035,9 @@ def install(return 0if not _triton_version_ok() or _kernels() is None:return 0- gemm, rotq = _gates(torch.cuda.get_device_capability(torch.cuda.current_device()))+ cap = torch.cuda.get_device_capability(torch.cuda.current_device())+ gemm, rotq = _gates(cap)+ gemm = gemm or unlisted_wanted(cap)if not (gemm or rotq):return 0except Exception: # noqa: BLE001@@ -970,7 +1133,26 @@ def _swap_on(from .diffusion_convrot_quant import rotquant_device_configgemm = device_config(index) is not None- rotq = any(r[1] is not None for _, r in recs) and rotquant_device_config(index) is not None+ rotated = any(r[1] is not None for _, r in recs)+ rotq = rotated and rotquant_device_config(index) is not None+ if gemm and rotated and not rotq and index in _UNLISTED_CARDS:+ # unlisted card: rotated Linears only with rotq_h16; else the stock fusions keep them, as on main+ rotq = _unlisted_rotated(index)+ if not rotq:+ recs = [(m, r) for m, r in recs if r[1] is None]+ if logger is not None:+ logger.info(+ "diffusion.int8_gemm: rotated Linears keep the stock path on cuda:%d (rotq_h16 off)",+ index,+ )+ if not gemm and logger is not None and _UNLISTED_TIMING.get(index):+ t = _UNLISTED_TIMING[index]+ logger.info(+ "diffusion.int8_gemm: fused-dequant GEMM kept off on cuda:%d (latency check: %.0f vs %.0f us cuBLAS)",+ index,+ t["fused"],+ t["stock"],+ )if not gemm:# no fused GEMM here: only rotated Linears gain (one kernel instead of rotation GEMM + act quant)recs = [(m, r) for m, r in recs if r[1] is not None] if rotq else []@@ -1018,11 +1200,15 @@ def _swap_on()if logger is not None and count:from .diffusion_convrot_quant import _H16_DEVICE+ timing = _UNLISTED_TIMING.get(index) if gemm else Nonelogger.info(- "diffusion.int8_gemm: %d int8 Linear(s) %s on sm_%d%d%s%s",+ "diffusion.int8_gemm: %d int8 Linear(s) %s on sm_%d%d%s%s%s",count,"run the fused-dequant GEMM (bf16 out)" if gemm else "keep cuBLAS",*torch.cuda.get_device_capability(index),+ f" (latency check: {timing['fused']:.0f} vs {timing['stock']:.0f} us cuBLAS)"+ if timing+ else "",(", ConvRot rotation fused into the act quant ("+ ("H16 x H16 kernel" if _H16_DEVICE.get(index) else "dense rotation kernel")diff --git a/studio/backend/core/inference/diffusion_qwenimage21_fused.py b/studio/backend/core/inference/diffusion_qwenimage21_fused.pyindex 272e420fc14..fb26c104933 100644--- a/studio/backend/core/inference/diffusion_qwenimage21_fused.py+++ b/studio/backend/core/inference/diffusion_qwenimage21_fused.py@@ -45,10 +45,13 @@ def arch_ok(device: Any = None) -> bool:index = torch.cuda.current_device()else:index = dev.index if dev.index is not None else torch.cuda.current_device()- from .diffusion_int8_gemm import _gates, int8_gemm_mode+ from .diffusion_int8_gemm import _gates, int8_gemm_mode, rotated_swappedif int8_gemm_mode() == "off":return True+ swapped = rotated_swapped(index)+ if swapped is not None:+ return not swappedgemm, rotq = _gates(torch.cuda.get_device_capability(index))return not (gemm or rotq)except Exception: # noqa: BLE001@@ -239,10 +242,13 @@ def _swap_covers(index: int) -> bool:if not torch.cuda.is_available() or is_rocm_torch(torch):return False- from .diffusion_int8_gemm import _gates, int8_gemm_mode+ from .diffusion_int8_gemm import _gates, int8_gemm_mode, rotated_swappedif int8_gemm_mode() == "off":return False+ swapped = rotated_swapped(index)+ if swapped is not None:+ return swapped_, rotq = _gates(torch.cuda.get_device_capability(index))if not rotq:return Falsediff --git a/studio/backend/tests/test_diffusion_int8_gemm_unlisted.py b/studio/backend/tests/test_diffusion_int8_gemm_unlisted.pynew file mode 100644index 00000000000..323fb7c9613--- /dev/null+++ b/studio/backend/tests/test_diffusion_int8_gemm_unlisted.py@@ -0,0 +1,290 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Unlisted-arch dispatch of the fused-dequant int8 GEMM (``diffusion_int8_gemm.unlisted_config``): an sm80+ arch with+no ``_ARCH_CONFIG`` tile runs ``_FALLBACK_CONFIG`` only when it passes the bit-exact probe and is not slower than+cuBLAS ``_int_mm``. Capability, probe and timer are faked, so these run on CPU."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_convrot_quant as cq+from core.inference import diffusion_int8_gemm as g8+from core.inference import diffusion_qwenimage21_fused as qf++_ENVS = (+ g8.INT8_GEMM_ENV,+ g8.INT8_GEMM_UNLISTED_ENV,+ g8.INT8_GEMM_CONVROT_ENV,+ cq.INT8_ROTQUANT_ENV,+)+++def _reset():+ for d in (+ g8._DEVICE_CFG,+ g8._DEVICE_TILES,+ g8._UNLISTED_PROBE,+ g8._UNLISTED_OK,+ g8._UNLISTED_TIMING,+ ):+ d.clear()+ g8._UNLISTED_CARDS.clear()++[email protected](autouse = True)+def _clean(monkeypatch):+ for name in _ENVS:+ monkeypatch.delenv(name, raising = False)+ _reset()+ yield+ _reset()+++class _Fake:+ def __init__(+ self,+ monkeypatch,+ cap,+ probe = True,+ timing = None,+ ):+ self.probes, self.timed = [], []+ self.timing = timing if timing is not None else {"fused": 10.0, "stock": 20.0}+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: cap)+ monkeypatch.setattr(g8, "is_rocm_torch", lambda t: False)+ monkeypatch.setattr(g8, "_triton_version_ok", lambda version = None: True)+ monkeypatch.setattr(g8, "_kernels", lambda: object())+ monkeypatch.setattr(g8, "_op", lambda: "op")+ monkeypatch.setattr(g8, "_probe", lambda index, cfg: self.probes.append(cfg) or probe)+ monkeypatch.setattr(g8, "_unlisted_latency", self._latency)++ def _latency(self, index, cfg):+ self.timed.append(cfg)+ if isinstance(self.timing, Exception):+ raise self.timing+ return self.timing+++ "cap, on",+ [+ ((7, 5), False),+ ((8, 0), False), # listed: own tile+ ((8, 6), True), # RTX 30xx / A6000 / A10+ ((8, 7), True),+ ((8, 9), False),+ ((9, 0), True), # H100+ ((10, 0), False), # measured slower than cuBLAS+ ((10, 3), False),+ ((12, 0), False),+ ((12, 1), True),+ ],+)+def test_arch_table(cap, on):+ assert g8.unlisted_wanted(cap) is on+ if cap >= (8, 0) and cap not in g8._UNLISTED_SKIP: # exactly one route per sm80+ arch+ assert (g8.arch_config(cap) is not None) is not on+++def test_fallback_tile_fits_a_99kb_part():+ bm, bn, bk, _gm, _warps, stages = g8._FALLBACK_CONFIG+ assert (bm * bk + bk * bn) * stages <= 99 * 1024++[email protected]("env", [g8.INT8_GEMM_UNLISTED_ENV, g8.INT8_GEMM_ENV])+def test_kill_switches(monkeypatch, env):+ monkeypatch.setenv(env, "0")+ fake = _Fake(monkeypatch, (8, 6))+ assert not g8.unlisted_wanted((8, 6))+ assert g8.device_config(0) is None+ assert fake.probes == [] and fake.timed == []+++ "timing, ok",+ [+ ({"fused": 10.0, "stock": 20.0}, True),+ ({"fused": 20.0, "stock": 20.0}, True), # a tie is not slower+ ({"fused": 30.0, "stock": 20.0}, False),+ (RuntimeError("capture failed"), False),+ ],+)+def test_latency_gate(monkeypatch, timing, ok):+ fake = _Fake(monkeypatch, (8, 6), timing = timing)+ assert g8.device_config(0) == (g8._FALLBACK_CONFIG if ok else None)+ assert fake.probes == [g8._FALLBACK_CONFIG] and fake.timed == [g8._FALLBACK_CONFIG]+ assert g8._DEVICE_TILES[0] == ()++[email protected]("env, name", [(g8.INT8_GEMM_UNLISTED_ENV, "1"), (g8.INT8_GEMM_ENV, "1")])+def test_force_skips_the_latency_check_not_the_probe(monkeypatch, env, name):+ monkeypatch.setenv(env, name)+ fake = _Fake(monkeypatch, (8, 6), timing = {"fused": 30.0, "stock": 20.0})+ assert g8.device_config(0) == g8._FALLBACK_CONFIG+ assert fake.probes == [g8._FALLBACK_CONFIG] and fake.timed == []+++def test_failed_probe_keeps_stock_and_never_times(monkeypatch):+ fake = _Fake(monkeypatch, (9, 0), probe = False)+ assert g8.device_config(0) is None+ assert fake.timed == []+++def test_decision_is_cached_per_device_and_the_switch_acts_between_loads(monkeypatch):+ fake = _Fake(monkeypatch, (8, 6))+ assert g8.device_config(0) == g8._FALLBACK_CONFIG+ assert g8.device_config(0) == g8._FALLBACK_CONFIG+ assert len(fake.probes) == 1 and len(fake.timed) == 1+ monkeypatch.setenv(g8.INT8_GEMM_UNLISTED_ENV, "0")+ assert g8.device_config(0) is None and g8._DEVICE_CFG[0] is None+ monkeypatch.delenv(g8.INT8_GEMM_UNLISTED_ENV)+ assert g8.device_config(0) == g8._FALLBACK_CONFIG+ assert len(fake.probes) == 1 and len(fake.timed) == 1+ assert g8.device_config(1) == g8._FALLBACK_CONFIG # another card: its own probe and check+ assert len(fake.probes) == 2 and len(fake.timed) == 2++[email protected]("cap", [(8, 9), (10, 0), (7, 5)])+def test_listed_skipped_and_pre_sm80_archs_never_time(monkeypatch, cap):+ fake = _Fake(monkeypatch, cap)+ g8.device_config(0)+ assert fake.timed == []+ if cap != (8, 9):+ assert fake.probes == [] and g8._DEVICE_CFG[0] is None+++def test_failed_launch_sticks_on_an_unlisted_card(monkeypatch):+ _Fake(monkeypatch, (8, 6))+ assert g8.device_config(0) == g8._FALLBACK_CONFIG+ g8._DEVICE_CFG[None] = g8._FALLBACK_CONFIG # CPU tensors report device index None+ g8._UNLISTED_CARDS.add(None)++ def boom(*args):+ raise RuntimeError("out of resources")++ monkeypatch.setattr(g8, "_launch", boom)+ monkeypatch.setattr(g8, "reference", lambda *args: "stock")+ a, w = torch.zeros(32, 1024, dtype = torch.int8), torch.zeros(64, 1024, dtype = torch.int8)+ assert g8._run(a, w, torch.ones(32), torch.ones(64), None) == "stock"+ assert g8._UNLISTED_PROBE[None] is False+ assert g8.unlisted_config(None) is None+++ "gemm, rot, swapped",+ [(True, True, True), (True, False, False), (False, True, False), (False, False, False)],+)+def test_rotated_linears_swap_only_with_both_kernels(monkeypatch, gemm, rot, swapped):+ fake = _Fake(monkeypatch, (8, 6), timing = {"fused": 10.0 if gemm else 30.0, "stock": 20.0})+ monkeypatch.setattr(g8, "_unlisted_rotated", lambda index: rot)+ monkeypatch.setattr(cq, "is_rocm_torch", lambda t: False)+ monkeypatch.setattr(qf, "is_rocm_torch", lambda t: False)+ assert g8.rotated_swapped(0) is swapped+ # Qwen-Image-2.1 stock fusions (fused QKV, rotq_h16 via arm_rotated) stand down only when the swap takes them+ assert qf.arch_ok(torch.device("cuda", 0)) is not swapped+ assert qf._swap_covers(0) is swapped+ assert len(fake.timed) <= 1++[email protected]("timing", [{"fused": 10.0, "stock": 20.0}, {"fused": 30.0, "stock": 20.0}])+def test_gate_takes_no_int8_fused_lock(monkeypatch, timing):+ """``diffusion_int8_fused._finalize`` asks ``rotated_ff`` (so this gate) while holding its lock: a gate that+ took it (``arm_rotated``) hung every load whose fused GEMM stayed off."""+ import threading++ from core.inference import diffusion_int8_fused as f8++ _Fake(monkeypatch, (8, 6), timing = timing)+ monkeypatch.setattr(f8, "_act_quant_device_ok", lambda index: True)+ monkeypatch.setattr(f8, "_act_quant_op", lambda: "actq")+ seen = []+ monkeypatch.setattr(+ cq, "unlisted_device_ok", lambda index, stock = None: seen.append(stock) or True+ )+ out = []+ with f8._LOCK:+ t = threading.Thread(target = lambda: out.append(g8.rotated_swapped(0)), daemon = True)+ t.start()+ t.join(10)+ assert not t.is_alive() and out == [timing["fused"] <= timing["stock"]]+ assert seen == ["actq"]++[email protected]("cap", [(8, 0), (8, 9), (12, 0)])+def test_listed_archs_keep_the_static_gate(monkeypatch, cap):+ _Fake(monkeypatch, cap)+ assert g8.rotated_swapped(0) is None+++def _model():+ lins = torch.nn.ModuleList([torch.nn.Linear(256, 256, bias = False) for _ in range(3)])+ return lins, {lins[0]: None, lins[1]: 256, lins[2]: 256} # one plain, two rotated+++def _swap(monkeypatch, gemm, rot):+ lins, groups = _model()+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (8, 6))+ monkeypatch.setattr(+ g8, "_eligible", lambda m: ("v2", groups[m], None, m.weight) if m in groups else None+ )+ monkeypatch.setattr(g8, "_split_cards", lambda t: ())+ monkeypatch.setattr(g8, "device_config", lambda index: g8._FALLBACK_CONFIG if gemm else None)+ monkeypatch.setattr(cq, "rotquant_device_config", lambda index: None)+ monkeypatch.setattr(g8, "_unlisted_rotated", lambda index: rot)+ monkeypatch.setattr(g8, "_op", lambda: "op")+ g8._UNLISTED_CARDS.add(0)+ count = g8._swap_on(lins, None, 0)+ recs = [m.__dict__.get(g8._REC) for m in lins]+ g8.uninstall(lins)+ return count, recs+++def test_swap_unlisted_with_rotq_h16_takes_every_linear(monkeypatch):+ count, recs = _swap(monkeypatch, gemm = True, rot = True)+ assert count == 3+ assert [r[2] for r in recs] == [+ False,+ True,+ True,+ ] # rotation fused into the act quant on the rotated ones+ assert all(r[4] for r in recs) # fused GEMM on all+++def test_swap_unlisted_without_rotq_h16_leaves_rotated_linears_to_main(monkeypatch):+ count, recs = _swap(monkeypatch, gemm = True, rot = False)+ assert count == 1+ assert recs[0] is not None and recs[0][4] and recs[1] is None and recs[2] is None+++def test_swap_unlisted_slower_gemm_swaps_nothing(monkeypatch):+ count, recs = _swap(monkeypatch, gemm = False, rot = True)+ assert count == 0 and recs == [None, None, None]++[email protected]("env, deferred", [(None, True), ("0", False)])+def test_deferred_install_gates_in_an_unlisted_arch(monkeypatch, env, deferred):+ from core.inference import diffusion_int8_fused as f8++ _Fake(monkeypatch, (8, 6))+ if env is not None:+ monkeypatch.setenv(g8.INT8_GEMM_UNLISTED_ENV, env)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(f8, "resident_cuda_device", lambda t: None)+ monkeypatch.setattr(g8, "candidates", lambda t, rotated_only = False: 0 if rotated_only else 3)+ queued = []+ monkeypatch.setattr(f8, "run_on_first_call", lambda t, name, fn: queued.append(name))+ assert g8.install(torch.nn.Linear(64, 64)) == (3 if deferred else 0)+ assert queued == (["int8_gemm"] if deferred else [])+++def test_switch_joins_the_compile_cache_key(monkeypatch):+ from core.inference import diffusion_compile_cache as cc++ assert g8.INT8_GEMM_UNLISTED_ENV not in cc.graph_switches()+ monkeypatch.setenv(g8.INT8_GEMM_UNLISTED_ENV, "0")+ assert cc.graph_switches()[g8.INT8_GEMM_UNLISTED_ENV] == "off"