Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: run the fused-dequant int8 GEMM on untuned sm80+ archs behind a probe and latency check (#15654)

Daniel Han36bd1af5964b
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index ac823469d82..66277e50c47 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -221,6 +221,7 @@ _GRAPH_SWITCHES = {
"UNSLOTH_DIFFUSION_INT8_GEMM": "auto",
"UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT": "on",
"UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED": "on",
+ "UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED": "auto",
"UNSLOTH_DIFFUSION_INT8_ROTQUANT": "auto",
"UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED": "auto",
"UNSLOTH_DIFFUSION_INT8_FUSED": "on",
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index 109cdb29b33..af744793d03 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""int8 GEMM with torchao dequant epilogue in Triton, bf16 out; sm80/sm89/sm120 only, else stock
+"""int8 GEMM with torchao dequant epilogue in Triton, bf16 out; sm80/sm89/sm120 measured, other sm80+ behind a
+bit-exact probe and a latency check (UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED=0 keeps them stock), sm100 / sm103 stock
(ROCm and CPU never reach it); Linear output bit-identical, compiled block not end to end;
UNSLOTH_DIFFUSION_INT8_GEMM=0 disables."""
@@ -19,6 +20,7 @@ INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"
INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"
INT8_GEMM_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_TILES"
INT8_SWIGLU_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED"
+INT8_GEMM_UNLISTED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED"
_MIN_TRITON = (3, 2)
# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge.
_MIN_ROWS = 17
@@ -50,8 +52,10 @@ _ARCH_CONFIG = {
(8, 9): (256, 128, 128, 8, 8, 3), # L4
(12, 0): (128, 128, 64, 8, 4, 4), # RTX PRO 6000
}
-# When the arch tile does not fit this part's shared memory.
+# When the arch tile does not fit this part's shared memory, and the unlisted archs' tile (64 KB of stages: fits 99 KB).
_FALLBACK_CONFIG = (128, 128, 64, 8, 4, 4)
+# Measured slower than cuBLAS: not probed unless forced.
+_UNLISTED_SKIP = ((10, 0), (10, 3))
# First match wins, bounds inclusive. No L4 rule on purpose: power-capped clocks lose end to end.
# (major, minor) -> ((N_lo, N_hi, K_lo, K_hi, tile), ...), M >= _SHAPE_MIN_M only.
@@ -109,6 +113,33 @@ def _triton_version_ok(version: Optional[str] = None) -> bool:
return bool(match) and (int(match.group(1)), int(match.group(2))) >= _MIN_TRITON
+def unlisted_mode() -> str:
+ """'off', 'force' (no latency check) or 'auto' for the sm80+ archs without an ``_ARCH_CONFIG`` tile."""
+ gemm = int8_gemm_mode()
+ if gemm != "auto":
+ return gemm
+ raw = (os.environ.get(INT8_GEMM_UNLISTED_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no"):
+ return "off"
+ if raw in ("1", "on", "true", "yes", "force"):
+ return "force"
+ return "auto"
+
+
+def unlisted_arch(capability: Optional[tuple]) -> bool:
+ if capability is None:
+ return False
+ cap = (int(capability[0]), int(capability[1]))
+ return cap >= (8, 0) and cap not in _ARCH_CONFIG
+
+
+def unlisted_wanted(capability: Optional[tuple], mode: Optional[str] = None) -> bool:
+ mode = unlisted_mode() if mode is None else mode
+ if mode == "off" or not unlisted_arch(capability):
+ return False
+ return mode == "force" or (int(capability[0]), int(capability[1])) not in _UNLISTED_SKIP
+
+
def arch_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:
"""The tile for this compute capability, or None where the lever stays off."""
mode = int8_gemm_mode() if mode is None else mode
@@ -308,6 +339,8 @@ def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
_DEVICE_TILES[index] = () # drop the shape tiles, keep the probed default
return _run(a, w, xs, ws, bias)
_DEVICE_CFG[index] = None
+ if index in _UNLISTED_CARDS:
+ _UNLISTED_PROBE[index] = False
return reference(a, w, xs, ws, bias)
@@ -352,7 +385,10 @@ def shape_tiles(capability: Optional[tuple]) -> tuple:
def device_config(index: int) -> Optional[tuple]:
"""Probe once per device: arch gate, Triton, op registration, then a launch that must match the eager torchao
epilogue bit for bit (bf16 and fp32 activation scales, bias, ragged M / N / K). None keeps the stock path. Each
- per-shape tile is probed the same way; one that fails leaves its shapes on the default tile."""
+ per-shape tile is probed the same way; one that fails leaves its shapes on the default tile. Unlisted archs:
+ ``unlisted_config``, re-read per call so its switch acts between loads."""
+ if index in _UNLISTED_CARDS:
+ return unlisted_config(index)
if index in _DEVICE_CFG:
return _DEVICE_CFG[index]
cfg = None
@@ -367,6 +403,9 @@ def device_config(index: int) -> Optional[tuple]:
and _op() is not None
):
cap = torch.cuda.get_device_capability(index)
+ if unlisted_arch(cap):
+ _UNLISTED_CARDS.add(index)
+ return unlisted_config(index)
want = arch_config(cap)
for cand in (want, _FALLBACK_CONFIG) if want is not None else ():
if cand is not None and _probe(index, cand):
@@ -386,6 +425,99 @@ def device_config(index: int) -> Optional[tuple]:
return cfg
+_UNLISTED_CARDS: set = set()
+_UNLISTED_PROBE: dict = {}
+_UNLISTED_OK: dict = {}
+_UNLISTED_TIMING: dict = {}
+# (M, N, K): Qwen-Image-2.1 attention projection, MLP in, MLP down
+_UNLISTED_LATENCY_SHAPES = ((2048, 4096, 4096), (2048, 12288, 4096), (2048, 4096, 12288))
+
+
+def _unlisted_latency(index: int, cfg: tuple) -> dict:
+ """{"fused": us, "stock": us} summed over ``_UNLISTED_LATENCY_SHAPES``, each arm captured in a CUDA graph and
+ replayed, best of seven interleaved rounds. Stock = cuBLAS ``_int_mm``; its epilogue fuses into the consumer, so
+ each arm ends in one read of its output (int32 for stock, bf16 here) standing in for that consumer."""
+ import torch
+ from .diffusion_cuda_graph import CAPTURE_MODE, _capturing, synchronize_outside_capture
+
+ dev = torch.device("cuda", index)
+ g = torch.Generator(device = "cpu").manual_seed(5)
+ ops = []
+ for m, n, k in _UNLISTED_LATENCY_SHAPES:
+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)
+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)
+ xs = torch.full((m,), 1e-3, device = dev, dtype = torch.bfloat16)
+ ws = torch.full((n,), 1e-3, device = dev, dtype = torch.bfloat16)
+ ops.append((a, w, xs, ws))
+ fns = {
+ "fused": lambda a, w, xs, ws: _launch(a, w, xs, ws, None, cfg).amax(),
+ "stock": lambda a, w, xs, ws: torch._int_mm(a, w.t()).amax(),
+ }
+ reps = 2
+ graphs = {}
+ with torch.cuda.device(dev):
+ side = torch.cuda.Stream(dev)
+ side.wait_stream(torch.cuda.current_stream(dev))
+ with torch.cuda.stream(side):
+ for fn in fns.values():
+ for op in ops:
+ fn(*op) # compile / autotune outside the capture
+ torch.cuda.current_stream(dev).wait_stream(side)
+ synchronize_outside_capture(dev)
+ for name, fn in fns.items():
+ graph = torch.cuda.CUDAGraph()
+ with _capturing(), torch.cuda.graph(graph, capture_error_mode = CAPTURE_MODE):
+ for _ in range(reps):
+ for op in ops:
+ fn(*op)
+ graphs[name] = graph
+ for graph in graphs.values():
+ graph.replay()
+ synchronize_outside_capture(dev)
+ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
+ best = {name: float("inf") for name in graphs}
+ for _ in range(7):
+ for name, graph in graphs.items():
+ start.record()
+ graph.replay()
+ end.record()
+ end.synchronize()
+ best[name] = min(best[name], start.elapsed_time(end) * 1000.0 / reps)
+ del graphs
+ return best
+
+
+def unlisted_config(index: int) -> Optional[tuple]:
+ """Unlisted sm80+ card: ``_FALLBACK_CONFIG`` when it passed the bit-exact probe (once per device) and the latency
+ check (once per (device, mode); skipped under force), else None. Outside any trace / capture."""
+ import torch
+
+ mode = unlisted_mode()
+ cfg = None
+ try:
+ if unlisted_wanted(torch.cuda.get_device_capability(index), mode):
+ if index not in _UNLISTED_PROBE:
+ _UNLISTED_PROBE[index] = _probe(index, _FALLBACK_CONFIG)
+ if _UNLISTED_PROBE[index]:
+ key = (index, mode)
+ if key not in _UNLISTED_OK:
+ ok = True
+ if mode != "force":
+ try:
+ timing = _unlisted_latency(index, _FALLBACK_CONFIG)
+ _UNLISTED_TIMING[index] = {k: round(v, 1) for k, v in timing.items()}
+ ok = timing["fused"] <= timing["stock"]
+ except Exception: # noqa: BLE001 - main's path stays
+ ok = False
+ _UNLISTED_OK[key] = ok
+ cfg = _FALLBACK_CONFIG if _UNLISTED_OK[key] else None
+ except Exception: # noqa: BLE001
+ cfg = None
+ _DEVICE_TILES[index] = ()
+ _DEVICE_CFG[index] = cfg
+ return cfg
+
+
# (M, N, K, bias, fp32 scales). Ragged N / K stay on 16: an off-16 probe compiles the spilling variant (see ``_aligned``).
_PROBE_SHAPES = (
(257, 384, 512, False, False),
@@ -847,6 +979,35 @@ def _gates(capability: Optional[tuple]) -> tuple:
return gemm, rotq
+def _unlisted_rotated(index: int) -> bool:
+ """Unlisted card: ``rotq_h16`` armed for it (probe + its own latency check). Takes no ``diffusion_int8_fused``
+ lock: its SwiGLU fusion reaches this through ``rotated_ff`` while holding it."""
+ if not convrot_enabled():
+ return False
+ try:
+ from . import diffusion_convrot_quant as cq
+ from .diffusion_int8_fused import _act_quant_device_ok, _act_quant_op
+
+ actq = None
+ if (index, cq.unlisted_mode()) not in cq._UNLISTED_DEVICE:
+ actq = _act_quant_op() if _act_quant_device_ok(index) else None
+ return bool(cq.unlisted_device_ok(index, actq))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def rotated_swapped(index: int) -> Optional[bool]:
+ """On an unlisted card (outside any trace): True when rotated Linears go through this swap (fused GEMM and the
+ ``rotq_h16`` rotation + act quant both on), so the Qwen-Image-2.1 / Z-Image stock fusions stand down. False keeps
+ them, exactly as on main. None on a listed arch (``_gates`` decides)."""
+ import torch
+
+ cap = torch.cuda.get_device_capability(index)
+ if not unlisted_arch(cap) or _gates(cap)[1] or int8_gemm_mode() == "off":
+ return None
+ return _unlisted_rotated(index) and device_config(index) is not None
+
+
def install(
transformer: Any,
logger: Any = None,
@@ -874,7 +1035,9 @@ def install(
return 0
if not _triton_version_ok() or _kernels() is None:
return 0
- gemm, rotq = _gates(torch.cuda.get_device_capability(torch.cuda.current_device()))
+ cap = torch.cuda.get_device_capability(torch.cuda.current_device())
+ gemm, rotq = _gates(cap)
+ gemm = gemm or unlisted_wanted(cap)
if not (gemm or rotq):
return 0
except Exception: # noqa: BLE001
@@ -970,7 +1133,26 @@ def _swap_on(
from .diffusion_convrot_quant import rotquant_device_config
gemm = device_config(index) is not None
- rotq = any(r[1] is not None for _, r in recs) and rotquant_device_config(index) is not None
+ rotated = any(r[1] is not None for _, r in recs)
+ rotq = rotated and rotquant_device_config(index) is not None
+ if gemm and rotated and not rotq and index in _UNLISTED_CARDS:
+ # unlisted card: rotated Linears only with rotq_h16; else the stock fusions keep them, as on main
+ rotq = _unlisted_rotated(index)
+ if not rotq:
+ recs = [(m, r) for m, r in recs if r[1] is None]
+ if logger is not None:
+ logger.info(
+ "diffusion.int8_gemm: rotated Linears keep the stock path on cuda:%d (rotq_h16 off)",
+ index,
+ )
+ if not gemm and logger is not None and _UNLISTED_TIMING.get(index):
+ t = _UNLISTED_TIMING[index]
+ logger.info(
+ "diffusion.int8_gemm: fused-dequant GEMM kept off on cuda:%d (latency check: %.0f vs %.0f us cuBLAS)",
+ index,
+ t["fused"],
+ t["stock"],
+ )
if not gemm:
# no fused GEMM here: only rotated Linears gain (one kernel instead of rotation GEMM + act quant)
recs = [(m, r) for m, r in recs if r[1] is not None] if rotq else []
@@ -1018,11 +1200,15 @@ def _swap_on(
)
if logger is not None and count:
from .diffusion_convrot_quant import _H16_DEVICE
+ timing = _UNLISTED_TIMING.get(index) if gemm else None
logger.info(
- "diffusion.int8_gemm: %d int8 Linear(s) %s on sm_%d%d%s%s",
+ "diffusion.int8_gemm: %d int8 Linear(s) %s on sm_%d%d%s%s%s",
count,
"run the fused-dequant GEMM (bf16 out)" if gemm else "keep cuBLAS",
*torch.cuda.get_device_capability(index),
+ f" (latency check: {timing['fused']:.0f} vs {timing['stock']:.0f} us cuBLAS)"
+ if timing
+ else "",
(
", ConvRot rotation fused into the act quant ("
+ ("H16 x H16 kernel" if _H16_DEVICE.get(index) else "dense rotation kernel")
diff --git a/studio/backend/core/inference/diffusion_qwenimage21_fused.py b/studio/backend/core/inference/diffusion_qwenimage21_fused.py
index 272e420fc14..fb26c104933 100644
--- a/studio/backend/core/inference/diffusion_qwenimage21_fused.py
+++ b/studio/backend/core/inference/diffusion_qwenimage21_fused.py
@@ -45,10 +45,13 @@ def arch_ok(device: Any = None) -> bool:
index = torch.cuda.current_device()
else:
index = dev.index if dev.index is not None else torch.cuda.current_device()
- from .diffusion_int8_gemm import _gates, int8_gemm_mode
+ from .diffusion_int8_gemm import _gates, int8_gemm_mode, rotated_swapped
if int8_gemm_mode() == "off":
return True
+ swapped = rotated_swapped(index)
+ if swapped is not None:
+ return not swapped
gemm, rotq = _gates(torch.cuda.get_device_capability(index))
return not (gemm or rotq)
except Exception: # noqa: BLE001
@@ -239,10 +242,13 @@ def _swap_covers(index: int) -> bool:
if not torch.cuda.is_available() or is_rocm_torch(torch):
return False
- from .diffusion_int8_gemm import _gates, int8_gemm_mode
+ from .diffusion_int8_gemm import _gates, int8_gemm_mode, rotated_swapped
if int8_gemm_mode() == "off":
return False
+ swapped = rotated_swapped(index)
+ if swapped is not None:
+ return swapped
_, rotq = _gates(torch.cuda.get_device_capability(index))
if not rotq:
return False
diff --git a/studio/backend/tests/test_diffusion_int8_gemm_unlisted.py b/studio/backend/tests/test_diffusion_int8_gemm_unlisted.py
new file mode 100644
index 00000000000..323fb7c9613
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_int8_gemm_unlisted.py
@@ -0,0 +1,290 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Unlisted-arch dispatch of the fused-dequant int8 GEMM (``diffusion_int8_gemm.unlisted_config``): an sm80+ arch with
+no ``_ARCH_CONFIG`` tile runs ``_FALLBACK_CONFIG`` only when it passes the bit-exact probe and is not slower than
+cuBLAS ``_int_mm``. Capability, probe and timer are faked, so these run on CPU."""
+
+from __future__ import annotations
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_convrot_quant as cq
+from core.inference import diffusion_int8_gemm as g8
+from core.inference import diffusion_qwenimage21_fused as qf
+
+_ENVS = (
+ g8.INT8_GEMM_ENV,
+ g8.INT8_GEMM_UNLISTED_ENV,
+ g8.INT8_GEMM_CONVROT_ENV,
+ cq.INT8_ROTQUANT_ENV,
+)
+
+
+def _reset():
+ for d in (
+ g8._DEVICE_CFG,
+ g8._DEVICE_TILES,
+ g8._UNLISTED_PROBE,
+ g8._UNLISTED_OK,
+ g8._UNLISTED_TIMING,
+ ):
+ d.clear()
+ g8._UNLISTED_CARDS.clear()
+
+
[email protected](autouse = True)
+def _clean(monkeypatch):
+ for name in _ENVS:
+ monkeypatch.delenv(name, raising = False)
+ _reset()
+ yield
+ _reset()
+
+
+class _Fake:
+ def __init__(
+ self,
+ monkeypatch,
+ cap,
+ probe = True,
+ timing = None,
+ ):
+ self.probes, self.timed = [], []
+ self.timing = timing if timing is not None else {"fused": 10.0, "stock": 20.0}
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: cap)
+ monkeypatch.setattr(g8, "is_rocm_torch", lambda t: False)
+ monkeypatch.setattr(g8, "_triton_version_ok", lambda version = None: True)
+ monkeypatch.setattr(g8, "_kernels", lambda: object())
+ monkeypatch.setattr(g8, "_op", lambda: "op")
+ monkeypatch.setattr(g8, "_probe", lambda index, cfg: self.probes.append(cfg) or probe)
+ monkeypatch.setattr(g8, "_unlisted_latency", self._latency)
+
+ def _latency(self, index, cfg):
+ self.timed.append(cfg)
+ if isinstance(self.timing, Exception):
+ raise self.timing
+ return self.timing
+
+
+ "cap, on",
+ [
+ ((7, 5), False),
+ ((8, 0), False), # listed: own tile
+ ((8, 6), True), # RTX 30xx / A6000 / A10
+ ((8, 7), True),
+ ((8, 9), False),
+ ((9, 0), True), # H100
+ ((10, 0), False), # measured slower than cuBLAS
+ ((10, 3), False),
+ ((12, 0), False),
+ ((12, 1), True),
+ ],
+)
+def test_arch_table(cap, on):
+ assert g8.unlisted_wanted(cap) is on
+ if cap >= (8, 0) and cap not in g8._UNLISTED_SKIP: # exactly one route per sm80+ arch
+ assert (g8.arch_config(cap) is not None) is not on
+
+
+def test_fallback_tile_fits_a_99kb_part():
+ bm, bn, bk, _gm, _warps, stages = g8._FALLBACK_CONFIG
+ assert (bm * bk + bk * bn) * stages <= 99 * 1024
+
+
[email protected]("env", [g8.INT8_GEMM_UNLISTED_ENV, g8.INT8_GEMM_ENV])
+def test_kill_switches(monkeypatch, env):
+ monkeypatch.setenv(env, "0")
+ fake = _Fake(monkeypatch, (8, 6))
+ assert not g8.unlisted_wanted((8, 6))
+ assert g8.device_config(0) is None
+ assert fake.probes == [] and fake.timed == []
+
+
+ "timing, ok",
+ [
+ ({"fused": 10.0, "stock": 20.0}, True),
+ ({"fused": 20.0, "stock": 20.0}, True), # a tie is not slower
+ ({"fused": 30.0, "stock": 20.0}, False),
+ (RuntimeError("capture failed"), False),
+ ],
+)
+def test_latency_gate(monkeypatch, timing, ok):
+ fake = _Fake(monkeypatch, (8, 6), timing = timing)
+ assert g8.device_config(0) == (g8._FALLBACK_CONFIG if ok else None)
+ assert fake.probes == [g8._FALLBACK_CONFIG] and fake.timed == [g8._FALLBACK_CONFIG]
+ assert g8._DEVICE_TILES[0] == ()
+
+
[email protected]("env, name", [(g8.INT8_GEMM_UNLISTED_ENV, "1"), (g8.INT8_GEMM_ENV, "1")])
+def test_force_skips_the_latency_check_not_the_probe(monkeypatch, env, name):
+ monkeypatch.setenv(env, name)
+ fake = _Fake(monkeypatch, (8, 6), timing = {"fused": 30.0, "stock": 20.0})
+ assert g8.device_config(0) == g8._FALLBACK_CONFIG
+ assert fake.probes == [g8._FALLBACK_CONFIG] and fake.timed == []
+
+
+def test_failed_probe_keeps_stock_and_never_times(monkeypatch):
+ fake = _Fake(monkeypatch, (9, 0), probe = False)
+ assert g8.device_config(0) is None
+ assert fake.timed == []
+
+
+def test_decision_is_cached_per_device_and_the_switch_acts_between_loads(monkeypatch):
+ fake = _Fake(monkeypatch, (8, 6))
+ assert g8.device_config(0) == g8._FALLBACK_CONFIG
+ assert g8.device_config(0) == g8._FALLBACK_CONFIG
+ assert len(fake.probes) == 1 and len(fake.timed) == 1
+ monkeypatch.setenv(g8.INT8_GEMM_UNLISTED_ENV, "0")
+ assert g8.device_config(0) is None and g8._DEVICE_CFG[0] is None
+ monkeypatch.delenv(g8.INT8_GEMM_UNLISTED_ENV)
+ assert g8.device_config(0) == g8._FALLBACK_CONFIG
+ assert len(fake.probes) == 1 and len(fake.timed) == 1
+ assert g8.device_config(1) == g8._FALLBACK_CONFIG # another card: its own probe and check
+ assert len(fake.probes) == 2 and len(fake.timed) == 2
+
+
[email protected]("cap", [(8, 9), (10, 0), (7, 5)])
+def test_listed_skipped_and_pre_sm80_archs_never_time(monkeypatch, cap):
+ fake = _Fake(monkeypatch, cap)
+ g8.device_config(0)
+ assert fake.timed == []
+ if cap != (8, 9):
+ assert fake.probes == [] and g8._DEVICE_CFG[0] is None
+
+
+def test_failed_launch_sticks_on_an_unlisted_card(monkeypatch):
+ _Fake(monkeypatch, (8, 6))
+ assert g8.device_config(0) == g8._FALLBACK_CONFIG
+ g8._DEVICE_CFG[None] = g8._FALLBACK_CONFIG # CPU tensors report device index None
+ g8._UNLISTED_CARDS.add(None)
+
+ def boom(*args):
+ raise RuntimeError("out of resources")
+
+ monkeypatch.setattr(g8, "_launch", boom)
+ monkeypatch.setattr(g8, "reference", lambda *args: "stock")
+ a, w = torch.zeros(32, 1024, dtype = torch.int8), torch.zeros(64, 1024, dtype = torch.int8)
+ assert g8._run(a, w, torch.ones(32), torch.ones(64), None) == "stock"
+ assert g8._UNLISTED_PROBE[None] is False
+ assert g8.unlisted_config(None) is None
+
+
+ "gemm, rot, swapped",
+ [(True, True, True), (True, False, False), (False, True, False), (False, False, False)],
+)
+def test_rotated_linears_swap_only_with_both_kernels(monkeypatch, gemm, rot, swapped):
+ fake = _Fake(monkeypatch, (8, 6), timing = {"fused": 10.0 if gemm else 30.0, "stock": 20.0})
+ monkeypatch.setattr(g8, "_unlisted_rotated", lambda index: rot)
+ monkeypatch.setattr(cq, "is_rocm_torch", lambda t: False)
+ monkeypatch.setattr(qf, "is_rocm_torch", lambda t: False)
+ assert g8.rotated_swapped(0) is swapped
+ # Qwen-Image-2.1 stock fusions (fused QKV, rotq_h16 via arm_rotated) stand down only when the swap takes them
+ assert qf.arch_ok(torch.device("cuda", 0)) is not swapped
+ assert qf._swap_covers(0) is swapped
+ assert len(fake.timed) <= 1
+
+
[email protected]("timing", [{"fused": 10.0, "stock": 20.0}, {"fused": 30.0, "stock": 20.0}])
+def test_gate_takes_no_int8_fused_lock(monkeypatch, timing):
+ """``diffusion_int8_fused._finalize`` asks ``rotated_ff`` (so this gate) while holding its lock: a gate that
+ took it (``arm_rotated``) hung every load whose fused GEMM stayed off."""
+ import threading
+
+ from core.inference import diffusion_int8_fused as f8
+
+ _Fake(monkeypatch, (8, 6), timing = timing)
+ monkeypatch.setattr(f8, "_act_quant_device_ok", lambda index: True)
+ monkeypatch.setattr(f8, "_act_quant_op", lambda: "actq")
+ seen = []
+ monkeypatch.setattr(
+ cq, "unlisted_device_ok", lambda index, stock = None: seen.append(stock) or True
+ )
+ out = []
+ with f8._LOCK:
+ t = threading.Thread(target = lambda: out.append(g8.rotated_swapped(0)), daemon = True)
+ t.start()
+ t.join(10)
+ assert not t.is_alive() and out == [timing["fused"] <= timing["stock"]]
+ assert seen == ["actq"]
+
+
[email protected]("cap", [(8, 0), (8, 9), (12, 0)])
+def test_listed_archs_keep_the_static_gate(monkeypatch, cap):
+ _Fake(monkeypatch, cap)
+ assert g8.rotated_swapped(0) is None
+
+
+def _model():
+ lins = torch.nn.ModuleList([torch.nn.Linear(256, 256, bias = False) for _ in range(3)])
+ return lins, {lins[0]: None, lins[1]: 256, lins[2]: 256} # one plain, two rotated
+
+
+def _swap(monkeypatch, gemm, rot):
+ lins, groups = _model()
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (8, 6))
+ monkeypatch.setattr(
+ g8, "_eligible", lambda m: ("v2", groups[m], None, m.weight) if m in groups else None
+ )
+ monkeypatch.setattr(g8, "_split_cards", lambda t: ())
+ monkeypatch.setattr(g8, "device_config", lambda index: g8._FALLBACK_CONFIG if gemm else None)
+ monkeypatch.setattr(cq, "rotquant_device_config", lambda index: None)
+ monkeypatch.setattr(g8, "_unlisted_rotated", lambda index: rot)
+ monkeypatch.setattr(g8, "_op", lambda: "op")
+ g8._UNLISTED_CARDS.add(0)
+ count = g8._swap_on(lins, None, 0)
+ recs = [m.__dict__.get(g8._REC) for m in lins]
+ g8.uninstall(lins)
+ return count, recs
+
+
+def test_swap_unlisted_with_rotq_h16_takes_every_linear(monkeypatch):
+ count, recs = _swap(monkeypatch, gemm = True, rot = True)
+ assert count == 3
+ assert [r[2] for r in recs] == [
+ False,
+ True,
+ True,
+ ] # rotation fused into the act quant on the rotated ones
+ assert all(r[4] for r in recs) # fused GEMM on all
+
+
+def test_swap_unlisted_without_rotq_h16_leaves_rotated_linears_to_main(monkeypatch):
+ count, recs = _swap(monkeypatch, gemm = True, rot = False)
+ assert count == 1
+ assert recs[0] is not None and recs[0][4] and recs[1] is None and recs[2] is None
+
+
+def test_swap_unlisted_slower_gemm_swaps_nothing(monkeypatch):
+ count, recs = _swap(monkeypatch, gemm = False, rot = True)
+ assert count == 0 and recs == [None, None, None]
+
+
[email protected]("env, deferred", [(None, True), ("0", False)])
+def test_deferred_install_gates_in_an_unlisted_arch(monkeypatch, env, deferred):
+ from core.inference import diffusion_int8_fused as f8
+
+ _Fake(monkeypatch, (8, 6))
+ if env is not None:
+ monkeypatch.setenv(g8.INT8_GEMM_UNLISTED_ENV, env)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(f8, "resident_cuda_device", lambda t: None)
+ monkeypatch.setattr(g8, "candidates", lambda t, rotated_only = False: 0 if rotated_only else 3)
+ queued = []
+ monkeypatch.setattr(f8, "run_on_first_call", lambda t, name, fn: queued.append(name))
+ assert g8.install(torch.nn.Linear(64, 64)) == (3 if deferred else 0)
+ assert queued == (["int8_gemm"] if deferred else [])
+
+
+def test_switch_joins_the_compile_cache_key(monkeypatch):
+ from core.inference import diffusion_compile_cache as cc
+
+ assert g8.INT8_GEMM_UNLISTED_ENV not in cc.graph_switches()
+ monkeypatch.setenv(g8.INT8_GEMM_UNLISTED_ENV, "0")
+ assert cc.graph_switches()[g8.INT8_GEMM_UNLISTED_ENV] == "off"