Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: route untuned archs (sm86 / 90 / 100 / 121) onto the factored ConvRot kernel behind a probe and latency check (#14988)

Daniel Han1d9c522c1b08
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index 1f12fab98b9..4c3e87803a7 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -298,6 +298,7 @@ _GRAPH_SWITCHES = {
"UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT": "on",
"UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED": "on",
"UNSLOTH_DIFFUSION_INT8_ROTQUANT": "auto",
+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED": "auto",
"UNSLOTH_DIFFUSION_INT8_FUSED": "on",
"UNSLOTH_DIFFUSION_ZIMAGE_FUSED": "on",
"UNSLOTH_DIFFUSION_Q21_REAL_ROPE": "on",
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.py
index 0e88ff4f60d..30be47ac5d8 100644
--- a/studio/backend/core/inference/diffusion_convrot_quant.py
+++ b/studio/backend/core/inference/diffusion_convrot_quant.py
@@ -27,6 +27,10 @@ H16 ROTATION (``rotq_h16``, default where ``rotq_i8`` passed): ``H256 = H16 (x)
everywhere. A different fp32 summation order: ~1 code in 10^6 differs by one step, scales equal; the probe gates it per
device at that tolerance (``codes_close``). ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_H16=0`` keeps the bit-identical dense kernel.
+UNLISTED ARCHS (sm86, sm87, sm90, sm100, sm103, sm121, ...: no ``_ROTQ_CONFIG`` entry, so no dense tile to probe):
+``rotq_h16`` needs no tile, so ``unlisted_device_ok`` probes it there directly, then a latency check against the two
+kernels it replaces (see below). Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED=0``.
+
SwiGLU PROLOGUE (``rotq_h16`` only): the down projection's codes come straight from the gate / value halves, the
product computed in-kernel like the compiled stock MLP; the [M, hidden] HBM round trip was ConvRot's largest cost over
plain int8 at Qwen-Image-2.1 1024px. Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_SWIGLU=0``.
@@ -836,6 +840,10 @@ def rotquant_device_config(index: int) -> Optional[tuple]:
sw = _swiglu_probe(index)
except Exception: # noqa: BLE001 - the dense kernel stays, no SwiGLU prologue
h16 = sw = False
+ elif _UNLISTED_DEVICE.get((index, unlisted_mode())):
+ h16 = (
+ _UNLISTED_PROBE.get(index) or False
+ ) # no dense tile: keep the verdict unlisted_device_ok armed
_H16_DEVICE[index] = h16
_SWIGLU_DEVICE[index] = sw
if cfg is not None:
@@ -1017,3 +1025,124 @@ def convrot_swiglu_act_quant(g2d: Any, v2d: Any, group: int, v2: bool) -> Option
if not _h16_ok(g2d, group, v2d):
return None
return _SWIGLU_HANDLE(g2d, v2d, group, v2)
+
+
+# Unlisted archs: main runs the rotation as a dense bf16 GEMM (1/8 the int8 rate on GA102) + a separate act quant.
+# rotq_h16 must pass the probe and be no slower than both, timed from CUDA-graph replays (the compiled denoiser replays
+# kernels; host launch cost must not count). ``..._UNLISTED=1`` skips the latency check, not the probe.
+INT8_ROTQUANT_UNLISTED_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED"
+_UNLISTED_DEVICE: dict = {}
+_UNLISTED_TIMING: dict = {}
+_UNLISTED_PROBE: dict = {} # index -> _h16_probe result (mode independent)
+_UNLISTED_LATENCY_SHAPES = ((2048, 4096), (1024, 12288))
+
+
+def unlisted_mode() -> str:
+ if rotquant_mode() == "off" or not rotquant_h16_enabled():
+ return "off"
+ return _env_mode(INT8_ROTQUANT_UNLISTED_ENV)
+
+
+def unlisted_wanted(capability: Optional[tuple], mode: Optional[str] = None) -> bool:
+ """True on a CUDA sm80+ arch with no dense ``rotq_i8`` tile (the listed archs keep their probed path)."""
+ mode = unlisted_mode() if mode is None else mode
+ if mode == "off" or capability is None:
+ return False
+ cap = (int(capability[0]), int(capability[1]))
+ return cap >= (8, 0) and rotquant_config(cap) is None
+
+
+def _unlisted_latency(
+ index: int,
+ tf32x3: bool,
+ stock_quant: Any = None,
+) -> dict:
+ """{"h16": us, "stock": us} over ``_UNLISTED_LATENCY_SHAPES``: each arm captured in a CUDA graph (two passes)
+ and replayed, best of seven interleaved rounds. Stock = the rotation GEMM, then ``stock_quant`` of its output when
+ given, else the GEMM alone (a lower bound)."""
+ import torch
+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
+
+ dev = torch.device("cuda", index)
+ xs = [torch.randn(m, k, device = dev).to(torch.bfloat16) for m, k in _UNLISTED_LATENCY_SHAPES]
+ h = build_convrot_hadamard(256, device = dev, dtype = torch.bfloat16)
+
+ def stock(x):
+ r = rotate_convrot_activation(x, h, 256)
+ return r if stock_quant is None else stock_quant(r)
+
+ fns = {"h16": lambda x: _h16_launch(x, "v2", tf32x3 = tf32x3), "stock": stock}
+ reps = 2
+ graphs = {}
+ with torch.cuda.device(dev):
+ side = torch.cuda.Stream(dev)
+ side.wait_stream(torch.cuda.current_stream(dev))
+ with torch.cuda.stream(side):
+ for fn in fns.values():
+ for x in xs:
+ fn(x) # compile / autotune outside the capture
+ torch.cuda.current_stream(dev).wait_stream(side)
+ torch.cuda.synchronize(dev)
+ for name, fn in fns.items():
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ for _ in range(reps):
+ for x in xs:
+ fn(x)
+ graphs[name] = graph
+ for graph in graphs.values():
+ graph.replay()
+ torch.cuda.synchronize(dev)
+ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
+ best = {name: float("inf") for name in graphs}
+ for _ in range(7):
+ for name, graph in graphs.items():
+ start.record()
+ graph.replay()
+ end.record()
+ end.synchronize()
+ best[name] = min(best[name], start.elapsed_time(end) * 1000.0 / reps)
+ del graphs
+ return best
+
+
+def unlisted_device_ok(index: int, stock_quant: Any = None) -> bool:
+ """Once per (device, mode), outside any trace / capture: arch gate, ``_h16_probe`` (once per device), latency
+ check (auto only). Keyed by mode so switching ``..._UNLISTED`` between loads takes effect without a restart."""
+ global _ROTQ_HANDLE
+ mode = unlisted_mode()
+ if mode == "off": # read per load: the switches act without a restart
+ return False
+ key = (index, mode)
+ if key not in _UNLISTED_DEVICE:
+ ok = False
+ try:
+ import torch
+ if (
+ torch.cuda.is_available()
+ and not is_rocm_torch(torch)
+ and _triton_ok()
+ and unlisted_wanted(torch.cuda.get_device_capability(index), mode)
+ and _kernels() is not None
+ and _rotq_op() is not None
+ ):
+ if index not in _UNLISTED_PROBE:
+ _UNLISTED_PROBE[index] = _h16_probe(index)
+ h16 = _UNLISTED_PROBE[index]
+ if h16:
+ ok = True
+ if mode != "force":
+ timing = _unlisted_latency(index, h16 == "tf32x3", stock_quant)
+ _UNLISTED_TIMING[index] = {k: round(v, 2) for k, v in timing.items()}
+ ok = timing["h16"] <= timing["stock"]
+ except Exception: # noqa: BLE001 - main's path stays
+ ok = False
+ _UNLISTED_DEVICE[key] = ok
+ ok = _UNLISTED_DEVICE[key]
+ h16 = _UNLISTED_PROBE.get(index)
+ if ok:
+ _H16_DEVICE[index] = h16
+ _ROTQ_HANDLE = _rotq_op()
+ elif h16: # probed unlisted card the latency check now rejects: drop a verdict a forced load left behind
+ _H16_DEVICE.pop(index, None)
+ return ok
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.py
index c230729c6c5..0418636dd1d 100644
--- a/studio/backend/core/inference/diffusion_int8_fused.py
+++ b/studio/backend/core/inference/diffusion_int8_fused.py
@@ -22,6 +22,8 @@ w2's own (rotating) forward. Same ops on the same values as the three stock rota
quant + ``_int_mm`` + epilogue without torchao 0.17's always-on zero-point pass, and the act quant itself as one probed
Triton kernel (``int8_act_quant_rows``, bit-identical to torchao 0.17 and 0.18): Inductor's codegen of the same per-row
quant ran 2-3x slower on B200 (Z-Image shapes, M 4224: K 3840 24-32 -> 11 us, K 10240 53-70 -> 32 us).
+A rotated input (ConvRot) is rotated and quantized by one kernel (``_rotated_act_quant``: ``rotq_h16`` of
+``diffusion_convrot_quant``) on an unlisted-arch card it was armed for, else rotation GEMM + act quant.
Kill switch: ``UNSLOTH_DIFFUSION_INT8_FUSED=0`` (all of the above).
"""
@@ -52,6 +54,8 @@ _OP_HANDLE: Any = None
_ACTQ_HANDLE: Any = None
# device indices whose own probe passed; the kernel never runs on a device that failed or was never probed
_ACTQ_DEVICES: frozenset = frozenset()
+_RQ_HANDLE: Any = None
+_RQ_DEVICES: frozenset = frozenset()
# UNSLOTH_DIFFUSION_INT8_FUSED=0 at the last install: int8_linear is then module(x)
_LINEAR_OFF = False
# torchao's safe_int_mm home, resolved at install (outside any trace) for the same reason.
@@ -1309,8 +1313,9 @@ def int8_linear_core(module: Any, x: Any) -> Any:
):
return _stock_call(module, x)
if rotated:
- x2d = _rotate(x2d, module.convrot_groupsize)
- xq, xs = _fast_act_quant(x2d, weight)
+ xq, xs = _rotated_act_quant(x2d, module.convrot_groupsize, weight)
+ else:
+ xq, xs = _fast_act_quant(x2d, weight)
y = _torchao_epilogue(
_int_mm(xq.reshape(-1, xq.shape[-1]), weight), xs, weight.scale, module.bias, x.dtype
)
@@ -1331,6 +1336,48 @@ def _rotate(x2d: Any, group: int) -> Any:
)
+def _rotated_act_quant(x2d: Any, group: int, weight: Any) -> tuple:
+ """``_fast_act_quant(_rotate(x2d, group), weight)``: one fused rotation + quant kernel (``rotq_h16``) on a device
+ it was armed for (an arch the tiled ``rotq_i8`` does not cover), else the rotation GEMM + act quant."""
+ import torch
+
+ if (
+ _RQ_HANDLE is not None
+ and group == 256
+ and x2d.is_cuda
+ and x2d.device.index in _RQ_DEVICES
+ and x2d.dtype == torch.bfloat16
+ and x2d.dim() == 2
+ and x2d.shape[0] > 0
+ and x2d.shape[1] % 256 == 0
+ ):
+ q, s = _RQ_HANDLE(x2d, group, True)
+ return q, s.reshape(-1, 1)
+ return _fast_act_quant(_rotate(x2d, group), weight)
+
+
+def arm_rotated(index: int) -> bool:
+ """Arm ``rotq_h16`` for card ``index`` (outside any trace); False keeps the rotation GEMM + act quant."""
+ global _RQ_HANDLE, _RQ_DEVICES
+ if int8_fused_disabled():
+ return False
+ try:
+ from .diffusion_convrot_quant import _rotq_op, unlisted_device_ok
+
+ actq = _act_quant_op() if _act_quant_device_ok(index) else None
+ ok = unlisted_device_ok(index, actq)
+ handle = _rotq_op() if ok else None
+ except Exception: # noqa: BLE001 - optimisation only
+ ok, handle = False, None
+ with _LOCK:
+ if ok and handle is not None:
+ _RQ_HANDLE = handle
+ _RQ_DEVICES = _RQ_DEVICES | {index}
+ return True
+ _RQ_DEVICES = _RQ_DEVICES - {index}
+ return False
+
+
def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:
"""SwiGLU MLP with plain int8 weights: one act quant, one GEMM over [gate | value], fused SiLU-product quant.
ConvRot parts: the shared input rotates once; a rotated down projection gets the bf16 product (see module doc)."""
@@ -1357,8 +1404,7 @@ def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) ->
return type(self).forward(self, hidden_states, *args, **kwargs)
down = _get(self, down_name)
if in_group:
- x2d = _rotate(x2d, in_group)
- xq, xs = _fast_act_quant(x2d, fused_in.weight)
+ xq, xs = _rotated_act_quant(x2d, in_group, fused_in.weight)
else:
xq, xs = _act_quant(x2d, fused_in.weight)
c = _int_mm(xq.reshape(-1, xq.shape[-1]), fused_in.weight)
@@ -1569,11 +1615,12 @@ def install(
offload_active: bool = False,
) -> int:
"""Idempotent; returns the (candidate) count. Must run before the first compiled forward, which traces ``forward``."""
- global _ACTQ_HANDLE, _ACTQ_DEVICES, _LINEAR_OFF
+ global _ACTQ_HANDLE, _ACTQ_DEVICES, _RQ_DEVICES, _LINEAR_OFF
if int8_fused_disabled():
# process-global: a previous load's probe must not outlive the kill switch
_ACTQ_HANDLE = None
_ACTQ_DEVICES = frozenset()
+ _RQ_DEVICES = frozenset()
_LINEAR_OFF = True
return 0
_LINEAR_OFF = False
@@ -1655,6 +1702,14 @@ def _swiglu_eligible(module: Any) -> bool:
return False
+def _has_rotated(transformer: Any) -> bool:
+ try:
+ from .diffusion_convrot import is_rotated_linear
+ return any(is_rotated_linear(m) for m in transformer.modules())
+ except Exception: # noqa: BLE001
+ return False
+
+
def _has_eligible(transformer: Any) -> bool:
return any(
_MARK in m.__dict__ or _ff_eligible(m) or _swiglu_eligible(m) or _flux_single_eligible(m)
@@ -1693,6 +1748,9 @@ def _finalize(transformer: Any, logger: Any = None) -> int:
indices = [dev.index if dev.index is not None else torch.cuda.current_device()]
count = 0
rearm = False
+ if _has_rotated(transformer):
+ for index in indices:
+ arm_rotated(index)
actq_ok = {index: _act_quant_device_ok(index) for index in indices}
actq = _act_quant_op() if any(actq_ok.values()) else None
with _LOCK:
@@ -1800,10 +1858,13 @@ def relocate_after_layer_split(transformer: Any, logger: Any = None) -> dict:
index = _module_card(module)
if index is not None:
cards.setdefault(index, []).append(module)
+ rotated = _has_rotated(transformer)
for index, modules in cards.items():
if _device_ok(index):
if _act_quant_device_ok(index) and _ACTQ_HANDLE is not None:
_ACTQ_DEVICES = _ACTQ_DEVICES | {index}
+ if rotated:
+ arm_rotated(index)
continue
for module in modules:
_restore_module(module)
diff --git a/studio/backend/core/inference/diffusion_qwenimage21_fused.py b/studio/backend/core/inference/diffusion_qwenimage21_fused.py
index 31b9fb1d691..2da69b7315b 100644
--- a/studio/backend/core/inference/diffusion_qwenimage21_fused.py
+++ b/studio/backend/core/inference/diffusion_qwenimage21_fused.py
@@ -4,9 +4,11 @@
"""Qwen-Image-2.1 ConvRot int8: one rotation + one act quant per shared input (QKV, SwiGLU gate/value) in the compile.
Rotation is on the shared input axis and int8 scales are per row, so fused outputs equal the separate rotated ones bit
-for bit. The fused QKV Linear: only where ``diffusion_int8_gemm`` leaves rotated Linears on the stock path (e.g. sm90,
-sm100), resident only. Where it swaps them, the patched helper routes Q / K / V through
-``diffusion_int8_gemm.shared_rotq_linears`` (Inductor does not merge the opaque kernel); resident or streamed.
+for bit. The fused QKV Linear: only where ``diffusion_int8_gemm`` leaves rotated Linears on the stock path (e.g. sm86,
+sm90, sm100), resident only; there the rotation and the act quant run as one ``rotq_h16`` kernel on a card that passed
+``diffusion_convrot_quant.unlisted_device_ok``, else as the rotation GEMM + act quant. Where it swaps them, the
+patched helper routes Q / K / V through ``diffusion_int8_gemm.shared_rotq_linears`` (Inductor does not merge the
+opaque kernel); resident or streamed.
Kill switch: ``UNSLOTH_DIFFUSION_Q21_CONVROT_FUSED=0``.
"""
@@ -290,6 +292,23 @@ def install(
return {"fused_qkv": shared or None, "out": None}
+def _rotq_note(cards: Any, armed: list) -> str:
+ """Log suffix: whether the fused rotq_h16 rotation + quant serves these cards, with its latency check."""
+ try:
+ from .diffusion_convrot_quant import _UNLISTED_TIMING
+ timing = {i: _UNLISTED_TIMING.get(i) for i in cards if _UNLISTED_TIMING.get(i)}
+ except Exception: # noqa: BLE001
+ timing = {}
+ shown = "; ".join(
+ f"cuda:{i} {t['h16']:.0f} vs {t['stock']:.0f} us stock"
+ for i, t in timing.items()
+ if "h16" in t and "stock" in t
+ )
+ if armed:
+ return ", rotation fused into the act quant (rotq_h16" + (f": {shown})" if shown else ")")
+ return f", fused rotation kept off ({shown})" if shown else ""
+
+
def install_modules(root: Any, logger: Any = None) -> dict:
result = {"fused_qkv": 0, "out": 0}
if q21_convrot_fused_disabled():
@@ -306,6 +325,10 @@ def install_modules(root: Any, logger: Any = None) -> dict:
todo = _rotated_attention(root)
if not todo or not _patch_prepare(logger):
return result
+ from .diffusion_int8_fused import arm_rotated
+
+ armed = [index for index in cards if arm_rotated(index)]
+ note = _rotq_note(cards, armed)
with _LOCK:
for module, parts in todo:
if _QKV_ATTR not in module.__dict__ and all(_plain(p) for p in parts):
@@ -324,9 +347,10 @@ def install_modules(root: Any, logger: Any = None) -> dict:
if logger is not None:
logger.info(
"diffusion.q21_convrot_fused: one ConvRot rotation + act quant for QKV on %d attention module(s), "
- "probed act quant on %d output projection(s)",
+ "probed act quant on %d output projection(s)%s",
result["fused_qkv"],
result["out"],
+ note,
)
return result
diff --git a/studio/backend/tests/test_diffusion_convrot_unlisted.py b/studio/backend/tests/test_diffusion_convrot_unlisted.py
new file mode 100644
index 00000000000..d31e5e4df4a
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_convrot_unlisted.py
@@ -0,0 +1,336 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for the unlisted-arch dispatch of ``rotq_h16`` (``diffusion_convrot_quant.unlisted_device_ok`` and
+``diffusion_int8_fused._rotated_act_quant``): every CUDA sm80+ arch without a dense ``rotq_i8`` tile gets the fused
+rotation + act quant when it passes the probe and is not slower than the rotation GEMM + act quant. The gate tests mock
+the capability and run anywhere; the kernel tests need NVIDIA CUDA, Triton and torchao."""
+
+from __future__ import annotations
+
+import pytest
+
+from core.inference import diffusion_convrot_quant as cq
+from core.inference import diffusion_int8_fused as f8
+
+torch = pytest.importorskip("torch")
+
+needs_cuda = pytest.mark.skipif(
+ not torch.cuda.is_available() or getattr(torch.version, "hip", None) is not None,
+ reason = "NVIDIA CUDA required",
+)
+
+_ENVS = (
+ cq.INT8_ROTQUANT_UNLISTED_ENV,
+ cq.INT8_ROTQUANT_H16_ENV,
+ cq.INT8_ROTQUANT_ENV,
+ "UNSLOTH_DIFFUSION_INT8_GEMM",
+)
+
+
[email protected](autouse = True)
+def _clean(monkeypatch):
+ for name in _ENVS:
+ monkeypatch.delenv(name, raising = False)
+ cq._UNLISTED_DEVICE.clear()
+ cq._UNLISTED_TIMING.clear()
+ cq._UNLISTED_PROBE.clear()
+ saved = dict(cq._H16_DEVICE)
+ cq._H16_DEVICE.clear() # a real-card verdict from an earlier test file must not leak in
+ yield
+ cq._UNLISTED_DEVICE.clear()
+ cq._UNLISTED_TIMING.clear()
+ cq._UNLISTED_PROBE.clear()
+ cq._H16_DEVICE.clear()
+ cq._H16_DEVICE.update(saved)
+
+
+ "cap, on",
+ [
+ ((7, 5), False), # T4: no bf16 tensor cores for rotq_h16's first dot
+ ((8, 0), False), # A100: dense rotq_i8 tile (and rotq_h16 on top of it)
+ ((8, 6), True), # RTX 30xx / A6000 / A10
+ ((8, 7), True), # Orin
+ ((8, 9), False), # L4 / RTX 40xx: tiled
+ ((9, 0), True), # H100
+ ((10, 0), True), # B200
+ ((10, 3), True), # B300
+ ((12, 0), False), # RTX PRO 6000 / RTX 50xx: tiled
+ ((12, 1), True), # DGX Spark
+ ],
+)
+def test_every_sm80_plus_arch_without_a_tile_is_covered(cap, on):
+ assert cq.unlisted_wanted(cap) is on
+ if cap >= (8, 0): # exactly one route to the fused rotation per sm80+ arch
+ assert (cq.rotquant_config(cap) is not None) is not on
+
+
+ "env",
+ [
+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED",
+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT_H16",
+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT",
+ "UNSLOTH_DIFFUSION_INT8_GEMM",
+ ],
+)
+def test_kill_switches(monkeypatch, env):
+ monkeypatch.setenv(env, "0")
+ for cap in ((8, 6), (9, 0), (10, 0), (12, 1)):
+ assert not cq.unlisted_wanted(cap)
+
+
+def test_forcing_the_dense_tile_takes_the_listed_route(monkeypatch):
+ monkeypatch.setenv(
+ cq.INT8_ROTQUANT_ENV, "1"
+ ) # dense fallback tile on every unmeasured sm80+ part
+ assert not cq.unlisted_wanted((8, 6)) and not cq.unlisted_wanted((10, 0))
+
+
+def _fake_device(monkeypatch, cap):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: cap)
+ monkeypatch.setattr(cq, "_triton_ok", lambda: True)
+ monkeypatch.setattr(cq, "_kernels", lambda: object())
+ monkeypatch.setattr(cq, "_rotq_op", lambda: "op")
+ monkeypatch.setattr(cq, "is_rocm_torch", lambda t: False)
+ monkeypatch.setattr(cq, "_ROTQ_HANDLE", None)
+
+
+ "timing, mode, ok",
+ [
+ ({"h16": 10.0, "stock": 20.0}, None, True),
+ ({"h16": 20.0, "stock": 20.0}, None, True), # a tie is not slower
+ (
+ {"h16": 30.0, "stock": 20.0},
+ None,
+ False,
+ ), # slower than the two kernels it replaces: main's path
+ ({"h16": 30.0, "stock": 20.0}, "1", True), # forced: no latency check
+ ],
+)
+def test_latency_gate(monkeypatch, timing, mode, ok):
+ _fake_device(monkeypatch, (8, 6))
+ if mode is not None:
+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, mode)
+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "ieee")
+ monkeypatch.setattr(cq, "_unlisted_latency", lambda index, tf32x3, stock = None: timing)
+ assert cq.unlisted_device_ok(0) is ok
+ assert (cq._H16_DEVICE.get(0) == "ieee") is ok
+ assert (cq._ROTQ_HANDLE == "op") is ok
+
+
+def test_latency_check_uses_the_probed_mode_and_the_callers_quant(monkeypatch):
+ _fake_device(monkeypatch, (9, 0))
+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "tf32x3")
+ seen = []
+ monkeypatch.setattr(
+ cq,
+ "_unlisted_latency",
+ lambda index, tf32x3, stock = None: seen.append((tf32x3, stock))
+ or {"h16": 1.0, "stock": 2.0},
+ )
+ quant = object()
+ assert (
+ cq.unlisted_device_ok(0, quant)
+ and seen == [(True, quant)]
+ and cq._H16_DEVICE[0] == "tf32x3"
+ )
+
+
+def test_failed_probe_keeps_main(monkeypatch):
+ _fake_device(monkeypatch, (10, 0))
+ monkeypatch.setattr(cq, "_h16_probe", lambda index: False)
+ monkeypatch.setattr(
+ cq, "_unlisted_latency", lambda *a, **k: pytest.fail("timed after a failed probe")
+ )
+ assert cq.unlisted_device_ok(0) is False and 0 not in cq._H16_DEVICE
+
+
[email protected]("cap", [(7, 5), (8, 0), (8, 9), (12, 0)])
+def test_listed_archs_and_sm75_never_probe(monkeypatch, cap):
+ _fake_device(monkeypatch, cap)
+ monkeypatch.setattr(cq, "_h16_probe", lambda index: pytest.fail("probed"))
+ assert cq.unlisted_device_ok(0) is False
+
+
+def test_kill_switch_acts_on_a_cached_device(monkeypatch):
+ _fake_device(monkeypatch, (8, 6))
+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "ieee")
+ monkeypatch.setattr(
+ cq, "_unlisted_latency", lambda index, tf32x3, stock = None: {"h16": 1.0, "stock": 2.0}
+ )
+ assert cq.unlisted_device_ok(0)
+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "0")
+ assert not cq.unlisted_device_ok(0)
+ monkeypatch.delenv(cq.INT8_ROTQUANT_UNLISTED_ENV)
+ assert cq.unlisted_device_ok(0)
+
+
+def _counting_slow_device(monkeypatch):
+ _fake_device(monkeypatch, (8, 6))
+ calls = {"probe": 0, "time": 0}
+
+ def probe(index):
+ calls["probe"] += 1
+ return "ieee"
+
+ def latency(
+ index,
+ tf32x3,
+ stock = None,
+ ):
+ calls["time"] += 1
+ return {"h16": 30.0, "stock": 20.0} # slower: auto rejects
+
+ monkeypatch.setattr(cq, "_h16_probe", probe)
+ monkeypatch.setattr(cq, "_unlisted_latency", latency)
+ return calls
+
+
+def test_force_after_an_auto_rejection_enables_the_card(monkeypatch):
+ calls = _counting_slow_device(monkeypatch)
+ assert not cq.unlisted_device_ok(0) and 0 not in cq._H16_DEVICE
+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "1")
+ assert cq.unlisted_device_ok(0) and cq._H16_DEVICE[0] == "ieee"
+ assert calls == {"probe": 1, "time": 1} # probed once per device, never timed under force
+
+
+def test_auto_after_a_forced_load_runs_the_latency_check(monkeypatch):
+ calls = _counting_slow_device(monkeypatch)
+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "1")
+ assert cq.unlisted_device_ok(0) and cq._H16_DEVICE[0] == "ieee"
+ monkeypatch.delenv(cq.INT8_ROTQUANT_UNLISTED_ENV)
+ assert not cq.unlisted_device_ok(0) and 0 not in cq._H16_DEVICE
+ assert not cq.unlisted_device_ok(0) # repeat auto load: cached, not re-timed
+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "1")
+ assert cq.unlisted_device_ok(0) and cq._H16_DEVICE[0] == "ieee"
+ assert calls == {"probe": 1, "time": 1}
+
+
+def test_arm_rotated_follows_the_gate(monkeypatch):
+ monkeypatch.setattr(f8, "_RQ_DEVICES", frozenset())
+ monkeypatch.setattr(f8, "_RQ_HANDLE", None)
+ monkeypatch.setattr(f8, "_act_quant_device_ok", lambda index: False)
+ monkeypatch.setattr(cq, "unlisted_device_ok", lambda index, quant = None: True)
+ monkeypatch.setattr(cq, "_rotq_op", lambda: "op")
+ assert f8.arm_rotated(3) and f8._RQ_DEVICES == frozenset({3}) and f8._RQ_HANDLE == "op"
+ monkeypatch.setattr(cq, "unlisted_device_ok", lambda index, quant = None: False)
+ assert not f8.arm_rotated(3) and f8._RQ_DEVICES == frozenset()
+
+
+ "timing, armed_after",
+ [({"h16": 10.0, "stock": 20.0}, "ieee"), ({"h16": 30.0, "stock": 20.0}, False)],
+)
+def test_dense_route_init_keeps_the_unlisted_verdict(monkeypatch, timing, armed_after):
+ # Z-Image order: install_int8_fused arms rotq_h16, then install_int8_gemm's _swap_on runs rotquant_device_config,
+ # which finds no dense tile on an unlisted arch and must not drop the verdict (no later arm_rotated restores it).
+ _fake_device(monkeypatch, (8, 6))
+ monkeypatch.setattr(cq, "_ROTQ_DEVICE", {})
+ monkeypatch.setattr(cq, "_ROTQ_DEVICE_K", {})
+ monkeypatch.setattr(cq, "_SWIGLU_DEVICE", {})
+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "ieee")
+ monkeypatch.setattr(cq, "_unlisted_latency", lambda index, tf32x3, stock = None: timing)
+ monkeypatch.setattr(cq, "_rotq_probe", lambda index, cfg: pytest.fail("no dense tile to probe"))
+ monkeypatch.setattr(f8, "_RQ_DEVICES", frozenset())
+ monkeypatch.setattr(f8, "_RQ_HANDLE", None)
+ monkeypatch.setattr(f8, "_act_quant_device_ok", lambda index: False)
+ assert f8.arm_rotated(0) is bool(armed_after)
+ assert cq.rotquant_device_config(0) is None
+ assert cq._H16_DEVICE.get(0, False) == armed_after
+ assert (f8._RQ_DEVICES == frozenset({0})) is bool(armed_after)
+ assert not cq._SWIGLU_DEVICE.get(0)
+
+
+# CUDA
+
+
+def _act(m, k, seed):
+ g = torch.Generator().manual_seed(seed)
+ x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)
+ x[:, :5] *= 80
+ x[min(1, m - 1)] = 0
+ return x.to(torch.bfloat16).cuda()
+
+
+def armed(monkeypatch):
+ """The unlisted route forced on whatever card runs the tests (the gate's arch check mocked)."""
+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "1")
+ monkeypatch.setattr(cq, "unlisted_wanted", lambda cap, mode = None: True)
+ monkeypatch.setattr(f8, "_RQ_DEVICES", frozenset())
+ monkeypatch.setattr(f8, "_RQ_HANDLE", None)
+ index = torch.cuda.current_device()
+ if not f8.arm_rotated(index):
+ pytest.skip("rotq_h16 probe refused this device")
+ return index
+
+
+@needs_cuda
[email protected]("m, k", [(300, 256 * 3), (4352, 4096), (4352, 12288), (19, 256 * 128)])
+def test_rotated_act_quant_runs_rotq_h16_once_armed(armed, m, k):
+ x = _act(m, k, m + k)
+ before = cq.h16_call_count()
+ q, s = f8._rotated_act_quant(x, 256, None)
+ assert cq.h16_call_count() == before + 1
+ rq, rs = cq.rotquant_reference(x, 256, "v2")
+ assert q.shape == rq.shape and s.shape == (m, 1)
+ assert cq.codes_close(q, s.reshape(-1), rq, rs)
+
+
+@needs_cuda
+def test_rotated_act_quant_keeps_main_on_an_unarmed_card(monkeypatch):
+ monkeypatch.setattr(f8, "_RQ_DEVICES", frozenset())
+ called = []
+ monkeypatch.setattr(f8, "_rotate", lambda x, g: called.append(g) or x)
+ monkeypatch.setattr(f8, "_fast_act_quant", lambda x, w: ("q", "s"))
+ assert f8._rotated_act_quant(_act(64, 512, 1), 256, None) == ("q", "s") and called == [256]
+
+
+@needs_cuda
+def test_latency_check_is_graph_replayed_and_repeatable():
+ """Kernel time, not host launch time: two runs agree within 25% per arm."""
+ index = torch.cuda.current_device()
+ runs = [cq._unlisted_latency(index, False) for _ in range(2)]
+ for name in ("h16", "stock"):
+ a, b = runs[0][name], runs[1][name]
+ assert 0 < a < float("inf") and abs(a - b) <= 0.25 * max(a, b)
+
+
+@needs_cuda
+def test_compiled_no_graph_break_no_recompile_and_cuda_graph(armed):
+ """One graph and no recompile across token counts (a new prompt length); the op captures in a CUDA graph with
+ replay equal to eager."""
+ import torch._dynamo
+ from torch._dynamo.utils import counters
+
+ torch._dynamo.reset()
+ counters.clear()
+
+ def fn(x):
+ q, s = f8._rotated_act_quant(x, 256, None)
+ return q.float() * s
+
+ cfn = torch.compile(fn, fullgraph = True, dynamic = True)
+ for m in (300, 517, 4352):
+ x = _act(m, 4096, m)
+ assert torch.equal(cfn(x), fn(x))
+ assert counters["stats"].get("unique_graphs", 0) <= 1
+ assert not counters["graph_break"]
+
+ static = _act(4352, 4096, 9)
+ stream = torch.cuda.Stream()
+ stream.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(stream):
+ fn(static)
+ torch.cuda.current_stream().wait_stream(stream)
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ out = fn(static)
+ static.copy_(_act(4352, 4096, 10))
+ graph.replay()
+ torch.cuda.synchronize()
+ assert torch.equal(out, fn(static))
diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.py
index c55fb4b0183..388d0837749 100644
--- a/studio/backend/tests/test_diffusion_int8_fused.py
+++ b/studio/backend/tests/test_diffusion_int8_fused.py
@@ -73,6 +73,9 @@ def _require_int8tensor(module):
@pytest.fixture(autouse = True)
def _clean_env(monkeypatch):
monkeypatch.delenv(fused.INT8_FUSED_ENV, raising = False)
+ # pins the stock rotation; rotq_h16 dispatch: test_diffusion_convrot_unlisted.py
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED", "0")
+ monkeypatch.setattr(fused, "_RQ_DEVICES", frozenset())
yield
diff --git a/studio/backend/tests/test_diffusion_qwenimage21_fused.py b/studio/backend/tests/test_diffusion_qwenimage21_fused.py
index c4fc42be75d..6405e353f85 100644
--- a/studio/backend/tests/test_diffusion_qwenimage21_fused.py
+++ b/studio/backend/tests/test_diffusion_qwenimage21_fused.py
@@ -36,6 +36,9 @@ needs_cuda = pytest.mark.skipif(
@pytest.fixture(autouse = True)
def _restore(monkeypatch):
monkeypatch.delenv(qf.Q21_CONVROT_FUSED_ENV, raising = False)
+ # pins the stock rotation; rotq_h16 dispatch: test_diffusion_convrot_unlisted.py
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED", "0")
+ monkeypatch.setattr(i8f, "_RQ_DEVICES", frozenset())
yield
qf.uninstall()