Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: route untuned archs (sm86 / 90 / 100 / 121) onto the factored ConvRot kernel behind a probe and latency check (#14988)
Daniel Han1d9c522c1b08
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex 1f12fab98b9..4c3e87803a7 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -298,6 +298,7 @@ _GRAPH_SWITCHES = {"UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT": "on","UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED": "on","UNSLOTH_DIFFUSION_INT8_ROTQUANT": "auto",+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED": "auto","UNSLOTH_DIFFUSION_INT8_FUSED": "on","UNSLOTH_DIFFUSION_ZIMAGE_FUSED": "on","UNSLOTH_DIFFUSION_Q21_REAL_ROPE": "on",diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.pyindex 0e88ff4f60d..30be47ac5d8 100644--- a/studio/backend/core/inference/diffusion_convrot_quant.py+++ b/studio/backend/core/inference/diffusion_convrot_quant.py@@ -27,6 +27,10 @@ H16 ROTATION (``rotq_h16``, default where ``rotq_i8`` passed): ``H256 = H16 (x)everywhere. A different fp32 summation order: ~1 code in 10^6 differs by one step, scales equal; the probe gates it perdevice at that tolerance (``codes_close``). ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_H16=0`` keeps the bit-identical dense kernel.+UNLISTED ARCHS (sm86, sm87, sm90, sm100, sm103, sm121, ...: no ``_ROTQ_CONFIG`` entry, so no dense tile to probe):+``rotq_h16`` needs no tile, so ``unlisted_device_ok`` probes it there directly, then a latency check against the two+kernels it replaces (see below). Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED=0``.+SwiGLU PROLOGUE (``rotq_h16`` only): the down projection's codes come straight from the gate / value halves, theproduct computed in-kernel like the compiled stock MLP; the [M, hidden] HBM round trip was ConvRot's largest cost overplain int8 at Qwen-Image-2.1 1024px. Kill switch ``UNSLOTH_DIFFUSION_INT8_ROTQUANT_SWIGLU=0``.@@ -836,6 +840,10 @@ def rotquant_device_config(index: int) -> Optional[tuple]:sw = _swiglu_probe(index)except Exception: # noqa: BLE001 - the dense kernel stays, no SwiGLU prologueh16 = sw = False+ elif _UNLISTED_DEVICE.get((index, unlisted_mode())):+ h16 = (+ _UNLISTED_PROBE.get(index) or False+ ) # no dense tile: keep the verdict unlisted_device_ok armed_H16_DEVICE[index] = h16_SWIGLU_DEVICE[index] = swif cfg is not None:@@ -1017,3 +1025,124 @@ def convrot_swiglu_act_quant(g2d: Any, v2d: Any, group: int, v2: bool) -> Optionif not _h16_ok(g2d, group, v2d):return Nonereturn _SWIGLU_HANDLE(g2d, v2d, group, v2)+++# Unlisted archs: main runs the rotation as a dense bf16 GEMM (1/8 the int8 rate on GA102) + a separate act quant.+# rotq_h16 must pass the probe and be no slower than both, timed from CUDA-graph replays (the compiled denoiser replays+# kernels; host launch cost must not count). ``..._UNLISTED=1`` skips the latency check, not the probe.+INT8_ROTQUANT_UNLISTED_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED"+_UNLISTED_DEVICE: dict = {}+_UNLISTED_TIMING: dict = {}+_UNLISTED_PROBE: dict = {} # index -> _h16_probe result (mode independent)+_UNLISTED_LATENCY_SHAPES = ((2048, 4096), (1024, 12288))+++def unlisted_mode() -> str:+ if rotquant_mode() == "off" or not rotquant_h16_enabled():+ return "off"+ return _env_mode(INT8_ROTQUANT_UNLISTED_ENV)+++def unlisted_wanted(capability: Optional[tuple], mode: Optional[str] = None) -> bool:+ """True on a CUDA sm80+ arch with no dense ``rotq_i8`` tile (the listed archs keep their probed path)."""+ mode = unlisted_mode() if mode is None else mode+ if mode == "off" or capability is None:+ return False+ cap = (int(capability[0]), int(capability[1]))+ return cap >= (8, 0) and rotquant_config(cap) is None+++def _unlisted_latency(+ index: int,+ tf32x3: bool,+ stock_quant: Any = None,+) -> dict:+ """{"h16": us, "stock": us} over ``_UNLISTED_LATENCY_SHAPES``: each arm captured in a CUDA graph (two passes)+ and replayed, best of seven interleaved rounds. Stock = the rotation GEMM, then ``stock_quant`` of its output when+ given, else the GEMM alone (a lower bound)."""+ import torch+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation++ dev = torch.device("cuda", index)+ xs = [torch.randn(m, k, device = dev).to(torch.bfloat16) for m, k in _UNLISTED_LATENCY_SHAPES]+ h = build_convrot_hadamard(256, device = dev, dtype = torch.bfloat16)++ def stock(x):+ r = rotate_convrot_activation(x, h, 256)+ return r if stock_quant is None else stock_quant(r)++ fns = {"h16": lambda x: _h16_launch(x, "v2", tf32x3 = tf32x3), "stock": stock}+ reps = 2+ graphs = {}+ with torch.cuda.device(dev):+ side = torch.cuda.Stream(dev)+ side.wait_stream(torch.cuda.current_stream(dev))+ with torch.cuda.stream(side):+ for fn in fns.values():+ for x in xs:+ fn(x) # compile / autotune outside the capture+ torch.cuda.current_stream(dev).wait_stream(side)+ torch.cuda.synchronize(dev)+ for name, fn in fns.items():+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ for _ in range(reps):+ for x in xs:+ fn(x)+ graphs[name] = graph+ for graph in graphs.values():+ graph.replay()+ torch.cuda.synchronize(dev)+ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)+ best = {name: float("inf") for name in graphs}+ for _ in range(7):+ for name, graph in graphs.items():+ start.record()+ graph.replay()+ end.record()+ end.synchronize()+ best[name] = min(best[name], start.elapsed_time(end) * 1000.0 / reps)+ del graphs+ return best+++def unlisted_device_ok(index: int, stock_quant: Any = None) -> bool:+ """Once per (device, mode), outside any trace / capture: arch gate, ``_h16_probe`` (once per device), latency+ check (auto only). Keyed by mode so switching ``..._UNLISTED`` between loads takes effect without a restart."""+ global _ROTQ_HANDLE+ mode = unlisted_mode()+ if mode == "off": # read per load: the switches act without a restart+ return False+ key = (index, mode)+ if key not in _UNLISTED_DEVICE:+ ok = False+ try:+ import torch+ if (+ torch.cuda.is_available()+ and not is_rocm_torch(torch)+ and _triton_ok()+ and unlisted_wanted(torch.cuda.get_device_capability(index), mode)+ and _kernels() is not None+ and _rotq_op() is not None+ ):+ if index not in _UNLISTED_PROBE:+ _UNLISTED_PROBE[index] = _h16_probe(index)+ h16 = _UNLISTED_PROBE[index]+ if h16:+ ok = True+ if mode != "force":+ timing = _unlisted_latency(index, h16 == "tf32x3", stock_quant)+ _UNLISTED_TIMING[index] = {k: round(v, 2) for k, v in timing.items()}+ ok = timing["h16"] <= timing["stock"]+ except Exception: # noqa: BLE001 - main's path stays+ ok = False+ _UNLISTED_DEVICE[key] = ok+ ok = _UNLISTED_DEVICE[key]+ h16 = _UNLISTED_PROBE.get(index)+ if ok:+ _H16_DEVICE[index] = h16+ _ROTQ_HANDLE = _rotq_op()+ elif h16: # probed unlisted card the latency check now rejects: drop a verdict a forced load left behind+ _H16_DEVICE.pop(index, None)+ return okdiff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.pyindex c230729c6c5..0418636dd1d 100644--- a/studio/backend/core/inference/diffusion_int8_fused.py+++ b/studio/backend/core/inference/diffusion_int8_fused.py@@ -22,6 +22,8 @@ w2's own (rotating) forward. Same ops on the same values as the three stock rotaquant + ``_int_mm`` + epilogue without torchao 0.17's always-on zero-point pass, and the act quant itself as one probedTriton kernel (``int8_act_quant_rows``, bit-identical to torchao 0.17 and 0.18): Inductor's codegen of the same per-rowquant ran 2-3x slower on B200 (Z-Image shapes, M 4224: K 3840 24-32 -> 11 us, K 10240 53-70 -> 32 us).+A rotated input (ConvRot) is rotated and quantized by one kernel (``_rotated_act_quant``: ``rotq_h16`` of+``diffusion_convrot_quant``) on an unlisted-arch card it was armed for, else rotation GEMM + act quant.Kill switch: ``UNSLOTH_DIFFUSION_INT8_FUSED=0`` (all of the above)."""@@ -52,6 +54,8 @@ _OP_HANDLE: Any = None_ACTQ_HANDLE: Any = None# device indices whose own probe passed; the kernel never runs on a device that failed or was never probed_ACTQ_DEVICES: frozenset = frozenset()+_RQ_HANDLE: Any = None+_RQ_DEVICES: frozenset = frozenset()# UNSLOTH_DIFFUSION_INT8_FUSED=0 at the last install: int8_linear is then module(x)_LINEAR_OFF = False# torchao's safe_int_mm home, resolved at install (outside any trace) for the same reason.@@ -1309,8 +1313,9 @@ def int8_linear_core(module: Any, x: Any) -> Any:):return _stock_call(module, x)if rotated:- x2d = _rotate(x2d, module.convrot_groupsize)- xq, xs = _fast_act_quant(x2d, weight)+ xq, xs = _rotated_act_quant(x2d, module.convrot_groupsize, weight)+ else:+ xq, xs = _fast_act_quant(x2d, weight)y = _torchao_epilogue(_int_mm(xq.reshape(-1, xq.shape[-1]), weight), xs, weight.scale, module.bias, x.dtype)@@ -1331,6 +1336,48 @@ def _rotate(x2d: Any, group: int) -> Any:)+def _rotated_act_quant(x2d: Any, group: int, weight: Any) -> tuple:+ """``_fast_act_quant(_rotate(x2d, group), weight)``: one fused rotation + quant kernel (``rotq_h16``) on a device+ it was armed for (an arch the tiled ``rotq_i8`` does not cover), else the rotation GEMM + act quant."""+ import torch++ if (+ _RQ_HANDLE is not None+ and group == 256+ and x2d.is_cuda+ and x2d.device.index in _RQ_DEVICES+ and x2d.dtype == torch.bfloat16+ and x2d.dim() == 2+ and x2d.shape[0] > 0+ and x2d.shape[1] % 256 == 0+ ):+ q, s = _RQ_HANDLE(x2d, group, True)+ return q, s.reshape(-1, 1)+ return _fast_act_quant(_rotate(x2d, group), weight)+++def arm_rotated(index: int) -> bool:+ """Arm ``rotq_h16`` for card ``index`` (outside any trace); False keeps the rotation GEMM + act quant."""+ global _RQ_HANDLE, _RQ_DEVICES+ if int8_fused_disabled():+ return False+ try:+ from .diffusion_convrot_quant import _rotq_op, unlisted_device_ok++ actq = _act_quant_op() if _act_quant_device_ok(index) else None+ ok = unlisted_device_ok(index, actq)+ handle = _rotq_op() if ok else None+ except Exception: # noqa: BLE001 - optimisation only+ ok, handle = False, None+ with _LOCK:+ if ok and handle is not None:+ _RQ_HANDLE = handle+ _RQ_DEVICES = _RQ_DEVICES | {index}+ return True+ _RQ_DEVICES = _RQ_DEVICES - {index}+ return False++def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:"""SwiGLU MLP with plain int8 weights: one act quant, one GEMM over [gate | value], fused SiLU-product quant.ConvRot parts: the shared input rotates once; a rotated down projection gets the bf16 product (see module doc)."""@@ -1357,8 +1404,7 @@ def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) ->return type(self).forward(self, hidden_states, *args, **kwargs)down = _get(self, down_name)if in_group:- x2d = _rotate(x2d, in_group)- xq, xs = _fast_act_quant(x2d, fused_in.weight)+ xq, xs = _rotated_act_quant(x2d, in_group, fused_in.weight)else:xq, xs = _act_quant(x2d, fused_in.weight)c = _int_mm(xq.reshape(-1, xq.shape[-1]), fused_in.weight)@@ -1569,11 +1615,12 @@ def install(offload_active: bool = False,) -> int:"""Idempotent; returns the (candidate) count. Must run before the first compiled forward, which traces ``forward``."""- global _ACTQ_HANDLE, _ACTQ_DEVICES, _LINEAR_OFF+ global _ACTQ_HANDLE, _ACTQ_DEVICES, _RQ_DEVICES, _LINEAR_OFFif int8_fused_disabled():# process-global: a previous load's probe must not outlive the kill switch_ACTQ_HANDLE = None_ACTQ_DEVICES = frozenset()+ _RQ_DEVICES = frozenset()_LINEAR_OFF = Truereturn 0_LINEAR_OFF = False@@ -1655,6 +1702,14 @@ def _swiglu_eligible(module: Any) -> bool:return False+def _has_rotated(transformer: Any) -> bool:+ try:+ from .diffusion_convrot import is_rotated_linear+ return any(is_rotated_linear(m) for m in transformer.modules())+ except Exception: # noqa: BLE001+ return False++def _has_eligible(transformer: Any) -> bool:return any(_MARK in m.__dict__ or _ff_eligible(m) or _swiglu_eligible(m) or _flux_single_eligible(m)@@ -1693,6 +1748,9 @@ def _finalize(transformer: Any, logger: Any = None) -> int:indices = [dev.index if dev.index is not None else torch.cuda.current_device()]count = 0rearm = False+ if _has_rotated(transformer):+ for index in indices:+ arm_rotated(index)actq_ok = {index: _act_quant_device_ok(index) for index in indices}actq = _act_quant_op() if any(actq_ok.values()) else Nonewith _LOCK:@@ -1800,10 +1858,13 @@ def relocate_after_layer_split(transformer: Any, logger: Any = None) -> dict:index = _module_card(module)if index is not None:cards.setdefault(index, []).append(module)+ rotated = _has_rotated(transformer)for index, modules in cards.items():if _device_ok(index):if _act_quant_device_ok(index) and _ACTQ_HANDLE is not None:_ACTQ_DEVICES = _ACTQ_DEVICES | {index}+ if rotated:+ arm_rotated(index)continuefor module in modules:_restore_module(module)diff --git a/studio/backend/core/inference/diffusion_qwenimage21_fused.py b/studio/backend/core/inference/diffusion_qwenimage21_fused.pyindex 31b9fb1d691..2da69b7315b 100644--- a/studio/backend/core/inference/diffusion_qwenimage21_fused.py+++ b/studio/backend/core/inference/diffusion_qwenimage21_fused.py@@ -4,9 +4,11 @@"""Qwen-Image-2.1 ConvRot int8: one rotation + one act quant per shared input (QKV, SwiGLU gate/value) in the compile.Rotation is on the shared input axis and int8 scales are per row, so fused outputs equal the separate rotated ones bit-for bit. The fused QKV Linear: only where ``diffusion_int8_gemm`` leaves rotated Linears on the stock path (e.g. sm90,-sm100), resident only. Where it swaps them, the patched helper routes Q / K / V through-``diffusion_int8_gemm.shared_rotq_linears`` (Inductor does not merge the opaque kernel); resident or streamed.+for bit. The fused QKV Linear: only where ``diffusion_int8_gemm`` leaves rotated Linears on the stock path (e.g. sm86,+sm90, sm100), resident only; there the rotation and the act quant run as one ``rotq_h16`` kernel on a card that passed+``diffusion_convrot_quant.unlisted_device_ok``, else as the rotation GEMM + act quant. Where it swaps them, the+patched helper routes Q / K / V through ``diffusion_int8_gemm.shared_rotq_linears`` (Inductor does not merge the+opaque kernel); resident or streamed.Kill switch: ``UNSLOTH_DIFFUSION_Q21_CONVROT_FUSED=0``."""@@ -290,6 +292,23 @@ def install(return {"fused_qkv": shared or None, "out": None}+def _rotq_note(cards: Any, armed: list) -> str:+ """Log suffix: whether the fused rotq_h16 rotation + quant serves these cards, with its latency check."""+ try:+ from .diffusion_convrot_quant import _UNLISTED_TIMING+ timing = {i: _UNLISTED_TIMING.get(i) for i in cards if _UNLISTED_TIMING.get(i)}+ except Exception: # noqa: BLE001+ timing = {}+ shown = "; ".join(+ f"cuda:{i} {t['h16']:.0f} vs {t['stock']:.0f} us stock"+ for i, t in timing.items()+ if "h16" in t and "stock" in t+ )+ if armed:+ return ", rotation fused into the act quant (rotq_h16" + (f": {shown})" if shown else ")")+ return f", fused rotation kept off ({shown})" if shown else ""++def install_modules(root: Any, logger: Any = None) -> dict:result = {"fused_qkv": 0, "out": 0}if q21_convrot_fused_disabled():@@ -306,6 +325,10 @@ def install_modules(root: Any, logger: Any = None) -> dict:todo = _rotated_attention(root)if not todo or not _patch_prepare(logger):return result+ from .diffusion_int8_fused import arm_rotated++ armed = [index for index in cards if arm_rotated(index)]+ note = _rotq_note(cards, armed)with _LOCK:for module, parts in todo:if _QKV_ATTR not in module.__dict__ and all(_plain(p) for p in parts):@@ -324,9 +347,10 @@ def install_modules(root: Any, logger: Any = None) -> dict:if logger is not None:logger.info("diffusion.q21_convrot_fused: one ConvRot rotation + act quant for QKV on %d attention module(s), "- "probed act quant on %d output projection(s)",+ "probed act quant on %d output projection(s)%s",result["fused_qkv"],result["out"],+ note,)return resultdiff --git a/studio/backend/tests/test_diffusion_convrot_unlisted.py b/studio/backend/tests/test_diffusion_convrot_unlisted.pynew file mode 100644index 00000000000..d31e5e4df4a--- /dev/null+++ b/studio/backend/tests/test_diffusion_convrot_unlisted.py@@ -0,0 +1,336 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for the unlisted-arch dispatch of ``rotq_h16`` (``diffusion_convrot_quant.unlisted_device_ok`` and+``diffusion_int8_fused._rotated_act_quant``): every CUDA sm80+ arch without a dense ``rotq_i8`` tile gets the fused+rotation + act quant when it passes the probe and is not slower than the rotation GEMM + act quant. The gate tests mock+the capability and run anywhere; the kernel tests need NVIDIA CUDA, Triton and torchao."""++from __future__ import annotations++import pytest++from core.inference import diffusion_convrot_quant as cq+from core.inference import diffusion_int8_fused as f8++torch = pytest.importorskip("torch")++needs_cuda = pytest.mark.skipif(+ not torch.cuda.is_available() or getattr(torch.version, "hip", None) is not None,+ reason = "NVIDIA CUDA required",+)++_ENVS = (+ cq.INT8_ROTQUANT_UNLISTED_ENV,+ cq.INT8_ROTQUANT_H16_ENV,+ cq.INT8_ROTQUANT_ENV,+ "UNSLOTH_DIFFUSION_INT8_GEMM",+)++[email protected](autouse = True)+def _clean(monkeypatch):+ for name in _ENVS:+ monkeypatch.delenv(name, raising = False)+ cq._UNLISTED_DEVICE.clear()+ cq._UNLISTED_TIMING.clear()+ cq._UNLISTED_PROBE.clear()+ saved = dict(cq._H16_DEVICE)+ cq._H16_DEVICE.clear() # a real-card verdict from an earlier test file must not leak in+ yield+ cq._UNLISTED_DEVICE.clear()+ cq._UNLISTED_TIMING.clear()+ cq._UNLISTED_PROBE.clear()+ cq._H16_DEVICE.clear()+ cq._H16_DEVICE.update(saved)+++ "cap, on",+ [+ ((7, 5), False), # T4: no bf16 tensor cores for rotq_h16's first dot+ ((8, 0), False), # A100: dense rotq_i8 tile (and rotq_h16 on top of it)+ ((8, 6), True), # RTX 30xx / A6000 / A10+ ((8, 7), True), # Orin+ ((8, 9), False), # L4 / RTX 40xx: tiled+ ((9, 0), True), # H100+ ((10, 0), True), # B200+ ((10, 3), True), # B300+ ((12, 0), False), # RTX PRO 6000 / RTX 50xx: tiled+ ((12, 1), True), # DGX Spark+ ],+)+def test_every_sm80_plus_arch_without_a_tile_is_covered(cap, on):+ assert cq.unlisted_wanted(cap) is on+ if cap >= (8, 0): # exactly one route to the fused rotation per sm80+ arch+ assert (cq.rotquant_config(cap) is not None) is not on+++ "env",+ [+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED",+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT_H16",+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT",+ "UNSLOTH_DIFFUSION_INT8_GEMM",+ ],+)+def test_kill_switches(monkeypatch, env):+ monkeypatch.setenv(env, "0")+ for cap in ((8, 6), (9, 0), (10, 0), (12, 1)):+ assert not cq.unlisted_wanted(cap)+++def test_forcing_the_dense_tile_takes_the_listed_route(monkeypatch):+ monkeypatch.setenv(+ cq.INT8_ROTQUANT_ENV, "1"+ ) # dense fallback tile on every unmeasured sm80+ part+ assert not cq.unlisted_wanted((8, 6)) and not cq.unlisted_wanted((10, 0))+++def _fake_device(monkeypatch, cap):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: cap)+ monkeypatch.setattr(cq, "_triton_ok", lambda: True)+ monkeypatch.setattr(cq, "_kernels", lambda: object())+ monkeypatch.setattr(cq, "_rotq_op", lambda: "op")+ monkeypatch.setattr(cq, "is_rocm_torch", lambda t: False)+ monkeypatch.setattr(cq, "_ROTQ_HANDLE", None)+++ "timing, mode, ok",+ [+ ({"h16": 10.0, "stock": 20.0}, None, True),+ ({"h16": 20.0, "stock": 20.0}, None, True), # a tie is not slower+ (+ {"h16": 30.0, "stock": 20.0},+ None,+ False,+ ), # slower than the two kernels it replaces: main's path+ ({"h16": 30.0, "stock": 20.0}, "1", True), # forced: no latency check+ ],+)+def test_latency_gate(monkeypatch, timing, mode, ok):+ _fake_device(monkeypatch, (8, 6))+ if mode is not None:+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, mode)+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "ieee")+ monkeypatch.setattr(cq, "_unlisted_latency", lambda index, tf32x3, stock = None: timing)+ assert cq.unlisted_device_ok(0) is ok+ assert (cq._H16_DEVICE.get(0) == "ieee") is ok+ assert (cq._ROTQ_HANDLE == "op") is ok+++def test_latency_check_uses_the_probed_mode_and_the_callers_quant(monkeypatch):+ _fake_device(monkeypatch, (9, 0))+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "tf32x3")+ seen = []+ monkeypatch.setattr(+ cq,+ "_unlisted_latency",+ lambda index, tf32x3, stock = None: seen.append((tf32x3, stock))+ or {"h16": 1.0, "stock": 2.0},+ )+ quant = object()+ assert (+ cq.unlisted_device_ok(0, quant)+ and seen == [(True, quant)]+ and cq._H16_DEVICE[0] == "tf32x3"+ )+++def test_failed_probe_keeps_main(monkeypatch):+ _fake_device(monkeypatch, (10, 0))+ monkeypatch.setattr(cq, "_h16_probe", lambda index: False)+ monkeypatch.setattr(+ cq, "_unlisted_latency", lambda *a, **k: pytest.fail("timed after a failed probe")+ )+ assert cq.unlisted_device_ok(0) is False and 0 not in cq._H16_DEVICE++[email protected]("cap", [(7, 5), (8, 0), (8, 9), (12, 0)])+def test_listed_archs_and_sm75_never_probe(monkeypatch, cap):+ _fake_device(monkeypatch, cap)+ monkeypatch.setattr(cq, "_h16_probe", lambda index: pytest.fail("probed"))+ assert cq.unlisted_device_ok(0) is False+++def test_kill_switch_acts_on_a_cached_device(monkeypatch):+ _fake_device(monkeypatch, (8, 6))+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "ieee")+ monkeypatch.setattr(+ cq, "_unlisted_latency", lambda index, tf32x3, stock = None: {"h16": 1.0, "stock": 2.0}+ )+ assert cq.unlisted_device_ok(0)+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "0")+ assert not cq.unlisted_device_ok(0)+ monkeypatch.delenv(cq.INT8_ROTQUANT_UNLISTED_ENV)+ assert cq.unlisted_device_ok(0)+++def _counting_slow_device(monkeypatch):+ _fake_device(monkeypatch, (8, 6))+ calls = {"probe": 0, "time": 0}++ def probe(index):+ calls["probe"] += 1+ return "ieee"++ def latency(+ index,+ tf32x3,+ stock = None,+ ):+ calls["time"] += 1+ return {"h16": 30.0, "stock": 20.0} # slower: auto rejects++ monkeypatch.setattr(cq, "_h16_probe", probe)+ monkeypatch.setattr(cq, "_unlisted_latency", latency)+ return calls+++def test_force_after_an_auto_rejection_enables_the_card(monkeypatch):+ calls = _counting_slow_device(monkeypatch)+ assert not cq.unlisted_device_ok(0) and 0 not in cq._H16_DEVICE+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "1")+ assert cq.unlisted_device_ok(0) and cq._H16_DEVICE[0] == "ieee"+ assert calls == {"probe": 1, "time": 1} # probed once per device, never timed under force+++def test_auto_after_a_forced_load_runs_the_latency_check(monkeypatch):+ calls = _counting_slow_device(monkeypatch)+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "1")+ assert cq.unlisted_device_ok(0) and cq._H16_DEVICE[0] == "ieee"+ monkeypatch.delenv(cq.INT8_ROTQUANT_UNLISTED_ENV)+ assert not cq.unlisted_device_ok(0) and 0 not in cq._H16_DEVICE+ assert not cq.unlisted_device_ok(0) # repeat auto load: cached, not re-timed+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "1")+ assert cq.unlisted_device_ok(0) and cq._H16_DEVICE[0] == "ieee"+ assert calls == {"probe": 1, "time": 1}+++def test_arm_rotated_follows_the_gate(monkeypatch):+ monkeypatch.setattr(f8, "_RQ_DEVICES", frozenset())+ monkeypatch.setattr(f8, "_RQ_HANDLE", None)+ monkeypatch.setattr(f8, "_act_quant_device_ok", lambda index: False)+ monkeypatch.setattr(cq, "unlisted_device_ok", lambda index, quant = None: True)+ monkeypatch.setattr(cq, "_rotq_op", lambda: "op")+ assert f8.arm_rotated(3) and f8._RQ_DEVICES == frozenset({3}) and f8._RQ_HANDLE == "op"+ monkeypatch.setattr(cq, "unlisted_device_ok", lambda index, quant = None: False)+ assert not f8.arm_rotated(3) and f8._RQ_DEVICES == frozenset()+++ "timing, armed_after",+ [({"h16": 10.0, "stock": 20.0}, "ieee"), ({"h16": 30.0, "stock": 20.0}, False)],+)+def test_dense_route_init_keeps_the_unlisted_verdict(monkeypatch, timing, armed_after):+ # Z-Image order: install_int8_fused arms rotq_h16, then install_int8_gemm's _swap_on runs rotquant_device_config,+ # which finds no dense tile on an unlisted arch and must not drop the verdict (no later arm_rotated restores it).+ _fake_device(monkeypatch, (8, 6))+ monkeypatch.setattr(cq, "_ROTQ_DEVICE", {})+ monkeypatch.setattr(cq, "_ROTQ_DEVICE_K", {})+ monkeypatch.setattr(cq, "_SWIGLU_DEVICE", {})+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "ieee")+ monkeypatch.setattr(cq, "_unlisted_latency", lambda index, tf32x3, stock = None: timing)+ monkeypatch.setattr(cq, "_rotq_probe", lambda index, cfg: pytest.fail("no dense tile to probe"))+ monkeypatch.setattr(f8, "_RQ_DEVICES", frozenset())+ monkeypatch.setattr(f8, "_RQ_HANDLE", None)+ monkeypatch.setattr(f8, "_act_quant_device_ok", lambda index: False)+ assert f8.arm_rotated(0) is bool(armed_after)+ assert cq.rotquant_device_config(0) is None+ assert cq._H16_DEVICE.get(0, False) == armed_after+ assert (f8._RQ_DEVICES == frozenset({0})) is bool(armed_after)+ assert not cq._SWIGLU_DEVICE.get(0)+++# CUDA+++def _act(m, k, seed):+ g = torch.Generator().manual_seed(seed)+ x = torch.randn(m, k, generator = g) * (torch.rand(1, k, generator = g) * 4)+ x[:, :5] *= 80+ x[min(1, m - 1)] = 0+ return x.to(torch.bfloat16).cuda()+++def armed(monkeypatch):+ """The unlisted route forced on whatever card runs the tests (the gate's arch check mocked)."""+ monkeypatch.setenv(cq.INT8_ROTQUANT_UNLISTED_ENV, "1")+ monkeypatch.setattr(cq, "unlisted_wanted", lambda cap, mode = None: True)+ monkeypatch.setattr(f8, "_RQ_DEVICES", frozenset())+ monkeypatch.setattr(f8, "_RQ_HANDLE", None)+ index = torch.cuda.current_device()+ if not f8.arm_rotated(index):+ pytest.skip("rotq_h16 probe refused this device")+ return index+++@needs_cuda[email protected]("m, k", [(300, 256 * 3), (4352, 4096), (4352, 12288), (19, 256 * 128)])+def test_rotated_act_quant_runs_rotq_h16_once_armed(armed, m, k):+ x = _act(m, k, m + k)+ before = cq.h16_call_count()+ q, s = f8._rotated_act_quant(x, 256, None)+ assert cq.h16_call_count() == before + 1+ rq, rs = cq.rotquant_reference(x, 256, "v2")+ assert q.shape == rq.shape and s.shape == (m, 1)+ assert cq.codes_close(q, s.reshape(-1), rq, rs)+++@needs_cuda+def test_rotated_act_quant_keeps_main_on_an_unarmed_card(monkeypatch):+ monkeypatch.setattr(f8, "_RQ_DEVICES", frozenset())+ called = []+ monkeypatch.setattr(f8, "_rotate", lambda x, g: called.append(g) or x)+ monkeypatch.setattr(f8, "_fast_act_quant", lambda x, w: ("q", "s"))+ assert f8._rotated_act_quant(_act(64, 512, 1), 256, None) == ("q", "s") and called == [256]+++@needs_cuda+def test_latency_check_is_graph_replayed_and_repeatable():+ """Kernel time, not host launch time: two runs agree within 25% per arm."""+ index = torch.cuda.current_device()+ runs = [cq._unlisted_latency(index, False) for _ in range(2)]+ for name in ("h16", "stock"):+ a, b = runs[0][name], runs[1][name]+ assert 0 < a < float("inf") and abs(a - b) <= 0.25 * max(a, b)+++@needs_cuda+def test_compiled_no_graph_break_no_recompile_and_cuda_graph(armed):+ """One graph and no recompile across token counts (a new prompt length); the op captures in a CUDA graph with+ replay equal to eager."""+ import torch._dynamo+ from torch._dynamo.utils import counters++ torch._dynamo.reset()+ counters.clear()++ def fn(x):+ q, s = f8._rotated_act_quant(x, 256, None)+ return q.float() * s++ cfn = torch.compile(fn, fullgraph = True, dynamic = True)+ for m in (300, 517, 4352):+ x = _act(m, 4096, m)+ assert torch.equal(cfn(x), fn(x))+ assert counters["stats"].get("unique_graphs", 0) <= 1+ assert not counters["graph_break"]++ static = _act(4352, 4096, 9)+ stream = torch.cuda.Stream()+ stream.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(stream):+ fn(static)+ torch.cuda.current_stream().wait_stream(stream)+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ out = fn(static)+ static.copy_(_act(4352, 4096, 10))+ graph.replay()+ torch.cuda.synchronize()+ assert torch.equal(out, fn(static))diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.pyindex c55fb4b0183..388d0837749 100644--- a/studio/backend/tests/test_diffusion_int8_fused.py+++ b/studio/backend/tests/test_diffusion_int8_fused.py@@ -73,6 +73,9 @@ def _require_int8tensor(module):@pytest.fixture(autouse = True)def _clean_env(monkeypatch):monkeypatch.delenv(fused.INT8_FUSED_ENV, raising = False)+ # pins the stock rotation; rotq_h16 dispatch: test_diffusion_convrot_unlisted.py+ monkeypatch.setenv("UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED", "0")+ monkeypatch.setattr(fused, "_RQ_DEVICES", frozenset())yielddiff --git a/studio/backend/tests/test_diffusion_qwenimage21_fused.py b/studio/backend/tests/test_diffusion_qwenimage21_fused.pyindex c4fc42be75d..6405e353f85 100644--- a/studio/backend/tests/test_diffusion_qwenimage21_fused.py+++ b/studio/backend/tests/test_diffusion_qwenimage21_fused.py@@ -36,6 +36,9 @@ needs_cuda = pytest.mark.skipif(@pytest.fixture(autouse = True)def _restore(monkeypatch):monkeypatch.delenv(qf.Q21_CONVROT_FUSED_ENV, raising = False)+ # pins the stock rotation; rotq_h16 dispatch: test_diffusion_convrot_unlisted.py+ monkeypatch.setenv("UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED", "0")+ monkeypatch.setattr(i8f, "_RQ_DEVICES", frozenset())yieldqf.uninstall()